Hadoop DataNode启动之DataBlockScanner

  存储在HDFS中的数据块每隔一段时间就会进行校验，因为由于硬件故障，系统BUG，异常断电等问题会导致数据块损坏，这时就需要HDFS采用其他块副本来修复。Hadoop为了感应块的损坏会启动一个后台线程周期性的对数据块进行检测，这个原理和客户端读取数据时的校验规则一样，以块为单位，每读取一个block，都要对其内的每一个chunk进行校验，默认情况下一个block大小为64M，每个chunk大小为512字节，下面我们从启动开始分析下这个过程

 


 DataBlockScanner在DN启动时创建，在startDataNode函数中

void startDataNode(Configuration conf,
AbstractList<File> dataDirs, SecureResources resources
) throws IOException {
......
//initialize periodic block scanner
String reason = null;
//获得扫描周期，默认为三周，参数单位为小时
if (conf.getInt("dfs.datanode.scan.period.hours", 0) < 0) {
reason = "verification is turned off by configuration";
} else if ( !(data instanceof FSDataset) ) {
reason = "verifcation is supported only with FSDataset";
}
if ( reason == null ) {
//开始创建块扫描器，但并为启动
blockScanner = new DataBlockScanner(this, (FSDataset)data, conf);
} else {
LOG.info("Periodic Block Verification is disabled because " +
reason + ".");
}
......
}

块扫描器的启动在DN主循环的offerService中

public void offerService() throws Exception {
LOG.info("using BLOCKREPORT_INTERVAL of " + blockReportInterval + "msec" +
" Initial delay: " + initialBlockReportDelay + "msec");
//
// Now loop for a long time....
//
while (shouldRun) {
try {
....
....
// start block scanner
if (blockScanner != null && blockScannerThread == null &&
upgradeManager.isUpgradeCompleted()) {
LOG.info("Starting Periodic block scanner.");
//创建后台守护线程并启动
blockScannerThread = new Daemon(blockScanner);
blockScannerThread.start();
}
....
....
} // offerService

 块校验的具体流程包含在线程执行体中，大致分为三个部分 1、块集合初始化  2、更新block校验时间 3、循环校验负责规则的block。中间还有节流器控制

public void run() {
try {
//1、块集合初始化
init();

// 2、更新block校验时间
if (!assignInitialVerificationTimes()) {
return;
}
//节流器配置
adjustThrottler();
//3、循环校验负责规则的block
while (datanode.shouldRun && !Thread.interrupted()) {
long now = System.currentTimeMillis();
synchronized (this) {
if ( now >= (currentPeriodStart + scanPeriod)) {
//开启新的校验周期
startNewPeriod();
}
}
if ( (now - getEarliestScanTime()) >= scanPeriod ) {
//校验一个数据块
verifyFirstBlock();
} else {
try {
//循环间隔1秒
Thread.sleep(1000);
} catch (InterruptedException ignored) {}
}
}
} catch (RuntimeException e) {
LOG.warn("RuntimeException during DataBlockScanner.run() : " +
StringUtils.stringifyException(e));
throw e;
} finally {
shutdown();
LOG.info("Exiting DataBlockScanner thread.");
}
}

  先看初始化部分，他会获得当前块报告，构建扫描信息类，获得块校验日志文件

private void init() {
// 获得块报告数组，并打乱顺序，以使块扫描不会总发生在同一个卷组上
Block arr[] = dataset.getBlockReport();
Collections.shuffle(Arrays.asList(arr));
//创建存放扫描信息的结构
blockInfoSet = new TreeSet<BlockScanInfo>();
blockMap = new HashMap<Block, BlockScanInfo>();
//填充扫描信息集合
long scanTime = -1;
for (Block block : arr) {
BlockScanInfo info = new BlockScanInfo( block );
info.lastScanTime = scanTime--;
//still keep 'info.lastScanType' to NONE.
addBlockInfo(info);
}

//获得扫描日志
File dir = null;
FSDataset.FSVolume[] volumes = dataset.volumes.volumes;
for(FSDataset.FSVolume vol : volumes) {
if (LogFileHandler.isFilePresent(vol.getDir(), verificationLogFile)) {
dir = vol.getDir();
break;
}
}
if (dir == null) {
dir = volumes[0].getDir();
}

try {
// 创建日志记录器
verificationLog = new LogFileHandler(dir, verificationLogFile, 100);
} catch (IOException e) {
LOG.warn("Could not open verfication log. " +
"Verification times are not stored.");
}

synchronized (this) {
//创建节流器
throttler = new BlockTransferThrottler(200, MAX_SCAN_RATE);
}
}

  下面看更新block校验时间的部分assignInitialVerificationTimes

private boolean assignInitialVerificationTimes() {
int numBlocks = 1;
synchronized (this) {
//获得块数量
numBlocks = Math.max(blockMap.size(), 1);
}

//构建日志读取器
LogFileHandler.Reader logReader = null;
try {
if (verificationLog != null) {
logReader = verificationLog.new Reader(false);
}
} catch (IOException e) {
LOG.warn("Could not read previous verification times : " +
StringUtils.stringifyException(e));
}
//循环读取日志，并更新当前行数
if (verificationLog != null) {
verificationLog.updateCurNumLines();
}

try {
// 根据校验日志更新块的校验时间
while (logReader != null && logReader.hasNext()) {
if (!datanode.shouldRun || Thread.interrupted()) {
return false;
}
//获取日志条目并更新
LogEntry entry = LogEntry.parseEntry(logReader.next());
if (entry != null) {
updateBlockInfo(entry);
}
}
} finally {
//更新完毕后关闭日志流
IOUtils.closeStream(logReader);
}

//两个块之间的校验间隔，避免频繁块扫描
long verifyInterval = (long) (Math.min( scanPeriod/2.0/numBlocks,
10*60*1000 ));
//最后校验时间
long lastScanTime = System.currentTimeMillis() - scanPeriod;

/* Before this loop, entries in blockInfoSet that are not
* updated above have lastScanTime of <= 0 . Loop until first entry has
* lastModificationTime > 0.
*/
synchronized (this) {
if (blockInfoSet.size() > 0 ) {
BlockScanInfo info;
//循环更新块校验时间和间隔
while ((info =  blockInfoSet.first()).lastScanTime < 0) {
delBlockInfo(info);
info.lastScanTime = lastScanTime;
lastScanTime += verifyInterval;
addBlockInfo(info);
}
}
}
return true;
}

下面看块校验部分，这也是blockscanner的主体部分，包含了校验逻辑，其实这个逻辑和上一篇dataXceiverServer中块读取的逻辑一样，只不过这里块数据读取后并不发送到客户端，而是发送到一个空流里，我们先看如何开始一个新的扫描周期

private synchronized void startNewPeriod() {
LOG.info("Starting a new period : work left in prev period : " +
String.format("%.2f%%", (bytesLeft * 100.0)/totalBytesToScan));
// reset the byte counts :
bytesLeft = totalBytesToScan; //记录需要扫描的字节数
currentPeriodStart = System.currentTimeMillis();//记录本周期的启动时间
}

块校验函数如下

private void verifyFirstBlock() {
Block block = null;
synchronized (this) {
if ( blockInfoSet.size() > 0 ) {
//获取一个block，所有blockinfo放在一个TreeSet中，所里内部是排序过的
block = blockInfoSet.first().block;
}
}

if ( block != null ) {
verifyBlock(block);//开始校验
}
}  

在blockSender.sendBlock函数中会走上一篇dataXceiverServer中一样的块读取逻辑

private void verifyBlock(Block block) {
BlockSender blockSender = null;
//安全起见，如果第一次校验失败则会再校验校验一次
for (int i=0; i<2; i++) {
boolean second = (i > 0);
try {
adjustThrottler();
//构建BlockSender
blockSender = new BlockSender(block, 0, -1, false,
false, true, datanode);
//构建空的数据输出流
DataOutputStream out =
new DataOutputStream(new IOUtils.NullOutputStream());
//发送一个块
blockSender.sendBlock(out, null, throttler);
//当块更新完毕后则记录日志
LOG.info((second ? "Second " : "") +
"Verification succeeded for " + block);

if ( second ) {
totalTransientErrors++;
}
//更新扫描状态
updateScanStatus(block, ScanType.VERIFICATION_SCAN, true);

return;
} catch (IOException e) {
........
} finally {
IOUtils.closeStream(blockSender);
datanode.getMetrics().incrBlocksVerified();
totalScans++;
totalVerifications++;
}
}
}

一旦块扫描完毕，就会更新扫描状态，通过函数updateScanStatus实现，内部调用updateScanStatusInternal

private synchronized void updateScanStatusInternal(Block block,
ScanType type, boolean scanOk, boolean updateOnly) {

if (!isInitialized()) {
return;
}
BlockScanInfo info = blockMap.get(block);

if ( info != null ) {
delBlockInfo(info);//删除旧的扫描信息
} else {
if (updateOnly) {
return;
}
// It might already be removed. Thats ok, it will be caught next time.
info = new BlockScanInfo(block);
}
//构建新的扫描信息
long now = System.currentTimeMillis();
info.lastScanType = type;
info.lastScanTime = now;
info.lastScanOk = scanOk;
addBlockInfo(info);

if (type == ScanType.REMOTE_READ) {
totalVerifications++;
}

// Don't update meta data too often in case of REMOTE_READ
// of if the verification failed.
long diff = now - info.lastLogTime;
if (!scanOk || (type == ScanType.REMOTE_READ &&
diff < scanPeriod/3 && diff < ONE_DAY)) {
return;
}

info.lastLogTime = now;
LogFileHandler log = verificationLog;
if (log != null) {
//记录扫描日志
log.appendLine(LogEntry.newEnry(block, now));
}
}

校验日志位于current目录下，文件名为dncp_block_verification.log.curr，校验日志格式如下：
date="2013-09-2316:49:22,824" 校验时间：日期时间格式，毫秒数

time="1379926162824"          与上面表示同一时间，此处为时间戳格式

genstamp="1002"               块标记

id="-8049298064173279059"     块ID

通过上面的信息就可以确定一个块精确到毫秒的校验时间，具体时间通过System.currentTimeMillis()来获得。