NutchBean详解之初始化

NutchBean是实现nutch查询的一个入口，就像Nutch中的crawl一样。在NutchBean的mian函数中，以下一行代码实现了初始化：

final NutchBean bean = new NutchBean(conf);

这行代码涉及到的东西很多，包括初始化了LuceneSearchBean、IndexSearcher、FetchedSegments以及lucene中的查询类IndexSearcher(注意：前一个IndexSearcher是nutch的，而后一个是Lucene的)。

代码展示如下：

final NutchBeanbean = new NutchBean(conf);

public NutchBean(Configuration conf) throws IOException {

this(conf, null);

}

public NutchBean(Configuration conf, Path dir) throws IOException {

。。。。。。//省略

searchBean = new LuceneSearchBean(conf, indexDir, indexesDir);

。。。。。。//省略

segmentBean = new FetchedSegments(conf, segments);

}

初始化LuceneSearchBena的代码：

publicLuceneSearchBean(Configuration conf, Path indexDir, Path indexesDir)

throws IOException {

this.conf = conf;

this.fs =FileSystem.get(this.conf);

init(indexDir, indexesDir);

}

privatevoid init(Path indexDir, Path indexesDir)

throws IOException {

Path absIndexDir =indexDir.makeQualified(indexDir.getFileSystem(conf));

Path absIndexesDir =indexesDir.makeQualified(indexesDir.getFileSystem(conf));

//TODO

Path indexDirAddtion=newPath("E:/out/index");

Path[] indexes=new Path[2];

if (this.fs.exists(indexDir)) {

LOG.info("opening mergedindex in " + absIndexDir.toUri());

//TODO

indexes[0]=indexDir;

indexes[1]=indexDirAddtion;

//this.searcher = newIndexSearcher(indexDir, this.conf);

//TODO

/*此处做了修改，将IndexSearcher中的参数Path，改为Path[],用于

* 对多个索引路径进行查询*/

this.searcher=newIndexSearcher(indexes,this.conf);

} else {

if (!this.fs.exists(indexesDir)){

// should throw exception ?

LOG.warn("Neither "+ absIndexDir.toUri() + " nor " +

absIndexesDir.toUri()+ " found!");

} else {

LOG.info("openingindexes in " + absIndexesDir.toUri());

}

List<Path> vDirs = newArrayList<Path>();

FileStatus[] fstats =fs.listStatus(indexesDir, HadoopFSUtil.getPassDirectoriesFilter(fs));

Path[] directories =HadoopFSUtil.getPaths(fstats);

for(int i = 0; i <directories.length; i++) {

Path indexdone = newPath(directories[i], Indexer.DONE_NAME);

if(fs.isFile(indexdone)) {

vDirs.add(directories[i]);

}

}

directories = new Path[vDirs.size() ];

for(int i = 0;vDirs.size()>0; i++) {

directories[i] =vDirs.remove(0);

}

this.searcher = newIndexSearcher(directories, this.conf);

}

}

IndexSearcher的代码如下：

public IndexSearcher(Path[] indexDirs, Configuration conf) throwsIOException {

IndexReader[] readers = newIndexReader[indexDirs.length];

this.conf = conf;

this.fs = FileSystem.get(conf);

for (int i = 0; i <indexDirs.length; i++) {

readers[i] =IndexReader.open(getDirectory(indexDirs[i]));

}

init(new MultiReader(readers),conf);

}

private void init(IndexReader reader, Configuration conf) throwsIOException {

this.reader = reader;

this.luceneSearcher = new

org.apache.lucene.search.IndexSearcher(reader);

this.luceneSearcher.setSimilarity(new NutchSimilarity());

this.optimizer = newLuceneQueryOptimizer(conf);

this.queryFilters = newQueryFilters(conf);

}

初始化luceneIndexSearcher

privateIndexSearcher(IndexReader r, boolean closeReader) {

reader = r;

this.closeReader = closeReader;

List<IndexReader> subReadersList =new ArrayList<IndexReader>();

gatherSubReaders(subReadersList, reader);

subReaders = subReadersList.toArray(newIndexReader[subReadersList.size()]);

docStarts = new int[subReaders.length];

int maxDoc = 0;

for (int i = 0; i < subReaders.length;i++) {

docStarts[i] = maxDoc;

maxDoc += subReaders[i].maxDoc();

}

}

初始化segmentBean

public FetchedSegments(Configurationconf, Path[] segmentsDir)

throws IOException {

this.conf = conf;

this.fs = FileSystem.get(this.conf);

final FileStatus[] fstats = fs.listStatus(segmentsDir,

HadoopFSUtil.getPassDirectoriesFilter(fs));

final Path[] segmentDirs = HadoopFSUtil.getPaths(fstats);

this.summarizer = new SummarizerFactory(this.conf).getSummarizer();

this.segmentsDir = segmentsDir;

this.segUpdater = new SegmentUpdater();

if (segmentDirs != null) {

for (final Path segmentDir : segmentDirs){

segments.put(segmentDir.getName(),

new Segment(this.fs, segmentDir, this.conf));

}

}

this.segUpdater.start();

}

注意：在实现跨索引查询的时候，除了修改luceneSearchBean中的初始化代码，还得修改sgementBean中的代码。