走进lucene - 创建索引、检索

看过王老师的信息检索导论，一直想学lucene，看看从实现的角度，搜索引擎是如何工作的。

正好工作中用到一点lucene，尝试使用，为后续分析做好准备。

截止到现在，lucene已经更新到4.8了，适配Java8，并做了很多封装。

自己会用3.6来进行学习。因为阅读lucene源码还是4版本一下的更为合适，而且一些简单查询，3.6也足够用了。

1 构建索引

public boolean indexBuild(String indexPath, String inputFile) {
boolean suc = true;
try {
Directory dir = FSDirectory.open(new File(indexPath));

Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_36);
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_36,
analyzer);

boolean create = true;
if (create) {
// Create a new index in the directory,
// removing any previously indexed documents:
iwc.setOpenMode(OpenMode.CREATE);
} else {
// Add new documents to an existing index:
iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
}

// Optional: for better indexing performance, if you
// are indexing many documents, increase the RAM
// buffer. But if you do this, increase the max heap
// size to the JVM (eg add -Xmx512m or -Xmx1g):
// iwc.setRAMBufferSizeMB(256.0);

IndexWriter writer = new IndexWriter(dir, iwc);
FileInputStream fis = new FileInputStream(inputFile);

// make a new, empty document
Document doc = new Document();
// Add the path of the file as a field named "path". Use a
// field that is indexed (i.e. searchable), but don't tokenize
// the field into separate words and don't index term frequency
// or positional information:
BufferedReader reader = new BufferedReader(new InputStreamReader(
fis, "UTF-8"));
String line = null;
while ((line = reader.readLine()) != null) {
int beginPos = line.indexOf("\t");
String hid = line.substring(0, beginPos);
String tags = line.substring(beginPos+1);
Field hidField = new Field("hid", hid, Field.Store.YES,
Field.Index.NOT_ANALYZED_NO_NORMS);
hidField.setIndexOptions(IndexOptions.DOCS_ONLY);
doc.add(hidField);

Field tagsField = new Field("tags", tags, Field.Store.YES,
Field.Index.NOT_ANALYZED_NO_NORMS);
tagsField.setIndexOptions(IndexOptions.DOCS_ONLY);
doc.add(tagsField);

if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
// New index, so we just add the document (no old document
// can be there):
System.out.println("adding " + inputFile);
writer.addDocument(doc);
} else {
// Existing index (an old copy of this document may have
// been indexed) so
// we use updateDocument instead to replace the old one
// matching the exact
// path, if present:
writer.updateDocument(new Term("path", inputFile), doc);
}
}
writer.close();
} catch (IOException e) {
e.printStackTrace();
}
return suc;
}

2 检索

最开始的时候使用QueryParser生成Query，发现不需要分析的字段，检索不出结果来。改为有Term生成Query就OK了。

private IndexReader reader = null;
private Analyzer analyzer = null;
private IndexSearcher searcher = null;

public boolean init(String indexPath) {
boolean suc = true;
try {
reader = IndexReader.open(FSDirectory.open(new File(indexPath)));
searcher = new IndexSearcher(reader);
analyzer = new StandardAnalyzer(Version.LUCENE_36);
} catch (CorruptIndexException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}

return suc;
}

public String queryTags(String hid) {
String result = null;
//		QueryParser parser = new QueryParser(Version.LUCENE_36, "hid", analyzer);
try {
Term term = new Term("hid", hid);
Query query = new TermQuery(term);

int hitsPerPage = 1;
// Collect enough docs to show 5 pages
TopDocs results = searcher.search(query, 5 * hitsPerPage);
ScoreDoc[] hits = results.scoreDocs;

int numTotalHits = results.totalHits;
System.out.println(numTotalHits + " total matching documents");

String str;
int start = 0;
// end pos for return docs
int end = Math.min(numTotalHits, hitsPerPage);

boolean raw = false;
for (int i = start; i < end; i++) {
if (raw) { // output raw format
String log = "doc=" + hits[i].doc +	" score=" + hits[i].score;
System.out.println(log);
continue;
}

Document doc = searcher.doc(hits[i].doc);
String tags = doc.get("tags");
if (tags != null) {
//just for work
result = tags;
break;
//System.out.println((i + 1) + ". " + tags);
} else {
String log = (i + 1) + " No tags for " + hid;
System.out.println(log);
}
}
} catch (CorruptIndexException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
return result;
}

public void close() {
try {
searcher.close();
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}