基于Lucene的文件检索Demo

通过Lucene实现了简单的文件检索功能的Demo。这个Demo支持基于文件内容的检索，支持中文分词和高亮显示。

下面简单的介绍下核心的类

1）索引相关的类

1.FileIndexBuilder ---建立索引

package uap.pub.bap.fs.search.indexer;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.wltea.analyzer.lucene.IKAnalyzer;

import uap.pub.bap.fs.search.util.FileSearchUtils;

/**
* 文件索引生成器
*
* @author chenfeic
*
*/
public class FileIndexBuilder {

/**
* 需要索引的文件列表
*/
private List<File> fileList = new ArrayList<File>();

private IndexWriter writer;

/**
*
* @param fileDir
*            文件位置
* @param indexDir
*            索引位置
*/
public void generateIndexer(String fileDir, String indexDir) {
if (StringUtils.isEmpty(indexDir) || StringUtils.isEmpty(fileDir)) {
System.out.println("文件和索引路径都不能为空");
throw new RuntimeException("文件和索引路径都不能为空");
}
Directory d = null;
try {
// 初始化IndexWriter
d = FSDirectory.open(new File(indexDir));
initWriter(indexDir, d);
// 创建索引文档
initIndex(fileDir);
System.out.println("索引创建成功！");
} catch (Exception e) {
System.out.println("创建索引失败");
System.out.println(e);
} finally {
FileSearchUtils.closeIndexWriter(writer);
FileSearchUtils.closeDirectory(d);
}
}

/**
* 初始化 Lucene Index Writer 步骤1： Directory创建索引存放的位置 步骤2：创建分析器Analyzer
* 步骤3：创建IndexWriterConfig,使用分析器Analyzer 步骤4：创建IndexWriter
*
* @param indexDir
* @param directory
* @throws IOException
*/
private void initWriter(String indexDir, Directory directory)
throws IOException {
Analyzer analyzer = new IKAnalyzer();
IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_46,
analyzer);
conf.setOpenMode(OpenMode.CREATE_OR_APPEND);
writer = new IndexWriter(directory, conf);
}

/**
* 初始化索引文档
*
* @param fileDir
*            文档目录
* @return
*/
private int initIndex(String fileDir) {
getAllSubFile(new File(fileDir));
TextFileFilter fileter = new TextFileFilter();
for (File file : fileList) {
if (fileter.accept(file)) {
try {
DocumentBuilder db = new DocumentBuilder(file);
Document doc = db.createDocument();
writer.addDocument(doc);
} catch (FileNotFoundException e) {
System.out.println("创建索引失败，文件不存在:" + e.getMessage());
} catch (IOException e) {
e.printStackTrace();
}
}
}
return writer.numDocs();

}

private void getAllSubFile(File file) {
File[] listFiles = file.listFiles();
if (ArrayUtils.isEmpty(listFiles)) {
return;
}
for (File subfile : listFiles) {
if (subfile.isDirectory()) {
getAllSubFile(subfile);
} else {
fileList.add(subfile);
}
}
}

public static void main(String[] args) {
String fileDir = "E:\\lucene\\data";
String indexDir = "E:\\lucene\\index";
FileIndexBuilder indexer = new FileIndexBuilder();
indexer.generateIndexer(fileDir, indexDir);
}

}

2. DocumentBuilder --索引内容生成器

package uap.pub.bap.fs.search.indexer;

import java.io.File;
import java.io.IOException;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;

import uap.pub.bap.fs.search.IFileSearchConst;
import uap.pub.bap.fs.search.util.FileSearchUtils;

/**
* Document生成器
*
* @author chenfeic
*
*/
public class DocumentBuilder {

private File file = null;

private IContextConverter contextConverter;

public DocumentBuilder(File file) {
this.file = file;
initConverter();
}

/**
* 初始化内容转换器
*/
private void initConverter() {
String fileType = FileSearchUtils.getFileType(file.getName());
// 1.word
if ("docx".equalsIgnoreCase(fileType)
|| "doc".equalsIgnoreCase(fileType)) {
contextConverter = new WordContextConverter();
}
// 2. excel
else if ("xlsx".equalsIgnoreCase(fileType)
|| "xls".equalsIgnoreCase(fileType)) {
contextConverter = new ExcelContextConverter();
}
// 3.pdf
else if ("pdf".equalsIgnoreCase(fileType)) {
contextConverter = new PdfContextConverter();
}
// 4.txt(log)
else {
contextConverter = new TextContextConverter();
}
}

public Document createDocument() {
if (file == null || !file.exists()) {
return null;
}
Document doc = new Document();
try {
doc.add(new TextField(IFileSearchConst.CONTENT_TYPE,
contextConverter.context2String(file), Field.Store.YES));
doc.add(new StringField(IFileSearchConst.FILENAM_TYPE, file
.getName(), Field.Store.YES));
doc.add(new StringField(IFileSearchConst.PATH_TYPE, file
.getCanonicalPath(), Field.Store.YES));
} catch (IOException e) {
e.printStackTrace();
}
return doc;
}
}

3： IContextConverter--文件内容转换器接口，将文件内容装换为字符串

package uap.pub.bap.fs.search.indexer;

import java.io.File;

/**
* 内容转换器，将文本内容转换成字符串
*
* @author chenfeic
*
*/
public interface IContextConverter {

/**
* 文件内容转换成字符串
*
* @param file 文件
* @return
*/
public String context2String(File file);

}

4：AbstractContextConverter--这个类主要利用第三方开源包cpdetector获取文件编码格式

package uap.pub.bap.fs.search.indexer;

import info.monitorenter.cpdetector.io.ASCIIDetector;
import info.monitorenter.cpdetector.io.CodepageDetectorProxy;
import info.monitorenter.cpdetector.io.JChardetFacade;
import info.monitorenter.cpdetector.io.ParsingDetector;
import info.monitorenter.cpdetector.io.UnicodeDetector;

import java.io.File;

public abstract class AbstractContextConverter implements IContextConverter {

/**
* 利用第三方开源包cpdetector获取文件编码格式
*
* @param path
*            要判断文件编码格式的源文件的路径
* @author huanglei
* @version 2012-7-12 14:05
*/
protected String getFileEncode(String path) {
/*
* detector是探测器，它把探测任务交给具体的探测实现类的实例完成。
* cpDetector内置了一些常用的探测实现类，这些探测实现类的实例可以通过add方法 加进来，如ParsingDetector、
* JChardetFacade、ASCIIDetector、UnicodeDetector。
* detector按照“谁最先返回非空的探测结果，就以该结果为准”的原则返回探测到的
* 字符集编码。使用需要用到三个第三方JAR包：antlr.jar、chardet.jar和cpdetector.jar
* cpDetector是基于统计学原理的，不保证完全正确。
*/
CodepageDetectorProxy detector = CodepageDetectorProxy.getInstance();
/*
* ParsingDetector可用于检查HTML、XML等文件或字符流的编码,构造方法中的参数用于
* 指示是否显示探测过程的详细信息，为false不显示。
*/
detector.add(new ParsingDetector(false));
/*
* JChardetFacade封装了由Mozilla组织提供的JChardet，它可以完成大多数文件的编码
* 测定。所以，一般有了这个探测器就可满足大多数项目的要求，如果你还不放心，可以
* 再多加几个探测器，比如下面的ASCIIDetector、UnicodeDetector等。
*/
detector.add(JChardetFacade.getInstance());// 用到antlr.jar、chardet.jar
// ASCIIDetector用于ASCII编码测定
detector.add(ASCIIDetector.getInstance());
// UnicodeDetector用于Unicode家族编码的测定
detector.add(UnicodeDetector.getInstance());
java.nio.charset.Charset charset = null;
File f = new File(path);
try {
charset = detector.detectCodepage(f.toURI().toURL());
} catch (Exception ex) {
ex.printStackTrace();
}
if (charset != null)
return charset.name();
else
return null;
}

}

4.TextContextConverter

package uap.pub.bap.fs.search.indexer;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;

import uap.pub.bap.fs.search.util.FileSearchUtils;

/**
* txt、配置文件、log等文本的Document生成器
*
* @author chenfeic
*
*/
public class TextContextConverter extends AbstractContextConverter {

@Override
public String context2String(File file) {
StringBuilder sb = new StringBuilder();
BufferedReader reader = null;
InputStream in = null;
try {
String encoding = getFileEncode(file.getCanonicalPath());
in = new FileInputStream(file);
if (encoding != null && !"".equals(encoding.trim())) {
reader = new BufferedReader(new InputStreamReader(in, encoding));
} else {
reader = new BufferedReader(new InputStreamReader(in));
}
// 将输入流写入输出流
String line = "";
while ((line = reader.readLine()) != null) {
sb.append(line + "\n");
}
} catch (Exception e) {
e.printStackTrace();
} finally {
FileSearchUtils.closeInputStream(in);
FileSearchUtils.closeReader(reader);
}
return sb.toString();
}

}

下面的两个类主要是读取excel、word等office 办公软件的内容，用到的第三方插件为poi

5.WordContextConverter

package uap.pub.bap.fs.search.indexer;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;

import org.apache.poi.POIXMLDocument;
import org.apache.poi.hwpf.extractor.WordExtractor;
import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
import org.apache.xmlbeans.XmlException;

import uap.pub.bap.fs.search.util.FileSearchUtils;

/**
* word文档内容转换器
*
* @author chenfeic
*
*/
public class WordContextConverter extends AbstractContextConverter {

@Override
public String context2String(File file) {
if (isWord2003(file)) {
return readWord2003(file);
} else {
return readWord2007(file);
}
}

/**
* 判断是否是Word 97(-2003)版本
*
* @param file
* @return
*/
private boolean isWord2003(File file) {
String fileType = FileSearchUtils.getFileType(file.getName());
return "doc".equalsIgnoreCase(fileType);
}

/**
* 读取Word 97(-2003)文件内容
*
* @param file
* @return
*/
private String readWord2003(File file) {
InputStream inputStream = null;
String context = null;
try {
inputStream = new FileInputStream(file);
WordExtractor extractor = new WordExtractor(inputStream);
context = extractor.getText();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
FileSearchUtils.closeInputStream(inputStream);
}
return context;
}

private String readWord2007(File file) {
String text = null;
OPCPackage openPackage = null;
try {
// 得到.docx文件提取器
openPackage = POIXMLDocument.openPackage(file.getCanonicalPath());
XWPFWordExtractor docx = new XWPFWordExtractor(openPackage);
// 提取.docx正文文本
text = docx.getText();
} catch (IOException e) {
e.printStackTrace();
} catch (XmlException e) {
e.printStackTrace();
} catch (OpenXML4JException e) {
e.printStackTrace();
} finally {
if (openPackage != null) {
try {
openPackage.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return text;
}

}

6.ExcelContextConverter

package uap.pub.bap.fs.search.indexer;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;

import org.apache.poi.hssf.extractor.ExcelExtractor;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFRow;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;

import uap.pub.bap.fs.search.util.FileSearchUtils;

/**
* excel内容转换器
*
* @author chenfeic
*
*/
public class ExcelContextConverter extends AbstractContextConverter {

@Override
public String context2String(File file) {
if (isExcel2003(file)) {
return readExcel2003(file);
} else {
return readExcel2007(file);
}
}

/**
* 判断是否是Excel 97(-2003)版本
*
* @param file
* @return
*/
private boolean isExcel2003(File file) {
String fileType = FileSearchUtils.getFileType(file.getName());
return "xls".equalsIgnoreCase(fileType);
}

public String readExcel2003(File file) {
InputStream inputStream = null;
String content = null;
try {
inputStream = new FileInputStream(file.getCanonicalPath());
HSSFWorkbook wb = new HSSFWorkbook(inputStream);
ExcelExtractor extractor = new ExcelExtractor(wb);
extractor.setFormulasNotResults(true);
extractor.setIncludeSheetNames(false);
content = extractor.getText();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
FileSearchUtils.closeInputStream(inputStream);
}
return content;
}

public String readExcel2007(File file) {
StringBuffer content = new StringBuffer();
InputStream inputStream = null;
try {
inputStream = new FileInputStream(file.getCanonicalPath());
XSSFWorkbook xwb = new XSSFWorkbook(inputStream);
// 循环工作表Sheet
for (int numSheet = 0; numSheet < xwb.getNumberOfSheets(); numSheet++) {
XSSFSheet xSheet = xwb.getSheetAt(numSheet);
if (xSheet == null) {
continue;
}
// 循环行Row
for (int rowNum = 0; rowNum <= xSheet.getLastRowNum(); rowNum++) {
XSSFRow xRow = xSheet.getRow(rowNum);
if (xRow == null) {
continue;
}
// 循环列Cell
for (int cellNum = 0; cellNum <= xRow.getLastCellNum(); cellNum++) {
XSSFCell xCell = xRow.getCell(cellNum);
if (xCell == null) {
continue;
}
if (xCell.getCellType() == XSSFCell.CELL_TYPE_BOOLEAN) {
content.append(xCell.getBooleanCellValue());
} else if (xCell.getCellType() == XSSFCell.CELL_TYPE_NUMERIC) {
content.append(xCell.getNumericCellValue());
} else {
content.append(xCell.getStringCellValue());
}
}
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
FileSearchUtils.closeInputStream(inputStream);
}

return content.toString();
}
}

2）检索相关的类

FileSearchServiceImpl --查询关键字

package uap.pub.bap.fs.search.service;

import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang.StringUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexNotFoundException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.wltea.analyzer.lucene.IKAnalyzer;

import uap.pub.bap.fs.search.IFileSearchConst;
import uap.pub.bap.fs.search.SearchResult;
import uap.pub.bap.fs.search.util.FileSearchUtils;

public class FileSearchServiceImpl implements IFileSerachService {

private int count = 0;

@Override

public List<SearchResult> search(String type, String key) {
//type:查询的类型---标题，文件内容等
//key:查询关键字
List<SearchResult> results = new ArrayList<SearchResult>();
if (StringUtils.isEmpty(key)) {
return results;
}
// TODO chenfeic
String indexDir = "E:\\lucene\\index";
IndexReader reader = null;
Directory directory = null;
try {
directory = FSDirectory.open(new File(indexDir));
reader = DirectoryReader.open(directory);
IndexSearcher search = new IndexSearcher(reader);
// 使用QueryParser查询分析器构造Query对象
Analyzer analyzer = new IKAnalyzer();
QueryParser qp = new QueryParser(Version.LUCENE_46, type, analyzer);
qp.setDefaultOperator(QueryParser.AND_OPERATOR);
Query query = qp.parse(key);
// 一个简单的指针容器，指向前N个排名的搜索结果
TopDocs hits = search.search(query, null, 100);
count = hits.totalHits;
for (ScoreDoc soreDoc : hits.scoreDocs) {
Document doc = search.doc(soreDoc.doc);
String summary = toHighlighter(query, doc,
IFileSearchConst.CONTENT_TYPE, analyzer);
String title = doc.get(IFileSearchConst.FILENAM_TYPE);
String path = doc.get(IFileSearchConst.PATH_TYPE);
SearchResult result = new SearchResult();
result.setPath(path);
result.setTitle(title);
if (!StringUtils.isEmpty(summary)) {
result.setSummary(summary);
}
results.add(result);
}

} catch (IndexNotFoundException e1) {
System.out.println("无查询结果,没有此词条的索引");
}catch (IOException e) {
System.out.println("无查询结果！");
e.printStackTrace();
} catch (ParseException e) {
e.printStackTrace();
} finally {
FileSearchUtils.closeIndexReader(reader);
FileSearchUtils.closeDirectory(directory);
}
return results;
}

/**
* 高亮显示
*
* @param query
* @param doc
* @param field
* @return
*/
private String toHighlighter(Query query, Document doc, String field,
Analyzer analyzer) {
try {
SimpleHTMLFormatter simpleHtmlFormatter = new SimpleHTMLFormatter(
"<font color=\"red\">", "</font>");
Highlighter highlighter = new Highlighter(simpleHtmlFormatter,
new QueryScorer(query));
// highlighter.setTextFragmenter(new
// SimpleFragmenter(20));//显示20个字符，默认是100个
TokenStream tokenStream = analyzer.tokenStream(field,
new StringReader(doc.get(field)));
String highlighterStr = highlighter.getBestFragment(tokenStream,
doc.get(field));
return highlighterStr == null ? doc.get(field) : highlighterStr;
} catch (IOException e) {
e.printStackTrace();

} catch (InvalidTokenOffsetsException e) {
e.printStackTrace();
}
return null;
}

@Override
public int getCount() {
return this.count;
}

}

上述基本上就是此Demo的核心类，其他的一些工具类和jsp、servlet处理类就没多写了

基本上用的都是第三方的开源工具，比如支持中文分词的IK_Analyzer(版本IK Analyzer 2012FF_hf1)(注：开始的时候想要的是paoding作为中文的分词器，用户之后发现paoding不支持Lucene4.0版本，估计3.X就已经不支持了，原因是因为坑跌的Lucene总是变化的他的实现和结构，使得有些方法变为final这样paoding中重写了此方法，导致编译出错)，为了支持基于文件内容的检索，所以需要对文件内容进行索引并保存，所以用到了poi用于对ms office的处理，对于txt等读取时通过cpdetector检测文件的编码格式。代码都可以参照上面。代码有些是自己写的，有些是参照网上其他同仁的，再次一并谢过

相关jar包列表为