搜索引擎luence之目录索引

/*
* Created on 2004-11-20
*
* index a dir file
*/
package demo;

/**使用lucence生成目录索引
*
*/
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import java.io.BufferedReader;
import java.io.FileReader;

import java.io.File;

public class DirSpider {
private String indexDir;
private String sSourceDir;
//控制显示配置参数
private boolean verbose;
//控制是否增量索引
private boolean incremental;

private IndexWriter index;
//从主函数的参数中得到所需配置信息
public static void main(String[] args) {
try {
DirSpider s = new DirSpider(args);
s.go();
} catch (Exception e) {
e.printStackTrace();
}
}

public DirSpider(String argv[]) throws Exception  {
verbose = false;
incremental = false;

for (int i = 0; i < argv.length; i++) {
if (argv[i].equals("-s"))
sSourceDir = argv[++i];
if(argv[i].equals("-d"))
indexDir = argv[++i];
else if(argv[i].equals("-v"))
verbose = true;
else if(argv[i].equals("-a"))
incremental = true;
}

if (sSourceDir == null)
throw new IllegalArgumentException("Missing required argument: -s [SourceDir dir]");

if (indexDir == null)
throw new IllegalArgumentException("Missing required argument: -d [index dir]");
}

public void go() throws Exception  {
long start = System.currentTimeMillis();

// create the index directory -- or append to existing
if (verbose) {
System.out.println("Creating index in: " + indexDir);
if (incremental) System.out.println("    - using incremental mode");
}
index = new IndexWriter(new File(indexDir), new StandardAnalyzer(),
!incremental);

File dir = new File(sSourceDir);

indexDir(dir);

index.optimize();
index.close();
if(verbose)
System.out.println("index complete in :"+(System.currentTimeMillis() - start)/1000);
}

private void indexDir(File dir)
{
File[] files = dir.listFiles();

for (int i = 0; i < files.length; i++) {
File f = files[i];
if (f.isDirectory()) {
indexDir(f);  // recurse 递归调用
} else if (f.getName().endsWith(".txt")) {//现在只对txt文本文件索引
indexFile(f);
}
}
}

private void indexFile(File item) {
if (verbose) System.out.println("Adding FILE: " + item);

News news = loadFile(item);

if ( news!= null && news.body != null) {
Document doc = new Document();
Field f = new Field("url", news.URL ,
Field.Store.YES, Field.Index.UN_TOKENIZED,
Field.TermVector.NO);
doc.add(f);

f = new Field("title", news.title ,
Field.Store.YES, Field.Index.TOKENIZED,
Field.TermVector.WITH_POSITIONS_OFFSETS);
doc.add(f);

f = new Field("content", news.body.toString() ,
Field.Store.YES, Field.Index.TOKENIZED,
Field.TermVector.WITH_POSITIONS_OFFSETS);
doc.add(f);
System.out.println(news);
//要清晰异常是什么,该怎样去处理。
try{
index.addDocument(doc);
}
catch(Exception e)
{
e.printStackTrace();
//System.exit(0)和System.exit(1)分别表示正常退出和异常退出
System.exit(-1);
}
}else{
System.out.println("索引数据为空!");
}
}

private static News loadFile(File sSourceFile){
News news = new News();

try
{	//注意附加URL的方式
//news.URL = "http://www.lietu.com/segtest/"+sSourceFile;
news.URL="http://localhost:8080/Chapter2WebPart/"+sSourceFile;

BufferedReader br = new BufferedReader(new FileReader(sSourceFile));
String s;

if ( (s = br.readLine()) != null )
{
news.title = s;
System.out.println(s);
while( (s = br.readLine()) != null ) {
news.body.append(s);
news.body.append('\n');
}
}
br.close();
}
catch (Exception e)
{
e.printStackTrace();
}

return news;
}
}
class News {
public String URL;
public String title;
public StringBuffer body;

public News(){
this.URL = "";
this.title = "";
this.body = new StringBuffer();
}

public String toString(){
return "URL :"+URL+" title :" + title +" body :"+ body.toString() ;
}
}

package com.lietu.web.test;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;

import com.lietu.web.bean.example.Article;

/**
* 根据luence索引搜索数据
*
*/
public class MySearche {

public MySearche() {
super();
}

public static void main(String[] args) {

new MySearche().indexSeaches("中");
}
// 索引查询
public Collection indexSeaches(String str) {
System.out.println("str:" + str);
Collection collections = new ArrayList();
try {
// 创建搜索对象
IndexSearcher indexSearcher = new IndexSearcher("C:/index");
// 创建索引读取对象
IndexReader reader = IndexReader.open("C:/index");
Query titleQuery = null;
// 创建查询分析器，对File属性title进行查询,采用的分析器是StandardAnalyzer
QueryParser queryparser = new QueryParser("title",
new StandardAnalyzer());
try {
// 分析用户输入的字符串
Query query = queryparser.parse(str);
// 根据用户的输入开始搜索
Hits hit = indexSearcher.search(query);
System.out.println("hit.length():" + hit.length());
for (int i = 0; i < hit.length(); i++) {
Article article = new Article();
// 得到title
article.setTitle(hit.doc(i).get("title"));
// 得到content
article.setContent(hit.doc(i).get("content"));
// 把document添加到集合中,并且返回
collections.add(article);

System.out.println("标题：" + hit.doc(i).get("title"));
System.out.println("内容:" + hit.doc(i).get("content"));
}
} catch (ParseException e) {

e.printStackTrace();
}
} catch (CorruptIndexException e) {

} catch (IOException e) {

e.printStackTrace();
}
return collections;
}
}