使用paoding lucene分词和网页爬虫实现的简易网页搜索

1、首先配置好paoding分词，配置过程参照：
http://zyj177484.blog.163.com/blog/static/18379423120127614414110/
2、配置好分词器后，会自动将输入的文字进行分词，如搜索百科全书，会自动分为 百科 全书两个词，分别在网页中查找这两个词

3、利用一个简单的网页爬虫，就可以进行关键词的搜索啦！

package com.boco.paoding;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.net.HttpURLConnection;
import java.net.URL;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;

import net.paoding.analysis.analyzer.PaodingAnalyzer;

/**
* 利用paoding lucene分词和网页爬虫实现的简易搜索
* @author dup
*
*/
public class PaodingAnalyzer1 {

/**
* @param args
* @throws IOException
*/
@SuppressWarnings("deprecation")
public static void main(String[] args) throws IOException {
new PaodingAnalyzer1().paodingAnalysis("http://www.baidu.com");
}

/**
* 分词
* @param url
*/
public void paodingAnalysis(String url){
Analyzer analyzer = new PaodingAnalyzer();
String  indexStr = "百科全书"; //分词器会分为百科、全书两个词，并在网页中分别查找
StringReader reader = new StringReader(indexStr);
TokenStream ts = analyzer.tokenStream(indexStr, reader);
Token t;
try {
t = ts.next();
while(t != null) {
System.out.print(t.termText()+"  ");
String webHtml = new PaodingAnalyzer1().methodPa(url);
System.out.println(webHtml.trim().contains(t.termText()));
t = ts.next();
}
}catch(IOException e){
e.printStackTrace();
}
}

/**
* 解析html
* @param strURL
* @return
*/
public String methodPa(String strURL){
String temp;
String temps="";
try{
URL url = new URL(strURL);
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
InputStreamReader isr = new InputStreamReader(conn.getInputStream(),"utf-8");
BufferedReader br = new BufferedReader(isr);
while((temp = br.readLine()) != null){
System.out.println(temp);
temps+=temp+" ";
}
br.close();
isr.close();
}catch(Exception e){
e.printStackTrace();
}
return temps;
}

}

jar包依赖：

paoding-analysis.jar

lucene-core-2.4.0.jar

可去我博客资源中进行下载，顺便赚几个积分哈哈，都是白菜价！