webcrawler-----自制爬虫搜索种子和电影（jsoup）

文章来源：开源中国社区http://www.oschina.net/code/snippet_778875_48198

自制的爬虫可以抓取网上的电影和种子，设置爬虫的深度足够可以爬取所有的资源。这里以163开始并无大碍，以任何一个好点的网站开始都是可以的。

需要的jar包：
IKAnalyzer2012FF_u1.jar
jsoup-1.8.2.jar
lucene-core-4.10.2.jar
lucene-highlighter-5.1.0.jar
目录中生成的文件，如下图所示：



把目录中生成的文件全部导入到sor4.7.2中利用solr的web管理界面可以进行检索，检索结果如下图所示：



java代码如下所示：

package tzj;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Stack;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.QueryBuilder;
import org.apache.lucene.util.Version;
import org.jsoup.Connection;
import org.jsoup.Connection.Response;
import org.jsoup.helper.HttpConnection;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.wltea.analyzer.lucene.IKAnalyzer;

public class PTest {

File dirc=new File("dirc2");

//当前层的URL
Stack<String[]> stackcurt=new Stack<String[]>();

//下一层的URL和名称
Stack<String[]> stacknext=new Stack<String[]>();

//控制递归层数
final int deep=6;

Analyzer analyzer=new IKAnalyzer();

FSDirectory directory;

IndexWriterConfig config;

IndexWriter indexWriter;

final String userAgent="Mozilla/5.0 (Windows NT 6.1; WOW64) "
+ "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.101 Safari/537.36";

public PTest(){
try {
directory=FSDirectory.open(dirc);
config=new IndexWriterConfig(Version.LATEST, analyzer);
indexWriter=new IndexWriter(directory, config);
config.setMaxBufferedDocs(10);
} catch (IOException e) {
e.printStackTrace();
}
}

public static void main(String[] args) throws IOException{

PTest pTest=new PTest();
String starturl="http://www.163.com";
pTest.startWork(starturl);

System.out.println(pTest.search("妻子"));
//      System.out.println(
//      pTest.isIndexed("http://www.9amhg.com/?intr=806"));

}

/** 使用方式并不是递归得到URL，而是逐层解析。
* 得到页面内容
* @param url
* @param deep
*/
public void getPageContent(String[] urlname){

Connection connection=HttpConnection.connect(urlname[0]);
connection.ignoreContentType(true);
connection.ignoreHttpErrors(true);
connection.userAgent(userAgent);
connection.referrer(urlname[2]);

org.jsoup.nodes.Document document=null;
Response response=null;
boolean store=false;

//如果本身就是资源就直接索引
if(isResource(urlname[0])){
store=true;
}else
{

try {
response=connection.execute();
//如果这个链接返回的是视频就直接索引
if(isMP4(response.contentType()))
store=true;

else
{
document = response.parse();
}
} catch (IOException e) {
e.printStackTrace();
}

}

try {
indexPageContent(urlname[0],urlname[1],store);
} catch (IOException e) {
e.printStackTrace();
}

if(document!=null){
Elements elements=document.getElementsByTag("a");

Iterator<Element> iterator=elements.iterator();

while(iterator.hasNext()){
Element element=iterator.next();
String attrurl=element.attr("href");
attrurl=processUrl(attrurl, urlname[0]);
try {
if(!isIndexed(attrurl))
stacknext.push(new String[]{attrurl,element.text(),urlname[0]});
} catch (IOException e) {
e.printStackTrace();
}

}
}

}
/**
* 还原url链接
* @param url
* @param parenturl
* @return
*/
public String processUrl(String url,String parenturl){

Pattern pattern_host=Pattern.compile("http://\\w+\\.\\w+\\.\\w+");

Matcher matcher_host=pattern_host.matcher(parenturl);
String host=null;

if(matcher_host.find())
host=matcher_host.group();

if(host==null)
return url;

if(url.startsWith("/"))
return host+url;
else if(url.startsWith("../")){

while(url.startsWith("../")){
url=url.substring(3);
parenturl=parenturl.substring(0, parenturl.lastIndexOf("/")-1);
parenturl=parenturl.substring(0, parenturl.lastIndexOf("/"));
}

return parenturl+"/"+url;
}

return url;
}

/**
* 判断返回类型是不是视频
* @param contenttype
* @return
*/
public boolean isMP4(String contenttype){

if(contenttype.startsWith("video")){
return true;
}

return false;
}

/**
* 判断这个链接是不是资源
* @param url
* @return
*/
public boolean isResource(String url){

//添加资源识别的种类
if(url.endsWith(".mp4")
||url.endsWith(".torrent")){
return true;
}

return false;

}

/**
* 开始工作
* @param urls
*/
public void startWork(String...urls){

int deep=1;

for (int i = 0; i < urls.length; i++) {
stackcurt.push(new String[]{urls[i],"",""});
}

while(this.deep>deep)
{
if(stackcurt.isEmpty())
{
stackcurt=stacknext;
stacknext=new Stack<String[]>();
deep++;
}

if(stackcurt.isEmpty())
break;

try {
getPageContent(stackcurt.pop());
} catch (RuntimeException e) {
//e.printStackTrace();
}
}

}

/**
* 索引链接
* @param url
* @throws IOException
*/
public void indexPageContent(String url,String title,boolean store) throws IOException{

System.out.println("索引"+url+"\t"+title);
Document document=new Document();
document.add(new TextField("url",url,Store.YES));
document.add(new TextField("md5code",md5(url),Store.YES));

document.add(new TextField("title", title,Store.YES));

indexWriter.addDocument(document);

indexWriter.commit();

}
/**
* 关键字分组
* @param content
* @return
*/
public boolean setGroup(String content){

return false;
}

/**
* 判断URL是不是已经被索引过了
* @param url
* @return
* @throws IOException
*/
public boolean isIndexed(String url) throws IOException{

if(!directory.fileExists("segments.gen"))
return false;

IndexReader indexReader= DirectoryReader.open(directory);

IndexSearcher indexSearcher=new IndexSearcher(indexReader);
TermQuery query=new TermQuery(new Term("md5code", md5(url)));

TopDocs docs= indexSearcher.search(query, 1);

return docs.scoreDocs.length>0?true:false;
}

/**
* 查询
* @param keyword
* @return
* @throws IOException
*/
public List<String> search(String keyword) throws IOException{

IndexSearcher indexSearcher=new IndexSearcher(DirectoryReader.open(directory));

Query booleanQuery=
new QueryBuilder(analyzer).createBooleanQuery("title", keyword);

TopDocs docs=indexSearcher.search(booleanQuery, 10);

List<String> urls=new ArrayList<String>();

Highlighter highlighter=new Highlighter(new SimpleHTMLFormatter(),new QueryScorer(booleanQuery));

ScoreDoc[] docss=docs.scoreDocs;
if(docs!=null&&docss.length>0){
for (int i = 0; i < docss.length; i++) {
Document document=indexSearcher.doc(docss[i].doc);
urls.add(document.get("url"));

TokenStream tokenStream = analyzer.tokenStream("title", new StringReader(document.get("title")));
try {
String str = highlighter.getBestFragment(tokenStream,  document.get("title"));
System.out.println(str);
} catch (InvalidTokenOffsetsException e) {
e.printStackTrace();
}
System.out.println("\t"+document.get("url")+"\t"+document.get("title")+"\t");

System.out.println("\n");
}
}

return urls;
}

/**
* MD5 加密
* @param plainText
* @return
*/
public static String md5(String plainText) {
try {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(plainText.getBytes());
byte b[] = md.digest();

int i;

StringBuffer buf = new StringBuffer();
for (int offset = 0; offset < b.length; offset++) {
i = b[offset];
if (i < 0)
i += 256;
if (i < 16)
buf.append("0");
buf.append(Integer.toHexString(i));
}

return buf.toString().substring(8, 24);
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
return  null;
}
}