HTMLPARSER 爬取 html网页 获取标题 关键字 内容 url

package com.bonc.pure.util;

import java.awt.List;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileInputStream;

import java.io.InputStreamReader;

import java.sql.Blob;

import java.sql.Clob;

import java.util.Date;

import org.hibernate.Hibernate;

import org.htmlparser.Node;

import org.htmlparser.NodeFilter;

import org.htmlparser.Parser;

import org.htmlparser.beans.StringBean;

import org.htmlparser.filters.NodeClassFilter;

import org.htmlparser.filters.TagNameFilter;

import org.htmlparser.tags.BaseHrefTag;

import org.htmlparser.tags.LinkTag;

import org.htmlparser.tags.MetaTag;

import org.htmlparser.tags.TitleTag;

import org.htmlparser.util.NodeList;

import org.htmlparser.util.ParserException;

import org.htmlparser.visitors.TextExtractingVisitor;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

import org.springframework.web.util.HtmlUtils;

import com.bonc.pure.util1.HtmlText;

/**

* 解析HTML工具类

* <p>Title:ZolParser </p>

* @author 施海洲

* @date 2015-10-29上午11:27:21

*/

public class ZolParser {

private static Logger log = LoggerFactory.getLogger(ZolParser.class);

HtmlText htmlText=new HtmlText();

String path ="D://bonc_test_bin//黑客hackjd.com.htm";

public HtmlText test() throws Exception

{

/** 开始时间 */

long now = System.currentTimeMillis();

// 获取源代码

String htmlCode = setHtmlCode();

/** 获取标题 */

String ss=getTitle(htmlCode);

htmlText.setHtml_Title(ss);

/** 获取关键字 */

htmlText.setHtml_Keyword(getkeywords(htmlCode));

/** 获取源URL */

htmlText.setHtml_Url(geturl(htmlCode));

/**上传时间*/

Date time=new Date();

htmlText.setHtml_date(time);

/**获取内容*/

String s= getPlainText(htmlCode);

@SuppressWarnings("deprecation")

Clob clob = (Clob) Hibernate.createBlob(s.getBytes("GBK"));

// htmlText.setHtml_Content(clob);

/**获取文件名*/

htmlText.setHtml_name(getName());

/** 结束时间 */

long current = System.currentTimeMillis();

// 消耗时间

log.info("全文消耗时间：" + (current - now));

return htmlText;

}

/**

* 获取HTML代码

*

* @return

* @throws Exception

*/

public String setHtmlCode() throws Exception

{

/** 通过HttpParser解析器解析HTML */

StringBuffer abstr = new StringBuffer();

BufferedReader reader = new BufferedReader(new InputStreamReader( new FileInputStream(path), "GBK"));

String temp="";

while((temp=reader.readLine())!=null){

abstr.append(temp);

abstr.append("\r\n");

}

String result =abstr.toString(); // 在这里得到文件中所有的字符串

//Parser parsers = Parser.createParser(result, "GB2312");// creat a parser ,which you want to parse.

Parser parsers = new Parser(result);

// 设置编码

parsers.setEncoding("utf-8");

// 设置标签过滤器

NodeFilter filter = new TagNameFilter("html");

// 过滤标签

NodeList nList = parsers.extractAllNodesThatMatch(filter);

// 返回字符串

return nList.toHtml();

}

/**

* 获取文件名

* @param

* @return fileName

* @throws ParserException

*/

public String getName() throws ParserException

{

File tempFile =new File( path.trim());

String fileName = tempFile.getName();

System.out.println("fileName = " + fileName);

return fileName;

}

/**

* 获取标题信息

*

* @param htmlCode

* @return

* @throws ParserException

*/

public String getTitle(String htmlCode) throws ParserException

{

Parser parser = new Parser();

parser.setInputHTML(htmlCode);

List list=new List();

/* 配置过滤器 */

NodeFilter newFilter = new NodeClassFilter() {

public boolean accept(Node node) {

if (node instanceof TitleTag) {

TitleTag mt = (TitleTag) node;

if (mt.getTagName() != null) {

return true;

} else {

return false;

}

}

return false;

}

};

String title=null;

NodeList Title = parser.extractAllNodesThatMatch(newFilter);

for (int i = 0; i < Title.size(); i++) {

if (Title.elementAt(i) instanceof TitleTag) {

TitleTag tt = (TitleTag) Title.elementAt(i);

title=tt.getTitle();

}

}

return title;

}

/**

* 获取关键字信息

*

* @param htmlCode

* @throws ParserException

*/

public String getkeywords(String htmlCode) throws ParserException

{

Parser parser = new Parser();

parser.setInputHTML(htmlCode);

List list=new List();

// Map<String, String> map = new HashMap<String, String>();

/* 配置过滤器 */

NodeFilter newFilter = new NodeClassFilter() { // 将Meta下面的keywords和description过滤出来

public boolean accept(Node node) {

if (node instanceof MetaTag) {

MetaTag mt = (MetaTag) node;

if (mt.getMetaTagName() != null) {

return true;

} else {

return false;

}

} else if(node instanceof TitleTag ) {

return true;

}

return false;

}

};

String t=null;

NodeList keywords = parser.extractAllNodesThatMatch(newFilter);

for (int i = 0; i < keywords.size(); i++) {

if (keywords.elementAt(i) instanceof TitleTag) {

TitleTag tt = (TitleTag) keywords.elementAt(i);

} else {

MetaTag mt = (MetaTag) keywords.elementAt(i);

if (mt.getMetaTagName().equals("keywords") | mt.getMetaTagName().equals("Keywords")) {

log.info("keywords:"+ mt.getMetaContent());

t=mt.getMetaContent();

}

}

}

return t;

}

/**

*获取URL

*

* @param htmlCode

* @throws ParserException

*/

@SuppressWarnings("serial")

public String geturl(String htmlCode) throws ParserException

{

Parser parser = new Parser();

parser.setInputHTML(htmlCode);

/* 配置过滤器 */

NodeFilter newFilter = new NodeClassFilter() {

public boolean accept(Node node) {

if (node instanceof BaseHrefTag) {

BaseHrefTag mt = (BaseHrefTag) node;

if (mt.getTagName()!= null) {

return true;

} else {

return false;

}

}

return false;

}

};

String url=null;

NodeList URL = parser.extractAllNodesThatMatch(newFilter);

for (int i = 0; i < URL.size(); i++) {

if (URL.elementAt(i) instanceof LinkTag) {

LinkTag tag = (LinkTag) URL.elementAt(i);

url= HtmlUtils.htmlUnescape(tag.getLink());

}

}

return url;

}

/**

* 获取纯文本信息

*

* @param htmlCode

* @return

* @throws Exception */

public String getPlainText(String htmlCode) throws Exception {

Parser parser = new Parser();

parser.setInputHTML(htmlCode);

StringBean sb = new StringBean();

TextExtractingVisitor visitor = new TextExtractingVisitor();

// 设置不需要得到页面所包含的链接信息

sb.setLinks(false);

// 设置将不间断空格由正规空格所替代

sb.setReplaceNonBreakingSpaces(true);

// 设置将一序列空格由一个单一空格所代替

sb.setCollapse(true);

parser.visitAllNodesWith(visitor);

String html = visitor.getExtractedText();

return html;

}

}