您的位置：首页 > Web前端 > HTML

htmlparser实现从网页上抓取数据（收集）

2012-07-09 10:26 357 查看

http://luoye1989hzx.blog.163.com/blog/static/1699218892010828364066/

本文引用自luoye1989《htmlparser实现从网页上抓取数据（收集）》

package parser;

import java.io.BufferedReader;

import java.io.BufferedWriter;

import java.io.FileWriter;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.net.MalformedURLException;

import java.net.URL;

/**

* 基本能实现网页抓取，不过要手动输入URL 将整个html内容保存到指定文件

*

* @author chenguoyong

*

*/

public class ScrubSelectedWeb {

private final static String CRLF = System.getProperty("line.separator");

/**

* @param args

*/

public static void main(String[] args) {

try {

URL ur = new URL("http://10.249.187.199:8083/injs100/");

InputStream instr = ur.openStream();

String s, str;

BufferedReader in = new BufferedReader(new InputStreamReader(instr));

StringBuffer sb = new StringBuffer();

BufferedWriter out = new BufferedWriter(new FileWriter(

"D:/outPut.txt"));

while ((s = in.readLine()) != null) {

sb.append(s + CRLF);

}

System.out.println(sb);

str = new String(sb);

out.write(str);

out.close();

in.close();

} catch (MalformedURLException e) {

e.printStackTrace();

} catch (IOException e) {

e.printStackTrace();

}

}

}

基本能实现网页抓取，不过要手动输入URL，此外没有重构。只是一个简单的思路。

1.htmlparser 使用

htmlparser是一个纯的java写的html解析的库，htmlparser不依赖于其它的java库，htmlparser主要用于改造或提取html。htmlparser能超高速解析html，而且不会出错。毫不夸张地说，htmlparser就是目前最好的html解析和分析的工具。无论你是想抓取网页数据还是改造html的内容，用了htmlparser绝对会忍不住称赞。由于htmlparser 结构设计精良，所以扩展htmlparser 非常便利。

Htmlparser中文论坛. http://bbs.hexiao.cn/thread.php?fid=6
Constructor Summary

Parser()

Parser(URLConnection connection)

Construct a parser using the provided URLConnection.

Method:

static Parser createParser(String html, String charset)

Creates the parser on an input string.

void visitAllNodesWith(NodeVisitor visitor)

Apply the given visitor to the current page.

HtmlPage(Parser parser)

NodeList

getBody()

TableTag[]

getTables()

String

getTitle()

void

setTitle(String title)

void

visitTag(Tag tag)

Called for each Tag visited.

Constructor Summary

NodeList()

NodeList(Node node)

Create a one element node list.

NodeList extractAllNodesThatMatch(NodeFilter filter)

Filter the list with the given filter non-recursively.

NodeList extractAllNodesThatMatch(NodeFilter filter, boolean recursive)

Filter the list with the given filter.

Node elementAt(int i)

1. html代码里面所有的链接地址和链接名称

package parser;

import org.htmlparser.Parser;

import org.htmlparser.Node;

import org.htmlparser.NodeFilter;

import org.htmlparser.Parser;

import org.htmlparser.filters.TagNameFilter;

import org.htmlparser.tags.LinkTag;

import org.htmlparser.tags.TableTag;

import org.htmlparser.util.NodeList;

import org.htmlparser.util.ParserException;

import org.htmlparser.visitors.HtmlPage;

/**

* htmlparser取得一段html代码里面所有的链接地址和链接名称

*

* @author chenguoyong

*

*/

public class Testhtmlparser {

/**

* @param args

*/

public static void main(String[] args) {

String htmlcode = "<HTML><HEAD><TITLE>AAA</TITLE></HEAD><BODY>"

+ "<a href='http://topic.csdn.net/u/20080522/14/0ff402ef-c382-499a-8213-ba6b2f550425.html'>连接1</a>"

+ "<a href='http://topic.csdn.net'>连接2</a></BODY></HTML>";

// 创建Parser对象根据传给字符串和指定的编码

Parser parser = Parser.createParser(htmlcode, "GBK");

// 创建HtmlPage对象HtmlPage(Parser parser)

HtmlPage page = new HtmlPage(parser);

try {

// HtmlPage extends visitor,Apply the given visitor to the current

// page.

parser.visitAllNodesWith(page);

} catch (ParserException e1) {

e1 = null;

}

// 所有的节点

NodeList nodelist = page.getBody();

// 建立一个节点filter用于过滤节点

NodeFilter filter = new TagNameFilter("A");

// 得到所有过滤后，想要的节点

nodelist = nodelist.extractAllNodesThatMatch(filter, true);

for (int i = 0; i < nodelist.size(); i++) {

LinkTag link = (LinkTag) nodelist.elementAt(i);

// 链接地址

System.out.println(link.getAttribute("href") + "\n");

// 链接名称

System.out.println(link.getStringText());

}

}

}

结果如下：
http://topic.csdn.net/u/20080522/14/0ff402ef-c382-499a-8213-ba6b2f550425.html
连接1
http://topic.csdn.net
连接2

2. 使用HtmlParser抓去网页内容

package parser;

import org.htmlparser.Parser;

import org.htmlparser.beans.StringBean;

import org.htmlparser.filters.NodeClassFilter;

import org.htmlparser.parserapplications.StringExtractor;

import org.htmlparser.tags.BodyTag;

import org.htmlparser.util.NodeList;

import org.htmlparser.util.ParserException;

/**

* 使用HtmlParser抓去网页内容: 要抓去页面的内容最方便的方法就是使用StringBean. 里面有几个控制页面内容的几个参数.

* 在后面的代码中会有说明. Htmlparser包中还有一个示例StringExtractor 里面有个直接得到内容的方法,

* 其中也是使用了StringBean . 另外直接解析Parser的每个标签也可以的.

*

* @author chenguoyong

*

*/

public class GetContent {

public void getContentUsingStringBean(String url) {

StringBean sb = new StringBean();

sb.setLinks(true); // 是否显示web页面的连接(Links)

// 为了取得页面的整洁美观一般设置上面两项为true , 如果要保持页面的原有格式, 如代码页面的空格缩进可以设置为false

sb.setCollapse(true); // 如果是true的话把一系列空白字符用一个字符替代.

sb.setReplaceNonBreakingSpaces(true);// If true regular space

sb

.setURL("http://www.blogjava.net/51AOP/archive/2006/07/19/59064.html");

System.out.println("The Content is :\n" + sb.getStrings());

}

public void getContentUsingStringExtractor(String url, boolean link) {

// StringExtractor内部机制和上面的一样.做了一下包装

StringExtractor se = new StringExtractor(url);

String text = null;

try {

text = se.extractStrings(link);

System.out.println("The content is :\n" + text);

} catch (ParserException e) {

e.printStackTrace();

}

}

public void getContentUsingParser(String url) {

NodeList nl;

try {

Parser p = new Parser(url);

nl = p.parse(new NodeClassFilter(BodyTag.class));

BodyTag bt = (BodyTag) nl.elementAt(0);

System.out.println(bt.toPlainTextString()); // 保留原来的内容格式. 包含js代码

} catch (ParserException e) {

e.printStackTrace();

}

}

/**

* @param args

*/

public static void main(String[] args) {

String url = "http://www.blogjava.net/51AOP/archive/2006/07/19/59064.html";

//new GetContent().getContentUsingParser(url);

//--------------------------------------------------

new GetContent().getContentUsingStringBean(url);

}

}

3.将整个html内容保存到指定文件

package parser;

import java.io.BufferedReader;

import java.io.BufferedWriter;

import java.io.FileWriter;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.net.MalformedURLException;

import java.net.URL;

/**

* 基本能实现网页抓取，不过要手动输入URL 将整个html内容保存到指定文件

*

* @author chenguoyong

*

*/

public class ScrubSelectedWeb {

private final static String CRLF = System.getProperty("line.separator");

/**

* @param args

*/

public static void main(String[] args) {

try {

URL ur = new URL("http://www.google.cn/");

InputStream instr = ur.openStream();

String s, str;

BufferedReader in = new BufferedReader(new InputStreamReader(instr));

StringBuffer sb = new StringBuffer();

BufferedWriter out = new BufferedWriter(new FileWriter(

"D:/outPut.txt"));

while ((s = in.readLine()) != null) {

sb.append(s + CRLF);

}

System.out.println(sb);

str = new String(sb);

out.write(str);

out.close();

in.close();

} catch (MalformedURLException e) {

e.printStackTrace();

} catch (IOException e) {

e.printStackTrace();

}

}

}

4利用htmlparser提取网页纯文本的例子

package parser;

import org.htmlparser.Node;

import org.htmlparser.NodeFilter;

import org.htmlparser.Parser;

import org.htmlparser.filters.TagNameFilter;

import org.htmlparser.tags.TableTag;

import org.htmlparser.util.NodeList;

/**

* 标题:利用htmlparser提取网页纯文本的例子

*/

public class TestHTMLParser2 {

/**

* 读取目标html内容

*

*/

public static void testHtml() {

try {

String sCurrentLine;

String sTotalString;

sCurrentLine = "";

sTotalString = "";

java.io.InputStream l_urlStream;

java.net.URL l_url = new java.net.URL(

"http://10.249.187.199:8083/injs100/");

java.net.HttpURLConnection l_connection = (java.net.HttpURLConnection) l_url

.openConnection();

l_connection.connect();

l_urlStream = l_connection.getInputStream();

java.io.BufferedReader l_reader = new java.io.BufferedReader(

new java.io.InputStreamReader(l_urlStream));

while ((sCurrentLine = l_reader.readLine()) != null) {

sTotalString += sCurrentLine + "\r\n";

}

String testText = extractText(sTotalString);

} catch (Exception e) {

e.printStackTrace();

}

}

/**

* 抽取纯文本信息

* @param inputHtml：html文本

* @return

* @throws Exception

*/

public static String extractText(String inputHtml) throws Exception {

StringBuffer text = new StringBuffer();

Parser parser = Parser.createParser(new String(inputHtml.getBytes(),

"GBK"), "GBK");

// 遍历所有的节点

NodeList nodes = parser.extractAllNodesThatMatch(new NodeFilter() {

public boolean accept(Node node) {

return true;

}

});

System.out.println(nodes.size());

for (int i = 0; i < nodes.size(); i++) {

Node nodet = nodes.elementAt(i);

//字符串的代表性节点:节点的描述

text.append(new String(nodet.toPlainTextString().getBytes("GBK"))

+ "\r\n");

}

return text.toString();

}

/**

* 读取文件的方式/utl 来分析内容. filePath也可以是一个Url.

* @param resource :文件/Url

* @throws Exception

*/

public static void test5(String resource) throws Exception {

Parser myParser = new Parser(resource);

myParser.setEncoding("GBK");

String filterStr = "table";

NodeFilter filter = new TagNameFilter(filterStr);

NodeList nodeList = myParser.extractAllNodesThatMatch(filter);

/*for(int i=0;i<nodeList.size();i++)

{

TableTag tabletag = (TableTag) nodeList.elementAt(i);

//标签名称

System.out.println(tabletag.getTagName());

System.out.println(tabletag.getText());

}*/

TableTag tabletag = (TableTag) nodeList.elementAt(1);

}

public static void main(String[] args) throws Exception {

test5("http://10.249.187.199:8083/injs100/");

//testHtml();

}

}

5.html解析table

package parser;

import org.apache.log4j.Logger;

import org.htmlparser.NodeFilter;

import org.htmlparser.Parser;

import org.htmlparser.filters.NodeClassFilter;

import org.htmlparser.filters.OrFilter;

import org.htmlparser.filters.TagNameFilter;

import org.htmlparser.tags.TableColumn;

import org.htmlparser.tags.TableRow;

import org.htmlparser.tags.TableTag;

import org.htmlparser.util.NodeList;

import org.htmlparser.util.ParserException;

import junit.framework.TestCase;

public class ParserTestCase extends TestCase {

private static final Logger logger = Logger.getLogger(ParserTestCase.class);

public ParserTestCase(String name) {

super(name);

}

/**

* 测试对<table>

* <tr>

* <td></td>

* </tr>

* </table>的解析

*/

public void testTable() {

Parser myParser;

NodeList nodeList = null;

myParser = Parser

.createParser(

"<body> "

+ "<table id=’table1′ >"

+ "<tr id='tro1'><td>1-11</td><td>1-12</td><td>1-13</td></tr>"

+ "<tr id='tro2'><td>1-21</td><td>1-22</td><td>1-23</td></tr>"

+ "<tr id='tro3'><td>1-31</td><td>1-32</td><td>1-33</td></tr></table>"

+ "<table id=’table2′ >"

+ "<tr id='tro4'><td>2-11</td><td>2-12</td><td>2-13</td></tr>"

+ "<tr id='tro5'><td>2-21</td><td>2-22</td><td>2-23</td></tr>"

+ "<tr id='tro6'><td>2-31</td><td>2-32</td><td>2-33</td></tr></table>"

+ "</body>", "GBK");

NodeFilter tableFilter = new NodeClassFilter(TableTag.class);

OrFilter lastFilter = new OrFilter();

lastFilter.setPredicates(new NodeFilter[] { tableFilter });

try {

nodeList = myParser.parse(lastFilter);

for (int i = 0; i <= nodeList.size(); i++) {

if (nodeList.elementAt(i) instanceof TableTag) {

TableTag tag = (TableTag) nodeList.elementAt(i);

TableRow[] rows = tag.getRows();

for (int j = 0; j < rows.length; j++) {

TableRow tr = (TableRow) rows[j];

System.out.println(tr.getAttribute("id"));

if (tr.getAttribute("id").equalsIgnoreCase("tro1")) {

TableColumn[] td = tr.getColumns();

for (int k = 0; k < td.length; k++) {

// logger.fatal("<td>" +

// td[k].toPlainTextString());

System.out.println("<td>"

+ td[k].toPlainTextString());

}

}

}

}

}

} catch (ParserException e) {

e.printStackTrace();

}

}

/**

* 得到目标数据

*

* @param url:目标url

* @throws Exception

*/

public static void getDatabyUrl(String url) throws Exception {

Parser myParser = new Parser(url);

NodeList nodeList = null;

myParser.setEncoding("gb2312");

NodeFilter tableFilter = new NodeClassFilter(TableTag.class);

OrFilter lastFilter = new OrFilter();

lastFilter.setPredicates(new NodeFilter[] { tableFilter });

try {

nodeList = myParser.parse(lastFilter);

// 可以从数据table的size:19-21开始到结束

for (int i = 15; i <= nodeList.size(); i++) {

if (nodeList.elementAt(i) instanceof TableTag) {

TableTag tag = (TableTag) nodeList.elementAt(i);

TableRow[] rows = tag.getRows();

for (int j = 0; j < rows.length; j++) {

TableRow tr = (TableRow) rows[j];

if (tr.getAttribute("id") != null

&& tr.getAttribute("id").equalsIgnoreCase(

"tr02")) {

TableColumn[] td = tr.getColumns();

// 对不起，没有你要查询的记录！

if (td.length == 1) {

System.out.println("对不起，没有你要查询的记录");

} else {

for (int k = 0; k < td.length; k++) {

System.out.println("<td>内容："

+ td[k].toPlainTextString().trim());

}

}

}

}

}

}

} catch (ParserException e) {

e.printStackTrace();

}

}

/**

* 测试已经得出有数据时table:22个，没有数据时table:19个

*

* @param args

*/

public static void main(String[] args) {

try {

// getDatabyUrl("http://gd.12530.com/user/querytonebytype.do?field=tonecode&condition=619505000000008942&type=1006&pkValue=619505000000008942");

getDatabyUrl("http://gd.12530.com/user/querytonebytype.do?field=tonecode&condition=619272000000001712&type=1006&pkValue=619272000000001712");

} catch (Exception e) {

e.printStackTrace();

}

}

}

6.html解析常用

package com.jscud.test;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileInputStream;

import java.io.InputStreamReader;

import org.htmlparser.Node;

import org.htmlparser.NodeFilter;

import org.htmlparser.Parser;

import org.htmlparser.filters.NodeClassFilter;

import org.htmlparser.filters.OrFilter;

import org.htmlparser.nodes.TextNode;

import org.htmlparser.tags.LinkTag;

import org.htmlparser.util.NodeList;

import org.htmlparser.util.ParserException;

import org.htmlparser.visitors.HtmlPage;

import org.htmlparser.visitors.TextExtractingVisitor;

import com.jscud.util.LogMan; //一个日志记录类

/**

* 演示了Html Parse的应用.

*

* @author scud http://www.jscud.com (http://www.jscud.com/)

*/

public class ParseHtmlTest

{

public static void main(String[] args) throws Exception

{

String aFile = "e:/jscud/temp/test.htm";

String content = readTextFile(aFile, "GBK");

test1(content);

System.out.println("====================================");

test2(content);

System.out.println("====================================");

test3(content);

System.out.println("====================================");

test4(content);

System.out.println("====================================");

test5(aFile);

System.out.println("====================================");

//访问外部资源,相对慢

test5("http://www.jscud.com (http://www.jscud.com/)");

System.out.println("====================================");

}

/**

* 读取文件的方式来分析内容.

* filePath也可以是一个Url.

*

* @param resource 文件/Url

*/

public static void test5(String resource) throws Exception

{

Parser myParser = new Parser(resource);

//设置编码

myParser.setEncoding("GBK");

HtmlPage visitor = new HtmlPage(myParser);

myParser.visitAllNodesWith(visitor);

String textInPage = visitor.getTitle();

System.out.println(textInPage);

}

/**

* 按页面方式处理.对一个标准的Html页面,推荐使用此种方式.

*/

public static void test4(String content) throws Exception

{

Parser myParser;

myParser = Parser.createParser(content, "GBK");

HtmlPage visitor = new HtmlPage(myParser);

myParser.visitAllNodesWith(visitor);

String textInPage = visitor.getTitle();

System.out.println(textInPage);

}

/**

* 利用Visitor模式解析html页面.

*

* 小优点:翻译了<>等符号

* 缺点:好多空格,无法提取link

*

*/

public static void test3(String content) throws Exception

{

Parser myParser;

myParser = Parser.createParser(content, "GBK");

TextExtractingVisitor visitor = new TextExtractingVisitor();

myParser.visitAllNodesWith(visitor);

String textInPage = visitor.getExtractedText();

System.out.println(textInPage);

}

/**

* 得到普通文本和链接的内容.

*

* 使用了过滤条件.

*/

public static void test2(String content) throws ParserException

{

Parser myParser;

NodeList nodeList = null;

myParser = Parser.createParser(content, "GBK");

NodeFilter textFilter = new NodeClassFilter(TextNode.class);

NodeFilter linkFilter = new NodeClassFilter(LinkTag.class);

//暂时不处理 meta

//NodeFilter metaFilter = new NodeClassFilter(MetaTag.class);

OrFilter lastFilter = new OrFilter();

lastFilter.setPredicates(new NodeFilter[] { textFilter, linkFilter });

nodeList = myParser.parse(lastFilter);

Node[] nodes = nodeList.toNodeArray();

for (int i = 0; i < nodes.length; i++)

{

Node anode = (Node) nodes[i];

String line = "";

if (anode instanceof TextNode)

{

TextNode textnode = (TextNode) anode;

//line = textnode.toPlainTextString().trim();

line = textnode.getText();

}

else if (anode instanceof LinkTag)

{

LinkTag linknode = (LinkTag) anode;

line = linknode.getLink();

//@todo ("") 过滤jsp标签:可以自己实现这个函数

//line = StringFunc.replace(line, "<%.*%>", "");

}

if (isTrimEmpty(line))

continue;

System.out.println(line);

}

}

/**

* 解析普通文本节点.

*

* @param content

* @throws ParserException

*/

public static void test1(String content) throws ParserException

{

Parser myParser;

Node[] nodes = null;

myParser = Parser.createParser(content, null);

nodes = myParser.extractAllNodesThatAre(TextNode.class); //exception could be thrown here

for (int i = 0; i < nodes.length; i++)

{

TextNode textnode = (TextNode) nodes[i];

String line = textnode.toPlainTextString().trim();

if (line.equals(""))

continue;

System.out.println(line);

}

}

/**

* 读取一个文件到字符串里.

*

* @param sFileName 文件名

* @param sEncode String

* @return 文件内容

*/

public static String readTextFile(String sFileName, String sEncode)

{

StringBuffer sbStr = new StringBuffer();

try

{

File ff = new File(sFileName);

InputStreamReader read = new InputStreamReader(new FileInputStream(ff),

sEncode);

BufferedReader ins = new BufferedReader(read);

String dataLine = "";

while (null != (dataLine = ins.readLine()))

{

sbStr.append(dataLine);

sbStr.append("\r\n");

}

ins.close();

}

catch (Exception e)

{

LogMan.error("read Text File Error", e);

}

return sbStr.toString();

}

/**

* 去掉左右空格后字符串是否为空

* @param astr String

* @return boolean

*/

public static boolean isTrimEmpty(String astr)

{

if ((null == astr) || (astr.length() == 0))

{

return true;

}

if (isBlank(astr.trim()))

{

return true;

}

return false;

}

/**

* 字符串是否为空:null或者长度为0.

* @param astr 源字符串.

* @return boolean

*/

public static boolean isBlank(String astr)

{

if ((null == astr) || (astr.length() == 0))

{

return true;

}

else

{

return false;

}

}

}

内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理

标签：

相关文章推荐

新的分享

章节导航