您的位置：首页 > Web前端 > HTML

使用HtmlAgilityPack批量抓取网页数据

2011-01-07 17:31 453 查看

转：/article/4787733.html

登录的处理。因为有些网页数据需要登陆后才能提取。这里要使用ieHTTPHeaders来提取登录时的提交信息。

抓取网页

HtmlAgilityPack.HtmlDocument htmlDoc;

if (!string.IsNullOrEmpty(登录URL))
else

ArrayList list = new ArrayList();
list.add("//table/tr[1]/td");
list.add("//table/tr[2]/td");
//获取循环的节点的xpath，比如：//table/tr
HtmlNodeCollection repeatNodes = htmlDoc.DocumentNode.SelectNodes("//table/tr");

//循环节点
foreach (HtmlNode node in repeatNodes)
htmlWeb.DefaultEncoding = System.Text.Encoding.GetEncoding(strEncode);
-------------------------------------------------------------------------------------------

using System;

using System.Collections.Generic;

using System.Text;

using Microsoft.VisualStudio.TestTools.WebTesting;

using HtmlAgilityPack;

public class WebTest1Coded : WebTest

{

public override IEnumerator<WebTestRequest> GetRequestEnumerator()

{

WebTestRequest request1 = new WebTestRequest("http://www.microsoft.com/");

request1.ValidateResponse += new EventHandler<ValidationEventArgs>(request1_ValidateResponse);

yield return request1;

}

void request1_ValidateResponse(object sender, ValidationEventArgs e)

{

//load the response body string as an HtmlAgilityPack.HtmlDocument

HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();

doc.LoadHtml(e.Response.BodyString);

//locate the "Nav" element

HtmlNode navNode = doc.GetElementbyId("Nav");

//pick the first <li> element

HtmlNode firstNavItemNode = navNode.SelectSingleNode(".//li");

//validate the first list item in the Nav element says "Windows"

e.IsValid = firstNavItemNode.InnerText == "Windows";

}

}

if ($ != jQuery) {
$ = jQuery.noConflict();
}

内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理

标签：

相关文章推荐

新的分享

章节导航