C#程序抓取网页实例

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Text;
using System.Windows.Forms;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Collections;
namespace CopyHtml
{
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
}
private void button1_Click(object sender, EventArgs e)
{
//获取指定网页中的源数据
string rl;
WebRequest Request = WebRequest.Create(textBox1.Text.Trim());
WebResponse Response = Request.GetResponse();
Stream resStream = Response.GetResponseStream();
StreamReader sr = new StreamReader(resStream, Encoding.Default);
StringBuilder sb = new StringBuilder();
while ((rl = sr.ReadLine()) != null)
{
sb.Append(rl);
}
textBox2.Text = sb.ToString();//抓取得到的源网页
string he = textBox2.Text.ToString();
textBox3.Text = stripHtml(he);//去除html标签后得到的源网页
Match TitleMatch = Regex.Match(he, "<title>([^<]*)</title>", RegexOptions.IgnoreCase | RegexOptions.Multiline);//获取网页的标题
string title = TitleMatch.Groups[1].Value;
textBox4.Text = ("网页的标题是：" + title );
}
/// <summary>
/// 去掉网页中的html标签
/// </summary>
/// <param name="strHtml">待转化的字符串</param>
/// <returns></returns>
private string stripHtml(string strHtml)
{
Regex objRegExp = new Regex("<(.|/n)+?>");
string strOutput = objRegExp.Replace(strHtml, "");
strOutput = strOutput.Replace("<", "<");
strOutput = strOutput.Replace(">", ">");
return strOutput;
}
// 提取HTML代码中的网址
public static ArrayList GetHyperLinks(string htmlCode)
{
ArrayList al = new ArrayList();
string strRegex = @"(href)[ ]*=[ ]*[""'][^""'#>]+[""']";
Regex r = new Regex(strRegex, RegexOptions.IgnoreCase);
MatchCollection m = r.Matches(htmlCode);
for (int i = 0; i <= m.Count - 1; i++)
{
bool rep = false;
string strNew = m[i].ToString();
// 过滤重复的URL
foreach (string str in al)
{
if (strNew == str)
{
rep = true;
break;
}
}
if (!rep) al.Add(strNew);
}
al.Sort();
return al;
}
}
}