封装一个 HttpWebRequest 和 HttpWebResponse 常用的操作类

using System;
using System.Text;
using System.Net;
using System.IO;
using System.Globalization;
using System.Collections.Specialized;
using System.Web;

namespace DearBruce.WebRequestDemo.CoreLib
{
/// <summary>
/// WebClient 帮助器
/// </summary>
public class HttpClientHelper
{
/// <summary>
/// 创建默认的
/// </summary>
/// <param name="url">请求的地址</param>
/// <returns></returns>
public static HttpWebRequest CreateHttpWebRequest(string url)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = FakeUserAgentRepository.InternetExplorer8.UserAgent;
request.AllowAutoRedirect = true;
//request.Method = "GET";
request.KeepAlive = true;
request.Referer = null;
request.Timeout = 100000; // 默认 100 秒
request.Headers["Accept-Language"] = CultureInfo.CreateSpecificCulture("zh-CN").Name;

return request;
}

/// <summary>
/// 向 HttpWebRequest 请求流写入 POST 数据
/// </summary>
/// <param name="request">HttpWebRequest 对象</param>
/// <param name="postVariables">提交的数据。比如：UserName=bruce&Password=123456 。</param>
/// <param name="encoding">数据编码，可为 null。如果为 null ，则默认 UTF8 编码 </param>
public static void WritePostData(HttpWebRequest request, NameValueCollection postVariables, Encoding encoding)
{
string postData = GetHttpPostVars(postVariables);
WritePostData(request, postData, encoding);
}

/// <summary>
/// 向 HttpWebRequest 请求流写入 POST 数据
/// </summary>
/// <param name="request">HttpWebRequest 对象</param>
/// <param name="postData">提交的数据。比如：UserName=bruce&Password=123456 。注意：如果 IsNullOrEmpty 则不会向 HttpWebRequest 请求流中写入数据</param>
/// <param name="encoding">数据编码，可为 null。如果为 null ，则默认 UTF8 编码 </param>
public static void WritePostData(HttpWebRequest request, string postData, Encoding encoding)
{
if (request == null)
{
throw new ArgumentNullException("request");
}
if (string.IsNullOrEmpty(postData))
{
return;
}
if (encoding == null)
{
encoding = Encoding.UTF8;
}
if (request.Method != "POST")
{
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded; charset=" + encoding.WebName;
}
using (BinaryWriter bw = new BinaryWriter(request.GetRequestStream()))
{
bw.Write(encoding.GetBytes(postData));
bw.Flush();
}
}

/// <summary>
/// 从 NameValueCollection 中得到 PostData
/// </summary>
/// <param name="variables">集合</param>
/// <returns>PostData</returns>
public static string GetHttpPostVars(NameValueCollection variables)
{
StringBuilder builder = new StringBuilder();
for (int i = 0; i < variables.Count; i++)
{
string key = variables.GetKey(i);
string[] values = variables.GetValues(i);
if (values != null)
{
foreach (string value in values)
{
builder.AppendFormat("{0}={1}", HttpUtility.UrlEncode(key), HttpUtility.UrlEncode(value));
}
}
if (i < variables.Count - 1)
{
builder.Append("&");
}
}
return builder.ToString();
}

/// <summary>
/// 根据 HttpWebRequest 请求对象得到响应的 Html 代码
/// </summary>
/// <param name="request">HttpWebRequest 对象</param>
/// <returns></returns>
public static string GetResponseText(HttpWebRequest request)
{
if (request == null)
{
throw new ArgumentNullException("request");
}
HttpWebResponse response = null;

try
{
response = (HttpWebResponse)request.GetResponse();
}
catch (WebException wex)
{
if (wex.Response == null)
{
return wex.Message;
}
response = (HttpWebResponse)wex.Response;
}
using (response)
{
Stream strem = response.GetResponseStream();
using (StreamReader reader = new StreamReader(strem, Encoding.GetEncoding(response.CharacterSet)))
{
return reader.ReadToEnd();
}
}
}

/// <summary>
/// 发送 Get 请求，得到 Html 代码
/// </summary>
/// <param name="url">请求地址</param>
/// <param name="cookieContainer">Cookie 容器，可为 null。响应时远程服务器返回的 Cookie 也自动包含其中</param>
/// <returns>Html 代码</returns>
public static string SendHttpGet(string url, CookieContainer cookieContainer)
{
HttpWebRequest request = CreateHttpWebRequest(url);

if (cookieContainer != null)
{
request.CookieContainer = cookieContainer;
}
return GetResponseText(request);
}

/// <summary>
/// 发送 POST 请求，得到 Html 代码
/// </summary>
/// <param name="url">请求地址</param>
/// <param name="postData">提交的数据。</param>
/// <param name="cookieContainer">Cookie 容器，可为 null。响应时远程服务器返回的 Cookie 也自动包含其中</param>
/// <returns>Html 代码</returns>
public static string SendHttpPost(string url, string postData, CookieContainer cookieContainer)
{
HttpWebRequest request = CreateHttpWebRequest(url);

if (cookieContainer != null)
{
request.CookieContainer = cookieContainer;
}
WritePostData(request, postData, Encoding.UTF8);

return GetResponseText(request);
}

/// <summary>
/// 发送 POST 请求，得到 Html 代码
/// </summary>
/// <param name="url">请求地址</param>
/// <param name="postVariables">提交的数据。</param>
/// <param name="cookieContainer">Cookie 容器，可为 null。响应时远程服务器返回的 Cookie 也自动包含其中</param>
/// <returns>Html 代码</returns>
public static string SendHttpPost(string url, NameValueCollection postVariables, CookieContainer cookieContainer)
{
string postData = GetHttpPostVars(postVariables);
return SendHttpPost(url, postData, cookieContainer);
}
}
}

public class FakeUserAgentRepository
{
// Fields
public static readonly FakeUserAgent Chrome =
new FakeUserAgent("Chrome", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.98 Safari/534.13");
public static readonly FakeUserAgent Firefox11 = new
FakeUserAgent("Firefox", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:11.0) Gecko/20100101 Firefox/11.0");
public static readonly FakeUserAgent InternetExplorer8 =
new FakeUserAgent("Internet Explorer 8", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; )");
}

using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Linq;
using System.Web;
using System.Web.UI;
using System.Web.UI.WebControls;
using DearBruce.WebRequestDemo.CoreLib;
using System.Net;
using System.IO;

namespace DearBruce.WebRequestDemo.WebFormUI
{
public partial class TestHttpClientHelper : System.Web.UI.Page
{

private static readonly string LoginUrl = "http://passport.cnblogs.com/login.aspx?ReturnUrl=http://www.cnblogs.com/";
private static readonly string CategoryListUrl = "http://www.cnblogs.com/";

protected void Page_Load(object sender, EventArgs e)
{

}

protected override void OnPreRender(EventArgs e)
{
TestLogin();
DownloadFile();
base.OnPreRender(e);
}

private void TestLogin()
{
// 创建一个CookieContainer实例，供多次请求之间共享Cookie
CookieContainer cookieContainer = new CookieContainer();

// 首先去登录页面登录
//string responseHtml = HttpClientHelper.SendHttpPost(LoginUrl, "tbUserName=bruce&tbPassword=123456", cookieContainer);
// 或者向下面这样调用
NameValueCollection postDataCollection = new NameValueCollection()
{
{ "tbUserName", "bruce" },
{ "tbPassword", "123456" }
};
string responseHtml = HttpClientHelper.SendHttpPost(LoginUrl, postDataCollection, cookieContainer);

// 如果登录成功 responseHtml 是新闻列表页面的 源代码
// 此时cookieContainer已经包含了服务端生成的登录Cookie
TestNewsList(cookieContainer);

}

private void TestNewsList(CookieContainer cookieContainer)
{
// 再去访问要请求的页面。
string html = HttpClientHelper.SendHttpGet(CategoryListUrl, cookieContainer);

if (html.IndexOf("博客园") > 0)
Response.Write("调用成功。");
else
Response.Write("页面结果不符合预期。");

// 如果还要访问其它的受限页面，可以继续调用。
}

private void DownloadFile()
{
string requstImgUrl = "http://pic002.cnblogs.com/images/2012/148372/2012061410561396.png";
string imgName = Path.GetFileName(requstImgUrl);

string imgFileName = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, Guid.NewGuid() + ".jpg");
WebClient webClient = new WebClient();

webClient.Headers.Set("Referer", "http://www.baidu.com");
webClient.Headers.Set("User-Agent", FakeUserAgentRepository.InternetExplorer8.UserAgent);

webClient.DownloadFileAsync(new Uri(requstImgUrl), imgFileName, imgName);
webClient.DownloadFileCompleted += new System.ComponentModel.AsyncCompletedEventHandler(webClient_DownloadFileCompleted);

System.Threading.Thread.Sleep(3000);
}

static void webClient_DownloadFileCompleted(object sender, System.ComponentModel.AsyncCompletedEventArgs e)
{
WebClient webClient = (WebClient)sender;
if (e.Error != null)
{
string imgName = (string)e.UserState;
Console.WriteLine("文件 {0} 下载失败", imgName);
}
string contentType = webClient.ResponseHeaders.Get("Content-Type");
Console.WriteLine("文件下载成功！");
Console.WriteLine("文件类型：" + contentType);
webClient.Dispose();
}
}
}