C#网页抓取 HttpWebRequest

protected void Button1_Click(object sender, EventArgs e)
{
FileStream fs = new FileStream(Server.MapPath("~/tmp1.htm"),FileMode.OpenOrCreate);
TextWriter tw = new StreamWriter(fs, System.Text.Encoding.UTF8);
//TextWriter tw = new StreamWriter(fs, System.Text.Encoding.GetEncoding("gb2312"));
HttpRuntime.ProcessRequest(new MyRequest("TestFormView.aspx", "info=hello中", tw));     //第二个参数info，相当于请求

的url后带的参数。
sh();
tw.Close();
Response.Write("ok");
}

public class MyRequest : SimpleWorkerRequest
{
private TextWriter Output;
public MyRequest(string a1, string a2, TextWriter a3):base(a1, a2, a3)
{
Output = a3;
}
public override void SendResponseFromMemory(byte[] data, int length)
{
Output.Write(System.Text.Encoding.UTF8.GetChars(data, 0, length));
}
}

// Set the 'Method' property of the 'Webrequest' to 'POST'.
myHttpWebRequest.Method = "POST";
Console.WriteLine ("\nPlease enter the data to be posted to the (http://www.contoso.com/codesnippets/next.asp) Uri :");

// Create a new string object to POST data to the Url.
string inputData = Console.ReadLine ();

string postData = "firstone=" + inputData;
ASCIIEncoding encoding = new ASCIIEncoding ();
byte[] byte1 = encoding.GetBytes (postData);

// Set the content type of the data being posted.
myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";

// Set the content length of the string being posted.
myHttpWebRequest.ContentLength = byte1.Length;

Stream newStream = myHttpWebRequest.GetRequestStream ();

newStream.Write (byte1, 0, byte1.Length);
Console.WriteLine ("The value of 'ContentLength' property after sending the data is {0}", myHttpWebRequest.ContentLength);

// Close the Stream object.
newStream.Close ();

如果要向指定的页面提交参数，webrequest提供了一个流，朝里面写就可以了

public virtual Stream GetRequestStream()
这里有两个地方注意下。第一，如果是以GET方式提交的话，参数直接写到WebRequest构造函数的URL里，如果是以POST方式提交，那就获取这

个流，把参数写进流里，注意在写之前必须指定Method 为POST。第二，写入之后要关闭这个流。

public class Test
{
// Specify the URL to receive the request.
public static void Main (string[] args)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create (args[0]);

// Set some reasonable limits on resources used by this request
request.MaximumAutomaticRedirections = 4;
request.MaximumResponseHeadersLength = 4;
// Set credentials to use for this request.
request.Credentials = CredentialCache.DefaultCredentials;
HttpWebResponse response = (HttpWebResponse)request.GetResponse ();

Console.WriteLine ("Content length is {0}", response.ContentLength);
Console.WriteLine ("Content type is {0}", response.ContentType);

// Get the stream associated with the response.
Stream receiveStream = response.GetResponseStream ();

// Pipes the stream to a higher level stream reader with the required encoding format.
StreamReader readStream = new StreamReader (receiveStream, Encoding.UTF8);

Console.WriteLine ("Response stream received.");
Console.WriteLine (readStream.ReadToEnd ());
response.Close ();
readStream.Close ();
}
}