使用代理(WebProxy）爬虫

/// <summary>
/// 发送get请求
/// </summary>
/// <param name="strUrl"></param>
/// <param name="isRetry"></param>
/// <returns></returns>
private string GetHtmlByUrl(string strUrl, bool isRetry = false, WebProxy daili = null)
{
currentDailiExecCount++;
if (currentDailiExecCount > dailiExecMaxCount)
{
logclass.Debug("======当前代理======" + currentdaili.Address.Authority + ":" + currentdaili.Address.Port + "==跑的次数超过了设置的最大次数（" + dailiExecMaxCount.ToString()+"）");
}
try
{
HttpWebResponse response = new webframework.common.HttpHelper()
{
URL = string.Format("{0}", strUrl),
//Proxy = daili == null ? currentdaili : daili,
//Proxy = new WebProxy("218.204.140.97:8118", true),
Proxy = daili == null ? (currentDailiExecCount > dailiExecMaxCount ? GetOneDaili() : currentdaili) : daili,
Timeout = 5 * 1000,
}.CreateGetHttpResponse();

return response.HttpString(Encoding.UTF8);
}
catch (Exception)
{
//重试请求
if (!isRetry)
return GetHtmlByUrl(strUrl, true, GetOneDaili());
else
throw null;
}

}

/// <summary>
/// 发送post请求
/// </summary>
/// <param name="strUrl"></param>
/// <param name="isRetry"></param>
/// <returns></returns>
private string PostHtmlByUrl(string strUrl, string strPostString, bool isRetry = false, WebProxy daili = null)
{
currentDailiExecCount++;
if (currentDailiExecCount > dailiExecMaxCount)
{
logclass.Debug("======当前代理======" + currentdaili.Address.Authority + ":" + currentdaili.Address.Port + "==跑的次数超过了设置的最大次数（" + dailiExecMaxCount.ToString()+"）");
}
try
{
HttpWebResponse response = new HttpHelper()
{
URL = strUrl,
PostString = strPostString,
//Proxy = new WebProxy("218.204.140.97:8118", true),
Proxy = daili == null ? (currentDailiExecCount>dailiExecMaxCount?GetOneDaili(): currentdaili) : daili,
//Proxy = daili == null ? currentdaili : daili,
PostEncoding = Encoding.UTF8,
Timeout = 5 * 1000,
}.CreatePostHttpResponse();

return response.HttpString(Encoding.UTF8);
}
catch (Exception)
{
//重试请求
if (!isRetry)
return PostHtmlByUrl(strUrl, strPostString, true, GetOneDaili());
else
throw null;
}

}