.net爬虫了解一下

using System;
//添加selenium的引用
using OpenQA.Selenium.PhantomJS;
using OpenQA.Selenium.Chrome;

using OpenQA.Selenium.Support.UI;
using OpenQA.Selenium;
using System.IO;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Net;
using HtmlAgilityPack;
namespace ConsoleApplication1
{
class requests
{

public static string UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36";

public static string AcceptLang = "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3";

public static bool AddProxy = false;

public static bool ProxyPort = false;
public static string ProxyHost = null;
private static string Status;
private static string ResponseUri;
private static HtmlDocument Document=new HtmlDocument();

static PhantomJSDriver driver { get; set; }

static void Main(string[] args)
{
//PhantomJSDriverService driverService = PhantomJSDriverService.CreateDefaultService();
////driverService.IgnoreSslErrors = true; //"any" also works
//driverService.HideCommandPromptWindow = true;
//driverService.ProxyType = "none";
//driverService.SslProtocol = "any";
//driverService.MaxDiskCacheSize = 1000;
//driverService.DiskCache = true;
//driverService.WebSecurity = false;
//PhantomJSOptions phs = new PhantomJSOptions();
//phs.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36");
var req = get(new Uri("http://www.baidu.com"));
var resp = response(req);
var root = GetResult(resp);
var nodelist=parse(root,"//div");
Console.WriteLine(root.DocumentNode.InnerHtml);
ChromeOptions options = new ChromeOptions();
options.AddArgument("--headless");
options.AddArgument("--nogpu");
List<String> tagNmaeList = new List<string>(); ;
//using (driver = new PhantomJSDriver(driverService, phs))
//{
using (var driver = new ChromeDriver(options))
{
driver.Manage().Window.Maximize();
driver.Navigate().GoToUrl(@"https://qiita.com/ttake/items/2e470462f8d90e76e996");
Thread.Sleep(5000);
string source = driver.PageSource;
Console.WriteLine(driver.Url);
var html = driver.FindElement(By.TagName("html"));
try
{
var div = driver.FindElement(By.Id("global-nav"));

}
catch (NoSuchElementException)
{
Console.WriteLine("找不到该元素"); ;
}
IJavaScriptExecutor js = (IJavaScriptExecutor)driver;

var divnode = driver.FindElement(By.ClassName("highlight"));

var bkgimg = js.ExecuteScript("arguments[0].style.getPropertyValue('background-image')", divnode);
if (bkgimg == null)
{
var outerhtml = divnode.GetAttribute("outerHTML");  //outerhtml
var outerText = divnode.GetAttribute("outerText");
var innerText = divnode.GetAttribute("innerText");
var children = divnode.GetAttribute("children");
var textContent = divnode.GetAttribute("textContent");
var childElementCount = divnode.GetAttribute("childElementCount");
}

var image = driver.FindElement(By.ClassName("it-Header_authorImage"));

var width = image.Size.Width;
var height = image.Size.Height;
var elem = js.ExecuteScript("return document.defaultView.getComputedStyle(arguments[0])", image);

Screenshot screenshot = ((ITakesScreenshot)driver).GetScreenshot();
var dirpath = @"D:\temp";
if (!Directory.Exists(dirpath))
{

Directory.CreateDirectory(dirpath);

}
screenshot.SaveAsFile(@"D:\temp\seleniumScreenshot.png", ScreenshotImageFormat.Png);
}
}
private static HttpWebRequest get(Uri uri)
{
HttpWebRequest httpWebRequest = WebRequest.Create(uri) as HttpWebRequest;

if (httpWebRequest != null)
{
httpWebRequest.AllowAutoRedirect = true;
httpWebRequest.AutomaticDecompression = (DecompressionMethods.GZip | DecompressionMethods.Deflate);
httpWebRequest.UserAgent = UserAgent;
if (!string.IsNullOrEmpty(AcceptLang))
{
httpWebRequest.Headers.Add("Accept-Language", AcceptLang);
}
if (AddProxy)
{
WebProxy proxy = new WebProxy(ProxyHost, ProxyPort);
httpWebRequest.Proxy = proxy;
}
}
return httpWebRequest;
}
private static HttpWebResponse response(WebRequest request)
{
return (HttpWebResponse)request.GetResponse();
}
private static HtmlDocument GetResult(HttpWebResponse response)
{
if (response != null)
{
Status = response.StatusCode.ToString().ToUpper();
ResponseUri = response.ResponseUri.AbsoluteUri;
using (Stream responseStream = response.GetResponseStream())
{
byte[] array = new byte[1024];
using (MemoryStream memoryStream = new MemoryStream())
{
int num;
do
{
num = responseStream.Read(array, 0, array.Length);
if (num > 0)
{
memoryStream.Write(array, 0, num);
}
}
while (num > 0);
memoryStream.Seek(0L, SeekOrigin.Begin);
Document.Load(memoryStream,Encoding.UTF8);
}
}
}
return Document;
}
private static HtmlNodeCollection parse(HtmlDocument doc,string xpath)
{

var nodelist=doc.DocumentNode.SelectNodes(xpath);
return nodelist;

}

}

}