您的位置：首页 > 编程语言 > C#

C# 网页信息采集（Form.cs）

2008-12-07 16:33 911 查看

当当全新正版图书，9周年店庆，特价销售，全场免运费！

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Text;
using System.Windows.Forms;

using System.Web;
using System.Net;
using System.IO;

using System.Data.SqlClient;
//using Microsoft.Office.Interop.Excel;
using System.Threading;
using WebBee;
using System.Text.RegularExpressions;

namespace WebBee
{
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
}
/// <summary>
/// kijiji 网
/// </summary>
/// <param ></param>
/// <param ></param>
private void button1_Click(object sender, EventArgs e)
{

//try
//{
// Microsoft.Office.Interop.Excel.Application objApp = new Microsoft.Office.Interop.Excel.Application();
// Microsoft.Office.Interop.Excel.Workbooks workbooks = objApp.Workbooks;

// //判断目标文件是否存在，若不存在则先创建再打开，若存在则打开；

// string strDestFileFullName = "D://datadb.xls";
// FileInfo destFile = new FileInfo(strDestFileFullName);
// destFile = new FileInfo(strDestFileFullName);
// object missing = System.Type.Missing;
// objApp.DisplayAlerts = false;
// workbooks.Open(strDestFileFullName, missing, missing, missing, missing, missing, missing, missing, missing, missing, missing, missing, missing, missing, missing);

// Microsoft.Office.Interop.Excel.Sheets objSheets = objApp.Worksheets;
// Microsoft.Office.Interop.Excel._Worksheet objSheet;
// objSheet = (Microsoft.Office.Interop.Excel._Worksheet)objSheets.get_Item(1);

// int fromId = Convert.ToInt32(this.fromID.Text);
// int toId = Convert.ToInt32(this.toId.Text);

// for (int j = fromId; j <= toId; j++)
// {
// string Url = fixTextBox.Text;
// Url = Url + j;

// int i = j - fromId;
// this.listBox1.Items.Add(Url);

// //得到指定Url的源码
// Encoding encoding = Encoding.GetEncoding("utf-8");
// string strWebContent = GetWebContent(Url, encoding);
// if (strWebContent == "") continue;

// //只取数据部分的Html代码
// int titleStart = strWebContent.IndexOf("<title>", 0);
// int titleEnd = strWebContent.IndexOf("</title>", 0);
// string temp_title = strWebContent.Substring(titleStart, titleEnd - titleStart + 8);
// int bodyStart = strWebContent.IndexOf("<div main_view/">", 0);

// //没有想要的信息，继续下一个
// if (bodyStart == -1) continue;
// int bodyEnd = strWebContent.IndexOf("</body>", 0);
// string needString = strWebContent.Substring(bodyStart, bodyEnd - bodyStart);
// needString = temp_title + needString;
// try
// {
// //生成HtmlDocument
// WebBrowser webb = new WebBrowser();
// webb.Navigate("about:blank");

// HtmlDocument htmldoc = webb.Document.OpenNew(false);
// htmldoc.Write(needString);

// string title = htmldoc.Title;
// string content = htmldoc.GetElementById("view_content").InnerText;

// string[] firstlinestrs = title.Split('|');

// //如果数据不完整,放弃这个文件
// if (firstlinestrs.Length != 3) continue;
// string[] areas = firstlinestrs[1].Split(',');
// string[] sorts ={ "", "", "" };

// if (firstlinestrs[2] != null)
// {
// sorts = firstlinestrs[2].Split(',');
// }
// else
// {

// }

// objSheet.Cells[i + 1, 1] = firstlinestrs[0];

// //区域--------------------------------------------
// objSheet.Cells[i + 1, 2] = areas[0];
// if (areas.Length >= 2)
// {
// objSheet.Cells[i + 1, 3] = areas[1];
// }
// else
// {
// objSheet.Cells[i + 1, 3] = "";
// }
// if (areas.Length >= 3)
// {
// objSheet.Cells[i + 1, 4] = areas[2];
// }
// else
// {
// objSheet.Cells[i + 1, 4] = "";
// }

// //分类-----------------------------------------
// objSheet.Cells[i + 1, 5] = sorts[0];
// if (sorts.Length >= 2)
// {
// objSheet.Cells[i + 1, 6] = sorts[1];
// }
// else
// {
// objSheet.Cells[i + 1, 6] = "";
// }
// if (sorts.Length >= 3)
// {
// objSheet.Cells[i + 1, 7] = sorts[2];
// }
// else
// {
// objSheet.Cells[i + 1, 7] = "";
// }

// //内容
// objSheet.Cells[i + 1, 8] = content;

// }
// catch (Exception ex)
// {

// }

// }

// string filename = "d://dd.xls";
// objSheet.SaveAs(filename, missing, missing, missing, missing, missing, missing, missing, missing, missing);

// workbooks.Close();
// objApp.Quit();

//}
//catch (Exception ex)
//{

//}
}

//根据Url地址得到网页的html源码
private string GetWebContent(string Url, Encoding encoding)
{
string strResult = "";
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
//声明一个HttpWebRequest请求
request.Timeout = 30000;
//设置连接超时时间
request.Headers.Set("Pragma", "no-cache");
// request.Headers.Set("KeepAlive", "true");
request.CookieContainer = new CookieContainer();
request.Credentials = CredentialCache.DefaultCredentials;
request.Referer = Url;

request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";

HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream streamReceive = response.GetResponseStream();

StreamReader streamReader = new StreamReader(streamReceive, encoding);
strResult = streamReader.ReadToEnd();
streamReceive.Close();
streamReader.Close();
streamReceive = null;
streamReader = null;
}
catch
{
return "";
}
return strResult;
}

/// <summary>
/// 点评网
/// </summary>
/// <param ></param>
/// <param ></param>
private void button2_Click(object sender, EventArgs e)
{

int fromID = Convert.ToInt32(this.fromID.Text);
int toId = Convert.ToInt32(this.toId.Text);
for (int i = fromID; i < toId; i++)
{

string Url = this.fixTextBox.Text + i;

//得到指定Url的源码
Encoding encoding = Encoding.GetEncoding("utf-8");
string strWebContent = GetWebContent(Url, encoding);
if (strWebContent.IndexOf("该商户不存在</title>") != -1) continue;
string needstr = "";
int infostart = strWebContent.IndexOf("<div Main/">", 0);
int infoend = strWebContent.IndexOf("<div Votes/">", 0);

needstr = strWebContent.Substring(infostart, infoend - infostart);

//

try
{
//生成HtmlDocument
WebBrowser webb = new WebBrowser();
webb.Navigate("about:blank");

HtmlDocument htmldoc = webb.Document.OpenNew(false);
htmldoc.Write(needstr);
ShopInfo shopInfo = GetShopInfo(htmldoc);
shopInfo.Shopurl = Url;

this.insertData(shopInfo);

}
catch (Exception ex)
{

}
}
}

/// <summary>
/// 取出列表页面的店铺URL
/// </summary>
/// <param ></param>
/// <returns></returns>

private string[] getNeedpages(HtmlDocument htmldoc)
{
string[] s = new string[20];
for (int i = 0; i < 20; i++)
{
string idname = "_ctl0__ctl" + i + "_HShopName";
string url = htmldoc.GetElementById(idname).GetAttribute("href");
url = url.Substring(11, url.Length - 11);
url = "http://www.dianping.com/" + url;
s[i] = url;
}

return s;
}

/// <summary>
/// 取出shopInfo对象
/// </summary>
/// <param ></param>
/// <returns></returns>
private ShopInfo GetShopInfo(HtmlDocument htmldoc)
{
string bodyHtml = htmldoc.Body.InnerHtml;

ShopInfo Si = new ShopInfo();

HtmlElementCollection areas_hc = htmldoc.GetElementsByTagName("div");

if (areas_hc != null)
{

Si.Areas = areas_hc[2].InnerText;
Si.Areas = Si.Areas.Replace(">", "|");

}

HtmlElementCollection h1_hc = htmldoc.GetElementsByTagName("h1");

if (h1_hc.Count != 0)
{
Si.Name = h1_hc[0].InnerText;
Si.Name = Si.Name.Replace(" ", " ").Trim(); ;

}
else
{
Si.Name = "";
}

HtmlElementCollection span_hc = htmldoc.GetElementsByTagName("span");
if (span_hc.Count != 0)
{
Si.ChildName = span_hc[0].InnerText;
Si.ChildName = Si.ChildName.Replace(" ", " ");
}
else
{
Si.ChildName = "";
}
int startAdress = bodyHtml.IndexOf("<B>地址:</B>", 0);
int endAddress = 0;

if (startAdress != -1)
{
endAddress = bodyHtml.IndexOf("<BR>", startAdress, 200);
Si.Address = bodyHtml.Substring(startAdress + 10, endAddress - startAdress - 10);
Si.Address = Si.Address.Replace(" ", " ").Trim();
if (Si.Address.IndexOf("<A class") != -1)
{
Si.Address = Si.Address.Substring(0, Si.Address.IndexOf("<A class"));

}
}
else
{
Si.Address = "";
}
//------------------------------
startAdress = bodyHtml.IndexOf("<B>电话:</B>", 0);

if (startAdress != -1)
{
endAddress = bodyHtml.IndexOf("<BR>", startAdress, 100);
Si.Phone = bodyHtml.Substring(startAdress + 10, endAddress - startAdress - 10);
Si.Phone = Si.Phone.Replace(" ", " ").Trim();
}
else
{
Si.Phone = "";
}

//------------------------------
startAdress = bodyHtml.IndexOf("<B>别名:</B>", 0);

if (startAdress != -1)
{
endAddress = bodyHtml.IndexOf("<BR>", startAdress, 100);
Si.Bm = bodyHtml.Substring(startAdress + 10, endAddress - startAdress - 10);
Si.Bm = Si.Bm.Replace(" ", " ").Trim();
}
else
{
Si.Bm = "";
}

//-----------------------------
if (htmldoc.GetElementById("ShopTag") != null)
{

Si.Tag = htmldoc.GetElementById("ShopTag").InnerText;
Si.Tag = parseTag(Si.Tag);
}
else
{

Si.Tag = "";

}
//-----------------------------ShopDishs
if (htmldoc.GetElementById("ShopComment") != null)
{

Si.Content = htmldoc.GetElementById("ShopComment").InnerText;
}
else
{

Si.Content = "";

}
//-----------------------------
if (htmldoc.GetElementById("ShopDish") != null)
{

Si.Dish = htmldoc.GetElementById("ShopDish").InnerText;
Si.Dish = parseTag(Si.Dish);
}
else
{

Si.Dish = "";

}
if (htmldoc.GetElementById("ShopDish") != null)
{
Si.Imagesurl = htmldoc.GetElementById("ShopPhoto").GetElementsByTagName("IMG")[0].GetAttribute("src");

}

htmldoc = null;
bodyHtml = null;
areas_hc = null;
span_hc = null;
return Si;
}

/// <summary>
/// 解析字符串
/// </summary>
/// <param ></param>
/// <returns></returns>

private string parseTag(string tagStr)
{
string[] temp = tagStr.Split(' ');
string returnStr = "";
for (int i = 0; i < temp.Length; i++)
{
if (temp[i] != "")
{
returnStr = returnStr + temp[i].Substring(0, temp[i].IndexOf("(")) + "|";
}

}
if (returnStr.EndsWith("|")) returnStr = returnStr.Substring(0, returnStr.Length - 1);

return returnStr;

}

private void insertData(ShopInfo si)
{

StringBuilder strSql = new StringBuilder();
strSql.Append("insert into getTempData(");
strSql.Append("name,bm,childName,phone,address,tag,dish,content,areas,imagesurl,url");
strSql.Append(") values (");
strSql.Append("@name,@bm,@childName,@phone,@address,@tag,@dish,@content,@areas,@imagesurl,@url)");

SqlParameter[] parameters = {
new SqlParameter("@name", SqlDbType.VarChar,128),
new SqlParameter("@bm",SqlDbType.VarChar,50),
new SqlParameter("@childName", SqlDbType.VarChar,50),
new SqlParameter("@phone", SqlDbType.VarChar,100),
new SqlParameter("@address", SqlDbType.VarChar,200),
new SqlParameter("@tag", SqlDbType.VarChar,256),
new SqlParameter("@dish", SqlDbType.VarChar,256),
new SqlParameter("@content", SqlDbType.VarChar,4096),
new SqlParameter("@areas", SqlDbType.VarChar,256),
new SqlParameter("@imagesurl", SqlDbType.VarChar,128),
new SqlParameter("@url", SqlDbType.VarChar,64)
};
parameters[0].Value = si.Name;
parameters[1].Value = si.Bm;
parameters[2].Value = si.ChildName;
parameters[3].Value = si.Phone;
parameters[4].Value = si.Address;
parameters[5].Value = si.Tag;
parameters[6].Value = si.Dish;
parameters[7].Value = si.Content;
parameters[8].Value = si.Areas;
parameters[9].Value = si.Imagesurl;
parameters[10].Value = si.Shopurl;

DBHelper.ExecuteSql(strSql.ToString(), parameters);
}
private bool mIsRunCtrip = false;

public bool IsRunCtrip
{
get { return mIsRunCtrip = false; }
set { mIsRunCtrip = value; }
}
delegate void dFrist(int pIndext);
delegate void d8JMain(int i);
dFrist dfrist = null;
d8JMain d8jmain = null;
private void button3_Click(object sender, EventArgs e)
{
progressBar1.Maximum = Convert.ToInt32(this.toId.Text) - Convert.ToInt32(this.fromID.Text);
this.Text = "正在采集携程网，请等待。。。";
dfrist = new dFrist(showmsg);
Thread tCtrip = new Thread(StartCtrip);
tCtrip.IsBackground = true;
tCtrip.Start();

// this.fixTextBox.Text = "已完成携程网采集，请根目录查看日志";

}
void showmsg(int i)
{

progressBar1.Value++;
label4.Text = "现在ID:" + i.ToString() + ",已完成：" + (progressBar1.Value / (float)progressBar1.Maximum).ToString("p");
if (progressBar1.Value == progressBar1.Maximum)
MessageBox.Show("finish");
}
void StartCtrip()
{
CtripInfo ci = null;
int fromID = Convert.ToInt32(this.fromID.Text);
int toId = Convert.ToInt32(this.toId.Text);
for (int i = fromID; i <= toId; i++)
{
this.BeginInvoke(dfrist, new object[] { i });
try
{
ci = new CtripInfo().GetCtripInfoByHotelID(i);
if (ci != null)
{
int System.Windows.Forms.Application.DoEvents();
if (null != ci.HotelPrice)
{
for (int ii = 1; ii < ci.HotelPrice.Count; ii++)
{
ci.AddHotelPrice(ci.HotelPrice[ii], id);
}
}
else
{
continue;
}
}
else
{
continue;
}
ci = null;

}
catch (Exception ex)
{

TextWriter tw = new StreamWriter(System.Windows.Forms.Application.StartupPath + "//CtripErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误ID: " + i.ToString() + "/r/n 错误原因：" + ex.Message);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;
ci = null;
continue;
}
}

}
void GetCtripHotel(int pStart, int pEnd)
{
IsRunCtrip = false;
}

private void button4_Click(object sender, EventArgs e)
{
//System.Data.DataSet dt = DBHelper.Query("select * from hotel price");
//string sql = "update hotelprice2 set RoomNewPrice='{0}',BroadBand='{1}',BedType='{2}' where ";
//if (dt != null)
//{
// for (int i = 0; i < dt.Tables[0].Rows.Count; i++)
// {
// string id"].ToString();
// string Oldprice = dt.Tables[0].Rows[i]["RoomNewPrice"].ToString();
// string Oldband = dt.Tables[0].Rows[i]["BroadBand"].ToString();
// string OldBedType = dt.Tables[0].Rows[i]["BroadBand"].ToString();
// if (Oldprice.IndexOf(" ") > 1)
// {
// int start = Oldprice.IndexOf(" ");
// string newprice = Oldprice.Substring(0, start);
// string newband = Oldprice.Substring(0, start + 5);

// sql = string.Format(sql, newprice, newband, Oldband);
// if (DBHelper.ExecuteSql(sql) > 0)
// {
// newband = newprice = null;
// continue;
// }
// }
// else
// {
// continue;
// }

// }

//}

//System.Data.DataSet dt = DBHelper.Query("select * from hotelinfo");
//string sql = "update hotelinfo set subshop='{0}',Name='{1}' where ;
//if (dt != null)
//{
// for (int i = 0; i < dt.Tables[0].Rows.Count; i++)
// {
// sql = "update hotelinfo set subshop='{0}',Name='{1}' where ;
// string id"].ToString();
// string OldName = dt.Tables[0].Rows[i]["Name"].ToString();
// if (OldName.IndexOf("(") > 1)
// {
// int start = OldName.IndexOf("(");
// string newName = OldName.Substring(0, start);
// string subshop = OldName.Substring(start + 1, OldName.Length - start - 2);
// sql = string.Format(sql, subshop, newName, id);

// if (DBHelper.ExecuteSql(sql) > 0)
// {
// = newName = subshop = null;
// continue;
// }
// }
// else
// {
// continue;
// }

// }

//}
// 北京 > 朝阳区 > 燕莎/酒仙桥/丽都
System.Data.DataSet dt = DBHelper.Query("select * from hotelinfo");
string sql = "update hotelinfo set position='{0}' where ;
if (dt != null)
{
for (int i = 37; i < dt.Tables[0].Rows.Count; i++)
{
sql = "update hotelinfo set position='{0}' where ;
string id"].ToString();
string OldName = dt.Tables[0].Rows[i]["area"].ToString();
if (OldName.IndexOf(">") > 1)
{
OldName = OldName.Replace(" ", "");
int start = OldName.IndexOf(">");
start++;
int end = OldName.IndexOf(">", start);
if (end > -1)
OldName = OldName.Substring(start, end - start);
else
OldName = OldName.Substring(start);
sql = string.Format(sql, OldName, id);

if (DBHelper.ExecuteSql(sql) > 0)
{
= null;
continue;
}
}
else
{
continue;
}
}
}
MessageBox.Show("i am ok");

}
/// <summary>
/// 删除HTML标识
/// </summary>
/// <param ></param>
/// <returns></returns>
public string DropHTMLTag(string htmlString)
{
htmlString = Regex.Replace(htmlString, @"<(.[^>]*)>", "", RegexOptions.IgnoreCase);
htmlString = Regex.Replace(htmlString, @"([/r/n])[/s]+", "", RegexOptions.IgnoreCase);
htmlString = Regex.Replace(htmlString, @"-->", "", RegexOptions.IgnoreCase);
htmlString = Regex.Replace(htmlString, @"&(lt|#60);", "<", RegexOptions.IgnoreCase);
htmlString = Regex.Replace(htmlString, @"&(gt|#62);", ">", RegexOptions.IgnoreCase);
htmlString = Regex.Replace(htmlString, @"");
end = line.IndexOf("", start);
line = line.Substring(start, end - start);
line = line.Replace("/r", "");
line = line.Replace("/n", "");
line = line.Replace("/t", "");
line = line.Replace("<a", "ウ");
line = line.Replace("/a>", "ウ");
return DropHTMLTag(line);

}
/// <summary>
/// 页的所有酒店
/// </summary>
/// <param ></param>
/// <returns>string[]</returns>
string[] GetHtml(string line)
{
try
{

string tmp = null;
int start = 0, end = 0;
List<string> sss = new List<string>();
while (line.IndexOf("href=/"", start) > 0)
{
line = line.Replace(" ", "");
start = line.IndexOf("href=/"", start);
if (start < 1)
continue;

start = start + 6;
end = line.IndexOf("/"target", start);
if (end < 1)
continue;
tmp = line.Substring(start, end - start);
if (sss.Count == 0)
{

sss.Add(tmp);
}
else
{
if (sss[sss.Count - 1] == tmp)
{
continue;
}
else
{
sss.Add(tmp);
}
}

start = end;
}
return sss.ToArray();
}
catch (Exception ex)
{
return null;

}

}
/// <summary>
/// 获得酒店总页数
/// </summary>
/// <param ></param>
/// <returns></returns>
int GetAreaHotelCount(string line)
{
int start = 0, end = 0;
start = line.IndexOf("");
if (start < 1) return 0;

end = line.IndexOf("", start);
if (end < 1) return 0;
line = line.Substring(start, end - start);
line = DropHTMLTag(line);
start = line.IndexOf("共");
if (start < 1) return 0;
start++;
end = line.IndexOf("页", start);
if (end < 1) return 0;
return int.Parse(line.Substring(start, end - start));

}
#endregion

#region 悟能详细页

private void button5_Click(object sender, EventArgs e)
{
progressBar1.Maximum = int.Parse(DBHelper.GetSingle("select count(*) from a").ToString());
this.Text = "正在采集悟能，请等待。。。";
dfrist = new dFrist(showmsg);
Thread tCtrip = new Thread(beginGetHotel);
tCtrip.IsBackground = true;
tCtrip.Start();
// beginGetHotel();
}
void beginGetHotel()
{
System.Data.DataSet dt = DBHelper.Query("select * from a");
string url = null, tmp = null;
string tel = null, jAdd = null, onlyfoot = null, Address = null, postcode = null, area = null, tag = null, siteurl = null, remark = null;
for (int i = 0; i < dt.Tables[0].Rows.Count; i++)
{

try
{
this.BeginInvoke(dfrist, new object[] { i });
jAdd = url = dt.Tables[0].Rows[i][1].ToString();
url = GetWebContent(url, Encoding.UTF8);
url = url.Replace("/r", "");
url = url.Replace("/n", "");
url = url.Replace("/t", "");
if (null != url && string.Empty != url)
{
int start = 0, end = 0;
start = url.IndexOf("");
if (start < 1)
{
continue;
}
end = url.IndexOf("", start);
tmp = url.Substring(start, end - start);
start = tmp.IndexOf("<strong>");
if (start < 1)
{
;
}
else
{
start += 8;
end = tmp.IndexOf("</strong>", start);
end - start);
}

start = tmp.IndexOf("<strong>", end);
start += 8;
end = tmp.IndexOf("</strong>", start);
tel = tmp.Substring(start, end - start);
start = tmp.IndexOf("<p>", end);
start += 3;
end = tmp.IndexOf("<br/>", start);
Address = tmp.Substring(start, end - start);
Address = ReplaceSingleQuotes(Address);
start = end;
start += 5;
end = tmp.IndexOf("</p>", end);
postcode = tmp.Substring(start, end - start);
//area=dt.Tables[0].Rows[i][2].ToString();
start = tmp.IndexOf("区域：", end);
if (start < 1)
{
area = "未知";
}
else
{
start += 3;
end = tmp.IndexOf("</p>", start);
area = DropHTMLTag(tmp.Substring(start, end - start));
area = ReplaceSingleQuotes(area);
}

start = tmp.IndexOf("标签：", end);
if (start < 1)
{
tag = "未知";
}
else
{
start += 3;
end = tmp.IndexOf("</p>", start);
tag = DropHTMLTag(tmp.Substring(start, end - start));
tag = ReplaceSingleQuotes(tag);
}

start = tmp.IndexOf("网址：", end);
if (start < 1)
{
siteurl = "未知";
}
else
{
start += 3;
end = tmp.IndexOf("</p>", start);
siteurl = DropHTMLTag(tmp.Substring(start, end - start));
}
start = url.IndexOf("特色推荐：");
if (start < 1)
{
onlyfoot = "未知";
}
else
{
start += 5;
end = url.IndexOf("</p>", start);
onlyfoot = url.Substring(start, end - start);
onlyfoot = ReplaceSingleQuotes(onlyfoot);
}

start = url.IndexOf("");
if (start < 1)
{
remark = "未知";
}
else
{

end = url.IndexOf("", start);
tmp = url.Substring(start, end - start);
if (tmp.IndexOf("简介") < 1)
{
remark = "未知";

}
else
{
start = tmp.IndexOf("简介");
end = tmp.IndexOf("</h4>", start);
remark = DropHTMLTag(tmp.Substring(start, end - start));
remark = ReplaceSingleQuotes(remark);
}

}

inser8jHotlInfo(name, tel, onlyfoot, Address, postcode, area, tag, siteurl, remark, jAdd);
= onlyfoot = Address = postcode = area = tag = siteurl = remark = null;
}
}
catch (Exception ex)
{

TextWriter tw = new StreamWriter(System.Windows.Forms.Application.StartupPath + "//CtripErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误ID: " + i.ToString() + "/r/n 错误原因：" + ex.Message);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;
continue;
}
}

}
/// <summary>
/// 替换单引号成中文的单引号
/// </summary>
/// <param ></param>
/// <returns></returns>
public string ReplaceSingleQuotes(string hTML)
{
return hTML.Replace("/'", "`");
}
void inser8jHotlInfo(string name, string tel, string onlyfoot, string Address, string postcode, string area, string tag, string siteurl, string remark, string jAdd)
{
string sql = "INSERT INTO [HotelInfo]([name], [Address], [Tel], [Postcode], [area], [tag], [siteurl], [onlyfoot], [remark],[8jAddress]) VALUES('{0}','{1}','{2}','{3}','{4}','{5}','{6}','{7}','{8}','{9}')";
sql = string.Format(sql, name, Address, tel, postcode, area, tag, siteurl, onlyfoot, remark, jAdd);
DBHelper.ExecuteSql(sql);
}

#endregion

private void Form1_Load(object sender, EventArgs e)
{

}

private void button6_Click(object sender, EventArgs e)
{
System.Data.DataSet dt1 = DBHelper.Query("select star5,city from [00]");
System.Data.DataSet dt2 = DBHelper.Query("select id,city from ctriptwo");
string sql = null;
for (int i = 0; i < dt2.Tables[0].Rows.Count; i++)
{
for (int ii = 0; ii < dt1.Tables[0].Rows.Count; ii++)
{
if (dt2.Tables[0].Rows[i]["city"].ToString() == dt1.Tables[0].Rows[ii]["city"].ToString())
{
sql = "UPDATE ctriptwo set startLevelcount0={0} where ;
sql = string.Format(sql, dt1.Tables[0].Rows[ii]["star5"], dt2.Tables[0].Rows[i]["id"]);
DBHelper.ExecuteSql(sql);
}
else
continue;

}

/*
INSERT INTO [LocalTest].[dbo].[ctripTwo]
([startLevelcount0]
,[startLevel5count4]
,[startLevel5count5]
,[startLevel5count3]
,[startLevel5count2]
,[startLevel5count1]
,[City])
VALUES
*/

}
MessageBox.Show("ok");
}

private void button7_Click(object sender, EventArgs e)
{

this.Text = "正在采集点评,请等待...";
label5.Text=System.DateTime.Now.ToString("hh时mm分ss秒");
Thread tCtrip = new Thread(Shop);
tCtrip.IsBackground = true;
tCtrip.Start();

}
public void Shop()
{
int shopid;
int log = 0; int iDivStart = 0;
string StrSql="";
string ShopTag = "";
string DianpingInfo="";

string ShopCity = ""; string ShopArea = ""; string ShopID = ""; string ShopName = ""; string ShopSort = ""; string ShopAddress = ""; string ShopPhone = "";
try
{
for (shopid = 1612436; shopid < 2700000; shopid++)
{
string Url = "http://www.dianping.com/shop/" + shopid;
string strResult = GetHtmlCode(Url);
if (ReturnMsg(strResult))
{
try
{
string divStart = @"<div ShopGuide"">";
string divEnd = @"<div Reviews"">";
iDivStart = strResult.IndexOf(divStart);
int iDivEnd = strResult.IndexOf(divEnd);
DianpingInfo = strResult.Substring(iDivStart, iDivEnd - iDivStart);
}
catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}
//提取导航条中信息
try
{
string NavigationInfo = @"<div Shop"">";
int NavigationEnd = strResult.IndexOf(NavigationInfo);
string Navigation = strResult.Substring(iDivStart, NavigationEnd - iDivStart);
string str = DropHTMLTag(Navigation).Replace(" ", "ml").Replace(">", "ml");
string ml = "ml";
string[] resultString = Regex.Split(str, ml, RegexOptions.IgnoreCase);
string str1 = (resultString.Length).ToString();
string str2 = "5";
string str3 = "7";
string str4 = "6";

if (str1 == str2)
{
try
{
//提取地址ShopAddress
int AddressEnd = DianpingInfo.IndexOf("地图");
int AddressStart = DianpingInfo.IndexOf("<b>地址:");
if (AddressEnd > 0)
{
ShopAddress = DropHTMLTag(DianpingInfo.Substring(AddressStart, AddressEnd - AddressStart)).Replace("地址:", "").Replace(" ", "").Replace("地图", "");
}
else if ((AddressEnd = DianpingInfo.IndexOf("电话:")) > 0)
{
ShopAddress = DropHTMLTag(DianpingInfo.Substring(AddressStart, AddressEnd - AddressStart)).Replace("地址:", "").Replace(" ", "").Replace("地图", "");
}
else
{
AddressEnd = DianpingInfo.IndexOf(@"<div Update"">");
ShopAddress = DropHTMLTag(DianpingInfo.Substring(AddressStart, AddressEnd - AddressStart)).Replace("地址:", "").Replace(" ", "").Replace("地图", "");
}
}
catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}

//商店ID名ShopID
ShopID = shopid.ToString();
//提取所在城市ShopCity
ShopCity = resultString[0].ToString();
//商店名ShopName
ShopName = resultString[4].ToString().Replace("'", ".");
//商店所在区
ShopArea = resultString[2].ToString();
//商店属于购物类别
ShopSort = resultString[3].ToString();

try
{ //商店联系电话
int PhoneStart = strResult.IndexOf("电话:");
//有电话走这边,没电话不添加
if (PhoneStart.ToString() != "-1")
{
int PhoneEnd = strResult.IndexOf(@"<div Update"">");
ShopPhone = DropHTMLTag(strResult.Substring(PhoneStart, PhoneEnd - PhoneStart)).Replace("电话:", "").Replace(" ", "");

}
else
{
ShopPhone = "无联系电话";
}
}
catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}

try
{
//分类标签ShopTag
int TagStart = strResult.IndexOf(@"<div ShopTag"">");
int TagEnd = strResult.IndexOf(@"<div Reviews"">");

ShopTag = DropHTMLTag(strResult.Substring(TagStart, TagEnd - TagStart)).Replace(" ", "");
}

catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}

}
else if (str1 == str3)
{

try
{
//提取地址ShopAddress
int AddressEnd = DianpingInfo.IndexOf("地图");
int AddressStart = DianpingInfo.IndexOf("<b>地址:");
if (AddressEnd > 0)
{
ShopAddress = DropHTMLTag(DianpingInfo.Substring(AddressStart, AddressEnd - AddressStart)).Replace("地址:", "").Replace(" ", "").Replace("地图", "");
}
else if ((AddressEnd = DianpingInfo.IndexOf("电话:")) > 0)
{
ShopAddress = DropHTMLTag(DianpingInfo.Substring(AddressStart, AddressEnd - AddressStart)).Replace("地址:", "").Replace(" ", "").Replace("地图", "");
}
else
{
AddressEnd = DianpingInfo.IndexOf(@"<div Update"">");
ShopAddress = DropHTMLTag(DianpingInfo.Substring(AddressStart, AddressEnd - AddressStart)).Replace("地址:", "").Replace(" ", "").Replace("地图", "");
}
}
catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}

//商店ID名ShopID
ShopID = shopid.ToString();
//提取所在城市ShopCity
ShopCity = resultString[0].ToString();
//商店名ShopName
ShopName = resultString[5].ToString().Replace("'", ".");
//商店所在区
ShopArea = resultString[2].ToString();
//商店属于购物类别
ShopSort = resultString[4].ToString();

try
{ //商店联系电话
int PhoneStart = strResult.IndexOf("电话:");
//有电话走这边,没电话不添加
if (PhoneStart.ToString() != "-1")
{
int PhoneEnd = strResult.IndexOf(@"<div Update"">");
ShopPhone = DropHTMLTag(strResult.Substring(PhoneStart, PhoneEnd - PhoneStart)).Replace("电话:", "").Replace(" ", "");

}
else
{
ShopPhone = "无联系电话";
}

}
catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}

try
{
//分类标签ShopTag
int TagStart = strResult.IndexOf(@"<div ShopTag"">");
int TagEnd = strResult.IndexOf(@"<div Reviews"">");

ShopTag = DropHTMLTag(strResult.Substring(TagStart, TagEnd - TagStart)).Replace(" ", "");
}

catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}

}
else if (str1 == str4)
{

try
{
//提取地址ShopAddress
int AddressEnd = DianpingInfo.IndexOf("地图");
int AddressStart = DianpingInfo.IndexOf("<b>地址:");
if (AddressEnd > 0)
{
ShopAddress = DropHTMLTag(DianpingInfo.Substring(AddressStart, AddressEnd - AddressStart)).Replace("地址:", "").Replace(" ", "").Replace("地图", "");
}
else if ((AddressEnd = DianpingInfo.IndexOf("电话:")) > 0)
{
ShopAddress = DropHTMLTag(DianpingInfo.Substring(AddressStart, AddressEnd - AddressStart)).Replace("地址:", "").Replace(" ", "").Replace("地图", "");
}
else
{
AddressEnd = DianpingInfo.IndexOf(@"<div Update"">");
ShopAddress = DropHTMLTag(DianpingInfo.Substring(AddressStart, AddressEnd - AddressStart)).Replace("地址:", "").Replace(" ", "").Replace("地图", "");
}
}
catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}

//商店ID名ShopID
ShopID = shopid.ToString();
//提取所在城市ShopCity
ShopCity = resultString[0].ToString();
//商店名ShopName
ShopName = resultString[5].ToString().Replace("'", ".");
//商店所在区
ShopArea = resultString[3].ToString();
//商店属于购物类别
ShopSort = resultString[4].ToString();

try
{ //商店联系电话
int PhoneStart = strResult.IndexOf("电话:");
//有电话走这边,没电话不添加
if (PhoneStart.ToString() != "-1")
{
int PhoneEnd = strResult.IndexOf(@"<div Update"">");
ShopPhone = DropHTMLTag(strResult.Substring(PhoneStart, PhoneEnd - PhoneStart)).Replace("电话:", "").Replace(" ", "");

}
else
{
ShopPhone = "无联系电话";
}

}
catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}

try
{
//分类标签ShopTag
int TagStart = strResult.IndexOf(@"<div ShopTag"">");
int TagEnd = strResult.IndexOf(@"<div Reviews"">");

ShopTag = DropHTMLTag(strResult.Substring(TagStart, TagEnd - TagStart)).Replace(" ", "");
}

catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}

}
else
{
try
{
//提取地址ShopAddress
int AddressEnd = DianpingInfo.IndexOf("地图");
int AddressStart = DianpingInfo.IndexOf("<b>地址:");
if (AddressEnd > 0)
{
ShopAddress = DropHTMLTag(DianpingInfo.Substring(AddressStart, AddressEnd - AddressStart)).Replace("地址:", "").Replace(" ", "").Replace("地图", "");
}
else if ((AddressEnd = DianpingInfo.IndexOf("电话:")) > 0)
{
ShopAddress = DropHTMLTag(DianpingInfo.Substring(AddressStart, AddressEnd - AddressStart)).Replace("地址:", "").Replace(" ", "").Replace("地图", "");
}
else
{
AddressEnd = DianpingInfo.IndexOf(@"<div Update"">");
ShopAddress = DropHTMLTag(DianpingInfo.Substring(AddressStart, AddressEnd - AddressStart)).Replace("地址:", "").Replace(" ", "").Replace("地图", "");
}
}
catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}

//商店ID名ShopID
ShopID = shopid.ToString();
//提取所在城市ShopCity
ShopCity = resultString[0].ToString();
//商店名ShopName
ShopName = resultString[3].ToString().Replace("'", ".");
//商店没有所在区
ShopArea = "无";
//商店属于购物类别
ShopSort = resultString[2].ToString();

try
{ //商店联系电话
int PhoneStart = strResult.IndexOf("电话:");
//有电话走这边,没电话不添加
if (PhoneStart.ToString() != "-1")
{
int PhoneEnd = strResult.IndexOf(@"<div Update"">");
ShopPhone = DropHTMLTag(strResult.Substring(PhoneStart, PhoneEnd - PhoneStart)).Replace("电话:", "").Replace(" ", "");

}
else
{
ShopPhone = "无联系电话";
}
}
catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}

try
{
//分类标签ShopTag
int TagStart = strResult.IndexOf(@"<div ShopTag"">");
int TagEnd = strResult.IndexOf(@"<div Reviews"">");

ShopTag = DropHTMLTag(strResult.Substring(TagStart, TagEnd - TagStart)).Replace(" ", "");
}

catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}

}

}
catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message + Url);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}

//插入数据库
StrSql = "insert into ShopInfo values(" + ShopID + ",'" + ShopName + "','" + ShopAddress + "','" + ShopPhone + "','" + ShopCity + "','" + ShopArea + "','" + ShopSort + "','" + ShopTag + "')";
log = DBHelper.ExecuteSql(StrSql) + log;

}
else
{

}
continue;
}
}
catch (Exception ex)
{

TextWriter tw = new StreamWriter("D://WebBee//ShoppingErrorlog.log", true);
tw.Flush();
tw.WriteLine("错误原因：" + ex.Message);
tw.WriteLine("-------------------------------------------");
tw.Flush();
tw.Close();
tw = null;

}
finally
{

MessageBox.Show("完成" + System.DateTime.Now.ToString("hh时mm分ss秒"));
}

}
//获取网源码方法
public string GetHtmlCode(string Url)
{
Encoding encoding = Encoding.GetEncoding("utf-8");
string strResult = "";
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
request.Timeout = 30000;
request.Headers.Set("Pragma", "no-cache");
request.CookieContainer = new CookieContainer();
request.Credentials = CredentialCache.DefaultCredentials;
request.Referer = Url;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream streamReceive = response.GetResponseStream();
StreamReader streamReader = new StreamReader(streamReceive, encoding);
strResult = streamReader.ReadToEnd();
streamReceive.Close();
streamReader.Close();
streamReceive = null;
streamReader = null;
}
catch (Exception ex)
{

}
return strResult;
}
//提供BooL判断是否继续
public bool ReturnMsg(string strResult)
{ //获取ＩＤ
int iTitleStart = strResult.IndexOf("<title>");
int iTitleEnd = strResult.IndexOf("</title>");
string StrWeb = strResult.Substring(iTitleStart, iTitleEnd - iTitleStart);
string StrTitle = DropHTMLTag(StrWeb);
//获取时候是购物类型网StrSort
string StrSort="";

try
{
string divStart = @"<div ShopGuide"">";
string divEnd = @"<div Reviews"">";
int iDivStart = strResult.IndexOf(divStart);
int iDivEnd = strResult.IndexOf(divEnd);
string NavigationInfo = @"<div Shop"">";
int NavigationEnd = strResult.IndexOf(NavigationInfo);
string Navigation = strResult.Substring(iDivStart, NavigationEnd - iDivStart);
string str = DropHTMLTag(Navigation).Replace(" ", "ml").Replace(">", "ml");
string ml = "ml";
string[] resultString= Regex.Split(str, ml, RegexOptions.IgnoreCase);
StrSort = resultString[1].ToString();
}
catch
{
StrSort = "出错";

}
finally
{

}
if (StrSort!="购物"||StrTitle == "该商户不存在或已被删除" )
{
return false;
}
else
{
return true;
}

}

} 　　　

}

内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理

标签：

相关文章推荐

新的分享

章节导航