盖得化工----requests/bs4---采集二级网址

# -*- coding: utf-8 -*-
"""
Created on Mon May  9 09:14:32 2016

@author: Administrator
"""
import requests,bs4,csv,time,selenium,random
from selenium import webdriver
from selenium.webdriver.common.keys import Keys

site_hubei="http://china.guidechem.com/suppliers/list_catid-21_area-%E6%B9%96%E5%8C%97"
pages_hubei=31
#所有公司的链接网址_存储列表
list_corporation_link=[]

def Get_sites(site,pages):
list_pages=[]
for page in range(1,pages+1):
thePage=site+"-"+"p"+str(page)+".html"
list_pages.append(thePage)

return list_pages

#采集一页所有公司的二级网址
def Get_secondLinks(pageSite):
res=requests.get(pageSite)
soup=bs4.BeautifulSoup(res.text,"lxml")
linkElems=soup.select(".dblue")
secondLinks=[i.get("href") for i in linkElems]

return secondLinks

#了正确格式写入网页到csv，  row内容添加一个索引号
#'http://show.guidechem.com/hbsthcsales/'处理为[1,'http://show.guidechem.com/hbsthcsales/']
def List_process(list1):
len_list1=len(list1)
list2=[]
for i in range(len_list1):
content=(i+1,list1[i])
list2.append(content)
return list2

#把一页内容写入csv文档 ,list_tableContent为二维列表[[a],[b],[c]]
def Write_table_to_csv(fileName,list_tableContent):
#对列表格式修改，字符串写入的格式不对
file=open(fileName,'w',newline='')
writer1=csv.writer(file)
writer1.writerows(list_tableContent)
file.close()

#构造所有湖北页面网址
list_pages_hubei=Get_sites(site_hubei,pages_hubei)
#首页
firstPage=list_pages_hubei[0]

#获取所有二级网址，先测试前十，所有页数：len(list_pages_hubei)
#正常语句for i in range(len(list_pages_hubei))：
for i in range(20,len(list_pages_hubei)):
pageSite=list_pages_hubei[i]
secondLinks=Get_secondLinks(pageSite)
secondLinks1=List_process(secondLinks)
fileName=str(i+1)+".csv"
Write_table_to_csv(fileName,secondLinks1)
list_corporation_link.append(secondLinks1)
time.sleep(random.randint(5,10))