200个化工网站批量爬取

# -*- coding: utf-8 -*-
"""
Created on Tue Mar 15 08:53:08 2016
采集化工标准补录项目
@author: Administrator
"""
import requests,bs4,openpyxl,time
from openpyxl.cell import get_column_letter,column_index_from_string
#开始时间
timeBegin=time.clock()

excelName="hb_sites.xlsx"
sheetName="Sheet1"
wb1=openpyxl.load_workbook(excelName)
sheet=wb1.get_sheet_by_name(sheetName)
start=1

del_content1="标准编号："
del_content2="发布部门："
del_content3="实施日期："

#excel的行数
sheet.get_highest_row()
#excel的列数
sheet.get_highest_column()

requests.codes.ok

#每个网站爬取相应数据
def Craw(site):
content_list=[]
res=requests.get(site)
res.encoding = 'gbk'
soup1=bs4.BeautifulSoup(res.text,"lxml")
StandardCode=soup1.select('h5')
for i in StandardCode:
content=i.getText()
content_list.append(content)

for i in content_list:
if "标准编号" in i:
i=i.strip(del_content1)
sheet['B'+str(row)].value=i
if "发布部门" in i:
i=i.strip(del_content2)
sheet['C'+str(row)].value=i
if "实施日期" in i:
i=i.strip(del_content3)
sheet['D'+str(row)].value=i

def TimeCount():
timeComsuming=timeEnd-timeBegin
print ("time Comsuming:%f seconds" % timeComsuming)
return timeComsuming

for row in range(2,200+1):
site=sheet['A'+str(row)].value
try:
Craw(site)
except:
continue

wb1.save(excelName)

#结束时间
timeEnd=time.clock()
timeComsuming=TimeCount()