python从零开始写爬虫（4）-- 整合代码

把前面代码整合下，抽取出到方法里，一只新浪新闻爬虫需要代码如下：

import requests
from bs4 import BeautifulSoup
import re
import json

#获取新闻链接
def getNewsURLs(url):
newsURL = []
newurl = url
res = requests.get(newurl)
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text,'html.parser')
for news in soup.select('.news-item'):
if len(news.select('h2'))>0:
newsURL.append(news.select('h2 a')[0]['href'])
return newsURL

#获取新闻详细信息
def getNewsDetail(newsurl):
result = {}
res = requests.get(newsurl)
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text,'html.parser')
m = re.search('doc-i(.*).shtml',newsurl)
result['newsid'] = m.group(1)
if len(soup.select('#artibodyTitle'))>0:
result['title'] = soup.select('#artibodyTitle')[0].text
else:
result['title'] = ''
if len(soup.select('#navtimeSource'))>0:
result['newssource'] = soup.select('#navtimeSource')[0].contents[1].text.strip()
else:
result['newssource'] = ''
if len(soup.select('#navtimeSource'))>0:
result['dt'] = soup.select('#navtimeSource')[0].contents[0].strip()
else:
result['dt'] = ''
if len(soup.select('#artibody p'))>0:
result['article'] = ''.join(p.text.strip() for p in soup.select('#artibody p')[:-1])
else:
result['article'] = ''
if len(soup.select('.article-editor'))>0:
result['editor'] = soup.select('.article-editor')[0].text.lstrip('责任编辑：')
else:
result['editor'] = ''
return result

#获取评论数
def getCommentCount(newsid):
commentURL = "http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=gn&newsid=comos-{}&group=&compress=0&ie=utf-8&oe=utf-8&page=1&page_size=20"
comments = requests.get(commentURL.format(newsid))
jd = json.loads(comments.text.strip('var date='))
return jd['result']['count']['total']

测试下：（因为获取的数据太多，我就直接获取newsurl[0]第一条数据做下测试）{



补充下：import re出错的话，在cmd跑下：pip install re;