python爬虫练习5：博客阅读量助手

目标与环境

爬取csdn博客如“http://blog.csdn.net/nima1994”的文章列表，获取阅读量等，与上次进行比较

使用eclipse+pydev编写，win10 64位，python 3.6.2；用到requests，beautifulsoup4，pandas等库。

代码

'''
Created on 2017年9月30日

@author: Harry
'''
import requests
import re
from bs4 import BeautifulSoup
import pandas as pd
import datetime
import os

#content:文本内容
def analysis(content):

b=BeautifulSoup(content,"lxml").select("#article_list > div.article_item")
for page in b:
url=host+page.select_one("div.article_title > h1 > span > a").attrs['href']
title=page.select_one("div.article_title > h1 > span").get_text().strip()
time=page.select_one("div.article_manage > span.link_postdate").get_text()
read_count=page.select_one("div.article_manage > span.link_view > a").next_sibling[1:-1]
comment=page.select_one("div.article_manage > span.link_comments > a").next_sibling[1:-1]
dt=[url,title,time,read_count,comment];
#global datas
datas.append(dt)
def getData():
r=requests.get(mysite)
page=int(re.findall(r"共(.*)页", BeautifulSoup(r.text,"lxml").select_one("#papelist > span").get_text())[0])
analysis(r.text)
for i in range(2,page+1):
r=requests.get(mysite+"/article/list/"+str(i))
analysis(r.text)
def calculation():
df=pd.DataFrame(datas)
df_last=pd.read_csv("d:\\temp.csv", encoding = "utf-8")

res=[]
col=["标题","浏览量","上次浏览量","浏览量增加","新增评论"]
s1=0
s2=0
s4=0
for i in range(0,df[0].size):
index=list(df_last["0"]).index(df[0][i])
#单条增加
s1+=int(df[3][index])
s2+=int(df_last["3"][index])
des=int(df[3][index])-int(df_last["3"][index])
pinglu=int(df[4][index])-int(df_last["4"][index])
s4+=pinglu
if des>0 or pinglu>0:
r=[df[1][i],df[3][index],df_last["3"][index],des,pinglu]
res.append(r)

spd=pd.DataFrame(res,columns=col)

print(spd)
s3=s1-s2
print("合计")
print("总浏览量：%d 上次总浏览量：%d 浏览量增加:%d 评论增加：%d" % (s1,s2,s3,s4))
write2csv()
def write2csv():
s=input("是否更新工作表:y?")
if(s!="y"):
print("未更新，程序已退出：",str(datetime.datetime.now()))
return
df=pd.DataFrame(datas)
df.to_csv("d:\\temp.csv",encoding = "utf-8")
print("写入成功：",str(datetime.datetime.now()))

datas=[]
host="http://blog.csdn.net"
mysite="http://blog.csdn.net/nima1994"
if __name__ == '__main__':
pd.set_option('display.max_rows',None)
getData()
if(os.path.exists("d:\\temp.csv")==False):
df=pd.DataFrame(datas)
df.to_csv("d:\\temp.csv",encoding = "utf-8")
print("第一次写入，无运行结果，程序退出")
exit()
calculation()

运行结果

以下文档写于：2018年03月07日 17:32:00

由于csdn博客升级，爬虫部分变量更改，简略代码如下：

import requests
import re
from bs4 import BeautifulSoup

#content:文本内容
def analysis(content):

b=BeautifulSoup(content,"lxml").select("#article_list > div.article_item")
for page in b:
url=page.select_one(".article_title .link_title a").attrs['href']
title=page.select_one(".article_title .link_title").get_text().strip()
time=page.select_one(".link_postdate").get_text()
read_count=str(page.select_one(".link_view").get_text()).strip()[3:-1]
comment=str(page.select_one(".link_comments a").next_sibling).strip()[1:-1]
dt=[url,title,time,read_count,comment];
#global datas
print(dt)
datas.append(dt)
def getData():
r=requests.get(mysite)
analysis(r.text)
while(True):
rp = BeautifulSoup(r.text, 'lxml').select("#papelist > div > nav > ul > li")
q = re.search(r'href="(.*?)".*?rel="(.*?)"', str(rp[len(rp) - 1]))
if q==None:
break
r=requests.get(q.group(1))
analysis(r.text)
datas=[]
host="http://blog.csdn.net"
mysite="http://blog.csdn.net/nima1994"
if __name__ == '__main__':
getData()
# for i in datas:
#     print(i)