csdn博客爬虫更新

# -*- coding:gbk -*-
import sys
import requests
import re
from lxml import etree
from lxml import html as ht

def download(url):
headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"}
html=requests.get(url,headers=headers).text
return html

html2=download("http://blog.csdn.net/Joliph")
selector2=etree.HTML(html2)
pagelist=selector2.xpath('//*[@id="papelist"]/a[last()-2]/text()')[0]
#这里有有个潜在的问题，在我博客写到5页以上时出现...后无法判断页数
pagelist=int(pagelist)
for page in range(1,pagelist+1):
url="http://blog.csdn.net/Joliph/article/list/"+str(page)
html=download(url)
selector=etree.HTML(html)
titlelist=selector.xpath('//*[@class="link_title"]/a/text()')
datelist=selector.xpath('//*[@class="article_manage"]/span[1]/text()')
#/text()!!!!!!!!!!!!!!!!!!!!!!!
number=len(titlelist)
for i in range(1,number+1):
tree=ht.fromstring(titlelist[i-1])
strcom=tree.xpath('string(.)')
print(datelist[i-1]+"----"+strcom)