欢迎使用CSDN-markdown编辑器

百度贴吧 美女图片爬虫的制作

#coding=utf-8
import urllib,urllib2
from lxml import etree
headers = {"User-Agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"}

def network(url):
request=urllib2.Request(url,headers = headers)
html=urllib2.urlopen(request).read()
rerurn html

def tiebaSpider(fullurl, beginPage, endPage):
for page in range(beginPage, endPage + 1):
pn = (page - 1) * 50
#filename = "第" + str(page) + "页.html"
fullurl = url + "&pn=" + str(pn)
#print fullurl
loadPage(fullurl)
#print html

print "谢谢使用"

def loadPage(fullurl):
"""
作用：根据url发送请求，获取服务器响应文件
url: 需要爬取的url地址
"""
html=network(fullurl)
#此时是今天的重点 实用lxml规则找到自己想要的内容
content=etree.HTML(html)
linklist=content.xpath('//div[@class="t_con cleafix"]/div/div/div/a/@href')
for link in linklist:
fulllink="http://tieba.baidu.com" + link
print fulllink
loadimage(fulllink)
def loadimage(link):
html=network(link)
content=etree.HTML(html)
imglisturl=content.xpath('//img[@class="BDE_Image"]/@src')
for imgurl in imglisturl:
print imgurl
writeimage(imgurl)
def writeimg(imgurl):
"""
作用：将html内容写入到本地
link：图片保存
"""
#切片设置图片名称
filename = link[-10:]
img=network(imgurl)
with open(filename,'wb') as  f:
f.write(img)
printe '保存成功'

if __name__ == "__main__":
kw = raw_input("请输入需要爬取的贴吧名:")
beginPage = int(raw_input("请输入起始页："))
endPage = int(raw_input("请输入结束页："))

url = "http://tieba.baidu.com/f?"
key = urllib.urlencode({"kw": kw})
fullurl = url + key
tiebaSpider(fullurl, beginPage, endPage)