百度贴吧图片区通用爬虫

　　很多百度贴吧都有图片区，这里以最喜欢的能年犬的贴吧为例，见下图。

　　


　　

　　根据观察发现它的网址为＂http://tieba.baidu.com/photo/g?kw=” + tieba_name + “&ie=utf-8＂。会发现有不同的主题，大触能年，杂志等等。需要注意的是，在这个页面，每个主题最多显示三个图册，若该图册有多于三个主题，则需要进入主题url才能进入。这里点击Blog，经观察发现其url为“ http://tieba.baidu.com/f?kw=%E8%83%BD%E5%B9%B4%E7%8E%B2%E5%A5%88&tab=album&subtab=album_good&cat_id=d13370a98226cffc766d8a82b9014a90f603ea35 ”，构造网址需要的信息是“cat id”，它可由图片区的html代码得到。

　　


　　

　　所写爬虫在运行后，可以指定贴吧，会返回该贴吧图片区有几个主题，选择主题后，会返回该主题所包含的图册名，选择后，即可对该图册进行下载，依然保存在all文件夹中。代码如下。

# -*- coding:utf-8 -*-
__author__ = 'fybhp'
import requests, json
from bs4 import BeautifulSoup
import os
import urllib
from allfiledir import allfilrdir

#requests用法,一个全局的会话对象.
s = requests.session()

#得到BeautifulSoup对象.
def get_soup(url):
h = s.get(url)
html = h.content
soup = BeautifulSoup(html, "html.parser")
return soup

#创建好按图册分类及图册名安排的目录.
def path(f, z, y):
#将z的相对路径加入f之后.
dir_name = os.path.join(f, z)
if not os.path.exists(dir_name):
print u'---正在创建目录\"' + z + '\"---'
os.makedirs(dir_name)
dir_name2 = os.path.join(dir_name, y)
if not os.path.exists(dir_name2):
print u'---正在创建目录\"' + y + '\"---'
os.makedirs(dir_name2)
return dir_name2

def check_xuanze_zhuti(xuanze_zhuti, zhuti):
add_url = ''
for yigezhuti in zhuti:
if xuanze_zhuti == yigezhuti[0]:
add_url = yigezhuti[1]
if add_url == '':
print u'该主题不存在，请重新输入。\n'
xuanze_zhuti = raw_input(u'请选择主题：\n').decode('utf-8')
check_xuanze_zhuti(xuanze_zhuti, zhuti)
else:
return add_url, xuanze_zhuti

def check_xuanze_tuce(xuanze_tuce, tuce):
tid = ''
pe = ''
for yigetuce in tuce:
if xuanze_tuce == yigetuce[0]:
tid = yigetuce[1]
pe = yigetuce[2][:-1]
if tid == '':
print u'图册不存在，请重新输入。\n'
xuanze_tuce = raw_input(u'请选择图册：\n').decode('utf-8')
check_xuanze_tuce(xuanze_tuce, tuce)
else:
return tid, pe, xuanze_tuce

def get_zhuti_neirong(main_url):
# 得到1，主题名；2，主题url；3，某主题图册数
zhuti_neirong = []
soup = get_soup(main_url)
zhuti_tag = soup.find_all('div', class_="grbh_left")
for item in zhuti_tag:
yige_zhutiming = item.a.text
yige_zhutiurl = item.a['href']
yige_tuceshu = item.span.text
zhuti_neirong.append([yige_zhutiming, yige_zhutiurl, yige_tuceshu])
return zhuti_neirong

def get_tuce_neirong(zhuti_url):
# 得到1，图册名；2，图册id；3，某图册中图片个数
tuce_neirong = []
soup = get_soup(zhuti_url)
tuce_tag = soup.find_all('div', class_="grbm_ele_wrapper")
for item in tuce_tag:
yige_tuceming = item.div.a.text
# 若图册最后为...，在创建目录时会出现差异，导致错误，在这里将...去掉。
if yige_tuceming[-3:] == '...':
yige_tuceming = yige_tuceming[:-3]
yige_tuceid = item.a['href']
yige_tupiangeshu = item.span.text
tuce_neirong.append([yige_tuceming, yige_tuceid, yige_tupiangeshu])
return tuce_neirong

# 处理json，得到所需要信息。
def get_pic_address(json_url):
pic_addres = []
h = s.get(json_url)
html = h.content.decode('unicode-escape')
target = json.loads(html)
for item in target['data']['pic_list']:
pic_addres.append(item['purl'])
return pic_addres

def pic_download(pic_address, dir_name2):
i = 1
for pic in pic_address:
pic1 = pic[:30]
pic2 = pic[-44:]
new_pic = pic1 + 'pic/item/' + pic2  # 获得高清图地址
basename = str(i) + '.jpg'
filename = os.path.join(dir_name2, basename)
if not os.path.exists(filename):
print 'Downloading ' + filename + '......'
urllib.urlretrieve(new_pic, filename)
else:
print filename + u'已存在，略过'
i += 1

def main():
tieba_name = raw_input(u'请输入贴吧名称：\n').decode('utf-8')
#构造图片区url.
main_url = 'http://tieba.baidu.com/photo/g?kw=' + tieba_name + '&ie=utf-8'
zhuti = get_zhuti_neirong(main_url)
if zhuti == []:
print u'抱歉，该贴吧图片区是空的。\n'
exit()
# print zhuti
for yigezhuti in zhuti:
print yigezhuti[0] + '   ' + yigezhuti[2] + u'个图册'
xuanze_zhuti = raw_input(u'请选择主题：\n').decode('utf-8')
add_url, xuanze_zhuti = check_xuanze_zhuti(xuanze_zhuti, zhuti)
#构造主题页面url.
zhuti_url = 'http://tieba.baidu.com/photo/g?kw=' + tieba_name + '&ie=utf-8&cat_id=' + str(add_url[10:])
tuce = get_tuce_neirong(zhuti_url)
# print tuce
for yigetuce in tuce:
print yigetuce[0] + '   ' + yigetuce[2] + u'图片'
xuanze_tuce = raw_input(u'请选择图册：\n').decode('utf-8')
tid, pe, xuanze_tuce = check_xuanze_tuce(xuanze_tuce, tuce)
json_url = 'http://tieba.baidu.com/photo/g/bw/picture/list?kw=' + tieba_name + '&alt=jview&rn=200&tid=' + str(
tid[3:]) + '&pn=1&ps=1&pe=' + str(pe) + '&info=1'
# print json_url
pic_address = get_pic_address(json_url)
# print  pic_address
dir_name2 = path(allfilrdir+'/'+tieba_name, xuanze_zhuti, xuanze_tuce)
pic_download(pic_address, dir_name2)

if __name__ == '__main__':
main()

　　试了能年犬的贴吧，还有犬夜叉的贴吧，均运行无误。

　　效果如下图：

　　


　　

　　这个爬虫并不是一般地对贴吧某图片帖中的图片进行爬取，它爬虫具有通用性，且可交互，但可能并未测试尽所有可能的情况，若在使用过程中出现问题，欢迎指正！