python 写爬虫 美女照片（一）

无意中可看到一个爬取煎蛋网的美女照片 博客，感觉很有趣，就仿照写了个，还几个

一、安装必要的库

1.1 beautifulsoup

pip install beautifulsoup4

1.2

pip install requests

1.3

pip install lxml

几个很重要的函数

1. request.urlretrieve(url,local) 把远程数据下载到本地
2.  req = requests.get(url = url,headers = headers)
req.encoding = 'utf-8'
xhtml = req.text
获取网页的文本信息

二、python 正则表达

获取图片网址

reg = '[^"]*?\.jpg' #表示获取jpg结尾的文本
imgre = re.compile(reg)
imgList = imgre.findall(html)

浏览器的头文件

headers = {
"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
}

防止被服务禁止下载，模拟浏览器访问

opener=request.build_opener()
opener.addheaders=[('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36')]
request.install_opener(opener)
request.urlretrieve(imgurl,'F:/python3/webImage/%s_%s.jpg'%(page,x))

三、写爬虫

import re
import requests
import time
from urllib import request
from urllib.request import urlretrieve ,HTTPError ,urlopen,URLError
url = ""
#获取网页文本
def getHtml(url):
#注释中，使用默认的读取网页,和下面 二选一
#file =  urlopen(url)
#html = file.read()
#xhtml=html.decode('utf-8')
#len(xhtml)

headers = {
"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
}
req = requests.get(url = url,headers = headers)
req.encoding = 'utf-8'
xhtml = req.text
#print(xhtml)
#bf = BeautifulSoup(html, 'lxml')
return xhtml
#分析网页，获取图片的网址
def getImageList(html):
reg = '[^"]*?\.jpg'
imgre = re.compile(reg)
imgList = imgre.findall(html)
#re.findall(imgre,html)
imgList= sorted(set(imgList),key=imgList.index)#去除重复的字段
List=[]
for list in imgList:
# 判断网址是否存在http：
if 'http' not in list:
List.append(('http://'+list).replace('////','//'))
else:
List.append(list)
return List
#下载图片
def download(imgList, page):
x = 1
for imgurl in imgList:
imgurl=imgurl.replace('\/', '/')
#print(imgurl)
try:
print(imgurl)
#模拟浏览器读取图片，防止被服务器 禁止访问
opener=request.build_opener()
opener.addheaders=[('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36')]
request.install_opener(opener)
request.urlretrieve(imgurl,'F:/python3/webImage/%s_%s.jpg'%(page,x))
except HTTPError as e:
print('e.value HTTPError')
print(e.code)
print(e.read())
except URLError as e:
print('e.value  URLError')
print(e.reason  )
except TimeoutError as e:
print("e.value")
x+=1
time.sleep(1) #推迟调用线程一秒

def downImageNum(pagenum):
page = 1
pageNumber = pagenum
while(page <= pageNumber):
html = getHtml(url)#获得url指向的html内容
imageList = getImageList(html)#获得所有图片的地址，返回列表
download(imageList,page)#下载所有的图片
page=page+1

if __name__ == '__main__':
for num in range(1,88):
if num == 1:
url = 'http://jandan.net/ooxx'
else:
url = 'http://jandan.net/ooxx/page-%d#comments' % num
downImageNum(num)

参考

python3网络爬虫：爬取煎蛋网美女照片写了下面这个爬虫，把写的记录下来。

from bs4 import BeautifulSoup
from urllib.request import urlretrieve
import requests
import os
import time
import re

if __name__ == '__main__':
list_url = []
list_url_1 = []

for num in range(1,50):
try:
url = 'http://www.rs05.com/movie/?p=%d' % num

headers = {
"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
}
req = requests.get(url = url,headers = headers)
req.encoding = 'utf-8'
html = req.text
bf = BeautifulSoup(html, 'html.parser')
targets_url = bf.find_all(class_="movie-thumbnails")
except TimeoutError as e:
print("e.value")
for each in targets_url:
list_url.append(each.get('href'))

for each in list_url:
try:
req = requests.get(url = each,headers = headers)
req.encoding = 'utf-8'
html = req.text
bf = BeautifulSoup(html, 'html.parser')
targets_url = bf.find_all(class_="edui-img")

for i in targets_url:
list_url_1.append(i.get('src'))

for target_url_1 in list_url_1:
try:
filename =str(num )+'-'+str(time.localtime())+'.jpg'
headers = {
"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
}
img_req = requests.get(url = target_url_1,headers = headers)
img_req.encoding = 'utf-8'
img_html = img_req.text
img_bf = BeautifulSoup(img_html, 'lxml')

urlretrieve(url = target_url_1,filename = 'E:/cache-work/python3/images1/'+ filename )
#print(filename)
time.sleep(1)
except TimeoutError as e:
print(e.value)
list_url_1.clear()
except TimeoutError as e:
print("e.value")
#print(len(list_url_1))
if len(list_url)>0:
list_url.clear()
if num%7==0:
time.sleep(500)
else:
time.sleep(100)
print('下载完成！')

四、总结

新手上路，总是遇到很多bug，感谢发达的信息时代，遇到问题，很快就能查到结果

1、tabERr，。原来回车换行 的时候，是按照tab空行的，要用四个空格代替；

2、有些网站 ，图片下载 403错误，下载的时候，添加投，模拟浏览器就ok了。

3、开始的时候，没加try‘’except 异常环节，很容易终止操作。

4、目前下载的图片存在重复，因为不同网址 对应相同的图片，正在想办法解决。