python 爬虫图片

#!/usr/bin/env python
# encoding: utf-8
'''
@author: caopeng
@license: (C) Copyright 2013-2017, Node Supply Chain Manager Corporation Limited.
@contact: deamoncao100@gmail.com
@software: garner
@file: movie1.py
@time: 2017/9/16 0016 14:49
@desc:
'''
import threading
import time
import urllib
import urllib.request
import os,queue,re
from bs4 import BeautifulSoup
def getUrl(name,hostUrls,girlsUrls,flag):
while  not flag.isSet():
user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
headers = { 'User-Agent' : user_agent }
try:
hostUrl=hostUrls.get(timeout=2)
except queue.Empty:
print("queue empty")
return
request=urllib.request.Request(hostUrl,headers=headers)
response=urllib.request.urlopen(request)
data=response.read().decode('gbk')
soup=BeautifulSoup(data)
tag_lady=soup.find_all("a",attrs={"class":"lady-avatar"})
for tag_href in tag_lady:
girlsUrls.put("https:"+tag_href['href'])
print("录入：https:"+tag_href['href'])
hostUrls.task_done()
print("getUrl is working")

def getImg(name,girlsUrls,flag):
while not flag.isSet():
user_agent ='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240'
headers={'User-Agent':user_agent}
try:
ur=girlsUrls.get(timeout=5)
except queue.Empty:
print(name+" imgqueue empty")
return
pattern=re.compile(r"/(\d+).htm")
items=pattern.findall(ur)
girlUrl="https://mm.taobao.com/self/aiShow.htm?userId="+items[0]
request=urllib.request.Request(girlUrl,headers=headers)
response=urllib.request.urlopen(request)
data=response.read()
soup=BeautifulSoup(data)
fileName=soup.head.title.contents
fileName[0]=fileName[0].rstrip()
tag_div=soup.find('div',attrs={"class":"mm-aixiu-content"})
imgs=tag_div.find_all("img",attrs={})
if len(imgs)==0:
girlsUrls.task_done()
return
path=cdir+'/'+str(fileName[0])
if not os.path.exists(path):
os.makedirs(path)
n=0
for img in imgs:
n=n+1
link=img.get('src')
if link:
s="http:"+str(link)
i=link[link.rfind('.'):]
try:
request=urllib.request.Request(s)
response=urllib.request.urlopen(request)
imgData=response.read()
pathfile=path+r'/'+str(n)+i
with open(pathfile,'wb') as f:
f.write(imgData)
f.close()
print("thread "+name+" write:"+pathfile)
except:
print(str(name)+" thread write false:"+s)
girlsUrls.task_done()

#start=time.time()
if __name__=='__main__':
start=time.time()
hostUrls=queue.Queue()
girlsUrls=queue.Queue()
cdir=os.getcwd()
url='https://mm.taobao.com/json/request_top_list.htm?page='
flag_girl=threading.Event()
flag_img=threading.Event()
for i in range(1,3):
u=url+str(i)
hostUrls.put(u)
threads_girl = threading.Thread(target=getUrl, args=(str(1), hostUrls,girlsUrls,flag_girl))
threads_img = [threading.Thread(target=getImg, args=(str(i+1), girlsUrls,flag_img))
for i in range(8)]
threads_girl.start()
while(girlsUrls.empty()):
print("wait..")
time.sleep(0.1)
for t in threads_img:
t.start()
hostUrls.join()
flag_girl.set()
girlsUrls.join()
flag_img.set()
for t in threads_img:
t.join()
end=time.time()
print("run time:"+str(end-start))