python 爬取GKGY会员多线程demo

# -*- coding: utf-8 -*-
__author__ = 'wangjingyao'
import urllib
import  urllib2
import re
import sys
import threading, Queue, time
import  user_agents,random,time

reload(sys)
sys.setdefaultencoding('utf8')#设置默认编码
_DATA = []
FILE_LOCK = threading.Lock()
SHARE_Q = Queue.Queue()  #构造一个不限制大小的的队列
_WORKER_THREAD_NUM = 10  #设置线程的个数

class MyThread(threading.Thread) :

def __init__(self, func) :
super(MyThread, self).__init__()  #调用父类的构造函数
self.func = func  #传入线程函数逻辑

def run(self) :
self.func()

def worker() :
global SHARE_Q
while not SHARE_Q.empty():
url = SHARE_Q.get() #获得任务
my_page = get_page(url)
getPageItems(my_page)  #获得当前页面的电影名
#write_into_file(temp_data)
time.sleep(1)
SHARE_Q.task_done()

def get_page(url) :
"""
根据所给的url爬取网页HTML
Args:
url: 表示当前要爬取页面的url
Returns:
返回抓取到整个页面的HTML(unicode编码)
Raises:
URLError:url引发的异常
"""
try :
user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36'
headers={'User-Agent' : user_agent}
request = urllib2.Request(url,headers = headers)
response = urllib2.urlopen(request)
my_page =  response.read().encode('gbk','ignore')
return my_page
except urllib2.URLError, e :
if hasattr(e, "code"):
print "The server couldn't fulfill the request."
print "Error code: %s" % e.code
return  None
elif hasattr(e, "reason"):
print "We failed to reach a server. Please check your url and read the Reason"
print "Reason: %s" % e.reason
return  None

def getPageItems(pageCode) :
"""
通过返回的整个网页HTML, 正则匹配前100的电影名称
Args:
my_page: 传入页面的HTML文本用于正则匹配
"""
if not pageCode:
print 'pageCode init error'
return  None
# 作者爬取
pattern = re.compile('<span itemprop="author">(.*?)</span>')
items = re.findall(pattern,pageCode)
for item in items:
_DATA.append(item)
print "authorSpider------"
# 评论会员爬取
partternComment = re.compile('<div class="comment-detail"><a href=".*?">(.*?)</a>')
itemcomments= re.findall(partternComment,pageCode)
for itemcomment in itemcomments:
if itemcomment.decode('gbk') != '极客漫游者':
_DATA.append(itemcomment)
print "commentSpider------"

def main() :
global SHARE_Q
threads = []
gkgy_url ="http://www.geekpark.net/topics/{page}"
#向队列中放入任务, 真正使用时, 应该设置为可持续的放入任务
for index in xrange(210714,213394) :
SHARE_Q.put(gkgy_url.format(page = index))
for i in xrange(_WORKER_THREAD_NUM) :
thread = MyThread(worker)
thread.start()  #线程开始处理任务
threads.append(thread)
for thread in threads :
thread.join()
SHARE_Q.join()
_DATAs=list(set(_DATA))
   with open("outGKGY.txt", "w+") as my_file :
for page in _DATAs :
my_file.write(page + "\t")
   print "Spider Successful!!!"