学习python写网络爬虫(四)

python自带的robotparser模块可以解析robots.txt文件，可以使用urllib2来支持代理，但是python的http模块的requests来实现该功能会更友好。

可以通过使用datetime模块和time模块在两次下载之间添加延时，可以防止被封禁。

在爬取动态网页时，由于一些网站会动态生成页面内容，导致出现无限多网页，这样可能会无止境的链接下去，这种情况被称作 爬虫陷阱，只要给爬虫设置一个爬取的深度就可以避免。

# coding=utf-8
import urllib2
import re
import urlparse
import robotparser
import datetime
import time

#记录每个域名上次访问的时间，来对爬虫进行限速
class Throttle:
"""Add a delay between download to the same domain"""
def __init__(self, delay):
self.delay = delay
self.domains = {}

def wait(self, url):
domain = urlparse.urlparse(url).netloc
last_accessed = self.domains.get(domain)

if self.delay > 0 and last_accessed is not None:
sleep_secs = self.delay - (datetime.datetime.now() - last_accessed).seconds

if sleep_secs > 0:
time.sleep(sleep_secs)

self.domains[domain] = datetime.datetime.now()

def download(url, user_agent='wswp', proxy = None, num_retries=2):
print 'Downloading:', url
headers = {'User-agent': user_agent}
request = urllib2.Request(url, headers=headers)
#设置代理
opener = urllib2.build_opener()
if proxy:
proxy_params = {urlparse.urlparse(url).scheme : proxy}
opener.add_handler(urllib2.ProxyHandler(proxy_params))

try:
#html = urllib2.urlopen(request).read()
html = opener.open(request).read()
except urllib2.URLError as e:
print 'Download error:', e.reason
html = None
if num_retries > 0:
if hasattr(e, 'code') and 500 <= e.code < 600:
#return download(url, num_retries - 1)
html = download(url,user_agent,proxy,num_retries-1)
return html

def link_crawler(seed_url, link_regex):
"""Crawl from the given seed URL follwoing links matched by link_regex"""
crawl_queue = [seed_url]
#防止抓取重复的
seen = set(crawl_queue)
#robotparser模块解析robots.txt文件,避免下载禁止爬取的页面
rp = robotparser.RobotFileParser()
rp.set_url('http://example.webscraping.com/robots.txt')
rp.read()

throttle = Throttle(5)
while crawl_queue:
url = crawl_queue.pop()
#限速在这里
throttle.wait(url)
if rp.can_fetch('wswp', url):
html = download(url)
for link in get_links(html):
if re.match(link_regex, link):
link = urlparse.urljoin(seed_url, link)
if link not in seen:
seen.add(link)
crawl_queue.append(link)
#测试一下，打印出抓取到的url
print link
else:
print 'blocked by robots.txt:',url

def get_links(html):
"""Return a list of links from html"""
webpage_regex = re.compile('<a[^>]+href=["\'](.*?)["\']',re.IGNORECASE)
return webpage_regex.findall(html)

link_crawler('http://example.webscraping.com/','/(index|view)')