python多进程抓取猫眼电影top

import requests

import re

import json

import multiprocessing

from requests import RequestException

# 获取要抓取网页源代码
def get_html(url):
try:
response = requests.get(url)
return response.text
except RequestException:
return None

# 解析单个网页并用正则表达式提取电影信息，用yield构造生成器对象
def parse_html(html):
pattern = re.compile('<dd>.*?board-index.*?(\d+).*?data-src="(.*?)".*?name.*?'
+'><a.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>', re.S)
movie_list = re.findall(pattern, html)
for item in movie_list:
yield{
"排名": item[0],
"电影名": item[2],
"封面链接": item[1],
"主演": item[3].strip(),
"上映时间":item[4],

}
# 将提取到的信息格式化为json文件。
def write_to_file(text):
with open('moviestop100.txt','a', encoding='utf-8') as f:
f.write(json.dumps(text, ensure_ascii=False)+'\n')
# 主方法，构建请求URL，并实现单个页面网页的提取
def main(i):

url = 'http://maoyan.com/board/4?offset='+str(i)
html = get_html(url)
for item in parse_html(html):
write_to_file(item)

# 程序执行的入口，创建一个进程池对象，并使用map函数实现多进程
if __name__ == "__main__":
pool = multiprocessing.Pool()
pool.map(main, [i*10 for i in range(12)])
#for i in range(10):
#   main(i*10)