【Python】抓取人人都是产品经理的文章

简介

代码

# -*- coding: utf-8 -*-
#-------------------------------------
# author: maqingxiong
# date:   2017-10-09
# desc:   抓取人人都是产品经理最新文章
#-------------------------------------

import requests
import sys
import io
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='gb18030')

from pybloom import ScalableBloomFilter
from fake_useragent import UserAgent

class PmSpider(object):
def __init__(self):
self.session = requests.session()
self.ua = UserAgent()
self.sbf = ScalableBloomFilter()
self.load_url_in_bloomfilter()

def load_url_in_bloomfilter(self):
with open('crawled_url.txt', 'r') as f:
for line in f:
self.sbf.add(line.replace('\n', ''))

def construct_all_url(self):
url_list = []
base_url = 'http://www.woshipm.com/__api/v1/stream-list?paged={}&action=laodpost'
for i in range(1, 11):
url_list.append(base_url.format(str(i)))
return url_list

def get_req_headers(self):
headers = {
'User-Agent': self.ua.random,
'Host': 'www.woshipm.com',
'Referer': 'http://www.woshipm.com/'
}
return headers

def get_html(self, url):
headers = self.get_req_headers()
try:
response = self.session.get(url=url, headers=headers)
if response.status_code == 200:
return response.json()
else:
return None
except Exception as e:
return self.get_html(url=url)

def parse_html(self, html):
for item in html['payload']:
print(item['id'])
print(item['title'])
print(item['permalink'])
print(item['date'])
print(item['image'])

def run(self):
f = open("crawled_url.txt", 'w')
url_list = self.construct_all_url()
for url in url_list:
if url in self.sbf:
continue
else:
f.write(url+'\n')
html = self.get_html(url=url)
if html:
self.parse_html(html)
f.close()

if __name__ == "__main__":
pm_spider = PmSpider()
pm_spider.run()