python抓取伯乐在线的全部文章，对标题分词后存入mongodb中

依赖包：

1.pymongo

2.jieba

# -*- coding: utf-8 -*-

"""

@author: jiangfuqiang

"""

from HTMLParser import HTMLParser

import urllib2

import sys

import pymongo

import time

import jieba

import traceback

default_encoding = 'utf-8'

if sys.getdefaultencoding() != default_encoding:

reload(sys)

sys.setdefaultencoding(default_encoding)

class FetchJobble(HTMLParser):

def __init__(self):

HTMLParser.__init__(self)

self.isPostThumb = False

self.isPostMeta = False

self.isMetaTitle = False

self.isCategoryTag = False

self.isComment = False

self.isexcerpt = False

self.isReadMore = False

self.isPicture = False

self.data = {}

self.result = []

def handle_starttag(self,tag,attrs):

if tag == 'div':

for key,value in attrs:

if key == 'class':

if value == 'post-thumb':

self.isPostThumb = True

elif value == 'meta-title':

self.isMetaTitle = True

elif tag == 'a' and self.isPostThumb == True:

for key, value in attrs:

if self.isReadMore:

if key == 'href':

self.data['redmoreLink'] = value

self.data['keyword'] = ",".join(jieba.cut(self.data['title']))

self.result.append(self.data)

self.isPostThumb = False

self.isMetaTitle = False

self.isReadMore = False

self.isCategoryTag = False

self.isComment = False

self.isexcerpt = False

self.isPicture = False

self.data = {}

else:

if key == 'class':

if value == 'meta-title':

self.isMetaTitle = True

elif key == 'rel':

if value == 'category tag':

self.isCategoryTag = True

elif key =='href':

if value.find('#respond') > 0:

self.isComment = True

elif tag == 'span' and self.isComment == True:

for key, value in attrs:

if key == 'class' and value == 'excerpt':

self.isexcerpt = True

elif key == 'class' and value == 'read-more':

self.isReadMore = True

elif tag == 'img' and self.isPostThumb and self.isPostMeta == False:

for key, value in attrs:

if key == 'src':

self.data['imgSrc'] = value

def handle_endtag(self,tag):

pass

def handle_data(self,data):

if self.isMetaTitle:

self.data['title'] = data

self.isMetaTitle = False

elif self.isCategoryTag:

ct = ''

if 'tag' in self.data.keys() :

ct = self.data['tag'] + "," + data

else:

ct = data

self.data['tag'] = ct

self.isCategoryTag = False

elif self.isComment and 'comment' not in self.data.keys():

self.data['comment'] = data.split(" ")[0]

elif self.isexcerpt:

self.data['desc'] = data

self.isexcerpt = False

def getResult(self):

return self.result

if __name__ == "__main__":

con = pymongo.Connection('localhost', 27017)

db = con.blog

fetchblog = db.fetch_blog

url = "http://blog.jobbole.com/all-posts/page/%d"

count = 1

flag = False

headers={

'User-Agent':'Mozilla/5.0 （Windows； U； Windows NT 6.1； en-US； rv：1.9.1.6） Gecko/20091201 Firefox/3.5.6'}

while flag == False:

try:

req = urllib2.Request(url%count,headers=headers)

request = urllib2.urlopen(req)

data = request.read()

fj = FetchJobble()

fj.feed(data)

result = fj.getResult()

if len(result) < 1:

flag = True

else:

for doc in result:

fetchblog.insert(doc)

print "page is %d"%count

count += 1

time.sleep(5)

except Exception, e:

traceback.print_exc()

print "parse error",e