scrapy爬取智联招聘中python工程师的招聘信息

# -*- coding:utf-8 -*-
'''
import sys
import codecs
import chardet
import scrapy
from scrapy import log
from scrapy.selector import Selector

from tutorial.items import DmozItem
'''
import pymongo
import scrapy
from scrapy.http import Request
#import sys
import codecs
import re
#from tutorial.items import DmozItem

class DmozSpider(scrapy.Spider):
name = "dmoz"
allowed_domains = ["zhaopin.com"]
start_urls = ["http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E5%8C%97%E4%BA%AC&kw=python&sm=0&sg=3e5d326a89a44a8594d2904a3a207b9e&p=1"]

def parse2(self,response):
f = codecs.open(r'F:\test\tutorial\a','a','utf-8')
#职位待遇
try:
pay_data = response.xpath('//body/div[@class="terminalpage clearfix"]/div[@class="terminalpage-left"]/ul[@class="terminal-ul clearfix"]/li')
pay = ''
for sel in pay_data:
pay_middle_data = sel.xpath('strong').extract()
pay = pay + re.sub(r'<.*?>','',pay_middle_data[0]) + ' '
f.write(pay+'\n')
except:
pass
#任职要求
try:
require_data = response.xpath('//body/div[@class="terminalpage clearfix"]/div[@class="terminalpage-left"]/div[@class="terminalpage-main clearfix"]/div[@class="tab-cont-box"]/div[1]/p').extract()
require_data_middle = ''
for i in require_data:
i_middle = re.sub(r'<.*?>',r'',i,re.S)
require_data_middle = require_data_middle + re.sub(r'\s*',r'',i_middle,re.S)
f.write(require_data_middle+'\n')
except:
pass
#公司地址
try:
company_data = response.xpath('//body/div[@class="terminalpage clearfix"]/div[@class="terminalpage-left"]/div[@class="terminalpage-main clearfix"]/div[@class="tab-cont-box"]/div[@class="tab-inner-cont"]/h2').extract()
company_data = re.search(r'<h2>\s*(.*?)\s*<a',company_data[0],re.S).group(1)
f.write(company_data+'\n')
except:
pass
#招聘信息地址
company_url = str(response)[5:-1]
f.write(company_url+'\n\n')

f.close()

conn = pymongo.MongoClient('192.168.1.5',27017)
db = conn.test
employee = db.employee
employee.insert({"url":company_url,"pay":pay,"require_data":require_data_middle,"company_data":company_data})

def parse(self,response):
#获取本页职位链接
url_data = response.xpath('//div[@id="newlist_list_content_table"]//td[@class="zwmc"]//a/@href').extract()
#        f = codecs.open(r'F:\test\tutorial\a','a','utf-8')
#        for company_url in url_data:
#            f.write(company_url+'\n')
#        f.close()
for company_url in url_data:
yield Request(company_url,callback=self.parse2)
#跳转到下一页
next_page_url =  response.xpath('//body/div[@class="main"]/div[@class="search_newlist_main"]/div[@class="newlist_main"]/form[@name="frmMain"]/div[@class="clearfix"]/div[@class="newlist_wrap fl"]/div[@class="pagesDown"]/ul/li[@class="pagesDown-pos"]/a/@href').extract()
yield Request(next_page_url[0],callback=self.parse)

'''
def parse(self,response):
f = codecs.open(r'F:\test\tutorial\a','a','utf-8')
#职位待遇
pay_data = response.xpath('//body/div[@class="terminalpage clearfix"]/div[@class="terminalpage-left"]/ul[@class="terminal-ul clearfix"]/li')
pay = ''
for sel in pay_data:
try:
pay_middle_data = sel.xpath('strong').extract()
pay = pay + re.sub(r'<.*?>','',pay_middle_data[0]) + ' '
except:
pass
f.write(pay+'\n')
#任职要求
require_data = response.xpath('//body/div[@class="terminalpage clearfix"]/div[@class="terminalpage-left"]/div[@class="terminalpage-main clearfix"]/div[@class="tab-cont-box"]/div[1]/p').extract()
require_data_middle = ''
for i in require_data:
i_middle = re.sub(r'<.*?>',r'',i,re.S)
require_data_middle = require_data_middle + re.sub(r'\s*',r'',i_middle,re.S)
f.write(require_data_middle+'\n')
#公司地址
company_data = response.xpath('//body/div[@class="terminalpage clearfix"]/div[@class="terminalpage-left"]/div[@class="terminalpage-main clearfix"]/div[@class="tab-cont-box"]/div[@class="tab-inner-cont"]/h2').extract()
company_data = re.search(r'<h2>\s*(.*?)\s*<a',company_data[0],re.S).group(1)
f.write(company_data+'\n')
#招聘信息地址
company_url = str(response)[5:-1]
f.write(company_url+'\n\n')
f.close()
'''