使用python抓取网站代码，并下载里面的url

方法一

#!/usr/bin/env python
#-*- coding:utf-8 -*-
#python2.7的代码

import os
import urllib,urllib2
from lxml import etree

rename_dic={}

header = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
"Referer": "http://www.baidu.com/"
}
get_url = 'http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/lefte.htm'
req = urllib2.Request(get_url,data=None,headers=header)
web_code = urllib2.urlopen(req)
content = web_code.read()

#通过xpath进行标签匹配
econtent = etree.HTML(content)
li_list = econtent.xpath("//ul/li/a")

for line in li_list:
if 'xls' in line.attrib['href']:
new_url ='http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/%s' % line.attrib['href']
name1 = '%s.xls' %line.text
name = new_url.rsplit('/',1)[1]
rename_dic[name] = name1
print(new_url,name)
#cmd = 'wget %s -O %s &> /dev/null' %(new_url,name)
#os.system(cmd)
#os.system('wget %s -O %s  &> /dev/null' %(new_url,name))
os.system('wget %s  &> /dev/null' %new_url)
#urllib.urlretrieve(new_url,name)   不要通过这种方式下载会有问题，里面的内容非法的URL地址访问

for oldname,newname in rename_dic.items():
os.rename(oldname,newname)

效果图：



备注：

1、xpath 匹配语法

// 是代表所有的

/ 代表层级

[] 筛选

@ 筛选条件

2、 xpath 匹配到的内容我们又三个方法

tag 返回对象标签名称

attrib 返回对象属性

text 返回对象当中的文本

示例：
//div[@id='content']    匹配所有div标签下，id='content'的标签
//ul/li/a               匹配所有 ul 标签下的 li 标签下的 a 标签

通过urlretrieve方法获取到的文本内容：



方法二

使用BeautifulSoup处理html元素

#!/usr/bin/env python
#-*- coding:utf-8 -*-

import requests
from bs4 import BeautifulSoup
'''pip install bs4'''
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
"Referer": "http://www.baidu.com/"
}

def get_url(url):
web_data = requests.get(url=url, headers=headers)
Soup = BeautifulSoup(web_data.content, 'lxml')
data = Soup.find('ul')
data = data.findAll('a')
data_url = []
data_name = []
for item in data:
data_url.append('http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/{}'.format(item['href']))
data_name.append(item.get_text())
return data_url, data_name

def Download_data(url, name):
for url_item, name_item in zip(url, name):
print('正在下载{}。。。'.format(name_item))
with open(r'data\{}.xls'.format(name_item), 'ab') as fl:
fl.write(requests.get(url_item, headers=headers).content)

if __name__ == '__main__':
url, name = get_url('http://www.stats.gov.cn/tjsj/pcsj/rkpc/6rp/lefte.htm')
Download_data(url, name)

抓取房天下数据

#!/usr/bin/env python
#-*- coding:utf-8 -*-

import requests
from bs4 import BeautifulSoup
res = requests.get('http://land.fang.com/market/210100________1_0_1.html')
soup = BeautifulSoup(res.text,'html.parser')
l = []
for message in soup.select('.list28_text'):
link = 'http://land.fang.com' + message.select('a')[0]['href']
l.append(link)
for link in l:
detail = requests.get(link)
soup1 = BeautifulSoup(detail.text,'html.parser')
messes=[]
for mess in soup1.select('.banbox tr td'):
messes.append(mess.text)
print(messes[1:3])