python3实践-从网站获取数据(Carbon Market Data-BJ) （pandas，bs4）

自己边看边实践一些简单的实际应用，下面的程序是从某个网站上获取需要的数据。

在编写的过程中，通过学习陆续了解到一些方法，发现Python真的是很便捷。

尤其是用pandas获取网页中的表格数据，真的是太方便了！！！

程序写的可能并不好，但基本上实现了自己的需求。

希望有高手来指点下~~

Version 04 （Jan 12 2017）【对于获取表格信息，推荐使用该方法】

# Code based on Python 3.x
# _*_ coding: utf-8 _*_
# __Author: "LEMON"

import pandas as pd

url2 = 'http://www.bjets.com.cn/article/jyxx/?'
links = []
for n in range(2, 40):
# 页面总数为39页，需要自己先从网页判断，也可以从页面抓取，后续可以完善
link = url2 + str(n)
links.append(link)
links.insert(0, url2)

df2 = pd.DataFrame()  # creates a new dataframe that's empty
for url in links:
# 利用pandas获取数据，需要安装 html5lib模块
dfs = pd.read_html(url, header=0)
for df in dfs:
df2= df2.append(df, ignore_index= True)

# df2.to_excel('MktDataBJ.xlsx') # 将数据存储在excel文件里
df2.to_csv('MktDataBJ-1.csv')  # 将数据存储在csv文件里

Version 03 （Jan 12 2017）

# Code based on Python 3.x
# _*_ coding: utf-8 _*_
# __Author: "LEMON"

from bs4 import BeautifulSoup
import requests
import csv

url2 = 'http://www.bjets.com.cn/article/jyxx/?'
links = []
for n in range(2, 40):
# 页面总数为39页，需要自己先从网页判断，也可以从页面抓取，后续可以完善
link = url2 + str(n)
links.append(link)
links.insert(0, url2)

for url in links:
rep = requests.get(url)
# content = rep.text.encode(rep.encoding).decode('utf-8')
# # 直接用requests时，中文内容需要转码

soup = BeautifulSoup(rep.content, 'html.parser')

# table = soup.table
table = soup.find('table')  # 两种方式都可以

trs = table.find_all('tr')
trs2 = trs[1:len(trs)]
list1 = []
for tr in trs2:
td = tr.find_all('td')
row = [i.text for i in td]
list1.append(row)

with open('MktDataBJ.csv', 'a', errors='ignore', newline='') as f:
f_csv = csv.writer(f)
f_csv.writerows(list1)

Version 02 （Jan 09 2017）

# Code based on Python 3.x
# _*_ coding: utf-8 _*_
# __Author: "LEMON"

from bs4 import BeautifulSoup
import requests
import csv

url2 = 'http://www.bjets.com.cn/article/jyxx/?'
links = []
for n in range(2, 40):
# 页面总数为39页，需要自己先从网页判断，也可以从页面抓取，后续可以完善
link = url2 + str(n)
links.append(link)
links.insert(0, url2)
# print(links)

for url in links:
rep = requests.get(url)
# content = rep.text.encode(rep.encoding).decode('utf-8')
# # 直接用requests时，中文内容需要转码

soup = BeautifulSoup(rep.content, 'html.parser')
body = soup.body
data = body.find('div', {'class': 'list_right'})

quotes = data.find_all('tr')
quotes1 = quotes[1:len(quotes)]

list1 = []
for x in quotes1:
list2 = []
for y in x.find_all('td'):
list2.append(y.text)  # 每日的数据做一个单独的list
list1.append(list2)
# print(list1)  # list1为每日数据的总列表
with open('MktDataBJ.csv', 'a', errors='ignore', newline='') as f:
f_csv = csv.writer(f)
f_csv.writerows(list1)

Version 01 （Jan 08 2017）

# Code based on Python 3.x
# _*_ coding: utf-8 _*_
# __Author: "LEMON"

from bs4 import BeautifulSoup
import requests
import csv

urllink = 'http://www.bjets.com.cn/article/jyxx/?'
links = []
for n in range(2, 40):
#页面总数为39页，需要自己先从网页判断，也可以从页面抓取，后续可以完善
link = urllink + str(n)
links.append(link)
links.insert(0, urllink)
# print(links)

for url in links:

rep = requests.get(url)
# content = rep.text.encode(rep.encoding).decode('utf-8')
# # 直接用requests时，中文内容需要转码

soup = BeautifulSoup(rep.content, 'html.parser')

# print(soup.prettify())
# # prettify()

body = soup.body
data = body.find('div', {'class': 'list_right'})

# table title
titles = data.find_all('th')

title = []
for x in titles:
title.append(x.text)
# print(title)

quotes = data.find_all('tr')
quotes1 = quotes[1:len(quotes)]
# print(quotes1)

list1 = []
for x in quotes1:
for y in x.find_all('td'):
list1.append(y.text)
# print(list1)  # list为每日数据的总列表

date = []
volumes = []
meanprice = []
totalmoney = []

for i in range(0, len(list1)):
if i % 4 == 0:
date.append(list1[i])
elif i % 4 == 1:
volumes.append(list1[i])
elif i % 4 == 2:
meanprice.append(list1[i])
else:
totalmoney.append(list1[i])

# print(date)
# print(volumes)
# print(meanprice)
# print(totalmoney)

final = []
for i in range(0, len(date)):
temp = [date[i], volumes[i], meanprice[i], totalmoney[i]]
final.append(temp)
# print(final)
with open('bj_carbon.csv', 'a', errors='ignore', newline='') as f:
f_csv = csv.writer(f)
f_csv.writerows(final)