您的位置：首页 > 编程语言 > Python开发

使用python进行收据搜集示例之different_format_data_processing

2016-12-16 23:54 627 查看

这里是用jupyter notebook写的关于使用python进行数据收集的基本知识，包括crawl_and_parse、different_format_data_processing、feature_engineering_example和python_regular_expression等。之前课程里提供的资料，移植到了python3+windows环境上。代码上传到csdn资源啦：ABC of data_collection 。

为了方便查看，代码分开4篇博客里。

下面是jupyter notebook代码导出的md文件。

2.different_format_data_processing

多种格式数据加载、处理与存储

实际的场景中，我们会在不同的地方遇到各种不同的数据格式（比如大家熟悉的csv与txt，比如网页HTML格式，比如XML格式），我们来一起看看python如何和这些格式的数据打交道。

寒小阳

2016-08

from __future__ import division
from numpy.random import randn
import numpy as np
import os
import sys
import matplotlib.pyplot as plt
np.random.seed(12345)
plt.rc('figure', figsize=(10, 6))
from pandas import Series, DataFrame
import pandas as pd
np.set_printoptions(precision=4)

%pwd

‘C:\\Users\\NodYoung\\Desktop\\Python_Data_Analysis-master\\Python_Data_Analysis-master\\Lesson4_data_collection\\python3\\different_data_formats’

## 1.各式各样的文本数据

### 1.1 CSV与TXT读取

注意：windows下没有cat命令

# !cat data1.csv

df = pd.read_csv('data1.csv')
df


	a	b	c	d	message
0	1	2	3	4	hello
1	5	6	7	8	world
2	9	10	11	12	foo

pd.read_table('data1.csv', sep=',')


	a	b	c	d	message
0	1	2	3	4	hello
1	5	6	7	8	world
2	9	10	11	12	foo

# !cat data2.csv

pd.read_csv('data2.csv', header=None)


	0	1	2	3	4
0	1	2	3	4	hello
1	5	6	7	8	world
2	9	10	11	12	foo

pd.read_csv('data2.csv', names=['a', 'b', 'c', 'd', 'message'])


	a	b	c	d	message
0	1	2	3	4	hello
1	5	6	7	8	world
2	9	10	11	12	foo

names = ['a', 'b', 'c', 'd', 'message']
pd.read_csv('data2.csv', names=names, index_col='message')


	a	b	c	d
message
hello	1	2	3	4
world	5	6	7	8
foo	9	10	11	12

# !cat csv_mindex.csv
parsed = pd.read_csv('csv_mindex.csv', index_col=['key1', 'key2'])
parsed


		value1	value2
key1	key2
one	a	1	2
b	3	4
c	5	6
d	7	8
two	a	9	10
b	11	12
c	13	14
d	15	16

list(open('data3.txt'))

[’ A B C\n’,
‘aaa -0.264438 -1.026059 -0.619500\n’,
‘bbb 0.927272 0.302904 -0.032399\n’,
‘ccc -0.264273 -0.386314 -0.217601\n’,
‘ddd -0.871858 -0.348382 1.100491\n’]

result = pd.read_table('data3.txt', sep='\s+')
result


	A	B	C
aaa	-0.264438	-1.026059	-0.619500
bbb	0.927272	0.302904	-0.032399
ccc	-0.264273	-0.386314	-0.217601
ddd	-0.871858	-0.348382	1.100491

# !cat data4.csv
pd.read_csv('data4.csv', skiprows=[0, 2, 3])


	a	b	c	d	message
0	1	2	3	4	hello
1	5	6	7	8	world
2	9	10	11	12	foo

# !cat data5.csv
result = pd.read_csv('data5.csv')
print(result)
pd.isnull(result)

something a b c d message
0 one 1 2 3.0 4 NaN
1 two 5 6 NaN 8 world
2 three 9 10 11.0 12 foo


	something	a	b	c	d	message
0	False	False	False	False	False	True
1	False	False	False	True	False	False
2	False	False	False	False	False	False

result = pd.read_csv('data5.csv', na_values=['NULL'])
result


	something	a	b	c	d	message
0	one	1	2	3.0	4	NaN
1	two	5	6	NaN	8	world
2	three	9	10	11.0	12	foo

sentinels = {'message': ['foo', 'NA'], 'something': ['two']}
pd.read_csv('data5.csv', na_values=sentinels)


	something	a	b	c	d	message
0	one	1	2	3.0	4	NaN
1	NaN	5	6	NaN	8	world
2	three	9	10	11.0	12	NaN

### 1.2 分片/块读取文本数据

result = pd.read_csv('data6.csv')
result


	one	two	three	four	key
0	0.467976	-0.038649	-0.295344	-1.824726	L
1	-0.358893	1.404453	0.704965	-0.200638	B
2	-0.501840	0.659254	-0.421691	-0.057688	G
3	0.204886	1.074134	1.388361	-0.982404	R
4	0.354628	-0.133116	0.283763	-0.837063	Q
5	1.817480	0.742273	0.419395	-2.251035	Q
6	-0.776764	0.935518	-0.332872	-1.875641	U
7	-0.913135	1.530624	-0.572657	0.477252	K
8	0.358480	-0.497572	-0.367016	0.507702	S
9	-1.740877	-1.160417	-1.637830	2.172201	G
10	0.240564	-0.328249	1.252155	1.072796	8
11	0.764018	1.165476	-0.639544	1.495258	R
12	0.571035	-0.310537	0.582437	-0.298765	1
13	2.317658	0.430710	-1.334216	0.199679	P
14	1.547771	-1.119753	-2.277634	0.329586	J
15	-1.310608	0.401719	-1.000987	1.156708	E
16	-0.088496	0.634712	0.153324	0.415335	B
17	-0.018663	-0.247487	-1.446522	0.750938	A
18	-0.070127	-1.579097	0.120892	0.671432	F
19	-0.194678	-0.492039	2.359605	0.319810	H
20	-0.248618	0.868707	-0.492226	-0.717959	W
21	-1.091549	-0.867110	-0.647760	-0.832562	C
22	0.641404	-0.138822	-0.621963	-0.284839	C
23	1.216408	0.992687	0.165162	-0.069619	V
24	-0.564474	0.792832	0.747053	0.571675	I
25	1.759879	-0.515666	-0.230481	1.362317	S
26	0.126266	0.309281	0.382820	-0.239199	L
27	1.334360	-0.100152	-0.840731	-0.643967	6
28	-0.737620	0.278087	-0.053235	-0.950972	J
29	-1.148486	-0.986292	-0.144963	0.124362	Y
…	…	…	…	…	…
9970	0.633495	-0.186524	0.927627	0.143164	4
9971	0.308636	-0.112857	0.762842	-1.072977	1
9972	-1.627051	-0.978151	0.154745	-1.229037	Z
9973	0.314847	0.097989	0.199608	0.955193	P
9974	1.666907	0.992005	0.496128	-0.686391	S
9975	0.010603	0.708540	-1.258711	0.226541	K
9976	0.118693	-0.714455	-0.501342	-0.254764	K
9977	0.302616	-2.011527	-0.628085	0.768827	H
9978	-0.098572	1.769086	-0.215027	-0.053076	A
9979	-0.019058	1.964994	0.738538	-0.883776	F
9980	-0.595349	0.001781	-1.423355	-1.458477	M
9981	1.392170	-1.396560	-1.425306	-0.847535	H
9982	-0.896029	-0.152287	1.924483	0.365184	6
9983	-2.274642	-0.901874	1.500352	0.996541	N
9984	-0.301898	1.019906	1.102160	2.624526	I
9985	-2.548389	-0.585374	1.496201	-0.718815	D
9986	-0.064588	0.759292	-1.568415	-0.420933	E
9987	-0.143365	-1.111760	-1.815581	0.435274	2
9988	-0.070412	-1.055921	0.338017	-0.440763	X
9989	0.649148	0.994273	-1.384227	0.485120	Q
9990	-0.370769	0.404356	-1.051628	-1.050899	8
9991	-0.409980	0.155627	-0.818990	1.277350	W
9992	0.301214	-1.111203	0.668258	0.671922	A
9993	1.821117	0.416445	0.173874	0.505118	X
9994	0.068804	1.322759	0.802346	0.223618	H
9995	2.311896	-0.417070	-1.409599	-0.515821	L
9996	-0.479893	-0.650419	0.745152	-0.646038	E
9997	0.523331	0.787112	0.486066	1.093156	K
9998	-0.362559	0.598894	-1.843201	0.887292	G
9999	-0.096376	-1.012999	-0.657431	-0.573315	0

10000 rows × 5 columns

pd.read_csv('data6.csv', nrows=5)


	one	two	three	four	key
0	0.467976	-0.038649	-0.295344	-1.824726	L
1	-0.358893	1.404453	0.704965	-0.200638	B
2	-0.501840	0.659254	-0.421691	-0.057688	G
3	0.204886	1.074134	1.388361	-0.982404	R
4	0.354628	-0.133116	0.283763	-0.837063	Q

chunker = pd.read_csv('data6.csv', chunksize=100)
chunker

chunker = pd.read_csv('data6.csv', chunksize=100)

tot = Series([])
for piece in chunker:
tot = tot.add(piece['key'].value_counts(), fill_value=0)

tot = tot.sort_values(ascending=False)

tot[:10]

E 368.0
X 364.0
L 346.0
O 343.0
Q 340.0
M 338.0
J 337.0
F 335.0
K 334.0
H 330.0
dtype: float64

### 1.3 把数据写入文本格式

data = pd.read_csv('data5.csv')
data


	something	a	b	c	d	message
0	one	1	2	3.0	4	NaN
1	two	5	6	NaN	8	world
2	three	9	10	11.0	12	foo

data.to_csv('out.csv')
# !cat out.csv

data.to_csv(sys.stdout, sep='|')

|something|a|b|c|d|message
0|one|1|2|3.0|4|
1|two|5|6||8|world
2|three|9|10|11.0|12|foo

data.to_csv(sys.stdout, na_rep='NULL')

,something,a,b,c,d,message
0,one,1,2,3.0,4,NULL
1,two,5,6,NULL,8,world
2,three,9,10,11.0,12,foo

data.to_csv(sys.stdout, index=False, header=False)

one,1,2,3.0,4,
two,5,6,,8,world
three,9,10,11.0,12,foo

data.to_csv(sys.stdout, index=False, columns=['a', 'b', 'c'])

a,b,c
1,2,3.0
5,6,
9,10,11.0

dates = pd.date_range('1/1/2000', periods=7)
print(dates)
ts = Series(np.arange(7), index=dates)
print(ts)
ts.to_csv('tseries.csv')
# !cat tseries.csv

DatetimeIndex([‘2000-01-01’, ‘2000-01-02’, ‘2000-01-03’, ‘2000-01-04’,
‘2000-01-05’, ‘2000-01-06’, ‘2000-01-07’],
dtype=’datetime64[ns]’, freq=’D’)
2000-01-01 0
2000-01-02 1
2000-01-03 2
2000-01-04 3
2000-01-05 4
2000-01-06 5
2000-01-07 6
Freq: D, dtype: int32

Series.from_csv('tseries.csv', parse_dates=True)

2000-01-01 0
2000-01-02 1
2000-01-03 2
2000-01-04 3
2000-01-05 4
2000-01-06 5
2000-01-07 6
dtype: int64

### 1.4 手动读写数据（按要求）

# !cat data7.csv

import csv
f = open('data7.csv')

reader = csv.reader(f)

for line in reader:
print(line)

[‘a’, ‘b’, ‘c’]
[‘1’, ‘2’, ‘3’]
[‘1’, ‘2’, ‘3’, ‘4’]

lines = list(csv.reader(open('data7.csv')))
header, values = lines[0], lines[1:]
print(values)
data_dict = {h: v for h, v in zip(header, zip(*values))}
print(data_dict)

[[‘1’, ‘2’, ‘3’], [‘1’, ‘2’, ‘3’, ‘4’]]
{‘c’: (‘3’, ‘3’), ‘a’: (‘1’, ‘1’), ‘b’: (‘2’, ‘2’)}

class my_dialect(csv.Dialect):
lineterminator = '\n'
delimiter = ';'
quotechar = '"'
quoting = csv.QUOTE_MINIMAL

with open('mydata.csv', 'w') as f:
writer = csv.writer(f, dialect=my_dialect)
writer.writerow(('one', 'two', 'three'))
writer.writerow(('1', '2', '3'))
writer.writerow(('4', '5', '6'))
writer.writerow(('7', '8', '9'))

# %cat mydata.csv
pd.read_csv('mydata.csv')


	one;two;three
0	1;2;3
1	4;5;6
2	7;8;9

### 1.5 JSON格式的数据

obj = \
"""
{"姓名": "张三",
"住处": ["天朝", "挖煤国", "万恶的资本主义日不落帝国"],
"宠物": null,
"兄弟": [{"姓名": "李四", "年龄": 25, "宠物": "汪星人"},
{"姓名": "王五", "年龄": 23, "宠物": "喵星人"}]
}
"""

import json
result = json.loads(obj)
result

{‘住处’: [‘天朝’, ‘挖煤国’, ‘万恶的资本主义日不落帝国’],
‘兄弟’: [{‘姓名’: ‘李四’, ‘宠物’: ‘汪星人’, ‘年龄’: 25},
{‘姓名’: ‘王五’, ‘宠物’: ‘喵星人’, ‘年龄’: 23}],
‘姓名’: ‘张三’,
‘宠物’: None}

print(json.dumps(result, ensure_ascii=False))

{“兄弟”: [{“年龄”: 25, “宠物”: “汪星人”, “姓名”: “李四”}, {“年龄”: 23, “宠物”: “喵星人”, “姓名”: “王五”}], “宠物”: null, “住处”: [“天朝”, “挖煤国”, “万恶的资本主义日不落帝国”], “姓名”: “张三”}

result[u"兄弟"][0]

{‘姓名’: ‘李四’, ‘宠物’: ‘汪星人’, ‘年龄’: 25}

print(json.dumps(result[u"兄弟"][0], ensure_ascii=False))

{“年龄”: 25, “宠物”: “汪星人”, “姓名”: “李四”}

asjson = json.dumps(result)

brothers = DataFrame(result[u'兄弟'], columns=[u'姓名', u'年龄'])
brothers


	姓名	年龄
0	李四	25
1	王五	23

### 1.6 人人都爱爬虫，人人都要解析XML 和 HTML

from lxml.html import parse
from urllib.request import urlopen

parsed = parse(urlopen('https://ask.julyedu.com/'))

doc = parsed.getroot()

doc

找到’’

links = doc.findall('.//a')
links[15:20]

[,
,
,
,
]

lnk = links[14]
lnk
lnk.get('href')
print(lnk.text_content())

全部问题

urls = [lnk.get('href') for lnk in doc.findall('.//a')]
urls[-10:]

[‘https://ask.julyedu.com/people/lucheng918’,
‘https://ask.julyedu.com/people/lucheng918’,
‘http://weibo.com/askjulyedu’,
None,
‘https://www.julyedu.com/help/index/about’,
‘https://www.julyedu.com/help/index/contact’,
‘https://www.julyedu.com/help/index/join’,
‘http://ask.julyedu.com/question/55’,
‘http://www.julyapp.com’,
‘http://www.miitbeian.gov.cn/’]

spans = doc.findall('.//span')
len(spans)

135

def _unpack(spans):
return [val.text_content() for val in spans]

contents = _unpack(spans)
for content in contents:
print(content)

通知设置
贡献
回复了问题 • 2 人关注 • 1 个回复 • 5 次浏览 • 2 小时前
• 来自相关主题
贡献
回复了问题 • 2 人关注 • 1 个回复 • 3 次浏览 • 2 小时前
• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 14 次浏览 • 6 小时前

• 来自相关主题
贡献
回复了问题 • 2 人关注 • 1 个回复 • 15 次浏览 • 1 天前
• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 23 次浏览 • 1 天前

• 来自相关主题
贡献
回复了问题 • 1 人关注 • 2 个回复 • 21 次浏览 • 1 天前
• 来自相关主题
贡献
回复了问题 • 2 人关注 • 2 个回复 • 23 次浏览 • 1 天前
• 来自相关主题
贡献
回复了问题 • 2 人关注 • 1 个回复 • 35 次浏览 • 1 天前
• 来自相关主题
贡献
回复了问题 • 2 人关注 • 2 个回复 • 55 次浏览 • 2 天前
• 来自相关主题
贡献
回复了问题 • 3 人关注 • 2 个回复 • 84 次浏览 • 2 天前
• 来自相关主题
发起了问题 • 2 人关注 • 0 个回复 • 67 次浏览 • 3 天前

• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 63 次浏览 • 3 天前

• 来自相关主题
贡献
回复了问题 • 3 人关注 • 2 个回复 • 53 次浏览 • 3 天前
• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 309 次浏览 • 3 天前

• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 71 次浏览 • 4 天前

• 来自相关主题
贡献
回复了问题 • 6 人关注 • 8 个回复 • 149 次浏览 • 5 天前
• 来自相关主题
发起了问题 • 2 人关注 • 0 个回复 • 49 次浏览 • 5 天前

• 来自相关主题
贡献
回复了问题 • 3 人关注 • 5 个回复 • 395 次浏览 • 5 天前
• 来自相关主题
贡献
回复了问题 • 34 人关注 • 16 个回复 • 4679 次浏览 • 6 天前
• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 42 次浏览 • 6 天前

• 来自相关主题
发起了问题 • 3 人关注 • 0 个回复 • 78 次浏览 • 2016-12-07 18:41

• 来自相关主题
回复了问题 • 1 人关注 • 1 个回复 • 308 次浏览 • 2016-12-07 18:31
• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 40 次浏览 • 2016-12-07 17:52

• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 39 次浏览 • 2016-12-07 17:37

• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 40 次浏览 • 2016-12-07 15:47

• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 55 次浏览 • 2016-12-06 22:39

• 来自相关主题
发起了问题 • 2 人关注 • 0 个回复 • 65 次浏览 • 2016-12-06 22:37

• 来自相关主题
贡献
回复了问题 • 2 人关注 • 2 个回复 • 63 次浏览 • 2016-12-06 19:37
• 来自相关主题
回复了问题 • 2 人关注 • 2 个回复 • 322 次浏览 • 2016-12-06 19:08
• 来自相关主题
贡献
回复了问题 • 1 人关注 • 2 个回复 • 66 次浏览 • 2016-12-06 17:55
• 来自相关主题
贡献
回复了问题 • 7 人关注 • 4 个回复 • 2726 次浏览 • 2016-12-06 17:48
• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 44 次浏览 • 2016-12-06 11:42

• 来自相关主题
贡献
回复了问题 • 3 人关注 • 1 个回复 • 71 次浏览 • 2016-12-06 10:05
• 来自相关主题
贡献
回复了问题 • 3 人关注 • 2 个回复 • 84 次浏览 • 2016-12-05 11:23
• 来自相关主题
贡献
回复了问题 • 19 人关注 • 22 个回复 • 1221 次浏览 • 2016-12-04 21:22
• 来自相关主题
贡献
回复了问题 • 3 人关注 • 1 个回复 • 73 次浏览 • 2016-12-04 15:51
• 来自相关主题
贡献
回复了问题 • 3 人关注 • 2 个回复 • 63 次浏览 • 2016-12-04 00:20
• 来自相关主题
发起了问题 • 2 人关注 • 0 个回复 • 431 次浏览 • 2016-12-03 17:34

• 来自相关主题
贡献
回复了问题 • 2 人关注 • 3 个回复 • 91 次浏览 • 2016-12-03 11:38
• 来自相关主题
贡献
回复了问题 • 2 人关注 • 1 个回复 • 51 次浏览 • 2016-12-02 10:45
• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 74 次浏览 • 2016-12-02 10:08

• 来自相关主题
贡献
回复了问题 • 2 人关注 • 1 个回复 • 58 次浏览 • 2016-12-01 22:16
• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 68 次浏览 • 2016-12-01 10:47

• 来自相关主题
贡献
回复了问题 • 47 人关注 • 101 个回复 • 4228 次浏览 • 2016-12-01 10:00
• 来自相关主题
贡献
回复了问题 • 2 人关注 • 2 个回复 • 50 次浏览 • 2016-12-01 09:58
• 来自相关主题
贡献
回复了问题 • 4 人关注 • 6 个回复 • 373 次浏览 • 2016-11-30 21:56
• 来自相关主题
贡献
回复了问题 • 2 人关注 • 1 个回复 • 79 次浏览 • 2016-11-30 21:36
• 来自相关主题
贡献
回复了问题 • 2 人关注 • 1 个回复 • 43 次浏览 • 2016-11-30 21:30
• 来自相关主题
发起了问题 • 1 人关注 • 0 个回复 • 74 次浏览 • 2016-11-30 10:11

• 来自相关主题
贡献
回复了问题 • 2 人关注 • 1 个回复 • 95 次浏览 • 2016-11-29 15:59
• 来自相关主题
返回顶部

var cnzz_protocol = ((“https:” == document.location.protocol) ? ” https://” : ” http://”);document.write(unescape(“%3Cspan id=’cnzz_stat_icon_1259748782’%3E%3C/span%3E%3Cscript src=’” + cnzz_protocol + “s11.cnzz.com/z_stat.php%3Fid%3D1259748782%26show%3Dpic’ type=’text/javascript’%3E%3C/script%3E”));

questions = doc.findall('.//h4')
len(questions)

contents = _unpack(questions)
for content in contents:
print(content)

《自动聊天机器人项目班》

七月在线app中下载的视频声音有杂音，听不清？

狂欢第三天：转盘直抽MacBook Air和锤子 M1

怎么分享活动页面呢

python爬虫报错 UnicodeEncodeError: ‘ascii’

量化课以及机器学习应用班的代码

七月在线双12三天大狂欢之第二天：转盘直播GTX 1080显卡和ipad mini4

进群核对名字没填咋办

How to buy some expired video classes from US?

七月在线双12三天大狂欢：转盘直抽iPhone 7、海量课程优惠券、全场课程限时特价秒杀

终于来了，《自然语言处理班》大纲讨论稿

今12.11之内抢购《Python数据分析班》，只需9元

人工神经网络输出至Softmax分类器的思考

mxnet examples 实验笔记(一)

今天12.10日之内，免费送《机器学习中的数学班》

希望能增加1-2倍的倍速播放

10月ML班第13讲作业图模型（违反直觉的贝叶斯…）

《PHP与后端架构实战班》大纲讨论稿

一起刷Kaggle（数据挖掘、机器学习在线比赛）

训练好后, tensorflow哪个方法是根据一个input输出一个output的…

双12三天大狂欢：转盘抽奖iPhone 7、定时抢券、课程限时秒杀

Python网络爬虫实战项目代码大全

python,theano的typeerror调试

EM推导中的一个问题

python有木有把灰度图像转float的方法.

数学微课II：《矩阵与凸优化》大纲讨论稿

数学微课I：《概率论与数理统计》大纲讨论稿

《自动聊天机器人项目班》，期待啥内容

如何入门机器学习

EM中参数求解

Mac Neural Conversational Model 自动聊天机器人实验

傅盛：未来的公司本质都是数据公司

七月在线讲的到底怎么样

【11月深度学习第三讲】第二层卷积操作的对象是否将第一层的输出合并为一张图？

《spark 机器学习班》，你最想听到啥内容

GPU云租用或攒GPU机

我想问下想人工智能机器学习这些要求是不是很高

python 爬虫笔记（二）：开源代理池，图片和社交网络数据的爬取

七月在线2016年度最火爆课程TOP 10

bagging分类时的概率计算问题

李开复：人工智能时代来临

云环境使用

tensorflow逻辑回归模型稀疏向量表示

一起来吐槽——对官网跟问答社区，有任何反馈或意见？

tf实现2元线性回归

Python爬虫项目班，大纲讨论稿

七月在线即将推出线上数据大赛

七月在线总学员人数突破9000

1 元限时秒杀《动态规划实战班》，仅限今11.30日

寒老师您好，请问训练好的word2vec模型，然后怎们能够输入某个词，然后得到这个词的向量表示呢！多谢您！！

### 1.7 解析XML

# !head -21 Performance_MNR.xml

from lxml import objectify

path = 'Performance_MNR.xml'
parsed = objectify.parse(open(path))
root = parsed.getroot()

data = []

skip_fields = ['PARENT_SEQ', 'INDICATOR_SEQ',
'DESIRED_CHANGE', 'DECIMAL_PLACES']

for elt in root.INDICATOR:
el_data = {}
for child in elt.getchildren():
if child.tag in skip_fields:
continue
el_data[child.tag] = child.pyval
data.append(el_data)

from pandas import DataFrame

perf = DataFrame(data)
perf


	AGENCY_NAME	CATEGORY	DESCRIPTION	FREQUENCY	INDICATOR_NAME	INDICATOR_UNIT	MONTHLY_ACTUAL	MONTHLY_TARGET	PERIOD_MONTH	PERIOD_YEAR	YTD_ACTUAL	YTD_TARGET
0	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.9	95	1	2008	96.9	95
1	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	95	95	2	2008	96	95
2	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.9	95	3	2008	96.3	95
3	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	98.3	95	4	2008	96.8	95
4	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	95.8	95	5	2008	96.6	95
5	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	94.4	95	6	2008	96.2	95
6	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96	95	7	2008	96.2	95
7	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.4	95	8	2008	96.2	95
8	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	93.7	95	9	2008	95.9	95
9	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.4	95	10	2008	96	95
10	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.9	95	11	2008	96.1	95
11	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	95.1	95	12	2008	96	95
12	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	92.6	96.2	1	2009	92.6	96.2
13	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.8	96.2	2	2009	94.6	96.2
14	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.9	96.2	3	2009	95.4	96.2
15	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	97.1	96.2	4	2009	95.9	96.2
16	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	97.8	96.2	5	2009	96.2	96.2
17	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	97.3	96.2	6	2009	96.4	96.2
18	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.7	96.2	7	2009	96.5	96.2
19	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	95.7	96.2	8	2009	96.4	96.2
20	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.1	96.2	9	2009	96.3	96.2
21	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	94.8	96.2	10	2009	96.2	96.2
22	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	95.7	96.2	11	2009	96.1	96.2
23	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	95	96.2	12	2009	96	96.2
24	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	98	96.3	1	2010	98	96.3
25	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	93	96.3	2	2010	95.6	96.3
26	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.9	96.3	3	2010	96.1	96.3
27	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	98.1	96.3	4	2010	96.6	96.3
28	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	97.6	96.3	5	2010	96.8	96.3
29	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	97.4	96.3	6	2010	96.9	96.3
…	…	…	…	…	…	…	…	…	…	…	…	…
618	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	94		7	2009	95.14
619	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	97		8	2009	95.38
620	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	98.3		9	2009	95.7
621	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	98.7		10	2009	96
622	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	98.1		11	2009	96.21
623	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100		12	2009	96.5
624	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	97.95	97	1	2010	97.95	97
625	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	2	2010	98.92	97
626	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	3	2010	99.29	97
627	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	4	2010	99.47	97
628	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	5	2010	99.58	97
629	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	91.21	97	6	2010	98.19	97
630	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	7	2010	98.46	97
631	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	8	2010	98.69	97
632	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	95.2	97	9	2010	98.3	97
633	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	90.91	97	10	2010	97.55	97
634	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	96.67	97	11	2010	97.47	97
635	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	90.03	97	12	2010	96.84	97
636	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	1	2011	100	97
637	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	2	2011	100	97
638	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	97.07	97	3	2011	98.86	97
639	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	98.18	97	4	2011	98.76	97
640	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	79.18	97	5	2011	90.91	97
641	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	6	2011		97
642	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	7	2011		97
643	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	8	2011		97
644	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	9	2011		97
645	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	10	2011		97
646	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	11	2011		97
647	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	12	2011		97

648 rows × 12 columns

root

root.get('href')

root.text

## 二进制格式的数据

import pandas as pd

frame = pd.read_csv('data1.csv')
frame
frame.to_pickle('frame_pickle')

pd.read_pickle('frame_pickle')


	a	b	c	d	message
0	1	2	3	4	hello
1	5	6	7	8	world
2	9	10	11	12	foo

### 使用HDF5格式

store = pd.HDFStore('mydata.h5')
store['obj1'] = frame
store['obj1_col'] = frame['a']
store

store['obj1']


	a	b	c	d	message
0	1	2	3	4	hello
1	5	6	7	8	world
2	9	10	11	12	foo

import os
store.close()
os.remove('mydata.h5')

### HTML与API交互

import requests
url = 'https://api.github.com/repos/pydata/pandas/milestones/28/labels'
resp = requests.get(url)
resp

data[:5]

[{‘AGENCY_NAME’: ‘Metro-North Railroad’,
‘CATEGORY’: ‘Service Indicators’,
‘DESCRIPTION’: ‘Percent of commuter trains that arrive at their destinations within 5 minutes and 59 seconds of the scheduled time. West of Hudson services include the Pascack Valley and Port Jervis lines. Metro-North Railroad contracts with New Jersey Transit to operate service on these lines.\n’,
‘FREQUENCY’: ‘M’,
‘INDICATOR_NAME’: ‘On-Time Performance (West of Hudson)’,
‘INDICATOR_UNIT’: ‘%’,
‘MONTHLY_ACTUAL’: 96.9,
‘MONTHLY_TARGET’: 95.0,
‘PERIOD_MONTH’: 1,
‘PERIOD_YEAR’: 2008,
‘YTD_ACTUAL’: 96.9,
‘YTD_TARGET’: 95.0},
{‘AGENCY_NAME’: ‘Metro-North Railroad’,
‘CATEGORY’: ‘Service Indicators’,
‘DESCRIPTION’: ‘Percent of commuter trains that arrive at their destinations within 5 minutes and 59 seconds of the scheduled time. West of Hudson services include the Pascack Valley and Port Jervis lines. Metro-North Railroad contracts with New Jersey Transit to operate service on these lines.\n’,
‘FREQUENCY’: ‘M’,
‘INDICATOR_NAME’: ‘On-Time Performance (West of Hudson)’,
‘INDICATOR_UNIT’: ‘%’,
‘MONTHLY_ACTUAL’: 95.0,
‘MONTHLY_TARGET’: 95.0,
‘PERIOD_MONTH’: 2,
‘PERIOD_YEAR’: 2008,
‘YTD_ACTUAL’: 96.0,
‘YTD_TARGET’: 95.0},
{‘AGENCY_NAME’: ‘Metro-North Railroad’,
‘CATEGORY’: ‘Service Indicators’,
‘DESCRIPTION’: ‘Percent of commuter trains that arrive at their destinations within 5 minutes and 59 seconds of the scheduled time. West of Hudson services include the Pascack Valley and Port Jervis lines. Metro-North Railroad contracts with New Jersey Transit to operate service on these lines.\n’,
‘FREQUENCY’: ‘M’,
‘INDICATOR_NAME’: ‘On-Time Performance (West of Hudson)’,
‘INDICATOR_UNIT’: ‘%’,
‘MONTHLY_ACTUAL’: 96.9,
‘MONTHLY_TARGET’: 95.0,
‘PERIOD_MONTH’: 3,
‘PERIOD_YEAR’: 2008,
‘YTD_ACTUAL’: 96.3,
‘YTD_TARGET’: 95.0},
{‘AGENCY_NAME’: ‘Metro-North Railroad’,
‘CATEGORY’: ‘Service Indicators’,
‘DESCRIPTION’: ‘Percent of commuter trains that arrive at their destinations within 5 minutes and 59 seconds of the scheduled time. West of Hudson services include the Pascack Valley and Port Jervis lines. Metro-North Railroad contracts with New Jersey Transit to operate service on these lines.\n’,
‘FREQUENCY’: ‘M’,
‘INDICATOR_NAME’: ‘On-Time Performance (West of Hudson)’,
‘INDICATOR_UNIT’: ‘%’,
‘MONTHLY_ACTUAL’: 98.3,
‘MONTHLY_TARGET’: 95.0,
‘PERIOD_MONTH’: 4,
‘PERIOD_YEAR’: 2008,
‘YTD_ACTUAL’: 96.8,
‘YTD_TARGET’: 95.0},
{‘AGENCY_NAME’: ‘Metro-North Railroad’,
‘CATEGORY’: ‘Service Indicators’,
‘DESCRIPTION’: ‘Percent of commuter trains that arrive at their destinations within 5 minutes and 59 seconds of the scheduled time. West of Hudson services include the Pascack Valley and Port Jervis lines. Metro-North Railroad contracts with New Jersey Transit to operate service on these lines.\n’,
‘FREQUENCY’: ‘M’,
‘INDICATOR_NAME’: ‘On-Time Performance (West of Hudson)’,
‘INDICATOR_UNIT’: ‘%’,
‘MONTHLY_ACTUAL’: 95.8,
‘MONTHLY_TARGET’: 95.0,
‘PERIOD_MONTH’: 5,
‘PERIOD_YEAR’: 2008,
‘YTD_ACTUAL’: 96.6,
‘YTD_TARGET’: 95.0}]

issue_labels = DataFrame(data)
issue_labels


	AGENCY_NAME	CATEGORY	DESCRIPTION	FREQUENCY	INDICATOR_NAME	INDICATOR_UNIT	MONTHLY_ACTUAL	MONTHLY_TARGET	PERIOD_MONTH	PERIOD_YEAR	YTD_ACTUAL	YTD_TARGET
0	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.9	95	1	2008	96.9	95
1	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	95	95	2	2008	96	95
2	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.9	95	3	2008	96.3	95
3	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	98.3	95	4	2008	96.8	95
4	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	95.8	95	5	2008	96.6	95
5	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	94.4	95	6	2008	96.2	95
6	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96	95	7	2008	96.2	95
7	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.4	95	8	2008	96.2	95
8	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	93.7	95	9	2008	95.9	95
9	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.4	95	10	2008	96	95
10	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.9	95	11	2008	96.1	95
11	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	95.1	95	12	2008	96	95
12	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	92.6	96.2	1	2009	92.6	96.2
13	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.8	96.2	2	2009	94.6	96.2
14	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.9	96.2	3	2009	95.4	96.2
15	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	97.1	96.2	4	2009	95.9	96.2
16	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	97.8	96.2	5	2009	96.2	96.2
17	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	97.3	96.2	6	2009	96.4	96.2
18	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.7	96.2	7	2009	96.5	96.2
19	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	95.7	96.2	8	2009	96.4	96.2
20	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.1	96.2	9	2009	96.3	96.2
21	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	94.8	96.2	10	2009	96.2	96.2
22	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	95.7	96.2	11	2009	96.1	96.2
23	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	95	96.2	12	2009	96	96.2
24	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	98	96.3	1	2010	98	96.3
25	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	93	96.3	2	2010	95.6	96.3
26	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	96.9	96.3	3	2010	96.1	96.3
27	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	98.1	96.3	4	2010	96.6	96.3
28	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	97.6	96.3	5	2010	96.8	96.3
29	Metro-North Railroad	Service Indicators	Percent of commuter trains that arrive at thei…	M	On-Time Performance (West of Hudson)	%	97.4	96.3	6	2010	96.9	96.3
…	…	…	…	…	…	…	…	…	…	…	…	…
618	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	94		7	2009	95.14
619	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	97		8	2009	95.38
620	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	98.3		9	2009	95.7
621	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	98.7		10	2009	96
622	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	98.1		11	2009	96.21
623	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100		12	2009	96.5
624	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	97.95	97	1	2010	97.95	97
625	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	2	2010	98.92	97
626	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	3	2010	99.29	97
627	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	4	2010	99.47	97
628	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	5	2010	99.58	97
629	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	91.21	97	6	2010	98.19	97
630	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	7	2010	98.46	97
631	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	8	2010	98.69	97
632	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	95.2	97	9	2010	98.3	97
633	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	90.91	97	10	2010	97.55	97
634	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	96.67	97	11	2010	97.47	97
635	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	90.03	97	12	2010	96.84	97
636	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	1	2011	100	97
637	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	100	97	2	2011	100	97
638	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	97.07	97	3	2011	98.86	97
639	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	98.18	97	4	2011	98.76	97
640	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%	79.18	97	5	2011	90.91	97
641	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	6	2011		97
642	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	7	2011		97
643	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	8	2011		97
644	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	9	2011		97
645	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	10	2011		97
646	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	11	2011		97
647	Metro-North Railroad	Service Indicators	Percent of the time that escalators are operat…	M	Escalator Availability	%		97	12	2011		97

648 rows × 12 columns

## 2.数据库相关操作

## 2.1 sqlite数据库

import sqlite3

query = """
CREATE TABLE test
(a VARCHAR(20), b VARCHAR(20),
c REAL,        d INTEGER
);"""

con = sqlite3.connect(':memory:')
con.execute(query)
con.commit()

data = [('Atlanta', 'Georgia', 1.25, 6),
('Tallahassee', 'Florida', 2.6, 3),
('Sacramento', 'California', 1.7, 5)]
stmt = "INSERT INTO test VALUES(?, ?, ?, ?)"

con.executemany(stmt, data)
con.commit()

cursor = con.execute('select * from test')
rows = cursor.fetchall()
rows

[(‘Atlanta’, ‘Georgia’, 1.25, 6),
(‘Tallahassee’, ‘Florida’, 2.6, 3),
(‘Sacramento’, ‘California’, 1.7, 5)]

cursor.description

((‘a’, None, None, None, None, None, None),
(‘b’, None, None, None, None, None, None),
(‘c’, None, None, None, None, None, None),
(‘d’, None, None, None, None, None, None))

DataFrame(rows, columns=next(zip(*cursor.description)))


	a	b	c	d
0	Atlanta	Georgia	1.25	6
1	Tallahassee	Florida	2.60	3
2	Sacramento	California	1.70	5

import pandas.io.sql as sql
sql.read_sql('select * from test', con)


	a	b	c	d
0	Atlanta	Georgia	1.25	6
1	Tallahassee	Florida	2.60	3
2	Sacramento	California	1.70	5

3.2 MySQL数据库

#coding=utf-8
import pymysql

conn= pymysql.connect(
host='localhost',
port = 3306,
user='root',
passwd='123456',
db ='test',
)
cur = conn.cursor()

#创建数据表
#cur.execute("create table student(id int ,name varchar(20),class varchar(30),age varchar(10))")

#插入一条数据
#cur.execute("insert into student values('2','Tom','3 year 2 class','9')")

#修改查询条件的数据
#cur.execute("update student set class='3 year 1 class' where name = 'Tom'")

#删除查询条件的数据
#cur.execute("delete from student where age='9'")

cur.close()
conn.commit()
conn.close()

---------------------------------------------------------------------------

ConnectionRefusedError                    Traceback (most recent call last)

C:\Program Files\Anaconda3\lib\site-packages\pymysql\connections.py in connect(self, sock)
889                             sock = socket.create_connection(
--> 890                                 (self.host, self.port), self.connect_timeout)
891                             break

C:\Program Files\Anaconda3\lib\socket.py in create_connection(address, timeout, source_address)
710     if err is not None:
--> 711         raise err
712     else:

C:\Program Files\Anaconda3\lib\socket.py in create_connection(address, timeout, source_address)
701                 sock.bind(source_address)
--> 702             sock.connect(sa)
703             return sock

ConnectionRefusedError: [WinError 10061] 由于目标计算机积极拒绝，无法连接。

During handling of the above exception, another exception occurred:

OperationalError                          Traceback (most recent call last)

<ipython-input-46-1caebd09ae3e> in <module>()
7         user='root',
8         passwd='123456',
----> 9         db ='test',
10         )
11 cur = conn.cursor()

C:\Program Files\Anaconda3\lib\site-packages\pymysql\__init__.py in Connect(*args, **kwargs)
88     """
89     from .connections import Connection
---> 90     return Connection(*args, **kwargs)
91
92 from pymysql import connections as _orig_conn

C:\Program Files\Anaconda3\lib\site-packages\pymysql\connections.py in __init__(self, host, user, password, database, port, unix_socket, charset, sql_mode, read_default_file, conv, use_unicode, client_flag, cursorclass, init_command, connect_timeout, ssl, read_default_group, compress, named_pipe, no_delay, autocommit, db, passwd, local_infile, max_allowed_packet, defer_connect, auth_plugin_map, read_timeout, write_timeout)
686             self._sock = None
687         else:
--> 688             self.connect()
689
690     def _create_ssl_ctx(self, sslp):

C:\Program Files\Anaconda3\lib\site-packages\pymysql\connections.py in connect(self, sock)
935                 exc.traceback = traceback.format_exc()
936                 if DEBUG: print(exc.traceback)
--> 937                 raise exc
938
939             # If e is neither DatabaseError or IOError, It's a bug.

OperationalError: (2003, "Can't connect to MySQL server on 'localhost' ([WinError 10061] 由于目标计算机积极拒绝，无法连接。)")

3.3 Memcache

#coding:utf8
import memcache

class MemcachedClient():
''' python memcached 客户端操作示例 '''

def __init__(self, hostList):
self.__mc = memcache.Client(hostList);

def set(self, key, value):
result = self.__mc.set("name", "NieYong")
return result

def get(self, key):
name = self.__mc.get("name")
return name

def delete(self, key):
result = self.__mc.delete("name")
return result

if __name__ == '__main__':
mc = MemcachedClient(["127.0.0.1:11511", "127.0.0.1:11512"])
key = "name"
result = mc.set(key, "NieYong")
print("set的结果：", result)
name = mc.get(key)
print("get的结果：", name)
result = mc.delete(key)
print("delete的结果：", result)

---------------------------------------------------------------------------

ImportError                               Traceback (most recent call last)

<ipython-input-48-1c58d1aac9bf> in <module>()
1
----> 2 import memcache
3
4 class MemcachedClient():
5     ''' python memcached 客户端操作示例 '''

ImportError: No module named 'memcache'

3.4 MongoDB

#encoding:utf=8
import pymongo

connection=pymongo.Connection('10.32.38.50',27017)

#选择myblog库
db=connection.myblog

# 使用users集合
collection=db.users

# 添加单条数据到集合中
user = {"name":"cui","age":"10"}
collection.insert(user)

#同时添加多条数据到集合中
users=[{"name":"cui","age":"9"},{"name":"cui","age":"11"}]
collection.insert(users)

#查询单条记录
print(collection.find_one()  )

#查询所有记录
for data in collection.find():
print(data  )

#查询此集合中数据条数
print(collection.count())

#简单参数查询
for data in collection.find({"name":"1"}):
print(data)

#使用find_one获取一条记录
print(collection.find_one({"name":"1"}))

#高级查询
print("__________________________________________")
print('''''collection.find({"age":{"$gt":"10"}})''')
print("__________________________________________")
for data in collection.find({"age":{"$gt":"10"}}).sort("age"):
print(data)

# 查看db下的所有集合
print(db.collection_names())

---------------------------------------------------------------------------

ImportError                               Traceback (most recent call last)

<ipython-input-49-7bcb5fe4f264> in <module>()
1
----> 2 import pymongo
3
4 connection=pymongo.Connection('10.32.38.50',27017)
5

ImportError: No module named 'pymongo'

内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理

标签：

相关文章推荐

新的分享

章节导航