python 爬取数据，并写入数据库

# coding=utf-8

import json
import requests
from bs4 import BeautifulSoup

import db

# 获取data
def get_data(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'};
response = requests.post(url, headers=headers, verify=False);
response.encoding = 'utf-8';
text = response.text;
# text = text.replace(' ', '');
# text = text.replace('	', '');
text = text.replace('\n', '');
text = text.replace('\r', '');
return text;

# 数据写入文件
def writeData(listData):
db.dbInsert(listData)
# file = open("jd.txt", "a+", encoding='utf-8');
# for data in listData:
#     writeData = json.dumps(data, ensure_ascii=False);
#     file.writelines(writeData);
#     file.writelines('\n');
# file.flush();
# file.close();

# 分析出想要的数据
# 返回的数据格式数list<Object>
def analyzeData(html):
listData = [];
soup = BeautifulSoup(html, 'lxml');
contentList = soup.select('div.p-name');
for content in contentList:
detail = {};
detail['key'] = content.select('a')[0].attrs.get('href');
detail['value'] = content.text.strip();
listData.append(detail);
return listData;

page = 0;
lastKey = '';
while True:
# 分页获取数据
page = page + 1;
url = "";
data = get_data(url);
listData = analyzeData(data);
lastKeyTemp = listData[len(listData) - 1].get('key');
if lastKeyTemp == lastKey:
break;
else:
lastKey = lastKeyTemp;
writeData(listData);

import psycopg2def dbInsert(listData):conn = psycopg2.connect(database='', user='', password='',host='',port='5432')cur = conn.cursor()for data in listData:try:cur.execute(r"""INSERT INTO t_data ("key", "value") VALUES('""" + data['key'] + """', '""" + data['value'] + """')""")except psycopg2.Error:continueconn.commit()# print('insert successfully')conn.close()