python爬虫——爬去淘宝商品页面，总是跳转到登录界面

考虑到cookie的问题
但是在headers={}中添加cookie 的信息，仍然跳转。时而跳转时而不跳转

在成功3次后，再次重定向到登录界面
考虑可能原因：淘宝反爬机制

import requests
import re
def getHTMLText(url):
try:
headers = {
"user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36",
"cookie": "_uab_collina=157258226625083933457665; thw=cn; t=36d1cd24cf0143fb6accdf025534d197; enc=97GhrHhKkErSIlgzQuOf4gDv8yB1IDMrzS%2FqNp8OhQXosfA5%2Bpm6Vj4%2B%2FjCYIIsIglI%2FeakHaMTRg2bsOCGe%2Fg%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; cookie2=130c64bc2ccdc80f7d3858f7c4143707; _tb_token_=ede063be6e3ee; XSRF-TOKEN=a5e7e17c-f1d8-4ece-8f63-20dd0adc170c; mt=ci=0_0; cna=PpdBFupPaykCAbdAPqcqw59D; isg=BFlZdWsdY9ah6DznWNPShWvxaEwz5k2YGu7yC3sO5wD_gngUwTA4a72QgAZROuXQ; l=cBLufjhqqvUwqS6yBOCZhurza7799IRAguPzaNbMi_5CU6L65G7Oovk9xFp6cjWdOrYp4-ERe5p9-eteiNF7dhspXUJ1."
}
r=requests.get(url,timeout=30,headers=headers)
r.raise_for_status()
r.encoding=r.apparent_encoding
return r.text
except:
return ""
def parsePage(ilt,html):
try:
plt=re.findall(r'\"view_price\"\:\"[\d\.]*\"',html)
tlt=re.findall(r'\"raw_title\"\:\".*?\"',html)
for i in range(len(plt)):
price=eval(plt[i].split(':')[1])
title=eval(tlt[i].split(':')[1])
ilt.append([price,title])
except:
return ""
def printGoodsList(ilt):
tply="{:4}\t{:8}\t{:16}"
print(tply.format("序号","价格","商品价格"))
count=1
for g in ilt:
count=count+1
print(tply.format(count,g[0],g[1]))
def main():
goods="狗屎"
depth=3
strat_url='https://s.taobao.com/search?q='+goods
infoList=[]
for i in  range(depth):
try:
url=strat_url+"&s="+str(44*i)
html=getHTMLText(url)
parsePage(infoList,html)
except:
continue
printGoodsList(infoList)
main()

• 点赞
• 收藏
• 分享
• 文章举报