对获取简单网页数据方法进行封装

对获取简单网页数据方法进行封装

# coding: utf-8
import urllib2
import urllib
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
import re

class BasicsQequest():

def __init__(self):
self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
self.headers = {'User-Agent': self.user_agent}

def basics_quest(self, url):
try:
# 构建请求的request
request = urllib2.Request(url, headers=self.headers)
# 利用urlopen获取页面代码
response = urllib2.urlopen(request)
# 将页面转化为UTF-8编码
pageCode = response.read().decode('utf-8')
# print pageCode
return pageCode
except urllib2.URLError, e:
if hasattr(e, "reason"):
print u"连接失败,错误原因", e.reason
return None

def basics_re(self, url, str_re):
pageCode = self.basics_quest(url)
if not pageCode:
print "页面加载失败...."
return None
pattern = re.compile(str_re, re.S)
items = pattern.findall(pageCode)
# for item in items:
# print item
return list(items)