用Python实现应用Last-Modified和ETag避免下载重复内容

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Created on Nov 9, 2011

@author: li3huo
'''

import urllib, urllib2
import sys
import time
class DefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
"""用来保证请求中记录Http状态
"""
def http_error_default(self, req, fp, code, msg, headers):
result = urllib2.HTTPError(
req.get_full_url(), code, msg, headers, fp)
result.status = code
return result

class Sample():
"""a sample is the url i want to download
"""
url = None
contentLength = 0
etag = None
lastModified = None
data = None
path = None

def __init__(self, url, contentLength=0, etag=None, lastModified=None):
self.url = url
self.contentLength = 0
self.etag = etag
self.lastModified = lastModified
self.status = 200
self.file = file
def __repr__(self):
return repr("Http Status=%d; Length=%d; Last Modified Time=%s; eTag=%s" % (self.status, self.contentLength, self.lastModified, self.etag))

def downloadSample(self):
request = urllib2.Request(self.url)
request.add_header('User-Agent', "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en)")
if self.lastModified:
request.add_header('If-Modified-Since', self.lastModified)
if self.etag:
request.add_header('If-None-Match', self.etag)
conn = urllib2.build_opener(DefaultErrorHandler()).open(request)

if hasattr(conn, 'headers'):
# save ETag, if the server sent one
self.etag = conn.headers.get('ETag')
# save Last-Modified header, if the server sent one
self.lastModified = conn.headers.get('Last-Modified')

self.contentLength = conn.headers.get("content-length")

if hasattr(conn, 'status'):
self.status = conn.status
print "status=%d" % self.status

self.data = conn.read()

if self.status == 304:
print "the content is same, so return nothing!"

if not self.contentLength:
self.contentLength = len(self.data)

conn.close()

if __name__ == '__main__':
url = 'http://www.sina.com.cn'
sample = Sample(url)
sample.downloadSample()
print sample
sample.downloadSample()
print sample