pool=eventlet.GreenPool(20)

默认是1000,可能是我在 ubuntu下unlimit -n没有调 ，或是python的httplib有问题，抓同一个站就抱错

r = h.getresponse()
File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
response.begin()
File "/usr/lib/python2.6/httplib.py", line 391, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
func(*args, **kwargs)
File "batch_download.py", line 17, in fetch
html=urllib2.urlopen(url).read()
File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.6/urllib2.py", line 391, in open
response = self._open(req, data)
File "/usr/lib/python2.6/urllib2.py", line 409, in _open
'_open', req)
File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
r = h.getresponse()
File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
response.begin()
File "/usr/lib/python2.6/httplib.py", line 391, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
func(*args, **kwargs)
File "batch_download.py", line 17, in fetch
html=urllib2.urlopen(url).read()
File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.6/urllib2.py", line 391, in open
response = self._open(req, data)
File "/usr/lib/python2.6/urllib2.py", line 409, in _open
'_open', req)
File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
r = h.getresponse()
File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
response.begin()
File "/usr/lib/python2.6/httplib.py", line 391, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
func(*args, **kwargs)
File "batch_download.py", line 17, in fetch
html=urllib2.urlopen(url).read()
File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.6/urllib2.py", line 391, in open
response = self._open(req, data)
File "/usr/lib/python2.6/urllib2.py", line 409, in _open
'_open', req)
File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
r = h.getresponse()
File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
response.begin()
File "/usr/lib/python2.6/httplib.py", line 391, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
func(*args, **kwargs)
File "batch_download.py", line 17, in fetch
html=urllib2.urlopen(url).read()
File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.6/urllib2.py", line 391, in open
response = self._open(req, data)
File "/usr/lib/python2.6/urllib2.py", line 409, in _open
'_open', req)
File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
r = h.getresponse()
File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
response.begin()
File "/usr/lib/python2.6/httplib.py", line 391, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
func(*args, **kwargs)
File "batch_download.py", line 17, in fetch
html=urllib2.urlopen(url).read()
File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.6/urllib2.py", line 391, in open
response = self._open(req, data)
File "/usr/lib/python2.6/urllib2.py", line 409, in _open
'_open', req)
File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
r = h.getresponse()
File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
response.begin()
File "/usr/lib/python2.6/httplib.py", line 391, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
func(*args, **kwargs)
File "batch_download.py", line 17, in fetch
html=urllib2.urlopen(url).read()
File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.6/urllib2.py", line 391, in open
response = self._open(req, data)
File "/usr/lib/python2.6/urllib2.py", line 409, in _open
'_open', req)
File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
r = h.getresponse()
File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
response.begin()
File "/usr/lib/python2.6/httplib.py", line 391, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
func(*args, **kwargs)
File "batch_download.py", line 17, in fetch
html=urllib2.urlopen(url).read()
File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.6/urllib2.py", line 391, in open
response = self._open(req, data)
File "/usr/lib/python2.6/urllib2.py", line 409, in _open
'_open', req)
File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
r = h.getresponse()
File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
response.begin()
File "/usr/lib/python2.6/httplib.py", line 391, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
func(*args, **kwargs)
File "batch_download.py", line 17, in fetch
html=urllib2.urlopen(url).read()
File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.6/urllib2.py", line 391, in open
response = self._open(req, data)
File "/usr/lib/python2.6/urllib2.py", line 409, in _open
'_open', req)
File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
r = h.getresponse()
File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
response.begin()
File "/usr/lib/python2.6/httplib.py", line 391, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
raise BadStatusLine(line)
BadStatusLine
Traceback (most recent call last):
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
func(*args, **kwargs)
File "batch_download.py", line 17, in fetch
html=urllib2.urlopen(url).read()
File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.6/urllib2.py", line 391, in open
response = self._open(req, data)
File "/usr/lib/python2.6/urllib2.py", line 409, in _open
'_open', req)
File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.6/urllib2.py", line 1134, in do_open
r = h.getresponse()
File "/usr/lib/python2.6/httplib.py", line 986, in getresponse
response.begin()
File "/usr/lib/python2.6/httplib.py", line 391, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.6/httplib.py", line 355, in _read_status
raise BadStatusLine(line)
BadStatusLine
^CTraceback (most recent call last):
File "batch_download.py", line 29, in <module>
pool.spawn_n(fetch, url)
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 104, in spawn_n
self.sem.acquire()
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/semaphore.py", line 71, in acquire
hubs.get_hub().switch()
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/hubs/hub.py", line 135, in switch
return self.greenlet.switch()
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/hubs/hub.py", line 169, in run
self.fire_timers(self.clock())
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/hubs/hub.py", line 285, in fire_timers
timer()
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/hubs/timer.py", line 56, in __call__
cb(*args, **kw)
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
func(*args, **kwargs)
File "batch_download.py", line 17, in fetch
html=urllib2.urlopen(url).read()
File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.6/urllib2.py", line 391, in open
response = self._open(req, data)
File "/usr/lib/python2.6/urllib2.py", line 409, in _open
'_open', req)
File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.6/urllib2.py", line 1133, in do_open
h.request(req.get_method(), req.get_selector(), req.data, headers)
File "/usr/lib/python2.6/httplib.py", line 910, in request
^[[A self._send_request(method, url, body, headers)
File "/usr/lib/python2.6/httplib.py", line 947, in _send_request
self.endheaders()
File "/usr/lib/python2.6/httplib.py", line 904, in endheaders
self._send_output()
File "/usr/lib/python2.6/httplib.py", line 776, in _send_output
self.send(msg)
File "/usr/lib/python2.6/httplib.py", line 735, in send
self.connect()
File "/usr/lib/python2.6/httplib.py", line 716, in connect
self.timeout)
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/green/socket.py", line 78, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
KeyboardInterrupt
mlzboy@mlzboy-mac:~/my/idea2/download/hqs$ python batch_download.py ^C
mlzboy@mlzboy-mac:~/my/idea2/download/hqs$ python batch_download.py
Traceback (most recent call last):
File "/usr/local/lib/python2.6/dist-packages/eventlet-0.9.9-py2.6.egg/eventlet/greenpool.py", line 80, in _spawn_n_impl
func(*args, **kwargs)
File "batch_download.py", line 17, in fetch
html=urllib2.urlopen(url).read()
File "/usr/lib/python2.6/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/usr/lib/python2.6/urllib2.py", line 391, in open
response = self._open(req, data)
File "/usr/lib/python2.6/urllib2.py", line 409, in _open
'_open', req)
File "/usr/lib/python2.6/urllib2.py", line 369, in _call_chain
result = func(*args)
File "/usr/lib/python2.6/urllib2.py", line 1161, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "/usr/lib/python2.6/urllib2.py", line 1136, in do_open
raise URLError(err)
URLError: <urlopen error [Errno 110] ETIMEDOUT>

刚发现设pool的一个好处是一个坏了，不影响其它的，只是速度慢一点，对我而言，少一两个页面也没事，