批量查询URL是否被收录(Python多线程)

#coding=utf-8

import urllib,urllib2,re,random

def get_proxies():

    html = urllib2.urlopen('http://www.xici.net.co/nn/').read()

    pa = re.compile(r'<td>(\d+.\d+.\d+.\d+)</td>[\s\S]+?<td>(\d+)</td>[\s\S]+?<div title="(\d\.\d+?)秒" class="bar">')

    ip_list = pa.findall(html)

    return [x for x in ip_list if x[2] < '3']

ip_list = get_proxies()

print 'collect %s ip' % len(ip_list)

def is_index(line): 

    while 1:

        try:

            ip = random.choice(ip_list)

            proxies = {}

            proxies['http'] = 'http://%s:%s' % (ip[0], ip[1])

            proxy_handler = urllib2.ProxyHandler(proxies)

            opener = urllib2.build_opener(proxy_handler)

            html=opener.open('http://www.baidu.com/s?wd=%s' % urllib.quote_plus(line), timeout=15).read()

            if '="http://verify.baidu.com' in html:

                ip_list.remove(ip)

                print ("yzm show", ip[0])

                continue

            if '<div class="nors"><p>抱歉，没有找到与' in html or 'class=f14>没有找到该URL。您可以直接访问' in html:

                return (0, ip[0])

            else:

                return (1, ip[0])

        except:

            ip_list.remove(ip)

            is_index(line)

            

for line in open('url.txt'):

    line = line.strip()

    print line + '\t' + '%s\t%s' % is_index(line)