java 网络爬虫之多线程抓取文件

记得这个是去年的东西了，今天重新拿出来重温，一些知识都模糊了很多。

一共六个类文件加上一个jar包，Demo文件是主文件；DownloadFile文件的作用是从网络URL上下载文件下来，别人已经封装好了拿来用；DownloadThread文件作用是多线程爬取文件下来，速度快；HttpUtils文件作用是将URL网页装换为可操作的document文件，也是别人已经封装好的；MD5不用我说了吧；Task是处理文件的类；

1 Demo.java

import java.util.ArrayList;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Demo {

/**
* @param args
*/

public static ArrayList<Task> arr = new ArrayList<Task>();

public static void main(String[] args) {

GeiALLimgUrl("http://www.csdn.net"); // 封装目标url

int maxindex = 2; // 设置的多线程个数，修改多少个随你

DownloadThread[] d = new DownloadThread[maxindex];

for (int i = 0; i < maxindex; i++) {

d[i] = new DownloadThread(i);

d[i].start();

}
}

public static void GeiALLimgUrl(String url) {
try {
String result = HttpUtils.doGet(url);

Document doc = Jsoup.parse(result);

Elements links = doc.select("img");

for (Element imgs : links) {

System.out.println(imgs.attr("src")); // 抓取的当前URL页面上的图片img

arr.add(new Task(imgs.attr("src"))); // 先存放在集合里，后续再操作
}
} catch (Exception e) {

e.printStackTrace();
}
}

public static Task getTask() {
for (Task s : arr) {
if (!s.hasDownloaded) {
s.hasDownloaded = true;
return s;
}
}
return null;
}
}

2 Task.java

public class Task {

//图片地址
public String imageUrl="";

//图片是否被下载了？
public  boolean hasDownloaded=false;

//图片的名字
public String filename;

//构造函数，提供图片的URL就可以了
public Task(String url){

imageUrl=url;

filename=MD5.string2MD5(url);  //对图片加密，利于爬取的各种操作

int last=imageUrl.lastIndexOf(".");

String ext=imageUrl.substring(last+1);

filename=filename +"."+ext;

System.out.println("文件名："+filename);
}

}

3 DownloadThread.java

import java.io.IOException;

public class DownloadThread extends Thread{

//当前ID号
public int ID;

public boolean exit=false;

public DownloadThread(int id){

ID=id;
}

@Override
public void run() {
// TODO Auto-generated method stub
//super.run();

DownloadFile download=new DownloadFile();

while(!exit){

//从任务列表中读取一个没有被下载的任务
Task target=Demo.getTask();

if(target!=null){

//下载
System.out.println(ID);
try {

download.downLoadFromUrl(target.imageUrl, target.filename, "c:\\images");

} catch (IOException e) {

e.printStackTrace();
}

}
else{

System.out.println("我是第"+ID+"个线程，我现在没有任务");

//没有任务，休息一下
try {
Thread.sleep(1000);
} catch (InterruptedException e) {

e.printStackTrace();
}
}

}

}

}

4 DownloadFile.java

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;

public class DownloadFile {

/**
* 从网络Url中下载文件
* @param urlStr
* @param fileName
* @param savePath
* @throws IOException
*/
public  void  downLoadFromUrl(String urlStr,String fileName,String savePath) throws IOException{
URL url = new URL(urlStr);
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
//设置超时间为3秒
conn.setConnectTimeout(3*1000);
//防止屏蔽程序抓取而返回403错误
conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");

//得到输入流
InputStream inputStream = conn.getInputStream();
//获取自己数组
byte[] getData = readInputStream(inputStream);

//文件保存位置
File saveDir = new File(savePath);
if(!saveDir.exists()){
saveDir.mkdir();
}

File file = new File(saveDir+File.separator+fileName);

if(file.exists())
{
System.out.println("文件已存在，不用重复下载");
return;
}

FileOutputStream fos = new FileOutputStream(file);
fos.write(getData);
if(fos!=null){
fos.close();
}
if(inputStream!=null){
inputStream.close();
}

System.out.println("info:"+url+" download success");

}

/**
* 从输入流中获取字节数组
* @param inputStream
* @return
* @throws IOException
*/
public   byte[] readInputStream(InputStream inputStream) throws IOException {
byte[] buffer = new byte[1024];
int len = 0;
ByteArrayOutputStream bos = new ByteArrayOutputStream();
while((len = inputStream.read(buffer)) != -1) {
bos.write(buffer, 0, len);
}
bos.close();
return bos.toByteArray();
}

}

5 HttpUtils.java

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.zip.GZIPInputStream;

public class HttpUtils {

//根据url访问服务器，返回服务器响应文本
public static String doGet(String url) throws Exception
{
//创建一个URL对象，URL
URL localURL = new URL(url);

//设置代理服务器
System.setProperty("http.proxyHost", "127.0.0.1");
System.setProperty("http.proxyPort", "8888");

URLConnection connection = localURL.openConnection();
HttpURLConnection httpURLConnection = (HttpURLConnection)connection;

//设置请求头部的属性
httpURLConnection.setRequestProperty("User-Agent","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E");

//保存输入输出流的对象
InputStream inputStream = null;
InputStreamReader inputStreamReader = null;
BufferedReader reader = null;
StringBuffer resultBuffer = new StringBuffer();
String tempLine = null;

//302强制浏览器跳转，200 ok
if (httpURLConnection.getResponseCode() >= 300) {
throw new Exception("HTTP Request is not success, Response code is " + httpURLConnection.getResponseCode());
}

try {
inputStream = httpURLConnection.getInputStream();

//get header by 'key'
String encoding = httpURLConnection.getHeaderField("Content-Encoding");

//如果返回的是压缩HTML代码
if(encoding!=null && encoding.equals("gzip"))
{
System.out.println("这是一个压缩的HTML\n");
GZIPInputStream gzin;
gzin = new GZIPInputStream(inputStream);
//对返回页面内容进行utf-8解码，从而中文不会乱码
inputStreamReader = new InputStreamReader(gzin,"gbk");

}
else
{
inputStreamReader = new InputStreamReader(inputStream,"gbk");
}
reader = new BufferedReader(inputStreamReader);

while ((tempLine = reader.readLine()) != null) {
resultBuffer.append(tempLine+"\n");
}

} finally {

if (reader != null) {
reader.close();
}

if (inputStreamReader != null) {
inputStreamReader.close();
}

if (inputStream != null) {
inputStream.close();
}

}

return resultBuffer.toString();
}

/*
* currentBase当前搜索网页的URL
* target是从网页标签提取出来的URL(例如href等)
* */
public static String getURL(String currentUrl,String targetUrl)
{
String temp=targetUrl;
//当前页面的路径
//例如：http://www.gdmec.cn/cs/csnew/index.html
//应该要分析出:http://www.gdmec.cn/cs/csnew/
String currentBase="";

String resultURL="";

if(currentUrl.endsWith("/"))
{
currentBase=currentUrl;
}
else
{
int lastPos=currentUrl.lastIndexOf("/");
currentBase=currentUrl.substring(0,lastPos+1);
}

System.out.println("currentBase:"+currentBase);

if(temp.startsWith("http"))
{
return resultURL;
}
else if(temp.startsWith("../"))
{
//resultURL=currentBase+temp.substring(2);

}
else if(temp.startsWith("./"))
{
resultURL=currentBase+temp.substring(2);
}
else if(temp.startsWith("//"))
{
resultURL="http:"+temp;
}
else if(temp.startsWith("/"))
{
resultURL=currentBase+temp.substring(1);
}
else
{
resultURL=currentBase+temp;
}
return resultURL;

}
}

6 MD5.java

import java.security.MessageDigest;

public class MD5 {

/***
* MD5加码 生成32位md5码
*/
public static String string2MD5(String inStr){
MessageDigest md5 = null;
try{
md5 = MessageDigest.getInstance("MD5");
}catch (Exception e){
System.out.println(e.toString());
e.printStackTrace();
return "";
}
char[] charArray = inStr.toCharArray();
byte[] byteArray = new byte[charArray.length];

for (int i = 0; i < charArray.length; i++)
byteArray[i] = (byte) charArray[i];
byte[] md5Bytes = md5.digest(byteArray);
StringBuffer hexValue = new StringBuffer();
for (int i = 0; i < md5Bytes.length; i++){
int val = ((int) md5Bytes[i]) & 0xff;
if (val < 16)
hexValue.append("0");
hexValue.append(Integer.toHexString(val));
}
return hexValue.toString();

}

}

 jar包 jsoup-1.9.2.jar

这里是爬取网络上指定url的图片，其他的比如爬取兼职信息，天气信息等也可以，当然，爬取过多随时会被墙掉，而且一些网页会使用get 或者post来获取信息，这时就要适当修改爬取的方式了，还有一些网页是异步加载，就留给你们自己尝试了。