java爬虫实战简单用Jsoup框架进行网页爬虫（如抓取网页图片）

package com.jiaxun.spider;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.net.URL;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class JsoupSpider {

// http://www.nipic.com/photo/jingguan/shanshui/index.html?page= 后面的数字逐个加1
// 直到2010
public static String getAllURL(String url) {
StringBuffer sb = null;
try {
sb = new StringBuffer();
for (int i = 1; i <= 2010; i++) {
sb.append(url + i).append("\r\n");
}

} catch (Exception e) {
// TODO: handle exception
}
return sb.toString();
}

// 在大图的网址获取详细的图片地址
public static String getPicURL(String picHrefUrl) {
String picURL = null;
if (picHrefUrl != null) {
try {
Document doc = Jsoup.connect(picHrefUrl).get();
Element element = doc.getElementById("J_worksImg");
picURL = element.attr("src");
} catch (Exception e) {
// TODO Auto-generated catch block
new RuntimeException(" ");
}
}
return picURL;
}

// 下载图片到本地磁盘
public static void downloadPicToLocal(String picSourceURL, String picDestPath) {
BufferedOutputStream bos = null;
BufferedInputStream bis = null;
try {
URL url = new URL(picSourceURL);
bis = new BufferedInputStream(url.openStream());
bos = new BufferedOutputStream(new FileOutputStream(picDestPath));
byte[] b = new byte[1024 * 1024];
int len = 0;
while ((len = bis.read(b)) != -1) {
bos.write(b, 0, len);
bos.flush();
}
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (Exception e) {
// TODO Auto-generated catch block
new RuntimeException(" ");
} finally {
try {
bos.close();
} catch (Exception e) {
// TODO Auto-generated catch block
}
try {
bis.close();
} catch (Exception e) {
// TODO Auto-generated catch block
}
}
}

// 获取文件后缀名
public static String getNameExtension(String picURL) {
String extension = null;

try {
int lastIndexOf = 0;
if (picURL != null && !picURL.equals("")) {
if (picURL.endsWith(".jpg"))
lastIndexOf = picURL.lastIndexOf(".jpg");
if (picURL.endsWith(".png"))
lastIndexOf = picURL.lastIndexOf(".png");
extension = picURL.substring(lastIndexOf);
}
} catch (Exception e) {
// TODO: handle exception
}
return extension;
}

// 通过Jsoup获取网页的单个图片地址_通过传入网页地址和生成图片目录
// 目标网址为http://www.nipic.com/photo/jingguan/shanshui/index.html?page=1
public static boolean getOnePic(String webURL, String picDestURL) {
boolean flag = true;
try {
// webURL="http://www.nipic.com/photo/jingguan/shanshui/index.html?page="
// 后面的数字逐个加1 直到2010
// 创建Document对象，拿到元素对象再操作属性。
File file = new File(picDestURL);
if (!file.exists()) {
file.mkdir();
}
int count = 2220;
for (int i = 112; i <= 2010; i++) {
// 分2010次拿到每页的网址
try {
String mainURL = webURL + i;
Document doc = Jsoup.connect(mainURL).get();
// 拿到每个页面的每个class元素
Elements elements = doc.getElementsByClass("relative block works-detail hover-none works-img-box");
for (Element element : elements) {
try {
// 获取每个页面的大图的网址
String picHrefUrl = element.attr("href");
// 获取每个大图的详细地址
String picURL = getPicURL(picHrefUrl);
// 从图片的详细地址开始下载单个图片到本地目标路径
downloadPicToLocal(picUR
9ad9
L, picDestURL + "\\" + (++count) + getNameExtension(picURL));
System.out.println("已抓取" + count + "张图片。");
} catch (Exception e) {
// TODO: handle exception
}

}
} catch (Exception e) {
// TODO: handle exception
}

}
// 计数器归零
count = 0;
} catch (Exception e) {
// TODO Auto-generated catch block
flag = false;
new RuntimeException("在服务器找不到对应图片，正在寻找下一个图片中。。。");
}
return flag;
}

public static void main(String[] args) {
boolean result = getOnePic("http://www.nipic.com/photo/jingguan/shanshui/index.html?page=",
"G:\\PictureSpider");
System.out.println(result);

}
}