您的位置：首页 > 编程语言 > Java开发

java爬虫入门--用jsoup爬取汽车之家的新闻

2017-10-15 22:32 543 查看 
public static boolean isContainChinese(String str) {
Pattern p = Pattern.compile("[\u4e00-\u9fa5]");
Matcher m = p.matcher(str);
if (m.find()) {
return true;
}
return false;
}

/**
* 从汽车之家抓新闻
* @param size
* @param baseUrl
* @param domainName
* @param newsListId
* @param newsContentClass
* @param titleTagOrClass
* @param limitHref
* @param dateTag
* @param needDeleteAlt
* @return
*/
public static ArrayList<News> getNewsFromCarHome(int size,String baseUrl,String domainName,String newsListId,
String newsContentClass,String titleTag,String dateTag,String needDeleteAlt){
ArrayList<News> newsList = new ArrayList<News>();
Document doc;
Elements elements =null;
Element title =null;
News news = null;
try {
doc = Jsoup.connect(baseUrl).get();
elements = (Elements) doc.getElementById(newsListId).children();
if(elements!=null&&elements.size()>0){
for(Element ele:elements){
news = new News();
title = ele.select("a").first();
if(title==null){
continue;
}
news.setTitle(title.getElementsByTag(titleTag).text());
if(news.getTitle()==null||news.getTitle().equals("")){
continue;
}
news.setHref(domainName+title.attr("href"));
if(dateTag!=null){
String date=ele.select("i").text();
news.setDate(date);
}
String newsUrl =news.getHref();
if (isContainChinese(news.getHref())) {
newsUrl = URLEncoder.encode(news.getHref(), "utf-8").toLowerCase().replace("%3a", ":").replace("%2f", "/");
}
Document newsDoc = Jsoup.connect(newsUrl).get();
String text=newsDoc.getElementsByClass(newsContentClass).html();
if(text.indexOf("未经许可")>0||text.indexOf("禁止转载")>0||text.indexOf("公众号")>0||text.indexOf("公众账号")>0){
continue;
}
text=replaceImgSrcFromDataSrc(text,true,needDeleteAlt);
int index=text.lastIndexOf("（");
if(index>0){
text=text.substring(0,index);
}
StringBuffer textBuffer=new StringBuffer(5);
textBuffer.append("<!DOCTYPE html><html><head><meta name=\"content-type\" content=\"text/html; charset=UTF-8\">");
textBuffer.append("</head><body>");
textBuffer.append(deleteSource(text));
textBuffer.append("</body></html>");
news.setContent(textBuffer.toString());
news.setContent(textBuffer.toString());
System.out.println("标题====="+news.getTitle());
System.out.println("href====="+news.getHref());
System.out.println("content====="+news.getContent());
newsList.add(news);
if(newsList.size()==size){
break;
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
return newsList;
}

private static String deleteImg(String text) {
return text.replaceAll("<img [^>]*>", "");
}
private static String deleteSource(String text) {
return text.replaceAll("\\(.*?\\)|\\[.*?]", "");
}
/**
* 删除a标签中的href
* @param content
* @return
*/
public static String removeHref(String content){
Document document = Jsoup.parse(content);
Elements elements = document.select("a[href]");
for(Element el:elements){
el.removeAttr("href");
}
return document.html();
}

/**
* 将htmlBody中所有img标签中的src内容替换为原data-src的内容, <br/>
* 如果不报含data-src,则src的内容不会被替换 <br/>
* @param htmlBody html内容
* @param needDeleteAlt 需要剔除的图片的alt信息
* @param imgUrlNeedAddProtocolPrefix 图片的url是否需要添加http协议前缀
* @return 返回替换后的内容
*/
public static String replaceImgSrcFromDataSrc(String htmlBody,boolean imgUrlNeedAddProtocolPrefix,String needDeleteAlt) {
Document document = Jsoup.parseBodyFragment(htmlBody);
List<Element> nodes = document.select("img");
int nodeLenth = nodes.size();
if(nodeLenth==0){
return htmlBody;
}
for (int i = 0; i < nodeLenth; i++) {
Element e = nodes.get(i);
String dataSrc = e.attr("data-src");
if (StringUtils.isNotBlank(dataSrc)) {
e.attr("src", dataSrc);
e.removeAttr("data-src");
}
}
if (htmlBody.contains("<html>")) {
if(needDeleteAlt==null&&!imgUrlNeedAddProtocolPrefix){
return document.toString();
}else if(needDeleteAlt==null&&imgUrlNeedAddProtocolPrefix){
return document.toString().replace("src=\"//", "src=\"http://");
}else if(needDeleteAlt!=null&&imgUrlNeedAddProtocolPrefix){
return document.toString().replace("src=\"//", "src=\"http://").replace("alt="+needDeleteAlt, "");
}
return document.toString().replace("alt="+needDeleteAlt, "");
} else {
if(needDeleteAlt==null&&!imgUrlNeedAddProtocolPrefix){
return document.select("body>*").toString();
}else if(needDeleteAlt==null&&imgUrlNeedAddProtocolPrefix){
return document.select("body>*").toString().replace("src=\"//", "src=\"http://");
}else if(needDeleteAlt!=null&&imgUrlNeedAddProtocolPrefix){
return document.select("body>*").toString().replace("src=\"//", "src=\"http://").replace("alt="+needDeleteAlt, "");
}
return document.select("body>*").toString().replace("alt="+needDeleteAlt, "");
}

}
public static void main(String[] args) throws Exception{
getNewsFromCarHome(1,"http://m.autohome.com.cn/channel","http://m.autohome.com.cn","list","details","h4","time","汽车之家");
}

内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理 点击这里给我发消息
标签：  java爬虫入门 jsoup入门 jsoup爬取网页教程
相关文章推荐
新的分享
章节导航