爬虫记录（1）——简单爬取一个页面的内容并写入到文本中

package com.dyw.crawler.util;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;

/**
* 爬虫工具类
* Created by dyw on 2017/9/1.
*/
public class CrawlerUtils {

/**
* 获取html内容转成string输出。
*
* @param url url链接
* @return 整个网页转成String字符串
*/
public static String getHtml(String url) throws Exception {
URL url1 = new URL(url);//使用java.net.URL
URLConnection connection = url1.openConnection();//打开链接
InputStream in = connection.getInputStream();//获取输入流
InputStreamReader isr = new InputStreamReader(in);//流的包装
BufferedReader br = new BufferedReader(isr);

String line;
StringBuffer sb = new StringBuffer();
while ((line = br.readLine()) != null) {//整行读取
sb.append(line, 0, line.length());//添加到StringBuffer中
sb.append('\n');//添加换行符
}
//关闭各种流，先声明的后关闭
br.close();
isr.close();
in.close();
return sb.toString();
}

}

package com.dyw.crawler.util;

import java.io.File;
import java.io.FileOutputStream;

/**
* IO工具类
* Created by dyw on 2017/9/1.
*/
public class IOUtils {

/**
* 创建文件
*
* @param file File类型
*/
public static void createFile(File file) throws Exception {
try {
if (!file.exists()) {
file.createNewFile();
}
} catch (Exception e) {
throw new Exception("创建文件的时候错误！", e);
}
}

/**
* 写入String到file中
*
* @param content  写入内容
* @param fileName 写入位置
*/
public static void writeFile(String content, File fileName) throws Exception {
FileOutputStream o;
try {
o = new FileOutputStream(fileName);
o.write(content.getBytes("Utf-8"));
o.close();
} catch (Exception e) {
throw new Exception("写入文件的时候错误！", e);
}
}
}

package com.dyw.crawler.project;

import com.dyw.crawler.util.CrawlerUtils;
import com.dyw.crawler.util.IOUtils;

import java.io.File;

/**
* 此包中的main方法
* Created by dyw on 2017/9/1.
*/
public class Project {

public static void main(String[] args) {
//文件放置的路径
String path = "C:\\Users\\dyw\\Desktop\\crawler";
//爬取的网站地址
String url = "http://blog.csdn.net/juewang_love";
String fileRealName = path + "/index.html";
File file = new File(fileRealName);
//创建文件
try {
IOUtils.createFile(file);
} catch (Exception e) {
throw new RuntimeException("创建文件失败!", e);
}
//获取内容
String htmlContent = null;
try {
htmlContent = CrawlerUtils.getHtml(url);
} catch (Exception e) {
throw new RuntimeException("获取内容失败!", e);
}
//写入内容
try {
IOUtils.writeFile(htmlContent, file);
} catch (Exception e) {
throw new RuntimeException("内容写入文件失败!", e);
}
}
}