爬虫抓取5大门户网站和电商数据day1:基础环境搭建

最新想用爬虫实现抓取五大门户网站（搜狐、新浪、网易、腾讯、凤凰网）和电商数据（天猫，京东，聚美等）， 今天第一天先搭建下环境和测试。

采用maven+xpath+ HttpClient+正则表达式。

maven pom.xml配置文件信息

<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
<version>1.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.10</artifactId>
<version>1.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.10</artifactId>
<version>1.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
<version>1.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka_2.10</artifactId>
<version>1.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-graphx_2.10</artifactId>
<version>1.6.0</version>
</dependency>
<!-- httpclient4.4 -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.4</version>
</dependency>
<!-- htmlcleaner -->
<dependency>
<groupId>net.sourceforge.htmlcleaner</groupId>
<artifactId>htmlcleaner</artifactId>
<version>2.10</version>
</dependency>
<!-- json -->
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20140107</version>
</dependency>

<!-- hbase -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>0.96.1.1-hadoop2</version>
</dependency>

<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>0.96.1.1-hadoop2</version>
</dependency>

<!-- redis 2.7.0-->
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.7.0</version>
</dependency>

<!-- slf4j -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.10</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.10</version>
</dependency>
<!-- quartz1.8.4 -->
<dependency>
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz</artifactId>
<version>1.8.4</version>
</dependency>
<!-- curator -->
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<version>2.7.1</version>
</dependency>

新建一个测试类：SpiderTest

/**
* url 入口，下载页面
* @param url
*/
public  static String downLoadCrawlurl(String url){
String context = null;
Logger logger = LoggerFactory.getLogger(SpiderTest.class);
HttpClientBuilder create = HttpClientBuilder.create();
HttpGet httpGet = new HttpGet(url);
CloseableHttpClient build = create.build();
try {
CloseableHttpResponse response = build.execute( httpGet);
HttpEntity entity = response.getEntity();
context = EntityUtils.toString( entity );
System.out.println("context:" + context);
}
catch ( ClientProtocolException e ) {
e.printStackTrace();
}
catch ( IOException e ) {
logger.info("download...." );
}
return context;
}

public static void main( String[] args ) {
　　
String url = "http://money.163.com/";
downLoadCrawlurl(url);

}