nodejs 采集新闻数据

var express = require('express');
var $ = require('jQuery');
var app = express();
var colors = require('colors');
var message_list = require('./zui/message_list');
var findData = require('./findData');

//设置全局跨域访问
app.all('*', function(req, res, next) {
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
res.header("Access-Control-Allow-Methods","PUT,POST,GET,DELETE,OPTIONS");
res.header("X-Powered-By",' 3.2.1')
res.header("Content-Type", "application/json;charset=utf-8");
next();
});

//输出主页列表数据
app.get('/getIndexData', function (req, res) {
findData.indexData(req, res);
});

/*
*输出列表数据。支持范围选择
*demo：http://localhost:3000/getList/0,9 前9条数据 从0开始
*demo: http://localhost:3000/getList/0  所有的数据  从0开始
*/
app.get(/^\/getList?(?:\/(\d+)(?:\,(\d+))?)?/, function (req, res) {
var start=undefined,end=undefined;
if(req.params.length==1){
start=req.params[0];
}else if(req.params.length==2){
start=req.params[0];
end=req.params[1];
}
findData.list(req, res,start,end,function(data){
for(var i=0,j=data.length;i<j;i++){
data[i].orderby="1";
}
return data;
});
});

//数据列表
var newsListData=undefined;
app.get('/getNewsList', function (req, res) {
if(typeof newsListData=="undefined"){
findData.list(req, res,undefined,undefined,function(data,length){
for(var i=0,j=data.length;i<j;i++){
data[i].orderby=Math.round(Math.random()*6);
data[i].summary="暂无摘要";
data[i].updateUserName="admin";

}
newsListData=data;
getNewsList(req, res);
});
}else{
getNewsList(req, res);
}
});

//缓存数据列表
function getNewsList(req, res){
var pageNum=req.query.pageNum,pageSize=req.query.pageSize;
var start=(pageNum-1)*pageSize,end=pageNum*pageSize;
var newData=newsListData.slice(start, end);
res.send(JSON.stringify(newData));
}

app.listen(3000);
console.log("nodejs was start".green);

var http = require('http'),
fs = require('fs'),
jquery = fs.readFileSync("lib/jquery.min.js", "utf-8"),
jsdom = require('jsdom');

function get(url, callback) {
//使用代理
var opt = {
host: '127.0.0.1',
port: '7070',
method: 'get', //这里是发送的方法
path: url
}
//以下是接受数据的代码
var req = http.request(opt, function(res) {
res.setEncoding('utf8');
var html = '';
res.on('data', function(d) {
html += d;
}).on('end', function() {
jsdom.env({
html: html,
src: [jquery],
done: function(errors, window) {
var $ = window.$;
callback && callback(errors, $);
window.close(); // 释放window相关资源，否则将会占用很高的内存
}
});
});
}).on('error', function(e) {
console.log("Got error: " + e.message);
})
req.end();
}

//导出list
exports.indexData = function(request, response, start, end) {
get('http://www.news.cn/edu/index.htm', function(errors, $) {
var $list = $(".list");
var data = [];
$list.each(function(index) {
var $this = $(this),
$li = $this.find("li"),
li_data = [];
$li.each(function(index) {
var $a = $(this).find("a");
var href=$a.attr("href").replace("http://news.xinhuanet.com/edu/","");
var temp_href="/"+href.replace(/[^c]*/,"");
var date=href.replace(temp_href,"").replace("/","-");
if(date.length>10){
date="2013-01-04";
}

li_data.push({
title: $a.text(),
href: $a.attr("href"),
date:date,
id: index
});
});
data.push({
id: index,
list: li_data
});
});

if (typeof start != "undefined" && typeof end == "undefined") {
data = data.slice(start);
} else if (typeof start != "undefined" && typeof end != "undefined") {
data = data.slice(start, end);
}
response.send(JSON.stringify(data));
});
}

//导出list
exports.list = function(request, response, start, end,callback) {
get('http://www.news.cn/edu/index.htm', function(errors, $) {
var $li = $(".list li");
var data = [];
$li.each(function(index) {
var $a = $(this).find("a");
var href=$a.attr("href").replace("http://news.xinhuanet.com/edu/","");
var temp_href="/"+href.replace(/[^c]*/,"");
var date=href.replace(temp_href,"").replace("/","-");
if(date.length>10){
date="2013-01-04";
}

data.push({
id: index,
title: $a.text(),
attachment: $a.attr("href"),
updateTime:date
});
});

var newData=data;
if (typeof start != "undefined" && typeof end == "undefined") {
newData = data.slice(start);
} else if (typeof start != "undefined" && typeof end != "undefined") {
newData = data.slice(start, end);
}
if(typeof callback=="function"){
newData=callback(newData,data.length);
}
//response.header("Access-Control-Allow-Origin", "*");   //设置跨域访问
response.send(JSON.stringify(newData));
});
}

http://localhost:3000/getIndexData http://localhost:3000/getList/0,4 http://localhost:3000/getNewsList?pageNum=1&pageSize=10