Linux企业级项目实践之网络爬虫（4）——主程序流程

当我们设计好程序框架之后就要开始实现它了。第一步当然是要实现主程序的流程框架。之后我们逐渐填充每个流程的细节和其需要调用的模块。

主程序的流程如下：

1、 解析命令行参数，并根据参数跳转到相应的处理分支

2、 解析配置文件

3、 载入处理模块

4、 加载种子URL

5、 启动抓取任务

主程序的代码如下：

int main(int argc, void *argv[])
{
struct epoll_event events[10];
int daemonized = 0;
char ch;

while ((ch = getopt(argc, (char* const*)argv, "vhd")) != -1) {
switch(ch) {
case 'v':
version();
break;
case 'd':
daemonized = 1;
break;
case 'h':
case '?':
default:
usage();
}
}

g_conf = initconfig();
loadconfig(g_conf);

set_nofile(1024);

vector<char *>::iterator it = g_conf->modules.begin();
for(; it != g_conf->modules.end(); it++) {
dso_load(g_conf->module_path, *it);
}

if (g_conf->seeds == NULL) {
SPIDER_LOG(SPIDER_LEVEL_ERROR, "Wehave no seeds, Buddy!");
} else {
int c = 0;
char ** splits =strsplit(g_conf->seeds, ',', &c, 0);
while (c--) {
Surl * surl = (Surl*)malloc(sizeof(Surl));
surl->url =url_normalized(strdup(splits[c]));
surl->level = 0;
surl->type = TYPE_HTML;
if (surl->url != NULL)
push_surlqueue(surl);
}
}

if (daemonized)
daemonize();

chdir("download");

int err = -1;
if ((err = create_thread(urlparser, NULL, NULL, NULL)) < 0) {
SPIDER_LOG(SPIDER_LEVEL_ERROR,"Create urlparser thread fail: %s", strerror(err));
}

int try_num = 1;
while(try_num < 8 && is_ourlqueue_empty())
usleep((10000 << try_num++));

if (try_num >= 8) {
SPIDER_LOG(SPIDER_LEVEL_ERROR, "NOourl! DNS parse error?");
}

if (g_conf->stat_interval > 0) {
signal(SIGALRM, stat);
set_ticker(g_conf->stat_interval);
}

int ourl_num = 0;
g_epfd = epoll_create(g_conf->max_job_num);

while(ourl_num++ < g_conf->max_job_num) {
if (attach_epoll_task() < 0)
break;
}

int n, i;
while(1) {
n = epoll_wait(g_epfd, events, 10,2000);
printf("epoll:%d\n",n);
if (n == -1)
printf("epollerrno:%s\n",strerror(errno));
fflush(stdout);

if (n <= 0) {
if (g_cur_thread_num <= 0&& is_ourlqueue_empty() && is_surlqueue_empty()) {
sleep(1);
if (g_cur_thread_num <= 0&& is_ourlqueue_empty() && is_surlqueue_empty())
break;
}
}

for (i = 0; i < n; i++) {
evso_arg * arg = (evso_arg*)(events[i].data.ptr);
if ((events[i].events &EPOLLERR) ||
(events[i].events &EPOLLHUP) ||
(!(events[i].events &EPOLLIN))) {
SPIDER_LOG(SPIDER_LEVEL_WARN,"epoll fail, close socket %d",arg->fd);
close(arg->fd);
continue;
}
epoll_ctl(g_epfd, EPOLL_CTL_DEL,arg->fd, &events[i]); /* del event */

printf("helloepoll:event=%d\n",events[i].events);
fflush(stdout);
create_thread(recv_response, arg,NULL, NULL);
}
}

SPIDER_LOG(SPIDER_LEVEL_DEBUG, "Task done!");
close(g_epfd);
return 0;
}