Linux企业级项目实践之网络爬虫（15）——区分文本文件和二进制文件

static char * BIN_SUFFIXES = ".jpg.jpeg.gif.png.ico.bmp.swf";
static int is_bin_url(char *url)
{
char *p = NULL;
if ((p = strrchr(url, '.')) != NULL) {
if (strstr(BIN_SUFFIXES, p) == NULL)
return 0;
else
return 1;
}
return 0;
}

char * attach_domain(char *url, const char *domain)
{
if (url == NULL)
return NULL;

if (strncmp(url, "http", 4) == 0) {
return url;

} else if (*url == '/') {
int i;
int ulen = strlen(url);
int dlen = strlen(domain);
char *tmp = (char *)malloc(ulen+dlen+1);
for (i = 0; i < dlen; i++)
tmp[i] = domain[i];
for (i = 0; i < ulen; i++)
tmp[i+dlen] = url[i];
tmp[ulen+dlen] = '\0';
free(url);
return tmp;

} else {
//do nothing
free(url);
return NULL;
}
}

char * url2fn(const Url * url)
{
int i = 0;
int l1 = strlen(url->domain);
int l2 = strlen(url->path);
char *fn = (char *)malloc(l1+l2+2);

for (i = 0; i < l1; i++)
fn[i] = url->domain[i];

fn[l1++] = '_';

for (i = 0; i < l2; i++)
fn[l1+i] = (url->path[i] == '/' ? '_' : url->path[i]);

fn[l1+l2] = '\0';

return fn;
}