[并发并行]_[pthread]_[使用线程池并发复制文件]

场景:

1.在复制多个文件时线程池也有用武之地，多线程复制文件是受io影响的，在设置一定的缓存时有一定效果。

2.或者在Linux下有一定效果，但是在windows上使用fopen效果不太明显。有知道的高手请指教。

3.参考qt并发复制文件，并沿用之前的线程池实现。

4.商用的话建议可以用boost的 http://sourceforge.net/projects/threadpool/?source=directory，如果是想用c的线程池实现，可以修改我写的。

Qt的调用实现:

QList<QString> keys = m_Files.keys();  
    int num_files = keys.count();  
    QFutureSynchronizer<tuple<QString, QString> > sync;  
  
    for (int i = 0; i < num_files; ++i) {  
        QString id = keys.at(i);  
        sync.addFuture(QtConcurrent::run(  
                           this,  
                           &ImportEPUB::LoadOneFile,  
                           m_Files.value(id),  
                           m_FileMimetypes.value(id)));  
    }  
  
    sync.waitForFinished();

我自己的实现从效果上其实复制速度还是有下降的。就和在windows上同时复制多个大文件不如先复制一个文件快。

main.cpp

/*
 * main.cpp
 *
 *  Created on: 2013-3-13
 *  Author: Sai
 */

#include <stdio.h>
#include <string.h>
#include <string>
#include <iostream>
#include <io.h>
using namespace std;

#include "dh_thread_pool.h"
#include "dh_thread_task.h"

#include "dh_download_manager.h"

typedef struct DiskData
{
	FILE* file;
} DiskData;

size_t WriteToDisk(void *contents, size_t size, size_t nmemb, void *userp)
{
	DiskData* dd = (DiskData*) userp;
	size_t number = nmemb * size;
	size_t writed_num = fwrite(contents, 1, number, dd->file);
	return writed_num;
}

void *RunTaskFunc(void * arg)
{
	int* i = (int*) arg;
	cout << "thread index: " << *i << endl;
	DhDownloadManager* manager = new DhDownloadManager();
	static const char* url =
			"http://www.istonsoft.com/downloads/iston-video-converter.exe";

	DiskData dd;
	char path[8];
	memset(path, 0, sizeof(path));
	sprintf(path, "%d.exe", *i);
	dd.file = fopen(path, "wb");
	manager->Process(url, &WriteToDisk, &dd);

	fclose(dd.file);
	return NULL;
}

void Download()
{
	printf("Hello, world\n");
	DhThreadPool *pool = new DhThreadPool(5);
	pool->Activate();

	for (int o = 0; o < 10; ++o)
	{
		int *i = new int;
		*i = o;
		pool->AddAsynTask(&RunTaskFunc, i);
	}
	getchar();
	pool->Destroy();
	delete pool;
}

void CopyOneToAnother(const char* source, const char* dest)
{
	FILE* file_r = fopen(source, "rb");
	FILE* file_w = fopen(dest, "wb");

	char buf[1025];
	buf[1024] = 0;
	while (!feof(file_r))
	{
		memset(buf, 0, 1024);
		int readed = fread(buf, 1, 1024, file_r);
		fwrite(buf, 1, readed, file_w);
	}
	fclose(file_r);
	fclose(file_w);
}

void CopyFileWithOneThread()
{
	double time = clock();
	static string dir("dist/");
	static string d_dir("dist/Debug/");
	for (int i = 1; i < 11; ++i)
	{
		char file_name[8];
		memset(file_name, 0, sizeof(file_name));
		sprintf(file_name, "%d.bin", i);
		string path(dir);
		path.append(file_name);
		string d_path(d_dir);
		d_path.append(file_name);
		CopyOneToAnother(path.c_str(), d_path.c_str());
	}

	cout << "单线程复制文件 耗时 单位(毫秒): " << clock() - time << endl;
}

void *CopyFile(void * arg)
{
	int* data = (int*) arg;
	static string dir("dist/");
	static string d_dir("dist/Debug/");

	char file_name[8];
	memset(file_name, 0, sizeof(file_name));
	sprintf(file_name, "%d.bin", *data);
	string path(dir);
	path.append(file_name);
	string d_path(d_dir);
	d_path.append(file_name);
	CopyOneToAnother(path.c_str(), d_path.c_str());

	free(data);
	return NULL;
}

void CopyFileWithConcurrentThread()
{
	double time = clock();

	DhThreadPool *pool = new DhThreadPool(5);
	pool->SetWaitTime(10);//设置扫描时间是10毫秒
	for (int o = 1; o < 11; ++o)
	{
		int *i = (int*) malloc(sizeof(int));
		*i = o;
		pool->AddAsynTask(&CopyFile, i);
	}
	pool->Activate();
	pool->WaitTaskFinishAndDestroy();
	delete pool;

	cout << "并发复制文件 耗时 单位(毫秒): " << clock() - time << endl;
}

int IsFileExist(const char* path)
{
	return !access(path, F_OK);
}

int CreateFixSizeFile(const char* file_path, uint64_t file_size)
{
	FILE* file = fopen(file_path, "wb");
	char buffer[1025];
	memset(buffer, 1, sizeof(buffer));
	uint64_t size = file_size;

	size_t num = 0;
	while (size)
	{
		num = fwrite(buffer, 1, 1024, file);
		size = size - num;
	}

	fclose(file);
	return 0;
}

int main(int argc, char *argv[])
{
	setbuf(stdout, (char*) NULL);
	setbuf(stderr, (char*) NULL);

	//1.创建文件
	string dir("dist/");
	for (int i = 1; i < 11; ++i)
	{
		char file_name[8];
		memset(file_name, 0, sizeof(file_name));
		sprintf(file_name, "%d.bin", i);
		string path(dir);
		path.append(file_name);
		if (!IsFileExist(path.c_str()))
		{
			CreateFixSizeFile(path.c_str(), 1024 * 1024 * 10 * i);
		}
	}
        
        //耗时
	//41500
	//42489
	//41368
	//1.单个线程复制文件
//	CopyFileWithOneThread();
        
        //耗时
	//85396
	//1.并发复制文件
	CopyFileWithConcurrentThread();

	return 0;
}

参考线程池: http://blog.csdn.net/infoworld/article/details/8670951