给10^7个有重复的整数排序(败者树)

　　参考July博文：程序员编程艺术：第十章、如何给10^7个数据量的磁盘文件排序，感谢July。

　　给10^7个无重复的整数排序请看另一篇博文：10^7个无重复的整数排序


　　对于给10^7个有重复的整数排序，我们不能用位图法来做，位图法只适用于无重复的数字，那么假设我们没有足够的内存去存储这1千万个整数，我们该如何去排序呢？还是分治法，把大化为小。比如：我们可以把这1千万个整数化为10份，用10个文件存储，分别为data1.txt到data10.txt，并且我们的内存足够存储每一份数据，即每一个dataX.txt，这样，我们就可以依次对这10个文件读取进内存，并利用内部排序，如快速排序，对每一个文件进行排序，然后在对这10个有序的文件进行归并排序，这样就达到我们的要求，即对这10^7个有重复的整数排序了。

　　下面请看代码，我把这10^7个整数分为10份，存储在10个文件中，依次对每一个文件进行快速排序，然后在对这10个文件进行归并排序，在归并的时候，只是采用类似选择排序的方法选择最小值，故比较次数与文件成线性关系。

const int FILE_NUM = 10;
const int MAX_PART = 1000000;
FILE *fpreads[FILE_NUM];

int cmp(const void* a, const void *b)
{
return *((int*)a) - *((int*)b);
}

//从unsort_data.txt中读取数据
int read_data(FILE *fp, int *array, int N)
{
int length = 0;
int num;
for (int i = 0; i < MAX_PART && (EOF != fscanf(fp, "%d", &num)); i++)
{
length++;
array[i] = num;
}
return length;
}

//打开data1.txt - data10.txt这10个文件
FILE* open_file(int count, char *mode)
{
FILE *fpwrite;
char filename[20];
memset(filename, 0, 20);
sprintf(filename, "data%d.txt", count);
fpwrite = fopen(filename, mode);
assert(fpwrite != NULL);
return fpwrite;
}

//向data1.txt - data10.txt这10个文件写入排好序的数据
void write_data(int *array, int N, int count)
{
FILE *fpwrite = open_file(count, "w");
for (int i = 0; i < N; i++)
{
fprintf(fpwrite, "%d ", array[i]);
}
fclose(fpwrite);
}

//内部排序，调用10次快速排序，产生data1.txt - data10.txt这10个有序文件
void interior_sort(void)
{
clock_t begin = clock();
FILE *fpread = fopen("unsort_data.txt", "r");
assert(fpread != NULL);

int count = 1;
int *array = new int[MAX_PART];
assert(array != NULL);
while (1)
{
memset(array, 0, sizeof(int) * MAX_PART);
int length = read_data(fpread, array, MAX_PART);
if (length == 0)
{
break;
}
qsort(array, length, sizeof(int), cmp);
write_data(array, length, count);
count++;
}
delete [] array;
fclose(fpread);
clock_t end = clock();
cout<<"10次快速排序所需时间为: "<<(end - begin)/CLK_TCK << "s"<<endl;
}

//对data1.txt - data10.txt这10个有序文件进行归并
void merge_sort()
{
clock_t begin = clock();
FILE *fpreads[FILE_NUM];      //10个文件的描述符
int data[FILE_NUM];           //10个文件的10个当前最小数据
bool flag[FILE_NUM] = {0};    //标记10个文件，是否已到EOF
FILE *fpwrite = fopen("sort_data.txt", "w");
assert(fpwrite != NULL);

for (int i = 0; i < FILE_NUM; i++)
{
fpreads[i] = open_file(i + 1, "r");
}
for (int i = 0; i < FILE_NUM; i++)
{
fscanf(fpreads[i], "%d", &data[i]);
}

while (1)
{
int count = 0;
while (count < FILE_NUM && flag[count])
{
count++;
}
if (count == FILE_NUM)
{
break;
}
int min_data = data[count];
int index = count;
for (int i = index; i < FILE_NUM; i++)  //在10个文件中找最小的数
{
if (!flag[i] && min_data > data[i])
{
min_data = data[i];
index = i;
}
}
fprintf(fpwrite, "%d ", min_data);
if (EOF == fscanf(fpreads[index],"%d", &data[index]))
{
flag[index] = true;
}
}
for (int i = 0; i < FILE_NUM; i++)
{
fclose(fpreads[i]);
}
fclose(fpwrite);
clock_t end = clock();
cout<<"10路归并排序所需时间为: "<<(end - begin)/CLK_TCK << "s"<<endl;
}

int _tmain(int argc, _TCHAR* argv[])
{
interior_sort();
merge_sort();
return 0;
}

　　对于上述归并排序，我们可以用败者树来刷选最小值，这样比较次数就从上述的线性级降到对数级，在归并数多的情况下，效率要比上述的要好，代码如下：

//利用败者树
const int N = 10000000;
const int FILE_NUM = 10;
const int MAX_PART = 1000000;
FILE *fpreads[FILE_NUM];
const int MIN = -1;     //最小值,必须比要排序数字的最小值要小，否则出错
const int MAX = N + 1;  //最大值,必须比要排序数字的最大值要大，否则出错

int cmp(const void* a, const void *b)
{
return *((int*)a) - *((int*)b);
}

//从unsort_data.txt中读取数据
int read_data(FILE *fp, int *array, int N)
{
int length = 0;
int num;
for (int i = 0; i < MAX_PART && (EOF != fscanf(fp, "%d", &num)); i++)
{
length++;
array[i] = num;
}
return length;
}

//打开data0.txt - data9.txt这10个文件
FILE* open_file(int count, char *mode)
{
FILE *fpwrite;
char filename[20];
memset(filename, 0, 20);
sprintf(filename, "data%d.txt", count);
fpwrite = fopen(filename, mode);
assert(fpwrite != NULL);
return fpwrite;
}

//向data0.txt - data9.txt这10个文件写入排好序的数据
void write_data(int *array, int N, int count)
{
FILE *fpwrite = open_file(count, "w");
for (int i = 0; i < N; i++)
{
fprintf(fpwrite, "%d ", array[i]);
}
fprintf(fpwrite, "%d", MAX);  //在每个文件最后写入一个最大值，表示文件结束
fclose(fpwrite);
}

//内部排序，调用10次快速排序，产生data0.txt - data9.txt这10个有序文件
void interior_sort(void)
{
clock_t begin = clock();
FILE *fpread = fopen("unsort_data.txt", "r");
assert(fpread != NULL);

int count = 0;
int *array = new int[MAX_PART];
assert(array != NULL);
while (1)
{
memset(array, 0, sizeof(int) * MAX_PART);
int length = read_data(fpread, array, MAX_PART);
if (length == 0)
{
break;
}
qsort(array, length, sizeof(int), cmp);
write_data(array, length, count);
count++;
}
delete [] array;
fclose(fpread);
clock_t end = clock();
cout<<"10次快速排序所需时间为: "<<(end - begin)/CLK_TCK << "s"<<endl;
}

//调整
void adjust(int ls[], int data[], int s)
{
int t = (s + FILE_NUM)/2;
while (t)
{
if (data[s] > data[ls[t]])
{
int temp = s;
s = ls[t];
ls[t] = temp;
}
t /= 2;
}
ls[0] = s;
}

void create_loser_tree(int ls[], int data[])
{
data[FILE_NUM] = MIN;
for (int i = 0; i < FILE_NUM; i++)
{
ls[i] = FILE_NUM;
}
for (int i = FILE_NUM - 1; i >= 0; i--)
{
adjust(ls, data, i);
}
}

void merge_sort_by_losertree()
{
clock_t begin = clock();
FILE *fpreads[FILE_NUM];      //10个文件的描述符
int data[FILE_NUM + 1];       //10个文件的10个当前最小数据
int ls[FILE_NUM];             //存放败者索引的节点
int index;
FILE *fpwrite = fopen("sort_data_by_losertree.txt", "w");
assert(fpwrite != NULL);

for (int i = 0; i < FILE_NUM; i++)
{
fpreads[i] = open_file(i, "r");
}
for (int i = 0; i < FILE_NUM; i++)
{
fscanf(fpreads[i], "%d", &data[i]);
}

create_loser_tree(ls, data); //创建败者树
while (data[ls[0]] != MAX)
{
index = ls[0];
fprintf(fpwrite, "%d ", data[index]);
fscanf(fpreads[index], "%d", &data[index]);
adjust(ls, data, index);
}
for (int i = 0; i < FILE_NUM; i++)
{
fclose(fpreads[i]);
}
fclose(fpwrite);
clock_t end = clock();
cout<<"10路归并排序所需时间为: "<<(end - begin)/CLK_TCK << "s"<<endl;
}

int _tmain(int argc, _TCHAR* argv[])
{
interior_sort();
merge_sort_by_losertree();
return 0;
}

　　未排序的数据如下：



　　利用归并排序后的文件如下：





　　2013年1月24日 venow 完