大文件字符串替换

前一段时间,在导数据的时候,常常要替换掉导出数据中的特殊的字符,而且一些字符是不可打印的,如ASCII码为0×05,0xFF等构成的字符串. 而且处理的文件都是几百G以上.故写了一个字符串替换的小程序予以自乐.在字符串替换的过程中,很重要的一点就是字符匹配. 先看看基于内存的匹配方法.

1. 无限内存中的字符串匹配
假设内存足够大,所以待处理的文本和要匹配的串都可以完全放入内存,且处理的文本为text[tsize], 要匹配的串为pat[psize],要替换的串为val[vsize] ,此时字符串匹配最简单的方式伪代码如下:

input: text[tsize] //待处理的文本大小为tsize
pat[psize] //要匹配的模式
val[vsize] //要替换的值
output: ret //处理好后的文本
begin
int i, j;
for(i=0; i<tsize; i++)
begin
for(j=0; j<psize; j++)
begin
if(text[i]!=pat[j])//不等
break;//跳出循环
end for
if(j == psize)
用val替换串text[i-j,i];
并调整i的位置到下一个匹配位置,继续匹配
end for
end

但是往往文件中的文本特别大,比如几百G以上,很难完全容人内存,所以问题在于处理在串在有限内存下的情形.

2. 有限内存中的大字符串匹配

当内存有限时,最极端的情况是,每一次从文件中读一个byte,且和pattern进行匹配,这样就会增加读文件的I/O次数,故因该设置in_buff 用于读入一段适合内存的数据,当为空了后才再次读入in_buff大小的数据, out_buff 把处理好的数据放入其中,当out_buff满了才输出.这样就可以有效的减少读数据和写数据的I/O次数.

实验表明,在同一个(不详细说明环境的配置)实验环境下,一个字节比1024个字节缓存读取和写入数据的的情形多花3倍的时间.

相关的c语言实现的code如下:

/*
* replace.c
*
* Created on: Jan 27, 2010
* Author: grand
*/

//为了处理大文件定义的预编译的定义
#define _FILE_OFFSET_BITS 64
#define __USE_FILE_OFFSET64
#define __USE_LARGEFILE64
#define _LARGEFILE64_SOURCE

#include <stdio.h>
#include <string.h>
#include <stdlib.h>

#define BUFF_SIZE 1024 //缓存大小,可以调整从1个byte到多个
#define true 1
#define false 0

char in_buff[BUFF_SIZE];
char out_buff[BUFF_SIZE];

//the pattern string
char* pPat = NULL;
unsigned int len_pat = 0;
//the new value for pattern string
char* pVal = NULL;
unsigned int len_val = 0;

//statics result
unsigned long long inTotal = 0;
unsigned long long match = 0;
unsigned long long outTotal = 0;

void usage()
{
printf("replace <srcFile> <destFile> <pat> <val> /n");
}

void outBuff(FILE * pDest, char* pBuff, int in_start, int in_end, int flush)
{

static unsigned int outPos = 0;
//force to write data to file.
if (flush == true)
{
outTotal += fwrite(out_buff, sizeof(char), outPos, pDest);
return;
}
//write data into out_buffer
if (BUFF_SIZE-outPos > in_end – in_start)
{
memcpy(out_buff + outPos, pBuff + in_start, in_end – in_start);
outPos += in_end – in_start;
}
//the out_buff is full, so write data to file.
else
{
outTotal += fwrite(out_buff, sizeof(char), outPos, pDest);
outTotal += fwrite(pBuff + in_start, sizeof(char), in_end – in_start,
pDest);
outPos = 0;
}
}

int inBuff(FILE * pSrc)
{
int c;
c = fread(in_buff, sizeof(char), BUFF_SIZE,pSrc);
inTotal += c;
return c;
}

int adjust(int patPos, FILE* pDest)
{
int help = patPos;
int temp = 1;
int beg = 1;
patPos = 0;
while (temp < help)
{
if (pPat[patPos] == pPat[temp])
{
patPos++;
temp++;
}
else
{
beg++;
temp = beg;
patPos = 0;
}
}
outBuff(pDest, pPat, 0, help – patPos, false);
return patPos;

}

void matchPat(FILE* pSrc, FILE* pDest)
{
int inPos = 0;
int patPos = 0;
int in_len = 0;
int i;
in_len = inBuff(pSrc);
while (true)
{
for (i = inPos; i < in_len && patPos < len_pat;)
{
if (*(in_buff + i) != *(pPat + patPos))
{
if (i – patPos < 0)
{
patPos = adjust(patPos, pDest);
}
else
{
i = i – patPos + 1;
patPos = 0;
}
}
else
{
i++;
patPos++;
}
}
// reach the end of the in_buff
if (i == in_len)
{
if (in_len – patPos > inPos)
outBuff(pDest, in_buff, inPos, in_len – patPos, false);
inPos = 0;
if (feof(pSrc))
{
outBuff(pDest, NULL,0, 0, true);
return;
}
in_len = inBuff(pSrc);
}
// succeed in matching a pattern
else
{
if (i – patPos > inPos)
outBuff(pDest, in_buff, inPos, i – patPos, false);
inPos = i;
outBuff(pDest, pVal, 0, len_val, false);
patPos = 0;
match++;
}
}
}

int main(int argc, char** argv)
{
FILE* pSrc = NULL;
FILE* pDest = NULL;

if (argc != 5)
{
usage();
return 1;
}
if ((pSrc = fopen(argv[1], "rb")) == NULL)
{
printf("can not open the src file:%s /n", argv[1]);
return 1;
}
if ((pDest = fopen(argv[2], "wb")) == NULL)
{
printf("can not open the dest file:%s /n", argv[2]);
return 1;
}

if ((pPat = malloc(strlen(argv[3]) * sizeof(char))) == NULL)
{
printf("encounter error when allocating memory for pattern/n");
return 1;
}

len_pat = strlen(argv[3]);
memcpy(pPat, argv[3], len_pat);

if ((pVal = malloc(strlen(argv[4]) * sizeof(char))) == NULL)
{
printf("encounter error when allocating memory for value/n");
return 1;
}
len_val = strlen(argv[4]);
memcpy(pVal, argv[4], len_val);
// replace start
matchPat(pSrc, pDest);
// replace end
// clean the resource
fclose(pSrc);
fclose(pDest);
free(pPat);
free(pVal);
printf("===============================================================/n");
printf("buffer size: %u bytes/n", BUFF_SIZE);
printf("src file size: %u bytes/n", inTotal);
printf("dest file size: %u bytes/n", outTotal);
printf("length of pattern: %u bytes/n", len_pat);
printf("length of replace value: %u bytes/n", len_val);
printf("match count: %u /n", match);
printf("===============================================================/n");
return 0;
}