2013年阿里巴巴一道笔试题（大文件处理）

现有一个亿级别数据量的文件，其中有按key升序的记录，现要求通过输入key查找对应的记录。

对于这种大文件读取，在读取时一般要采用内存文件映射， 另外，通常的处理操作就是对文件进行分隔。 把文件分隔为若干小文件后，记录下每个小文件中最小的key值，然后把输入值与这些key值依次比较便可以找到key对应的记录所在的小文件，然后把小文件读入内存，进行二分查找。

下面是所有的程序代码，为方便，文件中仅仅记录了key。

/*
* 分别采用分割文件，在子文件中进行二分查找的方法 和 直接进行键值查找的方法进行时间对比
*/
package com.alibaba;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.IntBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.TreeMap;
import java.util.Map.Entry;

public class SortBigFile {

//单个文件大小
private static final int FILE_SIZE = 1024*1024*10;

//整数个数
private static final int numOfInt = 100000000;

//分隔文件存储目录
private static final String divPath = "E:\\divide";

//记录分隔文件后每个文件的最小值，及对应的文件路径
private TreeMap<Integer,String> fileMap = new TreeMap<Integer,String>();

//待查找的大文件路径
private String filePath;

public SortBigFile(String path){
this.filePath = path;
}

//写入大文件，做测试
public void writeInt()
{
FileChannel fc = null;
IntBuffer ib = null;
try {
fc = new RandomAccessFile(filePath,"rw").getChannel();
ib = fc.map(FileChannel.MapMode.READ_WRITE, 0, numOfInt*4).asIntBuffer();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

for(int i=0;i<numOfInt;++i)
ib.put(i);
try {
fc.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
//对大文件进行分隔
public void splitFile()
{
FileChannel fc = null;
IntBuffer out = null;
try {
fc = new RandomAccessFile(filePath,"r").getChannel();
out = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()).asIntBuffer();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
//对文件进行分隔
try {
for(int i=0;i<fc.size()/FILE_SIZE+1;++i)
{
String path = divPath+"\\"+System.currentTimeMillis()+".tmp";
DataOutputStream dos = null;
try {
dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(new File(path))));
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
for(int j=0;j<FILE_SIZE/4;++j)
{
if(out.hasRemaining())
{
int num = out.get();
if(j==0)
{
fileMap.put(num, path);
}
dos.writeInt(num);
}
}
dos.close();
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
fc.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}

//对大文件进行键值查找
public String findKey(int key)
{
int fnum;
String fpath="";
boolean flag = false;
for(Entry<Integer,String> entry:fileMap.entrySet())
{
int num = entry.getKey();
String path = entry.getValue();
if(num==key)
return path+":first num";
else if(num<key)
{
fnum = num; fpath = path;
}
else
{
flag = true;
break;
}
}

if(fpath.isEmpty()||!flag)
{
return "find nothing";
}

int index = binarySearch(fpath, key);
if(index==-1)
return "find nothing";
else
return fpath+":"+index+" num";
}

//对子文件进行二分查找
private int binarySearch(String path, int key)
{
int index = 0;
ArrayList<Integer> nums = new ArrayList<Integer>();
DataInputStream dos = null;
try {
dos = new DataInputStream(new BufferedInputStream(new FileInputStream(new File(path))));
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
while(true)
{
int num;
try {
num = dos.readInt();
nums.add(num);
} catch (EOFException e) {
// TODO Auto-generated catch block
//文件结尾
break;
}
catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
int start =0, end = nums.size()-1;
while(start<end)
{
int mid = start + (end-start)/2;
if(nums.get(mid)==key)
return mid;
else if(nums.get(mid)>key)
{
end = mid;
}
else
{
start = mid;
}
}
return -1;

}

//采用直接内存映射读入文件，然后进行比较得到键值的方法
public int dirFind(int key)
{
FileChannel fc = null;
IntBuffer out = null;
try {
fc = new RandomAccessFile(filePath,"r").getChannel();
out = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()).asIntBuffer();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
int index = 0;
while(out.hasRemaining())
{
int num = out.get();
if(num==key)
return index;
else if(num<key)
{
++index;
}
else
{
return -1;
}
}
return -1;
}
/**
* @param args
*/
public static void main(String[] args) {
// TODO Auto-generated method stub
SortBigFile bigfile = new SortBigFile("E:\\test.tmp");
long start = 0;
long end = 0;
long withoutDiv = 0;

bigfile.writeInt();

//计算文件分割，然后二分查找键值的时间
start = System.currentTimeMillis();
bigfile.splitFile();
withoutDiv = System.currentTimeMillis();
System.out.println(bigfile.findKey(88732723));
end = System.currentTimeMillis();
System.out.println("with divide file time:"+(end-start)/1000);
System.out.println("without divide file time:"+(end-withoutDiv)/1000);

System.out.println("---------------");
//不进行分割，直接查找的时间
start = System.currentTimeMillis();
System.out.println("Index:"+ bigfile.dirFind(88732723));
end = System.currentTimeMillis();
System.out.println((end-start)/1000);
}
}