TF-IDF计算三

逆向文件频率（inverse document frequency，IDF）是一个词语普遍重要性的度量。某一特定词语的IDF，可以由总文件数目除以包含该词语之文件的数目，再将得到的商取对数.

IDF i = log N / n i

N 是代表语料库中文件数量的总数, n i 是代表包含词语n i 的文件数目所包含的词 i 的总数.

class IDFs

{

HashMap<String,Double> IDFsList = new HashMap<String, Double>();

ArrayList<ArrayList<String>> IDFsMainFileList = new ArrayList<ArrayList<String>>();

ArrayList<String> IDFsave = new ArrayList<String>();

int Ncount;

public IDFs(ArrayList<ArrayList<String>> idf)

{

IDFsMainFileList = idf;

Ncount = IDFsMainFileList.size();

}

public HashMap<String,Double> PrintIDFs()

{

for(int i=0; i<IDFsMainFileList.size(); i++)

{

ArrayList<String> IDFsSubFileList = IDFsMainFileList.get(i);

ArrayList<String> list = new ArrayList<String>();

for(int j=0; j<IDFsSubFileList.size(); j++)

{

if(!list.contains(IDFsSubFileList.get(j)))

{

list.add(IDFsSubFileList.get(j));

//Take elements from arraylist<arraylist<string>>

if(!IDFsList.containsKey(IDFsSubFileList.get(j)))

{

IDFsList.put(IDFsSubFileList.get(j),1.0);//save elements to arraylist<hashmap<String,Double>>

IDFsave.add(IDFsSubFileList.get(j));//take elements from hashmap

}

else

{

double value = IDFsList.get(IDFsSubFileList.get(j));

value++;

IDFsList.put(IDFsSubFileList.get(j),value);

}

}

}

}

for(int k=0; k<IDFsave.size(); k++)

{

double nTotal = IDFsList.get(IDFsave.get(k));

double temp = Ncount / nTotal;

double idfs = Math.log(temp);

IDFsList.put(IDFsave.get(k), idfs);

}

return IDFsList;

}

}