使用WVTool进行文本分类

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Calendar;
import java.util.List;

import edu.udo.cs.wvtool.config.WVTConfiguration;
import edu.udo.cs.wvtool.config.WVTConfigurationFact;
import edu.udo.cs.wvtool.generic.output.WordVectorWriter;
import edu.udo.cs.wvtool.generic.stemmer.DummyStemmer;
import edu.udo.cs.wvtool.generic.stemmer.WVTStemmer;
import edu.udo.cs.wvtool.generic.tokenizer.WVTTokenizer;
import edu.udo.cs.wvtool.generic.vectorcreation.TFIDF;
import edu.udo.cs.wvtool.generic.wordfilter.DummyWordFilter;
import edu.udo.cs.wvtool.generic.wordfilter.WVTWordFilter;
import edu.udo.cs.wvtool.main.WVTDocumentInfo;
import edu.udo.cs.wvtool.main.WVTFileInputList;
import edu.udo.cs.wvtool.main.WVTWordVector;
import edu.udo.cs.wvtool.main.WVTool;
import edu.udo.cs.wvtool.wordlist.WVTWordList;

public class MyTest{

public static void main(String[] args) throws Exception {
//初始化一个WVTool对象
WVTool wvt = new WVTool(false);

//初始化一个configuration对象
WVTConfiguration config = new WVTConfiguration();

WVTStemmer stemmer = new DummyStemmer();
WVTTokenizer tk = new ChineseTokenizer();

//DummyStopWordFilter filter = new DummyStopWordFilter();
WVTWordFilter filter = new DummyWordFilter();

config.setConfigurationRule(WVTConfiguration.STEP_TOKENIZER, new WVTConfigurationFact(tk));
config.setConfigurationRule(WVTConfiguration.STEP_STEMMER, new WVTConfigurationFact(stemmer));
config.setConfigurationRule(WVTConfiguration.STEP_WORDFILTER, new WVTConfigurationFact(filter));

WVTFileInputList list = new WVTFileInputList(2);

// Add entries
//为输入添加一个文档信息对象 （WVTDocumentInfo）,其中sourceName对象可以是一个文件夹的名称，也可以是一个文件名称, 最后一个0这个文档信息对象的类别
//样本数据
//list.addEntry(new WVTDocumentInfo("a.txt", "txt", "", "", 0));
//list.addEntry(new WVTDocumentInfo("b.txt", "txt", "", "", 1));
list.addEntry(new WVTDocumentInfo("D:/temp/1", "txt", "", "chinese", 0));
list.addEntry(new WVTDocumentInfo("D:/temp/2", "txt", "", "chinese", 1));

//生成wordList
WVTWordList wordList = wvt.createWordList(list, config);
//对wordList中词频做出一个限制，即词频在1<n<5之间
wordList.pruneByFrequency(1, 5);

//生成词组文件
wordList.storePlain(new FileWriter("wordlist.txt"));

// 生成词频文件
wordList.store(new FileWriter("wordVector.txt"));

//将生成的文本向量空间写入一个特定的文件
FileWriter outFile = new FileWriter("wv.txt");

//DummyWordVectorWriter wvw = new DummyWordVectorWriter(outFile, true);
WordVectorWriter wvw = new WordVectorWriter(outFile,true);

config.setConfigurationRule(WVTConfiguration.STEP_OUTPUT, new WVTConfigurationFact(wvw));
config.setConfigurationRule(WVTConfiguration.STEP_VECTOR_CREATION, new WVTConfigurationFact(new TFIDF()));

//Create the vectors
WVTWordVector[] vectors = wvt.createVectors(list, config, wordList,null);

//Close the output file
wvw.close();
outFile.close();

// 一个使用wordList构建文本空间向量的实例
//WVTWordVector q = wvt.createVector("cmu harvard net", wordList);

//测试的文本
WVTDocumentInfo d = new WVTDocumentInfo("", "txt", "", "chinese");
//测试文本的内容
String txt = getContent("test.txt");
//根据wordlist和config 生成向量
WVTWordVector q = wvt.createVector(txt, d, config, wordList);

FileWriter outFile1 = new FileWriter("test_wv1.txt");
WordVectorWriter wvw1 = new WordVectorWriter(outFile1, true);
wvw1.write(q);
wvw1.close();
outFile1.close();

//knn算法分类
KNN knn = new KNN();
//分类结果
List result = knn.LazyLearning(q, vectors, list.getNumClasses());
for(int i=0;i<result.size();i++){
CategoryResult cr = (CategoryResult) result.get(i);
System.out.println("rs:"+cr.getCategoryName()+" "+cr.getSimilarity());
}

}

public static String getContent(String file) throws IOException{
File myfile = new File(file);
if (!myfile.exists()) {
return "";
}
File f=new File(file);
InputStreamReader  read = new InputStreamReader (new FileInputStream(f));
BufferedReader reader = new BufferedReader(read);
String line;
String strContent = "";
while((line=reader.readLine())!=null){
strContent+=line;
}
return strContent;

}

}