C# 中文分词[基于统计的朴素贝叶斯算法]

主要思想:

1.要有一个语料库

2.统计每个词出现的频率,一会来做朴素贝叶斯候选

3.举例:中国人民共和国的

其中语料库中有中国,人民,中国人,共和国等等的词组.

现在输入:中国人都爱中华人民共和国;

分词的时候取max(各种分发得到的score);

例如:solution1：中国人_都爱中华人民_共和国

solution2：中国_人_都爱中华人民_共和国

solution3：中国_人_都爱_中华_人民_共和国

bestSegSolution=max(solutions(segSlution[i]));

4.对于一句汉字的分词可以看做

seg(StringIn)=firPart+seg(StringIn–firPart);//我用score来衡量当前分词结果的好坏

6。朴素贝叶斯的意思就是:分词后的,两个词之间是相互独立的,也就是后者的出现与前者无关

5.这个只是初级版,很简单,需要再加点东西,结果会更加的完美..当然,按照做事情的原则,都是从简单开始做的,再努力

usingSystem;
usingSystem.Collections.Generic;
usingSystem.Text;
usingSystem.Collections;
usingSystem.Windows.Forms;
usingSystem.IO;
usingSystem.Diagnostics;

namespaceChineseWordSeg
{
classNaiveBayes
{
privatestringwordLibPath="../WordLib/pku_training.txt";//所用的训练库是pku的语料库.

booltrained=false;
privateDictionary<string,long>wordLib=newDictionary<string,long>();
privateDictionary<string,long>singleWordLib=newDictionary<string,long>();
intmaxLen=0;
longmaxScore=0;
privatestringsegPos="";//记录单句的分割点,按照标点等非汉字的字符分开
privatestringsegSentence="";//记录整个段落的

//是不是中文字符

boolisChineseWord(charchr){
if(chr>=0x4E00&&chr<=0x9FFF)returntrue;
returnfalse;
}

publicvoidtrainDate(stringpath){
//统计每个词出现的次数

//1.统计每个词组频率,naiveBayes消歧.将一个组合不同的方式取得较大概率的那个分组方式.
//难道每个词还是hash一下么?
//2.统计每个字的频率,就像向心力那样...看看到底哪两个字比较容易联系到一起这个是一句废话,因为我没这么去做
wordLib.Clear();

DirectoryInfodirInfo=newDirectoryInfo(path);
DirectoryInfotmpDir=dirInfo.Parent;
stringsavePath=tmpDir.FullName;
FileInfofInfo=newFileInfo(wordLibPath);
stringfileNamePre=fInfo.Name;
savePath+="\\"+fileNamePre+"_trained";
FileInfoinfoOfDB=newFileInfo(savePath);

if(File.Exists(savePath)&&infoOfDB.Length>0){

StreamReadersr1=
newStreamReader(@savePath);
char[]sep={''};

while(sr1.Peek()!=-1)
{
string[]keyValue=sr1.ReadLine().Split(sep);

wordLib[keyValue[0]]=Convert.ToInt32(keyValue[1]);

}

return;
}

if(!File.Exists(path)){
MessageBox.Show("ÓïÁÏ¿â·¾¶ÓÐ´í£¬Çë¼ì²é");
return;
}

Stopwatchtm=newStopwatch();
tm.Start();
StreamReadersr=
newStreamReader(@path,
System.Text.Encoding.GetEncoding("gb2312"));

chartmpChar;
stringtmpStr;
char[]tmpCArray=newchar[100];

{
tmpStr="";
boolflag=false;
longtmpVal=0;
while(sr.Peek()!=-1){
tmpChar=(char)sr.Read();

if(isChineseWord(tmpChar))
{
flag=true;
/*
if(flag==true)
{
stringsingleWord=(tmpChar).ToString();
if(singleWordLib.ContainsKey(singleWord))
{
singleWordLib.TryGetValue(singleWord,outtmpVal);
singleWordLib[singleWord]=tmpVal+1;
}
else
singleWordLib.Add(singleWord,1);
//ͳ¼Æÿ¸ö×ÖµÄ

}*/

tmpStr+=(char)tmpChar;
}
else
{
tmpStr=tmpStr.Trim();
if(flag==true)
{
if(tmpStr.Length>1){
if(wordLib.ContainsKey(tmpStr))
{
wordLib.TryGetValue(tmpStr,outtmpVal);
wordLib[tmpStr]=tmpVal+1;
}
else
wordLib.Add(tmpStr,1);
}
else{
if(singleWordLib.ContainsKey(tmpStr))
{
singleWordLib.TryGetValue(tmpStr,outtmpVal);
singleWordLib[tmpStr]=tmpVal+1;
}
else
singleWordLib.Add(tmpStr,1);
}
//ͳ¼Æÿ¸ö´Ê×éµÄ
}
tmpStr="";
flag=false;
}

if(maxLen<tmpStr.Length)
{
maxLen=tmpStr.Length;
//¼Ç¼µ¥´Ê×î´óµÄ³¤¶È...
}
}
}
sr.Close();

StreamWritersw=newStreamWriter(savePath);

foreach(stringkeyinwordLib.Keys){
sw.WriteLine(key+""+wordLib[key]);
}

sw.Close();

tm.Stop();

MessageBox.Show(tm.Elapsed.Milliseconds.ToString(),"trainingdone");
}

//将分段好的结果传回.
publicstringgetSegedString(stringstrIn){
char[]seprator={'s'};
string[]segSplit=segSentence.Split(seprator);
List<int>segP=newList<int>();
segP.Clear();
intj,i;
intcntSegPos=0;

for(i=0;i<segSplit.Length;i++){
if(segSplit[i].Length>0)
{
segP.Add(Convert.ToInt16(segSplit[i]));
cntSegPos++;
}
}

char[]cArray=newchar[512];
cArray=strIn.ToCharArray();

stringstrOut="";
boolflag=true;

for(i=0,j=0;i<strIn.Length;i++)
{

while(j<cntSegPos&&segP.Contains(i))
{
segP.Remove(i);
flag=!flag;
if(flag)
strOut+=")";
elsestrOut+="(";
j++;
}
strOut+=cArray[i];
}
if(j<cntSegPos)strOut+=")";
returnstrOut;
}

//恩,做朴素贝叶斯分词
publicstringdoNaiveBayesSegmentation(stringstrIn,stringtrainDataPath){

if(!trained)
{
trained=true;
trainDate(trainDataPath);
}

stringstrTmp="";
char[]charBuffer=newchar[4096];
charBuffer=strIn.ToCharArray();
inti=0,len=strIn.Length;

while(i<len)
{
while(i<len&&isChineseWord(charBuffer[i]))strTmp+=charBuffer[i++];

{
if(strTmp.Length>0)
{
maxScore=0;
segPos="";
naviveBayesSeg(strTmp,0,"",i-strTmp.Length);
segSentence+=segPos;
}
strTmp="";
}

while(i<len&&!isChineseWord(charBuffer[i]))i++;
}

returngetSegedString(strIn);

}

//分词的具体实现,bestSegSolution=max(solutions(segSlution[i]));

对于一句汉字的分词可以看做seg(StringIn)=firPart+seg(StringIn–firPart);

我用score来衡量当前分词结果的好坏
publicvoidnaviveBayesSeg(stringstrIn,longscore,stringseg,inttPos){

if(true){
if(score>maxScore){
segPos=seg;
maxScore=score;

}
//return;
}
intstrLen=strIn.Length;
stringfirStr="";
inti=0;
for(i=1;i<=strIn.Length;i++){
firStr=strIn.Substring(0,i);
if(wordLib.ContainsKey(firStr))
{
naviveBayesSeg(strIn.Substring(i),score+wordLib[firStr],seg+(tPos+i-firStr.Length).ToString()+"s"+(tPos+i).ToString()+"s",tPos+i);
}
}

if(i>strIn.Length&&i>1)
naviveBayesSeg(strIn.Substring(1),score,seg,tPos+1);

}
}
}