c++实现文本中英文单词和汉字字符的统计

源代码下载：http://download.csdn.net/detail/nuptboyzhb/4987141

1.统计文本中汉字的频数，为后续的文本分类做基础。对于汉字的统计，需要判断读取的是否为汉字。源代码如下：

[C++ code]

/*
*@author:郑海波 http://blog.csdn.net/NUPTboyZHB *参考：实验室小熊
*注：有删改
*/
#pragma warning(disable:4786)
#include <iostream>
#include <vector>
#include <fstream>
#include <string>
#include <map>
#include <queue>
#include <ctime>
using namespace std;
void topK(const int &K)
{
double t=clock();

ifstream infile("test.txt");
if (!infile)
cout<<"can not open file"<<endl;

string s="";
map<string,int>wordcount;
unsigned char temp[2];
while(true)//国标2312
{
infile>>temp[0];
if(infile.eof()) break;
if (temp[0]>=0xB0)//GB2312下的汉字，最小是0XB0
{
s+=temp[0];
infile>>temp[1];
s+=temp[1];
}
else//非汉字字符不统计
{
s="";
continue;
}
wordcount[s]++;
s="";
}
cout<<"单词种类："<<wordcount.size()<<endl;
//优先队列使用小顶堆，排在前面的数量少，使用">";
priority_queue< pair< int,string >,vector< pair< int,string > >,greater< pair< int,string> > > queueK;
for (map<string,int>::iterator iter=wordcount.begin(); iter!=wordcount.end(); iter++)
{
queueK.push(make_pair(iter->second,iter->first));
if(queueK.size()>K)
queueK.pop();
}
pair<int,string>tmp;
//将排在后面的数量少，排在前面的数量多
priority_queue< pair< int,string >,vector< pair< int,string > >,less< pair< int,string> > > queueKless;
while (!queueK.empty())
{
tmp=queueK.top();
queueK.pop();
queueKless.push(tmp);
}
while(!queueKless.empty())
{
tmp=queueKless.top();
queueKless.pop();
cout<<tmp.second<<"\t"<<tmp.first<<endl;
}
cout<<"< Elapsed Time: "<<(clock()-t)/CLOCKS_PER_SEC<<" s>"<<endl;
}

int main()
{
int k=0;
cout<<"http://blog.csdn.net/NUPTboyZHB\n";
while (true)
{
cout<<"查看前K个频率最高的汉字，K=";
cin>>k;
if(k<=0)break;
topK(k);
}
return 0;
}

[图1]



2.统计英文单词的出现频率。这比统计汉字更加的容易，因为单词和单词之间是用空格分开的，所以，直接将单词保存到string中即可。

[c++ code]

/*
*@author:郑海波 http://blog.csdn.net/NUPTboyZHB *参考：实验室小熊
*注：有删改
*/
#pragma warning(disable:4786)
#include <iostream>
#include <vector>
#include <fstream>
#include <string>
#include <map>
#include <queue>
#include <ctime>
using namespace std;
void topK(const int &K)
{
double t=clock();

ifstream infile;
infile.open("test.txt");
if (!infile)
cout<<"can not open file"<<endl;
string s;
map<string,int>wordcount;

while(true)
{
infile>>s;
if(infile.eof()) break;
wordcount[s]++;
}
cout<<"单词种类："<<wordcount.size()<<endl;
//优先队列使用小顶堆，排在前面的数量少，使用">";
priority_queue< pair< int,string >,vector< pair< int,string > >,greater< pair< int,string> > > queueK;
for (map<string,int>::iterator iter=wordcount.begin(); iter!=wordcount.end(); iter++)
{
queueK.push(make_pair(iter->second,iter->first));
if(queueK.size()>K)
queueK.pop();
}
pair<int,string>tmp;
priority_queue< pair< int,string >,vector< pair< int,string > >,less< pair< int,string> > > queueKless;
while (!queueK.empty())
{
tmp=queueK.top();
queueK.pop();
queueKless.push(tmp);
}
while(!queueKless.empty())
{
tmp=queueKless.top();
queueKless.pop();
cout<<tmp.second<<"\t"<<tmp.first<<endl;
}
cout<<"< Elapsed Time: "<<(clock()-t)/CLOCKS_PER_SEC<<" >"<<endl;
}
int main()
{
int k=0;
cout<<"http://blog.csdn.net/NUPTboyZHB\n";
while (true)
{
cout<<"PUT IN K: ";
cin>>k;
if(k<=0)break;
topK(k);
}
return 0;
}

[图2]



参考：实验室小熊