Python编程入门-第11章 案例研究：文本统计

#先定义一个规整文本字符的函数
def normalize(s):
keep={'a','b','c','d','e','f','g','h','i','j','k','l',
'm','n','o','p','q','r','s','t','u','v','w','x',
'y','z',' ','-',"'"}
result=""
for c in s.lower():
if c in keep:
result+=c
return result

#接受一个字符串s并生成一个字典，该字典的键为s中的单词，值为对应单词出现的次数
def make_freq_dict(s):
s=normalize(s)
words=s.split()
d={}
for w in words:
if w in d:
d[w]+=1
else:
d[w]=1
return d

def print_file_stats(fname):
s=open(fname,'r').read()
#在规整字符串前先统计字符数量
num_chars=len(s)
#在规整字符串前先统计行数
num_lines=s.count('\n')

d=make_freq_dict(s)
#统计单词数量
num_words=sum(d[w] for w in d)

#创建一个列表，元素为单词及对应出现次数组成的元组
lst=[(d[w],w) for w in d]
lst.sort()
lst.reverse()
num_words0=len(normalize(s).split())

print("The file '%s' has: "% fname)
print("%s characters"% num_chars)
print("%s lines"% num_lines)
print("%s words"% num_words)
print("%s words"% num_words0)
print("\nThe top 10 most frequent words are:")
i=1
for count,word in lst[:10]:
print("%2d. %2s %s"%(i,count,word))
i+=1

print_file_stats("3.txt")

#这里只需要通过求字典长度即可得知不同单词总数
print("There are %s different words."% len(d))

#计算单词平均长度
m=0
for w in d:
m+=len(w)*d[w]
average_length=m/num_words

#计算罕用语数量
n=0
for w in d:
if d[w]==1:
n+=1

#建立一个排除词集合
stop_words={"the","a","i","of","is","in"}
#接受一个字符串s并生成一个字典，该字典的键为s中的单词，并且不在排除集合中
#值为对应单词出现的次数
def make_freq_dict1(s):
d0=make_freq_dict(s)
d={}
for w in d0:
if w in stop_words:
continue
else:
e={w:d0[w]}
d.update(e)
return d

#先定义一个规整文本字符的函数
def normalize(s):
keep={'a','b','c','d','e','f','g','h','i','j','k','l',
'm','n','o','p','q','r','s','t','u','v','w','x',
'y','z',' ','-',"'"}
result=""
for c in s.lower():
if c in keep:
result+=c
return result

#采用逐行读取的方式统计
def print_file_stats(fname):
f=open(fname,'r')
#先定义并初始化统计数据
num_chars=0
num_words=0
num_lines=0

#逐行统计
for line in f:
num_chars+=len(line)
num_words+=len(normalize(line).split())
d=make_freq_dict(line,d)
num_lines+=1

print("The file '%s' has: "% fname)
print("%s characters"% num_chars)
print("%s lines"% num_lines)
print("%s words"% num_words)
print_file_stats("3.txt")