python 自然语言处理（二）____获得文本语料和词汇资源

1>>>importnltk
2>>>nltk.corpus.gutenberg.fileids()
3['austen-emma.txt','austen-persuasion.txt','austen-sense.txt','bible-kjv.txt'
4,'blake-poems.txt','bryant-stories.txt','burgess-busterbrown.txt','carroll-a
5lice.txt','chesterton-ball.txt','chesterton-brown.txt','chesterton-thursday.t
6xt','edgeworth-parents.txt','melville-moby_dick.txt','milton-paradise.txt','
7shakespeare-caesar.txt','shakespeare-hamlet.txt','shakespeare-macbeth.txt','w
8hitman-leaves.txt']
9>>>

1>>>emma=nltk.corpus.gutenberg.words('austen-emma.txt')
2>>>len(emma)
3192427
4>>>

1>>>emma=nltk.Text(nltk.corpus.gutenberg.words('austen-emma.txt'))
2>>>emma.concordance("surprise")
3Displaying1of1matches:
4thatEmmacouldnotbutfeelsomesurprise,andalittledispleasure,onhe
5>>>

279>>>forfileidingutenberg.fileids():
280...raw=gutenberg.raw(fileid)
281...num_chars=len(raw)
282...words=gutenberg.words(fileid)
283...num_words=len(words)
284...sents=gutenberg.sents(fileid)
285...num_sents=len(sents)
286...vocab=set([w.lower()forwingutenberg.words(fileid)])
287...num_vocab=len(vocab)
288...print("%d%d%d%s"%(num_chars,num_words,num_sents,fileid))
289...
2908870711924277752austen-emma.txt
291466292981713747austen-persuasion.txt
2926730221415764999austen-sense.txt
2934332554101065430103bible-kjv.txt
294381538354438blake-poems.txt
295249439555632863bryant-stories.txt
29684663189631054burgess-busterbrown.txt
297144395341101703carroll-alice.txt
298457450969964779chesterton-ball.txt
299406629860633806chesterton-brown.txt
300320525692133742chesterton-thursday.txt
30193515821066310230edgeworth-parents.txt
302124299026081910059melville-moby_dick.txt
303468220968251851milton-paradise.txt
304112310258332163shakespeare-caesar.txt
305162881373603106shakespeare-hamlet.txt
306100351231401907shakespeare-macbeth.txt
3077112151548834250whitman-leaves.txt
308
309>>>raw[:1000]
310"[LeavesofGrassbyWaltWhitman1855]\n\n\nCome,saidmysoul,\nSuchversesfo
311rmyBodyletuswrite,(forweareone,)\nThatshouldIafterreturn,\nOr,long
312,longhence,inotherspheres,\nTheretosomegroupofmatesthechantsresumin
313g,\n(TallyingEarth'ssoil,trees,winds,tumultuouswaves,)\nEverwithpleas'd
314smileImaykeepon,\nEverandeveryettheversesowning--as,first,Ihereand
315now\nSigningforSoulandBody,settothemmyname,\n\nWaltWhitman\n\n\n\n[BO
316OKI.INSCRIPTIONS]\n\n}One's-SelfISing\n\nOne's-selfIsing,asimplesepa
317rateperson,\nYetutterthewordDemocratic,thewordEn-Masse.\n\nOfphysiology
318fromtoptotoeIsing,\nNotphysiognomyalonenorbrainaloneisworthyforth
319eMuse,Isay\ntheFormcompleteisworthierfar,\nTheFemaleequallywitht
320heMaleIsing.\n\nOfLifeimmenseinpassion,pulse,andpower,\nCheerful,for
321freestactionform'dunderthelawsdivine,\nTheModernManIsing.\n\n\n\n}As
322IPonder'dinSilence\n\nAsIponder'dinsilence,\nReturninguponmypoems,c"
323>>>
324>>>words
325['[','Leaves','of','Grass','by','Walt','Whitman',...]
326>>>sents
327[['[','Leaves','of','Grass','by','Walt','Whitman','1855',']'],['Come',
328',','said','my','soul',',','Such','verses','for','my','Body','let','u
329s','write',',','(','for','we','are','one',',)','That','should','I','
330after','return',',','Or',',','long',',','long','hence',',','in','othe
331r','spheres',',','There','to','some','group','of','mates','the','chant
332s','resuming',',','(','Tallying','Earth',"'",'s','soil',',','trees','
333,','winds',',','tumultuous','waves',',)','Ever','with','pleas',"'",'d'
334,'smile','I','may','keep','on',',','Ever','and','ever','yet','the','
335verses','owning','--','as',',','first',',','I','here','and','now','Si
336gning','for','Soul','and','Body',',','set','to','them','my','name',',
337'],...]

1>>>forfileidinwebtext.fileids():
2...print("%s%s..."%(fileid,webtext.raw(fileid)[:65]))
3...
4firefox.txtCookieManager:"Don'tallowsitesthatsetremovedcookiestose
5...
6grail.txtSCENE1:[wind][clopclopclop]
7KINGARTHUR:Whoathere![clop...
8overheard.txtWhiteguy:So,doyouhaveanyplansforthisevening?
9Asiangirl...
10pirates.txtPIRATESOFTHECARRIBEAN:DEADMAN'SCHEST,byTedElliott&Terr
11...
12singles.txt25SEXYMALE,seeksattracoldersinglelady,fordiscreetencoun
13...
14wine.txtLovelydelicate,fragrantRhonewine.Polishedleatherandstrawb...
15
16>>>

1>>>fromnltk.corpusimportbrown
2>>>brown.categories()
3['adventure','belles_lettres','editorial','fiction','government','hobbies',
4'humor','learned','lore','mystery','new','news','religion','reviews','r
5omance','science_fiction']
6>>>brown.words(categories='news')
7['The','Fulton','County','Grand','Jury','said',...]
9>>>brown.words(fileids=['cg22'])
10['Does','our','society','have','a','runaway',',',...]
11>>>brown.sents(categories=['news','editorial','reviews',])
12[['The','Fulton','County','Grand','Jury','said','Friday','an','investiga
13tion','of',"Atlanta's",'recent','primary','election','produced','``','no
14','evidence',"''",'that','any','irregularities','took','place','.'],['T
15he','jury','further','said','in','term-end','presentments','that','the',
16'City','Executive','Committee',',','which','had','over-all','charge','o
17f','the','election',',','``','deserves','the','praise','and','thanks',
18'of','the','City','of','Atlanta',"''",'for','the','manner','in','which
19','the','election','was','conducted','.'],...]
20>>>

1>>>fromnltk.corpusimportbrown
2>>>news_text=brown.words(categories='news')
3>>>fdist=nltk.FreqDist([w.lower()forwinnews_text])
4>>>modals=['can','could','may','might','must','will']
5>>>forminmodals:
6...print("%s:%d"%(m,fdist[m]))
7...
8can:94
9could:87
10may:93
11might:38
12must:53
13will:389
14>>>

1>>>fromnltk.corpusimportudhr
2>>>languages=['Chickasaw','English','German_Deutsch','Greenlandic_Inuktikut'
3,'Hungarian_Magyar','Ibibio_Efik']
4>>>
5>>>cfd=nltk.ConditionalFreqDist(
6...(lang,len(word))
7...forlanginlanguages
8...forwordinudhr.words(lang+'-Latin1'))
9>>>cfd.plot(cumulative=True)
10>>>

1>>>fromnltk.corpusimport*
2>>>corpus_root=r"E:\corpora"//本地存放文本的目录，原始的nltk数据库存放目录为D：\
3>>>wordlists=PlaintextCorpusReader(corpus_root,'.*')
4>>>wordlists.fileids()//获取文件列表
5['README','aaaaaaaaaaa.txt','austen-emma.txt','austen-persuasion.txt','auste//其中的aaaaaaaaaaa.txt是自定义的文件
6n-sense.txt','bible-kjv.txt','blake-poems.txt','bryant-stories.txt','burgess
7-busterbrown.txt','carroll-alice.txt','chesterton-ball.txt','chesterton-brown
8.txt','chesterton-thursday.txt','edgeworth-parents.txt','luo.txt','melville-
9moby_dick.txt','milton-paradise.txt','shakespeare-caesar.txt','shakespeare-ha
10mlet.txt','shakespeare-macbeth.txt','whitman-leaves.txt']
11>>>