把NLTK中提取的unigram 特征转换成0,1向量表示方式

weka中arff格式文件 需要给出特征向量才可读取。NLTK中不需要，但是可以把features转成 向量的形式表示出来。

本人代码设计能力实在太弱，折腾了一上午才调试出来可用的script 是在python下完成的。

""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""

import csv

import re

def replaceTwoOrMore(s):

# pattern to look for three or more repetitions of any character, including 改写如 loveeeeeee的词 为 love

# newlines.

pattern = re.compile(r"(.)\1{1,}", re.DOTALL)

return pattern.sub(r"\1\1", s)

inpfile = open("stopwords.txt", "r") #把读入的tweets去掉 stopwords

line1 = inpfile.readline()

stopWords = []

while line1:

word1 = line1.strip()

stopWords.append(word1)

line1 = inpfile.readline()

inpfile.close()

inpfile = open("selected_features.txt", "r") #读取feature list

line2 = inpfile.readline()

selected_features = []

while line2:

word2 = line2.strip()

selected_features.append(word2)

line2 = inpfile.readline()

inpfile.close()

posWords = []

negWords = []

file1 = 'positive.csv'

file2 = 'negative.csv'

posdata = open(file1, 'rb')

negdata = open(file2, 'rb')

initial_value = 0

reader1 = csv.reader(posdata, delimiter=',', quotechar='"', escapechar='\\')

for row in reader1:

line = []

array = [initial_value for i in range(2000)]

label = row[0]

word = (row[1]) # row[5] for 600000 data set row[1] for my own data

words = word.split()

for w in words:

w = replaceTwoOrMore(w)

w = w.strip('\'"?,.')

val = re.search(r"^[a-zA-Z][a-zA-Z0-9]*[a-zA-Z]+[a-zA-Z0-9]*$", w)

if(w in stopWords or val is None):

continue

else:

line.append(w.lower())

for i in range(0,2000): #和已经选取的feature list 进行比较，如果feature list中 含有 tweets中的词 标注1 否则 标注0， feature list有多长，就是多少维 #的向量 本例中为 2000

if(selected_features[i] in line):

array[i] = "1,"

else:

array[i] = "0,"

array.append('pos')

posWords.append(array)

#posWords = list(posWords)

reader2 = csv.reader(negdata , delimiter=',', quotechar='"', escapechar='\\')

for row in reader2:

line = []

array = [initial_value for i in range(2000)]

label = row[0]

word = (row[1]) # row[5] for 600000 data set row[1] for my own data

words = word.split()

for w in words:

w = replaceTwoOrMore(w)

w = w.strip('\'"?,.')

val = re.search(r"^[a-zA-Z][a-zA-Z0-9]*[a-zA-Z]+[a-zA-Z0-9]*$", w)

if(w in stopWords or val is None):

continue

else:

line.append(w.lower())

for i in range(0,2000):

if(selected_features[i] in line):

array[i] = "1,"

else:

array[i] = "0,"

array.append('neg')

negWords.append(array)

pos = posWords

neg = negWords

f1=open('pos_array.txt','w') #将转换完成的向量 输出

for i in pos:

k=' '.join([str(j) for j in i])

f1.write(k+"\n")

f1.close()

f2=open('neg_array.txt','w')

for i in neg:

k=' '.join([str(j) for j in i])

f2.write(k+"\n")

f2.close()

""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""