上班之后完成的第一个项目

今天下午，终于在3月上班后，完成了第一个项目。

项目的大致内容是，将wikipedia上的某个特定分类（n多个子分类）的所有内容抓取下来，放到gensim中的word2vector训练处模型。

  再将不同的keywords的simword叠加起来，后面的similarity相加，取前topN个，生成一个重复的simword template。

中间进行了很多种尝试，从公司的ES，到google search，最后选定Wikipedia作为语料库。技术都不难，但是过程中趟了一些坑。比如说语料库太大，怎么训练模型等等。

粘上代码，用以将来再看的时候，能会心一笑。

# -*- coding: utf-8 -*-
from __future__ import absolute_import
from nltk.corpus import stopwords
from util.logger import cluster_log
from util.date_util import DateUtil
import settings as job_settings
import gensim

class FilterTemplate(object):
def __init__(self, category):
# TODO: not defined campaign category
self.category = category
self.topN_similarity = job_settings.topN_similarity
self.topN_simword = job_settings.topN_simword
self.X = job_settings.X
self.model_root_path = job_settings.model_root_path

self.date_util = DateUtil()

def model_path(self):
# campaign category has different model path
# TODO: not defined all the category
campaign_category = {
"technology": self.model_root_path + "/technology/technology_model",
"manifacturing": self.model_root_path + "/manifacturing/manifacturing_model"
}
if self.category not in campaign_category.keys():
# if category don't exist process throws an exception
raise Exception("Not defined category: {0}, failed to get model path".format(self.category))
else:
return campaign_category[self.category]

def model_keyword_format(self, keywords_list):
# generate keywords format model needed
english_stopwords = stopwords.words('english')
word_bag = []
for keyword in keywords_list:
try:
if keyword.split(' ') > 1:
word_bag.extend([word for word in keyword.split(' ')])
else:
word_bag.extend(keyword)
except:
cluster_log.info("ERROR KEYWORD: " + keyword)
continue
# dispose of replace word
return [[word] for word in set(word_bag) if word not in english_stopwords and word.isalpha()]

def simword_similarity(self, word_list):
# get simword and similarity of the word in word2vector model
model = gensim.models.Word2Vec.load(self.model_path())
all_simword_list = []
cluster_log.info("strat to find simword and similarity in word2vector model, timestamp: {timstamp}".format(
timstamp = self.date_util.get_current_time_str()
))
for word in word_list:
try:
# topN_simword means to get the top n simword
simword_similarity = model.most_similar(word, topn = self.topN_simword)
# save the word (word similarity = 1) into simword similarity list
word_similarity = (word, 1)
simword_similarity.append(word_similarity)
all_simword_list.append(simword_similarity)
except:
cluster_log.info("{0} can't find simword in word2vector model".format(word))
continue
cluster_log.info("finish to find simword and similarity in word2vector model, timestamp: {timstamp}".format(
timstamp = self.date_util.get_current_time_str()
))
return all_simword_list

def clean_simword(self, simword):
# simword may be number or mojibake, needs to clean up
return True if simword[0].isalpha() and not simword[0].isdigit() and simword[1] != 1 else False

def non_replace_simword(self, all_simword_list):
# generate non replace simword and its total similarity in all simword list
# topN_similarity means get the top N similarity
non_replace_simword = {}
for simword_list in all_simword_list:
word = [w[0] for w in simword_list if w[1] == 1]
try:
for simword in simword_list:
if self.clean_simword(simword):
if not non_replace_simword.has_key(simword[0]):
non_replace_simword[simword[0]] = simword[1]
else:
non_replace_simword[simword[0]] += simword[1]
else:
continue
except:
cluster_log.info("{0} error in all_simword_list ".format(word[0]))
continue
return sorted(non_replace_simword.iteritems(), key = lambda simword: simword[1], reverse = True)[
:self.topN_similarity]

def global_template(self, sorted_set):
# generate global template
global_template = ""
for simword in sorted_set:
try:
if simword[1] != 0:
frequency = int(simword[1] * self.X)
else:
frequency = 1
global_template += (simword[0] + ' ') * frequency
except:
cluster_log.info("Simword: {0} ERROR".format(simword[0]))
continue
return global_template

def generate_global_template(self, keywords_list):
global_template = self.global_template(
self.non_replace_simword(
self.simword_similarity(
self.model_keyword_format(keywords_list)
)))
if len(global_template) == 0:
cluster_log.error("failed to generate global template")
return global_template