Word2vec 计算两个文本之间相似度

安装gensim 并且有pyemd，详情见下

def wmdistance(self, document1, document2):
"""
Compute the Word Mover's Distance between two documents. When using this
code, please consider citing the following papers:

.. Ofir Pele and Michael Werman, "A linear time histogram metric for improved SIFT matching".
.. Ofir Pele and Michael Werman, "Fast and robust earth mover's distances".
.. Matt Kusner et al. "From Word Embeddings To Document Distances".

Note that if one of the documents have no words that exist in the
Word2Vec vocab, `float('inf')` (i.e. infinity) will be returned.

This method only works if `pyemd` is installed (can be installed via pip, but requires a C compiler).

Example:
>>> # Train word2vec model.
>>> model = Word2Vec(sentences)

>>> # Some sentences to test.
>>> sentence_obama = 'Obama speaks to the media in Illinois'.lower().split()
>>> sentence_president = 'The president greets the press in Chicago'.lower().split()

>>> # Remove their stopwords.
>>> from nltk.corpus import stopwords
>>> stopwords = nltk.corpus.stopwords.words('english')
>>> sentence_obama = [w for w in sentence_obama if w not in stopwords]
>>> sentence_president = [w for w in sentence_president if w not in stopwords]

>>> # Compute WMD.
>>> distance = model.wmdistance(sentence_obama, sentence_president)
"""

if not PYEMD_EXT:
raise ImportError("Please install pyemd Python package to compute WMD.")

# Remove out-of-vocabulary words.
len_pre_oov1 = len(document1)
len_pre_oov2 = len(document2)
document1 = [token for token in document1 if token in self]
document2 = [token for token in document2 if token in self]
diff1 = len_pre_oov1 - len(document1)
diff2 = len_pre_oov2 - len(document2)
if diff1 > 0 or diff2 > 0:
logger.info('Removed %d and %d OOV words from document 1 and 2 (respectively).',
diff1, diff2)

if len(document1) == 0 or len(document2) == 0:
logger.info('At least one of the documents had no words that were'
'in the vocabulary. Aborting (returning inf).')
return float('inf')

dictionary = Dictionary(documents=[document1, document2])
vocab_len = len(dictionary)

if vocab_len == 1:
# Both documents are composed by a single unique token
return 0.0

# Sets for faster look-up.
docset1 = set(document1)
docset2 = set(document2)

# Compute distance matrix.
distance_matrix = zeros((vocab_len, vocab_len), dtype=double)
for i, t1 in dictionary.items():
for j, t2 in dictionary.items():
if not t1 in docset1 or not t2 in docset2:
continue
# Compute Euclidean distance between word vectors.
distance_matrix[i, j] = sqrt(np_sum((self[t1] - self[t2])**2))

if np_sum(distance_matrix) == 0.0:
# `emd` gets stuck if the distance matrix contains only zeros.
logger.info('The distance matrix is all zeros. Aborting (returning inf).')
return float('inf')

def nbow(document):
d = zeros(vocab_len, dtype=double)
nbow = dictionary.doc2bow(document)  # Word frequencies.
doc_len = len(document)
for idx, freq in nbow:
d[idx] = freq / float(doc_len)  # Normalized word frequencies.
return d

# Compute nBOW representation of documents.
d1 = nbow(document1)
d2 = nbow(document2)

# Compute WMD.
return emd(d1, d2, distance_matrix)