基于编辑距离和最长公共子串计算字符串相似度

基于编辑距离和最长公共子串实现的文本相似度计算：

计算公式为

s=lc/(led+lc)

lc为最长公共子串的长度

led为编辑距离

考虑到汉字和字符的不同，增加了Str2Word()进行字符串分词,实现单字的比较。代码写了一段时间了，有些遗忘。

ld（）为编辑距离求解

lcsLen()为动态规划求解最长公共子串问题

lcs()输出最长公共子串

Str2Word()进行字符串分词

# -*- coding: utf-8 -*-
from __future__ import division
import re

def ld(firstStr,secondStr):
if len(firstStr)>len(secondStr):
firstStr,secondStr=secondStr,firstStr
if len(firstStr)==0:
return len(secondStr),0
if len(secondStr)==0:
return len(firstStr),0
matrix=[range(len(secondStr)+1) for x in range(len(firstStr)+1)]
#print matrix
for i in range(1,len(firstStr)+1):
for j in range(1,len(secondStr)+1):
ld1=matrix[i-1][j]+1
ld2=matrix[i][j-1]+1
ld3=matrix[i-1][j-1]
if firstStr[i-1]!=secondStr[j-1]:
ld3+=1
matrix[i][j]=min(ld1,ld2,ld3)
#print matrix
ld=matrix[len(firstStr)][len(secondStr)]
#print len(secondStr)
return ld

def lcsLen(firstStr,secondStr):
if len(firstStr)>len(secondStr):
firstStr,secondStr=secondStr,firstStr
if len(firstStr)==0:
return len(secondStr),0
if len(secondStr)==0:
return len(firstStr),0
matrix=[range(len(secondStr)+1) for x in range(len(firstStr)+1)]
#print matrix
for m in range(0,len(firstStr)+1):
matrix[m][0]=0
for k in range(1,len(secondStr)+1):
matrix[0][k]=0
for i in range(1,len(firstStr)+1):
for j in range(1,len(secondStr)+1):
lcs1=matrix[i-1][j]
lcs2=matrix[i][j-1]
lcs3=matrix[i-1][j-1]
if firstStr[i-1]==secondStr[j-1]:
lcs3+=1
matrix[i][j]=max(lcs1,lcs2,lcs3)
#print matrix
lcslen=matrix[len(firstStr)][len(secondStr)]
return lcslen

def midleI(i,firstStr,secondStr):
strr=''
j=0
while i<len(firstStr) and j<len(secondStr):
while j<len(secondStr) and i<len(firstStr):
if firstStr[i]==secondStr[j]:
strr+=secondStr[j]
#print 'strr=',strr
i+=1
j+=1
#print 'j=',j
return strr

def lcs(firstStr,secondStr):
sen=[]
lenMax=0
result=''
lenId=0
for i in range(len(firstStr)):
result=midleI(i,firstStr,secondStr)
sen.append(result)
if len(result)>lenMax:
lenMax=len(result)
lenId=i
lc=sen[lenId]
#lcsLen=len(lcs)
return lc

def similarity(firstStr,secondStr):
s1=Str2Word(firstStr)
s2=Str2Word(secondStr)
lc=len(lcs(s1,s2))
led=ld(s1,s2)
s=lc/(led+lc)
return s

def preProcess(sen,edcode='utf-8'):
sen=sen.decode(edcode)
sen=re.sub(u"[。，、.,！……!《》<>\"':：？\?、\|“”‘’；]","",sen)
#print sen
return sen

def Str2Word(sen,edcode='utf-8'):
i=0
result=[]
sen=preProcess(sen)
length=len(sen)
while i<length:
# find ab...xyz and AB...YZ and 01..89
tempi=i
tok=sen[i:i+1]
while re.search("[0-9A-Za-z\-\+#@_]{1}",tok)<>None:
i+=1
tok=sen[i:i+1]
if i-tempi>0:
#print sen[tempi:i].lower()
result.append(sen[tempi:i].lower().encode(edcode))
#find chinese word
leftword = sen[i:i+1]
if leftword<>" " and i<length:
result.append(leftword)
i+=1
return result

if __name__ == '__main__':
first='一个全球性问题尽管大部分移民并没有跨出国界 '
second='气候移民是一个全球性问题移民并没有跨出国界但其影响却是跨国界的'
s=similarity(first,second)
print '相似度为：',s