熵相关计算

#借用下python机器学习中的数据集

import numpy as np
from collections import Counter
from math import  log

def CreateDataSet():
dataset = np.array([[1, 1, 'yes' ],
[1, 1, 'yes' ],
[1, 0, 'no'],
[0, 1, 'no'],
[0, 1, 'no']])
return dataset

'''熵  h(x)  sum(-p(x)*log p(x) )'''
def cal_entropy(dataset,m): #m表示计算dataset中第k列的熵
feature_cnt=Counter(dataset[:,m])    #引入Counter进行频度统计，或者采用字典的方式进行累加计算也可以；
length=len(dataset)
res= sum( [ -(v/length)*log(v/length) for v in feature_cnt.values() ])
return res

datasets=CreateDataSet()  #这里label其实没什么用
print(cal_entropy(datasets,2))

'''联合熵，将两列特征拼接好后进行计算 h(x,y)  -p(x,y)*log p(x,y)'''

def cal_union_entropy(dataset,m,n):  #计算两个特征的联合熵
col1=dataset[:,m]
col2=dataset[:,n]
feature_cnt=Counter([str(col1[i])+col2[i] for i in range(len(col1))])
length=len(dataset)
res= sum( [ -(v/length)*log(v/length) for v in feature_cnt.values() ])
return res

print(cal_union_entropy(datasets,1,2))

'''条件熵   h(x,y)-h(x)  x确定时，y的不确定性 '''

def cal_condition_entropy(dataset,m,n):
return  cal_union_entropy(dataset,m,n)-cal_entropy(dataset,n)

print(cal_condition_entropy(datasets,1,2))

'''交叉熵 h(p,q)=sum(-p*log(q))'''

def cal_cross_entropy(dataset,m):
feature_cnt=Counter(dataset[:,m])    #引入Counter进行频度统计，或者采用字典的方式进行累加计算也可以；
length=len(dataset)
prob=[0.5,0.5]                  ###模拟一个q分布
value=list(feature_cnt.values())
res=sum([ - (value[i]/length)*log(prob[i]) for i in range(len(value)) ])
return res

print(cal_cross_entropy(datasets,2))

'''相对熵 kl(p||q)=sum(-p*log(q/p))'''

def cal_cross_entropy(dataset,m):
feature_cnt=Counter(dataset[:,m])    #引入Counter进行频度统计，或者采用字典的方式进行累加计算也可以；
length=len(dataset)
prob=[0.5,0.5]                  ###模拟一个q分布
value=list(feature_cnt.values())
res=sum([ - (value[i]/length)*log(prob[i]*length/value[i]) for i in range(len(value)) ])
return res

print(cal_cross_entropy(datasets,2))