【数据挖掘】决策树之ID3算法

    在决策树中，如何衡量数据集的有序度至关重要。划分数据集的最大规则：将无序的数据变得更加有序，划分数据集前后信息发生的变化称为信息增益。香农熵表现为信息的期望值，熵越大，数据越混乱。另外一种度量集合是否无序的方法时基尼不纯度。

def calc_entropy(dataset):#香农熵的计算
lines = len(dataset)
lables={}
for curvect in dataset:
curlable = curvect[-1]#分类
labels.setdefault(curlable,0)
labels[curlable]+=1
shannonEntroy=0.0
for key in lables.keys():
prob = float(lables[key])/lines
shannonEntroy-=math.log(prob,2)*prob
return shannonEntroy

#划分数据集：从dataset中挑选特征cIndex中值为cValue的集合,返回集合中不包括cIndex对应的特征
def splitDataset(dataset,cIndex,cValue):
rset=[]
for featureset in dataset:
if featureset[cIndex] == cValue:
tmpset = featureset[:cIndex]
tmpset.append(featureset[cIndex+1:])
rset.extend(tmpset)
return rset

#ID3算法核心思想：遍历每个特征，找到最好的特征作为划分
def choose_best_feature(dataset):
features = len(dataset[0]) - 1
baseEntropy = calc_entropy(dataset)
bestfearue = -1
bestGain = 0.0
for i in range(features):
#从第i个特征的香农熵
featurelist = [example[i] for example in dataset]
uniqfeatureval = set(featurelist)
newEntropy = 0.0 #第
#计算第i个特征每个特征值中的香农熵,计算信息增益
for val in uniqfeatureval:
subdataset = splitDataset(dataset,i,val)
prob = len(subdataset)/float(len(dataset))
newEntropy += prob * calc_entropy(subdataset)
infoGain = baseEntropy - newEntropy
#保存最佳的特征
if infoGain > bestGain:
bestGain = infoGain
bestfearue = i
return bestfearue

def majortiyCnt(classList):
classCount={}
for vote in classList:
classCount.setdefault(vote,0)
classCount[vote]+=1
sortedclassCount = sorted(classCount.iteritems(),
key=operator.itemgetter(1),reverse=True)
return sortedclassCount[0][0]

#构建决策树，递归终止的条件：1)类别完全相同 2)只剩下一个特征
def createTree(dataset,lables):
classList = [example[-1] for example in dataset]
#划分类别完全相同
if classList.count(classList[0]) == len(classList):
return classList[0]
if len(dataset[0]) == 1:
return majortiyCnt(classList)
bestfeature = choose_best_feature(dataset)
bestlable = labels[bestfeature]
myTree = {bestlable:{}}
del labels[bestfeature]
#递归构建决策树
featrueVals = [example[bestfeature] for example in dataset]
uniqfeature = set(featrueVals)
#为第featrueVals特征的每个特征值构建子决策树
for value in uniqfeature:
sublables = lables[:]#复制
subdataset = splitDataset(dataset,bestfeature,value)
myTree[bestlable][value]=	createTree(subdataset,sublables)
return myTree