ikanalyzer分词，计算信息熵排序分词结果

因需求，现需分词接口，故记录之。

1、需要依赖：

1 public JSONArray entropy(String content, Integer quantity) throws Exception {
2         List<String> words = extract(DelHtmlTagUtil.delHTMLTag(content), quantity);
3         JSONArray array = calculateWordEntropy(words);
4         return array;
5     }
6
7 /**
8      * 传入String类型的文章，智能提取单词放入list
9      *
10      * @param content  传入分词的内容
11      * @param quantity 截取关键字在几个单词以上的数量，默认为1
12      * @return
13      */
14     private List<String> extract(String content, Integer quantity) throws IOException {
15         List<String> list = Lists.newArrayList();
16         StringReader reader = new StringReader(content);
17         IKSegmenter ik = new IKSegmenter(reader, true);
18         Lexeme lex = null;
19         while ((lex = ik.next()) != null) {
20             //String typeString = lex.getLexemeTypeString();  词语类型
21             String word = lex.getLexemeText();
22             if (word.length() > quantity) {//判断截取关键字在几个单词以上的数量
23                 list.add(word);
24             }
25         }
26         return list;
27     }
28
29     private JSONArray calculateWordEntropy(List<String> words) throws Exception{
30
31         int length = words.size();
32         ArrayList<String[]> wordList = new ArrayList<String[]>();
33         // 将分好的词每3个一组存到数组中
34         for (int i = 0; i < length; i++) {
35
36             String[] wordSeg = new String[3];
37             if (i == 0) {
38                 wordSeg[0] = "null";
39                 wordSeg[1] = words.get(i);
40                 wordSeg[2] = words.get(i + 1);
41             } else if (i == length - 1) {
42                 wordSeg[0] = words.get(i - 1);
43                 wordSeg[1] = words.get(i);
44                 wordSeg[2] = "null";
45             } else {
46                 wordSeg[0] = words.get(i - 1);
47                 wordSeg[1] = words.get(i);
48                 wordSeg[2] = words.get(i + 1);
49             }
50
51             wordList.add(wordSeg);
52
53         }
54         // 去除重复的词
55         List<String> lists = Lists.newArrayList();
56         for (int l = 0; l < length; l++) {
57             lists.add(words.get(l));
58         }
59         List<String> tempList = Lists.newArrayList();
60         for (String str : lists) {
61             if (!(tempList.contains(str))) {
62                 tempList.add(str);
63             }
64         }
65         String[] wordClean = new String[tempList.size()];
66         for (int m = 0; m < tempList.size(); m++) {
67             wordClean[m] = tempList.get(m);
68         }
69         // 统计每个词的词频
70         int[] frequent = new int[wordClean.length];
71         for (int j = 0; j < wordClean.length; j++) {
72             int count = 0;
73             for (int k = 0; k < words.size(); k++) {
74                 if (wordClean[j].equals(words.get(k))) {
75                     count++;
76                 }
77             }
78             frequent[j] = count;
79         }
80         // 将三元组中中间的那个词相同的存到一个list中，然后计算该词的信息熵
81         double[] allEntropy = new double[wordClean.length];
82         for (int n = 0; n < wordClean.length; n++) {
83             ArrayList<String[]> wordSegList = new ArrayList<String[]>();
84             int count = 1;
85             for (int p = 0; p < wordList.size(); p++) {
86                 String[] wordSegStr = wordList.get(p);
87                 if (wordSegStr[1].equals(wordClean
)) {
88                     count++;
89                     wordSegList.add(wordSegStr);
90                 }
91             }
92             String[] leftword = new String[wordSegList.size()];
93             String[] rightword = new String[wordSegList.size()];
94             // 计算左信息熵
95             for (int i = 0; i < wordSegList.size(); i++) {
96                 String[] left = wordSegList.get(i);
97                 leftword[i] = left[0];
98             }
99             // 去除左边重复的词
100             List<String> listsLeft = new ArrayList<String>();
101             for (int l = 0; l < leftword.length; l++) {
102                 listsLeft.add(leftword[l]);
103             }
104             List<String> tempListLeft = new ArrayList<String>();
105             for (String str : listsLeft) {
106                 if (!(tempListLeft.contains(str))) {
107                     tempListLeft.add(str);
108                 }
109             }
110             String[] leftWordClean = new String[tempListLeft.size()];
111             for (int m = 0; m < tempListLeft.size(); m++) {
112                 leftWordClean[m] = tempListLeft.get(m);
113             }
114             // 统计左边每个词的词频
115             int[] leftFrequent = new int[leftWordClean.length];
116             for (int j = 0; j < leftWordClean.length; j++) {
117                 int leftcount = 0;
118                 for (int k = 0; k < leftword.length; k++) {
119                     if (leftWordClean[j].equals(leftword[k])) {
120                         leftcount++;
121                     }
122                 }
123                 leftFrequent[j] = leftcount;
124             }
125             // 计算左熵值
126             double leftEntropy = 0;
127             for (int i = 0; i < leftFrequent.length; i++) {
128                 double a = (double) leftFrequent[i] / count;
129                 double b = Math.log((double) leftFrequent[i] / count);
130                 leftEntropy += -a * b;
131                 // leftEntropy +=
132                 // (-(double)(leftFrequent[i]/count))*Math.log((double)(leftFrequent[i]/count));
133             }
134             // 计算右信息熵
135             for (int i = 0; i < wordSegList.size(); i++) {
136                 String[] right = wordSegList.get(i);
137                 rightword[i] = right[2];
138             }
139             // 去除右边重复的词
140             List<String> listsRight = new ArrayList<String>();
141             for (int l = 0; l < rightword.length; l++) {
142                 listsRight.add(rightword[l]);
143             }
144             List<String> tempListRight = new ArrayList<String>();
145             for (String str : listsRight) {
146                 if (!(tempListRight.contains(str))) {
147                     tempListRight.add(str);
148                 }
149             }
150             String[] rightWordClean = new String[tempListRight.size()];
151             for (int m = 0; m < tempListRight.size(); m++) {
152                 rightWordClean[m] = tempListRight.get(m);
153             }
154             // 统计右边每个词的词频
155             int[] rightFrequent = new int[rightWordClean.length];
156             for (int j = 0; j < rightWordClean.length; j++) {
157                 int rightcount = 0;
158                 for (int k = 0; k < rightword.length; k++) {
159                     if (rightWordClean[j].equals(rightword[k])) {
160                         rightcount++;
161                     }
162                 }
163                 rightFrequent[j] = rightcount;
164             }
165             // 计算右熵值
166             double rightEntropy = 0.0;
167             for (int i = 0; i < rightFrequent.length; i++) {
168                 double a = (double) rightFrequent[i] / count;
169                 double b = Math.log((double) rightFrequent[i] / count);
170                 rightEntropy += -a * b;
171                 // rightEntropy +=
172                 // (-(double)(rightFrequent[i]/count))*Math.log((double)(rightFrequent[i]/count));
173             }
174             // 计算词的总信息熵
175             double wordEntropy = leftEntropy + rightEntropy;
176             allEntropy
 = wordEntropy;
177
178         }
179         JSONArray list = new JSONArray();
180         for (int i = 0; i < allEntropy.length; i++) {
181             JSONObject obj = new JSONObject();
182             obj.put("name", wordClean[i]);
183             obj.put("entropy", allEntropy[i]);
184             list.add(obj);
185         }
186         Collections.sort(list, (o1, o2) -> {
187             Double d1 = ((JSONObject) o1).getDouble("entropy");
188             Double d2 = ((JSONObject) o2).getDouble("entropy");
189             return d2.compareTo(d1);
190         });
191
192         return list;
193     }