敏感词汇过滤DFA算法

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace SensitiveWordFilter
{
public class SensitiveWord
{
private static readonly char IsEndChar = '$';

/**
* 初始化敏感词库<br>
* 将敏感词加入到HashMap中<br>
* 构建DFA算法模型
*
* @author dxm
*
*/
public class SensitiveWordInit
{

// 字符编码
private static readonly  String ENCODING = "UTF-8";

/**
* 初始化敏感字库
*
* @return
*/
public Dictionary<char, object> initKeyWord()
{

// 读取敏感词库
HashSet<String> wordSet = readSensitiveWordFile();

// 将敏感词库加入到HashMap中
return addSensitiveWordToHashMap(wordSet);
}

/**
* 读取敏感词库，将敏感词放入HashSet中，构建一个DFA算法模型：<br>
* 中 = {
*       isEnd = 0
*       国 = {
*             isEnd = 1
*             人 = {
*                   isEnd = 0
*                   民 = {
*                         isEnd = 1
*                   }
*             }
*             男 = {
*                   isEnd = 0
*                   人 = {
*                         isEnd = 1
*                   }
*             }
*       }
* }
* 五 = {
*       isEnd = 0
*       星 = {
*             isEnd = 0
*             红 = {
*                    isEnd = 0
*                    旗 = {
*                           isEnd = 1
*                    }
*              }
*       }
* }
*/
private Dictionary<char, object> addSensitiveWordToHashMap(HashSet<String> wordSet)
{

// 初始化敏感词容器，减少扩容操作
Dictionary<char, object> wordMap = new Dictionary<char, object>(wordSet.Count);

foreach (String word in wordSet)
{
IDictionary<char, object> nowMap = wordMap;
for (int i = 0; i < word.Length; i++)
{

// 转换成char型
char keyChar = word[i];

if (keyChar == IsEndChar)
continue;

Object tempMap;
// 获取
nowMap.TryGetValue(keyChar, out tempMap);

// 如果存在该key，直接赋值
if (tempMap != null)
{
nowMap = (Dictionary<char, object>)tempMap;
}

// 不存在则，则构建一个map，同时将isEnd设置为0，因为他不是最后一个
else {

// 设置标志位
Dictionary<char, object> newMap = new Dictionary<char, object>();
newMap.Add(IsEndChar, "0");

// 添加到集合
nowMap.Add(keyChar, newMap);
nowMap = newMap;
}

// 最后一个
if (i == word.Length - 1)
{
nowMap[IsEndChar] = "1";
}
}
}

return wordMap;
}

/**
* 读取敏感词库中的内容，将内容添加到SortedSet集合中
*
* @return
* @throws Exception
*/
private HashSet<String> readSensitiveWordFile()
{
HashSet<String> wordSet = new HashSet<string>();
string content = File.ReadAllText("dic.txt", Encoding.GetEncoding(ENCODING));
using (StringReader sr = new StringReader(content))
{
string s;
while ((s = sr.ReadLine()) != null)
{
wordSet.Add(s);
}
}
return wordSet;
}
}

public class SensitivewordFilter
{

private Dictionary<char, object> sensitiveWordMap = null;

// 最小匹配规则
public static int minMatchTYpe = 1;

// 最大匹配规则
public static int maxMatchType = 2;

// 单例
private static SensitivewordFilter inst = null;

/**
* 构造函数，初始化敏感词库
*/
private SensitivewordFilter()
{
sensitiveWordMap = new SensitiveWordInit().initKeyWord();
}

/**
* 获取单例
*
* @return
*/
public static SensitivewordFilter getInstance()
{
if (null == inst)
{
inst = new SensitivewordFilter();
}
return inst;
}

/**
* 判断文字是否包含敏感字符
*
* @param txt
* @param matchType
* @return
*/
public bool isContaintSensitiveWord(String txt, int matchType = 1)
{
bool flag = false;
for (int i = 0; i < txt.Length; i++)
{

// 判断是否包含敏感字符
int matchFlag = this.CheckSensitiveWord(txt, i, matchType);

// 大于0存在，返回true
if (matchFlag > 0)
{
flag = true;
}
}
return flag;
}

/**
* 获取文字中的敏感词
*
* @param txt
* @param matchType
* @return
*/
public HashSet<String> getSensitiveWord(String txt, int matchType = 1)
{
HashSet<String> sensitiveWordList = new HashSet<String>();

for (int i = 0; i < txt.Length; i++)
{

// 判断是否包含敏感字符
int length = CheckSensitiveWord(txt, i, matchType);

// 存在,加入list中
if (length > 0)
{
sensitiveWordList.Add(txt.Substring(i, length));

// 减1的原因，是因为for会自增
i = i + length - 1;
}
}

return sensitiveWordList;
}

/**
* 替换敏感字字符
*
* @param txt
* @param matchType
* @param replaceChar
* @return
*/
public String replaceSensitiveWord(String txt, String replaceChar, int matchType = 1)
{
StringBuilder sb = new StringBuilder(txt);
for (int i = 0; i < txt.Length; i++)
{

// 判断是否包含敏感字符
int length = CheckSensitiveWord(txt, i, matchType);

// 存在,加入list中
if (length > 0)
{
var ttxt = txt.Substring(i, length);
sb.Replace(ttxt, getReplaceChars(replaceChar, ttxt.Length), i, length);

// 减1的原因，是因为for会自增
i = i + length - 1;
}
}

return sb.ToString();
}

/**
* 获取替换字符串
*
* @param replaceChar
* @param length
* @return
*/
private String getReplaceChars(String replaceChar, int length)
{
StringBuilder sb = new StringBuilder();
for (int i = 0; i < length; i++)
{
sb.Append(replaceChar);
}

return sb.ToString();
}

/**
* 检查文字中是否包含敏感字符，检查规则如下：<br>
* 如果存在，则返回敏感词字符的长度，不存在返回0
*
* @param txt
* @param beginIndex
* @param matchType
* @return
*/
public int CheckSensitiveWord(String txt, int beginIndex, int matchType)
{

// 敏感词结束标识位：用于敏感词只有1位的情况
bool flag = false;

// 匹配标识数默认为0
int matchFlag = 0;
Dictionary<char, object> nowMap = sensitiveWordMap;
int tempFlag = 0;
Dictionary<char, object> tempMapForBack = new Dictionary<char, object>();
int len = txt.Length;
for (int i = beginIndex; i < len; i++)
{
char word = txt[i];

if (word == IsEndChar)
continue;

// 获取指定key
Object tempMap;
// 获取
nowMap.TryGetValue(word, out tempMap);

if (tempFlag == 0)
tempMapForBack = nowMap;

// 如果存在该key，直接赋值
if (tempMap != null)
{
nowMap = (Dictionary<char, object>)tempMap;
}
else
{
if (tempFlag > 0)
{
matchFlag = matchFlag - (i - tempFlag);
i = tempFlag - 1;
nowMap = tempMapForBack;
continue;
}
else
{
nowMap = null;
}
}

// 存在，则判断是否为最后一个
if (nowMap != null)
{

// 找到相应key，匹配标识+1
matchFlag++;

object value;

if (nowMap.TryGetValue(IsEndChar, out value))
{
if (value is string)
{
// 如果为最后一个匹配规则,结束循环，返回匹配标识数
if ("1" == (string)value)
{
if (nowMap.Keys.Count == 1 || tempFlag != 0 || i == len - 1)
{
// 结束标志位为true
flag = true;

// 最小规则，直接返回,最大规则还需继续查找
if (SensitivewordFilter.minMatchTYpe == matchType)
{
break;
}
}
else
{
tempFlag = i;
}
}
}
}
}
// 不存在，直接返回
else
{
break;
}
}

// 长度必须大于等于1，为词
if (matchFlag < 2 || !flag)
{
matchFlag = 0;
}
return matchFlag;
}
}
}
}

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace SensitiveWordFilter
{
class Program
{
static void Main(string[] args)
{
SensitiveWord.SensitivewordFilter filter = SensitiveWord.SensitivewordFilter.getInstance();
String txt = "$fuckfuck you你麻痹e菜太菜了fuckyou从飞啊 fuck you";
String hou = filter.replaceSensitiveWord(txt, "*");
Console.WriteLine("替换前的文字为：" + txt);
Console.WriteLine("替换后的文字为：" + hou);
Console.ReadKey();
}
}
}