编写工具展示lucene分词内部分析过程

此代码工具可以展示：对应的语汇单元还有多个属性没有在代码中展示，可查看包org.apache.lucene.analysis.tokenattributes里所有的attribute

package com.liu.lucene.pro;

import java.io.IOException;

import java.io.Reader;

import org.apache.lucene.analysis.Analyzer;

import org.apache.lucene.analysis.TokenStream;

import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;

import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;

public class AnalyzerUtils {

public static void displayTokens(Analyzer analyzer,Reader reader){

try {

TokenStream tokenStream = analyzer.tokenStream("path", reader);

tokenStream.reset();

CharTermAttribute term = tokenStream.addAttribute(CharTermAttribute.class);

PositionIncrementAttribute posIncrAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);

while(tokenStream.incrementToken()){

System.out.print("["+term.toString()+"]");

}

} catch (IOException e) {

// TODO Auto-generated catch block

e.printStackTrace();

}

}

}