javacc xxxTokenManager类作用 以及重要方法解析

/* Generated By:JavaCC: Do not edit this line. QueryParserTokenManager.java */
import java.io.StringReader;
/**
*@ summary xxxTokenManager重要方法介绍
*@ xxxTokenManager类根据词法状态和截取字符窜并生成不同类型的token对象
*  提供给xxxParser类进行语法分析,xxxParser通过jj_consume_token(token_kind);
*  方法进行消费指定的token然后xxxParser有xxxTokenManager对象的引用可以拿到
*  通过调用xxxTokenManager.getNextToken()方法得到下一个应该出现的字符对象
*  如果记号管理器返回的token对象和需要的token对象类型不统一说明字符窜
*  有语法错误
*/
/** Token Manager. */
public class QueryParserTokenManager implements QueryParserConstants
{

/** 输出调试信息的输出流. */
public static  java.io.PrintStream debugStream = System.out;
/** 设置调试输出流. */
public static  void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }
// ...
static final int[] jjnextStates = {
};

/** 当前在Token或Special_Token中定义的词法状态名字数组*/
/** Lexer state names. */
public static final String[] lexStateNames = {
"DEFAULT",
"IN_SINGLE_LINE_COMMENT",
"IN_FORMAL_COMMENT",
"IN_MULTI_LINE_COMMENT",
};

/** Lex State array. */
public static final int[] jjnewLexState = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 2, 3, 0, 0, 0, -1, -1,
};
static final long[] jjtoToken = {
0x101e1L,
};
static final long[] jjtoSkip = {
0x701eL,
};
static final long[] jjtoSpecial = {
0x7000L,
};
static final long[] jjtoMore = {
0x8e00L,
};
static protected SimpleCharStream input_stream;/** 用户输入的字符窜字节流*/
static private final int[] jjrounds = new int[4];
static private final int[] jjstateSet = new int[8];
private static final StringBuilder jjimage = new StringBuilder();
private static StringBuilder image = jjimage;
private static int jjimageLen;
private static int lengthOfMatch;
static protected char curChar;
/** Constructor. */
public QueryParserTokenManager(SimpleCharStream stream){
if (input_stream != null)
throw new TokenMgrError("ERROR: Second call to constructor of static lexer. You must use ReInit() to initialize the static variables.", TokenMgrError.STATIC_LEXER_ERROR);
input_stream = stream;
}

/** Constructor. */
public QueryParserTokenManager(SimpleCharStream stream, int lexState){
this(stream);
SwitchTo(lexState);
}

/** Reinitialise parser. */
static public void ReInit(SimpleCharStream stream)
{
jjmatchedPos = jjnewStateCnt = 0;
curLexState = defaultLexState;
input_stream = stream;
ReInitRounds();
}

/** 将当前token对象补充完整*/
static protected Token jjFillToken()
{
final Token t;
final String curTokenImage;
final int beginLine;
final int endLine;
final int beginColumn;
final int endColumn;
String im = jjstrLiteralImages[jjmatchedKind];
curTokenImage = (im == null) ? input_stream.GetImage() : im;
beginLine = input_stream.getBeginLine();
beginColumn = input_stream.getBeginColumn();
endLine = input_stream.getEndLine();
endColumn = input_stream.getEndColumn();
t = Token.newToken(jjmatchedKind, curTokenImage);

t.beginLine = beginLine;
t.endLine = endLine;
t.beginColumn = beginColumn;
t.endColumn = endColumn;

return t;
}

static int curLexState = 0;
static int defaultLexState = 0;
static int jjnewStateCnt;
static int jjround;
static int jjmatchedPos;
static int jjmatchedKind;

/** Get the next Token. */
/** 记号管理器如何生成token*/
public static Token getNextToken()
{
Token specialToken = null;/** 在匹配过程中可以生成的特殊记号*/
Token matchedToken; /** 匹配到的记号*/
int curPos = 0;

EOFLoop :
for (;;)
{
try
{
curChar = input_stream.BeginToken();/** 从输入的字节流中读取一个字符*/
}
catch(java.io.IOException e)
{
jjmatchedKind = 0;
matchedToken = jjFillToken();
matchedToken.specialToken = specialToken;
return matchedToken;
}

image = jjimage;
image.setLength(0);
jjimageLen = 0;

for (;;)
{
switch(curLexState)/**根据词法状态找到当前字符所对应的token_kind类型 */
{
case 0:/** 默认的词法状态*/
try {
input_stream.backup(0);
while (curChar <= 32 && (0x100002600L & (1L << curChar)) != 0L)
curChar = input_stream.BeginToken();
}
catch (java.io.IOException e1) {
continue EOFLoop;
}

jjmatchedKind = 0x7fffffff;
jjmatchedPos = 0;
curPos = jjMoveStringLiteralDfa0_0();
break;
case 1:
jjmatchedKind = 0x7fffffff;
jjmatchedPos = 0;
curPos = jjMoveStringLiteralDfa0_1();
if (jjmatchedPos == 0 && jjmatchedKind > 15)
{
jjmatchedKind = 15;
}
break;
case 2:
jjmatchedKind = 0x7fffffff;
jjmatchedPos = 0;
curPos = jjMoveStringLiteralDfa0_2();
if (jjmatchedPos == 0 && jjmatchedKind > 15)
{
jjmatchedKind = 15;
}
break;
case 3:
jjmatchedKind = 0x7fffffff;
jjmatchedPos = 0;
curPos = jjMoveStringLiteralDfa0_3();
if (jjmatchedPos == 0 && jjmatchedKind > 15)
{
jjmatchedKind = 15;
}
break;
}

if (jjmatchedKind != 0x7fffffff)
{
if (jjmatchedPos + 1 < curPos)
input_stream.backup(curPos - jjmatchedPos - 1);
if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L)
{
matchedToken = jjFillToken();
matchedToken.specialToken = specialToken;/** 如果有特殊字符就保存到token对象的specialtoken属性中*/
if (jjnewLexState[jjmatchedKind] != -1)
curLexState = jjnewLexState[jjmatchedKind];
return matchedToken;
}
else if ((jjtoSkip[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L)
{
if ((jjtoSpecial[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L)
{
matchedToken = jjFillToken();
if (specialToken == null)
specialToken = matchedToken;
else
{
matchedToken.specialToken = specialToken;
specialToken = (specialToken.next = matchedToken);
}
SkipLexicalActions(matchedToken);/** 忽略跳过指定的字符*/
}
else
SkipLexicalActions(null);
if (jjnewLexState[jjmatchedKind] != -1)
curLexState = jjnewLexState[jjmatchedKind];
continue EOFLoop;
}
/** 处理更多这样的字符的方法*/
MoreLexicalActions();

if (jjnewLexState[jjmatchedKind] != -1)
curLexState = jjnewLexState[jjmatchedKind];
curPos = 0;
jjmatchedKind = 0x7fffffff;
try {
curChar = input_stream.readChar();
continue;
}
catch (java.io.IOException e1) { }
}
int error_line = input_stream.getEndLine();
int error_column = input_stream.getEndColumn();
String error_after = null;
boolean EOFSeen = false;
try { input_stream.readChar(); input_stream.backup(1); }
catch (java.io.IOException e1) {
EOFSeen = true;
error_after = curPos <= 1 ? "" : input_stream.GetImage();
if (curChar == '\n' || curChar == '\r') {
error_line++;
error_column = 0;
}
else
error_column++;
}
if (!EOFSeen) {
input_stream.backup(1);
error_after = curPos <= 1 ? "" : input_stream.GetImage();
}
throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
}
}
}

/** 忽略的字符需要调用的方法 */
static void SkipLexicalActions(Token matchedToken)
{
switch(jjmatchedKind)
{
default :
break;
}
}
/** 更多这样的单词处理 */
static void MoreLexicalActions()
{
jjimageLen += (lengthOfMatch = jjmatchedPos + 1);
switch(jjmatchedKind)
{
case 10 :
image.append(input_stream.GetSuffix(jjimageLen));
jjimageLen = 0;
input_stream.backup(1);
break;
default :
break;
}
}

}