编译原理-词法分析（lexical analysis）初识（续） 推荐

这一篇与前一篇的思路是一样的，只不过这个例子稍微能够处理多行注释的问题。

原来的思路是这样的：

使用Java的正则表达式功能，并对Tiny源程序代码做了一定的前提条件所完成的 Tiny 语言，其实就是模仿教材中的最终输出文件解析出每一个 Token。

默认每个Token之间都以空格“ ”隔开，因此可用Java中的正则表达式将每一行依此规律拆分为一个个Token，然后再对每一个Token进行类别匹配（也用到正则表达式），最后按类别打印输出。以下实现但是不能够处理多行注释的问题。

对教材中的源程序样例代码修改如下（加了不少空格变成sample2.tny）：




扫描之后的输出结果：




代码实现：

package lexical_analysis;

import java.io.BufferedReader;
import java.io.FileReader;
import java.util.regex.Pattern;

public class OriginalLexicalAnalyser {

private static final int RESERVE_DWORD = 1;
private static final int ARITHMETIC_SYMBOLS = 2;
private static final int ID = 3;
private static final int NUM = 4;

// 保留字
private String[] reservedWords = new String[] { "read", "if", "then",
"repeat", "until", "write",
"end" };
// 数学运算符
private String[] arithmeticSymbols = new String[] { "+", "-", "*", "/",
"%", ":=", "=", "<",
">", "<=", ">=" };
// 源程序文件输入流
private BufferedReader sourceFile;
// 代码行数
private int lineCount = 0;
boolean commentFlag = false;

public OriginalLexicalAnalyser(String sourceFilePath) throws Exception {
// 创建并加载源程序文件输入流
this.sourceFile = new BufferedReader(new FileReader(sourceFilePath));
}

public void scan() throws Exception {
String eachLine = "";

while ((eachLine = this.sourceFile.readLine()) != null) {
++lineCount;
System.out.printf("%2d: %s\n", lineCount, eachLine);

int start = 0;
int end = 0;
int lineLen = eachLine.length();

String nextChar;
String token = "";

if("}".equals(eachLine)) {
commentFlag = false;
printToken(eachLine);
continue;
}

while (end < lineLen - 1) {
nextChar = eachLine.substring(end, end + 1);

// 上一行是多行注释开始,即 {
if (commentFlag == true) {
end = processComment(eachLine);
token = eachLine.substring(start, end);
printToken(token);

} else {
if (" ".equals(nextChar)) {
token = eachLine.substring(start, end);
printToken(token);
start = end + 1;
end = start;
} else if (";".equals(nextChar)) {
token = eachLine.substring(start, end);
printToken(token);
printToken(";");
break;
} else if("{".equals(nextChar)){
commentFlag = true;
start = end + 1;
end = start;
} else {
end++;
}
}
}
}
}

private int processComment(String eachLine) {
String ch;
int start = 0;
int lineLen = eachLine.length();
for (int i = 1; i < lineLen; ++i) {
ch = eachLine.substring(start, i);
start++;
if ("}".equals(ch)) {
commentFlag = false;
return i;
}
}
return lineLen - 1;
}

private void printToken(String token) {
if(isArithmeticSymbol(token)) {     // 数学运算符
System.out.println("    " + lineCount + ": " + token);
} else if(isReservedWord(token)) {  // 保留字
if(lineCount == 7) {
System.out.println("==========" + token + "===");
}
System.out.println("    " + lineCount + ": " + "reserved word: " + token);
// 源程序文件结束符
if("end".equals(token)) {
System.out.printf("%2d: %s\n", ++lineCount, "EOF");
}
} else if(";".equals(token)) {      // 行结束符,即分号
System.out.println("    " + lineCount + ": " + token);
}  else if(isID(token)) {           // 自定义标识符ID
System.out.println("    " + lineCount + ": " + "ID, name= " + token);
} else if(isNum(token)) {           // 数值NUM
System.out.println("    " + lineCount + ": " + "NUM, val= " + token);
}
}

/**
* 判断是否为“保留字”
* @param token
* @return
*/
private boolean isReservedWord(String token) {
int size = this.reservedWords.length;
for(int i = 0; i < size; i++) {
if(token.equals(reservedWords[i])) {
return true;
}
}
return false;
}

/**
* 判断是否为“数学运算符”
* @param token
* @return
*/
private boolean isArithmeticSymbol(String token) {
int size = this.arithmeticSymbols.length;
for(int i = 0; i < size; i++) {
if(token.equals(arithmeticSymbols[i])) {
return true;
}
}
return false;
}

/**
* 判断是否为“数值NUM”
* @param token
* @return
*/
private boolean isNum(String token) {
boolean flag = Pattern.matches("\\d+?", token);
return flag;
}

/**
* 判断是否为“ID”
* @param token
* @return
*/
private boolean isID(String token) {
boolean flag = Pattern.matches("[a-zA-Z]+?", token);
return flag;
}

/**
* “词法分析程序”的启动入口
* @param args
*/
public static void main(String[] args) throws Exception {
String sourceFilePath = "sample2.tny";
OriginalLexicalAnalyser lexicalAnalyser = new OriginalLexicalAnalyser(sourceFilePath);
lexicalAnalyser.scan();
}

}