2015-04-04 15:09:43 +03:00
|
|
|
package com.annimon.everlastingsummer;
|
|
|
|
|
2015-04-14 16:53:44 +03:00
|
|
|
import java.util.ArrayList;
|
|
|
|
import java.util.HashMap;
|
2015-04-04 15:09:43 +03:00
|
|
|
import java.util.List;
|
2015-04-14 16:53:44 +03:00
|
|
|
import java.util.Locale;
|
|
|
|
import java.util.Map;
|
2015-04-04 15:09:43 +03:00
|
|
|
|
|
|
|
/**
|
|
|
|
* @author aNNiMON
|
|
|
|
*/
|
|
|
|
public final class Lexer {
|
|
|
|
|
|
|
|
public static List<Token> tokenize(String input) {
|
|
|
|
return new Lexer().process(input).getTokens();
|
|
|
|
}
|
|
|
|
|
2015-04-16 14:00:56 +03:00
|
|
|
private static final String OPERATOR_CHARS = "=+-()[]!$:";
|
2015-04-04 15:09:43 +03:00
|
|
|
private static final TokenType[] OPERATOR_TYPES = new TokenType[] {
|
|
|
|
TokenType.EQ,
|
|
|
|
TokenType.PLUS, TokenType.MINUS,
|
|
|
|
TokenType.LPAREN, TokenType.RPAREN, TokenType.LBRACKET, TokenType.RBRACKET,
|
2015-04-16 14:00:56 +03:00
|
|
|
TokenType.EXCL, TokenType.COMMAND, TokenType.COLON,
|
2015-04-04 15:09:43 +03:00
|
|
|
};
|
2015-04-14 16:53:44 +03:00
|
|
|
|
|
|
|
private static final Map<String, TokenType> KEYWORDS;
|
|
|
|
static {
|
|
|
|
KEYWORDS = new HashMap<String, TokenType>();
|
|
|
|
KEYWORDS.put("play", TokenType.PLAY);
|
|
|
|
KEYWORDS.put("stop", TokenType.STOP);
|
|
|
|
KEYWORDS.put("music", TokenType.MUSIC);
|
|
|
|
KEYWORDS.put("ambience", TokenType.AMBIENCE);
|
|
|
|
KEYWORDS.put("sound", TokenType.SOUND);
|
|
|
|
KEYWORDS.put("sound_loop", TokenType.SOUNDLOOP);
|
|
|
|
KEYWORDS.put("fadein", TokenType.FADEIN);
|
|
|
|
KEYWORDS.put("fadeout", TokenType.FADEOUT);
|
2015-04-04 15:09:43 +03:00
|
|
|
|
2015-04-14 16:53:44 +03:00
|
|
|
KEYWORDS.put("scene", TokenType.SCENE);
|
|
|
|
KEYWORDS.put("anim", TokenType.ANIM);
|
|
|
|
KEYWORDS.put("bg", TokenType.BG);
|
|
|
|
KEYWORDS.put("cg", TokenType.CG);
|
|
|
|
KEYWORDS.put("at", TokenType.AT);
|
|
|
|
KEYWORDS.put("window", TokenType.WINDOW);
|
|
|
|
KEYWORDS.put("hide", TokenType.HIDE);
|
|
|
|
KEYWORDS.put("show", TokenType.SHOW);
|
|
|
|
KEYWORDS.put("with", TokenType.WITH);
|
2015-04-12 00:02:44 +03:00
|
|
|
|
2015-04-14 16:53:44 +03:00
|
|
|
KEYWORDS.put("return", TokenType.RETURN);
|
|
|
|
KEYWORDS.put("menu", TokenType.MENU);
|
|
|
|
KEYWORDS.put("endmenu", TokenType.ENDMENU);
|
|
|
|
KEYWORDS.put("jump", TokenType.JUMP);
|
|
|
|
KEYWORDS.put("label", TokenType.LABEL);
|
2015-04-04 15:09:43 +03:00
|
|
|
|
2015-04-15 23:40:18 +03:00
|
|
|
KEYWORDS.put("if", TokenType.IF);
|
2015-04-16 00:00:27 +03:00
|
|
|
KEYWORDS.put("else", TokenType.ELSE);
|
2015-04-15 23:40:18 +03:00
|
|
|
KEYWORDS.put("endif", TokenType.ENDIF);
|
2015-04-16 15:23:10 +03:00
|
|
|
KEYWORDS.put("not", TokenType.NOT);
|
2015-04-15 23:40:18 +03:00
|
|
|
|
2015-04-14 16:53:44 +03:00
|
|
|
KEYWORDS.put("renpy.pause", TokenType.RENPY_PAUSE);
|
|
|
|
KEYWORDS.put("persistent.sprite_time", TokenType.PERSISTENT_SPRITE_TIME);
|
|
|
|
KEYWORDS.put("prolog_time", TokenType.PROLOG_TIME);
|
|
|
|
KEYWORDS.put("day_time", TokenType.DAY_TIME);
|
|
|
|
KEYWORDS.put("sunset_time", TokenType.SUNSET_TIME);
|
|
|
|
KEYWORDS.put("night_time", TokenType.NIGHT_TIME);
|
|
|
|
}
|
2015-04-04 15:09:43 +03:00
|
|
|
|
2015-04-14 19:28:29 +03:00
|
|
|
private final List<Token> tokens;
|
|
|
|
private final StringBuilder buffer;
|
|
|
|
|
2015-04-04 15:09:43 +03:00
|
|
|
private TokenizeState state;
|
|
|
|
private int pos;
|
|
|
|
|
|
|
|
private enum TokenizeState {
|
|
|
|
DEFAULT, NUMBER, OPERATOR, WORD, TEXT, COMMENT
|
|
|
|
}
|
|
|
|
|
|
|
|
private Lexer() {
|
2015-04-14 16:52:00 +03:00
|
|
|
tokens = new ArrayList<Token>();
|
2015-04-04 15:09:43 +03:00
|
|
|
buffer = new StringBuilder();
|
|
|
|
state = TokenizeState.DEFAULT;
|
|
|
|
}
|
|
|
|
|
|
|
|
public List<Token> getTokens() {
|
|
|
|
return tokens;
|
|
|
|
}
|
|
|
|
|
|
|
|
public Lexer process(String input) {
|
|
|
|
final int length = input.length();
|
|
|
|
for (pos = 0; pos < length; pos++) {
|
|
|
|
tokenize(input.charAt(pos));
|
|
|
|
}
|
|
|
|
tokenize('\0');// EOF
|
|
|
|
addToken(TokenType.EOF, false);
|
2015-04-14 19:14:54 +03:00
|
|
|
input = null;
|
2015-04-04 15:09:43 +03:00
|
|
|
return this;
|
|
|
|
}
|
|
|
|
|
|
|
|
private void tokenize(char ch) {
|
|
|
|
switch (state) {
|
|
|
|
case DEFAULT: tokenizeDefault(ch); break;
|
|
|
|
case WORD: tokenizeWord(ch); break;
|
|
|
|
case NUMBER: tokenizeNumber(ch); break;
|
|
|
|
case OPERATOR: tokenizeOperator(ch); break;
|
|
|
|
case TEXT: tokenizeText(ch); break;
|
|
|
|
case COMMENT: tokenizeComment(ch); break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private void tokenizeDefault(char ch) {
|
|
|
|
if (Character.isLetter(ch)) {
|
|
|
|
// Слово (ключевое слово или команда)
|
|
|
|
buffer.append(ch);
|
|
|
|
state = TokenizeState.WORD;
|
|
|
|
} else if (Character.isDigit(ch)) {
|
|
|
|
// Число
|
|
|
|
buffer.append(ch);
|
|
|
|
state = TokenizeState.NUMBER;
|
|
|
|
} else if (ch == '"') {
|
|
|
|
// Текст в "кавычках"
|
|
|
|
state = TokenizeState.TEXT;
|
|
|
|
} else if (ch == '#') {
|
|
|
|
clearBuffer();
|
|
|
|
state = TokenizeState.COMMENT;
|
|
|
|
} else {
|
|
|
|
// Операторы и спецсимволы
|
|
|
|
tokenizeOperator(ch);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private void tokenizeWord(char ch) {
|
|
|
|
if (Character.isLetterOrDigit(ch) || (ch == '_') || (ch == '.')) {
|
|
|
|
buffer.append(ch);
|
|
|
|
} else {
|
2015-04-14 16:53:44 +03:00
|
|
|
final String word = buffer.toString().toLowerCase(Locale.ENGLISH);
|
|
|
|
addToken(KEYWORDS.containsKey(word) ? KEYWORDS.get(word) : TokenType.WORD);
|
2015-04-04 15:09:43 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private void tokenizeNumber(char ch) {
|
|
|
|
// Целое или вещественное число.
|
|
|
|
if (ch == '.') {
|
|
|
|
// Пропускаем десятичные точки, если они уже были в числе.
|
|
|
|
if (buffer.indexOf(".") == -1) buffer.append(ch);
|
|
|
|
} else if (Character.isDigit(ch)) {
|
|
|
|
buffer.append(ch);
|
|
|
|
} else {
|
|
|
|
addToken(TokenType.NUMBER);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private void tokenizeOperator(char ch) {
|
|
|
|
final int index = OPERATOR_CHARS.indexOf(ch);
|
|
|
|
if (index != -1) {
|
|
|
|
addToken(OPERATOR_TYPES[index], false);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private void tokenizeText(char ch) {
|
|
|
|
if (ch == '"') {
|
|
|
|
final int len = buffer.length();
|
|
|
|
// Добавляем токен, если не было экранирования символа кавычки.
|
|
|
|
if (len == 0 ||
|
|
|
|
( (len > 0) && (buffer.charAt(len - 1) != '\\') )) {
|
|
|
|
addToken(TokenType.TEXT, false);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
// Экранируем символ кавычки.
|
|
|
|
if (len > 0) {
|
|
|
|
buffer.setCharAt(len - 1, '\"');
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
buffer.append(ch);
|
|
|
|
}
|
|
|
|
|
|
|
|
private void tokenizeComment(char ch) {
|
|
|
|
if (ch == '\n' || ch == '\r') {
|
|
|
|
state = TokenizeState.DEFAULT;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private void addToken(TokenType type) {
|
|
|
|
addToken(type, true);
|
|
|
|
}
|
|
|
|
|
|
|
|
private void addToken(TokenType type, boolean reprocessLastChar) {
|
|
|
|
tokens.add(new Token(buffer.toString(), type));
|
|
|
|
clearBuffer();
|
|
|
|
if (reprocessLastChar) pos--;
|
|
|
|
state = TokenizeState.DEFAULT;
|
|
|
|
}
|
|
|
|
|
|
|
|
private void clearBuffer() {
|
|
|
|
buffer.setLength(0);
|
|
|
|
}
|
|
|
|
}
|