java-projects/Own-Programming-Language-Tutorial
1
0
Fork 0
mirror of https://github.com/aNNiMON/Own-Programming-Language-Tutorial.git synced 2024-09-20 00:34:20 +03:00
Code Issues Packages Projects Releases Wiki Activity

Optimize Lexer

Browse Source
This commit is contained in:
aNNiMON 2024-08-07 22:07:10 +03:00
parent f12f58c73d
commit 34490813a6
1 changed files with 40 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
ownlang-parser/src/main/java/com/annimon/ownlang/parser/Lexer.java
View File

@ -118,7 +118,7 @@ public final class Lexer {
private final int length; private final int length;
private final List<Token> tokens; private final List<Token> tokens;
private final StringBuilder buffer; private final StringBuilder globalBuffer;
private int pos; private int pos;
private int row, col; private int row, col;
@ -128,7 +128,7 @@ public final class Lexer {
length = input.length(); length = input.length();
tokens = new ArrayList<>(); tokens = new ArrayList<>();
buffer = new StringBuilder(40); globalBuffer = new StringBuilder(40);
row = col = 1; row = col = 1;
} }
@ -184,7 +184,7 @@ public final class Lexer {
if (decimal) { if (decimal) {
addToken(TokenType.DECIMAL_NUMBER, buffer.toString(), startPos); addToken(TokenType.DECIMAL_NUMBER, buffer.toString(), startPos);
} else if (current == 'L') { } else if (current == 'L') {
next(); skip();
addToken(TokenType.LONG_NUMBER, buffer.toString(), startPos); addToken(TokenType.LONG_NUMBER, buffer.toString(), startPos);
} else { } else {
addToken(TokenType.NUMBER, buffer.toString(), startPos); addToken(TokenType.NUMBER, buffer.toString(), startPos);
@ -192,11 +192,11 @@ public final class Lexer {
} }
private int subTokenizeScientificNumber(Pos startPos) { private int subTokenizeScientificNumber(Pos startPos) {
int sign = 1; int sign = switch (next()) {
switch (next()) { case '-' -> { skip(); yield -1; }
case '-': sign = -1; case '+' -> { skip(); yield 1; }
case '+': skip(); break; default -> 1;
} };
boolean hasValue = false; boolean hasValue = false;
char current = peek(0); char current = peek(0);
@ -237,7 +237,7 @@ public final class Lexer {
if (buffer.isEmpty()) throw error("Empty HEX value", startPos); if (buffer.isEmpty()) throw error("Empty HEX value", startPos);
if (peek(-1) == '_') throw error("HEX value cannot end with _", startPos, markEndPos()); if (peek(-1) == '_') throw error("HEX value cannot end with _", startPos, markEndPos());
if (current == 'L') { if (current == 'L') {
next(); skip();
addToken(TokenType.HEX_LONG_NUMBER, buffer.toString(), startPos); addToken(TokenType.HEX_LONG_NUMBER, buffer.toString(), startPos);
} else { } else {
addToken(TokenType.HEX_NUMBER, buffer.toString(), startPos); addToken(TokenType.HEX_NUMBER, buffer.toString(), startPos);
@ -320,35 +320,37 @@ public final class Lexer {
while (true) { while (true) {
if (current == '\\') { if (current == '\\') {
current = next(); current = next();
switch (current) { if ("\r\n\0".indexOf(current) != -1) {
case '\\': current = next(); buffer.append('\\'); continue; throw error("Reached end of line while parsing extended word.", startPos, markEndPos());
case '"': current = next(); buffer.append('"'); continue; }
case '0': current = next(); buffer.append('\0'); continue;
case 'b': current = next(); buffer.append('\b'); continue; int idx = "\\0\"bfnrt".indexOf(current);
case 'f': current = next(); buffer.append('\f'); continue; if (idx != -1) {
case 'n': current = next(); buffer.append('\n'); continue; current = next();
case 'r': current = next(); buffer.append('\r'); continue; buffer.append("\\\0\"\b\f\n\r\t".charAt(idx));
case 't': current = next(); buffer.append('\t'); continue; continue;
case 'u': // http://docs.oracle.com/javase/specs/jls/se8/html/jls-3.html#jls-3.3 }
int rollbackPosition = pos; if (current == 'u') {
while (current == 'u') current = next(); // http://docs.oracle.com/javase/specs/jls/se8/html/jls-3.html#jls-3.3
int escapedValue = 0; int rollbackPosition = pos;
for (int i = 12; i >= 0 && escapedValue != -1; i -= 4) { while (current == 'u') current = next();
if (isHexNumber(current)) { int escapedValue = 0;
escapedValue |= (Character.digit(current, 16) << i); for (int i = 12; i >= 0 && escapedValue != -1; i -= 4) {
} else { if (isHexNumber(current)) {
escapedValue = -1; escapedValue |= (Character.digit(current, 16) << i);
}
current = next();
}
if (escapedValue >= 0) {
buffer.append((char) escapedValue);
} else { } else {
// rollback escapedValue = -1;
buffer.append("\\u");
pos = rollbackPosition;
} }
continue; current = next();
}
if (escapedValue >= 0) {
buffer.append((char) escapedValue);
} else {
// rollback
buffer.append("\\u");
pos = rollbackPosition;
}
continue;
} }
buffer.append('\\'); buffer.append('\\');
continue; continue;
@ -396,8 +398,8 @@ public final class Lexer {
} }
private StringBuilder createBuffer() { private StringBuilder createBuffer() {
buffer.setLength(0); globalBuffer.setLength(0);
return buffer; return globalBuffer;
} }
private Pos markPos() { private Pos markPos() {