mirror of
https://github.com/aNNiMON/Own-Programming-Language-Tutorial.git
synced 2024-09-20 00:34:20 +03:00
Introduce DECIMAL_NUMBER token, fix HEX numbers with 'E' incorrectly treated as decimal
This commit is contained in:
parent
f6c3f3fe17
commit
923f0edbcb
@ -159,12 +159,15 @@ public final class Lexer {
|
||||
tokenizeHexNumber(2);
|
||||
return;
|
||||
}
|
||||
boolean decimal = false;
|
||||
boolean hasDot = false;
|
||||
while (true) {
|
||||
if (current == '.') {
|
||||
decimal = true;
|
||||
if (hasDot) throw error("Invalid float number " + buffer);
|
||||
hasDot = true;
|
||||
} else if (current == 'e' || current == 'E') {
|
||||
decimal = true;
|
||||
int exp = subTokenizeScientificNumber();
|
||||
buffer.append(current).append(exp);
|
||||
break;
|
||||
@ -174,7 +177,11 @@ public final class Lexer {
|
||||
buffer.append(current);
|
||||
current = next();
|
||||
}
|
||||
addToken(TokenType.NUMBER, buffer.toString(), startPos);
|
||||
if (decimal) {
|
||||
addToken(TokenType.DECIMAL_NUMBER, buffer.toString(), startPos);
|
||||
} else {
|
||||
addToken(TokenType.NUMBER, buffer.toString(), startPos);
|
||||
}
|
||||
}
|
||||
|
||||
private int subTokenizeScientificNumber() {
|
||||
|
@ -389,10 +389,15 @@ public final class Parser {
|
||||
MatchExpression.Pattern pattern = null;
|
||||
final Token current = get(0);
|
||||
if (match(TokenType.NUMBER)) {
|
||||
// case 0.5:
|
||||
// case 20:
|
||||
pattern = new MatchExpression.ConstantPattern(
|
||||
NumberValue.of(createNumber(current.text(), 10))
|
||||
);
|
||||
} else if (match(TokenType.DECIMAL_NUMBER)) {
|
||||
// case 0.5:
|
||||
pattern = new MatchExpression.ConstantPattern(
|
||||
NumberValue.of(createDecimalNumber(current.text()))
|
||||
);
|
||||
} else if (match(TokenType.HEX_NUMBER)) {
|
||||
// case #FF:
|
||||
pattern = new MatchExpression.ConstantPattern(
|
||||
@ -856,6 +861,9 @@ public final class Parser {
|
||||
if (match(TokenType.NUMBER)) {
|
||||
return new ValueExpression(createNumber(current.text(), 10));
|
||||
}
|
||||
if (match(TokenType.DECIMAL_NUMBER)) {
|
||||
return new ValueExpression(createDecimalNumber(current.text()));
|
||||
}
|
||||
if (match(TokenType.HEX_NUMBER)) {
|
||||
return new ValueExpression(createNumber(current.text(), 16));
|
||||
}
|
||||
@ -882,10 +890,6 @@ public final class Parser {
|
||||
}
|
||||
|
||||
private Number createNumber(String text, int radix) {
|
||||
// Double
|
||||
if (text.contains(".") || text.contains("e") || text.contains("E")) {
|
||||
return Double.parseDouble(text);
|
||||
}
|
||||
// Integer
|
||||
try {
|
||||
return Integer.parseInt(text, radix);
|
||||
@ -894,6 +898,11 @@ public final class Parser {
|
||||
}
|
||||
}
|
||||
|
||||
private Number createDecimalNumber(String text) {
|
||||
// Double
|
||||
return Double.parseDouble(text);
|
||||
}
|
||||
|
||||
private Token consume(TokenType expectedType) {
|
||||
final Token actual = get(0);
|
||||
if (expectedType != actual.type()) {
|
||||
|
@ -7,6 +7,7 @@ package com.annimon.ownlang.parser;
|
||||
public enum TokenType {
|
||||
|
||||
NUMBER,
|
||||
DECIMAL_NUMBER,
|
||||
HEX_NUMBER,
|
||||
WORD,
|
||||
TEXT,
|
||||
|
@ -50,7 +50,7 @@ class LexerPositionsTest {
|
||||
/*
|
||||
line2
|
||||
line*/a =/*
|
||||
*/3
|
||||
*/3.1
|
||||
""".stripIndent();
|
||||
List<Token> result = Lexer.tokenize(input);
|
||||
|
||||
@ -58,7 +58,7 @@ class LexerPositionsTest {
|
||||
.hasSize(3)
|
||||
.extracting(s -> s.pos().row(), s -> s.pos().col(), Token::type)
|
||||
.containsExactly(
|
||||
tuple(3, 9, WORD), tuple(3, 11, EQ), tuple(4, 3, NUMBER)
|
||||
tuple(3, 9, WORD), tuple(3, 11, EQ), tuple(4, 3, DECIMAL_NUMBER)
|
||||
);
|
||||
}
|
||||
}
|
@ -44,18 +44,18 @@ public class LexerTest {
|
||||
public void testNumbers() {
|
||||
String input = "0 3.1415 0xCAFEBABE 0Xf7_d6_c5 #FFFF";
|
||||
List<Token> result = Lexer.tokenize(input);
|
||||
assertTokens(result, NUMBER, NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER);
|
||||
assertTokens(result, NUMBER, DECIMAL_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER);
|
||||
assertThat(result)
|
||||
.extracting(Token::text)
|
||||
.containsExactly("0", "3.1415", "CAFEBABE", "f7d6c5", "FFFF");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFloatNumbersExponent() {
|
||||
public void testDecimalNumbersExponent() {
|
||||
String input = "4e+7 0.3E-19 2e0 5e0000000000000200 5E-000000089";
|
||||
List<Token> result = Lexer.tokenize(input);
|
||||
assertThat(result)
|
||||
.allMatch(t -> t.type().equals(NUMBER))
|
||||
.allMatch(t -> t.type().equals(DECIMAL_NUMBER))
|
||||
.extracting(Token::text)
|
||||
.containsExactly("4e7", "0.3E-19", "2e0", "5e200", "5E-89");
|
||||
}
|
||||
|
@ -23,11 +23,14 @@ public class LexerValidDataProvider {
|
||||
private static List<Arguments> numbers() {
|
||||
return List.of(
|
||||
Arguments.of("Numbers",
|
||||
"12 7.8 90000000 10.03",
|
||||
List.of(NUMBER, NUMBER, NUMBER, NUMBER)),
|
||||
"12 90000000",
|
||||
List.of(NUMBER, NUMBER)),
|
||||
Arguments.of("Decimal numbers",
|
||||
"7.8 10.03",
|
||||
List.of(DECIMAL_NUMBER, DECIMAL_NUMBER)),
|
||||
Arguments.of("Float notation",
|
||||
"7e8",
|
||||
List.of(NUMBER)),
|
||||
List.of(DECIMAL_NUMBER)),
|
||||
Arguments.of("Hex numbers",
|
||||
"#FF 0xCA 0x12fb 0xFF",
|
||||
List.of(HEX_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER))
|
||||
|
Loading…
Reference in New Issue
Block a user