From 923f0edbcb92d03a94e5eeedaaf20f9fcc707202 Mon Sep 17 00:00:00 2001 From: aNNiMON Date: Sun, 17 Sep 2023 19:00:17 +0300 Subject: [PATCH] Introduce DECIMAL_NUMBER token, fix HEX numbers with 'E' incorrectly treated as decimal --- .../com/annimon/ownlang/parser/Lexer.java | 9 ++++++++- .../com/annimon/ownlang/parser/Parser.java | 19 ++++++++++++++----- .../com/annimon/ownlang/parser/TokenType.java | 1 + .../ownlang/parser/LexerPositionsTest.java | 4 ++-- .../com/annimon/ownlang/parser/LexerTest.java | 6 +++--- .../parser/LexerValidDataProvider.java | 9 ++++++--- 6 files changed, 34 insertions(+), 14 deletions(-) diff --git a/ownlang-parser/src/main/java/com/annimon/ownlang/parser/Lexer.java b/ownlang-parser/src/main/java/com/annimon/ownlang/parser/Lexer.java index 28ee57e..9a11d54 100644 --- a/ownlang-parser/src/main/java/com/annimon/ownlang/parser/Lexer.java +++ b/ownlang-parser/src/main/java/com/annimon/ownlang/parser/Lexer.java @@ -159,12 +159,15 @@ public final class Lexer { tokenizeHexNumber(2); return; } + boolean decimal = false; boolean hasDot = false; while (true) { if (current == '.') { + decimal = true; if (hasDot) throw error("Invalid float number " + buffer); hasDot = true; } else if (current == 'e' || current == 'E') { + decimal = true; int exp = subTokenizeScientificNumber(); buffer.append(current).append(exp); break; @@ -174,7 +177,11 @@ public final class Lexer { buffer.append(current); current = next(); } - addToken(TokenType.NUMBER, buffer.toString(), startPos); + if (decimal) { + addToken(TokenType.DECIMAL_NUMBER, buffer.toString(), startPos); + } else { + addToken(TokenType.NUMBER, buffer.toString(), startPos); + } } private int subTokenizeScientificNumber() { diff --git a/ownlang-parser/src/main/java/com/annimon/ownlang/parser/Parser.java b/ownlang-parser/src/main/java/com/annimon/ownlang/parser/Parser.java index f4df30d..668894c 100644 --- a/ownlang-parser/src/main/java/com/annimon/ownlang/parser/Parser.java +++ b/ownlang-parser/src/main/java/com/annimon/ownlang/parser/Parser.java @@ -389,10 +389,15 @@ public final class Parser { MatchExpression.Pattern pattern = null; final Token current = get(0); if (match(TokenType.NUMBER)) { - // case 0.5: + // case 20: pattern = new MatchExpression.ConstantPattern( NumberValue.of(createNumber(current.text(), 10)) ); + } else if (match(TokenType.DECIMAL_NUMBER)) { + // case 0.5: + pattern = new MatchExpression.ConstantPattern( + NumberValue.of(createDecimalNumber(current.text())) + ); } else if (match(TokenType.HEX_NUMBER)) { // case #FF: pattern = new MatchExpression.ConstantPattern( @@ -856,6 +861,9 @@ public final class Parser { if (match(TokenType.NUMBER)) { return new ValueExpression(createNumber(current.text(), 10)); } + if (match(TokenType.DECIMAL_NUMBER)) { + return new ValueExpression(createDecimalNumber(current.text())); + } if (match(TokenType.HEX_NUMBER)) { return new ValueExpression(createNumber(current.text(), 16)); } @@ -882,10 +890,6 @@ public final class Parser { } private Number createNumber(String text, int radix) { - // Double - if (text.contains(".") || text.contains("e") || text.contains("E")) { - return Double.parseDouble(text); - } // Integer try { return Integer.parseInt(text, radix); @@ -894,6 +898,11 @@ public final class Parser { } } + private Number createDecimalNumber(String text) { + // Double + return Double.parseDouble(text); + } + private Token consume(TokenType expectedType) { final Token actual = get(0); if (expectedType != actual.type()) { diff --git a/ownlang-parser/src/main/java/com/annimon/ownlang/parser/TokenType.java b/ownlang-parser/src/main/java/com/annimon/ownlang/parser/TokenType.java index 87d15eb..cf6a71d 100644 --- a/ownlang-parser/src/main/java/com/annimon/ownlang/parser/TokenType.java +++ b/ownlang-parser/src/main/java/com/annimon/ownlang/parser/TokenType.java @@ -7,6 +7,7 @@ package com.annimon.ownlang.parser; public enum TokenType { NUMBER, + DECIMAL_NUMBER, HEX_NUMBER, WORD, TEXT, diff --git a/ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerPositionsTest.java b/ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerPositionsTest.java index 5f0c799..aac808e 100644 --- a/ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerPositionsTest.java +++ b/ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerPositionsTest.java @@ -50,7 +50,7 @@ class LexerPositionsTest { /* line2 line*/a =/* - */3 + */3.1 """.stripIndent(); List result = Lexer.tokenize(input); @@ -58,7 +58,7 @@ class LexerPositionsTest { .hasSize(3) .extracting(s -> s.pos().row(), s -> s.pos().col(), Token::type) .containsExactly( - tuple(3, 9, WORD), tuple(3, 11, EQ), tuple(4, 3, NUMBER) + tuple(3, 9, WORD), tuple(3, 11, EQ), tuple(4, 3, DECIMAL_NUMBER) ); } } \ No newline at end of file diff --git a/ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerTest.java b/ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerTest.java index 92b39c9..d3bd0ab 100644 --- a/ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerTest.java +++ b/ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerTest.java @@ -44,18 +44,18 @@ public class LexerTest { public void testNumbers() { String input = "0 3.1415 0xCAFEBABE 0Xf7_d6_c5 #FFFF"; List result = Lexer.tokenize(input); - assertTokens(result, NUMBER, NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER); + assertTokens(result, NUMBER, DECIMAL_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER); assertThat(result) .extracting(Token::text) .containsExactly("0", "3.1415", "CAFEBABE", "f7d6c5", "FFFF"); } @Test - public void testFloatNumbersExponent() { + public void testDecimalNumbersExponent() { String input = "4e+7 0.3E-19 2e0 5e0000000000000200 5E-000000089"; List result = Lexer.tokenize(input); assertThat(result) - .allMatch(t -> t.type().equals(NUMBER)) + .allMatch(t -> t.type().equals(DECIMAL_NUMBER)) .extracting(Token::text) .containsExactly("4e7", "0.3E-19", "2e0", "5e200", "5E-89"); } diff --git a/ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerValidDataProvider.java b/ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerValidDataProvider.java index c90cb44..891eab2 100644 --- a/ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerValidDataProvider.java +++ b/ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerValidDataProvider.java @@ -23,11 +23,14 @@ public class LexerValidDataProvider { private static List numbers() { return List.of( Arguments.of("Numbers", - "12 7.8 90000000 10.03", - List.of(NUMBER, NUMBER, NUMBER, NUMBER)), + "12 90000000", + List.of(NUMBER, NUMBER)), + Arguments.of("Decimal numbers", + "7.8 10.03", + List.of(DECIMAL_NUMBER, DECIMAL_NUMBER)), Arguments.of("Float notation", "7e8", - List.of(NUMBER)), + List.of(DECIMAL_NUMBER)), Arguments.of("Hex numbers", "#FF 0xCA 0x12fb 0xFF", List.of(HEX_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER))