Introduce DECIMAL_NUMBER token, fix HEX numbers with 'E' incorrectly treated as decimal

ownlang-parser/src/main/java/com/annimon/ownlang/parser/Lexer.java

@@ -159,12 +159,15 @@ public final class Lexer {
             tokenizeHexNumber(2);
             return;
         }
+        boolean decimal = false;
         boolean hasDot = false;
         while (true) {
             if (current == '.') {
+                decimal = true;
                 if (hasDot) throw error("Invalid float number " + buffer);
                 hasDot = true;
             } else if (current == 'e' || current == 'E') {
+                decimal = true;
                 int exp = subTokenizeScientificNumber();
                 buffer.append(current).append(exp);
                 break;
@@ -174,7 +177,11 @@ public final class Lexer {
             buffer.append(current);
             current = next();
         }
-        addToken(TokenType.NUMBER, buffer.toString(), startPos);
+        if (decimal) {
+            addToken(TokenType.DECIMAL_NUMBER, buffer.toString(), startPos);
+        } else {
+            addToken(TokenType.NUMBER, buffer.toString(), startPos);
+        }
     }
 
     private int subTokenizeScientificNumber() {

ownlang-parser/src/main/java/com/annimon/ownlang/parser/Parser.java

@@ -389,10 +389,15 @@ public final class Parser {
             MatchExpression.Pattern pattern = null;
             final Token current = get(0);
             if (match(TokenType.NUMBER)) {
-                // case 0.5:
+                // case 20:
                 pattern = new MatchExpression.ConstantPattern(
                         NumberValue.of(createNumber(current.text(), 10))
                 );
+            } else if (match(TokenType.DECIMAL_NUMBER)) {
+                // case 0.5:
+                pattern = new MatchExpression.ConstantPattern(
+                        NumberValue.of(createDecimalNumber(current.text()))
+                );
             } else if (match(TokenType.HEX_NUMBER)) {
                 // case #FF:
                 pattern = new MatchExpression.ConstantPattern(
@@ -856,6 +861,9 @@ public final class Parser {
         if (match(TokenType.NUMBER)) {
             return new ValueExpression(createNumber(current.text(), 10));
         }
+        if (match(TokenType.DECIMAL_NUMBER)) {
+            return new ValueExpression(createDecimalNumber(current.text()));
+        }
         if (match(TokenType.HEX_NUMBER)) {
             return new ValueExpression(createNumber(current.text(), 16));
         }
@@ -882,10 +890,6 @@ public final class Parser {
     }
 
     private Number createNumber(String text, int radix) {
-        // Double
-        if (text.contains(".") || text.contains("e") || text.contains("E")) {
-            return Double.parseDouble(text);
-        }
         // Integer
         try {
             return Integer.parseInt(text, radix);
@@ -894,6 +898,11 @@ public final class Parser {
         }
     }
 
+    private Number createDecimalNumber(String text) {
+        // Double
+        return Double.parseDouble(text);
+    }
+
     private Token consume(TokenType expectedType) {
         final Token actual = get(0);
         if (expectedType != actual.type()) {

ownlang-parser/src/main/java/com/annimon/ownlang/parser/TokenType.java

@@ -7,6 +7,7 @@ package com.annimon.ownlang.parser;
 public enum TokenType {
 
     NUMBER,
+    DECIMAL_NUMBER,
     HEX_NUMBER,
     WORD,
     TEXT,

ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerPositionsTest.java

@@ -50,7 +50,7 @@ class LexerPositionsTest {
                 /*
                   line2
                   line*/a =/*
-                */3
+                */3.1
                 """.stripIndent();
         List<Token> result = Lexer.tokenize(input);
 
@@ -58,7 +58,7 @@ class LexerPositionsTest {
                 .hasSize(3)
                 .extracting(s -> s.pos().row(), s -> s.pos().col(), Token::type)
                 .containsExactly(
-                        tuple(3, 9, WORD), tuple(3, 11, EQ), tuple(4, 3, NUMBER)
+                        tuple(3, 9, WORD), tuple(3, 11, EQ), tuple(4, 3, DECIMAL_NUMBER)
                 );
     }
 }

ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerTest.java

@@ -44,18 +44,18 @@ public class LexerTest {
     public void testNumbers() {
         String input = "0 3.1415 0xCAFEBABE 0Xf7_d6_c5 #FFFF";
         List<Token> result = Lexer.tokenize(input);
-        assertTokens(result, NUMBER, NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER);
+        assertTokens(result, NUMBER, DECIMAL_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER);
         assertThat(result)
                 .extracting(Token::text)
                 .containsExactly("0", "3.1415", "CAFEBABE", "f7d6c5", "FFFF");
     }
 
     @Test
-    public void testFloatNumbersExponent() {
+    public void testDecimalNumbersExponent() {
         String input = "4e+7 0.3E-19 2e0 5e0000000000000200 5E-000000089";
         List<Token> result = Lexer.tokenize(input);
         assertThat(result)
-                .allMatch(t -> t.type().equals(NUMBER))
+                .allMatch(t -> t.type().equals(DECIMAL_NUMBER))
                 .extracting(Token::text)
                 .containsExactly("4e7", "0.3E-19", "2e0", "5e200", "5E-89");
     }

ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerValidDataProvider.java

@@ -23,11 +23,14 @@ public class LexerValidDataProvider {
     private static List<Arguments> numbers() {
         return List.of(
                 Arguments.of("Numbers",
-                        "12 7.8 90000000 10.03",
-                        List.of(NUMBER, NUMBER, NUMBER, NUMBER)),
+                        "12 90000000",
+                        List.of(NUMBER, NUMBER)),
+                Arguments.of("Decimal numbers",
+                        "7.8 10.03",
+                        List.of(DECIMAL_NUMBER, DECIMAL_NUMBER)),
                 Arguments.of("Float notation",
                         "7e8",
-                        List.of(NUMBER)),
+                        List.of(DECIMAL_NUMBER)),
                 Arguments.of("Hex numbers",
                         "#FF 0xCA 0x12fb 0xFF",
                         List.of(HEX_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER))