Добавлены тесты, обновлён лексер

nbproject/project.properties

@@ -48,7 +48,9 @@ javac.source=1.8
 javac.target=1.8
 javac.test.classpath=\
     ${javac.classpath}:\
-    ${build.classes.dir}
+    ${build.classes.dir}:\
+    ${libs.junit_4.classpath}:\
+    ${libs.hamcrest.classpath}
 javac.test.processorpath=\
     ${javac.test.classpath}
 javadoc.additionalparam=

src/com/annimon/ownlang/Main.java

@@ -64,7 +64,7 @@ public final class Main {
     private static void run(String input, boolean showTokens, boolean showAst, boolean showMeasurements) {
         final TimeMeasurement measurement = new TimeMeasurement();
         measurement.start("Tokenize time");
-        final List<Token> tokens = new Lexer(input).tokenize();
+        final List<Token> tokens = Lexer.tokenize(input);
         measurement.stop("Tokenize time");
         if (showTokens) {
             for (int i = 0; i < tokens.size(); i++) {

src/com/annimon/ownlang/parser/Lexer.java

@@ -12,6 +12,10 @@ import java.util.Map;
  */
 public final class Lexer {
     
+    public static List<Token> tokenize(String input) {
+        return new Lexer(input).tokenize();
+    }
+    
     private static final String OPERATOR_CHARS = "+-*/%()[]{}=<>!&|.,^~?:";
     
     private static final Map<String, TokenType> OPERATORS;
@@ -136,6 +140,12 @@ public final class Lexer {
     private void tokenizeNumber() {
         clearBuffer();
         char current = peek(0);
+        if (current == '0' && (peek(1) == 'x' || (peek(1) == 'X'))) {
+            next();
+            next();
+            tokenizeHexNumber();
+            return;
+        }
         while (true) {
             if (current == '.') {
                 if (buffer.indexOf(".") != -1) throw error("Invalid float number");
@@ -151,11 +161,16 @@ public final class Lexer {
     private void tokenizeHexNumber() {
         clearBuffer();
         char current = peek(0);
-        while (isHexNumber(current)) {
-            buffer.append(current);
+        while (isHexNumber(current) || (current == '_')) {
+            if (current != '_') {
+                // allow _ symbol
+                buffer.append(current);
+            }
             current = next();
         }
-        addToken(TokenType.HEX_NUMBER, buffer.toString());
+        if (buffer.length() > 0) {
+            addToken(TokenType.HEX_NUMBER, buffer.toString());
+        }
     }
 
     private static boolean isHexNumber(char current) {

src/com/annimon/ownlang/parser/Parser.java

@@ -16,6 +16,15 @@ import java.util.Map;
  */
 public final class Parser {
     
+    public static Statement parse(List<Token> tokens) {
+        final Parser parser = new Parser(tokens);
+        final Statement program = parser.parse();
+        if (parser.getParseErrors().hasErrors()) {
+            throw new ParseException();
+        }
+        return program;
+    }
+    
     private static final Token EOF = new Token(TokenType.EOF, "", -1, -1);
     
     private static final Map<TokenType, BinaryExpression.Operator> assignOperator;

src/com/annimon/ownlang/parser/ast/IncludeStatement.java

@@ -23,7 +23,7 @@ public final class IncludeStatement implements Statement {
     public void execute() {
         try {
             final String input = SourceLoader.readSource(expression.eval().asString());
-            final List<Token> tokens = new Lexer(input).tokenize();
+            final List<Token> tokens = Lexer.tokenize(input);
             final Parser parser = new Parser(tokens);
             final Statement program = parser.parse();
             if (!parser.getParseErrors().hasErrors()) {

test/com/annimon/ownlang/parser/LexerTest.java

@@ -0,0 +1,150 @@
+package com.annimon.ownlang.parser;
+
+import com.annimon.ownlang.exceptions.LexerException;
+import static com.annimon.ownlang.parser.TokenType.*;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author aNNiMON
+ */
+public class LexerTest {
+    
+    @Test
+    public void testNumbers() {
+        String input = "0 3.1415 0xCAFEBABE 0Xf7_d6_c5 #FFFF #";
+        List<Token> expList = list(NUMBER, NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER);
+        List<Token> result = Lexer.tokenize(input);
+        assertTokens(expList, result);
+        assertEquals("0", result.get(0).getText());
+        assertEquals("3.1415", result.get(1).getText());
+        assertEquals("CAFEBABE", result.get(2).getText());
+        assertEquals("f7d6c5", result.get(3).getText());
+    }
+    
+    @Test(expected = LexerException.class)
+    public void testNumbersError() {
+        String input = "3.14.15 0Xf7_p6_s5";
+        Lexer.tokenize(input);
+    }
+    
+    @Test
+    public void testArithmetic() {
+        String input = "x = -1 + 2 * 3 % 4 / 5";
+        List<Token> expList = list(WORD, EQ, MINUS, NUMBER, PLUS, NUMBER, STAR, NUMBER, PERCENT, NUMBER, SLASH, NUMBER);
+        List<Token> result = Lexer.tokenize(input);
+        assertTokens(expList, result);
+        assertEquals("x", result.get(0).getText());
+    }
+    
+    @Test
+    public void testKeywords() {
+        String input = "if else while for include";
+        List<Token> expList = list(IF, ELSE, WHILE, FOR, INCLUDE);
+        List<Token> result = Lexer.tokenize(input);
+        assertTokens(expList, result);
+    }
+    
+    @Test
+    public void testWord() {
+        String input = "if bool include \"text\n\ntext\"";
+        List<Token> expList = list(IF, WORD, INCLUDE, TEXT);
+        List<Token> result = Lexer.tokenize(input);
+        assertTokens(expList, result);
+    }
+    
+    @Test
+    public void testString() {
+        String input = "\"1\\\"2\"";
+        List<Token> expList = list(TEXT);
+        List<Token> result = Lexer.tokenize(input);
+        assertTokens(expList, result);
+        assertEquals("1\"2", result.get(0).getText());
+    }
+    
+    @Test
+    public void testEmptyString() {
+        String input = "\"\"";
+        List<Token> expList = list(TEXT);
+        List<Token> result = Lexer.tokenize(input);
+        assertTokens(expList, result);
+        assertEquals("", result.get(0).getText());
+    }
+    
+    @Test(expected = LexerException.class)
+    public void testStringError() {
+        String input = "\"1\"\"";
+        List<Token> expList = list(TEXT);
+        List<Token> result = Lexer.tokenize(input);
+        assertTokens(expList, result);
+        assertEquals("1", result.get(0).getText());
+    }
+    
+    @Test
+    public void testOperators() {
+        String input = "=+-*/%<>!&|";
+        List<Token> expList = list(EQ, PLUS, MINUS, STAR, SLASH, PERCENT, LT, GT, EXCL, AMP, BAR);
+        List<Token> result = Lexer.tokenize(input);
+        assertTokens(expList, result);
+    }
+    
+    @Test
+    public void testOperators2Char() {
+        String input = "== != <= >= && || ==+ >=- ->";
+        List<Token> expList = list(EQEQ, EXCLEQ, LTEQ, GTEQ, AMPAMP, BARBAR,
+                EQEQ, PLUS,   GTEQ, MINUS,  MINUS, GT);
+        List<Token> result = Lexer.tokenize(input);
+        assertTokens(expList, result);
+    }
+    
+    @Test
+    public void testComments() {
+        String input = "// 1234 \n /* */ 123 /* \n 12345 \n\n\n */";
+        List<Token> expList = list(NUMBER);
+        List<Token> result = Lexer.tokenize(input);
+        assertTokens(expList, result);
+        assertEquals("123", result.get(0).getText());
+    }
+    
+    @Test
+    public void testComments2() {
+        String input = "// /* 1234 \n */";
+        List<Token> expList = list(STAR, SLASH);
+        List<Token> result = Lexer.tokenize(input);
+        assertTokens(expList, result);
+    }
+    
+    @Test(expected = LexerException.class)
+    public void testCommentsError() {
+        String input = "/* 1234 \n";
+        Lexer.tokenize(input);
+    }
+    
+    private static void assertTokens(List<Token> expList, List<Token> result) {
+        final int length = expList.size();
+        assertEquals(length, result.size());
+        for (int i = 0; i < length; i++) {
+            assertEquals(expList.get(i).getType(), result.get(i).getType());
+        }
+    }
+    
+    private static List<Token> list(TokenType... types) {
+        final List<Token> list = new ArrayList<Token>();
+        for (TokenType t : types) {
+            list.add(token(t));
+        }
+        return list;
+    }
+    
+    private static Token token(TokenType type) {
+        return token(type, "", 0, 0);
+    }
+    
+    private static Token token(TokenType type, String text, int row, int col) {
+        return new Token(type, text, row, col);
+    }
+    
+}

test/com/annimon/ownlang/parser/ParserTest.java

@@ -0,0 +1,61 @@
+package com.annimon.ownlang.parser;
+
+import com.annimon.ownlang.lib.Value;
+import com.annimon.ownlang.lib.Variables;
+import com.annimon.ownlang.parser.ast.*;
+import static com.annimon.ownlang.parser.ast.ASTHelper.*;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * @author aNNiMON
+ */
+public class ParserTest {
+    
+    @Test
+    public void testParsePrimary() {
+        assertEval(number(2), "2", value(2));
+        assertEval(string("test"), "\"test\"", value("test"));
+    }
+    
+    @Test
+    public void testParseAdditive() {
+        assertEval( number(5), "2 + 3", operator(BinaryExpression.Operator.ADD, value(2), value(3)) );
+        assertEval( number(-1), "2 - 3", operator(BinaryExpression.Operator.SUBTRACT, value(2), value(3)) );
+    }
+    
+    @Test
+    public void testParseMultiplicative() {
+        assertEval( number(6), "2 * 3", operator(BinaryExpression.Operator.MULTIPLY, value(2), value(3)) );
+        assertEval( number(4), "12 / 3", operator(BinaryExpression.Operator.DIVIDE, value(12), value(3)) );
+        assertEval( number(2), "12 % 5", operator(BinaryExpression.Operator.REMAINDER, value(12), value(5)) );
+    }
+    
+    private static void assertEval(Value expectedValue, String input, Expression expected) {
+        BlockStatement program = assertExpression(input, expected);
+        program.execute();
+        final Value actual = Variables.get("a");
+        assertEquals(expectedValue.asNumber(), actual.asNumber(), 0.001);
+        assertEquals(expectedValue.asString(), actual.asString());
+    }
+    
+    private static BlockStatement assertExpression(String input, Expression expected) {
+        return assertProgram("a = " + input, block(assign("a", expected)));
+    }
+    
+    private static BlockStatement assertProgram(String input, BlockStatement actual) {
+        BlockStatement result = (BlockStatement) parse(input);
+        assertStatements(result, actual);
+        return result;
+    }
+    
+    private static void assertStatements(BlockStatement expected, BlockStatement actual) {
+        for (int i = 0; i < expected.statements.size(); i++) {
+            assertEquals(expected.statements.get(i).getClass(), actual.statements.get(i).getClass());
+        }
+    }
+
+    private static Statement parse(String input) {
+        return Parser.parse(Lexer.tokenize(input));
+    }
+}

test/com/annimon/ownlang/parser/ast/ASTHelper.java

@@ -0,0 +1,73 @@
+package com.annimon.ownlang.parser.ast;
+
+import com.annimon.ownlang.lib.NumberValue;
+import com.annimon.ownlang.lib.StringValue;
+import com.annimon.ownlang.lib.Types;
+import com.annimon.ownlang.lib.Value;
+import static org.junit.Assert.*;
+
+/**
+ * Helper for build and test AST nodes.
+ * @author aNNiMON
+ */
+public final class ASTHelper {
+    
+    public static void assertValue(NumberValue expected, Value actual) {
+        assertEquals(Types.NUMBER, actual.type());
+        if (expected.raw() instanceof Double) {
+            assertEquals(expected.asNumber(), actual.asNumber(), 0.001);
+        }
+        assertEquals(expected.asInt(), actual.asInt());
+    }
+    
+    public static void assertValue(StringValue expected, Value actual) {
+        assertEquals(Types.STRING, actual.type());
+        assertEquals(expected.asString(), actual.asString());
+    }
+    
+    
+    public static BlockStatement block(Statement... statements) {
+        final BlockStatement result = new BlockStatement();
+        for (Statement statement : statements) {
+            result.add(statement);
+        }
+        return result;
+    }
+    
+    public static AssignmentExpression assign(String variable, Expression expr) {
+        return assign(var(variable), expr);
+    }
+    
+    public static AssignmentExpression assign(Accessible accessible, Expression expr) {
+        return assign(null, accessible, expr);
+    }
+    
+    public static AssignmentExpression assign(BinaryExpression.Operator op, Accessible accessible, Expression expr) {
+        return new AssignmentExpression(op, accessible, expr);
+    }
+    
+    public static BinaryExpression operator(BinaryExpression.Operator op, Expression left, Expression right) {
+        return new BinaryExpression(op, left, right);
+    }
+    
+    public static ValueExpression value(Number value) {
+        return new ValueExpression(value);
+    }
+    
+    public static ValueExpression value(String value) {
+        return new ValueExpression(value);
+    }
+    
+    public static VariableExpression var(String value) {
+        return new VariableExpression(value);
+    }
+    
+    
+    public static NumberValue number(Number value) {
+        return new NumberValue(value);
+    }
+    
+    public static StringValue string(String value) {
+        return new StringValue(value);
+    }
+}

test/com/annimon/ownlang/parser/ast/OperatorExpressionTest.java

@@ -0,0 +1,78 @@
+package com.annimon.ownlang.parser.ast;
+
+import static com.annimon.ownlang.parser.ast.ASTHelper.*;
+import static com.annimon.ownlang.parser.ast.BinaryExpression.Operator.*;
+import org.junit.Test;
+
+/**
+ * @author aNNiMON
+ */
+public class OperatorExpressionTest {
+    
+    @Test
+    public void testAddition() {
+        assertValue(number(4), operator(ADD, value(2), value(2)).eval());
+        assertValue(number(6), operator(ADD, value(1), operator(ADD, value(2), value(3))).eval());
+        assertValue(string("ABCD"), operator(ADD, value("AB"), value("CD")).eval());
+        assertValue(string("AB12"), operator(ADD, value("AB"), operator(ADD, value(10), value(2))).eval());
+    }
+    
+    @Test
+    public void testSubtraction() {
+        assertValue(number(0), operator(SUBTRACT, value(2), value(2)).eval());
+        assertValue(number(110), operator(SUBTRACT, value(100), operator(SUBTRACT, value(20), value(30))).eval());
+    }
+    
+    @Test
+    public void testMultiplication() {
+        assertValue(number(4), operator(MULTIPLY, value(2), value(2)).eval());
+        assertValue(number(30), operator(MULTIPLY, value(5), operator(MULTIPLY, value(-2), value(-3))).eval());
+        assertValue(string("ABABAB"), operator(MULTIPLY, value("AB"), value(3)).eval());
+    }
+    
+    @Test
+    public void testDivision() {
+        assertValue(number(3), operator(DIVIDE, value(6), value(2)).eval());
+        assertValue(number(30), operator(DIVIDE, value(-900), operator(DIVIDE, value(60), value(-2))).eval());
+    }
+    
+    @Test()
+    public void testDivisionZero() {
+        assertValue(number(Double.POSITIVE_INFINITY), operator(DIVIDE, value(2.0), value(0.0)).eval());
+    }
+    
+    @Test(expected = RuntimeException.class)
+    public void testDivisionZeroOnIntegers() {
+        operator(DIVIDE, value(2), value(0)).eval();
+    }
+    
+    @Test
+    public void testRemainder() {
+        assertValue(number(2), operator(REMAINDER, value(10), value(4)).eval());
+        assertValue(number(5), operator(REMAINDER, value(15), operator(REMAINDER, value(40), value(30))).eval());
+    }
+    
+    @Test()
+    public void testRemainderZero() {
+        assertValue(number(Double.NaN), operator(REMAINDER, value(2.0), value(0.0)).eval());
+    }
+    
+    @Test(expected = RuntimeException.class)
+    public void testRemainderZeroOnIntegers() {
+        operator(REMAINDER, value(2), value(0)).eval();
+    }
+    
+    @Test
+    public void testAND() {
+        assertValue(number(0x04), operator(AND, value(0x04), value(0x0F)).eval());
+        assertValue(number(0x00), operator(AND, value(0x04), value(0x08)).eval());
+        assertValue(number(8), operator(AND, value(12), value(9)).eval());
+    }
+    
+    @Test
+    public void testOR() {
+        assertValue(number(12), operator(OR, value(4), value(8)).eval());
+        assertValue(number(0x0F), operator(OR, value(3), value(12)).eval());
+        assertValue(number(0x0E), operator(OR, value(10), value(4)).eval());
+    }
+}

test/com/annimon/ownlang/parser/ast/ValueExpressionTest.java

@@ -0,0 +1,17 @@
+package com.annimon.ownlang.parser.ast;
+
+import static com.annimon.ownlang.parser.ast.ASTHelper.*;
+import org.junit.Test;
+
+/**
+ *
+ * @author aNNiMON
+ */
+public class ValueExpressionTest {
+    
+    @Test
+    public void testValue() {
+        assertValue(number(4), value(4).eval());
+        assertValue(string("ABCD"), value("ABCD").eval());
+    }
+}

test/com/annimon/ownlang/parser/ast/VariableExpressionTest.java

@@ -0,0 +1,33 @@
+package com.annimon.ownlang.parser.ast;
+
+import static com.annimon.ownlang.parser.ast.ASTHelper.*;
+import org.junit.Test;
+
+/**
+ *
+ * @author aNNiMON
+ */
+public class VariableExpressionTest {
+    
+    @Test
+    public void testVariable() {
+        assign("a", value(4)).execute();
+        assign("b", value("ABCD")).execute();
+        
+        assertValue(number(4), var("a").eval());
+        assertValue(string("ABCD"), var("b").eval());
+    }
+    
+    @Test
+    public void testVariableReplace() {
+        assign("a", value(4)).execute();
+        assign("a", value(8)).execute();
+        
+        assertValue(number(8), var("a").eval());
+    }
+    
+    @Test(expected = RuntimeException.class)
+    public void testUnknownVariable() {
+        var("a").eval();
+    }
+}