From 24ae7dae1dc606d332bd60b5881494a353661273 Mon Sep 17 00:00:00 2001 From: Victor Date: Fri, 19 Feb 2016 16:56:05 +0200 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D1=8B=20=D1=82=D0=B5=D1=81=D1=82=D1=8B,=20=D0=BE=D0=B1?= =?UTF-8?q?=D0=BD=D0=BE=D0=B2=D0=BB=D1=91=D0=BD=20=D0=BB=D0=B5=D0=BA=D1=81?= =?UTF-8?q?=D0=B5=D1=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nbproject/project.properties | 4 +- src/com/annimon/ownlang/Main.java | 2 +- src/com/annimon/ownlang/parser/Lexer.java | 21 ++- src/com/annimon/ownlang/parser/Parser.java | 9 ++ .../ownlang/parser/ast/IncludeStatement.java | 2 +- .../com/annimon/ownlang/parser/LexerTest.java | 150 ++++++++++++++++++ .../annimon/ownlang/parser/ParserTest.java | 61 +++++++ .../annimon/ownlang/parser/ast/ASTHelper.java | 73 +++++++++ .../parser/ast/OperatorExpressionTest.java | 78 +++++++++ .../parser/ast/ValueExpressionTest.java | 17 ++ .../parser/ast/VariableExpressionTest.java | 33 ++++ 11 files changed, 444 insertions(+), 6 deletions(-) create mode 100644 test/com/annimon/ownlang/parser/LexerTest.java create mode 100644 test/com/annimon/ownlang/parser/ParserTest.java create mode 100644 test/com/annimon/ownlang/parser/ast/ASTHelper.java create mode 100644 test/com/annimon/ownlang/parser/ast/OperatorExpressionTest.java create mode 100644 test/com/annimon/ownlang/parser/ast/ValueExpressionTest.java create mode 100644 test/com/annimon/ownlang/parser/ast/VariableExpressionTest.java diff --git a/nbproject/project.properties b/nbproject/project.properties index 0815ae9..c72e54c 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -48,7 +48,9 @@ javac.source=1.8 javac.target=1.8 javac.test.classpath=\ ${javac.classpath}:\ - ${build.classes.dir} + ${build.classes.dir}:\ + ${libs.junit_4.classpath}:\ + ${libs.hamcrest.classpath} javac.test.processorpath=\ ${javac.test.classpath} javadoc.additionalparam= diff --git a/src/com/annimon/ownlang/Main.java b/src/com/annimon/ownlang/Main.java index 499de79..ae1fd81 100644 --- a/src/com/annimon/ownlang/Main.java +++ b/src/com/annimon/ownlang/Main.java @@ -64,7 +64,7 @@ public final class Main { private static void run(String input, boolean showTokens, boolean showAst, boolean showMeasurements) { final TimeMeasurement measurement = new TimeMeasurement(); measurement.start("Tokenize time"); - final List tokens = new Lexer(input).tokenize(); + final List tokens = Lexer.tokenize(input); measurement.stop("Tokenize time"); if (showTokens) { for (int i = 0; i < tokens.size(); i++) { diff --git a/src/com/annimon/ownlang/parser/Lexer.java b/src/com/annimon/ownlang/parser/Lexer.java index f37c32c..ebb9428 100644 --- a/src/com/annimon/ownlang/parser/Lexer.java +++ b/src/com/annimon/ownlang/parser/Lexer.java @@ -12,6 +12,10 @@ import java.util.Map; */ public final class Lexer { + public static List tokenize(String input) { + return new Lexer(input).tokenize(); + } + private static final String OPERATOR_CHARS = "+-*/%()[]{}=<>!&|.,^~?:"; private static final Map OPERATORS; @@ -136,6 +140,12 @@ public final class Lexer { private void tokenizeNumber() { clearBuffer(); char current = peek(0); + if (current == '0' && (peek(1) == 'x' || (peek(1) == 'X'))) { + next(); + next(); + tokenizeHexNumber(); + return; + } while (true) { if (current == '.') { if (buffer.indexOf(".") != -1) throw error("Invalid float number"); @@ -151,11 +161,16 @@ public final class Lexer { private void tokenizeHexNumber() { clearBuffer(); char current = peek(0); - while (isHexNumber(current)) { - buffer.append(current); + while (isHexNumber(current) || (current == '_')) { + if (current != '_') { + // allow _ symbol + buffer.append(current); + } current = next(); } - addToken(TokenType.HEX_NUMBER, buffer.toString()); + if (buffer.length() > 0) { + addToken(TokenType.HEX_NUMBER, buffer.toString()); + } } private static boolean isHexNumber(char current) { diff --git a/src/com/annimon/ownlang/parser/Parser.java b/src/com/annimon/ownlang/parser/Parser.java index 5710c15..5bc70f3 100644 --- a/src/com/annimon/ownlang/parser/Parser.java +++ b/src/com/annimon/ownlang/parser/Parser.java @@ -16,6 +16,15 @@ import java.util.Map; */ public final class Parser { + public static Statement parse(List tokens) { + final Parser parser = new Parser(tokens); + final Statement program = parser.parse(); + if (parser.getParseErrors().hasErrors()) { + throw new ParseException(); + } + return program; + } + private static final Token EOF = new Token(TokenType.EOF, "", -1, -1); private static final Map assignOperator; diff --git a/src/com/annimon/ownlang/parser/ast/IncludeStatement.java b/src/com/annimon/ownlang/parser/ast/IncludeStatement.java index 1570d68..f43ec54 100644 --- a/src/com/annimon/ownlang/parser/ast/IncludeStatement.java +++ b/src/com/annimon/ownlang/parser/ast/IncludeStatement.java @@ -23,7 +23,7 @@ public final class IncludeStatement implements Statement { public void execute() { try { final String input = SourceLoader.readSource(expression.eval().asString()); - final List tokens = new Lexer(input).tokenize(); + final List tokens = Lexer.tokenize(input); final Parser parser = new Parser(tokens); final Statement program = parser.parse(); if (!parser.getParseErrors().hasErrors()) { diff --git a/test/com/annimon/ownlang/parser/LexerTest.java b/test/com/annimon/ownlang/parser/LexerTest.java new file mode 100644 index 0000000..6633a23 --- /dev/null +++ b/test/com/annimon/ownlang/parser/LexerTest.java @@ -0,0 +1,150 @@ +package com.annimon.ownlang.parser; + +import com.annimon.ownlang.exceptions.LexerException; +import static com.annimon.ownlang.parser.TokenType.*; +import java.util.ArrayList; +import java.util.List; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * + * @author aNNiMON + */ +public class LexerTest { + + @Test + public void testNumbers() { + String input = "0 3.1415 0xCAFEBABE 0Xf7_d6_c5 #FFFF #"; + List expList = list(NUMBER, NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER); + List result = Lexer.tokenize(input); + assertTokens(expList, result); + assertEquals("0", result.get(0).getText()); + assertEquals("3.1415", result.get(1).getText()); + assertEquals("CAFEBABE", result.get(2).getText()); + assertEquals("f7d6c5", result.get(3).getText()); + } + + @Test(expected = LexerException.class) + public void testNumbersError() { + String input = "3.14.15 0Xf7_p6_s5"; + Lexer.tokenize(input); + } + + @Test + public void testArithmetic() { + String input = "x = -1 + 2 * 3 % 4 / 5"; + List expList = list(WORD, EQ, MINUS, NUMBER, PLUS, NUMBER, STAR, NUMBER, PERCENT, NUMBER, SLASH, NUMBER); + List result = Lexer.tokenize(input); + assertTokens(expList, result); + assertEquals("x", result.get(0).getText()); + } + + @Test + public void testKeywords() { + String input = "if else while for include"; + List expList = list(IF, ELSE, WHILE, FOR, INCLUDE); + List result = Lexer.tokenize(input); + assertTokens(expList, result); + } + + @Test + public void testWord() { + String input = "if bool include \"text\n\ntext\""; + List expList = list(IF, WORD, INCLUDE, TEXT); + List result = Lexer.tokenize(input); + assertTokens(expList, result); + } + + @Test + public void testString() { + String input = "\"1\\\"2\""; + List expList = list(TEXT); + List result = Lexer.tokenize(input); + assertTokens(expList, result); + assertEquals("1\"2", result.get(0).getText()); + } + + @Test + public void testEmptyString() { + String input = "\"\""; + List expList = list(TEXT); + List result = Lexer.tokenize(input); + assertTokens(expList, result); + assertEquals("", result.get(0).getText()); + } + + @Test(expected = LexerException.class) + public void testStringError() { + String input = "\"1\"\""; + List expList = list(TEXT); + List result = Lexer.tokenize(input); + assertTokens(expList, result); + assertEquals("1", result.get(0).getText()); + } + + @Test + public void testOperators() { + String input = "=+-*/%<>!&|"; + List expList = list(EQ, PLUS, MINUS, STAR, SLASH, PERCENT, LT, GT, EXCL, AMP, BAR); + List result = Lexer.tokenize(input); + assertTokens(expList, result); + } + + @Test + public void testOperators2Char() { + String input = "== != <= >= && || ==+ >=- ->"; + List expList = list(EQEQ, EXCLEQ, LTEQ, GTEQ, AMPAMP, BARBAR, + EQEQ, PLUS, GTEQ, MINUS, MINUS, GT); + List result = Lexer.tokenize(input); + assertTokens(expList, result); + } + + @Test + public void testComments() { + String input = "// 1234 \n /* */ 123 /* \n 12345 \n\n\n */"; + List expList = list(NUMBER); + List result = Lexer.tokenize(input); + assertTokens(expList, result); + assertEquals("123", result.get(0).getText()); + } + + @Test + public void testComments2() { + String input = "// /* 1234 \n */"; + List expList = list(STAR, SLASH); + List result = Lexer.tokenize(input); + assertTokens(expList, result); + } + + @Test(expected = LexerException.class) + public void testCommentsError() { + String input = "/* 1234 \n"; + Lexer.tokenize(input); + } + + private static void assertTokens(List expList, List result) { + final int length = expList.size(); + assertEquals(length, result.size()); + for (int i = 0; i < length; i++) { + assertEquals(expList.get(i).getType(), result.get(i).getType()); + } + } + + private static List list(TokenType... types) { + final List list = new ArrayList(); + for (TokenType t : types) { + list.add(token(t)); + } + return list; + } + + private static Token token(TokenType type) { + return token(type, "", 0, 0); + } + + private static Token token(TokenType type, String text, int row, int col) { + return new Token(type, text, row, col); + } + +} diff --git a/test/com/annimon/ownlang/parser/ParserTest.java b/test/com/annimon/ownlang/parser/ParserTest.java new file mode 100644 index 0000000..dfe031e --- /dev/null +++ b/test/com/annimon/ownlang/parser/ParserTest.java @@ -0,0 +1,61 @@ +package com.annimon.ownlang.parser; + +import com.annimon.ownlang.lib.Value; +import com.annimon.ownlang.lib.Variables; +import com.annimon.ownlang.parser.ast.*; +import static com.annimon.ownlang.parser.ast.ASTHelper.*; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * @author aNNiMON + */ +public class ParserTest { + + @Test + public void testParsePrimary() { + assertEval(number(2), "2", value(2)); + assertEval(string("test"), "\"test\"", value("test")); + } + + @Test + public void testParseAdditive() { + assertEval( number(5), "2 + 3", operator(BinaryExpression.Operator.ADD, value(2), value(3)) ); + assertEval( number(-1), "2 - 3", operator(BinaryExpression.Operator.SUBTRACT, value(2), value(3)) ); + } + + @Test + public void testParseMultiplicative() { + assertEval( number(6), "2 * 3", operator(BinaryExpression.Operator.MULTIPLY, value(2), value(3)) ); + assertEval( number(4), "12 / 3", operator(BinaryExpression.Operator.DIVIDE, value(12), value(3)) ); + assertEval( number(2), "12 % 5", operator(BinaryExpression.Operator.REMAINDER, value(12), value(5)) ); + } + + private static void assertEval(Value expectedValue, String input, Expression expected) { + BlockStatement program = assertExpression(input, expected); + program.execute(); + final Value actual = Variables.get("a"); + assertEquals(expectedValue.asNumber(), actual.asNumber(), 0.001); + assertEquals(expectedValue.asString(), actual.asString()); + } + + private static BlockStatement assertExpression(String input, Expression expected) { + return assertProgram("a = " + input, block(assign("a", expected))); + } + + private static BlockStatement assertProgram(String input, BlockStatement actual) { + BlockStatement result = (BlockStatement) parse(input); + assertStatements(result, actual); + return result; + } + + private static void assertStatements(BlockStatement expected, BlockStatement actual) { + for (int i = 0; i < expected.statements.size(); i++) { + assertEquals(expected.statements.get(i).getClass(), actual.statements.get(i).getClass()); + } + } + + private static Statement parse(String input) { + return Parser.parse(Lexer.tokenize(input)); + } +} diff --git a/test/com/annimon/ownlang/parser/ast/ASTHelper.java b/test/com/annimon/ownlang/parser/ast/ASTHelper.java new file mode 100644 index 0000000..f6ea28b --- /dev/null +++ b/test/com/annimon/ownlang/parser/ast/ASTHelper.java @@ -0,0 +1,73 @@ +package com.annimon.ownlang.parser.ast; + +import com.annimon.ownlang.lib.NumberValue; +import com.annimon.ownlang.lib.StringValue; +import com.annimon.ownlang.lib.Types; +import com.annimon.ownlang.lib.Value; +import static org.junit.Assert.*; + +/** + * Helper for build and test AST nodes. + * @author aNNiMON + */ +public final class ASTHelper { + + public static void assertValue(NumberValue expected, Value actual) { + assertEquals(Types.NUMBER, actual.type()); + if (expected.raw() instanceof Double) { + assertEquals(expected.asNumber(), actual.asNumber(), 0.001); + } + assertEquals(expected.asInt(), actual.asInt()); + } + + public static void assertValue(StringValue expected, Value actual) { + assertEquals(Types.STRING, actual.type()); + assertEquals(expected.asString(), actual.asString()); + } + + + public static BlockStatement block(Statement... statements) { + final BlockStatement result = new BlockStatement(); + for (Statement statement : statements) { + result.add(statement); + } + return result; + } + + public static AssignmentExpression assign(String variable, Expression expr) { + return assign(var(variable), expr); + } + + public static AssignmentExpression assign(Accessible accessible, Expression expr) { + return assign(null, accessible, expr); + } + + public static AssignmentExpression assign(BinaryExpression.Operator op, Accessible accessible, Expression expr) { + return new AssignmentExpression(op, accessible, expr); + } + + public static BinaryExpression operator(BinaryExpression.Operator op, Expression left, Expression right) { + return new BinaryExpression(op, left, right); + } + + public static ValueExpression value(Number value) { + return new ValueExpression(value); + } + + public static ValueExpression value(String value) { + return new ValueExpression(value); + } + + public static VariableExpression var(String value) { + return new VariableExpression(value); + } + + + public static NumberValue number(Number value) { + return new NumberValue(value); + } + + public static StringValue string(String value) { + return new StringValue(value); + } +} diff --git a/test/com/annimon/ownlang/parser/ast/OperatorExpressionTest.java b/test/com/annimon/ownlang/parser/ast/OperatorExpressionTest.java new file mode 100644 index 0000000..239756c --- /dev/null +++ b/test/com/annimon/ownlang/parser/ast/OperatorExpressionTest.java @@ -0,0 +1,78 @@ +package com.annimon.ownlang.parser.ast; + +import static com.annimon.ownlang.parser.ast.ASTHelper.*; +import static com.annimon.ownlang.parser.ast.BinaryExpression.Operator.*; +import org.junit.Test; + +/** + * @author aNNiMON + */ +public class OperatorExpressionTest { + + @Test + public void testAddition() { + assertValue(number(4), operator(ADD, value(2), value(2)).eval()); + assertValue(number(6), operator(ADD, value(1), operator(ADD, value(2), value(3))).eval()); + assertValue(string("ABCD"), operator(ADD, value("AB"), value("CD")).eval()); + assertValue(string("AB12"), operator(ADD, value("AB"), operator(ADD, value(10), value(2))).eval()); + } + + @Test + public void testSubtraction() { + assertValue(number(0), operator(SUBTRACT, value(2), value(2)).eval()); + assertValue(number(110), operator(SUBTRACT, value(100), operator(SUBTRACT, value(20), value(30))).eval()); + } + + @Test + public void testMultiplication() { + assertValue(number(4), operator(MULTIPLY, value(2), value(2)).eval()); + assertValue(number(30), operator(MULTIPLY, value(5), operator(MULTIPLY, value(-2), value(-3))).eval()); + assertValue(string("ABABAB"), operator(MULTIPLY, value("AB"), value(3)).eval()); + } + + @Test + public void testDivision() { + assertValue(number(3), operator(DIVIDE, value(6), value(2)).eval()); + assertValue(number(30), operator(DIVIDE, value(-900), operator(DIVIDE, value(60), value(-2))).eval()); + } + + @Test() + public void testDivisionZero() { + assertValue(number(Double.POSITIVE_INFINITY), operator(DIVIDE, value(2.0), value(0.0)).eval()); + } + + @Test(expected = RuntimeException.class) + public void testDivisionZeroOnIntegers() { + operator(DIVIDE, value(2), value(0)).eval(); + } + + @Test + public void testRemainder() { + assertValue(number(2), operator(REMAINDER, value(10), value(4)).eval()); + assertValue(number(5), operator(REMAINDER, value(15), operator(REMAINDER, value(40), value(30))).eval()); + } + + @Test() + public void testRemainderZero() { + assertValue(number(Double.NaN), operator(REMAINDER, value(2.0), value(0.0)).eval()); + } + + @Test(expected = RuntimeException.class) + public void testRemainderZeroOnIntegers() { + operator(REMAINDER, value(2), value(0)).eval(); + } + + @Test + public void testAND() { + assertValue(number(0x04), operator(AND, value(0x04), value(0x0F)).eval()); + assertValue(number(0x00), operator(AND, value(0x04), value(0x08)).eval()); + assertValue(number(8), operator(AND, value(12), value(9)).eval()); + } + + @Test + public void testOR() { + assertValue(number(12), operator(OR, value(4), value(8)).eval()); + assertValue(number(0x0F), operator(OR, value(3), value(12)).eval()); + assertValue(number(0x0E), operator(OR, value(10), value(4)).eval()); + } +} diff --git a/test/com/annimon/ownlang/parser/ast/ValueExpressionTest.java b/test/com/annimon/ownlang/parser/ast/ValueExpressionTest.java new file mode 100644 index 0000000..50d41e8 --- /dev/null +++ b/test/com/annimon/ownlang/parser/ast/ValueExpressionTest.java @@ -0,0 +1,17 @@ +package com.annimon.ownlang.parser.ast; + +import static com.annimon.ownlang.parser.ast.ASTHelper.*; +import org.junit.Test; + +/** + * + * @author aNNiMON + */ +public class ValueExpressionTest { + + @Test + public void testValue() { + assertValue(number(4), value(4).eval()); + assertValue(string("ABCD"), value("ABCD").eval()); + } +} diff --git a/test/com/annimon/ownlang/parser/ast/VariableExpressionTest.java b/test/com/annimon/ownlang/parser/ast/VariableExpressionTest.java new file mode 100644 index 0000000..1478368 --- /dev/null +++ b/test/com/annimon/ownlang/parser/ast/VariableExpressionTest.java @@ -0,0 +1,33 @@ +package com.annimon.ownlang.parser.ast; + +import static com.annimon.ownlang.parser.ast.ASTHelper.*; +import org.junit.Test; + +/** + * + * @author aNNiMON + */ +public class VariableExpressionTest { + + @Test + public void testVariable() { + assign("a", value(4)).execute(); + assign("b", value("ABCD")).execute(); + + assertValue(number(4), var("a").eval()); + assertValue(string("ABCD"), var("b").eval()); + } + + @Test + public void testVariableReplace() { + assign("a", value(4)).execute(); + assign("a", value(8)).execute(); + + assertValue(number(8), var("a").eval()); + } + + @Test(expected = RuntimeException.class) + public void testUnknownVariable() { + var("a").eval(); + } +}