1
0
mirror of https://github.com/aNNiMON/HotaruFX.git synced 2024-09-19 14:14:21 +03:00

Add parser

This commit is contained in:
Victor 2017-08-22 12:47:43 +03:00
parent 45749bf471
commit 43a52fb049
22 changed files with 946 additions and 0 deletions

View File

@ -0,0 +1,14 @@
package com.annimon.hotarufx.exceptions;
import com.annimon.hotarufx.lexer.SourcePosition;
public class ParseException extends RuntimeException {
public ParseException() {
super();
}
public ParseException(String string, SourcePosition pos) {
super(string + " at " + pos.toString());
}
}

View File

@ -0,0 +1,141 @@
package com.annimon.hotarufx.lib;
public class NumberValue implements Value {
public static final NumberValue MINUS_ONE, ZERO, ONE;
private static final int CACHE_MIN = -128, CACHE_MAX = 127;
private static final NumberValue[] NUMBER_CACHE;
static {
final int length = CACHE_MAX - CACHE_MIN + 1;
NUMBER_CACHE = new NumberValue[length];
int value = CACHE_MIN;
for (int i = 0; i < length; i++) {
NUMBER_CACHE[i] = new NumberValue(value++);
}
final int zeroIndex = -CACHE_MIN;
MINUS_ONE = NUMBER_CACHE[zeroIndex - 1];
ZERO = NUMBER_CACHE[zeroIndex];
ONE = NUMBER_CACHE[zeroIndex + 1];
}
public static NumberValue fromBoolean(boolean b) {
return b ? ONE : ZERO;
}
public static NumberValue of(int value) {
if (CACHE_MIN <= value && value <= CACHE_MAX) {
return NUMBER_CACHE[-CACHE_MIN + value];
}
return new NumberValue(value);
}
public static NumberValue of(Number value) {
return new NumberValue(value);
}
private final Number value;
private NumberValue(Number value) {
this.value = value;
}
@Override
public int type() {
return Types.NUMBER;
}
@Override
public Number raw() {
return value;
}
public boolean asBoolean() {
return value.intValue() != 0;
}
public byte asByte() {
return value.byteValue();
}
public short asShort() {
return value.shortValue();
}
@Override
public int asInt() {
return value.intValue();
}
public long asLong() {
return value.longValue();
}
public float asFloat() {
return value.floatValue();
}
public double asDouble() {
return value.doubleValue();
}
@Override
public double asNumber() {
return value.doubleValue();
}
@Override
public String asString() {
return value.toString();
}
@Override
public int hashCode() {
int hash = 3;
hash = 71 * hash + value.hashCode();
return hash;
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass())
return false;
final Number other = ((NumberValue) obj).value;
if (value instanceof Double || other instanceof Double) {
return Double.compare(value.doubleValue(), other.doubleValue()) == 0;
}
if (value instanceof Float || other instanceof Float) {
return Float.compare(value.floatValue(), other.floatValue()) == 0;
}
if (value instanceof Long || other instanceof Long) {
return Long.compare(value.longValue(), other.longValue()) == 0;
}
return Integer.compare(value.intValue(), other.intValue()) == 0;
}
@Override
public int compareTo(Value o) {
if (o.type() == Types.NUMBER) {
final Number other = ((NumberValue) o).value;
if (value instanceof Double || other instanceof Double) {
return Double.compare(value.doubleValue(), other.doubleValue());
}
if (value instanceof Float || other instanceof Float) {
return Float.compare(value.floatValue(), other.floatValue());
}
if (value instanceof Long || other instanceof Long) {
return Long.compare(value.longValue(), other.longValue());
}
return Integer.compare(value.intValue(), other.intValue());
}
return asString().compareTo(o.asString());
}
@Override
public String toString() {
return asString();
}
}

View File

@ -0,0 +1,81 @@
package com.annimon.hotarufx.lib;
import java.util.Objects;
public class StringValue implements Value {
public static final StringValue EMPTY = new StringValue("");
private final String value;
public StringValue(String value) {
this.value = value;
}
public int length() {
return value.length();
}
@Override
public int type() {
return Types.STRING;
}
@Override
public Object raw() {
return value;
}
@Override
public int asInt() {
try {
return Integer.parseInt(value);
} catch (NumberFormatException e) {
return 0;
}
}
@Override
public double asNumber() {
try {
return Double.parseDouble(value);
} catch (NumberFormatException e) {
return 0;
}
}
@Override
public String asString() {
return value;
}
@Override
public int hashCode() {
int hash = 3;
hash = 97 * hash + Objects.hashCode(this.value);
return hash;
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass())
return false;
final StringValue other = (StringValue) obj;
return Objects.equals(this.value, other.value);
}
@Override
public int compareTo(Value o) {
if (o.type() == Types.STRING) {
return value.compareTo(((StringValue) o).value);
}
return asString().compareTo(o.asString());
}
@Override
public String toString() {
return asString();
}
}

View File

@ -0,0 +1,11 @@
package com.annimon.hotarufx.lib;
public class Types {
public static final int
OBJECT = 0,
NUMBER = 1,
STRING = 2,
MAP = 3,
FUNCTION = 4;
}

View File

@ -0,0 +1,14 @@
package com.annimon.hotarufx.lib;
public interface Value extends Comparable<Value> {
Object raw();
int asInt();
double asNumber();
String asString();
int type();
}

View File

@ -0,0 +1,218 @@
package com.annimon.hotarufx.parser;
import com.annimon.hotarufx.exceptions.ParseException;
import com.annimon.hotarufx.lexer.HotaruTokenId;
import com.annimon.hotarufx.lexer.Token;
import com.annimon.hotarufx.parser.ast.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.val;
public class HotaruParser extends Parser {
public static Node parse(List<Token> tokens) {
val parser = new HotaruParser(tokens);
val program = parser.parse();
if (parser.getParseErrors().hasErrors()) {
throw new ParseException();
}
return program;
}
public HotaruParser(List<Token> tokens) {
super(tokens);
}
private Node block() {
val block = new BlockNode();
block.start(getSourcePosition());
consume(HotaruTokenId.LBRACE);
while (!match(HotaruTokenId.RBRACE)) {
block.add(statement());
}
block.end(getSourcePosition());
return block;
}
private Node statementOrBlock() {
if (lookMatch(0, HotaruTokenId.LBRACE)) {
return block();
}
return statement();
}
@Override
protected Node statement() {
return assignmentStatement();
}
private Node assignmentStatement() {
return expression();
}
private Node functionChain(Node qualifiedNameExpr) {
// f1()()() || f1().f2().f3() || f1().key
val expr = function(qualifiedNameExpr);
if (lookMatch(0, HotaruTokenId.LPAREN)) {
return functionChain(expr);
}
if (lookMatch(0, HotaruTokenId.DOT)) {
final List<Node> indices = variableSuffix();
if (indices == null || indices.isEmpty()) return expr;
if (lookMatch(0, HotaruTokenId.LPAREN)) {
// next function call
return functionChain(new AccessNode(expr, indices));
}
// container access
return new AccessNode(expr, indices);
}
return expr;
}
private FunctionNode function(Node qualifiedNameExpr) {
// function(arg1, arg2, ...)
consume(HotaruTokenId.LPAREN);
val function = new FunctionNode(qualifiedNameExpr);
while (!match(HotaruTokenId.RPAREN)) {
function.addArgument(expression());
match(HotaruTokenId.COMMA);
}
return function;
}
private Node map() {
// {key1 : value1, key2 : value2, ...}
consume(HotaruTokenId.LBRACE);
final Map<String, Node> elements = new HashMap<>();
while (!match(HotaruTokenId.RBRACE)) {
val key = consume(HotaruTokenId.WORD).getText();
consume(HotaruTokenId.COLON);
val value = expression();
elements.put(key, value);
match(HotaruTokenId.COMMA);
}
return new MapNode(elements);
}
private Node expression() {
return assignment();
}
private Node assignment() {
val assignment = assignmentStrict();
if (assignment != null) {
return assignment;
}
return unary();
}
private Node assignmentStrict() {
final int position = pos;
val startSourcePosition = getSourcePosition();
val targetExpr = qualifiedName();
if ((targetExpr == null) || !(targetExpr instanceof Accessible)) {
pos = position;
return null;
}
if (!match(HotaruTokenId.EQ)) {
pos = position;
return null;
}
return new AssignNode((Accessible) targetExpr, expression())
.start(startSourcePosition)
.end(getSourcePosition());
}
private Node unary() {
if (match(HotaruTokenId.MINUS)) {
return new UnaryNode(UnaryNode.Operator.NEGATE, primary());
}
if (match(HotaruTokenId.PLUS)) {
return primary();
}
return primary();
}
private Node primary() {
if (match(HotaruTokenId.LPAREN)) {
val result = expression();
match(HotaruTokenId.RPAREN);
return result;
}
return variable();
}
private Node variable() {
// function(...
if (lookMatch(0, HotaruTokenId.WORD) && lookMatch(1, HotaruTokenId.LPAREN)) {
return functionChain(new ValueNode(consume(HotaruTokenId.WORD).getText()));
}
final Node qualifiedNameExpr = qualifiedName();
if (qualifiedNameExpr != null) {
// variable(args) || arr["key"](args) || obj.key(args)
if (lookMatch(0, HotaruTokenId.LPAREN)) {
return functionChain(qualifiedNameExpr);
}
return qualifiedNameExpr;
}
if (lookMatch(0, HotaruTokenId.LBRACE)) {
return map();
}
return value();
}
private Node qualifiedName() {
// var || var.key[index].key2
final Token current = get(0);
if (!match(HotaruTokenId.WORD)) return null;
final List<Node> indices = variableSuffix();
if ((indices == null) || indices.isEmpty()) {
return new VariableNode(current.getText());
}
return new AccessNode(current.getText(), indices);
}
private List<Node> variableSuffix() {
final List<Node> indices = new ArrayList<>();
while (lookMatch(0, HotaruTokenId.DOT)) {
if (match(HotaruTokenId.DOT)) {
val fieldName = consume(HotaruTokenId.WORD).getText();
val key = new ValueNode(fieldName);
indices.add(key);
}
}
return indices;
}
private Node value() {
val current = get(0);
if (match(HotaruTokenId.NUMBER)) {
return new ValueNode(createNumber(current.getText(), 10));
}
if (match(HotaruTokenId.TEXT)) {
return new ValueNode(current.getText());
}
throw new ParseException("Unknown expression: " + current, getSourcePosition());
}
private Number createNumber(String text, int radix) {
// Double
if (text.contains(".")) {
return Double.parseDouble(text);
}
// Integer
try {
return Integer.parseInt(text, radix);
} catch (NumberFormatException nfe) {
return Long.parseLong(text, radix);
}
}
}

View File

@ -0,0 +1,27 @@
package com.annimon.hotarufx.parser;
import com.annimon.hotarufx.lexer.SourcePosition;
public class ParseError {
private final Exception exception;
private final SourcePosition pos;
public ParseError(Exception exception, SourcePosition pos) {
this.exception = exception;
this.pos = pos;
}
public Exception getException() {
return exception;
}
public SourcePosition getPosition() {
return pos;
}
@Override
public String toString() {
return "ParseError " + exception.getMessage() + " at " + pos.toString();
}
}

View File

@ -0,0 +1,46 @@
package com.annimon.hotarufx.parser;
import com.annimon.hotarufx.lexer.SourcePosition;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Stream;
public class ParseErrors implements Iterable<ParseError> {
private final List<ParseError> errors;
public ParseErrors() {
errors = new ArrayList<>();
}
public void clear() {
errors.clear();
}
public void add(Exception ex, SourcePosition pos) {
errors.add(new ParseError(ex, pos));
}
public boolean hasErrors() {
return !errors.isEmpty();
}
@Override
public Iterator<ParseError> iterator() {
return errors.iterator();
}
public Stream<ParseError> errorsStream() {
return errors.stream();
}
@Override
public String toString() {
final StringBuilder result = new StringBuilder();
for (ParseError error : errors) {
result.append(error).append(System.lineSeparator());
}
return result.toString();
}
}

View File

@ -0,0 +1,105 @@
package com.annimon.hotarufx.parser;
import com.annimon.hotarufx.exceptions.ParseException;
import com.annimon.hotarufx.lexer.HotaruTokenId;
import com.annimon.hotarufx.lexer.SourcePosition;
import com.annimon.hotarufx.lexer.Token;
import com.annimon.hotarufx.parser.ast.BlockNode;
import com.annimon.hotarufx.parser.ast.Node;
import java.util.List;
import lombok.val;
public abstract class Parser {
private static final Token EOF = new Token(HotaruTokenId.EOF, "",
0, new SourcePosition(-1, -1, -1));
private final List<Token> tokens;
private final int size;
private final ParseErrors parseErrors;
private Node parsedNode;
protected int pos;
public Parser(List<Token> tokens) {
this.tokens = tokens;
this.size = tokens.size();
parseErrors = new ParseErrors();
pos = 0;
}
public Node getParsedNode() {
return parsedNode;
}
public ParseErrors getParseErrors() {
return parseErrors;
}
public Node parse() {
parseErrors.clear();
val result = new BlockNode();
result.start(getSourcePosition());
while (!match(HotaruTokenId.EOF)) {
try {
result.add(statement());
} catch (Exception ex) {
parseErrors.add(ex, getSourcePosition());
recover();
}
}
result.end(getSourcePosition());
parsedNode = result;
return result;
}
protected abstract Node statement();
protected SourcePosition getSourcePosition() {
if (size == 0) return new SourcePosition(0, 0, 0);
if (pos >= size) return tokens.get(size - 1).getPosition();
return tokens.get(pos).getPosition();
}
private void recover() {
int preRecoverPosition = pos;
for (int i = preRecoverPosition; i <= size; i++) {
pos = i;
try {
statement();
// successfully parsed,
pos = i; // restore position
return;
} catch (Exception ex) {
// fail
}
}
}
protected Token consume(HotaruTokenId type) {
val current = get(0);
if (type != current.getType()) {
throw new ParseException("Token " + current + " doesn't match " + type, current.getPosition());
}
pos++;
return current;
}
protected boolean match(HotaruTokenId type) {
val current = get(0);
if (type != current.getType()) return false;
pos++;
return true;
}
protected boolean lookMatch(int pos, HotaruTokenId type) {
return get(pos).getType() == type;
}
protected Token get(int relativePosition) {
val position = pos + relativePosition;
if (position >= size) return EOF;
return tokens.get(position);
}
}

View File

@ -0,0 +1,19 @@
package com.annimon.hotarufx.parser.ast;
import com.annimon.hotarufx.lexer.SourcePosition;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
@Accessors(fluent = true)
public abstract class ASTNode implements Node {
@Getter @Setter
private SourcePosition start;
@Getter @Setter
private SourcePosition end;
public String getSourceRange() {
return start + " .. " + end;
}
}

View File

@ -0,0 +1,33 @@
package com.annimon.hotarufx.parser.ast;
import com.annimon.hotarufx.lib.Value;
import java.util.List;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class AccessNode extends ASTNode implements Accessible {
@Getter
public final Node root;
public final List<Node> indices;
public AccessNode(String variable, List<Node> indices) {
this(new VariableNode(variable), indices);
}
@Override
public Value get() {
return null;
}
@Override
public Value set(Value value) {
return null;
}
@Override
public <R, T> R accept(ResultVisitor<R, T> visitor, T input) {
return visitor.visit(this, input);
}
}

View File

@ -0,0 +1,10 @@
package com.annimon.hotarufx.parser.ast;
import com.annimon.hotarufx.lib.Value;
public interface Accessible extends Node {
Value get();
Value set(Value value);
}

View File

@ -0,0 +1,15 @@
package com.annimon.hotarufx.parser.ast;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class AssignNode extends ASTNode {
public final Accessible target;
public final Node value;
@Override
public <R, T> R accept(ResultVisitor<R, T> visitor, T input) {
return visitor.visit(this, input);
}
}

View File

@ -0,0 +1,18 @@
package com.annimon.hotarufx.parser.ast;
import java.util.ArrayList;
import java.util.List;
public class BlockNode extends ASTNode {
public final List<Node> statements = new ArrayList<>();
public void add(Node node) {
statements.add(node);
}
@Override
public <R, T> R accept(ResultVisitor<R, T> visitor, T t) {
return visitor.visit(this, t);
}
}

View File

@ -0,0 +1,24 @@
package com.annimon.hotarufx.parser.ast;
import java.util.ArrayList;
import java.util.List;
public class FunctionNode extends ASTNode {
public final Node functionNode;
public final List<Node> arguments;
public FunctionNode(Node functionNode) {
this.functionNode = functionNode;
arguments = new ArrayList<>();
}
public void addArgument(Node arg) {
arguments.add(arg);
}
@Override
public <R, T> R accept(ResultVisitor<R, T> visitor, T input) {
return visitor.visit(this, input);
}
}

View File

@ -0,0 +1,15 @@
package com.annimon.hotarufx.parser.ast;
import java.util.Map;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class MapNode extends ASTNode {
public final Map<String, Node> elements;
@Override
public <R, T> R accept(ResultVisitor<R, T> visitor, T input) {
return visitor.visit(this, input);
}
}

View File

@ -0,0 +1,6 @@
package com.annimon.hotarufx.parser.ast;
public interface Node {
<R, T> R accept(ResultVisitor<R, T> visitor, T input);
}

View File

@ -0,0 +1,14 @@
package com.annimon.hotarufx.parser.ast;
public interface ResultVisitor<R, T> {
R visit(AccessNode node, T t);
R visit(AssignNode node, T t);
R visit(BlockNode node, T t);
R visit(FunctionNode node, T t);
R visit(MapNode node, T t);
R visit(UnaryNode node, T t);
R visit(ValueNode node, T t);
R visit(VariableNode node, T t);
}

View File

@ -0,0 +1,17 @@
package com.annimon.hotarufx.parser.ast;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class UnaryNode extends ASTNode {
public enum Operator { NEGATE };
public final Operator operator;
public final Node node;
@Override
public <R, T> R accept(ResultVisitor<R, T> visitor, T input) {
return visitor.visit(this, input);
}
}

View File

@ -0,0 +1,25 @@
package com.annimon.hotarufx.parser.ast;
import com.annimon.hotarufx.lib.NumberValue;
import com.annimon.hotarufx.lib.StringValue;
import com.annimon.hotarufx.lib.Value;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class ValueNode extends ASTNode {
public final Value value;
public ValueNode(Number value) {
this(NumberValue.of(value));
}
public ValueNode(String value) {
this(new StringValue(value));
}
@Override
public <R, T> R accept(ResultVisitor<R, T> visitor, T input) {
return visitor.visit(this, input);
}
}

View File

@ -0,0 +1,25 @@
package com.annimon.hotarufx.parser.ast;
import com.annimon.hotarufx.lib.Value;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class VariableNode extends ASTNode implements Accessible {
public final String name;
@Override
public Value get() {
return null;
}
@Override
public Value set(Value value) {
return value;
}
@Override
public <R, T> R accept(ResultVisitor<R, T> visitor, T input) {
return visitor.visit(this, input);
}
}

View File

@ -0,0 +1,68 @@
package com.annimon.hotarufx.parser;
import com.annimon.hotarufx.exceptions.ParseException;
import com.annimon.hotarufx.lexer.HotaruLexer;
import com.annimon.hotarufx.parser.ast.AssignNode;
import com.annimon.hotarufx.parser.ast.BlockNode;
import com.annimon.hotarufx.parser.ast.FunctionNode;
import com.annimon.hotarufx.parser.ast.MapNode;
import com.annimon.hotarufx.parser.ast.Node;
import com.annimon.hotarufx.parser.ast.ValueNode;
import com.annimon.hotarufx.parser.ast.VariableNode;
import org.junit.jupiter.api.Test;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.junit.jupiter.api.Assertions.assertThrows;
class HotaruParserTest {
static Node p(String input) {
return HotaruParser.parse(HotaruLexer.tokenize(input));
}
@Test
void testParseNodeAssignment() {
String input = "A = node({x: -1, text:'hello'})";
Node node = p(input);
assertThat(node, instanceOf(BlockNode.class));
BlockNode block = (BlockNode) node;
assertThat(block.statements.size(), is(1));
assertThat(block.start().getPosition(), is(1));
assertThat(block.end().getPosition(), is(input.length()));
Node firstNode = block.statements.get(0);
assertThat(firstNode, instanceOf(AssignNode.class));
AssignNode assignNode = (AssignNode) firstNode;
assertThat(assignNode.target, instanceOf(VariableNode.class));
assertThat(assignNode.value, instanceOf(FunctionNode.class));
VariableNode target = (VariableNode) assignNode.target;
assertThat(target.name, is("A"));
FunctionNode value = (FunctionNode) assignNode.value;
assertThat(value.functionNode, instanceOf(ValueNode.class));
assertThat(((ValueNode)value.functionNode).value.asString(), is("node"));
assertThat(value.arguments.size(), is(1));
Node argument = value.arguments.get(0);
assertThat(argument, instanceOf(MapNode.class));
MapNode mapNode = (MapNode) argument;
assertThat(mapNode.elements.size(), is(2));
assertThat(mapNode.elements, allOf(
hasKey("x"), hasKey("text")
));
}
@Test
void testParseErrors() {
assertThrows(ParseException.class, () -> p("A ="));
assertThrows(ParseException.class, () -> p("1 = A"));
assertThrows(ParseException.class, () -> p("{A = B}"));
}
}