java-projects/Own-Programming-Language-Tutorial
1
0
Fork 0
mirror of https://github.com/aNNiMON/Own-Programming-Language-Tutorial.git synced 2024-09-20 00:34:20 +03:00
Code Issues Packages Projects Releases Wiki Activity

Support for long number declaration

Browse Source
This commit is contained in:
aNNiMON 2024-02-10 23:30:37 +02:00
parent 31945471dd
commit 7acad44211
9 changed files with 102 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/docs/en/changelog.md
View File

@ -10,6 +10,7 @@
### Changes
- Introducing Constants. Constant can be imported only when using a module.
- Support for long number declaration: `700L`, `0xABL`
- Fixed variables scope in shadowing.
- Better error visualizing. Parse errors shows exact line in which an error occurs. Same for Linter and Runtime errors.
- Semantic linter as a required stage.

1
docs/docs/ru/changelog.md
View File

@ -10,6 +10,7 @@
### Изменения
- Добавлены константы. Константа может быть импортирована только при подключении модуля.
- Возможность задать числа типа long: `700L`, `0xABL`
- Исправлена область видимости переменных при шедоуинге.
- Улучшена визуализация ошибок. Ошибки парсинга показывают конкретное место, где возникла ошибка. То же самое с линтером и ошибками времени исполнения.
- Семантический линтер как обязательный этап работы интерпретатора.

11
ownlang-parser/src/main/java/com/annimon/ownlang/parser/Lexer.java
View File

@ -167,7 +167,8 @@ public final class Lexer {
while (true) {
if (current == '.') {
decimal = true;
if (hasDot) throw error("Invalid float number " + buffer, startPos);
if (hasDot)
throw error("Invalid float number " + buffer, startPos);
hasDot = true;
} else if (current == 'e' || current == 'E') {
decimal = true;
@ -182,6 +183,9 @@ public final class Lexer {
}
if (decimal) {
addToken(TokenType.DECIMAL_NUMBER, buffer.toString(), startPos);
} else if (current == 'L') {
next();
addToken(TokenType.LONG_NUMBER, buffer.toString(), startPos);
} else {
addToken(TokenType.NUMBER, buffer.toString(), startPos);
}
@ -232,8 +236,13 @@ public final class Lexer {
if (buffer.isEmpty()) throw error("Empty HEX value", startPos);
if (peek(-1) == '_') throw error("HEX value cannot end with _", startPos, markEndPos());
if (current == 'L') {
next();
addToken(TokenType.HEX_LONG_NUMBER, buffer.toString(), startPos);
} else {
addToken(TokenType.HEX_NUMBER, buffer.toString(), startPos);
}
}
private static boolean isNumber(char current) {
return ('0' <= current && current <= '9');

79
ownlang-parser/src/main/java/com/annimon/ownlang/parser/Parser.java
View File

@ -50,6 +50,14 @@ public final class Parser {
ASSIGN_OPERATORS.put(TokenType.ATEQ, BinaryExpression.Operator.AT);
}
private static final EnumSet<TokenType> NUMBER_TOKEN_TYPES = EnumSet.of(
TokenType.NUMBER,
TokenType.LONG_NUMBER,
TokenType.DECIMAL_NUMBER,
TokenType.HEX_NUMBER,
TokenType.HEX_LONG_NUMBER
);
private final List<Token> tokens;
private final int size;
private final ParseErrors parseErrors;
@ -347,7 +355,7 @@ public final class Parser {
}
if (lookMatch(0, TokenType.DOT)) {
final List<Node> indices = variableSuffix();
if (indices == null || indices.isEmpty()) {
if (indices.isEmpty()) {
return expr;
}
@ -411,20 +419,10 @@ public final class Parser {
consume(TokenType.CASE);
MatchExpression.Pattern pattern = null;
final Token current = get(0);
if (match(TokenType.NUMBER)) {
// case 20:
if (isNumberToken(current.type())) {
// case 20: / case 0.5: / case #FF:
pattern = new MatchExpression.ConstantPattern(
NumberValue.of(createNumber(current.text(), 10))
);
} else if (match(TokenType.DECIMAL_NUMBER)) {
// case 0.5:
pattern = new MatchExpression.ConstantPattern(
NumberValue.of(createDecimalNumber(current.text()))
);
} else if (match(TokenType.HEX_NUMBER)) {
// case #FF:
pattern = new MatchExpression.ConstantPattern(
NumberValue.of(createNumber(current.text(), 16))
NumberValue.of(getAsNumber(current))
);
} else if (match(TokenType.TEXT)) {
// case "text":
@ -859,7 +857,7 @@ public final class Parser {
final List<Node> indices = variableSuffix();
final var variable = new VariableExpression(current.text());
variable.setRange(getRange(startTokenIndex, index - 1));
if (indices == null || indices.isEmpty()) {
if (indices.isEmpty()) {
return variable;
} else {
return new ContainerAccessExpression(variable, indices, variable.getRange());
@ -869,7 +867,7 @@ public final class Parser {
private List<Node> variableSuffix() {
// .key1.arr1[expr1][expr2].key2
if (!lookMatch(0, TokenType.DOT) && !lookMatch(0, TokenType.LBRACKET)) {
return null;
return Collections.emptyList();
}
final List<Node> indices = new ArrayList<>();
while (lookMatch(0, TokenType.DOT) || lookMatch(0, TokenType.LBRACKET)) {
@ -888,19 +886,21 @@ public final class Parser {
private Node value() {
final Token current = get(0);
if (match(TokenType.NUMBER)) {
return new ValueExpression(createNumber(current.text(), 10));
}
if (match(TokenType.DECIMAL_NUMBER)) {
return new ValueExpression(createDecimalNumber(current.text()));
}
if (match(TokenType.HEX_NUMBER)) {
return new ValueExpression(createNumber(current.text(), 16));
if (isNumberToken(current.type())) {
return new ValueExpression(getAsNumber(current));
}
if (match(TokenType.TEXT)) {
final ValueExpression strExpr = new ValueExpression(current.text());
// "text".property || "text".func()
if (lookMatch(0, TokenType.DOT)) {
return stringProperty(strExpr);
}
return strExpr;
}
throw error("Unknown expression: " + current);
}
private Node stringProperty(ValueExpression strExpr) {
if (lookMatch(1, TokenType.WORD) && lookMatch(2, TokenType.LPAREN)) {
match(TokenType.DOT);
return functionChain(new ContainerAccessExpression(
@ -910,14 +910,33 @@ public final class Parser {
));
}
final List<Node> indices = variableSuffix();
if (indices == null || indices.isEmpty()) {
if (indices.isEmpty()) {
return strExpr;
}
return new ContainerAccessExpression(strExpr, indices, getRange());
}
return strExpr;
private boolean isNumberToken(TokenType type) {
return NUMBER_TOKEN_TYPES.contains(type);
}
throw error("Unknown expression: " + current);
private Number getAsNumber(Token current) {
if (match(TokenType.NUMBER)) {
return createNumber(current.text(), 10);
}
if (match(TokenType.LONG_NUMBER)) {
return createLongNumber(current.text(), 10);
}
if (match(TokenType.DECIMAL_NUMBER)) {
return createDecimalNumber(current.text());
}
if (match(TokenType.HEX_NUMBER)) {
return createNumber(current.text(), 16);
}
if (match(TokenType.HEX_LONG_NUMBER)) {
return createLongNumber(current.text(), 16);
}
throw error("Unknown number expression: " + current);
}
private Number createNumber(String text, int radix) {
@ -925,10 +944,14 @@ public final class Parser {
try {
return Integer.parseInt(text, radix);
} catch (NumberFormatException nfe) {
return Long.parseLong(text, radix);
return createLongNumber(text, radix);
}
}
private Number createLongNumber(String text, int radix) {
return Long.parseLong(text, radix);
}
private Number createDecimalNumber(String text) {
// Double
return Double.parseDouble(text);

2
ownlang-parser/src/main/java/com/annimon/ownlang/parser/TokenType.java
View File

@ -7,8 +7,10 @@ package com.annimon.ownlang.parser;
public enum TokenType {
NUMBER,
LONG_NUMBER,
DECIMAL_NUMBER,
HEX_NUMBER,
HEX_LONG_NUMBER,
WORD,
TEXT,

22
ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerTest.java
View File

@ -41,17 +41,17 @@ public class LexerTest {
}
@Test
public void testNumbers() {
String input = "0 3.1415 0xCAFEBABE 0Xf7_d6_c5 #FFFF";
void testNumbers() {
String input = "0 800L 3.1415 0xCAFEBABE 0Xf7_d6_c5 #FFFF 0x7FL";
List<Token> result = Lexer.tokenize(input);
assertTokens(result, NUMBER, DECIMAL_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER);
assertTokens(result, NUMBER, LONG_NUMBER, DECIMAL_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_LONG_NUMBER);
assertThat(result)
.extracting(Token::text)
.containsExactly("0", "3.1415", "CAFEBABE", "f7d6c5", "FFFF");
.containsExactly("0", "800", "3.1415", "CAFEBABE", "f7d6c5", "FFFF", "7F");
}
@Test
public void testDecimalNumbersExponent() {
void testDecimalNumbersExponent() {
String input = "4e+7 0.3E-19 2e0 5e0000000000000200 5E-000000089";
List<Token> result = Lexer.tokenize(input);
assertThat(result)
@ -61,7 +61,7 @@ public class LexerTest {
}
@Test
public void testString() {
void testString() {
String input = "\"1\\\"2\"";
List<Token> result = Lexer.tokenize(input);
assertTokens(result, TEXT);
@ -69,7 +69,7 @@ public class LexerTest {
}
@Test
public void testEscapeString() {
void testEscapeString() {
String input = """
"\\\\/\\\\"
""".stripIndent();
@ -79,7 +79,7 @@ public class LexerTest {
}
@Test
public void testEmptyString() {
void testEmptyString() {
String input = "\"\"";
List<Token> result = Lexer.tokenize(input);
assertTokens(result, TEXT);
@ -87,7 +87,7 @@ public class LexerTest {
}
@Test
public void testComments() {
void testComments() {
String input = "// 1234 \n /* */ 123 /* \n 12345 \n\n\n */";
List<Token> result = Lexer.tokenize(input);
assertTokens(result, NUMBER);
@ -96,7 +96,7 @@ public class LexerTest {
@ParameterizedTest
@MethodSource("validData")
public void testValidInput(String name, String input, List<TokenType> tokenTypes) throws IOException {
void testValidInput(String name, String input, List<TokenType> tokenTypes) throws IOException {
List<Token> result = Lexer.tokenize(input);
assertThat(result)
.hasSize(tokenTypes.size())
@ -106,7 +106,7 @@ public class LexerTest {
@ParameterizedTest
@MethodSource("invalidData")
public void testInvalidInput(String name, String input) throws IOException {
void testInvalidInput(String name, String input) throws IOException {
assertThatThrownBy(() -> Lexer.tokenize(input))
.isInstanceOf(OwnLangParserException.class);
}

5
ownlang-parser/src/test/java/com/annimon/ownlang/parser/LexerValidDataProvider.java
View File

@ -33,7 +33,10 @@ public class LexerValidDataProvider {
List.of(DECIMAL_NUMBER)),
Arguments.of("Hex numbers",
"#FF 0xCA 0x12fb 0xFF",
List.of(HEX_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER))
List.of(HEX_NUMBER, HEX_NUMBER, HEX_NUMBER, HEX_NUMBER)),
Arguments.of("Long numbers",
"680L #80L 0x700L",
List.of(LONG_NUMBER, HEX_LONG_NUMBER, HEX_LONG_NUMBER))
);
}

2
ownlang-parser/src/test/java/com/annimon/ownlang/parser/ProgramsTest.java
View File

@ -50,7 +50,7 @@ public class ProgramsTest {
@ParameterizedTest
@MethodSource("data")
public void testProgram(InputSource inputSource) {
void testProgram(InputSource inputSource) {
final StagesDataMap stagesData = new StagesDataMap();
try {
testPipeline.perform(stagesData, inputSource);

10
ownlang-parser/src/test/resources/expressions/binaryExpressionOnNumbers.own
View File

@ -14,6 +14,16 @@ def testMultiplicationOnNumbers() {
assertEquals(30, 5 * (-2 * -3))
}
def testMultiplicationOverflowOnNumbers() {
assertNotEquals(1234567890000L, 1234567890 * 1000)
assertNotEquals(0xFFFFFF00, 0x100 * 0xFFFFFF)
}
def testMultiplicationOnLongNumbers() {
assertEquals(1234567890000L, 1234567890 * 1000L)
assertEquals(0xFFFFFF00L, 0x100L * 0xFFFFFF)
}
def testDivisionOnNumbers() {
assertEquals(3, 6 / 2)
assertEquals(30, -900 / (60 / -2))