Парсинг строк в юникоде (u"text" или u'text')

2015-04-18 16:49:34 +03:00 · 2015-04-18 16:49:34 +03:00 · a9d99744b3
commit a9d99744b3
parent 37d02f5ad9
1 changed files with 7 additions and 0 deletions
--- a/src/com/annimon/everlastingsummer/Lexer.java
+++ b/src/com/annimon/everlastingsummer/Lexer.java
@ -137,6 +137,13 @@ public final class Lexer {
        if (Character.isLetterOrDigit(ch) || (ch == '_') || (ch == '.')) {
            buffer.append(ch);
        } else {
+            if ( (ch == '"' || ch == '\'') && (buffer.length() == 1) && (buffer.charAt(0) == 'u') ) {
+                // Строка в юникоде u"текст" или u'текст'
+                clearBuffer();
+                textStartChar = ch;
+                state = TokenizeState.TEXT;
+                return;
+            }
            final String word = buffer.toString().toLowerCase(Locale.ENGLISH);
            addToken(KEYWORDS.containsKey(word) ? KEYWORDS.get(word) : TokenType.WORD);
        }