Парсинг строк в юникоде (u"text" или u'text')
This commit is contained in:
parent
37d02f5ad9
commit
a9d99744b3
@ -137,6 +137,13 @@ public final class Lexer {
|
|||||||
if (Character.isLetterOrDigit(ch) || (ch == '_') || (ch == '.')) {
|
if (Character.isLetterOrDigit(ch) || (ch == '_') || (ch == '.')) {
|
||||||
buffer.append(ch);
|
buffer.append(ch);
|
||||||
} else {
|
} else {
|
||||||
|
if ( (ch == '"' || ch == '\'') && (buffer.length() == 1) && (buffer.charAt(0) == 'u') ) {
|
||||||
|
// Строка в юникоде u"текст" или u'текст'
|
||||||
|
clearBuffer();
|
||||||
|
textStartChar = ch;
|
||||||
|
state = TokenizeState.TEXT;
|
||||||
|
return;
|
||||||
|
}
|
||||||
final String word = buffer.toString().toLowerCase(Locale.ENGLISH);
|
final String word = buffer.toString().toLowerCase(Locale.ENGLISH);
|
||||||
addToken(KEYWORDS.containsKey(word) ? KEYWORDS.get(word) : TokenType.WORD);
|
addToken(KEYWORDS.containsKey(word) ? KEYWORDS.get(word) : TokenType.WORD);
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user