aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/tools/cython/Cython/Compiler/Lexicon.py
diff options
context:
space:
mode:
authoralexv-smirnov <alex@ydb.tech>2023-03-15 19:59:12 +0300
committeralexv-smirnov <alex@ydb.tech>2023-03-15 19:59:12 +0300
commit056bb284ccf8dd6793ec3a54ffa36c4fb2b9ad11 (patch)
tree4740980126f32e3af7937ba0ca5f83e59baa4ab0 /contrib/tools/cython/Cython/Compiler/Lexicon.py
parent269126dcced1cc8b53eb4398b4a33e5142f10290 (diff)
downloadydb-056bb284ccf8dd6793ec3a54ffa36c4fb2b9ad11.tar.gz
add library/cpp/actors, ymake build to ydb oss export
Diffstat (limited to 'contrib/tools/cython/Cython/Compiler/Lexicon.py')
-rw-r--r--contrib/tools/cython/Cython/Compiler/Lexicon.py138
1 files changed, 138 insertions, 0 deletions
diff --git a/contrib/tools/cython/Cython/Compiler/Lexicon.py b/contrib/tools/cython/Cython/Compiler/Lexicon.py
new file mode 100644
index 0000000000..72c9ceaefd
--- /dev/null
+++ b/contrib/tools/cython/Cython/Compiler/Lexicon.py
@@ -0,0 +1,138 @@
+# cython: language_level=3, py2_import=True
+#
+# Cython Scanner - Lexical Definitions
+#
+
+from __future__ import absolute_import, unicode_literals
+
+raw_prefixes = "rR"
+bytes_prefixes = "bB"
+string_prefixes = "fFuU" + bytes_prefixes
+char_prefixes = "cC"
+any_string_prefix = raw_prefixes + string_prefixes + char_prefixes
+IDENT = 'IDENT'
+
+
+def make_lexicon():
+ from ..Plex import \
+ Str, Any, AnyBut, AnyChar, Rep, Rep1, Opt, Bol, Eol, Eof, \
+ TEXT, IGNORE, State, Lexicon
+ from .Scanning import Method
+
+ letter = Any("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_")
+ digit = Any("0123456789")
+ bindigit = Any("01")
+ octdigit = Any("01234567")
+ hexdigit = Any("0123456789ABCDEFabcdef")
+ indentation = Bol + Rep(Any(" \t"))
+
+ def underscore_digits(d):
+ return Rep1(d) + Rep(Str("_") + Rep1(d))
+
+ decimal = underscore_digits(digit)
+ dot = Str(".")
+ exponent = Any("Ee") + Opt(Any("+-")) + decimal
+ decimal_fract = (decimal + dot + Opt(decimal)) | (dot + decimal)
+
+ name = letter + Rep(letter | digit)
+ intconst = decimal | (Str("0") + ((Any("Xx") + underscore_digits(hexdigit)) |
+ (Any("Oo") + underscore_digits(octdigit)) |
+ (Any("Bb") + underscore_digits(bindigit)) ))
+ intsuffix = (Opt(Any("Uu")) + Opt(Any("Ll")) + Opt(Any("Ll"))) | (Opt(Any("Ll")) + Opt(Any("Ll")) + Opt(Any("Uu")))
+ intliteral = intconst + intsuffix
+ fltconst = (decimal_fract + Opt(exponent)) | (decimal + exponent)
+ imagconst = (intconst | fltconst) + Any("jJ")
+
+ # invalid combinations of prefixes are caught in p_string_literal
+ beginstring = Opt(Rep(Any(string_prefixes + raw_prefixes)) |
+ Any(char_prefixes)
+ ) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
+ two_oct = octdigit + octdigit
+ three_oct = octdigit + octdigit + octdigit
+ two_hex = hexdigit + hexdigit
+ four_hex = two_hex + two_hex
+ escapeseq = Str("\\") + (two_oct | three_oct |
+ Str('N{') + Rep(AnyBut('}')) + Str('}') |
+ Str('u') + four_hex | Str('x') + two_hex |
+ Str('U') + four_hex + four_hex | AnyChar)
+
+ bra = Any("([{")
+ ket = Any(")]}")
+ punct = Any(":,;+-*/|&<>=.%`~^?!@")
+ diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "//",
+ "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=",
+ "<<=", ">>=", "**=", "//=", "->", "@=")
+ spaces = Rep1(Any(" \t\f"))
+ escaped_newline = Str("\\\n")
+ lineterm = Eol + Opt(Str("\n"))
+
+ comment = Str("#") + Rep(AnyBut("\n"))
+
+ return Lexicon([
+ (name, IDENT),
+ (intliteral, Method('strip_underscores', symbol='INT')),
+ (fltconst, Method('strip_underscores', symbol='FLOAT')),
+ (imagconst, Method('strip_underscores', symbol='IMAG')),
+ (punct | diphthong, TEXT),
+
+ (bra, Method('open_bracket_action')),
+ (ket, Method('close_bracket_action')),
+ (lineterm, Method('newline_action')),
+
+ (beginstring, Method('begin_string_action')),
+
+ (comment, IGNORE),
+ (spaces, IGNORE),
+ (escaped_newline, IGNORE),
+
+ State('INDENT', [
+ (comment + lineterm, Method('commentline')),
+ (Opt(spaces) + Opt(comment) + lineterm, IGNORE),
+ (indentation, Method('indentation_action')),
+ (Eof, Method('eof_action'))
+ ]),
+
+ State('SQ_STRING', [
+ (escapeseq, 'ESCAPE'),
+ (Rep1(AnyBut("'\"\n\\")), 'CHARS'),
+ (Str('"'), 'CHARS'),
+ (Str("\n"), Method('unclosed_string_action')),
+ (Str("'"), Method('end_string_action')),
+ (Eof, 'EOF')
+ ]),
+
+ State('DQ_STRING', [
+ (escapeseq, 'ESCAPE'),
+ (Rep1(AnyBut('"\n\\')), 'CHARS'),
+ (Str("'"), 'CHARS'),
+ (Str("\n"), Method('unclosed_string_action')),
+ (Str('"'), Method('end_string_action')),
+ (Eof, 'EOF')
+ ]),
+
+ State('TSQ_STRING', [
+ (escapeseq, 'ESCAPE'),
+ (Rep1(AnyBut("'\"\n\\")), 'CHARS'),
+ (Any("'\""), 'CHARS'),
+ (Str("\n"), 'NEWLINE'),
+ (Str("'''"), Method('end_string_action')),
+ (Eof, 'EOF')
+ ]),
+
+ State('TDQ_STRING', [
+ (escapeseq, 'ESCAPE'),
+ (Rep1(AnyBut('"\'\n\\')), 'CHARS'),
+ (Any("'\""), 'CHARS'),
+ (Str("\n"), 'NEWLINE'),
+ (Str('"""'), Method('end_string_action')),
+ (Eof, 'EOF')
+ ]),
+
+ (Eof, Method('eof_action'))
+ ],
+
+ # FIXME: Plex 1.9 needs different args here from Plex 1.1.4
+ #debug_flags = scanner_debug_flags,
+ #debug_file = scanner_dump_file
+ )
+