summaryrefslogtreecommitdiffstats
path: root/contrib/tools/cython/Cython/Compiler/Lexicon.py
diff options
context:
space:
mode:
authorAnton Samokhvalov <[email protected]>2022-02-10 16:45:15 +0300
committerDaniil Cherednik <[email protected]>2022-02-10 16:45:15 +0300
commit72cb13b4aff9bc9cf22e49251bc8fd143f82538f (patch)
treeda2c34829458c7d4e74bdfbdf85dff449e9e7fb8 /contrib/tools/cython/Cython/Compiler/Lexicon.py
parent778e51ba091dc39e7b7fcab2b9cf4dbedfb6f2b5 (diff)
Restoring authorship annotation for Anton Samokhvalov <[email protected]>. Commit 1 of 2.
Diffstat (limited to 'contrib/tools/cython/Cython/Compiler/Lexicon.py')
-rw-r--r--contrib/tools/cython/Cython/Compiler/Lexicon.py248
1 files changed, 124 insertions, 124 deletions
diff --git a/contrib/tools/cython/Cython/Compiler/Lexicon.py b/contrib/tools/cython/Cython/Compiler/Lexicon.py
index 72c9ceaefd5..3b79d869c5e 100644
--- a/contrib/tools/cython/Cython/Compiler/Lexicon.py
+++ b/contrib/tools/cython/Cython/Compiler/Lexicon.py
@@ -1,138 +1,138 @@
-# cython: language_level=3, py2_import=True
-#
-# Cython Scanner - Lexical Definitions
-#
-
+# cython: language_level=3, py2_import=True
+#
+# Cython Scanner - Lexical Definitions
+#
+
from __future__ import absolute_import, unicode_literals
-
-raw_prefixes = "rR"
-bytes_prefixes = "bB"
+
+raw_prefixes = "rR"
+bytes_prefixes = "bB"
string_prefixes = "fFuU" + bytes_prefixes
-char_prefixes = "cC"
-any_string_prefix = raw_prefixes + string_prefixes + char_prefixes
-IDENT = 'IDENT'
-
-
-def make_lexicon():
- from ..Plex import \
- Str, Any, AnyBut, AnyChar, Rep, Rep1, Opt, Bol, Eol, Eof, \
- TEXT, IGNORE, State, Lexicon
- from .Scanning import Method
-
- letter = Any("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_")
- digit = Any("0123456789")
- bindigit = Any("01")
- octdigit = Any("01234567")
- hexdigit = Any("0123456789ABCDEFabcdef")
- indentation = Bol + Rep(Any(" \t"))
-
+char_prefixes = "cC"
+any_string_prefix = raw_prefixes + string_prefixes + char_prefixes
+IDENT = 'IDENT'
+
+
+def make_lexicon():
+ from ..Plex import \
+ Str, Any, AnyBut, AnyChar, Rep, Rep1, Opt, Bol, Eol, Eof, \
+ TEXT, IGNORE, State, Lexicon
+ from .Scanning import Method
+
+ letter = Any("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_")
+ digit = Any("0123456789")
+ bindigit = Any("01")
+ octdigit = Any("01234567")
+ hexdigit = Any("0123456789ABCDEFabcdef")
+ indentation = Bol + Rep(Any(" \t"))
+
def underscore_digits(d):
return Rep1(d) + Rep(Str("_") + Rep1(d))
decimal = underscore_digits(digit)
- dot = Str(".")
- exponent = Any("Ee") + Opt(Any("+-")) + decimal
- decimal_fract = (decimal + dot + Opt(decimal)) | (dot + decimal)
-
- name = letter + Rep(letter | digit)
+ dot = Str(".")
+ exponent = Any("Ee") + Opt(Any("+-")) + decimal
+ decimal_fract = (decimal + dot + Opt(decimal)) | (dot + decimal)
+
+ name = letter + Rep(letter | digit)
intconst = decimal | (Str("0") + ((Any("Xx") + underscore_digits(hexdigit)) |
(Any("Oo") + underscore_digits(octdigit)) |
(Any("Bb") + underscore_digits(bindigit)) ))
- intsuffix = (Opt(Any("Uu")) + Opt(Any("Ll")) + Opt(Any("Ll"))) | (Opt(Any("Ll")) + Opt(Any("Ll")) + Opt(Any("Uu")))
- intliteral = intconst + intsuffix
- fltconst = (decimal_fract + Opt(exponent)) | (decimal + exponent)
- imagconst = (intconst | fltconst) + Any("jJ")
-
+ intsuffix = (Opt(Any("Uu")) + Opt(Any("Ll")) + Opt(Any("Ll"))) | (Opt(Any("Ll")) + Opt(Any("Ll")) + Opt(Any("Uu")))
+ intliteral = intconst + intsuffix
+ fltconst = (decimal_fract + Opt(exponent)) | (decimal + exponent)
+ imagconst = (intconst | fltconst) + Any("jJ")
+
# invalid combinations of prefixes are caught in p_string_literal
beginstring = Opt(Rep(Any(string_prefixes + raw_prefixes)) |
- Any(char_prefixes)
- ) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
- two_oct = octdigit + octdigit
- three_oct = octdigit + octdigit + octdigit
- two_hex = hexdigit + hexdigit
- four_hex = two_hex + two_hex
- escapeseq = Str("\\") + (two_oct | three_oct |
- Str('N{') + Rep(AnyBut('}')) + Str('}') |
- Str('u') + four_hex | Str('x') + two_hex |
- Str('U') + four_hex + four_hex | AnyChar)
-
- bra = Any("([{")
- ket = Any(")]}")
- punct = Any(":,;+-*/|&<>=.%`~^?!@")
- diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "//",
- "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=",
- "<<=", ">>=", "**=", "//=", "->", "@=")
- spaces = Rep1(Any(" \t\f"))
- escaped_newline = Str("\\\n")
- lineterm = Eol + Opt(Str("\n"))
-
- comment = Str("#") + Rep(AnyBut("\n"))
-
- return Lexicon([
- (name, IDENT),
+ Any(char_prefixes)
+ ) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
+ two_oct = octdigit + octdigit
+ three_oct = octdigit + octdigit + octdigit
+ two_hex = hexdigit + hexdigit
+ four_hex = two_hex + two_hex
+ escapeseq = Str("\\") + (two_oct | three_oct |
+ Str('N{') + Rep(AnyBut('}')) + Str('}') |
+ Str('u') + four_hex | Str('x') + two_hex |
+ Str('U') + four_hex + four_hex | AnyChar)
+
+ bra = Any("([{")
+ ket = Any(")]}")
+ punct = Any(":,;+-*/|&<>=.%`~^?!@")
+ diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "//",
+ "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=",
+ "<<=", ">>=", "**=", "//=", "->", "@=")
+ spaces = Rep1(Any(" \t\f"))
+ escaped_newline = Str("\\\n")
+ lineterm = Eol + Opt(Str("\n"))
+
+ comment = Str("#") + Rep(AnyBut("\n"))
+
+ return Lexicon([
+ (name, IDENT),
(intliteral, Method('strip_underscores', symbol='INT')),
(fltconst, Method('strip_underscores', symbol='FLOAT')),
(imagconst, Method('strip_underscores', symbol='IMAG')),
- (punct | diphthong, TEXT),
-
- (bra, Method('open_bracket_action')),
- (ket, Method('close_bracket_action')),
- (lineterm, Method('newline_action')),
-
- (beginstring, Method('begin_string_action')),
-
- (comment, IGNORE),
- (spaces, IGNORE),
- (escaped_newline, IGNORE),
-
- State('INDENT', [
- (comment + lineterm, Method('commentline')),
- (Opt(spaces) + Opt(comment) + lineterm, IGNORE),
- (indentation, Method('indentation_action')),
- (Eof, Method('eof_action'))
- ]),
-
- State('SQ_STRING', [
- (escapeseq, 'ESCAPE'),
- (Rep1(AnyBut("'\"\n\\")), 'CHARS'),
- (Str('"'), 'CHARS'),
- (Str("\n"), Method('unclosed_string_action')),
- (Str("'"), Method('end_string_action')),
- (Eof, 'EOF')
- ]),
-
- State('DQ_STRING', [
- (escapeseq, 'ESCAPE'),
- (Rep1(AnyBut('"\n\\')), 'CHARS'),
- (Str("'"), 'CHARS'),
- (Str("\n"), Method('unclosed_string_action')),
- (Str('"'), Method('end_string_action')),
- (Eof, 'EOF')
- ]),
-
- State('TSQ_STRING', [
- (escapeseq, 'ESCAPE'),
- (Rep1(AnyBut("'\"\n\\")), 'CHARS'),
- (Any("'\""), 'CHARS'),
- (Str("\n"), 'NEWLINE'),
- (Str("'''"), Method('end_string_action')),
- (Eof, 'EOF')
- ]),
-
- State('TDQ_STRING', [
- (escapeseq, 'ESCAPE'),
- (Rep1(AnyBut('"\'\n\\')), 'CHARS'),
- (Any("'\""), 'CHARS'),
- (Str("\n"), 'NEWLINE'),
- (Str('"""'), Method('end_string_action')),
- (Eof, 'EOF')
- ]),
-
- (Eof, Method('eof_action'))
- ],
-
- # FIXME: Plex 1.9 needs different args here from Plex 1.1.4
- #debug_flags = scanner_debug_flags,
- #debug_file = scanner_dump_file
- )
-
+ (punct | diphthong, TEXT),
+
+ (bra, Method('open_bracket_action')),
+ (ket, Method('close_bracket_action')),
+ (lineterm, Method('newline_action')),
+
+ (beginstring, Method('begin_string_action')),
+
+ (comment, IGNORE),
+ (spaces, IGNORE),
+ (escaped_newline, IGNORE),
+
+ State('INDENT', [
+ (comment + lineterm, Method('commentline')),
+ (Opt(spaces) + Opt(comment) + lineterm, IGNORE),
+ (indentation, Method('indentation_action')),
+ (Eof, Method('eof_action'))
+ ]),
+
+ State('SQ_STRING', [
+ (escapeseq, 'ESCAPE'),
+ (Rep1(AnyBut("'\"\n\\")), 'CHARS'),
+ (Str('"'), 'CHARS'),
+ (Str("\n"), Method('unclosed_string_action')),
+ (Str("'"), Method('end_string_action')),
+ (Eof, 'EOF')
+ ]),
+
+ State('DQ_STRING', [
+ (escapeseq, 'ESCAPE'),
+ (Rep1(AnyBut('"\n\\')), 'CHARS'),
+ (Str("'"), 'CHARS'),
+ (Str("\n"), Method('unclosed_string_action')),
+ (Str('"'), Method('end_string_action')),
+ (Eof, 'EOF')
+ ]),
+
+ State('TSQ_STRING', [
+ (escapeseq, 'ESCAPE'),
+ (Rep1(AnyBut("'\"\n\\")), 'CHARS'),
+ (Any("'\""), 'CHARS'),
+ (Str("\n"), 'NEWLINE'),
+ (Str("'''"), Method('end_string_action')),
+ (Eof, 'EOF')
+ ]),
+
+ State('TDQ_STRING', [
+ (escapeseq, 'ESCAPE'),
+ (Rep1(AnyBut('"\'\n\\')), 'CHARS'),
+ (Any("'\""), 'CHARS'),
+ (Str("\n"), 'NEWLINE'),
+ (Str('"""'), Method('end_string_action')),
+ (Eof, 'EOF')
+ ]),
+
+ (Eof, Method('eof_action'))
+ ],
+
+ # FIXME: Plex 1.9 needs different args here from Plex 1.1.4
+ #debug_flags = scanner_debug_flags,
+ #debug_file = scanner_dump_file
+ )
+