aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/tools/cython/Cython/Plex/Scanners.py
diff options
context:
space:
mode:
authorAnton Samokhvalov <pg83@yandex.ru>2022-02-10 16:45:15 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:45:15 +0300
commit72cb13b4aff9bc9cf22e49251bc8fd143f82538f (patch)
treeda2c34829458c7d4e74bdfbdf85dff449e9e7fb8 /contrib/tools/cython/Cython/Plex/Scanners.py
parent778e51ba091dc39e7b7fcab2b9cf4dbedfb6f2b5 (diff)
downloadydb-72cb13b4aff9bc9cf22e49251bc8fd143f82538f.tar.gz
Restoring authorship annotation for Anton Samokhvalov <pg83@yandex.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/tools/cython/Cython/Plex/Scanners.py')
-rw-r--r--contrib/tools/cython/Cython/Plex/Scanners.py108
1 files changed, 54 insertions, 54 deletions
diff --git a/contrib/tools/cython/Cython/Plex/Scanners.py b/contrib/tools/cython/Cython/Plex/Scanners.py
index 88f7e2da3b..ee5fea728e 100644
--- a/contrib/tools/cython/Cython/Plex/Scanners.py
+++ b/contrib/tools/cython/Cython/Plex/Scanners.py
@@ -1,57 +1,57 @@
# cython: auto_pickle=False
-#=======================================================================
-#
-# Python Lexical Analyser
-#
-#
-# Scanning an input stream
-#
-#=======================================================================
-
-from __future__ import absolute_import
-
-import cython
-
-cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object)
-
-from . import Errors
-from .Regexps import BOL, EOL, EOF
-
-NOT_FOUND = object()
-
-
-class Scanner(object):
+#=======================================================================
+#
+# Python Lexical Analyser
+#
+#
+# Scanning an input stream
+#
+#=======================================================================
+
+from __future__ import absolute_import
+
+import cython
+
+cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object)
+
+from . import Errors
+from .Regexps import BOL, EOL, EOF
+
+NOT_FOUND = object()
+
+
+class Scanner(object):
"""
A Scanner is used to read tokens from a stream of characters
using the token set specified by a Plex.Lexicon.
-
+
Constructor:
-
+
Scanner(lexicon, stream, name = '')
-
+
See the docstring of the __init__ method for details.
-
+
Methods:
-
+
See the docstrings of the individual methods for more
information.
-
+
read() --> (value, text)
Reads the next lexical token from the stream.
-
+
position() --> (name, line, col)
Returns the position of the last token read using the
read() method.
-
+
begin(state_name)
Causes scanner to change state.
-
+
produce(value [, text])
Causes return of a token value to the caller of the
Scanner.
-
+
"""
-
+
# lexicon = None # Lexicon
# stream = None # file-like object
# name = ''
@@ -69,22 +69,22 @@ class Scanner(object):
# state_name = '' # Name of initial state
# queue = None # list of tokens to be returned
# trace = 0
-
+
def __init__(self, lexicon, stream, name='', initial_pos=None):
"""
Scanner(lexicon, stream, name = '')
-
+
|lexicon| is a Plex.Lexicon instance specifying the lexical tokens
to be recognised.
-
+
|stream| can be a file object or anything which implements a
compatible read() method.
-
+
|name| is optional, and may be the name of the file being
scanned or any other identifying string.
"""
self.trace = 0
-
+
self.buffer = u''
self.buf_start_pos = 0
self.next_pos = 0
@@ -95,7 +95,7 @@ class Scanner(object):
self.start_col = 0
self.text = None
self.state_name = None
-
+
self.lexicon = lexicon
self.stream = stream
self.name = name
@@ -109,7 +109,7 @@ class Scanner(object):
self.input_state = 1
if initial_pos is not None:
self.cur_line, self.cur_line_start = initial_pos[1], -initial_pos[2]
-
+
def read(self):
"""
Read the next lexical token from the stream and return a
@@ -130,7 +130,7 @@ class Scanner(object):
result = queue[0]
del queue[0]
return result
-
+
def scan_a_token(self):
"""
Read the next input sequence recognised by the machine
@@ -156,7 +156,7 @@ class Scanner(object):
if self.cur_char is None or self.cur_char is EOF:
return (u'', None)
raise Errors.UnrecognizedInput(self, self.state_name)
-
+
def run_machine_inlined(self):
"""
Inlined version of run_machine for speed.
@@ -171,7 +171,7 @@ class Scanner(object):
buffer = self.buffer
buf_start_pos = self.buf_start_pos
buf_len = len(buffer)
- b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \
+ b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \
None, 0, 0, 0, u'', 0, 0
trace = self.trace
while 1:
@@ -267,7 +267,7 @@ class Scanner(object):
input_state = self.input_state
if self.trace:
print("Scanner: next: %s [%d] %d" % (" " * 20, input_state, self.cur_pos))
- if input_state == 1:
+ if input_state == 1:
self.cur_pos = self.next_pos
c = self.read_char()
if c == u'\n':
@@ -276,24 +276,24 @@ class Scanner(object):
elif not c:
self.cur_char = EOL
self.input_state = 4
- else:
+ else:
self.cur_char = c
- elif input_state == 2:
+ elif input_state == 2:
self.cur_char = u'\n'
self.input_state = 3
- elif input_state == 3:
+ elif input_state == 3:
self.cur_line += 1
self.cur_line_start = self.cur_pos = self.next_pos
self.cur_char = BOL
self.input_state = 1
- elif input_state == 4:
+ elif input_state == 4:
self.cur_char = EOF
self.input_state = 5
else: # input_state = 5
self.cur_char = u''
if self.trace:
print("--> [%d] %d %r" % (input_state, self.cur_pos, self.cur_char))
-
+
def position(self):
"""
Return a tuple (name, line, col) representing the location of
@@ -304,24 +304,24 @@ class Scanner(object):
(0-based).
"""
return (self.name, self.start_line, self.start_col)
-
+
def get_position(self):
"""Python accessible wrapper around position(), only for error reporting.
"""
return self.position()
-
+
def begin(self, state_name):
"""Set the current state of the scanner to the named state."""
self.initial_state = (
self.lexicon.get_initial_state(state_name))
self.state_name = state_name
-
+
def produce(self, value, text=None):
"""
Called from an action procedure, causes |value| to be returned
as the token value from read(). If |text| is supplied, it is
returned in place of the scanned text.
-
+
produce() can be called more than once during a single call to an action
procedure, in which case the tokens are queued up and returned one
at a time by subsequent calls to read(), until the queue is empty,
@@ -330,7 +330,7 @@ class Scanner(object):
if text is None:
text = self.text
self.queue.append((value, text))
-
+
def eof(self):
"""
Override this method if you want something to be done at