diff options
author | Anton Samokhvalov <pg83@yandex.ru> | 2022-02-10 16:45:17 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:17 +0300 |
commit | d3a398281c6fd1d3672036cb2d63f842d2cb28c5 (patch) | |
tree | dd4bd3ca0f36b817e96812825ffaf10d645803f2 /contrib/tools/cython/Cython/Plex/Scanners.py | |
parent | 72cb13b4aff9bc9cf22e49251bc8fd143f82538f (diff) | |
download | ydb-d3a398281c6fd1d3672036cb2d63f842d2cb28c5.tar.gz |
Restoring authorship annotation for Anton Samokhvalov <pg83@yandex.ru>. Commit 2 of 2.
Diffstat (limited to 'contrib/tools/cython/Cython/Plex/Scanners.py')
-rw-r--r-- | contrib/tools/cython/Cython/Plex/Scanners.py | 108 |
1 files changed, 54 insertions, 54 deletions
diff --git a/contrib/tools/cython/Cython/Plex/Scanners.py b/contrib/tools/cython/Cython/Plex/Scanners.py index ee5fea728e..88f7e2da3b 100644 --- a/contrib/tools/cython/Cython/Plex/Scanners.py +++ b/contrib/tools/cython/Cython/Plex/Scanners.py @@ -1,57 +1,57 @@ # cython: auto_pickle=False -#======================================================================= -# -# Python Lexical Analyser -# -# -# Scanning an input stream -# -#======================================================================= - -from __future__ import absolute_import - -import cython - -cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object) - -from . import Errors -from .Regexps import BOL, EOL, EOF - -NOT_FOUND = object() - - -class Scanner(object): +#======================================================================= +# +# Python Lexical Analyser +# +# +# Scanning an input stream +# +#======================================================================= + +from __future__ import absolute_import + +import cython + +cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object) + +from . import Errors +from .Regexps import BOL, EOL, EOF + +NOT_FOUND = object() + + +class Scanner(object): """ A Scanner is used to read tokens from a stream of characters using the token set specified by a Plex.Lexicon. - + Constructor: - + Scanner(lexicon, stream, name = '') - + See the docstring of the __init__ method for details. - + Methods: - + See the docstrings of the individual methods for more information. - + read() --> (value, text) Reads the next lexical token from the stream. - + position() --> (name, line, col) Returns the position of the last token read using the read() method. - + begin(state_name) Causes scanner to change state. - + produce(value [, text]) Causes return of a token value to the caller of the Scanner. - + """ - + # lexicon = None # Lexicon # stream = None # file-like object # name = '' @@ -69,22 +69,22 @@ class Scanner(object): # state_name = '' # Name of initial state # queue = None # list of tokens to be returned # trace = 0 - + def __init__(self, lexicon, stream, name='', initial_pos=None): """ Scanner(lexicon, stream, name = '') - + |lexicon| is a Plex.Lexicon instance specifying the lexical tokens to be recognised. - + |stream| can be a file object or anything which implements a compatible read() method. - + |name| is optional, and may be the name of the file being scanned or any other identifying string. """ self.trace = 0 - + self.buffer = u'' self.buf_start_pos = 0 self.next_pos = 0 @@ -95,7 +95,7 @@ class Scanner(object): self.start_col = 0 self.text = None self.state_name = None - + self.lexicon = lexicon self.stream = stream self.name = name @@ -109,7 +109,7 @@ class Scanner(object): self.input_state = 1 if initial_pos is not None: self.cur_line, self.cur_line_start = initial_pos[1], -initial_pos[2] - + def read(self): """ Read the next lexical token from the stream and return a @@ -130,7 +130,7 @@ class Scanner(object): result = queue[0] del queue[0] return result - + def scan_a_token(self): """ Read the next input sequence recognised by the machine @@ -156,7 +156,7 @@ class Scanner(object): if self.cur_char is None or self.cur_char is EOF: return (u'', None) raise Errors.UnrecognizedInput(self, self.state_name) - + def run_machine_inlined(self): """ Inlined version of run_machine for speed. @@ -171,7 +171,7 @@ class Scanner(object): buffer = self.buffer buf_start_pos = self.buf_start_pos buf_len = len(buffer) - b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \ + b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \ None, 0, 0, 0, u'', 0, 0 trace = self.trace while 1: @@ -267,7 +267,7 @@ class Scanner(object): input_state = self.input_state if self.trace: print("Scanner: next: %s [%d] %d" % (" " * 20, input_state, self.cur_pos)) - if input_state == 1: + if input_state == 1: self.cur_pos = self.next_pos c = self.read_char() if c == u'\n': @@ -276,24 +276,24 @@ class Scanner(object): elif not c: self.cur_char = EOL self.input_state = 4 - else: + else: self.cur_char = c - elif input_state == 2: + elif input_state == 2: self.cur_char = u'\n' self.input_state = 3 - elif input_state == 3: + elif input_state == 3: self.cur_line += 1 self.cur_line_start = self.cur_pos = self.next_pos self.cur_char = BOL self.input_state = 1 - elif input_state == 4: + elif input_state == 4: self.cur_char = EOF self.input_state = 5 else: # input_state = 5 self.cur_char = u'' if self.trace: print("--> [%d] %d %r" % (input_state, self.cur_pos, self.cur_char)) - + def position(self): """ Return a tuple (name, line, col) representing the location of @@ -304,24 +304,24 @@ class Scanner(object): (0-based). """ return (self.name, self.start_line, self.start_col) - + def get_position(self): """Python accessible wrapper around position(), only for error reporting. """ return self.position() - + def begin(self, state_name): """Set the current state of the scanner to the named state.""" self.initial_state = ( self.lexicon.get_initial_state(state_name)) self.state_name = state_name - + def produce(self, value, text=None): """ Called from an action procedure, causes |value| to be returned as the token value from read(). If |text| is supplied, it is returned in place of the scanned text. - + produce() can be called more than once during a single call to an action procedure, in which case the tokens are queued up and returned one at a time by subsequent calls to read(), until the queue is empty, @@ -330,7 +330,7 @@ class Scanner(object): if text is None: text = self.text self.queue.append((value, text)) - + def eof(self): """ Override this method if you want something to be done at |