diff options
author | ilezhankin <ilezhankin@yandex-team.ru> | 2022-02-10 16:45:55 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:55 +0300 |
commit | 1d125034f06575234f83f24f08677955133f140e (patch) | |
tree | ec05fbbd61dc118d5de37f206ab978cff58774bd /contrib/python/Pygments/py3/pygments/scanner.py | |
parent | 3a7a498715ef1b66f5054455421b845e45e3a653 (diff) | |
download | ydb-1d125034f06575234f83f24f08677955133f140e.tar.gz |
Restoring authorship annotation for <ilezhankin@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/python/Pygments/py3/pygments/scanner.py')
-rw-r--r-- | contrib/python/Pygments/py3/pygments/scanner.py | 200 |
1 files changed, 100 insertions, 100 deletions
diff --git a/contrib/python/Pygments/py3/pygments/scanner.py b/contrib/python/Pygments/py3/pygments/scanner.py index 5f32a22c3c..a11b24bdcb 100644 --- a/contrib/python/Pygments/py3/pygments/scanner.py +++ b/contrib/python/Pygments/py3/pygments/scanner.py @@ -1,104 +1,104 @@ -""" - pygments.scanner - ~~~~~~~~~~~~~~~~ - - This library implements a regex based scanner. Some languages - like Pascal are easy to parse but have some keywords that - depend on the context. Because of this it's impossible to lex - that just by using a regular expression lexer like the - `RegexLexer`. - - Have a look at the `DelphiLexer` to get an idea of how to use - this scanner. - +""" + pygments.scanner + ~~~~~~~~~~~~~~~~ + + This library implements a regex based scanner. Some languages + like Pascal are easy to parse but have some keywords that + depend on the context. Because of this it's impossible to lex + that just by using a regular expression lexer like the + `RegexLexer`. + + Have a look at the `DelphiLexer` to get an idea of how to use + this scanner. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" -import re - - -class EndOfText(RuntimeError): - """ - Raise if end of text is reached and the user - tried to call a match function. - """ - - + :license: BSD, see LICENSE for details. +""" +import re + + +class EndOfText(RuntimeError): + """ + Raise if end of text is reached and the user + tried to call a match function. + """ + + class Scanner: - """ - Simple scanner - - All method patterns are regular expression strings (not - compiled expressions!) - """ - - def __init__(self, text, flags=0): - """ - :param text: The text which should be scanned - :param flags: default regular expression flags - """ - self.data = text - self.data_length = len(text) - self.start_pos = 0 - self.pos = 0 - self.flags = flags - self.last = None - self.match = None - self._re_cache = {} - - def eos(self): - """`True` if the scanner reached the end of text.""" - return self.pos >= self.data_length - eos = property(eos, eos.__doc__) - - def check(self, pattern): - """ - Apply `pattern` on the current position and return - the match object. (Doesn't touch pos). Use this for - lookahead. - """ - if self.eos: - raise EndOfText() - if pattern not in self._re_cache: - self._re_cache[pattern] = re.compile(pattern, self.flags) - return self._re_cache[pattern].match(self.data, self.pos) - - def test(self, pattern): - """Apply a pattern on the current position and check + """ + Simple scanner + + All method patterns are regular expression strings (not + compiled expressions!) + """ + + def __init__(self, text, flags=0): + """ + :param text: The text which should be scanned + :param flags: default regular expression flags + """ + self.data = text + self.data_length = len(text) + self.start_pos = 0 + self.pos = 0 + self.flags = flags + self.last = None + self.match = None + self._re_cache = {} + + def eos(self): + """`True` if the scanner reached the end of text.""" + return self.pos >= self.data_length + eos = property(eos, eos.__doc__) + + def check(self, pattern): + """ + Apply `pattern` on the current position and return + the match object. (Doesn't touch pos). Use this for + lookahead. + """ + if self.eos: + raise EndOfText() + if pattern not in self._re_cache: + self._re_cache[pattern] = re.compile(pattern, self.flags) + return self._re_cache[pattern].match(self.data, self.pos) + + def test(self, pattern): + """Apply a pattern on the current position and check if it patches. Doesn't touch pos. """ - return self.check(pattern) is not None - - def scan(self, pattern): - """ - Scan the text for the given pattern and update pos/match - and related fields. The return value is a boolen that - indicates if the pattern matched. The matched value is - stored on the instance as ``match``, the last value is - stored as ``last``. ``start_pos`` is the position of the - pointer before the pattern was matched, ``pos`` is the - end position. - """ - if self.eos: - raise EndOfText() - if pattern not in self._re_cache: - self._re_cache[pattern] = re.compile(pattern, self.flags) - self.last = self.match - m = self._re_cache[pattern].match(self.data, self.pos) - if m is None: - return False - self.start_pos = m.start() - self.pos = m.end() - self.match = m.group() - return True - - def get_char(self): - """Scan exactly one char.""" - self.scan('.') - - def __repr__(self): - return '<%s %d/%d>' % ( - self.__class__.__name__, - self.pos, - self.data_length - ) + return self.check(pattern) is not None + + def scan(self, pattern): + """ + Scan the text for the given pattern and update pos/match + and related fields. The return value is a boolen that + indicates if the pattern matched. The matched value is + stored on the instance as ``match``, the last value is + stored as ``last``. ``start_pos`` is the position of the + pointer before the pattern was matched, ``pos`` is the + end position. + """ + if self.eos: + raise EndOfText() + if pattern not in self._re_cache: + self._re_cache[pattern] = re.compile(pattern, self.flags) + self.last = self.match + m = self._re_cache[pattern].match(self.data, self.pos) + if m is None: + return False + self.start_pos = m.start() + self.pos = m.end() + self.match = m.group() + return True + + def get_char(self): + """Scan exactly one char.""" + self.scan('.') + + def __repr__(self): + return '<%s %d/%d>' % ( + self.__class__.__name__, + self.pos, + self.data_length + ) |