diff options
author | ilezhankin <ilezhankin@yandex-team.ru> | 2022-02-10 16:45:56 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:56 +0300 |
commit | 62a805381e41500fbc7914c37c71ab040a098f4e (patch) | |
tree | 1a2c5ffcf89eb53ecd79dbc9bc0a195c27404d0c /contrib/python/Pygments/py3/pygments/token.py | |
parent | 1d125034f06575234f83f24f08677955133f140e (diff) | |
download | ydb-62a805381e41500fbc7914c37c71ab040a098f4e.tar.gz |
Restoring authorship annotation for <ilezhankin@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'contrib/python/Pygments/py3/pygments/token.py')
-rw-r--r-- | contrib/python/Pygments/py3/pygments/token.py | 368 |
1 files changed, 184 insertions, 184 deletions
diff --git a/contrib/python/Pygments/py3/pygments/token.py b/contrib/python/Pygments/py3/pygments/token.py index 2fcb0f4466..9013acb709 100644 --- a/contrib/python/Pygments/py3/pygments/token.py +++ b/contrib/python/Pygments/py3/pygments/token.py @@ -1,212 +1,212 @@ -""" - pygments.token - ~~~~~~~~~~~~~~ - - Basic token types and the standard tokens. - +""" + pygments.token + ~~~~~~~~~~~~~~ + + Basic token types and the standard tokens. + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - - -class _TokenType(tuple): - parent = None - - def split(self): - buf = [] - node = self - while node is not None: - buf.append(node) - node = node.parent - buf.reverse() - return buf - - def __init__(self, *args): - # no need to call super.__init__ - self.subtypes = set() - - def __contains__(self, val): - return self is val or ( - type(val) is self.__class__ and - val[:len(self)] == self - ) - - def __getattr__(self, val): - if not val or not val[0].isupper(): - return tuple.__getattribute__(self, val) - new = _TokenType(self + (val,)) - setattr(self, val, new) - self.subtypes.add(new) - new.parent = self - return new - - def __repr__(self): - return 'Token' + (self and '.' or '') + '.'.join(self) - + :license: BSD, see LICENSE for details. +""" + + +class _TokenType(tuple): + parent = None + + def split(self): + buf = [] + node = self + while node is not None: + buf.append(node) + node = node.parent + buf.reverse() + return buf + + def __init__(self, *args): + # no need to call super.__init__ + self.subtypes = set() + + def __contains__(self, val): + return self is val or ( + type(val) is self.__class__ and + val[:len(self)] == self + ) + + def __getattr__(self, val): + if not val or not val[0].isupper(): + return tuple.__getattribute__(self, val) + new = _TokenType(self + (val,)) + setattr(self, val, new) + self.subtypes.add(new) + new.parent = self + return new + + def __repr__(self): + return 'Token' + (self and '.' or '') + '.'.join(self) + def __copy__(self): # These instances are supposed to be singletons return self - + def __deepcopy__(self, memo): # These instances are supposed to be singletons return self - + Token = _TokenType() -# Special token types +# Special token types Text = Token.Text Whitespace = Text.Whitespace Escape = Token.Escape Error = Token.Error -# Text that doesn't belong to this lexer (e.g. HTML in PHP) +# Text that doesn't belong to this lexer (e.g. HTML in PHP) Other = Token.Other - -# Common token types for source code + +# Common token types for source code Keyword = Token.Keyword Name = Token.Name Literal = Token.Literal String = Literal.String Number = Literal.Number -Punctuation = Token.Punctuation +Punctuation = Token.Punctuation Operator = Token.Operator Comment = Token.Comment - -# Generic types for non-source code + +# Generic types for non-source code Generic = Token.Generic - + # String and some others are not direct children of Token. -# alias them: -Token.Token = Token -Token.String = String -Token.Number = Number - - -def is_token_subtype(ttype, other): - """ - Return True if ``ttype`` is a subtype of ``other``. - - exists for backwards compatibility. use ``ttype in other`` now. - """ - return ttype in other - - -def string_to_tokentype(s): - """ - Convert a string into a token type:: - - >>> string_to_token('String.Double') - Token.Literal.String.Double - >>> string_to_token('Token.Literal.Number') - Token.Literal.Number - >>> string_to_token('') - Token - - Tokens that are already tokens are returned unchanged: - - >>> string_to_token(String) - Token.Literal.String - """ - if isinstance(s, _TokenType): - return s - if not s: - return Token - node = Token - for item in s.split('.'): - node = getattr(node, item) - return node - - -# Map standard token types to short names, used in CSS class naming. -# If you add a new item, please be sure to run this file to perform -# a consistency check for duplicate values. -STANDARD_TYPES = { - Token: '', - - Text: '', - Whitespace: 'w', - Escape: 'esc', - Error: 'err', - Other: 'x', - - Keyword: 'k', - Keyword.Constant: 'kc', - Keyword.Declaration: 'kd', - Keyword.Namespace: 'kn', - Keyword.Pseudo: 'kp', - Keyword.Reserved: 'kr', - Keyword.Type: 'kt', - - Name: 'n', - Name.Attribute: 'na', - Name.Builtin: 'nb', - Name.Builtin.Pseudo: 'bp', - Name.Class: 'nc', - Name.Constant: 'no', - Name.Decorator: 'nd', - Name.Entity: 'ni', - Name.Exception: 'ne', - Name.Function: 'nf', +# alias them: +Token.Token = Token +Token.String = String +Token.Number = Number + + +def is_token_subtype(ttype, other): + """ + Return True if ``ttype`` is a subtype of ``other``. + + exists for backwards compatibility. use ``ttype in other`` now. + """ + return ttype in other + + +def string_to_tokentype(s): + """ + Convert a string into a token type:: + + >>> string_to_token('String.Double') + Token.Literal.String.Double + >>> string_to_token('Token.Literal.Number') + Token.Literal.Number + >>> string_to_token('') + Token + + Tokens that are already tokens are returned unchanged: + + >>> string_to_token(String) + Token.Literal.String + """ + if isinstance(s, _TokenType): + return s + if not s: + return Token + node = Token + for item in s.split('.'): + node = getattr(node, item) + return node + + +# Map standard token types to short names, used in CSS class naming. +# If you add a new item, please be sure to run this file to perform +# a consistency check for duplicate values. +STANDARD_TYPES = { + Token: '', + + Text: '', + Whitespace: 'w', + Escape: 'esc', + Error: 'err', + Other: 'x', + + Keyword: 'k', + Keyword.Constant: 'kc', + Keyword.Declaration: 'kd', + Keyword.Namespace: 'kn', + Keyword.Pseudo: 'kp', + Keyword.Reserved: 'kr', + Keyword.Type: 'kt', + + Name: 'n', + Name.Attribute: 'na', + Name.Builtin: 'nb', + Name.Builtin.Pseudo: 'bp', + Name.Class: 'nc', + Name.Constant: 'no', + Name.Decorator: 'nd', + Name.Entity: 'ni', + Name.Exception: 'ne', + Name.Function: 'nf', Name.Function.Magic: 'fm', - Name.Property: 'py', - Name.Label: 'nl', - Name.Namespace: 'nn', - Name.Other: 'nx', - Name.Tag: 'nt', - Name.Variable: 'nv', - Name.Variable.Class: 'vc', - Name.Variable.Global: 'vg', - Name.Variable.Instance: 'vi', + Name.Property: 'py', + Name.Label: 'nl', + Name.Namespace: 'nn', + Name.Other: 'nx', + Name.Tag: 'nt', + Name.Variable: 'nv', + Name.Variable.Class: 'vc', + Name.Variable.Global: 'vg', + Name.Variable.Instance: 'vi', Name.Variable.Magic: 'vm', - - Literal: 'l', - Literal.Date: 'ld', - - String: 's', + + Literal: 'l', + Literal.Date: 'ld', + + String: 's', String.Affix: 'sa', - String.Backtick: 'sb', - String.Char: 'sc', + String.Backtick: 'sb', + String.Char: 'sc', String.Delimiter: 'dl', - String.Doc: 'sd', - String.Double: 's2', - String.Escape: 'se', - String.Heredoc: 'sh', - String.Interpol: 'si', - String.Other: 'sx', - String.Regex: 'sr', - String.Single: 's1', - String.Symbol: 'ss', - - Number: 'm', - Number.Bin: 'mb', - Number.Float: 'mf', - Number.Hex: 'mh', - Number.Integer: 'mi', - Number.Integer.Long: 'il', - Number.Oct: 'mo', - - Operator: 'o', - Operator.Word: 'ow', - - Punctuation: 'p', - - Comment: 'c', - Comment.Hashbang: 'ch', - Comment.Multiline: 'cm', - Comment.Preproc: 'cp', - Comment.PreprocFile: 'cpf', - Comment.Single: 'c1', - Comment.Special: 'cs', - - Generic: 'g', - Generic.Deleted: 'gd', - Generic.Emph: 'ge', - Generic.Error: 'gr', - Generic.Heading: 'gh', - Generic.Inserted: 'gi', - Generic.Output: 'go', - Generic.Prompt: 'gp', - Generic.Strong: 'gs', - Generic.Subheading: 'gu', - Generic.Traceback: 'gt', -} + String.Doc: 'sd', + String.Double: 's2', + String.Escape: 'se', + String.Heredoc: 'sh', + String.Interpol: 'si', + String.Other: 'sx', + String.Regex: 'sr', + String.Single: 's1', + String.Symbol: 'ss', + + Number: 'm', + Number.Bin: 'mb', + Number.Float: 'mf', + Number.Hex: 'mh', + Number.Integer: 'mi', + Number.Integer.Long: 'il', + Number.Oct: 'mo', + + Operator: 'o', + Operator.Word: 'ow', + + Punctuation: 'p', + + Comment: 'c', + Comment.Hashbang: 'ch', + Comment.Multiline: 'cm', + Comment.Preproc: 'cp', + Comment.PreprocFile: 'cpf', + Comment.Single: 'c1', + Comment.Special: 'cs', + + Generic: 'g', + Generic.Deleted: 'gd', + Generic.Emph: 'ge', + Generic.Error: 'gr', + Generic.Heading: 'gh', + Generic.Inserted: 'gi', + Generic.Output: 'go', + Generic.Prompt: 'gp', + Generic.Strong: 'gs', + Generic.Subheading: 'gu', + Generic.Traceback: 'gt', +} |