intermediate changes

ref:cde9a383711a11544ce7e107a78147fb96cc4029

1 files changed, 213 insertions, 0 deletions

diff --git a/contrib/python/Pygments/py2/pygments/token.py b/contrib/python/Pygments/py2/pygments/token.py
new file mode 100644
index 0000000000..5c30eb4665
--- /dev/null
+++ b/contrib/python/Pygments/py2/pygments/token.py
@@ -0,0 +1,213 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.token
+    ~~~~~~~~~~~~~~
+
+    Basic token types and the standard tokens.
+
+    :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+
+class _TokenType(tuple):
+    parent = None
+
+    def split(self):
+        buf = []
+        node = self
+        while node is not None:
+            buf.append(node)
+            node = node.parent
+        buf.reverse()
+        return buf
+
+    def __init__(self, *args):
+        # no need to call super.__init__
+        self.subtypes = set()
+
+    def __contains__(self, val):
+        return self is val or (
+            type(val) is self.__class__ and
+            val[:len(self)] == self
+        )
+
+    def __getattr__(self, val):
+        if not val or not val[0].isupper():
+            return tuple.__getattribute__(self, val)
+        new = _TokenType(self + (val,))
+        setattr(self, val, new)
+        self.subtypes.add(new)
+        new.parent = self
+        return new
+
+    def __repr__(self):
+        return 'Token' + (self and '.' or '') + '.'.join(self)
+
+    def __copy__(self):
+        # These instances are supposed to be singletons
+        return self
+
+    def __deepcopy__(self, memo):
+        # These instances are supposed to be singletons
+        return self
+
+
+Token = _TokenType()
+
+# Special token types
+Text = Token.Text
+Whitespace = Text.Whitespace
+Escape = Token.Escape
+Error = Token.Error
+# Text that doesn't belong to this lexer (e.g. HTML in PHP)
+Other = Token.Other
+
+# Common token types for source code
+Keyword = Token.Keyword
+Name = Token.Name
+Literal = Token.Literal
+String = Literal.String
+Number = Literal.Number
+Punctuation = Token.Punctuation
+Operator = Token.Operator
+Comment = Token.Comment
+
+# Generic types for non-source code
+Generic = Token.Generic
+
+# String and some others are not direct children of Token.
+# alias them:
+Token.Token = Token
+Token.String = String
+Token.Number = Number
+
+
+def is_token_subtype(ttype, other):
+    """
+    Return True if ``ttype`` is a subtype of ``other``.
+
+    exists for backwards compatibility. use ``ttype in other`` now.
+    """
+    return ttype in other
+
+
+def string_to_tokentype(s):
+    """
+    Convert a string into a token type::
+
+        >>> string_to_token('String.Double')
+        Token.Literal.String.Double
+        >>> string_to_token('Token.Literal.Number')
+        Token.Literal.Number
+        >>> string_to_token('')
+        Token
+
+    Tokens that are already tokens are returned unchanged:
+
+        >>> string_to_token(String)
+        Token.Literal.String
+    """
+    if isinstance(s, _TokenType):
+        return s
+    if not s:
+        return Token
+    node = Token
+    for item in s.split('.'):
+        node = getattr(node, item)
+    return node
+
+
+# Map standard token types to short names, used in CSS class naming.
+# If you add a new item, please be sure to run this file to perform
+# a consistency check for duplicate values.
+STANDARD_TYPES = {
+    Token:                         '',
+
+    Text:                          '',
+    Whitespace:                    'w',
+    Escape:                        'esc',
+    Error:                         'err',
+    Other:                         'x',
+
+    Keyword:                       'k',
+    Keyword.Constant:              'kc',
+    Keyword.Declaration:           'kd',
+    Keyword.Namespace:             'kn',
+    Keyword.Pseudo:                'kp',
+    Keyword.Reserved:              'kr',
+    Keyword.Type:                  'kt',
+
+    Name:                          'n',
+    Name.Attribute:                'na',
+    Name.Builtin:                  'nb',
+    Name.Builtin.Pseudo:           'bp',
+    Name.Class:                    'nc',
+    Name.Constant:                 'no',
+    Name.Decorator:                'nd',
+    Name.Entity:                   'ni',
+    Name.Exception:                'ne',
+    Name.Function:                 'nf',
+    Name.Function.Magic:           'fm',
+    Name.Property:                 'py',
+    Name.Label:                    'nl',
+    Name.Namespace:                'nn',
+    Name.Other:                    'nx',
+    Name.Tag:                      'nt',
+    Name.Variable:                 'nv',
+    Name.Variable.Class:           'vc',
+    Name.Variable.Global:          'vg',
+    Name.Variable.Instance:        'vi',
+    Name.Variable.Magic:           'vm',
+
+    Literal:                       'l',
+    Literal.Date:                  'ld',
+
+    String:                        's',
+    String.Affix:                  'sa',
+    String.Backtick:               'sb',
+    String.Char:                   'sc',
+    String.Delimiter:              'dl',
+    String.Doc:                    'sd',
+    String.Double:                 's2',
+    String.Escape:                 'se',
+    String.Heredoc:                'sh',
+    String.Interpol:               'si',
+    String.Other:                  'sx',
+    String.Regex:                  'sr',
+    String.Single:                 's1',
+    String.Symbol:                 'ss',
+
+    Number:                        'm',
+    Number.Bin:                    'mb',
+    Number.Float:                  'mf',
+    Number.Hex:                    'mh',
+    Number.Integer:                'mi',
+    Number.Integer.Long:           'il',
+    Number.Oct:                    'mo',
+
+    Operator:                      'o',
+    Operator.Word:                 'ow',
+
+    Punctuation:                   'p',
+
+    Comment:                       'c',
+    Comment.Hashbang:              'ch',
+    Comment.Multiline:             'cm',
+    Comment.Preproc:               'cp',
+    Comment.PreprocFile:           'cpf',
+    Comment.Single:                'c1',
+    Comment.Special:               'cs',
+
+    Generic:                       'g',
+    Generic.Deleted:               'gd',
+    Generic.Emph:                  'ge',
+    Generic.Error:                 'gr',
+    Generic.Heading:               'gh',
+    Generic.Inserted:              'gi',
+    Generic.Output:                'go',
+    Generic.Prompt:                'gp',
+    Generic.Strong:                'gs',
+    Generic.Subheading:            'gu',
+    Generic.Traceback:             'gt',
+}