intermediate changes

ref:2d4f292087954c9344efdabb7b2a67f466263c65

1 files changed, 149 insertions, 136 deletions

diff --git a/contrib/python/Pygments/py3/pygments/lexers/jvm.py b/contrib/python/Pygments/py3/pygments/lexers/jvm.py
index 4ffc5c7fdf7..d053e3732e4 100644
--- a/contrib/python/Pygments/py3/pygments/lexers/jvm.py
+++ b/contrib/python/Pygments/py3/pygments/lexers/jvm.py
@@ -4,7 +4,7 @@
 
     Pygments lexers for JVM languages.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -13,7 +13,7 @@ import re
 from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \
     this, combined, default, words
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
-    Number, Punctuation
+    Number, Punctuation, Whitespace
 from pygments.util import shebang_matches
 from pygments import unistring as uni
 
@@ -25,22 +25,23 @@ __all__ = ['JavaLexer', 'ScalaLexer', 'GosuLexer', 'GosuTemplateLexer',
 
 class JavaLexer(RegexLexer):
     """
-    For `Java <https://www.oracle.com/technetwork/java/>`_ source code.
+    For Java source code.
     """
 
     name = 'Java'
+    url = 'https://www.oracle.com/technetwork/java/'
     aliases = ['java']
     filenames = ['*.java']
     mimetypes = ['text/x-java']
 
-    flags = re.MULTILINE | re.DOTALL | re.UNICODE
+    flags = re.MULTILINE | re.DOTALL
 
     tokens = {
         'root': [
             (r'(^\s*)((?:(?:public|private|protected|static|strictfp)(?:\s+))*)(record)\b',
-             bygroups(Text, using(this), Keyword.Declaration), 'class'),
-            (r'[^\S\n]+', Text),
-            (r'//.*?\n', Comment.Single),
+             bygroups(Whitespace, using(this), Keyword.Declaration), 'class'),
+            (r'[^\S\n]+', Whitespace),
+            (r'(//.*?)(\n)', bygroups(Comment.Single, Whitespace)),
             (r'/\*.*?\*/', Comment.Multiline),
             # keywords: go before method names to avoid lexing "throw new XYZ"
             # as a method signature
@@ -51,26 +52,25 @@ class JavaLexer(RegexLexer):
             (r'((?:(?:[^\W\d]|\$)[\w.\[\]$<>]*\s+)+?)'  # return arguments
              r'((?:[^\W\d]|\$)[\w$]*)'                  # method name
              r'(\s*)(\()',                              # signature start
-             bygroups(using(this), Name.Function, Text, Punctuation)),
+             bygroups(using(this), Name.Function, Whitespace, Punctuation)),
             (r'@[^\W\d][\w.]*', Name.Decorator),
             (r'(abstract|const|enum|extends|final|implements|native|private|'
              r'protected|public|sealed|static|strictfp|super|synchronized|throws|'
              r'transient|volatile|yield)\b', Keyword.Declaration),
             (r'(boolean|byte|char|double|float|int|long|short|void)\b',
              Keyword.Type),
-            (r'(package)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
+            (r'(package)(\s+)', bygroups(Keyword.Namespace, Whitespace), 'import'),
             (r'(true|false|null)\b', Keyword.Constant),
             (r'(class|interface)\b', Keyword.Declaration, 'class'),
-            (r'(var)(\s+)', bygroups(Keyword.Declaration, Text),
-             'var'),
-            (r'(import(?:\s+static)?)(\s+)', bygroups(Keyword.Namespace, Text),
+            (r'(var)(\s+)', bygroups(Keyword.Declaration, Whitespace), 'var'),
+            (r'(import(?:\s+static)?)(\s+)', bygroups(Keyword.Namespace, Whitespace),
              'import'),
             (r'"', String, 'string'),
             (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char),
             (r'(\.)((?:[^\W\d]|\$)[\w$]*)', bygroups(Punctuation,
                                                      Name.Attribute)),
-            (r'^(\s*)(default)(:)', bygroups(Text, Keyword, Punctuation)),
-            (r'^(\s*)((?:[^\W\d]|\$)[\w$]*)(:)', bygroups(Text, Name.Label,
+            (r'^(\s*)(default)(:)', bygroups(Whitespace, Keyword, Punctuation)),
+            (r'^(\s*)((?:[^\W\d]|\$)[\w$]*)(:)', bygroups(Whitespace, Name.Label,
                                                           Punctuation)),
             (r'([^\W\d]|\$)[\w$]*', Name),
             (r'([0-9][0-9_]*\.([0-9][0-9_]*)?|'
@@ -87,7 +87,7 @@ class JavaLexer(RegexLexer):
             (r'0|[1-9][0-9_]*[lL]?', Number.Integer),
             (r'[~^*!%&\[\]<>|+=/?-]', Operator),
             (r'[{}();:.,]', Punctuation),
-            (r'\n', Text)
+            (r'\n', Whitespace)
         ],
         'class': [
             (r'\s+', Text),
@@ -111,12 +111,13 @@ class JavaLexer(RegexLexer):
 
 class AspectJLexer(JavaLexer):
     """
-    For `AspectJ <http://www.eclipse.org/aspectj/>`_ source code.
+    For AspectJ source code.
 
     .. versionadded:: 1.6
     """
 
     name = 'AspectJ'
+    url = 'http://www.eclipse.org/aspectj/'
     aliases = ['aspectj']
     filenames = ['*.aj']
     mimetypes = ['text/x-aspectj']
@@ -150,10 +151,11 @@ class AspectJLexer(JavaLexer):
 
 class ScalaLexer(RegexLexer):
     """
-    For `Scala <http://www.scala-lang.org>`_ source code.
+    For Scala source code.
     """
 
     name = 'Scala'
+    url = 'http://www.scala-lang.org'
     aliases = ['scala']
     filenames = ['*.scala']
     mimetypes = ['text/x-scala']
@@ -221,7 +223,7 @@ class ScalaLexer(RegexLexer):
 
         # Includes:
         'whitespace': [
-            (r'\s+', Text),
+            (r'\s+', Whitespace),
         ],
         'comments': [
             (r'//.*?\n', Comment.Single),
@@ -231,12 +233,12 @@ class ScalaLexer(RegexLexer):
             (r'^#!([^\n]*)$', Comment.Hashbang),
         ],
         'imports': [
-            (r'\b(import)(\s+)', bygroups(Keyword, Text), 'import-path'),
+            (r'\b(import)(\s+)', bygroups(Keyword, Whitespace), 'import-path'),
         ],
         'exports': [
             (r'\b(export)(\s+)(given)(\s+)',
-                bygroups(Keyword, Text, Keyword, Text), 'export-path'),
-            (r'\b(export)(\s+)', bygroups(Keyword, Text), 'export-path'),
+                bygroups(Keyword, Whitespace, Keyword, Whitespace), 'export-path'),
+            (r'\b(export)(\s+)', bygroups(Keyword, Whitespace), 'export-path'),
         ],
         'storage-modifiers': [
             (words(storage_modifiers, prefix=r'\b', suffix=r'\b'), Keyword),
@@ -253,56 +255,56 @@ class ScalaLexer(RegexLexer):
         'using': [
             # using is a soft keyword, can only be used in the first position of
             # a parameter or argument list.
-            (r'(\()(\s*)(using)(\s)', bygroups(Punctuation, Text, Keyword, Text)),
+            (r'(\()(\s*)(using)(\s)', bygroups(Punctuation, Whitespace, Keyword, Whitespace)),
         ],
         'declarations': [
             (r'\b(def)\b(\s*)%s(%s)?' % (notStartOfComment, anyId),
-             bygroups(Keyword, Text, Name.Function)),
+             bygroups(Keyword, Whitespace, Name.Function)),
             (r'\b(trait)\b(\s*)%s(%s)?' % (notStartOfComment, anyId),
-                bygroups(Keyword, Text, Name.Class)),
+                bygroups(Keyword, Whitespace, Name.Class)),
             (r'\b(?:(case)(\s+))?(class|object|enum)\b(\s*)%s(%s)?' %
                 (notStartOfComment, anyId),
-                bygroups(Keyword, Text, Keyword, Text, Name.Class)),
+                bygroups(Keyword, Whitespace, Keyword, Whitespace, Name.Class)),
             (r'(?<!\.)\b(type)\b(\s*)%s(%s)?' % (notStartOfComment, anyId),
-                bygroups(Keyword, Text, Name.Class)),
+                bygroups(Keyword, Whitespace, Name.Class)),
             (r'\b(val|var)\b', Keyword.Declaration),
             (r'\b(package)(\s+)(object)\b(\s*)%s(%s)?' %
                 (notStartOfComment, anyId),
-                bygroups(Keyword, Text, Keyword, Text, Name.Namespace)),
-            (r'\b(package)(\s+)', bygroups(Keyword, Text), 'package'),
+                bygroups(Keyword, Whitespace, Keyword, Whitespace, Name.Namespace)),
+            (r'\b(package)(\s+)', bygroups(Keyword, Whitespace), 'package'),
             (r'\b(given)\b(\s*)(%s)' % idUpper,
-                bygroups(Keyword, Text, Name.Class)),
+                bygroups(Keyword, Whitespace, Name.Class)),
             (r'\b(given)\b(\s*)(%s)?' % anyId, 
-                bygroups(Keyword, Text, Name)),
+                bygroups(Keyword, Whitespace, Name)),
         ],
         'inheritance': [
             (r'\b(extends|with|derives)\b(\s*)'
              r'(%s|%s|(?=\([^\)]+=>)|(?=%s)|(?="))?' %
                 (idUpper, backQuotedId, plainid),
-                bygroups(Keyword, Text, Name.Class)),
+                bygroups(Keyword, Whitespace, Name.Class)),
         ],
         'extension': [
-            (r'\b(extension)(\s+)(?=[\[\(])', bygroups(Keyword, Text)),
+            (r'\b(extension)(\s+)(?=[\[\(])', bygroups(Keyword, Whitespace)),
         ],
         'end': [
             # end is a soft keyword, should only be highlighted in certain cases
             (r'\b(end)(\s+)(if|while|for|match|new|extension|val|var)\b',
-                bygroups(Keyword, Text, Keyword)),
+                bygroups(Keyword, Whitespace, Keyword)),
             (r'\b(end)(\s+)(%s)%s' % (idUpper, endOfLineMaybeWithComment),
-                bygroups(Keyword, Text, Name.Class)),
+                bygroups(Keyword, Whitespace, Name.Class)),
             (r'\b(end)(\s+)(%s|%s)?%s' %
                 (backQuotedId, plainid, endOfLineMaybeWithComment),
-                bygroups(Keyword, Text, Name.Namespace)),
+                bygroups(Keyword, Whitespace, Name.Namespace)),
         ],
         'punctuation': [
             (r'[{}()\[\];,.]', Punctuation),
-            (r'(?<!:):(?!:)', Punctuation),  
+            (r'(?<!:):(?!:)', Punctuation),
         ],
         'keywords': [
             (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword),
         ],
         'operators': [
-            (r'(%s{2,})(\s+)' % opchar, bygroups(Operator, Text)),
+            (r'(%s{2,})(\s+)' % opchar, bygroups(Operator, Whitespace)),
             (r'/(?![/*])', Operator),
             (words(operators), Operator),
             (r'(?<!%s)(!|&&|\|\|)(?!%s)' % (opchar, opchar), Operator),
@@ -334,7 +336,7 @@ class ScalaLexer(RegexLexer):
             (r'(\.)(type)\b', bygroups(Punctuation, Keyword)),
         ],
         'inline': [
-            # inline is a soft modifer, only highlighted if followed by if, 
+            # inline is a soft modifier, only highlighted if followed by if,
             # match or parameters.
             (r'\b(inline)(?=\s+(%s|%s)\s*:)' % (plainid, backQuotedId),
                 Keyword),
@@ -422,7 +424,7 @@ class ScalaLexer(RegexLexer):
             (r'(%s)(\.)' % anyId, bygroups(Name.Namespace, Punctuation)),
             (r'\.', Punctuation),
             (anyId, Name),
-            (r'[^\S\n]+', Text),
+            (r'[^\S\n]+', Whitespace),
         ],
         'interpolated-string-common': [
             (r'[^"$\\]+', String),
@@ -455,8 +457,8 @@ class GosuLexer(RegexLexer):
             (r'^(\s*(?:[a-zA-Z_][\w.\[\]]*\s+)+?)'  # modifiers etc.
              r'([a-zA-Z_]\w*)'                       # method name
              r'(\s*)(\()',                           # signature start
-             bygroups(using(this), Name.Function, Text, Operator)),
-            (r'[^\S\n]+', Text),
+             bygroups(using(this), Name.Function, Whitespace, Operator)),
+            (r'[^\S\n]+', Whitespace),
             (r'//.*?\n', Comment.Single),
             (r'/\*.*?\*/', Comment.Multiline),
             (r'@[a-zA-Z_][\w.]*', Name.Decorator),
@@ -467,15 +469,15 @@ class GosuLexer(RegexLexer):
             (r'(var|delegate|construct|function|private|internal|protected|'
              r'public|abstract|override|final|static|extends|transient|'
              r'implements|represents|readonly)\b', Keyword.Declaration),
-            (r'(property\s+)(get|set)?', Keyword.Declaration),
+            (r'(property)(\s+)(get|set)?', bygroups(Keyword.Declaration, Whitespace, Keyword.Declaration)),
             (r'(boolean|byte|char|double|float|int|long|short|void|block)\b',
              Keyword.Type),
-            (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)),
+            (r'(package)(\s+)', bygroups(Keyword.Namespace, Whitespace)),
             (r'(true|false|null|NaN|Infinity)\b', Keyword.Constant),
             (r'(class|interface|enhancement|enum)(\s+)([a-zA-Z_]\w*)',
-             bygroups(Keyword.Declaration, Text, Name.Class)),
+             bygroups(Keyword.Declaration, Whitespace, Name.Class)),
             (r'(uses)(\s+)([\w.]+\*?)',
-             bygroups(Keyword.Namespace, Text, Name.Namespace)),
+             bygroups(Keyword.Namespace, Whitespace, Name.Namespace)),
             (r'"', String, 'string'),
             (r'(\??[.#])([a-zA-Z_]\w*)',
              bygroups(Operator, Name.Attribute)),
@@ -485,7 +487,7 @@ class GosuLexer(RegexLexer):
             (r'and|or|not|[\\~^*!%&\[\](){}<>|+=:;,./?-]', Operator),
             (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
             (r'[0-9]+', Number.Integer),
-            (r'\n', Text)
+            (r'\n', Whitespace)
         ],
         'templateText': [
             (r'(\\<)|(\\\$)', String),
@@ -534,12 +536,13 @@ class GosuTemplateLexer(Lexer):
 
 class GroovyLexer(RegexLexer):
     """
-    For `Groovy <http://groovy.codehaus.org/>`_ source code.
+    For Groovy source code.
 
     .. versionadded:: 1.5
     """
 
     name = 'Groovy'
+    url = 'https://groovy-lang.org/'
     aliases = ['groovy']
     filenames = ['*.groovy','*.gradle']
     mimetypes = ['text/x-groovy']
@@ -553,8 +556,8 @@ class GroovyLexer(RegexLexer):
             default('base'),
         ],
         'base': [
-            (r'[^\S\n]+', Text),
-            (r'//.*?\n', Comment.Single),
+            (r'[^\S\n]+', Whitespace),
+            (r'(//.*?)(\n)', bygroups(Comment.Single, Whitespace)),
             (r'/\*.*?\*/', Comment.Multiline),
             # keywords: go before method names to avoid lexing "throw new XYZ"
             # as a method signature
@@ -569,18 +572,18 @@ class GroovyLexer(RegexLexer):
              r"|'(?:\\\\|\\[^\\]|[^'\\])*'"         # or single-quoted method name
              r')'
              r'(\s*)(\()',                          # signature start
-             bygroups(using(this), Name.Function, Text, Operator)),
+             bygroups(using(this), Name.Function, Whitespace, Operator)),
             (r'@[a-zA-Z_][\w.]*', Name.Decorator),
             (r'(abstract|const|enum|extends|final|implements|native|private|'
              r'protected|public|static|strictfp|super|synchronized|throws|'
              r'transient|volatile)\b', Keyword.Declaration),
             (r'(def|boolean|byte|char|double|float|int|long|short|void)\b',
              Keyword.Type),
-            (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)),
+            (r'(package)(\s+)', bygroups(Keyword.Namespace, Whitespace)),
             (r'(true|false|null)\b', Keyword.Constant),
-            (r'(class|interface)(\s+)', bygroups(Keyword.Declaration, Text),
+            (r'(class|interface)(\s+)', bygroups(Keyword.Declaration, Whitespace),
              'class'),
-            (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
+            (r'(import)(\s+)', bygroups(Keyword.Namespace, Whitespace), 'import'),
             (r'""".*?"""', String.Double),
             (r"'''.*?'''", String.Single),
             (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
@@ -595,7 +598,7 @@ class GroovyLexer(RegexLexer):
             (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
             (r'0x[0-9a-fA-F]+', Number.Hex),
             (r'[0-9]+L?', Number.Integer),
-            (r'\n', Text)
+            (r'\n', Whitespace)
         ],
         'class': [
             (r'[a-zA-Z_]\w*', Name.Class, '#pop')
@@ -611,12 +614,13 @@ class GroovyLexer(RegexLexer):
 
 class IokeLexer(RegexLexer):
     """
-    For `Ioke <http://ioke.org/>`_ (a strongly typed, dynamic,
+    For Ioke (a strongly typed, dynamic,
     prototype based programming language) source.
 
     .. versionadded:: 1.4
     """
     name = 'Ioke'
+    url = 'https://ioke.org/'
     filenames = ['*.ik']
     aliases = ['ioke', 'ik']
     mimetypes = ['text/x-iokesrc']
@@ -665,8 +669,8 @@ class IokeLexer(RegexLexer):
         ],
 
         'root': [
-            (r'\n', Text),
-            (r'\s+', Text),
+            (r'\n', Whitespace),
+            (r'\s+', Whitespace),
 
             # Comments
             (r';(.*?)\n', Comment),
@@ -684,7 +688,7 @@ class IokeLexer(RegexLexer):
             # Documentation
             (r'((?<=fn\()|(?<=fnx\()|(?<=method\()|(?<=macro\()|(?<=lecro\()'
              r'|(?<=syntax\()|(?<=dmacro\()|(?<=dlecro\()|(?<=dlecrox\()'
-             r'|(?<=dsyntax\())\s*"', String.Doc, 'documentation'),
+             r'|(?<=dsyntax\())(\s*)"', String.Doc, 'documentation'),
 
             # Text
             (r'"', String, 'text'),
@@ -802,13 +806,14 @@ class IokeLexer(RegexLexer):
 
 class ClojureLexer(RegexLexer):
     """
-    Lexer for `Clojure <http://clojure.org/>`_ source code.
+    Lexer for Clojure source code.
 
     .. versionadded:: 0.11
     """
     name = 'Clojure'
+    url = 'http://clojure.org/'
     aliases = ['clojure', 'clj']
-    filenames = ['*.clj']
+    filenames = ['*.clj', '*.cljc']
     mimetypes = ['text/x-clojure', 'application/x-clojure']
 
     special_forms = (
@@ -889,10 +894,12 @@ class ClojureLexer(RegexLexer):
             (r';.*$', Comment.Single),
 
             # whitespaces - usually not relevant
-            (r'[,\s]+', Text),
+            (r',+', Text),
+            (r'\s+', Whitespace),
 
             # numbers
             (r'-?\d+\.\d+', Number.Float),
+            (r'-?\d+/\d+', Number),
             (r'-?\d+', Number.Integer),
             (r'0x-?[abcdef\d]+', Number.Hex),
 
@@ -940,12 +947,12 @@ class ClojureLexer(RegexLexer):
 
 class ClojureScriptLexer(ClojureLexer):
     """
-    Lexer for `ClojureScript <http://clojure.org/clojurescript>`_
-    source code.
+    Lexer for ClojureScript source code.
 
     .. versionadded:: 2.0
     """
     name = 'ClojureScript'
+    url = 'http://clojure.org/clojurescript'
     aliases = ['clojurescript', 'cljs']
     filenames = ['*.cljs']
     mimetypes = ['text/x-clojurescript', 'application/x-clojurescript']
@@ -953,7 +960,7 @@ class ClojureScriptLexer(ClojureLexer):
 
 class TeaLangLexer(RegexLexer):
     """
-    For `Tea <http://teatrove.org/>`_ source code. Only used within a
+    For Tea source code. Only used within a
     TeaTemplateLexer.
 
     .. versionadded:: 1.5
@@ -967,17 +974,17 @@ class TeaLangLexer(RegexLexer):
             (r'^(\s*(?:[a-zA-Z_][\w\.\[\]]*\s+)+?)'  # return arguments
              r'([a-zA-Z_]\w*)'                       # method name
              r'(\s*)(\()',                           # signature start
-             bygroups(using(this), Name.Function, Text, Operator)),
-            (r'[^\S\n]+', Text),
-            (r'//.*?\n', Comment.Single),
+             bygroups(using(this), Name.Function, Whitespace, Operator)),
+            (r'[^\S\n]+', Whitespace),
+            (r'(//.*?)(\n)', bygroups(Comment.Single, Whitespace)),
             (r'/\*.*?\*/', Comment.Multiline),
             (r'@[a-zA-Z_][\w\.]*', Name.Decorator),
             (r'(and|break|else|foreach|if|in|not|or|reverse)\b',
              Keyword),
             (r'(as|call|define)\b', Keyword.Declaration),
             (r'(true|false|null)\b', Keyword.Constant),
-            (r'(template)(\s+)', bygroups(Keyword.Declaration, Text), 'template'),
-            (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
+            (r'(template)(\s+)', bygroups(Keyword.Declaration, Whitespace), 'template'),
+            (r'(import)(\s+)', bygroups(Keyword.Namespace, Whitespace), 'import'),
             (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
             (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
             (r'(\.)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)),
@@ -987,7 +994,7 @@ class TeaLangLexer(RegexLexer):
             (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
             (r'0x[0-9a-fA-F]+', Number.Hex),
             (r'[0-9]+L?', Number.Integer),
-            (r'\n', Text)
+            (r'\n', Whitespace)
         ],
         'template': [
             (r'[a-zA-Z_]\w*', Name.Class, '#pop')
@@ -1000,12 +1007,13 @@ class TeaLangLexer(RegexLexer):
 
 class CeylonLexer(RegexLexer):
     """
-    For `Ceylon <http://ceylon-lang.org/>`_ source code.
+    For Ceylon source code.
 
     .. versionadded:: 1.6
     """
 
     name = 'Ceylon'
+    url = 'http://ceylon-lang.org/'
     aliases = ['ceylon']
     filenames = ['*.ceylon']
     mimetypes = ['text/x-ceylon']
@@ -1021,9 +1029,9 @@ class CeylonLexer(RegexLexer):
             (r'^(\s*(?:[a-zA-Z_][\w.\[\]]*\s+)+?)'  # return arguments
              r'([a-zA-Z_]\w*)'                      # method name
              r'(\s*)(\()',                          # signature start
-             bygroups(using(this), Name.Function, Text, Operator)),
-            (r'[^\S\n]+', Text),
-            (r'//.*?\n', Comment.Single),
+             bygroups(using(this), Name.Function, Whitespace, Operator)),
+            (r'[^\S\n]+', Whitespace),
+            (r'(//.*?)(\n)', bygroups(Comment.Single, Whitespace)),
             (r'/\*', Comment.Multiline, 'comment'),
             (r'(shared|abstract|formal|default|actual|variable|deprecated|small|'
              r'late|literal|doc|by|see|throws|optional|license|tagged|final|native|'
@@ -1035,11 +1043,11 @@ class CeylonLexer(RegexLexer):
              r'super|given|of|out|assign)\b', Keyword.Declaration),
             (r'(function|value|void|new)\b',
              Keyword.Type),
-            (r'(assembly|module|package)(\s+)', bygroups(Keyword.Namespace, Text)),
+            (r'(assembly|module|package)(\s+)', bygroups(Keyword.Namespace, Whitespace)),
             (r'(true|false|null)\b', Keyword.Constant),
             (r'(class|interface|object|alias)(\s+)',
-             bygroups(Keyword.Declaration, Text), 'class'),
-            (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
+             bygroups(Keyword.Declaration, Whitespace), 'class'),
+            (r'(import)(\s+)', bygroups(Keyword.Namespace, Whitespace), 'import'),
             (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
             (r"'\\.'|'[^\\]'|'\\\{#[0-9a-fA-F]{4}\}'", String.Char),
             (r'(\.)([a-z_]\w*)',
@@ -1059,7 +1067,7 @@ class CeylonLexer(RegexLexer):
             (r'\$[01]+', Number.Bin),
             (r'\d{1,3}(_\d{3})+[kMGTP]?', Number.Integer),
             (r'[0-9]+[kMGTP]?', Number.Integer),
-            (r'\n', Text)
+            (r'\n', Whitespace)
         ],
         'class': [
             (r'[A-Za-z_]\w*', Name.Class, '#pop')
@@ -1079,18 +1087,18 @@ class CeylonLexer(RegexLexer):
 
 class KotlinLexer(RegexLexer):
     """
-    For `Kotlin <http://kotlinlang.org/>`_
-    source code.
+    For Kotlin source code.
 
     .. versionadded:: 1.5
     """
 
     name = 'Kotlin'
+    url = 'http://kotlinlang.org/'
     aliases = ['kotlin']
     filenames = ['*.kt', '*.kts']
     mimetypes = ['text/x-kotlin']
 
-    flags = re.MULTILINE | re.DOTALL | re.UNICODE
+    flags = re.MULTILINE | re.DOTALL
 
     kt_name = ('@?[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' +
                '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc', 'Cf',
@@ -1111,13 +1119,13 @@ class KotlinLexer(RegexLexer):
     tokens = {
         'root': [
             # Whitespaces
-            (r'[^\S\n]+', Text),
-            (r'\s+', Text),
-            (r'\\\n', Text),  # line continuation
-            (r'\n', Text),
+            (r'[^\S\n]+', Whitespace),
+            (r'\s+', Whitespace),
+            (r'\\$', String.Escape),  # line continuation
+            (r'\n', Whitespace),
             # Comments
-            (r'//.*?\n', Comment.Single),
-            (r'^#!/.+?\n', Comment.Single),  # shebang for kotlin scripts
+            (r'(//.*?)(\n)', bygroups(Comment.Single, Whitespace)),
+            (r'^(#!/.+?)(\n)', bygroups(Comment.Single, Whitespace)),  # shebang for kotlin scripts
             (r'/[*].*?[*]/', Comment.Multiline),
             # Keywords
             (r'as\?', Keyword),
@@ -1133,7 +1141,7 @@ class KotlinLexer(RegexLexer):
             # Constants
             (r'(true|false|null)\b', Keyword.Constant),
             # Imports
-            (r'(package|import)(\s+)(\S+)', bygroups(Keyword, Text, Name.Namespace)),
+            (r'(package|import)(\s+)(\S+)', bygroups(Keyword, Whitespace, Name.Namespace)),
             # Dot access
             (r'(\?\.)((?:[^\W\d]|\$)[\w$]*)', bygroups(Operator, Name.Attribute)),
             (r'(\.)((?:[^\W\d]|\$)[\w$]*)', bygroups(Punctuation, Name.Attribute)),
@@ -1142,18 +1150,18 @@ class KotlinLexer(RegexLexer):
             # Labels
             (r'[^\W\d][\w.]+@', Name.Decorator),
             # Object expression
-            (r'(object)(\s+)(:)(\s+)', bygroups(Keyword, Text, Punctuation, Text), 'class'),
+            (r'(object)(\s+)(:)(\s+)', bygroups(Keyword, Whitespace, Punctuation, Whitespace), 'class'),
             # Types
             (r'((?:(?:' + modifiers + r'|fun)\s+)*)(class|interface|object)(\s+)',
-             bygroups(using(this, state='modifiers'), Keyword.Declaration, Text), 'class'),
+             bygroups(using(this, state='modifiers'), Keyword.Declaration, Whitespace), 'class'),
             # Variables
-            (r'(var|val)(\s+)(\()', bygroups(Keyword.Declaration, Text, Punctuation),
+            (r'(var|val)(\s+)(\()', bygroups(Keyword.Declaration, Whitespace, Punctuation),
              'destructuring_assignment'),
             (r'((?:(?:' + modifiers + r')\s+)*)(var|val)(\s+)',
-             bygroups(using(this, state='modifiers'), Keyword.Declaration, Text), 'variable'),
+             bygroups(using(this, state='modifiers'), Keyword.Declaration, Whitespace), 'variable'),
             # Functions
             (r'((?:(?:' + modifiers + r')\s+)*)(fun)(\s+)',
-             bygroups(using(this, state='modifiers'), Keyword.Declaration, Text), 'function'),
+             bygroups(using(this, state='modifiers'), Keyword.Declaration, Whitespace), 'function'),
             # Operators
             (r'::|!!|\?[:.]', Operator),
             (r'[~^*!%&\[\]<>|+=/?-]', Operator),
@@ -1177,9 +1185,9 @@ class KotlinLexer(RegexLexer):
         ],
         'destructuring_assignment': [
             (r',', Punctuation),
-            (r'\s+', Text),
+            (r'\s+', Whitespace),
             (kt_id, Name.Variable),
-            (r'(:)(\s+)(' + kt_id + ')', bygroups(Punctuation, Text, Name)),
+            (r'(:)(\s+)(' + kt_id + ')', bygroups(Punctuation, Whitespace, Name)),
             (r'<', Operator, 'generic'),
             (r'\)', Punctuation, '#pop')
         ],
@@ -1189,16 +1197,16 @@ class KotlinLexer(RegexLexer):
             (kt_id, Name.Function, '#pop')
         ],
         'generic': [
-            (r'(>)(\s*)', bygroups(Operator, Text), '#pop'),
+            (r'(>)(\s*)', bygroups(Operator, Whitespace), '#pop'),
             (r':', Punctuation),
             (r'(reified|out|in)\b', Keyword),
             (r',', Punctuation),
-            (r'\s+', Text),
+            (r'\s+', Whitespace),
             (kt_id, Name)
         ],
         'modifiers': [
             (r'\w+', Keyword.Declaration),
-            (r'\s+', Text),
+            (r'\s+', Whitespace),
             default('#pop')
         ],
         'string': [
@@ -1213,7 +1221,7 @@ class KotlinLexer(RegexLexer):
         'string_common': [
             (r'\\\\', String),  # escaped backslash
             (r'\\"', String),  # escaped quote
-            (r'\\', String),  # bare backslash 
+            (r'\\', String),  # bare backslash
             (r'\$\{', String.Interpol, 'interpolation'),
             (r'(\$)(\w+)', bygroups(String.Interpol, Name)),
             (r'[^\\"$]+', String)
@@ -1235,12 +1243,13 @@ class KotlinLexer(RegexLexer):
 
 class XtendLexer(RegexLexer):
     """
-    For `Xtend <http://xtend-lang.org/>`_ source code.
+    For Xtend source code.
 
     .. versionadded:: 1.6
     """
 
     name = 'Xtend'
+    url = 'https://www.eclipse.org/xtend/'
     aliases = ['xtend']
     filenames = ['*.xtend']
     mimetypes = ['text/x-xtend']
@@ -1253,9 +1262,9 @@ class XtendLexer(RegexLexer):
             (r'^(\s*(?:[a-zA-Z_][\w.\[\]]*\s+)+?)'  # return arguments
              r'([a-zA-Z_$][\w$]*)'                  # method name
              r'(\s*)(\()',                          # signature start
-             bygroups(using(this), Name.Function, Text, Operator)),
-            (r'[^\S\n]+', Text),
-            (r'//.*?\n', Comment.Single),
+             bygroups(using(this), Name.Function, Whitespace, Operator)),
+            (r'[^\S\n]+', Whitespace),
+            (r'(//.*?)(\n)', bygroups(Comment.Single, Whitespace)),
             (r'/\*.*?\*/', Comment.Multiline),
             (r'@[a-zA-Z_][\w.]*', Name.Decorator),
             (r'(assert|break|case|catch|continue|default|do|else|finally|for|'
@@ -1267,11 +1276,11 @@ class XtendLexer(RegexLexer):
              r'transient|volatile)\b', Keyword.Declaration),
             (r'(boolean|byte|char|double|float|int|long|short|void)\b',
              Keyword.Type),
-            (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)),
+            (r'(package)(\s+)', bygroups(Keyword.Namespace, Whitespace)),
             (r'(true|false|null)\b', Keyword.Constant),
-            (r'(class|interface)(\s+)', bygroups(Keyword.Declaration, Text),
+            (r'(class|interface)(\s+)', bygroups(Keyword.Declaration, Whitespace),
              'class'),
-            (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
+            (r'(import)(\s+)', bygroups(Keyword.Namespace, Whitespace), 'import'),
             (r"(''')", String, 'template'),
             (r'(\u00BB)', String, 'template'),
             (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
@@ -1282,7 +1291,7 @@ class XtendLexer(RegexLexer):
             (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
             (r'0x[0-9a-fA-F]+', Number.Hex),
             (r'[0-9]+L?', Number.Integer),
-            (r'\n', Text)
+            (r'\n', Whitespace)
         ],
         'class': [
             (r'[a-zA-Z_]\w*', Name.Class, '#pop')
@@ -1300,12 +1309,13 @@ class XtendLexer(RegexLexer):
 
 class PigLexer(RegexLexer):
     """
-    For `Pig Latin <https://pig.apache.org/>`_ source code.
+    For Pig Latin source code.
 
     .. versionadded:: 2.0
     """
 
     name = 'Pig'
+    url = 'https://pig.apache.org/'
     aliases = ['pig']
     filenames = ['*.pig']
     mimetypes = ['text/x-pig']
@@ -1314,10 +1324,10 @@ class PigLexer(RegexLexer):
 
     tokens = {
         'root': [
-            (r'\s+', Text),
+            (r'\s+', Whitespace),
             (r'--.*', Comment),
             (r'/\*[\w\W]*?\*/', Comment.Multiline),
-            (r'\\\n', Text),
+            (r'\\$', String.Escape),
             (r'\\', Text),
             (r'\'(?:\\[ntbrf\\\']|\\u[0-9a-f]{4}|[^\'\\\n\r])*\'', String),
             include('keywords'),
@@ -1328,9 +1338,9 @@ class PigLexer(RegexLexer):
             (r'[0-9]*\.[0-9]+(e[0-9]+)?[fd]?', Number.Float),
             (r'0x[0-9a-f]+', Number.Hex),
             (r'[0-9]+L?', Number.Integer),
-            (r'\n', Text),
+            (r'\n', Whitespace),
             (r'([a-z_]\w*)(\s*)(\()',
-             bygroups(Name.Function, Text, Punctuation)),
+             bygroups(Name.Function, Whitespace, Punctuation)),
             (r'[()#:]', Text),
             (r'[^(:#\'")\s]+', Text),
             (r'\S+\s+', Text)   # TODO: make tests pass without \s+
@@ -1367,18 +1377,19 @@ class PigLexer(RegexLexer):
 
 class GoloLexer(RegexLexer):
     """
-    For `Golo <http://golo-lang.org/>`_ source code.
+    For Golo source code.
 
     .. versionadded:: 2.0
     """
 
     name = 'Golo'
+    url = 'http://golo-lang.org/'
     filenames = ['*.golo']
     aliases = ['golo']
 
     tokens = {
         'root': [
-            (r'[^\S\n]+', Text),
+            (r'[^\S\n]+', Whitespace),
 
             (r'#.*$', Comment),
 
@@ -1390,19 +1401,19 @@ class GoloLexer(RegexLexer):
             (r'[]{}|(),[]', Punctuation),
 
             (r'(module|import)(\s+)',
-                bygroups(Keyword.Namespace, Text),
+                bygroups(Keyword.Namespace, Whitespace),
                 'modname'),
             (r'\b([a-zA-Z_][\w$.]*)(::)',  bygroups(Name.Namespace, Punctuation)),
             (r'\b([a-zA-Z_][\w$]*(?:\.[a-zA-Z_][\w$]*)+)\b', Name.Namespace),
 
             (r'(let|var)(\s+)',
-                bygroups(Keyword.Declaration, Text),
+                bygroups(Keyword.Declaration, Whitespace),
                 'varname'),
             (r'(struct)(\s+)',
-                bygroups(Keyword.Declaration, Text),
+                bygroups(Keyword.Declaration, Whitespace),
                 'structname'),
             (r'(function)(\s+)',
-                bygroups(Keyword.Declaration, Text),
+                bygroups(Keyword.Declaration, Whitespace),
                 'funcname'),
 
             (r'(null|true|false)\b', Keyword.Constant),
@@ -1480,12 +1491,13 @@ class GoloLexer(RegexLexer):
 
 class JasminLexer(RegexLexer):
     """
-    For `Jasmin <http://jasmin.sourceforge.net/>`_ assembly code.
+    For Jasmin assembly code.
 
     .. versionadded:: 2.0
     """
 
     name = 'Jasmin'
+    url = 'http://jasmin.sourceforge.net/'
     aliases = ['jasmin', 'jasminxt']
     filenames = ['*.j']
 
@@ -1498,12 +1510,12 @@ class JasminLexer(RegexLexer):
 
     tokens = {
         'default': [
-            (r'\n', Text, '#pop'),
+            (r'\n', Whitespace, '#pop'),
             (r"'", String.Single, ('#pop', 'quote')),
             (r'"', String.Double, 'string'),
             (r'=', Punctuation),
             (r':', Punctuation, 'label'),
-            (_ws, Text),
+            (_ws, Whitespace),
             (r';.*', Comment.Single),
             (r'(\$[-+])?0x-?[\da-fA-F]+%s' % _break, Number.Hex),
             (r'(\$[-+]|\+)?-?\d+%s' % _break, Number.Integer),
@@ -1591,27 +1603,27 @@ class JasminLexer(RegexLexer):
             (r'[^"\\]+', String.Double)
         ],
         'root': [
-            (r'\n+', Text),
+            (r'\n+', Whitespace),
             (r"'", String.Single, 'quote'),
             include('default'),
             (r'(%s)([ \t\r]*)(:)' % _name,
-             bygroups(Name.Label, Text, Punctuation)),
+             bygroups(Name.Label, Whitespace, Punctuation)),
             (_name, String.Other)
         ],
         'annotation': [
-            (r'\n', Text, ('#pop', 'annotation-body')),
+            (r'\n', Whitespace, ('#pop', 'annotation-body')),
             (r'default%s' % _break, Keyword.Reserved,
              ('#pop', 'annotation-default')),
             include('default')
         ],
         'annotation-body': [
-            (r'\n+', Text),
+            (r'\n+', Whitespace),
             (r'\.end%s' % _break, Keyword.Reserved, '#pop'),
             include('default'),
             (_name, String.Other, ('annotation-items', 'descriptor/no-dots'))
         ],
         'annotation-default': [
-            (r'\n+', Text),
+            (r'\n+', Whitespace),
             (r'\.end%s' % _break, Keyword.Reserved, '#pop'),
             include('default'),
             default(('annotation-items', 'descriptor/no-dots'))
@@ -1665,7 +1677,7 @@ class JasminLexer(RegexLexer):
             default('descriptor/convert-dots')
         ],
         'enclosing-method': [
-            (_ws, Text),
+            (_ws, Whitespace),
             (r'(?=[^%s]*\()' % _separator, Text, ('#pop', 'invocation')),
             default(('#pop', 'class/convert-dots'))
         ],
@@ -1711,7 +1723,7 @@ class JasminLexer(RegexLexer):
              bygroups(Name.Namespace, Name.Class, Name.Variable.Class), '#pop')
         ],
         'table': [
-            (r'\n+', Text),
+            (r'\n+', Whitespace),
             (r'default%s' % _break, Keyword.Reserved, '#pop'),
             include('default'),
             (_name, Name.Label)
@@ -1744,12 +1756,13 @@ class JasminLexer(RegexLexer):
 
 class SarlLexer(RegexLexer):
     """
-    For `SARL <http://www.sarl.io>`_ source code.
+    For SARL source code.
 
     .. versionadded:: 2.4
     """
 
     name = 'SARL'
+    url = 'http://www.sarl.io'
     aliases = ['sarl']
     filenames = ['*.sarl']
     mimetypes = ['text/x-sarl']
@@ -1762,9 +1775,9 @@ class SarlLexer(RegexLexer):
             (r'^(\s*(?:[a-zA-Z_][\w.\[\]]*\s+)+?)'  # return arguments
              r'([a-zA-Z_$][\w$]*)'                      # method name
              r'(\s*)(\()',                             # signature start
-             bygroups(using(this), Name.Function, Text, Operator)),
-            (r'[^\S\n]+', Text),
-            (r'//.*?\n', Comment.Single),
+             bygroups(using(this), Name.Function, Whitespace, Operator)),
+            (r'[^\S\n]+', Whitespace),
+            (r'(//.*?)(\n)', bygroups(Comment.Single, Whitespace)),
             (r'/\*.*?\*/', Comment.Multiline),
             (r'@[a-zA-Z_][\w.]*', Name.Decorator),
             (r'(as|break|case|catch|default|do|else|extends|extension|finally|'
@@ -1776,12 +1789,12 @@ class SarlLexer(RegexLexer):
              Keyword.Declaration),
             (r'(boolean|byte|char|double|float|int|long|short|void)\b',
              Keyword.Type),
-            (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)),
+            (r'(package)(\s+)', bygroups(Keyword.Namespace, Whitespace)),
             (r'(false|it|null|occurrence|this|true|void)\b', Keyword.Constant),
             (r'(agent|annotation|artifact|behavior|capacity|class|enum|event|'
-             r'interface|skill|space)(\s+)', bygroups(Keyword.Declaration, Text),
+             r'interface|skill|space)(\s+)', bygroups(Keyword.Declaration, Whitespace),
              'class'),
-            (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'),
+            (r'(import)(\s+)', bygroups(Keyword.Namespace, Whitespace), 'import'),
             (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
             (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
             (r'[a-zA-Z_]\w*:', Name.Label),
@@ -1790,7 +1803,7 @@ class SarlLexer(RegexLexer):
             (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
             (r'0x[0-9a-fA-F]+', Number.Hex),
             (r'[0-9]+L?', Number.Integer),
-            (r'\n', Text)
+            (r'\n', Whitespace)
         ],
         'class': [
             (r'[a-zA-Z_]\w*', Name.Class, '#pop')