1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
|
"""
pygments.lexers.futhark
~~~~~~~~~~~~~~~~~~~~~~~
Lexer for the Futhark language
:copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
from pygments.lexer import RegexLexer, bygroups
from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
Number, Punctuation, Whitespace
from pygments import unistring as uni
__all__ = ['FutharkLexer']
line_re = re.compile('.*?\n')
class FutharkLexer(RegexLexer):
"""
A Futhark lexer
.. versionadded:: 2.8
"""
name = 'Futhark'
aliases = ['futhark']
filenames = ['*.fut']
mimetypes = ['text/x-futhark']
flags = re.MULTILINE | re.UNICODE
num_types = ('i8', 'i16', 'i32', 'i64', 'u8', 'u16', 'u32', 'u64', 'f32', 'f64')
other_types = ('bool', )
reserved = ('if', 'then', 'else', 'let', 'loop', 'in', 'with', 'type',
'val', 'entry', 'for', 'while', 'do', 'case', 'match',
'include', 'import', 'module', 'open', 'local', 'assert', '_')
ascii = ('NUL', 'SOH', '[SE]TX', 'EOT', 'ENQ', 'ACK',
'BEL', 'BS', 'HT', 'LF', 'VT', 'FF', 'CR', 'S[OI]', 'DLE',
'DC[1-4]', 'NAK', 'SYN', 'ETB', 'CAN',
'EM', 'SUB', 'ESC', '[FGRU]S', 'SP', 'DEL')
num_postfix = r'(%s)?' % '|'.join(num_types)
identifier_re = '[a-zA-Z_][a-zA-Z_0-9\']*'
# opstart_re = '+\-\*/%=\!><\|&\^'
tokens = {
'root': [
(r'--(.*?)$', Comment.Single),
(r'\s+', Whitespace),
(r'\(\)', Punctuation),
(r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved),
(r'\b(%s)(?!\')\b' % '|'.join(num_types + other_types), Keyword.Type),
# Identifiers
(r'#\[([a-zA-Z_\(\) ]*)\]', Comment.Preproc),
(r'!?(%s\.)*%s' % (identifier_re, identifier_re), Name),
(r'\\', Operator),
(r'[-+/%=!><|&*^][-+/%=!><|&*^.]*', Operator),
(r'[][(),:;`{}]', Punctuation),
# Numbers
(r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*_*[pP][+-]?\d(_*\d)*' + num_postfix,
Number.Float),
(r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*\.[\da-fA-F](_*[\da-fA-F])*'
r'(_*[pP][+-]?\d(_*\d)*)?' + num_postfix, Number.Float),
(r'\d(_*\d)*_*[eE][+-]?\d(_*\d)*' + num_postfix, Number.Float),
(r'\d(_*\d)*\.\d(_*\d)*(_*[eE][+-]?\d(_*\d)*)?' + num_postfix, Number.Float),
(r'0[bB]_*[01](_*[01])*' + num_postfix, Number.Bin),
(r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*' + num_postfix, Number.Hex),
(r'\d(_*\d)*' + num_postfix, Number.Integer),
# Character/String Literals
(r"'", String.Char, 'character'),
(r'"', String, 'string'),
# Special
(r'\[[a-zA-Z_\d]*\]', Keyword.Type),
(r'\(\)', Name.Builtin),
],
'character': [
# Allows multi-chars, incorrectly.
(r"[^\\']'", String.Char, '#pop'),
(r"\\", String.Escape, 'escape'),
("'", String.Char, '#pop'),
],
'string': [
(r'[^\\"]+', String),
(r"\\", String.Escape, 'escape'),
('"', String, '#pop'),
],
'escape': [
(r'[abfnrtv"\'&\\]', String.Escape, '#pop'),
(r'\^[][' + uni.Lu + r'@^_]', String.Escape, '#pop'),
('|'.join(ascii), String.Escape, '#pop'),
(r'o[0-7]+', String.Escape, '#pop'),
(r'x[\da-fA-F]+', String.Escape, '#pop'),
(r'\d+', String.Escape, '#pop'),
(r'(\s+)(\\)', bygroups(Whitespace, String.Escape), '#pop'),
],
}
|
