blob: 192f2f55e6a897729a80a565a3d955a5d7437e06 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
|
#pragma once
#include <Core/Defines.h>
#include <Parsers/Lexer.h>
#include <cassert>
#include <vector>
namespace DB
{
/** Parser operates on lazy stream of tokens.
* It could do lookaheads of any depth.
*/
/** Used as an input for parsers.
* All whitespace and comment tokens are transparently skipped.
*/
class Tokens
{
private:
std::vector<Token> data;
std::size_t last_accessed_index = 0;
public:
Tokens(const char * begin, const char * end, size_t max_query_size = 0, bool skip_insignificant = true);
ALWAYS_INLINE inline const Token & operator[](size_t index)
{
assert(index < data.size());
last_accessed_index = std::max(last_accessed_index, index);
return data[index];
}
ALWAYS_INLINE inline const Token & max() { return data[last_accessed_index]; }
};
/// To represent position in a token stream.
class TokenIterator
{
private:
Tokens * tokens;
size_t index = 0;
public:
explicit TokenIterator(Tokens & tokens_) : tokens(&tokens_) {}
ALWAYS_INLINE const Token & get() { return (*tokens)[index]; }
ALWAYS_INLINE const Token & operator*() { return get(); }
ALWAYS_INLINE const Token * operator->() { return &get(); }
ALWAYS_INLINE TokenIterator & operator++()
{
++index;
return *this;
}
ALWAYS_INLINE TokenIterator & operator--()
{
--index;
return *this;
}
ALWAYS_INLINE bool operator<(const TokenIterator & rhs) const { return index < rhs.index; }
ALWAYS_INLINE bool operator<=(const TokenIterator & rhs) const { return index <= rhs.index; }
ALWAYS_INLINE bool operator==(const TokenIterator & rhs) const { return index == rhs.index; }
ALWAYS_INLINE bool operator!=(const TokenIterator & rhs) const { return index != rhs.index; }
ALWAYS_INLINE bool isValid() { return get().type < TokenType::EndOfStream; }
/// Rightmost token we had looked.
ALWAYS_INLINE const Token & max() { return tokens->max(); }
};
/// Returns positions of unmatched parentheses.
using UnmatchedParentheses = std::vector<Token>;
UnmatchedParentheses checkUnmatchedParentheses(TokenIterator begin);
}
|
