1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
#pragma once
#include <yql/essentials/parser/proto_ast/common.h>
#ifdef ERROR
#undef ERROR
#endif
#include <contrib/libs/antlr4_cpp_runtime/src/antlr4-runtime.h>
namespace antlr4 {
class ANTLR4CPP_PUBLIC YqlErrorListener : public BaseErrorListener {
NProtoAST::IErrorCollector* errors;
bool* error;
public:
YqlErrorListener(NProtoAST::IErrorCollector* errors, bool* error);
virtual void syntaxError(Recognizer *recognizer, Token * offendingSymbol, size_t line, size_t charPositionInLine,
const std::string &msg, std::exception_ptr e) override;
};
}
namespace NProtoAST {
template <>
inline void InvalidToken<antlr4::Token>(IOutputStream& err, const antlr4::Token* token) {
if (token) {
if (token->getInputStream()) {
err << " '" << token->getText() << "'";
} else {
err << ABSENCE;
}
}
}
template <typename TParser, typename TLexer>
class TProtoASTBuilder4 {
public:
TProtoASTBuilder4(TStringBuf data, const TString& queryName = "query", google::protobuf::Arena* arena = nullptr)
: QueryName(queryName)
, InputStream(data)
, Lexer(&InputStream)
, TokenStream(&Lexer)
, Parser(&TokenStream, arena)
{
}
google::protobuf::Message* BuildAST(IErrorCollector& errors) {
// TODO: find a better way to break on lexer errors
typename antlr4::YqlErrorListener listener(&errors, &Parser.error);
Parser.removeErrorListeners();
Parser.addErrorListener(&listener);
try {
auto result = Parser.Parse(&errors);
Parser.removeErrorListener(&listener);
Parser.error = false;
return result;
} catch (const TTooManyErrors&) {
Parser.removeErrorListener(&listener);
Parser.error = false;
return nullptr;
} catch (...) {
errors.Error(0, 0, CurrentExceptionMessage());
Parser.removeErrorListener(&listener);
Parser.error = false;
return nullptr;
}
}
private:
TString QueryName;
antlr4::ANTLRInputStream InputStream;
TLexer Lexer;
antlr4::CommonTokenStream TokenStream;
TParser Parser;
};
template <typename TLexer>
class TLexerTokensCollector4 {
public:
TLexerTokensCollector4(TStringBuf data, const TString& queryName = "query")
: QueryName(queryName)
, InputStream(std::string(data))
, Lexer(&InputStream)
{
}
void CollectTokens(IErrorCollector& errors, const NSQLTranslation::ILexer::TTokenCallback& onNextToken) {
try {
bool error = false;
typename antlr4::YqlErrorListener listener(&errors, &error);
Lexer.removeErrorListeners();
Lexer.addErrorListener(&listener);
for (;;) {
auto token = Lexer.nextToken();
auto type = token->getType();
const bool isEOF = type == TLexer::EOF;
NSQLTranslation::TParsedToken last;
last.Name = GetTokenName(type);
last.Content = token->getText();
last.Line = token->getLine();
last.LinePos = token->getCharPositionInLine();
onNextToken(std::move(last));
if (isEOF) {
break;
}
}
} catch (const TTooManyErrors&) {
} catch (...) {
errors.Error(0, 0, CurrentExceptionMessage());
}
}
private:
TString GetTokenName(size_t type) const {
auto res = Lexer.getVocabulary().getSymbolicName(type);
if (res != ""){
return TString(res);
}
return TString(INVALID_TOKEN_NAME);
}
TString QueryName;
antlr4::ANTLRInputStream InputStream;
TLexer Lexer;
};
} // namespace NProtoAST
|
