1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
|
#include "lexer.h"
#include <yql/essentials/public/issue/yql_issue.h>
#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h>
#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h>
#include <yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h>
#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h>
#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Lexer.h>
#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h>
#include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h>
#include <util/string/ascii.h>
#if defined(_tsan_enabled_)
#include <util/system/mutex.h>
#endif
namespace NALPDefault {
extern ANTLR_UINT8 *SQLv1ParserTokenNames[];
}
namespace NALPAnsi {
extern ANTLR_UINT8 *SQLv1ParserTokenNames[];
}
namespace NSQLTranslationV1 {
namespace {
#if defined(_tsan_enabled_)
TMutex SanitizerSQLTranslationMutex;
#endif
using NSQLTranslation::ILexer;
class TV1Lexer : public ILexer {
public:
explicit TV1Lexer(bool ansi, bool antlr4)
: Ansi(ansi), Antlr4(antlr4)
{
}
bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) override {
NYql::TIssues newIssues;
#if defined(_tsan_enabled_)
TGuard<TMutex> grd(SanitizerSQLTranslationMutex);
#endif
NSQLTranslation::TErrorCollectorOverIssues collector(newIssues, maxErrors, "");
if (Ansi && !Antlr4) {
NProtoAST::TLexerTokensCollector3<NALPAnsi::SQLv1Lexer> tokensCollector(query, (const char**)NALPAnsi::SQLv1ParserTokenNames, queryName);
tokensCollector.CollectTokens(collector, onNextToken);
} else if (!Ansi && !Antlr4) {
NProtoAST::TLexerTokensCollector3<NALPDefault::SQLv1Lexer> tokensCollector(query, (const char**)NALPDefault::SQLv1ParserTokenNames, queryName);
tokensCollector.CollectTokens(collector, onNextToken);
} else if (Ansi && Antlr4) {
NProtoAST::TLexerTokensCollector4<NALPAnsiAntlr4::SQLv1Antlr4Lexer> tokensCollector(query, queryName);
tokensCollector.CollectTokens(collector, onNextToken);
} else {
NProtoAST::TLexerTokensCollector4<NALPDefaultAntlr4::SQLv1Antlr4Lexer> tokensCollector(query, queryName);
tokensCollector.CollectTokens(collector, onNextToken);
}
issues.AddIssues(newIssues);
return !AnyOf(newIssues.begin(), newIssues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; });
}
private:
const bool Ansi;
const bool Antlr4;
};
} // namespace
NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi, bool antlr4) {
return NSQLTranslation::ILexer::TPtr(new TV1Lexer(ansi, antlr4));
}
bool IsProbablyKeyword(const NSQLTranslation::TParsedToken& token) {
return AsciiEqualsIgnoreCase(token.Name, token.Content);
}
} // namespace NSQLTranslationV1
|
