aboutsummaryrefslogtreecommitdiffstats
path: root/yql/essentials/parser/lexer_common/lexer.h
blob: 4de7eedb8366abea8551adc5b90d7750efdc3d6f (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
#pragma once

#include <util/generic/string.h>
#include <util/generic/vector.h>
#include <util/stream/output.h>

#include <functional>

namespace NYql {

class TIssues;

}

namespace NSQLTranslation {

struct TParsedToken {
    // TODO: TStringBuf for Name & Content
    TString Name;
    TString Content;
    // Position of first token byte/symbol
    // When antlr3 lexer is used, LinePos is a position as in a byte array,
    // but when antlr4 lexer is used, LinePos is a position as in a symbol array,
    ui32 Line = 0;    // starts from 1
    ui32 LinePos = 0; // starts from 0
};

class ILexer {
public:
    using TPtr = THolder<ILexer>;
    using TTokenCallback = std::function<void(TParsedToken&& token)>;

    virtual bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) = 0;
    virtual ~ILexer() = default;
};

using TParsedTokenList = TVector<TParsedToken>;

IOutputStream& OutputTokens(IOutputStream& out, TParsedTokenList::const_iterator begin, TParsedTokenList::const_iterator end);
bool Tokenize(ILexer& lexer, const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors);

class ILexerFactory : public TThrRefBase {
public:
    virtual ~ILexerFactory() = default;

    virtual ILexer::TPtr MakeLexer() const = 0;
};

using TLexerFactoryPtr = TIntrusivePtr<ILexerFactory>;

TLexerFactoryPtr MakeDummyLexerFactory(const TString& name);

}