library/cpp/unicode/set/unicode_set_lexer.h


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

#pragma once

#include "unicode_set_token.h"

#include <util/generic/strbuf.h>
#include <util/system/yassert.h>

namespace NUnicode {
    namespace NPrivate {
        class TUnicodeSetLexer {
        private:
            const TWtringBuf& Data;

            int cs;
            //int* stack;
            //int top;
            int act;
            const wchar16* ts;
            const wchar16* te;
            const wchar16* p;
            const wchar16* pe;
            const wchar16* eof;

            TUnicodeSetToken LastToken;
            bool UseLast;

        private:
            EUnicodeSetTokenType YieldToken(EUnicodeSetTokenType type);
            EUnicodeSetTokenType YieldToken(EUnicodeSetTokenType type, wchar16 symbol);
            EUnicodeSetTokenType YieldToken(EUnicodeSetTokenType type, const wchar16* dataBegin, size_t dataSize);
            void Reset();

        public:
            explicit TUnicodeSetLexer(const TWtringBuf& data);

            EUnicodeSetTokenType GetToken();

            const TUnicodeSetToken& GetLastToken() {
                return LastToken;
            }

            inline void PushBack() {
                Y_VERIFY(!UseLast, "Double TUnicodeSetLexer::PushBack()");
                UseLast = true;
            }
        };

    }
}