blob: 25e42ab9fc3dc1cf4cb5a5e9a75c6acd547638b5 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
|
#pragma once
#include "public.h"
#include "lexer.h"
namespace NYson {
////////////////////////////////////////////////////////////////////////////////
class TTokenizer {
public:
explicit TTokenizer(const TStringBuf& input);
bool ParseNext();
const TToken& CurrentToken() const;
ETokenType GetCurrentType() const;
TStringBuf GetCurrentSuffix() const;
const TStringBuf& CurrentInput() const;
private:
TStringBuf Input;
TToken Token;
TStatelessLexer Lexer;
size_t Parsed;
};
////////////////////////////////////////////////////////////////////////////////
} // namespace NYson
|
