intermediate changes

ref:cde9a383711a11544ce7e107a78147fb96cc4029

1 files changed, 28 insertions, 0 deletions

diff --git a/library/cpp/yson/tokenizer.h b/library/cpp/yson/tokenizer.h
new file mode 100644
index 0000000000..0576aace95
--- /dev/null
+++ b/library/cpp/yson/tokenizer.h
@@ -0,0 +1,28 @@
+#pragma once
+
+#include "public.h"
+#include "lexer.h"
+
+namespace NYson {
+    ////////////////////////////////////////////////////////////////////////////////
+
+    class TTokenizer {
+    public:
+        explicit TTokenizer(const TStringBuf& input);
+
+        bool ParseNext();
+        const TToken& CurrentToken() const;
+        ETokenType GetCurrentType() const;
+        TStringBuf GetCurrentSuffix() const;
+        const TStringBuf& CurrentInput() const;
+
+    private:
+        TStringBuf Input;
+        TToken Token;
+        TStatelessLexer Lexer;
+        size_t Parsed;
+    };
+
+    ////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYson