intermediate changes

ref:cde9a383711a11544ce7e107a78147fb96cc4029

1 files changed, 37 insertions, 0 deletions

diff --git a/library/cpp/yson/tokenizer.cpp b/library/cpp/yson/tokenizer.cpp
new file mode 100644
index 0000000000..06760170d4
--- /dev/null
+++ b/library/cpp/yson/tokenizer.cpp
@@ -0,0 +1,37 @@
+#include "tokenizer.h"
+
+namespace NYson {
+    ////////////////////////////////////////////////////////////////////////////////
+
+    TTokenizer::TTokenizer(const TStringBuf& input)
+        : Input(input)
+        , Parsed(0)
+    {
+    }
+
+    bool TTokenizer::ParseNext() {
+        Input = Input.Tail(Parsed);
+        Token.Reset();
+        Parsed = Lexer.GetToken(Input, &Token);
+        return !CurrentToken().IsEmpty();
+    }
+
+    const TToken& TTokenizer::CurrentToken() const {
+        return Token;
+    }
+
+    ETokenType TTokenizer::GetCurrentType() const {
+        return CurrentToken().GetType();
+    }
+
+    TStringBuf TTokenizer::GetCurrentSuffix() const {
+        return Input.Tail(Parsed);
+    }
+
+    const TStringBuf& TTokenizer::CurrentInput() const {
+        return Input;
+    }
+
+    ////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYson