Intermediate changes

commit_hash:5aaa14131e1a44bd5ff318aaed16345258d4c42e

6 files changed, 196 insertions, 0 deletions

diff --git a/yql/essentials/utils/docs/markdown.cpp b/yql/essentials/utils/docs/markdown.cpp
new file mode 100644
index 00000000000..4af1357f1fe
--- /dev/null
+++ b/yql/essentials/utils/docs/markdown.cpp
@@ -0,0 +1,74 @@
+#include "markdown.h"
+
+#include <yql/essentials/utils/yql_panic.h>
+
+#include <contrib/libs/re2/re2/re2.h>
+
+#include <util/generic/yexception.h>
+
+namespace NSQLComplete {
+
+    class TMarkdownParser {
+    public:
+        void Parse(IInputStream& markdown, TMarkdownCallback&& onSection) {
+            for (TString line; markdown.ReadLine(line) != 0;) {
+                if (IsSkipping_) {
+                    if (IsSectionHeader(line)) {
+                        ResetSection(std::move(line));
+                        IsSkipping_ = false;
+                    } else {
+                        // Skip
+                    }
+                } else {
+                    if (IsSectionHeader(line)) {
+                        onSection(std::move(Section_));
+                        ResetSection(std::move(line));
+                    } else {
+                        Section_.Body.append(std::move(line));
+                    }
+                }
+            }
+
+            if (!IsSkipping_) {
+                onSection(std::move(Section_));
+            }
+        }
+
+    private:
+        void ResetSection(TString&& line) {
+            Section_ = TMarkdownSection();
+
+            TString content;
+            std::optional<TString> dummy;
+            std::optional<TString> anchor;
+            YQL_ENSURE(
+                RE2::FullMatch(line, SectionHeaderRegex_, &content, &dummy, &anchor),
+                "line '" << line << "' does not match regex '"
+                         << SectionHeaderRegex_.pattern() << "'");
+
+            Section_.Header.Content = std::move(content);
+            if (anchor) {
+                Section_.Header.Anchor = std::move(*anchor);
+            }
+        }
+
+        bool IsSectionHeader(TStringBuf line) const {
+            return HeaderDepth(line) == 2;
+        }
+
+        size_t HeaderDepth(TStringBuf line) const {
+            size_t pos = line.find_first_not_of('#');
+            return pos != TStringBuf::npos ? pos : 0;
+        }
+
+        RE2 SectionHeaderRegex_{R"re(## ([^#]+)(\s+{(#[a-z\-_]+)})?)re"};
+        bool IsSkipping_ = true;
+        TMarkdownSection Section_;
+    };
+
+    void ParseMarkdown(IInputStream& markdown, TMarkdownCallback&& onSection) {
+        TMarkdownParser parser;
+        parser.Parse(markdown, std::forward<TMarkdownCallback>(onSection));
+    }
+
+} // namespace NSQLComplete
diff --git a/yql/essentials/utils/docs/markdown.h b/yql/essentials/utils/docs/markdown.h
new file mode 100644
index 00000000000..648c46e4187
--- /dev/null
+++ b/yql/essentials/utils/docs/markdown.h
@@ -0,0 +1,23 @@
+#pragma once
+
+#include <util/generic/string.h>
+#include <util/generic/maybe.h>
+#include <util/stream/input.h>
+
+namespace NSQLComplete {
+
+    struct TMarkdownHeader {
+        TString Content;
+        TMaybe<TString> Anchor;
+    };
+
+    struct TMarkdownSection {
+        TMarkdownHeader Header;
+        TString Body;
+    };
+
+    using TMarkdownCallback = std::function<void(TMarkdownSection&&)>;
+
+    void ParseMarkdown(IInputStream& markdown, TMarkdownCallback&& onSection);
+
+} // namespace NSQLComplete
diff --git a/yql/essentials/utils/docs/markdown_ut.cpp b/yql/essentials/utils/docs/markdown_ut.cpp
new file mode 100644
index 00000000000..3f9d7acb2e3
--- /dev/null
+++ b/yql/essentials/utils/docs/markdown_ut.cpp
@@ -0,0 +1,75 @@
+#include "markdown.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NSQLComplete;
+
+Y_UNIT_TEST_SUITE(MarkdownParserTests) {
+
+    Y_UNIT_TEST(ParseMarkdown) {
+        TString markdown = R"(
+# Basic built-in functions
+
+Below are the general-purpose functions.
+
+## COALESCE {#coalesce}
+
+Iterates through the arguments from left to right.
+
+#### Examples
+
+```yql
+SELECT COALESCE(
+  maybe_empty_column,
+  "it's empty!"
+) FROM my_table;
+```
+
+## Random... {#random}
+
+Generates a pseudorandom number:
+
+* `Random()`: A floating point number (Double) from 0 to 1.
+* `RandomNumber()`: An integer from the complete Uint64 range.
+* `RandomUuid()`: [Uuid version 4](https://tools.ietf.org/html/rfc4122#section-4.4).
+
+#### Signatures
+
+```yql
+Random(T1[, T2, ...])->Double
+RandomNumber(T1[, T2, ...])->Uint64
+RandomUuid(T1[, T2, ...])->Uuid
+```
+
+No arguments are used for random number generation.
+
+#### Examples
+
+```yql
+SELECT
+    Random(key) -- [0, 1)
+FROM my_table;
+```
+)";
+        TVector<TMarkdownSection> sections;
+
+        TStringStream input(markdown);
+        ParseMarkdown(input, [&](TMarkdownSection&& section) {
+            sections.emplace_back(std::move(section));
+        });
+
+        UNIT_ASSERT_VALUES_EQUAL(sections.size(), 2);
+
+        UNIT_ASSERT_STRING_CONTAINS(sections[0].Header.Content, "COALESCE");
+        UNIT_ASSERT_VALUES_EQUAL(sections[0].Header.Anchor, "#coalesce");
+        UNIT_ASSERT_STRING_CONTAINS(sections[0].Body, "Iterates");
+        UNIT_ASSERT_STRING_CONTAINS(sections[0].Body, "COALESCE");
+
+        UNIT_ASSERT_STRING_CONTAINS(sections[1].Header.Content, "Random");
+        UNIT_ASSERT_VALUES_EQUAL(sections[1].Header.Anchor, "#random");
+        UNIT_ASSERT_STRING_CONTAINS(sections[1].Body, "Generates");
+        UNIT_ASSERT_STRING_CONTAINS(sections[1].Body, "Random");
+        UNIT_ASSERT_STRING_CONTAINS(sections[1].Body, "Random");
+    }
+
+} // Y_UNIT_TEST_SUITE(MarkdownParserTests)
diff --git a/yql/essentials/utils/docs/ut/ya.make b/yql/essentials/utils/docs/ut/ya.make
new file mode 100644
index 00000000000..7a88c936b8a
--- /dev/null
+++ b/yql/essentials/utils/docs/ut/ya.make
@@ -0,0 +1,7 @@
+UNITTEST_FOR(yql/essentials/utils/docs)
+
+SRCS(
+    markdown_ut.cpp
+)
+
+END()
diff --git a/yql/essentials/utils/docs/ya.make b/yql/essentials/utils/docs/ya.make
new file mode 100644
index 00000000000..1122a09b58a
--- /dev/null
+++ b/yql/essentials/utils/docs/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+SRCS(
+    markdown.cpp
+)
+
+PEERDIR(
+    yql/essentials/utils
+    contrib/libs/re2
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+    ut
+)
diff --git a/yql/essentials/utils/ya.make b/yql/essentials/utils/ya.make
index 769428e3550..d0e1b58200e 100644
--- a/yql/essentials/utils/ya.make
+++ b/yql/essentials/utils/ya.make
@@ -66,6 +66,7 @@ RECURSE_FOR_TESTS(
 IF (OPENSOURCE_PROJECT != "yt")
     RECURSE(
         backtrace
+        docs
         failure_injector
         fetch
         log