Intermediate changes

commit_hash:bf6ef092a6b0e517485efa54cfebca15283d6de2

4 files changed, 34 insertions, 2 deletions

diff --git a/yql/essentials/utils/docs/name.cpp b/yql/essentials/utils/docs/name.cpp
index 9ebf04d06bd..af4f64a3532 100644
--- a/yql/essentials/utils/docs/name.cpp
+++ b/yql/essentials/utils/docs/name.cpp
@@ -4,6 +4,7 @@
 #include <yql/essentials/core/sql_types/normalize_name.h>
 
 #include <util/string/split.h>
+#include <util/charset/utf8.h>
 
 namespace NYql::NDocs {
 
@@ -39,8 +40,8 @@ TMaybe<std::pair<TString, TString>> SplitUDF(TString name) {
     StringSplitter(name).SplitByString("::").Collect(&words);
     YQL_ENSURE(words.size() == 2, "Invalid UDF pattern: " << name);
 
-    TMaybe<TString> module = NormalizedName(std::move(words[0]));
-    TMaybe<TString> function = NormalizedName(std::move(words[1]));
+    TMaybe<TString> module = ToLowerUTF8(std::move(words[0]));
+    TMaybe<TString> function = ToLowerUTF8(std::move(words[1]));
     YQL_ENSURE(module && function, "Unable to normalize " << name);
 
     return std::make_pair(*module, *function);
diff --git a/yql/essentials/utils/docs/page.cpp b/yql/essentials/utils/docs/page.cpp
index f0b1671a9dd..542d7f72c64 100644
--- a/yql/essentials/utils/docs/page.cpp
+++ b/yql/essentials/utils/docs/page.cpp
@@ -42,6 +42,12 @@ TString ExtendedSyntaxRemoved(TString text) {
     return text;
 }
 
+TString CodeListingsTagRemoved(TString text) {
+    static const RE2 regex(R"re(```[a-z0-9]{1,16})re");
+    RE2::GlobalReplace(&text, regex, "```");
+    return text;
+}
+
 TMarkdownPage ExtendedSyntaxRemoved(TMarkdownPage page) {
     page.Text = ExtendedSyntaxRemoved(page.Text);
     for (auto& [_, section] : page.SectionsByAnchor) {
@@ -50,6 +56,14 @@ TMarkdownPage ExtendedSyntaxRemoved(TMarkdownPage page) {
     return page;
 }
 
+TMarkdownPage CodeListingsTagRemoved(TMarkdownPage page) {
+    page.Text = CodeListingsTagRemoved(page.Text);
+    for (auto& [_, section] : page.SectionsByAnchor) {
+        section.Body = CodeListingsTagRemoved(std::move(section.Body));
+    }
+    return page;
+}
+
 TPages ParsePages(TResourcesByRelativePath resources) {
     TPages pages;
     for (auto& [path, resource] : resources) {
@@ -73,4 +87,11 @@ TPages ExtendedSyntaxRemoved(TPages pages) {
     return pages;
 }
 
+TPages CodeListingsTagRemoved(TPages pages) {
+    for (auto& [_, page] : pages) {
+        page = CodeListingsTagRemoved(std::move(page));
+    }
+    return pages;
+}
+
 } // namespace NYql::NDocs
diff --git a/yql/essentials/utils/docs/page.h b/yql/essentials/utils/docs/page.h
index e01939a9292..cb68e1f02d4 100644
--- a/yql/essentials/utils/docs/page.h
+++ b/yql/essentials/utils/docs/page.h
@@ -13,4 +13,6 @@ TPages Resolved(TPages pages, TStringBuf baseURL);
 
 TPages ExtendedSyntaxRemoved(TPages pages);
 
+TPages CodeListingsTagRemoved(TPages pages);
+
 } // namespace NYql::NDocs
diff --git a/yql/essentials/utils/docs/page_ut.cpp b/yql/essentials/utils/docs/page_ut.cpp
index a11d7784a5b..37503c7a171 100644
--- a/yql/essentials/utils/docs/page_ut.cpp
+++ b/yql/essentials/utils/docs/page_ut.cpp
@@ -32,12 +32,17 @@ If one of the compared arguments is 0.0, the function always returns false.
 
 {% endnote %}
 
+```yql
+SELECT 1;
+```
+
 End.
 )";
 
     TPages pages = {{"builtins/window", ParseMarkdownPage(markdown)}};
     pages = Resolved(std::move(pages), "https://ytsaurus.tech/docs/en/yql");
     pages = ExtendedSyntaxRemoved(std::move(pages));
+    pages = CodeListingsTagRemoved(std::move(pages));
 
     TVector<TString> changes = {
         "[separate article](https://ytsaurus.tech/docs/en/yql/builtins/window/../../syntax/window)",
@@ -55,6 +60,9 @@ End.
     UNIT_ASSERT_STRING_CONTAINS(pages["builtins/window"].Text, "End.");
     UNIT_ASSERT(!pages["builtins/window"].Text.Contains("{% note alert %}"));
     UNIT_ASSERT(!pages["builtins/window"].Text.Contains("{% endnote %}"));
+
+    UNIT_ASSERT(!pages["builtins/window"].Text.Contains("```yql\nSELECT"));
+    UNIT_ASSERT_STRING_CONTAINS(pages["builtins/window"].Text, "```\nSELECT");
 }
 
 } // Y_UNIT_TEST_SUITE(PageTests)