YQL-20496: Fix an option highlighting pattern

There was a bug with highlighting an identifier a in expression a == b
as an option. So for an option match we expect an exactly = token. So
all hints are enumerated and the solid regex is built.
commit_hash:8df3cd082a7eaa99fc06daa282ba6dd9f0798f6f

4 files changed, 59 insertions, 14 deletions

diff --git a/yql/essentials/sql/v1/highlight/data_language_json.cpp b/yql/essentials/sql/v1/highlight/data_language_json.cpp
new file mode 100644
index 00000000000..224a2630a02
--- /dev/null
+++ b/yql/essentials/sql/v1/highlight/data_language_json.cpp
@@ -0,0 +1,34 @@
+#include "data_language_json.h"
+
+#include <library/cpp/json/json_reader.h>
+#include <library/cpp/resource/resource.h>
+
+namespace NSQLHighlight {
+
+TVector<TString> LoadTypes() {
+    TString resource = NResource::Find("types.json");
+    NJson::TJsonValue json = NJson::ReadJsonFastTree(resource);
+
+    TVector<TString> types;
+    for (const NJson::TJsonValue& value : json.GetArraySafe()) {
+        types.emplace_back(value["name"].GetStringSafe());
+    }
+    return types;
+}
+
+TVector<TString> LoadHints() {
+    TString resource = NResource::Find("statements_opensource.json");
+    NJson::TJsonValue json = NJson::ReadJsonFastTree(resource);
+
+    TVector<TString> hints;
+    for (const auto& [statement, services] : json.GetMapSafe()) {
+        for (const auto& [service, kinds] : services.GetMapSafe()) {
+            for (const auto& hint : kinds["hints"].GetArraySafe()) {
+                hints.emplace_back(hint["name"].GetStringSafe());
+            }
+        }
+    }
+    return hints;
+}
+
+} // namespace NSQLHighlight
diff --git a/yql/essentials/sql/v1/highlight/data_language_json.h b/yql/essentials/sql/v1/highlight/data_language_json.h
new file mode 100644
index 00000000000..1e2a7fdb9aa
--- /dev/null
+++ b/yql/essentials/sql/v1/highlight/data_language_json.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#include <util/generic/vector.h>
+#include <util/generic/string.h>
+
+namespace NSQLHighlight {
+
+TVector<TString> LoadTypes();
+
+TVector<TString> LoadHints();
+
+} // namespace NSQLHighlight
diff --git a/yql/essentials/sql/v1/highlight/sql_highlight.cpp b/yql/essentials/sql/v1/highlight/sql_highlight.cpp
index 3f17934cef4..9ad8c6b383b 100644
--- a/yql/essentials/sql/v1/highlight/sql_highlight.cpp
+++ b/yql/essentials/sql/v1/highlight/sql_highlight.cpp
@@ -1,10 +1,11 @@
 #include "sql_highlight.h"
 
+#include "data_language_json.h"
+
 #include <yql/essentials/sql/v1/lexer/regex/regex.h>
 
 #include <contrib/libs/re2/re2/re2.h>
 
-#include <library/cpp/json/json_reader.h>
 #include <library/cpp/resource/resource.h>
 
 #include <util/generic/algorithm.h>
@@ -104,6 +105,11 @@ TUnit MakeUnit<EUnitKind::BindParameterIdentifier>(Syntax& s) {
 
 template <>
 TUnit MakeUnit<EUnitKind::OptionIdentifier>(Syntax& s) {
+    TVector<NSQLTranslationV1::TRegexPattern> hints;
+    for (const TString& type : LoadHints()) {
+        hints.emplace_back(CaseInsensitive(type));
+    }
+
     return {
         .Kind = EUnitKind::OptionIdentifier,
         .Patterns = {
@@ -113,25 +119,16 @@ TUnit MakeUnit<EUnitKind::OptionIdentifier>(Syntax& s) {
                 .Before = TStringBuilder() << "PRAGMA" << s.Get("WS"),
                 .IsCaseInsensitive = true,
             },
-            {
-                .Body = s.Get("ID_PLAIN"),
-                .Before = TStringBuilder() << "WITH" << s.Get("WS"),
-                .IsCaseInsensitive = true,
-            },
-            {
-                .Body = s.Get("ID_PLAIN"),
-                .After = " ?" + s.Get("EQUALS"),
-                .IsCaseInsensitive = true,
-            }},
+            {Merged(std::move(hints))},
+        },
     };
 }
 
 template <>
 TUnit MakeUnit<EUnitKind::TypeIdentifier>(Syntax& s) {
     TVector<NSQLTranslationV1::TRegexPattern> types;
-    NJson::TJsonValue json = NJson::ReadJsonFastTree(NResource::Find("types.json"));
-    for (const NJson::TJsonValue& value : json.GetArraySafe()) {
-        types.emplace_back(CaseInsensitive(value["name"].GetStringSafe()));
+    for (const TString& type : LoadTypes()) {
+        types.emplace_back(CaseInsensitive(type));
     }
 
     return {
diff --git a/yql/essentials/sql/v1/highlight/ya.make b/yql/essentials/sql/v1/highlight/ya.make
index 103f1908aa0..d6ede67108f 100644
--- a/yql/essentials/sql/v1/highlight/ya.make
+++ b/yql/essentials/sql/v1/highlight/ya.make
@@ -1,6 +1,7 @@
 LIBRARY()
 
 SRCS(
+    data_language_json.cpp
     sql_highlight_json.cpp
     sql_highlight.cpp
     sql_highlighter.cpp
@@ -16,6 +17,7 @@ PEERDIR(
 RESOURCE(
     yql/essentials/sql/v1/highlight/ut/suite.json suite.json
     yql/essentials/data/language/types.json types.json
+    yql/essentials/data/language/statements_opensource.json statements_opensource.json
 )
 
 END()