YQL-19616: Improve TextMate and Vim grammars

- Fixed multiline tokens
- Fixed REAL number display
- Disabled punctuation highlighting
- Refactored by extracting properties
  to core highlighting
commit_hash:a2d1eb6e4e49b1cb785b90accbdecebe977faa13

8 files changed, 69 insertions, 85 deletions

diff --git a/yql/essentials/sql/v1/highlight/sql_highlight.cpp b/yql/essentials/sql/v1/highlight/sql_highlight.cpp
index a95d29e5aeb..97cd0e2f8ce 100644
--- a/yql/essentials/sql/v1/highlight/sql_highlight.cpp
+++ b/yql/essentials/sql/v1/highlight/sql_highlight.cpp
@@ -72,6 +72,8 @@ namespace NSQLHighlight {
         }
 
         unit.Patterns = {Merged(std::move(unit.Patterns))};
+        unit.IsPlain = false;
+        unit.IsCodeGenExcluded = true;
         return unit;
     }
 
@@ -82,6 +84,7 @@ namespace NSQLHighlight {
             .Patterns = {
                 {s.Get("ID_QUOTED")},
             },
+            .IsPlain = false,
         };
     }
 
@@ -92,6 +95,7 @@ namespace NSQLHighlight {
             .Patterns = {
                 {s.Concat({"DOLLAR", "ID_PLAIN"})},
             },
+            .IsPlain = false,
         };
     }
 
@@ -168,9 +172,9 @@ namespace NSQLHighlight {
         return {
             .Kind = EUnitKind::Literal,
             .Patterns = {
-                {s.Get("DIGITS")},
-                {s.Get("INTEGER_VALUE")},
                 {s.Get("REAL")},
+                {s.Get("INTEGER_VALUE")},
+                {s.Get("DIGITS")},
             },
         };
     }
@@ -183,6 +187,11 @@ namespace NSQLHighlight {
             .PatternsANSI = TVector<TRegexPattern>{
                 TRegexPattern{s.Get("STRING_VALUE", /* ansi = */ true)},
             },
+            .RangePattern = TRangePattern{
+                .Begin = "@@",
+                .End = "@@",
+            },
+            .IsPlain = false,
         };
     }
 
@@ -192,6 +201,11 @@ namespace NSQLHighlight {
             .Kind = EUnitKind::Comment,
             .Patterns = {{s.Get("COMMENT")}},
             .PatternsANSI = Nothing(),
+            .RangePattern = TRangePattern{
+                .Begin = R"re(/\*)re",
+                .End = R"re(\*/)re",
+            },
+            .IsPlain = false,
         };
     }
 
@@ -202,6 +216,8 @@ namespace NSQLHighlight {
             .Patterns = {
                 {s.Get("WS")},
             },
+            .IsPlain = false,
+            .IsCodeGenExcluded = true,
         };
     }
 
@@ -228,12 +244,12 @@ namespace NSQLHighlight {
 
         THighlighting h;
         h.Units.emplace_back(MakeUnit<EUnitKind::Comment>(s));
-        h.Units.emplace_back(MakeUnit<EUnitKind::Keyword>(s));
         h.Units.emplace_back(MakeUnit<EUnitKind::Punctuation>(s));
+        h.Units.emplace_back(MakeUnit<EUnitKind::TypeIdentifier>(s));
+        h.Units.emplace_back(MakeUnit<EUnitKind::Keyword>(s));
         h.Units.emplace_back(MakeUnit<EUnitKind::QuotedIdentifier>(s));
         h.Units.emplace_back(MakeUnit<EUnitKind::BindParameterIdentifier>(s));
         h.Units.emplace_back(MakeUnit<EUnitKind::FunctionIdentifier>(s));
-        h.Units.emplace_back(MakeUnit<EUnitKind::TypeIdentifier>(s));
         h.Units.emplace_back(MakeUnit<EUnitKind::Identifier>(s));
         h.Units.emplace_back(MakeUnit<EUnitKind::Literal>(s));
         h.Units.emplace_back(MakeUnit<EUnitKind::StringLiteral>(s));
diff --git a/yql/essentials/sql/v1/highlight/sql_highlight.h b/yql/essentials/sql/v1/highlight/sql_highlight.h
index 43c46c39d53..3d3fcdf7f3e 100644
--- a/yql/essentials/sql/v1/highlight/sql_highlight.h
+++ b/yql/essentials/sql/v1/highlight/sql_highlight.h
@@ -27,10 +27,18 @@ namespace NSQLHighlight {
         Error,
     };
 
+    struct TRangePattern {
+        TString Begin;
+        TString End;
+    };
+
     struct TUnit {
         EUnitKind Kind;
         TVector<NSQLTranslationV1::TRegexPattern> Patterns;
         TMaybe<TVector<NSQLTranslationV1::TRegexPattern>> PatternsANSI;
+        TMaybe<TRangePattern> RangePattern;
+        bool IsPlain = true;
+        bool IsCodeGenExcluded = false;
     };
 
     struct THighlighting {
diff --git a/yql/essentials/sql/v1/highlight/ut/suite.json b/yql/essentials/sql/v1/highlight/ut/suite.json
index 26fb2f3f4f4..04e961a6f91 100644
--- a/yql/essentials/sql/v1/highlight/ut/suite.json
+++ b/yql/essentials/sql/v1/highlight/ut/suite.json
@@ -47,11 +47,11 @@
     ["SELECT id, alias from users", "KKKKKK#_#II#P#_#IIIII#_#KKKK#_#IIIII"],
     ["INSERT INTO users (id, alias) VALUES (12, \"tester\")", "KKKKKK#_#KKKK#_#IIIII#_#P#II#P#_#IIIII#P#_#KKKKKK#_#P#LL#P#_#SSSSSSSS#P"],
     ["SELECT 123467, \"HeLLo, {name}!\", (1 + (5 * 1 / 0)), MIN(identifier) FROM `local/test/space/table` JOIN test;", "KKKKKK#_#LLLLLL#P#_#SSSSSSSSSSSSSSSS#P#_#P#L#_#P#_#P#L#_#P#_#L#_#P#_#L#P#P#P#_#FFF#P#IIIIIIIIII#P#_#KKKK#_#QQQQQQQQQQQQQQQQQQQQQQQQ#_#KKKK#_#IIII#P"],
-    ["SELECT Bool(phone) FROM customer", "KKKKKK#_#FFFF#P#IIIII#P#_#KKKK#_#IIIIIIII"]
+    ["SELECT Bool(phone) FROM customer", "KKKKKK#_#TTTT#P#IIIII#P#_#KKKK#_#IIIIIIII"]
   ],
   "TypeIdentifier": [
     ["Bool", "TTTT"],
-    ["Bool(value)", "FFFF#P#IIIII#P"]
+    ["Bool(value)", "TTTT#P#IIIII#P"]
   ],
   "Identifier": [
     ["test", "IIII"]
diff --git a/yql/essentials/tools/yql_highlight/generate.cpp b/yql/essentials/tools/yql_highlight/generate.cpp
deleted file mode 100644
index ba623cec9f6..00000000000
--- a/yql/essentials/tools/yql_highlight/generate.cpp
+++ /dev/null
@@ -1,17 +0,0 @@
-#include "generate.h"
-
-namespace NSQLHighlight {
-
-    bool IsPlain(EUnitKind kind) {
-        return (kind != EUnitKind::Comment) &&
-               (kind != EUnitKind::StringLiteral) &&
-               (kind != EUnitKind::QuotedIdentifier) &&
-               (kind != EUnitKind::BindParameterIdentifier);
-    }
-
-    bool IsIgnored(EUnitKind kind) {
-        return kind == EUnitKind::Whitespace ||
-               kind == EUnitKind::Error;
-    }
-
-} // namespace NSQLHighlight
diff --git a/yql/essentials/tools/yql_highlight/generate.h b/yql/essentials/tools/yql_highlight/generate.h
deleted file mode 100644
index 384590d2eb5..00000000000
--- a/yql/essentials/tools/yql_highlight/generate.h
+++ /dev/null
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <yql/essentials/sql/v1/highlight/sql_highlight.h>
-
-namespace NSQLHighlight {
-
-    bool IsPlain(EUnitKind kind);
-
-    bool IsIgnored(EUnitKind kind);
-
-} // namespace NSQLHighlight
diff --git a/yql/essentials/tools/yql_highlight/generate_textmate.cpp b/yql/essentials/tools/yql_highlight/generate_textmate.cpp
index 76677563f38..28244c323d4 100644
--- a/yql/essentials/tools/yql_highlight/generate_textmate.cpp
+++ b/yql/essentials/tools/yql_highlight/generate_textmate.cpp
@@ -1,6 +1,5 @@
 #include "generate_textmate.h"
 
-#include "generate.h"
 #include "json.h"
 
 #include <library/cpp/json/json_value.h>
@@ -12,14 +11,14 @@ namespace NSQLHighlight {
 
     namespace {
 
-        TString ToTextMateRegex(EUnitKind kind, const NSQLTranslationV1::TRegexPattern& pattern) {
+        TString ToTextMateRegex(const TUnit& unit, const NSQLTranslationV1::TRegexPattern& pattern) {
             TStringBuilder regex;
 
             if (pattern.IsCaseInsensitive) {
                 regex << "(?i)";
             }
 
-            if (IsPlain(kind)) {
+            if (unit.IsPlain) {
                 regex << R"re(\b)re";
             }
 
@@ -29,7 +28,7 @@ namespace NSQLHighlight {
                 regex << "(?=" << pattern.After << ")";
             }
 
-            if (IsPlain(kind)) {
+            if (unit.IsPlain) {
                 regex << R"re(\b)re";
             }
 
@@ -41,23 +40,23 @@ namespace NSQLHighlight {
                 case EUnitKind::Keyword:
                     return "keyword.control";
                 case EUnitKind::Punctuation:
-                    return "keyword.operator.custom";
+                    return "keyword.operator";
                 case EUnitKind::QuotedIdentifier:
-                    return "string.quoted.double.custom";
+                    return "string.interpolated";
                 case EUnitKind::BindParameterIdentifier:
-                    return "variable.other.dollar.custom";
+                    return "variable.parameter";
                 case EUnitKind::TypeIdentifier:
                     return "entity.name.type";
                 case EUnitKind::FunctionIdentifier:
                     return "entity.name.function";
                 case EUnitKind::Identifier:
-                    return "variable.other.custom";
+                    return "variable.other";
                 case EUnitKind::Literal:
-                    return "constant.numeric.custom";
+                    return "constant.numeric";
                 case EUnitKind::StringLiteral:
-                    return "string.quoted.double.custom";
+                    return "string.quoted.double";
                 case EUnitKind::Comment:
-                    return "comment.block.custom";
+                    return "comment.block";
                 case EUnitKind::Whitespace:
                     return "";
                 case EUnitKind::Error:
@@ -65,37 +64,23 @@ namespace NSQLHighlight {
             }
         }
 
-        TMaybe<std::tuple<TStringBuf, TStringBuf>> TextMateRange(EUnitKind kind) {
-            switch (kind) {
-                case EUnitKind::Comment: {
-                    return std::make_tuple(R"re(/\*)re", R"re(\*/)re");
-                } break;
-                case EUnitKind::StringLiteral: {
-                    return std::make_tuple("@@", "@@");
-                } break;
-                default: {
-                    return Nothing();
-                } break;
-            }
-        }
-
-        TMaybe<NJson::TJsonMap> TextMateMultilinePattern(EUnitKind kind) {
-            auto range = TextMateRange(kind);
+        TMaybe<NJson::TJsonMap> TextMateMultilinePattern(const TUnit& unit) {
+            auto range = unit.RangePattern;
             if (!range) {
                 return Nothing();
             }
 
             return NJson::TJsonMap({
-                {"begin", std::get<0>(*range)},
-                {"end", std::get<1>(*range)},
-                {"name", ToTextMateGroup(kind)},
+                {"begin", range->Begin},
+                {"end", range->End},
+                {"name", ToTextMateGroup(unit.Kind)},
             });
         }
 
-        NJson::TJsonMap ToTextMatePattern(EUnitKind kind, const NSQLTranslationV1::TRegexPattern& pattern) {
+        NJson::TJsonMap ToTextMatePattern(const TUnit& unit, const NSQLTranslationV1::TRegexPattern& pattern) {
             return NJson::TJsonMap({
-                {"match", ToTextMateRegex(kind, pattern)},
-                {"name", ToTextMateGroup(kind)},
+                {"match", ToTextMateRegex(unit, pattern)},
+                {"name", ToTextMateGroup(unit.Kind)},
             });
         }
 
@@ -110,10 +95,10 @@ namespace NSQLHighlight {
         root["$schema"] = "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json";
         root["name"] = "yql";
         root["scopeName"] = "source.yql";
-        root["fileTypes"] = NJson::TJsonArray({"sql", "yql"});
+        root["fileTypes"] = NJson::TJsonArray({"yql"});
 
         for (const TUnit& unit : highlighting.Units) {
-            if (IsIgnored(unit.Kind)) {
+            if (unit.IsCodeGenExcluded) {
                 continue;
             }
 
@@ -124,11 +109,11 @@ namespace NSQLHighlight {
             }));
 
             for (const NSQLTranslationV1::TRegexPattern& pattern : unit.Patterns) {
-                auto textmate = ToTextMatePattern(unit.Kind, pattern);
+                auto textmate = ToTextMatePattern(unit, pattern);
                 root["repository"][name]["patterns"].AppendValue(std::move(textmate));
             }
 
-            if (auto textmate = TextMateMultilinePattern(unit.Kind)) {
+            if (auto textmate = TextMateMultilinePattern(unit)) {
                 root["repository"][name]["patterns"].AppendValue(*textmate);
             }
         }
diff --git a/yql/essentials/tools/yql_highlight/generate_vim.cpp b/yql/essentials/tools/yql_highlight/generate_vim.cpp
index 579513c3cc3..c93442cf5f4 100644
--- a/yql/essentials/tools/yql_highlight/generate_vim.cpp
+++ b/yql/essentials/tools/yql_highlight/generate_vim.cpp
@@ -1,7 +1,5 @@
 #include "generate_vim.h"
 
-#include "generate.h"
-
 #include <yql/essentials/utils/yql_panic.h>
 
 #include <contrib/libs/re2/re2/re2.h>
@@ -29,13 +27,13 @@ namespace NSQLHighlight {
             return regex;
         }
 
-        TString ToVim(EUnitKind kind, const NSQLTranslationV1::TRegexPattern& pattern) {
+        TString ToVim(const TUnit& unit, const NSQLTranslationV1::TRegexPattern& pattern) {
             TStringBuilder vim;
 
             vim << R"(")";
             vim << R"(\v)";
 
-            if (IsPlain(kind)) {
+            if (unit.IsPlain) {
                 vim << R"(<)";
             }
 
@@ -49,7 +47,7 @@ namespace NSQLHighlight {
                 vim << "(" << ToVim(pattern.After) << ")@=";
             }
 
-            if (IsPlain(kind)) {
+            if (unit.IsPlain) {
                 vim << R"(>)";
             }
 
@@ -89,9 +87,15 @@ namespace NSQLHighlight {
 
         void PrintRules(IOutputStream& out, const TUnit& unit) {
             TString name = ToVimName(unit.Kind);
-            for (const NSQLTranslationV1::TRegexPattern& pattern : unit.Patterns) {
+            for (const auto& pattern : std::ranges::reverse_view(unit.Patterns)) {
                 out << "syn match " << ToVimName(unit.Kind) << " "
-                    << ToVim(unit.Kind, pattern) << '\n';
+                    << ToVim(unit, pattern) << '\n';
+            }
+            if (auto range = unit.RangePattern) {
+                out << "syntax region " << name << "Multiline" << " "
+                    << "start=\"" << range->Begin << "\" "
+                    << "end=\"" << range->End << "\""
+                    << '\n';
             }
         }
 
@@ -127,15 +131,13 @@ namespace NSQLHighlight {
     } // namespace
 
     void GenerateVim(IOutputStream& out, const THighlighting& highlighting) {
-        const auto units = std::ranges::reverse_view(highlighting.Units);
-
         out << "if exists(\"b:current_syntax\")" << '\n';
         out << "  finish" << '\n';
         out << "endif" << '\n';
         out << '\n';
 
-        for (const TUnit& unit : units) {
-            if (IsIgnored(unit.Kind)) {
+        for (const TUnit& unit : std::ranges::reverse_view(highlighting.Units)) {
+            if (unit.IsCodeGenExcluded) {
                 continue;
             }
 
@@ -144,9 +146,11 @@ namespace NSQLHighlight {
 
         out << '\n';
 
-        for (const TUnit& unit : units) {
+        for (const TUnit& unit : std::ranges::reverse_view(highlighting.Units)) {
+            TString name = ToVimName(unit.Kind);
             for (TStringBuf group : ToVimGroups(unit.Kind)) {
-                out << "highlight default link " << ToVimName(unit.Kind) << " " << group << '\n';
+                out << "highlight default link " << name << "Multiline" << " " << group << '\n';
+                out << "highlight default link " << name << " " << group << '\n';
             }
         }
 
diff --git a/yql/essentials/tools/yql_highlight/ya.make b/yql/essentials/tools/yql_highlight/ya.make
index f846c88e988..c255bcef95d 100644
--- a/yql/essentials/tools/yql_highlight/ya.make
+++ b/yql/essentials/tools/yql_highlight/ya.make
@@ -12,7 +12,6 @@ PEERDIR(
 SRCS(
     generate_textmate.cpp
     generate_vim.cpp
-    generate.cpp
     json.cpp
     yql_highlight.cpp
 )