YQL-19616: Generate YQLs syntax highlighting

- Support `Before` at core `TRegexPattern` and `IGenericLexer`.
- Added `Name` and `Extension` to core `THighlighting`.
- Added `Tighlighting` for `YQLs` factory method.
- Added `--language` option to `yql_highlight`.
- Added `artifact` targets for `YQLs`.

Yes, using the `NSQLTranslation::THighlighting` for
`YQLs` is not correct, but much simplier than generalize
this infrastructure just for a `YQLs`. So here is a
trade-off between development time and a clean code.

Results:
- JetBrains: https://nda.ya.ru/t/PXkZVE8m7H5wHS.
- Vim: https://nda.ya.ru/t/Am-6ZHQa7H5wJi.
- TextMate: https://nda.ya.ru/t/wH0YggAf7H5wKw.
- yql_highlight: https://nda.ya.ru/t/3FaCm57q7H7QSF.
commit_hash:f0e1abb8e7f1b083df531d761b357330bd514cb0

10 files changed, 225 insertions, 39 deletions

diff --git a/yql/essentials/sql/v1/highlight/sql_highlight.h b/yql/essentials/sql/v1/highlight/sql_highlight.h
index 3d3fcdf7f3e..79a53629614 100644
--- a/yql/essentials/sql/v1/highlight/sql_highlight.h
+++ b/yql/essentials/sql/v1/highlight/sql_highlight.h
@@ -42,6 +42,8 @@ namespace NSQLHighlight {
     };
 
     struct THighlighting {
+        TString Name = "YQL";
+        TString Extension = "yql";
         TVector<TUnit> Units;
     };
 
diff --git a/yql/essentials/sql/v1/lexer/regex/generic.cpp b/yql/essentials/sql/v1/lexer/regex/generic.cpp
index 926c50dde2c..3603a31f690 100644
--- a/yql/essentials/sql/v1/lexer/regex/generic.cpp
+++ b/yql/essentials/sql/v1/lexer/regex/generic.cpp
@@ -41,16 +41,27 @@ namespace NSQLTranslationV1 {
             }
 
             while (pos < text.size() && errors < maxErrors) {
-                TGenericToken matched = Match(TStringBuf(text, pos));
-                matched.Begin = pos;
+                TMaybe<TGenericToken> prev;
+                TGenericToken next = Match(TStringBuf(text, pos));
 
-                pos += matched.Content.size();
+                size_t skipped = next.Begin;
+                next.Begin = skipped + pos;
 
-                if (matched.Name == TGenericToken::Error) {
+                if (skipped != 0) {
+                    prev = Match(TStringBuf(text, pos, skipped));
+                    prev->Begin = pos;
+                }
+
+                pos += skipped + next.Content.size();
+
+                if (next.Name == TGenericToken::Error) {
                     errors += 1;
                 }
 
-                onNext(std::move(matched));
+                if (prev) {
+                    onNext(std::move(*prev));
+                }
+                onNext(std::move(next));
             }
 
             if (errors == maxErrors) {
@@ -100,15 +111,18 @@ namespace NSQLTranslationV1 {
         RE2::Options options;
         options.set_case_sensitive(!regex.IsCaseInsensitive);
 
-        return [bodyRe = MakeAtomicShared<RE2>(regex.Body, options),
+        return [beforeRe = MakeAtomicShared<RE2>(regex.Before, options),
+                bodyRe = MakeAtomicShared<RE2>(regex.Body, options),
                 afterRe = MakeAtomicShared<RE2>(regex.After, options),
                 name = std::move(name)](TStringBuf prefix) -> TMaybe<TGenericToken> {
-            TMaybe<TStringBuf> body, after;
-            if ((body = Match(prefix, *bodyRe)) &&
-                (after = Match(prefix.Tail(body->size()), *afterRe))) {
+            TMaybe<TStringBuf> before, body, after;
+            if ((before = Match(prefix, *beforeRe)) &&
+                (body = Match(prefix.Tail(before->size()), *bodyRe)) &&
+                (after = Match(prefix.Tail(before->size() + body->size()), *afterRe))) {
                 return TGenericToken{
                     .Name = name,
                     .Content = *body,
+                    .Begin = before->size(),
                 };
             }
             return Nothing();
@@ -120,8 +134,8 @@ namespace NSQLTranslationV1 {
 
         const TRegexPattern& sample = patterns.back();
         Y_ENSURE(AllOf(patterns, [&](const TRegexPattern& pattern) {
-            return std::tie(pattern.After, pattern.IsCaseInsensitive) ==
-                   std::tie(sample.After, sample.IsCaseInsensitive);
+            return std::tie(pattern.After, pattern.Before, pattern.IsCaseInsensitive) ==
+                   std::tie(sample.After, sample.Before, sample.IsCaseInsensitive);
         }));
 
         Sort(patterns, [](const TRegexPattern& lhs, const TRegexPattern& rhs) {
@@ -143,6 +157,7 @@ namespace NSQLTranslationV1 {
         return TRegexPattern{
             .Body = std::move(body),
             .After = sample.After,
+            .Before = sample.Before,
             .IsCaseInsensitive = sample.IsCaseInsensitive,
         };
     }
diff --git a/yql/essentials/sql/v1/lexer/regex/generic.h b/yql/essentials/sql/v1/lexer/regex/generic.h
index efbac67315a..60c2a53207c 100644
--- a/yql/essentials/sql/v1/lexer/regex/generic.h
+++ b/yql/essentials/sql/v1/lexer/regex/generic.h
@@ -39,6 +39,7 @@ namespace NSQLTranslationV1 {
     struct TRegexPattern {
         TString Body;
         TString After = "";
+        TString Before = "";
         bool IsCaseInsensitive = false;
     };
 
diff --git a/yql/essentials/tools/yql_highlight/artifact/ya.make b/yql/essentials/tools/yql_highlight/artifact/ya.make
index b9d54aefba3..6a9a393fa19 100644
--- a/yql/essentials/tools/yql_highlight/artifact/ya.make
+++ b/yql/essentials/tools/yql_highlight/artifact/ya.make
@@ -11,18 +11,38 @@ RUN_PROGRAM(
 )
 
 RUN_PROGRAM(
+    yql/essentials/tools/yql_highlight --language="yqls" --generate="tmlanguage"
+    STDOUT YQLs.tmLanguage.json
+)
+
+RUN_PROGRAM(
     yql/essentials/tools/yql_highlight --generate="vim"
     STDOUT yql_new.vim
 )
 
 RUN_PROGRAM(
+    yql/essentials/tools/yql_highlight --language="yqls" --generate="vim"
+    STDOUT yqls.vim
+)
+
+RUN_PROGRAM(
     yql/essentials/tools/yql_highlight --generate="tmbundle" --output="${BINDIR}/YQL.tmbundle.zip"
     OUT ${BINDIR}/YQL.tmbundle.zip
 )
 
 RUN_PROGRAM(
+    yql/essentials/tools/yql_highlight -l "yqls" -g "tmbundle" -o "${BINDIR}/YQLs.tmbundle.zip"
+    OUT ${BINDIR}/YQLs.tmbundle.zip
+)
+
+RUN_PROGRAM(
     yql/essentials/tools/yql_highlight --generate="tmbundle" --output="${BINDIR}/YQL.tmbundle.tar"
     OUT ${BINDIR}/YQL.tmbundle.tar
 )
 
+RUN_PROGRAM(
+    yql/essentials/tools/yql_highlight -l "yqls" -g "tmbundle" -o "${BINDIR}/YQLs.tmbundle.tar"
+    OUT ${BINDIR}/YQLs.tmbundle.tar
+)
+
 END()
diff --git a/yql/essentials/tools/yql_highlight/generator_textmate.cpp b/yql/essentials/tools/yql_highlight/generator_textmate.cpp
index 7a0a66c0e3d..2c90cde77b9 100644
--- a/yql/essentials/tools/yql_highlight/generator_textmate.cpp
+++ b/yql/essentials/tools/yql_highlight/generator_textmate.cpp
@@ -49,6 +49,10 @@ namespace NSQLHighlight {
                 regex << R"re(\b)re";
             }
 
+            if (!pattern.Before.empty()) {
+                regex << "(?<=" << pattern.Before << ")";
+            }
+
             regex << "(" << pattern.Body << ")";
 
             if (!pattern.After.empty()) {
@@ -123,9 +127,9 @@ namespace NSQLHighlight {
 
     NTextMate::TLanguage ToTextMateLanguage(const THighlighting& highlighting) {
         NTextMate::TLanguage language = {
-            .Name = "YQL",
-            .ScopeName = "source.yql",
-            .FileTypes = {"yql"},
+            .Name = highlighting.Name,
+            .ScopeName = "source." + highlighting.Extension,
+            .FileTypes = {highlighting.Extension},
         };
 
         for (const TUnit& unit : highlighting.Units) {
@@ -166,6 +170,7 @@ namespace NSQLHighlight {
         root["$schema"] = "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json";
         root["name"] = language.Name;
         root["scopeName"] = language.ScopeName;
+        root["scope"] = language.ScopeName;
 
         for (const TString& type : language.FileTypes) {
             root["fileTypes"].AppendValue(type);
@@ -219,14 +224,15 @@ namespace NSQLHighlight {
         Print(out, ToJson(ToTextMateLanguage(highlighting)));
     }
 
+    static const THashMap<TString, TString> UUID = {
+        {"InfoYQL", "059de4a7-ff49-4dbd-8a9d-a8114b77c4b9"},
+        {"SyntaxYQL", "bb7a80e5-733c-4ea6-9654-40db0675950c"},
+        {"InfoYQLs", "7f536d44-2667-430e-b145-540992400cb3"},
+        {"SyntaxYQLs", "6e62e13a-487b-4333-bbb2-9453d0783f8f"},
+    };
+
     class TTextMateBundleGenerator: public IGenerator {
     private:
-        static constexpr TStringBuf InfoUUID = "9BB0DBAF-E65C-4E14-A6A7-467D4AA535E0";
-        static constexpr TStringBuf SyntaxUUID = "1C3868E4-F96B-4E55-B204-1DCB5A20748B";
-        static constexpr TStringBuf BundleDir = "YQL.tmbundle";
-        static constexpr TStringBuf InfoFile = "info.plist";
-        static constexpr TStringBuf SyntaxFile = "Syntaxes/YQL.tmLanguage";
-
         template <class TWriter>
         void Write(
             NTar::TArchiveWriter& acrhive,
@@ -242,28 +248,40 @@ namespace NSQLHighlight {
 
     public:
         void Write(IOutputStream& out, const THighlighting& highlighting) final {
-            out << "File " << BundleDir << "/" << InfoFile << ":" << '\n';
+            const auto [bundle, info, syntax] = Paths(highlighting);
+
+            out << "File " << bundle << "/" << info << ":" << '\n';
             WriteInfo(out, ToTextMateLanguage(highlighting));
-            out << "File " << BundleDir << "/" << SyntaxFile << ":" << '\n';
+            out << "File " << bundle << "/" << syntax << ":" << '\n';
             WriteSyntax(out, ToTextMateLanguage(highlighting));
         }
 
         void Write(const TFsPath& path, const THighlighting& highlighting) final {
-            if (TString name = path.GetName(); !name.StartsWith(BundleDir)) {
+            const auto [bundle, info, syntax] = Paths(highlighting);
+
+            if (TString name = path.GetName(); !name.StartsWith(bundle)) {
                 ythrow yexception()
                     << "Invalid path '" << name
-                    << "', expected '" << BundleDir << "' "
+                    << "', expected '" << bundle << "' "
                     << "as an archive name";
             }
 
             NTextMate::TLanguage language = ToTextMateLanguage(highlighting);
 
             NTar::TArchiveWriter archive(path);
-            Write(archive, InfoFile, WriteInfo, language);
-            Write(archive, SyntaxFile, WriteSyntax, language);
+            Write(archive, info, WriteInfo, language);
+            Write(archive, syntax, WriteSyntax, language);
         }
 
     private:
+        static std::tuple<TString, TString, TString> Paths(const THighlighting& h) {
+            return {
+                TStringBuilder() << h.Name << ".tmbundle",
+                TStringBuilder() << "info.plist",
+                TStringBuilder() << "Syntaxes/" << h.Name << ".tmLanguage",
+            };
+        }
+
         static void WriteInfo(IOutputStream& out, const NTextMate::TLanguage& language) {
             out << R"(<?xml version="1.0" encoding="UTF-8"?>)" << '\n';
             out << R"(<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">)" << '\n';
@@ -272,7 +290,7 @@ namespace NSQLHighlight {
             out << R"(    <key>name</key>)" << '\n';
             out << R"(    <string>)" << language.Name << R"(</string>)" << '\n';
             out << R"(    <key>uuid</key>)" << '\n';
-            out << R"(    <string>)" << InfoUUID << R"(</string>)" << '\n';
+            out << R"(    <string>)" << UUID.at("Info" + language.Name) << R"(</string>)" << '\n';
             out << R"(</dict>)" << '\n';
             out << R"(</plist>)" << '\n';
         }
@@ -280,7 +298,7 @@ namespace NSQLHighlight {
         static void WriteSyntax(IOutputStream& out, const NTextMate::TLanguage& language) {
             NJson::TJsonValue json = ToJson(language);
             json.EraseValue("$schema");
-            json["uuid"] = SyntaxUUID;
+            json["uuid"] = UUID.at("Syntax" + language.Name);
 
             out << R"(<?xml version="1.0" encoding="UTF-8"?>)" << '\n';
             out << R"(<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">)" << '\n';
diff --git a/yql/essentials/tools/yql_highlight/generator_vim.cpp b/yql/essentials/tools/yql_highlight/generator_vim.cpp
index bb33157ce2e..9804b818ead 100644
--- a/yql/essentials/tools/yql_highlight/generator_vim.cpp
+++ b/yql/essentials/tools/yql_highlight/generator_vim.cpp
@@ -41,6 +41,10 @@ namespace NSQLHighlight {
                 vim << R"(\c)";
             }
 
+            if (!pattern.Before.empty()) {
+                vim << "(" << ToVim(pattern.Before) << ")@<=";
+            }
+
             vim << "(" << ToVim(pattern.Body) << ")";
 
             if (!pattern.After.empty()) {
@@ -161,7 +165,7 @@ namespace NSQLHighlight {
 
         out << '\n';
 
-        out << "let b:current_syntax = \"yql\"" << '\n';
+        out << "let b:current_syntax = \"" << highlighting.Extension << "\"" << '\n';
         out.Flush();
     }
 
diff --git a/yql/essentials/tools/yql_highlight/ya.make b/yql/essentials/tools/yql_highlight/ya.make
index 710d7ff40e6..ef13d815b6b 100644
--- a/yql/essentials/tools/yql_highlight/ya.make
+++ b/yql/essentials/tools/yql_highlight/ya.make
@@ -16,6 +16,7 @@ IF (NOT EXPORT_CMAKE OR NOT OPENSOURCE OR OPENSOURCE_PROJECT != "yt")
         generator.cpp
         json.cpp
         yql_highlight.cpp
+        yqls_highlight.cpp
     )
 
     END()
diff --git a/yql/essentials/tools/yql_highlight/yql_highlight.cpp b/yql/essentials/tools/yql_highlight/yql_highlight.cpp
index 4361ee6e106..43cb0d4b9c6 100644
--- a/yql/essentials/tools/yql_highlight/yql_highlight.cpp
+++ b/yql/essentials/tools/yql_highlight/yql_highlight.cpp
@@ -1,6 +1,7 @@
 #include "generator_json.h"
 #include "generator_textmate.h"
 #include "generator_vim.h"
+#include "yqls_highlight.h"
 
 #include <yql/essentials/sql/v1/highlight/sql_highlight.h>
 #include <yql/essentials/sql/v1/highlight/sql_highlighter.h>
@@ -13,10 +14,17 @@
 
 using namespace NSQLHighlight;
 
-using TGeneratorFactory = std::function<IGenerator::TPtr()>;
+using THighlightingFactory = std::function<THighlighting()>;
+using THighlightingMap = THashMap<TString, THighlightingFactory>;
 
+using TGeneratorFactory = std::function<IGenerator::TPtr()>;
 using TGeneratorMap = THashMap<TString, TGeneratorFactory>;
 
+const THighlightingMap highlightings = {
+    {"yql", [] { return MakeHighlighting(); }},
+    {"yqls", [] { return MakeYQLsHighlighting(); }},
+};
+
 const TGeneratorMap generators = {
     {"json", MakeJsonGenerator},
     {"tmlanguage", MakeTextMateJsonGenerator},
@@ -24,15 +32,16 @@ const TGeneratorMap generators = {
     {"vim", MakeVimGenerator},
 };
 
-const TVector<TString> targets = []() {
+template <class TMap>
+TVector<TString> Keys(const TMap& map) {
     TVector<TString> result;
-    for (const auto& [name, _] : generators) {
+    for (const auto& [name, _] : map) {
         result.push_back(name);
     }
     return result;
-}();
+}
 
-int RunHighlighter() {
+int RunHighlighter(const THighlighting& highlighting) {
     THashMap<EUnitKind, NColorizer::EAnsiCode> ColorByKind = {
         {EUnitKind::Keyword, NColorizer::BLUE},
         {EUnitKind::Punctuation, NColorizer::DARK_WHITE},
@@ -50,7 +59,6 @@ int RunHighlighter() {
 
     TString query = Cin.ReadAll();
 
-    THighlighting highlighting = MakeHighlighting();
     IHighlighter::TPtr highlighter = MakeHighlighter(highlighting);
     TVector<TToken> tokens = Tokenize(*highlighter, query);
 
@@ -63,13 +71,19 @@ int RunHighlighter() {
 }
 
 int Run(int argc, char* argv[]) {
+    TString syntax;
     TString target;
     TString path;
 
     NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default();
+    opts.AddLongOption('l', "language", "choice a syntax")
+        .RequiredArgument("syntax")
+        .Choices(Keys(highlightings))
+        .DefaultValue("yql")
+        .StoreResult(&syntax);
     opts.AddLongOption('g', "generate", "generate a highlighting configuration")
         .RequiredArgument("target")
-        .Choices(targets)
+        .Choices(Keys(generators))
         .StoreResult(&target);
     opts.AddLongOption('o', "output", "path to output file")
         .OptionalArgument("path")
@@ -78,21 +92,27 @@ int Run(int argc, char* argv[]) {
     opts.AddHelpOption();
 
     NLastGetopt::TOptsParseResult res(&opts, argc, argv);
+
+    const THighlightingFactory* factory = highlightings.FindPtr(syntax);
+    Y_ENSURE(factory, "No highlighting for syntax '" << syntax << "'");
+
+    THighlighting highlighting = (*factory)();
+
     if (res.Has("generate")) {
         const TGeneratorFactory* generator = generators.FindPtr(target);
         Y_ENSURE(generator, "No generator for target '" << target << "'");
 
         if (res.Has("output")) {
             TFsPath stdpath(path.c_str());
-            (*generator)()->Write(stdpath, MakeHighlighting());
+            (*generator)()->Write(stdpath, highlighting);
         } else {
-            (*generator)()->Write(Cout, MakeHighlighting());
+            (*generator)()->Write(Cout, highlighting);
         }
 
         return 0;
     }
 
-    return RunHighlighter();
+    return RunHighlighter(highlighting);
 }
 
 int main(int argc, char* argv[]) try {
diff --git a/yql/essentials/tools/yql_highlight/yqls_highlight.cpp b/yql/essentials/tools/yql_highlight/yqls_highlight.cpp
new file mode 100644
index 00000000000..40ffb2cf4ce
--- /dev/null
+++ b/yql/essentials/tools/yql_highlight/yqls_highlight.cpp
@@ -0,0 +1,96 @@
+#include "yqls_highlight.h"
+
+namespace NSQLHighlight {
+
+    using TRe = NSQLTranslationV1::TRegexPattern;
+    using NSQLTranslationV1::Merged;
+
+    THighlighting MakeYQLsHighlighting() {
+        TString id = R"re([A-Za-z_\-0-9]+)re";
+
+        TRe keywords = Merged({
+            {"let"},
+            {"return"},
+            {"quote"},
+            {"block"},
+            {"lambda"},
+            {"declare"},
+            {"import"},
+            {"export"},
+            {"library"},
+            {"override_library"},
+            {"package"},
+            {"set_package_version"},
+        });
+        keywords.Before = R"re(\()re";
+
+        return {
+            .Name = "YQLs",
+            .Extension = "yqls",
+            .Units = {
+                TUnit{
+                    .Kind = EUnitKind::Comment,
+                    .Patterns = {TRe{R"re(#.*)re"}},
+                    .IsPlain = false,
+                },
+                TUnit{
+                    .Kind = EUnitKind::Keyword,
+                    .Patterns = {keywords},
+                },
+                TUnit{
+                    .Kind = EUnitKind::BindParameterIdentifier,
+                    .Patterns = {TRe{"world"}},
+                },
+                TUnit{
+                    .Kind = EUnitKind::QuotedIdentifier,
+                    .Patterns = {
+                        TRe{.Body = id + "!", .Before = R"re(\()re"},
+                    },
+                    .IsPlain = false,
+                },
+                TUnit{
+                    .Kind = EUnitKind::FunctionIdentifier,
+                    .Patterns = {
+                        TRe{.Body = id, .Before = R"re(\()re"},
+                        TRe{.Body = "'" + id + "\\." + id},
+                    },
+                    .IsPlain = false,
+                },
+                TUnit{
+                    .Kind = EUnitKind::Literal,
+                    .Patterns = {TRe{"'" + id}},
+                    .IsPlain = false,
+                },
+                TUnit{
+                    .Kind = EUnitKind::Identifier,
+                    .Patterns = {TRe{id}},
+                },
+                TUnit{
+                    .Kind = EUnitKind::StringLiteral,
+                    .Patterns = {
+                        TRe{R"re(\"[^\"\n]*\")re"},
+                        TRe{R"re(\@\@(.|\n)*\@\@)re"},
+                    },
+                    .RangePattern = TRangePattern{
+                        .Begin = "@@",
+                        .End = "@@",
+                    },
+                    .IsPlain = false,
+                },
+                TUnit{
+                    .Kind = EUnitKind::Punctuation,
+                    .Patterns = {TRe{R"re(['\(\)])re"}},
+                    .IsPlain = false,
+                    .IsCodeGenExcluded = true,
+                },
+                TUnit{
+                    .Kind = EUnitKind::Whitespace,
+                    .Patterns = {TRe{R"re(\s+)re"}},
+                    .IsPlain = false,
+                    .IsCodeGenExcluded = true,
+                },
+            },
+        };
+    }
+
+} // namespace NSQLHighlight
diff --git a/yql/essentials/tools/yql_highlight/yqls_highlight.h b/yql/essentials/tools/yql_highlight/yqls_highlight.h
new file mode 100644
index 00000000000..d803015331b
--- /dev/null
+++ b/yql/essentials/tools/yql_highlight/yqls_highlight.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#include <yql/essentials/sql/v1/highlight/sql_highlight.h>
+
+namespace NSQLHighlight {
+
+    THighlighting MakeYQLsHighlighting();
+
+} // namespace NSQLHighlight