#9227 Switch YDB CLI highlighting engine (#18813)

Signed-off-by: vityaman <vityaman.dev@yandex.ru>

7 files changed, 62 insertions, 572 deletions

diff --git a/ydb/apps/ydb/CHANGELOG.md b/ydb/apps/ydb/CHANGELOG.md
index 222b4bcdf80..68e5b89db36 100644
--- a/ydb/apps/ydb/CHANGELOG.md
+++ b/ydb/apps/ydb/CHANGELOG.md
@@ -1,4 +1,4 @@
-
+* Switched highlighting engine
 * Added `ydb admin cluster config verion` command to show configuration version (V1/V2) on nodes.
 * Removed `--executor` option from `ydb workload run` commands. Use always `generic`.
 * Added object names completion in interactive mode
diff --git a/ydb/public/lib/ydb_cli/commands/interactive/highlight/ya.make b/ydb/public/lib/ydb_cli/commands/interactive/highlight/ya.make
index b6a4045ef8d..f8f1bade66c 100644
--- a/ydb/public/lib/ydb_cli/commands/interactive/highlight/ya.make
+++ b/ydb/public/lib/ydb_cli/commands/interactive/highlight/ya.make
@@ -2,15 +2,11 @@ LIBRARY()
 
 SRCS(
     yql_highlighter.cpp
-    yql_position.cpp
 )
 
 PEERDIR(
     contrib/restricted/patched/replxx
-    yql/essentials/sql/v1/lexer
-    yql/essentials/sql/v1/lexer/antlr4
-    yql/essentials/sql/v1/lexer/antlr4_ansi
-    yql/essentials/sql/settings
+    yql/essentials/sql/v1/highlight
     ydb/public/lib/ydb_cli/commands/interactive/highlight/color
 )
 
diff --git a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.cpp b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.cpp
index d219629b3d7..9b2eafe0acb 100644
--- a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.cpp
+++ b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.cpp
@@ -1,12 +1,7 @@
 #include "yql_highlighter.h"
 
-#include "yql_position.h"
-
-#include <yql/essentials/public/issue/yql_issue.h>
-#include <yql/essentials/sql/settings/translation_settings.h>
-#include <yql/essentials/sql/v1/lexer/lexer.h>
-#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
-#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/highlight/sql_highlight.h>
+#include <yql/essentials/sql/v1/highlight/sql_highlighter.h>
 
 #include <util/charset/utf8.h>
 #include <util/string/strip.h>
@@ -14,258 +9,83 @@
 #include <regex>
 
 namespace NYdb::NConsoleClient {
-    using NSQLTranslation::ILexer;
-    using NSQLTranslation::ParseTranslationSettings;
-    using NSQLTranslation::SQL_MAX_PARSER_ERRORS;
-    using NSQLTranslation::TParsedToken;
-    using NSQLTranslation::TParsedTokenList;
-    using NSQLTranslation::TTranslationSettings;
-    using NSQLTranslationV1::IsProbablyKeyword;
-    using NSQLTranslationV1::MakeLexer;
-    using NYql::TIssues;
-
-    using std::regex_constants::ECMAScript;
-    using std::regex_constants::icase;
-
-    NSQLTranslationV1::TLexers MakeLexers() {
-        NSQLTranslationV1::TLexers lexers;
-        lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
-        lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
-        return lexers;
-    }
-
-    constexpr const char* builtinFunctionPattern = ( //
-        "^("
-        "abs|aggregate_by|aggregate_list|aggregate_list_distinct|agg_list|agg_list_distinct|"
-        "as_table|avg|avg_if|adaptivedistancehistogram|adaptivewardhistogram|adaptiveweighthistogram|"
-        "addmember|addtimezone|aggregateflatten|aggregatetransforminput|aggregatetransformoutput|"
-        "aggregationfactory|asatom|asdict|asdictstrict|asenum|aslist|asliststrict|asset|assetstrict|"
-        "asstruct|astagged|astuple|asvariant|atomcode|bitcast|bit_and|bit_or|bit_xor|bool_and|"
-        "bool_or|bool_xor|bottom|bottom_by|blockwardhistogram|blockweighthistogram|cast|coalesce|"
-        "concat|concat_strict|correlation|count|count_if|covariance|covariance_population|"
-        "covariance_sample|callableargument|callableargumenttype|callableresulttype|callabletype|"
-        "callabletypecomponents|callabletypehandle|choosemembers|combinemembers|countdistinctestimate|"
-        "currentauthenticateduser|currentoperationid|currentoperationsharedid|currenttzdate|"
-        "currenttzdatetime|currenttztimestamp|currentutcdate|currentutcdatetime|currentutctimestamp|"
-        "dense_rank|datatype|datatypecomponents|datatypehandle|dictaggregate|dictcontains|dictcreate|"
-        "dicthasitems|dictitems|dictkeytype|dictkeys|dictlength|dictlookup|dictpayloadtype|dictpayloads|"
-        "dicttype|dicttypecomponents|dicttypehandle|each|each_strict|emptydicttype|emptydicttypehandle|"
-        "emptylisttype|emptylisttypehandle|endswith|ensure|ensureconvertibleto|ensuretype|enum|"
-        "evaluateatom|evaluatecode|evaluateexpr|evaluatetype|expandstruct|filter|filter_strict|find|"
-        "first_value|folder|filecontent|filepath|flattenmembers|forceremovemember|forceremovemembers|"
-        "forcerenamemembers|forcespreadmembers|formatcode|formattype|frombytes|frompg|funccode|"
-        "greatest|grouping|gathermembers|generictype|histogram|hll|hoppingwindowpgcast|hyperloglog|"
-        "if|if_strict|instanceof|json_exists|json_query|json_value|jointablerow|just|lag|last_value|"
-        "lead|least|len|length|like|likely|like_strict|lambdaargumentscount|lambdacode|"
-        "lambdaoptionalargumentscount|linearhistogram|listaggregate|listall|listany|listavg|listcode|"
-        "listcollect|listconcat|listcreate|listdistinct|listenumerate|listextend|listextendstrict|"
-        "listextract|listfilter|listflatmap|listflatten|listfold|listfold1|listfold1map|listfoldmap|"
-        "listfromrange|listfromtuple|listhas|listhasitems|listhead|listindexof|listitemtype|listlast|"
-        "listlength|listmap|listmax|listmin|listnotnull|listreplicate|listreverse|listskip|"
-        "listskipwhile|listskipwhileinclusive|listsort|listsortasc|listsortdesc|listsum|listtake|"
-        "listtakewhile|listtakewhileinclusive|listtotuple|listtype|listtypehandle|listunionall|"
-        "listuniq|listzip|listzipall|loghistogram|logarithmichistogram|max|max_by|max_of|median|"
-        "min|min_by|min_of|mode|multi_aggregate_by|nanvl|nvl|nothing|nulltype|nulltypehandle|"
-        "optionalitemtype|optionaltype|optionaltypehandle|percentile|parsefile|parsetype|"
-        "parsetypehandle|pgand|pgarray|pgcall|pgconst|pgnot|pgop|pgor|pickle|quotecode|range|"
-        "range_strict|rank|regexp|regexp_strict|rfind|row_number|random|randomnumber|randomuuid|"
-        "removemember|removemembers|removetimezone|renamemembers|replacemember|reprcode|resourcetype|"
-        "resourcetypehandle|resourcetypetag|some|stddev|stddev_population|stddev_sample|substring|"
-        "sum|sum_if|sessionstart|sessionwindow|setcreate|setdifference|setincludes|setintersection|"
-        "setisdisjoint|setsymmetricdifference|setunion|spreadmembers|stablepickle|startswith|staticmap|"
-        "staticzip|streamitemtype|streamtype|streamtypehandle|structmembertype|structmembers|"
-        "structtypecomponents|structtypehandle|subqueryextend|subqueryextendfor|subquerymerge|"
-        "subquerymergefor|subqueryunionall|subqueryunionallfor|subqueryunionmerge|subqueryunionmergefor"
-        "|top|topfreq|top_by|tablename|tablepath|tablerecordindex|tablerow|tablerows|taggedtype|"
-        "taggedtypecomponents|taggedtypehandle|tobytes|todict|tomultidict|topg|toset|tosorteddict|"
-        "tosortedmultidict|trymember|tupleelementtype|tupletype|tupletypecomponents|tupletypehandle|"
-        "typehandle|typekind|typeof|udaf|udf|unittype|unpickle|untag|unwrap|variance|variance_population|"
-        "variance_sample|variant|varianttype|varianttypehandle|variantunderlyingtype|voidtype|"
-        "voidtypehandle|way|worldcode|weakfield"
-        ")$");
-
-    constexpr const char* typePattern = ( //
-        "^("
-        "bool|date|datetime|decimal|double|float|int16|int32|int64|int8|interval|json|jsondocument|"
-        "string|timestamp|tzdate|tzdatetime|tztimestamp|uint16|uint32|uint64|uint8|utf8|uuid|yson|"
-        "text|bytes"
-        ")$");
-
-    bool IsAnsiQuery(const TString& queryUtf8) {
-        TTranslationSettings settings;
-        TIssues issues;
-        ParseTranslationSettings(queryUtf8, settings, issues);
-        return settings.AnsiLexer;
-    }
+    using NSQLHighlight::MakeHighlighter;
+    using NSQLHighlight::MakeHighlighting;
 
     class TYQLHighlighter: public IYQLHighlighter {
+    private:
+        static constexpr size_t InvalidPosition = Max<std::ptrdiff_t>();
+
     public:
         explicit TYQLHighlighter(TColorSchema color)
             : Coloring(color)
-            , BuiltinFunctionRegex(builtinFunctionPattern, ECMAScript | icase)
-            , TypeRegex(typePattern, ECMAScript | icase)
-            , CppLexer(MakeLexer(MakeLexers(), /* ansi = */ false, /* antlr4 = */ true))
-            , ANSILexer(MakeLexer(MakeLexers(), /* ansi = */ true, /* antlr4 = */ true))
+            , Highlighter(MakeHighlighter(MakeHighlighting()))
         {
         }
 
-    public:
-        void Apply(TStringBuf queryUtf8, TColors& colors) {
-            Tokens = Tokenize(TString(queryUtf8));
-            auto mapping = TYQLPositionMapping::Build(TString(queryUtf8));
+        void Apply(TStringBuf query, TColors& colors) const override {
+            const TVector<std::ptrdiff_t> symbolIndexByByte = BuildSymbolByByteIndex(query);
+
+            Highlighter->Tokenize(query, [&](NSQLHighlight::TToken&& token) {
+                if (token.Kind == NSQLHighlight::EUnitKind::Error) {
+                    return;
+                }
 
-            for (std::size_t i = 0; i < Tokens.size() - 1; ++i) {
-                const auto& token = Tokens.at(i);
-                const auto color = ColorOf(token, i);
+                TStringBuf content = query.SubString(token.Begin, token.Length);
 
-                const std::ptrdiff_t start = mapping.RawPos(token);
-                const std::ptrdiff_t length = GetNumberOfUTF8Chars(token.Content);
+                const std::ptrdiff_t start = symbolIndexByByte[token.Begin];
+                const size_t length = GetNumberOfUTF8Chars(content);
                 const std::ptrdiff_t stop = start + length;
 
                 std::fill(std::next(std::begin(colors), start),
-                          std::next(std::begin(colors), stop), color);
-            }
+                          std::next(std::begin(colors), stop),
+                          Color(token.Kind));
+            });
         }
 
     private:
-        TParsedTokenList Tokenize(const TString& queryUtf8) {
-            ILexer& lexer = *SuitableLexer(queryUtf8);
-
-            TParsedTokenList tokens;
-            TIssues issues;
-            NSQLTranslation::Tokenize(lexer, queryUtf8, "Query", tokens, issues, SQL_MAX_PARSER_ERRORS);
-            return tokens;
-        }
-
-        ILexer::TPtr& SuitableLexer(const TString& queryUtf8) {
-            if (IsAnsiQuery(queryUtf8)) {
-                return ANSILexer;
+        TColor Color(const NSQLHighlight::EUnitKind& kind) const {
+            switch (kind) {
+                case NSQLHighlight::EUnitKind::Keyword:
+                    return Coloring.keyword;
+                case NSQLHighlight::EUnitKind::Punctuation:
+                    return Coloring.operation;
+                case NSQLHighlight::EUnitKind::QuotedIdentifier:
+                    return Coloring.identifier.quoted;
+                case NSQLHighlight::EUnitKind::BindParamterIdentifier:
+                    return Coloring.identifier.variable;
+                case NSQLHighlight::EUnitKind::TypeIdentifier:
+                    return Coloring.identifier.type;
+                case NSQLHighlight::EUnitKind::FunctionIdentifier:
+                    return Coloring.identifier.function;
+                case NSQLHighlight::EUnitKind::Identifier:
+                    return Coloring.identifier.variable;
+                case NSQLHighlight::EUnitKind::Literal:
+                    return Coloring.number;
+                case NSQLHighlight::EUnitKind::StringLiteral:
+                    return Coloring.string;
+                case NSQLHighlight::EUnitKind::Comment:
+                    return Coloring.comment;
+                case NSQLHighlight::EUnitKind::Whitespace:
+                    return Coloring.unknown;
+                case NSQLHighlight::EUnitKind::Error:
+                    return Coloring.unknown;
+                default:
+                    return Coloring.unknown;
             }
-            return CppLexer;
         }
 
-        TColor ColorOf(const TParsedToken& token, size_t index) {
-            if (IsString(token)) {
-                return Coloring.string;
-            }
-            if (IsFunctionIdentifier(token, index)) {
-                return Coloring.identifier.function;
-            }
-            if (IsTypeIdentifier(token)) {
-                return Coloring.identifier.type;
-            }
-            if (IsVariableIdentifier(token)) {
-                return Coloring.identifier.variable;
-            }
-            if (IsQuotedIdentifier(token)) {
-                return Coloring.identifier.quoted;
-            }
-            if (IsNumber(token)) {
-                return Coloring.number;
-            }
-            if (IsOperation(token)) {
-                return Coloring.operation;
-            }
-            if (IsKeyword(token)) {
-                return Coloring.keyword;
+        static TVector<std::ptrdiff_t> BuildSymbolByByteIndex(TStringBuf text) {
+            TVector<std::ptrdiff_t> index(text.size(), InvalidPosition);
+            for (size_t i = 0, j = 0; i < text.size(); i += UTF8RuneLen(text[i]), j += 1) {
+                index[i] = j;
             }
-            if (IsComment(token)) {
-                return Coloring.comment;
-            }
-            return Coloring.unknown;
-        }
-
-        bool IsKeyword(const TParsedToken& token) const {
-            return IsProbablyKeyword(token);
-        }
-
-        bool IsOperation(const TParsedToken& token) const {
-            return (
-                token.Name == "EQUALS" ||
-                token.Name == "EQUALS2" ||
-                token.Name == "NOT_EQUALS" ||
-                token.Name == "NOT_EQUALS2" ||
-                token.Name == "LESS" ||
-                token.Name == "LESS_OR_EQ" ||
-                token.Name == "GREATER" ||
-                token.Name == "GREATER_OR_EQ" ||
-                token.Name == "SHIFT_LEFT" ||
-                token.Name == "ROT_LEFT" ||
-                token.Name == "AMPERSAND" ||
-                token.Name == "PIPE" ||
-                token.Name == "DOUBLE_PIPE" ||
-                token.Name == "STRUCT_OPEN" ||
-                token.Name == "STRUCT_CLOSE" ||
-                token.Name == "PLUS" ||
-                token.Name == "MINUS" ||
-                token.Name == "TILDA" ||
-                token.Name == "ASTERISK" ||
-                token.Name == "SLASH" ||
-                token.Name == "PERCENT" ||
-                token.Name == "SEMICOLON" ||
-                token.Name == "DOT" ||
-                token.Name == "COMMA" ||
-                token.Name == "LPAREN" ||
-                token.Name == "RPAREN" ||
-                token.Name == "QUESTION" ||
-                token.Name == "COLON" ||
-                token.Name == "COMMAT" ||
-                token.Name == "DOUBLE_COMMAT" ||
-                token.Name == "DOLLAR" ||
-                token.Name == "LBRACE_CURLY" ||
-                token.Name == "RBRACE_CURLY" ||
-                token.Name == "CARET" ||
-                token.Name == "NAMESPACE" ||
-                token.Name == "ARROW" ||
-                token.Name == "RBRACE_SQUARE" ||
-                token.Name == "LBRACE_SQUARE");
-        }
-
-        bool IsFunctionIdentifier(const TParsedToken& token, size_t index) {
-            if (token.Name != "ID_PLAIN") {
-                return false;
-            }
-            return std::regex_search(token.Content.begin(), token.Content.end(), BuiltinFunctionRegex) ||
-                   (2 <= index && Tokens.at(index - 1).Name == "NAMESPACE" && Tokens.at(index - 2).Name == "ID_PLAIN") ||
-                   (index < Tokens.size() - 1 && Tokens.at(index + 1).Name == "NAMESPACE");
-        }
-
-        bool IsTypeIdentifier(const TParsedToken& token) const {
-            return token.Name == "ID_PLAIN" && std::regex_search(token.Content.begin(), token.Content.end(), TypeRegex);
-        }
-
-        bool IsVariableIdentifier(const TParsedToken& token) const {
-            return token.Name == "ID_PLAIN";
-        }
-
-        bool IsQuotedIdentifier(const TParsedToken& token) const {
-            return token.Name == "ID_QUOTED";
-        }
-
-        bool IsString(const TParsedToken& token) const {
-            return token.Name == "STRING_VALUE";
-        }
-
-        bool IsNumber(const TParsedToken& token) const {
-            return token.Name == "DIGITS" ||
-                   token.Name == "INTEGER_VALUE" ||
-                   token.Name == "REAL" ||
-                   token.Name == "BLOB";
-        }
-
-        bool IsComment(const TParsedToken& token) const {
-            return token.Name == "COMMENT";
+            return index;
         }
 
     private:
         TColorSchema Coloring;
-        std::regex BuiltinFunctionRegex;
-        std::regex TypeRegex;
-
-        ILexer::TPtr CppLexer;
-        ILexer::TPtr ANSILexer;
-
-        TParsedTokenList Tokens;
+        NSQLHighlight::IHighlighter::TPtr Highlighter;
     };
 
     IYQLHighlighter::TPtr MakeYQLHighlighter(TColorSchema color) {
diff --git a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.h b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.h
index 4d39208605f..a0cccc1878d 100644
--- a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.h
+++ b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter.h
@@ -15,7 +15,7 @@ namespace NYdb::NConsoleClient {
     public:
         using TPtr = THolder<IYQLHighlighter>;
 
-        virtual void Apply(TStringBuf queryUtf8, TColors& colors) = 0;
+        virtual void Apply(TStringBuf queryUtf8, TColors& colors) const = 0;
         virtual ~IYQLHighlighter() = default;
     };
 
diff --git a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter_ut.cpp b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter_ut.cpp
index 1afe38f5654..e57208bca19 100644
--- a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter_ut.cpp
+++ b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_highlighter_ut.cpp
@@ -65,17 +65,9 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) {
                             SymbolsFromColors(actual));
     }
 
-    Y_UNIT_TEST(Blank) {
+    Y_UNIT_TEST(Empty) {
         auto highlight = MakeYQLHighlighter(Coloring);
         Check(highlight, "", "");
-        Check(highlight, " ", " ");
-        Check(highlight, "   ", "   ");
-        Check(highlight, "\n", " ");
-        Check(highlight, "\n\n", "  ");
-        Check(highlight, "\r\n", "  ");
-        Check(highlight, "\r", " ");
-        Check(highlight, "\r\n\n", "   ");
-        Check(highlight, "\r\n\r\n", "    ");
     }
 
     Y_UNIT_TEST(Invalid) {
@@ -84,148 +76,8 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) {
         Check(highlight, "й", "u");
         Check(highlight, "编", "u");
         Check(highlight, "\xF0\x9F\x98\x8A", "u");
-        Check(highlight, "!select", "uuvvvvv");
-        Check(highlight, "!sselect", "uukkkkkk");
-    }
-
-    Y_UNIT_TEST(Keyword) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-        Check(highlight, "SELECT", "kkkkkk");
-        Check(highlight, "select", "kkkkkk");
-        Check(highlight, "ALTER", "kkkkk");
-        Check(highlight, "GROUP BY", "kkkkk kk");
-        Check(highlight, "INSERT", "kkkkkk");
-    }
-
-    Y_UNIT_TEST(Operation) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-        Check(highlight, "(1 + 21 / 4)", "on o nn o no");
-        Check(highlight, "(1+21/4)", "ononnono");
-    }
-
-    Y_UNIT_TEST(FunctionIdentifier) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-
-        Check(highlight, "MIN", "fff");
-        Check(highlight, "min", "fff");
-        Check(highlight, "MIN(123, 65)", "fffonnno nno");
-        Check(highlight, "MIN", "fff");
-
-        Check(highlight, "minimum", "vvvvvvv");
-        Check(highlight, "MINimum", "vvvvvvv");
-
-        Check(highlight, "Math::Sin", "ffffoofff");
-        Check(highlight, "Math", "vvvv");
-        Check(highlight, "Math::", "ffffoo");
-        Check(highlight, "::Sin", "oovvv");
-    }
-
-    Y_UNIT_TEST(TypeIdentifier) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-        Check(highlight, "Bool", "tttt");
-        Check(highlight, "Bool(value)", "ttttovvvvvo");
-    }
-
-    Y_UNIT_TEST(VariableIdentifier) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-        Check(highlight, "test", "vvvv");
-    }
-
-    Y_UNIT_TEST(QuotedIdentifier) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-        Check(highlight, "`/cluster/database`", "qqqqqqqqqqqqqqqqqqq");
-        Check(highlight, "`test`select", "qqqqqqkkkkkk");
-        Check(highlight, "`/cluster", "uuuuuuuuu");
-        Check(highlight, "`\xF0\x9F\x98\x8A`", "qqq");
-    }
-
-    Y_UNIT_TEST(String) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-        Check(highlight, "\"\"", "ss");
-        Check(highlight, "\"test\"", "ssssss");
-        Check(highlight, "\"", "u");
-        Check(highlight, "\"\"\"", "ssu");
-        Check(highlight, "\"\\\"", "uuu");
-        Check(highlight, "\"test select from", "uuuuu uuuuuu uuuu");
-        Check(highlight, "\"\\\"\"", "ssss");
-        Check(highlight, "\"select\"select", "sssssssssvvvvv");
-        Check(highlight, "\"select\"group", "sssssssskkkkk");
-        Check(highlight, "SELECT \\\"\xF0\x9F\x98\x8A\\\" FROM test", "kkkkkk uuuuu uuuu uuuu");
-    }
-
-    Y_UNIT_TEST(MultilineString) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-
-        Check(highlight, "@@", "oo");
-        Check(highlight, "@@@", "ooo");
-        Check(highlight, "@@@@", "ssss");
-        Check(highlight, "@@@@@", "sssss");
-        Check(highlight, "@@test@@@", "sssssssss");
-
-        Check(
-            highlight,
-            ("$txt = @@some\n"
-             "multiline\n"
-             "text@@;"),
-            ("ovvv o sssssss"
-             "ssssssssss"
-             "sssssso"));
-        Check(
-            highlight,
-            ("$txt = @@some\n"
-             "multiline with double at: @@@@\n"
-             "text@@;"),
-            ("ovvv o sssssss"
-             "sssssssssssssssssssssssssssssss"
-             "sssssso"));
-    }
-
-    Y_UNIT_TEST(TypedString) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-        Check(
-            highlight,
-            "SELECT \"foo\"u, '[1;2]'y, @@{\"a\":null}@@j;",
-            "kkkkkk sssssso sssssssso ssssssssssssssso");
-    }
-
-    Y_UNIT_TEST(Number) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-
-        Check(highlight, "1234", "nnnn");
-        Check(highlight, "-123", "onnn");
-
-        Check(
-            highlight,
-            ("SELECT "
-             "123l AS `Int64`, "
-             "0b01u AS `Uint32`, "
-             "0xfful AS `Uint64`, "
-             "0o7ut AS `Uint8`, "
-             "456s AS `Int16`, "
-             "1.2345f AS `Float`;"),
-            ("kkkkkk "
-             "nnnn kk qqqqqqqo "
-             "nnnnn kk qqqqqqqqo "
-             "nnnnnn kk qqqqqqqqo "
-             "nnnnn kk qqqqqqqo "
-             "nnnn kk qqqqqqqo "
-             "nnnnnnn kk qqqqqqqo"));
-    }
-
-    Y_UNIT_TEST(SQL) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-        Check(highlight, "SELECT id, alias from users",
-              "kkkkkk vvo vvvvv kkkk vvvvv");
-        Check(highlight, "INSERT INTO users (id, alias) VALUES (12, \"tester\")",
-              "kkkkkk kkkk vvvvv ovvo vvvvvo kkkkkk onno sssssssso");
-        Check(
-            highlight,
-            "SELECT 123467, \"Hello, {name}!\", (1 + (5 * 1 / 0)), MIN(identifier)"
-            "FROM `local/test/space/table` JOIN test;",
-            "kkkkkk nnnnnno sssssssssssssssso on o on o n o nooo fffovvvvvvvvvvo"
-            "kkkk qqqqqqqqqqqqqqqqqqqqqqqq kkkk vvvvo");
-        Check(highlight, "SELECT Bool(phone) FROM customer",
-              "kkkkkk ttttovvvvvo kkkk vvvvvvvv");
+        Check(highlight, "!select", "ukkkkkk");
+        Check(highlight, "!sselect", "uvvvvvvv");
     }
 
     Y_UNIT_TEST(Emoji) {
@@ -242,127 +94,18 @@ Y_UNIT_TEST_SUITE(YqlHighlightTests) {
     Y_UNIT_TEST(Typing) {
         const TString query =
             "SELECT \n"
-            "  123467, \"Hello, {name}!\", \n"
+            "  123467, \"Hello, друг, {name} \xF0\x9F\x98\x8A!\", \n"
             "  (1 + (5 * 1 / 0)), MIN(identifier), \n"
             "  Bool(field), Math::Sin(var) \n"
             "FROM `local/test/space/table` JOIN test;";
 
-        const TString pattern =
-            "kkkkkk  "
-            "  nnnnnno sssssssssssssssso  "
-            "  on o on o n o nooo fffovvvvvvvvvvoo  "
-            "  ttttovvvvvoo ffffoofffovvvo  "
-            "kkkk qqqqqqqqqqqqqqqqqqqqqqqq kkkk vvvvo";
-
         auto highlight = MakeYQLHighlighter(Coloring);
         for (std::size_t size = 0; size <= query.size(); ++size) {
             const TStringBuf prefix(query, 0, size);
 
             auto colors = Apply(*highlight, prefix);
             Y_DO_NOT_OPTIMIZE_AWAY(colors);
-
-            if (size == query.size() || IsSpace(pattern[size])) {
-                const TStringBuf pattern_prefix(pattern, 0, size);
-                Check(highlight, prefix, pattern_prefix);
-            }
         }
     }
 
-    Y_UNIT_TEST(Comment) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-        Check(highlight, "- select", "o kkkkkk");
-        Check(highlight, "select -- select", "kkkkkk ccccccccc");
-        Check(highlight, "-- select\nselect", "cccccccccckkkkkk");
-        Check(highlight, "/* select */", "cccccccccccc");
-        Check(highlight, "select /* select */ select", "kkkkkk cccccccccccc kkkkkk");
-        Check(highlight, "/**/ --", "cccc cc");
-        Check(highlight, "/*/**/*/", "ccccccoo");
-    }
-
-    Y_UNIT_TEST(Multiline) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-        Check(
-            highlight,
-            "SELECT *\n"
-            "FROM test",
-            "kkkkkk o kkkk vvvv");
-        Check(
-            highlight,
-            "SELECT *\n"
-            "\n"
-            "\r\n"
-            "FROM test",
-            "kkkkkk o    kkkk vvvv");
-        Check(
-            highlight,
-            "SELECT *\r\n"
-            "FROM test",
-            "kkkkkk o  kkkk vvvv");
-        Check(
-            highlight,
-            "SELECT *\r\n"
-            "FROM test\n",
-            "kkkkkk o  kkkk vvvv ");
-    }
-
-    Y_UNIT_TEST(ANSI) {
-        auto highlight = MakeYQLHighlighter(Coloring);
-
-        Check(
-            highlight,
-            "--!ansi_lexer\n"
-            "SELECT * FROM T; /* this is a comment /* this is a nested comment */ */",
-            "cccccccccccccc"
-            "kkkkkk o kkkk vo cccccccccccccccccccccccccccccccccccccccccccccccccccccc");
-        Check(
-            highlight,
-            "--!ansi_lexer\n"
-            "SELECT 1 as \"column with \"\" double quote\";",
-            "cccccccccccccc"
-            "kkkkkk n kk ssssssssssssssssssssssssssssso");
-        Check(
-            highlight,
-            "--!ansi_lexer\n"
-            "SELECT 'string with '' quote';",
-            "cccccccccccccc"
-            "kkkkkk sssssssssssssssssssssso");
-
-        Check(
-            highlight,
-            " \t\n --!ansi_lexer \n"
-            "/* /* */ */",
-            "    ccccccccccccccc"
-            "ccccccccccc");
-
-        Check(
-            highlight,
-            (
-                "\n"
-                "\t --!ansi_lexer \n"
-                "-- Some comment\n"
-                "pragma SimpleColumns;\n"
-                "select 1, '''' as empty;"),
-            (
-                " "
-                "  ccccccccccccccc"
-                "cccccccccccccccc"
-                "kkkkkk vvvvvvvvvvvvvo "
-                "kkkkkk no ssss kk kkkkko"));
-        Check(
-            highlight,
-            (
-                "$str = '\n"
-                "--!ansi_lexer\n"
-                "--!syntax_v1\n"
-                "';\n"
-                "\n"
-                "select 1, $str, \"\" as empty;"),
-            (
-                "ovvv o ss"
-                "ssssssssssssss"
-                "sssssssssssss"
-                "so "
-                " "
-                "kkkkkk no ovvvo ss kk kkkkko"));
-    }
 } // Y_UNIT_TEST_SUITE(YqlHighlightTests)
diff --git a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_position.cpp b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_position.cpp
deleted file mode 100644
index 15fada3f46c..00000000000
--- a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_position.cpp
+++ /dev/null
@@ -1,40 +0,0 @@
-#include "yql_position.h"
-
-#include <yql/essentials/utils/line_split.h>
-
-#include <util/charset/utf8.h>
-#include <util/stream/str.h>
-
-namespace NYdb::NConsoleClient {
-
-    ui32 TYQLPositionMapping::RawPos(const TParsedToken& token) const {
-        return SymbolsCountBeforeLine.at(token.Line) + token.LinePos;
-    }
-
-    TYQLPositionMapping TYQLPositionMapping::Build(const TString& queryUtf8) {
-        TVector<ui32> symbolsCountBeforeLine = {0, 0};
-
-        TStringStream stream(queryUtf8);
-        TLineSplitter lines(stream);
-
-        size_t read;
-        for (TString line; (read = lines.Next(line)) != 0;) {
-            const auto index = symbolsCountBeforeLine.size();
-            const auto previous = symbolsCountBeforeLine.at(index - 1);
-
-            const auto newlineWidth = read - line.size();
-            Y_ASSERT(0 <= newlineWidth && newlineWidth <= 2);
-
-            const auto current = GetNumberOfUTF8Chars(line) + newlineWidth;
-            symbolsCountBeforeLine.emplace_back(previous + current);
-        }
-
-        return TYQLPositionMapping(std::move(symbolsCountBeforeLine));
-    }
-
-    TYQLPositionMapping::TYQLPositionMapping(TVector<ui32> SymbolsCountBeforeLine)
-        : SymbolsCountBeforeLine(std::move(SymbolsCountBeforeLine))
-    {
-    }
-
-} // namespace NYdb::NConsoleClient
diff --git a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_position.h b/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_position.h
deleted file mode 100644
index 4a9dddec82c..00000000000
--- a/ydb/public/lib/ydb_cli/commands/interactive/highlight/yql_position.h
+++ /dev/null
@@ -1,29 +0,0 @@
-#pragma once
-
-#include <yql/essentials/parser/lexer_common/lexer.h>
-
-#include <util/system/types.h>
-#include <util/generic/string.h>
-#include <util/generic/vector.h>
-
-namespace NYdb::NConsoleClient {
-
-    using NSQLTranslation::TParsedToken;
-
-    class TYQLPositionMapping final {
-    public:
-        // Translates (Line, LinePos) position into RawPos that
-        // is an absolute symbol position in utf8 symbols array
-        ui32 RawPos(const TParsedToken& token) const;
-
-    public:
-        static TYQLPositionMapping Build(const TString& queryUtf8);
-
-    private:
-        explicit TYQLPositionMapping(TVector<ui32> SymbolsCountBeforeLine);
-
-    private:
-        TVector<ui32> SymbolsCountBeforeLine;
-    };
-
-} // namespace NYdb::NConsoleClient