[YQL-14290] Support MERGE() JOIN hint

ref:20808fd01246b3b64a2f93eb7434691c4393d6d6

12 files changed, 71 insertions, 25 deletions

diff --git a/CMakeLists.darwin.txt b/CMakeLists.darwin.txt
index 40f5c20757e..fea6b301ca7 100644
--- a/CMakeLists.darwin.txt
+++ b/CMakeLists.darwin.txt
@@ -471,11 +471,12 @@ add_subdirectory(ydb/library/yql/parser/proto_ast/collect_issues)
 add_subdirectory(ydb/library/yql/parser/proto_ast/gen/v0)
 add_subdirectory(ydb/library/yql/sql/v0/lexer)
 add_subdirectory(ydb/library/yql/sql/v1)
+add_subdirectory(ydb/library/yql/parser/lexer_common)
 add_subdirectory(ydb/library/yql/parser/proto_ast/gen/v1)
 add_subdirectory(ydb/library/yql/parser/proto_ast/gen/v1_proto)
 add_subdirectory(ydb/library/yql/parser/proto_ast/gen/v1_ansi)
-add_subdirectory(ydb/library/yql/sql/v1/proto_parser)
 add_subdirectory(ydb/library/yql/sql/v1/lexer)
+add_subdirectory(ydb/library/yql/sql/v1/proto_parser)
 add_subdirectory(ydb/library/yql/core/common_opt)
 add_subdirectory(ydb/library/yql/core/peephole_opt)
 add_subdirectory(ydb/library/yql/core/type_ann)
@@ -1169,7 +1170,6 @@ add_subdirectory(ydb/library/yql/core/file_storage/http_download)
 add_subdirectory(ydb/library/yql/core/file_storage/http_download/proto)
 add_subdirectory(ydb/library/yql/core/issue/ut)
 add_subdirectory(ydb/library/yql/minikql/ut)
-add_subdirectory(ydb/library/yql/parser/lexer_common)
 add_subdirectory(ydb/library/yql/utils/ut)
 add_subdirectory(ydb/library/yql/utils/threading)
 add_subdirectory(ydb/library/yql/utils/failure_injector/ut)
diff --git a/CMakeLists.linux.txt b/CMakeLists.linux.txt
index 314468b7c2a..708a6f310a4 100644
--- a/CMakeLists.linux.txt
+++ b/CMakeLists.linux.txt
@@ -551,11 +551,12 @@ add_subdirectory(ydb/library/yql/parser/proto_ast/collect_issues)
 add_subdirectory(ydb/library/yql/parser/proto_ast/gen/v0)
 add_subdirectory(ydb/library/yql/sql/v0/lexer)
 add_subdirectory(ydb/library/yql/sql/v1)
+add_subdirectory(ydb/library/yql/parser/lexer_common)
 add_subdirectory(ydb/library/yql/parser/proto_ast/gen/v1)
 add_subdirectory(ydb/library/yql/parser/proto_ast/gen/v1_proto)
 add_subdirectory(ydb/library/yql/parser/proto_ast/gen/v1_ansi)
-add_subdirectory(ydb/library/yql/sql/v1/proto_parser)
 add_subdirectory(ydb/library/yql/sql/v1/lexer)
+add_subdirectory(ydb/library/yql/sql/v1/proto_parser)
 add_subdirectory(ydb/library/yql/core/common_opt)
 add_subdirectory(ydb/library/yql/core/peephole_opt)
 add_subdirectory(ydb/library/yql/core/type_ann)
@@ -1264,7 +1265,6 @@ add_subdirectory(ydb/library/yql/core/file_storage/http_download)
 add_subdirectory(ydb/library/yql/core/file_storage/http_download/proto)
 add_subdirectory(ydb/library/yql/core/issue/ut)
 add_subdirectory(ydb/library/yql/minikql/ut)
-add_subdirectory(ydb/library/yql/parser/lexer_common)
 add_subdirectory(ydb/library/yql/utils/ut)
 add_subdirectory(ydb/library/yql/utils/threading)
 add_subdirectory(ydb/library/yql/utils/failure_injector/ut)
diff --git a/ydb/library/yql/core/issue/yql_issue.h b/ydb/library/yql/core/issue/yql_issue.h
index 9ec7ab14bb3..ed7f8d2d122 100644
--- a/ydb/library/yql/core/issue/yql_issue.h
+++ b/ydb/library/yql/core/issue/yql_issue.h
@@ -4,6 +4,12 @@
 #include <ydb/library/yql/public/issue/yql_issue.h>
 #include <ydb/library/yql/public/issue/yql_issue_id.h>
 
+#ifdef _win_
+#ifdef GetMessage
+#undef GetMessage
+#endif
+#endif
+
 namespace NYql {
 
 extern const char IssueMapResource[14];
diff --git a/ydb/library/yql/public/issue/yql_issue_id.h b/ydb/library/yql/public/issue/yql_issue_id.h
index 9f4300f06be..8214876cf98 100644
--- a/ydb/library/yql/public/issue/yql_issue_id.h
+++ b/ydb/library/yql/public/issue/yql_issue_id.h
@@ -13,6 +13,12 @@
 #include <util/generic/yexception.h>
 #include <util/string/subst.h>
 
+#ifdef _win_
+#ifdef GetMessage
+#undef GetMessage
+#endif
+#endif
+
 namespace NYql {
 
 using TIssueCode = ui32;
diff --git a/ydb/library/yql/sql/sql.cpp b/ydb/library/yql/sql/sql.cpp
index 00adc841177..5a99fbf00ff 100644
--- a/ydb/library/yql/sql/sql.cpp
+++ b/ydb/library/yql/sql/sql.cpp
@@ -131,7 +131,7 @@ namespace NSQLTranslation {
         }
     }
 
-    NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const TTranslationSettings& settings) {
+    NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings) {
         NYql::TAstParseResult result;
         switch (settings.SyntaxVersion) {
             case 0:
@@ -149,7 +149,7 @@ namespace NSQLTranslation {
 
                 return NSQLTranslationV0::SqlASTToYql(protoAst, settings);
             case 1:
-                return NSQLTranslationV1::SqlASTToYql(protoAst, settings);
+                return NSQLTranslationV1::SqlASTToYql(protoAst, hints, settings);
             default:
                 result.Issues.AddIssue(NYql::YqlIssue(NYql::TPosition(), NYql::TIssuesIds::DEFAULT_ERROR,
                     TStringBuilder() << "Unknown SQL syntax version: " << settings.SyntaxVersion));
diff --git a/ydb/library/yql/sql/sql.h b/ydb/library/yql/sql/sql.h
index a2d2d0d3911..db699ebece1 100644
--- a/ydb/library/yql/sql/sql.h
+++ b/ydb/library/yql/sql/sql.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <ydb/library/yql/parser/lexer_common/hints.h>
 #include <ydb/library/yql/parser/lexer_common/lexer.h>
 #include <ydb/library/yql/parser/proto_ast/proto_ast.h>
 #include <ydb/library/yql/public/issue/yql_warning.h>
@@ -19,6 +20,6 @@ namespace NSQLTranslation {
     google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NYql::TIssues& issues, size_t maxErrors,
         const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr);
     ILexer::TPtr SqlLexer(const TString& query, NYql::TIssues& issues, const TTranslationSettings& settings = {}, ui16* actualSyntaxVersion = nullptr);
-    NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const TTranslationSettings& settings);
+    NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const TSQLHints& hints, const TTranslationSettings& settings);
 
 }  // namespace NSQLTranslationV0
diff --git a/ydb/library/yql/sql/v1/CMakeLists.txt b/ydb/library/yql/sql/v1/CMakeLists.txt
index 7794fc4b730..b9330a5e7da 100644
--- a/ydb/library/yql/sql/v1/CMakeLists.txt
+++ b/ydb/library/yql/sql/v1/CMakeLists.txt
@@ -24,12 +24,14 @@ target_link_libraries(yql-sql-v1 PUBLIC
   library-yql-core
   yql-core-issue
   core-issue-protos
+  yql-parser-lexer_common
   yql-parser-proto_ast
   parser-proto_ast-collect_issues
   proto_ast-gen-v1
   proto_ast-gen-v1_ansi
   proto_ast-gen-v1_proto
   yql-parser-pg_catalog
+  sql-v1-lexer
   sql-v1-proto_parser
   tools-enum_parser-enum_serialization_runtime
 )
diff --git a/ydb/library/yql/sql/v1/context.cpp b/ydb/library/yql/sql/v1/context.cpp
index 43e4cce0331..69a0557d118 100644
--- a/ydb/library/yql/sql/v1/context.cpp
+++ b/ydb/library/yql/sql/v1/context.cpp
@@ -67,10 +67,12 @@ THashMap<TStringBuf, TPragmaMaybeField> CTX_PRAGMA_MAYBE_FIELDS = {
 } // namespace
 
 TContext::TContext(const NSQLTranslation::TTranslationSettings& settings,
+                   const NSQLTranslation::TSQLHints& hints,
                    TIssues& issues)
     : ClusterMapping(settings.ClusterMapping)
     , PathPrefix(settings.PathPrefix)
     , ClusterPathPrefixes(settings.ClusterPathPrefixes)
+    , SQLHints(hints)
     , Settings(settings)
     , Pool(new TMemoryPool(4096))
     , Issues(issues)
@@ -163,6 +165,21 @@ void TContext::SetWarningPolicyFor(NYql::TIssueCode code, NYql::EWarningAction a
     WarningPolicy.AddRule(rule);
 }
 
+TVector<NSQLTranslation::TSQLHint> TContext::PullHintForToken(NYql::TPosition tokenPos) {
+    TVector<NSQLTranslation::TSQLHint> result;
+    auto it = SQLHints.find(tokenPos);
+    if (it == SQLHints.end()) {
+        return result;
+    }
+    result = std::move(it->second);
+    SQLHints.erase(it);
+    return result;
+}
+
+void TContext::WarnUnusedHints() {
+    // TODO
+}
+
 IOutputStream& TContext::MakeIssue(ESeverity severity, TIssueCode code, NYql::TPosition pos) {
     if (severity == TSeverityIds::S_WARNING) {
         auto action = WarningPolicy.GetAction(code);
diff --git a/ydb/library/yql/sql/v1/context.h b/ydb/library/yql/sql/v1/context.h
index 8c09bef194d..396b95daea6 100644
--- a/ydb/library/yql/sql/v1/context.h
+++ b/ydb/library/yql/sql/v1/context.h
@@ -80,6 +80,7 @@ namespace NSQLTranslationV1 {
     class TContext {
     public:
         TContext(const NSQLTranslation::TTranslationSettings& settings,
+                 const NSQLTranslation::TSQLHints& hints,
                  NYql::TIssues& issues);
 
         virtual ~TContext();
@@ -184,6 +185,9 @@ namespace NSQLTranslationV1 {
             return TopLevelColumnReferenceState;
         }
 
+        TVector<NSQLTranslation::TSQLHint> PullHintForToken(NYql::TPosition tokenPos);
+        void WarnUnusedHints();
+
     private:
         IOutputStream& MakeIssue(NYql::ESeverity severity, NYql::TIssueCode code, NYql::TPosition pos);
 
@@ -195,6 +199,7 @@ namespace NSQLTranslationV1 {
         THashMap<TString, TString> ProviderPathPrefixes;
         THashMap<TString, TString> ClusterPathPrefixes;
         bool IntoHeading = true;
+        NSQLTranslation::TSQLHints SQLHints;
 
         friend class TColumnRefScope;
 
diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp
index 9b96d0e2889..b13ae40ff15 100644
--- a/ydb/library/yql/sql/v1/sql.cpp
+++ b/ydb/library/yql/sql/v1/sql.cpp
@@ -4,12 +4,14 @@
 #include "node.h"
 #include "sql_call_param.h"
 #include "ydb/library/yql/ast/yql_ast.h"
+#include <ydb/library/yql/parser/lexer_common/hints.h>
 #include <ydb/library/yql/parser/proto_ast/collect_issues/collect_issues.h>
 #include <ydb/library/yql/parser/proto_ast/gen/v1/SQLv1Lexer.h>
 #include <ydb/library/yql/parser/proto_ast/gen/v1/SQLv1Parser.h>
 #include <ydb/library/yql/parser/proto_ast/gen/v1_ansi/SQLv1Lexer.h>
 #include <ydb/library/yql/parser/proto_ast/gen/v1_ansi/SQLv1Parser.h>
 
+#include <ydb/library/yql/sql/v1/lexer/lexer.h>
 #include <ydb/library/yql/sql/v1/proto_parser/proto_parser.h>
 
 #include <ydb/library/yql/ast/yql_expr.h>
@@ -5586,9 +5588,9 @@ bool TSqlSelect::JoinOp(ISource* join, const TRule_join_source::TBlock3& block,
                 return false;
             }
             TString joinOp("Inner");
-            // TODO: custom join hints/settings should be here
+            auto hints = Ctx.PullHintForToken(Ctx.TokenPosition(alt.GetToken3()));
             TJoinLinkSettings linkSettings;
-            linkSettings.ForceSortedMerge = false;
+            linkSettings.ForceSortedMerge = AnyOf(hints, [](const NSQLTranslation::TSQLHint& hint) { return to_lower(hint.Name) == "merge"; });
             switch (alt.GetBlock2().Alt_case()) {
                 case TRule_join_op::TAlt2::TBlock2::kAlt1:
                     if (alt.GetBlock2().GetAlt1().HasBlock1()) {
@@ -10155,15 +10157,18 @@ void SqlASTToYqlImpl(NYql::TAstParseResult& res, const google::protobuf::Message
             ctx.Error() << "Error occurred on parse SQL query, but no error is collected" <<
                 ", please send this request over bug report into YQL interface or write on yql@ maillist";
         }
+    } else {
+        ctx.WarnUnusedHints();
     }
 }
 
 NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst,
+    const NSQLTranslation::TSQLHints& hints,
     const NSQLTranslation::TTranslationSettings& settings)
 {
     YQL_ENSURE(IsQueryMode(settings.Mode));
     TAstParseResult res;
-    TContext ctx(settings, res.Issues);
+    TContext ctx(settings, hints, res.Issues);
     SqlASTToYqlImpl(res, protoAst, ctx);
     return res;
 }
@@ -10171,10 +10176,19 @@ NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst,
 NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules)
 {
     TAstParseResult res;
-    TContext ctx(settings, res.Issues);
+    const TString queryName = "query";
+
+    NSQLTranslation::TSQLHints hints;
+    auto lexer = MakeLexer(settings.AnsiLexer);
+    YQL_ENSURE(lexer);
+    if (!CollectSqlHints(*lexer, query, queryName, settings.File, hints, res.Issues, settings.MaxErrors)) {
+        return res;
+    }
+
+    TContext ctx(settings, hints, res.Issues);
     NSQLTranslation::TErrorCollectorOverIssues collector(res.Issues, settings.MaxErrors, settings.File);
 
-    google::protobuf::Message* ast(SqlAST(query, "query", collector, settings.AnsiLexer, settings.Arena));
+    google::protobuf::Message* ast(SqlAST(query, queryName, collector, settings.AnsiLexer, settings.Arena));
     if (ast) {
         SqlASTToYqlImpl(res, *ast, ctx);
     } else {
diff --git a/ydb/library/yql/sql/v1/sql.h b/ydb/library/yql/sql/v1/sql.h
index a59f0f3c23c..82e5c97a697 100644
--- a/ydb/library/yql/sql/v1/sql.h
+++ b/ydb/library/yql/sql/v1/sql.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <ydb/library/yql/ast/yql_ast.h>
+#include <ydb/library/yql/parser/lexer_common/hints.h>
 #include <ydb/library/yql/parser/proto_ast/proto_ast.h>
 #include <ydb/library/yql/public/issue/yql_warning.h>
 #include <ydb/library/yql/public/issue/yql_issue_manager.h>
@@ -15,6 +16,6 @@ namespace NSQLTranslation {
 namespace NSQLTranslationV1 {
 
     NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules = nullptr);
-    NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const NSQLTranslation::TTranslationSettings& settings);
+    NYql::TAstParseResult SqlASTToYql(const google::protobuf::Message& protoAst, const NSQLTranslation::TSQLHints& hints, const NSQLTranslation::TTranslationSettings& settings);
 
 }  // namespace NSQLTranslationV1
diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp
index 37c5fdc167e..ef3a3297bba 100644
--- a/ydb/library/yql/sql/v1/sql_ut.cpp
+++ b/ydb/library/yql/sql/v1/sql_ut.cpp
@@ -1735,10 +1735,8 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) {
         UNIT_ASSERT(!res1.Root);
         UNIT_ASSERT(!res2.Root);
 
-        UNIT_ASSERT_NO_DIFF(Err2Str(res1), "<main>:1:15: Error: Unexpected character : syntax error...\n\n"
-                                           "<main>:1:12: Error: Unexpected token 'b' : cannot match to any predicted input...\n\n");
-        UNIT_ASSERT_NO_DIFF(Err2Str(res2), "<main>:1:15: Error: Unexpected character : syntax error...\n\n"
-                                           "<main>:1:12: Error: Unexpected token 'b' : cannot match to any predicted input...\n\n");
+        UNIT_ASSERT_NO_DIFF(Err2Str(res1), "<main>:1:15: Error: Unexpected character : syntax error...\n\n");
+        UNIT_ASSERT_NO_DIFF(Err2Str(res2), "<main>:1:15: Error: Unexpected character : syntax error...\n\n");
     }
 
     Y_UNIT_TEST(InvalidHexInStringLiteral) {
@@ -1764,8 +1762,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) {
     Y_UNIT_TEST(InvalidDoubleAtString) {
         NYql::TAstParseResult res = SqlToYql("select @@@@@@");
         UNIT_ASSERT(!res.Root);
-        UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Unexpected character : syntax error...\n\n"
-                                          "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n");
+        UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:13: Error: Unexpected character : syntax error...\n\n");
     }
 
     Y_UNIT_TEST(InvalidDoubleAtStringWhichWasAcceptedEarlier) {
@@ -1783,8 +1780,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) {
     Y_UNIT_TEST(InvalidDoubleAtStringFromTable) {
         NYql::TAstParseResult res = SqlToYql("select @@@@@@ from plato.foo");
         UNIT_ASSERT(!res.Root);
-        UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unexpected character : syntax error...\n\n"
-                                          "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n");
+        UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:28: Error: Unexpected character : syntax error...\n\n");
     }
 
     Y_UNIT_TEST(SelectInvalidSyntax) {
@@ -3959,8 +3955,7 @@ Y_UNIT_TEST_SUITE(AnsiIdentsNegative) {
         UNIT_ASSERT(res.Root);
         res = SqlToYqlWithAnsiLexer(req);
         UNIT_ASSERT(!res.Root);
-        UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Unexpected character : syntax error...\n\n"
-                                          "<main>:1:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n");
+        UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:16: Error: Unexpected character : syntax error...\n\n");
 
         req = "/*\n"
               "--/*\n"
@@ -3969,8 +3964,7 @@ Y_UNIT_TEST_SUITE(AnsiIdentsNegative) {
         UNIT_ASSERT(res.Root);
         res = SqlToYqlWithAnsiLexer(req);
         UNIT_ASSERT(!res.Root);
-        UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:12: Error: Unexpected character : syntax error...\n\n"
-                                          "<main>:3:0: Error: Unexpected token absence : cannot match to any predicted input...\n\n");
+        UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:3:12: Error: Unexpected character : syntax error...\n\n");
 
         req = "/*\n"
               "/*\n"