Intermediate changes

commit_hash:e2b7b2f9068bb480c3779b387c101673cc6cf7a7

6 files changed, 69 insertions, 34 deletions

diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
index 68fd6f50de..83939478d6 100644
--- a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
+++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
@@ -75,19 +75,6 @@ const ui32 PRETTY_FLAGS = NYql::TAstPrintFlags::PerLine | NYql::TAstPrintFlags::
         NYql::TAstPrintFlags::AdaptArbitraryContent;
 
 template <typename TMessage>
-THolder<TMessage> ParseProtoConfig(const TString& cfgFile) {
-    auto config = MakeHolder<TMessage>();
-    TString configData = TFileInput(cfgFile).ReadAll();
-
-    using ::google::protobuf::TextFormat;
-    if (!TextFormat::ParseFromString(configData, config.Get())) {
-        throw yexception() << "Bad format of config file " << cfgFile;
-    }
-
-    return config;
-}
-
-template <typename TMessage>
 THolder<TMessage> ParseProtoFromResource(TStringBuf resourceName) {
     if (!NResource::Has(resourceName)) {
         return {};
@@ -190,6 +177,15 @@ void TFacadeRunOptions::PrintInfo(const TString& msg) {
     }
 }
 
+void TFacadeRunOptions::ParseProtoConfig(const TString& cfgFile, google::protobuf::Message* config) {
+    TString configData = TFileInput(cfgFile).ReadAll();
+
+    using ::google::protobuf::TextFormat;
+    if (!TextFormat::ParseFromString(configData, config)) {
+        throw yexception() << "Bad format of config file " << cfgFile;
+    }
+}
+
 void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
     User = GetUsername();
 
@@ -207,7 +203,7 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
     NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default();
 
     opts.AddHelpOption();
-    opts.AddLongOption('p', "program", "Program file (use - to read from stdin)").Required().RequiredArgument("FILE")
+    opts.AddLongOption('p', "program", "Program file (use - to read from stdin)").Optional().RequiredArgument("FILE")
         .Handler1T<TString>([this](const TString& file) {
             ProgramFile = file;
             if (ProgramFile == "-") {
@@ -216,14 +212,13 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
             } else {
                 ProgramText = TFileInput(ProgramFile).ReadAll();
             }
-            User = GetUsername();
         });
     opts.AddLongOption('s', "sql", "Program is SQL query").NoArgument().StoreValue(&ProgramType, EProgramType::Sql);
     if (PgSupport) {
         opts.AddLongOption("pg", "Program has PG syntax").NoArgument().StoreValue(&ProgramType, EProgramType::Pg);
         opts.AddLongOption("pg-ext", "Pg extensions config file").Optional().RequiredArgument("FILE")
             .Handler1T<TString>([this](const TString& file) {
-                PgExtConfig = ParseProtoConfig<NProto::TPgExtensions>(file);
+                PgExtConfig = TFacadeRunOptions::ParseProtoConfig<NProto::TPgExtensions>(file);
             });
     }
     opts.AddLongOption('f', "file", "Additional files").RequiredArgument("name@path")
@@ -250,14 +245,16 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
 
     opts.AddLongOption('m', "mounts", "Mount points config file.").Optional().RequiredArgument("FILE")
         .Handler1T<TString>([this](const TString& file) {
-            MountConfig = ParseProtoConfig<NYqlMountConfig::TMountConfig>(file);
+            MountConfig = TFacadeRunOptions::ParseProtoConfig<NYqlMountConfig::TMountConfig>(file);
         });
     opts.AddLongOption("params-file", "Query parameters values in YSON format").Optional().RequiredArgument("FILE")
         .Handler1T<TString>([this](const TString& file) {
             Params = TFileInput(file).ReadAll();
         });
+    opts.AddLongOption("yson-attrs", "Provide operation yson attribues").Optional().RequiredArgument("VALUE").StoreResult(&YsonAttrs);
     opts.AddLongOption('G', "gateways", TStringBuilder() << "Used gateways, available: " << JoinSeq(",", SupportedGateways_)).DefaultValue(JoinSeq(",", GatewayTypes))
         .Handler1T<TString>([this](const TString& gateways) {
+            GatewayTypes.clear();
             ::StringSplitter(gateways).Split(',').Consume([&](const TStringBuf& val) {
                 if (!SupportedGateways_.contains(val)) {
                     throw yexception() << "Unsupported gateway \"" << val << '"';
@@ -267,7 +264,7 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
         });
     opts.AddLongOption("gateways-cfg", "Gateways configuration file").Optional().RequiredArgument("FILE")
         .Handler1T<TString>([this](const TString& file) {
-            GatewaysConfig = ParseProtoConfig<TGatewaysConfig>(file);
+            GatewaysConfig = TFacadeRunOptions::ParseProtoConfig<TGatewaysConfig>(file);
         });
     opts.AddLongOption("fs-cfg", "Fs configuration file").Optional().RequiredArgument("FILE")
         .Handler1T<TString>([this](const TString& file) {
@@ -378,7 +375,6 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
     opts.AddLongOption("syntax-version", "SQL syntax version").StoreResult(&SyntaxVersion).DefaultValue(1);
     opts.AddLongOption("ansi-lexer", "Use ansi lexer").NoArgument().SetFlag(&AnsiLexer);
     opts.AddLongOption("assume-ydb-on-slash", "Assume YDB provider if cluster name starts with '/'").NoArgument().SetFlag(&AssumeYdbOnClusterWithSlash);
-    opts.AddLongOption("test-antlr4", "Check antlr4 parser").NoArgument().SetFlag(&TestAntlr4);
 
     opts.AddLongOption("with-final-issues", "Include some final messages (like statistic) in issues").NoArgument().SetFlag(&WithFinalIssues);
     if (FailureInjectionSupport) {
@@ -431,6 +427,16 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
                 }
                 QPlayerMode = EQPlayerMode::Replay;
             });
+        opts.AddLongOption("gateways-patch", "QPlayer patch for gateways conf").Optional().RequiredArgument("FILE")
+            .Handler1T<TString>([this](const TString& file) {
+                GatewaysPatch = TFileInput(file).ReadAll();
+            });
+    }
+
+    if (CustomTests) {
+        opts.AddLongOption("test-antlr4", "Check antlr4 parser").NoArgument().SetFlag(&TestAntlr4);
+        opts.AddLongOption("test-format", "Compare formatted query's AST with the original query's AST (only syntaxVersion=1 is supported)").NoArgument().SetFlag(&TestSqlFormat);
+        opts.AddLongOption("validate-result-format", "Check that result-format can parse Result").NoArgument().SetFlag(&ValidateResultFormat);
     }
 
     opts.SetFreeArgsMax(0);
@@ -453,6 +459,12 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
             QPlayerContext = TQContext(QPlayerStorage_->MakeWriter(OperationId, {}));
         }
     }
+    if (EQPlayerMode::Replay != QPlayerMode && !ProgramText) {
+        throw yexception() << "Either program or replay option should be specified";
+    }
+    if (GatewaysPatch && EQPlayerMode::Replay != QPlayerMode) {
+        throw yexception() << "gateways-patch can only be used with replay option";
+    }
 
     if (Mode >= ERunMode::Validate && GatewayTypes.empty()) {
         throw yexception() << "At least one gateway from the list " << JoinSeq(",", SupportedGateways_).Quote() << " must be specified";
@@ -539,8 +551,10 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) {
         NPg::GetSqlLanguageParser()->Freeze();
     }
 
-    FuncRegistry_ = NKikimr::NMiniKQL::CreateFunctionRegistry(&NYql::NBacktrace::KikimrBackTrace,
-        NKikimr::NMiniKQL::CreateBuiltinRegistry(), true, RunOptions_.UdfsPaths);
+    auto funcRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(&NYql::NBacktrace::KikimrBackTrace,
+        NKikimr::NMiniKQL::CreateBuiltinRegistry(), true, RunOptions_.UdfsPaths)->Clone();
+    NKikimr::NMiniKQL::FillStaticModules(*funcRegistry);
+    FuncRegistry_ = funcRegistry;
 
     NSQLTranslationV1::TLexers lexers;
     lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
@@ -595,17 +609,19 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) {
     IUdfResolver::TPtr udfResolver;
     TUdfIndex::TPtr udfIndex;
     if (FileStorage_ && RunOptions_.ScanUdfs) {
-        if (!RunOptions_.UdfResolverPath) {
+        if (!RunOptions_.UdfResolverPath && EQPlayerMode::Replay != RunOptions_.QPlayerMode) {
             Cerr << "udf-resolver path must be specified when use 'scan-udfs'";
             return -1;
         }
 
         udfResolver = NCommon::CreateOutProcUdfResolver(FuncRegistry_.Get(), FileStorage_, RunOptions_.UdfResolverPath, {}, {}, RunOptions_.UdfResolverFilterSyscalls, {});
 
-        RunOptions_.PrintInfo(TStringBuilder() << TInstant::Now().ToStringLocalUpToSeconds() << " Udf scanning started for " << RunOptions_.UdfsPaths.size() << " udfs ...");
         udfIndex = new TUdfIndex();
-        LoadRichMetadataToUdfIndex(*udfResolver, RunOptions_.UdfsPaths, false, TUdfIndex::EOverrideMode::RaiseError, *udfIndex);
-        RunOptions_.PrintInfo(TStringBuilder() << TInstant::Now().ToStringLocalUpToSeconds() << " UdfIndex done.");
+        if (EQPlayerMode::Replay != RunOptions_.QPlayerMode) {
+            RunOptions_.PrintInfo(TStringBuilder() << TInstant::Now().ToStringLocalUpToSeconds() << " Udf scanning started for " << RunOptions_.UdfsPaths.size() << " udfs ...");
+            LoadRichMetadataToUdfIndex(*udfResolver, RunOptions_.UdfsPaths, false, TUdfIndex::EOverrideMode::RaiseError, *udfIndex);
+            RunOptions_.PrintInfo(TStringBuilder() << TInstant::Now().ToStringLocalUpToSeconds() << " UdfIndex done.");
+        }
 
         udfResolver = NCommon::CreateUdfResolverWithIndex(udfIndex, udfResolver, FileStorage_);
         RunOptions_.PrintInfo(TStringBuilder() << TInstant::Now().ToStringLocalUpToSeconds() << " Udfs scanned");
@@ -657,11 +673,15 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) {
 
 int TFacadeRunner::DoRun(TProgramFactory& factory) {
 
-    TProgramPtr program = factory.Create(RunOptions_.ProgramFile, RunOptions_.ProgramText, RunOptions_.OperationId, EHiddenMode::Disable, RunOptions_.QPlayerContext);;
+    TProgramPtr program = factory.Create(RunOptions_.ProgramFile, RunOptions_.ProgramText, RunOptions_.OperationId, EHiddenMode::Disable, RunOptions_.QPlayerContext, RunOptions_.GatewaysPatch);
     if (RunOptions_.Params) {
         program->SetParametersYson(RunOptions_.Params);
     }
 
+    if (RunOptions_.YsonAttrs) {
+        program->SetOperationAttrsYson(RunOptions_.YsonAttrs);
+    }
+
     if (RunOptions_.EnableResultPosition) {
         program->EnableResultPosition();
     }
@@ -686,6 +706,7 @@ int TFacadeRunner::DoRun(TProgramFactory& factory) {
         settings.TestAntlr4 = RunOptions_.TestAntlr4;
         settings.V0Behavior = NSQLTranslation::EV0Behavior::Report;
         settings.AssumeYdbOnClusterWithSlash = RunOptions_.AssumeYdbOnClusterWithSlash;
+        settings.Bindings = RunOptions_.Bindings;
         if (ERunMode::Discover == RunOptions_.Mode) {
             settings.Mode = NSQLTranslation::ESqlMode::DISCOVERY;
         }
diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.h b/yql/essentials/tools/yql_facade_run/yql_facade_run.h
index b586370c2d..d6037766c6 100644
--- a/yql/essentials/tools/yql_facade_run/yql_facade_run.h
+++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <yql/essentials/sql/settings/translation_settings.h>
 #include <yql/essentials/core/file_storage/defs/downloader.h>
 #include <yql/essentials/core/file_storage/file_storage.h>
 #include <yql/essentials/core/credentials/yql_credentials.h>
@@ -84,6 +85,8 @@ public:
     bool AnsiLexer = false;
     bool TestAntlr4 = false;
     bool AssumeYdbOnClusterWithSlash = false;
+    bool TestSqlFormat = false;
+    THashMap<TString, NSQLTranslation::TTableBindingSettings> Bindings;
 
     bool PrintAst = false;
     bool FullExpr = false;
@@ -92,6 +95,7 @@ public:
     int Verbosity = TLOG_ERR;
     bool ShowLog = false;
     bool WithFinalIssues = false;
+    bool ValidateResultFormat = false;
 
     IOutputStream* TraceOptStream = nullptr;
 
@@ -104,6 +108,7 @@ public:
     NYql::TUserDataTable DataTable;
     TVector<TString> UdfsPaths;
     TString Params;
+    TString YsonAttrs;
     NUdf::EValidateMode ValidateMode = NUdf::EValidateMode::Greedy;
     TCredentials::TPtr Credentials = MakeIntrusive<TCredentials>();
 
@@ -115,6 +120,7 @@ public:
     THolder<TGatewaysConfig> GatewaysConfig;
     THolder<TFileStorageConfig> FsConfig;
     THolder<NProto::TPgExtensions> PgExtConfig;
+    TMaybe<TString> GatewaysPatch;
 
     // No command line options for these settings. Should be configured in the inherited class
     bool NoDebug = false;
@@ -122,12 +128,11 @@ public:
     bool FailureInjectionSupport = false;
     bool UseRepeatableRandomAndTimeProviders = false;
     bool UseMetaFromGrpah = false;
-    bool TestSqlFormat = false;
-    bool ValidateResultFormat = false;
     bool EnableResultPosition = false;
     bool EnableCredentials = false;
     bool EnableQPlayer = false;
     bool OptimizeLibs = true;
+    bool CustomTests = false;
 
     void Parse(int argc, const char *argv[]);
 
@@ -145,6 +150,15 @@ public:
 
     void PrintInfo(const TString& msg);
 
+    static void ParseProtoConfig(const TString& cfgFile, google::protobuf::Message* config);
+
+    template <typename TMessage>
+    static THolder<TMessage> ParseProtoConfig(const TString& cfgFile) {
+        auto config = MakeHolder<TMessage>();
+        ParseProtoConfig(cfgFile, config.Get());
+        return config;
+    }
+
 private:
     std::vector<std::function<void(NLastGetopt::TOpts&)>> OptExtenders_;
     std::vector<std::function<void(const NLastGetopt::TOptsParseResult&)>> OptHandlers_;
diff --git a/yql/tools/yqlrun/lib/yqlrun_lib.cpp b/yql/tools/yqlrun/lib/yqlrun_lib.cpp
index 5006bd5b40..0eba257d9a 100644
--- a/yql/tools/yqlrun/lib/yqlrun_lib.cpp
+++ b/yql/tools/yqlrun/lib/yqlrun_lib.cpp
@@ -48,6 +48,7 @@ TYqlRunTool::TYqlRunTool()
 {
     GetRunOptions().UseRepeatableRandomAndTimeProviders = true;
     GetRunOptions().ResultsFormat = NYson::EYsonFormat::Pretty;
+    GetRunOptions().CustomTests = true;
 
     GetRunOptions().AddOptExtension([this](NLastGetopt::TOpts& opts) {
         opts.AddLongOption('t', "table", "Table mapping").RequiredArgument("table@file")
@@ -90,8 +91,6 @@ TYqlRunTool::TYqlRunTool()
                 });
             });
         opts.AddLongOption("tmp-dir", "Directory for temporary tables").RequiredArgument("DIR").StoreResult(&TmpDir_);
-        opts.AddLongOption("test-format", "Compare formatted query's AST with the original query's AST (only syntaxVersion=1 is supported)").NoArgument().SetFlag(&GetRunOptions().TestSqlFormat);
-        opts.AddLongOption("validate-result-format", "Check that result-format can parse Result").NoArgument().SetFlag(&GetRunOptions().ValidateResultFormat);
     });
 
     GetRunOptions().AddOptHandler([this](const NLastGetopt::TOptsParseResult& res) {
diff --git a/yt/yql/tools/ytrun/lib/ytrun_lib.cpp b/yt/yql/tools/ytrun/lib/ytrun_lib.cpp
index 9945d5ad67..547cdc0d88 100644
--- a/yt/yql/tools/ytrun/lib/ytrun_lib.cpp
+++ b/yt/yql/tools/ytrun/lib/ytrun_lib.cpp
@@ -105,10 +105,10 @@ TYtRunTool::TYtRunTool(TString name)
                         << ", current stage: " << progress.Stage.first << Endl;
                 });
             });
-        opts.AddLongOption("threads", "gateway threads")
+        opts.AddLongOption("yt-threads", "YT gateway threads")
             .Optional()
             .RequiredArgument("COUNT")
-            .StoreResult(&NumThreads_);
+            .StoreResult(&NumYtThreads_);
         opts.AddLongOption("keep-temp", "keep temporary tables")
             .Optional()
             .NoArgument()
@@ -134,7 +134,7 @@ TYtRunTool::TYtRunTool(TString name)
         }
 
         auto ytConfig = GetRunOptions().GatewaysConfig->MutableYt();
-        ytConfig->SetGatewayThreads(NumThreads_);
+        ytConfig->SetGatewayThreads(NumYtThreads_);
         if (MrJobBin_.empty()) {
             ytConfig->ClearMrJobBin();
         } else {
diff --git a/yt/yql/tools/ytrun/lib/ytrun_lib.h b/yt/yql/tools/ytrun/lib/ytrun_lib.h
index cb72e40304..edbc7c12ec 100644
--- a/yt/yql/tools/ytrun/lib/ytrun_lib.h
+++ b/yt/yql/tools/ytrun/lib/ytrun_lib.h
@@ -30,7 +30,7 @@ protected:
 protected:
     TString MrJobBin_;
     TString MrJobUdfsDir_;
-    size_t NumThreads_ = 1;
+    size_t NumYtThreads_ = 1;
     bool KeepTemp_ = false;
     TString DefYtServer_;
     NFmr::IFmrWorker::TPtr FmrWorker_;
diff --git a/yt/yql/tools/ytrun/ytrun.cpp b/yt/yql/tools/ytrun/ytrun.cpp
index d405eb419e..e3036198c7 100644
--- a/yt/yql/tools/ytrun/ytrun.cpp
+++ b/yt/yql/tools/ytrun/ytrun.cpp
@@ -1,4 +1,5 @@
 #include <yt/yql/tools/ytrun/lib/ytrun_lib.h>
+#include <util/generic/yexception.h>
 
 int main(int argc, const char *argv[]) {
     try {