diff options
author | grigoriypisar <grigoriypisar@yandex-team.com> | 2023-11-08 13:43:08 +0300 |
---|---|---|
committer | grigoriypisar <grigoriypisar@yandex-team.com> | 2023-11-08 15:02:15 +0300 |
commit | 52e23495a9390cac8f528198780a559df1cb5ea3 (patch) | |
tree | b6484f753c3019321a90087b159be4f1a02835d7 | |
parent | 07edcf8c7ece3559597b0db82f1d0d103a8ed4be (diff) | |
download | ydb-52e23495a9390cac8f528198780a559df1cb5ea3.tar.gz |
added iam token passing
Added iam token authentification
24 files changed, 1106 insertions, 5 deletions
diff --git a/.mapping.json b/.mapping.json index bbc838ef39..5fc5d1e849 100644 --- a/.mapping.json +++ b/.mapping.json @@ -9805,6 +9805,16 @@ "ydb/tests/tools/idx_test/CMakeLists.linux-x86_64.txt":"", "ydb/tests/tools/idx_test/CMakeLists.txt":"", "ydb/tests/tools/idx_test/CMakeLists.windows-x86_64.txt":"", + "ydb/tests/tools/kqprun/CMakeLists.darwin-x86_64.txt":"", + "ydb/tests/tools/kqprun/CMakeLists.linux-aarch64.txt":"", + "ydb/tests/tools/kqprun/CMakeLists.linux-x86_64.txt":"", + "ydb/tests/tools/kqprun/CMakeLists.txt":"", + "ydb/tests/tools/kqprun/CMakeLists.windows-x86_64.txt":"", + "ydb/tests/tools/kqprun/src/CMakeLists.darwin-x86_64.txt":"", + "ydb/tests/tools/kqprun/src/CMakeLists.linux-aarch64.txt":"", + "ydb/tests/tools/kqprun/src/CMakeLists.linux-x86_64.txt":"", + "ydb/tests/tools/kqprun/src/CMakeLists.txt":"", + "ydb/tests/tools/kqprun/src/CMakeLists.windows-x86_64.txt":"", "yt/CMakeLists.txt":"", "yt/cpp/CMakeLists.txt":"", "yt/cpp/mapreduce/CMakeLists.txt":"", diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index c690a131c2..786808273e 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -855,10 +855,10 @@ namespace Tests { auto databaseResolverActorId = NFq::MakeDatabaseResolverActorId(); Runtime->RegisterService( databaseResolverActorId, - Runtime->Register(NFq::CreateDatabaseResolver(httpProxyActorId, nullptr), nodeIdx), + Runtime->Register(NFq::CreateDatabaseResolver(httpProxyActorId, Settings->CredentialsFactory), nodeIdx), nodeIdx ); - + std::shared_ptr<NFq::TDatabaseAsyncResolverImpl> databaseAsyncResolver; if (queryServiceConfig.GetGeneric().HasMdbGateway() && queryServiceConfig.HasMdbTransformHost()) { databaseAsyncResolver = std::make_shared<NFq::TDatabaseAsyncResolverImpl>( @@ -873,7 +873,7 @@ namespace Tests { federatedQuerySetupFactory = std::make_shared<NKikimr::NKqp::TKqpFederatedQuerySetupFactoryMock>( NYql::IHTTPGateway::Make(&queryServiceConfig.GetHttpGateway()), NYql::NConnector::MakeClientGRPC(queryServiceConfig.GetGeneric().GetConnector()), - nullptr, + Settings->CredentialsFactory, databaseAsyncResolver, queryServiceConfig.GetS3(), queryServiceConfig.GetGeneric() diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index 5b44a07b8f..e8b96c9549 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -139,6 +139,7 @@ namespace Tests { TString MeteringFilePath; TString AwsRegion; NKqp::IKqpFederatedQuerySetupFactory::TPtr FederatedQuerySetupFactory = std::make_shared<NKqp::TKqpFederatedQuerySetupFactoryNoop>(); + NYql::ISecuredServiceAccountCredentialsFactory::TPtr CredentialsFactory; bool InitializeFederatedQuerySetupFactory = false; std::function<IActor*(const NKikimrProto::TAuthConfig&)> CreateTicketParser = NKikimr::CreateTicketParser; @@ -183,6 +184,7 @@ namespace Tests { TServerSettings& SetMeteringFilePath(const TString& path) { EnableMetering = true; MeteringFilePath = path; return *this; } TServerSettings& SetAwsRegion(const TString& value) { AwsRegion = value; return *this; } TServerSettings& SetFederatedQuerySetupFactory(NKqp::IKqpFederatedQuerySetupFactory::TPtr value) { FederatedQuerySetupFactory = value; return *this; } + TServerSettings& SetCredentialsFactory(NYql::ISecuredServiceAccountCredentialsFactory::TPtr credentialsFactory) { CredentialsFactory = std::move(credentialsFactory); return *this; } TServerSettings& SetInitializeFederatedQuerySetupFactory(bool value) { InitializeFederatedQuerySetupFactory = value; return *this; } TServerSettings& SetPersQueueGetReadSessionsInfoWorkerFactory( std::shared_ptr<NKikimr::NMsgBusProxy::IPersQueueGetReadSessionsInfoWorkerFactory> factory diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp index fe79fb197b..4b3c98ee53 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_settings.cpp @@ -3,7 +3,6 @@ #include <ydb/library/yql/providers/common/structured_token/yql_token_builder.h> #include <ydb/library/yql/utils/log/log.h> -#include <util/system/env.h> namespace NYql { @@ -84,7 +83,7 @@ namespace NYql { const auto iamToken = credentials->FindCredentialContent( "default_" + cluster.name(), "default_generic", - cluster.GetToken() ? cluster.GetToken() : GetEnv("YQL_TOKEN")); + cluster.GetToken()); if (iamToken) { return b.SetIAMToken(iamToken).ToJson(); } diff --git a/ydb/tests/tools/CMakeLists.txt b/ydb/tests/tools/CMakeLists.txt index 3ec7cd1baa..9ed8c2439a 100644 --- a/ydb/tests/tools/CMakeLists.txt +++ b/ydb/tests/tools/CMakeLists.txt @@ -7,3 +7,4 @@ add_subdirectory(idx_test) +add_subdirectory(kqprun) diff --git a/ydb/tests/tools/kqprun/CMakeLists.darwin-x86_64.txt b/ydb/tests/tools/kqprun/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..a7019ec48e --- /dev/null +++ b/ydb/tests/tools/kqprun/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,38 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(src) + +add_executable(kqprun) +target_compile_options(kqprun PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(kqprun PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + library-cpp-getopt + tools-kqprun-src + datetime2_udf + string_udf + yson2_udf +) +target_link_options(kqprun PRIVATE + -Wl,-platform_version,macos,11.0,11.0 + -fPIC + -fPIC + -framework + CoreFoundation +) +target_sources(kqprun PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/tests/tools/kqprun/kqprun.cpp +) +target_allocator(kqprun + system_allocator +) +vcs_info(kqprun) diff --git a/ydb/tests/tools/kqprun/CMakeLists.linux-aarch64.txt b/ydb/tests/tools/kqprun/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..b3c1be0871 --- /dev/null +++ b/ydb/tests/tools/kqprun/CMakeLists.linux-aarch64.txt @@ -0,0 +1,41 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(src) + +add_executable(kqprun) +target_compile_options(kqprun PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(kqprun PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-getopt + tools-kqprun-src + datetime2_udf + string_udf + yson2_udf +) +target_link_options(kqprun PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(kqprun PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/tests/tools/kqprun/kqprun.cpp +) +target_allocator(kqprun + cpp-malloc-jemalloc +) +vcs_info(kqprun) diff --git a/ydb/tests/tools/kqprun/CMakeLists.linux-x86_64.txt b/ydb/tests/tools/kqprun/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..b86593352c --- /dev/null +++ b/ydb/tests/tools/kqprun/CMakeLists.linux-x86_64.txt @@ -0,0 +1,43 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(src) + +add_executable(kqprun) +target_compile_options(kqprun PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(kqprun PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + library-cpp-getopt + tools-kqprun-src + datetime2_udf + string_udf + yson2_udf +) +target_link_options(kqprun PRIVATE + -ldl + -lrt + -Wl,--no-as-needed + -fPIC + -fPIC + -lpthread + -lrt + -ldl +) +target_sources(kqprun PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/tests/tools/kqprun/kqprun.cpp +) +target_allocator(kqprun + cpp-malloc-tcmalloc + libs-tcmalloc-no_percpu_cache +) +vcs_info(kqprun) diff --git a/ydb/tests/tools/kqprun/CMakeLists.txt b/ydb/tests/tools/kqprun/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/tests/tools/kqprun/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/tests/tools/kqprun/CMakeLists.windows-x86_64.txt b/ydb/tests/tools/kqprun/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..d9a54cf808 --- /dev/null +++ b/ydb/tests/tools/kqprun/CMakeLists.windows-x86_64.txt @@ -0,0 +1,31 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(src) + +add_executable(kqprun) +target_compile_options(kqprun PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(kqprun PUBLIC + contrib-libs-cxxsupp + yutil + library-cpp-cpuid_check + library-cpp-getopt + tools-kqprun-src + datetime2_udf + string_udf + yson2_udf +) +target_sources(kqprun PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/tests/tools/kqprun/kqprun.cpp +) +target_allocator(kqprun + system_allocator +) +vcs_info(kqprun) diff --git a/ydb/tests/tools/kqprun/kqprun.cpp b/ydb/tests/tools/kqprun/kqprun.cpp new file mode 100644 index 0000000000..3594b60772 --- /dev/null +++ b/ydb/tests/tools/kqprun/kqprun.cpp @@ -0,0 +1,211 @@ +#include "src/kqp_runner.h" + +#include <contrib/libs/protobuf/src/google/protobuf/text_format.h> + +#include <library/cpp/colorizer/colors.h> +#include <library/cpp/getopt/last_getopt.h> + +#include <util/stream/file.h> +#include <util/system/env.h> + +#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h> +#include <ydb/library/yql/utils/backtrace/backtrace.h> + + +struct TExecutionOptions { + TString ScriptQuery; + TString SchemeQuery; + + NKikimrKqp::EQueryAction ScriptQueryAction = NKikimrKqp::QUERY_ACTION_EXECUTE; +}; + + +void RunScript(const TExecutionOptions& executionOptions, const NKqpRun::TRunnerOptions& runnerOptions) { + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + + Cout << colors.Yellow() << "Initialization of kqp runner..." << colors.Default() << Endl; + NKqpRun::TKqpRunner runner(runnerOptions); + + if (executionOptions.SchemeQuery) { + Cout << colors.Yellow() << "Executing scheme query..." << colors.Default() << Endl; + if (!runner.ExecuteSchemeQuery(executionOptions.SchemeQuery)) { + ythrow yexception() << "Scheme query execution failed"; + } + } + + if (executionOptions.ScriptQuery) { + Cout << colors.Yellow() << "Executing script..." << colors.Default() << Endl; + if (!runner.ExecuteScript(executionOptions.ScriptQuery, executionOptions.ScriptQueryAction)) { + ythrow yexception() << "Script execution failed"; + } + } + + if (executionOptions.ScriptQueryAction == NKikimrKqp::QUERY_ACTION_EXECUTE) { + Cout << colors.Yellow() << "Writing script results..." << colors.Default() << Endl; + if (!runner.WriteScriptResults()) { + ythrow yexception() << "Writing script results failed"; + } + } +} + + +THolder<TFileOutput> SetupDefaultFileOutput(const TString& filePath, IOutputStream*& stream) { + THolder<TFileOutput> fileHolder; + if (filePath == "-") { + stream = &Cout; + } else if (filePath) { + fileHolder.Reset(new TFileOutput(filePath)); + stream = fileHolder.Get(); + } + return fileHolder; +} + + +void RunMain(int argc, const char* argv[]) { + TString scriptQueryFile; + TString schemeQueryFile; + TString resultOutputFile = "-"; + TString schemeQueryAstFile; + TString scriptQueryAstFile; + TString scriptQueryPlanFile; + TString logFile = "-"; + + TString scriptQueryAction = "execute"; + TString planOutputFormat = "pretty"; + i64 resultsRowsLimit = 1000; + + TVector<TString> udfsPaths; + TString udfsDirectory; + + NLastGetopt::TOpts options = NLastGetopt::TOpts::Default(); + options.AddLongOption('p', "script-query", "Script query to execute") + .Optional() + .RequiredArgument("FILE") + .StoreResult(&scriptQueryFile); + options.AddLongOption('s', "scheme-query", "Scheme query to execute") + .Optional() + .RequiredArgument("FILE") + .StoreResult(&schemeQueryFile); + + options.AddLongOption("log-file", "File with execution logs (use '-' to write in stderr)") + .Optional() + .RequiredArgument("FILE") + .StoreResult(&logFile); + options.AddLongOption("result-file", "File with script execution results (use '-' to write in stdout)") + .Optional() + .RequiredArgument("FILE") + .StoreResult(&resultOutputFile); + options.AddLongOption("scheme-ast-file", "File with scheme query ast (use '-' to write in stdout)") + .Optional() + .RequiredArgument("FILE") + .StoreResult(&schemeQueryAstFile); + options.AddLongOption("script-ast-file", "File with script query ast (use '-' to write in stdout)") + .Optional() + .RequiredArgument("FILE") + .StoreResult(&scriptQueryAstFile); + options.AddLongOption("script-plan-file", "File with script query plan (use '-' to write in stdout)") + .Optional() + .RequiredArgument("FILE") + .StoreResult(&scriptQueryPlanFile); + + options.AddLongOption("script-action", "Script query execute action, one of { execute | explain }") + .Optional() + .RequiredArgument("STR") + .DefaultValue(scriptQueryAction) + .StoreResult(&scriptQueryAction); + options.AddLongOption("plan-format", "Script query plan format, one of { pretty | table | json }") + .Optional() + .RequiredArgument("STR") + .DefaultValue(planOutputFormat) + .StoreResult(&planOutputFormat); + options.AddLongOption("results-limit", "Rows limit for script execution results") + .Optional() + .RequiredArgument("INT") + .DefaultValue(resultsRowsLimit) + .StoreResult(&resultsRowsLimit); + + options.AddLongOption("udf", "Load shared library with UDF by given path") + .Optional() + .RequiredArgument("FILE") + .AppendTo(&udfsPaths); + options.AddLongOption("udfs-dir", "Load all shared libraries with UDFs found in given directory") + .Optional() + .RequiredArgument("PATH") + .StoreResult(&udfsDirectory); + + NLastGetopt::TOptsParseResult parsedOptions(&options, argc, argv); + + // Execution options + + TExecutionOptions executionOptions; + + if (!schemeQueryFile && !scriptQueryFile) { + ythrow yexception() << "Nothing to execute"; + } + if (schemeQueryFile) { + executionOptions.SchemeQuery = TFileInput(schemeQueryFile).ReadAll(); + } + if (scriptQueryFile) { + executionOptions.ScriptQuery = TFileInput(scriptQueryFile).ReadAll(); + } + + executionOptions.ScriptQueryAction = + (scriptQueryAction == TStringBuf("execute")) ? NKikimrKqp::QUERY_ACTION_EXECUTE + : (scriptQueryAction == TStringBuf("explain")) ? NKikimrKqp::QUERY_ACTION_EXPLAIN + : NKikimrKqp::QUERY_ACTION_EXECUTE; + + // Runner options + + NKqpRun::TRunnerOptions runnerOptions; + + if (resultsRowsLimit >= 0) { + runnerOptions.ResultsRowsLimit = resultsRowsLimit; + } else { + ythrow yexception() << "Results rows limit less than zero"; + } + + THolder<TFileOutput> resultFileHolder = SetupDefaultFileOutput(resultOutputFile, runnerOptions.ResultOutput); + THolder<TFileOutput> schemeQueryAstFileHolder = SetupDefaultFileOutput(schemeQueryAstFile, runnerOptions.SchemeQueryAstOutput); + THolder<TFileOutput> scriptQueryAstFileHolder = SetupDefaultFileOutput(scriptQueryAstFile, runnerOptions.ScriptQueryAstOutput); + THolder<TFileOutput> scriptQueryPlanFileHolder = SetupDefaultFileOutput(scriptQueryPlanFile, runnerOptions.ScriptQueryPlanOutput); + + runnerOptions.PlanOutputFormat = + (planOutputFormat == TStringBuf("pretty")) ? NYdb::NConsoleClient::EOutputFormat::Pretty + : (planOutputFormat == TStringBuf("table")) ? NYdb::NConsoleClient::EOutputFormat::PrettyTable + : (planOutputFormat == TStringBuf("json")) ? NYdb::NConsoleClient::EOutputFormat::JsonUnicode + : NYdb::NConsoleClient::EOutputFormat::Default; + + // Ydb settings + + if (logFile != "-") { + runnerOptions.YdbSettings.LogOutputFile = logFile; + } + + runnerOptions.YdbSettings.YqlToken = GetEnv("YQL_TOKEN"); + + NKikimr::NMiniKQL::FindUdfsInDir(udfsDirectory, &udfsPaths); + auto functionRegistry = NKikimr::NMiniKQL::CreateFunctionRegistry(&NYql::NBacktrace::KikimrBackTrace, NKikimr::NMiniKQL::CreateBuiltinRegistry(), false, udfsPaths)->Clone(); + NKikimr::NMiniKQL::FillStaticModules(*functionRegistry); + runnerOptions.YdbSettings.FunctionRegistry = functionRegistry.Get(); + + TString appConfigData = TFileInput("./configuration/app_config.conf").ReadAll(); + if (!google::protobuf::TextFormat::ParseFromString(appConfigData, &runnerOptions.YdbSettings.AppConfig)) { + ythrow yexception() << "Bad format of app configuration"; + } + + RunScript(executionOptions, runnerOptions); +} + + +int main(int argc, const char* argv[]) { + try { + RunMain(argc, argv); + } catch (...) { + NColorizer::TColors colors = NColorizer::AutoColors(Cerr); + + Cerr << colors.Red() << CurrentExceptionMessage() << colors.Default() << Endl; + return 1; + } + + return 0; +} diff --git a/ydb/tests/tools/kqprun/src/CMakeLists.darwin-x86_64.txt b/ydb/tests/tools/kqprun/src/CMakeLists.darwin-x86_64.txt new file mode 100644 index 0000000000..54954b2566 --- /dev/null +++ b/ydb/tests/tools/kqprun/src/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(tools-kqprun-src) +target_compile_options(tools-kqprun-src PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(tools-kqprun-src PUBLIC + contrib-libs-cxxsupp + yutil + ydb-core-testlib + yql-parser-pg_wrapper + yql-sql-pg +) +target_sources(tools-kqprun-src PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/tests/tools/kqprun/src/kqp_runner.cpp + ${CMAKE_SOURCE_DIR}/ydb/tests/tools/kqprun/src/ydb_setup.cpp +) diff --git a/ydb/tests/tools/kqprun/src/CMakeLists.linux-aarch64.txt b/ydb/tests/tools/kqprun/src/CMakeLists.linux-aarch64.txt new file mode 100644 index 0000000000..f129ed449d --- /dev/null +++ b/ydb/tests/tools/kqprun/src/CMakeLists.linux-aarch64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(tools-kqprun-src) +target_compile_options(tools-kqprun-src PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(tools-kqprun-src PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + ydb-core-testlib + yql-parser-pg_wrapper + yql-sql-pg +) +target_sources(tools-kqprun-src PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/tests/tools/kqprun/src/kqp_runner.cpp + ${CMAKE_SOURCE_DIR}/ydb/tests/tools/kqprun/src/ydb_setup.cpp +) diff --git a/ydb/tests/tools/kqprun/src/CMakeLists.linux-x86_64.txt b/ydb/tests/tools/kqprun/src/CMakeLists.linux-x86_64.txt new file mode 100644 index 0000000000..f129ed449d --- /dev/null +++ b/ydb/tests/tools/kqprun/src/CMakeLists.linux-x86_64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(tools-kqprun-src) +target_compile_options(tools-kqprun-src PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(tools-kqprun-src PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + ydb-core-testlib + yql-parser-pg_wrapper + yql-sql-pg +) +target_sources(tools-kqprun-src PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/tests/tools/kqprun/src/kqp_runner.cpp + ${CMAKE_SOURCE_DIR}/ydb/tests/tools/kqprun/src/ydb_setup.cpp +) diff --git a/ydb/tests/tools/kqprun/src/CMakeLists.txt b/ydb/tests/tools/kqprun/src/CMakeLists.txt new file mode 100644 index 0000000000..f8b31df0c1 --- /dev/null +++ b/ydb/tests/tools/kqprun/src/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/ydb/tests/tools/kqprun/src/CMakeLists.windows-x86_64.txt b/ydb/tests/tools/kqprun/src/CMakeLists.windows-x86_64.txt new file mode 100644 index 0000000000..54954b2566 --- /dev/null +++ b/ydb/tests/tools/kqprun/src/CMakeLists.windows-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(tools-kqprun-src) +target_compile_options(tools-kqprun-src PRIVATE + -DUSE_CURRENT_UDF_ABI_VERSION +) +target_link_libraries(tools-kqprun-src PUBLIC + contrib-libs-cxxsupp + yutil + ydb-core-testlib + yql-parser-pg_wrapper + yql-sql-pg +) +target_sources(tools-kqprun-src PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/tests/tools/kqprun/src/kqp_runner.cpp + ${CMAKE_SOURCE_DIR}/ydb/tests/tools/kqprun/src/ydb_setup.cpp +) diff --git a/ydb/tests/tools/kqprun/src/common.h b/ydb/tests/tools/kqprun/src/common.h new file mode 100644 index 0000000000..fd7b021bbf --- /dev/null +++ b/ydb/tests/tools/kqprun/src/common.h @@ -0,0 +1,36 @@ +#pragma once + +#include <ydb/core/protos/config.pb.h> + +#include <ydb/library/yql/minikql/mkql_function_registry.h> + +#include <ydb/public/lib/ydb_cli/common/formats.h> + + +namespace NKqpRun { + +struct TYdbSetupSettings { + TString DomainName = "Root"; + + TMaybe<TString> LogOutputFile; + + TString YqlToken; + NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry = nullptr; + NKikimrConfig::TAppConfig AppConfig; +}; + + +struct TRunnerOptions { + i64 ResultsRowsLimit = 1000; + + IOutputStream* ResultOutput = &Cout; + IOutputStream* SchemeQueryAstOutput = nullptr; + IOutputStream* ScriptQueryAstOutput = nullptr; + IOutputStream* ScriptQueryPlanOutput = nullptr; + + NYdb::NConsoleClient::EOutputFormat PlanOutputFormat = NYdb::NConsoleClient::EOutputFormat::Default; + + TYdbSetupSettings YdbSettings; +}; + +} // namespace NKqpRun diff --git a/ydb/tests/tools/kqprun/src/kqp_runner.cpp b/ydb/tests/tools/kqprun/src/kqp_runner.cpp new file mode 100644 index 0000000000..1d118a93d0 --- /dev/null +++ b/ydb/tests/tools/kqprun/src/kqp_runner.cpp @@ -0,0 +1,135 @@ +#include "kqp_runner.h" +#include "ydb_setup.h" + +#include <library/cpp/colorizer/colors.h> + +#include <ydb/public/lib/json_value/ydb_json_value.h> +#include <ydb/public/lib/ydb_cli/common/format.h> + + +namespace NKqpRun { + +//// TKqpRunner::TImpl + +class TKqpRunner::TImpl { +public: + explicit TImpl(const TRunnerOptions& options) + : Options_(options) + , YdbSetup_(options.YdbSettings) + , CerrColors_(NColorizer::AutoColors(Cerr)) + , CoutColors_(NColorizer::AutoColors(Cout)) + {} + + bool ExecuteSchemeQuery(const TString& query) const { + TSchemeMeta meta; + TRequestResult status = YdbSetup_.SchemeQueryRequest(query, meta); + + if (!status.IsSuccess()) { + Cerr << CerrColors_.Red() << "Failed to execute scheme query, reason:" << CerrColors_.Default() << Endl << status.ToString() << Endl; + return false; + } + + if (Options_.SchemeQueryAstOutput) { + Cout << CoutColors_.Cyan() << "Writing scheme query ast" << CoutColors_.Default() << Endl; + Options_.SchemeQueryAstOutput->Write(meta.Ast); + } + + return true; + } + + bool ExecuteScript(const TString& script, NKikimrKqp::EQueryAction action) { + TRequestResult status = YdbSetup_.ScriptQueryRequest(script, action, ExecutionOperation_); + + if (!status.IsSuccess()) { + Cerr << CerrColors_.Red() << "Failed to start script execution, reason:" << CerrColors_.Default() << Endl << status.ToString() << Endl; + return false; + } + + return WaitScriptExecutionOperation(); + } + + bool WriteScriptResults() const { + for (i32 resultSetId = 0; resultSetId < ExecutionMeta_.ResultSetsCount; ++resultSetId) { + Ydb::ResultSet resultSet; + TRequestResult status = YdbSetup_.FetchScriptExecutionResultsRequest(ExecutionOperation_, resultSetId, Options_.ResultsRowsLimit, resultSet); + + if (!status.IsSuccess()) { + Cerr << CerrColors_.Red() << "Failed to fetch result set with id " << resultSetId << ", reason:" << CerrColors_.Default() << Endl << status.ToString() << Endl; + return false; + } + + Options_.ResultOutput->Write(NYdb::FormatResultSetJson(resultSet, NYdb::EBinaryStringEncoding::Unicode)); + } + + return true; + } + +private: + bool WaitScriptExecutionOperation() { + TRequestResult status; + while (true) { + status = YdbSetup_.GetScriptExecutionOperationRequest(ExecutionOperation_, ExecutionMeta_); + + if (ExecutionMeta_.Ready) { + break; + } + + if (!status.IsSuccess()) { + Cerr << CerrColors_.Red() << "Failed to get script execution operation, reason:" << CerrColors_.Default() << Endl << status.ToString() << Endl; + return false; + } + + Sleep(TDuration::Seconds(1)); + } + + if (Options_.ScriptQueryAstOutput) { + Cout << CoutColors_.Cyan() << "Writing script query ast" << CoutColors_.Default() << Endl; + Options_.ScriptQueryAstOutput->Write(ExecutionMeta_.Ast); + } + + if (!status.IsSuccess() || ExecutionMeta_.ExecutionStatus != NYdb::NQuery::EExecStatus::Completed) { + Cerr << CerrColors_.Red() << "Failed to execute script, invalid final status, reason:" << CerrColors_.Default() << Endl << status.ToString() << Endl; + return false; + } + + if (Options_.ScriptQueryPlanOutput) { + Cout << CoutColors_.Cyan() << "Writing script query plan" << CoutColors_.Default() << Endl; + + NYdb::NConsoleClient::TQueryPlanPrinter printer(Options_.PlanOutputFormat, true, *Options_.ScriptQueryPlanOutput); + printer.Print(ExecutionMeta_.Plan); + } + + return true; + } + +private: + TRunnerOptions Options_; + + TYdbSetup YdbSetup_; + NColorizer::TColors CerrColors_; + NColorizer::TColors CoutColors_; + + TString ExecutionOperation_; + TExecutionMeta ExecutionMeta_; +}; + + +//// TKqpRunner + +TKqpRunner::TKqpRunner(const TRunnerOptions& options) + : Impl_(new TImpl(options)) +{} + +bool TKqpRunner::ExecuteSchemeQuery(const TString& query) const { + return Impl_->ExecuteSchemeQuery(query); +} + +bool TKqpRunner::ExecuteScript(const TString& query, NKikimrKqp::EQueryAction action) const { + return Impl_->ExecuteScript(query, action); +} + +bool TKqpRunner::WriteScriptResults() const { + return Impl_->WriteScriptResults(); +} + +} // namespace NKqpRun diff --git a/ydb/tests/tools/kqprun/src/kqp_runner.h b/ydb/tests/tools/kqprun/src/kqp_runner.h new file mode 100644 index 0000000000..233cdd0475 --- /dev/null +++ b/ydb/tests/tools/kqprun/src/kqp_runner.h @@ -0,0 +1,23 @@ +#pragma once + +#include "common.h" + + +namespace NKqpRun { + +class TKqpRunner { +public: + explicit TKqpRunner(const TRunnerOptions& options); + + bool ExecuteSchemeQuery(const TString& query) const; + + bool ExecuteScript(const TString& script, NKikimrKqp::EQueryAction action) const; + + bool WriteScriptResults() const; + +private: + class TImpl; + std::shared_ptr<TImpl> Impl_; +}; + +} // namespace NKqpRun diff --git a/ydb/tests/tools/kqprun/src/ya.make b/ydb/tests/tools/kqprun/src/ya.make new file mode 100644 index 0000000000..98931dcd3f --- /dev/null +++ b/ydb/tests/tools/kqprun/src/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +SRCS( + kqp_runner.cpp + ydb_setup.cpp +) + +PEERDIR( + ydb/core/testlib + + ydb/library/yql/parser/pg_wrapper + ydb/library/yql/sql/pg +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/tests/tools/kqprun/src/ydb_setup.cpp b/ydb/tests/tools/kqprun/src/ydb_setup.cpp new file mode 100644 index 0000000000..31ff8f78b2 --- /dev/null +++ b/ydb/tests/tools/kqprun/src/ydb_setup.cpp @@ -0,0 +1,303 @@ +#include "ydb_setup.h" + +#include <ydb/core/kqp/common/kqp_script_executions.h> +#include <ydb/core/kqp/proxy_service/kqp_script_executions.h> + +#include <ydb/core/testlib/test_client.h> + + +namespace NKqpRun { + +namespace { + +class TStaticCredentialsProvider : public NYdb::ICredentialsProvider { +public: + TStaticCredentialsProvider(const TString& yqlToken) + : YqlToken_(yqlToken) + {} + + TString GetAuthInfo() const override { + return YqlToken_; + } + + bool IsValid() const override { + return true; + } + +private: + TString YqlToken_; +}; + +class TStaticCredentialsProviderFactory : public NYdb::ICredentialsProviderFactory { +public: + TStaticCredentialsProviderFactory(const TString& yqlToken) + : YqlToken_(yqlToken) + {} + + std::shared_ptr<NYdb::ICredentialsProvider> CreateProvider() const override { + return std::make_shared<TStaticCredentialsProvider>(YqlToken_); + } + +private: + TString YqlToken_; +}; + +class TStaticSecuredCredentialsFactory : public NYql::ISecuredServiceAccountCredentialsFactory { +public: + TStaticSecuredCredentialsFactory(const TString& yqlToken) + : YqlToken_(yqlToken) + {} + + std::shared_ptr<NYdb::ICredentialsProviderFactory> Create(const TString&, const TString&) override { + return std::make_shared<TStaticCredentialsProviderFactory>(YqlToken_); + } + +private: + TString YqlToken_; +}; + +} // anonymous namespace + + +//// TYdbSetup::TImpl + +class TYdbSetup::TImpl { +private: + void SetLoggerSettings(NKikimr::Tests::TServerSettings& serverSettings) const { + auto loggerInitializer = [this](NActors::TTestActorRuntime& runtime) { + if (Settings_.AppConfig.GetLogConfig().HasDefaultLevel()) { + auto priority = NActors::NLog::EPriority(Settings_.AppConfig.GetLogConfig().GetDefaultLevel()); + auto descriptor = NKikimrServices::EServiceKikimr_descriptor(); + for (int i = 0; i < descriptor->value_count(); ++i) { + runtime.SetLogPriority(static_cast<NKikimrServices::EServiceKikimr>(descriptor->value(i)->number()), priority); + } + } + + for (auto setting : Settings_.AppConfig.GetLogConfig().get_arr_entry()) { + NKikimrServices::EServiceKikimr service; + if (!NKikimrServices::EServiceKikimr_Parse(setting.GetComponent(), &service)) { + ythrow yexception() << "Invalid kikimr service name " << setting.GetComponent(); + } + + runtime.SetLogPriority(service, NActors::NLog::EPriority(setting.GetLevel())); + } + }; + + serverSettings.SetLoggerInitializer(loggerInitializer); + + if (Settings_.LogOutputFile) { + serverSettings.SetLogBackend(NActors::CreateFileBackend(*Settings_.LogOutputFile)); + } else { + serverSettings.SetLogBackend(NActors::CreateStderrBackend()); + } + } + + void SetFunctionRegistry(NKikimr::Tests::TServerSettings& serverSettings) const { + if (!Settings_.FunctionRegistry) { + return; + } + + auto functionRegistryFactory = [this](const NKikimr::NScheme::TTypeRegistry&) { + return Settings_.FunctionRegistry; + }; + + serverSettings.SetFrFactory(functionRegistryFactory); + } + + NKikimr::Tests::TServerSettings GetServerSettings() { + ui32 msgBusPort = PortManager_.GetPort(); + + NKikimr::Tests::TServerSettings serverSettings(msgBusPort); + serverSettings.SetNodeCount(1); + + serverSettings.SetDomainName(Settings_.DomainName); + serverSettings.SetAppConfig(Settings_.AppConfig); + serverSettings.SetFeatureFlags(Settings_.AppConfig.GetFeatureFlags()); + + serverSettings.SetCredentialsFactory(std::make_shared<TStaticSecuredCredentialsFactory>(Settings_.YqlToken)); + serverSettings.SetInitializeFederatedQuerySetupFactory(true); + + SetLoggerSettings(serverSettings); + SetFunctionRegistry(serverSettings); + + return serverSettings; + } + + void InitializeServer() { + NKikimr::Tests::TServerSettings serverSettings = GetServerSettings(); + + Server_ = MakeHolder<NKikimr::Tests::TServer>(serverSettings); + Server_->GetRuntime()->SetDispatchTimeout(TDuration::Max()); + + Client_ = MakeHolder<NKikimr::Tests::TClient>(serverSettings); + Client_->InitRootScheme(); + } + +public: + explicit TImpl(const TYdbSetupSettings& settings) + : Settings_(settings) + { + InitializeServer(); + } + + NKikimr::NKqp::TEvKqp::TEvQueryResponse::TPtr SchemeQueryRequest(const TString& query) const { + auto event = MakeHolder<NKikimr::NKqp::TEvKqp::TEvQueryRequest>(); + FillSchemeRequest(query, *event->Record.MutableRequest()); + + return RunKqpProxyRequest<NKikimr::NKqp::TEvKqp::TEvQueryRequest, NKikimr::NKqp::TEvKqp::TEvQueryResponse>(std::move(event)); + } + + NKikimr::NKqp::TEvKqp::TEvScriptResponse::TPtr ScriptQueryRequest(const TString& script, NKikimrKqp::EQueryAction action) const { + auto event = MakeHolder<NKikimr::NKqp::TEvKqp::TEvScriptRequest>(); + FillScriptRequest(script, action, *event->Record.MutableRequest()); + + return RunKqpProxyRequest<NKikimr::NKqp::TEvKqp::TEvScriptRequest, NKikimr::NKqp::TEvKqp::TEvScriptResponse>(std::move(event)); + } + + NKikimr::NKqp::TEvGetScriptExecutionOperationResponse::TPtr GetScriptExecutionOperationRequest(const TString& operation) const { + NKikimr::NOperationId::TOperationId operationId(operation); + auto event = MakeHolder<NKikimr::NKqp::TEvGetScriptExecutionOperation>(Settings_.DomainName, operationId); + + return RunKqpProxyRequest<NKikimr::NKqp::TEvGetScriptExecutionOperation, NKikimr::NKqp::TEvGetScriptExecutionOperationResponse>(std::move(event)); + } + + NKikimr::NKqp::TEvKqp::TEvFetchScriptResultsResponse::TPtr FetchScriptExecutionResultsRequest(const TString& operation, i32 resultSetId, i64 limit) const { + TString executionId = *NKikimr::NKqp::ScriptExecutionIdFromOperation(operation); + + NActors::TActorId edgeActor = GetRuntime()->AllocateEdgeActor(); + NActors::IActor* fetchActor = NKikimr::NKqp::CreateGetScriptExecutionResultActor(edgeActor, Settings_.DomainName, executionId, resultSetId, 0, limit); + + GetRuntime()->Register(fetchActor); + + return GetRuntime()->GrabEdgeEvent<NKikimr::NKqp::TEvKqp::TEvFetchScriptResultsResponse>(edgeActor); + } + + ~TImpl() { + Server_.Reset(); + Client_.Reset(); + } + +private: + NActors::TTestActorRuntime* GetRuntime() const { + return Server_->GetRuntime(); + } + + template <typename TRequest, typename TResponse> + typename TResponse::TPtr RunKqpProxyRequest(THolder<TRequest> event) const { + NActors::TActorId edgeActor = GetRuntime()->AllocateEdgeActor(); + NActors::TActorId kqpProxy = NKikimr::NKqp::MakeKqpProxyID(GetRuntime()->GetNodeId()); + + GetRuntime()->Send(kqpProxy, edgeActor, event.Release()); + + return GetRuntime()->GrabEdgeEvent<TResponse>(edgeActor); + } + +private: + void FillSchemeRequest(const TString& query, NKikimrKqp::TQueryRequest& request) const { + request.SetType(NKikimrKqp::QUERY_TYPE_SQL_DDL); + request.SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); + request.SetCollectStats(Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL); + + request.SetDatabase(Settings_.DomainName); + request.SetQuery(query); + } + + void FillScriptRequest(const TString& script, NKikimrKqp::EQueryAction action, NKikimrKqp::TQueryRequest& request) const { + if (action == NKikimrKqp::QUERY_ACTION_EXECUTE) { + request.MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write(); + request.MutableTxControl()->set_commit_tx(true); + } + + request.SetType(NKikimrKqp::QUERY_TYPE_SQL_GENERIC_SCRIPT); + request.SetAction(action); + request.SetCollectStats(Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL); + + request.SetDatabase(Settings_.DomainName); + request.SetQuery(script); + } + +private: + TYdbSetupSettings Settings_; + + THolder<NKikimr::Tests::TServer> Server_; + THolder<NKikimr::Tests::TClient> Client_; + TPortManager PortManager_; +}; + + +//// TRequestResult + +TRequestResult::TRequestResult() + : Status(Ydb::StatusIds::STATUS_CODE_UNSPECIFIED) +{} + +TRequestResult::TRequestResult(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues) + : Status(status) + , Issues(issues) +{} + +bool TRequestResult::IsSuccess() const { + return Status == Ydb::StatusIds::SUCCESS; +} + +TString TRequestResult::ToString() const { + return TStringBuilder() << "Request finished with status: " << Status << "\nIssues:\n" << Issues.ToString() << "\n"; +} + + +//// TYdbSetup + +TYdbSetup::TYdbSetup(const TYdbSetupSettings& settings) + : Impl_(new TImpl(settings)) +{} + +TRequestResult TYdbSetup::SchemeQueryRequest(const TString& query, TSchemeMeta& meta) const { + auto schemeQueryOperationResponse = Impl_->SchemeQueryRequest(query)->Get()->Record.GetRef(); + + meta.Ast = schemeQueryOperationResponse.GetResponse().GetQueryAst(); + + NYql::TIssues issues; + NYql::IssuesFromMessage(schemeQueryOperationResponse.GetResponse().GetQueryIssues(), issues); + + return TRequestResult(schemeQueryOperationResponse.GetYdbStatus(), issues); +} + +TRequestResult TYdbSetup::ScriptQueryRequest(const TString& script, NKikimrKqp::EQueryAction action, TString& operation) const { + auto scriptExecutionOperation = Impl_->ScriptQueryRequest(script, action); + + operation = scriptExecutionOperation->Get()->OperationId; + + return TRequestResult(scriptExecutionOperation->Get()->Status, scriptExecutionOperation->Get()->Issues); +} + +TRequestResult TYdbSetup::GetScriptExecutionOperationRequest(const TString& operation, TExecutionMeta& meta) const { + auto scriptExecutionOperation = Impl_->GetScriptExecutionOperationRequest(operation); + + meta.Ready = scriptExecutionOperation->Get()->Ready; + + auto serializedMeta = scriptExecutionOperation->Get()->Metadata; + if (serializedMeta) { + Ydb::Query::ExecuteScriptMetadata deserializedMeta; + serializedMeta->UnpackTo(&deserializedMeta); + + meta.ExecutionStatus = static_cast<NYdb::NQuery::EExecStatus>(deserializedMeta.exec_status()); + meta.ResultSetsCount = deserializedMeta.result_sets_meta_size(); + meta.Ast = deserializedMeta.exec_stats().query_ast(); + meta.Plan = deserializedMeta.exec_stats().query_plan(); + } + + return TRequestResult(scriptExecutionOperation->Get()->Status, scriptExecutionOperation->Get()->Issues); +} + +TRequestResult TYdbSetup::FetchScriptExecutionResultsRequest(const TString& operation, i32 resultSetId, i64 limit, Ydb::ResultSet& resultSet) const { + auto scriptExecutionResults = Impl_->FetchScriptExecutionResultsRequest(operation, resultSetId, limit)->Get()->Record; + + resultSet = scriptExecutionResults.GetResultSet(); + + NYql::TIssues issues; + NYql::IssuesFromMessage(scriptExecutionResults.GetIssues(), issues); + + return TRequestResult(scriptExecutionResults.GetStatus(), issues); +} + +} // namespace NKqpRun diff --git a/ydb/tests/tools/kqprun/src/ydb_setup.h b/ydb/tests/tools/kqprun/src/ydb_setup.h new file mode 100644 index 0000000000..c918ef25e3 --- /dev/null +++ b/ydb/tests/tools/kqprun/src/ydb_setup.h @@ -0,0 +1,57 @@ +#pragma once + +#include "common.h" + +#include <ydb/public/sdk/cpp/client/ydb_query/query.h> + + +namespace NKqpRun { + +struct TSchemeMeta { + TString Ast; +}; + + +struct TExecutionMeta { + bool Ready = false; + NYdb::NQuery::EExecStatus ExecutionStatus = NYdb::NQuery::EExecStatus::Unspecified; + + i32 ResultSetsCount = 0; + + TString Ast; + TString Plan; +}; + + +struct TRequestResult { + Ydb::StatusIds::StatusCode Status; + NYql::TIssues Issues; + + TRequestResult(); + + TRequestResult(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues); + + bool IsSuccess() const; + + TString ToString() const; +}; + + +class TYdbSetup { +public: + explicit TYdbSetup(const TYdbSetupSettings& settings); + + TRequestResult SchemeQueryRequest(const TString& query, TSchemeMeta& meta) const; + + TRequestResult ScriptQueryRequest(const TString& script, NKikimrKqp::EQueryAction action, TString& operation) const; + + TRequestResult GetScriptExecutionOperationRequest(const TString& operation, TExecutionMeta& meta) const; + + TRequestResult FetchScriptExecutionResultsRequest(const TString& operation, i32 resultSetId, i64 limit, Ydb::ResultSet& resultSet) const; + +private: + class TImpl; + std::shared_ptr<TImpl> Impl_; +}; + +} // namespace NKqpRun diff --git a/ydb/tests/tools/kqprun/ya.make b/ydb/tests/tools/kqprun/ya.make new file mode 100644 index 0000000000..0aefcbe444 --- /dev/null +++ b/ydb/tests/tools/kqprun/ya.make @@ -0,0 +1,21 @@ +PROGRAM() + +SRCS( + kqprun.cpp +) + +PEERDIR( + library/cpp/getopt + + ydb/tests/tools/kqprun/src +) + +PEERDIR( + ydb/library/yql/udfs/common/datetime2 + ydb/library/yql/udfs/common/string + ydb/library/yql/udfs/common/yson2 +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/tests/tools/ya.make b/ydb/tests/tools/ya.make index ec754ed566..f3d73bb51f 100644 --- a/ydb/tests/tools/ya.make +++ b/ydb/tests/tools/ya.make @@ -2,6 +2,7 @@ RECURSE( datastreams_helpers fq_runner idx_test + kqprun s3_recipe ydb_serializable ydb_serializable/replay |