diff options
author | brgayazov <bulat@ydb.tech> | 2023-02-02 16:46:09 +0300 |
---|---|---|
committer | brgayazov <bulat@ydb.tech> | 2023-02-02 16:46:09 +0300 |
commit | fed15f08e95162b9d4734c949bbd291f85d6a247 (patch) | |
tree | e5e87f40f1e10b2e8f7cf9f71852abaf7c075071 | |
parent | d9a051d043c63252e4178c8dfceab74bfffba606 (diff) | |
download | ydb-fed15f08e95162b9d4734c949bbd291f85d6a247.tar.gz |
Added parameters get from json file
82 files changed, 2151 insertions, 119 deletions
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp index df483057be5..6c11ba616af 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp @@ -29,19 +29,28 @@ void TCommandExecuteYqlScript::Config(TConfig& config) { config.Opts->AddLongOption("explain", "Explain query").Optional().StoreTrue(&Explain); config.Opts->AddLongOption("show-response-metadata", ResponseHeadersHelp).Optional().StoreTrue(&ShowHeaders); - AddParametersOption(config); + AddParametersOption(config, "script"); + + AddFormats(config, { + EOutputFormat::Pretty, + EOutputFormat::JsonUnicode, + EOutputFormat::JsonUnicodeArray, + EOutputFormat::JsonBase64, + EOutputFormat::JsonBase64Array + }); AddInputFormats(config, { EOutputFormat::JsonUnicode, EOutputFormat::JsonBase64 }); - AddFormats(config, { - EOutputFormat::Pretty, + AddStdinFormats(config, { EOutputFormat::JsonUnicode, - EOutputFormat::JsonUnicodeArray, EOutputFormat::JsonBase64, - EOutputFormat::JsonBase64Array + EOutputFormat::Raw, + }, { + EOutputFormat::NoFraming, + EOutputFormat::NewlineDelimited }); config.SetFreeArgsNum(0); @@ -73,7 +82,9 @@ void TCommandExecuteYqlScript::Parse(TConfig& config) { throw TMisuseException() << "Both mutually exclusive options \"Text of script\" (\"--script\", \"-s\") " << "and \"Path to file with script text\" (\"--file\", \"-f\") were provided."; } - ParseParameters(); + ValidateResult = MakeHolder<NScripting::TExplainYqlResult>( + ExplainQuery(config, Script, NScripting::ExplainYqlRequestMode::Validate)); + ParseParameters(config); } int TCommandExecuteYqlScript::Run(TConfig& config) { @@ -94,25 +105,30 @@ int TCommandExecuteYqlScript::Run(TConfig& config) { NScripting::TExecuteYqlRequestSettings settings; settings.CollectQueryStats(ParseQueryStatsMode(CollectStatsMode, NTable::ECollectQueryStatsMode::None)); - NScripting::TAsyncExecuteYqlResult asyncResult; - if (Parameters.size()) { - auto validateResult = ExplainQuery(config, Script, NScripting::ExplainYqlRequestMode::Validate); - asyncResult = client.ExecuteYqlScript( - Script, - BuildParams(validateResult.GetParameterTypes(), InputFormat), - FillSettings(settings) - ); + if (!Parameters.empty() || !IsStdinInteractive()) { + THolder<TParamsBuilder> paramBuilder; + while (GetNextParams(ValidateResult->GetParameterTypes(), InputFormat, StdinFormat, FramingFormat, paramBuilder)) { + auto asyncResult = client.ExecuteYqlScript( + Script, + paramBuilder->Build(), + FillSettings(settings) + ); + + auto result = asyncResult.GetValueSync(); + ThrowOnError(result); + PrintResponseHeader(result); + PrintResponse(result); + } } else { - asyncResult = client.ExecuteYqlScript( + auto asyncResult = client.ExecuteYqlScript( Script, FillSettings(settings) ); + auto result = asyncResult.GetValueSync(); + ThrowOnError(result); + PrintResponseHeader(result); + PrintResponse(result); } - auto result = asyncResult.GetValueSync(); - - ThrowOnError(result); - PrintResponseHeader(result); - PrintResponse(result); } return EXIT_SUCCESS; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp index 9b401976fd1..25d37d8986d 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp @@ -355,12 +355,7 @@ void TCommandExecuteQuery::Config(TConfig& config) { config.Opts->AddLongOption('f', "file", "Path to file with query text to execute") .RequiredArgument("PATH").StoreResult(&QueryFile); - AddParametersOption(config, "(for data & scan queries)"); - - AddInputFormats(config, { - EOutputFormat::JsonUnicode, - EOutputFormat::JsonBase64 - }); + AddParametersOption(config, "query", "(for data & scan queries)"); AddFormats(config, { EOutputFormat::Pretty, @@ -372,6 +367,20 @@ void TCommandExecuteQuery::Config(TConfig& config) { EOutputFormat::Tsv }); + AddInputFormats(config, { + EOutputFormat::JsonUnicode, + EOutputFormat::JsonBase64 + }); + + AddStdinFormats(config, { + EOutputFormat::JsonUnicode, + EOutputFormat::JsonBase64, + EOutputFormat::Raw, + }, { + EOutputFormat::NoFraming, + EOutputFormat::NewlineDelimited + }); + CheckExamples(config); config.SetFreeArgsNum(0); @@ -383,10 +392,13 @@ void TCommandExecuteQuery::Parse(TConfig& config) { if (BasicStats && CollectStatsMode) { throw TMisuseException() << "Both mutually exclusive options \"--stats\" and \"-s\" are provided."; } - if (ParameterOptions.size() && QueryType == "scheme") { + if ((!ParameterOptions.empty() || !ParameterFiles.empty() || !StdinParameters.empty() || IsStdinFormatSet || IsFramingFormatSet || + config.ParseResult->Has("batch")) && QueryType == "scheme") { throw TMisuseException() << "Scheme queries does not support parameters."; } - ParseParameters(); + ValidateResult = MakeHolder<NScripting::TExplainYqlResult>( + ExplainQuery(config, Query, NScripting::ExplainYqlRequestMode::Validate)); + ParseParameters(config); CheckQueryOptions(); } @@ -427,25 +439,52 @@ int TCommandExecuteQuery::ExecuteDataQuery(TConfig& config) { } } } + NTable::TTableClient client(CreateDriver(config)); NTable::TAsyncDataQueryResult asyncResult; - if (Parameters.size()) { - auto validateResult = ExplainQuery(config, Query, NScripting::ExplainYqlRequestMode::Validate); - asyncResult = GetSession(config).ExecuteDataQuery( - Query, - NTable::TTxControl::BeginTx(txSettings).CommitTx(), - BuildParams(validateResult.GetParameterTypes(), InputFormat), - FillSettings(settings) - ); + + if (!Parameters.empty() || !IsStdinInteractive()) { + THolder<TParamsBuilder> paramBuilder; + while (GetNextParams(ValidateResult->GetParameterTypes(), InputFormat, StdinFormat, FramingFormat, paramBuilder)) { + auto operation = [this, &txSettings, ¶mBuilder, &settings, &asyncResult](NTable::TSession session) { + auto promise = NThreading::NewPromise<NTable::TDataQueryResult>(); + asyncResult = promise.GetFuture(); + auto result = session.ExecuteDataQuery( + Query, + NTable::TTxControl::BeginTx(txSettings).CommitTx(), + paramBuilder->Build(), + FillSettings(settings) + ); + return result.Apply([promise](const NTable::TAsyncDataQueryResult& result) mutable { + promise.SetValue(result.GetValue()); + return static_cast<TStatus>(result.GetValue()); + }); + }; + auto retryResult = client.RetryOperation(std::move(operation)); + retryResult.GetValueSync(); + auto result = asyncResult.GetValueSync(); + ThrowOnError(result); + PrintDataQueryResponse(result); + } } else { - asyncResult = GetSession(config).ExecuteDataQuery( - Query, - NTable::TTxControl::BeginTx(txSettings).CommitTx(), - FillSettings(settings) - ); + auto operation = [this, &txSettings, &settings, &asyncResult](NTable::TSession session) { + auto promise = NThreading::NewPromise<NTable::TDataQueryResult>(); + asyncResult = promise.GetFuture(); + auto result = session.ExecuteDataQuery( + Query, + NTable::TTxControl::BeginTx(txSettings).CommitTx(), + FillSettings(settings) + ); + return result.Apply([promise](const NTable::TAsyncDataQueryResult& result) mutable { + promise.SetValue(result.GetValue()); + return static_cast<TStatus>(result.GetValue()); + }); + }; + auto retryResult = client.RetryOperation(std::move(operation)); + retryResult.GetValueSync(); + auto result = asyncResult.GetValueSync(); + ThrowOnError(result); + PrintDataQueryResponse(result); } - NTable::TDataQueryResult result = asyncResult.GetValueSync(); - ThrowOnError(result); - PrintDataQueryResponse(result); return EXIT_SUCCESS; } @@ -485,25 +524,56 @@ int TCommandExecuteQuery::ExecuteScanQuery(TConfig& config) { settings.CollectQueryStats(ParseQueryStatsMode(CollectStatsMode, defaultStatsMode)); NTable::TAsyncScanQueryPartIterator asyncResult; - if (Parameters.size()) { - auto validateResult = ExplainQuery(config, Query, NScripting::ExplainYqlRequestMode::Validate); - asyncResult = client.StreamExecuteScanQuery( - Query, - BuildParams(validateResult.GetParameterTypes(), InputFormat), - settings - ); + SetInterruptHandlers(); + if (!Parameters.empty() || !IsStdinInteractive()) { + THolder<TParamsBuilder> paramBuilder; + while (GetNextParams(ValidateResult->GetParameterTypes(), InputFormat, StdinFormat, FramingFormat, paramBuilder)) { + auto operation = [this, ¶mBuilder, &settings, &asyncResult](NTable::TTableClient client) { + auto promise = NThreading::NewPromise<NTable::TScanQueryPartIterator>(); + asyncResult = promise.GetFuture(); + auto result = client.StreamExecuteScanQuery( + Query, + paramBuilder->Build(), + settings + ); + return result.Apply([promise](const NTable::TAsyncScanQueryPartIterator& result) mutable { + promise.SetValue(result.GetValue()); + return static_cast<TStatus>(result.GetValue()); + }); + }; + auto retryResult = client.RetryOperation(std::move(operation)); + retryResult.GetValueSync(); + auto result = asyncResult.GetValueSync(); + ThrowOnError(result); + if (!PrintScanQueryResponse(result)) { + return EXIT_FAILURE; + } + } } else { - asyncResult = client.StreamExecuteScanQuery(Query, settings); + auto operation = [this, &settings, &asyncResult](NTable::TTableClient client) { + auto promise = NThreading::NewPromise<NTable::TScanQueryPartIterator>(); + asyncResult = promise.GetFuture(); + auto result = client.StreamExecuteScanQuery( + Query, + settings + ); + return result.Apply([promise](const NTable::TAsyncScanQueryPartIterator& result) mutable { + promise.SetValue(result.GetValue()); + return static_cast<TStatus>(result.GetValue()); + }); + }; + auto retryResult = client.RetryOperation(std::move(operation)); + retryResult.GetValueSync(); + auto result = asyncResult.GetValueSync(); + ThrowOnError(result); + if (!PrintScanQueryResponse(result)) { + return EXIT_FAILURE; + } } - - auto result = asyncResult.GetValueSync(); - ThrowOnError(result); - PrintScanQueryResponse(result); return EXIT_SUCCESS; } -void TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator& result) { - SetInterruptHandlers(); +bool TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator& result) { TMaybe<TString> stats; TMaybe<TString> fullStats; { @@ -546,7 +616,9 @@ void TCommandExecuteQuery::PrintScanQueryResponse(NTable::TScanQueryPartIterator if (IsInterrupted()) { Cerr << "<INTERRUPTED>" << Endl; + return false; } + return true; } TCommandExplain::TCommandExplain() diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h index 93ae2a6f0c2..771f4a80d25 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h @@ -106,7 +106,7 @@ public: int ExecuteSchemeQuery(TConfig& config); int ExecuteScanQuery(TConfig& config); - void PrintScanQueryResponse(NTable::TScanQueryPartIterator& result); + bool PrintScanQueryResponse(NTable::TScanQueryPartIterator& result); private: TString CollectStatsMode; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_yql.cpp b/ydb/public/lib/ydb_cli/commands/ydb_yql.cpp index 0ebb7faedc2..2a340481f29 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_yql.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_yql.cpp @@ -4,6 +4,7 @@ #include <ydb/public/lib/ydb_cli/common/pretty_table.h> #include <ydb/public/lib/ydb_cli/common/print_operation.h> #include <ydb/public/lib/ydb_cli/common/query_stats.h> +#include <util/generic/queue.h> namespace NYdb { namespace NConsoleClient { @@ -18,14 +19,9 @@ void TCommandYql::Config(TConfig& config) { config.Opts->AddLongOption("stats", "Collect statistics mode [none, basic, full]") .RequiredArgument("[String]").StoreResult(&CollectStatsMode); config.Opts->AddLongOption('s', "script", "Text of script to execute").RequiredArgument("[String]").StoreResult(&Script); - config.Opts->AddLongOption('f', "file", "[Required] Script file").RequiredArgument("PATH").StoreResult(&ScriptFile); + config.Opts->AddLongOption('f', "file", "Script file").RequiredArgument("PATH").StoreResult(&ScriptFile); - AddParametersOption(config); - - AddInputFormats(config, { - EOutputFormat::JsonUnicode, - EOutputFormat::JsonBase64 - }); + AddParametersOption(config, "script"); AddFormats(config, { EOutputFormat::Pretty, @@ -35,7 +31,21 @@ void TCommandYql::Config(TConfig& config) { EOutputFormat::JsonBase64Array, EOutputFormat::Csv, EOutputFormat::Tsv - }); + }); + + AddInputFormats(config, { + EOutputFormat::JsonUnicode, + EOutputFormat::JsonBase64 + }); + + AddStdinFormats(config, { + EOutputFormat::JsonUnicode, + EOutputFormat::JsonBase64, + EOutputFormat::Raw, + }, { + EOutputFormat::NoFraming, + EOutputFormat::NewlineDelimited + }); CheckExamples(config); @@ -56,40 +66,51 @@ void TCommandYql::Parse(TConfig& config) { if (ScriptFile) { Script = ReadFromFile(ScriptFile, "script"); } - ParseParameters(); + ValidateResult = MakeHolder<NScripting::TExplainYqlResult>( + ExplainQuery(config, Script, NScripting::ExplainYqlRequestMode::Validate)); + ParseParameters(config); } int TCommandYql::Run(TConfig& config) { TDriver driver = CreateDriver(config); NScripting::TScriptingClient client(driver); - NTable::TTableClient tableClient(driver); NScripting::TExecuteYqlRequestSettings settings; settings.CollectQueryStats(ParseQueryStatsMode(CollectStatsMode, NTable::ECollectQueryStatsMode::None)); - - NScripting::TAsyncYqlResultPartIterator asyncResult; - if (Parameters.size()) { - auto validateResult = ExplainQuery(config, Script, NScripting::ExplainYqlRequestMode::Validate); - asyncResult = client.StreamExecuteYqlScript( - Script, - BuildParams(validateResult.GetParameterTypes(), InputFormat), - FillSettings(settings) - ); + SetInterruptHandlers(); + if (!Parameters.empty() || !IsStdinInteractive()) { + THolder<TParamsBuilder> paramBuilder; + while (!IsInterrupted() && + GetNextParams(ValidateResult->GetParameterTypes(), InputFormat, StdinFormat, FramingFormat, paramBuilder)) { + + auto asyncResult = client.StreamExecuteYqlScript( + Script, + paramBuilder->Build(), + FillSettings(settings) + ); + + auto result = asyncResult.GetValueSync(); + ThrowOnError(result); + if (!PrintResponse(result)) { + return EXIT_FAILURE; + } + } } else { - asyncResult = client.StreamExecuteYqlScript( - Script, - FillSettings(settings) + auto asyncResult = client.StreamExecuteYqlScript( + Script, + FillSettings(settings) ); - } - NScripting::TYqlResultPartIterator result = asyncResult.GetValueSync(); - ThrowOnError(result); - PrintResponse(result); + auto result = asyncResult.GetValueSync(); + ThrowOnError(result); + if (!PrintResponse(result)) { + return EXIT_FAILURE; + } + } return EXIT_SUCCESS; } -void TCommandYql::PrintResponse(NScripting::TYqlResultPartIterator& result) { - SetInterruptHandlers(); +bool TCommandYql::PrintResponse(NScripting::TYqlResultPartIterator& result) { TStringStream statsStr; { ui32 currentIndex = 0; @@ -129,7 +150,9 @@ void TCommandYql::PrintResponse(NScripting::TYqlResultPartIterator& result) { if (IsInterrupted()) { Cerr << "<INTERRUPTED>" << Endl; + return false; } + return true; } } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_yql.h b/ydb/public/lib/ydb_cli/commands/ydb_yql.h index 3e238a414ba..054f48a2349 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_yql.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_yql.h @@ -21,7 +21,7 @@ public: virtual int Run(TConfig& config) override; private: - void PrintResponse(NScripting::TYqlResultPartIterator& result); + bool PrintResponse(NScripting::TYqlResultPartIterator& result); TString CollectStatsMode; TString Script; diff --git a/ydb/public/lib/ydb_cli/common/CMakeLists.darwin.txt b/ydb/public/lib/ydb_cli/common/CMakeLists.darwin.txt index 3c4fd1eb8bf..a02c35983f4 100644 --- a/ydb/public/lib/ydb_cli/common/CMakeLists.darwin.txt +++ b/ydb/public/lib/ydb_cli/common/CMakeLists.darwin.txt @@ -52,3 +52,8 @@ generate_enum_serilization(common INCLUDE_HEADERS ydb/public/lib/ydb_cli/common/formats.h ) +generate_enum_serilization(common + ${CMAKE_SOURCE_DIR}/ydb/public/lib/ydb_cli/common/parameters.h + INCLUDE_HEADERS + ydb/public/lib/ydb_cli/common/parameters.h +) diff --git a/ydb/public/lib/ydb_cli/common/CMakeLists.linux-aarch64.txt b/ydb/public/lib/ydb_cli/common/CMakeLists.linux-aarch64.txt index 82949275ba6..de4553e6944 100644 --- a/ydb/public/lib/ydb_cli/common/CMakeLists.linux-aarch64.txt +++ b/ydb/public/lib/ydb_cli/common/CMakeLists.linux-aarch64.txt @@ -53,3 +53,8 @@ generate_enum_serilization(common INCLUDE_HEADERS ydb/public/lib/ydb_cli/common/formats.h ) +generate_enum_serilization(common + ${CMAKE_SOURCE_DIR}/ydb/public/lib/ydb_cli/common/parameters.h + INCLUDE_HEADERS + ydb/public/lib/ydb_cli/common/parameters.h +) diff --git a/ydb/public/lib/ydb_cli/common/CMakeLists.linux.txt b/ydb/public/lib/ydb_cli/common/CMakeLists.linux.txt index 82949275ba6..de4553e6944 100644 --- a/ydb/public/lib/ydb_cli/common/CMakeLists.linux.txt +++ b/ydb/public/lib/ydb_cli/common/CMakeLists.linux.txt @@ -53,3 +53,8 @@ generate_enum_serilization(common INCLUDE_HEADERS ydb/public/lib/ydb_cli/common/formats.h ) +generate_enum_serilization(common + ${CMAKE_SOURCE_DIR}/ydb/public/lib/ydb_cli/common/parameters.h + INCLUDE_HEADERS + ydb/public/lib/ydb_cli/common/parameters.h +) diff --git a/ydb/public/lib/ydb_cli/common/format.cpp b/ydb/public/lib/ydb_cli/common/format.cpp index 28a30456dcd..d21494e6b4f 100644 --- a/ydb/public/lib/ydb_cli/common/format.cpp +++ b/ydb/public/lib/ydb_cli/common/format.cpp @@ -17,6 +17,15 @@ namespace { { EOutputFormat::Tsv, "Input in tsv format" }, }; + THashMap<EOutputFormat, TString> StdinFormatDescriptions = { + { EOutputFormat::JsonUnicode, "Parameter names and values in json unicode format" }, + { EOutputFormat::JsonBase64, "Parameter names and values in json unicode format, binary string parameter values are base64-encoded" }, + { EOutputFormat::NewlineDelimited, "Newline character delimits parameter sets on stdin and triggers " + "processing in accordance to \"batch\" option" }, + { EOutputFormat::Raw, "Binary value with no transformations or parsing, parameter name is set by an \"stdin-par\" option" }, + { EOutputFormat::NoFraming, "Data from stdin is taken as a single set of parameters" } + }; + THashMap<EOutputFormat, TString> FormatDescriptions = { { EOutputFormat::Pretty, "Human readable output" }, { EOutputFormat::Json, "Output in json format" }, @@ -78,10 +87,13 @@ void TCommandWithFormat::AddDeprecatedJsonOption(TClientCommand::TConfig& config .Hidden(); } -void TCommandWithFormat::AddInputFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats) { +void TCommandWithFormat::AddInputFormats(TClientCommand::TConfig& config, + const TVector<EOutputFormat>& allowedFormats, EOutputFormat defaultFormat) { TStringStream description; description << "Input format. Available options: "; NColorizer::TColors colors = NColorizer::AutoColors(Cout); + Y_VERIFY(std::find(allowedFormats.begin(), allowedFormats.end(), defaultFormat) != allowedFormats.end(), + "Couldn't find default format %s in allowed formats", (TStringBuilder() << defaultFormat).c_str()); for (const auto& format : allowedFormats) { auto findResult = InputFormatDescriptions.find(format); Y_VERIFY(findResult != InputFormatDescriptions.end(), @@ -89,15 +101,48 @@ void TCommandWithFormat::AddInputFormats(TClientCommand::TConfig& config, const description << "\n " << colors.BoldColor() << format << colors.OldColor() << "\n " << findResult->second; } + description << "\nDefault: " << colors.CyanColor() << "\"" << defaultFormat << "\"" << colors.OldColor() << "."; config.Opts->AddLongOption("input-format", description.Str()) .RequiredArgument("STRING").StoreResult(&InputFormat); AllowedInputFormats = allowedFormats; } -void TCommandWithFormat::AddFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats) { +void TCommandWithFormat::AddStdinFormats(TClientCommand::TConfig &config, const TVector<EOutputFormat>& allowedStdinFormats, + const TVector<EOutputFormat>& allowedFramingFormats) { + TStringStream description; + description << "Stdin parameters format and framing. Specify this option twice to select both.\n" + << "1. Parameters format. Available options: "; + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + for (const auto& format : allowedStdinFormats) { + auto findResult = StdinFormatDescriptions.find(format); + Y_VERIFY(findResult != StdinFormatDescriptions.end(), + "Couldn't find description for %s stdin format", (TStringBuilder() << format).c_str()); + description << "\n " << colors.BoldColor() << format << colors.OldColor() + << "\n " << findResult->second; + } + description << "\nDefault: " << colors.CyanColor() << "\"json-unicode\"" << colors.OldColor() << "."; + description << "\n2. Framing: defines how parameter sets are delimited on the stdin. Available options: "; + for (const auto& format : allowedFramingFormats) { + auto findResult = StdinFormatDescriptions.find(format); + Y_VERIFY(findResult != StdinFormatDescriptions.end(), + "Couldn't find description for %s framing format", (TStringBuilder() << format).c_str()); + description << "\n " << colors.BoldColor() << format << colors.OldColor() + << "\n " << findResult->second; + } + description << "\nDefault: " << colors.CyanColor() << "\"no-framing\"" << colors.OldColor() << "."; + config.Opts->AddLongOption("stdin-format", description.Str()) + .RequiredArgument("STRING").AppendTo(&StdinFormats); + AllowedStdinFormats = allowedStdinFormats; + AllowedFramingFormats = allowedFramingFormats; +} + +void TCommandWithFormat::AddFormats(TClientCommand::TConfig& config, + const TVector<EOutputFormat>& allowedFormats, EOutputFormat defaultFormat) { TStringStream description; description << "Output format. Available options: "; NColorizer::TColors colors = NColorizer::AutoColors(Cout); + Y_VERIFY(std::find(allowedFormats.begin(), allowedFormats.end(), defaultFormat) != allowedFormats.end(), + "Couldn't find default format %s in allowed formats", (TStringBuilder() << defaultFormat).c_str()); for (const auto& format : allowedFormats) { auto findResult = FormatDescriptions.find(format); Y_VERIFY(findResult != FormatDescriptions.end(), @@ -105,6 +150,7 @@ void TCommandWithFormat::AddFormats(TClientCommand::TConfig& config, const TVect description << "\n " << colors.BoldColor() << format << colors.OldColor() << "\n " << findResult->second; } + description << "\nDefault: " << colors.CyanColor() << "\"" << defaultFormat << "\"" << colors.OldColor() << "."; config.Opts->AddLongOption("format", description.Str()) .RequiredArgument("STRING").StoreResult(&OutputFormat); AllowedFormats = allowedFormats; @@ -133,6 +179,37 @@ void TCommandWithFormat::ParseFormats() { throw TMisuseException() << "Input format " << InputFormat << " is not available for this command"; } + if (!StdinFormats.empty()) { + for (const auto& format : AllowedInputFormats) { + Y_VERIFY(std::find(AllowedStdinFormats.begin(), AllowedStdinFormats.end(), format) != AllowedStdinFormats.end(), + "Allowed stdin formats should contain all allowed input formats"); + } + for (const auto& format : StdinFormats) { + if (format == EOutputFormat::Default) { + IsStdinFormatSet = true; + IsFramingFormatSet = true; + } else if (std::find(AllowedStdinFormats.begin(), AllowedStdinFormats.end(), format) != AllowedStdinFormats.end()) { + if (IsStdinFormatSet) { + throw TMisuseException() << "Formats " << StdinFormat << " and " << format + << " are mutually exclusive, choose only one of them."; + } + StdinFormat = format; + IsStdinFormatSet = true; + } else if (std::find(AllowedFramingFormats.begin(), AllowedFramingFormats.end(), format) != AllowedFramingFormats.end()) { + if (IsFramingFormatSet) { + throw TMisuseException() << "Formats " << StdinFormat << " and " << format + << " are mutually exclusive, choose only one of them."; + } + FramingFormat = format; + IsFramingFormatSet = true; + } else { + throw TMisuseException() << "Stdin format " << format << " is not available for this command"; + } + } + if (!IsStdinFormatSet) { + StdinFormat = InputFormat; + } + } if (OutputFormat == EOutputFormat::Default || DeprecatedOptionUsed) { return; diff --git a/ydb/public/lib/ydb_cli/common/format.h b/ydb/public/lib/ydb_cli/common/format.h index f8fb3413cf3..76f114ecd2a 100644 --- a/ydb/public/lib/ydb_cli/common/format.h +++ b/ydb/public/lib/ydb_cli/common/format.h @@ -21,8 +21,12 @@ protected: class TCommandWithFormat { protected: - void AddInputFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats); - void AddFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedFormats); + void AddInputFormats(TClientCommand::TConfig& config, + const TVector<EOutputFormat>& allowedFormats, EOutputFormat defaultFormat = EOutputFormat::JsonUnicode); + void AddStdinFormats(TClientCommand::TConfig& config, const TVector<EOutputFormat>& allowedStdinFormats, + const TVector<EOutputFormat>& allowedFramingFormats); + void AddFormats(TClientCommand::TConfig& config, + const TVector<EOutputFormat>& allowedFormats, EOutputFormat defaultFormat = EOutputFormat::Pretty); void AddMessagingFormats(TClientCommand::TConfig& config, const TVector<EMessagingFormat>& allowedFormats); void ParseFormats(); void ParseMessagingFormats(); @@ -35,13 +39,22 @@ protected: protected: EOutputFormat OutputFormat = EOutputFormat::Default; EOutputFormat InputFormat = EOutputFormat::Default; + EOutputFormat FramingFormat = EOutputFormat::Default; + EOutputFormat StdinFormat = EOutputFormat::Default; + TVector<EOutputFormat> StdinFormats; EMessagingFormat MessagingFormat = EMessagingFormat::SingleMessage; private: TVector<EOutputFormat> AllowedInputFormats; + TVector<EOutputFormat> AllowedStdinFormats; + TVector<EOutputFormat> AllowedFramingFormats; TVector<EOutputFormat> AllowedFormats; TVector<EMessagingFormat> AllowedMessagingFormats; bool DeprecatedOptionUsed = false; + +protected: + bool IsStdinFormatSet = false; + bool IsFramingFormatSet = false; }; class TResultSetPrinter { diff --git a/ydb/public/lib/ydb_cli/common/formats.h b/ydb/public/lib/ydb_cli/common/formats.h index 47565292737..25bb08421c2 100644 --- a/ydb/public/lib/ydb_cli/common/formats.h +++ b/ydb/public/lib/ydb_cli/common/formats.h @@ -17,6 +17,9 @@ enum class EOutputFormat { Csv /* "csv" */, Tsv /* "tsv" */, Parquet /* "parquet" */, + NoFraming /* "no-framing" */, + NewlineDelimited /* "newline-delimited" */, + Raw /* "raw" */ }; // EMessagingFormat to be used in both input and output when working with files/pipes in operations related to messaging diff --git a/ydb/public/lib/ydb_cli/common/parameters.cpp b/ydb/public/lib/ydb_cli/common/parameters.cpp index 3732e1837cd..a2425662a9d 100644 --- a/ydb/public/lib/ydb_cli/common/parameters.cpp +++ b/ydb/public/lib/ydb_cli/common/parameters.cpp @@ -1,11 +1,26 @@ #include "parameters.h" #include <ydb/public/lib/json_value/ydb_json_value.h> +#include <library/cpp/json/json_reader.h> +#include <library/cpp/threading/future/async.h> namespace NYdb { namespace NConsoleClient { -void TCommandWithParameters::ParseParameters() { + bool IsStdinInteractive() { +#if defined(_win32_) + errno = 0; + bool value = _isatty(_fileno(stdin)); + return value || (errno == EBADF); +#elif defined(_unix_) + errno = 0; + bool value = isatty(fileno(stdin)); + return value || (errno == EBADF); +#endif + return true; + } + +void TCommandWithParameters::ParseParameters(TClientCommand::TConfig& config) { for (const auto& parameterOption : ParameterOptions) { auto equalPos = parameterOption.find("="); if (equalPos == TString::npos) { @@ -21,12 +36,46 @@ void TCommandWithParameters::ParseParameters() { << parameterOption << "\". " << "Parameter name should start with '$' sign. Example (for linux):\n" << "--param '$input=1'"; } - + if (Parameters.find(paramName) != Parameters.end()) { + throw TMisuseException() << "Parameter $" << paramName << " value found in more than one source: \'--param\' option."; + } Parameters[paramName] = parameterOption.substr(equalPos + 1); + ParameterSources[paramName] = "\'--param\' option"; + } + + for (auto& file : ParameterFiles) { + TString data; + data = ReadFromFile(file, "param-file"); + TMap<TString, TString> params; + ParseJson(data, params, "param file " + file); + for (const auto& [name, value]: params) { + if (Parameters.find(name) != Parameters.end()) { + throw TMisuseException() << "Parameter " << name << " value found in more than one source: " + << "param file " << file << ", " << ParameterSources[name] << "."; + } + Parameters[name] = value; + ParameterSources[name] = "param file " + file; + } + } + + if (!StdinParameters.empty() && IsStdinInteractive()) { + throw TMisuseException() << "\"--stdin-par\" option is allowed only with non-interactive stdin."; + } + for (auto it = StdinParameters.begin(); it != StdinParameters.end(); ++it) { + if (std::find(StdinParameters.begin(), it, *it) != it) { + throw TMisuseException() << "Parameter $" << *it << " value found in more than one source: \'--stdin-par\' option."; + } + if (Parameters.find("$" + *it) != Parameters.end()) { + throw TMisuseException() << "Parameter $" << *it << " value found in more than one source: \'--stdin-par\' option, " + << ParameterSources["$" + *it] << "."; + } + } + if (BatchMode != EBatchMode::Adaptive && (config.ParseResult->Has("batch-limit") || config.ParseResult->Has("batch-max-delay"))) { + throw TMisuseException() << "Options \"--batch-limit\" and \"--batch-max-delay\" are allowed only in \"adaptive\" batch mode."; } } -void TCommandWithParameters::AddParametersOption(TClientCommand::TConfig& config, const TString& clarification) { +void TCommandWithParameters::AddParametersOption(TClientCommand::TConfig& config, const TString& requestString, const TString& clarification) { TStringStream descr; NColorizer::TColors colors = NColorizer::AutoColors(Cout); descr << "Query parameter[s]."; @@ -38,6 +87,27 @@ void TCommandWithParameters::AddParametersOption(TClientCommand::TConfig& config << "Escaping depends on operating system."; config.Opts->AddLongOption('p', "param", descr.Str()) .RequiredArgument("$name=value").AppendTo(&ParameterOptions); + descr.Clear(); + descr << "Batching mode for stdin parameters processing. Available options:\n " + << colors.BoldColor() << "iterative" << colors.OldColor() + << "\n Executes " << requestString << " for each parameter set (exactly one execution " + "when no framing specified in \"stdin-format\")\n " + << colors.BoldColor() << "full" << colors.OldColor() + << "\n Executes " << requestString << " once, with all parameter sets wrapped in json list, when EOF is reached on stdin\n " + << colors.BoldColor() << "adaptive" << colors.OldColor() + << "\n Executes " << requestString << " with a json list of parameter sets every time when its number reaches batch-limit, " + "or the waiting time reaches batch-max-delay." + "\nDefault: " << colors.CyanColor() << "\"iterative\"" << colors.OldColor() << "."; + config.Opts->AddLongOption("param-file", "File name with parameter names and values " + "in json format. You may specify this option repeatedly.") + .RequiredArgument("PATH").AppendTo(&ParameterFiles); + config.Opts->AddLongOption("stdin-par", "Parameter name on stdin, required/applicable when stdin-format implies values only.") + .RequiredArgument("STRING").AppendTo(&StdinParameters); + config.Opts->AddLongOption("batch", descr.Str()).RequiredArgument("STRING").StoreResult(&BatchMode); + config.Opts->AddLongOption("batch-limit", "Maximum size of list for adaptive batching mode").RequiredArgument("INT") + .StoreResult(&BatchLimit).DefaultValue(1000); + config.Opts->AddLongOption("batch-max-delay", "Maximum delay to process first item in the list for adaptive batching mode") + .RequiredArgument("VAL").StoreResult(&BatchMaxDelay).DefaultValue(TDuration::Seconds(1)); AddOptionExamples( "param", @@ -62,7 +132,7 @@ void TCommandWithParameters::AddParametersOption(TClientCommand::TConfig& config ); } -TParams TCommandWithParameters::BuildParams(const std::map<TString, TType>& paramTypes, EOutputFormat inputFormat) { +void TCommandWithParameters::AddParams(const std::map<TString, TType>& paramTypes, EOutputFormat inputFormat, TParamsBuilder& paramBuilder) { EBinaryStringEncoding encoding; switch (inputFormat) { case EOutputFormat::Default: @@ -76,16 +146,248 @@ TParams TCommandWithParameters::BuildParams(const std::map<TString, TType>& para throw TMisuseException() << "Unknown input format: " << inputFormat; } - TParamsBuilder paramBuilder; for (const auto&[name, value] : Parameters) { auto paramIt = paramTypes.find(name); if (paramIt == paramTypes.end()) { - throw TMisuseException() << "Query does not contain parameter \"" << name << "\"."; + if (ParameterSources[name] == "\'--param\' option") { + throw TMisuseException() << "Query does not contain parameter \"" << name << "\"."; + } else { + continue; + } } const TType& type = (*paramIt).second; paramBuilder.AddParam(name, JsonToYdbValue(value, type, encoding)); } - return paramBuilder.Build(); +} + +bool TCommandWithParameters::GetNextParams(const std::map<TString, TType>& paramTypes, EOutputFormat inputFormat, EOutputFormat encodingFormat, + EOutputFormat framingFormat, THolder<TParamsBuilder>& paramBuilder) { + paramBuilder = MakeHolder<TParamsBuilder>(); + AddParams(paramTypes, inputFormat, *paramBuilder); + if (IsStdinInteractive()) { + static bool firstEncounter = true; + if (!firstEncounter) { + return false; + } + firstEncounter = false; + return true; + } + EBinaryStringEncoding encoding; + switch (encodingFormat) { + case EOutputFormat::Default: + case EOutputFormat::Raw: + case EOutputFormat::JsonUnicode: + encoding = EBinaryStringEncoding::Unicode; + break; + case EOutputFormat::JsonBase64: + encoding = EBinaryStringEncoding::Base64; + break; + default: + throw TMisuseException() << "Unknown encoding format: " << encodingFormat; + } + if (BatchMode == EBatchMode::Iterative) { + TString data; + if (StdinParameters.empty()) { + if (encodingFormat == EOutputFormat::Raw) { + throw TMisuseException() << "For Raw format \"--stdin-par\" option should be used."; + } + auto data = ReadData(framingFormat); + if (!data.Defined()) { + return false; + } + if (!data->empty()) { + TMap<TString, TString> result; + ParseJson(*data, result, "stdin"); + ApplyParams(result, paramTypes, encoding, *paramBuilder, "stdin"); + } + } else { + for (const auto &name: StdinParameters) { + auto data = ReadData(framingFormat); + if (!data.Defined()) { + return false; + } + TString fullname = "$" + name; + auto paramIt = paramTypes.find(fullname); + if (paramIt == paramTypes.end()) { + throw TMisuseException() << "Query does not contain parameter \"" << fullname << "\"."; + } + + const TType &type = (*paramIt).second; + if (encodingFormat == EOutputFormat::Raw) { + TTypeParser parser(type); + if (parser.GetKind() != TTypeParser::ETypeKind::Primitive) { + throw TMisuseException() << "Wrong type of parameter \"" << fullname << "\"."; + } + + if (parser.GetPrimitive() == EPrimitiveType::String) { + paramBuilder->AddParam(fullname, TValueBuilder().String(*data).Build()); + } else if (parser.GetPrimitive() == EPrimitiveType::Utf8) { + paramBuilder->AddParam(fullname, TValueBuilder().Utf8(*data).Build()); + } else { + throw TMisuseException() << "Wrong type of parameter \"" << fullname << "\"."; + } + } else { + paramBuilder->AddParam(fullname, JsonToYdbValue(*data, type, encoding)); + } + } + } + } else if (BatchMode == EBatchMode::Full || BatchMode == EBatchMode::Adaptive) { + static bool isEndReached = false; + static auto pool = CreateThreadPool(2); + if (isEndReached) { + return false; + } + + if (StdinParameters.size() != 1) { + throw TMisuseException() << "Only one stdin parameter allowed in \"full\" batch mode."; + } + TString name = StdinParameters.front(); + TString fullname = "$" + name; + auto paramIt = paramTypes.find(fullname); + if (paramIt == paramTypes.end()) { + throw TMisuseException() << "Query does not contain parameter \"" << fullname << "\"."; + } + const TType &type = (*paramIt).second; + TTypeParser parser(type); + + if (parser.GetKind() != TTypeParser::ETypeKind::List) { + throw TMisuseException() << "Wrong type of parameter \"" << fullname << "\"."; + } + + TMaybe<TString> data; + size_t listSize = 0; + TInstant endTime; + auto ReadDataLambda = [framingFormat]() { + return ReadData(framingFormat); + }; + if (encodingFormat == EOutputFormat::Raw) { + TValueBuilder valueBuilder; + valueBuilder.BeginList(); + parser.OpenList(); + while (true) { + static NThreading::TFuture<TMaybe<TString>> futureData; + if (!futureData.Initialized() || listSize) { + futureData = NThreading::Async(ReadDataLambda, *pool); + } + if (BatchMode == EBatchMode::Adaptive && listSize && + ((BatchMaxDelay != TDuration::Zero() && !futureData.Wait(endTime)) || listSize == BatchLimit)) { + break; + } + data = futureData.GetValueSync(); + if (!data.Defined()) { + isEndReached = true; + break; + } + if (!listSize) { + endTime = Now() + BatchMaxDelay; + } + + if (parser.GetKind() != TTypeParser::ETypeKind::Primitive) { + throw TMisuseException() << "Wrong type of list \"" << fullname << "\" elements."; + } + if (parser.GetPrimitive() == EPrimitiveType::String) { + valueBuilder.AddListItem(TValueBuilder().String(*data).Build()); + } else if (parser.GetPrimitive() == EPrimitiveType::Utf8) { + valueBuilder.AddListItem(TValueBuilder().Utf8(*data).Build()); + } else { + throw TMisuseException() << "Wrong type of list \"" << fullname << "\" elements."; + } + ++listSize; + } + if (!listSize) { + return false; + } + valueBuilder.EndList(); + paramBuilder->AddParam(fullname, valueBuilder.Build()); + } else { + TString array = "["; + while (true) { + static NThreading::TFuture<TMaybe<TString>> futureData; + if (!futureData.Initialized() || listSize) { + futureData = NThreading::Async(ReadDataLambda, *pool); + } + if (BatchMode == EBatchMode::Adaptive && listSize && + ((BatchMaxDelay != TDuration::Zero() && !futureData.Wait(endTime)) || listSize == BatchLimit)) { + break; + } + data = futureData.GetValueSync(); + if (!data.Defined()) { + isEndReached = true; + break; + } + if (!listSize) { + endTime = Now() + BatchMaxDelay; + } else { + array += ","; + } + array += *data; + ++listSize; + } + if (!listSize) { + return false; + } + array += "]"; + paramBuilder->AddParam(fullname, JsonToYdbValue(array, type, encoding)); + } + + } else { + throw TMisuseException() << "Unknown batch format."; + } + + return true; +} + +void TCommandWithParameters::ParseJson(const TString& str, TMap<TString, TString>& result, const TString& source) { + NJson::TJsonValue jsonValue; + if (!NJson::ReadJsonTree(str, &jsonValue)) { + throw TMisuseException() << "Can't parse \"" << str << "\" as json."; + } + if (!jsonValue.IsMap()) { + throw TMisuseException() << "Json value \"" << str << "\" is not a map."; + } + for (const auto&[name, value] : jsonValue.GetMap()) { + TString fullname = "$" + name; + if (result.find(fullname) != result.end()) { + throw TMisuseException() << "Parameter " << fullname << " value found in more than one source: " << source << "."; + } + result[fullname] = ToString(value); + } +} + +void TCommandWithParameters::ApplyParams(const TMap<TString, TString> ¶ms, const std::map<TString, TType>& paramTypes, + EBinaryStringEncoding encoding, TParamsBuilder ¶mBuilder, const TString& source) { + for (const auto &[name, value]: params) { + auto paramIt = paramTypes.find(name); + if (paramIt == paramTypes.end()) { + continue; + } + if (Parameters.find(name) != Parameters.end()) { + throw TMisuseException() << "Parameter " << name << " value found in more than one source: " + << source << ", " << ParameterSources[name] << "."; + } + const TType &type = (*paramIt).second; + paramBuilder.AddParam(name, JsonToYdbValue(value, type, encoding)); + } +} + +TMaybe<TString> TCommandWithParameters::ReadData(EOutputFormat framingFormat) { + TString result; + if (framingFormat == EOutputFormat::Default || framingFormat == EOutputFormat::NoFraming) { + static bool isFirstEncounter = true; + if (!isFirstEncounter) { + return Nothing(); + } + result = Cin.ReadAll(); + isFirstEncounter = false; + } else if (framingFormat == EOutputFormat::NewlineDelimited) { + if (!Cin.ReadLine(result)) { + return Nothing(); + } + } else { + throw TMisuseException() << "Unknown framing format: " << framingFormat; + } + return result; + } } diff --git a/ydb/public/lib/ydb_cli/common/parameters.h b/ydb/public/lib/ydb_cli/common/parameters.h index 2a33734e16b..a9065f09810 100644 --- a/ydb/public/lib/ydb_cli/common/parameters.h +++ b/ydb/public/lib/ydb_cli/common/parameters.h @@ -4,20 +4,50 @@ #include "command.h" #include "formats.h" +#include <ydb/public/lib/ydb_cli/commands/ydb_command.h> #include <ydb/public/sdk/cpp/client/ydb_params/params.h> +#include <ydb/public/lib/json_value/ydb_json_value.h> + +#if defined(_win32_) +#include <io.h> +#elif defined(_unix_) +#include <unistd.h> +#endif namespace NYdb { namespace NConsoleClient { +enum class EBatchMode { + Iterative /* "iterative" */, + Full /* "full" */, + Adaptive /* "adaptive" */ +}; + +bool IsStdinInteractive(); + class TCommandWithParameters : public TCommandWithExamples { protected: - void ParseParameters(); - void AddParametersOption(TClientCommand::TConfig& config, const TString& clarification = ""); - TParams BuildParams(const std::map<TString, TType>& paramTypes, EOutputFormat inputFormat); + void ParseParameters(TClientCommand::TConfig& config); + void AddParametersOption(TClientCommand::TConfig& config, const TString& requestString, const TString& clarification = ""); + bool GetNextParams(const std::map<TString, TType>& paramTypes, EOutputFormat inputFormat, EOutputFormat encodingFormat, + EOutputFormat framingFormat, THolder<TParamsBuilder>& paramBuilder); + +private: + void AddParams(const std::map<TString, TType>& paramTypes, EOutputFormat inputFormat, TParamsBuilder& paramBuilder); + static void ParseJson(const TString& str, TMap<TString, TString>& result, const TString& source); + void ApplyParams(const TMap<TString, TString>& params, const std::map<TString, TType>& paramTypes, + EBinaryStringEncoding encoding, TParamsBuilder& paramBuilder, const TString& source); + + static TMaybe<TString> ReadData(EOutputFormat framingFormat); protected: - TVector<TString> ParameterOptions; + TVector<TString> ParameterOptions, ParameterFiles, StdinParameters; TMap<TString, TString> Parameters; + TMap<TString, TString> ParameterSources; + EBatchMode BatchMode = EBatchMode::Iterative; + size_t BatchLimit; + TDuration BatchMaxDelay; + THolder<NScripting::TExplainYqlResult> ValidateResult; }; } diff --git a/ydb/tests/functional/ydb_cli/canondata/result.json b/ydb/tests/functional/ydb_cli/canondata/result.json index 57776dab145..0833d2a4f1e 100644 --- a/ydb/tests/functional/ydb_cli/canondata/result.json +++ b/ydb/tests/functional/ydb_cli/canondata/result.json @@ -50,6 +50,102 @@ "test_ydb_scripting.TestExecuteScriptWithParams.test_uint64_and_string": { "uri": "file://test_ydb_scripting.TestExecuteScriptWithParams.test_uint64_and_string/result.output" }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_ignore_excess_parameters": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_ignore_excess_parameters/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_list": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_list/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_multiple_files": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_multiple_files/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_ignore_excess_parameters": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_ignore_excess_parameters/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_list": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_list/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_multiple_files": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_multiple_files/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_struct": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_struct/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_uint32": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_uint32/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_uint64_and_string": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_uint64_and_string/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_struct": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_struct/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_uint32": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_uint32/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_uint64_and_string": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_uint64_and_string/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_batching_adaptive": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_batching_adaptive/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_batching_full": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_batching_full/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_different_sources": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_different_sources/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_framing_newline_delimited": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_framing_newline_delimited/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_ignore_excess_parameters": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_ignore_excess_parameters/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_mix_json_and_binary": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_mix_json_and_binary/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_simple_json": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_simple_json/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_batching_adaptive": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_batching_adaptive/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_batching_full": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_batching_full/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_different_sources": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_different_sources/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_framing_newline_delimited": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_framing_newline_delimited/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_ignore_excess_parameters": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_ignore_excess_parameters/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_mix_json_and_binary": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_mix_json_and_binary/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_simple_json": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_simple_json/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_text_data": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_text_data/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_text_data_framing": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_text_data_framing/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_unnamed_json": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_unnamed_json/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_text_data": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_text_data/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_text_data_framing": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_text_data_framing/result.output" + }, + "test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_unnamed_json": { + "uri": "file://test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_unnamed_json/result.output" + }, "test_ydb_scripting.TestScriptingServiceHelp.test_help": { "uri": "file://test_ydb_scripting.TestScriptingServiceHelp.test_help/result.output" }, @@ -133,5 +229,101 @@ }, "test_ydb_table.TestExecuteQueryWithParams.test_uint64_and_string": { "uri": "file://test_ydb_table.TestExecuteQueryWithParams.test_uint64_and_string/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_ignore_excess_parameters": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_ignore_excess_parameters/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_list": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_list/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_multiple_files": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_multiple_files/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_struct": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_struct/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_uint32": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_uint32/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_uint64_and_string": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_uint64_and_string/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_ignore_excess_parameters": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_ignore_excess_parameters/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_list": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_list/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_multiple_files": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_multiple_files/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_struct": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_struct/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_uint32": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_uint32/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_uint64_and_string": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_uint64_and_string/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_ignore_excess_parameters": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_ignore_excess_parameters/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_batching_adaptive": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_batching_adaptive/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_batching_full": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_batching_full/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_different_sources": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_different_sources/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_framing_newline_delimited": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_framing_newline_delimited/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_mix_json_and_binary": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_mix_json_and_binary/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_simple_json": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_simple_json/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_text_data": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_text_data/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_text_data_framing": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_text_data_framing/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_unnamed_json": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_unnamed_json/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_ignore_excess_parameters": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_ignore_excess_parameters/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_batching_adaptive": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_batching_adaptive/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_batching_full": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_batching_full/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_different_sources": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_different_sources/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_framing_newline_delimited": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_framing_newline_delimited/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_mix_json_and_binary": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_mix_json_and_binary/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_simple_json": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_simple_json/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_text_data": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_text_data/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_text_data_framing": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_text_data_framing/result.output" + }, + "test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_unnamed_json": { + "uri": "file://test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_unnamed_json/result.output" } } diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_ignore_excess_parameters/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_ignore_excess_parameters/result.output new file mode 100644 index 00000000000..6daf3981029 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_ignore_excess_parameters/result.output @@ -0,0 +1,5 @@ +ββββββ +| a | +ββββββ€ +| 12 | +ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_list/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_list/result.output new file mode 100644 index 00000000000..09df5d83ab9 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_list/result.output @@ -0,0 +1,5 @@ +βββββββββββ +| values | +βββββββββββ€ +| [1,2,3] | +βββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_multiple_files/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_multiple_files/result.output new file mode 100644 index 00000000000..6319ad9c535 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_multiple_files/result.output @@ -0,0 +1,5 @@ +ββββββββββββββββββββ¬βββββββ¬βββββββββββββββ +| str | num | date | +ββββββββββββββββββββΌβββββββΌβββββββββββββββ€ +| "Π‘ΡΡΠΎΡΠΊΠ°" | 1542 | "2011-11-11" | +ββββββββββββββββββββ΄βββββββ΄βββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_ignore_excess_parameters/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_ignore_excess_parameters/result.output new file mode 100644 index 00000000000..6daf3981029 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_ignore_excess_parameters/result.output @@ -0,0 +1,5 @@ +ββββββ +| a | +ββββββ€ +| 12 | +ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_list/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_list/result.output new file mode 100644 index 00000000000..09df5d83ab9 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_list/result.output @@ -0,0 +1,5 @@ +βββββββββββ +| values | +βββββββββββ€ +| [1,2,3] | +βββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_multiple_files/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_multiple_files/result.output new file mode 100644 index 00000000000..6319ad9c535 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_multiple_files/result.output @@ -0,0 +1,5 @@ +ββββββββββββββββββββ¬βββββββ¬βββββββββββββββ +| str | num | date | +ββββββββββββββββββββΌβββββββΌβββββββββββββββ€ +| "Π‘ΡΡΠΎΡΠΊΠ°" | 1542 | "2011-11-11" | +ββββββββββββββββββββ΄βββββββ΄βββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_struct/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_struct/result.output new file mode 100644 index 00000000000..0a3e804bb40 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_struct/result.output @@ -0,0 +1,7 @@ +βββββββ¬ββββββββ +| key | value | +βββββββΌββββββββ€ +| 1 | "one" | +βββββββΌββββββββ€ +| 2 | "two" | +βββββββ΄ββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_uint32/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_uint32/result.output new file mode 100644 index 00000000000..ad14eb5d5f9 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_uint32/result.output @@ -0,0 +1,5 @@ +ββββββββ¬ββββββ¬ββββββββ +| id | key | value | +ββββββββΌββββββΌββββββββ€ +| 1111 | 1 | "one" | +ββββββββ΄ββββββ΄ββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_uint64_and_string/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_uint64_and_string/result.output new file mode 100644 index 00000000000..d3c530a7966 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_stream_uint64_and_string/result.output @@ -0,0 +1,7 @@ +ββββββββ¬ββββββ¬ββββββββββ +| id | key | value | +ββββββββΌββββββΌββββββββββ€ +| 2222 | 2 | "two" | +ββββββββΌββββββΌββββββββββ€ +| 7777 | 7 | "seven" | +ββββββββ΄ββββββ΄ββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_struct/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_struct/result.output new file mode 100644 index 00000000000..0a3e804bb40 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_struct/result.output @@ -0,0 +1,7 @@ +βββββββ¬ββββββββ +| key | value | +βββββββΌββββββββ€ +| 1 | "one" | +βββββββΌββββββββ€ +| 2 | "two" | +βββββββ΄ββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_uint32/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_uint32/result.output new file mode 100644 index 00000000000..ad14eb5d5f9 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_uint32/result.output @@ -0,0 +1,5 @@ +ββββββββ¬ββββββ¬ββββββββ +| id | key | value | +ββββββββΌββββββΌββββββββ€ +| 1111 | 1 | "one" | +ββββββββ΄ββββββ΄ββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_uint64_and_string/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_uint64_and_string/result.output new file mode 100644 index 00000000000..d3c530a7966 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromJson.test_uint64_and_string/result.output @@ -0,0 +1,7 @@ +ββββββββ¬ββββββ¬ββββββββββ +| id | key | value | +ββββββββΌββββββΌββββββββββ€ +| 2222 | 2 | "two" | +ββββββββΌββββββΌββββββββββ€ +| 7777 | 7 | "seven" | +ββββββββ΄ββββββ΄ββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_batching_adaptive/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_batching_adaptive/result.output new file mode 100644 index 00000000000..d01126cebc9 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_batching_adaptive/result.output @@ -0,0 +1,15 @@ +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +| arr | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€ +| [{"id":1,"s":"Line1"},{"id":2,"s":"Line2"},{"id":3,"s":"Line3"}] | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +| arr | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€ +| [{"id":4,"s":"Line4"},{"id":5,"s":"Line5"},{"id":6,"s":"Line6"}] | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +| arr | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€ +| [{"id":7,"s":"Line7"},{"id":8,"s":"Line8"},{"id":9,"s":"Line9"}] | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_batching_full/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_batching_full/result.output new file mode 100644 index 00000000000..49e822c46be --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_batching_full/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββββββββββββββ +| s | +βββββββββββββββββββββββββββββ€ +| ["Line1","Line2","Line3"] | +βββββββββββββββββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_different_sources/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_different_sources/result.output new file mode 100644 index 00000000000..81382040d2f --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_different_sources/result.output @@ -0,0 +1,5 @@ +ββββββββββββββββββββββββ¬βββββββββββββββ¬ββββββ +| s | date | val | +ββββββββββββββββββββββββΌβββββββββββββββΌββββββ€ +| "Π‘ΡΡΠΎΠΊΠ° utf-8" | "2000-09-01" | 100 | +ββββββββββββββββββββββββ΄βββββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_framing_newline_delimited/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_framing_newline_delimited/result.output new file mode 100644 index 00000000000..7992832afa8 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_framing_newline_delimited/result.output @@ -0,0 +1,15 @@ +βββββββββββββββ¬ββββββ +| s | num | +βββββββββββββββΌββββββ€ +| "Some text" | 1 | +βββββββββββββββ΄ββββββ +βββββββββββββββββββββββββββββββββββ¬ββββββ +| s | num | +βββββββββββββββββββββββββββββββββββΌββββββ€ +| "Π‘ΡΡΠΎΠΊΠ° 1\nΠ‘ΡΡΠΎΠΊΠ°2" | 2 | +βββββββββββββββββββββββββββββββββββ΄ββββββ +βββββββββββββ¬ββββββ +| s | num | +βββββββββββββΌββββββ€ +| "Abacaba" | 3 | +βββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_ignore_excess_parameters/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_ignore_excess_parameters/result.output new file mode 100644 index 00000000000..6daf3981029 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_ignore_excess_parameters/result.output @@ -0,0 +1,5 @@ +ββββββ +| a | +ββββββ€ +| 12 | +ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_mix_json_and_binary/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_mix_json_and_binary/result.output new file mode 100644 index 00000000000..7cfc6365a45 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_mix_json_and_binary/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ¬βββββββββββββββ¬ββββββ +| s | date | val | +βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββΌβββββββββββββββΌββββββ€ +| "\u00D0\u00A1\u00D1\u0082\u00D1\u0080\u00D0\u00BE\u00D0\u00BA\u00D0\u00B0 \u00D0\u00BD\u00D0\u00BE\u00D0\u00BC\u00D0\u00B5\u00D1\u0080 1\nString number 2\n\u00D0\u00A1\u00D1\u0082\u00D1\u0080\u00D0\u00BE\u00D0\u00BA\u00D0\u00B0 \u00D0\u00BD\u00D0\u00BE\u00D0\u00BC\u00D0\u00B5\u00D1\u0080 3" | "2044-08-21" | 32 | +βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ΄βββββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_simple_json/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_simple_json/result.output new file mode 100644 index 00000000000..b1acd7e95ab --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_simple_json/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββ¬ββββββ +| s | val | +βββββββββββββββββΌββββββ€ +| "Some_string" | 32 | +βββββββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_batching_adaptive/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_batching_adaptive/result.output new file mode 100644 index 00000000000..d01126cebc9 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_batching_adaptive/result.output @@ -0,0 +1,15 @@ +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +| arr | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€ +| [{"id":1,"s":"Line1"},{"id":2,"s":"Line2"},{"id":3,"s":"Line3"}] | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +| arr | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€ +| [{"id":4,"s":"Line4"},{"id":5,"s":"Line5"},{"id":6,"s":"Line6"}] | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +| arr | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€ +| [{"id":7,"s":"Line7"},{"id":8,"s":"Line8"},{"id":9,"s":"Line9"}] | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_batching_full/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_batching_full/result.output new file mode 100644 index 00000000000..49e822c46be --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_batching_full/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββββββββββββββ +| s | +βββββββββββββββββββββββββββββ€ +| ["Line1","Line2","Line3"] | +βββββββββββββββββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_different_sources/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_different_sources/result.output new file mode 100644 index 00000000000..81382040d2f --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_different_sources/result.output @@ -0,0 +1,5 @@ +ββββββββββββββββββββββββ¬βββββββββββββββ¬ββββββ +| s | date | val | +ββββββββββββββββββββββββΌβββββββββββββββΌββββββ€ +| "Π‘ΡΡΠΎΠΊΠ° utf-8" | "2000-09-01" | 100 | +ββββββββββββββββββββββββ΄βββββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_framing_newline_delimited/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_framing_newline_delimited/result.output new file mode 100644 index 00000000000..7992832afa8 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_framing_newline_delimited/result.output @@ -0,0 +1,15 @@ +βββββββββββββββ¬ββββββ +| s | num | +βββββββββββββββΌββββββ€ +| "Some text" | 1 | +βββββββββββββββ΄ββββββ +βββββββββββββββββββββββββββββββββββ¬ββββββ +| s | num | +βββββββββββββββββββββββββββββββββββΌββββββ€ +| "Π‘ΡΡΠΎΠΊΠ° 1\nΠ‘ΡΡΠΎΠΊΠ°2" | 2 | +βββββββββββββββββββββββββββββββββββ΄ββββββ +βββββββββββββ¬ββββββ +| s | num | +βββββββββββββΌββββββ€ +| "Abacaba" | 3 | +βββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_ignore_excess_parameters/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_ignore_excess_parameters/result.output new file mode 100644 index 00000000000..6daf3981029 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_ignore_excess_parameters/result.output @@ -0,0 +1,5 @@ +ββββββ +| a | +ββββββ€ +| 12 | +ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_mix_json_and_binary/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_mix_json_and_binary/result.output new file mode 100644 index 00000000000..7cfc6365a45 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_mix_json_and_binary/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ¬βββββββββββββββ¬ββββββ +| s | date | val | +βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββΌβββββββββββββββΌββββββ€ +| "\u00D0\u00A1\u00D1\u0082\u00D1\u0080\u00D0\u00BE\u00D0\u00BA\u00D0\u00B0 \u00D0\u00BD\u00D0\u00BE\u00D0\u00BC\u00D0\u00B5\u00D1\u0080 1\nString number 2\n\u00D0\u00A1\u00D1\u0082\u00D1\u0080\u00D0\u00BE\u00D0\u00BA\u00D0\u00B0 \u00D0\u00BD\u00D0\u00BE\u00D0\u00BC\u00D0\u00B5\u00D1\u0080 3" | "2044-08-21" | 32 | +βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ΄βββββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_simple_json/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_simple_json/result.output new file mode 100644 index 00000000000..b1acd7e95ab --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_simple_json/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββ¬ββββββ +| s | val | +βββββββββββββββββΌββββββ€ +| "Some_string" | 32 | +βββββββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_text_data/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_text_data/result.output new file mode 100644 index 00000000000..34c5637f0b1 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_text_data/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββββββββββββ +| s | +βββββββββββββββββββββββββββ€ +| "Line1\nLine2\nLine3\n" | +βββββββββββββββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_text_data_framing/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_text_data_framing/result.output new file mode 100644 index 00000000000..04624405d77 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_text_data_framing/result.output @@ -0,0 +1,15 @@ +βββββββββββ +| s | +βββββββββββ€ +| "Line1" | +βββββββββββ +βββββββββββ +| s | +βββββββββββ€ +| "Line2" | +βββββββββββ +βββββββββββ +| s | +βββββββββββ€ +| "Line3" | +βββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_unnamed_json/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_unnamed_json/result.output new file mode 100644 index 00000000000..3c7aab5f01a --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_stream_unnamed_json/result.output @@ -0,0 +1,5 @@ +βββββββββββββ +| arr | +βββββββββββββ€ +| [1,2,3,4] | +βββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_text_data/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_text_data/result.output new file mode 100644 index 00000000000..34c5637f0b1 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_text_data/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββββββββββββ +| s | +βββββββββββββββββββββββββββ€ +| "Line1\nLine2\nLine3\n" | +βββββββββββββββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_text_data_framing/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_text_data_framing/result.output new file mode 100644 index 00000000000..04624405d77 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_text_data_framing/result.output @@ -0,0 +1,15 @@ +βββββββββββ +| s | +βββββββββββ€ +| "Line1" | +βββββββββββ +βββββββββββ +| s | +βββββββββββ€ +| "Line2" | +βββββββββββ +βββββββββββ +| s | +βββββββββββ€ +| "Line3" | +βββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_unnamed_json/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_unnamed_json/result.output new file mode 100644 index 00000000000..3c7aab5f01a --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestExecuteScriptWithParamsFromStdin.test_unnamed_json/result.output @@ -0,0 +1,5 @@ +βββββββββββββ +| arr | +βββββββββββββ€ +| [1,2,3,4] | +βββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestScriptingServiceHelp.test_help/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestScriptingServiceHelp.test_help/result.output index cf897ae670c..71010f4e5f5 100644 --- a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestScriptingServiceHelp.test_help/result.output +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestScriptingServiceHelp.test_help/result.output @@ -19,11 +19,18 @@ Options: Several parameter options can be specified. To change input format use --input-format option. Escaping depends on operating system. - --input-format STRING Input format. Available options: - json-unicode - Input in json format, binary strings are decoded with unicode characters - json-base64 - Input in json format, binary strings are decoded with base64 + --param-file PATH File name with parameter names and values in json format. You may specify this option repeatedly. + --stdin-par STRING Parameter name on stdin, required/applicable when stdin-format implies values only. + --batch STRING Batching mode for stdin parameters processing. Available options: + iterative + Executes script for each parameter set (exactly one execution when no framing specified in "stdin-format") + full + Executes script once, with all parameter sets wrapped in json list, when EOF is reached on stdin + adaptive + Executes script with a json list of parameter sets every time when its number reaches batch-limit, or the waiting time reaches batch-max-delay. + Default: "iterative". + --batch-limit INT Maximum size of list for adaptive batching mode (default: 1000) + --batch-max-delay VAL Maximum delay to process first item in the list for adaptive batching mode (default: "1.000000s") --format STRING Output format. Available options: pretty Human readable output @@ -35,3 +42,25 @@ Options: Output in json format, binary strings are encoded with base64. Every row is a separate json on a separate line. json-base64-array Output in json format, binary strings are encoded with base64. Every resultset is a json array of rows. Every row is a separate json on a separate line. + Default: "pretty". + --input-format STRING Input format. Available options: + json-unicode + Input in json format, binary strings are decoded with unicode characters + json-base64 + Input in json format, binary strings are decoded with base64 + Default: "json-unicode". + --stdin-format STRING Stdin parameters format and framing. Specify this option twice to select both. + 1. Parameters format. Available options: + json-unicode + Parameter names and values in json unicode format + json-base64 + Parameter names and values in json unicode format, binary string parameter values are base64-encoded + raw + Binary value with no transformations or parsing, parameter name is set by an "stdin-par" option + Default: "json-unicode". + 2. Framing: defines how parameter sets are delimited on the stdin. Available options: + no-framing + Data from stdin is taken as a single set of parameters + newline-delimited + Newline character delimits parameter sets on stdin and triggers processing in accordance to "batch" option + Default: "no-framing". diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestScriptingServiceHelp.test_help_ex/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestScriptingServiceHelp.test_help_ex/result.output index 1ef91a3a24e..8a0e9e70c7d 100644 --- a/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestScriptingServiceHelp.test_help_ex/result.output +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_scripting.TestScriptingServiceHelp.test_help_ex/result.output @@ -31,11 +31,18 @@ Options: What cli expects: $values=[{"key":1,"value":"one"},{"key":2,"value":"two"}] How to pass in linux: --param '$values=[{"key":1,"value":"one"},{"key":2,"value":"two"}]' - --input-format STRING Input format. Available options: - json-unicode - Input in json format, binary strings are decoded with unicode characters - json-base64 - Input in json format, binary strings are decoded with base64 + --param-file PATH File name with parameter names and values in json format. You may specify this option repeatedly. + --stdin-par STRING Parameter name on stdin, required/applicable when stdin-format implies values only. + --batch STRING Batching mode for stdin parameters processing. Available options: + iterative + Executes script for each parameter set (exactly one execution when no framing specified in "stdin-format") + full + Executes script once, with all parameter sets wrapped in json list, when EOF is reached on stdin + adaptive + Executes script with a json list of parameter sets every time when its number reaches batch-limit, or the waiting time reaches batch-max-delay. + Default: "iterative". + --batch-limit INT Maximum size of list for adaptive batching mode (default: 1000) + --batch-max-delay VAL Maximum delay to process first item in the list for adaptive batching mode (default: "1.000000s") --format STRING Output format. Available options: pretty Human readable output @@ -47,6 +54,28 @@ Options: Output in json format, binary strings are encoded with base64. Every row is a separate json on a separate line. json-base64-array Output in json format, binary strings are encoded with base64. Every resultset is a json array of rows. Every row is a separate json on a separate line. + Default: "pretty". + --input-format STRING Input format. Available options: + json-unicode + Input in json format, binary strings are decoded with unicode characters + json-base64 + Input in json format, binary strings are decoded with base64 + Default: "json-unicode". + --stdin-format STRING Stdin parameters format and framing. Specify this option twice to select both. + 1. Parameters format. Available options: + json-unicode + Parameter names and values in json unicode format + json-base64 + Parameter names and values in json unicode format, binary string parameter values are base64-encoded + raw + Binary value with no transformations or parsing, parameter name is set by an "stdin-par" option + Default: "json-unicode". + 2. Framing: defines how parameter sets are delimited on the stdin. Available options: + no-framing + Data from stdin is taken as a single set of parameters + newline-delimited + Newline character delimits parameter sets on stdin and triggers processing in accordance to "batch" option + Default: "no-framing". Examples: 1) Execute script text diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_ignore_excess_parameters/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_ignore_excess_parameters/result.output new file mode 100644 index 00000000000..6daf3981029 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_ignore_excess_parameters/result.output @@ -0,0 +1,5 @@ +ββββββ +| a | +ββββββ€ +| 12 | +ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_list/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_list/result.output new file mode 100644 index 00000000000..09df5d83ab9 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_list/result.output @@ -0,0 +1,5 @@ +βββββββββββ +| values | +βββββββββββ€ +| [1,2,3] | +βββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_multiple_files/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_multiple_files/result.output new file mode 100644 index 00000000000..6319ad9c535 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_multiple_files/result.output @@ -0,0 +1,5 @@ +ββββββββββββββββββββ¬βββββββ¬βββββββββββββββ +| str | num | date | +ββββββββββββββββββββΌβββββββΌβββββββββββββββ€ +| "Π‘ΡΡΠΎΡΠΊΠ°" | 1542 | "2011-11-11" | +ββββββββββββββββββββ΄βββββββ΄βββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_struct/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_struct/result.output new file mode 100644 index 00000000000..0a3e804bb40 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_struct/result.output @@ -0,0 +1,7 @@ +βββββββ¬ββββββββ +| key | value | +βββββββΌββββββββ€ +| 1 | "one" | +βββββββΌββββββββ€ +| 2 | "two" | +βββββββ΄ββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_uint32/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_uint32/result.output new file mode 100644 index 00000000000..ad14eb5d5f9 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_uint32/result.output @@ -0,0 +1,5 @@ +ββββββββ¬ββββββ¬ββββββββ +| id | key | value | +ββββββββΌββββββΌββββββββ€ +| 1111 | 1 | "one" | +ββββββββ΄ββββββ΄ββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_uint64_and_string/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_uint64_and_string/result.output new file mode 100644 index 00000000000..d3c530a7966 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_data_query_uint64_and_string/result.output @@ -0,0 +1,7 @@ +ββββββββ¬ββββββ¬ββββββββββ +| id | key | value | +ββββββββΌββββββΌββββββββββ€ +| 2222 | 2 | "two" | +ββββββββΌββββββΌββββββββββ€ +| 7777 | 7 | "seven" | +ββββββββ΄ββββββ΄ββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_ignore_excess_parameters/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_ignore_excess_parameters/result.output new file mode 100644 index 00000000000..6daf3981029 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_ignore_excess_parameters/result.output @@ -0,0 +1,5 @@ +ββββββ +| a | +ββββββ€ +| 12 | +ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_list/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_list/result.output new file mode 100644 index 00000000000..09df5d83ab9 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_list/result.output @@ -0,0 +1,5 @@ +βββββββββββ +| values | +βββββββββββ€ +| [1,2,3] | +βββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_multiple_files/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_multiple_files/result.output new file mode 100644 index 00000000000..6319ad9c535 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_multiple_files/result.output @@ -0,0 +1,5 @@ +ββββββββββββββββββββ¬βββββββ¬βββββββββββββββ +| str | num | date | +ββββββββββββββββββββΌβββββββΌβββββββββββββββ€ +| "Π‘ΡΡΠΎΡΠΊΠ°" | 1542 | "2011-11-11" | +ββββββββββββββββββββ΄βββββββ΄βββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_struct/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_struct/result.output new file mode 100644 index 00000000000..0a3e804bb40 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_struct/result.output @@ -0,0 +1,7 @@ +βββββββ¬ββββββββ +| key | value | +βββββββΌββββββββ€ +| 1 | "one" | +βββββββΌββββββββ€ +| 2 | "two" | +βββββββ΄ββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_uint32/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_uint32/result.output new file mode 100644 index 00000000000..ad14eb5d5f9 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_uint32/result.output @@ -0,0 +1,5 @@ +ββββββββ¬ββββββ¬ββββββββ +| id | key | value | +ββββββββΌββββββΌββββββββ€ +| 1111 | 1 | "one" | +ββββββββ΄ββββββ΄ββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_uint64_and_string/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_uint64_and_string/result.output new file mode 100644 index 00000000000..d3c530a7966 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromJson.test_scan_query_uint64_and_string/result.output @@ -0,0 +1,7 @@ +ββββββββ¬ββββββ¬ββββββββββ +| id | key | value | +ββββββββΌββββββΌββββββββββ€ +| 2222 | 2 | "two" | +ββββββββΌββββββΌββββββββββ€ +| 7777 | 7 | "seven" | +ββββββββ΄ββββββ΄ββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_ignore_excess_parameters/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_ignore_excess_parameters/result.output new file mode 100644 index 00000000000..6daf3981029 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_ignore_excess_parameters/result.output @@ -0,0 +1,5 @@ +ββββββ +| a | +ββββββ€ +| 12 | +ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_batching_adaptive/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_batching_adaptive/result.output new file mode 100644 index 00000000000..d01126cebc9 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_batching_adaptive/result.output @@ -0,0 +1,15 @@ +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +| arr | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€ +| [{"id":1,"s":"Line1"},{"id":2,"s":"Line2"},{"id":3,"s":"Line3"}] | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +| arr | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€ +| [{"id":4,"s":"Line4"},{"id":5,"s":"Line5"},{"id":6,"s":"Line6"}] | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +| arr | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€ +| [{"id":7,"s":"Line7"},{"id":8,"s":"Line8"},{"id":9,"s":"Line9"}] | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_batching_full/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_batching_full/result.output new file mode 100644 index 00000000000..49e822c46be --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_batching_full/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββββββββββββββ +| s | +βββββββββββββββββββββββββββββ€ +| ["Line1","Line2","Line3"] | +βββββββββββββββββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_different_sources/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_different_sources/result.output new file mode 100644 index 00000000000..81382040d2f --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_different_sources/result.output @@ -0,0 +1,5 @@ +ββββββββββββββββββββββββ¬βββββββββββββββ¬ββββββ +| s | date | val | +ββββββββββββββββββββββββΌβββββββββββββββΌββββββ€ +| "Π‘ΡΡΠΎΠΊΠ° utf-8" | "2000-09-01" | 100 | +ββββββββββββββββββββββββ΄βββββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_framing_newline_delimited/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_framing_newline_delimited/result.output new file mode 100644 index 00000000000..7992832afa8 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_framing_newline_delimited/result.output @@ -0,0 +1,15 @@ +βββββββββββββββ¬ββββββ +| s | num | +βββββββββββββββΌββββββ€ +| "Some text" | 1 | +βββββββββββββββ΄ββββββ +βββββββββββββββββββββββββββββββββββ¬ββββββ +| s | num | +βββββββββββββββββββββββββββββββββββΌββββββ€ +| "Π‘ΡΡΠΎΠΊΠ° 1\nΠ‘ΡΡΠΎΠΊΠ°2" | 2 | +βββββββββββββββββββββββββββββββββββ΄ββββββ +βββββββββββββ¬ββββββ +| s | num | +βββββββββββββΌββββββ€ +| "Abacaba" | 3 | +βββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_mix_json_and_binary/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_mix_json_and_binary/result.output new file mode 100644 index 00000000000..7cfc6365a45 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_mix_json_and_binary/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ¬βββββββββββββββ¬ββββββ +| s | date | val | +βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββΌβββββββββββββββΌββββββ€ +| "\u00D0\u00A1\u00D1\u0082\u00D1\u0080\u00D0\u00BE\u00D0\u00BA\u00D0\u00B0 \u00D0\u00BD\u00D0\u00BE\u00D0\u00BC\u00D0\u00B5\u00D1\u0080 1\nString number 2\n\u00D0\u00A1\u00D1\u0082\u00D1\u0080\u00D0\u00BE\u00D0\u00BA\u00D0\u00B0 \u00D0\u00BD\u00D0\u00BE\u00D0\u00BC\u00D0\u00B5\u00D1\u0080 3" | "2044-08-21" | 32 | +βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ΄βββββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_simple_json/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_simple_json/result.output new file mode 100644 index 00000000000..b1acd7e95ab --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_simple_json/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββ¬ββββββ +| s | val | +βββββββββββββββββΌββββββ€ +| "Some_string" | 32 | +βββββββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_text_data/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_text_data/result.output new file mode 100644 index 00000000000..34c5637f0b1 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_text_data/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββββββββββββ +| s | +βββββββββββββββββββββββββββ€ +| "Line1\nLine2\nLine3\n" | +βββββββββββββββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_text_data_framing/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_text_data_framing/result.output new file mode 100644 index 00000000000..04624405d77 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_text_data_framing/result.output @@ -0,0 +1,15 @@ +βββββββββββ +| s | +βββββββββββ€ +| "Line1" | +βββββββββββ +βββββββββββ +| s | +βββββββββββ€ +| "Line2" | +βββββββββββ +βββββββββββ +| s | +βββββββββββ€ +| "Line3" | +βββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_unnamed_json/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_unnamed_json/result.output new file mode 100644 index 00000000000..3c7aab5f01a --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_data_query_unnamed_json/result.output @@ -0,0 +1,5 @@ +βββββββββββββ +| arr | +βββββββββββββ€ +| [1,2,3,4] | +βββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_ignore_excess_parameters/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_ignore_excess_parameters/result.output new file mode 100644 index 00000000000..6daf3981029 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_ignore_excess_parameters/result.output @@ -0,0 +1,5 @@ +ββββββ +| a | +ββββββ€ +| 12 | +ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_batching_adaptive/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_batching_adaptive/result.output new file mode 100644 index 00000000000..d01126cebc9 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_batching_adaptive/result.output @@ -0,0 +1,15 @@ +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +| arr | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€ +| [{"id":1,"s":"Line1"},{"id":2,"s":"Line2"},{"id":3,"s":"Line3"}] | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +| arr | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€ +| [{"id":4,"s":"Line4"},{"id":5,"s":"Line5"},{"id":6,"s":"Line6"}] | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ +| arr | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€ +| [{"id":7,"s":"Line7"},{"id":8,"s":"Line8"},{"id":9,"s":"Line9"}] | +ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_batching_full/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_batching_full/result.output new file mode 100644 index 00000000000..49e822c46be --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_batching_full/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββββββββββββββ +| s | +βββββββββββββββββββββββββββββ€ +| ["Line1","Line2","Line3"] | +βββββββββββββββββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_different_sources/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_different_sources/result.output new file mode 100644 index 00000000000..81382040d2f --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_different_sources/result.output @@ -0,0 +1,5 @@ +ββββββββββββββββββββββββ¬βββββββββββββββ¬ββββββ +| s | date | val | +ββββββββββββββββββββββββΌβββββββββββββββΌββββββ€ +| "Π‘ΡΡΠΎΠΊΠ° utf-8" | "2000-09-01" | 100 | +ββββββββββββββββββββββββ΄βββββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_framing_newline_delimited/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_framing_newline_delimited/result.output new file mode 100644 index 00000000000..7992832afa8 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_framing_newline_delimited/result.output @@ -0,0 +1,15 @@ +βββββββββββββββ¬ββββββ +| s | num | +βββββββββββββββΌββββββ€ +| "Some text" | 1 | +βββββββββββββββ΄ββββββ +βββββββββββββββββββββββββββββββββββ¬ββββββ +| s | num | +βββββββββββββββββββββββββββββββββββΌββββββ€ +| "Π‘ΡΡΠΎΠΊΠ° 1\nΠ‘ΡΡΠΎΠΊΠ°2" | 2 | +βββββββββββββββββββββββββββββββββββ΄ββββββ +βββββββββββββ¬ββββββ +| s | num | +βββββββββββββΌββββββ€ +| "Abacaba" | 3 | +βββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_mix_json_and_binary/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_mix_json_and_binary/result.output new file mode 100644 index 00000000000..7cfc6365a45 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_mix_json_and_binary/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ¬βββββββββββββββ¬ββββββ +| s | date | val | +βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββΌβββββββββββββββΌββββββ€ +| "\u00D0\u00A1\u00D1\u0082\u00D1\u0080\u00D0\u00BE\u00D0\u00BA\u00D0\u00B0 \u00D0\u00BD\u00D0\u00BE\u00D0\u00BC\u00D0\u00B5\u00D1\u0080 1\nString number 2\n\u00D0\u00A1\u00D1\u0082\u00D1\u0080\u00D0\u00BE\u00D0\u00BA\u00D0\u00B0 \u00D0\u00BD\u00D0\u00BE\u00D0\u00BC\u00D0\u00B5\u00D1\u0080 3" | "2044-08-21" | 32 | +βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ΄βββββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_simple_json/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_simple_json/result.output new file mode 100644 index 00000000000..b1acd7e95ab --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_simple_json/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββ¬ββββββ +| s | val | +βββββββββββββββββΌββββββ€ +| "Some_string" | 32 | +βββββββββββββββββ΄ββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_text_data/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_text_data/result.output new file mode 100644 index 00000000000..34c5637f0b1 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_text_data/result.output @@ -0,0 +1,5 @@ +βββββββββββββββββββββββββββ +| s | +βββββββββββββββββββββββββββ€ +| "Line1\nLine2\nLine3\n" | +βββββββββββββββββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_text_data_framing/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_text_data_framing/result.output new file mode 100644 index 00000000000..04624405d77 --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_text_data_framing/result.output @@ -0,0 +1,15 @@ +βββββββββββ +| s | +βββββββββββ€ +| "Line1" | +βββββββββββ +βββββββββββ +| s | +βββββββββββ€ +| "Line2" | +βββββββββββ +βββββββββββ +| s | +βββββββββββ€ +| "Line3" | +βββββββββββ diff --git a/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_unnamed_json/result.output b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_unnamed_json/result.output new file mode 100644 index 00000000000..3c7aab5f01a --- /dev/null +++ b/ydb/tests/functional/ydb_cli/canondata/test_ydb_table.TestExecuteQueryWithParamsFromStdin.test_scan_query_unnamed_json/result.output @@ -0,0 +1,5 @@ +βββββββββββββ +| arr | +βββββββββββββ€ +| [1,2,3,4] | +βββββββββββββ diff --git a/ydb/tests/functional/ydb_cli/test_ydb_scripting.py b/ydb/tests/functional/ydb_cli/test_ydb_scripting.py index 3aa49fc4a07..8d428920fb9 100644 --- a/ydb/tests/functional/ydb_cli/test_ydb_scripting.py +++ b/ydb/tests/functional/ydb_cli/test_ydb_scripting.py @@ -46,8 +46,8 @@ def create_table_with_data(session, path): class BaseTestScriptingService(object): @classmethod - def execute_ydb_cli_command(cls, args): - execution = yatest_common.execute([ydb_bin()] + args) + def execute_ydb_cli_command(cls, args, stdin=None): + execution = yatest_common.execute([ydb_bin()] + args, stdin=stdin) result = execution.std_out logger.debug("std_out:\n" + result.decode('utf-8')) return result @@ -79,13 +79,13 @@ class BaseTestScriptingServiceWithDatabase(BaseTestScriptingService): cls.cluster.stop() @classmethod - def execute_ydb_cli_command_with_db(cls, args): + def execute_ydb_cli_command_with_db(cls, args, stdin=None): return cls.execute_ydb_cli_command( [ "--endpoint", "grpc://localhost:%d" % cls.cluster.nodes[1].grpc_port, "--database", cls.root_dir ] + - args + args, stdin ) @@ -197,3 +197,385 @@ class TestExecuteScriptWithFormats(BaseTestScriptingServiceWithDatabase): def test_stream_yql_script_json_unicode_array(self): return self.stream_yql_script('json-unicode-array') + + +class TestExecuteScriptWithParamsFromJson(BaseTestScriptingServiceWithDatabase): + @classmethod + def setup_class(cls): + BaseTestScriptingServiceWithDatabase.setup_class() + + session = cls.driver.table_client.session().create() + cls.table_path = cls.root_dir + "/scripting_params_from_json" + create_table_with_data(session, cls.table_path) + + @staticmethod + def write_data(data, filename="params.json"): + with open(filename, "w") as file: + file.write(data) + + def uint32(self, command): + param_data = '{\n' \ + ' "par1": 1\n' \ + '}' + script = "DECLARE $par1 AS Uint32; SELECT * FROM `{}` WHERE key = $par1;".format(self.table_path) + self.write_data(param_data) + output = self.execute_ydb_cli_command_with_db(command + ["--param-file", "params.json", "-s", script]) + return self.canonical_result(output) + + def uint64_and_string(self, command): + param_data = '{\n' \ + ' "value": "seven",\n' \ + ' "id": 2222' \ + '}' + script = "DECLARE $id AS Uint64; "\ + "DECLARE $value AS String; "\ + "SELECT * FROM `{}` WHERE id = $id OR value = $value;".format(self.table_path) + self.write_data(param_data) + output = self.execute_ydb_cli_command_with_db(command + ["--param-file", "params.json", "-s", script]) + return self.canonical_result(output) + + def list(self, command): + param_data = '{\n' \ + ' "values": [1, 2, 3]\n' \ + '}' + script = "DECLARE $values AS List<Uint64?>; SELECT $values AS values;" + self.write_data(param_data) + output = self.execute_ydb_cli_command_with_db(command + ["--param-file", "params.json", "-s", script]) + return self.canonical_result(output) + + def struct(self, command): + param_data = '{\n' \ + ' "values": [\n' \ + ' {\n'\ + ' "key": 1,\n'\ + ' "value": "one"\n'\ + ' },\n'\ + ' {\n'\ + ' "key": 2,\n'\ + ' "value": "two"\n'\ + ' }\n'\ + ' ]\n'\ + '}' + script = "DECLARE $values AS List<Struct<key:Uint64, value:Utf8>>; "\ + "SELECT "\ + "Table.key AS key, "\ + "Table.value AS value "\ + "FROM (SELECT $values AS lst) FLATTEN BY lst AS Table;" + self.write_data(param_data) + output = self.execute_ydb_cli_command_with_db(command + ["--param-file", "params.json", "-s", script]) + return self.canonical_result(output) + + def multiple_files(self, command): + param_data1 = '{\n' \ + ' "str": "Π‘ΡΡΠΎΡΠΊΠ°"\n' \ + '}' + param_data2 = '{\n' \ + ' "num": 1542\n' \ + '}' + param_data3 = '{\n' \ + ' "date": "2011-11-11"\n' \ + '}' + script = "DECLARE $str AS Utf8; "\ + "DECLARE $num AS Uint64; "\ + "DECLARE $date AS Date; "\ + "SELECT $str AS str, $num as num, $date as date; " + self.write_data(param_data1, "param1.json") + self.write_data(param_data2, "param2.json") + self.write_data(param_data3, "param3.json") + output = self.execute_ydb_cli_command_with_db( + command + ["--param-file", "param1.json", "--param-file", "param2.json", "--param-file", "param3.json", "-s", script] + ) + return self.canonical_result(output) + + def ignore_excess_parameters(self, command): + param_data = '{\n' \ + ' "a": 12,\n' \ + ' "b": 34' \ + '}' + script = "DECLARE $a AS Uint64; " \ + "SELECT $a AS a; " + self.write_data(param_data) + output = self.execute_ydb_cli_command_with_db( + command + ["-s", script, "--param-file", "params.json"] + ) + return self.canonical_result(output) + + def test_uint32(self): + return self.uint32(["scripting", "yql"]) + + def test_uint64_and_string(self): + return self.uint64_and_string(["scripting", "yql"]) + + def test_list(self): + return self.list(["scripting", "yql"]) + + def test_struct(self): + return self.struct(["scripting", "yql"]) + + def test_multiple_files(self): + return self.multiple_files(["scripting", "yql"]) + + def test_ignore_excess_parameters(self): + return self.ignore_excess_parameters(["scripting", "yql"]) + + def test_stream_uint32(self): + return self.uint32(["yql"]) + + def test_stream_uint64_and_string(self): + return self.uint64_and_string(["yql"]) + + def test_stream_list(self): + return self.list(["yql"]) + + def test_stream_struct(self): + return self.struct(["yql"]) + + def test_stream_multiple_files(self): + return self.multiple_files(["yql"]) + + def test_stream_ignore_excess_parameters(self): + return self.ignore_excess_parameters(["yql"]) + + +class TestExecuteScriptWithParamsFromStdin(BaseTestScriptingServiceWithDatabase): + @classmethod + def setup_class(cls): + BaseTestScriptingServiceWithDatabase.setup_class() + + session = cls.driver.table_client.session().create() + cls.table_path = cls.root_dir + "/scripting_params_from_stdin" + create_table_with_data(session, cls.table_path) + + @staticmethod + def write_data(data, filename="stdin.txt"): + with open(filename, "w") as file: + file.write(data) + + @classmethod + def get_stdin(cls): + cls.stdin = open("stdin.txt", "r") + return cls.stdin + + @classmethod + def close_stdin(cls): + cls.stdin.close() + + def simple_json(self, command): + param_data = '{\n' \ + ' "s": "Some_string",\n' \ + ' "val": 32\n' \ + '}' + script = "DECLARE $s AS Utf8; "\ + "DECLARE $val AS Uint64; "\ + "SELECT $s AS s, $val AS val; " + self.write_data(param_data) + output = self.execute_ydb_cli_command_with_db(command + ["-s", script], self.get_stdin()) + self.close_stdin() + return self.canonical_result(output) + + def text_data(self, command): + param_data = 'Line1\n' \ + 'Line2\n' \ + 'Line3\n' + script = "DECLARE $s AS Utf8; " \ + "SELECT $s AS s; " + self.write_data(param_data) + output = self.execute_ydb_cli_command_with_db( + command + ["-s", script, "--stdin-format", "raw", "--stdin-par", "s"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def unnamed_json(self, command): + param_data = "[1, 2, 3, 4]" + script = "DECLARE $arr AS List<Uint64>; "\ + "SELECT $arr AS arr; " + self.write_data(param_data) + output = self.execute_ydb_cli_command_with_db( + command + ["-s", script, "--stdin-par", "arr"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def mix_json_and_binary(self, command): + param_data1 = 'Π‘ΡΡΠΎΠΊΠ° Π½ΠΎΠΌΠ΅Ρ 1\n' \ + 'String number 2\n' \ + 'Π‘ΡΡΠΎΠΊΠ° Π½ΠΎΠΌΠ΅Ρ 3' + param_data2 = '{\n' \ + ' "date": "2044-08-21",\n' \ + ' "val": 32\n' \ + '}' + script = "DECLARE $s AS String; " \ + "DECLARE $date AS Date; " \ + "DECLARE $val AS Uint64; " \ + "SELECT $s AS s, $date AS date, $val AS val; " + self.write_data(param_data1) + self.write_data(param_data2, "params.json") + output = self.execute_ydb_cli_command_with_db( + command + ["-s", script, "--stdin-par", "s", "--stdin-format", "raw", "--param-file", "params.json"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def different_sources(self, command): + param_data1 = '{\n' \ + ' "s": "Π‘ΡΡΠΎΠΊΠ° utf-8"\n' \ + '}' + param_data2 = '{\n' \ + ' "date": "2000-09-01"\n' \ + '}' + script = "DECLARE $s AS Utf8; " \ + "DECLARE $date AS Date; " \ + "DECLARE $val AS Uint64; " \ + "SELECT $s AS s, $date AS date, $val AS val; " + self.write_data(param_data1) + self.write_data(param_data2, "params.json") + output = self.execute_ydb_cli_command_with_db( + command + ["-s", script, "--param-file", "params.json", "--param", "$val=100"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def framing_newline_delimited(self, command): + param_data = '{"s": "Some text", "num": 1}\n' \ + '{"s": "Π‘ΡΡΠΎΠΊΠ° 1\\nΠ‘ΡΡΠΎΠΊΠ°2", "num": 2}\n' \ + '{"s": "Abacaba", "num": 3}\n' + script = "DECLARE $s AS Utf8; " \ + "DECLARE $num AS Uint64; " \ + "SELECT $s AS s, $num AS num; " + self.write_data(param_data) + output = self.execute_ydb_cli_command_with_db( + command + ["-s", script, "--stdin-format", "newline-delimited"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def text_data_framing(self, command): + param_data = 'Line1\n' \ + 'Line2\n' \ + 'Line3\n' + script = "DECLARE $s AS Utf8; " \ + "SELECT $s AS s; " + self.write_data(param_data) + output = self.execute_ydb_cli_command_with_db( + command + ["-s", script, "--stdin-format", "raw", "--stdin-par", "s", "--stdin-format", "newline-delimited"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def batching_full(self, command): + param_data = 'Line1\n' \ + 'Line2\n' \ + 'Line3\n' + script = "DECLARE $s AS List<Utf8>; " \ + "SELECT $s AS s; " + self.write_data(param_data) + output = self.execute_ydb_cli_command_with_db( + command + ["-s", script, "--stdin-format", "raw", "--stdin-par", "s", + "--stdin-format", "newline-delimited", "--batch", "full"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def batching_adaptive(self, command): + param_data = '{"s": "Line1", "id": 1}\n' \ + '{"s": "Line2", "id": 2}\n' \ + '{"s": "Line3", "id": 3}\n' \ + '{"s": "Line4", "id": 4}\n' \ + '{"s": "Line5", "id": 5}\n' \ + '{"s": "Line6", "id": 6}\n' \ + '{"s": "Line7", "id": 7}\n' \ + '{"s": "Line8", "id": 8}\n' \ + '{"s": "Line9", "id": 9}\n' + script = "DECLARE $arr AS List<Struct<s:Utf8, id:Uint64>>; " \ + "SELECT $arr AS arr; " + self.write_data(param_data) + output = self.execute_ydb_cli_command_with_db( + command + ["-s", script, "--stdin-par", "arr", "--stdin-format", "newline-delimited", + "--batch", "adaptive", "--batch-max-delay", "0", "--batch-limit", "3"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def ignore_excess_parameters(self, command): + param_data = '{\n' \ + ' "a": 12,\n' \ + ' "b": 34' \ + '}' + script = "DECLARE $a AS Uint64; " \ + "SELECT $a AS a; " + self.write_data(param_data) + output = self.execute_ydb_cli_command_with_db( + command + ["-s", script], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def test_simple_json(self): + return self.simple_json(["scripting", "yql"]) + + def test_text_data(self): + return self.text_data(["scripting", "yql"]) + + def test_unnamed_json(self): + return self.unnamed_json(["scripting", "yql"]) + + def test_mix_json_and_binary(self): + return self.mix_json_and_binary(["scripting", "yql"]) + + def test_different_sources(self): + return self.different_sources(["scripting", "yql"]) + + def test_framing_newline_delimited(self): + return self.framing_newline_delimited(["scripting", "yql"]) + + def test_text_data_framing(self): + return self.text_data_framing(["scripting", "yql"]) + + def test_batching_full(self): + return self.batching_full(["scripting", "yql"]) + + def test_batching_adaptive(self): + return self.batching_adaptive(["scripting", "yql"]) + + def test_ignore_excess_parameters(self): + return self.ignore_excess_parameters(["scripting", "yql"]) + + def test_stream_simple_json(self): + return self.simple_json(["yql"]) + + def test_stream_text_data(self): + return self.text_data(["yql"]) + + def test_stream_unnamed_json(self): + return self.unnamed_json(["yql"]) + + def test_stream_mix_json_and_binary(self): + return self.mix_json_and_binary(["yql"]) + + def test_stream_different_sources(self): + return self.different_sources(["yql"]) + + def test_stream_framing_newline_delimited(self): + return self.framing_newline_delimited(["yql"]) + + def test_stream_text_data_framing(self): + return self.text_data_framing(["yql"]) + + def test_stream_batching_full(self): + return self.batching_full(["yql"]) + + def test_stream_batching_adaptive(self): + return self.batching_adaptive(["yql"]) + + def test_stream_ignore_excess_parameters(self): + return self.ignore_excess_parameters(["yql"]) diff --git a/ydb/tests/functional/ydb_cli/test_ydb_table.py b/ydb/tests/functional/ydb_cli/test_ydb_table.py index 61f5c54c426..e7533d2a739 100644 --- a/ydb/tests/functional/ydb_cli/test_ydb_table.py +++ b/ydb/tests/functional/ydb_cli/test_ydb_table.py @@ -64,14 +64,14 @@ class BaseTestTableService(object): cls.cluster.stop() @classmethod - def execute_ydb_cli_command(cls, args): + def execute_ydb_cli_command(cls, args, stdin=None): execution = yatest_common.execute( [ ydb_bin(), "--endpoint", "grpc://localhost:%d" % cls.cluster.nodes[1].grpc_port, "--database", cls.root_dir ] + - args + args, stdin=stdin ) result = execution.std_out @@ -228,3 +228,396 @@ class TestExecuteQueryWithFormats(BaseTestTableService): def test_read_table_tsv(self): return self.execute_read_table('tsv') + + +class TestExecuteQueryWithParamsFromJson(BaseTestTableService): + @classmethod + def setup_class(cls): + BaseTestTableService.setup_class() + + session = cls.driver.table_client.session().create() + cls.table_path = cls.root_dir + "/table_params_from_json" + create_table_with_data(session, cls.table_path) + + @staticmethod + def write_data(data, filename="params.json"): + with open(filename, "w") as file: + file.write(data) + + def uint32(self, query_type): + param_data = '{\n' \ + ' "par1": 1\n' \ + '}' + query = "DECLARE $par1 AS Uint32; SELECT * FROM `{}` WHERE key = $par1;".format(self.table_path) + self.write_data(param_data) + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "--param-file", "params.json", "-q", query] + ) + return self.canonical_result(output) + + def uint64_and_string(self, query_type): + param_data = '{\n' \ + ' "value": "seven",\n' \ + ' "id": 2222' \ + '}' + query = "DECLARE $id AS Uint64; "\ + "DECLARE $value AS String; "\ + "SELECT * FROM `{}` WHERE id = $id OR value = $value;".format(self.table_path) + self.write_data(param_data) + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "--param-file", "params.json", "-q", query] + ) + return self.canonical_result(output) + + def list(self, query_type): + param_data = '{\n' \ + ' "values": [1, 2, 3]\n' \ + '}' + query = "DECLARE $values AS List<Uint64?>; SELECT $values AS values;" + self.write_data(param_data) + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "--param-file", "params.json", "-q", query] + ) + return self.canonical_result(output) + + def struct(self, query_type): + param_data = '{\n' \ + ' "values": [\n' \ + ' {\n'\ + ' "key": 1,\n'\ + ' "value": "one"\n'\ + ' },\n'\ + ' {\n'\ + ' "key": 2,\n'\ + ' "value": "two"\n'\ + ' }\n'\ + ' ]\n'\ + '}' + query = "DECLARE $values AS List<Struct<key:Uint64, value:Utf8>>; "\ + "SELECT "\ + "Table.key AS key, "\ + "Table.value AS value "\ + "FROM (SELECT $values AS lst) FLATTEN BY lst AS Table;" + self.write_data(param_data) + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "--param-file", "params.json", "-q", query] + ) + return self.canonical_result(output) + + def multiple_files(self, query_type): + param_data1 = '{\n' \ + ' "str": "Π‘ΡΡΠΎΡΠΊΠ°"\n' \ + '}' + param_data2 = '{\n' \ + ' "num": 1542\n' \ + '}' + param_data3 = '{\n' \ + ' "date": "2011-11-11"\n' \ + '}' + query = "DECLARE $str AS Utf8; "\ + "DECLARE $num AS Uint64; "\ + "DECLARE $date AS Date; "\ + "SELECT $str AS str, $num as num, $date as date; " + self.write_data(param_data1, "param1.json") + self.write_data(param_data2, "param2.json") + self.write_data(param_data3, "param3.json") + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "--param-file", "param1.json", "--param-file", + "param2.json", "--param-file", "param3.json", "-q", query] + ) + return self.canonical_result(output) + + def ignore_excess_parameters(self, query_type): + param_data = '{\n' \ + ' "a": 12,\n' \ + ' "b": 34' \ + '}' + query = "DECLARE $a AS Uint64; " \ + "SELECT $a AS a; " + self.write_data(param_data) + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "-q", query, "--param-file", "params.json"] + ) + return self.canonical_result(output) + + def test_data_query_uint32(self): + return self.uint32("data") + + def test_data_query_uint64_and_string(self): + return self.uint64_and_string("data") + + def test_data_query_list(self): + return self.list("data") + + def test_data_query_struct(self): + return self.struct("data") + + def test_data_query_multiple_files(self): + return self.multiple_files("data") + + def test_data_ignore_excess_parameters(self): + return self.ignore_excess_parameters("data") + + def test_scan_query_uint32(self): + return self.uint32("scan") + + def test_scan_query_uint64_and_string(self): + return self.uint64_and_string("scan") + + def test_scan_query_list(self): + return self.list("scan") + + def test_scan_query_struct(self): + return self.struct("scan") + + def test_scan_query_multiple_files(self): + return self.multiple_files("scan") + + def test_scan_ignore_excess_parameters(self): + return self.ignore_excess_parameters("scan") + + +class TestExecuteQueryWithParamsFromStdin(BaseTestTableService): + @classmethod + def setup_class(cls): + BaseTestTableService.setup_class() + + session = cls.driver.table_client.session().create() + cls.table_path = cls.root_dir + "/table_params_from_stdin" + create_table_with_data(session, cls.table_path) + + @staticmethod + def write_data(data, filename="stdin.txt"): + with open(filename, "w") as file: + file.write(data) + + @classmethod + def get_stdin(cls): + cls.stdin = open("stdin.txt", "r") + return cls.stdin + + @classmethod + def close_stdin(cls): + cls.stdin.close() + + def simple_json(self, query_type): + param_data = '{\n' \ + ' "s": "Some_string",\n' \ + ' "val": 32\n' \ + '}' + query = "DECLARE $s AS Utf8; "\ + "DECLARE $val AS Uint64; "\ + "SELECT $s AS s, $val AS val; " + self.write_data(param_data) + output = self.execute_ydb_cli_command(["table", "query", "execute", "-t", query_type, "-q", query], self.get_stdin()) + self.close_stdin() + return self.canonical_result(output) + + def text_data(self, query_type): + param_data = 'Line1\n' \ + 'Line2\n' \ + 'Line3\n' + query = "DECLARE $s AS Utf8; " \ + "SELECT $s AS s; " + self.write_data(param_data) + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "-q", query, "--stdin-format", "raw", "--stdin-par", "s"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def unnamed_json(self, query_type): + param_data = "[1, 2, 3, 4]" + query = "DECLARE $arr AS List<Uint64>; "\ + "SELECT $arr AS arr; " + self.write_data(param_data) + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "-q", query, "--stdin-par", "arr"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def mix_json_and_binary(self, query_type): + param_data1 = 'Π‘ΡΡΠΎΠΊΠ° Π½ΠΎΠΌΠ΅Ρ 1\n' \ + 'String number 2\n' \ + 'Π‘ΡΡΠΎΠΊΠ° Π½ΠΎΠΌΠ΅Ρ 3' + param_data2 = '{\n' \ + ' "date": "2044-08-21",\n' \ + ' "val": 32\n' \ + '}' + query = "DECLARE $s AS String; " \ + "DECLARE $date AS Date; " \ + "DECLARE $val AS Uint64; " \ + "SELECT $s AS s, $date AS date, $val AS val; " + self.write_data(param_data1) + self.write_data(param_data2, "params.json") + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "-q", query, "--stdin-par", "s", + "--stdin-format", "raw", "--param-file", "params.json"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def different_sources(self, query_type): + param_data1 = '{\n' \ + ' "s": "Π‘ΡΡΠΎΠΊΠ° utf-8"\n' \ + '}' + param_data2 = '{\n' \ + ' "date": "2000-09-01"\n' \ + '}' + query = "DECLARE $s AS Utf8; " \ + "DECLARE $date AS Date; " \ + "DECLARE $val AS Uint64; " \ + "SELECT $s AS s, $date AS date, $val AS val; " + self.write_data(param_data1) + self.write_data(param_data2, "params.json") + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "-q", query, "--param-file", "params.json", "--param", "$val=100"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def framing_newline_delimited(self, query_type): + param_data = '{"s": "Some text", "num": 1}\n' \ + '{"s": "Π‘ΡΡΠΎΠΊΠ° 1\\nΠ‘ΡΡΠΎΠΊΠ°2", "num": 2}\n' \ + '{"s": "Abacaba", "num": 3}\n' + query = "DECLARE $s AS Utf8; " \ + "DECLARE $num AS Uint64; " \ + "SELECT $s AS s, $num AS num; " + self.write_data(param_data) + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "-q", query, "--stdin-format", "newline-delimited"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def text_data_framing(self, query_type): + param_data = 'Line1\n' \ + 'Line2\n' \ + 'Line3\n' + query = "DECLARE $s AS Utf8; " \ + "SELECT $s AS s; " + self.write_data(param_data) + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "-q", query, "--stdin-format", "raw", + "--stdin-par", "s", "--stdin-format", "newline-delimited"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def batching_full(self, query_type): + param_data = 'Line1\n' \ + 'Line2\n' \ + 'Line3\n' + query = "DECLARE $s AS List<Utf8>; " \ + "SELECT $s AS s; " + self.write_data(param_data) + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "-q", query, "--stdin-format", "raw", "--stdin-par", "s", + "--stdin-format", "newline-delimited", "--batch", "full"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def batching_adaptive(self, query_type): + param_data = '{"s": "Line1", "id": 1}\n' \ + '{"s": "Line2", "id": 2}\n' \ + '{"s": "Line3", "id": 3}\n' \ + '{"s": "Line4", "id": 4}\n' \ + '{"s": "Line5", "id": 5}\n' \ + '{"s": "Line6", "id": 6}\n' \ + '{"s": "Line7", "id": 7}\n' \ + '{"s": "Line8", "id": 8}\n' \ + '{"s": "Line9", "id": 9}\n' + query = "DECLARE $arr as List<Struct<s:Utf8, id:Uint64>>; " \ + "SELECT $arr as arr; " + self.write_data(param_data) + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "-q", query, "--stdin-par", "arr", "--stdin-format", "newline-delimited", + "--batch", "adaptive", "--batch-max-delay", "0", "--batch-limit", "3"], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def ignore_excess_parameters(self, query_type): + param_data = '{\n' \ + ' "a": 12,\n' \ + ' "b": 34' \ + '}' + query = "DECLARE $a AS Uint64; " \ + "SELECT $a AS a; " + self.write_data(param_data) + output = self.execute_ydb_cli_command( + ["table", "query", "execute", "-t", query_type, "-q", query], + self.get_stdin() + ) + self.close_stdin() + return self.canonical_result(output) + + def test_data_query_simple_json(self): + return self.simple_json("data") + + def test_data_query_text_data(self): + return self.text_data("data") + + def test_data_query_unnamed_json(self): + return self.unnamed_json("data") + + def test_data_query_mix_json_and_binary(self): + return self.mix_json_and_binary("data") + + def test_data_query_different_sources(self): + return self.different_sources("data") + + def test_data_query_framing_newline_delimited(self): + return self.framing_newline_delimited("data") + + def test_data_query_text_data_framing(self): + return self.text_data_framing("data") + + def test_data_query_batching_full(self): + return self.batching_full("data") + + def test_data_query_batching_adaptive(self): + return self.batching_adaptive("data") + + def test_data_ignore_excess_parameters(self): + return self.ignore_excess_parameters("data") + + def test_scan_query_simple_json(self): + return self.simple_json("scan") + + def test_scan_query_text_data(self): + return self.text_data("scan") + + def test_scan_query_unnamed_json(self): + return self.unnamed_json("scan") + + def test_scan_query_mix_json_and_binary(self): + return self.mix_json_and_binary("scan") + + def test_scan_query_different_sources(self): + return self.different_sources("scan") + + def test_scan_query_framing_newline_delimited(self): + return self.framing_newline_delimited("scan") + + def test_scan_query_text_data_framing(self): + return self.text_data_framing("scan") + + def test_scan_query_batching_full(self): + return self.batching_full("scan") + + def test_scan_query_batching_adaptive(self): + return self.batching_adaptive("scan") + + def test_scan_ignore_excess_parameters(self): + return self.ignore_excess_parameters("scan") |