diff options
author | andrewproni <andrewproni@yandex-team.com> | 2023-06-27 15:00:12 +0300 |
---|---|---|
committer | andrewproni <andrewproni@yandex-team.com> | 2023-06-27 15:00:12 +0300 |
commit | 01f8e9fbbb21e162ecf9e59d2e79435ad4cee827 (patch) | |
tree | 9ed566da3444b5c2b22e1e3be05f7a05b7741290 | |
parent | 65276d5feee9170231b2ff6e033f715b2c82a589 (diff) | |
download | ydb-01f8e9fbbb21e162ecf9e59d2e79435ad4cee827.tar.gz |
Поддержка других режимов ExecMode
8 files changed, 149 insertions, 44 deletions
diff --git a/ydb/core/grpc_services/query/rpc_execute_script.cpp b/ydb/core/grpc_services/query/rpc_execute_script.cpp index 85a8ade5c2..c8a5140ae5 100644 --- a/ydb/core/grpc_services/query/rpc_execute_script.cpp +++ b/ydb/core/grpc_services/query/rpc_execute_script.cpp @@ -44,21 +44,30 @@ bool FillQueryContent(const Ydb::Query::ExecuteScriptRequest& req, NKikimrKqp::T } } + std::tuple<Ydb::StatusIds::StatusCode, NYql::TIssues> FillKqpRequest( const Ydb::Query::ExecuteScriptRequest& req, NKikimrKqp::TEvQueryRequest& kqpRequest) { kqpRequest.MutableRequest()->MutableYdbParameters()->insert(req.parameters().begin(), req.parameters().end()); + switch (req.exec_mode()) { case Ydb::Query::EXEC_MODE_EXECUTE: kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); break; + case Ydb::Query::EXEC_MODE_EXPLAIN: + kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN); + break; + // TODO: other modes default: { NYql::TIssues issues; - issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Unexpected query mode")); + issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, + req.exec_mode() == Ydb::Query::EXEC_MODE_UNSPECIFIED ? "Query mode is not specified" : "Query mode is not supported yet")); return {Ydb::StatusIds::BAD_REQUEST, std::move(issues)}; } } + + kqpRequest.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_GENERIC_SCRIPT); kqpRequest.MutableRequest()->SetKeepSession(false); diff --git a/ydb/core/kqp/proxy_service/kqp_script_executions.cpp b/ydb/core/kqp/proxy_service/kqp_script_executions.cpp index 3347992604..5d4540a695 100644 --- a/ydb/core/kqp/proxy_service/kqp_script_executions.cpp +++ b/ydb/core/kqp/proxy_service/kqp_script_executions.cpp @@ -411,41 +411,41 @@ private: size_t TablesCreating = 0; }; +Ydb::Query::ExecMode GetExecModeFromAction(NKikimrKqp::EQueryAction action) { + switch (action) { + case NKikimrKqp::QUERY_ACTION_EXECUTE: + return Ydb::Query::EXEC_MODE_EXECUTE; + case NKikimrKqp::QUERY_ACTION_EXPLAIN: + return Ydb::Query::EXEC_MODE_EXPLAIN; + case NKikimrKqp::QUERY_ACTION_VALIDATE: + return Ydb::Query::EXEC_MODE_VALIDATE; + case NKikimrKqp::QUERY_ACTION_PARSE: + return Ydb::Query::EXEC_MODE_PARSE; + case NKikimrKqp::QUERY_ACTION_PREPARE: + [[fallthrough]]; + case NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED: + [[fallthrough]]; + case NKikimrKqp::QUERY_ACTION_BEGIN_TX: + [[fallthrough]]; + case NKikimrKqp::QUERY_ACTION_COMMIT_TX: + [[fallthrough]]; + case NKikimrKqp::QUERY_ACTION_ROLLBACK_TX: + [[fallthrough]]; + case NKikimrKqp::QUERY_ACTION_TOPIC: + throw std::runtime_error(TStringBuilder() << "Unsupported query action: " << NKikimrKqp::EQueryAction_Name(action)); + } +} + class TCreateScriptOperationQuery : public TQueryBase { public: - TCreateScriptOperationQuery(const TString& executionId, const NActors::TActorId& runScriptActorId, const NKikimrKqp::TEvQueryRequest& req, TDuration leaseDuration) + TCreateScriptOperationQuery(const TString& executionId, const NActors::TActorId& runScriptActorId, const NKikimrKqp::TEvQueryRequest& req, TDuration leaseDuration = TDuration::Zero()) : ExecutionId(executionId) , RunScriptActorId(runScriptActorId) , Request(req) - , LeaseDuration(leaseDuration) + , LeaseDuration(leaseDuration ? leaseDuration : LEASE_DURATION) { } - - Ydb::Query::ExecMode GetExecMode() const { - switch (Request.GetRequest().GetAction()) { - case NKikimrKqp::QUERY_ACTION_EXECUTE: - return Ydb::Query::EXEC_MODE_EXECUTE; - case NKikimrKqp::QUERY_ACTION_EXPLAIN: - return Ydb::Query::EXEC_MODE_EXPLAIN; - case NKikimrKqp::QUERY_ACTION_VALIDATE: - return Ydb::Query::EXEC_MODE_VALIDATE; - case NKikimrKqp::QUERY_ACTION_PARSE: - return Ydb::Query::EXEC_MODE_PARSE; - case NKikimrKqp::QUERY_ACTION_PREPARE: - [[fallthrough]]; - case NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED: - [[fallthrough]]; - case NKikimrKqp::QUERY_ACTION_BEGIN_TX: - [[fallthrough]]; - case NKikimrKqp::QUERY_ACTION_COMMIT_TX: - [[fallthrough]]; - case NKikimrKqp::QUERY_ACTION_ROLLBACK_TX: - [[fallthrough]]; - case NKikimrKqp::QUERY_ACTION_TOPIC: - throw std::runtime_error(TStringBuilder() << "Unsupported query action: " << NKikimrKqp::EQueryAction_Name(Request.GetRequest().GetAction())); - } - } - + void OnRunQuery() override { TString sql = R"( DECLARE $database AS Text; @@ -481,7 +481,7 @@ public: .Int32(Ydb::Query::EXEC_STATUS_STARTING) .Build() .AddParam("$execution_mode") - .Int32(GetExecMode()) + .Int32(GetExecModeFromAction(Request.GetRequest().GetAction())) .Build() .AddParam("$query_text") .Utf8(Request.GetRequest().GetQuery()) @@ -536,7 +536,7 @@ struct TCreateScriptExecutionActor : public TActorBootstrapped<TCreateScriptExec void Handle(TEvPrivate::TEvCreateScriptOperationResponse::TPtr& ev) { if (ev->Get()->Status == Ydb::StatusIds::SUCCESS) { Send(RunScriptActorId, new NActors::TEvents::TEvWakeup()); - Send(Event->Sender, new TEvKqp::TEvScriptResponse(ScriptExecutionOperationFromExecutionId(ev->Get()->ExecutionId), ev->Get()->ExecutionId, Ydb::Query::EXEC_STATUS_STARTING, Ydb::Query::EXEC_MODE_EXECUTE)); + Send(Event->Sender, new TEvKqp::TEvScriptResponse(ScriptExecutionOperationFromExecutionId(ev->Get()->ExecutionId), ev->Get()->ExecutionId, Ydb::Query::EXEC_STATUS_STARTING, GetExecModeFromAction(Event->Get()->Record.GetRequest().GetAction()))); } else { Send(RunScriptActorId, new NActors::TEvents::TEvPoison()); Send(Event->Sender, new TEvKqp::TEvScriptResponse(ev->Get()->Status, std::move(ev->Get()->Issues))); @@ -656,19 +656,22 @@ class TScriptExecutionFinisherBase : public TQueryBase { public: using TQueryBase::TQueryBase; - void FinishScriptExecution(const TString& database, const TString& executionId, Ydb::StatusIds::StatusCode operationStatus, Ydb::Query::ExecStatus execStatus, const NYql::TIssues& issues = LeaseExpiredIssues(), TTxControl txControl = TTxControl::ContinueAndCommitTx()) { + void FinishScriptExecution(const TString& database, const TString& executionId, Ydb::StatusIds::StatusCode operationStatus, Ydb::Query::ExecStatus execStatus, + const NYql::TIssues& issues = LeaseExpiredIssues(), TTxControl txControl = TTxControl::ContinueAndCommitTx(), TString queryPlan = "{}") { TString sql = R"( DECLARE $database AS Text; DECLARE $execution_id AS Text; DECLARE $operation_status AS Int32; DECLARE $execution_status AS Int32; DECLARE $issues AS JsonDocument; + DECLARE $plan AS JsonDocument; UPDATE `.metadata/script_executions` SET operation_status = $operation_status, execution_status = $execution_status, issues = $issues, + plan = $plan, end_ts = CurrentUtcTimestamp() WHERE database = $database AND execution_id = $execution_id; @@ -692,6 +695,9 @@ public: .Build() .AddParam("$issues") .JsonDocument(SerializeIssues(issues)) + .Build() + .AddParam("$plan") + .JsonDocument(queryPlan) .Build(); RunDataQuery(sql, ¶ms, txControl); @@ -720,7 +726,8 @@ public: ui64 leaseGeneration, Ydb::StatusIds::StatusCode operationStatus, Ydb::Query::ExecStatus execStatus, - NYql::TIssues issues + NYql::TIssues issues, + TString queryPlan = "{}" ) : Database(database) , ExecutionId(executionId) @@ -728,6 +735,7 @@ public: , OperationStatus(operationStatus) , ExecStatus(execStatus) , Issues(std::move(issues)) + , QueryPlan(std::move(queryPlan)) { } @@ -777,7 +785,7 @@ public: return; } - FinishScriptExecution(Database, ExecutionId, OperationStatus, ExecStatus, Issues); + FinishScriptExecution(Database, ExecutionId, OperationStatus, ExecStatus, Issues, TTxControl::ContinueAndCommitTx(), QueryPlan); FinishWasRun = true; } else { Finish(); @@ -785,7 +793,8 @@ public: } void OnFinish(Ydb::StatusIds::StatusCode status, NYql::TIssues&& issues) override { - KQP_PROXY_LOG_D("Finish script execution operation. ExecutionId: " << ExecutionId << ". Lease generation: " << LeaseGeneration << ": " << Ydb::StatusIds::StatusCode_Name(status) << ". Issues: " << issues.ToOneLineString()); + KQP_PROXY_LOG_D("Finish script execution operation. ExecutionId: " << ExecutionId << ". Lease generation: " << + LeaseGeneration << ": " << Ydb::StatusIds::StatusCode_Name(status) << ". Issues: " << issues.ToOneLineString() << ". Plan: " << QueryPlan); Send(Owner, new TEvScriptExecutionFinished(status, std::move(issues))); } @@ -796,6 +805,7 @@ private: const Ydb::StatusIds::StatusCode OperationStatus; const Ydb::Query::ExecStatus ExecStatus; const NYql::TIssues Issues; + const TString QueryPlan; bool FinishWasRun = false; }; @@ -1053,8 +1063,9 @@ public: query_text, syntax, execution_mode, - issues, - result_set_metas + result_set_metas, + plan, + issues FROM `.metadata/script_executions` WHERE database = $database AND execution_id = $execution_id; @@ -1123,6 +1134,11 @@ public: metadata.set_exec_mode(static_cast<Ydb::Query::ExecMode>(*executionMode)); } + const TMaybe<TString> plan = result.ColumnParser("plan").GetOptionalJsonDocument(); + if (plan) { + metadata.mutable_exec_stats()->set_query_plan(*plan); + } + const TMaybe<TString> issuesSerialized = result.ColumnParser("issues").GetOptionalJsonDocument(); if (issuesSerialized) { Issues = DeserializeIssues(*issuesSerialized); @@ -1941,9 +1957,10 @@ NActors::IActor* CreateScriptExecutionFinisher( ui64 leaseGeneration, Ydb::StatusIds::StatusCode operationStatus, Ydb::Query::ExecStatus execStatus, - NYql::TIssues issues) + NYql::TIssues issues, + TString queryPlan) { - return new TScriptExecutionFinisher(executionId, database, leaseGeneration, operationStatus, execStatus, std::move(issues)); + return new TScriptExecutionFinisher(executionId, database, leaseGeneration, operationStatus, execStatus, std::move(issues), std::move(queryPlan)); } NActors::IActor* CreateForgetScriptExecutionOperationActor(TEvForgetScriptExecutionOperation::TPtr ev) { diff --git a/ydb/core/kqp/proxy_service/kqp_script_executions.h b/ydb/core/kqp/proxy_service/kqp_script_executions.h index 940ff189ed..33b2e3d070 100644 --- a/ydb/core/kqp/proxy_service/kqp_script_executions.h +++ b/ydb/core/kqp/proxy_service/kqp_script_executions.h @@ -33,7 +33,8 @@ NActors::IActor* CreateScriptExecutionFinisher( ui64 leaseGeneration, Ydb::StatusIds::StatusCode operationStatus, Ydb::Query::ExecStatus execStatus, - NYql::TIssues issues + NYql::TIssues issues, + TString queryPlan = "{}" ); // Updates lease deadline in database. diff --git a/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp b/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp index 3c66dd8380..7848fa7fd3 100644 --- a/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp +++ b/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp @@ -261,9 +261,13 @@ private: return; } auto& record = ev->Get()->Record.GetRef(); - + const auto& issueMessage = record.GetResponse().GetQueryIssues(); NYql::IssuesFromMessage(issueMessage, Issues); + + if (record.GetResponse().HasQueryPlan()) { + QueryPlan = record.GetResponse().GetQueryPlan(); + } Finish(record.GetYdbStatus()); } @@ -390,13 +394,15 @@ private: } } + void CheckSaveInflight() { if (Status == Ydb::StatusIds::SUCCESS && RunState == ERunState::Finishing && (SaveResultMetaInflight || SaveResultInflight)) { // wait for save completions return; } - Register(CreateScriptExecutionFinisher(ExecutionId, Database, LeaseGeneration, Status, GetExecStatusFromStatusCode(Status), Issues)); + Register(CreateScriptExecutionFinisher(ExecutionId, Database, LeaseGeneration, Status, GetExecStatusFromStatusCode(Status), + Issues, std::move(QueryPlan))); if (RunState == ERunState::Cancelling) { Issues.AddIssue("Script execution is cancelled"); } @@ -454,6 +460,7 @@ private: ui32 SaveResultInflight = 0; ui32 SaveResultMetaInflight = 0; bool PendingResultMeta = false; + TString QueryPlan = "{}"; }; } // namespace diff --git a/ydb/core/kqp/ut/service/kqp_query_service_ut.cpp b/ydb/core/kqp/ut/service/kqp_query_service_ut.cpp index 94a23904bd..b8d461abe5 100644 --- a/ydb/core/kqp/ut/service/kqp_query_service_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_query_service_ut.cpp @@ -286,7 +286,7 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr.GetDriver()); - UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecStatus, EExecStatus::Completed); + UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecMode, EExecMode::Execute); UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecutionId, scriptExecutionOperation.Metadata().ExecutionId); UNIT_ASSERT_STRING_CONTAINS(readyOp.Metadata().ScriptContent.Text, "SELECT 42"); @@ -341,6 +341,74 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { } } + Y_UNIT_TEST(ExplainScript) { + auto kikimr = DefaultKikimrRunner(); + auto db = kikimr.GetQueryClient(); + + auto settings = TExecuteScriptSettings().ExecMode(Ydb::Query::EXEC_MODE_EXPLAIN); + auto scriptExecutionOperation = db.ExecuteScript(R"( + SELECT 42 + )", settings).ExtractValueSync(); + + UNIT_ASSERT_EQUAL(scriptExecutionOperation.Metadata().ExecMode, EExecMode::Explain); + + NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr.GetDriver()); + UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecStatus, EExecStatus::Completed); + UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecMode, EExecMode::Explain); + UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecutionId, scriptExecutionOperation.Metadata().ExecutionId); + UNIT_ASSERT_STRING_CONTAINS(readyOp.Metadata().ScriptContent.Text, "SELECT 42"); + UNIT_ASSERT(!readyOp.Metadata().ExecStats.query_plan().empty()); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(readyOp.Metadata().ExecStats.query_plan(), &plan, true); + UNIT_ASSERT(ValidatePlanNodeIds(plan)); + } + + Y_UNIT_TEST(ParseScript) { + auto kikimr = DefaultKikimrRunner(); + auto db = kikimr.GetQueryClient(); + + auto settings = TExecuteScriptSettings().ExecMode(Ydb::Query::EXEC_MODE_PARSE); + auto scriptExecutionOperation = db.ExecuteScript(R"( + SELECT 42 + )", settings).ExtractValueSync(); + + // TODO: change when parse mode will be supported + UNIT_ASSERT_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::BAD_REQUEST, scriptExecutionOperation.Status().GetStatus()); + UNIT_ASSERT(scriptExecutionOperation.Status().GetIssues().Size() == 1); + UNIT_ASSERT_EQUAL_C(scriptExecutionOperation.Status().GetIssues().back().GetMessage(), "Query mode is not supported yet", scriptExecutionOperation.Status().GetIssues().ToString()); + + } + + Y_UNIT_TEST(ValidateScript) { + auto kikimr = DefaultKikimrRunner(); + auto db = kikimr.GetQueryClient(); + + auto settings = TExecuteScriptSettings().ExecMode(Ydb::Query::EXEC_MODE_VALIDATE); + auto scriptExecutionOperation = db.ExecuteScript(R"( + SELECT 42 + )", settings).ExtractValueSync(); + + // TODO: change when validate mode will be supported + UNIT_ASSERT_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::BAD_REQUEST, scriptExecutionOperation.Status().GetStatus()); + UNIT_ASSERT(scriptExecutionOperation.Status().GetIssues().Size() == 1); + UNIT_ASSERT_EQUAL_C(scriptExecutionOperation.Status().GetIssues().back().GetMessage(), "Query mode is not supported yet", scriptExecutionOperation.Status().GetIssues().ToString()); + } + + Y_UNIT_TEST(ExecuteScriptWithUnspecifiedMode) { + auto kikimr = DefaultKikimrRunner(); + auto db = kikimr.GetQueryClient(); + + auto settings = TExecuteScriptSettings().ExecMode(Ydb::Query::EXEC_MODE_UNSPECIFIED); + auto scriptExecutionOperation = db.ExecuteScript(R"( + SELECT 42 + )", settings).ExtractValueSync(); + + UNIT_ASSERT_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::BAD_REQUEST, scriptExecutionOperation.Status().GetStatus()); + UNIT_ASSERT(scriptExecutionOperation.Status().GetIssues().Size() == 1); + UNIT_ASSERT_EQUAL_C(scriptExecutionOperation.Status().GetIssues().back().GetMessage(), "Query mode is not specified", scriptExecutionOperation.Status().GetIssues().ToString()); + } + Y_UNIT_TEST(ListScriptExecutions) { auto kikimr = DefaultKikimrRunner(); auto db = kikimr.GetQueryClient(); diff --git a/ydb/public/sdk/cpp/client/draft/ydb_query/client.cpp b/ydb/public/sdk/cpp/client/draft/ydb_query/client.cpp index deb7d9e175..571ada740a 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_query/client.cpp +++ b/ydb/public/sdk/cpp/client/draft/ydb_query/client.cpp @@ -37,7 +37,7 @@ public: NThreading::TFuture<TScriptExecutionOperation> ExecuteScript(const TString& script, const TExecuteScriptSettings& settings) { using namespace Ydb::Query; auto request = MakeOperationRequest<ExecuteScriptRequest>(settings); - request.set_exec_mode(Ydb::Query::EXEC_MODE_EXECUTE); + request.set_exec_mode(settings.ExecMode_); request.mutable_script_content()->set_text(script); auto promise = NThreading::NewPromise<TScriptExecutionOperation>(); diff --git a/ydb/public/sdk/cpp/client/draft/ydb_query/query.cpp b/ydb/public/sdk/cpp/client/draft/ydb_query/query.cpp index 5cebf4989c..bb5d7e0e27 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_query/query.cpp +++ b/ydb/public/sdk/cpp/client/draft/ydb_query/query.cpp @@ -23,6 +23,7 @@ TScriptExecutionOperation::TScriptExecutionOperation(TStatus&& status, Ydb::Oper Metadata_.ExecutionId = metadata.execution_id(); Metadata_.ExecMode = static_cast<EExecMode>(metadata.exec_mode()); Metadata_.ExecStatus = static_cast<EExecStatus>(metadata.exec_status()); + Metadata_.ExecStats = metadata.exec_stats(); if (metadata.has_script_content()) { Metadata_.ScriptContent.Syntax = static_cast<ESyntax>(metadata.script_content().syntax()); diff --git a/ydb/public/sdk/cpp/client/draft/ydb_query/query.h b/ydb/public/sdk/cpp/client/draft/ydb_query/query.h index 790157bde2..51d57f8287 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_query/query.h +++ b/ydb/public/sdk/cpp/client/draft/ydb_query/query.h @@ -126,6 +126,7 @@ private: using TAsyncExecuteQueryResult = NThreading::TFuture<TExecuteQueryResult>; struct TExecuteScriptSettings : public TOperationRequestSettings<TExecuteScriptSettings> { + FLUENT_SETTING_DEFAULT(Ydb::Query::ExecMode, ExecMode, Ydb::Query::EXEC_MODE_EXECUTE); }; class TVersionedScriptId { @@ -175,6 +176,7 @@ public: // Not greater than one of SavedScriptId or QueryContent is set. std::optional<TVersionedScriptId> ScriptId; TQueryContent ScriptContent; + Ydb::TableStats::QueryStats ExecStats; }; using TOperation::TOperation; |