diff options
author | kelvich <kelvich@yandex-team.ru> | 2022-02-10 16:52:14 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:52:14 +0300 |
commit | 14c9296656b0377bde647d52a12c3aefa2fd58dd (patch) | |
tree | 97c811a3d83dce39c0d6a323c6f8bcc0710beca5 | |
parent | 986f48c33fde808e393e27d5dd0057658efbb660 (diff) | |
download | ydb-14c9296656b0377bde647d52a12c3aefa2fd58dd.tar.gz |
Restoring authorship annotation for <kelvich@yandex-team.ru>. Commit 1 of 2.
117 files changed, 4050 insertions, 4050 deletions
diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h index 44b25c4a5f9..91d9172b27c 100644 --- a/ydb/core/grpc_services/base/base.h +++ b/ydb/core/grpc_services/base/base.h @@ -135,7 +135,7 @@ struct TRpcServices { EvLongTxRollback, EvLongTxWrite, EvLongTxRead, - EvExplainYqlScript, + EvExplainYqlScript, EvImportData, EvAnalyticsReserved, EvDataStreamsCreateStream, diff --git a/ydb/core/grpc_services/rpc_commit_transaction.cpp b/ydb/core/grpc_services/rpc_commit_transaction.cpp index 3b85b5ef62d..c4bc057bc20 100644 --- a/ydb/core/grpc_services/rpc_commit_transaction.cpp +++ b/ydb/core/grpc_services/rpc_commit_transaction.cpp @@ -82,7 +82,7 @@ private: auto commitResult = TEvCommitTransactionRequest::AllocateResult<Ydb::Table::CommitTransactionResult>(Request_); if (kqpResponse.HasQueryStats()) { - FillQueryStats(*commitResult->mutable_query_stats(), kqpResponse); + FillQueryStats(*commitResult->mutable_query_stats(), kqpResponse); } ReplyWithResult(Ydb::StatusIds::SUCCESS, issueMessage, *commitResult, ctx); diff --git a/ydb/core/grpc_services/rpc_execute_data_query.cpp b/ydb/core/grpc_services/rpc_execute_data_query.cpp index 317a3488ce0..07ee2f550e8 100644 --- a/ydb/core/grpc_services/rpc_execute_data_query.cpp +++ b/ydb/core/grpc_services/rpc_execute_data_query.cpp @@ -171,8 +171,8 @@ public: static void ConvertQueryStats(const NKikimrKqp::TQueryResponse& from, Ydb::Table::ExecuteQueryResult* to) { if (from.HasQueryStats()) { - FillQueryStats(*to->mutable_query_stats(), from); - to->mutable_query_stats()->set_query_ast(from.GetQueryAst()); + FillQueryStats(*to->mutable_query_stats(), from); + to->mutable_query_stats()->set_query_ast(from.GetQueryAst()); return; } diff --git a/ydb/core/grpc_services/rpc_execute_yql_script.cpp b/ydb/core/grpc_services/rpc_execute_yql_script.cpp index 2650d32df50..9a1078e3163 100644 --- a/ydb/core/grpc_services/rpc_execute_yql_script.cpp +++ b/ydb/core/grpc_services/rpc_execute_yql_script.cpp @@ -100,9 +100,9 @@ public: ConvertKqpQueryResultsToDbResult(kqpResponse, queryResult); if (kqpResponse.HasQueryStats()) { - FillQueryStats(*queryResult->mutable_query_stats(), kqpResponse); - } else if (kqpResponse.HasQueryPlan()) { - queryResult->mutable_query_stats()->set_query_plan(kqpResponse.GetQueryPlan()); + FillQueryStats(*queryResult->mutable_query_stats(), kqpResponse); + } else if (kqpResponse.HasQueryPlan()) { + queryResult->mutable_query_stats()->set_query_plan(kqpResponse.GetQueryPlan()); } ReplyWithResult(Ydb::StatusIds::SUCCESS, issueMessage, *queryResult, ctx); diff --git a/ydb/core/grpc_services/rpc_explain_yql_script.cpp b/ydb/core/grpc_services/rpc_explain_yql_script.cpp index 1d4667feece..8c49ad9296f 100644 --- a/ydb/core/grpc_services/rpc_explain_yql_script.cpp +++ b/ydb/core/grpc_services/rpc_explain_yql_script.cpp @@ -1,112 +1,112 @@ #include "service_yql_scripting.h" -#include "rpc_kqp_base.h" -#include "rpc_common.h" - +#include "rpc_kqp_base.h" +#include "rpc_common.h" + #include <ydb/public/api/protos/ydb_scripting.pb.h> -namespace NKikimr { -namespace NGRpcService { - +namespace NKikimr { +namespace NGRpcService { + using TEvExplainYqlScriptRequest = TGrpcRequestOperationCall<Ydb::Scripting::ExplainYqlRequest, Ydb::Scripting::ExplainYqlResponse>; -using namespace Ydb; - -class TExplainYqlScriptRPC : public TRpcKqpRequestActor<TExplainYqlScriptRPC, TEvExplainYqlScriptRequest> { - using TBase = TRpcKqpRequestActor<TExplainYqlScriptRPC, TEvExplainYqlScriptRequest>; - -public: - using TResult = Ydb::Scripting::ExplainYqlResult; - +using namespace Ydb; + +class TExplainYqlScriptRPC : public TRpcKqpRequestActor<TExplainYqlScriptRPC, TEvExplainYqlScriptRequest> { + using TBase = TRpcKqpRequestActor<TExplainYqlScriptRPC, TEvExplainYqlScriptRequest>; + +public: + using TResult = Ydb::Scripting::ExplainYqlResult; + TExplainYqlScriptRPC(IRequestOpCtx* msg) - : TBase(msg) {} - - void Bootstrap(const TActorContext &ctx) { - TBase::Bootstrap(ctx); - - this->Become(&TExplainYqlScriptRPC::StateWork); - Proceed(ctx); - } - - void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { - switch (ev->GetTypeRewrite()) { - HFunc(NKqp::TEvKqp::TEvQueryResponse, Handle); - default: TBase::StateWork(ev, ctx); - } - } - - void Proceed(const TActorContext &ctx) { + : TBase(msg) {} + + void Bootstrap(const TActorContext &ctx) { + TBase::Bootstrap(ctx); + + this->Become(&TExplainYqlScriptRPC::StateWork); + Proceed(ctx); + } + + void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) { + switch (ev->GetTypeRewrite()) { + HFunc(NKqp::TEvKqp::TEvQueryResponse, Handle); + default: TBase::StateWork(ev, ctx); + } + } + + void Proceed(const TActorContext &ctx) { const auto req = GetProtoRequest(); - const auto traceId = Request_->GetTraceId(); - - auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); - SetAuthToken(ev, *Request_); - SetDatabase(ev, *Request_); - - if (traceId) { - ev->Record.SetTraceId(traceId.GetRef()); - } - - ev->Record.MutableRequest()->SetCancelAfterMs(GetCancelAfter().MilliSeconds()); - ev->Record.MutableRequest()->SetTimeoutMs(GetOperationTimeout().MilliSeconds()); - - auto& script = req->script(); - NYql::TIssues issues; - if (!CheckQuery(script, issues)) { - return Reply(Ydb::StatusIds::BAD_REQUEST, issues, ctx); - } - - switch (req->mode()) { + const auto traceId = Request_->GetTraceId(); + + auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>(); + SetAuthToken(ev, *Request_); + SetDatabase(ev, *Request_); + + if (traceId) { + ev->Record.SetTraceId(traceId.GetRef()); + } + + ev->Record.MutableRequest()->SetCancelAfterMs(GetCancelAfter().MilliSeconds()); + ev->Record.MutableRequest()->SetTimeoutMs(GetOperationTimeout().MilliSeconds()); + + auto& script = req->script(); + NYql::TIssues issues; + if (!CheckQuery(script, issues)) { + return Reply(Ydb::StatusIds::BAD_REQUEST, issues, ctx); + } + + switch (req->mode()) { // KIKIMR-10990 //case Ydb::Scripting::ExplainYqlRequest_Mode_PARSE: // ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_PARSE); // break; - case Ydb::Scripting::ExplainYqlRequest_Mode_VALIDATE: - ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_VALIDATE); - break; - case Ydb::Scripting::ExplainYqlRequest_Mode_PLAN: - ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN); - break; - default: - issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, TStringBuilder() - << "Unknown explain mode")); - return Reply(Ydb::StatusIds::BAD_REQUEST, issues, ctx); - } - - ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_SCRIPT); - ev->Record.MutableRequest()->SetQuery(script); - ev->Record.MutableRequest()->SetKeepSession(false); - - ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); - } - - void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record.GetRef(); + case Ydb::Scripting::ExplainYqlRequest_Mode_VALIDATE: + ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_VALIDATE); + break; + case Ydb::Scripting::ExplainYqlRequest_Mode_PLAN: + ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN); + break; + default: + issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, TStringBuilder() + << "Unknown explain mode")); + return Reply(Ydb::StatusIds::BAD_REQUEST, issues, ctx); + } + + ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_SCRIPT); + ev->Record.MutableRequest()->SetQuery(script); + ev->Record.MutableRequest()->SetKeepSession(false); + + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); + } + + void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { + const auto& record = ev->Get()->Record.GetRef(); AddServerHintsIfAny(record); - if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { - const auto& kqpResponse = record.GetResponse(); - const auto& issueMessage = kqpResponse.GetQueryIssues(); + if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { + const auto& kqpResponse = record.GetResponse(); + const auto& issueMessage = kqpResponse.GetQueryIssues(); const auto& queryParameters = kqpResponse.GetQueryParameters(); - - Ydb::Scripting::ExplainYqlResult queryResult; - queryResult.set_plan(kqpResponse.GetQueryPlan()); + + Ydb::Scripting::ExplainYqlResult queryResult; + queryResult.set_plan(kqpResponse.GetQueryPlan()); for (const auto& queryParameter : queryParameters) { Ydb::Type parameterType; ConvertMiniKQLTypeToYdbType(queryParameter.GetType(), parameterType); queryResult.mutable_parameters_types()->insert({ queryParameter.GetName(), parameterType }); } - - ReplyWithResult(Ydb::StatusIds::SUCCESS, issueMessage, queryResult, ctx); - } else { - return OnGenericQueryResponseError(record, ctx); - } - } -}; - + + ReplyWithResult(Ydb::StatusIds::SUCCESS, issueMessage, queryResult, ctx); + } else { + return OnGenericQueryResponseError(record, ctx); + } + } +}; + void DoExplainYqlScript(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) { TActivationContext::AsActorContext().Register(new TExplainYqlScriptRPC(p.release())); -} - -} // namespace NGRpcService -} // namespace NKikimr +} + +} // namespace NGRpcService +} // namespace NKikimr diff --git a/ydb/core/grpc_services/rpc_kqp_base.cpp b/ydb/core/grpc_services/rpc_kqp_base.cpp index 414543f5740..18a7fcc65bf 100644 --- a/ydb/core/grpc_services/rpc_kqp_base.cpp +++ b/ydb/core/grpc_services/rpc_kqp_base.cpp @@ -1,13 +1,13 @@ #include "rpc_kqp_base.h" #include <ydb/core/kqp/prepare/kqp_query_plan.h> - + namespace NKikimr { namespace NGRpcService { -void FillQueryStats(Ydb::TableStats::QueryStats& queryStats, const NKikimrKqp::TQueryResponse& kqpResponse) { - const auto& kqpStats = kqpResponse.GetQueryStats(); - +void FillQueryStats(Ydb::TableStats::QueryStats& queryStats, const NKikimrKqp::TQueryResponse& kqpResponse) { + const auto& kqpStats = kqpResponse.GetQueryStats(); + uint64_t totalCpuTimeUs = 0; uint64_t totalDurationUs = 0; @@ -65,8 +65,8 @@ void FillQueryStats(Ydb::TableStats::QueryStats& queryStats, const NKikimrKqp::T queryStats.set_process_cpu_time_us(kqpStats.GetWorkerCpuTimeUs()); queryStats.set_total_cpu_time_us(totalCpuTimeUs); queryStats.set_total_duration_us(totalDurationUs); - - queryStats.set_query_plan(kqpResponse.GetQueryPlan()); + + queryStats.set_query_plan(kqpResponse.GetQueryPlan()); } } // namespace NGRpcService diff --git a/ydb/core/grpc_services/rpc_kqp_base.h b/ydb/core/grpc_services/rpc_kqp_base.h index 2811a49e78a..acb1edfd931 100644 --- a/ydb/core/grpc_services/rpc_kqp_base.h +++ b/ydb/core/grpc_services/rpc_kqp_base.h @@ -45,7 +45,7 @@ inline NYql::NDqProto::EDqStatsMode GetKqpStatsMode(Ydb::Table::QueryStatsCollec switch (mode) { case Ydb::Table::QueryStatsCollection::STATS_COLLECTION_BASIC: return NYql::NDqProto::DQ_STATS_MODE_BASIC; - case Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL: + case Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL: return NYql::NDqProto::DQ_STATS_MODE_PROFILE; default: return NYql::NDqProto::DQ_STATS_MODE_NONE; @@ -70,7 +70,7 @@ inline bool CheckQuery(const TString& query, NYql::TIssues& issues) { return true; } -void FillQueryStats(Ydb::TableStats::QueryStats& queryStats, const NKikimrKqp::TQueryResponse& kqpResponse); +void FillQueryStats(Ydb::TableStats::QueryStats& queryStats, const NKikimrKqp::TQueryResponse& kqpResponse); inline void ConvertKqpQueryResultToDbResult(const NKikimrMiniKQL::TResult& from, Ydb::ResultSet* to) { const auto& type = from.GetType(); diff --git a/ydb/core/grpc_services/rpc_read_columns.cpp b/ydb/core/grpc_services/rpc_read_columns.cpp index 4baae606a97..bc57f76f887 100644 --- a/ydb/core/grpc_services/rpc_read_columns.cpp +++ b/ydb/core/grpc_services/rpc_read_columns.cpp @@ -429,10 +429,10 @@ private: } void Handle(NKqp::TEvKqpCompute::TEvScanError::TPtr& ev, const TActorContext& ctx) { - NYql::TIssues issues; - Ydb::StatusIds::StatusCode status = ev->Get()->Record.GetStatus(); - NYql::IssuesFromMessage(ev->Get()->Record.GetIssues(), issues); - + NYql::TIssues issues; + Ydb::StatusIds::StatusCode status = ev->Get()->Record.GetStatus(); + NYql::IssuesFromMessage(ev->Get()->Record.GetIssues(), issues); + ReplyWithError(status, issues, ctx); } diff --git a/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp b/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp index 5f95f263c6f..8daa2413094 100644 --- a/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp +++ b/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp @@ -139,12 +139,12 @@ bool FillKqpRequest(const Ydb::Experimental::ExecuteStreamQueryRequest& req, NKi return false; } - if (req.explain()) { - kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN); - } else { - kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); - } - + if (req.explain()) { + kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN); + } else { + kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); + } + kqpRequest.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_SCAN); kqpRequest.MutableRequest()->SetQuery(query); kqpRequest.MutableRequest()->SetKeepSession(false); @@ -174,21 +174,21 @@ bool FillKqpRequest(const Ydb::Table::ExecuteScanQueryRequest& req, NKikimrKqp:: return false; } - switch (req.mode()) { - case Ydb::Table::ExecuteScanQueryRequest::MODE_EXEC: - kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); - kqpRequest.MutableRequest()->SetStatsMode(GetKqpStatsMode(req.collect_stats())); - break; - case Ydb::Table::ExecuteScanQueryRequest::MODE_EXPLAIN: - kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN); - break; - default: { - NYql::TIssues issues; - issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Unexpected query mode")); - error = TParseRequestError(Ydb::StatusIds::BAD_REQUEST, issues); - return false; - } - } + switch (req.mode()) { + case Ydb::Table::ExecuteScanQueryRequest::MODE_EXEC: + kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); + kqpRequest.MutableRequest()->SetStatsMode(GetKqpStatsMode(req.collect_stats())); + break; + case Ydb::Table::ExecuteScanQueryRequest::MODE_EXPLAIN: + kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN); + break; + default: { + NYql::TIssues issues; + issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Unexpected query mode")); + error = TParseRequestError(Ydb::StatusIds::BAD_REQUEST, issues); + return false; + } + } kqpRequest.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_SCAN); kqpRequest.MutableRequest()->SetKeepSession(false); @@ -371,19 +371,19 @@ private: } void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { - auto& record = ev->Get()->Record.GetRef(); + auto& record = ev->Get()->Record.GetRef(); NYql::TIssues issues; - const auto& issueMessage = record.GetResponse().GetQueryIssues(); + const auto& issueMessage = record.GetResponse().GetQueryIssues(); NYql::IssuesFromMessage(issueMessage, issues); - - if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { - TResponse response; - TString out; - auto& kqpResponse = record.GetResponse(); - response.set_status(Ydb::StatusIds::SUCCESS); - - if constexpr (std::is_same_v<TResponse, Ydb::Table::ExecuteScanQueryPartialResponse>) { + + if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) { + TResponse response; + TString out; + auto& kqpResponse = record.GetResponse(); + response.set_status(Ydb::StatusIds::SUCCESS); + + if constexpr (std::is_same_v<TResponse, Ydb::Table::ExecuteScanQueryPartialResponse>) { bool reportStats = NeedReportStats(*Request_->GetProtoRequest()); bool reportPlan = reportStats && NeedReportPlan(*Request_->GetProtoRequest()); @@ -401,33 +401,33 @@ private: ExecutionProfiles_.clear(); } else if (reportPlan) { response.mutable_result()->mutable_query_stats()->set_query_plan(kqpResponse.GetQueryPlan()); - } - + } + if (reportPlan) { response.mutable_result()->mutable_query_stats()->set_query_ast(kqpResponse.GetQueryAst()); } Y_PROTOBUF_SUPPRESS_NODISCARD response.SerializeToString(&out); Request_->SendSerializedResult(std::move(out), record.GetYdbStatus()); - } - } else { - if (kqpResponse.HasQueryStats()) { - NKqpProto::TKqpStatsQuery queryStats; - for (const auto& execStats: ExecutionProfiles_) { - /* copy as ExecutionProfiles_ vector will be used and cleared later */ - queryStats.AddExecutions()->CopyFrom(*execStats); - } - response.mutable_result()->set_query_plan(SerializeAnalyzePlan(queryStats)); - } else { - response.mutable_result()->set_query_plan(kqpResponse.GetQueryPlan()); - } + } + } else { + if (kqpResponse.HasQueryStats()) { + NKqpProto::TKqpStatsQuery queryStats; + for (const auto& execStats: ExecutionProfiles_) { + /* copy as ExecutionProfiles_ vector will be used and cleared later */ + queryStats.AddExecutions()->CopyFrom(*execStats); + } + response.mutable_result()->set_query_plan(SerializeAnalyzePlan(queryStats)); + } else { + response.mutable_result()->set_query_plan(kqpResponse.GetQueryPlan()); + } Y_PROTOBUF_SUPPRESS_NODISCARD response.SerializeToString(&out); Request_->SendSerializedResult(std::move(out), record.GetYdbStatus()); - } - } - - ReplyFinishStream(record.GetYdbStatus(), issues, ctx); + } + } + + ReplyFinishStream(record.GetYdbStatus(), issues, ctx); } void Handle(NKqp::TEvKqp::TEvProcessResponse::TPtr& ev, const TActorContext& ctx) { diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make index 05b156bf043..cf0b96748ef 100644 --- a/ydb/core/grpc_services/ya.make +++ b/ydb/core/grpc_services/ya.make @@ -38,7 +38,7 @@ SRCS( rpc_execute_data_query.cpp rpc_execute_scheme_query.cpp rpc_execute_yql_script.cpp - rpc_explain_yql_script.cpp + rpc_explain_yql_script.cpp rpc_explain_data_query.cpp rpc_forget_operation.cpp rpc_get_operation.cpp diff --git a/ydb/core/kqp/common/kqp_gateway.h b/ydb/core/kqp/common/kqp_gateway.h index 1e06beca393..81e8eaef909 100644 --- a/ydb/core/kqp/common/kqp_gateway.h +++ b/ydb/core/kqp/common/kqp_gateway.h @@ -68,42 +68,42 @@ public: , Params(std::move(params)) {} }; - struct TKqpSnapshot { - ui64 Step; - ui64 TxId; - - constexpr TKqpSnapshot() - : Step(0) - , TxId(0) - {} - - TKqpSnapshot(ui64 step, ui64 txId) - : Step(step) - , TxId(txId) - {} - - bool IsValid() const { - return Step != 0 || TxId != 0; - } - - bool operator ==(const TKqpSnapshot &snapshot) const { - return snapshot.Step == Step && snapshot.TxId == TxId; - } - - size_t GetHash() const noexcept { - auto tuple = std::make_tuple(Step, TxId); - return THash<decltype(tuple)>()(tuple); - } - - static const TKqpSnapshot InvalidSnapshot; - }; - - struct TKqpSnapshotHandle : public IKqpGateway::TGenericResult { - TKqpSnapshot Snapshot; - NActors::TActorId ManagingActor; + struct TKqpSnapshot { + ui64 Step; + ui64 TxId; + + constexpr TKqpSnapshot() + : Step(0) + , TxId(0) + {} + + TKqpSnapshot(ui64 step, ui64 txId) + : Step(step) + , TxId(txId) + {} + + bool IsValid() const { + return Step != 0 || TxId != 0; + } + + bool operator ==(const TKqpSnapshot &snapshot) const { + return snapshot.Step == Step && snapshot.TxId == TxId; + } + + size_t GetHash() const noexcept { + auto tuple = std::make_tuple(Step, TxId); + return THash<decltype(tuple)>()(tuple); + } + + static const TKqpSnapshot InvalidSnapshot; + }; + + struct TKqpSnapshotHandle : public IKqpGateway::TGenericResult { + TKqpSnapshot Snapshot; + NActors::TActorId ManagingActor; NKikimrIssues::TStatusIds::EStatusCode Status = NKikimrIssues::TStatusIds::UNKNOWN; - }; - + }; + struct TExecPhysicalRequest : private TMoveOnly { TVector<TPhysicalTxData> Transactions; TVector<NYql::NDq::TMkqlValueRef> Locks; @@ -143,12 +143,12 @@ public: virtual NThreading::TFuture<TMkqlResult> PrepareMkql(const TString& cluster, const TString& program) = 0; - /* Snapshots */ - virtual NThreading::TFuture<TKqpSnapshotHandle> CreatePersistentSnapshot(const TVector<TString>& tablePaths, - TDuration queryTimeout) = 0; - - virtual void DiscardPersistentSnapshot(const TKqpSnapshotHandle& handle) = 0; - + /* Snapshots */ + virtual NThreading::TFuture<TKqpSnapshotHandle> CreatePersistentSnapshot(const TVector<TString>& tablePaths, + TDuration queryTimeout) = 0; + + virtual void DiscardPersistentSnapshot(const TKqpSnapshotHandle& handle) = 0; + virtual NThreading::TFuture<TKqpSnapshotHandle> AcquireMvccSnapshot(TDuration queryTimeout) = 0; /* Physical */ @@ -168,8 +168,8 @@ public: TKqpParamsMap&& params, const TAstQuerySettings& settings, const Ydb::Table::TransactionSettings& txSettings) = 0; - virtual NThreading::TFuture<TQueryResult> ExplainScanQueryAst(const TString& cluster, const TString& query) = 0; - + virtual NThreading::TFuture<TQueryResult> ExplainScanQueryAst(const TString& cluster, const TString& query) = 0; + virtual NThreading::TFuture<TQueryResult> ExecScanQueryAst(const TString& cluster, const TString& query, TKqpParamsMap&& params, const TAstQuerySettings& settings, ui64 rowsLimit) = 0; @@ -186,10 +186,10 @@ public: } // namespace NKqp } // namespace NKikimr - -template<> -struct THash<NKikimr::NKqp::IKqpGateway::TKqpSnapshot> { - inline size_t operator()(const NKikimr::NKqp::IKqpGateway::TKqpSnapshot& snapshot) const { - return snapshot.GetHash(); - } -}; + +template<> +struct THash<NKikimr::NKqp::IKqpGateway::TKqpSnapshot> { + inline size_t operator()(const NKikimr::NKqp::IKqpGateway::TKqpSnapshot& snapshot) const { + return snapshot.GetHash(); + } +}; diff --git a/ydb/core/kqp/common/kqp_transform.h b/ydb/core/kqp/common/kqp_transform.h index 457c4248011..53e8d415348 100644 --- a/ydb/core/kqp/common/kqp_transform.h +++ b/ydb/core/kqp/common/kqp_transform.h @@ -213,7 +213,7 @@ public: DeferredEffects.Clear(); ParamsState = MakeIntrusive<TParamsState>(); - SnapshotHandle.Snapshot = IKqpGateway::TKqpSnapshot::InvalidSnapshot; + SnapshotHandle.Snapshot = IKqpGateway::TKqpSnapshot::InvalidSnapshot; ForceNewEngineSettings = {}; } @@ -255,7 +255,7 @@ public: TDeferredEffects DeferredEffects; TIntrusivePtr<TParamsState> ParamsState; - IKqpGateway::TKqpSnapshotHandle SnapshotHandle; + IKqpGateway::TKqpSnapshotHandle SnapshotHandle; TKqpForceNewEngineState ForceNewEngineSettings; }; diff --git a/ydb/core/kqp/compile/kqp_mkql_compiler.cpp b/ydb/core/kqp/compile/kqp_mkql_compiler.cpp index 04130446986..e3ce04acc57 100644 --- a/ydb/core/kqp/compile/kqp_mkql_compiler.cpp +++ b/ydb/core/kqp/compile/kqp_mkql_compiler.cpp @@ -209,23 +209,23 @@ const TKikimrTableMetadata& TKqlCompileContext::GetTableMeta(const TKqpTable& ta TIntrusivePtr<IMkqlCallableCompiler> CreateKqlCompiler(const TKqlCompileContext& ctx) { auto compiler = MakeIntrusive<NCommon::TMkqlCommonCallableCompiler>(); - compiler->AddCallable(TKqpWideReadTable::CallableName(), - [&ctx](const TExprNode& node, TMkqlBuildContext& buildCtx) { - TKqpWideReadTable readTable(&node); - - const auto& tableMeta = ctx.GetTableMeta(readTable.Table()); + compiler->AddCallable(TKqpWideReadTable::CallableName(), + [&ctx](const TExprNode& node, TMkqlBuildContext& buildCtx) { + TKqpWideReadTable readTable(&node); + + const auto& tableMeta = ctx.GetTableMeta(readTable.Table()); auto keyRange = MakeKeyRange(readTable, ctx, buildCtx); - + auto result = ctx.PgmBuilder().KqpWideReadTable(MakeTableId(readTable.Table()), keyRange, GetKqpColumns(tableMeta, readTable.Columns(), true)); - + return result; }); - + compiler->AddCallable(TKqpWideReadTableRanges::CallableName(), [&ctx](const TExprNode& node, TMkqlBuildContext& buildCtx) { TKqpWideReadTableRanges readTableRanges(&node); - + const auto& tableMeta = ctx.GetTableMeta(readTableRanges.Table()); ValidateRangesType(readTableRanges.Ranges().Ref().GetTypeAnn(), tableMeta); @@ -247,12 +247,12 @@ TIntrusivePtr<IMkqlCallableCompiler> CreateKqlCompiler(const TKqlCompileContext& ValidateRangesType(readTable.Ranges().Ref().GetTypeAnn(), tableMeta); TKqpKeyRanges ranges = MakeComputedKeyRanges(readTable, ctx, buildCtx); - + // Return type depends on the process program, so it is built explicitly. TStringStream errorStream; auto returnType = NCommon::BuildType(*readTable.Ref().GetTypeAnn(), ctx.PgmBuilder(), errorStream); YQL_ENSURE(returnType, "Failed to build type: " << errorStream.Str()); - + // Process program for OLAP read is not present in MKQL, it is passed in range description // in physical plan directly to executer. Read callables in MKQL only used to associate // input stream of the graph with the external scans, so it doesn't make much sense to pass @@ -265,10 +265,10 @@ TIntrusivePtr<IMkqlCallableCompiler> CreateKqlCompiler(const TKqlCompileContext& GetKqpColumns(tableMeta, readTable.Columns(), true), returnType ); - - return result; - }); - + + return result; + }); + compiler->AddCallable(TKqpLookupTable::CallableName(), [&ctx](const TExprNode& node, TMkqlBuildContext& buildCtx) { TKqpLookupTable lookupTable(&node); diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp index f8e3cd0c772..ac3b3ec59d7 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp +++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp @@ -740,8 +740,8 @@ private: } void HandleExecute(TEvTxProxySchemeCache::TEvInvalidateTableResult::TPtr&) { - } - + } + void HandleExecute(TEvents::TEvUndelivered::TPtr& ev) { switch (ev->Get()->SourceType) { case TEvDataShard::TEvKqpScan::EventType: @@ -807,7 +807,7 @@ private: void SendStartScanRequest(TShardState& state, ui32 gen) { YQL_ENSURE(state.State == EShardState::Starting); - auto ev = MakeHolder<TEvDataShard::TEvKqpScan>(); + auto ev = MakeHolder<TEvDataShard::TEvKqpScan>(); ev->Record.SetLocalPathId(ScanData->TableId.PathId.LocalPathId); for (auto& column: ScanData->GetColumns()) { ev->Record.AddColumnTags(column.Tag); @@ -1012,7 +1012,7 @@ private: CA_LOG(prio, "Table: " << ScanData->TablePath << ", scan has not been started yet"); } } - + void PassAway() override { Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0)); for (ui32 nodeId : TrackingNodes) { diff --git a/ydb/core/kqp/counters/kqp_counters.cpp b/ydb/core/kqp/counters/kqp_counters.cpp index 26a28a0249a..cb120cd2de9 100644 --- a/ydb/core/kqp/counters/kqp_counters.cpp +++ b/ydb/core/kqp/counters/kqp_counters.cpp @@ -66,8 +66,8 @@ void TKqpCountersBase::Init() { KqpGroup->GetCounter("Requests/QueryCommitTx", true); QueryActionRequests[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX] = KqpGroup->GetCounter("Requests/QueryRollbackTx", true); - QueryActionRequests[NKikimrKqp::QUERY_ACTION_PARSE] = - KqpGroup->GetCounter("Requests/QueryParse", true); + QueryActionRequests[NKikimrKqp::QUERY_ACTION_PARSE] = + KqpGroup->GetCounter("Requests/QueryParse", true); OtherQueryRequests = KqpGroup->GetCounter("Requests/QueryOther", true); CloseSessionRequests = KqpGroup->GetCounter("Requests/CloseSession", true); @@ -84,26 +84,26 @@ void TKqpCountersBase::Init() { SqlV1Translations = KqpGroup->GetCounter("Requests/Sql/V1", true); SqlUnknownTranslations = KqpGroup->GetCounter("Requests/Sql/Unknown", true); - QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_UNDEFINED] = - KqpGroup->GetCounter("Request/QueryTypeUndefined", true); - QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DML] = - KqpGroup->GetCounter("Request/QueryTypeDml", true); - QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DDL] = - KqpGroup->GetCounter("Request/QueryTypeDdl", true); - QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_PREPARED_DML] = - KqpGroup->GetCounter("Request/QueryTypePreparedDml", true); - QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_DML] = - KqpGroup->GetCounter("Request/QueryTypeAstDml", true); - QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT] = - KqpGroup->GetCounter("Request/QueryTypeSqlScript", true); + QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_UNDEFINED] = + KqpGroup->GetCounter("Request/QueryTypeUndefined", true); + QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DML] = + KqpGroup->GetCounter("Request/QueryTypeDml", true); + QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DDL] = + KqpGroup->GetCounter("Request/QueryTypeDdl", true); + QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_PREPARED_DML] = + KqpGroup->GetCounter("Request/QueryTypePreparedDml", true); + QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_DML] = + KqpGroup->GetCounter("Request/QueryTypeAstDml", true); + QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT] = + KqpGroup->GetCounter("Request/QueryTypeSqlScript", true); QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT_STREAMING] = KqpGroup->GetCounter("Request/QueryTypeSqlScriptStreaming", true); - QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN] = - KqpGroup->GetCounter("Request/QueryTypeSqlScan", true); - QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_SCAN] = - KqpGroup->GetCounter("Request/QueryTypeAstScan", true); - OtherQueryTypes = KqpGroup->GetCounter("Requests/QueryTypeOther", true); - + QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN] = + KqpGroup->GetCounter("Request/QueryTypeSqlScan", true); + QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_SCAN] = + KqpGroup->GetCounter("Request/QueryTypeAstScan", true); + OtherQueryTypes = KqpGroup->GetCounter("Requests/QueryTypeOther", true); + QueriesWithRangeScan = KqpGroup->GetCounter("Query/WithRangeScan", true); QueriesWithFullScan = KqpGroup->GetCounter("Query/WithFullScan", true); @@ -242,15 +242,15 @@ void TKqpCountersBase::ReportQueryAction(NKikimrKqp::EQueryAction action) { } void TKqpCountersBase::ReportQueryType(NKikimrKqp::EQueryType type) { - auto counter = QueryTypes.FindPtr(type); - if (counter) { - (*counter)->Inc(); - return; - } - - OtherQueryTypes->Inc(); -} - + auto counter = QueryTypes.FindPtr(type); + if (counter) { + (*counter)->Inc(); + return; + } + + OtherQueryTypes->Inc(); +} + void TKqpCountersBase::ReportSessionShutdownRequest() { SessionBalancerShutdowns->Inc(); } diff --git a/ydb/core/kqp/counters/kqp_counters.h b/ydb/core/kqp/counters/kqp_counters.h index 53a8f7d2c23..4f9047adfeb 100644 --- a/ydb/core/kqp/counters/kqp_counters.h +++ b/ydb/core/kqp/counters/kqp_counters.h @@ -35,7 +35,7 @@ protected: void Init(); void ReportQueryAction(NKikimrKqp::EQueryAction action); - void ReportQueryType(NKikimrKqp::EQueryType type); + void ReportQueryType(NKikimrKqp::EQueryType type); void ReportSessionShutdownRequest(); void ReportCreateSession(ui64 requestSize); @@ -116,9 +116,9 @@ protected: NMonitoring::TDynamicCounters::TCounterPtr SqlV1Translations; NMonitoring::TDynamicCounters::TCounterPtr SqlUnknownTranslations; - THashMap<NKikimrKqp::EQueryType, NMonitoring::TDynamicCounters::TCounterPtr> QueryTypes; - NMonitoring::TDynamicCounters::TCounterPtr OtherQueryTypes; - + THashMap<NKikimrKqp::EQueryType, NMonitoring::TDynamicCounters::TCounterPtr> QueryTypes; + NMonitoring::TDynamicCounters::TCounterPtr OtherQueryTypes; + NMonitoring::TDynamicCounters::TCounterPtr QueriesWithRangeScan; NMonitoring::TDynamicCounters::TCounterPtr QueriesWithFullScan; NMonitoring::THistogramPtr QueryAffectedShardsCount; diff --git a/ydb/core/kqp/executer/kqp_executer_stats.cpp b/ydb/core/kqp/executer/kqp_executer_stats.cpp index 8c3605fd918..c40af5abb34 100644 --- a/ydb/core/kqp/executer/kqp_executer_stats.cpp +++ b/ydb/core/kqp/executer/kqp_executer_stats.cpp @@ -168,7 +168,7 @@ void TQueryExecutionStats::AddDatashardStats(NYql::NDqProto::TDqComputeActorStat tableAggr->SetWriteRows(tableAggr->GetWriteRows() + table.GetWriteRows()); tableAggr->SetWriteBytes(tableAggr->GetWriteBytes() + table.GetWriteBytes()); tableAggr->SetEraseRows(tableAggr->GetEraseRows() + table.GetEraseRows()); - + auto& shards = TableShards[table.GetTablePath()]; for (const auto& perShard : txStats.GetPerShardStats()) { shards.insert(perShard.GetShardId()); diff --git a/ydb/core/kqp/executer/kqp_planner.cpp b/ydb/core/kqp/executer/kqp_planner.cpp index 6bcfa85c66f..a5bdce84632 100644 --- a/ydb/core/kqp/executer/kqp_planner.cpp +++ b/ydb/core/kqp/executer/kqp_planner.cpp @@ -1,12 +1,12 @@ #include "kqp_planner.h" #include "kqp_planner_strategy.h" -#include "kqp_shards_resolver.h" +#include "kqp_shards_resolver.h" #include <ydb/core/base/appdata.h> #include <ydb/core/kqp/rm/kqp_rm.h> #include <ydb/core/kqp/rm/kqp_resource_estimation.h> -#include <util/generic/set.h> +#include <util/generic/set.h> namespace NKikimr::NKqp { @@ -28,8 +28,8 @@ TKqpPlanner::TKqpPlanner(ui64 txId, const TActorId& executer, TVector<NDqProto:: : TxId(txId) , ExecuterId(executer) , Tasks(std::move(tasks)) - , ScanTasks(std::move(scanTasks)) - , Snapshot(snapshot) + , ScanTasks(std::move(scanTasks)) + , Snapshot(snapshot) , Database(database) , UserToken(userToken) , Deadline(deadline) @@ -161,12 +161,12 @@ void TKqpPlanner::Process(const TVector<NKikimrKqp::TKqpNodeResources>& snapshot TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(), CalcSendMessageFlagsForNode(target.NodeId()))); } - - TVector<ui64> nodes; + + TVector<ui64> nodes; nodes.reserve(ScanTasks.size()); for (auto& [nodeId, _]: ScanTasks) { nodes.push_back(nodeId); - } + } for (ui64 nodeId: nodes) { auto ev = PrepareKqpNodeRequest({}); @@ -175,7 +175,7 @@ void TKqpPlanner::Process(const TVector<NKikimrKqp::TKqpNodeResources>& snapshot auto target = MakeKqpNodeServiceID(nodeId); TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(), CalcSendMessageFlagsForNode(target.NodeId()))); - } + } Y_VERIFY(ScanTasks.empty()); } else { auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::PRECONDITION_FAILED, @@ -198,12 +198,12 @@ void TKqpPlanner::RunLocal(const TVector<NKikimrKqp::TKqpNodeResources>& snapsho auto target = MakeKqpNodeServiceID(SelfId().NodeId()); TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(), IEventHandle::FlagTrackDelivery)); - - TVector<ui64> nodes; - for (const auto& pair: ScanTasks) { - nodes.push_back(pair.first); + + TVector<ui64> nodes; + for (const auto& pair: ScanTasks) { + nodes.push_back(pair.first); YQL_ENSURE(pair.first != SelfId().NodeId()); - } + } THashMap<ui64, size_t> nodeIdToIdx; for (size_t idx = 0; idx < snapshot.size(); ++idx) { @@ -211,15 +211,15 @@ void TKqpPlanner::RunLocal(const TVector<NKikimrKqp::TKqpNodeResources>& snapsho LOG_D("snapshot #" << idx << ": " << snapshot[idx].ShortDebugString()); } - for (auto nodeId: nodes) { + for (auto nodeId: nodes) { auto ev = PrepareKqpNodeRequest({}); AddScansToKqpNodeRequest(ev, nodeId); auto target = MakeKqpNodeServiceID(nodeId); TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(), CalcSendMessageFlagsForNode(target.NodeId()))); - } - Y_VERIFY(ScanTasks.size() == 0); - + } + Y_VERIFY(ScanTasks.size() == 0); + PassAway(); } @@ -321,7 +321,7 @@ IActor* CreateKqpPlanner(ui64 txId, const TActorId& executer, TVector<NDqProto:: const TString& database, const TMaybe<TString>& token, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode, bool disableLlvmForUdfStages, bool enableLlvm, bool withSpilling, const TMaybe<NKikimrKqp::TRlPath>& rlPath) { - return new TKqpPlanner(txId, executer, std::move(tasks), std::move(scanTasks), snapshot, + return new TKqpPlanner(txId, executer, std::move(tasks), std::move(scanTasks), snapshot, database, token, deadline, statsMode, disableLlvmForUdfStages, enableLlvm, withSpilling, rlPath); } diff --git a/ydb/core/kqp/executer/kqp_planner.h b/ydb/core/kqp/executer/kqp_planner.h index dc5e993a74d..418cdf28ba0 100644 --- a/ydb/core/kqp/executer/kqp_planner.h +++ b/ydb/core/kqp/executer/kqp_planner.h @@ -31,8 +31,8 @@ class TKqpPlanner : public TActorBootstrapped<TKqpPlanner> { }; public: - TKqpPlanner(ui64 txId, const TActorId& executer, TVector<NYql::NDqProto::TDqTask>&& tasks, - THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot, + TKqpPlanner(ui64 txId, const TActorId& executer, TVector<NYql::NDqProto::TDqTask>&& tasks, + THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot, const TString& database, const TMaybe<TString>& userToken, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode, bool disableLlvmForUdfStages, bool enableLlvm, bool withSpilling, const TMaybe<NKikimrKqp::TRlPath>& rlPath); @@ -58,8 +58,8 @@ private: const ui64 TxId; const TActorId ExecuterId; TVector<NYql::NDqProto::TDqTask> Tasks; - THashMap<ui64, TVector<NYql::NDqProto::TDqTask>> ScanTasks; - const IKqpGateway::TKqpSnapshot Snapshot; + THashMap<ui64, TVector<NYql::NDqProto::TDqTask>> ScanTasks; + const IKqpGateway::TKqpSnapshot Snapshot; TString Database; const TMaybe<TString> UserToken; const TInstant Deadline; @@ -72,7 +72,7 @@ private: }; IActor* CreateKqpPlanner(ui64 txId, const TActorId& executer, TVector<NYql::NDqProto::TDqTask>&& tasks, - THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot, + THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot, const TString& database, const TMaybe<TString>& userToken, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode, bool disableLlvmForUdfStages, bool enableLlvm, bool withSpilling, const TMaybe<NKikimrKqp::TRlPath>& rlPath); diff --git a/ydb/core/kqp/executer/kqp_result_channel.cpp b/ydb/core/kqp/executer/kqp_result_channel.cpp index 0fe235323e3..90416bfd22d 100644 --- a/ydb/core/kqp/executer/kqp_result_channel.cpp +++ b/ydb/core/kqp/executer/kqp_result_channel.cpp @@ -74,7 +74,7 @@ private: LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ", send ack to channelId: " << ChannelId << ", seqNo: " << seqNo - << ", enough: " << ev->Get()->Record.GetEnough() + << ", enough: " << ev->Get()->Record.GetEnough() << ", freeSpace: " << freeSpace << ", to: " << ComputeActor); @@ -82,7 +82,7 @@ private: ackEv->Record.SetSeqNo(seqNo); ackEv->Record.SetChannelId(ChannelId); ackEv->Record.SetFreeSpace(freeSpace); - ackEv->Record.SetFinish(ev->Get()->Record.GetEnough()); + ackEv->Record.SetFinish(ev->Get()->Record.GetEnough()); Send(ComputeActor, ackEv.Release(), /* TODO: undelivery */ 0, /* cookie */ ChannelId); } diff --git a/ydb/core/kqp/executer/kqp_scan_executer.cpp b/ydb/core/kqp/executer/kqp_scan_executer.cpp index 07c29f64e21..d1f62a5d6d9 100644 --- a/ydb/core/kqp/executer/kqp_scan_executer.cpp +++ b/ydb/core/kqp/executer/kqp_scan_executer.cpp @@ -255,7 +255,7 @@ private: void HandleExecute(TEvKqpExecuter::TEvStreamDataAck::TPtr& ev) { LOG_T("Recv stream data ack, seqNo: " << ev->Get()->Record.GetSeqNo() << ", freeSpace: " << ev->Get()->Record.GetFreeSpace() - << ", enough: " << ev->Get()->Record.GetEnough() + << ", enough: " << ev->Get()->Record.GetEnough() << ", from: " << ev->Sender); if (ResultChannelProxies.empty()) { @@ -401,43 +401,43 @@ private: } }; - void BuildDatashardScanTasks(TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, - const NMiniKQL::TTypeEnvironment& typeEnv) - { + void BuildDatashardScanTasks(TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, + const NMiniKQL::TTypeEnvironment& typeEnv) + { Y_VERIFY_DEBUG(stageInfo.Meta.IsDatashard()); THashMap<ui64, TVector<ui64>> nodeTasks; // nodeId -> [taskId] - - auto getNodeTask = [&](ui64 nodeId, ui32 taskIdx) -> TTask& { - auto& tasks = nodeTasks[nodeId]; - - if (taskIdx < tasks.size()) { - return TasksGraph.GetTask(tasks[taskIdx]); - } else { - Y_ASSERT(taskIdx == tasks.size()); - auto& task = TasksGraph.AddTask(stageInfo); - task.Meta.NodeId = nodeId; - tasks.emplace_back(task.Id); - return task; - } - }; - - auto& stage = GetStage(stageInfo); - - const auto& table = TableKeys.GetTable(stageInfo.Meta.TableId); + + auto getNodeTask = [&](ui64 nodeId, ui32 taskIdx) -> TTask& { + auto& tasks = nodeTasks[nodeId]; + + if (taskIdx < tasks.size()) { + return TasksGraph.GetTask(tasks[taskIdx]); + } else { + Y_ASSERT(taskIdx == tasks.size()); + auto& task = TasksGraph.AddTask(stageInfo); + task.Meta.NodeId = nodeId; + tasks.emplace_back(task.Id); + return task; + } + }; + + auto& stage = GetStage(stageInfo); + + const auto& table = TableKeys.GetTable(stageInfo.Meta.TableId); const auto& keyTypes = table.KeyColumnTypes; - - for (auto& op : stage.GetTableOps()) { - Y_VERIFY_DEBUG(stageInfo.Meta.TablePath == op.GetTable().GetPath()); - + + for (auto& op : stage.GetTableOps()) { + Y_VERIFY_DEBUG(stageInfo.Meta.TablePath == op.GetTable().GetPath()); + auto columns = BuildKqpColumns(op, table); THashMap<ui64, TShardInfo> partitions; - - switch (op.GetTypeCase()) { + + switch (op.GetTypeCase()) { case NKqpProto::TKqpPhyTableOperation::kReadRanges: partitions = PrunePartitions(TableKeys, op.GetReadRanges(), stageInfo, holderFactory, typeEnv); break; - case NKqpProto::TKqpPhyTableOperation::kReadRange: + case NKqpProto::TKqpPhyTableOperation::kReadRange: partitions = PrunePartitions(TableKeys, op.GetReadRange(), stageInfo, holderFactory, typeEnv); break; case NKqpProto::TKqpPhyTableOperation::kLookup: @@ -447,7 +447,7 @@ private: YQL_ENSURE(false, "Unexpected table scan operation: " << (ui32) op.GetTypeCase()); break; } - + bool reverse = false; ui64 itemsLimit = 0; TString itemsLimitParamName; @@ -465,20 +465,20 @@ private: stageInfo.Meta.SkipNullKeys.assign(op.GetReadRange().GetSkipNullKeys().begin(), op.GetReadRange().GetSkipNullKeys().end()); } - + // TODO: take into account number of active scans on node bool heavyProgram = stage.GetProgram().GetSettings().GetHasSort() || stage.GetProgram().GetSettings().GetHasMapJoin(); const ui32 maxScansPerNode = heavyProgram ? 4 : 16; THashMap<ui64, ui32> nTasksOnNodes; // nodeId -> tasks count - + for (auto& [shardId, shardInfo] : partitions) { YQL_ENSURE(!shardInfo.KeyWriteRanges); - + ui64 nodeId = ShardIdToNodeId.at(shardId); ui32 nTasksOnNode = nTasksOnNodes[nodeId]++; auto& task = getNodeTask(nodeId, nTasksOnNode % maxScansPerNode); - + for (auto& [name, value] : shardInfo.Params) { auto ret = task.Meta.Params.emplace(name, std::move(value)); YQL_ENSURE(ret.second); @@ -487,65 +487,65 @@ private: auto retType = task.Meta.ParamTypes.emplace(name, typeIterator->second); YQL_ENSURE(retType.second); } - + TTaskMeta::TShardReadInfo readInfo = { .Ranges = std::move(*shardInfo.KeyReadRanges), // sorted & non-intersecting .Columns = columns, .ShardId = shardId, }; - + if (itemsLimit && !task.Meta.Params.contains(itemsLimitParamName)) { task.Meta.Params.emplace(itemsLimitParamName, itemsLimitBytes); task.Meta.ParamTypes.emplace(itemsLimitParamName, itemsLimitType); - } - + } + FillReadInfo(task.Meta, itemsLimit, reverse, TMaybe<::NKqpProto::TKqpPhyOpReadOlapRanges>()); if (!task.Meta.Reads) { task.Meta.Reads.ConstructInPlace(); - } + } task.Meta.Reads->emplace_back(std::move(readInfo)); - } - } - - LOG_D("Stage " << stageInfo.Id << " will be executed on " << nodeTasks.size() << " nodes."); - - for (const auto& pair : nodeTasks) { - for (const auto& taskIdx : pair.second) { - auto& task = TasksGraph.GetTask(taskIdx); - YQL_ENSURE(task.Meta.Reads.Defined()); - auto& taskReads = task.Meta.Reads.GetRef(); - - /* - * Sort read ranges so that sequential scan of that ranges produce sorted result. - * - * Partition pruner feed us with set of non-intersecting ranges with filled right boundary. + } + } + + LOG_D("Stage " << stageInfo.Id << " will be executed on " << nodeTasks.size() << " nodes."); + + for (const auto& pair : nodeTasks) { + for (const auto& taskIdx : pair.second) { + auto& task = TasksGraph.GetTask(taskIdx); + YQL_ENSURE(task.Meta.Reads.Defined()); + auto& taskReads = task.Meta.Reads.GetRef(); + + /* + * Sort read ranges so that sequential scan of that ranges produce sorted result. + * + * Partition pruner feed us with set of non-intersecting ranges with filled right boundary. * So we may sort ranges based solely on the their rightmost point. - */ - std::sort(taskReads.begin(), taskReads.end(), [&](const auto& lhs, const auto& rhs){ + */ + std::sort(taskReads.begin(), taskReads.end(), [&](const auto& lhs, const auto& rhs){ YQL_ENSURE(lhs.ShardId != rhs.ShardId); - + const std::pair<const TSerializedCellVec*, bool> k1 = lhs.Ranges.GetRightBorder(); const std::pair<const TSerializedCellVec*, bool> k2 = lhs.Ranges.GetRightBorder(); - const int cmp = CompareBorders<false, false>( + const int cmp = CompareBorders<false, false>( k1.first->GetCells(), k2.first->GetCells(), k1.second, k2.second, - keyTypes); - - return (cmp < 0); - }); - - LOG_D("Stage " << stageInfo.Id << " create datashard scan task: " << taskIdx - << ", node: " << pair.first - << ", meta: " << task.Meta.ToString(keyTypes, *AppData()->TypeRegistry)); - } - } - } - + keyTypes); + + return (cmp < 0); + }); + + LOG_D("Stage " << stageInfo.Id << " create datashard scan task: " << taskIdx + << ", node: " << pair.first + << ", meta: " << task.Meta.ToString(keyTypes, *AppData()->TypeRegistry)); + } + } + } + // Returns the list of ColumnShards that can store rows from the specified range // NOTE: Unlike OLTP tables that store data in DataShards, data in OLAP tables is not range // partitioned and multiple ColumnShards store data from the same key range @@ -755,10 +755,10 @@ private: return; } - // NodeId -> {Tasks} - THashMap<ui64, TVector<NYql::NDqProto::TDqTask>> scanTasks; - ui32 nShardScans = 0; - ui32 nScanTasks = 0; + // NodeId -> {Tasks} + THashMap<ui64, TVector<NYql::NDqProto::TDqTask>> scanTasks; + ui32 nShardScans = 0; + ui32 nScanTasks = 0; TVector<NYql::NDqProto::TDqTask> computeTasks; @@ -873,10 +873,10 @@ private: if (stageInfo.Meta.IsSysView()) { computeTasks.emplace_back(std::move(taskDesc)); - } else { + } else { scanTasks[task.Meta.NodeId].emplace_back(std::move(taskDesc)); nScanTasks++; - } + } } else { computeTasks.emplace_back(std::move(taskDesc)); } @@ -900,25 +900,25 @@ private: << ", " << nScanTasks << " scan tasks on " << scanTasks.size() << " nodes" << ", totalShardScans: " << nShardScans << ", execType: Scan" << ", snapshot: {" << Request.Snapshot.TxId << ", " << Request.Snapshot.Step << "}"); - + ExecuteScanTx(std::move(computeTasks), std::move(scanTasks)); - + Become(&TKqpScanExecuter::ExecuteState); - } - - void ExecuteScanTx(TVector<NYql::NDqProto::TDqTask>&& computeTasks, THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks) { + } + + void ExecuteScanTx(TVector<NYql::NDqProto::TDqTask>&& computeTasks, THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks) { LOG_D("Execute scan tx, computeTasks: " << computeTasks.size() << ", scanTasks: " << scanTasks.size()); - for (const auto& [_, tasks]: scanTasks) { - for (const auto& task : tasks) { - PendingComputeTasks.insert(task.GetId()); + for (const auto& [_, tasks]: scanTasks) { + for (const auto& task : tasks) { + PendingComputeTasks.insert(task.GetId()); } } - for (auto& taskDesc : computeTasks) { - PendingComputeTasks.insert(taskDesc.GetId()); - } - - auto planner = CreateKqpPlanner(TxId, SelfId(), std::move(computeTasks), + for (auto& taskDesc : computeTasks) { + PendingComputeTasks.insert(taskDesc.GetId()); + } + + auto planner = CreateKqpPlanner(TxId, SelfId(), std::move(computeTasks), std::move(scanTasks), Request.Snapshot, Database, UserToken, Deadline.GetOrElse(TInstant::Zero()), Request.StatsMode, Request.DisableLlvmForUdfStages, Request.LlvmEnabled, AppData()->EnableKqpSpilling, Request.RlPath); @@ -950,12 +950,12 @@ private: Stats->FinishTs = TInstant::Now(); Stats->Finish(); - + if (Request.StatsMode == NYql::NDqProto::DQ_STATS_MODE_PROFILE) { const auto& tx = Request.Transactions[0].Body; auto planWithStats = AddExecStatsToTxPlan(tx.GetPlan(), response.GetResult().GetStats()); response.MutableResult()->MutableStats()->AddTxPlansWithStats(planWithStats); - } + } } LOG_D("Sending response to: " << Target); diff --git a/ydb/core/kqp/executer/kqp_shards_resolver.cpp b/ydb/core/kqp/executer/kqp_shards_resolver.cpp index 506dfcf5730..644f7628c1b 100644 --- a/ydb/core/kqp/executer/kqp_shards_resolver.cpp +++ b/ydb/core/kqp/executer/kqp_shards_resolver.cpp @@ -46,8 +46,8 @@ public: auto tabletResolver = MakeTabletResolverID(); auto resolveFlags = GetResolveFlags(); - Y_ASSERT(ShardIds.size() > 0); - + Y_ASSERT(ShardIds.size() > 0); + for (ui64 tabletId : ShardIds) { LOG_T("Send request about tabletId: " << tabletId); bool sent = Send(tabletResolver, new TEvTabletResolver::TEvForward(tabletId, nullptr, resolveFlags)); diff --git a/ydb/core/kqp/executer/kqp_tasks_graph.h b/ydb/core/kqp/executer/kqp_tasks_graph.h index 7c356ab623f..ebe7647c1e9 100644 --- a/ydb/core/kqp/executer/kqp_tasks_graph.h +++ b/ydb/core/kqp/executer/kqp_tasks_graph.h @@ -95,11 +95,11 @@ struct TShardKeyRanges { std::pair<const TSerializedCellVec*, bool> GetRightBorder() const; }; -// TODO: use two different structs for scans and data queries +// TODO: use two different structs for scans and data queries struct TTaskMeta { ui64 ShardId = 0; // only in case of non-scans (data-query & legacy scans) ui64 NodeId = 0; // only in case of scans over persistent snapshots - + TMap<TString, NYql::NDqProto::TData> Params; TMap<TString, NKikimr::NMiniKQL::TType*> ParamTypes; diff --git a/ydb/core/kqp/executer/ya.make b/ydb/core/kqp/executer/ya.make index 3db082d5d0b..e1c4e8d12bf 100644 --- a/ydb/core/kqp/executer/ya.make +++ b/ydb/core/kqp/executer/ya.make @@ -15,7 +15,7 @@ SRCS( kqp_partition_helper.cpp kqp_planner.cpp kqp_planner_strategy.cpp - kqp_shards_resolver.cpp + kqp_shards_resolver.cpp kqp_result_channel.cpp kqp_table_resolver.cpp kqp_tasks_graph.cpp diff --git a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json index 2faa33d5e1d..33030cc3003 100644 --- a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json +++ b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json @@ -71,11 +71,11 @@ "Match": {"Type": "Callable", "Name": "KqpReadTable"} }, { - "Name": "TKqpWideReadTable", - "Base": "TKqlReadTableBase", - "Match": {"Type": "Callable", "Name": "KqpWideReadTable"} - }, - { + "Name": "TKqpWideReadTable", + "Base": "TKqlReadTableBase", + "Match": {"Type": "Callable", "Name": "KqpWideReadTable"} + }, + { "Name": "TKqlReadTableRangesBase", "Base": "TCallable", "Match": {"Type": "CallableBase"}, diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp index 44af6d11682..bf24f0831b2 100644 --- a/ydb/core/kqp/host/kqp_host.cpp +++ b/ydb/core/kqp/host/kqp_host.cpp @@ -118,7 +118,7 @@ void FillAstAndPlan(IKqpHost::TQueryResult& queryResult, const NKikimrKqp::TPrep } } -/* +/* * Validate YqlScript. */ class TAsyncValidateYqlResult : public TKqpAsyncResultBase<IKqpHost::TQueryResult> { @@ -145,49 +145,49 @@ private: }; /* - * Explain Yql/YqlScript. - */ + * Explain Yql/YqlScript. + */ class TAsyncExplainYqlResult : public TKqpAsyncResultBase<IKqpHost::TQueryResult> { public: using TResult = IKqpHost::TQueryResult; - TAsyncExplainYqlResult(TExprNode* queryRoot, TIntrusivePtr<TKikimrSessionContext> sessionCtx, - TExprContext& exprCtx, TAutoPtr<IGraphTransformer> transformer, + TAsyncExplainYqlResult(TExprNode* queryRoot, TIntrusivePtr<TKikimrSessionContext> sessionCtx, + TExprContext& exprCtx, TAutoPtr<IGraphTransformer> transformer, IPlanBuilder& planBuilder, TMaybe<TSqlVersion> sqlVersion, bool useDqExplain = false) : TKqpAsyncResultBase(queryRoot, exprCtx, *transformer.Get()) - , SessionCtx(sessionCtx) + , SessionCtx(sessionCtx) , Transformer(transformer) , PlanBuilder(planBuilder) - , SqlVersion(sqlVersion) - , UseDqExplain(useDqExplain) {} + , SqlVersion(sqlVersion) + , UseDqExplain(useDqExplain) {} void FillResult(TResult& queryResult) const override { - if (UseDqExplain) { - TVector<const TString> plans; - for (auto id : SessionCtx->Query().ExecutionOrder) { - auto result = SessionCtx->Query().Results.FindPtr(id); - if (result) { - plans.push_back(result->QueryPlan); - } - } - queryResult.QueryPlan = SerializeScriptPlan(plans); - } else { - FillAstAndPlan(queryResult, GetExprRoot().Get(), GetExprContext(), PlanBuilder); - } + if (UseDqExplain) { + TVector<const TString> plans; + for (auto id : SessionCtx->Query().ExecutionOrder) { + auto result = SessionCtx->Query().Results.FindPtr(id); + if (result) { + plans.push_back(result->QueryPlan); + } + } + queryResult.QueryPlan = SerializeScriptPlan(plans); + } else { + FillAstAndPlan(queryResult, GetExprRoot().Get(), GetExprContext(), PlanBuilder); + } queryResult.SqlVersion = SqlVersion; } private: - TIntrusivePtr<TKikimrSessionContext> SessionCtx; + TIntrusivePtr<TKikimrSessionContext> SessionCtx; TAutoPtr<IGraphTransformer> Transformer; IPlanBuilder& PlanBuilder; TMaybe<TSqlVersion> SqlVersion; - bool UseDqExplain; + bool UseDqExplain; }; -/* - * Execute Yql/SchemeQuery/YqlScript. - */ +/* + * Execute Yql/SchemeQuery/YqlScript. + */ class TAsyncExecuteYqlResult : public TKqpAsyncExecuteResultBase<IKqpHost::TQueryResult> { public: using TResult = IKqpHost::TQueryResult; @@ -215,18 +215,18 @@ public: } } - TVector<const TString> queryPlans; + TVector<const TString> queryPlans; for (auto id : SessionCtx->Query().ExecutionOrder) { auto result = SessionCtx->Query().Results.FindPtr(id); if (result) { - queryPlans.push_back(SerializeAnalyzePlan(result->QueryStats)); + queryPlans.push_back(SerializeAnalyzePlan(result->QueryStats)); AddQueryStats(queryResult.QueryStats, std::move(result->QueryStats)); } } FillAstAndPlan(queryResult, GetExprRoot().Get(), GetExprContext(), PlanBuilder); queryResult.SqlVersion = SqlVersion; - queryResult.QueryPlan = SerializeScriptPlan(queryPlans); + queryResult.QueryPlan = SerializeScriptPlan(queryPlans); } private: @@ -237,9 +237,9 @@ private: TMaybe<TSqlVersion> SqlVersion; }; -/* - * Execute prepared Yql/ScanQuery/DataQuery. - */ +/* + * Execute prepared Yql/ScanQuery/DataQuery. + */ class TAsyncExecutePreparedResult : public TKqpAsyncExecuteResultBase<IKqpHost::TQueryResult> { public: using TResult = IKqpHost::TQueryResult; @@ -283,14 +283,14 @@ public: queryResult.PreparedQuery = Query; } - /* - * Set stats and plan for DataQuery. In case of ScanQuery they will be set - * later in TStreamExecuteScanQueryRPC. - */ + /* + * Set stats and plan for DataQuery. In case of ScanQuery they will be set + * later in TStreamExecuteScanQueryRPC. + */ if (ExecuteCtx->QueryResults.size() == 1) { auto& execResult = ExecuteCtx->QueryResults[0]; queryResult.QueryStats.Swap(&execResult.QueryStats); - queryResult.QueryPlan = SerializeAnalyzePlan(queryResult.QueryStats); + queryResult.QueryPlan = SerializeAnalyzePlan(queryResult.QueryStats); } queryResult.SqlVersion = SqlVersion; @@ -304,9 +304,9 @@ private: TMaybe<TSqlVersion> SqlVersion; }; -/* +/* * Prepare ScanQuery/DataQuery by AST (when called through scripting). - */ + */ class TAsyncExecuteKqlResult : public TKqpAsyncExecuteResultBase<IKqpHost::TQueryResult> { public: using TResult = IKqpHost::TQueryResult; @@ -328,9 +328,9 @@ private: TExecuteContext& ExecuteCtx; }; -/* - * Prepare ScanQuery/DataQuery. - */ +/* + * Prepare ScanQuery/DataQuery. + */ class TAsyncPrepareYqlResult : public TKqpAsyncResultBase<IKqpHost::TQueryResult> { public: using TResult = IKqpHost::TQueryResult; @@ -349,7 +349,7 @@ public: prepareResult.PreparingQuery = std::move(QueryCtx->PreparingQuery); prepareResult.PreparingQuery->SetText(std::move(QueryText)); prepareResult.SqlVersion = SqlVersion; - + if (prepareResult.PreparingQuery->GetVersion() == NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1) { prepareResult.QueryPlan = SerializeExplainPlan(prepareResult.PreparingQuery->GetPhysicalQuery()); prepareResult.QueryAst = prepareResult.PreparingQuery->GetPhysicalQuery().GetQueryAst(); @@ -818,9 +818,9 @@ public: return KqpRunner->PrepareDataQuery(cluster, query, ctx, settings); case EKikimrQueryType::Scan: return KqpRunner->PrepareScanQuery(cluster, query, ctx, settings); - case EKikimrQueryType::YqlScript: + case EKikimrQueryType::YqlScript: case EKikimrQueryType::YqlScriptStreaming: - break; + break; default: YQL_ENSURE(false, "Unexpected query type for prepare action: " << queryType); return nullptr; @@ -856,7 +856,7 @@ public: future = Gateway->ExecScanQueryAst(Cluster, queryAstStr, CollectParameters(query), querySettings, rowsLimit); } - } else { + } else { Ydb::Table::TransactionSettings txSettings; txSettings.mutable_serializable_read_write(); if (SessionCtx->Query().PrepareOnly) { @@ -865,19 +865,19 @@ public: future = Gateway->ExecDataQueryAst(Cluster, queryAstStr, CollectParameters(query), querySettings, txSettings); } - } + } break; case EKikimrQueryType::YqlScriptStreaming: if (useScanQuery) { future = Gateway->StreamExecScanQueryAst(Cluster, queryAstStr, CollectParameters(query), querySettings, SessionCtx->Query().ReplyTarget); - } else { + } else { Ydb::Table::TransactionSettings txSettings; txSettings.mutable_serializable_read_write(); future = Gateway->StreamExecDataQueryAst(Cluster, queryAstStr, CollectParameters(query), querySettings, txSettings, SessionCtx->Query().ReplyTarget); - } + } break; default: @@ -1250,13 +1250,13 @@ public: }); } - IAsyncQueryResultPtr ExplainScanQuery(const TString& query, bool isSql) override { - return CheckedProcessQuery(*ExprCtx, - [this, &query, isSql] (TExprContext& ctx) { - return ExplainScanQueryInternal(query, isSql, ctx); - }); - } - + IAsyncQueryResultPtr ExplainScanQuery(const TString& query, bool isSql) override { + return CheckedProcessQuery(*ExprCtx, + [this, &query, isSql] (TExprContext& ctx) { + return ExplainScanQueryInternal(query, isSql, ctx); + }); + } + TQueryResult SyncExplainDataQuery(const TString& query, bool isSql) override { return CheckedSyncProcessQuery( [this, &query, isSql] () { @@ -1742,21 +1742,21 @@ private: if (prepared.PreparingQuery->GetVersion() == NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1) { explainResult.QueryPlan = std::move(prepared.QueryPlan); explainResult.QueryAst = std::move(*prepared.PreparingQuery->MutablePhysicalQuery()->MutableQueryAst()); - } else { + } else { FillAstAndPlan(explainResult, *prepared.PreparingQuery); - } + } explainResult.SqlVersion = prepared.SqlVersion; return MakeKikimrResultHolder(std::move(explainResult)); }); } - IAsyncQueryResultPtr ExplainScanQueryInternal(const TString& query, bool isSql, TExprContext& ctx) { - auto prepareResult = isSql - ? PrepareScanQueryInternal(query, ctx) - : PrepareScanQueryAstInternal(query, ctx); + IAsyncQueryResultPtr ExplainScanQueryInternal(const TString& query, bool isSql, TExprContext& ctx) { + auto prepareResult = isSql + ? PrepareScanQueryInternal(query, ctx) + : PrepareScanQueryAstInternal(query, ctx); return prepareResult; - } - + } + IAsyncQueryResultPtr PrepareDataQueryInternal(const TString& query, const TPrepareSettings& settings, TExprContext& ctx) { @@ -1812,12 +1812,12 @@ private: SessionCtx, *ExecuteCtx); } - IAsyncQueryResultPtr PrepareScanQueryInternal(const TString& query, TExprContext& ctx, EKikimrStatsMode statsMode = EKikimrStatsMode::None) { + IAsyncQueryResultPtr PrepareScanQueryInternal(const TString& query, TExprContext& ctx, EKikimrStatsMode statsMode = EKikimrStatsMode::None) { SetupYqlTransformer(nullptr); SessionCtx->Query().Type = EKikimrQueryType::Scan; SessionCtx->Query().PrepareOnly = true; - SessionCtx->Query().StatsMode = statsMode; + SessionCtx->Query().StatsMode = statsMode; SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>(); TMaybe<TSqlVersion> sqlVersion = 1; @@ -2045,21 +2045,21 @@ private: } IAsyncQueryResultPtr ExplainYqlScriptInternal(const TString& script, TExprContext& ctx) { - SetupYqlTransformer(nullptr); - - SessionCtx->Query().Type = EKikimrQueryType::YqlScript; - SessionCtx->Query().PrepareOnly = true; + SetupYqlTransformer(nullptr); + + SessionCtx->Query().Type = EKikimrQueryType::YqlScript; + SessionCtx->Query().PrepareOnly = true; SessionCtx->Query().SuppressDdlChecks = true; SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>(); - - TMaybe<TSqlVersion> sqlVersion; - auto scriptExpr = CompileYqlQuery(script, true, true, ctx, sqlVersion); - if (!scriptExpr) { - return nullptr; - } - - return MakeIntrusive<TAsyncExplainYqlResult>(scriptExpr.Get(), SessionCtx, ctx, YqlTransformer, - *PlanBuilder, sqlVersion, true /* UseDqExplain */); + + TMaybe<TSqlVersion> sqlVersion; + auto scriptExpr = CompileYqlQuery(script, true, true, ctx, sqlVersion); + if (!scriptExpr) { + return nullptr; + } + + return MakeIntrusive<TAsyncExplainYqlResult>(scriptExpr.Get(), SessionCtx, ctx, YqlTransformer, + *PlanBuilder, sqlVersion, true /* UseDqExplain */); } IAsyncQueryResultPtr ExecuteScanQueryInternal(const TString& query, bool isSql, @@ -2067,7 +2067,7 @@ private: const IKikimrQueryExecutor::TExecuteSettings& settings, TExprContext& ctx) { auto prepareResult = isSql - ? PrepareScanQueryInternal(query, ctx, settings.StatsMode) + ? PrepareScanQueryInternal(query, ctx, settings.StatsMode) : PrepareScanQueryAstInternal(query, ctx); if (!prepareResult) { diff --git a/ydb/core/kqp/host/kqp_host.h b/ydb/core/kqp/host/kqp_host.h index 11282d0ebe1..381031e7b4c 100644 --- a/ydb/core/kqp/host/kqp_host.h +++ b/ydb/core/kqp/host/kqp_host.h @@ -98,8 +98,8 @@ public: NKikimrMiniKQL::TParams&& parameters, const NActors::TActorId& target, const NYql::IKikimrQueryExecutor::TExecuteSettings& settings) = 0; - virtual IAsyncQueryResultPtr ExplainScanQuery(const TString& query, bool isSql) = 0; - + virtual IAsyncQueryResultPtr ExplainScanQuery(const TString& query, bool isSql) = 0; + /* Scripting */ virtual IAsyncQueryResultPtr ValidateYqlScript(const TString& script) = 0; virtual TQueryResult SyncValidateYqlScript(const TString& script) = 0; diff --git a/ydb/core/kqp/host/kqp_host_impl.h b/ydb/core/kqp/host/kqp_host_impl.h index fb1fb139cd5..1999cbdb7c8 100644 --- a/ydb/core/kqp/host/kqp_host_impl.h +++ b/ydb/core/kqp/host/kqp_host_impl.h @@ -292,11 +292,11 @@ TAutoPtr<NYql::IGraphTransformer> CreateKqpExecuteScanTransformer(TIntrusivePtr< const TString& cluster, TIntrusivePtr<TKqpTransactionState> txState, TIntrusivePtr<TKqlTransformContext> transformCtx); -TAutoPtr<NYql::IGraphTransformer> CreateKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, +TAutoPtr<NYql::IGraphTransformer> CreateKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, TIntrusivePtr<TKqlTransformContext> transformCtx, TIntrusivePtr<TKqpTransactionState> txState); - -TAutoPtr<NYql::IGraphTransformer> CreateKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, - TIntrusivePtr<TKqpTransactionState> txState); - + +TAutoPtr<NYql::IGraphTransformer> CreateKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, + TIntrusivePtr<TKqpTransactionState> txState); + } // namespace NKqp } // namespace NKikimr diff --git a/ydb/core/kqp/host/kqp_run_scan.cpp b/ydb/core/kqp/host/kqp_run_scan.cpp index 7c91ae82003..df88f621605 100644 --- a/ydb/core/kqp/host/kqp_run_scan.cpp +++ b/ydb/core/kqp/host/kqp_run_scan.cpp @@ -65,60 +65,60 @@ protected: } }; -class TKqpCreateSnapshotTransformer : public TGraphTransformerBase { -public: - TKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, TIntrusivePtr<TKqlTransformContext> transformCtx, +class TKqpCreateSnapshotTransformer : public TGraphTransformerBase { +public: + TKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, TIntrusivePtr<TKqlTransformContext> transformCtx, TIntrusivePtr<TKqpTransactionState> txState) - : Gateway(gateway) - , TransformCtx(transformCtx) - , TxState(txState) - {} - - TStatus DoTransform(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext&) { - output = input; - - THashSet<TString> tablesSet; - for (const auto& phyTx: TransformCtx->PhysicalQuery->GetTransactions()) { - for (const auto& stage: phyTx.GetStages()) { - for (const auto& tableOp: stage.GetTableOps()) { - tablesSet.insert(tableOp.GetTable().GetPath()); - } - } - } - TVector<TString> tables(tablesSet.begin(), tablesSet.end()); - - auto timeout = TransformCtx->QueryCtx->Deadlines.TimeoutAt - Gateway->GetCurrentTime(); - if (!timeout) { - // TODO: Just cancel request. - timeout = TDuration::MilliSeconds(1); - } - SnapshotFuture = Gateway->CreatePersistentSnapshot(tables, timeout); - - Promise = NewPromise(); - auto promise = Promise; - SnapshotFuture.Apply([promise](const TFuture<IKqpGateway::TKqpSnapshotHandle> future) mutable { - YQL_ENSURE(future.HasValue()); - promise.SetValue(); - }); - - return TStatus::Async; - } - - NThreading::TFuture<void> DoGetAsyncFuture(const NYql::TExprNode& ) { - return Promise.GetFuture(); - } - - TStatus DoApplyAsyncChanges(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext& ctx) { - output = input; - - auto handle = SnapshotFuture.ExtractValue(); - - if (!handle.Snapshot.IsValid()) { + : Gateway(gateway) + , TransformCtx(transformCtx) + , TxState(txState) + {} + + TStatus DoTransform(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext&) { + output = input; + + THashSet<TString> tablesSet; + for (const auto& phyTx: TransformCtx->PhysicalQuery->GetTransactions()) { + for (const auto& stage: phyTx.GetStages()) { + for (const auto& tableOp: stage.GetTableOps()) { + tablesSet.insert(tableOp.GetTable().GetPath()); + } + } + } + TVector<TString> tables(tablesSet.begin(), tablesSet.end()); + + auto timeout = TransformCtx->QueryCtx->Deadlines.TimeoutAt - Gateway->GetCurrentTime(); + if (!timeout) { + // TODO: Just cancel request. + timeout = TDuration::MilliSeconds(1); + } + SnapshotFuture = Gateway->CreatePersistentSnapshot(tables, timeout); + + Promise = NewPromise(); + auto promise = Promise; + SnapshotFuture.Apply([promise](const TFuture<IKqpGateway::TKqpSnapshotHandle> future) mutable { + YQL_ENSURE(future.HasValue()); + promise.SetValue(); + }); + + return TStatus::Async; + } + + NThreading::TFuture<void> DoGetAsyncFuture(const NYql::TExprNode& ) { + return Promise.GetFuture(); + } + + TStatus DoApplyAsyncChanges(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext& ctx) { + output = input; + + auto handle = SnapshotFuture.ExtractValue(); + + if (!handle.Snapshot.IsValid()) { YQL_CLOG(NOTICE, ProviderKqp) << "Failed to create persistent snapshot. " << "Status: " << NKikimrIssues::TStatusIds_EStatusCode_Name(handle.Status) << ", issues: " << handle.Issues().ToString(); - TIssue issue("Failed to create persistent snapshot"); + TIssue issue("Failed to create persistent snapshot"); switch (handle.Status) { case NKikimrIssues::TStatusIds::SCHEME_ERROR: issue.SetCode(NYql::TIssuesIds::KIKIMR_SCHEME_ERROR, NYql::TSeverityIds::S_ERROR); @@ -136,48 +136,48 @@ public: } for (const auto& subIssue: handle.Issues()) { - issue.AddSubIssue(MakeIntrusive<TIssue>(subIssue)); - } - ctx.AddError(issue); - return TStatus::Error; - } - - TxState->Tx().SnapshotHandle = handle; - return TStatus::Ok; - } - -private: - TIntrusivePtr<IKqpGateway> Gateway; - TIntrusivePtr<TKqlTransformContext> TransformCtx; - - NThreading::TFuture<IKqpGateway::TKqpSnapshotHandle> SnapshotFuture; - NThreading::TPromise<void> Promise; - TIntrusivePtr<TKqpTransactionState> TxState; -}; - -class TKqpReleaseSnapshotTransformer : public TSyncTransformerBase { -public: - TKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, TIntrusivePtr<TKqpTransactionState> txState) - : Gateway(gateway) - , TxState(txState) - {} - - TStatus DoTransform(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext&) { - output = input; - + issue.AddSubIssue(MakeIntrusive<TIssue>(subIssue)); + } + ctx.AddError(issue); + return TStatus::Error; + } + + TxState->Tx().SnapshotHandle = handle; + return TStatus::Ok; + } + +private: + TIntrusivePtr<IKqpGateway> Gateway; + TIntrusivePtr<TKqlTransformContext> TransformCtx; + + NThreading::TFuture<IKqpGateway::TKqpSnapshotHandle> SnapshotFuture; + NThreading::TPromise<void> Promise; + TIntrusivePtr<TKqpTransactionState> TxState; +}; + +class TKqpReleaseSnapshotTransformer : public TSyncTransformerBase { +public: + TKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, TIntrusivePtr<TKqpTransactionState> txState) + : Gateway(gateway) + , TxState(txState) + {} + + TStatus DoTransform(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext&) { + output = input; + if (TxState->Tx().GetSnapshot().IsValid()) { - Gateway->DiscardPersistentSnapshot(TxState->Tx().SnapshotHandle); - } - - return TStatus::Ok; - } - -private: - TIntrusivePtr<IKqpGateway> Gateway; - TIntrusivePtr<TKqpTransactionState> TxState; -}; - - + Gateway->DiscardPersistentSnapshot(TxState->Tx().SnapshotHandle); + } + + return TStatus::Ok; + } + +private: + TIntrusivePtr<IKqpGateway> Gateway; + TIntrusivePtr<TKqpTransactionState> TxState; +}; + + TAutoPtr<IGraphTransformer> CreateKqpExecuteScanTransformer(TIntrusivePtr<IKqpGateway> gateway, const TString& cluster, TIntrusivePtr<TKqpTransactionState> txState, TIntrusivePtr<TKqlTransformContext> transformCtx) @@ -185,17 +185,17 @@ TAutoPtr<IGraphTransformer> CreateKqpExecuteScanTransformer(TIntrusivePtr<IKqpGa return new TKqpExecuteScanTransformer(gateway, cluster, txState, transformCtx); } -TAutoPtr<IGraphTransformer> CreateKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, +TAutoPtr<IGraphTransformer> CreateKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, TIntrusivePtr<TKqlTransformContext> transformCtx, TIntrusivePtr<TKqpTransactionState> txState) -{ +{ return new TKqpCreateSnapshotTransformer(gateway, transformCtx, txState); -} - -TAutoPtr<IGraphTransformer> CreateKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, - TIntrusivePtr<TKqpTransactionState> txState) -{ - return new TKqpReleaseSnapshotTransformer(gateway, txState); -} - +} + +TAutoPtr<IGraphTransformer> CreateKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, + TIntrusivePtr<TKqpTransactionState> txState) +{ + return new TKqpReleaseSnapshotTransformer(gateway, txState); +} + } // namespace NKqp } // namespace NKikimr diff --git a/ydb/core/kqp/host/kqp_runner.cpp b/ydb/core/kqp/host/kqp_runner.cpp index 42b4eca7c66..6e49eafe151 100644 --- a/ydb/core/kqp/host/kqp_runner.cpp +++ b/ydb/core/kqp/host/kqp_runner.cpp @@ -245,7 +245,7 @@ public: ScanRunQueryTransformer = TTransformationPipeline(typesCtx) .Add(CreateKqpCreateSnapshotTransformer(Gateway, TransformCtx, TxState), "CreateSnapshot") .Add(CreateKqpExecuteScanTransformer(Gateway, Cluster, TxState, TransformCtx), "ExecuteScan") - .Add(CreateKqpReleaseSnapshotTransformer(Gateway, TxState), "ReleaseSnapshot") + .Add(CreateKqpReleaseSnapshotTransformer(Gateway, TxState), "ReleaseSnapshot") .Build(false); } @@ -553,15 +553,15 @@ private: auto& preparedQuery = *TransformCtx->QueryCtx->PreparingQuery; TKqpPhysicalQuery physicalQuery(transformedQuery); - auto compiler = CreateKqpQueryCompiler(Cluster, OptimizeCtx->Tables, FuncRegistry); + auto compiler = CreateKqpQueryCompiler(Cluster, OptimizeCtx->Tables, FuncRegistry); auto ret = compiler->CompilePhysicalQuery(physicalQuery, dataQuery.Operations(), *preparedQuery.MutablePhysicalQuery(), ctx); - if (!ret) { - ctx.AddError(TIssue(ctx.GetPosition(query->Pos()), "Failed to compile physical query.")); + if (!ret) { + ctx.AddError(TIssue(ctx.GetPosition(query->Pos()), "Failed to compile physical query.")); return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues())); } - preparedQuery.SetVersion(NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1); - // TODO(sk): only on stats mode or if explain-only + preparedQuery.SetVersion(NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1); + // TODO(sk): only on stats mode or if explain-only PreparedExplainTransformer->Rewind(); return MakeIntrusive<TPhysicalAsyncRunResult>(builtQuery, ctx, *PreparedExplainTransformer, *TransformCtx); } @@ -599,7 +599,7 @@ private: TransformCtx->ReplyTarget = target; Y_ASSERT(!TxState->Tx().GetSnapshot().IsValid()); - + return MakeIntrusive<TScanAsyncRunResult>(world, ctx, *ScanRunQueryTransformer); } diff --git a/ydb/core/kqp/kqp_compile_actor.cpp b/ydb/core/kqp/kqp_compile_actor.cpp index 27cf296a03d..20ef8b5e6cd 100644 --- a/ydb/core/kqp/kqp_compile_actor.cpp +++ b/ydb/core/kqp/kqp_compile_actor.cpp @@ -94,8 +94,8 @@ public: ctx.SelfID.NodeId(), counters, MakeMiniKQLCompileServiceID()); Gateway->SetToken(Query.Cluster, UserToken); - Config->FeatureFlags = AppData(ctx)->FeatureFlags; - + Config->FeatureFlags = AppData(ctx)->FeatureFlags; + KqpHost = CreateKqpHost(Gateway, Query.Cluster, Query.Database, Config, ModuleResolverState->ModuleResolver, AppData(ctx)->FunctionRegistry, false); diff --git a/ydb/core/kqp/kqp_ic_gateway.cpp b/ydb/core/kqp/kqp_ic_gateway.cpp index 47489a9bd54..14626ec9d3f 100644 --- a/ydb/core/kqp/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/kqp_ic_gateway.cpp @@ -32,7 +32,7 @@ #include <library/cpp/actors/core/hfunc.h> #include <util/string/split.h> -#include <util/string/vector.h> +#include <util/string/vector.h> namespace NKikimr { namespace NKqp { @@ -46,8 +46,8 @@ using namespace NKikimrSchemeOp; using NKikimrTxUserProxy::TMiniKQLTransaction; -constexpr const IKqpGateway::TKqpSnapshot IKqpGateway::TKqpSnapshot::InvalidSnapshot = TKqpSnapshot(); - +constexpr const IKqpGateway::TKqpSnapshot IKqpGateway::TKqpSnapshot::InvalidSnapshot = TKqpSnapshot(); + #define STATIC_ASSERT_STATE_EQUAL(name) \ static_assert(static_cast<ui32>(NYql::TIndexDescription::EIndexState::name) \ == NKikimrSchemeOp::EIndexState::EIndexState##name, \ @@ -208,7 +208,7 @@ public: } auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>(); - resp->Record.SetEnough(truncated); + resp->Record.SetEnough(truncated); resp->Record.SetSeqNo(ev->Get()->Record.GetSeqNo()); resp->Record.SetFreeSpace(ResultSetBytesLimit); ctx.Send(ev->Sender, resp.Release()); @@ -1742,32 +1742,32 @@ public: }); } - TFuture<TQueryResult> ExplainScanQueryAst(const TString& cluster, const TString& query) override - { - using TRequest = NKqp::TEvKqp::TEvQueryRequest; - using TResponse = NKqp::TEvKqp::TEvQueryResponse; - - auto ev = MakeHolder<TRequest>(); - if (UserToken) { - ev->Record.SetUserToken(UserToken->Serialized); - } - - ev->Record.MutableRequest()->SetDatabase(Database); - ev->Record.MutableRequest()->SetCluster(cluster); - ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN); - ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_AST_SCAN); - ev->Record.MutableRequest()->SetQuery(query); - ev->Record.MutableRequest()->SetKeepSession(false); - - return SendKqpScanQueryRequest(ev.Release(), 100, - [] (TPromise<TQueryResult> promise, TResponse&& responseEv) { - TQueryResult queryResult; - queryResult.ProtobufArenaPtr.reset(new google::protobuf::Arena()); - KqpResponseToQueryResult(responseEv.Record.GetRef(), queryResult); + TFuture<TQueryResult> ExplainScanQueryAst(const TString& cluster, const TString& query) override + { + using TRequest = NKqp::TEvKqp::TEvQueryRequest; + using TResponse = NKqp::TEvKqp::TEvQueryResponse; + + auto ev = MakeHolder<TRequest>(); + if (UserToken) { + ev->Record.SetUserToken(UserToken->Serialized); + } + + ev->Record.MutableRequest()->SetDatabase(Database); + ev->Record.MutableRequest()->SetCluster(cluster); + ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN); + ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_AST_SCAN); + ev->Record.MutableRequest()->SetQuery(query); + ev->Record.MutableRequest()->SetKeepSession(false); + + return SendKqpScanQueryRequest(ev.Release(), 100, + [] (TPromise<TQueryResult> promise, TResponse&& responseEv) { + TQueryResult queryResult; + queryResult.ProtobufArenaPtr.reset(new google::protobuf::Arena()); + KqpResponseToQueryResult(responseEv.Record.GetRef(), queryResult); promise.SetValue(std::move(queryResult)); - }); - } - + }); + } + TFuture<TQueryResult> ExecDataQueryAst(const TString& cluster, const TString& query, TKqpParamsMap&& params, const TAstQuerySettings& settings, const Ydb::Table::TransactionSettings& txSettings) override { @@ -1850,32 +1850,32 @@ public: return ExecutePhysicalQueryInternal(std::move(request), target, true); } - TFuture<TKqpSnapshotHandle> CreatePersistentSnapshot(const TVector<TString>& tablePaths, TDuration queryTimeout) override { + TFuture<TKqpSnapshotHandle> CreatePersistentSnapshot(const TVector<TString>& tablePaths, TDuration queryTimeout) override { auto* snapMgr = CreateKqpSnapshotManager(Database, queryTimeout); - auto snapMgrActorId = RegisterActor(snapMgr); - + auto snapMgrActorId = RegisterActor(snapMgr); + auto ev = MakeHolder<TEvKqpSnapshot::TEvCreateSnapshotRequest>(tablePaths); - - return SendActorRequest< + + return SendActorRequest< TEvKqpSnapshot::TEvCreateSnapshotRequest, TEvKqpSnapshot::TEvCreateSnapshotResponse, - IKqpGateway::TKqpSnapshotHandle> - ( - snapMgrActorId, - ev.Release(), - [snapMgrActorId](TPromise<IKqpGateway::TKqpSnapshotHandle> promise, + IKqpGateway::TKqpSnapshotHandle> + ( + snapMgrActorId, + ev.Release(), + [snapMgrActorId](TPromise<IKqpGateway::TKqpSnapshotHandle> promise, TEvKqpSnapshot::TEvCreateSnapshotResponse&& response) mutable - { - IKqpGateway::TKqpSnapshotHandle handle; - handle.Snapshot = response.Snapshot; - handle.ManagingActor = snapMgrActorId; + { + IKqpGateway::TKqpSnapshotHandle handle; + handle.Snapshot = response.Snapshot; + handle.ManagingActor = snapMgrActorId; handle.Status = response.Status; handle.AddIssues(response.Issues); - promise.SetValue(handle); - } - ); - } - + promise.SetValue(handle); + } + ); + } + NThreading::TFuture<TKqpSnapshotHandle> AcquireMvccSnapshot(TDuration queryTimeout) override { auto* snapMgr = CreateKqpSnapshotManager(Database, queryTimeout); auto snapMgrActorId = RegisterActor(snapMgr); @@ -1901,11 +1901,11 @@ public: ); } - void DiscardPersistentSnapshot(const TKqpSnapshotHandle& handle) override { + void DiscardPersistentSnapshot(const TKqpSnapshotHandle& handle) override { if (handle.ManagingActor) ActorSystem->Send(handle.ManagingActor, new TEvKqpSnapshot::TEvDiscardSnapshot(handle.Snapshot)); - } - + } + TInstant GetCurrentTime() const override { return TAppData::TimeProvider->Now(); } diff --git a/ydb/core/kqp/kqp_worker_actor.cpp b/ydb/core/kqp/kqp_worker_actor.cpp index cccb8a51e8e..5cd0f533dae 100644 --- a/ydb/core/kqp/kqp_worker_actor.cpp +++ b/ydb/core/kqp/kqp_worker_actor.cpp @@ -185,8 +185,8 @@ public: Gateway = CreateKikimrIcGateway(Settings.Cluster, Settings.Database, std::move(loader), ctx.ExecutorThread.ActorSystem, ctx.SelfID.NodeId(), RequestCounters, MakeMiniKQLCompileServiceID()); - Config->FeatureFlags = AppData(ctx)->FeatureFlags; - + Config->FeatureFlags = AppData(ctx)->FeatureFlags; + KqpHost = CreateKqpHost(Gateway, Settings.Cluster, Settings.Database, Config, ModuleResolverState->ModuleResolver, AppData(ctx)->FunctionRegistry, !Settings.LongSession); @@ -284,9 +284,9 @@ public: QueryState->RequestActorId = ActorIdFromProto(event.GetRequestActorId()); if (GetStatsMode(queryRequest, EKikimrStatsMode::None) > EKikimrStatsMode::Basic) { - QueryState->ReplyFlags |= NKikimrKqp::QUERY_REPLY_FLAG_AST; - } - + QueryState->ReplyFlags |= NKikimrKqp::QUERY_REPLY_FLAG_AST; + } + if (event.HasRlPath()) { QueryState->RlPath = event.GetRlPath(); } @@ -342,7 +342,7 @@ public: switch (queryRequest.GetAction()) { case NKikimrKqp::QUERY_ACTION_EXPLAIN: - case NKikimrKqp::QUERY_ACTION_PARSE: + case NKikimrKqp::QUERY_ACTION_PARSE: case NKikimrKqp::QUERY_ACTION_VALIDATE: case NKikimrKqp::QUERY_ACTION_PREPARE: if (QueryState->KeepSession && !Settings.LongSession) { @@ -773,7 +773,7 @@ private: switch (queryRequest.GetAction()) { case NKikimrKqp::QUERY_ACTION_EXECUTE: case NKikimrKqp::QUERY_ACTION_EXPLAIN: - case NKikimrKqp::QUERY_ACTION_PARSE: + case NKikimrKqp::QUERY_ACTION_PARSE: case NKikimrKqp::QUERY_ACTION_VALIDATE: case NKikimrKqp::QUERY_ACTION_PREPARE: case NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED: @@ -1352,19 +1352,19 @@ private: break; } - case NKikimrKqp::QUERY_TYPE_SQL_SCAN: - case NKikimrKqp::QUERY_TYPE_AST_SCAN: { - bool isSql = (type == NKikimrKqp::QUERY_TYPE_SQL_SCAN); - QueryState->AsyncQueryResult = KqpHost->ExplainScanQuery(query, isSql); - break; - } - + case NKikimrKqp::QUERY_TYPE_SQL_SCAN: + case NKikimrKqp::QUERY_TYPE_AST_SCAN: { + bool isSql = (type == NKikimrKqp::QUERY_TYPE_SQL_SCAN); + QueryState->AsyncQueryResult = KqpHost->ExplainScanQuery(query, isSql); + break; + } + case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT: case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: { - QueryState->AsyncQueryResult = KqpHost->ExplainYqlScript(query); - break; - } - + QueryState->AsyncQueryResult = KqpHost->ExplainYqlScript(query); + break; + } + default: QueryState->Error = "Unexpected query type."; return false; @@ -1729,7 +1729,7 @@ private: FillQueryProfile(stats, *record.MutableResponse()); record.MutableResponse()->MutableQueryStats()->Swap(&stats); - record.MutableResponse()->SetQueryPlan(queryResult.QueryPlan); + record.MutableResponse()->SetQueryPlan(queryResult.QueryPlan); } AddTrailingInfo(responseEv->Record.GetRef()); diff --git a/ydb/core/kqp/opt/kqp_opt_kql.cpp b/ydb/core/kqp/opt/kqp_opt_kql.cpp index 1eeb95daee4..ab9a57d584d 100644 --- a/ydb/core/kqp/opt/kqp_opt_kql.cpp +++ b/ydb/core/kqp/opt/kqp_opt_kql.cpp @@ -546,7 +546,7 @@ TExprNode::TPtr HandleReadTable(const TKiReadTable& read, TExprContext& ctx, con YQL_ENSURE(metadata, "unable to find metadata for index: " << indexName); YQL_ENSURE(state == TIndexDescription::EIndexState::Ready || state == TIndexDescription::EIndexState::WriteOnly); - + if (state != TIndexDescription::EIndexState::Ready) { auto err = TStringBuilder() << "Requested index: " << indexName diff --git a/ydb/core/kqp/prepare/kqp_query_plan.cpp b/ydb/core/kqp/prepare/kqp_query_plan.cpp index 346a8b15cbe..fe361969f17 100644 --- a/ydb/core/kqp/prepare/kqp_query_plan.cpp +++ b/ydb/core/kqp/prepare/kqp_query_plan.cpp @@ -10,12 +10,12 @@ #include <ydb/library/yql/dq/opt/dq_opt.h> #include <ydb/library/yql/dq/tasks/dq_tasks_graph.h> -#include <library/cpp/json/writer/json.h> -#include <library/cpp/json/json_reader.h> - +#include <library/cpp/json/writer/json.h> +#include <library/cpp/json/json_reader.h> + #include <util/generic/queue.h> #include <util/string/strip.h> -#include <util/string/vector.h> +#include <util/string/vector.h> #include <regex> @@ -24,7 +24,7 @@ namespace NKqp { using namespace NYql; using namespace NYql::NNodes; -using namespace NYql::NDq; +using namespace NYql::NDq; using namespace NClient; namespace { @@ -49,33 +49,33 @@ struct TTableInfo { TVector<TTableWrite> Writes; }; -struct TSerializerCtx { - TSerializerCtx(TExprContext& exprCtx, const TString& cluster, +struct TSerializerCtx { + TSerializerCtx(TExprContext& exprCtx, const TString& cluster, const TIntrusivePtr<NYql::TKikimrTablesData> tablesData, const TKikimrConfiguration::TPtr config, THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> pureTxResults) - : ExprCtx(exprCtx) - , Cluster(cluster) - , TablesData(tablesData) + : ExprCtx(exprCtx) + , Cluster(cluster) + , TablesData(tablesData) , Config(config) , PureTxResults(std::move(pureTxResults)) - {} - - TMap<TString, TTableInfo> Tables; - TMap<TString, TExprNode::TPtr> BindNameToStage; - TMap<TString, ui32> StageGuidToId; + {} + + TMap<TString, TTableInfo> Tables; + TMap<TString, TExprNode::TPtr> BindNameToStage; + TMap<TString, ui32> StageGuidToId; THashMap<ui32, TVector<std::pair<ui32, ui32>>> ParamBindings; THashSet<ui32> PrecomputePhases; - ui32 TxId = 0; + ui32 TxId = 0; ui32 PlanNodeId = 0; - - const TExprContext& ExprCtx; - const TString& Cluster; - const TIntrusivePtr<NYql::TKikimrTablesData> TablesData; + + const TExprContext& ExprCtx; + const TString& Cluster; + const TIntrusivePtr<NYql::TKikimrTablesData> TablesData; const TKikimrConfiguration::TPtr Config; THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> PureTxResults; -}; - +}; + TString GetExprStr(const TExprBase& scalar) { if (auto maybeData = scalar.Maybe<TCoDataCtor>()) { auto literal = TString(maybeData.Cast().Literal()); @@ -285,15 +285,15 @@ void FillTablesInfo(const TExprNode::TPtr& query, TMap<TString, TTableInfo>& tab class TxPlanSerializer { public: TxPlanSerializer(TSerializerCtx& serializerCtx, ui32 txId, const TKqpPhysicalTx& tx) : SerializerCtx(serializerCtx), TxId(txId), Tx(tx) {} - + void Serialize() { auto& phaseNode = QueryPlanNodes[++SerializerCtx.PlanNodeId]; phaseNode.TypeName = "Phase"; - + for (ui32 resId = 0; resId < Tx.Results().Size(); ++resId) { auto res = Tx.Results().Item(resId); auto& planNode = AddPlanNode(phaseNode); - + if (SerializerCtx.PrecomputePhases.find(TxId) != SerializerCtx.PrecomputePhases.end()) { planNode.TypeName = TStringBuilder() << "Precompute_" << TxId << "_" << resId; planNode.CteName = TStringBuilder() << "tx_result_binding_" << TxId << "_" << resId; @@ -301,7 +301,7 @@ public: planNode.TypeName = TStringBuilder() << "ResultSet_" << TxId << "_" << resId; planNode.Type = EPlanNodeType::ResultSet; } - + if (res.Maybe<TDqCnResult>()) { Visit(res.Cast<TDqCnResult>().Output().Stage(), planNode); } else if (res.Maybe<TDqCnValue>()) { @@ -310,28 +310,28 @@ public: Y_ENSURE(false, res.Ref().Content()); } } - + for (const auto& stage: Tx.Stages()) { if (stage.Cast<TDqStageBase>().Program().Body().Maybe<TKqpEffects>()) { auto &planNode = AddPlanNode(phaseNode); planNode.TypeName = "Effect"; Visit(TExprBase(stage), planNode); } - } + } /* set NodeId using reverse order */ for (auto& [id, planNode] : QueryPlanNodes) { planNode.NodeId = SerializerCtx.PlanNodeId - id + 1; } } - + void WriteToJson(NJsonWriter::TBuf& writer) const { if (!QueryPlanNodes.empty()) { auto phasePlanNode = QueryPlanNodes.begin(); WritePlanNodeToJson(phasePlanNode->second, writer); } - } - + } + private: struct TPredicate { TVector<TString> Args; @@ -367,20 +367,20 @@ private: writer.WriteKey("PlanNodeId").WriteInt(planNode.NodeId); writer.WriteKey("Node Type").WriteString(planNode.TypeName); writer.WriteKey("StageGuid").WriteString(planNode.Guid); - + if (auto type = GetPlanNodeType(planNode)) { writer.WriteKey("PlanNodeType").WriteString(type); } - + if (planNode.CteName) { writer.WriteKey("Parent Relationship").WriteString("InitPlan"); writer.WriteKey("Subplan Name").WriteString("CTE " + *planNode.CteName); } - + if (planNode.CteRefName) { writer.WriteKey("CTE Name").WriteString(*planNode.CteRefName); } - + for (const auto& [key, value] : planNode.NodeInfo) { writer.WriteKey(key); writer.WriteJsonValue(&value, true); @@ -417,18 +417,18 @@ private: if (!planNode.Plans.empty()) { writer.WriteKey("Plans"); writer.BeginList(); - + for (auto planId : planNode.Plans) { Y_ENSURE(QueryPlanNodes.contains(planId)); WritePlanNodeToJson(QueryPlanNodes.at(planId), writer); } - + writer.EndList(); } - + writer.EndObject(); } - + TString GetPlanNodeType(const TQueryPlanNode& planNode) const { switch (planNode.Type) { case EPlanNodeType::Connection: @@ -447,7 +447,7 @@ private: return TMaybe<std::pair<ui32, ui32>>( {FromString<ui32>(match[1].str()), FromString<ui32>(match[2].str())}); } - + return {}; } @@ -471,21 +471,21 @@ private: planNode.Operators.push_back(std::move(op)); return planNode.Operators.size() - 1; } - + TQueryPlanNode& GetParent(ui32 nodeId) { auto it = std::find_if(QueryPlanNodes.begin(), QueryPlanNodes.end(), [nodeId](const auto& node) { return node.second.Plans.contains(nodeId); }); - + Y_ENSURE(it != QueryPlanNodes.end()); return it->second; } - + TQueryPlanNode& AddPlanNode(TQueryPlanNode& planNode) { planNode.Plans.insert(++SerializerCtx.PlanNodeId); return QueryPlanNodes[SerializerCtx.PlanNodeId]; } - + TString ToStr(const TCoDataCtor& data) { TStringStream out; EscapeArbitraryAtom(data.Literal().Value(), '"', &out); @@ -640,12 +640,12 @@ private: void Visit(const TExprBase& expr, TQueryPlanNode& planNode) { if (expr.Maybe<TDqPhyStage>()) { auto stageGuid = NDq::TDqStageSettings::Parse(expr.Cast<TDqPhyStage>()).Id; - + if (VisitedStages.find(expr.Raw()) != VisitedStages.end()) { auto stageId = SerializerCtx.StageGuidToId[stageGuid]; Y_ENSURE(QueryPlanNodes.contains(stageId)); auto& commonNode = QueryPlanNodes[stageId]; - + auto& parentNode = GetParent(stageId); parentNode.Plans.erase(stageId); @@ -674,11 +674,11 @@ private: } else { stagePlanNode.TypeName = "Stage"; } - } - + } + for (const auto& input : expr.Cast<TDqStageBase>().Inputs()) { auto inputCn = input.Cast<TDqConnection>(); - + auto& inputPlanNode = AddPlanNode(stagePlanNode); inputPlanNode.Type = EPlanNodeType::Connection; @@ -715,17 +715,17 @@ private: operatorId = Visit(maybeReadRanges.Cast(), planNode); } else if (auto maybeLookup = TMaybeNode<TKqlLookupTableBase>(node)) { operatorId = Visit(maybeLookup.Cast(), planNode); - } else if (auto maybeFilter = TMaybeNode<TCoFilterBase>(node)) { + } else if (auto maybeFilter = TMaybeNode<TCoFilterBase>(node)) { operatorId = Visit(maybeFilter.Cast(), planNode); - } else if (auto maybeMapJoin = TMaybeNode<TCoMapJoinCore>(node)) { + } else if (auto maybeMapJoin = TMaybeNode<TCoMapJoinCore>(node)) { operatorId = Visit(maybeMapJoin.Cast(), planNode); } else if (TMaybeNode<TCoFlatMapBase>(node).Lambda().Body().Maybe<TCoMapJoinCore>() || TMaybeNode<TCoFlatMapBase>(node).Lambda().Body().Maybe<TCoMap>().Input().Maybe<TCoMapJoinCore>()) { - auto flatMap = TMaybeNode<TCoFlatMapBase>(node).Cast(); - auto join = TExprBase(FindNode(node, [](const TExprNode::TPtr& node) { - Y_ENSURE(!TMaybeNode<TDqConnection>(node).IsValid()); - return TMaybeNode<TCoMapJoinCore>(node).IsValid(); - })).Cast<TCoMapJoinCore>(); + auto flatMap = TMaybeNode<TCoFlatMapBase>(node).Cast(); + auto join = TExprBase(FindNode(node, [](const TExprNode::TPtr& node) { + Y_ENSURE(!TMaybeNode<TDqConnection>(node).IsValid()); + return TMaybeNode<TCoMapJoinCore>(node).IsValid(); + })).Cast<TCoMapJoinCore>(); operatorId = Visit(flatMap, join, planNode); node = join.Ptr(); } else if (auto maybeJoinDict = TMaybeNode<TCoJoinDict>(node)) { @@ -760,13 +760,13 @@ private: } else if (auto maybeDelete = TMaybeNode<TKqpDeleteRows>(node)) { operatorId = Visit(maybeDelete.Cast(), planNode); } - + TVector<ui32> inputIds; if (auto maybeEffects = TMaybeNode<TKqpEffects>(node)) { for (const auto& effect : maybeEffects.Cast().Args()) { auto ids = Visit(effect, planNode); inputIds.insert(inputIds.end(), ids.begin(), ids.end()); - } + } } else { for (const auto& child : node->Children()) { auto ids = Visit(child, planNode); @@ -781,14 +781,14 @@ private: return inputIds; } - + ui32 Visit(const TCoCondense1& /*condense*/, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "Aggregate"; return AddOperator(planNode, "Aggregate", std::move(op)); } - + ui32 Visit(const TCoCombineCore& combiner, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "Aggregate"; @@ -797,7 +797,7 @@ private: return AddOperator(planNode, "Aggregate", std::move(op)); } - + ui32 Visit(const TCoSort& sort, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "Sort"; @@ -805,7 +805,7 @@ private: return AddOperator(planNode, "Sort", std::move(op)); } - + ui32 Visit(const TCoTopSort& topSort, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "TopSort"; @@ -814,7 +814,7 @@ private: return AddOperator(planNode, "TopSort", std::move(op)); } - + ui32 Visit(const TCoTake& take, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "Limit"; @@ -822,7 +822,7 @@ private: return AddOperator(planNode, "Limit", std::move(op)); } - + ui32 Visit(const TCoSkip& skip, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "Offset"; @@ -830,14 +830,14 @@ private: return AddOperator(planNode, "Offset", std::move(op)); } - + ui32 Visit(const TCoExtendBase& /*extend*/, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "Union"; return AddOperator(planNode, "Union", std::move(op)); } - + ui32 Visit(const TCoIterator& iter, TQueryPlanNode& planNode) { const auto iterValue = PrettyExprStr(iter.List()); @@ -866,7 +866,7 @@ private: for (const auto& column : upsert.Columns()) { writeInfo.Columns.push_back(TString(column.Value())); } - + SerializerCtx.Tables[table].Writes.push_back(writeInfo); planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); return AddOperator(planNode, "Upsert", std::move(op)); @@ -882,7 +882,7 @@ private: TTableWrite writeInfo; writeInfo.Type = ETableWriteType::MultiErase; - + SerializerCtx.Tables[table].Writes.push_back(writeInfo); planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); return AddOperator(planNode, "Delete", std::move(op)); @@ -898,16 +898,16 @@ private: auto inputs = Visit(flatMap.Input().Ptr(), planNode); planNode.Operators[operatorId].Inputs.insert(inputs.begin(), inputs.end()); return operatorId; - } - + } + ui32 Visit(const TCoMapJoinCore& join, TQueryPlanNode& planNode) { const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (MapJoin)"; TOperator op; op.Properties["Name"] = name; return AddOperator(planNode, name, std::move(op)); - } - + } + ui32 Visit(const TCoFlatMapBase& flatMap, const TCoJoinDict& join, TQueryPlanNode& planNode) { const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (JoinDict)"; @@ -942,7 +942,7 @@ private: ui32 Visit(const TCoFilterBase& filter, TQueryPlanNode& planNode) { TOperator op; op.Properties["Name"] = "Filter"; - + auto pred = ExtractPredicate(filter.Lambda()); op.Properties["Predicate"] = pred.Body; @@ -952,12 +952,12 @@ private: return AddOperator(planNode, "Filter", std::move(op)); } - + ui32 Visit(const TKqlLookupTableBase& lookup, TQueryPlanNode& planNode) { auto table = TString(lookup.Table().Path().Value()); TTableRead readInfo; readInfo.Type = ETableReadType::Lookup; - + TOperator op; op.Properties["Name"] = "TablePointLookup"; auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table); @@ -967,7 +967,7 @@ private: readInfo.Columns.push_back(TString(col.Value())); columns.AppendValue(col.Value()); } - + SerializerCtx.Tables[table].Reads.push_back(readInfo); planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); return AddOperator(planNode, "TablePointLookup", std::move(op)); @@ -1103,10 +1103,10 @@ private: return (*result)[index].GetDataText(); } } - } + } return TString("n/a"); }; - + /* Collect info about scan range */ struct TKeyPartRange { TString From; @@ -1126,14 +1126,14 @@ private: for (ui32 i = 0; i < fromKey.ArgCount(); ++i) { scanRangeDescr[i].From = describeBoundary(fromKey.Arg(i)); - } + } for (ui32 i = 0; i < toKey.ArgCount(); ++i) { scanRangeDescr[i].To = describeBoundary(toKey.Arg(i)); } for (ui32 i = 0; i < scanRangeDescr.size(); ++i) { scanRangeDescr[i].ColumnName = tableData.Metadata->KeyColumnNames[i]; } - + TString leftParen = range.From().Maybe<TKqlKeyInc>().IsValid() ? "[" : "("; TString rightParen = range.To().Maybe<TKqlKeyInc>().IsValid() ? "]" : ")"; bool hasRangeScans = false; @@ -1161,8 +1161,8 @@ private: } ranges.AppendValue(rangeDescr); - } - + } + // Scan which fixes only few first members of compound primary key were called "Lookup" // by older explain version. We continue to do so. if (readInfo.LookupBy.size() > 0) { @@ -1170,7 +1170,7 @@ private: } else { readInfo.Type = hasRangeScans ? ETableReadType::Scan : ETableReadType::FullScan; } - } + } auto& columns = op.Properties["ReadColumns"]; for (auto const& col : read.Columns()) { @@ -1214,32 +1214,32 @@ private: } return operatorId; - } - + } + private: TSerializerCtx& SerializerCtx; const ui32 TxId; const TKqpPhysicalTx& Tx; - + TMap<ui32, TQueryPlanNode> QueryPlanNodes; TNodeSet VisitedStages; }; using ModifyFunction = std::function<void (NJson::TJsonValue& node)>; - + void ModifyPlan(NJson::TJsonValue& plan, const ModifyFunction& modify) { modify(plan); - + auto& map = plan.GetMapSafe(); - if (map.contains("Plans")) { + if (map.contains("Plans")) { for (auto& subplan : map.at("Plans").GetArraySafe()) { ModifyPlan(subplan, modify); - } - } -} - -void WriteCommonTablesInfo(NJsonWriter::TBuf& writer, TMap<TString, TTableInfo>& tables) { - + } + } +} + +void WriteCommonTablesInfo(NJsonWriter::TBuf& writer, TMap<TString, TTableInfo>& tables) { + for (auto& pair : tables) { auto& info = pair.second; @@ -1347,63 +1347,63 @@ void WriteCommonTablesInfo(NJsonWriter::TBuf& writer, TMap<TString, TTableInfo>& } writer.EndList(); - -} - -} // namespace - -void WriteKqlPlan(NJsonWriter::TBuf& writer, const TExprNode::TPtr& query) { - TMap<TString, TTableInfo> tables; - - writer.BeginObject(); - writer.WriteKey("meta"); - - writer.BeginObject(); - writer.WriteKey("version").WriteString("0.1"); - writer.WriteKey("type").WriteString("query"); + +} + +} // namespace + +void WriteKqlPlan(NJsonWriter::TBuf& writer, const TExprNode::TPtr& query) { + TMap<TString, TTableInfo> tables; + + writer.BeginObject(); + writer.WriteKey("meta"); + + writer.BeginObject(); + writer.WriteKey("version").WriteString("0.1"); + writer.WriteKey("type").WriteString("query"); writer.EndObject(); - writer.WriteKey("tables"); - FillTablesInfo(query, tables); - WriteCommonTablesInfo(writer, tables); - - writer.EndObject(); + writer.WriteKey("tables"); + FillTablesInfo(query, tables); + WriteCommonTablesInfo(writer, tables); + + writer.EndObject(); } -// TODO(sk): check prepared statements params in read ranges -// TODO(sk): check params from correlated subqueries // lookup join +// TODO(sk): check prepared statements params in read ranges +// TODO(sk): check params from correlated subqueries // lookup join void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const TKqpPhysicalQuery& query, THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> pureTxResults, TExprContext& ctx, const TString& cluster, const TIntrusivePtr<NYql::TKikimrTablesData> tablesData, TKikimrConfiguration::TPtr config) -{ +{ TSerializerCtx serializerCtx(ctx, cluster, tablesData, config, std::move(pureTxResults)); - - /* bindingName -> stage */ + + /* bindingName -> stage */ auto collectBindings = [&serializerCtx, &query] (auto id, const auto& phase) { - for (const auto& param: phase.ParamBindings()) { - if (auto maybeResultBinding = param.Binding().template Maybe<TKqpTxResultBinding>()) { - auto resultBinding = maybeResultBinding.Cast(); - auto txId = FromString<ui32>(resultBinding.TxIndex()); - auto resultId = FromString<ui32>(resultBinding.ResultIndex()); - auto result = query.Transactions().Item(txId).Results().Item(resultId); - if (auto maybeConnection = result.template Maybe<TDqConnection>()) { - auto stage = maybeConnection.Cast().Output().Stage(); - serializerCtx.BindNameToStage[param.Name().StringValue()] = stage.Ptr(); + for (const auto& param: phase.ParamBindings()) { + if (auto maybeResultBinding = param.Binding().template Maybe<TKqpTxResultBinding>()) { + auto resultBinding = maybeResultBinding.Cast(); + auto txId = FromString<ui32>(resultBinding.TxIndex()); + auto resultId = FromString<ui32>(resultBinding.ResultIndex()); + auto result = query.Transactions().Item(txId).Results().Item(resultId); + if (auto maybeConnection = result.template Maybe<TDqConnection>()) { + auto stage = maybeConnection.Cast().Output().Stage(); + serializerCtx.BindNameToStage[param.Name().StringValue()] = stage.Ptr(); serializerCtx.PrecomputePhases.insert(txId); serializerCtx.ParamBindings[id].push_back({txId, resultId}); - } - } - } - }; + } + } + } + }; ui32 id = 0; - for (const auto& tx: query.Transactions()) { + for (const auto& tx: query.Transactions()) { collectBindings(id++, tx); - } - + } + auto setPlan = [&serializerCtx](auto txId, const auto& tx, auto& txProto) { - NJsonWriter::TBuf txWriter; - txWriter.SetIndentSpaces(2); + NJsonWriter::TBuf txWriter; + txWriter.SetIndentSpaces(2); TxPlanSerializer txPlanSerializer(serializerCtx, txId, tx); if (!serializerCtx.PureTxResults.contains(txId)) { txPlanSerializer.Serialize(); @@ -1411,18 +1411,18 @@ void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const TKqpPhysicalQ txProto.SetPlan(txWriter.Str()); } }; - + id = 0; for (ui32 txId = 0; txId < query.Transactions().Size(); ++txId) { setPlan(id++, query.Transactions().Item(txId), (*queryProto.MutableTransactions())[txId]); - } - - NJsonWriter::TBuf writer; - writer.SetIndentSpaces(2); - WriteCommonTablesInfo(writer, serializerCtx.Tables); - queryProto.SetQueryPlan(writer.Str()); -} - + } + + NJsonWriter::TBuf writer; + writer.SetIndentSpaces(2); + WriteCommonTablesInfo(writer, serializerCtx.Tables); + queryProto.SetQueryPlan(writer.Str()); +} + TString AddExecStatsToTxPlan(const TString& txPlanJson, const NYql::NDqProto::TDqExecutionStats& stats) { if (txPlanJson.empty()) { return {}; @@ -1430,17 +1430,17 @@ TString AddExecStatsToTxPlan(const TString& txPlanJson, const NYql::NDqProto::TD THashMap<TProtoStringType, const NYql::NDqProto::TDqStageStats*> stages; for (const auto& stage : stats.GetStages()) { - stages[stage.GetStageGuid()] = &stage; - } - - NJson::TJsonValue root; - NJson::ReadJsonTree(txPlanJson, &root, true); - + stages[stage.GetStageGuid()] = &stage; + } + + NJson::TJsonValue root; + NJson::ReadJsonTree(txPlanJson, &root, true); + auto addStatsToPlanNode = [&stages](NJson::TJsonValue& node) { if (auto stageGuid = node.GetMapSafe().FindPtr("StageGuid")) { if (auto stat = stages.FindPtr(stageGuid->GetStringSafe())) { auto& stats = node["Stats"]; - + stats["TotalTasks"] = (*stat)->GetTotalTasksCount(); stats["TotalDurationMs"] = (*stat)->GetFinishTimeMs().GetMax() - (*stat)->GetFirstRowTimeMs().GetMin(); stats["TotalCpuTimeUs"] = (*stat)->GetCpuTimeUs().GetSum(); @@ -1448,44 +1448,44 @@ TString AddExecStatsToTxPlan(const TString& txPlanJson, const NYql::NDqProto::TD stats["TotalInputBytes"] = (*stat)->GetInputBytes().GetSum(); stats["TotalOutputRows"] = (*stat)->GetOutputRows().GetSum(); stats["TotalOutputBytes"] = (*stat)->GetOutputBytes().GetSum(); - } - } + } + } }; - + ModifyPlan(root, addStatsToPlanNode); NJsonWriter::TBuf txWriter; txWriter.WriteJsonValue(&root, true); - return txWriter.Str(); -} - -TString SerializeTxPlans(const TVector<const TString>& txPlans, const TString commonPlanInfo = "") { - NJsonWriter::TBuf writer; - writer.SetIndentSpaces(2); - - writer.BeginObject(); - writer.WriteKey("meta"); - writer.BeginObject(); - writer.WriteKey("version").WriteString("0.2"); - writer.WriteKey("type").WriteString("query"); - writer.EndObject(); - - if (!commonPlanInfo.Empty()) { - writer.WriteKey("tables"); - writer.UnsafeWriteValue(commonPlanInfo); - } - - writer.WriteKey("Plan"); - writer.BeginObject(); - writer.WriteKey("Node Type").WriteString("Query"); - writer.WriteKey("Plans"); - writer.BeginList(); - + return txWriter.Str(); +} + +TString SerializeTxPlans(const TVector<const TString>& txPlans, const TString commonPlanInfo = "") { + NJsonWriter::TBuf writer; + writer.SetIndentSpaces(2); + + writer.BeginObject(); + writer.WriteKey("meta"); + writer.BeginObject(); + writer.WriteKey("version").WriteString("0.2"); + writer.WriteKey("type").WriteString("query"); + writer.EndObject(); + + if (!commonPlanInfo.Empty()) { + writer.WriteKey("tables"); + writer.UnsafeWriteValue(commonPlanInfo); + } + + writer.WriteKey("Plan"); + writer.BeginObject(); + writer.WriteKey("Node Type").WriteString("Query"); + writer.WriteKey("Plans"); + writer.BeginList(); + auto removeStageGuid = [](NJson::TJsonValue& node) { auto& map = node.GetMapSafe(); if (map.contains("StageGuid")) { map.erase("StageGuid"); - } + } }; for (const auto& txPlan : txPlans) { @@ -1500,74 +1500,74 @@ TString SerializeTxPlans(const TVector<const TString>& txPlans, const TString co ModifyPlan(subplan, removeStageGuid); writer.WriteJsonValue(&subplan, true); } - } - - writer.EndList(); - writer.EndObject(); - writer.EndObject(); - - return writer.Str(); -} - -TString SerializeExplainPlan(const NKqpProto::TKqpPhyQuery& phyQuery) { - TVector<const TString> txPlans; + } + + writer.EndList(); + writer.EndObject(); + writer.EndObject(); + + return writer.Str(); +} + +TString SerializeExplainPlan(const NKqpProto::TKqpPhyQuery& phyQuery) { + TVector<const TString> txPlans; txPlans.reserve(phyQuery.GetTransactions().size()); - for (const auto& phyTx: phyQuery.GetTransactions()) { + for (const auto& phyTx: phyQuery.GetTransactions()) { txPlans.emplace_back(phyTx.GetPlan()); - } - return SerializeTxPlans(txPlans, phyQuery.GetQueryPlan()); -} - -TString SerializeAnalyzePlan(const NKqpProto::TKqpStatsQuery& queryStats) { - TVector<const TString> txPlans; - for (const auto& execStats: queryStats.GetExecutions()) { - for (const auto& txPlan: execStats.GetTxPlansWithStats()) { - txPlans.push_back(txPlan); - } - } - return SerializeTxPlans(txPlans); -} - -TString SerializeScriptPlan(const TVector<const TString>& queryPlans) { - NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE); - writer.SetIndentSpaces(2); - - writer.BeginObject(); - writer.WriteKey("meta"); - writer.BeginObject(); - writer.WriteKey("version").WriteString("0.2"); - writer.WriteKey("type").WriteString("script"); - writer.EndObject(); - - writer.WriteKey("queries"); - writer.BeginList(); - + } + return SerializeTxPlans(txPlans, phyQuery.GetQueryPlan()); +} + +TString SerializeAnalyzePlan(const NKqpProto::TKqpStatsQuery& queryStats) { + TVector<const TString> txPlans; + for (const auto& execStats: queryStats.GetExecutions()) { + for (const auto& txPlan: execStats.GetTxPlansWithStats()) { + txPlans.push_back(txPlan); + } + } + return SerializeTxPlans(txPlans); +} + +TString SerializeScriptPlan(const TVector<const TString>& queryPlans) { + NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE); + writer.SetIndentSpaces(2); + + writer.BeginObject(); + writer.WriteKey("meta"); + writer.BeginObject(); + writer.WriteKey("version").WriteString("0.2"); + writer.WriteKey("type").WriteString("script"); + writer.EndObject(); + + writer.WriteKey("queries"); + writer.BeginList(); + for (const auto& plan : queryPlans) { if (plan.empty()) { continue; } - NJson::TJsonValue root; - NJson::ReadJsonTree(plan, &root, true); - auto planMap = root.GetMapSafe(); - - writer.BeginObject(); - if (auto tableAccesses = planMap.FindPtr("tables")) { - writer.WriteKey("tables"); - writer.WriteJsonValue(tableAccesses); - } - if (auto dqPlan = planMap.FindPtr("Plan")) { - writer.WriteKey("Plan"); - writer.WriteJsonValue(dqPlan); - } - writer.EndObject(); - } - - writer.EndList(); - writer.EndObject(); - - return writer.Str(); -} - + NJson::TJsonValue root; + NJson::ReadJsonTree(plan, &root, true); + auto planMap = root.GetMapSafe(); + + writer.BeginObject(); + if (auto tableAccesses = planMap.FindPtr("tables")) { + writer.WriteKey("tables"); + writer.WriteJsonValue(tableAccesses); + } + if (auto dqPlan = planMap.FindPtr("Plan")) { + writer.WriteKey("Plan"); + writer.WriteJsonValue(dqPlan); + } + writer.EndObject(); + } + + writer.EndList(); + writer.EndObject(); + + return writer.Str(); +} + } // namespace NKqp } // namespace NKikimr diff --git a/ydb/core/kqp/prepare/kqp_query_plan.h b/ydb/core/kqp/prepare/kqp_query_plan.h index 4e299e14903..e3895f11d5a 100644 --- a/ydb/core/kqp/prepare/kqp_query_plan.h +++ b/ydb/core/kqp/prepare/kqp_query_plan.h @@ -1,38 +1,38 @@ -#pragma once - +#pragma once + #include <ydb/core/protos/kqp_physical.pb.h> #include <ydb/core/protos/kqp_stats.pb.h> #include <ydb/core/kqp/expr_nodes/kqp_expr_nodes.h> #include <ydb/core/kqp/provider/yql_kikimr_provider.h> - + #include <ydb/library/yql/ast/yql_expr.h> - -#include <library/cpp/json/writer/json.h> - -namespace NKikimr { -namespace NKqp { - -void WriteKqlPlan(NJsonWriter::TBuf& writer, const NYql::TExprNode::TPtr& query); - -/* - * Set dqPlan in each physical transaction (TKqpPhyQuery.Transactions[].Plan). Common query plan with all - * table accesses is stored in top-level TKqpPhyQuery.QueryPlan. - */ + +#include <library/cpp/json/writer/json.h> + +namespace NKikimr { +namespace NKqp { + +void WriteKqlPlan(NJsonWriter::TBuf& writer, const NYql::TExprNode::TPtr& query); + +/* + * Set dqPlan in each physical transaction (TKqpPhyQuery.Transactions[].Plan). Common query plan with all + * table accesses is stored in top-level TKqpPhyQuery.QueryPlan. + */ void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const NYql::NNodes::TKqpPhysicalQuery& query, THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> pureTxResults, NYql::TExprContext& ctx, const TString& cluster, const TIntrusivePtr<NYql::TKikimrTablesData> tablesData, NYql::TKikimrConfiguration::TPtr config); - -/* - * Fill stages in given txPlan with ExecutionStats fields. Each plan stage stores StageGuid which is - * used to find corresponding TKqpStatsExecution object. - */ + +/* + * Fill stages in given txPlan with ExecutionStats fields. Each plan stage stores StageGuid which is + * used to find corresponding TKqpStatsExecution object. + */ TString AddExecStatsToTxPlan(const TString& txPlan, const NYql::NDqProto::TDqExecutionStats& stats); - -TString SerializeExplainPlan(const NKqpProto::TKqpPhyQuery& phyQuery); - -TString SerializeAnalyzePlan(const NKqpProto::TKqpStatsQuery& queryStats); - -TString SerializeScriptPlan(const TVector<const TString>& queryPlans); - -} // namespace NKqp -} // namespace NKikimr + +TString SerializeExplainPlan(const NKqpProto::TKqpPhyQuery& phyQuery); + +TString SerializeAnalyzePlan(const NKqpProto::TKqpStatsQuery& queryStats); + +TString SerializeScriptPlan(const TVector<const TString>& queryPlans); + +} // namespace NKqp +} // namespace NKikimr diff --git a/ydb/core/kqp/prepare/kqp_type_ann.cpp b/ydb/core/kqp/prepare/kqp_type_ann.cpp index ee51c7349c2..caa6653538b 100644 --- a/ydb/core/kqp/prepare/kqp_type_ann.cpp +++ b/ydb/core/kqp/prepare/kqp_type_ann.cpp @@ -230,7 +230,7 @@ TStatus AnnotateReadTable(const TExprNode::TPtr& node, TExprContext& ctx, const } else { YQL_ENSURE(false, "Unexpected ReadTable callable: " << node->Content()); } - + return TStatus::Ok; } @@ -1056,8 +1056,8 @@ TAutoPtr<IGraphTransformer> CreateKqpTypeAnnotationTransformer(const TString& cl if (TKqlLookupTableBase::Match(input.Get())) { return AnnotateLookupTable(input, ctx, cluster, *tablesData, config->SystemColumnsEnabled()); - } - + } + if (TKqlKeyInc::Match(input.Get()) || TKqlKeyExc::Match(input.Get())) { return AnnotateKeyTuple(input, ctx); } diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.h b/ydb/core/kqp/provider/yql_kikimr_provider.h index 6903e6f1932..baec0739087 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider.h @@ -81,12 +81,12 @@ struct TKikimrQueryContext : TThrRefBase { TKikimrQueryContext& operator=(const TKikimrQueryContext&) = delete; bool PrepareOnly = false; - - /* - * Defuse DDL-prohibiting checks when PrepareOnly = true. Used in scripting query explain. - */ + + /* + * Defuse DDL-prohibiting checks when PrepareOnly = true. Used in scripting query explain. + */ bool SuppressDdlChecks = false; - + EKikimrStatsMode StatsMode = EKikimrStatsMode::None; EKikimrQueryType Type = EKikimrQueryType::Unspecified; TKikimrQueryDeadlines Deadlines; diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h index 530cfb28558..4c629b747da 100644 --- a/ydb/core/kqp/provider/yql_kikimr_settings.h +++ b/ydb/core/kqp/provider/yql_kikimr_settings.h @@ -146,8 +146,8 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi } TKikimrSettings::TConstPtr Snapshot() const; - - NKikimrConfig::TFeatureFlags FeatureFlags; + + NKikimrConfig::TFeatureFlags FeatureFlags; }; } diff --git a/ydb/core/kqp/rm/kqp_resource_estimation.cpp b/ydb/core/kqp/rm/kqp_resource_estimation.cpp index aeeef801775..671d3e9c5f8 100644 --- a/ydb/core/kqp/rm/kqp_resource_estimation.cpp +++ b/ydb/core/kqp/rm/kqp_resource_estimation.cpp @@ -5,8 +5,8 @@ namespace NKikimr::NKqp { using namespace NYql::NDqProto; using namespace NKikimrConfig; -TTaskResourceEstimation EstimateTaskResources(const TDqTask& task, int nScans, ui32 dsOnNodeCount, - const TTableServiceConfig::TResourceManager& config) +TTaskResourceEstimation EstimateTaskResources(const TDqTask& task, int nScans, ui32 dsOnNodeCount, + const TTableServiceConfig::TResourceManager& config) { TTaskResourceEstimation ret; EstimateTaskResources(task, nScans, dsOnNodeCount, config, ret); @@ -18,8 +18,8 @@ void EstimateTaskResources(const TDqTask& task, int nScans, ui32 dsOnNodeCount, { ret.TaskId = task.GetId(); - if (nScans > 0) { - ret.ScanBuffersCount = nScans; + if (nScans > 0) { + ret.ScanBuffersCount = nScans; ret.ScanBufferMemoryLimit = config.GetScanBufferSize(); if (dsOnNodeCount && ret.ScanBufferMemoryLimit * dsOnNodeCount > config.GetMaxTotalScanBuffersSize()) { diff --git a/ydb/core/kqp/rm/kqp_resource_estimation.h b/ydb/core/kqp/rm/kqp_resource_estimation.h index b69ac203514..e66dc52f121 100644 --- a/ydb/core/kqp/rm/kqp_resource_estimation.h +++ b/ydb/core/kqp/rm/kqp_resource_estimation.h @@ -30,7 +30,7 @@ struct TTaskResourceEstimation { } }; -TTaskResourceEstimation EstimateTaskResources(const NYql::NDqProto::TDqTask& task, int nScans, ui32 dsOnNodeCount, +TTaskResourceEstimation EstimateTaskResources(const NYql::NDqProto::TDqTask& task, int nScans, ui32 dsOnNodeCount, const NKikimrConfig::TTableServiceConfig::TResourceManager& config); void EstimateTaskResources(const NYql::NDqProto::TDqTask& task, int nScans, ui32 dsOnNodeCount, diff --git a/ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp b/ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp index 689c0ed86b7..5d76d1b777e 100644 --- a/ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp +++ b/ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp @@ -31,7 +31,7 @@ Y_UNIT_TEST(TestChannelSize) { auto* output = task.MutableOutputs()->Add(); output->MutableChannels()->Add(); - auto est = EstimateTaskResources(task, 0, 0, config); + auto est = EstimateTaskResources(task, 0, 0, config); UNIT_ASSERT_EQUAL(201, est.ChannelBuffersCount); UNIT_ASSERT_EQUAL(est.ChannelBufferMemoryLimit, config.GetChannelBufferSize()); @@ -41,7 +41,7 @@ Y_UNIT_TEST(TestChannelSize) { input->MutableChannels()->Add(); } - est = EstimateTaskResources(task, 0, 0, config); + est = EstimateTaskResources(task, 0, 0, config); UNIT_ASSERT_EQUAL(301, est.ChannelBuffersCount); UNIT_ASSERT(est.ChannelBufferMemoryLimit < config.GetChannelBufferSize()); diff --git a/ydb/core/kqp/rm/ya.make b/ydb/core/kqp/rm/ya.make index 0d8a7596391..4185e629129 100644 --- a/ydb/core/kqp/rm/ya.make +++ b/ydb/core/kqp/rm/ya.make @@ -4,7 +4,7 @@ OWNER(g:kikimr) SRCS( kqp_resource_estimation.cpp - kqp_snapshot_manager.cpp + kqp_snapshot_manager.cpp kqp_rm.cpp ) diff --git a/ydb/core/kqp/runtime/kqp_program_builder.cpp b/ydb/core/kqp/runtime/kqp_program_builder.cpp index b5169c19d82..3682fd4d4b2 100644 --- a/ydb/core/kqp/runtime/kqp_program_builder.cpp +++ b/ydb/core/kqp/runtime/kqp_program_builder.cpp @@ -141,9 +141,9 @@ TRuntimeNode TKqpProgramBuilder::KqpReadTable(const TTableId& tableId, const TKq return TRuntimeNode(builder.Build(), false); } -TRuntimeNode TKqpProgramBuilder::KqpWideReadTable(const TTableId& tableId, const TKqpKeyRange& range, +TRuntimeNode TKqpProgramBuilder::KqpWideReadTable(const TTableId& tableId, const TKqpKeyRange& range, const TArrayRef<TKqpTableColumn>& columns) -{ +{ auto rowType = GetRowType(*this, columns); auto structType = AS_TYPE(TStructType, rowType); auto returnType = MakeWideFlowType(*this, structType); @@ -152,18 +152,18 @@ TRuntimeNode TKqpProgramBuilder::KqpWideReadTable(const TTableId& tableId, const MKQL_ENSURE_S(returnType->IsFlow()); const auto itemType = AS_TYPE(TFlowType, returnType)->GetItemType(); MKQL_ENSURE_S(itemType->IsTuple()); - - TCallableBuilder builder(Env, __func__, returnType); - builder.Add(BuildTableIdLiteral(tableId, *this)); + + TCallableBuilder builder(Env, __func__, returnType); + builder.Add(BuildTableIdLiteral(tableId, *this)); builder.Add(BuildKeyRangeNode(*this, range)); - builder.Add(BuildColumnTags(*this, columns)); + builder.Add(BuildColumnTags(*this, columns)); builder.Add(BuildSkipNullKeysNode(*this, range)); builder.Add(range.ItemsLimit ? range.ItemsLimit : NewNull()); builder.Add(NewDataLiteral(range.Reverse)); - - return TRuntimeNode(builder.Build(), false); -} - + + return TRuntimeNode(builder.Build(), false); +} + TRuntimeNode TKqpProgramBuilder::KqpWideReadTableRanges(const TTableId& tableId, const TKqpKeyRanges& ranges, const TArrayRef<TKqpTableColumn>& columns, TType* returnType) { diff --git a/ydb/core/kqp/runtime/kqp_program_builder.h b/ydb/core/kqp/runtime/kqp_program_builder.h index 348d25eeea3..0884d68a74c 100644 --- a/ydb/core/kqp/runtime/kqp_program_builder.h +++ b/ydb/core/kqp/runtime/kqp_program_builder.h @@ -47,9 +47,9 @@ public: TRuntimeNode KqpReadTable(const TTableId& tableId, const TKqpKeyRange& range, const TArrayRef<TKqpTableColumn>& columns); - TRuntimeNode KqpWideReadTable(const TTableId& tableId, const TKqpKeyRange& range, - const TArrayRef<TKqpTableColumn>& columns); - + TRuntimeNode KqpWideReadTable(const TTableId& tableId, const TKqpKeyRange& range, + const TArrayRef<TKqpTableColumn>& columns); + TRuntimeNode KqpWideReadTableRanges(const TTableId& tableId, const TKqpKeyRanges& range, const TArrayRef<TKqpTableColumn>& columns, TType* returnType); diff --git a/ydb/core/kqp/runtime/kqp_read_table.cpp b/ydb/core/kqp/runtime/kqp_read_table.cpp index 358ebd0a3eb..9db590688b5 100644 --- a/ydb/core/kqp/runtime/kqp_read_table.cpp +++ b/ydb/core/kqp/runtime/kqp_read_table.cpp @@ -137,38 +137,38 @@ void ParseWideReadColumns(const TCallable& callable, const TRuntimeNode& tagsNod } } -TParseReadTableResult ParseWideReadTable(TCallable& callable) { +TParseReadTableResult ParseWideReadTable(TCallable& callable) { MKQL_ENSURE_S(callable.GetInputsCount() >= 4); - - TParseReadTableResult result; - + + TParseReadTableResult result; + result.CallableId = 0; // callable.GetUniqueId(); - - auto tableNode = callable.GetInput(0); - auto rangeNode = callable.GetInput(1); - auto tagsNode = callable.GetInput(2); - - result.TableId = NKqp::ParseTableId(tableNode); - - auto range = AS_VALUE(TTupleLiteral, rangeNode); - MKQL_ENSURE_S(range); - MKQL_ENSURE_S(range->GetValuesCount() >= 4); - - result.FromTuple = AS_VALUE(TTupleLiteral, range->GetValue(0)); - MKQL_ENSURE_S(result.FromTuple); - result.FromInclusive = AS_VALUE(TDataLiteral, range->GetValue(1))->AsValue().Get<bool>(); - result.ToTuple = AS_VALUE(TTupleLiteral, range->GetValue(2)); - MKQL_ENSURE_S(result.ToTuple); - result.ToInclusive = AS_VALUE(TDataLiteral, range->GetValue(3))->AsValue().Get<bool>(); - + + auto tableNode = callable.GetInput(0); + auto rangeNode = callable.GetInput(1); + auto tagsNode = callable.GetInput(2); + + result.TableId = NKqp::ParseTableId(tableNode); + + auto range = AS_VALUE(TTupleLiteral, rangeNode); + MKQL_ENSURE_S(range); + MKQL_ENSURE_S(range->GetValuesCount() >= 4); + + result.FromTuple = AS_VALUE(TTupleLiteral, range->GetValue(0)); + MKQL_ENSURE_S(result.FromTuple); + result.FromInclusive = AS_VALUE(TDataLiteral, range->GetValue(1))->AsValue().Get<bool>(); + result.ToTuple = AS_VALUE(TTupleLiteral, range->GetValue(2)); + MKQL_ENSURE_S(result.ToTuple); + result.ToInclusive = AS_VALUE(TDataLiteral, range->GetValue(3))->AsValue().Get<bool>(); + ParseWideReadColumns(callable, tagsNode, result.Columns, result.SystemColumns); - + auto skipNullKeys = AS_VALUE(TListLiteral, callable.GetInput(3)); result.SkipNullKeys.reserve(skipNullKeys->GetItemsCount()); for (ui32 i = 0; i < skipNullKeys->GetItemsCount(); ++i) { result.SkipNullKeys.push_back(AS_VALUE(TDataLiteral, skipNullKeys->GetItems()[i])->AsValue().Get<bool>()); } - + if (callable.GetInputsCount() >= 5) { auto node = callable.GetInput(4).GetNode(); if (node->GetType()->GetKind() == TType::EKind::Callable) { @@ -184,9 +184,9 @@ TParseReadTableResult ParseWideReadTable(TCallable& callable) { result.Reverse = AS_VALUE(TDataLiteral, callable.GetInput(5))->AsValue().Get<bool>(); } - return result; -} - + return result; +} + TParseReadTableRangesResult ParseWideReadTableRanges(TCallable& callable) { MKQL_ENSURE_S(callable.GetInputsCount() == 5); @@ -233,88 +233,88 @@ namespace { class TKqpScanWideReadTableWrapperBase : public TStatelessWideFlowCodegeneratorNode<TKqpScanWideReadTableWrapperBase> { using TBase = TStatelessWideFlowCodegeneratorNode<TKqpScanWideReadTableWrapperBase>; -public: +public: TKqpScanWideReadTableWrapperBase(TKqpScanComputeContext& computeCtx, std::vector<EValueRepresentation>&& representations) : TBase(this) , ComputeCtx(computeCtx) - , Representations(std::move(representations)) - {} - - EFetchResult DoCalculate(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const { - Y_UNUSED(ctx); - - if (!TableReader) { + , Representations(std::move(representations)) + {} + + EFetchResult DoCalculate(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const { + Y_UNUSED(ctx); + + if (!TableReader) { TableReader = ComputeCtx.ReadTable(GetCallableId()); - } - - return TableReader->Next(output); - } - -#ifndef MKQL_DISABLE_CODEGEN - ICodegeneratorInlineWideNode::TGenerateResult DoGenGetValues(const TCodegenContext& ctx, BasicBlock*& block) const { - auto& context = ctx.Codegen->GetContext(); + } + + return TableReader->Next(output); + } + +#ifndef MKQL_DISABLE_CODEGEN + ICodegeneratorInlineWideNode::TGenerateResult DoGenGetValues(const TCodegenContext& ctx, BasicBlock*& block) const { + auto& context = ctx.Codegen->GetContext(); const auto size = GetAllColumnsSize(); - ICodegeneratorInlineWideNode::TGettersList getters(size); - - const auto valueType = Type::getInt128Ty(context); - const auto valuesType = ArrayType::get(valueType, size); - const auto values = new AllocaInst(valuesType, 0U, "values", &ctx.Func->getEntryBlock().back()); - - const auto fieldsType = ArrayType::get(PointerType::getUnqual(valueType), size); - const auto fields = new AllocaInst(fieldsType, 0U, "fields", &ctx.Func->getEntryBlock().back()); - - const auto indexType = Type::getInt32Ty(context); - Value* init = UndefValue::get(fieldsType); - for (auto i = 0U; i < size; ++i) { - const auto pointer = GetElementPtrInst::CreateInBounds(values, - {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)}, - (TString("ptr_") += ToString(i)).c_str(), - &ctx.Func->getEntryBlock().back()); - init = InsertValueInst::Create(init, pointer, {i}, (TString("insert_") += ToString(i)).c_str(), &ctx.Func->getEntryBlock().back()); - - new StoreInst(ConstantInt::get(valueType, 0), pointer, &ctx.Func->getEntryBlock().back()); - - getters[i] = [i, indexType, values] (const TCodegenContext&, BasicBlock*& block) { - const auto loadPtr = GetElementPtrInst::CreateInBounds(values, - {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)}, - (TString("loadPtr_") += ToString(i)).c_str(), - block); - return new LoadInst(loadPtr, "load", block); - }; - } - - new StoreInst(init, fields, &ctx.Func->getEntryBlock().back()); - - for (ui32 i = 0U; i < size; ++i) { - const auto pointer = GetElementPtrInst::CreateInBounds(values, - {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)}, - (TString("ptr_") += ToString(i)).c_str(), block); - ValueCleanup(Representations[i], pointer, ctx, block); - new StoreInst(ConstantInt::get(valueType, 0), pointer, block); - } - - const auto ptrType = PointerType::getUnqual(StructType::get(context)); + ICodegeneratorInlineWideNode::TGettersList getters(size); + + const auto valueType = Type::getInt128Ty(context); + const auto valuesType = ArrayType::get(valueType, size); + const auto values = new AllocaInst(valuesType, 0U, "values", &ctx.Func->getEntryBlock().back()); + + const auto fieldsType = ArrayType::get(PointerType::getUnqual(valueType), size); + const auto fields = new AllocaInst(fieldsType, 0U, "fields", &ctx.Func->getEntryBlock().back()); + + const auto indexType = Type::getInt32Ty(context); + Value* init = UndefValue::get(fieldsType); + for (auto i = 0U; i < size; ++i) { + const auto pointer = GetElementPtrInst::CreateInBounds(values, + {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)}, + (TString("ptr_") += ToString(i)).c_str(), + &ctx.Func->getEntryBlock().back()); + init = InsertValueInst::Create(init, pointer, {i}, (TString("insert_") += ToString(i)).c_str(), &ctx.Func->getEntryBlock().back()); + + new StoreInst(ConstantInt::get(valueType, 0), pointer, &ctx.Func->getEntryBlock().back()); + + getters[i] = [i, indexType, values] (const TCodegenContext&, BasicBlock*& block) { + const auto loadPtr = GetElementPtrInst::CreateInBounds(values, + {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)}, + (TString("loadPtr_") += ToString(i)).c_str(), + block); + return new LoadInst(loadPtr, "load", block); + }; + } + + new StoreInst(init, fields, &ctx.Func->getEntryBlock().back()); + + for (ui32 i = 0U; i < size; ++i) { + const auto pointer = GetElementPtrInst::CreateInBounds(values, + {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)}, + (TString("ptr_") += ToString(i)).c_str(), block); + ValueCleanup(Representations[i], pointer, ctx, block); + new StoreInst(ConstantInt::get(valueType, 0), pointer, block); + } + + const auto ptrType = PointerType::getUnqual(StructType::get(context)); const auto func = ConstantInt::get(Type::getInt64Ty(context), GetMethodPtr(&TKqpScanWideReadTableWrapperBase::DoCalculate)); - const auto self = CastInst::Create(Instruction::IntToPtr, ConstantInt::get(Type::getInt64Ty(context), uintptr_t(this)), ptrType, "self", block); - const auto funcType = FunctionType::get(Type::getInt32Ty(context), { self->getType(), ctx.Ctx->getType(), fields->getType() }, false); - const auto funcPtr = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(funcType), "fetch_func", block); - const auto result = CallInst::Create(funcPtr, { self, ctx.Ctx, fields }, "fetch", block); - - for (ui32 i = 0U; i < size; ++i) { - const auto pointer = GetElementPtrInst::CreateInBounds(values, - {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)}, - (TString("ptr_") += ToString(i)).c_str(), block); - ValueRelease(Representations[i], pointer, ctx, block); - } - - return {result, std::move(getters)}; - } -#endif - + const auto self = CastInst::Create(Instruction::IntToPtr, ConstantInt::get(Type::getInt64Ty(context), uintptr_t(this)), ptrType, "self", block); + const auto funcType = FunctionType::get(Type::getInt32Ty(context), { self->getType(), ctx.Ctx->getType(), fields->getType() }, false); + const auto funcPtr = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(funcType), "fetch_func", block); + const auto result = CallInst::Create(funcPtr, { self, ctx.Ctx, fields }, "fetch", block); + + for (ui32 i = 0U; i < size; ++i) { + const auto pointer = GetElementPtrInst::CreateInBounds(values, + {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)}, + (TString("ptr_") += ToString(i)).c_str(), block); + ValueRelease(Representations[i], pointer, ctx, block); + } + + return {result, std::move(getters)}; + } +#endif + virtual ui32 GetCallableId() const = 0; virtual ui32 GetAllColumnsSize() const = 0; -private: +private: TKqpScanComputeContext& ComputeCtx; mutable TIntrusivePtr<IKqpTableReader> TableReader; const std::vector<EValueRepresentation> Representations; @@ -339,17 +339,17 @@ private: return ParseResult.Columns.size() + ParseResult.SystemColumns.size(); } - void RegisterDependencies() const { + void RegisterDependencies() const { FlowDependsOn(FromNode); FlowDependsOn(ToNode); - } - -private: - IComputationNode* FromNode; - IComputationNode* ToNode; + } + +private: + IComputationNode* FromNode; + IComputationNode* ToNode; TParseReadTableResult ParseResult; -}; - +}; + class TKqpScanWideReadTableRangesWrapper : public TKqpScanWideReadTableWrapperBase { public: TKqpScanWideReadTableRangesWrapper(TKqpScanComputeContext& computeCtx, const TParseReadTableRangesResult& parseResult, @@ -401,28 +401,28 @@ IComputationNode* WrapKqpScanWideReadTableRanges(TCallable& callable, const TCom return new TKqpScanWideReadTableRangesWrapper(computeCtx, parseResult, rangesNode, std::move(representations)); } -IComputationNode* WrapKqpScanWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx, - TKqpScanComputeContext& computeCtx) -{ - std::vector<EValueRepresentation> representations; - - auto parseResult = ParseWideReadTable(callable); - auto fromNode = LocateNode(ctx.NodeLocator, *parseResult.FromTuple); - auto toNode = LocateNode(ctx.NodeLocator, *parseResult.ToTuple); - - const auto type = callable.GetType()->GetReturnType(); - const auto returnItemType = type->IsFlow() ? - AS_TYPE(TFlowType, callable.GetType()->GetReturnType())->GetItemType(): - AS_TYPE(TStreamType, callable.GetType()->GetReturnType())->GetItemType(); - - const auto tupleType = AS_TYPE(TTupleType, returnItemType); - - representations.reserve(tupleType->GetElementsCount()); - for (ui32 i = 0U; i < tupleType->GetElementsCount(); ++i) - representations.emplace_back(GetValueRepresentation(tupleType->GetElementType(i))); - - return new TKqpScanWideReadTableWrapper(computeCtx, parseResult, fromNode, toNode, std::move(representations)); -} - +IComputationNode* WrapKqpScanWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx, + TKqpScanComputeContext& computeCtx) +{ + std::vector<EValueRepresentation> representations; + + auto parseResult = ParseWideReadTable(callable); + auto fromNode = LocateNode(ctx.NodeLocator, *parseResult.FromTuple); + auto toNode = LocateNode(ctx.NodeLocator, *parseResult.ToTuple); + + const auto type = callable.GetType()->GetReturnType(); + const auto returnItemType = type->IsFlow() ? + AS_TYPE(TFlowType, callable.GetType()->GetReturnType())->GetItemType(): + AS_TYPE(TStreamType, callable.GetType()->GetReturnType())->GetItemType(); + + const auto tupleType = AS_TYPE(TTupleType, returnItemType); + + representations.reserve(tupleType->GetElementsCount()); + for (ui32 i = 0U; i < tupleType->GetElementsCount(); ++i) + representations.emplace_back(GetValueRepresentation(tupleType->GetElementType(i))); + + return new TKqpScanWideReadTableWrapper(computeCtx, parseResult, fromNode, toNode, std::move(representations)); +} + } // namespace NMiniKQL } // namespace NKikimr diff --git a/ydb/core/kqp/runtime/kqp_read_table.h b/ydb/core/kqp/runtime/kqp_read_table.h index 99c9cc3f050..b13541251df 100644 --- a/ydb/core/kqp/runtime/kqp_read_table.h +++ b/ydb/core/kqp/runtime/kqp_read_table.h @@ -38,14 +38,14 @@ struct TParseReadTableRangesResult : TParseReadTableResultBase { void ParseReadColumns(const TType* readType, const TRuntimeNode& tagsNode, TSmallVec<TKqpComputeContextBase::TColumn>& columns, TSmallVec<TKqpComputeContextBase::TColumn>& systemColumns); -TParseReadTableResult ParseWideReadTable(TCallable& callable); +TParseReadTableResult ParseWideReadTable(TCallable& callable); TParseReadTableRangesResult ParseWideReadTableRanges(TCallable& callable); IComputationNode* WrapKqpScanWideReadTableRanges(TCallable& callable, const TComputationNodeFactoryContext& ctx, TKqpScanComputeContext& computeCtx); -IComputationNode* WrapKqpScanWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx, - TKqpScanComputeContext& computeCtx); - -} // namespace NMiniKQL -} // namespace NKikimr +IComputationNode* WrapKqpScanWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx, + TKqpScanComputeContext& computeCtx); +} // namespace NMiniKQL +} // namespace NKikimr + diff --git a/ydb/core/kqp/runtime/kqp_scan_data.cpp b/ydb/core/kqp/runtime/kqp_scan_data.cpp index c4d8e241966..56c49bf0f5f 100644 --- a/ydb/core/kqp/runtime/kqp_scan_data.cpp +++ b/ydb/core/kqp/runtime/kqp_scan_data.cpp @@ -445,7 +445,7 @@ TIntrusivePtr<IKqpTableReader> TKqpScanComputeContext::ReadTable(ui32) const { class TKqpTableReader : public IKqpTableReader { public: - TKqpTableReader(TKqpScanComputeContext::TScanData& scanData) + TKqpTableReader(TKqpScanComputeContext::TScanData& scanData) : ScanData(scanData) {} @@ -461,30 +461,30 @@ public: return NUdf::EFetchStatus::Ok; } - EFetchResult Next(NUdf::TUnboxedValue* const* result) override { + EFetchResult Next(NUdf::TUnboxedValue* const* result) override { if (ScanData.IsEmpty()) { if (ScanData.IsFinished()) { - return EFetchResult::Finish; - } + return EFetchResult::Finish; + } return EFetchResult::Yield; - } - + } + auto row = ScanData.TakeRow(); for (ui32 i = 0; i < ScanData.GetColumns().size() + ScanData.GetSystemColumns().size(); ++i) { - if (result[i]) { + if (result[i]) { *result[i] = std::move(row.GetElement(i)); - } - } - - return EFetchResult::One; - } - + } + } + + return EFetchResult::One; + } + private: TKqpScanComputeContext::TScanData& ScanData; }; -TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TScanData& scanData) { - return MakeIntrusive<TKqpTableReader>(scanData); +TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TScanData& scanData) { + return MakeIntrusive<TKqpTableReader>(scanData); } } // namespace NMiniKQL diff --git a/ydb/core/kqp/runtime/kqp_scan_data.h b/ydb/core/kqp/runtime/kqp_scan_data.h index c8e7d82b328..d3db62aecf0 100644 --- a/ydb/core/kqp/runtime/kqp_scan_data.h +++ b/ydb/core/kqp/runtime/kqp_scan_data.h @@ -10,7 +10,7 @@ #include <ydb/library/yql/dq/actors/protos/dq_stats.pb.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h> - + #include <library/cpp/actors/core/log.h> #include <contrib/libs/apache/arrow/cpp/src/arrow/api.h> @@ -31,7 +31,7 @@ public: virtual NUdf::EFetchStatus Next(NUdf::TUnboxedValue& result) = 0; virtual EFetchResult Next(NUdf::TUnboxedValue* const* output) = 0; }; - + class TKqpScanComputeContext : public TKqpComputeContextBase { public: class TScanData { @@ -54,17 +54,17 @@ public: const TSmallVec<TColumn>& GetColumns() const { return Columns; } - + const TSmallVec<TColumn>& GetSystemColumns() const { return SystemColumns; - } + } ui64 AddRows(const TVector<TOwnedCellVec>& batch, TMaybe<ui64> shardId, const THolderFactory& holderFactory); ui64 AddRows(const arrow::RecordBatch& batch, TMaybe<ui64> shardId, const THolderFactory& holderFactory); NUdf::TUnboxedValue TakeRow(); - + bool IsEmpty() const { return RowBatches.empty(); } @@ -84,7 +84,7 @@ public: void Clear() { RowBatches.clear(); } - + public: ui64 TaskId = 0; TTableId TableId; @@ -98,7 +98,7 @@ public: struct TBasicStats { size_t Rows = 0; size_t Bytes = 0; - ui32 AffectedShards = 0; + ui32 AffectedShards = 0; }; struct TProfileStats { @@ -156,7 +156,7 @@ private: TMap<ui32, TScanData> Scans; }; -TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TScanData& scanData); +TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TScanData& scanData); } // namespace NMiniKQL } // namespace NKikimr diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.cpp b/ydb/core/kqp/ut/common/kqp_ut_common.cpp index 9bc7889f92f..5659da5ecf8 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.cpp +++ b/ydb/core/kqp/ut/common/kqp_ut_common.cpp @@ -638,61 +638,61 @@ TString StreamResultToYson(NYdb::NScripting::TYqlResultPartIterator& it) { return out.Str(); } -template<typename TIterator> -TCollectedStreamResult CollectStreamResultImpl(TIterator& it) { - TCollectedStreamResult res; - - TStringStream out; +template<typename TIterator> +TCollectedStreamResult CollectStreamResultImpl(TIterator& it) { + TCollectedStreamResult res; + + TStringStream out; NYson::TYsonWriter resultSetWriter(&out, NYson::EYsonFormat::Text, ::NYson::EYsonType::Node, true); - resultSetWriter.OnBeginList(); - - for (;;) { - auto streamPart = it.ReadNext().GetValueSync(); - if (!streamPart.IsSuccess()) { - UNIT_ASSERT_C(streamPart.EOS(), streamPart.GetIssues().ToString()); - break; - } - + resultSetWriter.OnBeginList(); + + for (;;) { + auto streamPart = it.ReadNext().GetValueSync(); + if (!streamPart.IsSuccess()) { + UNIT_ASSERT_C(streamPart.EOS(), streamPart.GetIssues().ToString()); + break; + } + if constexpr (std::is_same_v<TIterator, NYdb::NTable::TScanQueryPartIterator>) { UNIT_ASSERT_C(streamPart.HasResultSet() || streamPart.HasQueryStats(), "Unexpected empty scan query response."); } - if (streamPart.HasResultSet()) { + if (streamPart.HasResultSet()) { auto resultSet = streamPart.ExtractResultSet(); PrintResultSet(resultSet, resultSetWriter); - } - - if constexpr (std::is_same_v<TIterator, NYdb::NTable::TScanQueryPartIterator>) { - if (streamPart.HasQueryStats() ) { - res.QueryStats = NYdb::TProtoAccessor::GetProto(streamPart.GetQueryStats()); + } + + if constexpr (std::is_same_v<TIterator, NYdb::NTable::TScanQueryPartIterator>) { + if (streamPart.HasQueryStats() ) { + res.QueryStats = NYdb::TProtoAccessor::GetProto(streamPart.GetQueryStats()); auto plan = res.QueryStats->query_plan(); if (!plan.empty()) { res.PlanJson = plan; } - } - } else { - if (streamPart.HasPlan()) { - res.PlanJson = streamPart.ExtractPlan(); - } - } - } - - resultSetWriter.OnEndList(); - - res.ResultSetYson = out.Str(); - return res; -} - -TCollectedStreamResult CollectStreamResult(NYdb::NExperimental::TStreamPartIterator& it) { - return CollectStreamResultImpl(it); -} - -TCollectedStreamResult CollectStreamResult(NYdb::NTable::TScanQueryPartIterator& it) { - return CollectStreamResultImpl(it); -} - + } + } else { + if (streamPart.HasPlan()) { + res.PlanJson = streamPart.ExtractPlan(); + } + } + } + + resultSetWriter.OnEndList(); + + res.ResultSetYson = out.Str(); + return res; +} + +TCollectedStreamResult CollectStreamResult(NYdb::NExperimental::TStreamPartIterator& it) { + return CollectStreamResultImpl(it); +} + +TCollectedStreamResult CollectStreamResult(NYdb::NTable::TScanQueryPartIterator& it) { + return CollectStreamResultImpl(it); +} + TString ReadTablePartToYson(NYdb::NTable::TSession session, const TString& table) { auto it = session.ReadTable(table).GetValueSync(); UNIT_ASSERT(it.IsSuccess()); @@ -710,9 +710,9 @@ TString ReadTablePartToYson(NYdb::NTable::TSession session, const TString& table ui32 CountPlanNodesByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value) { ui32 result = 0; - - if (plan.IsArray()) { - for (const auto &node: plan.GetArray()) { + + if (plan.IsArray()) { + for (const auto &node: plan.GetArray()) { result += CountPlanNodesByKv(node, key, value); } return result; @@ -747,29 +747,29 @@ ui32 CountPlanNodesByKv(const NJson::TJsonValue& plan, const TString& key, const NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value) { if (plan.IsArray()) { for (const auto &node: plan.GetArray()) { - auto stage = FindPlanNodeByKv(node, key, value); - if (stage.IsDefined()) { - return stage; - } - } - } else if (plan.IsMap()) { - auto map = plan.GetMap(); + auto stage = FindPlanNodeByKv(node, key, value); + if (stage.IsDefined()) { + return stage; + } + } + } else if (plan.IsMap()) { + auto map = plan.GetMap(); if (map.contains(key) && map.at(key).GetStringRobust() == value) { - return plan; - } - if (map.contains("Plans")) { - for (const auto &node: map["Plans"].GetArraySafe()) { - auto stage = FindPlanNodeByKv(node, key, value); - if (stage.IsDefined()) { - return stage; - } - } - } else if (map.contains("Plan")) { - auto stage = FindPlanNodeByKv(map.at("Plan"), key, value); - if (stage.IsDefined()) { - return stage; - } - } + return plan; + } + if (map.contains("Plans")) { + for (const auto &node: map["Plans"].GetArraySafe()) { + auto stage = FindPlanNodeByKv(node, key, value); + if (stage.IsDefined()) { + return stage; + } + } + } else if (map.contains("Plan")) { + auto stage = FindPlanNodeByKv(map.at("Plan"), key, value); + if (stage.IsDefined()) { + return stage; + } + } if (map.contains("Operators")) { for (const auto &node : map["Operators"].GetArraySafe()) { @@ -779,13 +779,13 @@ NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& } } } - } else { - Y_ASSERT(false); - } - - return NJson::TJsonValue(); -} - + } else { + Y_ASSERT(false); + } + + return NJson::TJsonValue(); +} + void CreateSampleTablesWithIndex(TSession& session) { auto res = session.ExecuteSchemeQuery(R"( --!syntax_v1 diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.h b/ydb/core/kqp/ut/common/kqp_ut_common.h index c1d64b65c3e..ac82850edbb 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.h +++ b/ydb/core/kqp/ut/common/kqp_ut_common.h @@ -12,7 +12,7 @@ #include <ydb/library/yql/core/issue/yql_issue.h> -#include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_reader.h> #include <library/cpp/testing/unittest/tests_data.h> #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/yson/writer.h> @@ -54,17 +54,17 @@ void N(NUnitTest::TTestContext&) template <bool UseNewEngine, bool ForceVersionV1 = false> -TString Query(const TString& tmpl) { +TString Query(const TString& tmpl) { return TStringBuilder() << (ForceVersionV1 ? "--!syntax_v1\n" : "") << "PRAGMA Kikimr.UseNewEngine = '" << (UseNewEngine ? "true" : "false") << "';" << Endl //<< (UseNewEngine ? "PRAGMA Kikimr.UseNewEngine = 'true';" : "") << tmpl; -} - +} + #define Q_(expr) Query<UseNewEngine, false>(expr) #define Q1_(expr) Query<UseNewEngine, true>(expr) - + namespace NKikimr { namespace NKqp { @@ -151,15 +151,15 @@ private: THolder<NYdb::TDriver> Driver; }; -struct TCollectedStreamResult { - TString ResultSetYson; +struct TCollectedStreamResult { + TString ResultSetYson; TMaybe<TString> PlanJson; TMaybe<Ydb::TableStats::QueryStats> QueryStats; -}; - -TCollectedStreamResult CollectStreamResult(NYdb::NExperimental::TStreamPartIterator& it); -TCollectedStreamResult CollectStreamResult(NYdb::NTable::TScanQueryPartIterator& it); - +}; + +TCollectedStreamResult CollectStreamResult(NYdb::NExperimental::TStreamPartIterator& it); +TCollectedStreamResult CollectStreamResult(NYdb::NTable::TScanQueryPartIterator& it); + enum class EIndexTypeSql { Global, GlobalSync, @@ -223,8 +223,8 @@ TString StreamResultToYson(NYdb::NTable::TScanQueryPartIterator& it); TString StreamResultToYson(NYdb::NScripting::TYqlResultPartIterator& it); ui32 CountPlanNodesByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value); -NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value); - +NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value); + TString ReadTablePartToYson(NYdb::NTable::TSession session, const TString& table); inline void AssertSuccessResult(const NYdb::TStatus& result) { diff --git a/ydb/core/kqp/ut/kqp_explain_ut.cpp b/ydb/core/kqp/ut/kqp_explain_ut.cpp index c3a06836367..e8bca37e9b0 100644 --- a/ydb/core/kqp/ut/kqp_explain_ut.cpp +++ b/ydb/core/kqp/ut/kqp_explain_ut.cpp @@ -1,141 +1,141 @@ #include <ydb/core/kqp/ut/common/kqp_ut_common.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> - + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> -namespace NKikimr { -namespace NKqp { - -using namespace NYdb; -using namespace NYdb::NTable; - -namespace { -void CreateSampleTables(TKikimrRunner& kikimr) { - kikimr.GetTestClient().CreateTable("/Root", R"( - Name: "FourShard" - Columns { Name: "Key", Type: "Uint64" } - Columns { Name: "Value1", Type: "String" } - Columns { Name: "Value2", Type: "String" } - KeyColumnNames: ["Key"], - SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 100 } } } } - SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 200 } } } } - SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 300 } } } } - )"); - - TTableClient tableClient{kikimr.GetDriver()}; - auto session = tableClient.CreateSession().GetValueSync().GetSession(); - - auto result = session.ExecuteDataQuery(R"( +namespace NKikimr { +namespace NKqp { + +using namespace NYdb; +using namespace NYdb::NTable; + +namespace { +void CreateSampleTables(TKikimrRunner& kikimr) { + kikimr.GetTestClient().CreateTable("/Root", R"( + Name: "FourShard" + Columns { Name: "Key", Type: "Uint64" } + Columns { Name: "Value1", Type: "String" } + Columns { Name: "Value2", Type: "String" } + KeyColumnNames: ["Key"], + SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 100 } } } } + SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 200 } } } } + SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 300 } } } } + )"); + + TTableClient tableClient{kikimr.GetDriver()}; + auto session = tableClient.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExecuteDataQuery(R"( REPLACE INTO `/Root/FourShard` (Key, Value1, Value2) VALUES - (1u, "Value-001", "1"), - (2u, "Value-002", "2"), - (101u, "Value-101", "101"), - (102u, "Value-102", "102"), - (201u, "Value-201", "201"), - (202u, "Value-202", "202"), - (301u, "Value-301", "301"), - (302u, "Value-302", "302") - )", TTxControl::BeginTx().CommitTx()).GetValueSync(); - - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - session.Close(); -} - -} - -Y_UNIT_TEST_SUITE(KqpExplain) { - - Y_UNIT_TEST(Explain) { - TKikimrRunner kikimr; + (1u, "Value-001", "1"), + (2u, "Value-002", "2"), + (101u, "Value-101", "101"), + (102u, "Value-102", "102"), + (201u, "Value-201", "201"), + (202u, "Value-202", "202"), + (301u, "Value-301", "301"), + (302u, "Value-302", "302") + )", TTxControl::BeginTx().CommitTx()).GetValueSync(); + + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + session.Close(); +} + +} + +Y_UNIT_TEST_SUITE(KqpExplain) { + + Y_UNIT_TEST(Explain) { + TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); TStreamExecScanQuerySettings settings; - settings.Explain(true); - + settings.Explain(true); + auto it = db.StreamExecuteScanQuery(R"( SELECT count(*) FROM `/Root/EightShard` AS t JOIN `/Root/KeyValue` AS kv ON t.Data = kv.Key; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT(res.PlanJson); - + Cerr << *res.PlanJson; - NJson::TJsonValue plan; + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - + auto join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan"); - UNIT_ASSERT(join.IsDefined()); + UNIT_ASSERT(join.IsDefined()); auto left = FindPlanNodeByKv(join, "Table", "EightShard"); - UNIT_ASSERT(left.IsDefined()); + UNIT_ASSERT(left.IsDefined()); auto right = FindPlanNodeByKv(join, "Table", "KeyValue"); - UNIT_ASSERT(right.IsDefined()); - } - - Y_UNIT_TEST(ExplainStream) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - TStreamExecScanQuerySettings settings; - settings.Explain(true); - - auto it = db.StreamExecuteScanQuery(R"( + UNIT_ASSERT(right.IsDefined()); + } + + Y_UNIT_TEST(ExplainStream) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + TStreamExecScanQuerySettings settings; + settings.Explain(true); + + auto it = db.StreamExecuteScanQuery(R"( SELECT count(*) FROM `/Root/EightShard` AS t JOIN `/Root/KeyValue` AS kv ON t.Data = kv.Key; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT(res.PlanJson); Cerr << *res.PlanJson; - - NJson::TJsonValue plan; + + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - + auto join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan"); - UNIT_ASSERT(join.IsDefined()); + UNIT_ASSERT(join.IsDefined()); auto left = FindPlanNodeByKv(join, "Table", "EightShard"); - UNIT_ASSERT(left.IsDefined()); + UNIT_ASSERT(left.IsDefined()); auto right = FindPlanNodeByKv(join, "Table", "KeyValue"); - UNIT_ASSERT(right.IsDefined()); - } - - Y_UNIT_TEST(ExplainScanQueryWithParams) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - TStreamExecScanQuerySettings settings; - settings.Explain(true); - - auto it = db.StreamExecuteScanQuery(R"( + UNIT_ASSERT(right.IsDefined()); + } + + Y_UNIT_TEST(ExplainScanQueryWithParams) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + TStreamExecScanQuerySettings settings; + settings.Explain(true); + + auto it = db.StreamExecuteScanQuery(R"( --SELECT count(*) FROM `/Root/EightShard` AS t JOIN `/Root/KeyValue` AS kv ON t.Data = kv.Key; - PRAGMA Kikimr.UseNewEngine = "false"; - DECLARE $value as Utf8; - SELECT $value as value; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - } - - Y_UNIT_TEST(AggGroupLimit) { - TKikimrRunner kikimr; + PRAGMA Kikimr.UseNewEngine = "false"; + DECLARE $value as Utf8; + SELECT $value as value; + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + } + + Y_UNIT_TEST(AggGroupLimit) { + TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); TStreamExecScanQuerySettings settings; - settings.Explain(true); - + settings.Explain(true); + auto it = db.StreamExecuteScanQuery(R"( SELECT min(Message), max(Message) FROM `/Root/Logs` WHERE Ts > 1 and Ts <= 4 or App="ydb" GROUP BY App; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT(res.PlanJson); Cerr << *res.PlanJson; - - NJson::TJsonValue plan; + + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - + auto read = FindPlanNodeByKv(plan, "Node Type", "Aggregate-Filter-TableFullScan"); - UNIT_ASSERT(read.IsDefined()); + UNIT_ASSERT(read.IsDefined()); auto tables = read.GetMapSafe().at("Tables").GetArraySafe(); UNIT_ASSERT(tables[0].GetStringSafe() == "Logs"); auto shuffle = FindPlanNodeByKv(plan, "Node Type", "HashShuffle"); @@ -147,94 +147,94 @@ Y_UNIT_TEST_SUITE(KqpExplain) { UNIT_ASSERT(aggregate.GetMapSafe().at("GroupBy").GetStringSafe() == "item.App"); UNIT_ASSERT(aggregate.GetMapSafe().at("Aggregation").GetStringSafe() == "{_yql_agg_0: MIN(item.Message),_yql_agg_1: MAX(item.Message)}"); - } - - Y_UNIT_TEST(ComplexJoin) { - TKikimrRunner kikimr; - CreateSampleTables(kikimr); + } + + Y_UNIT_TEST(ComplexJoin) { + TKikimrRunner kikimr; + CreateSampleTables(kikimr); auto db = kikimr.GetTableClient(); TStreamExecScanQuerySettings settings; - settings.Explain(true); - + settings.Explain(true); + auto it = db.StreamExecuteScanQuery(R"( - $join = ( - SELECT l.Key as Key, l.Text as Text, l.Data as Data, r.Value1 as Value1, r.Value2 as Value2 + $join = ( + SELECT l.Key as Key, l.Text as Text, l.Data as Data, r.Value1 as Value1, r.Value2 as Value2 FROM `/Root/EightShard` AS l JOIN `/Root/FourShard` AS r ON l.Key = r.Key - ); - SELECT Key, COUNT(*) AS Cnt - FROM $join - WHERE Cast(Data As Int64) < (Key - 100) and Value1 != 'Value-101' - GROUP BY Key - UNION ALL + ); + SELECT Key, COUNT(*) AS Cnt + FROM $join + WHERE Cast(Data As Int64) < (Key - 100) and Value1 != 'Value-101' + GROUP BY Key + UNION ALL (SELECT Key FROM `/Root/KeyValue` ORDER BY Key LIMIT 1) - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT(res.PlanJson); - - NJson::TJsonValue plan; + + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - + auto join = FindPlanNodeByKv( plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan" ); - UNIT_ASSERT(join.IsDefined()); + UNIT_ASSERT(join.IsDefined()); auto left = FindPlanNodeByKv(join, "Table", "EightShard"); - UNIT_ASSERT(left.IsDefined()); + UNIT_ASSERT(left.IsDefined()); auto right = FindPlanNodeByKv(join, "Table", "FourShard"); - UNIT_ASSERT(right.IsDefined()); - } - - Y_UNIT_TEST(PrecomputeRange) { - TKikimrRunner kikimr; + UNIT_ASSERT(right.IsDefined()); + } + + Y_UNIT_TEST(PrecomputeRange) { + TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); TStreamExecScanQuerySettings settings; - settings.Explain(true); - + settings.Explain(true); + auto it = db.StreamExecuteScanQuery(R"( SELECT * FROM `/Root/EightShard` WHERE Key BETWEEN 150 AND 266 ORDER BY Data LIMIT 4; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT(res.PlanJson); Cerr << *res.PlanJson; - - NJson::TJsonValue plan; + + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - + auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangesScan"); - UNIT_ASSERT(node.IsDefined()); + UNIT_ASSERT(node.IsDefined()); auto operators = node.GetMapSafe().at("Operators").GetArraySafe(); UNIT_ASSERT(operators[1].GetMapSafe().at("Name") == "TableRangesScan"); auto& readRanges = operators[1].GetMapSafe().at("ReadRanges").GetArraySafe(); UNIT_ASSERT(readRanges[0] == "Key [150, 266]"); - } - - Y_UNIT_TEST(CompoundKeyRange) { - TKikimrRunner kikimr; + } + + Y_UNIT_TEST(CompoundKeyRange) { + TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); TStreamExecScanQuerySettings settings; - settings.Explain(true); - + settings.Explain(true); + auto it = db.StreamExecuteScanQuery(R"( PRAGMA Kikimr.OptEnablePredicateExtract = "false"; SELECT * FROM `/Root/Logs` WHERE App = "new_app_1" AND Host < "xyz" AND Ts = (42+7) Limit 10; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT(res.PlanJson); Cerr << *res.PlanJson; - - NJson::TJsonValue plan; + + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - + auto read = FindPlanNodeByKv(plan, "Node Type", "Limit-TablePointLookup"); auto& operators = read.GetMapSafe().at("Operators").GetArraySafe(); UNIT_ASSERT(operators.size() == 2); @@ -242,134 +242,134 @@ Y_UNIT_TEST_SUITE(KqpExplain) { auto& lookup = operators[1].GetMapSafe(); UNIT_ASSERT(lookup.at("Name") == "TablePointLookup"); UNIT_ASSERT(lookup.at("ReadRange").GetArraySafe()[0] == "App (new_app_1)"); - } - - Y_UNIT_TEST(SortStage) { - TKikimrRunner kikimr; + } + + Y_UNIT_TEST(SortStage) { + TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); TStreamExecScanQuerySettings settings; - settings.Explain(true); - + settings.Explain(true); + auto it = db.StreamExecuteScanQuery(R"( SELECT * FROM `/Root/EightShard` WHERE Key BETWEEN 150 AND 266 ORDER BY Text; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT(res.PlanJson); Cerr << *res.PlanJson; - - NJson::TJsonValue plan; + + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); auto scanSort = FindPlanNodeByKv(plan, "Node Type", "Sort-TableRangesScan"); - UNIT_ASSERT(scanSort.IsDefined()); - } - - Y_UNIT_TEST(LimitOffset) { - TKikimrRunner kikimr; + UNIT_ASSERT(scanSort.IsDefined()); + } + + Y_UNIT_TEST(LimitOffset) { + TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); TStreamExecScanQuerySettings settings; - settings.Explain(true); - + settings.Explain(true); + auto it = db.StreamExecuteScanQuery(R"( SELECT * FROM `/Root/EightShard` ORDER BY Text LIMIT 10 OFFSET 15; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT(res.PlanJson); Cerr << *res.PlanJson; - - NJson::TJsonValue plan; + + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - - auto limit = FindPlanNodeByKv(plan, "Limit", "10"); - UNIT_ASSERT(limit.IsDefined()); - auto offset = FindPlanNodeByKv(plan, "Offset", "15"); - UNIT_ASSERT(offset.IsDefined()); - } - - Y_UNIT_TEST(SelfJoin3xSameLabels) { - TKikimrRunner kikimr; + + auto limit = FindPlanNodeByKv(plan, "Limit", "10"); + UNIT_ASSERT(limit.IsDefined()); + auto offset = FindPlanNodeByKv(plan, "Offset", "15"); + UNIT_ASSERT(offset.IsDefined()); + } + + Y_UNIT_TEST(SelfJoin3xSameLabels) { + TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); TStreamExecScanQuerySettings settings; - settings.Explain(true); - + settings.Explain(true); + auto it = db.StreamExecuteScanQuery(R"( - $foo = ( - SELECT t1.Key AS Key + $foo = ( + SELECT t1.Key AS Key FROM `/Root/KeyValue` AS t1 JOIN `/Root/KeyValue` AS t2 - ON t1.Key = t2.Key - GROUP BY t1.Key - ); - SELECT t1.Key AS Key - FROM $foo AS Foo + ON t1.Key = t2.Key + GROUP BY t1.Key + ); + SELECT t1.Key AS Key + FROM $foo AS Foo JOIN `/Root/KeyValue` AS t1 - ON t1.Key = Foo.Key - ORDER BY Key - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + ON t1.Key = Foo.Key + ORDER BY Key + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT(res.PlanJson); Cerr << *res.PlanJson; - - NJson::TJsonValue plan; + + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - + auto join1 = FindPlanNodeByKv(plan, "Node Type", "Sort-InnerJoin (MapJoin)-Filter-Aggregate-Sort"); - UNIT_ASSERT(join1.IsDefined()); + UNIT_ASSERT(join1.IsDefined()); auto join2 = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter"); - UNIT_ASSERT(join2.IsDefined()); - } - - Y_UNIT_TEST(PureExpr) { - TKikimrRunner kikimr; + UNIT_ASSERT(join2.IsDefined()); + } + + Y_UNIT_TEST(PureExpr) { + TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); TStreamExecScanQuerySettings settings; - settings.Explain(true); - + settings.Explain(true); + auto it = db.StreamExecuteScanQuery(R"( - SELECT 1,2,3 UNION ALL SELECT 4,5,6; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + SELECT 1,2,3 UNION ALL SELECT 4,5,6; + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT(res.PlanJson); Cerr << *res.PlanJson; - - NJson::TJsonValue plan; + + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - - auto constExpr = FindPlanNodeByKv(plan, "Node Type", "ConstantExpr"); - UNIT_ASSERT(constExpr.IsDefined()); - } - - Y_UNIT_TEST(MultiUsedStage) { - NKikimrConfig::TAppConfig appCfg; - auto* spilling = appCfg.MutableTableServiceConfig()->MutableSpillingServiceConfig()->MutableLocalFileConfig(); - spilling->SetEnable(true); - spilling->SetRoot("./spilling/"); - TKikimrRunner kikimr(appCfg); - + + auto constExpr = FindPlanNodeByKv(plan, "Node Type", "ConstantExpr"); + UNIT_ASSERT(constExpr.IsDefined()); + } + + Y_UNIT_TEST(MultiUsedStage) { + NKikimrConfig::TAppConfig appCfg; + auto* spilling = appCfg.MutableTableServiceConfig()->MutableSpillingServiceConfig()->MutableLocalFileConfig(); + spilling->SetEnable(true); + spilling->SetRoot("./spilling/"); + TKikimrRunner kikimr(appCfg); + auto db = kikimr.GetTableClient(); TStreamExecScanQuerySettings settings; - settings.Explain(true); - + settings.Explain(true); + auto it = db.StreamExecuteScanQuery(R"( select count(*) from `/Root/KeyValue` AS t1 join `/Root/KeyValue` AS t2 on t1.Key = t2.Key; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT(res.PlanJson); Cerr << *res.PlanJson; - - NJson::TJsonValue plan; + + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - + bool containCte = false; auto& plans = plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans"); for (auto& planNode : plans.GetArraySafe()) { @@ -381,90 +381,90 @@ Y_UNIT_TEST_SUITE(KqpExplain) { } UNIT_ASSERT(containCte); - } - - Y_UNIT_TEST(SqlIn) { - TKikimrRunner kikimr; + } + + Y_UNIT_TEST(SqlIn) { + TKikimrRunner kikimr; CreateSampleTables(kikimr); TStreamExecScanQuerySettings settings; - settings.Explain(true); + settings.Explain(true); auto db = kikimr.GetTableClient(); - - auto query = R"( + + auto query = R"( PRAGMA Kikimr.OptEnablePredicateExtract = "false"; - SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42) - ORDER BY Key - )"; - + SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42) + ORDER BY Key + )"; + auto it = db.StreamExecuteScanQuery(query, settings).GetValueSync(); - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT(res.PlanJson); Cerr << *res.PlanJson; - - NJson::TJsonValue plan; + + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - + auto unionNode = FindPlanNodeByKv(plan, "Node Type", "Sort-Union"); - UNIT_ASSERT_EQUAL(unionNode.GetMap().at("Plans").GetArraySafe().size(), 4); - } - - Y_UNIT_TEST(ExplainDataQueryOldEngine) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto result = session.ExplainDataQuery(R"( - SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42) ORDER BY Key; - )").ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - - NJson::TJsonValue plan; - NJson::ReadJsonTree(result.GetPlan(), &plan, true); - UNIT_ASSERT_EQUAL(plan.GetMapSafe().at("tables").GetArraySafe()[0].GetMapSafe().at("name").GetStringSafe(), "/Root/KeyValue"); - } - - Y_UNIT_TEST(ExplainDataQueryNewEngine) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto result = session.ExplainDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; - SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42) ORDER BY Key; - SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 4*2) ORDER BY Key; - SELECT count(distinct Value) FROM `/Root/KeyValue` WHERE Key > 20 and Key <= 120; - SELECT count(distinct Value) FROM `/Root/KeyValue`; - SELECT count(distinct Value) FROM `/Root/KeyValue` WHERE Key >= 10; - )").ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - - NJson::TJsonValue plan; - NJson::ReadJsonTree(result.GetPlan(), &plan, true); + UNIT_ASSERT_EQUAL(unionNode.GetMap().at("Plans").GetArraySafe().size(), 4); + } + + Y_UNIT_TEST(ExplainDataQueryOldEngine) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExplainDataQuery(R"( + SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42) ORDER BY Key; + )").ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(result.GetPlan(), &plan, true); + UNIT_ASSERT_EQUAL(plan.GetMapSafe().at("tables").GetArraySafe()[0].GetMapSafe().at("name").GetStringSafe(), "/Root/KeyValue"); + } + + Y_UNIT_TEST(ExplainDataQueryNewEngine) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExplainDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; + SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42) ORDER BY Key; + SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 4*2) ORDER BY Key; + SELECT count(distinct Value) FROM `/Root/KeyValue` WHERE Key > 20 and Key <= 120; + SELECT count(distinct Value) FROM `/Root/KeyValue`; + SELECT count(distinct Value) FROM `/Root/KeyValue` WHERE Key >= 10; + )").ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(result.GetPlan(), &plan, true); auto node = FindPlanNodeByKv(plan, "Name", "TableRangeScan"); UNIT_ASSERT_EQUAL(node.GetMapSafe().at("Table").GetStringSafe(), "KeyValue"); node = FindPlanNodeByKv(plan, "Name", "TableFullScan"); UNIT_ASSERT_EQUAL(node.GetMapSafe().at("Table").GetStringSafe(), "KeyValue"); node = FindPlanNodeByKv(plan, "Name", "TablePointLookup"); UNIT_ASSERT_EQUAL(node.GetMapSafe().at("Table").GetStringSafe(), "KeyValue"); - } - - Y_UNIT_TEST(FewEffects) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto result = session.ExplainDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; + } + + Y_UNIT_TEST(FewEffects) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExplainDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; UPDATE `/Root/EightShard` SET Data=Data+1; UPDATE `/Root/EightShard` SET Data=Data-1 WHERE Key In (100,200,300); DELETE FROM `/Root/EightShard` WHERE Key > 350; - )").ExtractValueSync(); + )").ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - NJson::TJsonValue plan; - NJson::ReadJsonTree(result.GetPlan(), &plan, true); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(result.GetPlan(), &plan, true); // Cerr << plan << Endl; @@ -502,28 +502,28 @@ Y_UNIT_TEST_SUITE(KqpExplain) { UNIT_ASSERT_VALUES_EQUAL(counter["FullScan"], fullScansCount); UNIT_ASSERT_VALUES_EQUAL(counter["Scan"], rangeScansCount); UNIT_ASSERT_VALUES_EQUAL(counter["Lookup"], lookupsCount); - } - - Y_UNIT_TEST(ExplainDataQueryWithParams) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto result = session.ExplainDataQuery(R"( - PRAGMA Kikimr.UseNewEngine = "false"; - DECLARE $value as Utf8; - SELECT $value as value; - )").ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - auto result2 = session.ExplainDataQuery(R"( - PRAGMA Kikimr.UseNewEngine = "true"; - DECLARE $value as Utf8; - SELECT $value as value; - )").ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), EStatus::SUCCESS, result2.GetIssues().ToString()); - } - + } + + Y_UNIT_TEST(ExplainDataQueryWithParams) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExplainDataQuery(R"( + PRAGMA Kikimr.UseNewEngine = "false"; + DECLARE $value as Utf8; + SELECT $value as value; + )").ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto result2 = session.ExplainDataQuery(R"( + PRAGMA Kikimr.UseNewEngine = "true"; + DECLARE $value as Utf8; + SELECT $value as value; + )").ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), EStatus::SUCCESS, result2.GetIssues().ToString()); + } + Y_UNIT_TEST(FullOuterJoin) { TKikimrRunner kikimr; CreateSampleTables(kikimr); @@ -712,9 +712,9 @@ Y_UNIT_TEST_SUITE(KqpExplain) { << " but received: " << filter.GetMapSafe().at("Predicate")); } } -} - -} // namespace NKqp -} // namespace NKikimr - - +} + +} // namespace NKqp +} // namespace NKikimr + + diff --git a/ydb/core/kqp/ut/kqp_join_ut.cpp b/ydb/core/kqp/ut/kqp_join_ut.cpp index 8fa59d31b36..3c28faa0e99 100644 --- a/ydb/core/kqp/ut/kqp_join_ut.cpp +++ b/ydb/core/kqp/ut/kqp_join_ut.cpp @@ -301,21 +301,21 @@ Y_UNIT_TEST_SUITE(KqpJoin) { AssertTableReads(result, "/Root/Join1_2", 3); } - Y_UNIT_TEST_NEW_ENGINE(LeftJoinWithNull) { + Y_UNIT_TEST_NEW_ENGINE(LeftJoinWithNull) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTables(session); - auto result = session.ExecuteDataQuery(Q_(R"( + auto result = session.ExecuteDataQuery(Q_(R"( SELECT * FROM `/Root/Join1_1` AS t1 INNER JOIN `/Root/Join1_2` AS t2 ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2 LEFT JOIN `/Root/Join1_3` AS t3 ON t2.Fk3 = t3.Key WHERE t1.Value == "Value5" AND t2.Value == "Value31"; - )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT(result.IsSuccess()); CompareYson(R"([[[108];["One"];[8];["Value5"];#;[108];["One"];["Value31"];#;#]])", @@ -762,14 +762,14 @@ Y_UNIT_TEST_SUITE(KqpJoin) { FormatResultSetYson(result.GetResultSet(0))); } - Y_UNIT_TEST_NEW_ENGINE(JoinAggregate) { + Y_UNIT_TEST_NEW_ENGINE(JoinAggregate) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); CreateSampleTables(session); { - auto result = session.ExecuteDataQuery(Q_(R"( + auto result = session.ExecuteDataQuery(Q_(R"( SELECT t1.Value, SUM(t3.Value) FROM `/Root/Join1_1` AS t1 INNER JOIN `/Root/Join1_2` AS t2 @@ -777,7 +777,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) { LEFT JOIN `/Root/Join1_3` AS t3 ON t2.Fk3 = t3.Key GROUP BY t1.Value; - )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); CompareYson(R"([[["Value1"];[3004]];[["Value2"];[1001]];[["Value3"];[2006]];[["Value5"];#]])", diff --git a/ydb/core/kqp/ut/kqp_newengine_ut.cpp b/ydb/core/kqp/ut/kqp_newengine_ut.cpp index 3737687d8d6..3fca42841a9 100644 --- a/ydb/core/kqp/ut/kqp_newengine_ut.cpp +++ b/ydb/core/kqp/ut/kqp_newengine_ut.cpp @@ -710,68 +710,68 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { } #endif - Y_UNIT_TEST(PureExpr) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); + Y_UNIT_TEST(PureExpr) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); NYdb::NTable::TExecDataQuerySettings execSettings; execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; - SELECT '42', 42, 5*5; + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; + SELECT '42', 42, 5*5; )", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx(), execSettings).ExtractValueSync(); - - CompareYson(R"([["42";42;25]])", FormatResultSetYson(result.GetResultSet(0))); + + CompareYson(R"([["42";42;25]])", FormatResultSetYson(result.GetResultSet(0))); auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); UNIT_ASSERT_EQUAL(stats.query_phases().size(), 1); UNIT_ASSERT(stats.query_phases(0).table_access().size() == 0); - } - - Y_UNIT_TEST(MultiStatement) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); + } + + Y_UNIT_TEST(MultiStatement) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); NYdb::NTable::TExecDataQuerySettings execSettings; execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; - SELECT 1; - SELECT 2; - SELECT 3; + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; + SELECT 1; + SELECT 2; + SELECT 3; )", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx(), execSettings).ExtractValueSync(); - - CompareYson(R"([[1]])", FormatResultSetYson(result.GetResultSet(0))); - CompareYson(R"([[2]])", FormatResultSetYson(result.GetResultSet(1))); - CompareYson(R"([[3]])", FormatResultSetYson(result.GetResultSet(2))); + + CompareYson(R"([[1]])", FormatResultSetYson(result.GetResultSet(0))); + CompareYson(R"([[2]])", FormatResultSetYson(result.GetResultSet(1))); + CompareYson(R"([[3]])", FormatResultSetYson(result.GetResultSet(2))); auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); UNIT_ASSERT_EQUAL(stats.query_phases().size(), 1); UNIT_ASSERT(stats.query_phases(0).table_access().size() == 0); - } - - Y_UNIT_TEST(MultiStatementMixPure) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; - SELECT 1; + } + + Y_UNIT_TEST(MultiStatementMixPure) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; + SELECT 1; SELECT Key FROM [/Root/TwoShard] ORDER BY Key DESC LIMIT 1; - SELECT 2; + SELECT 2; SELECT Key FROM [/Root/EightShard] ORDER BY Key ASC LIMIT 1; - )", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync(); - - CompareYson(R"([[1]])", FormatResultSetYson(result.GetResultSet(0))); - CompareYson(R"([[[4000000003u]]])", FormatResultSetYson(result.GetResultSet(1))); - CompareYson(R"([[2]])", FormatResultSetYson(result.GetResultSet(2))); - CompareYson(R"([[[101u]]])", FormatResultSetYson(result.GetResultSet(3))); - } - + )", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync(); + + CompareYson(R"([[1]])", FormatResultSetYson(result.GetResultSet(0))); + CompareYson(R"([[[4000000003u]]])", FormatResultSetYson(result.GetResultSet(1))); + CompareYson(R"([[2]])", FormatResultSetYson(result.GetResultSet(2))); + CompareYson(R"([[[101u]]])", FormatResultSetYson(result.GetResultSet(3))); + } + Y_UNIT_TEST(LocksSingleShard) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); diff --git a/ydb/core/kqp/ut/kqp_scan_ut.cpp b/ydb/core/kqp/ut/kqp_scan_ut.cpp index 347ca354412..f35b783bb9f 100644 --- a/ydb/core/kqp/ut/kqp_scan_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scan_ut.cpp @@ -1271,104 +1271,104 @@ Y_UNIT_TEST_SUITE(KqpScan) { UNIT_ASSERT_EQUAL_C(part.GetStatus(), EStatus::PRECONDITION_FAILED, part.GetIssues().ToString()); } - Y_UNIT_TEST(PureExpr) { - TKikimrRunner kikimr(AppCfg()); + Y_UNIT_TEST(PureExpr) { + TKikimrRunner kikimr(AppCfg()); auto db = kikimr.GetTableClient(); - + auto it = db.StreamExecuteScanQuery(R"( - SELECT '42', 42, 5*5; - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - auto res = StreamResultToYson(it); - CompareYson(R"([["42";42;25]])", res); - } - - Y_UNIT_TEST(UnionWithPureExpr) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - + SELECT '42', 42, 5*5; + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto res = StreamResultToYson(it); + CompareYson(R"([["42";42;25]])", res); + } + + Y_UNIT_TEST(UnionWithPureExpr) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + auto it = db.StreamExecuteScanQuery(R"( - SELECT '42', 42, 5*5 - UNION ALL - SELECT 'forty-two'; - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - auto res = StreamResultToYson(it); - CompareYson(R"([["42";[42];[25]];["forty-two";#;#]])", res); - } - - Y_UNIT_TEST(MiltiExprWithPure) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - + SELECT '42', 42, 5*5 + UNION ALL + SELECT 'forty-two'; + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto res = StreamResultToYson(it); + CompareYson(R"([["42";[42];[25]];["forty-two";#;#]])", res); + } + + Y_UNIT_TEST(MiltiExprWithPure) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + auto it = db.StreamExecuteScanQuery(R"( SELECT Key FROM `/Root/KeyValue` ORDER BY Key LIMIT 1; - SELECT 2; - )").GetValueSync(); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - auto part = it.ReadNext().GetValueSync(); - part.GetIssues().PrintTo(Cerr); - UNIT_ASSERT_EQUAL_C(part.GetStatus(), EStatus::PRECONDITION_FAILED, part.GetIssues().ToString()); - } - - Y_UNIT_TEST(UnionBasic) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - + SELECT 2; + )").GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto part = it.ReadNext().GetValueSync(); + part.GetIssues().PrintTo(Cerr); + UNIT_ASSERT_EQUAL_C(part.GetStatus(), EStatus::PRECONDITION_FAILED, part.GetIssues().ToString()); + } + + Y_UNIT_TEST(UnionBasic) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + auto it = db.StreamExecuteScanQuery(R"( (SELECT Key FROM `/Root/KeyValue` ORDER BY Key LIMIT 1) - UNION ALL + UNION ALL (SELECT Key FROM `/Root/EightShard` ORDER BY Key LIMIT 1); - )").GetValueSync(); - auto res = StreamResultToYson(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[[1u]];[[101u]]])", res); - } - - Y_UNIT_TEST(UnionMixed) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - + )").GetValueSync(); + auto res = StreamResultToYson(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[[1u]];[[101u]]])", res); + } + + Y_UNIT_TEST(UnionMixed) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + auto it = db.StreamExecuteScanQuery(R"( - SELECT 42 - UNION ALL + SELECT 42 + UNION ALL (SELECT Key FROM `/Root/EightShard` ORDER BY Key LIMIT 1); - )").GetValueSync(); - auto res = StreamResultToYson(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[[42];#];[#;[101u]]])", res); - } - - Y_UNIT_TEST(UnionThree) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - + )").GetValueSync(); + auto res = StreamResultToYson(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[[42];#];[#;[101u]]])", res); + } + + Y_UNIT_TEST(UnionThree) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + auto it = db.StreamExecuteScanQuery(R"( (SELECT Key FROM `/Root/KeyValue` ORDER BY Key LIMIT 1) - UNION ALL + UNION ALL (SELECT Key FROM `/Root/EightShard` ORDER BY Key LIMIT 1) - UNION ALL + UNION ALL (SELECT Key FROM `/Root/TwoShard` ORDER BY Key DESC LIMIT 1); - )").GetValueSync(); - auto res = StreamResultToYson(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[[1u]];[[101u]];[[4000000003u]]])", res); - } - - Y_UNIT_TEST(UnionSameTable) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - + )").GetValueSync(); + auto res = StreamResultToYson(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[[1u]];[[101u]];[[4000000003u]]])", res); + } + + Y_UNIT_TEST(UnionSameTable) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + auto it = db.StreamExecuteScanQuery(R"( (SELECT Key FROM `/Root/KeyValue` ORDER BY Key LIMIT 1) - UNION ALL + UNION ALL (SELECT Key FROM `/Root/KeyValue` ORDER BY Key LIMIT 1); - )").GetValueSync(); - auto res = StreamResultToYson(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[[1u]];[[1u]]])", res); - } - + )").GetValueSync(); + auto res = StreamResultToYson(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[[1u]];[[1u]]])", res); + } + Y_UNIT_TEST(UnionAggregate) { TKikimrRunner kikimr(AppCfg()); auto db = kikimr.GetTableClient(); @@ -1385,142 +1385,142 @@ Y_UNIT_TEST_SUITE(KqpScan) { CompareYson(R"([[[2u]];[[24u]];[[11000u]]])", res); } - Y_UNIT_TEST(CountDistinct) { - TKikimrRunner kikimr(AppCfg()); + Y_UNIT_TEST(CountDistinct) { + TKikimrRunner kikimr(AppCfg()); auto db = kikimr.GetTableClient(); - + auto it = db.StreamExecuteScanQuery(R"( - SELECT count(*) AS c1, count(DISTINCT Value) AS c2 + SELECT count(*) AS c1, count(DISTINCT Value) AS c2 FROM `/Root/KeyValue` GROUP BY Key; - )").GetValueSync(); - auto res = StreamResultToYson(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[1u;1u];[1u;1u]])", res); - } - - Y_UNIT_TEST(FullFrameWindow) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - + )").GetValueSync(); + auto res = StreamResultToYson(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[1u;1u];[1u;1u]])", res); + } + + Y_UNIT_TEST(FullFrameWindow) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + auto it = db.StreamExecuteScanQuery(R"( - SELECT group_total, count(*) FROM ( - SELECT Key, Text, SUM(Data) OVER w1 AS group_total + SELECT group_total, count(*) FROM ( + SELECT Key, Text, SUM(Data) OVER w1 AS group_total FROM `/Root/EightShard` WINDOW w1 AS (partition by Text) - ) GROUP BY group_total ORDER BY group_total; - )").GetValueSync(); - auto res = StreamResultToYson(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[[15];8u];[[16];8u];[[17];8u]])", res); - } - - Y_UNIT_TEST(SimpleWindow) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - + ) GROUP BY group_total ORDER BY group_total; + )").GetValueSync(); + auto res = StreamResultToYson(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[[15];8u];[[16];8u];[[17];8u]])", res); + } + + Y_UNIT_TEST(SimpleWindow) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + auto it = db.StreamExecuteScanQuery(R"( - SELECT Text, running_total FROM ( - SELECT Key, Text, SUM(Data) OVER w1 AS running_total + SELECT Text, running_total FROM ( + SELECT Key, Text, SUM(Data) OVER w1 AS running_total FROM `/Root/EightShard` - WHERE Text = 'Value2' - WINDOW w1 AS (partition by Text order by Key) - ) ORDER BY running_total; - )").GetValueSync(); - auto res = StreamResultToYson(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - CompareYson(R"([ - [["Value2"];[3]]; - [["Value2"];[4]]; - [["Value2"];[6]]; - [["Value2"];[9]]; - [["Value2"];[10]]; - [["Value2"];[12]]; - [["Value2"];[15]]; - [["Value2"];[16]] - ])", res); - } - - Y_UNIT_TEST(TwoAggregatesOneFullFrameWindow) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - + WHERE Text = 'Value2' + WINDOW w1 AS (partition by Text order by Key) + ) ORDER BY running_total; + )").GetValueSync(); + auto res = StreamResultToYson(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + CompareYson(R"([ + [["Value2"];[3]]; + [["Value2"];[4]]; + [["Value2"];[6]]; + [["Value2"];[9]]; + [["Value2"];[10]]; + [["Value2"];[12]]; + [["Value2"];[15]]; + [["Value2"];[16]] + ])", res); + } + + Y_UNIT_TEST(TwoAggregatesOneFullFrameWindow) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + auto it = db.StreamExecuteScanQuery(R"( - SELECT tot, avg, count(*) FROM ( - SELECT Key, Text, SUM(Data) OVER w1 AS tot, avg(Data) OVER w1 AS avg + SELECT tot, avg, count(*) FROM ( + SELECT Key, Text, SUM(Data) OVER w1 AS tot, avg(Data) OVER w1 AS avg FROM `/Root/EightShard` - WHERE Text = 'Value3' - WINDOW w1 AS (partition by Text) - ) GROUP BY tot, avg - )").GetValueSync(); - auto res = StreamResultToYson(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[[17];[2.125];8u]])", res); - } - - Y_UNIT_TEST(TwoAggregatesTwoWindows) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - + WHERE Text = 'Value3' + WINDOW w1 AS (partition by Text) + ) GROUP BY tot, avg + )").GetValueSync(); + auto res = StreamResultToYson(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[[17];[2.125];8u]])", res); + } + + Y_UNIT_TEST(TwoAggregatesTwoWindows) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + auto it = db.StreamExecuteScanQuery(R"( - SELECT Key, Text, sum(Data) OVER w1 AS tot, sum(Data) OVER w2 AS avg + SELECT Key, Text, sum(Data) OVER w1 AS tot, sum(Data) OVER w2 AS avg FROM `/Root/EightShard` - WHERE Text = 'Value2' - WINDOW w1 AS (partition by Text), - w2 AS (partition by Text order by Key) - ORDER BY Key; - )").GetValueSync(); - auto res = StreamResultToYson(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([ - [[102u];["Value2"];[16];[3]]; - [[202u];["Value2"];[16];[4]]; - [[302u];["Value2"];[16];[6]]; - [[402u];["Value2"];[16];[9]]; - [[502u];["Value2"];[16];[10]]; - [[602u];["Value2"];[16];[12]]; - [[702u];["Value2"];[16];[15]]; - [[802u];["Value2"];[16];[16]] - ])", res); - } - - Y_UNIT_TEST(CustomWindow) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - + WHERE Text = 'Value2' + WINDOW w1 AS (partition by Text), + w2 AS (partition by Text order by Key) + ORDER BY Key; + )").GetValueSync(); + auto res = StreamResultToYson(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([ + [[102u];["Value2"];[16];[3]]; + [[202u];["Value2"];[16];[4]]; + [[302u];["Value2"];[16];[6]]; + [[402u];["Value2"];[16];[9]]; + [[502u];["Value2"];[16];[10]]; + [[602u];["Value2"];[16];[12]]; + [[702u];["Value2"];[16];[15]]; + [[802u];["Value2"];[16];[16]] + ])", res); + } + + Y_UNIT_TEST(CustomWindow) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + auto it = db.StreamExecuteScanQuery(R"( - SELECT Key, sum_short_win FROM ( - SELECT Key, Text, SUM(Data) OVER w1 AS sum_short_win + SELECT Key, sum_short_win FROM ( + SELECT Key, Text, SUM(Data) OVER w1 AS sum_short_win FROM `/Root/EightShard` - WHERE Text = 'Value2' - WINDOW w1 AS (partition by Text order by Key ROWS BETWEEN CURRENT ROW and 2 FOLLOWING) - ) ORDER BY Key; - )").GetValueSync(); - auto res = StreamResultToYson(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([ - [[102u];[6]]; - [[202u];[6]]; - [[302u];[6]]; - [[402u];[6]]; - [[502u];[6]]; - [[602u];[6]]; - [[702u];[4]]; - [[802u];[1]] - ])", res); - } - - Y_UNIT_TEST(EmptySet) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - + WHERE Text = 'Value2' + WINDOW w1 AS (partition by Text order by Key ROWS BETWEEN CURRENT ROW and 2 FOLLOWING) + ) ORDER BY Key; + )").GetValueSync(); + auto res = StreamResultToYson(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([ + [[102u];[6]]; + [[202u];[6]]; + [[302u];[6]]; + [[402u];[6]]; + [[502u];[6]]; + [[602u];[6]]; + [[702u];[4]]; + [[802u];[1]] + ])", res); + } + + Y_UNIT_TEST(EmptySet) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + auto it = db.StreamExecuteScanQuery(R"( SELECT Key FROM `/Root/EightShard` WHERE false; - )").GetValueSync(); - auto res = StreamResultToYson(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson("[]",res); - } - + )").GetValueSync(); + auto res = StreamResultToYson(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson("[]",res); + } + Y_UNIT_TEST(RestrictSqlV0) { TKikimrRunner kikimr(AppCfg()); auto db = kikimr.GetTableClient(); @@ -1536,18 +1536,18 @@ Y_UNIT_TEST_SUITE(KqpScan) { UNIT_ASSERT_EQUAL_C(part.GetStatus(), EStatus::GENERIC_ERROR, part.GetStatus()); } - Y_UNIT_TEST(LongStringCombiner) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - - auto it = db.StreamExecuteScanQuery(R"( + Y_UNIT_TEST(LongStringCombiner) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + + auto it = db.StreamExecuteScanQuery(R"( SELECT min(Message), max(Message) FROM `/Root/Logs`; - )").GetValueSync(); - auto res = StreamResultToYson(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[[" GET /index.html HTTP/1.1"];["some very very very very long string"]]])",res); - } - + )").GetValueSync(); + auto res = StreamResultToYson(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[[" GET /index.html HTTP/1.1"];["some very very very very long string"]]])",res); + } + Y_UNIT_TEST(SqlInParameter) { TKikimrRunner kikimr(AppCfg()); auto db = kikimr.GetTableClient(); @@ -1638,46 +1638,46 @@ Y_UNIT_TEST_SUITE(KqpScan) { testFn(false); testFn(true); } - - Y_UNIT_TEST(SecondaryIndex) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - CreateSampleTablesWithIndex(session); - - // should work with ScanQuery - auto itOk = db.StreamExecuteScanQuery(R"( - PRAGMA AnsiInForEmptyOrNullableItemsCollections; - - SELECT Value - FROM `/Root/SecondaryComplexKeys` - WHERE (Fk1, Fk2) IN AsList((1, "Fk1"), (2, "Fk2"), (42, "Fk5"), (Null, "FkNull")) - ORDER BY Value; - )").GetValueSync(); - UNIT_ASSERT_C(itOk.IsSuccess(), itOk.GetIssues().ToString()); - CompareYson(R"([[["Payload1"]];[["Payload2"]]])", StreamResultToYson(itOk)); - - // should NOT work with explicit view - auto itIndex = db.StreamExecuteScanQuery(R"( - PRAGMA AnsiInForEmptyOrNullableItemsCollections; - SELECT Value - FROM `/Root/SecondaryComplexKeys` VIEW Index - WHERE (Fk1, Fk2) IN AsList((1, "Fk1"), (2, "Fk2"), (42, "Fk5"), (Null, "FkNull")) - ORDER BY Value; - )").GetValueSync(); - - NYdb::EStatus status = NYdb::EStatus::STATUS_UNDEFINED; - for (;;) { - auto streamPart = itIndex.ReadNext().GetValueSync(); - if (!streamPart.IsSuccess()) { - status = streamPart.GetStatus(); - break; - } - } + + Y_UNIT_TEST(SecondaryIndex) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + CreateSampleTablesWithIndex(session); + + // should work with ScanQuery + auto itOk = db.StreamExecuteScanQuery(R"( + PRAGMA AnsiInForEmptyOrNullableItemsCollections; + + SELECT Value + FROM `/Root/SecondaryComplexKeys` + WHERE (Fk1, Fk2) IN AsList((1, "Fk1"), (2, "Fk2"), (42, "Fk5"), (Null, "FkNull")) + ORDER BY Value; + )").GetValueSync(); + UNIT_ASSERT_C(itOk.IsSuccess(), itOk.GetIssues().ToString()); + CompareYson(R"([[["Payload1"]];[["Payload2"]]])", StreamResultToYson(itOk)); + + // should NOT work with explicit view + auto itIndex = db.StreamExecuteScanQuery(R"( + PRAGMA AnsiInForEmptyOrNullableItemsCollections; + SELECT Value + FROM `/Root/SecondaryComplexKeys` VIEW Index + WHERE (Fk1, Fk2) IN AsList((1, "Fk1"), (2, "Fk2"), (42, "Fk5"), (Null, "FkNull")) + ORDER BY Value; + )").GetValueSync(); + + NYdb::EStatus status = NYdb::EStatus::STATUS_UNDEFINED; + for (;;) { + auto streamPart = itIndex.ReadNext().GetValueSync(); + if (!streamPart.IsSuccess()) { + status = streamPart.GetStatus(); + break; + } + } UNIT_ASSERT_VALUES_EQUAL_C(status, EStatus::BAD_REQUEST, "ScanQuery with explicit index should fail"); - } - + } + Y_UNIT_TEST(BoolFlag) { TKikimrRunner kikimr(AppCfg()); @@ -1712,21 +1712,21 @@ Y_UNIT_TEST_SUITE(KqpScan) { [[4u];[400u];[%false]] ])", StreamResultToYson(it)); } - - Y_UNIT_TEST(Counters) { - TKikimrRunner kikimr(AppCfg()); - auto db = kikimr.GetTableClient(); - - auto it = db.StreamExecuteScanQuery(R"( + + Y_UNIT_TEST(Counters) { + TKikimrRunner kikimr(AppCfg()); + auto db = kikimr.GetTableClient(); + + auto it = db.StreamExecuteScanQuery(R"( SELECT * FROM `/Root/KeyValue` WHERE Key IN (1,2,3) LIMIT 10; - )").GetValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", StreamResultToYson(it)); - TKqpCounters counters(kikimr.GetTestServer().GetRuntime()->GetAppData().Counters); + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", StreamResultToYson(it)); + TKqpCounters counters(kikimr.GetTestServer().GetRuntime()->GetAppData().Counters); UNIT_ASSERT_VALUES_EQUAL(1, counters.GetQueryTypeCounter(NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN)->Val()); - } - + } + Y_UNIT_TEST(DropRedundantSortByPk) { TKikimrRunner kikimr(AppCfg()); auto db = kikimr.GetTableClient(); diff --git a/ydb/core/kqp/ut/kqp_scripting_ut.cpp b/ydb/core/kqp/ut/kqp_scripting_ut.cpp index 65e75ac303e..b76d06057f8 100644 --- a/ydb/core/kqp/ut/kqp_scripting_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scripting_ut.cpp @@ -266,85 +266,85 @@ Y_UNIT_TEST_SUITE(KqpScripting) { )").GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } - - Y_UNIT_TEST(ScriptExplain) { - TKikimrRunner kikimr; - TScriptingClient client(kikimr.GetDriver()); - - TExplainYqlRequestSettings settings; - settings.Mode(ExplainYqlRequestMode::Plan); - - auto result = client.ExplainYqlScript(R"( - PRAGMA Kikimr.UseNewEngine = "true"; - CREATE TABLE `/Root/ScriptingTest` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - COMMIT; - - PRAGMA Kikimr.UseNewEngine = "false"; - REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES - (1, "One"), - (2, "Two"); - COMMIT; - - PRAGMA Kikimr.UseNewEngine = "true"; - REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES - (3, "Three"), - (4, "Four"); - COMMIT; - - PRAGMA kikimr.ScanQuery = "true"; - PRAGMA Kikimr.UseNewEngine = "true"; - SELECT count(*) FROM `/Root/ScriptingTest`; - COMMIT; - - PRAGMA kikimr.ScanQuery = "false"; - PRAGMA Kikimr.UseNewEngine = "true"; - SELECT count(*) FROM `/Root/ScriptingTest`; - COMMIT; - - PRAGMA kikimr.ScanQuery = "true"; - PRAGMA Kikimr.UseNewEngine = "false"; - SELECT count(*) FROM `/Root/ScriptingTest`; - COMMIT; - - PRAGMA kikimr.ScanQuery = "false"; - PRAGMA Kikimr.UseNewEngine = "false"; - SELECT count(*) FROM `/Root/ScriptingTest`; - COMMIT; - - PRAGMA kikimr.ScanQuery = "true"; - SELECT 1*2*3*4*5; - COMMIT; - - PRAGMA kikimr.ScanQuery = "false"; - SELECT 1*2*3*4*5; - COMMIT; - )", settings).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - auto planJson = result.GetPlan(); - - NJson::TJsonValue plan; - NJson::ReadJsonTree(planJson, &plan, true); - UNIT_ASSERT_EQUAL(plan.GetMapSafe().at("queries").GetArraySafe().size(), 8); - } - - Y_UNIT_TEST(ScriptValidate) { - TKikimrRunner kikimr; - TScriptingClient client(kikimr.GetDriver()); - - TExplainYqlRequestSettings settings; - settings.Mode(ExplainYqlRequestMode::Validate); - - auto result = client.ExplainYqlScript(R"( + + Y_UNIT_TEST(ScriptExplain) { + TKikimrRunner kikimr; + TScriptingClient client(kikimr.GetDriver()); + + TExplainYqlRequestSettings settings; + settings.Mode(ExplainYqlRequestMode::Plan); + + auto result = client.ExplainYqlScript(R"( + PRAGMA Kikimr.UseNewEngine = "true"; + CREATE TABLE `/Root/ScriptingTest` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + COMMIT; + + PRAGMA Kikimr.UseNewEngine = "false"; + REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES + (1, "One"), + (2, "Two"); + COMMIT; + + PRAGMA Kikimr.UseNewEngine = "true"; + REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES + (3, "Three"), + (4, "Four"); + COMMIT; + + PRAGMA kikimr.ScanQuery = "true"; + PRAGMA Kikimr.UseNewEngine = "true"; + SELECT count(*) FROM `/Root/ScriptingTest`; + COMMIT; + + PRAGMA kikimr.ScanQuery = "false"; + PRAGMA Kikimr.UseNewEngine = "true"; + SELECT count(*) FROM `/Root/ScriptingTest`; + COMMIT; + + PRAGMA kikimr.ScanQuery = "true"; + PRAGMA Kikimr.UseNewEngine = "false"; + SELECT count(*) FROM `/Root/ScriptingTest`; + COMMIT; + + PRAGMA kikimr.ScanQuery = "false"; + PRAGMA Kikimr.UseNewEngine = "false"; + SELECT count(*) FROM `/Root/ScriptingTest`; + COMMIT; + + PRAGMA kikimr.ScanQuery = "true"; + SELECT 1*2*3*4*5; + COMMIT; + + PRAGMA kikimr.ScanQuery = "false"; + SELECT 1*2*3*4*5; + COMMIT; + )", settings).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + auto planJson = result.GetPlan(); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(planJson, &plan, true); + UNIT_ASSERT_EQUAL(plan.GetMapSafe().at("queries").GetArraySafe().size(), 8); + } + + Y_UNIT_TEST(ScriptValidate) { + TKikimrRunner kikimr; + TScriptingClient client(kikimr.GetDriver()); + + TExplainYqlRequestSettings settings; + settings.Mode(ExplainYqlRequestMode::Validate); + + auto result = client.ExplainYqlScript(R"( DECLARE $value1 as Utf8; DECLARE $value2 as UInt32; SELECT $value1 as value1, $value2 as value2; - )", settings).GetValueSync(); - + )", settings).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); auto paramTypes = result.GetParameterTypes(); UNIT_ASSERT_VALUES_EQUAL(paramTypes.size(), 2); @@ -361,76 +361,76 @@ Y_UNIT_TEST_SUITE(KqpScripting) { UNIT_ASSERT_EQUAL(parser2.GetKind(), TTypeParser::ETypeKind::Primitive); UNIT_ASSERT_EQUAL(parser2.GetPrimitive(), EPrimitiveType::Uint32); - UNIT_ASSERT_EQUAL(result.GetPlan(), ""); - } - - Y_UNIT_TEST(ScriptStats) { - TKikimrRunner kikimr; - TScriptingClient client(kikimr.GetDriver()); - - TExecuteYqlRequestSettings settings; + UNIT_ASSERT_EQUAL(result.GetPlan(), ""); + } + + Y_UNIT_TEST(ScriptStats) { + TKikimrRunner kikimr; + TScriptingClient client(kikimr.GetDriver()); + + TExecuteYqlRequestSettings settings; settings.CollectQueryStats(NYdb::NTable::ECollectQueryStatsMode::Full); - - auto result = client.ExecuteYqlScript(R"( - PRAGMA Kikimr.UseNewEngine = "true"; - CREATE TABLE `/Root/ScriptingTest` ( - Key Uint64, - Value String, - PRIMARY KEY (Key) - ); - COMMIT; - - PRAGMA Kikimr.UseNewEngine = "false"; - REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES - (1, "One"), - (2, "Two"); - COMMIT; - - PRAGMA Kikimr.UseNewEngine = "true"; - REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES - (3, "Three"), - (4, "Four"); - COMMIT; - - PRAGMA kikimr.ScanQuery = "true"; - PRAGMA Kikimr.UseNewEngine = "true"; - SELECT count(*) FROM `/Root/ScriptingTest`; - COMMIT; - - PRAGMA kikimr.ScanQuery = "false"; - PRAGMA Kikimr.UseNewEngine = "true"; - SELECT count(*) FROM `/Root/ScriptingTest`; - COMMIT; - - PRAGMA kikimr.ScanQuery = "true"; - PRAGMA Kikimr.UseNewEngine = "false"; - SELECT count(*) FROM `/Root/ScriptingTest`; - COMMIT; - - PRAGMA kikimr.ScanQuery = "false"; - PRAGMA Kikimr.UseNewEngine = "false"; - SELECT count(*) FROM `/Root/ScriptingTest`; - COMMIT; - - PRAGMA kikimr.ScanQuery = "true"; - SELECT 1*2*3*4*5; - COMMIT; - - PRAGMA kikimr.ScanQuery = "false"; - SELECT 1*2*3*4*5; - COMMIT; - )", settings).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 6); - - auto stats = result.GetStats().Get(); - auto planJson = NYdb::TProtoAccessor::GetProto(*stats).query_plan(); - - NJson::TJsonValue plan; - NJson::ReadJsonTree(planJson, &plan, true); + + auto result = client.ExecuteYqlScript(R"( + PRAGMA Kikimr.UseNewEngine = "true"; + CREATE TABLE `/Root/ScriptingTest` ( + Key Uint64, + Value String, + PRIMARY KEY (Key) + ); + COMMIT; + + PRAGMA Kikimr.UseNewEngine = "false"; + REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES + (1, "One"), + (2, "Two"); + COMMIT; + + PRAGMA Kikimr.UseNewEngine = "true"; + REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES + (3, "Three"), + (4, "Four"); + COMMIT; + + PRAGMA kikimr.ScanQuery = "true"; + PRAGMA Kikimr.UseNewEngine = "true"; + SELECT count(*) FROM `/Root/ScriptingTest`; + COMMIT; + + PRAGMA kikimr.ScanQuery = "false"; + PRAGMA Kikimr.UseNewEngine = "true"; + SELECT count(*) FROM `/Root/ScriptingTest`; + COMMIT; + + PRAGMA kikimr.ScanQuery = "true"; + PRAGMA Kikimr.UseNewEngine = "false"; + SELECT count(*) FROM `/Root/ScriptingTest`; + COMMIT; + + PRAGMA kikimr.ScanQuery = "false"; + PRAGMA Kikimr.UseNewEngine = "false"; + SELECT count(*) FROM `/Root/ScriptingTest`; + COMMIT; + + PRAGMA kikimr.ScanQuery = "true"; + SELECT 1*2*3*4*5; + COMMIT; + + PRAGMA kikimr.ScanQuery = "false"; + SELECT 1*2*3*4*5; + COMMIT; + )", settings).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 6); + + auto stats = result.GetStats().Get(); + auto planJson = NYdb::TProtoAccessor::GetProto(*stats).query_plan(); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(planJson, &plan, true); auto node = FindPlanNodeByKv(plan.GetMap().at("queries").GetArray()[2], "Node Type", "Aggregate-TableFullScan"); UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 1); - } + } Y_UNIT_TEST(StreamExecuteYqlScriptScan) { TKikimrRunner kikimr; diff --git a/ydb/core/kqp/ut/kqp_stats_ut.cpp b/ydb/core/kqp/ut/kqp_stats_ut.cpp index f7815c016ee..fb7a56e7c0f 100644 --- a/ydb/core/kqp/ut/kqp_stats_ut.cpp +++ b/ydb/core/kqp/ut/kqp_stats_ut.cpp @@ -1,73 +1,73 @@ #include <ydb/core/kqp/ut/common/kqp_ut_common.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> #include <ydb/public/sdk/cpp/client/resources/ydb_resources.h> - + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h> #include <cstdlib> -namespace NKikimr { -namespace NKqp { - -using namespace NYdb; -using namespace NYdb::NTable; - -Y_UNIT_TEST_SUITE(KqpStats) { - -Y_UNIT_TEST(MultiTxStatsFullExp) { - TKikimrRunner kikimr; - NExperimental::TStreamQueryClient db{kikimr.GetDriver()}; - auto settings = NExperimental::TExecuteStreamQuerySettings(); - settings.ProfileMode(NYdb::NExperimental::EStreamQueryProfileMode::Full); - - auto it = db.ExecuteStreamQuery(R"( +namespace NKikimr { +namespace NKqp { + +using namespace NYdb; +using namespace NYdb::NTable; + +Y_UNIT_TEST_SUITE(KqpStats) { + +Y_UNIT_TEST(MultiTxStatsFullExp) { + TKikimrRunner kikimr; + NExperimental::TStreamQueryClient db{kikimr.GetDriver()}; + auto settings = NExperimental::TExecuteStreamQuerySettings(); + settings.ProfileMode(NYdb::NExperimental::EStreamQueryProfileMode::Full); + + auto it = db.ExecuteStreamQuery(R"( SELECT * FROM `/Root/EightShard` WHERE Key BETWEEN 150 AND 266 ORDER BY Data LIMIT 4; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.ResultSetYson, R"([[[1];[202u];["Value2"]];[[2];[201u];["Value1"]];[[3];[203u];["Value3"]]])"); - + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.ResultSetYson, R"([[[1];[202u];["Value2"]];[[2];[201u];["Value1"]];[[3];[203u];["Value3"]]])"); + UNIT_ASSERT(res.PlanJson); - NJson::TJsonValue plan; + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangesScan"); UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 2); -} - -Y_UNIT_TEST(JoinNoStats) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - TStreamExecScanQuerySettings settings; - settings.CollectQueryStats(ECollectQueryStatsMode::None); - - auto it = db.StreamExecuteScanQuery(R"( +} + +Y_UNIT_TEST(JoinNoStats) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + TStreamExecScanQuerySettings settings; + settings.CollectQueryStats(ECollectQueryStatsMode::None); + + auto it = db.StreamExecuteScanQuery(R"( SELECT count(*) FROM `/Root/EightShard` AS t JOIN `/Root/KeyValue` AS kv ON t.Data = kv.Key; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(res.ResultSetYson, "[[16u]]"); - + UNIT_ASSERT(!res.QueryStats); UNIT_ASSERT(!res.PlanJson); -} - -Y_UNIT_TEST(JoinStatsBasic) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - TStreamExecScanQuerySettings settings; - settings.CollectQueryStats(ECollectQueryStatsMode::Basic); - - auto it = db.StreamExecuteScanQuery(R"( - SELECT count(*) FROM `/Root/EightShard` AS t JOIN `/Root/KeyValue` AS kv ON t.Data = kv.Key; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - - UNIT_ASSERT_VALUES_EQUAL(res.ResultSetYson, "[[16u]]"); - +} + +Y_UNIT_TEST(JoinStatsBasic) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + TStreamExecScanQuerySettings settings; + settings.CollectQueryStats(ECollectQueryStatsMode::Basic); + + auto it = db.StreamExecuteScanQuery(R"( + SELECT count(*) FROM `/Root/EightShard` AS t JOIN `/Root/KeyValue` AS kv ON t.Data = kv.Key; + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + UNIT_ASSERT_VALUES_EQUAL(res.ResultSetYson, "[[16u]]"); + UNIT_ASSERT(res.QueryStats); UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases().size(), 2); if (res.QueryStats->query_phases(0).table_access(0).name() == "/Root/KeyValue") { @@ -83,170 +83,170 @@ Y_UNIT_TEST(JoinStatsBasic) { } UNIT_ASSERT(!res.PlanJson); -} - -Y_UNIT_TEST(MultiTxStatsFull) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - TStreamExecScanQuerySettings settings; - settings.CollectQueryStats(ECollectQueryStatsMode::Full); - - auto it = db.StreamExecuteScanQuery(R"( - SELECT * FROM `/Root/EightShard` WHERE Key BETWEEN 150 AND 266 ORDER BY Data LIMIT 4; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); +} + +Y_UNIT_TEST(MultiTxStatsFull) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + TStreamExecScanQuerySettings settings; + settings.CollectQueryStats(ECollectQueryStatsMode::Full); + + auto it = db.StreamExecuteScanQuery(R"( + SELECT * FROM `/Root/EightShard` WHERE Key BETWEEN 150 AND 266 ORDER BY Data LIMIT 4; + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL( res.ResultSetYson, R"([[[1];[202u];["Value2"]];[[2];[201u];["Value1"]];[[3];[203u];["Value3"]]])" ); - + UNIT_ASSERT(res.QueryStats); UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases().size(), 2); UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases(1).table_access(0).name(), "/Root/EightShard"); UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases(1).table_access(0).partitions_count(), 2); UNIT_ASSERT(res.PlanJson); - NJson::TJsonValue plan; + NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangesScan"); UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 2); -} - -Y_UNIT_TEST(DeferredEffects) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - TString planJson; - NJson::TJsonValue plan; - - TExecDataQuerySettings settings; - settings.CollectQueryStats(ECollectQueryStatsMode::Full); - - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; - +} + +Y_UNIT_TEST(DeferredEffects) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString planJson; + NJson::TJsonValue plan; + + TExecDataQuerySettings settings; + settings.CollectQueryStats(ECollectQueryStatsMode::Full); + + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; + UPSERT INTO `/Root/TwoShard` SELECT Key + 100u AS Key, Value1 FROM `/Root/TwoShard` WHERE Key in (1,2,3,4,5); - )", TTxControl::BeginTx(), settings).ExtractValueSync(); + )", TTxControl::BeginTx(), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - // TODO(sk): do proper phase dependency tracking - // - // NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); - // auto node = FindPlanNodeByKv(plan, "Node Type", "TablePointLookup"); + + // TODO(sk): do proper phase dependency tracking + // + // NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); + // auto node = FindPlanNodeByKv(plan, "Node Type", "TablePointLookup"); // UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 1); - - auto tx = result.GetTransaction(); - UNIT_ASSERT(tx); - - auto params = db.GetParamsBuilder() - .AddParam("$key") - .Uint32(100) - .Build() - .AddParam("$value") - .String("New") - .Build() - .Build(); - - result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; - - DECLARE $key AS Uint32; - DECLARE $value AS String; - + + auto tx = result.GetTransaction(); + UNIT_ASSERT(tx); + + auto params = db.GetParamsBuilder() + .AddParam("$key") + .Uint32(100) + .Build() + .AddParam("$value") + .String("New") + .Build() + .Build(); + + result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; + + DECLARE $key AS Uint32; + DECLARE $value AS String; + UPSERT INTO `/Root/TwoShard` (Key, Value1) VALUES - ($key, $value); + ($key, $value); )", TTxControl::Tx(*tx).CommitTx(), std::move(params), settings).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); + + NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); UNIT_ASSERT_VALUES_EQUAL(plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans").GetArraySafe().size(), 3); - - result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; + + result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; SELECT * FROM `/Root/TwoShard`; UPDATE `/Root/TwoShard` SET Value1 = "XXX" WHERE Key in (3,600); )", TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); + result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); + + NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); UNIT_ASSERT_VALUES_EQUAL(plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans").GetArraySafe().size(), 4); auto ru = result.GetResponseMetadata().find(NYdb::YDB_CONSUMED_UNITS_HEADER); UNIT_ASSERT(ru != result.GetResponseMetadata().end()); UNIT_ASSERT(std::atoi(ru->second.c_str()) > 1); -} - -Y_UNIT_TEST(DataQueryWithEffects) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - TExecDataQuerySettings settings; - settings.CollectQueryStats(ECollectQueryStatsMode::Full); - - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; - +} + +Y_UNIT_TEST(DataQueryWithEffects) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + TExecDataQuerySettings settings; + settings.CollectQueryStats(ECollectQueryStatsMode::Full); + + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; + UPSERT INTO `/Root/TwoShard` SELECT Key + 1u AS Key, Value1 FROM `/Root/TwoShard`; - )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - AssertSuccessResult(result); - - NJson::TJsonValue plan; - NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); - + )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + AssertSuccessResult(result); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); + auto node = FindPlanNodeByKv(plan, "Node Type", "Upsert-ConstantExpr"); UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 2); -} - -Y_UNIT_TEST(DataQueryOldEngine) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - TExecDataQuerySettings settings; - settings.CollectQueryStats(ECollectQueryStatsMode::Full); - - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "false"; - +} + +Y_UNIT_TEST(DataQueryOldEngine) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + TExecDataQuerySettings settings; + settings.CollectQueryStats(ECollectQueryStatsMode::Full); + + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "false"; + UPSERT INTO `/Root/TwoShard` SELECT Key + 1u AS Key, Value1 FROM `/Root/TwoShard`; - )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - AssertSuccessResult(result); -} - -Y_UNIT_TEST(DataQueryMulti) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - TExecDataQuerySettings settings; - settings.CollectQueryStats(ECollectQueryStatsMode::Full); - - auto result = session.ExecuteDataQuery(R"( - PRAGMA kikimr.UseNewEngine = "true"; - - SELECT 1; - SELECT 2; - SELECT 3; - SELECT 4; - )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync(); - result.GetIssues().PrintTo(Cerr); - AssertSuccessResult(result); - - NJson::TJsonValue plan; - NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); + )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + AssertSuccessResult(result); +} + +Y_UNIT_TEST(DataQueryMulti) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + TExecDataQuerySettings settings; + settings.CollectQueryStats(ECollectQueryStatsMode::Full); + + auto result = session.ExecuteDataQuery(R"( + PRAGMA kikimr.UseNewEngine = "true"; + + SELECT 1; + SELECT 2; + SELECT 3; + SELECT 4; + )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + AssertSuccessResult(result); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); UNIT_ASSERT_EQUAL_C(plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans").GetArraySafe().size(), 0, result.GetQueryPlan()); -} - +} + Y_UNIT_TEST_NEW_ENGINE(RequestUnitForBadRequestExecute) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); @@ -300,7 +300,7 @@ Y_UNIT_TEST_NEW_ENGINE(RequestUnitForSuccessExplicitPrepare) { UNIT_ASSERT(result.GetConsumedRu() > 1); } -} // suite - -} // namespace NKqp -} // namespace NKikimr +} // suite + +} // namespace NKqp +} // namespace NKikimr diff --git a/ydb/core/kqp/ut/ya.make b/ydb/core/kqp/ut/ya.make index 609ec3237f8..c7cfb9eb4dd 100644 --- a/ydb/core/kqp/ut/ya.make +++ b/ydb/core/kqp/ut/ya.make @@ -23,7 +23,7 @@ SRCS( kqp_arrow_in_channels_ut.cpp kqp_document_api_ut.cpp kqp_effects_perf_ut.cpp - kqp_explain_ut.cpp + kqp_explain_ut.cpp kqp_flip_join_ut.cpp kqp_index_lookup_join_ut.cpp kqp_indexes_ut.cpp @@ -47,7 +47,7 @@ SRCS( kqp_scripting_ut.cpp kqp_service_ut.cpp kqp_sort_ut.cpp - kqp_stats_ut.cpp + kqp_stats_ut.cpp kqp_sqlin_ut.cpp kqp_sys_view_ut.cpp kqp_sys_col_ut.cpp diff --git a/ydb/core/protos/kqp.proto b/ydb/core/protos/kqp.proto index 9b46b720e11..98e9b0797c6 100644 --- a/ydb/core/protos/kqp.proto +++ b/ydb/core/protos/kqp.proto @@ -46,7 +46,7 @@ enum EQueryAction { QUERY_ACTION_BEGIN_TX = 6; QUERY_ACTION_COMMIT_TX = 7; QUERY_ACTION_ROLLBACK_TX = 8; - QUERY_ACTION_PARSE = 9; + QUERY_ACTION_PARSE = 9; }; enum EIsolationLevel { @@ -386,7 +386,7 @@ message TEvExecuterStreamData { message TEvExecuterStreamDataAck { optional uint64 SeqNo = 1; optional int64 FreeSpace = 2; - optional bool Enough = 3 [default = false]; + optional bool Enough = 3 [default = false]; }; message TEvExecuterStreamProfile { @@ -525,28 +525,28 @@ message TKqpNodeResources { repeated TMemory Memory = 8; optional uint32 ExecutionUnits = 9; } - + /// Scans -message TEvScanInitActor { - optional uint64 ScanId = 1; - optional NActorsProto.TActorId ScanActorId = 2; +message TEvScanInitActor { + optional uint64 ScanId = 1; + optional NActorsProto.TActorId ScanActorId = 2; optional uint32 Generation = 3; -}; - -message TEvScanError { - optional Ydb.StatusIds.StatusCode Status = 1; - repeated Ydb.Issue.IssueMessage Issues = 2; +}; + +message TEvScanError { + optional Ydb.StatusIds.StatusCode Status = 1; + repeated Ydb.Issue.IssueMessage Issues = 2; optional uint32 Generation = 3; -} - -message TEvRemoteScanData { - optional uint32 ScanId = 1; +} + +message TEvRemoteScanData { + optional uint32 ScanId = 1; optional uint64 CpuTimeUs = 2; - optional uint64 WaitTimeMs = 3; - optional uint32 PageFaults = 4; - optional bool PageFault = 5; - optional bool Finished = 6; - optional bytes LastKey = 7; + optional uint64 WaitTimeMs = 3; + optional uint32 PageFaults = 4; + optional bool PageFault = 5; + optional bool Finished = 6; + optional bytes LastKey = 7; optional uint32 Generation = 9; message TArrowBatch { @@ -557,14 +557,14 @@ message TEvRemoteScanData { // Only one of the fields Rows and ArrowBatch must be filled. However, we can not use oneof feature because Rows // field is repeated. Moving it into oneof is impossible. We may wrap it into a message but this would break // backwards comparability. - repeated bytes Rows = 8; + repeated bytes Rows = 8; optional TArrowBatch ArrowBatch = 10; -} - -message TEvRemoteScanDataAck { - optional uint64 FreeSpace = 1; +} + +message TEvRemoteScanDataAck { + optional uint64 FreeSpace = 1; optional uint32 Generation = 2; -} +} message TEvKillScanTablet { } diff --git a/ydb/core/protos/kqp_physical.proto b/ydb/core/protos/kqp_physical.proto index 44d8f587166..922d6f6c7c9 100644 --- a/ydb/core/protos/kqp_physical.proto +++ b/ydb/core/protos/kqp_physical.proto @@ -230,7 +230,7 @@ message TKqpPhyStage { uint32 OutputsCount = 5; repeated TKqpPhyTableOperation TableOps = 6; bool IsEffectsStage = 7; - string StageGuid = 8; + string StageGuid = 8; } message TKqpPhyResult { @@ -252,7 +252,7 @@ message TKqpPhyTx { repeated TKqpPhyStage Stages = 2; repeated TKqpPhyResult Results = 3; repeated TKqpPhyParamBinding ParamBindings = 4; - string Plan = 5; + string Plan = 5; bool HasEffects = 6; // at least one stage has flag TKqpPhyStage::IsEffectStage set } @@ -290,8 +290,8 @@ message TKqpPhyQuery { repeated TKqpPhyTx Transactions = 2; reserved 3; // Deprecated, (repeated TKqpPhyTx Effects = 3); repeated TKqpPhyResultBinding ResultBindings = 4; - string QueryPlan = 5; - string QueryAst = 6; + string QueryPlan = 5; + string QueryAst = 6; // StrictDml constraints info repeated TKqpTableOp TableOps = 7; diff --git a/ydb/core/protos/tx_datashard.proto b/ydb/core/protos/tx_datashard.proto index 89863b25400..1549fe524f5 100644 --- a/ydb/core/protos/tx_datashard.proto +++ b/ydb/core/protos/tx_datashard.proto @@ -221,8 +221,8 @@ message TKqpTransaction { optional NYql.NDqProto.TComputeRuntimeSettings RuntimeSettings = 4; reserved 5; - - optional NKikimrKqp.TKqpSnapshot Snapshot = 6; + + optional NKikimrKqp.TKqpSnapshot Snapshot = 6; } message TKqpTaskInfo { @@ -1373,20 +1373,20 @@ message TEvBuildIndexProgressResponse { optional uint64 RequestSeqNoGeneration = 12; optional uint64 RequestSeqNoRound = 13; } - -message TEvKqpScan { - optional uint64 TxId = 1; - optional uint64 ScanId = 2; - optional uint64 LocalPathId = 3; - optional string TablePath = 4; - optional uint64 SchemaVersion = 5; - repeated uint32 ColumnTags = 6; - repeated uint32 ColumnTypes = 7; - repeated bool SkipNullKeys = 8; + +message TEvKqpScan { + optional uint64 TxId = 1; + optional uint64 ScanId = 2; + optional uint64 LocalPathId = 3; + optional string TablePath = 4; + optional uint64 SchemaVersion = 5; + repeated uint32 ColumnTags = 6; + repeated uint32 ColumnTypes = 7; + repeated bool SkipNullKeys = 8; repeated NKikimrTx.TKeyRange Ranges = 9; - optional NKikimrKqp.TKqpSnapshot Snapshot = 10; + optional NKikimrKqp.TKqpSnapshot Snapshot = 10; reserved 11; // optional NKqpProto.EKqpStatsMode StatsMode = 11; - optional uint64 TimeoutMs = 12; + optional uint64 TimeoutMs = 12; optional uint32 Generation = 13; // Seralized OlapProgram in old format without parameters reserved 14; @@ -1396,7 +1396,7 @@ message TEvKqpScan { optional NYql.NDqProto.EDqStatsMode StatsMode = 18; optional bytes OlapProgram = 19; optional NKikimrSchemeOp.EOlapProgramType OlapProgramType = 20; -} +} message TEvCompactTable { optional NKikimrProto.TPathID PathId = 1; diff --git a/ydb/core/sys_view/common/scan_actor_base_impl.h b/ydb/core/sys_view/common/scan_actor_base_impl.h index 298a8efbc03..2c8a3f3fcd6 100644 --- a/ydb/core/sys_view/common/scan_actor_base_impl.h +++ b/ydb/core/sys_view/common/scan_actor_base_impl.h @@ -88,8 +88,8 @@ protected: << ", error: " << message); auto error = MakeHolder<NKqp::TEvKqpCompute::TEvScanError>(); - error->Record.SetStatus(status); - IssueToMessage(NYql::TIssue(message), error->Record.MutableIssues()->Add()); + error->Record.SetStatus(status); + IssueToMessage(NYql::TIssue(message), error->Record.MutableIssues()->Add()); TBase::Send(OwnerActorId, error.Release()); diff --git a/ydb/core/tx/columnshard/columnshard.h b/ydb/core/tx/columnshard/columnshard.h index 62486e3c33e..9f1da5d78c3 100644 --- a/ydb/core/tx/columnshard/columnshard.h +++ b/ydb/core/tx/columnshard/columnshard.h @@ -235,7 +235,7 @@ struct TEvColumnShard { } }; - using TEvScan = TEvDataShard::TEvKqpScan; + using TEvScan = TEvDataShard::TEvKqpScan; }; inline auto& Proto(TEvColumnShard::TEvProposeTransaction* ev) { diff --git a/ydb/core/tx/datashard/datashard.cpp b/ydb/core/tx/datashard/datashard.cpp index d86568b4a15..e44b05fa25a 100644 --- a/ydb/core/tx/datashard/datashard.cpp +++ b/ydb/core/tx/datashard/datashard.cpp @@ -1360,11 +1360,11 @@ void TDataShard::Handle(TEvDataShard::TEvStateChangedResult::TPtr& ev, const TAc bool TDataShard::CheckDataTxReject(const TString& opDescr, const TActorContext &ctx, NKikimrTxDataShard::TEvProposeTransactionResult::EStatus &rejectStatus, - TString &reason) + TString &reason) { bool reject = false; rejectStatus = NKikimrTxDataShard::TEvProposeTransactionResult::OVERLOADED; - TVector<TString> rejectReasons; + TVector<TString> rejectReasons; // In v0.5 reject all transactions on split Src after receiving EvSplit if (State == TShardState::SplitSrcWaitForNoTxInFlight || @@ -1372,19 +1372,19 @@ bool TDataShard::CheckDataTxReject(const TString& opDescr, State == TShardState::SplitSrcSendingSnapshot || State == TShardState::SplitSrcWaitForPartitioningChanged) { reject = true; - rejectReasons.push_back(TStringBuilder() - << "is in process of split opId " << SrcSplitOpId - << " state " << DatashardStateName(State) - << " (wrong shard state)"); + rejectReasons.push_back(TStringBuilder() + << "is in process of split opId " << SrcSplitOpId + << " state " << DatashardStateName(State) + << " (wrong shard state)"); } else if (State == TShardState::SplitDstReceivingSnapshot) { reject = true; - rejectReasons.push_back(TStringBuilder() - << "is in process of split opId " << DstSplitOpId - << " state " << DatashardStateName(State)); + rejectReasons.push_back(TStringBuilder() + << "is in process of split opId " << DstSplitOpId + << " state " << DatashardStateName(State)); } else if (State == TShardState::PreOffline || State == TShardState::Offline) { reject = true; rejectStatus = NKikimrTxDataShard::TEvProposeTransactionResult::ERROR; - rejectReasons.push_back("is in a pre/offline state assuming this is due to a finished split (wrong shard state)"); + rejectReasons.push_back("is in a pre/offline state assuming this is due to a finished split (wrong shard state)"); } else if (MvccSwitchState == TSwitchState::SWITCHING) { reject = true; rejectReasons.push_back(TStringBuilder() @@ -1394,7 +1394,7 @@ bool TDataShard::CheckDataTxReject(const TString& opDescr, if (Pipeline.HasDrop()) { reject = true; - rejectReasons.push_back("is in process of drop"); + rejectReasons.push_back("is in process of drop"); rejectStatus = NKikimrTxDataShard::TEvProposeTransactionResult::ERROR; } @@ -1402,24 +1402,24 @@ bool TDataShard::CheckDataTxReject(const TString& opDescr, TDuration lag = GetDataTxCompleteLag(); if (txInfly > 1 && lag > TDuration::MilliSeconds(MaxTxLagMilliseconds)) { reject = true; - rejectReasons.push_back(TStringBuilder() - << "lags behind, lag: " << lag - << " in-flight tx count: " << txInfly); + rejectReasons.push_back(TStringBuilder() + << "lags behind, lag: " << lag + << " in-flight tx count: " << txInfly); } const float rejectProbabilty = Executor()->GetRejectProbability(); if (!reject && rejectProbabilty > 0) { float rnd = AppData(ctx)->RandomProvider->GenRandReal2(); reject |= (rnd < rejectProbabilty); - if (reject) - rejectReasons.push_back("decided to reject due to given RejectProbability"); + if (reject) + rejectReasons.push_back("decided to reject due to given RejectProbability"); } size_t totalInFly = (TxInFly() + ImmediateInFly() + ProposeQueue.Size() + TxWaiting()); if (totalInFly > GetMaxTxInFly()) { - reject = true; - rejectReasons.push_back("MaxTxInFly was exceeded"); - } + reject = true; + rejectReasons.push_back("MaxTxInFly was exceeded"); + } if (!reject && Stopping) { reject = true; @@ -1437,13 +1437,13 @@ bool TDataShard::CheckDataTxReject(const TString& opDescr, } } - if (reject) { - reason = TStringBuilder() - << "Rejecting " << opDescr - << " because datashard " << TabletID() << ": " - << JoinSeq("; ", rejectReasons); - } - + if (reject) { + reason = TStringBuilder() + << "Rejecting " << opDescr + << " because datashard " << TabletID() << ": " + << JoinSeq("; ", rejectReasons); + } + return reject; } @@ -1463,8 +1463,8 @@ bool TDataShard::CheckDataTxRejectAndReply(TEvDataShard::TEvProposeTransaction* TString txDescr = TStringBuilder() << "data TxId " << msg->GetTxId(); NKikimrTxDataShard::TEvProposeTransactionResult::EStatus rejectStatus; - TString rejectReason; - bool reject = CheckDataTxReject(txDescr, ctx, rejectStatus, rejectReason); + TString rejectReason; + bool reject = CheckDataTxReject(txDescr, ctx, rejectStatus, rejectReason); if (reject) { THolder<TEvDataShard::TEvProposeTransactionResult> result = @@ -1473,8 +1473,8 @@ bool TDataShard::CheckDataTxRejectAndReply(TEvDataShard::TEvProposeTransaction* msg->GetTxId(), rejectStatus)); - result->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, rejectReason); - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, rejectReason); + result->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, rejectReason); + LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, rejectReason); ctx.Send(msg->GetSource(), result.Release()); IncCounter(COUNTER_PREPARE_OVERLOADED); diff --git a/ydb/core/tx/datashard/datashard.h b/ydb/core/tx/datashard/datashard.h index 5a93c035026..33cc99535f0 100644 --- a/ydb/core/tx/datashard/datashard.h +++ b/ydb/core/tx/datashard/datashard.h @@ -277,8 +277,8 @@ struct TEvDataShard { EvUnsafeUploadRowsRequest, EvUnsafeUploadRowsResponse, - EvKqpScan, - + EvKqpScan, + EvChangeS3UploadStatus, EvGetRemovedRowVersions, /* for tests */ @@ -1346,13 +1346,13 @@ struct TEvDataShard { TEvDataShard::EvBuildIndexProgressResponse> { }; - - struct TEvKqpScan - : public TEventPB<TEvKqpScan, - NKikimrTxDataShard::TEvKqpScan, - TEvDataShard::EvKqpScan> - { - }; + + struct TEvKqpScan + : public TEventPB<TEvKqpScan, + NKikimrTxDataShard::TEvKqpScan, + TEvDataShard::EvKqpScan> + { + }; struct TEvGetRemovedRowVersions : public TEventLocal<TEvGetRemovedRowVersions, EvGetRemovedRowVersions> { TPathId PathId; diff --git a/ydb/core/tx/datashard/datashard__kqp_scan.cpp b/ydb/core/tx/datashard/datashard__kqp_scan.cpp index b6b6b7e337d..e110f6be0ba 100644 --- a/ydb/core/tx/datashard/datashard__kqp_scan.cpp +++ b/ydb/core/tx/datashard/datashard__kqp_scan.cpp @@ -1,61 +1,61 @@ -#include "datashard_impl.h" +#include "datashard_impl.h" #include "range_ops.h" -#include <util/string/vector.h> - +#include <util/string/vector.h> + #include <ydb/core/actorlib_impl/long_timer.h> #include <ydb/core/formats/arrow_batch_builder.h> #include <ydb/core/kqp/kqp_compute.h> #include <ydb/core/tablet_flat/flat_row_celled.h> - + #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h> - -namespace NKikimr { + +namespace NKikimr { namespace NDataShard { - -// using namespace NTabletFlatExecutor; -using namespace NKqp; -using namespace NMiniKQL; - -constexpr ui64 MAX_BATCH_ROWS = 10'000; -constexpr ui64 INIT_BATCH_ROWS = 1'000; -constexpr ui64 MIN_BATCH_ROWS_ON_PAGEFAULT = 1'000; -constexpr ui64 MIN_BATCH_SIZE_ON_PAGEFAULT = 256_KB; -constexpr ui64 READAHEAD_LO = 32_KB; -constexpr ui64 READAHEAD_HI = 512_KB; -constexpr TDuration SCAN_HARD_TIMEOUT = TDuration::Minutes(10); -constexpr TDuration SCAN_HARD_TIMEOUT_GAP = TDuration::Seconds(5); - + +// using namespace NTabletFlatExecutor; +using namespace NKqp; +using namespace NMiniKQL; + +constexpr ui64 MAX_BATCH_ROWS = 10'000; +constexpr ui64 INIT_BATCH_ROWS = 1'000; +constexpr ui64 MIN_BATCH_ROWS_ON_PAGEFAULT = 1'000; +constexpr ui64 MIN_BATCH_SIZE_ON_PAGEFAULT = 256_KB; +constexpr ui64 READAHEAD_LO = 32_KB; +constexpr ui64 READAHEAD_HI = 512_KB; +constexpr TDuration SCAN_HARD_TIMEOUT = TDuration::Minutes(10); +constexpr TDuration SCAN_HARD_TIMEOUT_GAP = TDuration::Seconds(5); + class TKqpScanResult : public IDestructable {}; - -class TKqpScan : public TActor<TKqpScan>, public NTable::IScan { -public: + +class TKqpScan : public TActor<TKqpScan>, public NTable::IScan { +public: static constexpr auto ActorActivityType() { - return NKikimrServices::TActivity::KQP_TABLE_SCAN; - } - -public: - TKqpScan(const TActorId& computeActorId, const TActorId& datashardActorId, ui32 scanId, + return NKikimrServices::TActivity::KQP_TABLE_SCAN; + } + +public: + TKqpScan(const TActorId& computeActorId, const TActorId& datashardActorId, ui32 scanId, NDataShard::TUserTable::TCPtr tableInfo, const TSmallVec<TSerializedTableRange>&& tableRanges, const TSmallVec<NTable::TTag>&& columnTags, const TSmallVec<bool>&& skipNullKeys, const NYql::NDqProto::EDqStatsMode& statsMode, ui64 timeoutMs, ui32 generation, NKikimrTxDataShard::EScanDataFormat dataFormat) - : TActor(&TKqpScan::StateScan) - , ComputeActorId(computeActorId) - , DatashardActorId(datashardActorId) - , ScanId(scanId) - , TableInfo(tableInfo) + : TActor(&TKqpScan::StateScan) + , ComputeActorId(computeActorId) + , DatashardActorId(datashardActorId) + , ScanId(scanId) + , TableInfo(tableInfo) , TablePath(TableInfo->Path) , TableRanges(std::move(tableRanges)) , CurrentRange(0) , Tags(std::move(columnTags)) , SkipNullKeys(std::move(skipNullKeys)) - , StatsMode(statsMode) - , Deadline(TInstant::Now() + (timeoutMs ? TDuration::MilliSeconds(timeoutMs) + SCAN_HARD_TIMEOUT_GAP : SCAN_HARD_TIMEOUT)) + , StatsMode(statsMode) + , Deadline(TInstant::Now() + (timeoutMs ? TDuration::MilliSeconds(timeoutMs) + SCAN_HARD_TIMEOUT_GAP : SCAN_HARD_TIMEOUT)) , Generation(generation) , DataFormat(dataFormat) - , PeerFreeSpace(0) - , Sleep(true) - , IsLocal(computeActorId.NodeId() == datashardActorId.NodeId()) + , PeerFreeSpace(0) + , Sleep(true) + , IsLocal(computeActorId.NodeId() == datashardActorId.NodeId()) { if (DataFormat == NKikimrTxDataShard::EScanDataFormat::ARROW) { BatchBuilder = MakeHolder<NArrow::TArrowBatchBuilder>(); @@ -79,141 +79,141 @@ public: << DebugPrintRange(TableInfo->KeyColumnTypes, range.ToTableRange(), *AppData()->TypeRegistry)); } } - -private: - STATEFN(StateScan) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvKqpCompute::TEvScanDataAck, HandleScan); + +private: + STATEFN(StateScan) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvKqpCompute::TEvScanDataAck, HandleScan); hFunc(TEvKqpCompute::TEvKillScanTablet, HandleScan); - hFunc(TEvKqp::TEvAbortExecution, HandleScan); - hFunc(TEvents::TEvUndelivered, HandleScan); - hFunc(TEvents::TEvWakeup, HandleScan); - default: - Y_FAIL("TKqpScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); - } - } - - void HandleScan(TEvKqpCompute::TEvScanDataAck::TPtr& ev) { - if (!Driver) { - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got ScanDataAck while driver not set"); - PassAway(); - return; - } - - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got ScanDataAck" + hFunc(TEvKqp::TEvAbortExecution, HandleScan); + hFunc(TEvents::TEvUndelivered, HandleScan); + hFunc(TEvents::TEvWakeup, HandleScan); + default: + Y_FAIL("TKqpScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); + } + } + + void HandleScan(TEvKqpCompute::TEvScanDataAck::TPtr& ev) { + if (!Driver) { + LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got ScanDataAck while driver not set"); + PassAway(); + return; + } + + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got ScanDataAck" << ", at: " << ScanActorId << ", scanId: " << ScanId << ", table: " << TablePath << ", gen: " << ev->Get()->Generation << ", tablet: " << DatashardActorId - << ", freeSpace: " << ev->Get()->FreeSpace << ", prevFreeSpace: " << PeerFreeSpace); - + << ", freeSpace: " << ev->Get()->FreeSpace << ", prevFreeSpace: " << PeerFreeSpace); + YQL_ENSURE(ev->Get()->Generation == Generation, "expected: " << Generation << ", got: " << ev->Get()->Generation); - if (!ComputeActorId) { - ComputeActorId = ev->Sender; - } - - PeerFreeSpace = ev->Get()->FreeSpace; - if (PeerFreeSpace > 0) { - if (Y_UNLIKELY(IsProfile())) { + if (!ComputeActorId) { + ComputeActorId = ev->Sender; + } + + PeerFreeSpace = ev->Get()->FreeSpace; + if (PeerFreeSpace > 0) { + if (Y_UNLIKELY(IsProfile())) { StartWaitTime = TInstant::Now(); - } - if (Sleep) { - Sleep = false; - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Wakeup driver at: " << ScanActorId); - Driver->Touch(EScan::Feed); - } - } - } - + } + if (Sleep) { + Sleep = false; + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Wakeup driver at: " << ScanActorId); + Driver->Touch(EScan::Feed); + } + } + } + void HandleScan(TEvKqpCompute::TEvKillScanTablet::TPtr&) { LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Kill self tablet " << DatashardActorId); Send(DatashardActorId, new TEvents::TEvPoison); } - void HandleScan(TEvKqp::TEvAbortExecution::TPtr& ev) { - if (!Driver) { - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got AbortExecution while driver not set"); - PassAway(); - return; - } - - auto& msg = ev->Get()->Record; - - auto prio = msg.GetStatusCode() == Ydb::StatusIds::SUCCESS ? NActors::NLog::PRI_DEBUG : NActors::NLog::PRI_WARN; - LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::TX_DATASHARD, "Got AbortExecution" + void HandleScan(TEvKqp::TEvAbortExecution::TPtr& ev) { + if (!Driver) { + LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got AbortExecution while driver not set"); + PassAway(); + return; + } + + auto& msg = ev->Get()->Record; + + auto prio = msg.GetStatusCode() == Ydb::StatusIds::SUCCESS ? NActors::NLog::PRI_DEBUG : NActors::NLog::PRI_WARN; + LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::TX_DATASHARD, "Got AbortExecution" << ", at: " << ScanActorId << ", tablet: " << DatashardActorId - << ", scanId: " << ScanId << ", table: " << TablePath - << ", code: " << Ydb::StatusIds_StatusCode_Name(msg.GetStatusCode()) - << ", reason: " << msg.GetMessage()); - + << ", scanId: " << ScanId << ", table: " << TablePath + << ", code: " << Ydb::StatusIds_StatusCode_Name(msg.GetStatusCode()) + << ", reason: " << msg.GetMessage()); + AbortEvent = ev->Release(); - Driver->Touch(EScan::Final); - } - - void HandleScan(TEvents::TEvUndelivered::TPtr& ev) { - if (!Driver) { - PassAway(); - return; - } - + Driver->Touch(EScan::Final); + } + + void HandleScan(TEvents::TEvUndelivered::TPtr& ev) { + if (!Driver) { + PassAway(); + return; + } + LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Undelivered event: " << ev->GetTypeRewrite() << ", at: " << ScanActorId << ", tablet: " << DatashardActorId - << ", scanId: " << ScanId << ", table: " << TablePath); - - switch (ev->GetTypeRewrite()) { + << ", scanId: " << ScanId << ", table: " << TablePath); + + switch (ev->GetTypeRewrite()) { case TEvKqpCompute::TEvScanInitActor::EventType: case TEvKqpCompute::TEvScanData::EventType: - Driver->Touch(EScan::Final); - } - } - - void HandleScan(TEvents::TEvWakeup::TPtr&) { - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Guard execution timeout at: " << ScanActorId - << ", scanId: " << ScanId << ", table: " << TablePath); - - TimeoutActorId = {}; - - if (Driver) { - Driver->Touch(EScan::Final); - } else { - PassAway(); - } - } - -private: + Driver->Touch(EScan::Final); + } + } + + void HandleScan(TEvents::TEvWakeup::TPtr&) { + LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Guard execution timeout at: " << ScanActorId + << ", scanId: " << ScanId << ", table: " << TablePath); + + TimeoutActorId = {}; + + if (Driver) { + Driver->Touch(EScan::Final); + } else { + PassAway(); + } + } + +private: THello Prepare(IDriver* driver, TIntrusiveConstPtr<TScheme> scheme) noexcept final { - Y_VERIFY(scheme); - Y_VERIFY(driver); - - Driver = driver; - ScanActorId = TActivationContext::AsActorContext().RegisterWithSameMailbox(this); - - // propagate self actor id + Y_VERIFY(scheme); + Y_VERIFY(driver); + + Driver = driver; + ScanActorId = TActivationContext::AsActorContext().RegisterWithSameMailbox(this); + + // propagate self actor id Send(ComputeActorId, new TEvKqpCompute::TEvScanInitActor(ScanId, ScanActorId, Generation), IEventHandle::FlagTrackDelivery); - - Sleep = true; - - THello startConfig; - startConfig.Scan = EScan::Sleep; - startConfig.Conf.ReadAheadLo = READAHEAD_LO; - startConfig.Conf.ReadAheadHi = READAHEAD_HI; - - TimeoutActorId = CreateLongTimer(TlsActivationContext->AsActorContext(), Deadline - TInstant::Now(), - new IEventHandle(SelfId(), SelfId(), new TEvents::TEvWakeup)); - - if (Y_UNLIKELY(IsProfile())) { + + Sleep = true; + + THello startConfig; + startConfig.Scan = EScan::Sleep; + startConfig.Conf.ReadAheadLo = READAHEAD_LO; + startConfig.Conf.ReadAheadHi = READAHEAD_HI; + + TimeoutActorId = CreateLongTimer(TlsActivationContext->AsActorContext(), Deadline - TInstant::Now(), + new IEventHandle(SelfId(), SelfId(), new TEvents::TEvWakeup)); + + if (Y_UNLIKELY(IsProfile())) { StartWaitTime = TInstant::Now(); - } - + } + LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Start scan" << ", at: " << ScanActorId << ", tablet: " << DatashardActorId << ", scanId: " << ScanId << ", table: " << TablePath << ", gen: " << Generation - << ", deadline: " << Deadline); - - return startConfig; - } - - EScan Seek(TLead& lead, ui64 seq) noexcept final { + << ", deadline: " << Deadline); + + return startConfig; + } + + EScan Seek(TLead& lead, ui64 seq) noexcept final { YQL_ENSURE(seq == CurrentRange); if (CurrentRange == TableRanges.size()) { @@ -221,80 +221,80 @@ private: "TableRanges is over" << ", at: " << ScanActorId << ", scanId: " << ScanId << ", table: " << TablePath); - return EScan::Final; - } - + return EScan::Final; + } + auto& range = TableRanges[CurrentRange]; - int cmpFrom; - int cmpTo; - cmpFrom = CompareBorders<false, false>( + int cmpFrom; + int cmpTo; + cmpFrom = CompareBorders<false, false>( range.From.GetCells(), TableInfo->Range.From.GetCells(), range.FromInclusive, TableInfo->Range.FromInclusive, TableInfo->KeyColumnTypes); - - cmpTo = CompareBorders<true, true>( + + cmpTo = CompareBorders<true, true>( range.To.GetCells(), TableInfo->Range.To.GetCells(), range.ToInclusive, TableInfo->Range.ToInclusive, TableInfo->KeyColumnTypes); - - if (cmpFrom > 0) { + + if (cmpFrom > 0) { auto seek = range.FromInclusive ? NTable::ESeek::Lower : NTable::ESeek::Upper; lead.To(Tags, range.From.GetCells(), seek); - } else { - lead.To(Tags, {}, NTable::ESeek::Lower); - } - + } else { + lead.To(Tags, {}, NTable::ESeek::Lower); + } + if (cmpTo < 0) { lead.Until(range.To.GetCells(), range.ToInclusive); } - return EScan::Feed; - } - - EScan Feed(TArrayRef<const TCell> key, const TRow& row) noexcept final { - LastKey = TOwnedCellVec(key); - - Y_VERIFY(SkipNullKeys.size() <= key.size()); - for (ui32 i = 0; i < SkipNullKeys.size(); ++i) { - if (SkipNullKeys[i] && key[i].IsNull()) { - return EScan::Feed; - } - } - + return EScan::Feed; + } + + EScan Feed(TArrayRef<const TCell> key, const TRow& row) noexcept final { + LastKey = TOwnedCellVec(key); + + Y_VERIFY(SkipNullKeys.size() <= key.size()); + for (ui32 i = 0; i < SkipNullKeys.size(); ++i) { + if (SkipNullKeys[i] && key[i].IsNull()) { + return EScan::Feed; + } + } + MakeResult(); - - if (Y_UNLIKELY(IsProfile())) { - Result->WaitTime += TInstant::Now() - StartWaitTime; - } - - Y_DEFER { - if (Y_UNLIKELY(IsProfile())) { - StartWaitTime = TInstant::Now(); - } - }; - + + if (Y_UNLIKELY(IsProfile())) { + Result->WaitTime += TInstant::Now() - StartWaitTime; + } + + Y_DEFER { + if (Y_UNLIKELY(IsProfile())) { + StartWaitTime = TInstant::Now(); + } + }; + AddRow(row); - - auto sent = SendResult(/* pageFault */ false); - + + auto sent = SendResult(/* pageFault */ false); + if (!sent) { // There is free space in memory and results are not sent to caller return EScan::Feed; } - + if (PeerFreeSpace <= 0) { - Sleep = true; - return EScan::Sleep; - } - + Sleep = true; + return EScan::Sleep; + } + return EScan::Feed; // sent by rows limit, can send one more batch - } - + } + EScan Exhausted() noexcept override { LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Range " << CurrentRange << " of " << TableRanges.size() << " exhausted: try next one." @@ -311,34 +311,34 @@ private: return EScan::Reset; } - EScan PageFault() noexcept override final { - ++PageFaults; - if (Result && !Result->Rows.empty()) { - bool sent = SendResult(/* pageFault */ true); - - if (sent && PeerFreeSpace == 0) { - Sleep = true; - return EScan::Sleep; - } - - if (sent && PeerFreeSpace > 0 && Y_UNLIKELY(IsProfile())) { + EScan PageFault() noexcept override final { + ++PageFaults; + if (Result && !Result->Rows.empty()) { + bool sent = SendResult(/* pageFault */ true); + + if (sent && PeerFreeSpace == 0) { + Sleep = true; + return EScan::Sleep; + } + + if (sent && PeerFreeSpace > 0 && Y_UNLIKELY(IsProfile())) { StartWaitTime = TInstant::Now(); - } - } - return EScan::Feed; - } - -private: + } + } + return EScan::Feed; + } + +private: TAutoPtr<IDestructable> Finish(EAbort abort) noexcept final { - auto prio = abort == EAbort::None ? NActors::NLog::PRI_DEBUG : NActors::NLog::PRI_ERROR; - LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::TX_DATASHARD, "Finish scan" - << ", at: " << ScanActorId << ", scanId: " << ScanId + auto prio = abort == EAbort::None ? NActors::NLog::PRI_DEBUG : NActors::NLog::PRI_ERROR; + LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::TX_DATASHARD, "Finish scan" + << ", at: " << ScanActorId << ", scanId: " << ScanId << ", table: " << TablePath << ", reason: " << (int) abort << ", abortEvent: " << (AbortEvent ? AbortEvent->Record.ShortDebugString() : TString("<none>"))); - + if (abort != EAbort::None || AbortEvent) { auto ev = MakeHolder<TEvKqpCompute::TEvScanError>(Generation); - + if (AbortEvent) { ev->Record.SetStatus(AbortEvent->Record.GetStatusCode()); auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_OPERATION_ABORTED, TStringBuilder() @@ -352,30 +352,30 @@ private: } Send(ComputeActorId, ev.Release()); - } else { - if (Result) { + } else { + if (Result) { // TODO: // Result->CpuTime = CpuTime; - } else { + } else { Result = MakeHolder<TEvKqpCompute::TEvScanData>(ScanId, Generation); - } + } auto send = SendResult(Result->PageFault, true); Y_VERIFY_DEBUG(send); - } - - Driver = nullptr; - if (TimeoutActorId) { - Send(TimeoutActorId, new TEvents::TEvPoison); - } - PassAway(); - - return new TKqpScanResult(); - } - - void Describe(IOutputStream& out) const noexcept final { - out << "TExecuteKqpScanTxUnit, TKqpScan"; - } - + } + + Driver = nullptr; + if (TimeoutActorId) { + Send(TimeoutActorId, new TEvents::TEvPoison); + } + PassAway(); + + return new TKqpScanResult(); + } + + void Describe(IOutputStream& out) const noexcept final { + out << "TExecuteKqpScanTxUnit, TKqpScan"; + } + void MakeResult() { if (!Result) { Result = MakeHolder<TEvKqpCompute::TEvScanData>(ScanId, Generation); @@ -419,16 +419,16 @@ private: bool SendResult(bool pageFault, bool finish = false) noexcept { if (Rows >= MAX_BATCH_ROWS || CellvecBytes >= PeerFreeSpace || (pageFault && (Rows >= MIN_BATCH_ROWS_ON_PAGEFAULT || CellvecBytes >= MIN_BATCH_SIZE_ON_PAGEFAULT)) || finish) - { - Result->PageFault = pageFault; - Result->PageFaults = PageFaults; + { + Result->PageFault = pageFault; + Result->PageFaults = PageFaults; if (finish) { Result->Finished = true; } else { Result->LastKey = LastKey; } auto sendBytes = CellvecBytes; - + if (DataFormat == NKikimrTxDataShard::EScanDataFormat::ARROW) { FlushBatchToResult(); sendBytes = NArrow::GetBatchDataSize(Result->ArrowBatch); @@ -438,20 +438,20 @@ private: YQL_ENSURE(Result->Rows.size() == Rows); } - PageFaults = 0; - - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Send ScanData" - << ", from: " << ScanActorId << ", to: " << ComputeActorId - << ", scanId: " << ScanId << ", table: " << TablePath + PageFaults = 0; + + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Send ScanData" + << ", from: " << ScanActorId << ", to: " << ComputeActorId + << ", scanId: " << ScanId << ", table: " << TablePath << ", bytes: " << sendBytes << ", rows: " << Rows << ", page faults: " << Result->PageFaults - << ", finished: " << Result->Finished << ", pageFault: " << Result->PageFault); - + << ", finished: " << Result->Finished << ", pageFault: " << Result->PageFault); + if (PeerFreeSpace < sendBytes) { - PeerFreeSpace = 0; - } else { + PeerFreeSpace = 0; + } else { PeerFreeSpace -= sendBytes; - } - + } + if (sendBytes >= 48_MB) { LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Query size limit exceeded."); if (finish) { @@ -471,14 +471,14 @@ private: } } - Send(ComputeActorId, Result.Release(), IEventHandle::FlagTrackDelivery); - ReportDatashardStats(); - - return true; - } - return false; - } - + Send(ComputeActorId, Result.Release(), IEventHandle::FlagTrackDelivery); + ReportDatashardStats(); + + return true; + } + return false; + } + // Call only after MakeResult method. void FlushBatchToResult() { // FlushBatch reset Batch pointer in BatchBuilder only if some rows were added after. So we if we have already @@ -489,107 +489,107 @@ private: } } - void ReportDatashardStats() { + void ReportDatashardStats() { Send(DatashardActorId, new TDataShard::TEvPrivate::TEvScanStats(Rows, CellvecBytes)); - Rows = 0; + Rows = 0; CellvecBytes = 0; - } - - bool IsProfile() const { + } + + bool IsProfile() const { return StatsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE; - } - -private: - TActorId ComputeActorId; - const TActorId DatashardActorId; - const ui32 ScanId; + } + +private: + TActorId ComputeActorId; + const TActorId DatashardActorId; + const ui32 ScanId; const NDataShard::TUserTable::TCPtr TableInfo; - const TString TablePath; + const TString TablePath; const TSmallVec<TSerializedTableRange> TableRanges; ui32 CurrentRange; - const TSmallVec<NTable::TTag> Tags; + const TSmallVec<NTable::TTag> Tags; TSmallVec<NScheme::TTypeId> Types; - const TSmallVec<bool> SkipNullKeys; + const TSmallVec<bool> SkipNullKeys; const NYql::NDqProto::EDqStatsMode StatsMode; - const TInstant Deadline; + const TInstant Deadline; const ui32 Generation; const NKikimrTxDataShard::EScanDataFormat DataFormat; - ui64 PeerFreeSpace = 0; - bool Sleep; - const bool IsLocal; - - IDriver* Driver = nullptr; - TActorId ScanActorId; - TActorId TimeoutActorId; + ui64 PeerFreeSpace = 0; + bool Sleep; + const bool IsLocal; + + IDriver* Driver = nullptr; + TActorId ScanActorId; + TActorId TimeoutActorId; TAutoPtr<TEvKqp::TEvAbortExecution> AbortEvent; - + THolder<NArrow::TArrowBatchBuilder> BatchBuilder; - THolder<TEvKqpCompute::TEvScanData> Result; - ui64 Rows = 0; + THolder<TEvKqpCompute::TEvScanData> Result; + ui64 Rows = 0; ui64 CellvecBytes = 0; - ui32 PageFaults = 0; - TInstant StartWaitTime; - - // Key corresponding to the latest row added to RowsBuffer. Used to avoid row duplication on - // scan restarts. - TOwnedCellVec LastKey; -}; - + ui32 PageFaults = 0; + TInstant StartWaitTime; + + // Key corresponding to the latest row added to RowsBuffer. Used to avoid row duplication on + // scan restarts. + TOwnedCellVec LastKey; +}; + void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext&) { - auto& request = ev->Get()->Record; - auto scanComputeActor = ev->Sender; + auto& request = ev->Get()->Record; + auto scanComputeActor = ev->Sender; auto generation = request.GetGeneration(); - - auto infoIt = TableInfos.find(request.GetLocalPathId()); - + + auto infoIt = TableInfos.find(request.GetLocalPathId()); + auto reportError = [this, scanComputeActor, generation] (const TString& table, const TString& detailedReason) { auto ev = MakeHolder<TEvKqpCompute::TEvScanError>(generation); - ev->Record.SetStatus(Ydb::StatusIds::ABORTED); + ev->Record.SetStatus(Ydb::StatusIds::ABORTED); auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_SCHEME_MISMATCH, TStringBuilder() << "Table '" << table << "' scheme changed."); - IssueToMessage(issue, ev->Record.MutableIssues()->Add()); - Send(scanComputeActor, ev.Release(), IEventHandle::FlagTrackDelivery); - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, detailedReason); - }; - - if (infoIt == TableInfos.end()) { - reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "." - << " Can not find table '" << request.GetTablePath() << "'" - << " by LocalPathId " << request.GetLocalPathId() << " at " << TabletID()); - return; - } - + IssueToMessage(issue, ev->Record.MutableIssues()->Add()); + Send(scanComputeActor, ev.Release(), IEventHandle::FlagTrackDelivery); + LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, detailedReason); + }; + + if (infoIt == TableInfos.end()) { + reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "." + << " Can not find table '" << request.GetTablePath() << "'" + << " by LocalPathId " << request.GetLocalPathId() << " at " << TabletID()); + return; + } + auto tableInfo = infoIt->second; // copy table info ptr here auto& tableColumns = tableInfo->Columns; - Y_VERIFY(request.GetColumnTags().size() == request.GetColumnTypes().size()); - + Y_VERIFY(request.GetColumnTags().size() == request.GetColumnTypes().size()); + if (tableInfo->GetTableSchemaVersion() != 0 && request.GetSchemaVersion() != tableInfo->GetTableSchemaVersion()) - { - reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "." - << " Table '" << request.GetTablePath() << "'" - << " schema version changed at " << TabletID()); - return; - } - - - for (int i = 0; i < request.GetColumnTags().size(); ++i) { - auto* column = tableColumns.FindPtr(request.GetColumnTags(i)); - if (!column) { - reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "." - << " Cant find table '" << request.GetTablePath() << "'" - << " column " << request.GetColumnTags(i) << " at " << TabletID()); - return; - } - - if (column->Type != request.GetColumnTypes(i)) { - reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "." - << " Table '" << request.GetTablePath() << "'" - << " column " << request.GetColumnTags(i) << " type mismatch at " << TabletID()); - return; - } - } - + { + reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "." + << " Table '" << request.GetTablePath() << "'" + << " schema version changed at " << TabletID()); + return; + } + + + for (int i = 0; i < request.GetColumnTags().size(); ++i) { + auto* column = tableColumns.FindPtr(request.GetColumnTags(i)); + if (!column) { + reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "." + << " Cant find table '" << request.GetTablePath() << "'" + << " column " << request.GetColumnTags(i) << " at " << TabletID()); + return; + } + + if (column->Type != request.GetColumnTypes(i)) { + reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "." + << " Table '" << request.GetTablePath() << "'" + << " column " << request.GetColumnTags(i) << " type mismatch at " << TabletID()); + return; + } + } + if (request.HasOlapProgram()) { auto msg = TStringBuilder() << "TxId: " << request.GetTxId() << "." << " Unexpected process program in datashard scan at " << TabletID(); @@ -603,8 +603,8 @@ void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext& return; } - auto& snapshot = request.GetSnapshot(); - + auto& snapshot = request.GetSnapshot(); + auto snapshotKey = TSnapshotKey(PathOwnerId, request.GetLocalPathId(), snapshot.GetStep(), snapshot.GetTxId()); if (!SnapshotManager.FindAvailable(snapshotKey)) { reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "." @@ -612,8 +612,8 @@ void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext& return; } - Pipeline.StartStreamingTx(snapshot.GetTxId(), 1); - + Pipeline.StartStreamingTx(snapshot.GetTxId(), 1); + TSmallVec<TSerializedTableRange> ranges; ranges.reserve(request.RangesSize()); @@ -621,28 +621,28 @@ void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext& ranges.emplace_back(std::move(TSerializedTableRange(range))); } - auto* tableScan = new TKqpScan( - scanComputeActor, - SelfId(), - request.GetScanId(), - tableInfo, + auto* tableScan = new TKqpScan( + scanComputeActor, + SelfId(), + request.GetScanId(), + tableInfo, std::move(ranges), std::move(TSmallVec<NTable::TTag>(request.GetColumnTags().begin(), request.GetColumnTags().end())), std::move(TSmallVec<bool>(request.GetSkipNullKeys().begin(), request.GetSkipNullKeys().end())), - request.GetStatsMode(), + request.GetStatsMode(), request.GetTimeoutMs(), generation, request.GetDataFormat() - ); - - auto scanOptions = TScanOptions() - .DisableResourceBroker() - .SetReadPrio(TScanOptions::EReadPrio::Low) - .SetReadAhead(READAHEAD_LO, READAHEAD_HI) - .SetSnapshotRowVersion(TRowVersion(snapshot.GetStep(), snapshot.GetTxId())); - + ); + + auto scanOptions = TScanOptions() + .DisableResourceBroker() + .SetReadPrio(TScanOptions::EReadPrio::Low) + .SetReadAhead(READAHEAD_LO, READAHEAD_HI) + .SetSnapshotRowVersion(TRowVersion(snapshot.GetStep(), snapshot.GetTxId())); + Executor()->QueueScan(tableInfo->LocalTid, tableScan, snapshot.GetTxId(), scanOptions); -} - -} -} +} + +} +} diff --git a/ydb/core/tx/datashard/datashard__op_rows.cpp b/ydb/core/tx/datashard/datashard__op_rows.cpp index 3b3c9d9d6f9..fc77402f2e8 100644 --- a/ydb/core/tx/datashard/datashard__op_rows.cpp +++ b/ydb/core/tx/datashard/datashard__op_rows.cpp @@ -92,8 +92,8 @@ static void WrongShardState(NKikimrTxDataShard::TEvEraseRowsResponse& response) template <typename TEvResponse, typename TEvRequest> static bool MaybeReject(TDataShard* self, TEvRequest& ev, const TActorContext& ctx, const TString& txDesc, bool isWrite) { NKikimrTxDataShard::TEvProposeTransactionResult::EStatus rejectStatus; - TString rejectReason; - bool reject = self->CheckDataTxReject(txDesc, ctx, rejectStatus, rejectReason); + TString rejectReason; + bool reject = self->CheckDataTxReject(txDesc, ctx, rejectStatus, rejectReason); bool outOfSpace = false; if (!reject && isWrite) { @@ -116,7 +116,7 @@ static bool MaybeReject(TDataShard* self, TEvRequest& ev, const TActorContext& c LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, "Rejecting " << txDesc << " request on datashard" << ": tablet# " << self->TabletID() - << ", error# " << rejectReason); + << ", error# " << rejectReason); auto response = MakeHolder<TEvResponse>(); response->Record.SetTabletID(self->TabletID()); @@ -125,7 +125,7 @@ static bool MaybeReject(TDataShard* self, TEvRequest& ev, const TActorContext& c } else { WrongShardState(response->Record); } - response->Record.SetErrorDescription(rejectReason); + response->Record.SetErrorDescription(rejectReason); ctx.Send(ev->Sender, std::move(response)); return true; diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h index 11313474f80..d44eef218b8 100644 --- a/ydb/core/tx/datashard/datashard_impl.h +++ b/ydb/core/tx/datashard/datashard_impl.h @@ -229,7 +229,7 @@ class TDataShard friend class TTxS3Listing; friend class TExecuteKqpScanTxUnit; friend class TTableScan; - friend class TKqpScan; + friend class TKqpScan; friend class TTransQueue; friend class TOutReadSets; @@ -917,7 +917,7 @@ class TDataShard void Handle(TEvDataShard::TEvGetTableStats::TPtr& ev, const TActorContext& ctx); void Handle(TEvPrivate::TEvAsyncTableStats::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvS3ListingRequest::TPtr& ev, const TActorContext& ctx); - void Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext& ctx); + void Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvUploadRowsRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvEraseRowsRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvConditionalEraseRowsRequest::TPtr& ev, const TActorContext& ctx); @@ -1157,7 +1157,7 @@ public: bool CheckDataTxReject(const TString& opDescr, const TActorContext &ctx, NKikimrTxDataShard::TEvProposeTransactionResult::EStatus& rejectStatus, - TString &reason); + TString &reason); bool CheckDataTxRejectAndReply(TEvDataShard::TEvProposeTransaction* msg, const TActorContext& ctx); TSysLocks& SysLocksTable() { return SysLocks; } @@ -2179,7 +2179,7 @@ protected: HFunc(TEvDataShard::TEvGetTableStats, Handle); HFunc(TEvPrivate::TEvAsyncTableStats, Handle); HFunc(TEvDataShard::TEvS3ListingRequest, Handle); - HFunc(TEvDataShard::TEvKqpScan, Handle); + HFunc(TEvDataShard::TEvKqpScan, Handle); HFunc(TEvDataShard::TEvUploadRowsRequest, Handle); HFunc(TEvDataShard::TEvEraseRowsRequest, Handle); HFunc(TEvDataShard::TEvConditionalEraseRowsRequest, Handle); diff --git a/ydb/core/tx/datashard/datashard_kqp_compute.cpp b/ydb/core/tx/datashard/datashard_kqp_compute.cpp index edd6ca8e194..31df96e4a11 100644 --- a/ydb/core/tx/datashard/datashard_kqp_compute.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_compute.cpp @@ -27,7 +27,7 @@ typedef IComputationNode* (*TCallableDatashardBuilderFunc)(TCallable& callable, struct TKqpDatashardComputationMap { TKqpDatashardComputationMap() { - Map["KqpWideReadTable"] = &WrapKqpWideReadTable; + Map["KqpWideReadTable"] = &WrapKqpWideReadTable; Map["KqpWideReadTableRanges"] = &WrapKqpWideReadTableRanges; Map["KqpLookupTable"] = &WrapKqpLookupTable; Map["KqpUpsertRows"] = &WrapKqpUpsertRows; @@ -65,7 +65,7 @@ typedef IComputationNode* (*TCallableScanBuilderFunc)(TCallable& callable, struct TKqpScanComputationMap { TKqpScanComputationMap() { - Map["KqpWideReadTable"] = &WrapKqpScanWideReadTable; + Map["KqpWideReadTable"] = &WrapKqpScanWideReadTable; Map["KqpWideReadTableRanges"] = &WrapKqpScanWideReadTableRanges; } diff --git a/ydb/core/tx/datashard/datashard_kqp_compute.h b/ydb/core/tx/datashard/datashard_kqp_compute.h index 4bbb8fa0de7..141ae0a1a92 100644 --- a/ydb/core/tx/datashard/datashard_kqp_compute.h +++ b/ydb/core/tx/datashard/datashard_kqp_compute.h @@ -102,11 +102,11 @@ IComputationNode* WrapKqpDeleteRows(TCallable& callable, const TComputationNodeF TKqpDatashardComputeContext& computeCtx); IComputationNode* WrapKqpEffects(TCallable& callable, const TComputationNodeFactoryContext& ctx, TKqpDatashardComputeContext& computeCtx); -IComputationNode* WrapKqpWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx, - TKqpDatashardComputeContext& computeCtx); +IComputationNode* WrapKqpWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx, + TKqpDatashardComputeContext& computeCtx); TComputationNodeFactory GetKqpDatashardComputeFactory(TKqpDatashardComputeContext* computeCtx); TComputationNodeFactory GetKqpScanComputeFactory(TKqpScanComputeContext* computeCtx); -} // namespace NMiniKQL +} // namespace NMiniKQL } // namespace NKikimr diff --git a/ydb/core/tx/datashard/datashard_kqp_read_table.cpp b/ydb/core/tx/datashard/datashard_kqp_read_table.cpp index 9fbd8ef0ad2..9cc1bed0fd1 100644 --- a/ydb/core/tx/datashard/datashard_kqp_read_table.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_read_table.cpp @@ -74,18 +74,18 @@ TMaybe<TKeyRangesType> ParseKeyRangesType(const TTupleType* rangeTupleType) { }; } -TSerializedTableRange CreateTableRange(const TParseReadTableResult& parseResult, const IComputationNode* fromNode, - const IComputationNode* toNode, const TTypeEnvironment& typeEnv, TComputationContext& ctx) -{ - TVector<TCell> fromCells; - BuildKeyTupleCells(parseResult.FromTuple->GetType(), fromNode->GetValue(ctx), fromCells, typeEnv); - - TVector<TCell> toCells; - BuildKeyTupleCells(parseResult.ToTuple->GetType(), toNode->GetValue(ctx), toCells, typeEnv); - - return TSerializedTableRange(fromCells, parseResult.FromInclusive, toCells, parseResult.ToInclusive); -} - +TSerializedTableRange CreateTableRange(const TParseReadTableResult& parseResult, const IComputationNode* fromNode, + const IComputationNode* toNode, const TTypeEnvironment& typeEnv, TComputationContext& ctx) +{ + TVector<TCell> fromCells; + BuildKeyTupleCells(parseResult.FromTuple->GetType(), fromNode->GetValue(ctx), fromCells, typeEnv); + + TVector<TCell> toCells; + BuildKeyTupleCells(parseResult.ToTuple->GetType(), toNode->GetValue(ctx), toCells, typeEnv); + + return TSerializedTableRange(fromCells, parseResult.FromInclusive, toCells, parseResult.ToInclusive); +} + TSerializedTableRange BuildFullRange(ui32 keyColumnsSize) { /* Build range from NULL, ... NULL to +inf, ... +inf */ TVector<TCell> fromKeyValues(keyColumnsSize); @@ -183,51 +183,51 @@ TVector<TSerializedTableRange> CreateTableRanges(const TParseReadTableRangesResu return ranges; } -void CreateRangePoints(ui64 localTid, const TSerializedTableRange& serializedTableRange, TSmallVec<TRawTypeValue>& from, - TSmallVec<TRawTypeValue>& to, TKqpDatashardComputeContext& computeCtx) -{ - const auto* tableInfo = computeCtx.Database->GetScheme().GetTableInfo(localTid); - auto tableRange = serializedTableRange.ToTableRange(); - ConvertTableKeys(computeCtx.Database->GetScheme(), tableInfo, tableRange.From, from, nullptr); - ConvertTableKeys(computeCtx.Database->GetScheme(), tableInfo, tableRange.To, to, nullptr); -} - +void CreateRangePoints(ui64 localTid, const TSerializedTableRange& serializedTableRange, TSmallVec<TRawTypeValue>& from, + TSmallVec<TRawTypeValue>& to, TKqpDatashardComputeContext& computeCtx) +{ + const auto* tableInfo = computeCtx.Database->GetScheme().GetTableInfo(localTid); + auto tableRange = serializedTableRange.ToTableRange(); + ConvertTableKeys(computeCtx.Database->GetScheme(), tableInfo, tableRange.From, from, nullptr); + ConvertTableKeys(computeCtx.Database->GetScheme(), tableInfo, tableRange.To, to, nullptr); +} + template <bool IsReverse> class TKqpWideReadTableWrapperBase : public TStatelessWideFlowCodegeneratorNode<TKqpWideReadTableWrapperBase<IsReverse>> { -public: +public: TKqpWideReadTableWrapperBase(TKqpDatashardComputeContext& computeCtx, const TTypeEnvironment& typeEnv, const TSmallVec<TTag>& systemColumnTags, const TSmallVec<bool>& skipNullKeys) : TStatelessWideFlowCodegeneratorNode<TKqpWideReadTableWrapperBase<IsReverse>>(this) , ComputeCtx(computeCtx) - , TypeEnv(typeEnv) + , TypeEnv(typeEnv) , SystemColumnTags(systemColumnTags) , SkipNullKeys(skipNullKeys) , ShardTableStats(ComputeCtx.GetDatashardCounters()) , TaskTableStats(ComputeCtx.GetTaskCounters(ComputeCtx.GetCurrentTaskId())) { } - - EFetchResult DoCalculate(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const { - return this->ReadValue(ctx, output); - } - -#ifndef MKQL_DISABLE_CODEGEN - ICodegeneratorInlineWideNode::TGenerateResult DoGenGetValues(const TCodegenContext& ctx, BasicBlock*& block) const { - Y_UNUSED(ctx, block); - Y_FAIL("LLVM compilation is not implemented for OLTP-workload"); - } -#endif - + + EFetchResult DoCalculate(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const { + return this->ReadValue(ctx, output); + } + +#ifndef MKQL_DISABLE_CODEGEN + ICodegeneratorInlineWideNode::TGenerateResult DoGenGetValues(const TCodegenContext& ctx, BasicBlock*& block) const { + Y_UNUSED(ctx, block); + Y_FAIL("LLVM compilation is not implemented for OLTP-workload"); + } +#endif + protected: virtual EFetchResult ReadValue(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const = 0; - + EFetchResult ReadNext(NUdf::TUnboxedValue* const* output) const { bool breakLocks = false; - while (Iterator->Next(NTable::ENext::Data) == NTable::EReady::Data) { + while (Iterator->Next(NTable::ENext::Data) == NTable::EReady::Data) { if (!breakLocks && (breakLocks = bool(Iterator->Stats.InvisibleRowSkips))) { ComputeCtx.BreakSetLocks(); } - TDbTupleRef rowKey = Iterator->GetKey(); - + TDbTupleRef rowKey = Iterator->GetKey(); + ui64 deletedRowSkips = std::exchange(Iterator->Stats.DeletedRowSkips, 0); ui64 invisibleRowSkips = std::exchange(Iterator->Stats.InvisibleRowSkips, 0); @@ -238,47 +238,47 @@ protected: TaskTableStats.InvisibleRowSkips += invisibleRowSkips; Y_VERIFY(SkipNullKeys.size() <= rowKey.ColumnCount); - bool skipRow = false; + bool skipRow = false; for (ui32 i = 0; i < SkipNullKeys.size(); ++i) { if (SkipNullKeys[i] && rowKey.Columns[i].IsNull()) { - skipRow = true; - break; - } - } - if (skipRow) { - continue; - } - - TDbTupleRef rowValues = Iterator->GetValues(); - - size_t columnsCount = rowValues.ColumnCount + SystemColumnTags.size(); - - ui64 rowSize = 0; - for (ui32 i = 0; i < rowValues.ColumnCount; ++i) { - rowSize += rowValues.Columns[i].IsNull() ? 1 : rowValues.Columns[i].Size(); - if (auto out = *output++) { - *out = GetCellValue(rowValues.Cells()[i], rowValues.Types[i]); - } - } - - // Some per-row overhead to deal with the case when no columns were requested - rowSize = std::max(rowSize, (ui64) 8); - - for (ui32 i = rowValues.ColumnCount, j = 0; i < columnsCount; ++i, ++j) { - auto out = *output++; - if (!out) { - continue; - } - - switch (SystemColumnTags[j]) { - case TKeyDesc::EColumnIdDataShard: - *out = TUnboxedValue(TUnboxedValuePod(ComputeCtx.GetShardId())); - break; - default: - throw TSchemeErrorTabletException(); - } - } - + skipRow = true; + break; + } + } + if (skipRow) { + continue; + } + + TDbTupleRef rowValues = Iterator->GetValues(); + + size_t columnsCount = rowValues.ColumnCount + SystemColumnTags.size(); + + ui64 rowSize = 0; + for (ui32 i = 0; i < rowValues.ColumnCount; ++i) { + rowSize += rowValues.Columns[i].IsNull() ? 1 : rowValues.Columns[i].Size(); + if (auto out = *output++) { + *out = GetCellValue(rowValues.Cells()[i], rowValues.Types[i]); + } + } + + // Some per-row overhead to deal with the case when no columns were requested + rowSize = std::max(rowSize, (ui64) 8); + + for (ui32 i = rowValues.ColumnCount, j = 0; i < columnsCount; ++i, ++j) { + auto out = *output++; + if (!out) { + continue; + } + + switch (SystemColumnTags[j]) { + case TKeyDesc::EColumnIdDataShard: + *out = TUnboxedValue(TUnboxedValuePod(ComputeCtx.GetShardId())); + break; + default: + throw TSchemeErrorTabletException(); + } + } + if (Remains) { Remains = *Remains - 1; } @@ -289,9 +289,9 @@ protected: TaskTableStats.SelectRangeRows++; TaskTableStats.SelectRangeBytes += rowSize; - return EFetchResult::One; - } - + return EFetchResult::One; + } + if (!breakLocks && bool(Iterator->Stats.InvisibleRowSkips)) { ComputeCtx.BreakSetLocks(); } @@ -308,11 +308,11 @@ protected: if (Iterator->Last() == NTable::EReady::Page) { ComputeCtx.SetTabletNotReady(); return EFetchResult::Yield; - } - - return EFetchResult::Finish; - } - + } + + return EFetchResult::Finish; + } + protected: TKqpDatashardComputeContext& ComputeCtx; const TTypeEnvironment& TypeEnv; @@ -343,7 +343,7 @@ public: this->TaskTableStats.NSelectRange++; } -private: +private: EFetchResult ReadValue(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const final { if (!this->Iterator) { auto serializedTableRange = CreateTableRange(ParseResult, FromNode, ToNode, this->TypeEnv, ctx); @@ -378,21 +378,21 @@ private: } private: - void RegisterDependencies() const final { + void RegisterDependencies() const final { this->FlowDependsOn(FromNode); this->FlowDependsOn(ToNode); - } - -private: - TParseReadTableResult ParseResult; - IComputationNode* FromNode; - IComputationNode* ToNode; + } + +private: + TParseReadTableResult ParseResult; + IComputationNode* FromNode; + IComputationNode* ToNode; IComputationNode* ItemsLimit; - ui64 LocalTid; - TSmallVec<TTag> ColumnTags; + ui64 LocalTid; + TSmallVec<TTag> ColumnTags; ui64 TaskId; -}; - +}; + template <bool IsReverse> class TKqpWideReadTableRangesWrapper : public TKqpWideReadTableWrapperBase<IsReverse> { public: @@ -582,13 +582,13 @@ IComputationNode* WrapKqpWideReadTableRanges(TCallable& callable, const TComputa return new TKqpWideReadTableRangesWrapper<false>(computeCtx, ctx.Env, parseResult, rangesNode, itemsLimit); } -IComputationNode* WrapKqpWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx, - TKqpDatashardComputeContext& computeCtx) -{ - auto parseResult = ParseWideReadTable(callable); - auto fromNode = LocateNode(ctx.NodeLocator, *parseResult.FromTuple); - auto toNode = LocateNode(ctx.NodeLocator, *parseResult.ToTuple); - +IComputationNode* WrapKqpWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx, + TKqpDatashardComputeContext& computeCtx) +{ + auto parseResult = ParseWideReadTable(callable); + auto fromNode = LocateNode(ctx.NodeLocator, *parseResult.FromTuple); + auto toNode = LocateNode(ctx.NodeLocator, *parseResult.ToTuple); + auto keyColumns = computeCtx.GetKeyColumnsInfo(parseResult.TableId); ValidateKeyTuple(parseResult.FromTuple->GetType(), keyColumns); ValidateKeyTuple(parseResult.ToTuple->GetType(), keyColumns); @@ -603,7 +603,7 @@ IComputationNode* WrapKqpWideReadTable(TCallable& callable, const TComputationNo } return new TKqpWideReadTableWrapper<false>(computeCtx, ctx.Env, parseResult, fromNode, toNode, itemsLimit); -} - -} // namespace NMiniKQL +} + +} // namespace NMiniKQL } // namespace NKikimr diff --git a/ydb/core/tx/datashard/datashard_pipeline.h b/ydb/core/tx/datashard/datashard_pipeline.h index 67c35260b28..9f265580377 100644 --- a/ydb/core/tx/datashard/datashard_pipeline.h +++ b/ydb/core/tx/datashard/datashard_pipeline.h @@ -305,8 +305,8 @@ public: void HoldExecutionProfile(TOperation::TPtr op); void FillStoredExecutionProfiles(NKikimrTxDataShard::TEvGetSlowOpProfilesResponse &rec) const; - void StartStreamingTx(ui64 txId, ui32 count) { - ActiveStreamingTxs[txId] += count; + void StartStreamingTx(ui64 txId, ui32 count) { + ActiveStreamingTxs[txId] += count; } bool FinishStreamingTx(ui64 txId) { diff --git a/ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp b/ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp index b2b14de1904..db45e2820e1 100644 --- a/ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp +++ b/ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp @@ -3,29 +3,29 @@ #include <ydb/core/tx/datashard/datashard.h> #include <ydb/core/tx/datashard/datashard_ut_common_kqp.h> #include <ydb/core/tx/tx_proxy/proxy.h> - -namespace NKikimr { -namespace NKqp { - -using namespace NYdb; -using namespace NYdb::NTable; -using namespace NYdb::NScripting; -using namespace Tests; + +namespace NKikimr { +namespace NKqp { + +using namespace NYdb; +using namespace NYdb::NTable; +using namespace NYdb::NScripting; +using namespace Tests; using namespace NKikimr::NDataShard::NKqpHelpers; - -namespace { - - /* sum(value) == 596400 */ - TString FillTableQuery() { - TStringBuilder sql; - sql << "UPSERT INTO [/Root/table-1] (key, value) VALUES "; - for (int i = 1; i < 100; ++i) { - sql << " (" << i << ", " << i << i << "),"; - } - sql << " (100500, 100500);"; - return sql; - } - + +namespace { + + /* sum(value) == 596400 */ + TString FillTableQuery() { + TStringBuilder sql; + sql << "UPSERT INTO [/Root/table-1] (key, value) VALUES "; + for (int i = 1; i < 100; ++i) { + sql << " (" << i << ", " << i << i << "),"; + } + sql << " (100500, 100500);"; + return sql; + } + void EnableLogging(TTestActorRuntime& runtime) { runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG); //runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG); @@ -36,344 +36,344 @@ namespace { runtime.SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_TRACE); } -} - -Y_UNIT_TEST_SUITE(KqpScan) { - - - /* +} + +Y_UNIT_TEST_SUITE(KqpScan) { + + + /* * Start scan on shard with amount of rows bigger then ScanData capacity (~ 10 ScanData's to - * complete scan). Event capture filter will pass only first ScanData from one table actor and will - * kill tablet after that. So in order to complete scan ComputeActor need to handle scan restart after - * each ScanData. - */ + * complete scan). Event capture filter will pass only first ScanData from one table actor and will + * kill tablet after that. So in order to complete scan ComputeActor need to handle scan restart after + * each ScanData. + */ Y_UNIT_TEST_WITH_MVCC(ScanRetryRead) { - NKikimrConfig::TAppConfig appCfg; - - auto* rm = appCfg.MutableTableServiceConfig()->MutableResourceManager(); - rm->SetChannelBufferSize(100); - rm->SetMinChannelBufferSize(100); - rm->SetScanBufferSize(100); - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings.SetDomainName("Root") + NKikimrConfig::TAppConfig appCfg; + + auto* rm = appCfg.MutableTableServiceConfig()->MutableResourceManager(); + rm->SetChannelBufferSize(100); + rm->SetMinChannelBufferSize(100); + rm->SetScanBufferSize(100); + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings.SetDomainName("Root") .SetEnableMvcc(WithMvcc) - .SetNodeCount(2) - .SetAppConfig(appCfg) - .SetUseRealThreads(false); - - Tests::TServer::TPtr server = new TServer(serverSettings); - auto &runtime = *server->GetRuntime(); - auto sender = runtime.AllocateEdgeActor(); - + .SetNodeCount(2) + .SetAppConfig(appCfg) + .SetUseRealThreads(false); + + Tests::TServer::TPtr server = new TServer(serverSettings); + auto &runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + // EnableLogging(runtime); - - InitRoot(server, sender); + + InitRoot(server, sender); CreateShardedTable(server, sender, "/Root", "table-1", 1); - ExecSQL(server, sender, FillTableQuery()); - - TSet<TActorId> scans; + ExecSQL(server, sender, FillTableQuery()); + + TSet<TActorId> scans; TSet<TActorId> killedTablets; - - ui64 result = 0; - - auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> auto { - - switch (ev->GetTypeRewrite()) { - /* - * Trick executor to think that all datashard are located on node 1. - */ - case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: { - auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>(); - for (auto& [shardId, nodeId]: msg->ShardNodes) { + + ui64 result = 0; + + auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> auto { + + switch (ev->GetTypeRewrite()) { + /* + * Trick executor to think that all datashard are located on node 1. + */ + case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: { + auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>(); + for (auto& [shardId, nodeId]: msg->ShardNodes) { Cerr << "-- nodeId: " << nodeId << Endl; Cerr.Flush(); nodeId = runtime.GetNodeId(0); - } - break; - } - - case TEvDataShard::EvKqpScan: { - Cerr << (TStringBuilder() << "-- EvScan " << ev->Sender << " -> " << ev->Recipient << Endl); + } + break; + } + + case TEvDataShard::EvKqpScan: { + Cerr << (TStringBuilder() << "-- EvScan " << ev->Sender << " -> " << ev->Recipient << Endl); Cerr.Flush(); - break; - } - - /* - * Respond to streamData with acks. Without that execution pipeline will stop - * producing new tuples. - */ - case NKqp::TKqpExecuterEvents::EvStreamData: { - auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record; - - Cerr << (TStringBuilder() << "-- EvStreamData: " << record.AsJSON() << Endl); + break; + } + + /* + * Respond to streamData with acks. Without that execution pipeline will stop + * producing new tuples. + */ + case NKqp::TKqpExecuterEvents::EvStreamData: { + auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record; + + Cerr << (TStringBuilder() << "-- EvStreamData: " << record.AsJSON() << Endl); Cerr.Flush(); - - Y_ASSERT(record.GetResultSet().rows().size() == 1); - Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1); - result = record.GetResultSet().rows().at(0).items().at(0).uint64_value(); - - auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>(); - resp->Record.SetEnough(false); - resp->Record.SetSeqNo(ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record.GetSeqNo()); - resp->Record.SetFreeSpace(100); - runtime.Send(new IEventHandle(ev->Sender, sender, resp.Release())); - return TTestActorRuntime::EEventAction::DROP; - } - - /* Drop message and kill tablet if we already had seen this tablet */ - case NKqp::TKqpComputeEvents::EvScanData: { - if (scans.contains(ev->Sender)) { + + Y_ASSERT(record.GetResultSet().rows().size() == 1); + Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1); + result = record.GetResultSet().rows().at(0).items().at(0).uint64_value(); + + auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>(); + resp->Record.SetEnough(false); + resp->Record.SetSeqNo(ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record.GetSeqNo()); + resp->Record.SetFreeSpace(100); + runtime.Send(new IEventHandle(ev->Sender, sender, resp.Release())); + return TTestActorRuntime::EEventAction::DROP; + } + + /* Drop message and kill tablet if we already had seen this tablet */ + case NKqp::TKqpComputeEvents::EvScanData: { + if (scans.contains(ev->Sender)) { if (killedTablets.empty()) { // do only 1 kill per test runtime.Send(new IEventHandle(ev->Sender, ev->Sender, new NKqp::TEvKqpCompute::TEvKillScanTablet)); Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": hijack event, kill tablet " << ev->Sender << Endl); Cerr.Flush(); } - } else { - scans.insert(ev->Sender); + } else { + scans.insert(ev->Sender); runtime.EnableScheduleForActor(ev->Sender); - Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": pass" << Endl); + Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": pass" << Endl); Cerr.Flush(); - } - - break; - } - - default: - break; - } - return TTestActorRuntime::EEventAction::PROCESS; - }; - runtime.SetObserverFunc(captureEvents); - - auto streamSender = runtime.AllocateEdgeActor(); - SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, "SELECT sum(value) FROM `/Root/table-1`;", false)); - auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender); - - UNIT_ASSERT_VALUES_EQUAL(result, 596400); - } - - /* - * Force remote scans by meddling with EvShardsResolveStatus. Check that remote scan actually took place. - */ + } + + break; + } + + default: + break; + } + return TTestActorRuntime::EEventAction::PROCESS; + }; + runtime.SetObserverFunc(captureEvents); + + auto streamSender = runtime.AllocateEdgeActor(); + SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, "SELECT sum(value) FROM `/Root/table-1`;", false)); + auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender); + + UNIT_ASSERT_VALUES_EQUAL(result, 596400); + } + + /* + * Force remote scans by meddling with EvShardsResolveStatus. Check that remote scan actually took place. + */ Y_UNIT_TEST_WITH_MVCC(RemoteShardScan) { - NKikimrConfig::TAppConfig appCfg; - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings.SetDomainName("Root") + NKikimrConfig::TAppConfig appCfg; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings.SetDomainName("Root") .SetEnableMvcc(WithMvcc) - .SetNodeCount(2) - .SetAppConfig(appCfg) - .SetUseRealThreads(false); - - Tests::TServer::TPtr server = new TServer(serverSettings); - auto &runtime = *server->GetRuntime(); - auto sender = runtime.AllocateEdgeActor(); - + .SetNodeCount(2) + .SetAppConfig(appCfg) + .SetUseRealThreads(false); + + Tests::TServer::TPtr server = new TServer(serverSettings); + auto &runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + // EnableLogging(runtime); - - InitRoot(server, sender); + + InitRoot(server, sender); CreateShardedTable(server, sender, "/Root", "table-1", 7); - ExecSQL(server, sender, FillTableQuery()); - + ExecSQL(server, sender, FillTableQuery()); + ui64 firstNodeId = server->GetRuntime()->GetNodeId(0); // ui64 secondNodeId = server->GetRuntime()->GetNodeId(1); - bool remoteScanDetected = false; - - ui64 result = 0; - - auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) { - - switch (ev->GetTypeRewrite()) { - /* - * Trick executor to think that all datashard are located on node 1. - */ - case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: { - auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>(); - for (auto& [shardId, nodeId]: msg->ShardNodes) { + bool remoteScanDetected = false; + + ui64 result = 0; + + auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) { + + switch (ev->GetTypeRewrite()) { + /* + * Trick executor to think that all datashard are located on node 1. + */ + case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: { + auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>(); + for (auto& [shardId, nodeId]: msg->ShardNodes) { nodeId = firstNodeId; - } - break; - } - - /* - * Respond to streamData with acks. Without that execution pipeline will stop - * producing new tuples. - */ - case NKqp::TKqpExecuterEvents::EvStreamData: { - auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record; - - Cerr << (TStringBuilder() << "-- EvStreamData(from: " << ev->Sender << ", to: " << ev->Recipient << "): " << record.AsJSON() << Endl); - - Y_ASSERT(record.GetResultSet().rows().size() == 1); - Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1); - result = record.GetResultSet().rows().at(0).items().at(0).uint64_value(); - - auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>(); - resp->Record.SetEnough(false); - resp->Record.SetSeqNo(ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record.GetSeqNo()); - resp->Record.SetFreeSpace(100); - runtime.Send(new IEventHandle(ev->Sender, sender, resp.Release())); - return TTestActorRuntime::EEventAction::DROP; - } - - /* - * Check that remote scan actually happend. - */ - case NKqp::TKqpComputeEvents::EvScanData: { - remoteScanDetected = remoteScanDetected || ev->Sender.NodeId() != ev->Recipient.NodeId(); - break; - } - - default: - break; - } - return TTestActorRuntime::EEventAction::PROCESS; - }; - runtime.SetObserverFunc(captureEvents); - - auto streamSender = runtime.AllocateEdgeActor(); - SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, "SELECT sum(value) FROM `/Root/table-1`;", false)); - auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender); - - UNIT_ASSERT(remoteScanDetected); - UNIT_ASSERT_VALUES_EQUAL(result, 596400); - } - + } + break; + } + + /* + * Respond to streamData with acks. Without that execution pipeline will stop + * producing new tuples. + */ + case NKqp::TKqpExecuterEvents::EvStreamData: { + auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record; + + Cerr << (TStringBuilder() << "-- EvStreamData(from: " << ev->Sender << ", to: " << ev->Recipient << "): " << record.AsJSON() << Endl); + + Y_ASSERT(record.GetResultSet().rows().size() == 1); + Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1); + result = record.GetResultSet().rows().at(0).items().at(0).uint64_value(); + + auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>(); + resp->Record.SetEnough(false); + resp->Record.SetSeqNo(ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record.GetSeqNo()); + resp->Record.SetFreeSpace(100); + runtime.Send(new IEventHandle(ev->Sender, sender, resp.Release())); + return TTestActorRuntime::EEventAction::DROP; + } + + /* + * Check that remote scan actually happend. + */ + case NKqp::TKqpComputeEvents::EvScanData: { + remoteScanDetected = remoteScanDetected || ev->Sender.NodeId() != ev->Recipient.NodeId(); + break; + } + + default: + break; + } + return TTestActorRuntime::EEventAction::PROCESS; + }; + runtime.SetObserverFunc(captureEvents); + + auto streamSender = runtime.AllocateEdgeActor(); + SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, "SELECT sum(value) FROM `/Root/table-1`;", false)); + auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender); + + UNIT_ASSERT(remoteScanDetected); + UNIT_ASSERT_VALUES_EQUAL(result, 596400); + } + Y_UNIT_TEST_WITH_MVCC(ScanDuringSplit) { - NKikimrConfig::TAppConfig appCfg; - - auto* rm = appCfg.MutableTableServiceConfig()->MutableResourceManager(); - rm->SetChannelBufferSize(100); - rm->SetMinChannelBufferSize(100); - rm->SetScanBufferSize(100); - - TPortManager pm; - TServerSettings serverSettings(pm.GetPort(2134)); - serverSettings.SetDomainName("Root") + NKikimrConfig::TAppConfig appCfg; + + auto* rm = appCfg.MutableTableServiceConfig()->MutableResourceManager(); + rm->SetChannelBufferSize(100); + rm->SetMinChannelBufferSize(100); + rm->SetScanBufferSize(100); + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings.SetDomainName("Root") .SetEnableMvcc(WithMvcc) - .SetNodeCount(2) - .SetAppConfig(appCfg) - .SetUseRealThreads(false); - - Tests::TServer::TPtr server = new TServer(serverSettings); - auto &runtime = *server->GetRuntime(); - auto sender = runtime.AllocateEdgeActor(); - auto senderSplit = runtime.AllocateEdgeActor(); - + .SetNodeCount(2) + .SetAppConfig(appCfg) + .SetUseRealThreads(false); + + Tests::TServer::TPtr server = new TServer(serverSettings); + auto &runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + auto senderSplit = runtime.AllocateEdgeActor(); + // EnableLogging(runtime); - + SetSplitMergePartCountLimit(&runtime, -1); - InitRoot(server, sender); + InitRoot(server, sender); CreateShardedTable(server, sender, "/Root", "table-1", 1); - ExecSQL(server, sender, FillTableQuery()); - - auto shards = GetTableShards(server, sender, "/Root/table-1"); + ExecSQL(server, sender, FillTableQuery()); + + auto shards = GetTableShards(server, sender, "/Root/table-1"); for (const auto& shard: shards) { Cerr << (TStringBuilder() << "-- shardId=" << shard << Endl); Cerr.Flush(); } - - TSet<TActorId> scans; - TActorId firstScanActor; - ui64 tabletId = 0; - - ui64 result = 0; - + + TSet<TActorId> scans; + TActorId firstScanActor; + ui64 tabletId = 0; + + ui64 result = 0; + auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) { - switch (ev->GetTypeRewrite()) { - case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: { - auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>(); - for (auto& [shardId, nodeId]: msg->ShardNodes) { - tabletId = shardId; - Cerr << (TStringBuilder() << "-- tabletId= " << tabletId << Endl); + switch (ev->GetTypeRewrite()) { + case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: { + auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>(); + for (auto& [shardId, nodeId]: msg->ShardNodes) { + tabletId = shardId; + Cerr << (TStringBuilder() << "-- tabletId= " << tabletId << Endl); Cerr.Flush(); - } - break; - } - - case TEvDataShard::EvKqpScan: { - Cerr << (TStringBuilder() << "-- EvScan " << ev->Sender << " -> " << ev->Recipient << Endl); + } + break; + } + + case TEvDataShard::EvKqpScan: { + Cerr << (TStringBuilder() << "-- EvScan " << ev->Sender << " -> " << ev->Recipient << Endl); Cerr.Flush(); - break; - } - - /* - * Respond to streamData with acks. Without that execution pipeline will stop - * producing new tuples. - */ - case NKqp::TKqpExecuterEvents::EvStreamData: { - auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record; - - Cerr << (TStringBuilder() << "-- EvStreamData: " << record.AsJSON() << Endl); + break; + } + + /* + * Respond to streamData with acks. Without that execution pipeline will stop + * producing new tuples. + */ + case NKqp::TKqpExecuterEvents::EvStreamData: { + auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record; + + Cerr << (TStringBuilder() << "-- EvStreamData: " << record.AsJSON() << Endl); Cerr.Flush(); - - Y_ASSERT(record.GetResultSet().rows().size() == 1); - Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1); - result = record.GetResultSet().rows().at(0).items().at(0).uint64_value(); - - auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>(); - resp->Record.SetEnough(false); - resp->Record.SetSeqNo(ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record.GetSeqNo()); - resp->Record.SetFreeSpace(100); - - runtime.Send(new IEventHandle(ev->Sender, sender, resp.Release())); - - return TTestActorRuntime::EEventAction::DROP; - } - - /* Drop message and kill tablet if we already had seen this tablet */ - case NKqp::TKqpComputeEvents::EvScanData: { - if (!firstScanActor) { - firstScanActor = ev->Sender; - - // send split - ui32 splitKey = 55; - auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); - request->Record.SetExecTimeoutPeriod(Max<ui64>()); - auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme(); + + Y_ASSERT(record.GetResultSet().rows().size() == 1); + Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1); + result = record.GetResultSet().rows().at(0).items().at(0).uint64_value(); + + auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>(); + resp->Record.SetEnough(false); + resp->Record.SetSeqNo(ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record.GetSeqNo()); + resp->Record.SetFreeSpace(100); + + runtime.Send(new IEventHandle(ev->Sender, sender, resp.Release())); + + return TTestActorRuntime::EEventAction::DROP; + } + + /* Drop message and kill tablet if we already had seen this tablet */ + case NKqp::TKqpComputeEvents::EvScanData: { + if (!firstScanActor) { + firstScanActor = ev->Sender; + + // send split + ui32 splitKey = 55; + auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>(); + request->Record.SetExecTimeoutPeriod(Max<ui64>()); + auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme(); tx.SetOperationType(NKikimrSchemeOp::ESchemeOpSplitMergeTablePartitions); - auto &desc = *tx.MutableSplitMergeTablePartitions(); - desc.SetTablePath("/Root/table-1"); - desc.AddSourceTabletId(tabletId); - desc.AddSplitBoundary()->MutableKeyPrefix()->AddTuple()->MutableOptional()->SetUint32(splitKey); - - runtime.Send(new IEventHandle(MakeTxProxyID(), senderSplit, request.Release())); - - Cerr << (TStringBuilder() << "-- EvScanData from old tablet " << ev->Sender << ": pass and split" << Endl); + auto &desc = *tx.MutableSplitMergeTablePartitions(); + desc.SetTablePath("/Root/table-1"); + desc.AddSourceTabletId(tabletId); + desc.AddSplitBoundary()->MutableKeyPrefix()->AddTuple()->MutableOptional()->SetUint32(splitKey); + + runtime.Send(new IEventHandle(MakeTxProxyID(), senderSplit, request.Release())); + + Cerr << (TStringBuilder() << "-- EvScanData from old tablet " << ev->Sender << ": pass and split" << Endl); Cerr.Flush(); - } else if (firstScanActor == ev->Sender) { - // data from old table scan, drop it - Cerr << (TStringBuilder() << "-- EvScanData from old tablet " << ev->Sender << ": drop" << Endl); + } else if (firstScanActor == ev->Sender) { + // data from old table scan, drop it + Cerr << (TStringBuilder() << "-- EvScanData from old tablet " << ev->Sender << ": drop" << Endl); Cerr.Flush(); return TTestActorRuntime::EEventAction::DROP; - } else { - // data from new tablet scan, pass it - Cerr << (TStringBuilder() << "-- EvScanData from new tablet" << ev->Sender << ": pass" << Endl); + } else { + // data from new tablet scan, pass it + Cerr << (TStringBuilder() << "-- EvScanData from new tablet" << ev->Sender << ": pass" << Endl); Cerr.Flush(); - } - - break; - } - - default: - break; - } - return TTestActorRuntime::EEventAction::PROCESS; - }; - runtime.SetObserverFunc(captureEvents); - - auto streamSender = runtime.AllocateEdgeActor(); - SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, "SELECT sum(value) FROM `/Root/table-1`;", false)); - auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender); - - UNIT_ASSERT_VALUES_EQUAL(result, 596400); - } - + } + + break; + } + + default: + break; + } + return TTestActorRuntime::EEventAction::PROCESS; + }; + runtime.SetObserverFunc(captureEvents); + + auto streamSender = runtime.AllocateEdgeActor(); + SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, "SELECT sum(value) FROM `/Root/table-1`;", false)); + auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender); + + UNIT_ASSERT_VALUES_EQUAL(result, 596400); + } + Y_UNIT_TEST_WITH_MVCC(ScanRetryReadRanges) { Y_UNUSED(EnableLogging); @@ -514,8 +514,8 @@ Y_UNIT_TEST_SUITE(KqpScan) { UNIT_ASSERT_VALUES_EQUAL(incomingRangesSize, 3); } -} - -} // namespace NKqp -} // namespace NKikimr - +} + +} // namespace NKqp +} // namespace NKikimr + diff --git a/ydb/core/tx/datashard/datashard_ut_order.cpp b/ydb/core/tx/datashard/datashard_ut_order.cpp index 32472dce59e..b819d6e6074 100644 --- a/ydb/core/tx/datashard/datashard_ut_order.cpp +++ b/ydb/core/tx/datashard/datashard_ut_order.cpp @@ -3686,18 +3686,18 @@ void TestLateKqpQueryAfterColumnDrop(bool dataQuery, const TString& query, bool // } switch (ev->GetTypeRewrite()) { case TEvDataShard::EvProposeTransaction: { - auto &rec = ev->Get<TEvDataShard::TEvProposeTransaction>()->Record; - if (capturePropose && rec.GetTxKind() != NKikimrTxDataShard::TX_KIND_SNAPSHOT) { - Cerr << "---- capture EvProposeTransaction ---- type=" << rec.GetTxKind() << Endl; - eventsPropose.emplace_back(ev.Release()); - return TTestActorRuntime::EEventAction::DROP; - } - break; - } - - case TEvDataShard::EvKqpScan: { + auto &rec = ev->Get<TEvDataShard::TEvProposeTransaction>()->Record; + if (capturePropose && rec.GetTxKind() != NKikimrTxDataShard::TX_KIND_SNAPSHOT) { + Cerr << "---- capture EvProposeTransaction ---- type=" << rec.GetTxKind() << Endl; + eventsPropose.emplace_back(ev.Release()); + return TTestActorRuntime::EEventAction::DROP; + } + break; + } + + case TEvDataShard::EvKqpScan: { if (capturePropose) { - Cerr << "---- capture EvKqpScan ----" << Endl; + Cerr << "---- capture EvKqpScan ----" << Endl; eventsPropose.emplace_back(ev.Release()); return TTestActorRuntime::EEventAction::DROP; } diff --git a/ydb/core/tx/datashard/ya.make b/ydb/core/tx/datashard/ya.make index 9c98e8e79b0..b245e681ff8 100644 --- a/ydb/core/tx/datashard/ya.make +++ b/ydb/core/tx/datashard/ya.make @@ -64,7 +64,7 @@ SRCS( datashard__s3.cpp datashard__s3_download_txs.cpp datashard__s3_upload_txs.cpp - datashard__kqp_scan.cpp + datashard__kqp_scan.cpp datashard__snapshot_txs.cpp datashard__stats.cpp datashard__store_table_path.cpp diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp index 82f6c5812ac..72403fe9756 100644 --- a/ydb/library/yql/ast/yql_expr.cpp +++ b/ydb/library/yql/ast/yql_expr.cpp @@ -1910,8 +1910,8 @@ namespace { } } - template <class TPMap> - bool GatherParentsImpl(const TExprNode& node, TPMap& parentsMap, TNodeSet& visited, bool withLeaves) { + template <class TPMap> + bool GatherParentsImpl(const TExprNode& node, TPMap& parentsMap, TNodeSet& visited, bool withLeaves) { if (node.Type() == TExprNode::Arguments) { return false; } @@ -1927,7 +1927,7 @@ namespace { } node.ForEachChild([&](const TExprNode& child) { - if (GatherParentsImpl<TPMap>(child, parentsMap, visited, withLeaves)) { + if (GatherParentsImpl<TPMap>(child, parentsMap, visited, withLeaves)) { parentsMap[&child].emplace(&node); } }); @@ -3180,15 +3180,15 @@ bool CompareExprTreeParts(const TExprNode& one, const TExprNode& two, const TNod void GatherParents(const TExprNode& node, TParentsMap& parentsMap, bool withLeaves) { parentsMap.clear(); TNodeSet visisted; - GatherParentsImpl<TParentsMap>(node, parentsMap, visisted, withLeaves); -} - -void GatherParentsMulti(const TExprNode& node, TParentsMultiMap& parentsMap, bool withLeaves) { - parentsMap.clear(); - TNodeSet visisted; - GatherParentsImpl<TParentsMultiMap>(node, parentsMap, visisted, withLeaves); + GatherParentsImpl<TParentsMap>(node, parentsMap, visisted, withLeaves); } +void GatherParentsMulti(const TExprNode& node, TParentsMultiMap& parentsMap, bool withLeaves) { + parentsMap.clear(); + TNodeSet visisted; + GatherParentsImpl<TParentsMultiMap>(node, parentsMap, visisted, withLeaves); +} + void CheckCounts(const TExprNode& root) { TRefCountsMap refCounts; CalculateReferences(root, refCounts); diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h index d6eb544f9af..9d65574e919 100644 --- a/ydb/library/yql/ast/yql_expr.h +++ b/ydb/library/yql/ast/yql_expr.h @@ -2074,9 +2074,9 @@ using TNodeSet = std::unordered_set<const TExprNode*>; using TNodeOnNodeOwnedMap = TNodeMap<TExprNode::TPtr>; using TParentsMap = TNodeMap<TNodeSet>; -using TNodeMultiSet = std::unordered_multiset<const TExprNode*>; -using TParentsMultiMap = TNodeMap<TNodeMultiSet>; - +using TNodeMultiSet = std::unordered_multiset<const TExprNode*>; +using TParentsMultiMap = TNodeMap<TNodeMultiSet>; + template <> struct TMakeTypeImpl<TVoidExprType> { static const TVoidExprType* Make(TExprContext& ctx); @@ -2549,7 +2549,7 @@ bool CompareExprTrees(const TExprNode*& one, const TExprNode*& two); bool CompareExprTreeParts(const TExprNode& one, const TExprNode& two, const TNodeMap<ui32>& argsMap); void GatherParents(const TExprNode& node, TParentsMap& parentsMap, bool withLeaves = false); -void GatherParentsMulti(const TExprNode& node, TParentsMultiMap& parentsMap, bool withLeaves = false); +void GatherParentsMulti(const TExprNode& node, TParentsMultiMap& parentsMap, bool withLeaves = false); struct TConvertToAstSettings { ui32 AnnotationFlags = 0; diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json index edfcaa15302..aee1590d573 100644 --- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json +++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json @@ -198,11 +198,11 @@ "Match": {"Type": "Callable", "Name": "OrderedLMap"} }, { - "Name": "TCoNarrowMap", - "Base": "TCoMapBase", - "Match": {"Type": "Callable", "Name": "NarrowMap"} - }, - { + "Name": "TCoNarrowMap", + "Base": "TCoMapBase", + "Match": {"Type": "Callable", "Name": "NarrowMap"} + }, + { "Name": "TCoNarrowFlatMap", "Base": "TCoMapBase", "Match": {"Type": "Callable", "Name": "NarrowFlatMap"} @@ -213,11 +213,11 @@ "Match": {"Type": "Callable", "Name": "NarrowMultiMap"} }, { - "Name": "TCoWideMap", - "Base": "TCoMapBase", - "Match": {"Type": "Callable", "Name": "WideMap"} - }, - { + "Name": "TCoWideMap", + "Base": "TCoMapBase", + "Match": {"Type": "Callable", "Name": "WideMap"} + }, + { "Name": "TCoFilterBase", "Base": "TCoInputBase", "Match": {"Type": "CallableBase"}, @@ -237,11 +237,11 @@ "Match": {"Type": "Callable", "Name": "OrderedFilter"} }, { - "Name": "TCoWideFilter", - "Base": "TCoFilterBase", - "Match": {"Type": "Callable", "Name": "WideFilter"} - }, - { + "Name": "TCoWideFilter", + "Base": "TCoFilterBase", + "Match": {"Type": "Callable", "Name": "WideFilter"} + }, + { "Name": "TCoFilterNullMembersBase", "Base": "TCoInputBase", "Match": {"Type": "CallableBase"}, @@ -1365,19 +1365,19 @@ "Match": {"Type": "Callable", "Name": "EquiJoin"} }, { - "Name": "TCoMapJoinCore", + "Name": "TCoMapJoinCore", "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "MapJoinCore"}, - "Children": [ - {"Index": 0, "Name": "LeftInput", "Type": "TExprBase"}, - {"Index": 1, "Name": "RightDict", "Type": "TExprBase"}, - {"Index": 2, "Name": "JoinKind", "Type": "TCoAtom"}, - {"Index": 3, "Name": "LeftKeysColumns", "Type": "TCoAtomList"}, + "Match": {"Type": "Callable", "Name": "MapJoinCore"}, + "Children": [ + {"Index": 0, "Name": "LeftInput", "Type": "TExprBase"}, + {"Index": 1, "Name": "RightDict", "Type": "TExprBase"}, + {"Index": 2, "Name": "JoinKind", "Type": "TCoAtom"}, + {"Index": 3, "Name": "LeftKeysColumns", "Type": "TCoAtomList"}, {"Index": 4, "Name": "LeftRenames", "Type": "TCoAtomList"}, {"Index": 5, "Name": "RightRenames", "Type": "TCoAtomList"} - ] - }, - { + ] + }, + { "Name": "TCoJoinDict", "Base": "TCallable", "Match": {"Type": "Callable", "Name": "JoinDict"}, @@ -1645,18 +1645,18 @@ ] }, { - "Name": "TCoWideCombiner", - "Base": "TCoInputBase", - "Match": {"Type": "Callable", "Name": "WideCombiner"}, - "Children": [ - {"Index": 1, "Name": "MemLimit", "Type": "TCoAtom"}, - {"Index": 2, "Name": "KeyExtractor", "Type": "TCoLambda"}, - {"Index": 3, "Name": "InitHandler", "Type": "TCoLambda"}, - {"Index": 4, "Name": "UpdateHandler", "Type": "TCoLambda"}, - {"Index": 5, "Name": "FinishHandler", "Type": "TCoLambda"} - ] - }, - { + "Name": "TCoWideCombiner", + "Base": "TCoInputBase", + "Match": {"Type": "Callable", "Name": "WideCombiner"}, + "Children": [ + {"Index": 1, "Name": "MemLimit", "Type": "TCoAtom"}, + {"Index": 2, "Name": "KeyExtractor", "Type": "TCoLambda"}, + {"Index": 3, "Name": "InitHandler", "Type": "TCoLambda"}, + {"Index": 4, "Name": "UpdateHandler", "Type": "TCoLambda"}, + {"Index": 5, "Name": "FinishHandler", "Type": "TCoLambda"} + ] + }, + { "Name": "TCoGroupingCore", "Base": "TCoInputBase", "Match": {"Type": "Callable", "Name": "GroupingCore"}, diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h index deaadefb42c..4625b45f9f0 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h @@ -743,14 +743,14 @@ protected: protected: // virtual methods (TODO: replace with static_cast<TDerived*>(this)->Foo() - + virtual THolder<NKikimr::IDestructable> GetSourcesState() { return nullptr; - } - + } + virtual void PollSources(THolder<NKikimr::IDestructable> /* state */) { - } - + } + virtual void TerminateSources(const TString& /* message */, bool /* success */) { } @@ -1358,7 +1358,7 @@ private: if (TaskRunner) { TaskRunner->UpdateStats(); - + if (auto* taskStats = TaskRunner->GetStats()) { auto* protoTask = dst->AddTasks(); FillTaskRunnerStats(Task.GetId(), Task.GetStageId(), *taskStats, protoTask, (bool) ProfileStats); diff --git a/ydb/library/yql/dq/opt/dq_opt.cpp b/ydb/library/yql/dq/opt/dq_opt.cpp index f60f29dd460..1b05bca4dab 100644 --- a/ydb/library/yql/dq/opt/dq_opt.cpp +++ b/ydb/library/yql/dq/opt/dq_opt.cpp @@ -123,13 +123,13 @@ const TNodeSet& GetConsumers(const TExprBase& node, const TParentsMap& parentsMa return consumersIt->second; } -const TNodeMultiSet& GetConsumers(const TExprBase& node, const TParentsMultiMap& parentsMap) { - auto consumersIt = parentsMap.find(node.Raw()); - YQL_ENSURE(consumersIt != parentsMap.end()); - - return consumersIt->second; -} - +const TNodeMultiSet& GetConsumers(const TExprBase& node, const TParentsMultiMap& parentsMap) { + auto consumersIt = parentsMap.find(node.Raw()); + YQL_ENSURE(consumersIt != parentsMap.end()); + + return consumersIt->second; +} + ui32 GetConsumersCount(const TExprBase& node, const TParentsMap& parentsMap) { return GetConsumers(node, parentsMap).size(); } diff --git a/ydb/library/yql/dq/opt/dq_opt.h b/ydb/library/yql/dq/opt/dq_opt.h index 5b3146d0f42..8974d4b5db8 100644 --- a/ydb/library/yql/dq/opt/dq_opt.h +++ b/ydb/library/yql/dq/opt/dq_opt.h @@ -47,8 +47,8 @@ NNodes::TCoLambda BuildIdentityLambda(TPositionHandle pos, TExprContext& ctx); bool EnsureDqUnion(const NNodes::TExprBase& node, TExprContext& ctx); const TNodeSet& GetConsumers(const NNodes::TExprBase& node, const TParentsMap& parentsMap); -const TNodeMultiSet& GetConsumers(const NNodes::TExprBase& node, const TParentsMultiMap& parentsMap); - +const TNodeMultiSet& GetConsumers(const NNodes::TExprBase& node, const TParentsMultiMap& parentsMap); + ui32 GetConsumersCount(const NNodes::TExprBase& node, const TParentsMap& parentsMap); bool IsSingleConsumer(const NNodes::TExprBase& node, const TParentsMap& parentsMap); diff --git a/ydb/library/yql/dq/opt/dq_opt_join.cpp b/ydb/library/yql/dq/opt/dq_opt_join.cpp index 4097b49e24a..069530054f7 100644 --- a/ydb/library/yql/dq/opt/dq_opt_join.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_join.cpp @@ -495,7 +495,7 @@ TExprBase DqBuildPhyJoin(const TDqJoin& join, bool pushLeftStage, TExprContext& TMaybeNode<TDqCnBroadcast> rightBroadcast; TNodeOnNodeOwnedMap rightPrecomputes; - + if (rightCn) { auto collectRightStage = Build<TDqStage>(ctx, join.Pos()) .Inputs() @@ -575,7 +575,7 @@ TExprBase DqBuildPhyJoin(const TDqJoin& join, bool pushLeftStage, TExprContext& TCoArgument leftInputArg{ctx.NewArgument(join.Pos(), "_dq_join_left")}; TCoArgument rightInputArg{ctx.NewArgument(join.Pos(), "_dq_join_right")}; - + bool buildNewStage = !pushLeftStage; if (!rightCn) { // right input is DqPure expression, try to push down the join... @@ -594,7 +594,7 @@ TExprBase DqBuildPhyJoin(const TDqJoin& join, bool pushLeftStage, TExprContext& buildNewStage = GetStageOutputsCount(leftCn.Output().Stage(), true) > 1; } } - + TExprBase joinRightInput = buildNewStage ? (TExprBase) rightInputArg : (rightBroadcast diff --git a/ydb/library/yql/dq/opt/dq_opt_log.cpp b/ydb/library/yql/dq/opt/dq_opt_log.cpp index fed0f09f0f0..839a1f90c32 100644 --- a/ydb/library/yql/dq/opt/dq_opt_log.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_log.cpp @@ -84,69 +84,69 @@ TExprBase DqRewriteTakeSortToTopSort(TExprBase node, TExprContext& ctx, const TP .Done(); } -/* - * Enforce PARTITION COMPACT BY as it avoids generating join in favour of Fold1Map. - */ -TExprBase DqEnforceCompactPartition(TExprBase node, TExprList frames, TExprContext& ctx) { - - for (const auto &frameNode : frames.Ref().Children()) { - YQL_ENSURE(frameNode->IsCallable("WinOnRows")); - - auto frameSpec = frameNode->Child(0); - if (frameSpec->Type() == TExprNode::List) { - TVector<TExprBase> values; - bool compact = false; - - for (const auto& setting : frameSpec->Children()) { - const auto settingName = setting->Head().Content(); - if (settingName == "compact") { - compact = true; - break; - } - values.push_back(TExprBase(setting)); - } - - if (!compact) { - auto newFrameSpec = Build<TExprList>(ctx, frameNode->Pos()) - .Add(values) - .Add<TExprList>() - .Add<TCoAtom>() - .Value("compact") - .Build() - .Build() - .Done(); - - TNodeOnNodeOwnedMap replaces; - replaces[frameNode->Child(0)] = newFrameSpec.Ptr(); - node = TExprBase(ctx.ReplaceNodes(node.Ptr(), replaces)); - } - } - } - - return node; +/* + * Enforce PARTITION COMPACT BY as it avoids generating join in favour of Fold1Map. + */ +TExprBase DqEnforceCompactPartition(TExprBase node, TExprList frames, TExprContext& ctx) { + + for (const auto &frameNode : frames.Ref().Children()) { + YQL_ENSURE(frameNode->IsCallable("WinOnRows")); + + auto frameSpec = frameNode->Child(0); + if (frameSpec->Type() == TExprNode::List) { + TVector<TExprBase> values; + bool compact = false; + + for (const auto& setting : frameSpec->Children()) { + const auto settingName = setting->Head().Content(); + if (settingName == "compact") { + compact = true; + break; + } + values.push_back(TExprBase(setting)); + } + + if (!compact) { + auto newFrameSpec = Build<TExprList>(ctx, frameNode->Pos()) + .Add(values) + .Add<TExprList>() + .Add<TCoAtom>() + .Value("compact") + .Build() + .Build() + .Done(); + + TNodeOnNodeOwnedMap replaces; + replaces[frameNode->Child(0)] = newFrameSpec.Ptr(); + node = TExprBase(ctx.ReplaceNodes(node.Ptr(), replaces)); + } + } + } + + return node; } - -TExprBase DqExpandWindowFunctions(TExprBase node, TExprContext& ctx, bool enforceCompact) { + +TExprBase DqExpandWindowFunctions(TExprBase node, TExprContext& ctx, bool enforceCompact) { if (node.Maybe<TCoCalcOverWindowBase>() || node.Maybe<TCoCalcOverWindowGroup>()) { - if (enforceCompact) { + if (enforceCompact) { auto calcs = ExtractCalcsOverWindow(node.Ptr(), ctx); for (auto& c : calcs) { TCoCalcOverWindowTuple win(c); - node = DqEnforceCompactPartition(node, win.Frames(), ctx); - } - } + node = DqEnforceCompactPartition(node, win.Frames(), ctx); + } + } return TExprBase(ExpandCalcOverWindow(node.Ptr(), ctx)); - } else { - return node; - } -} - + } else { + return node; + } +} + static void CollectSinkStages(const NNodes::TDqQuery& dqQuery, THashSet<TExprNode::TPtr, TExprNode::TPtrHash>& sinkStages) { for (const auto& stage : dqQuery.SinkStages()) { sinkStages.insert(stage.Ptr()); } -} +} NNodes::TExprBase DqMergeQueriesWithSinks(NNodes::TExprBase dqQueryNode, TExprContext& ctx) { NNodes::TDqQuery dqQuery = dqQueryNode.Cast<NNodes::TDqQuery>(); diff --git a/ydb/library/yql/dq/opt/dq_opt_log.h b/ydb/library/yql/dq/opt/dq_opt_log.h index 8e9ed97b063..b42b492dd9e 100644 --- a/ydb/library/yql/dq/opt/dq_opt_log.h +++ b/ydb/library/yql/dq/opt/dq_opt_log.h @@ -18,10 +18,10 @@ NNodes::TExprBase DqRewriteTakeSortToTopSort(NNodes::TExprBase node, TExprContex NNodes::TExprBase DqRewriteEquiJoin(const NNodes::TExprBase& node, TExprContext& ctx); -NNodes::TExprBase DqEnforceCompactPartition(NNodes::TExprBase node, NNodes::TExprList frames, TExprContext& ctx); - -NNodes::TExprBase DqExpandWindowFunctions(NNodes::TExprBase node, TExprContext& ctx, bool enforceCompact); - +NNodes::TExprBase DqEnforceCompactPartition(NNodes::TExprBase node, NNodes::TExprList frames, TExprContext& ctx); + +NNodes::TExprBase DqExpandWindowFunctions(NNodes::TExprBase node, TExprContext& ctx, bool enforceCompact); + NNodes::TExprBase DqMergeQueriesWithSinks(NNodes::TExprBase dqQueryNode, TExprContext& ctx); NNodes::TExprBase DqFlatMapOverExtend(NNodes::TExprBase node, TExprContext& ctx); diff --git a/ydb/library/yql/dq/opt/dq_opt_phy.cpp b/ydb/library/yql/dq/opt/dq_opt_phy.cpp index cbe871059d7..0c15ea2dceb 100644 --- a/ydb/library/yql/dq/opt/dq_opt_phy.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_phy.cpp @@ -1338,94 +1338,94 @@ TExprBase DqRewriteLengthOfStageOutput(TExprBase node, TExprContext& ctx, IOptim .Done(); } -TExprBase DqBuildPureExprStage(TExprBase node, TExprContext& ctx) { - if (!IsDqPureExpr(node)) { - return node; - } - - auto stage = Build<TDqStage>(ctx, node.Pos()) - .Inputs() - .Build() - .Program() - .Args({}) - .Body<TCoToStream>() - .Input(node) - .Build() - .Build() +TExprBase DqBuildPureExprStage(TExprBase node, TExprContext& ctx) { + if (!IsDqPureExpr(node)) { + return node; + } + + auto stage = Build<TDqStage>(ctx, node.Pos()) + .Inputs() + .Build() + .Program() + .Args({}) + .Body<TCoToStream>() + .Input(node) + .Build() + .Build() .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) - .Done(); - - return Build<TDqCnUnionAll>(ctx, node.Pos()) - .Output() - .Stage(stage) - .Index().Build("0") - .Build() - .Done(); -} - -/* - * Move (Extend ...) into a separate stage. - * - * If overConnsOnly=true stage will be built only if all arguments have TDqConnection type. - * - * With overConnsOnly=false non-TDqConnection arguments are left in the same stage. This - * is needed for handling UNION ALL case, which generates top-level Extend where some arguments - * can be pure expressions not wrapped in DqStage (e.g. ... UNION ALL SELECT 1). - */ + .Done(); + + return Build<TDqCnUnionAll>(ctx, node.Pos()) + .Output() + .Stage(stage) + .Index().Build("0") + .Build() + .Done(); +} + +/* + * Move (Extend ...) into a separate stage. + * + * If overConnsOnly=true stage will be built only if all arguments have TDqConnection type. + * + * With overConnsOnly=false non-TDqConnection arguments are left in the same stage. This + * is needed for handling UNION ALL case, which generates top-level Extend where some arguments + * can be pure expressions not wrapped in DqStage (e.g. ... UNION ALL SELECT 1). + */ TExprBase DqBuildExtendStage(TExprBase node, TExprContext& ctx) { if (!node.Maybe<TCoExtendBase>()) { - return node; - } - + return node; + } + auto extend = node.Cast<TCoExtendBase>(); - TVector<TCoArgument> inputArgs; - TVector<TExprBase> inputConns; - TVector<TExprBase> extendArgs; - - for (const auto& arg: extend) { - if (arg.Maybe<TDqConnection>()) { - auto conn = arg.Cast<TDqConnection>(); - TCoArgument programArg = Build<TCoArgument>(ctx, conn.Pos()) - .Name("arg") - .Done(); - inputConns.push_back(conn); - inputArgs.push_back(programArg); - extendArgs.push_back(programArg); - } else if (IsDqPureExpr(arg)) { - // arg is deemed to be a pure expression so leave it inside (Extend ...) - extendArgs.push_back(Build<TCoToFlow>(ctx, arg.Pos()) - .Input(arg) - .Done()); - } else { - return node; - } - } - - if (inputConns.empty()) { - return node; - } - - auto stage = Build<TDqStage>(ctx, node.Pos()) - .Inputs() - .Add(inputConns) - .Build() - .Program() - .Args(inputArgs) - .Body<TCoExtend>() // TODO: check Extend effectiveness - .Add(extendArgs) - .Build() - .Build() + TVector<TCoArgument> inputArgs; + TVector<TExprBase> inputConns; + TVector<TExprBase> extendArgs; + + for (const auto& arg: extend) { + if (arg.Maybe<TDqConnection>()) { + auto conn = arg.Cast<TDqConnection>(); + TCoArgument programArg = Build<TCoArgument>(ctx, conn.Pos()) + .Name("arg") + .Done(); + inputConns.push_back(conn); + inputArgs.push_back(programArg); + extendArgs.push_back(programArg); + } else if (IsDqPureExpr(arg)) { + // arg is deemed to be a pure expression so leave it inside (Extend ...) + extendArgs.push_back(Build<TCoToFlow>(ctx, arg.Pos()) + .Input(arg) + .Done()); + } else { + return node; + } + } + + if (inputConns.empty()) { + return node; + } + + auto stage = Build<TDqStage>(ctx, node.Pos()) + .Inputs() + .Add(inputConns) + .Build() + .Program() + .Args(inputArgs) + .Body<TCoExtend>() // TODO: check Extend effectiveness + .Add(extendArgs) + .Build() + .Build() .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) - .Done(); - - return Build<TDqCnUnionAll>(ctx, node.Pos()) - .Output() - .Stage(stage) - .Index().Build("0") - .Build() - .Done(); -} - + .Done(); + + return Build<TDqCnUnionAll>(ctx, node.Pos()) + .Output() + .Stage(stage) + .Index().Build("0") + .Build() + .Done(); +} + /* * Precompute input value in a separate stage. */ diff --git a/ydb/library/yql/dq/opt/dq_opt_phy.h b/ydb/library/yql/dq/opt/dq_opt_phy.h index 70a9d70082b..51809112b46 100644 --- a/ydb/library/yql/dq/opt/dq_opt_phy.h +++ b/ydb/library/yql/dq/opt/dq_opt_phy.h @@ -74,11 +74,11 @@ NNodes::TExprBase DqBuildJoinDict(const NNodes::TDqJoin& join, TExprContext& ctx TMaybe<std::pair<NNodes::TExprBase, NNodes::TDqConnection>> ExtractPureExprStage(TExprNode::TPtr input, TExprContext& ctx); - -NNodes::TExprBase DqBuildPureExprStage(NNodes::TExprBase node, TExprContext& ctx); - + +NNodes::TExprBase DqBuildPureExprStage(NNodes::TExprBase node, TExprContext& ctx); + NNodes::TExprBase DqBuildExtendStage(NNodes::TExprBase node, TExprContext& ctx); - + NNodes::TExprBase DqBuildPrecomputeStage(NNodes::TExprBase node, TExprContext& ctx); NYql::NNodes::TExprBase DqBuildHasItems(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx, diff --git a/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp b/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp index a1c8cf10c2c..24ed88afb8e 100644 --- a/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp @@ -123,22 +123,22 @@ std::pair<TDqStage, TVector<TCoAtom>> ReplicateStageOutput(const TDqStage& stage struct TMultiUsedOutput { TDqOutput Output; - const TNodeMultiSet& Consumers; + const TNodeMultiSet& Consumers; - TMultiUsedOutput(const TDqOutput& output, const TNodeMultiSet& consumers) + TMultiUsedOutput(const TDqOutput& output, const TNodeMultiSet& consumers) : Output(output) , Consumers(consumers) {} }; struct TMultiUsedConnection { TDqConnection Connection; - const TNodeMultiSet& Consumers; + const TNodeMultiSet& Consumers; // set if Connection has single consumer, but it's Output has many consumers // in that case, Connection - is on of connections, references to this Output TMaybe<TMultiUsedOutput> Output; - TMultiUsedConnection(const TDqConnection& connection, const TNodeMultiSet& consumers) + TMultiUsedConnection(const TDqConnection& connection, const TNodeMultiSet& consumers) : Connection(connection) , Consumers(consumers) {} diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp index 10900352c3b..28aadfd7420 100644 --- a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp +++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp @@ -333,10 +333,10 @@ public: LOG(TStringBuilder() << "task: " << TaskId << ", program size: " << programSize << ", llvm: `" << Settings.OptLLVM << "`."); - if (Y_UNLIKELY(CollectProfileStats)) { - ProgramParsed.StatsRegistry = NMiniKQL::CreateDefaultStatsRegistry(); - } - + if (Y_UNLIKELY(CollectProfileStats)) { + ProgramParsed.StatsRegistry = NMiniKQL::CreateDefaultStatsRegistry(); + } + auto validatePolicy = Settings.TerminateOnError ? NUdf::EValidatePolicy::Fail : NUdf::EValidatePolicy::Exception; TComputationPatternOpts opts(Alloc().Ref(), typeEnv, Context.ComputationFactory, Context.FuncRegistry, NUdf::EValidateMode::None, validatePolicy, Settings.OptLLVM, diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.h b/ydb/library/yql/dq/runtime/dq_tasks_runner.h index a0cc816e91e..50cf0325f1b 100644 --- a/ydb/library/yql/dq/runtime/dq_tasks_runner.h +++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.h @@ -77,7 +77,7 @@ struct TDqTaskRunnerStats { // profile stats TDuration WaitTime; // wall time of waiting for input, scans & output TDuration WaitOutputTime; - + NMonitoring::IHistogramCollectorPtr ComputeCpuTimeByRun; // in millis THashMap<ui64, const TDqInputChannelStats*> InputChannels; // Channel id -> Channel stats diff --git a/ydb/library/yql/minikql/codegen/codegen.cpp b/ydb/library/yql/minikql/codegen/codegen.cpp index 6e4096af228..f37c356605a 100644 --- a/ydb/library/yql/minikql/codegen/codegen.cpp +++ b/ydb/library/yql/minikql/codegen/codegen.cpp @@ -337,18 +337,18 @@ public: } void TogglePerfJITEventListener() override { -#ifdef __linux__ - PerfListener_ = llvm::JITEventListener::createPerfJITEventListener(); - Engine_->RegisterJITEventListener(PerfListener_); -#endif - } - +#ifdef __linux__ + PerfListener_ = llvm::JITEventListener::createPerfJITEventListener(); + Engine_->RegisterJITEventListener(PerfListener_); +#endif + } + ~TCodegen() { -#ifdef __linux__ - if (PerfListener_) { - Engine_->UnregisterJITEventListener(PerfListener_); - } -#endif +#ifdef __linux__ + if (PerfListener_) { + Engine_->UnregisterJITEventListener(PerfListener_); + } +#endif Engine_->UnregisterJITEventListener(this); } @@ -404,9 +404,9 @@ public: void Compile(const TStringBuf compileOpts, TCompileStats* compileStats) override { - bool dumpTimers = compileOpts.Contains("time-passes"); - bool disableOpt = compileOpts.Contains("disable-opt"); - + bool dumpTimers = compileOpts.Contains("time-passes"); + bool disableOpt = compileOpts.Contains("disable-opt"); + #if defined(_msan_enabled_) ReverseGlobalMapping_[(const void*)&__emutls_get_address] = "__emutls_get_address"; #endif @@ -500,9 +500,9 @@ public: if (compileStats) { compileStats->ModulePassTime = (Now() - modulePassStart).MilliSeconds(); } - + AllocateTls(); - + auto finalizeStart = Now(); Engine_->finalizeObject(); if (compileStats) { @@ -729,7 +729,7 @@ private: std::string Diagnostic_; std::string Triple_; llvm::Module* Module_; - llvm::JITEventListener* PerfListener_ = nullptr; + llvm::JITEventListener* PerfListener_ = nullptr; std::unique_ptr<llvm::ExecutionEngine> Engine_; std::vector<std::pair<llvm::object::SectionRef, ui64>> CodeSections_; std::vector<std::pair<ui64, llvm::Function*>> SortedFuncs_; diff --git a/ydb/library/yql/minikql/codegen/codegen.h b/ydb/library/yql/minikql/codegen/codegen.h index 41326b6e0fe..b1ce8c50b11 100644 --- a/ydb/library/yql/minikql/codegen/codegen.h +++ b/ydb/library/yql/minikql/codegen/codegen.h @@ -1,5 +1,5 @@ #pragma once - + #include <memory> #include <util/generic/fwd.h> #include <util/generic/strbuf.h> @@ -58,7 +58,7 @@ public: virtual void ShowGeneratedFunctions(IOutputStream* out) = 0; virtual void LoadBitCode(TStringBuf bitcode, TStringBuf uniqId) = 0; virtual void AddGlobalMapping(TStringBuf name, const void* address) = 0; - virtual void TogglePerfJITEventListener() = 0; + virtual void TogglePerfJITEventListener() = 0; typedef std::unique_ptr<ICodegen> TPtr; static TPtr Make(ETarget target, ESanitize sanitize = ESanitize::Auto); diff --git a/ydb/library/yql/minikql/codegen/ya.make b/ydb/library/yql/minikql/codegen/ya.make index cdc45b7d261..b371147d309 100644 --- a/ydb/library/yql/minikql/codegen/ya.make +++ b/ydb/library/yql/minikql/codegen/ya.make @@ -33,12 +33,12 @@ PEERDIR( contrib/libs/llvm12/lib/Transforms/ObjCARC ) -IF (OS_LINUX) +IF (OS_LINUX) PEERDIR( contrib/libs/llvm12/lib/ExecutionEngine/PerfJITEvents ) -ENDIF() - +ENDIF() + END() RECURSE_FOR_TESTS( diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp index f2b188ae47c..8730f201c88 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp @@ -714,10 +714,10 @@ public: Cerr << "MaxFunctionInstructions: " << codegenStats.MaxFunctionInstructions << Endl; } - if (opts.OptLLVM.Contains("--dump-perf-map")) { - Codegen->TogglePerfJITEventListener(); - } - + if (opts.OptLLVM.Contains("--dump-perf-map")) { + Codegen->TogglePerfJITEventListener(); + } + if (codegenStats.TotalFunctions >= TotalFunctionsLimit || codegenStats.TotalInstructions >= TotalInstructionsLimit || codegenStats.MaxFunctionInstructions >= MaxFunctionInstructionsLimit) { @@ -782,19 +782,19 @@ public: TypeEnv = typeEnv; } - TStringBuf GetCompileOptions(const TString& s) { - const TString flag = "--compile-options"; - auto lpos = s.rfind(flag); - if (lpos == TString::npos) - return TStringBuf(); - lpos += flag.Size(); - auto rpos = s.find(" --", lpos); - if (rpos == TString::npos) - return TStringBuf(s, lpos); - else - return TStringBuf(s, lpos, rpos - lpos); - }; - + TStringBuf GetCompileOptions(const TString& s) { + const TString flag = "--compile-options"; + auto lpos = s.rfind(flag); + if (lpos == TString::npos) + return TStringBuf(); + lpos += flag.Size(); + auto rpos = s.find(" --", lpos); + if (rpos == TString::npos) + return TStringBuf(s, lpos); + else + return TStringBuf(s, lpos, rpos - lpos); + }; + THolder<IComputationGraph> Clone(const TComputationOptsFull& compOpts) final { return MakeHolder<TComputationGraph>(PatternNodes, compOpts); } diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h b/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h index 0a63ca9d349..11e19a11f51 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h @@ -567,7 +567,7 @@ private: THROW yexception() << "Failed to get representation kind."; } - NUdf::TUnboxedValue GetValue(TComputationContext&) const { + NUdf::TUnboxedValue GetValue(TComputationContext&) const { THROW yexception() << "Failed to get value from wide flow node."; } }; diff --git a/ydb/public/api/grpc/ydb_scripting_v1.proto b/ydb/public/api/grpc/ydb_scripting_v1.proto index c8743a07de3..cbabd54ef5c 100644 --- a/ydb/public/api/grpc/ydb_scripting_v1.proto +++ b/ydb/public/api/grpc/ydb_scripting_v1.proto @@ -11,5 +11,5 @@ service ScriptingService { // Executes yql request with streaming result. rpc StreamExecuteYql(Scripting.ExecuteYqlRequest) returns (stream Scripting.ExecuteYqlPartialResponse); - rpc ExplainYql(Scripting.ExplainYqlRequest) returns (Scripting.ExplainYqlResponse); + rpc ExplainYql(Scripting.ExplainYqlRequest) returns (Scripting.ExplainYqlResponse); } diff --git a/ydb/public/api/protos/ydb_experimental.proto b/ydb/public/api/protos/ydb_experimental.proto index 94eed9671b7..62bee205434 100644 --- a/ydb/public/api/protos/ydb_experimental.proto +++ b/ydb/public/api/protos/ydb_experimental.proto @@ -22,7 +22,7 @@ message ExecuteStreamQueryRequest { } ProfileMode profile_mode = 3; - bool explain = 4; + bool explain = 4; } message ExecuteStreamQueryResponse { @@ -39,6 +39,6 @@ message ExecuteStreamQueryResult { Ydb.ResultSet result_set = 1; string profile = 2; StreamQueryProgress progress = 3; - string query_plan = 4; + string query_plan = 4; } } diff --git a/ydb/public/api/protos/ydb_query_stats.proto b/ydb/public/api/protos/ydb_query_stats.proto index 300d5d9837c..5a7a4c107e9 100644 --- a/ydb/public/api/protos/ydb_query_stats.proto +++ b/ydb/public/api/protos/ydb_query_stats.proto @@ -39,8 +39,8 @@ message QueryStats { repeated QueryPhaseStats query_phases = 1; CompilationStats compilation = 2; uint64 process_cpu_time_us = 3; - string query_plan = 4; - string query_ast = 5; + string query_plan = 4; + string query_ast = 5; uint64 total_duration_us = 6; uint64 total_cpu_time_us = 7; } diff --git a/ydb/public/api/protos/ydb_scripting.proto b/ydb/public/api/protos/ydb_scripting.proto index 4c1a2954185..c2197e79e7c 100644 --- a/ydb/public/api/protos/ydb_scripting.proto +++ b/ydb/public/api/protos/ydb_scripting.proto @@ -29,7 +29,7 @@ message ExecuteYqlResult { repeated Ydb.ResultSet result_sets = 1; Ydb.TableStats.QueryStats query_stats = 2; } - + // Response for StreamExecuteYql is a stream of ExecuteYqlPartialResponse messages. // These responses can contain ExecuteYqlPartialResult messages with // results (or result parts) for data or scan queries in the script. @@ -52,24 +52,24 @@ message ExecuteYqlPartialResult { Ydb.TableStats.QueryStats query_stats = 3; } -message ExplainYqlRequest { - enum Mode { +message ExplainYqlRequest { + enum Mode { MODE_UNSPECIFIED = 0; // PARSE = 1; VALIDATE = 2; PLAN = 3; - } - - Ydb.Operations.OperationParams operation_params = 1; - string script = 2; - Mode mode = 3; -} - -message ExplainYqlResponse { - Ydb.Operations.Operation operation = 1; -} - -message ExplainYqlResult { - map<string, Type> parameters_types = 1; - string plan = 2; -} + } + + Ydb.Operations.OperationParams operation_params = 1; + string script = 2; + Mode mode = 3; +} + +message ExplainYqlResponse { + Ydb.Operations.Operation operation = 1; +} + +message ExplainYqlResult { + map<string, Type> parameters_types = 1; + string plan = 2; +} diff --git a/ydb/public/api/protos/ydb_table.proto b/ydb/public/api/protos/ydb_table.proto index c8063d9475c..bae30553cac 100644 --- a/ydb/public/api/protos/ydb_table.proto +++ b/ydb/public/api/protos/ydb_table.proto @@ -653,7 +653,7 @@ message QueryStatsCollection { STATS_COLLECTION_UNSPECIFIED = 0; STATS_COLLECTION_NONE = 1; // Stats collection is disabled STATS_COLLECTION_BASIC = 2; // Aggregated stats of reads, updates and deletes per table - STATS_COLLECTION_FULL = 3; // Add execution stats and plan on top of STATS_COLLECTION_BASIC + STATS_COLLECTION_FULL = 3; // Add execution stats and plan on top of STATS_COLLECTION_BASIC } } @@ -968,7 +968,7 @@ message BulkUpsertResult { message ExecuteScanQueryRequest { enum Mode { MODE_UNSPECIFIED = 0; - MODE_EXPLAIN = 1; + MODE_EXPLAIN = 1; // MODE_PREPARE = 2; MODE_EXEC = 3; } @@ -980,7 +980,7 @@ message ExecuteScanQueryRequest { reserved 5; // query_cache_policy Mode mode = 6; reserved 7; // report_progress - QueryStatsCollection.Mode collect_stats = 8; + QueryStatsCollection.Mode collect_stats = 8; } message ExecuteScanQueryPartialResponse { @@ -995,6 +995,6 @@ message ExecuteScanQueryPartialResult { reserved 3; // query_meta reserved 4; // query_progress reserved 5; // query_plan - Ydb.TableStats.QueryStats query_stats = 6; + Ydb.TableStats.QueryStats query_stats = 6; } diff --git a/ydb/public/lib/experimental/ydb_experimental.cpp b/ydb/public/lib/experimental/ydb_experimental.cpp index 8765c0c750a..4241ae73ee8 100644 --- a/ydb/public/lib/experimental/ydb_experimental.cpp +++ b/ydb/public/lib/experimental/ydb_experimental.cpp @@ -59,16 +59,16 @@ public: EStatus clientStatus = static_cast<EStatus>(self->Response_.status()); TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}}; TStatus status{std::move(plainStatus)}; - + if (self->Response_.result().has_result_set()) { promise.SetValue({TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), std::move(status)}); } else if (!self->Response_.result().profile().empty()) { promise.SetValue({std::move(*self->Response_.mutable_result()->mutable_profile()), std::move(status)}); - } else if (!self->Response_.result().query_plan().Empty()) { - TMaybe<TString> queryPlan = self->Response_.result().query_plan(); - promise.SetValue({queryPlan, std::move(status)}); + } else if (!self->Response_.result().query_plan().Empty()) { + TMaybe<TString> queryPlan = self->Response_.result().query_plan(); + promise.SetValue({queryPlan, std::move(status)}); } else if (self->Response_.result().has_progress()) { // skip, not supported yet } else { @@ -127,8 +127,8 @@ public: break; } - request.set_explain(settings.Explain_); - + request.set_explain(settings.Explain_); + auto promise = NewPromise<std::pair<TPlainStatus, TStreamProcessorPtr>>(); Connections_->StartReadStream< diff --git a/ydb/public/lib/experimental/ydb_experimental.h b/ydb/public/lib/experimental/ydb_experimental.h index 3cdd8f6dc8f..4e166427203 100644 --- a/ydb/public/lib/experimental/ydb_experimental.h +++ b/ydb/public/lib/experimental/ydb_experimental.h @@ -27,10 +27,10 @@ public: const TString& GetProfile() const { return *Profile_; } TString ExtractProfile() { return std::move(*Profile_); } - bool HasPlan() const { return Plan_.Defined(); } - const TString& GetPlan() const { return *Plan_; } - TString ExtractPlan() { return std::move(*Plan_); } - + bool HasPlan() const { return Plan_.Defined(); } + const TString& GetPlan() const { return *Plan_; } + TString ExtractPlan() { return std::move(*Plan_); } + TStreamPart(TStatus&& status) : TStreamPartStatus(std::move(status)) {} @@ -45,15 +45,15 @@ public: , Profile_(std::move(profile)) {} - TStreamPart(const TMaybe<TString>& plan, TStatus&& status) - : TStreamPartStatus(std::move(status)) - , Plan_(plan) - {} - + TStreamPart(const TMaybe<TString>& plan, TStatus&& status) + : TStreamPartStatus(std::move(status)) + , Plan_(plan) + {} + private: TMaybe<TResultSet> ResultSet_; TMaybe<TString> Profile_; - TMaybe<TString> Plan_; + TMaybe<TString> Plan_; }; using TAsyncStreamPart = NThreading::TFuture<TStreamPart>; @@ -81,7 +81,7 @@ struct TExecuteStreamQuerySettings : public TRequestSettings<TExecuteStreamQuery using TSelf = TExecuteStreamQuerySettings; FLUENT_SETTING_DEFAULT(EStreamQueryProfileMode, ProfileMode, EStreamQueryProfileMode::None); - FLUENT_SETTING_DEFAULT(bool, Explain, false); + FLUENT_SETTING_DEFAULT(bool, Explain, false); }; class TStreamQueryClient { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp index 2f94b2fc033..f1a510172f9 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp @@ -553,12 +553,12 @@ void TCommandExplain::Config(TConfig& config) { .RequiredArgument("PATH").StoreResult(&QueryFile); config.Opts->AddLongOption("ast", "Print query AST") .StoreTrue(&PrintAst); - - config.Opts->AddLongOption('t', "type", "Query type [data, scan]") - .RequiredArgument("[String]").DefaultValue("data").StoreResult(&QueryType); - config.Opts->AddLongOption("analyze", "Run query and collect execution statistics") - .NoArgument().SetFlag(&Analyze); - + + config.Opts->AddLongOption('t', "type", "Query type [data, scan]") + .RequiredArgument("[String]").DefaultValue("data").StoreResult(&QueryType); + config.Opts->AddLongOption("analyze", "Run query and collect execution statistics") + .NoArgument().SetFlag(&Analyze); + AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::JsonUnicode, @@ -577,77 +577,77 @@ void TCommandExplain::Parse(TConfig& config) { int TCommandExplain::Run(TConfig& config) { CheckQueryFile(); - TString planJson; - TString ast; - if (QueryType == "scan") { - NTable::TTableClient client(CreateDriver(config)); - NTable::TStreamExecScanQuerySettings settings; - - if (Analyze) { - settings.CollectQueryStats(NTable::ECollectQueryStatsMode::Full); - } else { - settings.Explain(true); - } - - auto result = client.StreamExecuteScanQuery(Query, settings).GetValueSync(); - ThrowOnError(result); - - SetInterruptHandlers(); + TString planJson; + TString ast; + if (QueryType == "scan") { + NTable::TTableClient client(CreateDriver(config)); + NTable::TStreamExecScanQuerySettings settings; + + if (Analyze) { + settings.CollectQueryStats(NTable::ECollectQueryStatsMode::Full); + } else { + settings.Explain(true); + } + + auto result = client.StreamExecuteScanQuery(Query, settings).GetValueSync(); + ThrowOnError(result); + + SetInterruptHandlers(); while (!IsInterrupted()) { - auto tablePart = result.ReadNext().GetValueSync(); - if (!tablePart.IsSuccess()) { - if (tablePart.EOS()) { - break; - } - ThrowOnError(tablePart); - } - if (tablePart.HasQueryStats() ) { + auto tablePart = result.ReadNext().GetValueSync(); + if (!tablePart.IsSuccess()) { + if (tablePart.EOS()) { + break; + } + ThrowOnError(tablePart); + } + if (tablePart.HasQueryStats() ) { auto proto = NYdb::TProtoAccessor::GetProto(tablePart.GetQueryStats()); - planJson = proto.query_plan(); - ast = proto.query_ast(); - } - } - + planJson = proto.query_plan(); + ast = proto.query_ast(); + } + } + if (IsInterrupted()) { Cerr << "<INTERRUPTED>" << Endl; } - } else if (QueryType == "data" && Analyze) { - NTable::TExecDataQuerySettings settings; - settings.CollectQueryStats(NTable::ECollectQueryStatsMode::Full); - - auto result = GetSession(config).ExecuteDataQuery( - Query, - NTable::TTxControl::BeginTx(NTable::TTxSettings::SerializableRW()).CommitTx(), - settings - ).ExtractValueSync(); - ThrowOnError(result); - planJson = result.GetQueryPlan(); + } else if (QueryType == "data" && Analyze) { + NTable::TExecDataQuerySettings settings; + settings.CollectQueryStats(NTable::ECollectQueryStatsMode::Full); + + auto result = GetSession(config).ExecuteDataQuery( + Query, + NTable::TTxControl::BeginTx(NTable::TTxSettings::SerializableRW()).CommitTx(), + settings + ).ExtractValueSync(); + ThrowOnError(result); + planJson = result.GetQueryPlan(); if (auto stats = result.GetStats()) { - auto proto = NYdb::TProtoAccessor::GetProto(*stats); - ast = proto.query_ast(); - } - } else if (QueryType == "data" && !Analyze) { - NTable::TExplainQueryResult result = GetSession(config).ExplainDataQuery( - Query, - FillSettings(NTable::TExplainDataQuerySettings()) - ).GetValueSync(); - ThrowOnError(result); - planJson = result.GetPlan(); - ast = result.GetAst(); - - } else { - throw TMissUseException() << "Unknown query type for explain."; - } - + auto proto = NYdb::TProtoAccessor::GetProto(*stats); + ast = proto.query_ast(); + } + } else if (QueryType == "data" && !Analyze) { + NTable::TExplainQueryResult result = GetSession(config).ExplainDataQuery( + Query, + FillSettings(NTable::TExplainDataQuerySettings()) + ).GetValueSync(); + ThrowOnError(result); + planJson = result.GetPlan(); + ast = result.GetAst(); + + } else { + throw TMissUseException() << "Unknown query type for explain."; + } + if (PrintAst) { Cout << "Query AST:" << Endl << ast << Endl; - } else { + } else { Cout << "Query Plan:" << Endl; TQueryPlanPrinter queryPlanPrinter(OutputFormat, Analyze); queryPlanPrinter.Print(planJson); - } - + } + return EXIT_SUCCESS; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h index 5cd2514aa02..d706fb8f65a 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h @@ -123,10 +123,10 @@ public: virtual int Run(TConfig& config) override; private: - + bool PrintAst = false; - TString QueryType; - bool Analyze = false; + TString QueryType; + bool Analyze = false; }; class TCommandReadTable : public TYdbCommand, public TCommandWithPath, diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp index 38b8c8123dd..905d236a42c 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp +++ b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp @@ -40,8 +40,8 @@ const TMaybe<NTable::TQueryStats>& TExecuteYqlResult::GetStats() const { return QueryStats_; } -//////////////////////////////////////////////////////////////////////////////// - +//////////////////////////////////////////////////////////////////////////////// + class TYqlResultPartIterator::TReaderImpl { public: using TSelf = TYqlResultPartIterator::TReaderImpl; @@ -124,10 +124,10 @@ TAsyncYqlResultPart TYqlResultPartIterator::ReadNext() { //////////////////////////////////////////////////////////////////////////////// TExplainYqlResult::TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map<TString, Ydb::Type>&& types, TString&& plan) - : TStatus(std::move(status)) + : TStatus(std::move(status)) , ParameterTypes_(std::move(types)) - , Plan_(plan) {} - + , Plan_(plan) {} + std::map<TString, TType> TExplainYqlResult::GetParameterTypes() const { std::map<TString, TType> typesMap; for (const auto& param : ParameterTypes_) { @@ -136,12 +136,12 @@ std::map<TString, TType> TExplainYqlResult::GetParameterTypes() const { return typesMap; } -const TString& TExplainYqlResult::GetPlan() const { - return Plan_; -} - -//////////////////////////////////////////////////////////////////////////////// - +const TString& TExplainYqlResult::GetPlan() const { + return Plan_; +} + +//////////////////////////////////////////////////////////////////////////////// + class TScriptingClient::TImpl : public TClientImplCommon<TScriptingClient::TImpl> { public: using TYqlScriptProcessorPtr = TYqlResultPartIterator::TReaderImpl::TStreamProcessorPtr; @@ -245,57 +245,57 @@ public: return promise.GetFuture(); } - TAsyncExplainYqlResult ExplainYqlScript(const TString& script, - const TExplainYqlRequestSettings& settings) - { - auto request = MakeOperationRequest<Ydb::Scripting::ExplainYqlRequest>(settings); - request.set_script(script); - - switch (settings.Mode_) { + TAsyncExplainYqlResult ExplainYqlScript(const TString& script, + const TExplainYqlRequestSettings& settings) + { + auto request = MakeOperationRequest<Ydb::Scripting::ExplainYqlRequest>(settings); + request.set_script(script); + + switch (settings.Mode_) { // KIKIMR-10990 //case ExplainYqlRequestMode::Parse: // request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_PARSE); // break; - case ExplainYqlRequestMode::Validate: - request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_VALIDATE); - break; - case ExplainYqlRequestMode::Plan: - request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_PLAN); - break; - } - - auto promise = NewPromise<TExplainYqlResult>(); - + case ExplainYqlRequestMode::Validate: + request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_VALIDATE); + break; + case ExplainYqlRequestMode::Plan: + request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_PLAN); + break; + } + + auto promise = NewPromise<TExplainYqlResult>(); + auto extractor = [promise] - (google::protobuf::Any* any, TPlainStatus status) mutable { - TString plan; + (google::protobuf::Any* any, TPlainStatus status) mutable { + TString plan; ::google::protobuf::Map<TString, Ydb::Type> types; - if (any) { - Ydb::Scripting::ExplainYqlResult result; - any->UnpackTo(&result); - - plan = result.plan(); + if (any) { + Ydb::Scripting::ExplainYqlResult result; + any->UnpackTo(&result); + + plan = result.plan(); types = result.parameters_types(); - } - + } + TExplainYqlResult explainResult(TStatus(std::move(status)), std::move(types), std::move(plan)); - promise.SetValue(std::move(explainResult)); - }; - - Connections_->RunDeferred<Ydb::Scripting::V1::ScriptingService, Ydb::Scripting::ExplainYqlRequest, - Ydb::Scripting::ExplainYqlResponse>( - std::move(request), - extractor, - &Ydb::Scripting::V1::ScriptingService::Stub::AsyncExplainYql, - DbDriverState_, - INITIAL_DEFERRED_CALL_DELAY, - TRpcRequestSettings::Make(settings), - settings.ClientTimeout_); - - return promise.GetFuture(); - } - + promise.SetValue(std::move(explainResult)); + }; + + Connections_->RunDeferred<Ydb::Scripting::V1::ScriptingService, Ydb::Scripting::ExplainYqlRequest, + Ydb::Scripting::ExplainYqlResponse>( + std::move(request), + extractor, + &Ydb::Scripting::V1::ScriptingService::Stub::AsyncExplainYql, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings), + settings.ClientTimeout_); + + return promise.GetFuture(); + } + private: template<typename TRequest> static void SetParams(::google::protobuf::Map<TString, Ydb::TypedValue>* params, TRequest* request) { @@ -373,11 +373,11 @@ TAsyncYqlResultPartIterator TScriptingClient::StreamExecuteYqlScript(const TStri return Impl_->StreamExecuteYqlScript(script, paramsPtr, settings); } -TAsyncExplainYqlResult TScriptingClient::ExplainYqlScript(const TString& script, - const TExplainYqlRequestSettings& settings) -{ - return Impl_->ExplainYqlScript(script, settings); -} - +TAsyncExplainYqlResult TScriptingClient::ExplainYqlScript(const TString& script, + const TExplainYqlRequestSettings& settings) +{ + return Impl_->ExplainYqlScript(script, settings); +} + } // namespace NScheme } // namespace NYdb diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting.h b/ydb/public/sdk/cpp/client/draft/ydb_scripting.h index e99f05daa24..d84290b8e20 100644 --- a/ydb/public/sdk/cpp/client/draft/ydb_scripting.h +++ b/ydb/public/sdk/cpp/client/draft/ydb_scripting.h @@ -82,21 +82,21 @@ private: std::shared_ptr<TReaderImpl> ReaderImpl_; }; -class TExplainYqlResult : public TStatus { -public: +class TExplainYqlResult : public TStatus { +public: TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map<TString, Ydb::Type>&& types, TString&& plan); - + std::map<TString, TType> GetParameterTypes() const; - const TString& GetPlan() const; - -private: + const TString& GetPlan() const; + +private: ::google::protobuf::Map<TString, Ydb::Type> ParameterTypes_; - TString Plan_; -}; - + TString Plan_; +}; + using TAsyncExecuteYqlResult = NThreading::TFuture<TExecuteYqlResult>; using TAsyncYqlResultPartIterator = NThreading::TFuture<TYqlResultPartIterator>; -using TAsyncExplainYqlResult = NThreading::TFuture<TExplainYqlResult>; +using TAsyncExplainYqlResult = NThreading::TFuture<TExplainYqlResult>; //////////////////////////////////////////////////////////////////////////////// @@ -104,18 +104,18 @@ struct TExecuteYqlRequestSettings : public TOperationRequestSettings<TExecuteYql FLUENT_SETTING_DEFAULT(NTable::ECollectQueryStatsMode, CollectQueryStats, NTable::ECollectQueryStatsMode::None); }; -enum class ExplainYqlRequestMode { +enum class ExplainYqlRequestMode { // Parse = 1, - Validate = 2, - Plan = 3, -}; - -struct TExplainYqlRequestSettings : public TOperationRequestSettings<TExplainYqlRequestSettings> { + Validate = 2, + Plan = 3, +}; + +struct TExplainYqlRequestSettings : public TOperationRequestSettings<TExplainYqlRequestSettings> { FLUENT_SETTING_DEFAULT(ExplainYqlRequestMode, Mode, ExplainYqlRequestMode::Validate); -}; - -//////////////////////////////////////////////////////////////////////////////// - +}; + +//////////////////////////////////////////////////////////////////////////////// + class TScriptingClient { class TImpl; @@ -143,9 +143,9 @@ public: TAsyncYqlResultPartIterator StreamExecuteYqlScript(const TString& script, TParams&& params, const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings()); - TAsyncExplainYqlResult ExplainYqlScript(const TString& script, - const TExplainYqlRequestSettings& settings = TExplainYqlRequestSettings()); - + TAsyncExplainYqlResult ExplainYqlScript(const TString& script, + const TExplainYqlRequestSettings& settings = TExplainYqlRequestSettings()); + private: std::shared_ptr<TImpl> Impl_; }; diff --git a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h index 355c271dc24..fcd371df126 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h +++ b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h @@ -31,9 +31,9 @@ class TYqlResultPartIterator; namespace NTable { enum class ECollectQueryStatsMode { - None = 0, // Stats collection is disabled - Basic = 1, // Aggregated stats of reads, updates and deletes per table - Full = 2 // Add per-stage execution profile and query plan on top of Basic mode + None = 0, // Stats collection is disabled + Basic = 1, // Aggregated stats of reads, updates and deletes per table + Full = 2 // Add per-stage execution profile and query plan on top of Basic mode }; class TQueryStats { @@ -41,7 +41,7 @@ class TQueryStats { friend class NYdb::TProtoAccessor; friend class NYdb::NScripting::TScriptingClient; friend class NYdb::NScripting::TYqlResultPartIterator; - friend class TScanQueryPartIterator; + friend class TScanQueryPartIterator; public: TString ToString(bool withPlan = false) const; diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index 610fec38847..f30e7402c09 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -1274,17 +1274,17 @@ public: // TODO: Add headers for streaming calls. TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}}; TStatus status{std::move(plainStatus)}; - TMaybe<TQueryStats> queryStats; - - if (self->Response_.result().has_query_stats()) { - queryStats = TQueryStats(self->Response_.result().query_stats()); - } - + TMaybe<TQueryStats> queryStats; + + if (self->Response_.result().has_query_stats()) { + queryStats = TQueryStats(self->Response_.result().query_stats()); + } + if (self->Response_.result().has_result_set()) { - promise.SetValue({std::move(status), - TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), queryStats}); + promise.SetValue({std::move(status), + TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), queryStats}); } else { - promise.SetValue({std::move(status), queryStats}); + promise.SetValue({std::move(status), queryStats}); } } }; @@ -2602,14 +2602,14 @@ public: *request.mutable_parameters() = *params; } - if (settings.Explain_) { - request.set_mode(Ydb::Table::ExecuteScanQueryRequest::MODE_EXPLAIN); - } else { - request.set_mode(Ydb::Table::ExecuteScanQueryRequest::MODE_EXEC); - } - - request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); + if (settings.Explain_) { + request.set_mode(Ydb::Table::ExecuteScanQueryRequest::MODE_EXPLAIN); + } else { + request.set_mode(Ydb::Table::ExecuteScanQueryRequest::MODE_EXEC); + } + request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); + auto promise = NewPromise<std::pair<TPlainStatus, TScanQueryProcessorPtr>>(); Connections_->StartReadStream< @@ -4192,14 +4192,14 @@ const TMaybe<TQueryStats>& TDataQueryResult::GetStats() const { return QueryStats_; } -const TString TDataQueryResult::GetQueryPlan() const { - if (QueryStats_.Defined()) { - return NYdb::TProtoAccessor::GetProto(*QueryStats_.Get()).query_plan(); - } else { - return ""; - } -} - +const TString TDataQueryResult::GetQueryPlan() const { + if (QueryStats_.Defined()) { + return NYdb::TProtoAccessor::GetProto(*QueryStats_.Get()).query_plan(); + } else { + return ""; + } +} + //////////////////////////////////////////////////////////////////////////////// TBeginTransactionResult::TBeginTransactionResult(TStatus&& status, TTransaction transaction) diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index 42d21d2fec7..aead5f6014e 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -891,14 +891,14 @@ struct TBulkUpsertSettings : public TOperationRequestSettings<TBulkUpsertSetting FLUENT_SETTING_DEFAULT(TString, FormatSettings, ""); }; -struct TStreamExecScanQuerySettings : public TRequestSettings<TStreamExecScanQuerySettings> { - // Return query plan without actual query execution - FLUENT_SETTING_DEFAULT(bool, Explain, false); - - // Collect runtime statistics with a given detalization mode - FLUENT_SETTING_DEFAULT(ECollectQueryStatsMode, CollectQueryStats, ECollectQueryStatsMode::None); -}; - +struct TStreamExecScanQuerySettings : public TRequestSettings<TStreamExecScanQuerySettings> { + // Return query plan without actual query execution + FLUENT_SETTING_DEFAULT(bool, Explain, false); + + // Collect runtime statistics with a given detalization mode + FLUENT_SETTING_DEFAULT(ECollectQueryStatsMode, CollectQueryStats, ECollectQueryStatsMode::None); +}; + class TSession; struct TRetryState; @@ -1728,8 +1728,8 @@ public: const TMaybe<TQueryStats>& GetStats() const; - const TString GetQueryPlan() const; - + const TString GetQueryPlan() const; + private: TMaybe<TTransaction> Transaction_; TVector<TResultSet> ResultSets_; @@ -1778,28 +1778,28 @@ public: const TResultSet& GetResultSet() const { return *ResultSet_; } TResultSet ExtractResultSet() { return std::move(*ResultSet_); } - bool HasQueryStats() const { return QueryStats_.Defined(); } - const TQueryStats& GetQueryStats() const { return *QueryStats_; } - TQueryStats ExtractQueryStats() { return std::move(*QueryStats_); } - + bool HasQueryStats() const { return QueryStats_.Defined(); } + const TQueryStats& GetQueryStats() const { return *QueryStats_; } + TQueryStats ExtractQueryStats() { return std::move(*QueryStats_); } + TScanQueryPart(TStatus&& status) : TStreamPartStatus(std::move(status)) {} - TScanQueryPart(TStatus&& status, const TMaybe<TQueryStats> &queryStats) - : TStreamPartStatus(std::move(status)) - , QueryStats_(queryStats) - {} - - TScanQueryPart(TStatus&& status, TResultSet&& resultSet, const TMaybe<TQueryStats> &queryStats) + TScanQueryPart(TStatus&& status, const TMaybe<TQueryStats> &queryStats) : TStreamPartStatus(std::move(status)) + , QueryStats_(queryStats) + {} + + TScanQueryPart(TStatus&& status, TResultSet&& resultSet, const TMaybe<TQueryStats> &queryStats) + : TStreamPartStatus(std::move(status)) , ResultSet_(std::move(resultSet)) - , QueryStats_(queryStats) + , QueryStats_(queryStats) {} private: TMaybe<TResultSet> ResultSet_; - TMaybe<TQueryStats> QueryStats_; + TMaybe<TQueryStats> QueryStats_; }; using TAsyncScanQueryPart = NThreading::TFuture<TScanQueryPart>; diff --git a/ydb/public/sdk/python/ydb/scripting.py b/ydb/public/sdk/python/ydb/scripting.py index 587630f3245..f4ea53b6f7a 100644 --- a/ydb/public/sdk/python/ydb/scripting.py +++ b/ydb/public/sdk/python/ydb/scripting.py @@ -1,6 +1,6 @@ from ydb.public.api.protos import ydb_scripting_pb2 from ydb.public.api.grpc import ydb_scripting_v1_pb2_grpc -from . import issues, convert, settings +from . import issues, convert, settings class TypedParameters(object): @@ -23,21 +23,21 @@ class ScriptingClientSettings(object): return self -class ExplainYqlScriptSettings(settings.BaseRequestSettings): +class ExplainYqlScriptSettings(settings.BaseRequestSettings): MODE_UNSPECIFIED = 0 MODE_PARSE = 1 MODE_VALIDATE = 2 MODE_EXPLAIN = 3 - - def __init__(self): - super(ExplainYqlScriptSettings, self).__init__() - self.mode = False - - def with_mode(self, val): - self.mode = val - return self - - + + def __init__(self): + super(ExplainYqlScriptSettings, self).__init__() + self.mode = False + + def with_mode(self, val): + self.mode = val + return self + + def _execute_yql_query_request_factory(script, tp=None, settings=None): params = ( None @@ -54,11 +54,11 @@ class YqlQueryResult(object): ) -class YqlExplainResult(object): - def __init__(self, result): - self.plan = result.plan - - +class YqlExplainResult(object): + def __init__(self, result): + self.plan = result.plan + + def _wrap_response(rpc_state, response, scripting_client_settings): issues._process_response(response.operation) message = ydb_scripting_pb2.ExecuteYqlResult() @@ -66,13 +66,13 @@ def _wrap_response(rpc_state, response, scripting_client_settings): return YqlQueryResult(message) -def _wrap_explain_response(rpc_state, response): - issues._process_response(response.operation) - message = ydb_scripting_pb2.ExplainYqlResult() - response.operation.result.Unpack(message) - return YqlExplainResult(message) - - +def _wrap_explain_response(rpc_state, response): + issues._process_response(response.operation) + message = ydb_scripting_pb2.ExplainYqlResult() + response.operation.result.Unpack(message) + return YqlExplainResult(message) + + class ScriptingClient(object): def __init__(self, driver, scripting_client_settings=None): self.driver = driver @@ -92,12 +92,12 @@ class ScriptingClient(object): settings=settings, wrap_args=(self.scripting_client_settings,), ) - - def explain_yql(self, script, settings=None): - return self.driver( - ydb_scripting_pb2.ExplainYqlRequest(script=script, mode=settings.mode), - ydb_scripting_v1_pb2_grpc.ScriptingServiceStub, + + def explain_yql(self, script, settings=None): + return self.driver( + ydb_scripting_pb2.ExplainYqlRequest(script=script, mode=settings.mode), + ydb_scripting_v1_pb2_grpc.ScriptingServiceStub, "ExplainYql", - _wrap_explain_response, + _wrap_explain_response, settings=settings, - ) + ) diff --git a/ydb/services/ydb/ydb_scripting.cpp b/ydb/services/ydb/ydb_scripting.cpp index 3596b17fd94..7c14425da08 100644 --- a/ydb/services/ydb/ydb_scripting.cpp +++ b/ydb/services/ydb/ydb_scripting.cpp @@ -57,18 +57,18 @@ void TGRpcYdbScriptingService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) { new TGrpcRequestOperationCall<ExecuteYqlRequest, ExecuteYqlResponse> (ctx, &DoExecuteYqlScript, TRequestAuxSettings{TRateLimiterMode::Ru, nullptr})); }) - + ADD_REQUEST(StreamExecuteYql, ExecuteYqlRequest, ExecuteYqlPartialResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TGrpcRequestNoOperationCall<ExecuteYqlRequest, ExecuteYqlPartialResponse> (ctx, &DoStreamExecuteYqlScript, TRequestAuxSettings{TRateLimiterMode::Rps, nullptr})); }) - ADD_REQUEST(ExplainYql, ExplainYqlRequest, ExplainYqlResponse, { + ADD_REQUEST(ExplainYql, ExplainYqlRequest, ExplainYqlResponse, { ActorSystem_->Send(GRpcRequestProxyId_, new TGrpcRequestOperationCall<ExplainYqlRequest, ExplainYqlResponse> (ctx, &DoExplainYqlScript, TRequestAuxSettings{TRateLimiterMode::Rps, nullptr})); - }) + }) #undef ADD_REQUEST } diff --git a/ydb/tests/functional/canonical/canondata/result.json b/ydb/tests/functional/canonical/canondata/result.json index d927ff0f908..567332cdf7c 100644 --- a/ydb/tests/functional/canonical/canondata/result.json +++ b/ydb/tests/functional/canonical/canondata/result.json @@ -91,14 +91,14 @@ "uri": "file://test_sql.TestCanonicalFolder1.test_case_dynumber_select_params.sql-result_sets_/dynumber_select_params.sql_scan_query.results" } }, - "test_sql.TestCanonicalFolder1.test_case[explain.script-script]": { - "script": { + "test_sql.TestCanonicalFolder1.test_case[explain.script-script]": { + "script": { "uri": "file://test_sql.TestCanonicalFolder1.test_case_explain.script-script_/explain.script.results" - }, - "script_plan": { + }, + "script_plan": { "uri": "file://test_sql.TestCanonicalFolder1.test_case_explain.script-script_/explain.script.plan" - } - }, + } + }, "test_sql.TestCanonicalFolder1.test_case[index/select_using_index.sql-plan]": { "plan": { "uri": "file://test_sql.TestCanonicalFolder1.test_case_index_select_using_index.sql-plan_/index_select_using_index.sql.plan" @@ -913,9 +913,9 @@ "script": { "uri": "file://test_sql.TestCanonicalFolder1.test_case_simple_ct.script-script_/simple_ct.script.results" }, - "script_plan": { + "script_plan": { "uri": "file://test_sql.TestCanonicalFolder1.test_case_simple_ct.script-script_/simple_ct.script.plan" - }, + }, "table_data_Questions": { "uri": "file://test_sql.TestCanonicalFolder1.test_case_simple_ct.script-script_/simple_ct.script_Questions.results" } @@ -1163,16 +1163,16 @@ "test_sql.TestCanonicalFolder1.test_case[simple/script_params.script-script]": { "script": { "uri": "file://test_sql.TestCanonicalFolder1.test_case_simple_script_params.script-script_/simple_script_params.script.results" - }, - "script_plan": { + }, + "script_plan": { "uri": "file://test_sql.TestCanonicalFolder1.test_case_simple_script_params.script-script_/simple_script_params.script.plan" } }, "test_sql.TestCanonicalFolder1.test_case[table_types.script-script]": { "script": { "uri": "file://test_sql.TestCanonicalFolder1.test_case_table_types.script-script_/table_types.script.results" - }, - "script_plan": { + }, + "script_plan": { "uri": "file://test_sql.TestCanonicalFolder1.test_case_table_types.script-script_/table_types.script.plan" } }, diff --git a/ydb/tests/functional/canonical/sql/explain.script b/ydb/tests/functional/canonical/sql/explain.script index 74d5722adb6..7b349cae29b 100644 --- a/ydb/tests/functional/canonical/sql/explain.script +++ b/ydb/tests/functional/canonical/sql/explain.script @@ -1,48 +1,48 @@ ---!syntax_v1 -PRAGMA Kikimr.UseNewEngine = "true"; -CREATE TABLE ScriptingTest ( - Key Uint64, - Value String, - PRIMARY KEY (Key) -); -COMMIT; - -PRAGMA Kikimr.UseNewEngine = "false"; -REPLACE INTO ScriptingTest (Key, Value) VALUES - (1, "One"), - (2, "Two"); -COMMIT; - -PRAGMA Kikimr.UseNewEngine = "true"; -REPLACE INTO ScriptingTest (Key, Value) VALUES - (3, "Three"), - (4, "Four"); -COMMIT; - -PRAGMA kikimr.ScanQuery = "true"; -PRAGMA Kikimr.UseNewEngine = "true"; -SELECT count(*) FROM ScriptingTest; -COMMIT; - -PRAGMA kikimr.ScanQuery = "false"; -PRAGMA Kikimr.UseNewEngine = "true"; -SELECT count(*) FROM ScriptingTest; -COMMIT; - -PRAGMA kikimr.ScanQuery = "true"; -PRAGMA Kikimr.UseNewEngine = "false"; -SELECT count(*) FROM ScriptingTest; -COMMIT; - -PRAGMA kikimr.ScanQuery = "false"; -PRAGMA Kikimr.UseNewEngine = "false"; -SELECT count(*) FROM ScriptingTest; -COMMIT; - -PRAGMA kikimr.ScanQuery = "true"; -SELECT 1*2*3*4*5; -COMMIT; - -PRAGMA kikimr.ScanQuery = "false"; -SELECT 1*2*3*4*5; -COMMIT;
\ No newline at end of file +--!syntax_v1 +PRAGMA Kikimr.UseNewEngine = "true"; +CREATE TABLE ScriptingTest ( + Key Uint64, + Value String, + PRIMARY KEY (Key) +); +COMMIT; + +PRAGMA Kikimr.UseNewEngine = "false"; +REPLACE INTO ScriptingTest (Key, Value) VALUES + (1, "One"), + (2, "Two"); +COMMIT; + +PRAGMA Kikimr.UseNewEngine = "true"; +REPLACE INTO ScriptingTest (Key, Value) VALUES + (3, "Three"), + (4, "Four"); +COMMIT; + +PRAGMA kikimr.ScanQuery = "true"; +PRAGMA Kikimr.UseNewEngine = "true"; +SELECT count(*) FROM ScriptingTest; +COMMIT; + +PRAGMA kikimr.ScanQuery = "false"; +PRAGMA Kikimr.UseNewEngine = "true"; +SELECT count(*) FROM ScriptingTest; +COMMIT; + +PRAGMA kikimr.ScanQuery = "true"; +PRAGMA Kikimr.UseNewEngine = "false"; +SELECT count(*) FROM ScriptingTest; +COMMIT; + +PRAGMA kikimr.ScanQuery = "false"; +PRAGMA Kikimr.UseNewEngine = "false"; +SELECT count(*) FROM ScriptingTest; +COMMIT; + +PRAGMA kikimr.ScanQuery = "true"; +SELECT 1*2*3*4*5; +COMMIT; + +PRAGMA kikimr.ScanQuery = "false"; +SELECT 1*2*3*4*5; +COMMIT;
\ No newline at end of file diff --git a/ydb/tests/functional/canonical/test_sql.py b/ydb/tests/functional/canonical/test_sql.py index af5cdc2b9c7..fcad5b399ac 100644 --- a/ydb/tests/functional/canonical/test_sql.py +++ b/ydb/tests/functional/canonical/test_sql.py @@ -425,12 +425,12 @@ class BaseCanonicalTest(object): rp = script_client.execute_yql(query, params) return self.wrap_result_sets(rp.result_sets) - def script_explain(self, query): - script_client = ydb.ScriptingClient(self.driver) - settings = ydb.ExplainYqlScriptSettings().with_mode(ydb.ExplainYqlScriptSettings.MODE_EXPLAIN) - rp = script_client.explain_yql(query, settings) - return rp.plan - + def script_explain(self, query): + script_client = ydb.ScriptingClient(self.driver) + settings = ydb.ExplainYqlScriptSettings().with_mode(ydb.ExplainYqlScriptSettings.MODE_EXPLAIN) + rp = script_client.explain_yql(query, settings) + return rp.plan + def compare_tables_test(self, canons, config, query_name): for table_name in config.get('compare_tables', []): canons['table_data_%s' % table_name] = self.canonical_results( @@ -486,7 +486,7 @@ class BaseCanonicalTest(object): pt = config.get('parameters_types', {}) result_sets = self.script_query(query, pv, pt) canons['script'] = self.canonical_results(query_name, self.pretty_json(result_sets)) - canons['script_plan'] = self.canonical_plan(query_name, self.script_explain(query)) + canons['script_plan'] = self.canonical_plan(query_name, self.script_explain(query)) self.compare_tables_test(canons, config, query_name) elif kind == 'plan': plan = self.explain(query) diff --git a/ydb/tests/functional/suite_tests/canondata/result.json b/ydb/tests/functional/suite_tests/canondata/result.json index ebfb6f81449..06e3464f701 100644 --- a/ydb/tests/functional/suite_tests/canondata/result.json +++ b/ydb/tests/functional/suite_tests/canondata/result.json @@ -19064,11 +19064,11 @@ "query_99": { "uri": "file://test_sql_logic.TestSQLLogic.test_sql_suite_results-select3-9.test_/query_99" } - }, - "test_stream_query.TestStreamQuery.test_sql_suite[plan-window.test]": {}, - "test_stream_query.TestStreamQuery.test_sql_suite[results-window.test]": { - "window.test.out": { + }, + "test_stream_query.TestStreamQuery.test_sql_suite[plan-window.test]": {}, + "test_stream_query.TestStreamQuery.test_sql_suite[results-window.test]": { + "window.test.out": { "uri": "file://test_stream_query.TestStreamQuery.test_sql_suite_results-window.test_/window.test.out" - } + } } } diff --git a/ydb/tests/functional/suite_tests/stream_query/window.test b/ydb/tests/functional/suite_tests/stream_query/window.test index 9704855ef81..4a21fc859a4 100644 --- a/ydb/tests/functional/suite_tests/stream_query/window.test +++ b/ydb/tests/functional/suite_tests/stream_query/window.test @@ -1,294 +1,294 @@ -statement ok -CREATE TABLE empsalary ( - depname String, - empno Int64, - salary Int64, - enroll_date Date, - primary key(empno) -); - -statement ok -INSERT INTO empsalary (depname, empno, salary, enroll_date) VALUES -('develop', 10, 5200, Date('2007-08-01')), -('sales', 1, 5000, Date('2006-10-01')), -('personnel', 5, 3500, Date('2007-12-10')), -('sales', 4, 4800, Date('2007-08-08')), -('personnel', 2, 3900, Date('2006-12-23')), -('develop', 7, 4200, Date('2008-01-01')), -('develop', 9, 4500, Date('2008-01-01')), -('sales', 3, 4800, Date('2007-08-01')), -('develop', 8, 6000, Date('2006-10-01')), -('develop', 11, 5200, Date('2007-08-15')); - -statement ok -CREATE TABLE tenk1 ( - unique1 Int64, - unique2 Int64, - two Int64, - four Int64, - ten Int64, - twenty Int64, - hundred Int64, - thousand Int64, - twothousand Int64, - fivethous Int64, - tenthous Int64, - odd Int64, - even Int64, - stringu1 String, - stringu2 String, - string4 String, - primary key (unique1, unique2) -); - -statement ok -CREATE TABLE int4_tbl(f1 Int32, primary key (f1)); - -statement ok -INSERT INTO int4_tbl(f1) VALUES (123456), (0); - -statement import table data -insert into tenk1 (unique1, unique2, two, four, ten, twenty, hundred, thousand, twothousand, fivethous, tenthous, odd, even, stringu1, stringu2, string4) values postgres/data/tenk.data - -statement stream query -SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname) ORDER BY depname, salary; - -statement stream query -SELECT depname, empno, salary, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary); - --- with GROUP BY -statement stream query -SELECT four, ten, SUM(SUM(four)) OVER w, AVG(ten) FROM tenk1 -GROUP BY four, ten WINDOW w AS (PARTITION BY four) ORDER BY four, ten; - -statement stream query -SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname); - -statement stream query -SELECT depname, empno, salary, rank() OVER w AS r FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary) ORDER BY r; - --- empty window specification -statement stream query -SELECT COUNT(*) OVER w FROM tenk1 WHERE unique2 < 10 WINDOW w AS (); - --- no window operation -statement stream query -SELECT four FROM tenk1 WHERE FALSE WINDOW w AS (PARTITION BY ten); - --- cumulative aggregate -statement stream query -SELECT sum(four) OVER w AS sum_1, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY ten ORDER BY unique2); - -statement stream query -SELECT row_number() OVER w FROM tenk1 WHERE unique2 < 10 WINDOW w AS (ORDER BY unique2); - -statement stream query -SELECT rank() OVER w AS rank_1, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); - -statement stream query -SELECT dense_rank() OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); - -statement error -SELECT percent_rank() OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); - -statement error -SELECT cume_dist() OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); - -statement error -SELECT ntile(3) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (ORDER BY ten, four); - -statement error -SELECT ntile(NULL) OVER w, ten, four FROM tenk1 WINDOW w AS (ORDER BY ten, four) LIMIT 2; - -statement stream query -SELECT lag(ten) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); - -statement error -SELECT lag(ten, four) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); - -statement error -SELECT lag(ten, four, 0) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); - -statement stream query -SELECT lead(ten) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); - -statement stream query -SELECT lead(ten * 2, 1) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); - -statement error -SELECT lead(ten * 2, 1, -1) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); - -statement stream query -SELECT first_value(ten) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); - --- last_value returns the last row of the frame, which is CURRENT ROW in ORDER BY window. --- XXX: different behavior w.r.t postgres -statement stream query -SELECT last_value(four) OVER w, last_value(four) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (ORDER BY ten); - -statement stream query -SELECT last_value(ten) OVER w, ten, four FROM - (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten LIMIT 1000) -WINDOW w AS (PARTITION BY four) -ORDER BY four, ten; - -statement error -SELECT nth_value(ten, four + 1) OVER w, ten, four FROM - (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten) -WINDOW w AS (PARTITION BY four); - -statement stream query -SELECT ten, two, sum(hundred) AS gsum, sum(sum(hundred)) OVER w AS wsum -FROM tenk1 GROUP BY ten, two WINDOW w AS (PARTITION BY two ORDER BY ten); - -statement stream query -SELECT count(*) OVER w, four FROM (SELECT * FROM tenk1 WHERE two = 1) WHERE unique2 < 10 WINDOW w AS (PARTITION BY four); - --- WRONG! -statement stream query -SELECT (count(*) OVER w + sum(hundred) OVER w) AS cntsum -FROM tenk1 WHERE unique2 < 10 -WINDOW w AS (PARTITION BY four ORDER BY ten); - --- opexpr with different windows evaluation. -statement stream query -SELECT * FROM( - SELECT count(*) OVER w1 + - sum(hundred) OVER w2 AS total, - count(*) OVER w1 AS fourcount, - sum(hundred) OVER w2 AS twosum - FROM tenk1 - WINDOW w1 AS (PARTITION BY four ORDER BY ten), w2 AS (PARTITION BY two ORDER BY ten) -) -WHERE total <> fourcount + twosum; - -statement stream query -SELECT avg(four) OVER w FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY thousand / 100); - -statement stream query -SELECT ten, two, sum(hundred) AS gsum, sum(sum(hundred)) OVER win AS wsum -FROM tenk1 GROUP BY ten, two WINDOW win AS (PARTITION BY two ORDER BY ten); - --- more than one window with GROUP BY -statement stream query -SELECT sum(salary), - row_number() OVER w1, - sum(sum(salary)) OVER w2 -FROM empsalary GROUP BY depname -WINDOW w1 AS (ORDER BY depname), w2 AS (ORDER BY depname DESC); - --- identical windows with different names -statement stream query -SELECT sum(salary) OVER w1, count(*) OVER w2 -FROM empsalary WINDOW w1 AS (ORDER BY salary), w2 AS (ORDER BY salary); - --- subplan -statement error -SELECT lead(ten, (SELECT two FROM tenk1 WHERE s.unique2 = unique2)) OVER w -FROM tenk1 s WHERE unique2 < 10 -WINDOW w AS (PARTITION BY four ORDER BY ten); - --- empty table -statement stream query -SELECT count(*) OVER w FROM (SELECT * FROM tenk1 WHERE FALSE) WINDOW w AS (PARTITION BY four); - --- mixture of agg/wfunc in the same window -statement stream query -SELECT sum(salary) OVER w, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary DESC); - --- strict aggs -statement stream query -SELECT empno, depname, salary, bonus, depadj, MIN(bonus) OVER w, MAX(depadj) OVER w FROM( - SELECT empsalary.*, - CASE WHEN enroll_date < DateTime::MakeDatetime(DateTime::Parse("%Y-%m-%d")('2007-08-01')) - THEN 2008 - DateTime::GetYear(enroll_date) - ELSE 42 END * 500 AS bonus, - CASE WHEN AVG(salary) OVER wi < salary - THEN 200 - ELSE 42 END AS depadj - FROM empsalary - WINDOW wi AS (PARTITION BY depname) -) -WINDOW w AS (ORDER BY empno); - --- window function over ungrouped agg over empty row set (bug before 9.1) -statement stream query -SELECT SUM(COUNT(f1)) OVER w FROM int4_tbl WHERE f1=42 WINDOW w AS (); - --- window function with ORDER BY an expression involving aggregates (9.1 bug) -statement stream query -select ten, - sum(unique1) + sum(unique2) as res, - rank() over w as rank -from tenk1 -group by ten -window w AS (order by sum(unique1) + sum(unique2)) -order by ten; - --- window and aggregate with GROUP BY expression (9.2 bug) -statement stream query -select first_value(max(x)) over w, y - from (select unique1 as x, ten+four as y from tenk1) - group by y - window w AS (); - --- test non-default frame specifications -statement stream query -SELECT four, ten, - sum(ten) over w, - last_value(ten) over w -FROM (select distinct ten, four from tenk1) -WINDOW w AS (partition by four order by ten); - -statement error -SELECT four, ten, - sum(ten) over w, - last_value(ten) over w -FROM (select distinct ten, four from tenk1) -WINDOW w AS (partition by four order by ten range between unbounded preceding and current row); - -statement error -SELECT four, ten, - sum(ten) over w, - last_value(ten) over w -FROM (select distinct ten, four from tenk1) -WINDOW w AS (partition by four order by ten range between unbounded preceding and unbounded following); - -statement error -SELECT four, ten/4 as two, - sum(ten/4) over w, - last_value(ten/4) over w -FROM (select distinct ten, four from tenk1) -WINDOW w AS (partition by four order by ten/4 range between unbounded preceding and current row); - -statement stream query -SELECT four, ten/4 as two, - sum(ten/4) over w, - last_value(ten/4) over w -FROM (select distinct ten, four from tenk1) -WINDOW w AS (partition by four order by ten/4 rows between unbounded preceding and current row) - -statement error -SELECT sum(unique1) over w, - unique1, four -FROM tenk1 WHERE unique1 < 10 -WINDOW w AS (order by four range between current row and unbounded following); - -statement stream query -SELECT sum(unique1) over w, - unique1, four -FROM tenk1 WHERE unique1 < 10 -WINDOW w AS (rows between current row and unbounded following); - -statement stream query -SELECT sum(unique1) over w, - unique1, four -FROM tenk1 WHERE unique1 < 10 -WINDOW w AS (rows between 2 preceding and 2 following); - -statement stream query -SELECT sum(unique1) over w, - unique1, four -FROM tenk1 WHERE unique1 < 10 -WINDOW w AS (rows between 2 preceding and 2 following exclude no others); - +statement ok +CREATE TABLE empsalary ( + depname String, + empno Int64, + salary Int64, + enroll_date Date, + primary key(empno) +); + +statement ok +INSERT INTO empsalary (depname, empno, salary, enroll_date) VALUES +('develop', 10, 5200, Date('2007-08-01')), +('sales', 1, 5000, Date('2006-10-01')), +('personnel', 5, 3500, Date('2007-12-10')), +('sales', 4, 4800, Date('2007-08-08')), +('personnel', 2, 3900, Date('2006-12-23')), +('develop', 7, 4200, Date('2008-01-01')), +('develop', 9, 4500, Date('2008-01-01')), +('sales', 3, 4800, Date('2007-08-01')), +('develop', 8, 6000, Date('2006-10-01')), +('develop', 11, 5200, Date('2007-08-15')); + +statement ok +CREATE TABLE tenk1 ( + unique1 Int64, + unique2 Int64, + two Int64, + four Int64, + ten Int64, + twenty Int64, + hundred Int64, + thousand Int64, + twothousand Int64, + fivethous Int64, + tenthous Int64, + odd Int64, + even Int64, + stringu1 String, + stringu2 String, + string4 String, + primary key (unique1, unique2) +); + +statement ok +CREATE TABLE int4_tbl(f1 Int32, primary key (f1)); + +statement ok +INSERT INTO int4_tbl(f1) VALUES (123456), (0); + +statement import table data +insert into tenk1 (unique1, unique2, two, four, ten, twenty, hundred, thousand, twothousand, fivethous, tenthous, odd, even, stringu1, stringu2, string4) values postgres/data/tenk.data + +statement stream query +SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname) ORDER BY depname, salary; + +statement stream query +SELECT depname, empno, salary, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary); + +-- with GROUP BY +statement stream query +SELECT four, ten, SUM(SUM(four)) OVER w, AVG(ten) FROM tenk1 +GROUP BY four, ten WINDOW w AS (PARTITION BY four) ORDER BY four, ten; + +statement stream query +SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname); + +statement stream query +SELECT depname, empno, salary, rank() OVER w AS r FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary) ORDER BY r; + +-- empty window specification +statement stream query +SELECT COUNT(*) OVER w FROM tenk1 WHERE unique2 < 10 WINDOW w AS (); + +-- no window operation +statement stream query +SELECT four FROM tenk1 WHERE FALSE WINDOW w AS (PARTITION BY ten); + +-- cumulative aggregate +statement stream query +SELECT sum(four) OVER w AS sum_1, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY ten ORDER BY unique2); + +statement stream query +SELECT row_number() OVER w FROM tenk1 WHERE unique2 < 10 WINDOW w AS (ORDER BY unique2); + +statement stream query +SELECT rank() OVER w AS rank_1, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); + +statement stream query +SELECT dense_rank() OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); + +statement error +SELECT percent_rank() OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); + +statement error +SELECT cume_dist() OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); + +statement error +SELECT ntile(3) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (ORDER BY ten, four); + +statement error +SELECT ntile(NULL) OVER w, ten, four FROM tenk1 WINDOW w AS (ORDER BY ten, four) LIMIT 2; + +statement stream query +SELECT lag(ten) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); + +statement error +SELECT lag(ten, four) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); + +statement error +SELECT lag(ten, four, 0) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); + +statement stream query +SELECT lead(ten) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); + +statement stream query +SELECT lead(ten * 2, 1) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); + +statement error +SELECT lead(ten * 2, 1, -1) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); + +statement stream query +SELECT first_value(ten) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten); + +-- last_value returns the last row of the frame, which is CURRENT ROW in ORDER BY window. +-- XXX: different behavior w.r.t postgres +statement stream query +SELECT last_value(four) OVER w, last_value(four) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (ORDER BY ten); + +statement stream query +SELECT last_value(ten) OVER w, ten, four FROM + (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten LIMIT 1000) +WINDOW w AS (PARTITION BY four) +ORDER BY four, ten; + +statement error +SELECT nth_value(ten, four + 1) OVER w, ten, four FROM + (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten) +WINDOW w AS (PARTITION BY four); + +statement stream query +SELECT ten, two, sum(hundred) AS gsum, sum(sum(hundred)) OVER w AS wsum +FROM tenk1 GROUP BY ten, two WINDOW w AS (PARTITION BY two ORDER BY ten); + +statement stream query +SELECT count(*) OVER w, four FROM (SELECT * FROM tenk1 WHERE two = 1) WHERE unique2 < 10 WINDOW w AS (PARTITION BY four); + +-- WRONG! +statement stream query +SELECT (count(*) OVER w + sum(hundred) OVER w) AS cntsum +FROM tenk1 WHERE unique2 < 10 +WINDOW w AS (PARTITION BY four ORDER BY ten); + +-- opexpr with different windows evaluation. +statement stream query +SELECT * FROM( + SELECT count(*) OVER w1 + + sum(hundred) OVER w2 AS total, + count(*) OVER w1 AS fourcount, + sum(hundred) OVER w2 AS twosum + FROM tenk1 + WINDOW w1 AS (PARTITION BY four ORDER BY ten), w2 AS (PARTITION BY two ORDER BY ten) +) +WHERE total <> fourcount + twosum; + +statement stream query +SELECT avg(four) OVER w FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY thousand / 100); + +statement stream query +SELECT ten, two, sum(hundred) AS gsum, sum(sum(hundred)) OVER win AS wsum +FROM tenk1 GROUP BY ten, two WINDOW win AS (PARTITION BY two ORDER BY ten); + +-- more than one window with GROUP BY +statement stream query +SELECT sum(salary), + row_number() OVER w1, + sum(sum(salary)) OVER w2 +FROM empsalary GROUP BY depname +WINDOW w1 AS (ORDER BY depname), w2 AS (ORDER BY depname DESC); + +-- identical windows with different names +statement stream query +SELECT sum(salary) OVER w1, count(*) OVER w2 +FROM empsalary WINDOW w1 AS (ORDER BY salary), w2 AS (ORDER BY salary); + +-- subplan +statement error +SELECT lead(ten, (SELECT two FROM tenk1 WHERE s.unique2 = unique2)) OVER w +FROM tenk1 s WHERE unique2 < 10 +WINDOW w AS (PARTITION BY four ORDER BY ten); + +-- empty table +statement stream query +SELECT count(*) OVER w FROM (SELECT * FROM tenk1 WHERE FALSE) WINDOW w AS (PARTITION BY four); + +-- mixture of agg/wfunc in the same window +statement stream query +SELECT sum(salary) OVER w, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary DESC); + +-- strict aggs +statement stream query +SELECT empno, depname, salary, bonus, depadj, MIN(bonus) OVER w, MAX(depadj) OVER w FROM( + SELECT empsalary.*, + CASE WHEN enroll_date < DateTime::MakeDatetime(DateTime::Parse("%Y-%m-%d")('2007-08-01')) + THEN 2008 - DateTime::GetYear(enroll_date) + ELSE 42 END * 500 AS bonus, + CASE WHEN AVG(salary) OVER wi < salary + THEN 200 + ELSE 42 END AS depadj + FROM empsalary + WINDOW wi AS (PARTITION BY depname) +) +WINDOW w AS (ORDER BY empno); + +-- window function over ungrouped agg over empty row set (bug before 9.1) +statement stream query +SELECT SUM(COUNT(f1)) OVER w FROM int4_tbl WHERE f1=42 WINDOW w AS (); + +-- window function with ORDER BY an expression involving aggregates (9.1 bug) +statement stream query +select ten, + sum(unique1) + sum(unique2) as res, + rank() over w as rank +from tenk1 +group by ten +window w AS (order by sum(unique1) + sum(unique2)) +order by ten; + +-- window and aggregate with GROUP BY expression (9.2 bug) +statement stream query +select first_value(max(x)) over w, y + from (select unique1 as x, ten+four as y from tenk1) + group by y + window w AS (); + +-- test non-default frame specifications +statement stream query +SELECT four, ten, + sum(ten) over w, + last_value(ten) over w +FROM (select distinct ten, four from tenk1) +WINDOW w AS (partition by four order by ten); + +statement error +SELECT four, ten, + sum(ten) over w, + last_value(ten) over w +FROM (select distinct ten, four from tenk1) +WINDOW w AS (partition by four order by ten range between unbounded preceding and current row); + +statement error +SELECT four, ten, + sum(ten) over w, + last_value(ten) over w +FROM (select distinct ten, four from tenk1) +WINDOW w AS (partition by four order by ten range between unbounded preceding and unbounded following); + +statement error +SELECT four, ten/4 as two, + sum(ten/4) over w, + last_value(ten/4) over w +FROM (select distinct ten, four from tenk1) +WINDOW w AS (partition by four order by ten/4 range between unbounded preceding and current row); + +statement stream query +SELECT four, ten/4 as two, + sum(ten/4) over w, + last_value(ten/4) over w +FROM (select distinct ten, four from tenk1) +WINDOW w AS (partition by four order by ten/4 rows between unbounded preceding and current row) + +statement error +SELECT sum(unique1) over w, + unique1, four +FROM tenk1 WHERE unique1 < 10 +WINDOW w AS (order by four range between current row and unbounded following); + +statement stream query +SELECT sum(unique1) over w, + unique1, four +FROM tenk1 WHERE unique1 < 10 +WINDOW w AS (rows between current row and unbounded following); + +statement stream query +SELECT sum(unique1) over w, + unique1, four +FROM tenk1 WHERE unique1 < 10 +WINDOW w AS (rows between 2 preceding and 2 following); + +statement stream query +SELECT sum(unique1) over w, + unique1, four +FROM tenk1 WHERE unique1 < 10 +WINDOW w AS (rows between 2 preceding and 2 following exclude no others); + diff --git a/ydb/tests/functional/suite_tests/test_base.py b/ydb/tests/functional/suite_tests/test_base.py index 0e6e9a35d99..cddbcae6c9f 100644 --- a/ydb/tests/functional/suite_tests/test_base.py +++ b/ydb/tests/functional/suite_tests/test_base.py @@ -45,7 +45,7 @@ class StatementTypes(enum.Enum): Ok = 'statement ok' Error = 'statement error' Query = 'statement query' - StreamQuery = 'statement stream query' + StreamQuery = 'statement stream query' ImportTableData = 'statement import table data' @@ -101,7 +101,7 @@ def get_single_statement(lines): statement = "\n".join(statement_lines) return statement statement_lines.append(line) - return "\n".join(statement_lines) + return "\n".join(statement_lines) class ParsedStatement(collections.namedtuple('ParsedStatement', ["at_line", "s_type", "suite_name", "text"])): @@ -206,36 +206,36 @@ def write_canonical_response(response, file): ) -def format_as_table(data): - nrows = len(data) - - if nrows == 0: - return "(0 rows)\n" - - keys = data[0].keys() - widths = [len(k) + 2 for k in keys] - for row in data: - for i, val in enumerate(row.values()): - widths[i] = max(widths[i], len(str(val)) + 2) - - col_names = ['{: ^{w}}'.format(str(k), w=widths[i]) for i, k in enumerate(keys)] - - table = "" - table += "|".join(col_names) - table += "\n" - table += "+".join(['-'*w for w in widths]) - table += "\n" - - for row in data: - vals = ['{: >{w}}'.format(str(val) + ' ', w=widths[i]) for i, val in enumerate(row.values())] - table += "|".join(vals) - table += "\n" - - table += "(%d row%s)\n" % (nrows, 's' if nrows != 1 else 0) - - return table - - +def format_as_table(data): + nrows = len(data) + + if nrows == 0: + return "(0 rows)\n" + + keys = data[0].keys() + widths = [len(k) + 2 for k in keys] + for row in data: + for i, val in enumerate(row.values()): + widths[i] = max(widths[i], len(str(val)) + 2) + + col_names = ['{: ^{w}}'.format(str(k), w=widths[i]) for i, k in enumerate(keys)] + + table = "" + table += "|".join(col_names) + table += "\n" + table += "+".join(['-'*w for w in widths]) + table += "\n" + + for row in data: + vals = ['{: >{w}}'.format(str(val) + ' ', w=widths[i]) for i, val in enumerate(row.values())] + table += "|".join(vals) + table += "\n" + + table += "(%d row%s)\n" % (nrows, 's' if nrows != 1 else 0) + + return table + + @six.add_metaclass(abc.ABCMeta) class BaseSuiteRunner(object): check_new_engine_plan = True @@ -317,7 +317,7 @@ class BaseSuiteRunner(object): from_type = { StatementTypes.Ok: self.assert_statement_ok, StatementTypes.Query: self.assert_statement_query, - StatementTypes.StreamQuery: self.assert_statement_stream_query, + StatementTypes.StreamQuery: self.assert_statement_stream_query, StatementTypes.Error: (lambda x: x), StatementTypes.ImportTableData: self.assert_statement_import_table_data, StatementTypes.Skipped: lambda x: x @@ -458,26 +458,26 @@ class BaseSuiteRunner(object): if retries == 0: raise - def assert_statement_stream_query(self, statement): - if self.plan: - return - - yql_text = format_yql_statement(statement.text, self.table_path_prefix) - yql_text = "--!syntax_v1\n" + yql_text + "\n\n" + def assert_statement_stream_query(self, statement): + if self.plan: + return + + yql_text = format_yql_statement(statement.text, self.table_path_prefix) + yql_text = "--!syntax_v1\n" + yql_text + "\n\n" result = self.execute_scan_query(yql_text) - file_name = statement.suite_name.split('/')[1] + '.out' - output_path = os.path.join(yatest_common.output_path(), file_name) - with open(output_path, 'a+') as w: - w.write(yql_text) + file_name = statement.suite_name.split('/')[1] + '.out' + output_path = os.path.join(yatest_common.output_path(), file_name) + with open(output_path, 'a+') as w: + w.write(yql_text) w.write(format_as_table(result)) - w.write("\n\n") - + w.write("\n\n") + self.files[file_name] = yatest_common.canonical_file( path=output_path, local=True, universal_lines=True, ) - + def is_probably_scheme(self, yql_text): lwr = yql_text.lower() return 'create table' in lwr or 'drop table' in lwr diff --git a/ydb/tests/functional/suite_tests/test_stream_query.py b/ydb/tests/functional/suite_tests/test_stream_query.py index dbc81c363a5..7e1a6f03cab 100644 --- a/ydb/tests/functional/suite_tests/test_stream_query.py +++ b/ydb/tests/functional/suite_tests/test_stream_query.py @@ -1,12 +1,12 @@ -# -*- coding: utf-8 -*- -import pytest +# -*- coding: utf-8 -*- +import pytest from test_base import BaseSuiteRunner, get_test_suites - - -class TestStreamQuery(BaseSuiteRunner): + + +class TestStreamQuery(BaseSuiteRunner): def execute_assert(self, left, right, message): assert left == right, message - @pytest.mark.parametrize(['kind', 'suite'], get_test_suites("stream_query")) - def test_sql_suite(self, kind, suite): - return self.run_sql_suite(kind, "stream_query", suite) + @pytest.mark.parametrize(['kind', 'suite'], get_test_suites("stream_query")) + def test_sql_suite(self, kind, suite): + return self.run_sql_suite(kind, "stream_query", suite) |