aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorkelvich <kelvich@yandex-team.ru>2022-02-10 16:52:14 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:52:14 +0300
commit14c9296656b0377bde647d52a12c3aefa2fd58dd (patch)
tree97c811a3d83dce39c0d6a323c6f8bcc0710beca5
parent986f48c33fde808e393e27d5dd0057658efbb660 (diff)
downloadydb-14c9296656b0377bde647d52a12c3aefa2fd58dd.tar.gz
Restoring authorship annotation for <kelvich@yandex-team.ru>. Commit 1 of 2.
-rw-r--r--ydb/core/grpc_services/base/base.h2
-rw-r--r--ydb/core/grpc_services/rpc_commit_transaction.cpp2
-rw-r--r--ydb/core/grpc_services/rpc_execute_data_query.cpp4
-rw-r--r--ydb/core/grpc_services/rpc_execute_yql_script.cpp6
-rw-r--r--ydb/core/grpc_services/rpc_explain_yql_script.cpp180
-rw-r--r--ydb/core/grpc_services/rpc_kqp_base.cpp12
-rw-r--r--ydb/core/grpc_services/rpc_kqp_base.h4
-rw-r--r--ydb/core/grpc_services/rpc_read_columns.cpp8
-rw-r--r--ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp98
-rw-r--r--ydb/core/grpc_services/ya.make2
-rw-r--r--ydb/core/kqp/common/kqp_gateway.h100
-rw-r--r--ydb/core/kqp/common/kqp_transform.h4
-rw-r--r--ydb/core/kqp/compile/kqp_mkql_compiler.cpp30
-rw-r--r--ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp8
-rw-r--r--ydb/core/kqp/counters/kqp_counters.cpp58
-rw-r--r--ydb/core/kqp/counters/kqp_counters.h8
-rw-r--r--ydb/core/kqp/executer/kqp_executer_stats.cpp2
-rw-r--r--ydb/core/kqp/executer/kqp_planner.cpp36
-rw-r--r--ydb/core/kqp/executer/kqp_planner.h10
-rw-r--r--ydb/core/kqp/executer/kqp_result_channel.cpp4
-rw-r--r--ydb/core/kqp/executer/kqp_scan_executer.cpp182
-rw-r--r--ydb/core/kqp/executer/kqp_shards_resolver.cpp4
-rw-r--r--ydb/core/kqp/executer/kqp_tasks_graph.h4
-rw-r--r--ydb/core/kqp/executer/ya.make2
-rw-r--r--ydb/core/kqp/expr_nodes/kqp_expr_nodes.json10
-rw-r--r--ydb/core/kqp/host/kqp_host.cpp158
-rw-r--r--ydb/core/kqp/host/kqp_host.h4
-rw-r--r--ydb/core/kqp/host/kqp_host_impl.h10
-rw-r--r--ydb/core/kqp/host/kqp_run_scan.cpp200
-rw-r--r--ydb/core/kqp/host/kqp_runner.cpp14
-rw-r--r--ydb/core/kqp/kqp_compile_actor.cpp4
-rw-r--r--ydb/core/kqp/kqp_ic_gateway.cpp102
-rw-r--r--ydb/core/kqp/kqp_worker_actor.cpp38
-rw-r--r--ydb/core/kqp/opt/kqp_opt_kql.cpp2
-rw-r--r--ydb/core/kqp/prepare/kqp_query_plan.cpp488
-rw-r--r--ydb/core/kqp/prepare/kqp_query_plan.h58
-rw-r--r--ydb/core/kqp/prepare/kqp_type_ann.cpp6
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_provider.h10
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_settings.h4
-rw-r--r--ydb/core/kqp/rm/kqp_resource_estimation.cpp8
-rw-r--r--ydb/core/kqp/rm/kqp_resource_estimation.h2
-rw-r--r--ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp4
-rw-r--r--ydb/core/kqp/rm/ya.make2
-rw-r--r--ydb/core/kqp/runtime/kqp_program_builder.cpp20
-rw-r--r--ydb/core/kqp/runtime/kqp_program_builder.h6
-rw-r--r--ydb/core/kqp/runtime/kqp_read_table.cpp262
-rw-r--r--ydb/core/kqp/runtime/kqp_read_table.h12
-rw-r--r--ydb/core/kqp/runtime/kqp_scan_data.cpp30
-rw-r--r--ydb/core/kqp/runtime/kqp_scan_data.h16
-rw-r--r--ydb/core/kqp/ut/common/kqp_ut_common.cpp148
-rw-r--r--ydb/core/kqp/ut/common/kqp_ut_common.h28
-rw-r--r--ydb/core/kqp/ut/kqp_explain_ut.cpp676
-rw-r--r--ydb/core/kqp/ut/kqp_join_ut.cpp12
-rw-r--r--ydb/core/kqp/ut/kqp_newengine_ut.cpp84
-rw-r--r--ydb/core/kqp/ut/kqp_scan_ut.cpp530
-rw-r--r--ydb/core/kqp/ut/kqp_scripting_ut.cpp286
-rw-r--r--ydb/core/kqp/ut/kqp_stats_ut.cpp368
-rw-r--r--ydb/core/kqp/ut/ya.make4
-rw-r--r--ydb/core/protos/kqp.proto52
-rw-r--r--ydb/core/protos/kqp_physical.proto8
-rw-r--r--ydb/core/protos/tx_datashard.proto30
-rw-r--r--ydb/core/sys_view/common/scan_actor_base_impl.h4
-rw-r--r--ydb/core/tx/columnshard/columnshard.h2
-rw-r--r--ydb/core/tx/datashard/datashard.cpp60
-rw-r--r--ydb/core/tx/datashard/datashard.h18
-rw-r--r--ydb/core/tx/datashard/datashard__kqp_scan.cpp722
-rw-r--r--ydb/core/tx/datashard/datashard__op_rows.cpp8
-rw-r--r--ydb/core/tx/datashard/datashard_impl.h8
-rw-r--r--ydb/core/tx/datashard/datashard_kqp_compute.cpp4
-rw-r--r--ydb/core/tx/datashard/datashard_kqp_compute.h6
-rw-r--r--ydb/core/tx/datashard/datashard_kqp_read_table.cpp216
-rw-r--r--ydb/core/tx/datashard/datashard_pipeline.h4
-rw-r--r--ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp640
-rw-r--r--ydb/core/tx/datashard/datashard_ut_order.cpp22
-rw-r--r--ydb/core/tx/datashard/ya.make2
-rw-r--r--ydb/library/yql/ast/yql_expr.cpp20
-rw-r--r--ydb/library/yql/ast/yql_expr.h8
-rw-r--r--ydb/library/yql/core/expr_nodes/yql_expr_nodes.json74
-rw-r--r--ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h12
-rw-r--r--ydb/library/yql/dq/opt/dq_opt.cpp14
-rw-r--r--ydb/library/yql/dq/opt/dq_opt.h4
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_join.cpp6
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_log.cpp104
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_log.h8
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_phy.cpp166
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_phy.h8
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp8
-rw-r--r--ydb/library/yql/dq/runtime/dq_tasks_runner.cpp8
-rw-r--r--ydb/library/yql/dq/runtime/dq_tasks_runner.h2
-rw-r--r--ydb/library/yql/minikql/codegen/codegen.cpp34
-rw-r--r--ydb/library/yql/minikql/codegen/codegen.h4
-rw-r--r--ydb/library/yql/minikql/codegen/ya.make6
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp34
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_impl.h2
-rw-r--r--ydb/public/api/grpc/ydb_scripting_v1.proto2
-rw-r--r--ydb/public/api/protos/ydb_experimental.proto4
-rw-r--r--ydb/public/api/protos/ydb_query_stats.proto4
-rw-r--r--ydb/public/api/protos/ydb_scripting.proto36
-rw-r--r--ydb/public/api/protos/ydb_table.proto8
-rw-r--r--ydb/public/lib/experimental/ydb_experimental.cpp12
-rw-r--r--ydb/public/lib/experimental/ydb_experimental.h22
-rw-r--r--ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp130
-rw-r--r--ydb/public/lib/ydb_cli/commands/ydb_service_table.h6
-rw-r--r--ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp118
-rw-r--r--ydb/public/sdk/cpp/client/draft/ydb_scripting.h46
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h8
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/table.cpp48
-rw-r--r--ydb/public/sdk/cpp/client/ydb_table/table.h44
-rw-r--r--ydb/public/sdk/python/ydb/scripting.py62
-rw-r--r--ydb/services/ydb/ydb_scripting.cpp6
-rw-r--r--ydb/tests/functional/canonical/canondata/result.json24
-rw-r--r--ydb/tests/functional/canonical/sql/explain.script96
-rw-r--r--ydb/tests/functional/canonical/test_sql.py14
-rw-r--r--ydb/tests/functional/suite_tests/canondata/result.json10
-rw-r--r--ydb/tests/functional/suite_tests/stream_query/window.test588
-rw-r--r--ydb/tests/functional/suite_tests/test_base.py92
-rw-r--r--ydb/tests/functional/suite_tests/test_stream_query.py16
117 files changed, 4050 insertions, 4050 deletions
diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h
index 44b25c4a5f9..91d9172b27c 100644
--- a/ydb/core/grpc_services/base/base.h
+++ b/ydb/core/grpc_services/base/base.h
@@ -135,7 +135,7 @@ struct TRpcServices {
EvLongTxRollback,
EvLongTxWrite,
EvLongTxRead,
- EvExplainYqlScript,
+ EvExplainYqlScript,
EvImportData,
EvAnalyticsReserved,
EvDataStreamsCreateStream,
diff --git a/ydb/core/grpc_services/rpc_commit_transaction.cpp b/ydb/core/grpc_services/rpc_commit_transaction.cpp
index 3b85b5ef62d..c4bc057bc20 100644
--- a/ydb/core/grpc_services/rpc_commit_transaction.cpp
+++ b/ydb/core/grpc_services/rpc_commit_transaction.cpp
@@ -82,7 +82,7 @@ private:
auto commitResult = TEvCommitTransactionRequest::AllocateResult<Ydb::Table::CommitTransactionResult>(Request_);
if (kqpResponse.HasQueryStats()) {
- FillQueryStats(*commitResult->mutable_query_stats(), kqpResponse);
+ FillQueryStats(*commitResult->mutable_query_stats(), kqpResponse);
}
ReplyWithResult(Ydb::StatusIds::SUCCESS, issueMessage, *commitResult, ctx);
diff --git a/ydb/core/grpc_services/rpc_execute_data_query.cpp b/ydb/core/grpc_services/rpc_execute_data_query.cpp
index 317a3488ce0..07ee2f550e8 100644
--- a/ydb/core/grpc_services/rpc_execute_data_query.cpp
+++ b/ydb/core/grpc_services/rpc_execute_data_query.cpp
@@ -171,8 +171,8 @@ public:
static void ConvertQueryStats(const NKikimrKqp::TQueryResponse& from, Ydb::Table::ExecuteQueryResult* to) {
if (from.HasQueryStats()) {
- FillQueryStats(*to->mutable_query_stats(), from);
- to->mutable_query_stats()->set_query_ast(from.GetQueryAst());
+ FillQueryStats(*to->mutable_query_stats(), from);
+ to->mutable_query_stats()->set_query_ast(from.GetQueryAst());
return;
}
diff --git a/ydb/core/grpc_services/rpc_execute_yql_script.cpp b/ydb/core/grpc_services/rpc_execute_yql_script.cpp
index 2650d32df50..9a1078e3163 100644
--- a/ydb/core/grpc_services/rpc_execute_yql_script.cpp
+++ b/ydb/core/grpc_services/rpc_execute_yql_script.cpp
@@ -100,9 +100,9 @@ public:
ConvertKqpQueryResultsToDbResult(kqpResponse, queryResult);
if (kqpResponse.HasQueryStats()) {
- FillQueryStats(*queryResult->mutable_query_stats(), kqpResponse);
- } else if (kqpResponse.HasQueryPlan()) {
- queryResult->mutable_query_stats()->set_query_plan(kqpResponse.GetQueryPlan());
+ FillQueryStats(*queryResult->mutable_query_stats(), kqpResponse);
+ } else if (kqpResponse.HasQueryPlan()) {
+ queryResult->mutable_query_stats()->set_query_plan(kqpResponse.GetQueryPlan());
}
ReplyWithResult(Ydb::StatusIds::SUCCESS, issueMessage, *queryResult, ctx);
diff --git a/ydb/core/grpc_services/rpc_explain_yql_script.cpp b/ydb/core/grpc_services/rpc_explain_yql_script.cpp
index 1d4667feece..8c49ad9296f 100644
--- a/ydb/core/grpc_services/rpc_explain_yql_script.cpp
+++ b/ydb/core/grpc_services/rpc_explain_yql_script.cpp
@@ -1,112 +1,112 @@
#include "service_yql_scripting.h"
-#include "rpc_kqp_base.h"
-#include "rpc_common.h"
-
+#include "rpc_kqp_base.h"
+#include "rpc_common.h"
+
#include <ydb/public/api/protos/ydb_scripting.pb.h>
-namespace NKikimr {
-namespace NGRpcService {
-
+namespace NKikimr {
+namespace NGRpcService {
+
using TEvExplainYqlScriptRequest =
TGrpcRequestOperationCall<Ydb::Scripting::ExplainYqlRequest, Ydb::Scripting::ExplainYqlResponse>;
-using namespace Ydb;
-
-class TExplainYqlScriptRPC : public TRpcKqpRequestActor<TExplainYqlScriptRPC, TEvExplainYqlScriptRequest> {
- using TBase = TRpcKqpRequestActor<TExplainYqlScriptRPC, TEvExplainYqlScriptRequest>;
-
-public:
- using TResult = Ydb::Scripting::ExplainYqlResult;
-
+using namespace Ydb;
+
+class TExplainYqlScriptRPC : public TRpcKqpRequestActor<TExplainYqlScriptRPC, TEvExplainYqlScriptRequest> {
+ using TBase = TRpcKqpRequestActor<TExplainYqlScriptRPC, TEvExplainYqlScriptRequest>;
+
+public:
+ using TResult = Ydb::Scripting::ExplainYqlResult;
+
TExplainYqlScriptRPC(IRequestOpCtx* msg)
- : TBase(msg) {}
-
- void Bootstrap(const TActorContext &ctx) {
- TBase::Bootstrap(ctx);
-
- this->Become(&TExplainYqlScriptRPC::StateWork);
- Proceed(ctx);
- }
-
- void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
- switch (ev->GetTypeRewrite()) {
- HFunc(NKqp::TEvKqp::TEvQueryResponse, Handle);
- default: TBase::StateWork(ev, ctx);
- }
- }
-
- void Proceed(const TActorContext &ctx) {
+ : TBase(msg) {}
+
+ void Bootstrap(const TActorContext &ctx) {
+ TBase::Bootstrap(ctx);
+
+ this->Become(&TExplainYqlScriptRPC::StateWork);
+ Proceed(ctx);
+ }
+
+ void StateWork(TAutoPtr<IEventHandle>& ev, const TActorContext& ctx) {
+ switch (ev->GetTypeRewrite()) {
+ HFunc(NKqp::TEvKqp::TEvQueryResponse, Handle);
+ default: TBase::StateWork(ev, ctx);
+ }
+ }
+
+ void Proceed(const TActorContext &ctx) {
const auto req = GetProtoRequest();
- const auto traceId = Request_->GetTraceId();
-
- auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>();
- SetAuthToken(ev, *Request_);
- SetDatabase(ev, *Request_);
-
- if (traceId) {
- ev->Record.SetTraceId(traceId.GetRef());
- }
-
- ev->Record.MutableRequest()->SetCancelAfterMs(GetCancelAfter().MilliSeconds());
- ev->Record.MutableRequest()->SetTimeoutMs(GetOperationTimeout().MilliSeconds());
-
- auto& script = req->script();
- NYql::TIssues issues;
- if (!CheckQuery(script, issues)) {
- return Reply(Ydb::StatusIds::BAD_REQUEST, issues, ctx);
- }
-
- switch (req->mode()) {
+ const auto traceId = Request_->GetTraceId();
+
+ auto ev = MakeHolder<NKqp::TEvKqp::TEvQueryRequest>();
+ SetAuthToken(ev, *Request_);
+ SetDatabase(ev, *Request_);
+
+ if (traceId) {
+ ev->Record.SetTraceId(traceId.GetRef());
+ }
+
+ ev->Record.MutableRequest()->SetCancelAfterMs(GetCancelAfter().MilliSeconds());
+ ev->Record.MutableRequest()->SetTimeoutMs(GetOperationTimeout().MilliSeconds());
+
+ auto& script = req->script();
+ NYql::TIssues issues;
+ if (!CheckQuery(script, issues)) {
+ return Reply(Ydb::StatusIds::BAD_REQUEST, issues, ctx);
+ }
+
+ switch (req->mode()) {
// KIKIMR-10990
//case Ydb::Scripting::ExplainYqlRequest_Mode_PARSE:
// ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_PARSE);
// break;
- case Ydb::Scripting::ExplainYqlRequest_Mode_VALIDATE:
- ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_VALIDATE);
- break;
- case Ydb::Scripting::ExplainYqlRequest_Mode_PLAN:
- ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN);
- break;
- default:
- issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, TStringBuilder()
- << "Unknown explain mode"));
- return Reply(Ydb::StatusIds::BAD_REQUEST, issues, ctx);
- }
-
- ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_SCRIPT);
- ev->Record.MutableRequest()->SetQuery(script);
- ev->Record.MutableRequest()->SetKeepSession(false);
-
- ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release());
- }
-
- void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) {
- const auto& record = ev->Get()->Record.GetRef();
+ case Ydb::Scripting::ExplainYqlRequest_Mode_VALIDATE:
+ ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_VALIDATE);
+ break;
+ case Ydb::Scripting::ExplainYqlRequest_Mode_PLAN:
+ ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN);
+ break;
+ default:
+ issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, TStringBuilder()
+ << "Unknown explain mode"));
+ return Reply(Ydb::StatusIds::BAD_REQUEST, issues, ctx);
+ }
+
+ ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_SCRIPT);
+ ev->Record.MutableRequest()->SetQuery(script);
+ ev->Record.MutableRequest()->SetKeepSession(false);
+
+ ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release());
+ }
+
+ void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) {
+ const auto& record = ev->Get()->Record.GetRef();
AddServerHintsIfAny(record);
- if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) {
- const auto& kqpResponse = record.GetResponse();
- const auto& issueMessage = kqpResponse.GetQueryIssues();
+ if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) {
+ const auto& kqpResponse = record.GetResponse();
+ const auto& issueMessage = kqpResponse.GetQueryIssues();
const auto& queryParameters = kqpResponse.GetQueryParameters();
-
- Ydb::Scripting::ExplainYqlResult queryResult;
- queryResult.set_plan(kqpResponse.GetQueryPlan());
+
+ Ydb::Scripting::ExplainYqlResult queryResult;
+ queryResult.set_plan(kqpResponse.GetQueryPlan());
for (const auto& queryParameter : queryParameters) {
Ydb::Type parameterType;
ConvertMiniKQLTypeToYdbType(queryParameter.GetType(), parameterType);
queryResult.mutable_parameters_types()->insert({ queryParameter.GetName(), parameterType });
}
-
- ReplyWithResult(Ydb::StatusIds::SUCCESS, issueMessage, queryResult, ctx);
- } else {
- return OnGenericQueryResponseError(record, ctx);
- }
- }
-};
-
+
+ ReplyWithResult(Ydb::StatusIds::SUCCESS, issueMessage, queryResult, ctx);
+ } else {
+ return OnGenericQueryResponseError(record, ctx);
+ }
+ }
+};
+
void DoExplainYqlScript(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider&) {
TActivationContext::AsActorContext().Register(new TExplainYqlScriptRPC(p.release()));
-}
-
-} // namespace NGRpcService
-} // namespace NKikimr
+}
+
+} // namespace NGRpcService
+} // namespace NKikimr
diff --git a/ydb/core/grpc_services/rpc_kqp_base.cpp b/ydb/core/grpc_services/rpc_kqp_base.cpp
index 414543f5740..18a7fcc65bf 100644
--- a/ydb/core/grpc_services/rpc_kqp_base.cpp
+++ b/ydb/core/grpc_services/rpc_kqp_base.cpp
@@ -1,13 +1,13 @@
#include "rpc_kqp_base.h"
#include <ydb/core/kqp/prepare/kqp_query_plan.h>
-
+
namespace NKikimr {
namespace NGRpcService {
-void FillQueryStats(Ydb::TableStats::QueryStats& queryStats, const NKikimrKqp::TQueryResponse& kqpResponse) {
- const auto& kqpStats = kqpResponse.GetQueryStats();
-
+void FillQueryStats(Ydb::TableStats::QueryStats& queryStats, const NKikimrKqp::TQueryResponse& kqpResponse) {
+ const auto& kqpStats = kqpResponse.GetQueryStats();
+
uint64_t totalCpuTimeUs = 0;
uint64_t totalDurationUs = 0;
@@ -65,8 +65,8 @@ void FillQueryStats(Ydb::TableStats::QueryStats& queryStats, const NKikimrKqp::T
queryStats.set_process_cpu_time_us(kqpStats.GetWorkerCpuTimeUs());
queryStats.set_total_cpu_time_us(totalCpuTimeUs);
queryStats.set_total_duration_us(totalDurationUs);
-
- queryStats.set_query_plan(kqpResponse.GetQueryPlan());
+
+ queryStats.set_query_plan(kqpResponse.GetQueryPlan());
}
} // namespace NGRpcService
diff --git a/ydb/core/grpc_services/rpc_kqp_base.h b/ydb/core/grpc_services/rpc_kqp_base.h
index 2811a49e78a..acb1edfd931 100644
--- a/ydb/core/grpc_services/rpc_kqp_base.h
+++ b/ydb/core/grpc_services/rpc_kqp_base.h
@@ -45,7 +45,7 @@ inline NYql::NDqProto::EDqStatsMode GetKqpStatsMode(Ydb::Table::QueryStatsCollec
switch (mode) {
case Ydb::Table::QueryStatsCollection::STATS_COLLECTION_BASIC:
return NYql::NDqProto::DQ_STATS_MODE_BASIC;
- case Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL:
+ case Ydb::Table::QueryStatsCollection::STATS_COLLECTION_FULL:
return NYql::NDqProto::DQ_STATS_MODE_PROFILE;
default:
return NYql::NDqProto::DQ_STATS_MODE_NONE;
@@ -70,7 +70,7 @@ inline bool CheckQuery(const TString& query, NYql::TIssues& issues) {
return true;
}
-void FillQueryStats(Ydb::TableStats::QueryStats& queryStats, const NKikimrKqp::TQueryResponse& kqpResponse);
+void FillQueryStats(Ydb::TableStats::QueryStats& queryStats, const NKikimrKqp::TQueryResponse& kqpResponse);
inline void ConvertKqpQueryResultToDbResult(const NKikimrMiniKQL::TResult& from, Ydb::ResultSet* to) {
const auto& type = from.GetType();
diff --git a/ydb/core/grpc_services/rpc_read_columns.cpp b/ydb/core/grpc_services/rpc_read_columns.cpp
index 4baae606a97..bc57f76f887 100644
--- a/ydb/core/grpc_services/rpc_read_columns.cpp
+++ b/ydb/core/grpc_services/rpc_read_columns.cpp
@@ -429,10 +429,10 @@ private:
}
void Handle(NKqp::TEvKqpCompute::TEvScanError::TPtr& ev, const TActorContext& ctx) {
- NYql::TIssues issues;
- Ydb::StatusIds::StatusCode status = ev->Get()->Record.GetStatus();
- NYql::IssuesFromMessage(ev->Get()->Record.GetIssues(), issues);
-
+ NYql::TIssues issues;
+ Ydb::StatusIds::StatusCode status = ev->Get()->Record.GetStatus();
+ NYql::IssuesFromMessage(ev->Get()->Record.GetIssues(), issues);
+
ReplyWithError(status, issues, ctx);
}
diff --git a/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp b/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp
index 5f95f263c6f..8daa2413094 100644
--- a/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp
+++ b/ydb/core/grpc_services/rpc_stream_execute_scan_query.cpp
@@ -139,12 +139,12 @@ bool FillKqpRequest(const Ydb::Experimental::ExecuteStreamQueryRequest& req, NKi
return false;
}
- if (req.explain()) {
- kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN);
- } else {
- kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
- }
-
+ if (req.explain()) {
+ kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN);
+ } else {
+ kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
+ }
+
kqpRequest.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_SCAN);
kqpRequest.MutableRequest()->SetQuery(query);
kqpRequest.MutableRequest()->SetKeepSession(false);
@@ -174,21 +174,21 @@ bool FillKqpRequest(const Ydb::Table::ExecuteScanQueryRequest& req, NKikimrKqp::
return false;
}
- switch (req.mode()) {
- case Ydb::Table::ExecuteScanQueryRequest::MODE_EXEC:
- kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
- kqpRequest.MutableRequest()->SetStatsMode(GetKqpStatsMode(req.collect_stats()));
- break;
- case Ydb::Table::ExecuteScanQueryRequest::MODE_EXPLAIN:
- kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN);
- break;
- default: {
- NYql::TIssues issues;
- issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Unexpected query mode"));
- error = TParseRequestError(Ydb::StatusIds::BAD_REQUEST, issues);
- return false;
- }
- }
+ switch (req.mode()) {
+ case Ydb::Table::ExecuteScanQueryRequest::MODE_EXEC:
+ kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE);
+ kqpRequest.MutableRequest()->SetStatsMode(GetKqpStatsMode(req.collect_stats()));
+ break;
+ case Ydb::Table::ExecuteScanQueryRequest::MODE_EXPLAIN:
+ kqpRequest.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN);
+ break;
+ default: {
+ NYql::TIssues issues;
+ issues.AddIssue(MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, "Unexpected query mode"));
+ error = TParseRequestError(Ydb::StatusIds::BAD_REQUEST, issues);
+ return false;
+ }
+ }
kqpRequest.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_SCAN);
kqpRequest.MutableRequest()->SetKeepSession(false);
@@ -371,19 +371,19 @@ private:
}
void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) {
- auto& record = ev->Get()->Record.GetRef();
+ auto& record = ev->Get()->Record.GetRef();
NYql::TIssues issues;
- const auto& issueMessage = record.GetResponse().GetQueryIssues();
+ const auto& issueMessage = record.GetResponse().GetQueryIssues();
NYql::IssuesFromMessage(issueMessage, issues);
-
- if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) {
- TResponse response;
- TString out;
- auto& kqpResponse = record.GetResponse();
- response.set_status(Ydb::StatusIds::SUCCESS);
-
- if constexpr (std::is_same_v<TResponse, Ydb::Table::ExecuteScanQueryPartialResponse>) {
+
+ if (record.GetYdbStatus() == Ydb::StatusIds::SUCCESS) {
+ TResponse response;
+ TString out;
+ auto& kqpResponse = record.GetResponse();
+ response.set_status(Ydb::StatusIds::SUCCESS);
+
+ if constexpr (std::is_same_v<TResponse, Ydb::Table::ExecuteScanQueryPartialResponse>) {
bool reportStats = NeedReportStats(*Request_->GetProtoRequest());
bool reportPlan = reportStats && NeedReportPlan(*Request_->GetProtoRequest());
@@ -401,33 +401,33 @@ private:
ExecutionProfiles_.clear();
} else if (reportPlan) {
response.mutable_result()->mutable_query_stats()->set_query_plan(kqpResponse.GetQueryPlan());
- }
-
+ }
+
if (reportPlan) {
response.mutable_result()->mutable_query_stats()->set_query_ast(kqpResponse.GetQueryAst());
}
Y_PROTOBUF_SUPPRESS_NODISCARD response.SerializeToString(&out);
Request_->SendSerializedResult(std::move(out), record.GetYdbStatus());
- }
- } else {
- if (kqpResponse.HasQueryStats()) {
- NKqpProto::TKqpStatsQuery queryStats;
- for (const auto& execStats: ExecutionProfiles_) {
- /* copy as ExecutionProfiles_ vector will be used and cleared later */
- queryStats.AddExecutions()->CopyFrom(*execStats);
- }
- response.mutable_result()->set_query_plan(SerializeAnalyzePlan(queryStats));
- } else {
- response.mutable_result()->set_query_plan(kqpResponse.GetQueryPlan());
- }
+ }
+ } else {
+ if (kqpResponse.HasQueryStats()) {
+ NKqpProto::TKqpStatsQuery queryStats;
+ for (const auto& execStats: ExecutionProfiles_) {
+ /* copy as ExecutionProfiles_ vector will be used and cleared later */
+ queryStats.AddExecutions()->CopyFrom(*execStats);
+ }
+ response.mutable_result()->set_query_plan(SerializeAnalyzePlan(queryStats));
+ } else {
+ response.mutable_result()->set_query_plan(kqpResponse.GetQueryPlan());
+ }
Y_PROTOBUF_SUPPRESS_NODISCARD response.SerializeToString(&out);
Request_->SendSerializedResult(std::move(out), record.GetYdbStatus());
- }
- }
-
- ReplyFinishStream(record.GetYdbStatus(), issues, ctx);
+ }
+ }
+
+ ReplyFinishStream(record.GetYdbStatus(), issues, ctx);
}
void Handle(NKqp::TEvKqp::TEvProcessResponse::TPtr& ev, const TActorContext& ctx) {
diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make
index 05b156bf043..cf0b96748ef 100644
--- a/ydb/core/grpc_services/ya.make
+++ b/ydb/core/grpc_services/ya.make
@@ -38,7 +38,7 @@ SRCS(
rpc_execute_data_query.cpp
rpc_execute_scheme_query.cpp
rpc_execute_yql_script.cpp
- rpc_explain_yql_script.cpp
+ rpc_explain_yql_script.cpp
rpc_explain_data_query.cpp
rpc_forget_operation.cpp
rpc_get_operation.cpp
diff --git a/ydb/core/kqp/common/kqp_gateway.h b/ydb/core/kqp/common/kqp_gateway.h
index 1e06beca393..81e8eaef909 100644
--- a/ydb/core/kqp/common/kqp_gateway.h
+++ b/ydb/core/kqp/common/kqp_gateway.h
@@ -68,42 +68,42 @@ public:
, Params(std::move(params)) {}
};
- struct TKqpSnapshot {
- ui64 Step;
- ui64 TxId;
-
- constexpr TKqpSnapshot()
- : Step(0)
- , TxId(0)
- {}
-
- TKqpSnapshot(ui64 step, ui64 txId)
- : Step(step)
- , TxId(txId)
- {}
-
- bool IsValid() const {
- return Step != 0 || TxId != 0;
- }
-
- bool operator ==(const TKqpSnapshot &snapshot) const {
- return snapshot.Step == Step && snapshot.TxId == TxId;
- }
-
- size_t GetHash() const noexcept {
- auto tuple = std::make_tuple(Step, TxId);
- return THash<decltype(tuple)>()(tuple);
- }
-
- static const TKqpSnapshot InvalidSnapshot;
- };
-
- struct TKqpSnapshotHandle : public IKqpGateway::TGenericResult {
- TKqpSnapshot Snapshot;
- NActors::TActorId ManagingActor;
+ struct TKqpSnapshot {
+ ui64 Step;
+ ui64 TxId;
+
+ constexpr TKqpSnapshot()
+ : Step(0)
+ , TxId(0)
+ {}
+
+ TKqpSnapshot(ui64 step, ui64 txId)
+ : Step(step)
+ , TxId(txId)
+ {}
+
+ bool IsValid() const {
+ return Step != 0 || TxId != 0;
+ }
+
+ bool operator ==(const TKqpSnapshot &snapshot) const {
+ return snapshot.Step == Step && snapshot.TxId == TxId;
+ }
+
+ size_t GetHash() const noexcept {
+ auto tuple = std::make_tuple(Step, TxId);
+ return THash<decltype(tuple)>()(tuple);
+ }
+
+ static const TKqpSnapshot InvalidSnapshot;
+ };
+
+ struct TKqpSnapshotHandle : public IKqpGateway::TGenericResult {
+ TKqpSnapshot Snapshot;
+ NActors::TActorId ManagingActor;
NKikimrIssues::TStatusIds::EStatusCode Status = NKikimrIssues::TStatusIds::UNKNOWN;
- };
-
+ };
+
struct TExecPhysicalRequest : private TMoveOnly {
TVector<TPhysicalTxData> Transactions;
TVector<NYql::NDq::TMkqlValueRef> Locks;
@@ -143,12 +143,12 @@ public:
virtual NThreading::TFuture<TMkqlResult> PrepareMkql(const TString& cluster, const TString& program) = 0;
- /* Snapshots */
- virtual NThreading::TFuture<TKqpSnapshotHandle> CreatePersistentSnapshot(const TVector<TString>& tablePaths,
- TDuration queryTimeout) = 0;
-
- virtual void DiscardPersistentSnapshot(const TKqpSnapshotHandle& handle) = 0;
-
+ /* Snapshots */
+ virtual NThreading::TFuture<TKqpSnapshotHandle> CreatePersistentSnapshot(const TVector<TString>& tablePaths,
+ TDuration queryTimeout) = 0;
+
+ virtual void DiscardPersistentSnapshot(const TKqpSnapshotHandle& handle) = 0;
+
virtual NThreading::TFuture<TKqpSnapshotHandle> AcquireMvccSnapshot(TDuration queryTimeout) = 0;
/* Physical */
@@ -168,8 +168,8 @@ public:
TKqpParamsMap&& params, const TAstQuerySettings& settings,
const Ydb::Table::TransactionSettings& txSettings) = 0;
- virtual NThreading::TFuture<TQueryResult> ExplainScanQueryAst(const TString& cluster, const TString& query) = 0;
-
+ virtual NThreading::TFuture<TQueryResult> ExplainScanQueryAst(const TString& cluster, const TString& query) = 0;
+
virtual NThreading::TFuture<TQueryResult> ExecScanQueryAst(const TString& cluster, const TString& query,
TKqpParamsMap&& params, const TAstQuerySettings& settings, ui64 rowsLimit) = 0;
@@ -186,10 +186,10 @@ public:
} // namespace NKqp
} // namespace NKikimr
-
-template<>
-struct THash<NKikimr::NKqp::IKqpGateway::TKqpSnapshot> {
- inline size_t operator()(const NKikimr::NKqp::IKqpGateway::TKqpSnapshot& snapshot) const {
- return snapshot.GetHash();
- }
-};
+
+template<>
+struct THash<NKikimr::NKqp::IKqpGateway::TKqpSnapshot> {
+ inline size_t operator()(const NKikimr::NKqp::IKqpGateway::TKqpSnapshot& snapshot) const {
+ return snapshot.GetHash();
+ }
+};
diff --git a/ydb/core/kqp/common/kqp_transform.h b/ydb/core/kqp/common/kqp_transform.h
index 457c4248011..53e8d415348 100644
--- a/ydb/core/kqp/common/kqp_transform.h
+++ b/ydb/core/kqp/common/kqp_transform.h
@@ -213,7 +213,7 @@ public:
DeferredEffects.Clear();
ParamsState = MakeIntrusive<TParamsState>();
- SnapshotHandle.Snapshot = IKqpGateway::TKqpSnapshot::InvalidSnapshot;
+ SnapshotHandle.Snapshot = IKqpGateway::TKqpSnapshot::InvalidSnapshot;
ForceNewEngineSettings = {};
}
@@ -255,7 +255,7 @@ public:
TDeferredEffects DeferredEffects;
TIntrusivePtr<TParamsState> ParamsState;
- IKqpGateway::TKqpSnapshotHandle SnapshotHandle;
+ IKqpGateway::TKqpSnapshotHandle SnapshotHandle;
TKqpForceNewEngineState ForceNewEngineSettings;
};
diff --git a/ydb/core/kqp/compile/kqp_mkql_compiler.cpp b/ydb/core/kqp/compile/kqp_mkql_compiler.cpp
index 04130446986..e3ce04acc57 100644
--- a/ydb/core/kqp/compile/kqp_mkql_compiler.cpp
+++ b/ydb/core/kqp/compile/kqp_mkql_compiler.cpp
@@ -209,23 +209,23 @@ const TKikimrTableMetadata& TKqlCompileContext::GetTableMeta(const TKqpTable& ta
TIntrusivePtr<IMkqlCallableCompiler> CreateKqlCompiler(const TKqlCompileContext& ctx) {
auto compiler = MakeIntrusive<NCommon::TMkqlCommonCallableCompiler>();
- compiler->AddCallable(TKqpWideReadTable::CallableName(),
- [&ctx](const TExprNode& node, TMkqlBuildContext& buildCtx) {
- TKqpWideReadTable readTable(&node);
-
- const auto& tableMeta = ctx.GetTableMeta(readTable.Table());
+ compiler->AddCallable(TKqpWideReadTable::CallableName(),
+ [&ctx](const TExprNode& node, TMkqlBuildContext& buildCtx) {
+ TKqpWideReadTable readTable(&node);
+
+ const auto& tableMeta = ctx.GetTableMeta(readTable.Table());
auto keyRange = MakeKeyRange(readTable, ctx, buildCtx);
-
+
auto result = ctx.PgmBuilder().KqpWideReadTable(MakeTableId(readTable.Table()), keyRange,
GetKqpColumns(tableMeta, readTable.Columns(), true));
-
+
return result;
});
-
+
compiler->AddCallable(TKqpWideReadTableRanges::CallableName(),
[&ctx](const TExprNode& node, TMkqlBuildContext& buildCtx) {
TKqpWideReadTableRanges readTableRanges(&node);
-
+
const auto& tableMeta = ctx.GetTableMeta(readTableRanges.Table());
ValidateRangesType(readTableRanges.Ranges().Ref().GetTypeAnn(), tableMeta);
@@ -247,12 +247,12 @@ TIntrusivePtr<IMkqlCallableCompiler> CreateKqlCompiler(const TKqlCompileContext&
ValidateRangesType(readTable.Ranges().Ref().GetTypeAnn(), tableMeta);
TKqpKeyRanges ranges = MakeComputedKeyRanges(readTable, ctx, buildCtx);
-
+
// Return type depends on the process program, so it is built explicitly.
TStringStream errorStream;
auto returnType = NCommon::BuildType(*readTable.Ref().GetTypeAnn(), ctx.PgmBuilder(), errorStream);
YQL_ENSURE(returnType, "Failed to build type: " << errorStream.Str());
-
+
// Process program for OLAP read is not present in MKQL, it is passed in range description
// in physical plan directly to executer. Read callables in MKQL only used to associate
// input stream of the graph with the external scans, so it doesn't make much sense to pass
@@ -265,10 +265,10 @@ TIntrusivePtr<IMkqlCallableCompiler> CreateKqlCompiler(const TKqlCompileContext&
GetKqpColumns(tableMeta, readTable.Columns(), true),
returnType
);
-
- return result;
- });
-
+
+ return result;
+ });
+
compiler->AddCallable(TKqpLookupTable::CallableName(),
[&ctx](const TExprNode& node, TMkqlBuildContext& buildCtx) {
TKqpLookupTable lookupTable(&node);
diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp
index f8e3cd0c772..ac3b3ec59d7 100644
--- a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp
+++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp
@@ -740,8 +740,8 @@ private:
}
void HandleExecute(TEvTxProxySchemeCache::TEvInvalidateTableResult::TPtr&) {
- }
-
+ }
+
void HandleExecute(TEvents::TEvUndelivered::TPtr& ev) {
switch (ev->Get()->SourceType) {
case TEvDataShard::TEvKqpScan::EventType:
@@ -807,7 +807,7 @@ private:
void SendStartScanRequest(TShardState& state, ui32 gen) {
YQL_ENSURE(state.State == EShardState::Starting);
- auto ev = MakeHolder<TEvDataShard::TEvKqpScan>();
+ auto ev = MakeHolder<TEvDataShard::TEvKqpScan>();
ev->Record.SetLocalPathId(ScanData->TableId.PathId.LocalPathId);
for (auto& column: ScanData->GetColumns()) {
ev->Record.AddColumnTags(column.Tag);
@@ -1012,7 +1012,7 @@ private:
CA_LOG(prio, "Table: " << ScanData->TablePath << ", scan has not been started yet");
}
}
-
+
void PassAway() override {
Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvUnlink(0));
for (ui32 nodeId : TrackingNodes) {
diff --git a/ydb/core/kqp/counters/kqp_counters.cpp b/ydb/core/kqp/counters/kqp_counters.cpp
index 26a28a0249a..cb120cd2de9 100644
--- a/ydb/core/kqp/counters/kqp_counters.cpp
+++ b/ydb/core/kqp/counters/kqp_counters.cpp
@@ -66,8 +66,8 @@ void TKqpCountersBase::Init() {
KqpGroup->GetCounter("Requests/QueryCommitTx", true);
QueryActionRequests[NKikimrKqp::QUERY_ACTION_ROLLBACK_TX] =
KqpGroup->GetCounter("Requests/QueryRollbackTx", true);
- QueryActionRequests[NKikimrKqp::QUERY_ACTION_PARSE] =
- KqpGroup->GetCounter("Requests/QueryParse", true);
+ QueryActionRequests[NKikimrKqp::QUERY_ACTION_PARSE] =
+ KqpGroup->GetCounter("Requests/QueryParse", true);
OtherQueryRequests = KqpGroup->GetCounter("Requests/QueryOther", true);
CloseSessionRequests = KqpGroup->GetCounter("Requests/CloseSession", true);
@@ -84,26 +84,26 @@ void TKqpCountersBase::Init() {
SqlV1Translations = KqpGroup->GetCounter("Requests/Sql/V1", true);
SqlUnknownTranslations = KqpGroup->GetCounter("Requests/Sql/Unknown", true);
- QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_UNDEFINED] =
- KqpGroup->GetCounter("Request/QueryTypeUndefined", true);
- QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DML] =
- KqpGroup->GetCounter("Request/QueryTypeDml", true);
- QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DDL] =
- KqpGroup->GetCounter("Request/QueryTypeDdl", true);
- QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_PREPARED_DML] =
- KqpGroup->GetCounter("Request/QueryTypePreparedDml", true);
- QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_DML] =
- KqpGroup->GetCounter("Request/QueryTypeAstDml", true);
- QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT] =
- KqpGroup->GetCounter("Request/QueryTypeSqlScript", true);
+ QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_UNDEFINED] =
+ KqpGroup->GetCounter("Request/QueryTypeUndefined", true);
+ QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DML] =
+ KqpGroup->GetCounter("Request/QueryTypeDml", true);
+ QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_DDL] =
+ KqpGroup->GetCounter("Request/QueryTypeDdl", true);
+ QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_PREPARED_DML] =
+ KqpGroup->GetCounter("Request/QueryTypePreparedDml", true);
+ QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_DML] =
+ KqpGroup->GetCounter("Request/QueryTypeAstDml", true);
+ QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT] =
+ KqpGroup->GetCounter("Request/QueryTypeSqlScript", true);
QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCRIPT_STREAMING] =
KqpGroup->GetCounter("Request/QueryTypeSqlScriptStreaming", true);
- QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN] =
- KqpGroup->GetCounter("Request/QueryTypeSqlScan", true);
- QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_SCAN] =
- KqpGroup->GetCounter("Request/QueryTypeAstScan", true);
- OtherQueryTypes = KqpGroup->GetCounter("Requests/QueryTypeOther", true);
-
+ QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN] =
+ KqpGroup->GetCounter("Request/QueryTypeSqlScan", true);
+ QueryTypes[NKikimrKqp::EQueryType::QUERY_TYPE_AST_SCAN] =
+ KqpGroup->GetCounter("Request/QueryTypeAstScan", true);
+ OtherQueryTypes = KqpGroup->GetCounter("Requests/QueryTypeOther", true);
+
QueriesWithRangeScan = KqpGroup->GetCounter("Query/WithRangeScan", true);
QueriesWithFullScan = KqpGroup->GetCounter("Query/WithFullScan", true);
@@ -242,15 +242,15 @@ void TKqpCountersBase::ReportQueryAction(NKikimrKqp::EQueryAction action) {
}
void TKqpCountersBase::ReportQueryType(NKikimrKqp::EQueryType type) {
- auto counter = QueryTypes.FindPtr(type);
- if (counter) {
- (*counter)->Inc();
- return;
- }
-
- OtherQueryTypes->Inc();
-}
-
+ auto counter = QueryTypes.FindPtr(type);
+ if (counter) {
+ (*counter)->Inc();
+ return;
+ }
+
+ OtherQueryTypes->Inc();
+}
+
void TKqpCountersBase::ReportSessionShutdownRequest() {
SessionBalancerShutdowns->Inc();
}
diff --git a/ydb/core/kqp/counters/kqp_counters.h b/ydb/core/kqp/counters/kqp_counters.h
index 53a8f7d2c23..4f9047adfeb 100644
--- a/ydb/core/kqp/counters/kqp_counters.h
+++ b/ydb/core/kqp/counters/kqp_counters.h
@@ -35,7 +35,7 @@ protected:
void Init();
void ReportQueryAction(NKikimrKqp::EQueryAction action);
- void ReportQueryType(NKikimrKqp::EQueryType type);
+ void ReportQueryType(NKikimrKqp::EQueryType type);
void ReportSessionShutdownRequest();
void ReportCreateSession(ui64 requestSize);
@@ -116,9 +116,9 @@ protected:
NMonitoring::TDynamicCounters::TCounterPtr SqlV1Translations;
NMonitoring::TDynamicCounters::TCounterPtr SqlUnknownTranslations;
- THashMap<NKikimrKqp::EQueryType, NMonitoring::TDynamicCounters::TCounterPtr> QueryTypes;
- NMonitoring::TDynamicCounters::TCounterPtr OtherQueryTypes;
-
+ THashMap<NKikimrKqp::EQueryType, NMonitoring::TDynamicCounters::TCounterPtr> QueryTypes;
+ NMonitoring::TDynamicCounters::TCounterPtr OtherQueryTypes;
+
NMonitoring::TDynamicCounters::TCounterPtr QueriesWithRangeScan;
NMonitoring::TDynamicCounters::TCounterPtr QueriesWithFullScan;
NMonitoring::THistogramPtr QueryAffectedShardsCount;
diff --git a/ydb/core/kqp/executer/kqp_executer_stats.cpp b/ydb/core/kqp/executer/kqp_executer_stats.cpp
index 8c3605fd918..c40af5abb34 100644
--- a/ydb/core/kqp/executer/kqp_executer_stats.cpp
+++ b/ydb/core/kqp/executer/kqp_executer_stats.cpp
@@ -168,7 +168,7 @@ void TQueryExecutionStats::AddDatashardStats(NYql::NDqProto::TDqComputeActorStat
tableAggr->SetWriteRows(tableAggr->GetWriteRows() + table.GetWriteRows());
tableAggr->SetWriteBytes(tableAggr->GetWriteBytes() + table.GetWriteBytes());
tableAggr->SetEraseRows(tableAggr->GetEraseRows() + table.GetEraseRows());
-
+
auto& shards = TableShards[table.GetTablePath()];
for (const auto& perShard : txStats.GetPerShardStats()) {
shards.insert(perShard.GetShardId());
diff --git a/ydb/core/kqp/executer/kqp_planner.cpp b/ydb/core/kqp/executer/kqp_planner.cpp
index 6bcfa85c66f..a5bdce84632 100644
--- a/ydb/core/kqp/executer/kqp_planner.cpp
+++ b/ydb/core/kqp/executer/kqp_planner.cpp
@@ -1,12 +1,12 @@
#include "kqp_planner.h"
#include "kqp_planner_strategy.h"
-#include "kqp_shards_resolver.h"
+#include "kqp_shards_resolver.h"
#include <ydb/core/base/appdata.h>
#include <ydb/core/kqp/rm/kqp_rm.h>
#include <ydb/core/kqp/rm/kqp_resource_estimation.h>
-#include <util/generic/set.h>
+#include <util/generic/set.h>
namespace NKikimr::NKqp {
@@ -28,8 +28,8 @@ TKqpPlanner::TKqpPlanner(ui64 txId, const TActorId& executer, TVector<NDqProto::
: TxId(txId)
, ExecuterId(executer)
, Tasks(std::move(tasks))
- , ScanTasks(std::move(scanTasks))
- , Snapshot(snapshot)
+ , ScanTasks(std::move(scanTasks))
+ , Snapshot(snapshot)
, Database(database)
, UserToken(userToken)
, Deadline(deadline)
@@ -161,12 +161,12 @@ void TKqpPlanner::Process(const TVector<NKikimrKqp::TKqpNodeResources>& snapshot
TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(),
CalcSendMessageFlagsForNode(target.NodeId())));
}
-
- TVector<ui64> nodes;
+
+ TVector<ui64> nodes;
nodes.reserve(ScanTasks.size());
for (auto& [nodeId, _]: ScanTasks) {
nodes.push_back(nodeId);
- }
+ }
for (ui64 nodeId: nodes) {
auto ev = PrepareKqpNodeRequest({});
@@ -175,7 +175,7 @@ void TKqpPlanner::Process(const TVector<NKikimrKqp::TKqpNodeResources>& snapshot
auto target = MakeKqpNodeServiceID(nodeId);
TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(),
CalcSendMessageFlagsForNode(target.NodeId())));
- }
+ }
Y_VERIFY(ScanTasks.empty());
} else {
auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(Ydb::StatusIds::PRECONDITION_FAILED,
@@ -198,12 +198,12 @@ void TKqpPlanner::RunLocal(const TVector<NKikimrKqp::TKqpNodeResources>& snapsho
auto target = MakeKqpNodeServiceID(SelfId().NodeId());
TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(), IEventHandle::FlagTrackDelivery));
-
- TVector<ui64> nodes;
- for (const auto& pair: ScanTasks) {
- nodes.push_back(pair.first);
+
+ TVector<ui64> nodes;
+ for (const auto& pair: ScanTasks) {
+ nodes.push_back(pair.first);
YQL_ENSURE(pair.first != SelfId().NodeId());
- }
+ }
THashMap<ui64, size_t> nodeIdToIdx;
for (size_t idx = 0; idx < snapshot.size(); ++idx) {
@@ -211,15 +211,15 @@ void TKqpPlanner::RunLocal(const TVector<NKikimrKqp::TKqpNodeResources>& snapsho
LOG_D("snapshot #" << idx << ": " << snapshot[idx].ShortDebugString());
}
- for (auto nodeId: nodes) {
+ for (auto nodeId: nodes) {
auto ev = PrepareKqpNodeRequest({});
AddScansToKqpNodeRequest(ev, nodeId);
auto target = MakeKqpNodeServiceID(nodeId);
TlsActivationContext->Send(new IEventHandle(target, ExecuterId, ev.Release(),
CalcSendMessageFlagsForNode(target.NodeId())));
- }
- Y_VERIFY(ScanTasks.size() == 0);
-
+ }
+ Y_VERIFY(ScanTasks.size() == 0);
+
PassAway();
}
@@ -321,7 +321,7 @@ IActor* CreateKqpPlanner(ui64 txId, const TActorId& executer, TVector<NDqProto::
const TString& database, const TMaybe<TString>& token, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode,
bool disableLlvmForUdfStages, bool enableLlvm, bool withSpilling, const TMaybe<NKikimrKqp::TRlPath>& rlPath)
{
- return new TKqpPlanner(txId, executer, std::move(tasks), std::move(scanTasks), snapshot,
+ return new TKqpPlanner(txId, executer, std::move(tasks), std::move(scanTasks), snapshot,
database, token, deadline, statsMode, disableLlvmForUdfStages, enableLlvm, withSpilling, rlPath);
}
diff --git a/ydb/core/kqp/executer/kqp_planner.h b/ydb/core/kqp/executer/kqp_planner.h
index dc5e993a74d..418cdf28ba0 100644
--- a/ydb/core/kqp/executer/kqp_planner.h
+++ b/ydb/core/kqp/executer/kqp_planner.h
@@ -31,8 +31,8 @@ class TKqpPlanner : public TActorBootstrapped<TKqpPlanner> {
};
public:
- TKqpPlanner(ui64 txId, const TActorId& executer, TVector<NYql::NDqProto::TDqTask>&& tasks,
- THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot,
+ TKqpPlanner(ui64 txId, const TActorId& executer, TVector<NYql::NDqProto::TDqTask>&& tasks,
+ THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot,
const TString& database, const TMaybe<TString>& userToken, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode,
bool disableLlvmForUdfStages, bool enableLlvm, bool withSpilling, const TMaybe<NKikimrKqp::TRlPath>& rlPath);
@@ -58,8 +58,8 @@ private:
const ui64 TxId;
const TActorId ExecuterId;
TVector<NYql::NDqProto::TDqTask> Tasks;
- THashMap<ui64, TVector<NYql::NDqProto::TDqTask>> ScanTasks;
- const IKqpGateway::TKqpSnapshot Snapshot;
+ THashMap<ui64, TVector<NYql::NDqProto::TDqTask>> ScanTasks;
+ const IKqpGateway::TKqpSnapshot Snapshot;
TString Database;
const TMaybe<TString> UserToken;
const TInstant Deadline;
@@ -72,7 +72,7 @@ private:
};
IActor* CreateKqpPlanner(ui64 txId, const TActorId& executer, TVector<NYql::NDqProto::TDqTask>&& tasks,
- THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot,
+ THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks, const IKqpGateway::TKqpSnapshot& snapshot,
const TString& database, const TMaybe<TString>& userToken, TInstant deadline, const NYql::NDqProto::EDqStatsMode& statsMode,
bool disableLlvmForUdfStages, bool enableLlvm, bool withSpilling, const TMaybe<NKikimrKqp::TRlPath>& rlPath);
diff --git a/ydb/core/kqp/executer/kqp_result_channel.cpp b/ydb/core/kqp/executer/kqp_result_channel.cpp
index 0fe235323e3..90416bfd22d 100644
--- a/ydb/core/kqp/executer/kqp_result_channel.cpp
+++ b/ydb/core/kqp/executer/kqp_result_channel.cpp
@@ -74,7 +74,7 @@ private:
LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId
<< ", send ack to channelId: " << ChannelId
<< ", seqNo: " << seqNo
- << ", enough: " << ev->Get()->Record.GetEnough()
+ << ", enough: " << ev->Get()->Record.GetEnough()
<< ", freeSpace: " << freeSpace
<< ", to: " << ComputeActor);
@@ -82,7 +82,7 @@ private:
ackEv->Record.SetSeqNo(seqNo);
ackEv->Record.SetChannelId(ChannelId);
ackEv->Record.SetFreeSpace(freeSpace);
- ackEv->Record.SetFinish(ev->Get()->Record.GetEnough());
+ ackEv->Record.SetFinish(ev->Get()->Record.GetEnough());
Send(ComputeActor, ackEv.Release(), /* TODO: undelivery */ 0, /* cookie */ ChannelId);
}
diff --git a/ydb/core/kqp/executer/kqp_scan_executer.cpp b/ydb/core/kqp/executer/kqp_scan_executer.cpp
index 07c29f64e21..d1f62a5d6d9 100644
--- a/ydb/core/kqp/executer/kqp_scan_executer.cpp
+++ b/ydb/core/kqp/executer/kqp_scan_executer.cpp
@@ -255,7 +255,7 @@ private:
void HandleExecute(TEvKqpExecuter::TEvStreamDataAck::TPtr& ev) {
LOG_T("Recv stream data ack, seqNo: " << ev->Get()->Record.GetSeqNo()
<< ", freeSpace: " << ev->Get()->Record.GetFreeSpace()
- << ", enough: " << ev->Get()->Record.GetEnough()
+ << ", enough: " << ev->Get()->Record.GetEnough()
<< ", from: " << ev->Sender);
if (ResultChannelProxies.empty()) {
@@ -401,43 +401,43 @@ private:
}
};
- void BuildDatashardScanTasks(TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory,
- const NMiniKQL::TTypeEnvironment& typeEnv)
- {
+ void BuildDatashardScanTasks(TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory,
+ const NMiniKQL::TTypeEnvironment& typeEnv)
+ {
Y_VERIFY_DEBUG(stageInfo.Meta.IsDatashard());
THashMap<ui64, TVector<ui64>> nodeTasks; // nodeId -> [taskId]
-
- auto getNodeTask = [&](ui64 nodeId, ui32 taskIdx) -> TTask& {
- auto& tasks = nodeTasks[nodeId];
-
- if (taskIdx < tasks.size()) {
- return TasksGraph.GetTask(tasks[taskIdx]);
- } else {
- Y_ASSERT(taskIdx == tasks.size());
- auto& task = TasksGraph.AddTask(stageInfo);
- task.Meta.NodeId = nodeId;
- tasks.emplace_back(task.Id);
- return task;
- }
- };
-
- auto& stage = GetStage(stageInfo);
-
- const auto& table = TableKeys.GetTable(stageInfo.Meta.TableId);
+
+ auto getNodeTask = [&](ui64 nodeId, ui32 taskIdx) -> TTask& {
+ auto& tasks = nodeTasks[nodeId];
+
+ if (taskIdx < tasks.size()) {
+ return TasksGraph.GetTask(tasks[taskIdx]);
+ } else {
+ Y_ASSERT(taskIdx == tasks.size());
+ auto& task = TasksGraph.AddTask(stageInfo);
+ task.Meta.NodeId = nodeId;
+ tasks.emplace_back(task.Id);
+ return task;
+ }
+ };
+
+ auto& stage = GetStage(stageInfo);
+
+ const auto& table = TableKeys.GetTable(stageInfo.Meta.TableId);
const auto& keyTypes = table.KeyColumnTypes;
-
- for (auto& op : stage.GetTableOps()) {
- Y_VERIFY_DEBUG(stageInfo.Meta.TablePath == op.GetTable().GetPath());
-
+
+ for (auto& op : stage.GetTableOps()) {
+ Y_VERIFY_DEBUG(stageInfo.Meta.TablePath == op.GetTable().GetPath());
+
auto columns = BuildKqpColumns(op, table);
THashMap<ui64, TShardInfo> partitions;
-
- switch (op.GetTypeCase()) {
+
+ switch (op.GetTypeCase()) {
case NKqpProto::TKqpPhyTableOperation::kReadRanges:
partitions = PrunePartitions(TableKeys, op.GetReadRanges(), stageInfo, holderFactory, typeEnv);
break;
- case NKqpProto::TKqpPhyTableOperation::kReadRange:
+ case NKqpProto::TKqpPhyTableOperation::kReadRange:
partitions = PrunePartitions(TableKeys, op.GetReadRange(), stageInfo, holderFactory, typeEnv);
break;
case NKqpProto::TKqpPhyTableOperation::kLookup:
@@ -447,7 +447,7 @@ private:
YQL_ENSURE(false, "Unexpected table scan operation: " << (ui32) op.GetTypeCase());
break;
}
-
+
bool reverse = false;
ui64 itemsLimit = 0;
TString itemsLimitParamName;
@@ -465,20 +465,20 @@ private:
stageInfo.Meta.SkipNullKeys.assign(op.GetReadRange().GetSkipNullKeys().begin(),
op.GetReadRange().GetSkipNullKeys().end());
}
-
+
// TODO: take into account number of active scans on node
bool heavyProgram = stage.GetProgram().GetSettings().GetHasSort() ||
stage.GetProgram().GetSettings().GetHasMapJoin();
const ui32 maxScansPerNode = heavyProgram ? 4 : 16;
THashMap<ui64, ui32> nTasksOnNodes; // nodeId -> tasks count
-
+
for (auto& [shardId, shardInfo] : partitions) {
YQL_ENSURE(!shardInfo.KeyWriteRanges);
-
+
ui64 nodeId = ShardIdToNodeId.at(shardId);
ui32 nTasksOnNode = nTasksOnNodes[nodeId]++;
auto& task = getNodeTask(nodeId, nTasksOnNode % maxScansPerNode);
-
+
for (auto& [name, value] : shardInfo.Params) {
auto ret = task.Meta.Params.emplace(name, std::move(value));
YQL_ENSURE(ret.second);
@@ -487,65 +487,65 @@ private:
auto retType = task.Meta.ParamTypes.emplace(name, typeIterator->second);
YQL_ENSURE(retType.second);
}
-
+
TTaskMeta::TShardReadInfo readInfo = {
.Ranges = std::move(*shardInfo.KeyReadRanges), // sorted & non-intersecting
.Columns = columns,
.ShardId = shardId,
};
-
+
if (itemsLimit && !task.Meta.Params.contains(itemsLimitParamName)) {
task.Meta.Params.emplace(itemsLimitParamName, itemsLimitBytes);
task.Meta.ParamTypes.emplace(itemsLimitParamName, itemsLimitType);
- }
-
+ }
+
FillReadInfo(task.Meta, itemsLimit, reverse, TMaybe<::NKqpProto::TKqpPhyOpReadOlapRanges>());
if (!task.Meta.Reads) {
task.Meta.Reads.ConstructInPlace();
- }
+ }
task.Meta.Reads->emplace_back(std::move(readInfo));
- }
- }
-
- LOG_D("Stage " << stageInfo.Id << " will be executed on " << nodeTasks.size() << " nodes.");
-
- for (const auto& pair : nodeTasks) {
- for (const auto& taskIdx : pair.second) {
- auto& task = TasksGraph.GetTask(taskIdx);
- YQL_ENSURE(task.Meta.Reads.Defined());
- auto& taskReads = task.Meta.Reads.GetRef();
-
- /*
- * Sort read ranges so that sequential scan of that ranges produce sorted result.
- *
- * Partition pruner feed us with set of non-intersecting ranges with filled right boundary.
+ }
+ }
+
+ LOG_D("Stage " << stageInfo.Id << " will be executed on " << nodeTasks.size() << " nodes.");
+
+ for (const auto& pair : nodeTasks) {
+ for (const auto& taskIdx : pair.second) {
+ auto& task = TasksGraph.GetTask(taskIdx);
+ YQL_ENSURE(task.Meta.Reads.Defined());
+ auto& taskReads = task.Meta.Reads.GetRef();
+
+ /*
+ * Sort read ranges so that sequential scan of that ranges produce sorted result.
+ *
+ * Partition pruner feed us with set of non-intersecting ranges with filled right boundary.
* So we may sort ranges based solely on the their rightmost point.
- */
- std::sort(taskReads.begin(), taskReads.end(), [&](const auto& lhs, const auto& rhs){
+ */
+ std::sort(taskReads.begin(), taskReads.end(), [&](const auto& lhs, const auto& rhs){
YQL_ENSURE(lhs.ShardId != rhs.ShardId);
-
+
const std::pair<const TSerializedCellVec*, bool> k1 = lhs.Ranges.GetRightBorder();
const std::pair<const TSerializedCellVec*, bool> k2 = lhs.Ranges.GetRightBorder();
- const int cmp = CompareBorders<false, false>(
+ const int cmp = CompareBorders<false, false>(
k1.first->GetCells(),
k2.first->GetCells(),
k1.second,
k2.second,
- keyTypes);
-
- return (cmp < 0);
- });
-
- LOG_D("Stage " << stageInfo.Id << " create datashard scan task: " << taskIdx
- << ", node: " << pair.first
- << ", meta: " << task.Meta.ToString(keyTypes, *AppData()->TypeRegistry));
- }
- }
- }
-
+ keyTypes);
+
+ return (cmp < 0);
+ });
+
+ LOG_D("Stage " << stageInfo.Id << " create datashard scan task: " << taskIdx
+ << ", node: " << pair.first
+ << ", meta: " << task.Meta.ToString(keyTypes, *AppData()->TypeRegistry));
+ }
+ }
+ }
+
// Returns the list of ColumnShards that can store rows from the specified range
// NOTE: Unlike OLTP tables that store data in DataShards, data in OLAP tables is not range
// partitioned and multiple ColumnShards store data from the same key range
@@ -755,10 +755,10 @@ private:
return;
}
- // NodeId -> {Tasks}
- THashMap<ui64, TVector<NYql::NDqProto::TDqTask>> scanTasks;
- ui32 nShardScans = 0;
- ui32 nScanTasks = 0;
+ // NodeId -> {Tasks}
+ THashMap<ui64, TVector<NYql::NDqProto::TDqTask>> scanTasks;
+ ui32 nShardScans = 0;
+ ui32 nScanTasks = 0;
TVector<NYql::NDqProto::TDqTask> computeTasks;
@@ -873,10 +873,10 @@ private:
if (stageInfo.Meta.IsSysView()) {
computeTasks.emplace_back(std::move(taskDesc));
- } else {
+ } else {
scanTasks[task.Meta.NodeId].emplace_back(std::move(taskDesc));
nScanTasks++;
- }
+ }
} else {
computeTasks.emplace_back(std::move(taskDesc));
}
@@ -900,25 +900,25 @@ private:
<< ", " << nScanTasks << " scan tasks on " << scanTasks.size() << " nodes"
<< ", totalShardScans: " << nShardScans << ", execType: Scan"
<< ", snapshot: {" << Request.Snapshot.TxId << ", " << Request.Snapshot.Step << "}");
-
+
ExecuteScanTx(std::move(computeTasks), std::move(scanTasks));
-
+
Become(&TKqpScanExecuter::ExecuteState);
- }
-
- void ExecuteScanTx(TVector<NYql::NDqProto::TDqTask>&& computeTasks, THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks) {
+ }
+
+ void ExecuteScanTx(TVector<NYql::NDqProto::TDqTask>&& computeTasks, THashMap<ui64, TVector<NYql::NDqProto::TDqTask>>&& scanTasks) {
LOG_D("Execute scan tx, computeTasks: " << computeTasks.size() << ", scanTasks: " << scanTasks.size());
- for (const auto& [_, tasks]: scanTasks) {
- for (const auto& task : tasks) {
- PendingComputeTasks.insert(task.GetId());
+ for (const auto& [_, tasks]: scanTasks) {
+ for (const auto& task : tasks) {
+ PendingComputeTasks.insert(task.GetId());
}
}
- for (auto& taskDesc : computeTasks) {
- PendingComputeTasks.insert(taskDesc.GetId());
- }
-
- auto planner = CreateKqpPlanner(TxId, SelfId(), std::move(computeTasks),
+ for (auto& taskDesc : computeTasks) {
+ PendingComputeTasks.insert(taskDesc.GetId());
+ }
+
+ auto planner = CreateKqpPlanner(TxId, SelfId(), std::move(computeTasks),
std::move(scanTasks), Request.Snapshot,
Database, UserToken, Deadline.GetOrElse(TInstant::Zero()), Request.StatsMode,
Request.DisableLlvmForUdfStages, Request.LlvmEnabled, AppData()->EnableKqpSpilling, Request.RlPath);
@@ -950,12 +950,12 @@ private:
Stats->FinishTs = TInstant::Now();
Stats->Finish();
-
+
if (Request.StatsMode == NYql::NDqProto::DQ_STATS_MODE_PROFILE) {
const auto& tx = Request.Transactions[0].Body;
auto planWithStats = AddExecStatsToTxPlan(tx.GetPlan(), response.GetResult().GetStats());
response.MutableResult()->MutableStats()->AddTxPlansWithStats(planWithStats);
- }
+ }
}
LOG_D("Sending response to: " << Target);
diff --git a/ydb/core/kqp/executer/kqp_shards_resolver.cpp b/ydb/core/kqp/executer/kqp_shards_resolver.cpp
index 506dfcf5730..644f7628c1b 100644
--- a/ydb/core/kqp/executer/kqp_shards_resolver.cpp
+++ b/ydb/core/kqp/executer/kqp_shards_resolver.cpp
@@ -46,8 +46,8 @@ public:
auto tabletResolver = MakeTabletResolverID();
auto resolveFlags = GetResolveFlags();
- Y_ASSERT(ShardIds.size() > 0);
-
+ Y_ASSERT(ShardIds.size() > 0);
+
for (ui64 tabletId : ShardIds) {
LOG_T("Send request about tabletId: " << tabletId);
bool sent = Send(tabletResolver, new TEvTabletResolver::TEvForward(tabletId, nullptr, resolveFlags));
diff --git a/ydb/core/kqp/executer/kqp_tasks_graph.h b/ydb/core/kqp/executer/kqp_tasks_graph.h
index 7c356ab623f..ebe7647c1e9 100644
--- a/ydb/core/kqp/executer/kqp_tasks_graph.h
+++ b/ydb/core/kqp/executer/kqp_tasks_graph.h
@@ -95,11 +95,11 @@ struct TShardKeyRanges {
std::pair<const TSerializedCellVec*, bool> GetRightBorder() const;
};
-// TODO: use two different structs for scans and data queries
+// TODO: use two different structs for scans and data queries
struct TTaskMeta {
ui64 ShardId = 0; // only in case of non-scans (data-query & legacy scans)
ui64 NodeId = 0; // only in case of scans over persistent snapshots
-
+
TMap<TString, NYql::NDqProto::TData> Params;
TMap<TString, NKikimr::NMiniKQL::TType*> ParamTypes;
diff --git a/ydb/core/kqp/executer/ya.make b/ydb/core/kqp/executer/ya.make
index 3db082d5d0b..e1c4e8d12bf 100644
--- a/ydb/core/kqp/executer/ya.make
+++ b/ydb/core/kqp/executer/ya.make
@@ -15,7 +15,7 @@ SRCS(
kqp_partition_helper.cpp
kqp_planner.cpp
kqp_planner_strategy.cpp
- kqp_shards_resolver.cpp
+ kqp_shards_resolver.cpp
kqp_result_channel.cpp
kqp_table_resolver.cpp
kqp_tasks_graph.cpp
diff --git a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json
index 2faa33d5e1d..33030cc3003 100644
--- a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json
+++ b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json
@@ -71,11 +71,11 @@
"Match": {"Type": "Callable", "Name": "KqpReadTable"}
},
{
- "Name": "TKqpWideReadTable",
- "Base": "TKqlReadTableBase",
- "Match": {"Type": "Callable", "Name": "KqpWideReadTable"}
- },
- {
+ "Name": "TKqpWideReadTable",
+ "Base": "TKqlReadTableBase",
+ "Match": {"Type": "Callable", "Name": "KqpWideReadTable"}
+ },
+ {
"Name": "TKqlReadTableRangesBase",
"Base": "TCallable",
"Match": {"Type": "CallableBase"},
diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp
index 44af6d11682..bf24f0831b2 100644
--- a/ydb/core/kqp/host/kqp_host.cpp
+++ b/ydb/core/kqp/host/kqp_host.cpp
@@ -118,7 +118,7 @@ void FillAstAndPlan(IKqpHost::TQueryResult& queryResult, const NKikimrKqp::TPrep
}
}
-/*
+/*
* Validate YqlScript.
*/
class TAsyncValidateYqlResult : public TKqpAsyncResultBase<IKqpHost::TQueryResult> {
@@ -145,49 +145,49 @@ private:
};
/*
- * Explain Yql/YqlScript.
- */
+ * Explain Yql/YqlScript.
+ */
class TAsyncExplainYqlResult : public TKqpAsyncResultBase<IKqpHost::TQueryResult> {
public:
using TResult = IKqpHost::TQueryResult;
- TAsyncExplainYqlResult(TExprNode* queryRoot, TIntrusivePtr<TKikimrSessionContext> sessionCtx,
- TExprContext& exprCtx, TAutoPtr<IGraphTransformer> transformer,
+ TAsyncExplainYqlResult(TExprNode* queryRoot, TIntrusivePtr<TKikimrSessionContext> sessionCtx,
+ TExprContext& exprCtx, TAutoPtr<IGraphTransformer> transformer,
IPlanBuilder& planBuilder, TMaybe<TSqlVersion> sqlVersion, bool useDqExplain = false)
: TKqpAsyncResultBase(queryRoot, exprCtx, *transformer.Get())
- , SessionCtx(sessionCtx)
+ , SessionCtx(sessionCtx)
, Transformer(transformer)
, PlanBuilder(planBuilder)
- , SqlVersion(sqlVersion)
- , UseDqExplain(useDqExplain) {}
+ , SqlVersion(sqlVersion)
+ , UseDqExplain(useDqExplain) {}
void FillResult(TResult& queryResult) const override {
- if (UseDqExplain) {
- TVector<const TString> plans;
- for (auto id : SessionCtx->Query().ExecutionOrder) {
- auto result = SessionCtx->Query().Results.FindPtr(id);
- if (result) {
- plans.push_back(result->QueryPlan);
- }
- }
- queryResult.QueryPlan = SerializeScriptPlan(plans);
- } else {
- FillAstAndPlan(queryResult, GetExprRoot().Get(), GetExprContext(), PlanBuilder);
- }
+ if (UseDqExplain) {
+ TVector<const TString> plans;
+ for (auto id : SessionCtx->Query().ExecutionOrder) {
+ auto result = SessionCtx->Query().Results.FindPtr(id);
+ if (result) {
+ plans.push_back(result->QueryPlan);
+ }
+ }
+ queryResult.QueryPlan = SerializeScriptPlan(plans);
+ } else {
+ FillAstAndPlan(queryResult, GetExprRoot().Get(), GetExprContext(), PlanBuilder);
+ }
queryResult.SqlVersion = SqlVersion;
}
private:
- TIntrusivePtr<TKikimrSessionContext> SessionCtx;
+ TIntrusivePtr<TKikimrSessionContext> SessionCtx;
TAutoPtr<IGraphTransformer> Transformer;
IPlanBuilder& PlanBuilder;
TMaybe<TSqlVersion> SqlVersion;
- bool UseDqExplain;
+ bool UseDqExplain;
};
-/*
- * Execute Yql/SchemeQuery/YqlScript.
- */
+/*
+ * Execute Yql/SchemeQuery/YqlScript.
+ */
class TAsyncExecuteYqlResult : public TKqpAsyncExecuteResultBase<IKqpHost::TQueryResult> {
public:
using TResult = IKqpHost::TQueryResult;
@@ -215,18 +215,18 @@ public:
}
}
- TVector<const TString> queryPlans;
+ TVector<const TString> queryPlans;
for (auto id : SessionCtx->Query().ExecutionOrder) {
auto result = SessionCtx->Query().Results.FindPtr(id);
if (result) {
- queryPlans.push_back(SerializeAnalyzePlan(result->QueryStats));
+ queryPlans.push_back(SerializeAnalyzePlan(result->QueryStats));
AddQueryStats(queryResult.QueryStats, std::move(result->QueryStats));
}
}
FillAstAndPlan(queryResult, GetExprRoot().Get(), GetExprContext(), PlanBuilder);
queryResult.SqlVersion = SqlVersion;
- queryResult.QueryPlan = SerializeScriptPlan(queryPlans);
+ queryResult.QueryPlan = SerializeScriptPlan(queryPlans);
}
private:
@@ -237,9 +237,9 @@ private:
TMaybe<TSqlVersion> SqlVersion;
};
-/*
- * Execute prepared Yql/ScanQuery/DataQuery.
- */
+/*
+ * Execute prepared Yql/ScanQuery/DataQuery.
+ */
class TAsyncExecutePreparedResult : public TKqpAsyncExecuteResultBase<IKqpHost::TQueryResult> {
public:
using TResult = IKqpHost::TQueryResult;
@@ -283,14 +283,14 @@ public:
queryResult.PreparedQuery = Query;
}
- /*
- * Set stats and plan for DataQuery. In case of ScanQuery they will be set
- * later in TStreamExecuteScanQueryRPC.
- */
+ /*
+ * Set stats and plan for DataQuery. In case of ScanQuery they will be set
+ * later in TStreamExecuteScanQueryRPC.
+ */
if (ExecuteCtx->QueryResults.size() == 1) {
auto& execResult = ExecuteCtx->QueryResults[0];
queryResult.QueryStats.Swap(&execResult.QueryStats);
- queryResult.QueryPlan = SerializeAnalyzePlan(queryResult.QueryStats);
+ queryResult.QueryPlan = SerializeAnalyzePlan(queryResult.QueryStats);
}
queryResult.SqlVersion = SqlVersion;
@@ -304,9 +304,9 @@ private:
TMaybe<TSqlVersion> SqlVersion;
};
-/*
+/*
* Prepare ScanQuery/DataQuery by AST (when called through scripting).
- */
+ */
class TAsyncExecuteKqlResult : public TKqpAsyncExecuteResultBase<IKqpHost::TQueryResult> {
public:
using TResult = IKqpHost::TQueryResult;
@@ -328,9 +328,9 @@ private:
TExecuteContext& ExecuteCtx;
};
-/*
- * Prepare ScanQuery/DataQuery.
- */
+/*
+ * Prepare ScanQuery/DataQuery.
+ */
class TAsyncPrepareYqlResult : public TKqpAsyncResultBase<IKqpHost::TQueryResult> {
public:
using TResult = IKqpHost::TQueryResult;
@@ -349,7 +349,7 @@ public:
prepareResult.PreparingQuery = std::move(QueryCtx->PreparingQuery);
prepareResult.PreparingQuery->SetText(std::move(QueryText));
prepareResult.SqlVersion = SqlVersion;
-
+
if (prepareResult.PreparingQuery->GetVersion() == NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1) {
prepareResult.QueryPlan = SerializeExplainPlan(prepareResult.PreparingQuery->GetPhysicalQuery());
prepareResult.QueryAst = prepareResult.PreparingQuery->GetPhysicalQuery().GetQueryAst();
@@ -818,9 +818,9 @@ public:
return KqpRunner->PrepareDataQuery(cluster, query, ctx, settings);
case EKikimrQueryType::Scan:
return KqpRunner->PrepareScanQuery(cluster, query, ctx, settings);
- case EKikimrQueryType::YqlScript:
+ case EKikimrQueryType::YqlScript:
case EKikimrQueryType::YqlScriptStreaming:
- break;
+ break;
default:
YQL_ENSURE(false, "Unexpected query type for prepare action: " << queryType);
return nullptr;
@@ -856,7 +856,7 @@ public:
future = Gateway->ExecScanQueryAst(Cluster, queryAstStr, CollectParameters(query),
querySettings, rowsLimit);
}
- } else {
+ } else {
Ydb::Table::TransactionSettings txSettings;
txSettings.mutable_serializable_read_write();
if (SessionCtx->Query().PrepareOnly) {
@@ -865,19 +865,19 @@ public:
future = Gateway->ExecDataQueryAst(Cluster, queryAstStr, CollectParameters(query),
querySettings, txSettings);
}
- }
+ }
break;
case EKikimrQueryType::YqlScriptStreaming:
if (useScanQuery) {
future = Gateway->StreamExecScanQueryAst(Cluster, queryAstStr, CollectParameters(query),
querySettings, SessionCtx->Query().ReplyTarget);
- } else {
+ } else {
Ydb::Table::TransactionSettings txSettings;
txSettings.mutable_serializable_read_write();
future = Gateway->StreamExecDataQueryAst(Cluster, queryAstStr, CollectParameters(query),
querySettings, txSettings, SessionCtx->Query().ReplyTarget);
- }
+ }
break;
default:
@@ -1250,13 +1250,13 @@ public:
});
}
- IAsyncQueryResultPtr ExplainScanQuery(const TString& query, bool isSql) override {
- return CheckedProcessQuery(*ExprCtx,
- [this, &query, isSql] (TExprContext& ctx) {
- return ExplainScanQueryInternal(query, isSql, ctx);
- });
- }
-
+ IAsyncQueryResultPtr ExplainScanQuery(const TString& query, bool isSql) override {
+ return CheckedProcessQuery(*ExprCtx,
+ [this, &query, isSql] (TExprContext& ctx) {
+ return ExplainScanQueryInternal(query, isSql, ctx);
+ });
+ }
+
TQueryResult SyncExplainDataQuery(const TString& query, bool isSql) override {
return CheckedSyncProcessQuery(
[this, &query, isSql] () {
@@ -1742,21 +1742,21 @@ private:
if (prepared.PreparingQuery->GetVersion() == NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1) {
explainResult.QueryPlan = std::move(prepared.QueryPlan);
explainResult.QueryAst = std::move(*prepared.PreparingQuery->MutablePhysicalQuery()->MutableQueryAst());
- } else {
+ } else {
FillAstAndPlan(explainResult, *prepared.PreparingQuery);
- }
+ }
explainResult.SqlVersion = prepared.SqlVersion;
return MakeKikimrResultHolder(std::move(explainResult));
});
}
- IAsyncQueryResultPtr ExplainScanQueryInternal(const TString& query, bool isSql, TExprContext& ctx) {
- auto prepareResult = isSql
- ? PrepareScanQueryInternal(query, ctx)
- : PrepareScanQueryAstInternal(query, ctx);
+ IAsyncQueryResultPtr ExplainScanQueryInternal(const TString& query, bool isSql, TExprContext& ctx) {
+ auto prepareResult = isSql
+ ? PrepareScanQueryInternal(query, ctx)
+ : PrepareScanQueryAstInternal(query, ctx);
return prepareResult;
- }
-
+ }
+
IAsyncQueryResultPtr PrepareDataQueryInternal(const TString& query, const TPrepareSettings& settings,
TExprContext& ctx)
{
@@ -1812,12 +1812,12 @@ private:
SessionCtx, *ExecuteCtx);
}
- IAsyncQueryResultPtr PrepareScanQueryInternal(const TString& query, TExprContext& ctx, EKikimrStatsMode statsMode = EKikimrStatsMode::None) {
+ IAsyncQueryResultPtr PrepareScanQueryInternal(const TString& query, TExprContext& ctx, EKikimrStatsMode statsMode = EKikimrStatsMode::None) {
SetupYqlTransformer(nullptr);
SessionCtx->Query().Type = EKikimrQueryType::Scan;
SessionCtx->Query().PrepareOnly = true;
- SessionCtx->Query().StatsMode = statsMode;
+ SessionCtx->Query().StatsMode = statsMode;
SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
TMaybe<TSqlVersion> sqlVersion = 1;
@@ -2045,21 +2045,21 @@ private:
}
IAsyncQueryResultPtr ExplainYqlScriptInternal(const TString& script, TExprContext& ctx) {
- SetupYqlTransformer(nullptr);
-
- SessionCtx->Query().Type = EKikimrQueryType::YqlScript;
- SessionCtx->Query().PrepareOnly = true;
+ SetupYqlTransformer(nullptr);
+
+ SessionCtx->Query().Type = EKikimrQueryType::YqlScript;
+ SessionCtx->Query().PrepareOnly = true;
SessionCtx->Query().SuppressDdlChecks = true;
SessionCtx->Query().PreparingQuery = std::make_unique<NKikimrKqp::TPreparedQuery>();
-
- TMaybe<TSqlVersion> sqlVersion;
- auto scriptExpr = CompileYqlQuery(script, true, true, ctx, sqlVersion);
- if (!scriptExpr) {
- return nullptr;
- }
-
- return MakeIntrusive<TAsyncExplainYqlResult>(scriptExpr.Get(), SessionCtx, ctx, YqlTransformer,
- *PlanBuilder, sqlVersion, true /* UseDqExplain */);
+
+ TMaybe<TSqlVersion> sqlVersion;
+ auto scriptExpr = CompileYqlQuery(script, true, true, ctx, sqlVersion);
+ if (!scriptExpr) {
+ return nullptr;
+ }
+
+ return MakeIntrusive<TAsyncExplainYqlResult>(scriptExpr.Get(), SessionCtx, ctx, YqlTransformer,
+ *PlanBuilder, sqlVersion, true /* UseDqExplain */);
}
IAsyncQueryResultPtr ExecuteScanQueryInternal(const TString& query, bool isSql,
@@ -2067,7 +2067,7 @@ private:
const IKikimrQueryExecutor::TExecuteSettings& settings, TExprContext& ctx)
{
auto prepareResult = isSql
- ? PrepareScanQueryInternal(query, ctx, settings.StatsMode)
+ ? PrepareScanQueryInternal(query, ctx, settings.StatsMode)
: PrepareScanQueryAstInternal(query, ctx);
if (!prepareResult) {
diff --git a/ydb/core/kqp/host/kqp_host.h b/ydb/core/kqp/host/kqp_host.h
index 11282d0ebe1..381031e7b4c 100644
--- a/ydb/core/kqp/host/kqp_host.h
+++ b/ydb/core/kqp/host/kqp_host.h
@@ -98,8 +98,8 @@ public:
NKikimrMiniKQL::TParams&& parameters, const NActors::TActorId& target,
const NYql::IKikimrQueryExecutor::TExecuteSettings& settings) = 0;
- virtual IAsyncQueryResultPtr ExplainScanQuery(const TString& query, bool isSql) = 0;
-
+ virtual IAsyncQueryResultPtr ExplainScanQuery(const TString& query, bool isSql) = 0;
+
/* Scripting */
virtual IAsyncQueryResultPtr ValidateYqlScript(const TString& script) = 0;
virtual TQueryResult SyncValidateYqlScript(const TString& script) = 0;
diff --git a/ydb/core/kqp/host/kqp_host_impl.h b/ydb/core/kqp/host/kqp_host_impl.h
index fb1fb139cd5..1999cbdb7c8 100644
--- a/ydb/core/kqp/host/kqp_host_impl.h
+++ b/ydb/core/kqp/host/kqp_host_impl.h
@@ -292,11 +292,11 @@ TAutoPtr<NYql::IGraphTransformer> CreateKqpExecuteScanTransformer(TIntrusivePtr<
const TString& cluster, TIntrusivePtr<TKqpTransactionState> txState,
TIntrusivePtr<TKqlTransformContext> transformCtx);
-TAutoPtr<NYql::IGraphTransformer> CreateKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway,
+TAutoPtr<NYql::IGraphTransformer> CreateKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway,
TIntrusivePtr<TKqlTransformContext> transformCtx, TIntrusivePtr<TKqpTransactionState> txState);
-
-TAutoPtr<NYql::IGraphTransformer> CreateKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway,
- TIntrusivePtr<TKqpTransactionState> txState);
-
+
+TAutoPtr<NYql::IGraphTransformer> CreateKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway,
+ TIntrusivePtr<TKqpTransactionState> txState);
+
} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/host/kqp_run_scan.cpp b/ydb/core/kqp/host/kqp_run_scan.cpp
index 7c91ae82003..df88f621605 100644
--- a/ydb/core/kqp/host/kqp_run_scan.cpp
+++ b/ydb/core/kqp/host/kqp_run_scan.cpp
@@ -65,60 +65,60 @@ protected:
}
};
-class TKqpCreateSnapshotTransformer : public TGraphTransformerBase {
-public:
- TKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, TIntrusivePtr<TKqlTransformContext> transformCtx,
+class TKqpCreateSnapshotTransformer : public TGraphTransformerBase {
+public:
+ TKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, TIntrusivePtr<TKqlTransformContext> transformCtx,
TIntrusivePtr<TKqpTransactionState> txState)
- : Gateway(gateway)
- , TransformCtx(transformCtx)
- , TxState(txState)
- {}
-
- TStatus DoTransform(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext&) {
- output = input;
-
- THashSet<TString> tablesSet;
- for (const auto& phyTx: TransformCtx->PhysicalQuery->GetTransactions()) {
- for (const auto& stage: phyTx.GetStages()) {
- for (const auto& tableOp: stage.GetTableOps()) {
- tablesSet.insert(tableOp.GetTable().GetPath());
- }
- }
- }
- TVector<TString> tables(tablesSet.begin(), tablesSet.end());
-
- auto timeout = TransformCtx->QueryCtx->Deadlines.TimeoutAt - Gateway->GetCurrentTime();
- if (!timeout) {
- // TODO: Just cancel request.
- timeout = TDuration::MilliSeconds(1);
- }
- SnapshotFuture = Gateway->CreatePersistentSnapshot(tables, timeout);
-
- Promise = NewPromise();
- auto promise = Promise;
- SnapshotFuture.Apply([promise](const TFuture<IKqpGateway::TKqpSnapshotHandle> future) mutable {
- YQL_ENSURE(future.HasValue());
- promise.SetValue();
- });
-
- return TStatus::Async;
- }
-
- NThreading::TFuture<void> DoGetAsyncFuture(const NYql::TExprNode& ) {
- return Promise.GetFuture();
- }
-
- TStatus DoApplyAsyncChanges(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext& ctx) {
- output = input;
-
- auto handle = SnapshotFuture.ExtractValue();
-
- if (!handle.Snapshot.IsValid()) {
+ : Gateway(gateway)
+ , TransformCtx(transformCtx)
+ , TxState(txState)
+ {}
+
+ TStatus DoTransform(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext&) {
+ output = input;
+
+ THashSet<TString> tablesSet;
+ for (const auto& phyTx: TransformCtx->PhysicalQuery->GetTransactions()) {
+ for (const auto& stage: phyTx.GetStages()) {
+ for (const auto& tableOp: stage.GetTableOps()) {
+ tablesSet.insert(tableOp.GetTable().GetPath());
+ }
+ }
+ }
+ TVector<TString> tables(tablesSet.begin(), tablesSet.end());
+
+ auto timeout = TransformCtx->QueryCtx->Deadlines.TimeoutAt - Gateway->GetCurrentTime();
+ if (!timeout) {
+ // TODO: Just cancel request.
+ timeout = TDuration::MilliSeconds(1);
+ }
+ SnapshotFuture = Gateway->CreatePersistentSnapshot(tables, timeout);
+
+ Promise = NewPromise();
+ auto promise = Promise;
+ SnapshotFuture.Apply([promise](const TFuture<IKqpGateway::TKqpSnapshotHandle> future) mutable {
+ YQL_ENSURE(future.HasValue());
+ promise.SetValue();
+ });
+
+ return TStatus::Async;
+ }
+
+ NThreading::TFuture<void> DoGetAsyncFuture(const NYql::TExprNode& ) {
+ return Promise.GetFuture();
+ }
+
+ TStatus DoApplyAsyncChanges(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext& ctx) {
+ output = input;
+
+ auto handle = SnapshotFuture.ExtractValue();
+
+ if (!handle.Snapshot.IsValid()) {
YQL_CLOG(NOTICE, ProviderKqp) << "Failed to create persistent snapshot. "
<< "Status: " << NKikimrIssues::TStatusIds_EStatusCode_Name(handle.Status)
<< ", issues: " << handle.Issues().ToString();
- TIssue issue("Failed to create persistent snapshot");
+ TIssue issue("Failed to create persistent snapshot");
switch (handle.Status) {
case NKikimrIssues::TStatusIds::SCHEME_ERROR:
issue.SetCode(NYql::TIssuesIds::KIKIMR_SCHEME_ERROR, NYql::TSeverityIds::S_ERROR);
@@ -136,48 +136,48 @@ public:
}
for (const auto& subIssue: handle.Issues()) {
- issue.AddSubIssue(MakeIntrusive<TIssue>(subIssue));
- }
- ctx.AddError(issue);
- return TStatus::Error;
- }
-
- TxState->Tx().SnapshotHandle = handle;
- return TStatus::Ok;
- }
-
-private:
- TIntrusivePtr<IKqpGateway> Gateway;
- TIntrusivePtr<TKqlTransformContext> TransformCtx;
-
- NThreading::TFuture<IKqpGateway::TKqpSnapshotHandle> SnapshotFuture;
- NThreading::TPromise<void> Promise;
- TIntrusivePtr<TKqpTransactionState> TxState;
-};
-
-class TKqpReleaseSnapshotTransformer : public TSyncTransformerBase {
-public:
- TKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, TIntrusivePtr<TKqpTransactionState> txState)
- : Gateway(gateway)
- , TxState(txState)
- {}
-
- TStatus DoTransform(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext&) {
- output = input;
-
+ issue.AddSubIssue(MakeIntrusive<TIssue>(subIssue));
+ }
+ ctx.AddError(issue);
+ return TStatus::Error;
+ }
+
+ TxState->Tx().SnapshotHandle = handle;
+ return TStatus::Ok;
+ }
+
+private:
+ TIntrusivePtr<IKqpGateway> Gateway;
+ TIntrusivePtr<TKqlTransformContext> TransformCtx;
+
+ NThreading::TFuture<IKqpGateway::TKqpSnapshotHandle> SnapshotFuture;
+ NThreading::TPromise<void> Promise;
+ TIntrusivePtr<TKqpTransactionState> TxState;
+};
+
+class TKqpReleaseSnapshotTransformer : public TSyncTransformerBase {
+public:
+ TKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway, TIntrusivePtr<TKqpTransactionState> txState)
+ : Gateway(gateway)
+ , TxState(txState)
+ {}
+
+ TStatus DoTransform(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext&) {
+ output = input;
+
if (TxState->Tx().GetSnapshot().IsValid()) {
- Gateway->DiscardPersistentSnapshot(TxState->Tx().SnapshotHandle);
- }
-
- return TStatus::Ok;
- }
-
-private:
- TIntrusivePtr<IKqpGateway> Gateway;
- TIntrusivePtr<TKqpTransactionState> TxState;
-};
-
-
+ Gateway->DiscardPersistentSnapshot(TxState->Tx().SnapshotHandle);
+ }
+
+ return TStatus::Ok;
+ }
+
+private:
+ TIntrusivePtr<IKqpGateway> Gateway;
+ TIntrusivePtr<TKqpTransactionState> TxState;
+};
+
+
TAutoPtr<IGraphTransformer> CreateKqpExecuteScanTransformer(TIntrusivePtr<IKqpGateway> gateway,
const TString& cluster, TIntrusivePtr<TKqpTransactionState> txState,
TIntrusivePtr<TKqlTransformContext> transformCtx)
@@ -185,17 +185,17 @@ TAutoPtr<IGraphTransformer> CreateKqpExecuteScanTransformer(TIntrusivePtr<IKqpGa
return new TKqpExecuteScanTransformer(gateway, cluster, txState, transformCtx);
}
-TAutoPtr<IGraphTransformer> CreateKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway,
+TAutoPtr<IGraphTransformer> CreateKqpCreateSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway,
TIntrusivePtr<TKqlTransformContext> transformCtx, TIntrusivePtr<TKqpTransactionState> txState)
-{
+{
return new TKqpCreateSnapshotTransformer(gateway, transformCtx, txState);
-}
-
-TAutoPtr<IGraphTransformer> CreateKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway,
- TIntrusivePtr<TKqpTransactionState> txState)
-{
- return new TKqpReleaseSnapshotTransformer(gateway, txState);
-}
-
+}
+
+TAutoPtr<IGraphTransformer> CreateKqpReleaseSnapshotTransformer(TIntrusivePtr<IKqpGateway> gateway,
+ TIntrusivePtr<TKqpTransactionState> txState)
+{
+ return new TKqpReleaseSnapshotTransformer(gateway, txState);
+}
+
} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/host/kqp_runner.cpp b/ydb/core/kqp/host/kqp_runner.cpp
index 42b4eca7c66..6e49eafe151 100644
--- a/ydb/core/kqp/host/kqp_runner.cpp
+++ b/ydb/core/kqp/host/kqp_runner.cpp
@@ -245,7 +245,7 @@ public:
ScanRunQueryTransformer = TTransformationPipeline(typesCtx)
.Add(CreateKqpCreateSnapshotTransformer(Gateway, TransformCtx, TxState), "CreateSnapshot")
.Add(CreateKqpExecuteScanTransformer(Gateway, Cluster, TxState, TransformCtx), "ExecuteScan")
- .Add(CreateKqpReleaseSnapshotTransformer(Gateway, TxState), "ReleaseSnapshot")
+ .Add(CreateKqpReleaseSnapshotTransformer(Gateway, TxState), "ReleaseSnapshot")
.Build(false);
}
@@ -553,15 +553,15 @@ private:
auto& preparedQuery = *TransformCtx->QueryCtx->PreparingQuery;
TKqpPhysicalQuery physicalQuery(transformedQuery);
- auto compiler = CreateKqpQueryCompiler(Cluster, OptimizeCtx->Tables, FuncRegistry);
+ auto compiler = CreateKqpQueryCompiler(Cluster, OptimizeCtx->Tables, FuncRegistry);
auto ret = compiler->CompilePhysicalQuery(physicalQuery, dataQuery.Operations(),
*preparedQuery.MutablePhysicalQuery(), ctx);
- if (!ret) {
- ctx.AddError(TIssue(ctx.GetPosition(query->Pos()), "Failed to compile physical query."));
+ if (!ret) {
+ ctx.AddError(TIssue(ctx.GetPosition(query->Pos()), "Failed to compile physical query."));
return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
}
- preparedQuery.SetVersion(NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1);
- // TODO(sk): only on stats mode or if explain-only
+ preparedQuery.SetVersion(NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1);
+ // TODO(sk): only on stats mode or if explain-only
PreparedExplainTransformer->Rewind();
return MakeIntrusive<TPhysicalAsyncRunResult>(builtQuery, ctx, *PreparedExplainTransformer, *TransformCtx);
}
@@ -599,7 +599,7 @@ private:
TransformCtx->ReplyTarget = target;
Y_ASSERT(!TxState->Tx().GetSnapshot().IsValid());
-
+
return MakeIntrusive<TScanAsyncRunResult>(world, ctx, *ScanRunQueryTransformer);
}
diff --git a/ydb/core/kqp/kqp_compile_actor.cpp b/ydb/core/kqp/kqp_compile_actor.cpp
index 27cf296a03d..20ef8b5e6cd 100644
--- a/ydb/core/kqp/kqp_compile_actor.cpp
+++ b/ydb/core/kqp/kqp_compile_actor.cpp
@@ -94,8 +94,8 @@ public:
ctx.SelfID.NodeId(), counters, MakeMiniKQLCompileServiceID());
Gateway->SetToken(Query.Cluster, UserToken);
- Config->FeatureFlags = AppData(ctx)->FeatureFlags;
-
+ Config->FeatureFlags = AppData(ctx)->FeatureFlags;
+
KqpHost = CreateKqpHost(Gateway, Query.Cluster, Query.Database, Config, ModuleResolverState->ModuleResolver,
AppData(ctx)->FunctionRegistry, false);
diff --git a/ydb/core/kqp/kqp_ic_gateway.cpp b/ydb/core/kqp/kqp_ic_gateway.cpp
index 47489a9bd54..14626ec9d3f 100644
--- a/ydb/core/kqp/kqp_ic_gateway.cpp
+++ b/ydb/core/kqp/kqp_ic_gateway.cpp
@@ -32,7 +32,7 @@
#include <library/cpp/actors/core/hfunc.h>
#include <util/string/split.h>
-#include <util/string/vector.h>
+#include <util/string/vector.h>
namespace NKikimr {
namespace NKqp {
@@ -46,8 +46,8 @@ using namespace NKikimrSchemeOp;
using NKikimrTxUserProxy::TMiniKQLTransaction;
-constexpr const IKqpGateway::TKqpSnapshot IKqpGateway::TKqpSnapshot::InvalidSnapshot = TKqpSnapshot();
-
+constexpr const IKqpGateway::TKqpSnapshot IKqpGateway::TKqpSnapshot::InvalidSnapshot = TKqpSnapshot();
+
#define STATIC_ASSERT_STATE_EQUAL(name) \
static_assert(static_cast<ui32>(NYql::TIndexDescription::EIndexState::name) \
== NKikimrSchemeOp::EIndexState::EIndexState##name, \
@@ -208,7 +208,7 @@ public:
}
auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>();
- resp->Record.SetEnough(truncated);
+ resp->Record.SetEnough(truncated);
resp->Record.SetSeqNo(ev->Get()->Record.GetSeqNo());
resp->Record.SetFreeSpace(ResultSetBytesLimit);
ctx.Send(ev->Sender, resp.Release());
@@ -1742,32 +1742,32 @@ public:
});
}
- TFuture<TQueryResult> ExplainScanQueryAst(const TString& cluster, const TString& query) override
- {
- using TRequest = NKqp::TEvKqp::TEvQueryRequest;
- using TResponse = NKqp::TEvKqp::TEvQueryResponse;
-
- auto ev = MakeHolder<TRequest>();
- if (UserToken) {
- ev->Record.SetUserToken(UserToken->Serialized);
- }
-
- ev->Record.MutableRequest()->SetDatabase(Database);
- ev->Record.MutableRequest()->SetCluster(cluster);
- ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN);
- ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_AST_SCAN);
- ev->Record.MutableRequest()->SetQuery(query);
- ev->Record.MutableRequest()->SetKeepSession(false);
-
- return SendKqpScanQueryRequest(ev.Release(), 100,
- [] (TPromise<TQueryResult> promise, TResponse&& responseEv) {
- TQueryResult queryResult;
- queryResult.ProtobufArenaPtr.reset(new google::protobuf::Arena());
- KqpResponseToQueryResult(responseEv.Record.GetRef(), queryResult);
+ TFuture<TQueryResult> ExplainScanQueryAst(const TString& cluster, const TString& query) override
+ {
+ using TRequest = NKqp::TEvKqp::TEvQueryRequest;
+ using TResponse = NKqp::TEvKqp::TEvQueryResponse;
+
+ auto ev = MakeHolder<TRequest>();
+ if (UserToken) {
+ ev->Record.SetUserToken(UserToken->Serialized);
+ }
+
+ ev->Record.MutableRequest()->SetDatabase(Database);
+ ev->Record.MutableRequest()->SetCluster(cluster);
+ ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXPLAIN);
+ ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_AST_SCAN);
+ ev->Record.MutableRequest()->SetQuery(query);
+ ev->Record.MutableRequest()->SetKeepSession(false);
+
+ return SendKqpScanQueryRequest(ev.Release(), 100,
+ [] (TPromise<TQueryResult> promise, TResponse&& responseEv) {
+ TQueryResult queryResult;
+ queryResult.ProtobufArenaPtr.reset(new google::protobuf::Arena());
+ KqpResponseToQueryResult(responseEv.Record.GetRef(), queryResult);
promise.SetValue(std::move(queryResult));
- });
- }
-
+ });
+ }
+
TFuture<TQueryResult> ExecDataQueryAst(const TString& cluster, const TString& query, TKqpParamsMap&& params,
const TAstQuerySettings& settings, const Ydb::Table::TransactionSettings& txSettings) override
{
@@ -1850,32 +1850,32 @@ public:
return ExecutePhysicalQueryInternal(std::move(request), target, true);
}
- TFuture<TKqpSnapshotHandle> CreatePersistentSnapshot(const TVector<TString>& tablePaths, TDuration queryTimeout) override {
+ TFuture<TKqpSnapshotHandle> CreatePersistentSnapshot(const TVector<TString>& tablePaths, TDuration queryTimeout) override {
auto* snapMgr = CreateKqpSnapshotManager(Database, queryTimeout);
- auto snapMgrActorId = RegisterActor(snapMgr);
-
+ auto snapMgrActorId = RegisterActor(snapMgr);
+
auto ev = MakeHolder<TEvKqpSnapshot::TEvCreateSnapshotRequest>(tablePaths);
-
- return SendActorRequest<
+
+ return SendActorRequest<
TEvKqpSnapshot::TEvCreateSnapshotRequest,
TEvKqpSnapshot::TEvCreateSnapshotResponse,
- IKqpGateway::TKqpSnapshotHandle>
- (
- snapMgrActorId,
- ev.Release(),
- [snapMgrActorId](TPromise<IKqpGateway::TKqpSnapshotHandle> promise,
+ IKqpGateway::TKqpSnapshotHandle>
+ (
+ snapMgrActorId,
+ ev.Release(),
+ [snapMgrActorId](TPromise<IKqpGateway::TKqpSnapshotHandle> promise,
TEvKqpSnapshot::TEvCreateSnapshotResponse&& response) mutable
- {
- IKqpGateway::TKqpSnapshotHandle handle;
- handle.Snapshot = response.Snapshot;
- handle.ManagingActor = snapMgrActorId;
+ {
+ IKqpGateway::TKqpSnapshotHandle handle;
+ handle.Snapshot = response.Snapshot;
+ handle.ManagingActor = snapMgrActorId;
handle.Status = response.Status;
handle.AddIssues(response.Issues);
- promise.SetValue(handle);
- }
- );
- }
-
+ promise.SetValue(handle);
+ }
+ );
+ }
+
NThreading::TFuture<TKqpSnapshotHandle> AcquireMvccSnapshot(TDuration queryTimeout) override {
auto* snapMgr = CreateKqpSnapshotManager(Database, queryTimeout);
auto snapMgrActorId = RegisterActor(snapMgr);
@@ -1901,11 +1901,11 @@ public:
);
}
- void DiscardPersistentSnapshot(const TKqpSnapshotHandle& handle) override {
+ void DiscardPersistentSnapshot(const TKqpSnapshotHandle& handle) override {
if (handle.ManagingActor)
ActorSystem->Send(handle.ManagingActor, new TEvKqpSnapshot::TEvDiscardSnapshot(handle.Snapshot));
- }
-
+ }
+
TInstant GetCurrentTime() const override {
return TAppData::TimeProvider->Now();
}
diff --git a/ydb/core/kqp/kqp_worker_actor.cpp b/ydb/core/kqp/kqp_worker_actor.cpp
index cccb8a51e8e..5cd0f533dae 100644
--- a/ydb/core/kqp/kqp_worker_actor.cpp
+++ b/ydb/core/kqp/kqp_worker_actor.cpp
@@ -185,8 +185,8 @@ public:
Gateway = CreateKikimrIcGateway(Settings.Cluster, Settings.Database, std::move(loader),
ctx.ExecutorThread.ActorSystem, ctx.SelfID.NodeId(), RequestCounters, MakeMiniKQLCompileServiceID());
- Config->FeatureFlags = AppData(ctx)->FeatureFlags;
-
+ Config->FeatureFlags = AppData(ctx)->FeatureFlags;
+
KqpHost = CreateKqpHost(Gateway, Settings.Cluster, Settings.Database, Config, ModuleResolverState->ModuleResolver,
AppData(ctx)->FunctionRegistry, !Settings.LongSession);
@@ -284,9 +284,9 @@ public:
QueryState->RequestActorId = ActorIdFromProto(event.GetRequestActorId());
if (GetStatsMode(queryRequest, EKikimrStatsMode::None) > EKikimrStatsMode::Basic) {
- QueryState->ReplyFlags |= NKikimrKqp::QUERY_REPLY_FLAG_AST;
- }
-
+ QueryState->ReplyFlags |= NKikimrKqp::QUERY_REPLY_FLAG_AST;
+ }
+
if (event.HasRlPath()) {
QueryState->RlPath = event.GetRlPath();
}
@@ -342,7 +342,7 @@ public:
switch (queryRequest.GetAction()) {
case NKikimrKqp::QUERY_ACTION_EXPLAIN:
- case NKikimrKqp::QUERY_ACTION_PARSE:
+ case NKikimrKqp::QUERY_ACTION_PARSE:
case NKikimrKqp::QUERY_ACTION_VALIDATE:
case NKikimrKqp::QUERY_ACTION_PREPARE:
if (QueryState->KeepSession && !Settings.LongSession) {
@@ -773,7 +773,7 @@ private:
switch (queryRequest.GetAction()) {
case NKikimrKqp::QUERY_ACTION_EXECUTE:
case NKikimrKqp::QUERY_ACTION_EXPLAIN:
- case NKikimrKqp::QUERY_ACTION_PARSE:
+ case NKikimrKqp::QUERY_ACTION_PARSE:
case NKikimrKqp::QUERY_ACTION_VALIDATE:
case NKikimrKqp::QUERY_ACTION_PREPARE:
case NKikimrKqp::QUERY_ACTION_EXECUTE_PREPARED:
@@ -1352,19 +1352,19 @@ private:
break;
}
- case NKikimrKqp::QUERY_TYPE_SQL_SCAN:
- case NKikimrKqp::QUERY_TYPE_AST_SCAN: {
- bool isSql = (type == NKikimrKqp::QUERY_TYPE_SQL_SCAN);
- QueryState->AsyncQueryResult = KqpHost->ExplainScanQuery(query, isSql);
- break;
- }
-
+ case NKikimrKqp::QUERY_TYPE_SQL_SCAN:
+ case NKikimrKqp::QUERY_TYPE_AST_SCAN: {
+ bool isSql = (type == NKikimrKqp::QUERY_TYPE_SQL_SCAN);
+ QueryState->AsyncQueryResult = KqpHost->ExplainScanQuery(query, isSql);
+ break;
+ }
+
case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT:
case NKikimrKqp::QUERY_TYPE_SQL_SCRIPT_STREAMING: {
- QueryState->AsyncQueryResult = KqpHost->ExplainYqlScript(query);
- break;
- }
-
+ QueryState->AsyncQueryResult = KqpHost->ExplainYqlScript(query);
+ break;
+ }
+
default:
QueryState->Error = "Unexpected query type.";
return false;
@@ -1729,7 +1729,7 @@ private:
FillQueryProfile(stats, *record.MutableResponse());
record.MutableResponse()->MutableQueryStats()->Swap(&stats);
- record.MutableResponse()->SetQueryPlan(queryResult.QueryPlan);
+ record.MutableResponse()->SetQueryPlan(queryResult.QueryPlan);
}
AddTrailingInfo(responseEv->Record.GetRef());
diff --git a/ydb/core/kqp/opt/kqp_opt_kql.cpp b/ydb/core/kqp/opt/kqp_opt_kql.cpp
index 1eeb95daee4..ab9a57d584d 100644
--- a/ydb/core/kqp/opt/kqp_opt_kql.cpp
+++ b/ydb/core/kqp/opt/kqp_opt_kql.cpp
@@ -546,7 +546,7 @@ TExprNode::TPtr HandleReadTable(const TKiReadTable& read, TExprContext& ctx, con
YQL_ENSURE(metadata, "unable to find metadata for index: " << indexName);
YQL_ENSURE(state == TIndexDescription::EIndexState::Ready
|| state == TIndexDescription::EIndexState::WriteOnly);
-
+
if (state != TIndexDescription::EIndexState::Ready) {
auto err = TStringBuilder()
<< "Requested index: " << indexName
diff --git a/ydb/core/kqp/prepare/kqp_query_plan.cpp b/ydb/core/kqp/prepare/kqp_query_plan.cpp
index 346a8b15cbe..fe361969f17 100644
--- a/ydb/core/kqp/prepare/kqp_query_plan.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_plan.cpp
@@ -10,12 +10,12 @@
#include <ydb/library/yql/dq/opt/dq_opt.h>
#include <ydb/library/yql/dq/tasks/dq_tasks_graph.h>
-#include <library/cpp/json/writer/json.h>
-#include <library/cpp/json/json_reader.h>
-
+#include <library/cpp/json/writer/json.h>
+#include <library/cpp/json/json_reader.h>
+
#include <util/generic/queue.h>
#include <util/string/strip.h>
-#include <util/string/vector.h>
+#include <util/string/vector.h>
#include <regex>
@@ -24,7 +24,7 @@ namespace NKqp {
using namespace NYql;
using namespace NYql::NNodes;
-using namespace NYql::NDq;
+using namespace NYql::NDq;
using namespace NClient;
namespace {
@@ -49,33 +49,33 @@ struct TTableInfo {
TVector<TTableWrite> Writes;
};
-struct TSerializerCtx {
- TSerializerCtx(TExprContext& exprCtx, const TString& cluster,
+struct TSerializerCtx {
+ TSerializerCtx(TExprContext& exprCtx, const TString& cluster,
const TIntrusivePtr<NYql::TKikimrTablesData> tablesData,
const TKikimrConfiguration::TPtr config,
THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> pureTxResults)
- : ExprCtx(exprCtx)
- , Cluster(cluster)
- , TablesData(tablesData)
+ : ExprCtx(exprCtx)
+ , Cluster(cluster)
+ , TablesData(tablesData)
, Config(config)
, PureTxResults(std::move(pureTxResults))
- {}
-
- TMap<TString, TTableInfo> Tables;
- TMap<TString, TExprNode::TPtr> BindNameToStage;
- TMap<TString, ui32> StageGuidToId;
+ {}
+
+ TMap<TString, TTableInfo> Tables;
+ TMap<TString, TExprNode::TPtr> BindNameToStage;
+ TMap<TString, ui32> StageGuidToId;
THashMap<ui32, TVector<std::pair<ui32, ui32>>> ParamBindings;
THashSet<ui32> PrecomputePhases;
- ui32 TxId = 0;
+ ui32 TxId = 0;
ui32 PlanNodeId = 0;
-
- const TExprContext& ExprCtx;
- const TString& Cluster;
- const TIntrusivePtr<NYql::TKikimrTablesData> TablesData;
+
+ const TExprContext& ExprCtx;
+ const TString& Cluster;
+ const TIntrusivePtr<NYql::TKikimrTablesData> TablesData;
const TKikimrConfiguration::TPtr Config;
THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> PureTxResults;
-};
-
+};
+
TString GetExprStr(const TExprBase& scalar) {
if (auto maybeData = scalar.Maybe<TCoDataCtor>()) {
auto literal = TString(maybeData.Cast().Literal());
@@ -285,15 +285,15 @@ void FillTablesInfo(const TExprNode::TPtr& query, TMap<TString, TTableInfo>& tab
class TxPlanSerializer {
public:
TxPlanSerializer(TSerializerCtx& serializerCtx, ui32 txId, const TKqpPhysicalTx& tx) : SerializerCtx(serializerCtx), TxId(txId), Tx(tx) {}
-
+
void Serialize() {
auto& phaseNode = QueryPlanNodes[++SerializerCtx.PlanNodeId];
phaseNode.TypeName = "Phase";
-
+
for (ui32 resId = 0; resId < Tx.Results().Size(); ++resId) {
auto res = Tx.Results().Item(resId);
auto& planNode = AddPlanNode(phaseNode);
-
+
if (SerializerCtx.PrecomputePhases.find(TxId) != SerializerCtx.PrecomputePhases.end()) {
planNode.TypeName = TStringBuilder() << "Precompute_" << TxId << "_" << resId;
planNode.CteName = TStringBuilder() << "tx_result_binding_" << TxId << "_" << resId;
@@ -301,7 +301,7 @@ public:
planNode.TypeName = TStringBuilder() << "ResultSet_" << TxId << "_" << resId;
planNode.Type = EPlanNodeType::ResultSet;
}
-
+
if (res.Maybe<TDqCnResult>()) {
Visit(res.Cast<TDqCnResult>().Output().Stage(), planNode);
} else if (res.Maybe<TDqCnValue>()) {
@@ -310,28 +310,28 @@ public:
Y_ENSURE(false, res.Ref().Content());
}
}
-
+
for (const auto& stage: Tx.Stages()) {
if (stage.Cast<TDqStageBase>().Program().Body().Maybe<TKqpEffects>()) {
auto &planNode = AddPlanNode(phaseNode);
planNode.TypeName = "Effect";
Visit(TExprBase(stage), planNode);
}
- }
+ }
/* set NodeId using reverse order */
for (auto& [id, planNode] : QueryPlanNodes) {
planNode.NodeId = SerializerCtx.PlanNodeId - id + 1;
}
}
-
+
void WriteToJson(NJsonWriter::TBuf& writer) const {
if (!QueryPlanNodes.empty()) {
auto phasePlanNode = QueryPlanNodes.begin();
WritePlanNodeToJson(phasePlanNode->second, writer);
}
- }
-
+ }
+
private:
struct TPredicate {
TVector<TString> Args;
@@ -367,20 +367,20 @@ private:
writer.WriteKey("PlanNodeId").WriteInt(planNode.NodeId);
writer.WriteKey("Node Type").WriteString(planNode.TypeName);
writer.WriteKey("StageGuid").WriteString(planNode.Guid);
-
+
if (auto type = GetPlanNodeType(planNode)) {
writer.WriteKey("PlanNodeType").WriteString(type);
}
-
+
if (planNode.CteName) {
writer.WriteKey("Parent Relationship").WriteString("InitPlan");
writer.WriteKey("Subplan Name").WriteString("CTE " + *planNode.CteName);
}
-
+
if (planNode.CteRefName) {
writer.WriteKey("CTE Name").WriteString(*planNode.CteRefName);
}
-
+
for (const auto& [key, value] : planNode.NodeInfo) {
writer.WriteKey(key);
writer.WriteJsonValue(&value, true);
@@ -417,18 +417,18 @@ private:
if (!planNode.Plans.empty()) {
writer.WriteKey("Plans");
writer.BeginList();
-
+
for (auto planId : planNode.Plans) {
Y_ENSURE(QueryPlanNodes.contains(planId));
WritePlanNodeToJson(QueryPlanNodes.at(planId), writer);
}
-
+
writer.EndList();
}
-
+
writer.EndObject();
}
-
+
TString GetPlanNodeType(const TQueryPlanNode& planNode) const {
switch (planNode.Type) {
case EPlanNodeType::Connection:
@@ -447,7 +447,7 @@ private:
return TMaybe<std::pair<ui32, ui32>>(
{FromString<ui32>(match[1].str()), FromString<ui32>(match[2].str())});
}
-
+
return {};
}
@@ -471,21 +471,21 @@ private:
planNode.Operators.push_back(std::move(op));
return planNode.Operators.size() - 1;
}
-
+
TQueryPlanNode& GetParent(ui32 nodeId) {
auto it = std::find_if(QueryPlanNodes.begin(), QueryPlanNodes.end(), [nodeId](const auto& node) {
return node.second.Plans.contains(nodeId);
});
-
+
Y_ENSURE(it != QueryPlanNodes.end());
return it->second;
}
-
+
TQueryPlanNode& AddPlanNode(TQueryPlanNode& planNode) {
planNode.Plans.insert(++SerializerCtx.PlanNodeId);
return QueryPlanNodes[SerializerCtx.PlanNodeId];
}
-
+
TString ToStr(const TCoDataCtor& data) {
TStringStream out;
EscapeArbitraryAtom(data.Literal().Value(), '"', &out);
@@ -640,12 +640,12 @@ private:
void Visit(const TExprBase& expr, TQueryPlanNode& planNode) {
if (expr.Maybe<TDqPhyStage>()) {
auto stageGuid = NDq::TDqStageSettings::Parse(expr.Cast<TDqPhyStage>()).Id;
-
+
if (VisitedStages.find(expr.Raw()) != VisitedStages.end()) {
auto stageId = SerializerCtx.StageGuidToId[stageGuid];
Y_ENSURE(QueryPlanNodes.contains(stageId));
auto& commonNode = QueryPlanNodes[stageId];
-
+
auto& parentNode = GetParent(stageId);
parentNode.Plans.erase(stageId);
@@ -674,11 +674,11 @@ private:
} else {
stagePlanNode.TypeName = "Stage";
}
- }
-
+ }
+
for (const auto& input : expr.Cast<TDqStageBase>().Inputs()) {
auto inputCn = input.Cast<TDqConnection>();
-
+
auto& inputPlanNode = AddPlanNode(stagePlanNode);
inputPlanNode.Type = EPlanNodeType::Connection;
@@ -715,17 +715,17 @@ private:
operatorId = Visit(maybeReadRanges.Cast(), planNode);
} else if (auto maybeLookup = TMaybeNode<TKqlLookupTableBase>(node)) {
operatorId = Visit(maybeLookup.Cast(), planNode);
- } else if (auto maybeFilter = TMaybeNode<TCoFilterBase>(node)) {
+ } else if (auto maybeFilter = TMaybeNode<TCoFilterBase>(node)) {
operatorId = Visit(maybeFilter.Cast(), planNode);
- } else if (auto maybeMapJoin = TMaybeNode<TCoMapJoinCore>(node)) {
+ } else if (auto maybeMapJoin = TMaybeNode<TCoMapJoinCore>(node)) {
operatorId = Visit(maybeMapJoin.Cast(), planNode);
} else if (TMaybeNode<TCoFlatMapBase>(node).Lambda().Body().Maybe<TCoMapJoinCore>() ||
TMaybeNode<TCoFlatMapBase>(node).Lambda().Body().Maybe<TCoMap>().Input().Maybe<TCoMapJoinCore>()) {
- auto flatMap = TMaybeNode<TCoFlatMapBase>(node).Cast();
- auto join = TExprBase(FindNode(node, [](const TExprNode::TPtr& node) {
- Y_ENSURE(!TMaybeNode<TDqConnection>(node).IsValid());
- return TMaybeNode<TCoMapJoinCore>(node).IsValid();
- })).Cast<TCoMapJoinCore>();
+ auto flatMap = TMaybeNode<TCoFlatMapBase>(node).Cast();
+ auto join = TExprBase(FindNode(node, [](const TExprNode::TPtr& node) {
+ Y_ENSURE(!TMaybeNode<TDqConnection>(node).IsValid());
+ return TMaybeNode<TCoMapJoinCore>(node).IsValid();
+ })).Cast<TCoMapJoinCore>();
operatorId = Visit(flatMap, join, planNode);
node = join.Ptr();
} else if (auto maybeJoinDict = TMaybeNode<TCoJoinDict>(node)) {
@@ -760,13 +760,13 @@ private:
} else if (auto maybeDelete = TMaybeNode<TKqpDeleteRows>(node)) {
operatorId = Visit(maybeDelete.Cast(), planNode);
}
-
+
TVector<ui32> inputIds;
if (auto maybeEffects = TMaybeNode<TKqpEffects>(node)) {
for (const auto& effect : maybeEffects.Cast().Args()) {
auto ids = Visit(effect, planNode);
inputIds.insert(inputIds.end(), ids.begin(), ids.end());
- }
+ }
} else {
for (const auto& child : node->Children()) {
auto ids = Visit(child, planNode);
@@ -781,14 +781,14 @@ private:
return inputIds;
}
-
+
ui32 Visit(const TCoCondense1& /*condense*/, TQueryPlanNode& planNode) {
TOperator op;
op.Properties["Name"] = "Aggregate";
return AddOperator(planNode, "Aggregate", std::move(op));
}
-
+
ui32 Visit(const TCoCombineCore& combiner, TQueryPlanNode& planNode) {
TOperator op;
op.Properties["Name"] = "Aggregate";
@@ -797,7 +797,7 @@ private:
return AddOperator(planNode, "Aggregate", std::move(op));
}
-
+
ui32 Visit(const TCoSort& sort, TQueryPlanNode& planNode) {
TOperator op;
op.Properties["Name"] = "Sort";
@@ -805,7 +805,7 @@ private:
return AddOperator(planNode, "Sort", std::move(op));
}
-
+
ui32 Visit(const TCoTopSort& topSort, TQueryPlanNode& planNode) {
TOperator op;
op.Properties["Name"] = "TopSort";
@@ -814,7 +814,7 @@ private:
return AddOperator(planNode, "TopSort", std::move(op));
}
-
+
ui32 Visit(const TCoTake& take, TQueryPlanNode& planNode) {
TOperator op;
op.Properties["Name"] = "Limit";
@@ -822,7 +822,7 @@ private:
return AddOperator(planNode, "Limit", std::move(op));
}
-
+
ui32 Visit(const TCoSkip& skip, TQueryPlanNode& planNode) {
TOperator op;
op.Properties["Name"] = "Offset";
@@ -830,14 +830,14 @@ private:
return AddOperator(planNode, "Offset", std::move(op));
}
-
+
ui32 Visit(const TCoExtendBase& /*extend*/, TQueryPlanNode& planNode) {
TOperator op;
op.Properties["Name"] = "Union";
return AddOperator(planNode, "Union", std::move(op));
}
-
+
ui32 Visit(const TCoIterator& iter, TQueryPlanNode& planNode) {
const auto iterValue = PrettyExprStr(iter.List());
@@ -866,7 +866,7 @@ private:
for (const auto& column : upsert.Columns()) {
writeInfo.Columns.push_back(TString(column.Value()));
}
-
+
SerializerCtx.Tables[table].Writes.push_back(writeInfo);
planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]);
return AddOperator(planNode, "Upsert", std::move(op));
@@ -882,7 +882,7 @@ private:
TTableWrite writeInfo;
writeInfo.Type = ETableWriteType::MultiErase;
-
+
SerializerCtx.Tables[table].Writes.push_back(writeInfo);
planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]);
return AddOperator(planNode, "Delete", std::move(op));
@@ -898,16 +898,16 @@ private:
auto inputs = Visit(flatMap.Input().Ptr(), planNode);
planNode.Operators[operatorId].Inputs.insert(inputs.begin(), inputs.end());
return operatorId;
- }
-
+ }
+
ui32 Visit(const TCoMapJoinCore& join, TQueryPlanNode& planNode) {
const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (MapJoin)";
TOperator op;
op.Properties["Name"] = name;
return AddOperator(planNode, name, std::move(op));
- }
-
+ }
+
ui32 Visit(const TCoFlatMapBase& flatMap, const TCoJoinDict& join, TQueryPlanNode& planNode) {
const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (JoinDict)";
@@ -942,7 +942,7 @@ private:
ui32 Visit(const TCoFilterBase& filter, TQueryPlanNode& planNode) {
TOperator op;
op.Properties["Name"] = "Filter";
-
+
auto pred = ExtractPredicate(filter.Lambda());
op.Properties["Predicate"] = pred.Body;
@@ -952,12 +952,12 @@ private:
return AddOperator(planNode, "Filter", std::move(op));
}
-
+
ui32 Visit(const TKqlLookupTableBase& lookup, TQueryPlanNode& planNode) {
auto table = TString(lookup.Table().Path().Value());
TTableRead readInfo;
readInfo.Type = ETableReadType::Lookup;
-
+
TOperator op;
op.Properties["Name"] = "TablePointLookup";
auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table);
@@ -967,7 +967,7 @@ private:
readInfo.Columns.push_back(TString(col.Value()));
columns.AppendValue(col.Value());
}
-
+
SerializerCtx.Tables[table].Reads.push_back(readInfo);
planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]);
return AddOperator(planNode, "TablePointLookup", std::move(op));
@@ -1103,10 +1103,10 @@ private:
return (*result)[index].GetDataText();
}
}
- }
+ }
return TString("n/a");
};
-
+
/* Collect info about scan range */
struct TKeyPartRange {
TString From;
@@ -1126,14 +1126,14 @@ private:
for (ui32 i = 0; i < fromKey.ArgCount(); ++i) {
scanRangeDescr[i].From = describeBoundary(fromKey.Arg(i));
- }
+ }
for (ui32 i = 0; i < toKey.ArgCount(); ++i) {
scanRangeDescr[i].To = describeBoundary(toKey.Arg(i));
}
for (ui32 i = 0; i < scanRangeDescr.size(); ++i) {
scanRangeDescr[i].ColumnName = tableData.Metadata->KeyColumnNames[i];
}
-
+
TString leftParen = range.From().Maybe<TKqlKeyInc>().IsValid() ? "[" : "(";
TString rightParen = range.To().Maybe<TKqlKeyInc>().IsValid() ? "]" : ")";
bool hasRangeScans = false;
@@ -1161,8 +1161,8 @@ private:
}
ranges.AppendValue(rangeDescr);
- }
-
+ }
+
// Scan which fixes only few first members of compound primary key were called "Lookup"
// by older explain version. We continue to do so.
if (readInfo.LookupBy.size() > 0) {
@@ -1170,7 +1170,7 @@ private:
} else {
readInfo.Type = hasRangeScans ? ETableReadType::Scan : ETableReadType::FullScan;
}
- }
+ }
auto& columns = op.Properties["ReadColumns"];
for (auto const& col : read.Columns()) {
@@ -1214,32 +1214,32 @@ private:
}
return operatorId;
- }
-
+ }
+
private:
TSerializerCtx& SerializerCtx;
const ui32 TxId;
const TKqpPhysicalTx& Tx;
-
+
TMap<ui32, TQueryPlanNode> QueryPlanNodes;
TNodeSet VisitedStages;
};
using ModifyFunction = std::function<void (NJson::TJsonValue& node)>;
-
+
void ModifyPlan(NJson::TJsonValue& plan, const ModifyFunction& modify) {
modify(plan);
-
+
auto& map = plan.GetMapSafe();
- if (map.contains("Plans")) {
+ if (map.contains("Plans")) {
for (auto& subplan : map.at("Plans").GetArraySafe()) {
ModifyPlan(subplan, modify);
- }
- }
-}
-
-void WriteCommonTablesInfo(NJsonWriter::TBuf& writer, TMap<TString, TTableInfo>& tables) {
-
+ }
+ }
+}
+
+void WriteCommonTablesInfo(NJsonWriter::TBuf& writer, TMap<TString, TTableInfo>& tables) {
+
for (auto& pair : tables) {
auto& info = pair.second;
@@ -1347,63 +1347,63 @@ void WriteCommonTablesInfo(NJsonWriter::TBuf& writer, TMap<TString, TTableInfo>&
}
writer.EndList();
-
-}
-
-} // namespace
-
-void WriteKqlPlan(NJsonWriter::TBuf& writer, const TExprNode::TPtr& query) {
- TMap<TString, TTableInfo> tables;
-
- writer.BeginObject();
- writer.WriteKey("meta");
-
- writer.BeginObject();
- writer.WriteKey("version").WriteString("0.1");
- writer.WriteKey("type").WriteString("query");
+
+}
+
+} // namespace
+
+void WriteKqlPlan(NJsonWriter::TBuf& writer, const TExprNode::TPtr& query) {
+ TMap<TString, TTableInfo> tables;
+
+ writer.BeginObject();
+ writer.WriteKey("meta");
+
+ writer.BeginObject();
+ writer.WriteKey("version").WriteString("0.1");
+ writer.WriteKey("type").WriteString("query");
writer.EndObject();
- writer.WriteKey("tables");
- FillTablesInfo(query, tables);
- WriteCommonTablesInfo(writer, tables);
-
- writer.EndObject();
+ writer.WriteKey("tables");
+ FillTablesInfo(query, tables);
+ WriteCommonTablesInfo(writer, tables);
+
+ writer.EndObject();
}
-// TODO(sk): check prepared statements params in read ranges
-// TODO(sk): check params from correlated subqueries // lookup join
+// TODO(sk): check prepared statements params in read ranges
+// TODO(sk): check params from correlated subqueries // lookup join
void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const TKqpPhysicalQuery& query,
THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> pureTxResults, TExprContext& ctx, const TString& cluster,
const TIntrusivePtr<NYql::TKikimrTablesData> tablesData, TKikimrConfiguration::TPtr config)
-{
+{
TSerializerCtx serializerCtx(ctx, cluster, tablesData, config, std::move(pureTxResults));
-
- /* bindingName -> stage */
+
+ /* bindingName -> stage */
auto collectBindings = [&serializerCtx, &query] (auto id, const auto& phase) {
- for (const auto& param: phase.ParamBindings()) {
- if (auto maybeResultBinding = param.Binding().template Maybe<TKqpTxResultBinding>()) {
- auto resultBinding = maybeResultBinding.Cast();
- auto txId = FromString<ui32>(resultBinding.TxIndex());
- auto resultId = FromString<ui32>(resultBinding.ResultIndex());
- auto result = query.Transactions().Item(txId).Results().Item(resultId);
- if (auto maybeConnection = result.template Maybe<TDqConnection>()) {
- auto stage = maybeConnection.Cast().Output().Stage();
- serializerCtx.BindNameToStage[param.Name().StringValue()] = stage.Ptr();
+ for (const auto& param: phase.ParamBindings()) {
+ if (auto maybeResultBinding = param.Binding().template Maybe<TKqpTxResultBinding>()) {
+ auto resultBinding = maybeResultBinding.Cast();
+ auto txId = FromString<ui32>(resultBinding.TxIndex());
+ auto resultId = FromString<ui32>(resultBinding.ResultIndex());
+ auto result = query.Transactions().Item(txId).Results().Item(resultId);
+ if (auto maybeConnection = result.template Maybe<TDqConnection>()) {
+ auto stage = maybeConnection.Cast().Output().Stage();
+ serializerCtx.BindNameToStage[param.Name().StringValue()] = stage.Ptr();
serializerCtx.PrecomputePhases.insert(txId);
serializerCtx.ParamBindings[id].push_back({txId, resultId});
- }
- }
- }
- };
+ }
+ }
+ }
+ };
ui32 id = 0;
- for (const auto& tx: query.Transactions()) {
+ for (const auto& tx: query.Transactions()) {
collectBindings(id++, tx);
- }
-
+ }
+
auto setPlan = [&serializerCtx](auto txId, const auto& tx, auto& txProto) {
- NJsonWriter::TBuf txWriter;
- txWriter.SetIndentSpaces(2);
+ NJsonWriter::TBuf txWriter;
+ txWriter.SetIndentSpaces(2);
TxPlanSerializer txPlanSerializer(serializerCtx, txId, tx);
if (!serializerCtx.PureTxResults.contains(txId)) {
txPlanSerializer.Serialize();
@@ -1411,18 +1411,18 @@ void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const TKqpPhysicalQ
txProto.SetPlan(txWriter.Str());
}
};
-
+
id = 0;
for (ui32 txId = 0; txId < query.Transactions().Size(); ++txId) {
setPlan(id++, query.Transactions().Item(txId), (*queryProto.MutableTransactions())[txId]);
- }
-
- NJsonWriter::TBuf writer;
- writer.SetIndentSpaces(2);
- WriteCommonTablesInfo(writer, serializerCtx.Tables);
- queryProto.SetQueryPlan(writer.Str());
-}
-
+ }
+
+ NJsonWriter::TBuf writer;
+ writer.SetIndentSpaces(2);
+ WriteCommonTablesInfo(writer, serializerCtx.Tables);
+ queryProto.SetQueryPlan(writer.Str());
+}
+
TString AddExecStatsToTxPlan(const TString& txPlanJson, const NYql::NDqProto::TDqExecutionStats& stats) {
if (txPlanJson.empty()) {
return {};
@@ -1430,17 +1430,17 @@ TString AddExecStatsToTxPlan(const TString& txPlanJson, const NYql::NDqProto::TD
THashMap<TProtoStringType, const NYql::NDqProto::TDqStageStats*> stages;
for (const auto& stage : stats.GetStages()) {
- stages[stage.GetStageGuid()] = &stage;
- }
-
- NJson::TJsonValue root;
- NJson::ReadJsonTree(txPlanJson, &root, true);
-
+ stages[stage.GetStageGuid()] = &stage;
+ }
+
+ NJson::TJsonValue root;
+ NJson::ReadJsonTree(txPlanJson, &root, true);
+
auto addStatsToPlanNode = [&stages](NJson::TJsonValue& node) {
if (auto stageGuid = node.GetMapSafe().FindPtr("StageGuid")) {
if (auto stat = stages.FindPtr(stageGuid->GetStringSafe())) {
auto& stats = node["Stats"];
-
+
stats["TotalTasks"] = (*stat)->GetTotalTasksCount();
stats["TotalDurationMs"] = (*stat)->GetFinishTimeMs().GetMax() - (*stat)->GetFirstRowTimeMs().GetMin();
stats["TotalCpuTimeUs"] = (*stat)->GetCpuTimeUs().GetSum();
@@ -1448,44 +1448,44 @@ TString AddExecStatsToTxPlan(const TString& txPlanJson, const NYql::NDqProto::TD
stats["TotalInputBytes"] = (*stat)->GetInputBytes().GetSum();
stats["TotalOutputRows"] = (*stat)->GetOutputRows().GetSum();
stats["TotalOutputBytes"] = (*stat)->GetOutputBytes().GetSum();
- }
- }
+ }
+ }
};
-
+
ModifyPlan(root, addStatsToPlanNode);
NJsonWriter::TBuf txWriter;
txWriter.WriteJsonValue(&root, true);
- return txWriter.Str();
-}
-
-TString SerializeTxPlans(const TVector<const TString>& txPlans, const TString commonPlanInfo = "") {
- NJsonWriter::TBuf writer;
- writer.SetIndentSpaces(2);
-
- writer.BeginObject();
- writer.WriteKey("meta");
- writer.BeginObject();
- writer.WriteKey("version").WriteString("0.2");
- writer.WriteKey("type").WriteString("query");
- writer.EndObject();
-
- if (!commonPlanInfo.Empty()) {
- writer.WriteKey("tables");
- writer.UnsafeWriteValue(commonPlanInfo);
- }
-
- writer.WriteKey("Plan");
- writer.BeginObject();
- writer.WriteKey("Node Type").WriteString("Query");
- writer.WriteKey("Plans");
- writer.BeginList();
-
+ return txWriter.Str();
+}
+
+TString SerializeTxPlans(const TVector<const TString>& txPlans, const TString commonPlanInfo = "") {
+ NJsonWriter::TBuf writer;
+ writer.SetIndentSpaces(2);
+
+ writer.BeginObject();
+ writer.WriteKey("meta");
+ writer.BeginObject();
+ writer.WriteKey("version").WriteString("0.2");
+ writer.WriteKey("type").WriteString("query");
+ writer.EndObject();
+
+ if (!commonPlanInfo.Empty()) {
+ writer.WriteKey("tables");
+ writer.UnsafeWriteValue(commonPlanInfo);
+ }
+
+ writer.WriteKey("Plan");
+ writer.BeginObject();
+ writer.WriteKey("Node Type").WriteString("Query");
+ writer.WriteKey("Plans");
+ writer.BeginList();
+
auto removeStageGuid = [](NJson::TJsonValue& node) {
auto& map = node.GetMapSafe();
if (map.contains("StageGuid")) {
map.erase("StageGuid");
- }
+ }
};
for (const auto& txPlan : txPlans) {
@@ -1500,74 +1500,74 @@ TString SerializeTxPlans(const TVector<const TString>& txPlans, const TString co
ModifyPlan(subplan, removeStageGuid);
writer.WriteJsonValue(&subplan, true);
}
- }
-
- writer.EndList();
- writer.EndObject();
- writer.EndObject();
-
- return writer.Str();
-}
-
-TString SerializeExplainPlan(const NKqpProto::TKqpPhyQuery& phyQuery) {
- TVector<const TString> txPlans;
+ }
+
+ writer.EndList();
+ writer.EndObject();
+ writer.EndObject();
+
+ return writer.Str();
+}
+
+TString SerializeExplainPlan(const NKqpProto::TKqpPhyQuery& phyQuery) {
+ TVector<const TString> txPlans;
txPlans.reserve(phyQuery.GetTransactions().size());
- for (const auto& phyTx: phyQuery.GetTransactions()) {
+ for (const auto& phyTx: phyQuery.GetTransactions()) {
txPlans.emplace_back(phyTx.GetPlan());
- }
- return SerializeTxPlans(txPlans, phyQuery.GetQueryPlan());
-}
-
-TString SerializeAnalyzePlan(const NKqpProto::TKqpStatsQuery& queryStats) {
- TVector<const TString> txPlans;
- for (const auto& execStats: queryStats.GetExecutions()) {
- for (const auto& txPlan: execStats.GetTxPlansWithStats()) {
- txPlans.push_back(txPlan);
- }
- }
- return SerializeTxPlans(txPlans);
-}
-
-TString SerializeScriptPlan(const TVector<const TString>& queryPlans) {
- NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE);
- writer.SetIndentSpaces(2);
-
- writer.BeginObject();
- writer.WriteKey("meta");
- writer.BeginObject();
- writer.WriteKey("version").WriteString("0.2");
- writer.WriteKey("type").WriteString("script");
- writer.EndObject();
-
- writer.WriteKey("queries");
- writer.BeginList();
-
+ }
+ return SerializeTxPlans(txPlans, phyQuery.GetQueryPlan());
+}
+
+TString SerializeAnalyzePlan(const NKqpProto::TKqpStatsQuery& queryStats) {
+ TVector<const TString> txPlans;
+ for (const auto& execStats: queryStats.GetExecutions()) {
+ for (const auto& txPlan: execStats.GetTxPlansWithStats()) {
+ txPlans.push_back(txPlan);
+ }
+ }
+ return SerializeTxPlans(txPlans);
+}
+
+TString SerializeScriptPlan(const TVector<const TString>& queryPlans) {
+ NJsonWriter::TBuf writer(NJsonWriter::HEM_UNSAFE);
+ writer.SetIndentSpaces(2);
+
+ writer.BeginObject();
+ writer.WriteKey("meta");
+ writer.BeginObject();
+ writer.WriteKey("version").WriteString("0.2");
+ writer.WriteKey("type").WriteString("script");
+ writer.EndObject();
+
+ writer.WriteKey("queries");
+ writer.BeginList();
+
for (const auto& plan : queryPlans) {
if (plan.empty()) {
continue;
}
- NJson::TJsonValue root;
- NJson::ReadJsonTree(plan, &root, true);
- auto planMap = root.GetMapSafe();
-
- writer.BeginObject();
- if (auto tableAccesses = planMap.FindPtr("tables")) {
- writer.WriteKey("tables");
- writer.WriteJsonValue(tableAccesses);
- }
- if (auto dqPlan = planMap.FindPtr("Plan")) {
- writer.WriteKey("Plan");
- writer.WriteJsonValue(dqPlan);
- }
- writer.EndObject();
- }
-
- writer.EndList();
- writer.EndObject();
-
- return writer.Str();
-}
-
+ NJson::TJsonValue root;
+ NJson::ReadJsonTree(plan, &root, true);
+ auto planMap = root.GetMapSafe();
+
+ writer.BeginObject();
+ if (auto tableAccesses = planMap.FindPtr("tables")) {
+ writer.WriteKey("tables");
+ writer.WriteJsonValue(tableAccesses);
+ }
+ if (auto dqPlan = planMap.FindPtr("Plan")) {
+ writer.WriteKey("Plan");
+ writer.WriteJsonValue(dqPlan);
+ }
+ writer.EndObject();
+ }
+
+ writer.EndList();
+ writer.EndObject();
+
+ return writer.Str();
+}
+
} // namespace NKqp
} // namespace NKikimr
diff --git a/ydb/core/kqp/prepare/kqp_query_plan.h b/ydb/core/kqp/prepare/kqp_query_plan.h
index 4e299e14903..e3895f11d5a 100644
--- a/ydb/core/kqp/prepare/kqp_query_plan.h
+++ b/ydb/core/kqp/prepare/kqp_query_plan.h
@@ -1,38 +1,38 @@
-#pragma once
-
+#pragma once
+
#include <ydb/core/protos/kqp_physical.pb.h>
#include <ydb/core/protos/kqp_stats.pb.h>
#include <ydb/core/kqp/expr_nodes/kqp_expr_nodes.h>
#include <ydb/core/kqp/provider/yql_kikimr_provider.h>
-
+
#include <ydb/library/yql/ast/yql_expr.h>
-
-#include <library/cpp/json/writer/json.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-void WriteKqlPlan(NJsonWriter::TBuf& writer, const NYql::TExprNode::TPtr& query);
-
-/*
- * Set dqPlan in each physical transaction (TKqpPhyQuery.Transactions[].Plan). Common query plan with all
- * table accesses is stored in top-level TKqpPhyQuery.QueryPlan.
- */
+
+#include <library/cpp/json/writer/json.h>
+
+namespace NKikimr {
+namespace NKqp {
+
+void WriteKqlPlan(NJsonWriter::TBuf& writer, const NYql::TExprNode::TPtr& query);
+
+/*
+ * Set dqPlan in each physical transaction (TKqpPhyQuery.Transactions[].Plan). Common query plan with all
+ * table accesses is stored in top-level TKqpPhyQuery.QueryPlan.
+ */
void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const NYql::NNodes::TKqpPhysicalQuery& query,
THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> pureTxResults, NYql::TExprContext& ctx, const TString& cluster,
const TIntrusivePtr<NYql::TKikimrTablesData> tablesData, NYql::TKikimrConfiguration::TPtr config);
-
-/*
- * Fill stages in given txPlan with ExecutionStats fields. Each plan stage stores StageGuid which is
- * used to find corresponding TKqpStatsExecution object.
- */
+
+/*
+ * Fill stages in given txPlan with ExecutionStats fields. Each plan stage stores StageGuid which is
+ * used to find corresponding TKqpStatsExecution object.
+ */
TString AddExecStatsToTxPlan(const TString& txPlan, const NYql::NDqProto::TDqExecutionStats& stats);
-
-TString SerializeExplainPlan(const NKqpProto::TKqpPhyQuery& phyQuery);
-
-TString SerializeAnalyzePlan(const NKqpProto::TKqpStatsQuery& queryStats);
-
-TString SerializeScriptPlan(const TVector<const TString>& queryPlans);
-
-} // namespace NKqp
-} // namespace NKikimr
+
+TString SerializeExplainPlan(const NKqpProto::TKqpPhyQuery& phyQuery);
+
+TString SerializeAnalyzePlan(const NKqpProto::TKqpStatsQuery& queryStats);
+
+TString SerializeScriptPlan(const TVector<const TString>& queryPlans);
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/prepare/kqp_type_ann.cpp b/ydb/core/kqp/prepare/kqp_type_ann.cpp
index ee51c7349c2..caa6653538b 100644
--- a/ydb/core/kqp/prepare/kqp_type_ann.cpp
+++ b/ydb/core/kqp/prepare/kqp_type_ann.cpp
@@ -230,7 +230,7 @@ TStatus AnnotateReadTable(const TExprNode::TPtr& node, TExprContext& ctx, const
} else {
YQL_ENSURE(false, "Unexpected ReadTable callable: " << node->Content());
}
-
+
return TStatus::Ok;
}
@@ -1056,8 +1056,8 @@ TAutoPtr<IGraphTransformer> CreateKqpTypeAnnotationTransformer(const TString& cl
if (TKqlLookupTableBase::Match(input.Get())) {
return AnnotateLookupTable(input, ctx, cluster, *tablesData, config->SystemColumnsEnabled());
- }
-
+ }
+
if (TKqlKeyInc::Match(input.Get()) || TKqlKeyExc::Match(input.Get())) {
return AnnotateKeyTuple(input, ctx);
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.h b/ydb/core/kqp/provider/yql_kikimr_provider.h
index 6903e6f1932..baec0739087 100644
--- a/ydb/core/kqp/provider/yql_kikimr_provider.h
+++ b/ydb/core/kqp/provider/yql_kikimr_provider.h
@@ -81,12 +81,12 @@ struct TKikimrQueryContext : TThrRefBase {
TKikimrQueryContext& operator=(const TKikimrQueryContext&) = delete;
bool PrepareOnly = false;
-
- /*
- * Defuse DDL-prohibiting checks when PrepareOnly = true. Used in scripting query explain.
- */
+
+ /*
+ * Defuse DDL-prohibiting checks when PrepareOnly = true. Used in scripting query explain.
+ */
bool SuppressDdlChecks = false;
-
+
EKikimrStatsMode StatsMode = EKikimrStatsMode::None;
EKikimrQueryType Type = EKikimrQueryType::Unspecified;
TKikimrQueryDeadlines Deadlines;
diff --git a/ydb/core/kqp/provider/yql_kikimr_settings.h b/ydb/core/kqp/provider/yql_kikimr_settings.h
index 530cfb28558..4c629b747da 100644
--- a/ydb/core/kqp/provider/yql_kikimr_settings.h
+++ b/ydb/core/kqp/provider/yql_kikimr_settings.h
@@ -146,8 +146,8 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi
}
TKikimrSettings::TConstPtr Snapshot() const;
-
- NKikimrConfig::TFeatureFlags FeatureFlags;
+
+ NKikimrConfig::TFeatureFlags FeatureFlags;
};
}
diff --git a/ydb/core/kqp/rm/kqp_resource_estimation.cpp b/ydb/core/kqp/rm/kqp_resource_estimation.cpp
index aeeef801775..671d3e9c5f8 100644
--- a/ydb/core/kqp/rm/kqp_resource_estimation.cpp
+++ b/ydb/core/kqp/rm/kqp_resource_estimation.cpp
@@ -5,8 +5,8 @@ namespace NKikimr::NKqp {
using namespace NYql::NDqProto;
using namespace NKikimrConfig;
-TTaskResourceEstimation EstimateTaskResources(const TDqTask& task, int nScans, ui32 dsOnNodeCount,
- const TTableServiceConfig::TResourceManager& config)
+TTaskResourceEstimation EstimateTaskResources(const TDqTask& task, int nScans, ui32 dsOnNodeCount,
+ const TTableServiceConfig::TResourceManager& config)
{
TTaskResourceEstimation ret;
EstimateTaskResources(task, nScans, dsOnNodeCount, config, ret);
@@ -18,8 +18,8 @@ void EstimateTaskResources(const TDqTask& task, int nScans, ui32 dsOnNodeCount,
{
ret.TaskId = task.GetId();
- if (nScans > 0) {
- ret.ScanBuffersCount = nScans;
+ if (nScans > 0) {
+ ret.ScanBuffersCount = nScans;
ret.ScanBufferMemoryLimit = config.GetScanBufferSize();
if (dsOnNodeCount && ret.ScanBufferMemoryLimit * dsOnNodeCount > config.GetMaxTotalScanBuffersSize()) {
diff --git a/ydb/core/kqp/rm/kqp_resource_estimation.h b/ydb/core/kqp/rm/kqp_resource_estimation.h
index b69ac203514..e66dc52f121 100644
--- a/ydb/core/kqp/rm/kqp_resource_estimation.h
+++ b/ydb/core/kqp/rm/kqp_resource_estimation.h
@@ -30,7 +30,7 @@ struct TTaskResourceEstimation {
}
};
-TTaskResourceEstimation EstimateTaskResources(const NYql::NDqProto::TDqTask& task, int nScans, ui32 dsOnNodeCount,
+TTaskResourceEstimation EstimateTaskResources(const NYql::NDqProto::TDqTask& task, int nScans, ui32 dsOnNodeCount,
const NKikimrConfig::TTableServiceConfig::TResourceManager& config);
void EstimateTaskResources(const NYql::NDqProto::TDqTask& task, int nScans, ui32 dsOnNodeCount,
diff --git a/ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp b/ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp
index 689c0ed86b7..5d76d1b777e 100644
--- a/ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp
+++ b/ydb/core/kqp/rm/kqp_resource_estimation_ut.cpp
@@ -31,7 +31,7 @@ Y_UNIT_TEST(TestChannelSize) {
auto* output = task.MutableOutputs()->Add();
output->MutableChannels()->Add();
- auto est = EstimateTaskResources(task, 0, 0, config);
+ auto est = EstimateTaskResources(task, 0, 0, config);
UNIT_ASSERT_EQUAL(201, est.ChannelBuffersCount);
UNIT_ASSERT_EQUAL(est.ChannelBufferMemoryLimit, config.GetChannelBufferSize());
@@ -41,7 +41,7 @@ Y_UNIT_TEST(TestChannelSize) {
input->MutableChannels()->Add();
}
- est = EstimateTaskResources(task, 0, 0, config);
+ est = EstimateTaskResources(task, 0, 0, config);
UNIT_ASSERT_EQUAL(301, est.ChannelBuffersCount);
UNIT_ASSERT(est.ChannelBufferMemoryLimit < config.GetChannelBufferSize());
diff --git a/ydb/core/kqp/rm/ya.make b/ydb/core/kqp/rm/ya.make
index 0d8a7596391..4185e629129 100644
--- a/ydb/core/kqp/rm/ya.make
+++ b/ydb/core/kqp/rm/ya.make
@@ -4,7 +4,7 @@ OWNER(g:kikimr)
SRCS(
kqp_resource_estimation.cpp
- kqp_snapshot_manager.cpp
+ kqp_snapshot_manager.cpp
kqp_rm.cpp
)
diff --git a/ydb/core/kqp/runtime/kqp_program_builder.cpp b/ydb/core/kqp/runtime/kqp_program_builder.cpp
index b5169c19d82..3682fd4d4b2 100644
--- a/ydb/core/kqp/runtime/kqp_program_builder.cpp
+++ b/ydb/core/kqp/runtime/kqp_program_builder.cpp
@@ -141,9 +141,9 @@ TRuntimeNode TKqpProgramBuilder::KqpReadTable(const TTableId& tableId, const TKq
return TRuntimeNode(builder.Build(), false);
}
-TRuntimeNode TKqpProgramBuilder::KqpWideReadTable(const TTableId& tableId, const TKqpKeyRange& range,
+TRuntimeNode TKqpProgramBuilder::KqpWideReadTable(const TTableId& tableId, const TKqpKeyRange& range,
const TArrayRef<TKqpTableColumn>& columns)
-{
+{
auto rowType = GetRowType(*this, columns);
auto structType = AS_TYPE(TStructType, rowType);
auto returnType = MakeWideFlowType(*this, structType);
@@ -152,18 +152,18 @@ TRuntimeNode TKqpProgramBuilder::KqpWideReadTable(const TTableId& tableId, const
MKQL_ENSURE_S(returnType->IsFlow());
const auto itemType = AS_TYPE(TFlowType, returnType)->GetItemType();
MKQL_ENSURE_S(itemType->IsTuple());
-
- TCallableBuilder builder(Env, __func__, returnType);
- builder.Add(BuildTableIdLiteral(tableId, *this));
+
+ TCallableBuilder builder(Env, __func__, returnType);
+ builder.Add(BuildTableIdLiteral(tableId, *this));
builder.Add(BuildKeyRangeNode(*this, range));
- builder.Add(BuildColumnTags(*this, columns));
+ builder.Add(BuildColumnTags(*this, columns));
builder.Add(BuildSkipNullKeysNode(*this, range));
builder.Add(range.ItemsLimit ? range.ItemsLimit : NewNull());
builder.Add(NewDataLiteral(range.Reverse));
-
- return TRuntimeNode(builder.Build(), false);
-}
-
+
+ return TRuntimeNode(builder.Build(), false);
+}
+
TRuntimeNode TKqpProgramBuilder::KqpWideReadTableRanges(const TTableId& tableId, const TKqpKeyRanges& ranges,
const TArrayRef<TKqpTableColumn>& columns, TType* returnType)
{
diff --git a/ydb/core/kqp/runtime/kqp_program_builder.h b/ydb/core/kqp/runtime/kqp_program_builder.h
index 348d25eeea3..0884d68a74c 100644
--- a/ydb/core/kqp/runtime/kqp_program_builder.h
+++ b/ydb/core/kqp/runtime/kqp_program_builder.h
@@ -47,9 +47,9 @@ public:
TRuntimeNode KqpReadTable(const TTableId& tableId, const TKqpKeyRange& range,
const TArrayRef<TKqpTableColumn>& columns);
- TRuntimeNode KqpWideReadTable(const TTableId& tableId, const TKqpKeyRange& range,
- const TArrayRef<TKqpTableColumn>& columns);
-
+ TRuntimeNode KqpWideReadTable(const TTableId& tableId, const TKqpKeyRange& range,
+ const TArrayRef<TKqpTableColumn>& columns);
+
TRuntimeNode KqpWideReadTableRanges(const TTableId& tableId, const TKqpKeyRanges& range,
const TArrayRef<TKqpTableColumn>& columns, TType* returnType);
diff --git a/ydb/core/kqp/runtime/kqp_read_table.cpp b/ydb/core/kqp/runtime/kqp_read_table.cpp
index 358ebd0a3eb..9db590688b5 100644
--- a/ydb/core/kqp/runtime/kqp_read_table.cpp
+++ b/ydb/core/kqp/runtime/kqp_read_table.cpp
@@ -137,38 +137,38 @@ void ParseWideReadColumns(const TCallable& callable, const TRuntimeNode& tagsNod
}
}
-TParseReadTableResult ParseWideReadTable(TCallable& callable) {
+TParseReadTableResult ParseWideReadTable(TCallable& callable) {
MKQL_ENSURE_S(callable.GetInputsCount() >= 4);
-
- TParseReadTableResult result;
-
+
+ TParseReadTableResult result;
+
result.CallableId = 0; // callable.GetUniqueId();
-
- auto tableNode = callable.GetInput(0);
- auto rangeNode = callable.GetInput(1);
- auto tagsNode = callable.GetInput(2);
-
- result.TableId = NKqp::ParseTableId(tableNode);
-
- auto range = AS_VALUE(TTupleLiteral, rangeNode);
- MKQL_ENSURE_S(range);
- MKQL_ENSURE_S(range->GetValuesCount() >= 4);
-
- result.FromTuple = AS_VALUE(TTupleLiteral, range->GetValue(0));
- MKQL_ENSURE_S(result.FromTuple);
- result.FromInclusive = AS_VALUE(TDataLiteral, range->GetValue(1))->AsValue().Get<bool>();
- result.ToTuple = AS_VALUE(TTupleLiteral, range->GetValue(2));
- MKQL_ENSURE_S(result.ToTuple);
- result.ToInclusive = AS_VALUE(TDataLiteral, range->GetValue(3))->AsValue().Get<bool>();
-
+
+ auto tableNode = callable.GetInput(0);
+ auto rangeNode = callable.GetInput(1);
+ auto tagsNode = callable.GetInput(2);
+
+ result.TableId = NKqp::ParseTableId(tableNode);
+
+ auto range = AS_VALUE(TTupleLiteral, rangeNode);
+ MKQL_ENSURE_S(range);
+ MKQL_ENSURE_S(range->GetValuesCount() >= 4);
+
+ result.FromTuple = AS_VALUE(TTupleLiteral, range->GetValue(0));
+ MKQL_ENSURE_S(result.FromTuple);
+ result.FromInclusive = AS_VALUE(TDataLiteral, range->GetValue(1))->AsValue().Get<bool>();
+ result.ToTuple = AS_VALUE(TTupleLiteral, range->GetValue(2));
+ MKQL_ENSURE_S(result.ToTuple);
+ result.ToInclusive = AS_VALUE(TDataLiteral, range->GetValue(3))->AsValue().Get<bool>();
+
ParseWideReadColumns(callable, tagsNode, result.Columns, result.SystemColumns);
-
+
auto skipNullKeys = AS_VALUE(TListLiteral, callable.GetInput(3));
result.SkipNullKeys.reserve(skipNullKeys->GetItemsCount());
for (ui32 i = 0; i < skipNullKeys->GetItemsCount(); ++i) {
result.SkipNullKeys.push_back(AS_VALUE(TDataLiteral, skipNullKeys->GetItems()[i])->AsValue().Get<bool>());
}
-
+
if (callable.GetInputsCount() >= 5) {
auto node = callable.GetInput(4).GetNode();
if (node->GetType()->GetKind() == TType::EKind::Callable) {
@@ -184,9 +184,9 @@ TParseReadTableResult ParseWideReadTable(TCallable& callable) {
result.Reverse = AS_VALUE(TDataLiteral, callable.GetInput(5))->AsValue().Get<bool>();
}
- return result;
-}
-
+ return result;
+}
+
TParseReadTableRangesResult ParseWideReadTableRanges(TCallable& callable) {
MKQL_ENSURE_S(callable.GetInputsCount() == 5);
@@ -233,88 +233,88 @@ namespace {
class TKqpScanWideReadTableWrapperBase : public TStatelessWideFlowCodegeneratorNode<TKqpScanWideReadTableWrapperBase> {
using TBase = TStatelessWideFlowCodegeneratorNode<TKqpScanWideReadTableWrapperBase>;
-public:
+public:
TKqpScanWideReadTableWrapperBase(TKqpScanComputeContext& computeCtx, std::vector<EValueRepresentation>&& representations)
: TBase(this)
, ComputeCtx(computeCtx)
- , Representations(std::move(representations))
- {}
-
- EFetchResult DoCalculate(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const {
- Y_UNUSED(ctx);
-
- if (!TableReader) {
+ , Representations(std::move(representations))
+ {}
+
+ EFetchResult DoCalculate(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const {
+ Y_UNUSED(ctx);
+
+ if (!TableReader) {
TableReader = ComputeCtx.ReadTable(GetCallableId());
- }
-
- return TableReader->Next(output);
- }
-
-#ifndef MKQL_DISABLE_CODEGEN
- ICodegeneratorInlineWideNode::TGenerateResult DoGenGetValues(const TCodegenContext& ctx, BasicBlock*& block) const {
- auto& context = ctx.Codegen->GetContext();
+ }
+
+ return TableReader->Next(output);
+ }
+
+#ifndef MKQL_DISABLE_CODEGEN
+ ICodegeneratorInlineWideNode::TGenerateResult DoGenGetValues(const TCodegenContext& ctx, BasicBlock*& block) const {
+ auto& context = ctx.Codegen->GetContext();
const auto size = GetAllColumnsSize();
- ICodegeneratorInlineWideNode::TGettersList getters(size);
-
- const auto valueType = Type::getInt128Ty(context);
- const auto valuesType = ArrayType::get(valueType, size);
- const auto values = new AllocaInst(valuesType, 0U, "values", &ctx.Func->getEntryBlock().back());
-
- const auto fieldsType = ArrayType::get(PointerType::getUnqual(valueType), size);
- const auto fields = new AllocaInst(fieldsType, 0U, "fields", &ctx.Func->getEntryBlock().back());
-
- const auto indexType = Type::getInt32Ty(context);
- Value* init = UndefValue::get(fieldsType);
- for (auto i = 0U; i < size; ++i) {
- const auto pointer = GetElementPtrInst::CreateInBounds(values,
- {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)},
- (TString("ptr_") += ToString(i)).c_str(),
- &ctx.Func->getEntryBlock().back());
- init = InsertValueInst::Create(init, pointer, {i}, (TString("insert_") += ToString(i)).c_str(), &ctx.Func->getEntryBlock().back());
-
- new StoreInst(ConstantInt::get(valueType, 0), pointer, &ctx.Func->getEntryBlock().back());
-
- getters[i] = [i, indexType, values] (const TCodegenContext&, BasicBlock*& block) {
- const auto loadPtr = GetElementPtrInst::CreateInBounds(values,
- {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)},
- (TString("loadPtr_") += ToString(i)).c_str(),
- block);
- return new LoadInst(loadPtr, "load", block);
- };
- }
-
- new StoreInst(init, fields, &ctx.Func->getEntryBlock().back());
-
- for (ui32 i = 0U; i < size; ++i) {
- const auto pointer = GetElementPtrInst::CreateInBounds(values,
- {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)},
- (TString("ptr_") += ToString(i)).c_str(), block);
- ValueCleanup(Representations[i], pointer, ctx, block);
- new StoreInst(ConstantInt::get(valueType, 0), pointer, block);
- }
-
- const auto ptrType = PointerType::getUnqual(StructType::get(context));
+ ICodegeneratorInlineWideNode::TGettersList getters(size);
+
+ const auto valueType = Type::getInt128Ty(context);
+ const auto valuesType = ArrayType::get(valueType, size);
+ const auto values = new AllocaInst(valuesType, 0U, "values", &ctx.Func->getEntryBlock().back());
+
+ const auto fieldsType = ArrayType::get(PointerType::getUnqual(valueType), size);
+ const auto fields = new AllocaInst(fieldsType, 0U, "fields", &ctx.Func->getEntryBlock().back());
+
+ const auto indexType = Type::getInt32Ty(context);
+ Value* init = UndefValue::get(fieldsType);
+ for (auto i = 0U; i < size; ++i) {
+ const auto pointer = GetElementPtrInst::CreateInBounds(values,
+ {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)},
+ (TString("ptr_") += ToString(i)).c_str(),
+ &ctx.Func->getEntryBlock().back());
+ init = InsertValueInst::Create(init, pointer, {i}, (TString("insert_") += ToString(i)).c_str(), &ctx.Func->getEntryBlock().back());
+
+ new StoreInst(ConstantInt::get(valueType, 0), pointer, &ctx.Func->getEntryBlock().back());
+
+ getters[i] = [i, indexType, values] (const TCodegenContext&, BasicBlock*& block) {
+ const auto loadPtr = GetElementPtrInst::CreateInBounds(values,
+ {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)},
+ (TString("loadPtr_") += ToString(i)).c_str(),
+ block);
+ return new LoadInst(loadPtr, "load", block);
+ };
+ }
+
+ new StoreInst(init, fields, &ctx.Func->getEntryBlock().back());
+
+ for (ui32 i = 0U; i < size; ++i) {
+ const auto pointer = GetElementPtrInst::CreateInBounds(values,
+ {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)},
+ (TString("ptr_") += ToString(i)).c_str(), block);
+ ValueCleanup(Representations[i], pointer, ctx, block);
+ new StoreInst(ConstantInt::get(valueType, 0), pointer, block);
+ }
+
+ const auto ptrType = PointerType::getUnqual(StructType::get(context));
const auto func = ConstantInt::get(Type::getInt64Ty(context), GetMethodPtr(&TKqpScanWideReadTableWrapperBase::DoCalculate));
- const auto self = CastInst::Create(Instruction::IntToPtr, ConstantInt::get(Type::getInt64Ty(context), uintptr_t(this)), ptrType, "self", block);
- const auto funcType = FunctionType::get(Type::getInt32Ty(context), { self->getType(), ctx.Ctx->getType(), fields->getType() }, false);
- const auto funcPtr = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(funcType), "fetch_func", block);
- const auto result = CallInst::Create(funcPtr, { self, ctx.Ctx, fields }, "fetch", block);
-
- for (ui32 i = 0U; i < size; ++i) {
- const auto pointer = GetElementPtrInst::CreateInBounds(values,
- {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)},
- (TString("ptr_") += ToString(i)).c_str(), block);
- ValueRelease(Representations[i], pointer, ctx, block);
- }
-
- return {result, std::move(getters)};
- }
-#endif
-
+ const auto self = CastInst::Create(Instruction::IntToPtr, ConstantInt::get(Type::getInt64Ty(context), uintptr_t(this)), ptrType, "self", block);
+ const auto funcType = FunctionType::get(Type::getInt32Ty(context), { self->getType(), ctx.Ctx->getType(), fields->getType() }, false);
+ const auto funcPtr = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(funcType), "fetch_func", block);
+ const auto result = CallInst::Create(funcPtr, { self, ctx.Ctx, fields }, "fetch", block);
+
+ for (ui32 i = 0U; i < size; ++i) {
+ const auto pointer = GetElementPtrInst::CreateInBounds(values,
+ {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, i)},
+ (TString("ptr_") += ToString(i)).c_str(), block);
+ ValueRelease(Representations[i], pointer, ctx, block);
+ }
+
+ return {result, std::move(getters)};
+ }
+#endif
+
virtual ui32 GetCallableId() const = 0;
virtual ui32 GetAllColumnsSize() const = 0;
-private:
+private:
TKqpScanComputeContext& ComputeCtx;
mutable TIntrusivePtr<IKqpTableReader> TableReader;
const std::vector<EValueRepresentation> Representations;
@@ -339,17 +339,17 @@ private:
return ParseResult.Columns.size() + ParseResult.SystemColumns.size();
}
- void RegisterDependencies() const {
+ void RegisterDependencies() const {
FlowDependsOn(FromNode);
FlowDependsOn(ToNode);
- }
-
-private:
- IComputationNode* FromNode;
- IComputationNode* ToNode;
+ }
+
+private:
+ IComputationNode* FromNode;
+ IComputationNode* ToNode;
TParseReadTableResult ParseResult;
-};
-
+};
+
class TKqpScanWideReadTableRangesWrapper : public TKqpScanWideReadTableWrapperBase {
public:
TKqpScanWideReadTableRangesWrapper(TKqpScanComputeContext& computeCtx, const TParseReadTableRangesResult& parseResult,
@@ -401,28 +401,28 @@ IComputationNode* WrapKqpScanWideReadTableRanges(TCallable& callable, const TCom
return new TKqpScanWideReadTableRangesWrapper(computeCtx, parseResult, rangesNode, std::move(representations));
}
-IComputationNode* WrapKqpScanWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx,
- TKqpScanComputeContext& computeCtx)
-{
- std::vector<EValueRepresentation> representations;
-
- auto parseResult = ParseWideReadTable(callable);
- auto fromNode = LocateNode(ctx.NodeLocator, *parseResult.FromTuple);
- auto toNode = LocateNode(ctx.NodeLocator, *parseResult.ToTuple);
-
- const auto type = callable.GetType()->GetReturnType();
- const auto returnItemType = type->IsFlow() ?
- AS_TYPE(TFlowType, callable.GetType()->GetReturnType())->GetItemType():
- AS_TYPE(TStreamType, callable.GetType()->GetReturnType())->GetItemType();
-
- const auto tupleType = AS_TYPE(TTupleType, returnItemType);
-
- representations.reserve(tupleType->GetElementsCount());
- for (ui32 i = 0U; i < tupleType->GetElementsCount(); ++i)
- representations.emplace_back(GetValueRepresentation(tupleType->GetElementType(i)));
-
- return new TKqpScanWideReadTableWrapper(computeCtx, parseResult, fromNode, toNode, std::move(representations));
-}
-
+IComputationNode* WrapKqpScanWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx,
+ TKqpScanComputeContext& computeCtx)
+{
+ std::vector<EValueRepresentation> representations;
+
+ auto parseResult = ParseWideReadTable(callable);
+ auto fromNode = LocateNode(ctx.NodeLocator, *parseResult.FromTuple);
+ auto toNode = LocateNode(ctx.NodeLocator, *parseResult.ToTuple);
+
+ const auto type = callable.GetType()->GetReturnType();
+ const auto returnItemType = type->IsFlow() ?
+ AS_TYPE(TFlowType, callable.GetType()->GetReturnType())->GetItemType():
+ AS_TYPE(TStreamType, callable.GetType()->GetReturnType())->GetItemType();
+
+ const auto tupleType = AS_TYPE(TTupleType, returnItemType);
+
+ representations.reserve(tupleType->GetElementsCount());
+ for (ui32 i = 0U; i < tupleType->GetElementsCount(); ++i)
+ representations.emplace_back(GetValueRepresentation(tupleType->GetElementType(i)));
+
+ return new TKqpScanWideReadTableWrapper(computeCtx, parseResult, fromNode, toNode, std::move(representations));
+}
+
} // namespace NMiniKQL
} // namespace NKikimr
diff --git a/ydb/core/kqp/runtime/kqp_read_table.h b/ydb/core/kqp/runtime/kqp_read_table.h
index 99c9cc3f050..b13541251df 100644
--- a/ydb/core/kqp/runtime/kqp_read_table.h
+++ b/ydb/core/kqp/runtime/kqp_read_table.h
@@ -38,14 +38,14 @@ struct TParseReadTableRangesResult : TParseReadTableResultBase {
void ParseReadColumns(const TType* readType, const TRuntimeNode& tagsNode,
TSmallVec<TKqpComputeContextBase::TColumn>& columns, TSmallVec<TKqpComputeContextBase::TColumn>& systemColumns);
-TParseReadTableResult ParseWideReadTable(TCallable& callable);
+TParseReadTableResult ParseWideReadTable(TCallable& callable);
TParseReadTableRangesResult ParseWideReadTableRanges(TCallable& callable);
IComputationNode* WrapKqpScanWideReadTableRanges(TCallable& callable, const TComputationNodeFactoryContext& ctx,
TKqpScanComputeContext& computeCtx);
-IComputationNode* WrapKqpScanWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx,
- TKqpScanComputeContext& computeCtx);
-
-} // namespace NMiniKQL
-} // namespace NKikimr
+IComputationNode* WrapKqpScanWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx,
+ TKqpScanComputeContext& computeCtx);
+} // namespace NMiniKQL
+} // namespace NKikimr
+
diff --git a/ydb/core/kqp/runtime/kqp_scan_data.cpp b/ydb/core/kqp/runtime/kqp_scan_data.cpp
index c4d8e241966..56c49bf0f5f 100644
--- a/ydb/core/kqp/runtime/kqp_scan_data.cpp
+++ b/ydb/core/kqp/runtime/kqp_scan_data.cpp
@@ -445,7 +445,7 @@ TIntrusivePtr<IKqpTableReader> TKqpScanComputeContext::ReadTable(ui32) const {
class TKqpTableReader : public IKqpTableReader {
public:
- TKqpTableReader(TKqpScanComputeContext::TScanData& scanData)
+ TKqpTableReader(TKqpScanComputeContext::TScanData& scanData)
: ScanData(scanData)
{}
@@ -461,30 +461,30 @@ public:
return NUdf::EFetchStatus::Ok;
}
- EFetchResult Next(NUdf::TUnboxedValue* const* result) override {
+ EFetchResult Next(NUdf::TUnboxedValue* const* result) override {
if (ScanData.IsEmpty()) {
if (ScanData.IsFinished()) {
- return EFetchResult::Finish;
- }
+ return EFetchResult::Finish;
+ }
return EFetchResult::Yield;
- }
-
+ }
+
auto row = ScanData.TakeRow();
for (ui32 i = 0; i < ScanData.GetColumns().size() + ScanData.GetSystemColumns().size(); ++i) {
- if (result[i]) {
+ if (result[i]) {
*result[i] = std::move(row.GetElement(i));
- }
- }
-
- return EFetchResult::One;
- }
-
+ }
+ }
+
+ return EFetchResult::One;
+ }
+
private:
TKqpScanComputeContext::TScanData& ScanData;
};
-TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TScanData& scanData) {
- return MakeIntrusive<TKqpTableReader>(scanData);
+TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TScanData& scanData) {
+ return MakeIntrusive<TKqpTableReader>(scanData);
}
} // namespace NMiniKQL
diff --git a/ydb/core/kqp/runtime/kqp_scan_data.h b/ydb/core/kqp/runtime/kqp_scan_data.h
index c8e7d82b328..d3db62aecf0 100644
--- a/ydb/core/kqp/runtime/kqp_scan_data.h
+++ b/ydb/core/kqp/runtime/kqp_scan_data.h
@@ -10,7 +10,7 @@
#include <ydb/library/yql/dq/actors/protos/dq_stats.pb.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
-
+
#include <library/cpp/actors/core/log.h>
#include <contrib/libs/apache/arrow/cpp/src/arrow/api.h>
@@ -31,7 +31,7 @@ public:
virtual NUdf::EFetchStatus Next(NUdf::TUnboxedValue& result) = 0;
virtual EFetchResult Next(NUdf::TUnboxedValue* const* output) = 0;
};
-
+
class TKqpScanComputeContext : public TKqpComputeContextBase {
public:
class TScanData {
@@ -54,17 +54,17 @@ public:
const TSmallVec<TColumn>& GetColumns() const {
return Columns;
}
-
+
const TSmallVec<TColumn>& GetSystemColumns() const {
return SystemColumns;
- }
+ }
ui64 AddRows(const TVector<TOwnedCellVec>& batch, TMaybe<ui64> shardId, const THolderFactory& holderFactory);
ui64 AddRows(const arrow::RecordBatch& batch, TMaybe<ui64> shardId, const THolderFactory& holderFactory);
NUdf::TUnboxedValue TakeRow();
-
+
bool IsEmpty() const {
return RowBatches.empty();
}
@@ -84,7 +84,7 @@ public:
void Clear() {
RowBatches.clear();
}
-
+
public:
ui64 TaskId = 0;
TTableId TableId;
@@ -98,7 +98,7 @@ public:
struct TBasicStats {
size_t Rows = 0;
size_t Bytes = 0;
- ui32 AffectedShards = 0;
+ ui32 AffectedShards = 0;
};
struct TProfileStats {
@@ -156,7 +156,7 @@ private:
TMap<ui32, TScanData> Scans;
};
-TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TScanData& scanData);
+TIntrusivePtr<IKqpTableReader> CreateKqpTableReader(TKqpScanComputeContext::TScanData& scanData);
} // namespace NMiniKQL
} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.cpp b/ydb/core/kqp/ut/common/kqp_ut_common.cpp
index 9bc7889f92f..5659da5ecf8 100644
--- a/ydb/core/kqp/ut/common/kqp_ut_common.cpp
+++ b/ydb/core/kqp/ut/common/kqp_ut_common.cpp
@@ -638,61 +638,61 @@ TString StreamResultToYson(NYdb::NScripting::TYqlResultPartIterator& it) {
return out.Str();
}
-template<typename TIterator>
-TCollectedStreamResult CollectStreamResultImpl(TIterator& it) {
- TCollectedStreamResult res;
-
- TStringStream out;
+template<typename TIterator>
+TCollectedStreamResult CollectStreamResultImpl(TIterator& it) {
+ TCollectedStreamResult res;
+
+ TStringStream out;
NYson::TYsonWriter resultSetWriter(&out, NYson::EYsonFormat::Text, ::NYson::EYsonType::Node, true);
- resultSetWriter.OnBeginList();
-
- for (;;) {
- auto streamPart = it.ReadNext().GetValueSync();
- if (!streamPart.IsSuccess()) {
- UNIT_ASSERT_C(streamPart.EOS(), streamPart.GetIssues().ToString());
- break;
- }
-
+ resultSetWriter.OnBeginList();
+
+ for (;;) {
+ auto streamPart = it.ReadNext().GetValueSync();
+ if (!streamPart.IsSuccess()) {
+ UNIT_ASSERT_C(streamPart.EOS(), streamPart.GetIssues().ToString());
+ break;
+ }
+
if constexpr (std::is_same_v<TIterator, NYdb::NTable::TScanQueryPartIterator>) {
UNIT_ASSERT_C(streamPart.HasResultSet() || streamPart.HasQueryStats(),
"Unexpected empty scan query response.");
}
- if (streamPart.HasResultSet()) {
+ if (streamPart.HasResultSet()) {
auto resultSet = streamPart.ExtractResultSet();
PrintResultSet(resultSet, resultSetWriter);
- }
-
- if constexpr (std::is_same_v<TIterator, NYdb::NTable::TScanQueryPartIterator>) {
- if (streamPart.HasQueryStats() ) {
- res.QueryStats = NYdb::TProtoAccessor::GetProto(streamPart.GetQueryStats());
+ }
+
+ if constexpr (std::is_same_v<TIterator, NYdb::NTable::TScanQueryPartIterator>) {
+ if (streamPart.HasQueryStats() ) {
+ res.QueryStats = NYdb::TProtoAccessor::GetProto(streamPart.GetQueryStats());
auto plan = res.QueryStats->query_plan();
if (!plan.empty()) {
res.PlanJson = plan;
}
- }
- } else {
- if (streamPart.HasPlan()) {
- res.PlanJson = streamPart.ExtractPlan();
- }
- }
- }
-
- resultSetWriter.OnEndList();
-
- res.ResultSetYson = out.Str();
- return res;
-}
-
-TCollectedStreamResult CollectStreamResult(NYdb::NExperimental::TStreamPartIterator& it) {
- return CollectStreamResultImpl(it);
-}
-
-TCollectedStreamResult CollectStreamResult(NYdb::NTable::TScanQueryPartIterator& it) {
- return CollectStreamResultImpl(it);
-}
-
+ }
+ } else {
+ if (streamPart.HasPlan()) {
+ res.PlanJson = streamPart.ExtractPlan();
+ }
+ }
+ }
+
+ resultSetWriter.OnEndList();
+
+ res.ResultSetYson = out.Str();
+ return res;
+}
+
+TCollectedStreamResult CollectStreamResult(NYdb::NExperimental::TStreamPartIterator& it) {
+ return CollectStreamResultImpl(it);
+}
+
+TCollectedStreamResult CollectStreamResult(NYdb::NTable::TScanQueryPartIterator& it) {
+ return CollectStreamResultImpl(it);
+}
+
TString ReadTablePartToYson(NYdb::NTable::TSession session, const TString& table) {
auto it = session.ReadTable(table).GetValueSync();
UNIT_ASSERT(it.IsSuccess());
@@ -710,9 +710,9 @@ TString ReadTablePartToYson(NYdb::NTable::TSession session, const TString& table
ui32 CountPlanNodesByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value) {
ui32 result = 0;
-
- if (plan.IsArray()) {
- for (const auto &node: plan.GetArray()) {
+
+ if (plan.IsArray()) {
+ for (const auto &node: plan.GetArray()) {
result += CountPlanNodesByKv(node, key, value);
}
return result;
@@ -747,29 +747,29 @@ ui32 CountPlanNodesByKv(const NJson::TJsonValue& plan, const TString& key, const
NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value) {
if (plan.IsArray()) {
for (const auto &node: plan.GetArray()) {
- auto stage = FindPlanNodeByKv(node, key, value);
- if (stage.IsDefined()) {
- return stage;
- }
- }
- } else if (plan.IsMap()) {
- auto map = plan.GetMap();
+ auto stage = FindPlanNodeByKv(node, key, value);
+ if (stage.IsDefined()) {
+ return stage;
+ }
+ }
+ } else if (plan.IsMap()) {
+ auto map = plan.GetMap();
if (map.contains(key) && map.at(key).GetStringRobust() == value) {
- return plan;
- }
- if (map.contains("Plans")) {
- for (const auto &node: map["Plans"].GetArraySafe()) {
- auto stage = FindPlanNodeByKv(node, key, value);
- if (stage.IsDefined()) {
- return stage;
- }
- }
- } else if (map.contains("Plan")) {
- auto stage = FindPlanNodeByKv(map.at("Plan"), key, value);
- if (stage.IsDefined()) {
- return stage;
- }
- }
+ return plan;
+ }
+ if (map.contains("Plans")) {
+ for (const auto &node: map["Plans"].GetArraySafe()) {
+ auto stage = FindPlanNodeByKv(node, key, value);
+ if (stage.IsDefined()) {
+ return stage;
+ }
+ }
+ } else if (map.contains("Plan")) {
+ auto stage = FindPlanNodeByKv(map.at("Plan"), key, value);
+ if (stage.IsDefined()) {
+ return stage;
+ }
+ }
if (map.contains("Operators")) {
for (const auto &node : map["Operators"].GetArraySafe()) {
@@ -779,13 +779,13 @@ NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString&
}
}
}
- } else {
- Y_ASSERT(false);
- }
-
- return NJson::TJsonValue();
-}
-
+ } else {
+ Y_ASSERT(false);
+ }
+
+ return NJson::TJsonValue();
+}
+
void CreateSampleTablesWithIndex(TSession& session) {
auto res = session.ExecuteSchemeQuery(R"(
--!syntax_v1
diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.h b/ydb/core/kqp/ut/common/kqp_ut_common.h
index c1d64b65c3e..ac82850edbb 100644
--- a/ydb/core/kqp/ut/common/kqp_ut_common.h
+++ b/ydb/core/kqp/ut/common/kqp_ut_common.h
@@ -12,7 +12,7 @@
#include <ydb/library/yql/core/issue/yql_issue.h>
-#include <library/cpp/json/json_reader.h>
+#include <library/cpp/json/json_reader.h>
#include <library/cpp/testing/unittest/tests_data.h>
#include <library/cpp/testing/unittest/registar.h>
#include <library/cpp/yson/writer.h>
@@ -54,17 +54,17 @@
void N(NUnitTest::TTestContext&)
template <bool UseNewEngine, bool ForceVersionV1 = false>
-TString Query(const TString& tmpl) {
+TString Query(const TString& tmpl) {
return TStringBuilder()
<< (ForceVersionV1 ? "--!syntax_v1\n" : "")
<< "PRAGMA Kikimr.UseNewEngine = '" << (UseNewEngine ? "true" : "false") << "';" << Endl
//<< (UseNewEngine ? "PRAGMA Kikimr.UseNewEngine = 'true';" : "")
<< tmpl;
-}
-
+}
+
#define Q_(expr) Query<UseNewEngine, false>(expr)
#define Q1_(expr) Query<UseNewEngine, true>(expr)
-
+
namespace NKikimr {
namespace NKqp {
@@ -151,15 +151,15 @@ private:
THolder<NYdb::TDriver> Driver;
};
-struct TCollectedStreamResult {
- TString ResultSetYson;
+struct TCollectedStreamResult {
+ TString ResultSetYson;
TMaybe<TString> PlanJson;
TMaybe<Ydb::TableStats::QueryStats> QueryStats;
-};
-
-TCollectedStreamResult CollectStreamResult(NYdb::NExperimental::TStreamPartIterator& it);
-TCollectedStreamResult CollectStreamResult(NYdb::NTable::TScanQueryPartIterator& it);
-
+};
+
+TCollectedStreamResult CollectStreamResult(NYdb::NExperimental::TStreamPartIterator& it);
+TCollectedStreamResult CollectStreamResult(NYdb::NTable::TScanQueryPartIterator& it);
+
enum class EIndexTypeSql {
Global,
GlobalSync,
@@ -223,8 +223,8 @@ TString StreamResultToYson(NYdb::NTable::TScanQueryPartIterator& it);
TString StreamResultToYson(NYdb::NScripting::TYqlResultPartIterator& it);
ui32 CountPlanNodesByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value);
-NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value);
-
+NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& key, const TString& value);
+
TString ReadTablePartToYson(NYdb::NTable::TSession session, const TString& table);
inline void AssertSuccessResult(const NYdb::TStatus& result) {
diff --git a/ydb/core/kqp/ut/kqp_explain_ut.cpp b/ydb/core/kqp/ut/kqp_explain_ut.cpp
index c3a06836367..e8bca37e9b0 100644
--- a/ydb/core/kqp/ut/kqp_explain_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_explain_ut.cpp
@@ -1,141 +1,141 @@
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-
-namespace {
-void CreateSampleTables(TKikimrRunner& kikimr) {
- kikimr.GetTestClient().CreateTable("/Root", R"(
- Name: "FourShard"
- Columns { Name: "Key", Type: "Uint64" }
- Columns { Name: "Value1", Type: "String" }
- Columns { Name: "Value2", Type: "String" }
- KeyColumnNames: ["Key"],
- SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 100 } } } }
- SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 200 } } } }
- SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 300 } } } }
- )");
-
- TTableClient tableClient{kikimr.GetDriver()};
- auto session = tableClient.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExecuteDataQuery(R"(
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+
+namespace {
+void CreateSampleTables(TKikimrRunner& kikimr) {
+ kikimr.GetTestClient().CreateTable("/Root", R"(
+ Name: "FourShard"
+ Columns { Name: "Key", Type: "Uint64" }
+ Columns { Name: "Value1", Type: "String" }
+ Columns { Name: "Value2", Type: "String" }
+ KeyColumnNames: ["Key"],
+ SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 100 } } } }
+ SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 200 } } } }
+ SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 300 } } } }
+ )");
+
+ TTableClient tableClient{kikimr.GetDriver()};
+ auto session = tableClient.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExecuteDataQuery(R"(
REPLACE INTO `/Root/FourShard` (Key, Value1, Value2) VALUES
- (1u, "Value-001", "1"),
- (2u, "Value-002", "2"),
- (101u, "Value-101", "101"),
- (102u, "Value-102", "102"),
- (201u, "Value-201", "201"),
- (202u, "Value-202", "202"),
- (301u, "Value-301", "301"),
- (302u, "Value-302", "302")
- )", TTxControl::BeginTx().CommitTx()).GetValueSync();
-
- UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
-
- session.Close();
-}
-
-}
-
-Y_UNIT_TEST_SUITE(KqpExplain) {
-
- Y_UNIT_TEST(Explain) {
- TKikimrRunner kikimr;
+ (1u, "Value-001", "1"),
+ (2u, "Value-002", "2"),
+ (101u, "Value-101", "101"),
+ (102u, "Value-102", "102"),
+ (201u, "Value-201", "201"),
+ (202u, "Value-202", "202"),
+ (301u, "Value-301", "301"),
+ (302u, "Value-302", "302")
+ )", TTxControl::BeginTx().CommitTx()).GetValueSync();
+
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+
+ session.Close();
+}
+
+}
+
+Y_UNIT_TEST_SUITE(KqpExplain) {
+
+ Y_UNIT_TEST(Explain) {
+ TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
TStreamExecScanQuerySettings settings;
- settings.Explain(true);
-
+ settings.Explain(true);
+
auto it = db.StreamExecuteScanQuery(R"(
SELECT count(*) FROM `/Root/EightShard` AS t JOIN `/Root/KeyValue` AS kv ON t.Data = kv.Key;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT(res.PlanJson);
-
+
Cerr << *res.PlanJson;
- NJson::TJsonValue plan;
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
-
+
auto join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan");
- UNIT_ASSERT(join.IsDefined());
+ UNIT_ASSERT(join.IsDefined());
auto left = FindPlanNodeByKv(join, "Table", "EightShard");
- UNIT_ASSERT(left.IsDefined());
+ UNIT_ASSERT(left.IsDefined());
auto right = FindPlanNodeByKv(join, "Table", "KeyValue");
- UNIT_ASSERT(right.IsDefined());
- }
-
- Y_UNIT_TEST(ExplainStream) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- TStreamExecScanQuerySettings settings;
- settings.Explain(true);
-
- auto it = db.StreamExecuteScanQuery(R"(
+ UNIT_ASSERT(right.IsDefined());
+ }
+
+ Y_UNIT_TEST(ExplainStream) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ TStreamExecScanQuerySettings settings;
+ settings.Explain(true);
+
+ auto it = db.StreamExecuteScanQuery(R"(
SELECT count(*) FROM `/Root/EightShard` AS t JOIN `/Root/KeyValue` AS kv ON t.Data = kv.Key;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT(res.PlanJson);
Cerr << *res.PlanJson;
-
- NJson::TJsonValue plan;
+
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
-
+
auto join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan");
- UNIT_ASSERT(join.IsDefined());
+ UNIT_ASSERT(join.IsDefined());
auto left = FindPlanNodeByKv(join, "Table", "EightShard");
- UNIT_ASSERT(left.IsDefined());
+ UNIT_ASSERT(left.IsDefined());
auto right = FindPlanNodeByKv(join, "Table", "KeyValue");
- UNIT_ASSERT(right.IsDefined());
- }
-
- Y_UNIT_TEST(ExplainScanQueryWithParams) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- TStreamExecScanQuerySettings settings;
- settings.Explain(true);
-
- auto it = db.StreamExecuteScanQuery(R"(
+ UNIT_ASSERT(right.IsDefined());
+ }
+
+ Y_UNIT_TEST(ExplainScanQueryWithParams) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ TStreamExecScanQuerySettings settings;
+ settings.Explain(true);
+
+ auto it = db.StreamExecuteScanQuery(R"(
--SELECT count(*) FROM `/Root/EightShard` AS t JOIN `/Root/KeyValue` AS kv ON t.Data = kv.Key;
- PRAGMA Kikimr.UseNewEngine = "false";
- DECLARE $value as Utf8;
- SELECT $value as value;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- }
-
- Y_UNIT_TEST(AggGroupLimit) {
- TKikimrRunner kikimr;
+ PRAGMA Kikimr.UseNewEngine = "false";
+ DECLARE $value as Utf8;
+ SELECT $value as value;
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ }
+
+ Y_UNIT_TEST(AggGroupLimit) {
+ TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
TStreamExecScanQuerySettings settings;
- settings.Explain(true);
-
+ settings.Explain(true);
+
auto it = db.StreamExecuteScanQuery(R"(
SELECT min(Message), max(Message) FROM `/Root/Logs` WHERE Ts > 1 and Ts <= 4 or App="ydb" GROUP BY App;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT(res.PlanJson);
Cerr << *res.PlanJson;
-
- NJson::TJsonValue plan;
+
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
-
+
auto read = FindPlanNodeByKv(plan, "Node Type", "Aggregate-Filter-TableFullScan");
- UNIT_ASSERT(read.IsDefined());
+ UNIT_ASSERT(read.IsDefined());
auto tables = read.GetMapSafe().at("Tables").GetArraySafe();
UNIT_ASSERT(tables[0].GetStringSafe() == "Logs");
auto shuffle = FindPlanNodeByKv(plan, "Node Type", "HashShuffle");
@@ -147,94 +147,94 @@ Y_UNIT_TEST_SUITE(KqpExplain) {
UNIT_ASSERT(aggregate.GetMapSafe().at("GroupBy").GetStringSafe() == "item.App");
UNIT_ASSERT(aggregate.GetMapSafe().at("Aggregation").GetStringSafe() ==
"{_yql_agg_0: MIN(item.Message),_yql_agg_1: MAX(item.Message)}");
- }
-
- Y_UNIT_TEST(ComplexJoin) {
- TKikimrRunner kikimr;
- CreateSampleTables(kikimr);
+ }
+
+ Y_UNIT_TEST(ComplexJoin) {
+ TKikimrRunner kikimr;
+ CreateSampleTables(kikimr);
auto db = kikimr.GetTableClient();
TStreamExecScanQuerySettings settings;
- settings.Explain(true);
-
+ settings.Explain(true);
+
auto it = db.StreamExecuteScanQuery(R"(
- $join = (
- SELECT l.Key as Key, l.Text as Text, l.Data as Data, r.Value1 as Value1, r.Value2 as Value2
+ $join = (
+ SELECT l.Key as Key, l.Text as Text, l.Data as Data, r.Value1 as Value1, r.Value2 as Value2
FROM `/Root/EightShard` AS l JOIN `/Root/FourShard` AS r ON l.Key = r.Key
- );
- SELECT Key, COUNT(*) AS Cnt
- FROM $join
- WHERE Cast(Data As Int64) < (Key - 100) and Value1 != 'Value-101'
- GROUP BY Key
- UNION ALL
+ );
+ SELECT Key, COUNT(*) AS Cnt
+ FROM $join
+ WHERE Cast(Data As Int64) < (Key - 100) and Value1 != 'Value-101'
+ GROUP BY Key
+ UNION ALL
(SELECT Key FROM `/Root/KeyValue` ORDER BY Key LIMIT 1)
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT(res.PlanJson);
-
- NJson::TJsonValue plan;
+
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
-
+
auto join = FindPlanNodeByKv(
plan,
"Node Type",
"Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan"
);
- UNIT_ASSERT(join.IsDefined());
+ UNIT_ASSERT(join.IsDefined());
auto left = FindPlanNodeByKv(join, "Table", "EightShard");
- UNIT_ASSERT(left.IsDefined());
+ UNIT_ASSERT(left.IsDefined());
auto right = FindPlanNodeByKv(join, "Table", "FourShard");
- UNIT_ASSERT(right.IsDefined());
- }
-
- Y_UNIT_TEST(PrecomputeRange) {
- TKikimrRunner kikimr;
+ UNIT_ASSERT(right.IsDefined());
+ }
+
+ Y_UNIT_TEST(PrecomputeRange) {
+ TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
TStreamExecScanQuerySettings settings;
- settings.Explain(true);
-
+ settings.Explain(true);
+
auto it = db.StreamExecuteScanQuery(R"(
SELECT * FROM `/Root/EightShard` WHERE Key BETWEEN 150 AND 266 ORDER BY Data LIMIT 4;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT(res.PlanJson);
Cerr << *res.PlanJson;
-
- NJson::TJsonValue plan;
+
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
-
+
auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangesScan");
- UNIT_ASSERT(node.IsDefined());
+ UNIT_ASSERT(node.IsDefined());
auto operators = node.GetMapSafe().at("Operators").GetArraySafe();
UNIT_ASSERT(operators[1].GetMapSafe().at("Name") == "TableRangesScan");
auto& readRanges = operators[1].GetMapSafe().at("ReadRanges").GetArraySafe();
UNIT_ASSERT(readRanges[0] == "Key [150, 266]");
- }
-
- Y_UNIT_TEST(CompoundKeyRange) {
- TKikimrRunner kikimr;
+ }
+
+ Y_UNIT_TEST(CompoundKeyRange) {
+ TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
TStreamExecScanQuerySettings settings;
- settings.Explain(true);
-
+ settings.Explain(true);
+
auto it = db.StreamExecuteScanQuery(R"(
PRAGMA Kikimr.OptEnablePredicateExtract = "false";
SELECT * FROM `/Root/Logs` WHERE App = "new_app_1" AND Host < "xyz" AND Ts = (42+7) Limit 10;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT(res.PlanJson);
Cerr << *res.PlanJson;
-
- NJson::TJsonValue plan;
+
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
-
+
auto read = FindPlanNodeByKv(plan, "Node Type", "Limit-TablePointLookup");
auto& operators = read.GetMapSafe().at("Operators").GetArraySafe();
UNIT_ASSERT(operators.size() == 2);
@@ -242,134 +242,134 @@ Y_UNIT_TEST_SUITE(KqpExplain) {
auto& lookup = operators[1].GetMapSafe();
UNIT_ASSERT(lookup.at("Name") == "TablePointLookup");
UNIT_ASSERT(lookup.at("ReadRange").GetArraySafe()[0] == "App (new_app_1)");
- }
-
- Y_UNIT_TEST(SortStage) {
- TKikimrRunner kikimr;
+ }
+
+ Y_UNIT_TEST(SortStage) {
+ TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
TStreamExecScanQuerySettings settings;
- settings.Explain(true);
-
+ settings.Explain(true);
+
auto it = db.StreamExecuteScanQuery(R"(
SELECT * FROM `/Root/EightShard` WHERE Key BETWEEN 150 AND 266 ORDER BY Text;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT(res.PlanJson);
Cerr << *res.PlanJson;
-
- NJson::TJsonValue plan;
+
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
auto scanSort = FindPlanNodeByKv(plan, "Node Type", "Sort-TableRangesScan");
- UNIT_ASSERT(scanSort.IsDefined());
- }
-
- Y_UNIT_TEST(LimitOffset) {
- TKikimrRunner kikimr;
+ UNIT_ASSERT(scanSort.IsDefined());
+ }
+
+ Y_UNIT_TEST(LimitOffset) {
+ TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
TStreamExecScanQuerySettings settings;
- settings.Explain(true);
-
+ settings.Explain(true);
+
auto it = db.StreamExecuteScanQuery(R"(
SELECT * FROM `/Root/EightShard` ORDER BY Text LIMIT 10 OFFSET 15;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT(res.PlanJson);
Cerr << *res.PlanJson;
-
- NJson::TJsonValue plan;
+
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
-
- auto limit = FindPlanNodeByKv(plan, "Limit", "10");
- UNIT_ASSERT(limit.IsDefined());
- auto offset = FindPlanNodeByKv(plan, "Offset", "15");
- UNIT_ASSERT(offset.IsDefined());
- }
-
- Y_UNIT_TEST(SelfJoin3xSameLabels) {
- TKikimrRunner kikimr;
+
+ auto limit = FindPlanNodeByKv(plan, "Limit", "10");
+ UNIT_ASSERT(limit.IsDefined());
+ auto offset = FindPlanNodeByKv(plan, "Offset", "15");
+ UNIT_ASSERT(offset.IsDefined());
+ }
+
+ Y_UNIT_TEST(SelfJoin3xSameLabels) {
+ TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
TStreamExecScanQuerySettings settings;
- settings.Explain(true);
-
+ settings.Explain(true);
+
auto it = db.StreamExecuteScanQuery(R"(
- $foo = (
- SELECT t1.Key AS Key
+ $foo = (
+ SELECT t1.Key AS Key
FROM `/Root/KeyValue` AS t1
JOIN `/Root/KeyValue` AS t2
- ON t1.Key = t2.Key
- GROUP BY t1.Key
- );
- SELECT t1.Key AS Key
- FROM $foo AS Foo
+ ON t1.Key = t2.Key
+ GROUP BY t1.Key
+ );
+ SELECT t1.Key AS Key
+ FROM $foo AS Foo
JOIN `/Root/KeyValue` AS t1
- ON t1.Key = Foo.Key
- ORDER BY Key
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ ON t1.Key = Foo.Key
+ ORDER BY Key
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT(res.PlanJson);
Cerr << *res.PlanJson;
-
- NJson::TJsonValue plan;
+
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
-
+
auto join1 = FindPlanNodeByKv(plan, "Node Type", "Sort-InnerJoin (MapJoin)-Filter-Aggregate-Sort");
- UNIT_ASSERT(join1.IsDefined());
+ UNIT_ASSERT(join1.IsDefined());
auto join2 = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter");
- UNIT_ASSERT(join2.IsDefined());
- }
-
- Y_UNIT_TEST(PureExpr) {
- TKikimrRunner kikimr;
+ UNIT_ASSERT(join2.IsDefined());
+ }
+
+ Y_UNIT_TEST(PureExpr) {
+ TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
TStreamExecScanQuerySettings settings;
- settings.Explain(true);
-
+ settings.Explain(true);
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT 1,2,3 UNION ALL SELECT 4,5,6;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ SELECT 1,2,3 UNION ALL SELECT 4,5,6;
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT(res.PlanJson);
Cerr << *res.PlanJson;
-
- NJson::TJsonValue plan;
+
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
-
- auto constExpr = FindPlanNodeByKv(plan, "Node Type", "ConstantExpr");
- UNIT_ASSERT(constExpr.IsDefined());
- }
-
- Y_UNIT_TEST(MultiUsedStage) {
- NKikimrConfig::TAppConfig appCfg;
- auto* spilling = appCfg.MutableTableServiceConfig()->MutableSpillingServiceConfig()->MutableLocalFileConfig();
- spilling->SetEnable(true);
- spilling->SetRoot("./spilling/");
- TKikimrRunner kikimr(appCfg);
-
+
+ auto constExpr = FindPlanNodeByKv(plan, "Node Type", "ConstantExpr");
+ UNIT_ASSERT(constExpr.IsDefined());
+ }
+
+ Y_UNIT_TEST(MultiUsedStage) {
+ NKikimrConfig::TAppConfig appCfg;
+ auto* spilling = appCfg.MutableTableServiceConfig()->MutableSpillingServiceConfig()->MutableLocalFileConfig();
+ spilling->SetEnable(true);
+ spilling->SetRoot("./spilling/");
+ TKikimrRunner kikimr(appCfg);
+
auto db = kikimr.GetTableClient();
TStreamExecScanQuerySettings settings;
- settings.Explain(true);
-
+ settings.Explain(true);
+
auto it = db.StreamExecuteScanQuery(R"(
select count(*) from `/Root/KeyValue` AS t1 join `/Root/KeyValue` AS t2 on t1.Key = t2.Key;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT(res.PlanJson);
Cerr << *res.PlanJson;
-
- NJson::TJsonValue plan;
+
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
-
+
bool containCte = false;
auto& plans = plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans");
for (auto& planNode : plans.GetArraySafe()) {
@@ -381,90 +381,90 @@ Y_UNIT_TEST_SUITE(KqpExplain) {
}
UNIT_ASSERT(containCte);
- }
-
- Y_UNIT_TEST(SqlIn) {
- TKikimrRunner kikimr;
+ }
+
+ Y_UNIT_TEST(SqlIn) {
+ TKikimrRunner kikimr;
CreateSampleTables(kikimr);
TStreamExecScanQuerySettings settings;
- settings.Explain(true);
+ settings.Explain(true);
auto db = kikimr.GetTableClient();
-
- auto query = R"(
+
+ auto query = R"(
PRAGMA Kikimr.OptEnablePredicateExtract = "false";
- SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42)
- ORDER BY Key
- )";
-
+ SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42)
+ ORDER BY Key
+ )";
+
auto it = db.StreamExecuteScanQuery(query, settings).GetValueSync();
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT(res.PlanJson);
Cerr << *res.PlanJson;
-
- NJson::TJsonValue plan;
+
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
-
+
auto unionNode = FindPlanNodeByKv(plan, "Node Type", "Sort-Union");
- UNIT_ASSERT_EQUAL(unionNode.GetMap().at("Plans").GetArraySafe().size(), 4);
- }
-
- Y_UNIT_TEST(ExplainDataQueryOldEngine) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExplainDataQuery(R"(
- SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42) ORDER BY Key;
- )").ExtractValueSync();
- result.GetIssues().PrintTo(Cerr);
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
- UNIT_ASSERT_EQUAL(plan.GetMapSafe().at("tables").GetArraySafe()[0].GetMapSafe().at("name").GetStringSafe(), "/Root/KeyValue");
- }
-
- Y_UNIT_TEST(ExplainDataQueryNewEngine) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExplainDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42) ORDER BY Key;
- SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 4*2) ORDER BY Key;
- SELECT count(distinct Value) FROM `/Root/KeyValue` WHERE Key > 20 and Key <= 120;
- SELECT count(distinct Value) FROM `/Root/KeyValue`;
- SELECT count(distinct Value) FROM `/Root/KeyValue` WHERE Key >= 10;
- )").ExtractValueSync();
- result.GetIssues().PrintTo(Cerr);
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+ UNIT_ASSERT_EQUAL(unionNode.GetMap().at("Plans").GetArraySafe().size(), 4);
+ }
+
+ Y_UNIT_TEST(ExplainDataQueryOldEngine) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExplainDataQuery(R"(
+ SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42) ORDER BY Key;
+ )").ExtractValueSync();
+ result.GetIssues().PrintTo(Cerr);
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+ UNIT_ASSERT_EQUAL(plan.GetMapSafe().at("tables").GetArraySafe()[0].GetMapSafe().at("name").GetStringSafe(), "/Root/KeyValue");
+ }
+
+ Y_UNIT_TEST(ExplainDataQueryNewEngine) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExplainDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 42) ORDER BY Key;
+ SELECT Key, Value FROM `/Root/KeyValue` WHERE Key IN (1, 2, 3, 4*2) ORDER BY Key;
+ SELECT count(distinct Value) FROM `/Root/KeyValue` WHERE Key > 20 and Key <= 120;
+ SELECT count(distinct Value) FROM `/Root/KeyValue`;
+ SELECT count(distinct Value) FROM `/Root/KeyValue` WHERE Key >= 10;
+ )").ExtractValueSync();
+ result.GetIssues().PrintTo(Cerr);
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
auto node = FindPlanNodeByKv(plan, "Name", "TableRangeScan");
UNIT_ASSERT_EQUAL(node.GetMapSafe().at("Table").GetStringSafe(), "KeyValue");
node = FindPlanNodeByKv(plan, "Name", "TableFullScan");
UNIT_ASSERT_EQUAL(node.GetMapSafe().at("Table").GetStringSafe(), "KeyValue");
node = FindPlanNodeByKv(plan, "Name", "TablePointLookup");
UNIT_ASSERT_EQUAL(node.GetMapSafe().at("Table").GetStringSafe(), "KeyValue");
- }
-
- Y_UNIT_TEST(FewEffects) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExplainDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+ }
+
+ Y_UNIT_TEST(FewEffects) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExplainDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
UPDATE `/Root/EightShard` SET Data=Data+1;
UPDATE `/Root/EightShard` SET Data=Data-1 WHERE Key In (100,200,300);
DELETE FROM `/Root/EightShard` WHERE Key > 350;
- )").ExtractValueSync();
+ )").ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetPlan(), &plan, true);
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetPlan(), &plan, true);
// Cerr << plan << Endl;
@@ -502,28 +502,28 @@ Y_UNIT_TEST_SUITE(KqpExplain) {
UNIT_ASSERT_VALUES_EQUAL(counter["FullScan"], fullScansCount);
UNIT_ASSERT_VALUES_EQUAL(counter["Scan"], rangeScansCount);
UNIT_ASSERT_VALUES_EQUAL(counter["Lookup"], lookupsCount);
- }
-
- Y_UNIT_TEST(ExplainDataQueryWithParams) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- auto result = session.ExplainDataQuery(R"(
- PRAGMA Kikimr.UseNewEngine = "false";
- DECLARE $value as Utf8;
- SELECT $value as value;
- )").ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- auto result2 = session.ExplainDataQuery(R"(
- PRAGMA Kikimr.UseNewEngine = "true";
- DECLARE $value as Utf8;
- SELECT $value as value;
- )").ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), EStatus::SUCCESS, result2.GetIssues().ToString());
- }
-
+ }
+
+ Y_UNIT_TEST(ExplainDataQueryWithParams) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ auto result = session.ExplainDataQuery(R"(
+ PRAGMA Kikimr.UseNewEngine = "false";
+ DECLARE $value as Utf8;
+ SELECT $value as value;
+ )").ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ auto result2 = session.ExplainDataQuery(R"(
+ PRAGMA Kikimr.UseNewEngine = "true";
+ DECLARE $value as Utf8;
+ SELECT $value as value;
+ )").ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), EStatus::SUCCESS, result2.GetIssues().ToString());
+ }
+
Y_UNIT_TEST(FullOuterJoin) {
TKikimrRunner kikimr;
CreateSampleTables(kikimr);
@@ -712,9 +712,9 @@ Y_UNIT_TEST_SUITE(KqpExplain) {
<< " but received: " << filter.GetMapSafe().at("Predicate"));
}
}
-}
-
-} // namespace NKqp
-} // namespace NKikimr
-
-
+}
+
+} // namespace NKqp
+} // namespace NKikimr
+
+
diff --git a/ydb/core/kqp/ut/kqp_join_ut.cpp b/ydb/core/kqp/ut/kqp_join_ut.cpp
index 8fa59d31b36..3c28faa0e99 100644
--- a/ydb/core/kqp/ut/kqp_join_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_join_ut.cpp
@@ -301,21 +301,21 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
AssertTableReads(result, "/Root/Join1_2", 3);
}
- Y_UNIT_TEST_NEW_ENGINE(LeftJoinWithNull) {
+ Y_UNIT_TEST_NEW_ENGINE(LeftJoinWithNull) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTables(session);
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT * FROM `/Root/Join1_1` AS t1
INNER JOIN `/Root/Join1_2` AS t2
ON t1.Fk21 == t2.Key1 AND t1.Fk22 == t2.Key2
LEFT JOIN `/Root/Join1_3` AS t3
ON t2.Fk3 = t3.Key
WHERE t1.Value == "Value5" AND t2.Value == "Value31";
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT(result.IsSuccess());
CompareYson(R"([[[108];["One"];[8];["Value5"];#;[108];["One"];["Value31"];#;#]])",
@@ -762,14 +762,14 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
FormatResultSetYson(result.GetResultSet(0)));
}
- Y_UNIT_TEST_NEW_ENGINE(JoinAggregate) {
+ Y_UNIT_TEST_NEW_ENGINE(JoinAggregate) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTables(session);
{
- auto result = session.ExecuteDataQuery(Q_(R"(
+ auto result = session.ExecuteDataQuery(Q_(R"(
SELECT t1.Value, SUM(t3.Value)
FROM `/Root/Join1_1` AS t1
INNER JOIN `/Root/Join1_2` AS t2
@@ -777,7 +777,7 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
LEFT JOIN `/Root/Join1_3` AS t3
ON t2.Fk3 = t3.Key
GROUP BY t1.Value;
- )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
+ )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
CompareYson(R"([[["Value1"];[3004]];[["Value2"];[1001]];[["Value3"];[2006]];[["Value5"];#]])",
diff --git a/ydb/core/kqp/ut/kqp_newengine_ut.cpp b/ydb/core/kqp/ut/kqp_newengine_ut.cpp
index 3737687d8d6..3fca42841a9 100644
--- a/ydb/core/kqp/ut/kqp_newengine_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_newengine_ut.cpp
@@ -710,68 +710,68 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
}
#endif
- Y_UNIT_TEST(PureExpr) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
+ Y_UNIT_TEST(PureExpr) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT '42', 42, 5*5;
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT '42', 42, 5*5;
)", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx(), execSettings).ExtractValueSync();
-
- CompareYson(R"([["42";42;25]])", FormatResultSetYson(result.GetResultSet(0)));
+
+ CompareYson(R"([["42";42;25]])", FormatResultSetYson(result.GetResultSet(0)));
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
UNIT_ASSERT_EQUAL(stats.query_phases().size(), 1);
UNIT_ASSERT(stats.query_phases(0).table_access().size() == 0);
- }
-
- Y_UNIT_TEST(MultiStatement) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
+ }
+
+ Y_UNIT_TEST(MultiStatement) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
NYdb::NTable::TExecDataQuerySettings execSettings;
execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT 1;
- SELECT 2;
- SELECT 3;
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT 1;
+ SELECT 2;
+ SELECT 3;
)", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx(), execSettings).ExtractValueSync();
-
- CompareYson(R"([[1]])", FormatResultSetYson(result.GetResultSet(0)));
- CompareYson(R"([[2]])", FormatResultSetYson(result.GetResultSet(1)));
- CompareYson(R"([[3]])", FormatResultSetYson(result.GetResultSet(2)));
+
+ CompareYson(R"([[1]])", FormatResultSetYson(result.GetResultSet(0)));
+ CompareYson(R"([[2]])", FormatResultSetYson(result.GetResultSet(1)));
+ CompareYson(R"([[3]])", FormatResultSetYson(result.GetResultSet(2)));
auto& stats = NYdb::TProtoAccessor::GetProto(*result.GetStats());
UNIT_ASSERT_EQUAL(stats.query_phases().size(), 1);
UNIT_ASSERT(stats.query_phases(0).table_access().size() == 0);
- }
-
- Y_UNIT_TEST(MultiStatementMixPure) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
- SELECT 1;
+ }
+
+ Y_UNIT_TEST(MultiStatementMixPure) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+ SELECT 1;
SELECT Key FROM [/Root/TwoShard] ORDER BY Key DESC LIMIT 1;
- SELECT 2;
+ SELECT 2;
SELECT Key FROM [/Root/EightShard] ORDER BY Key ASC LIMIT 1;
- )", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync();
-
- CompareYson(R"([[1]])", FormatResultSetYson(result.GetResultSet(0)));
- CompareYson(R"([[[4000000003u]]])", FormatResultSetYson(result.GetResultSet(1)));
- CompareYson(R"([[2]])", FormatResultSetYson(result.GetResultSet(2)));
- CompareYson(R"([[[101u]]])", FormatResultSetYson(result.GetResultSet(3)));
- }
-
+ )", TTxControl::BeginTx(TTxSettings::OnlineRO()).CommitTx()).ExtractValueSync();
+
+ CompareYson(R"([[1]])", FormatResultSetYson(result.GetResultSet(0)));
+ CompareYson(R"([[[4000000003u]]])", FormatResultSetYson(result.GetResultSet(1)));
+ CompareYson(R"([[2]])", FormatResultSetYson(result.GetResultSet(2)));
+ CompareYson(R"([[[101u]]])", FormatResultSetYson(result.GetResultSet(3)));
+ }
+
Y_UNIT_TEST(LocksSingleShard) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
diff --git a/ydb/core/kqp/ut/kqp_scan_ut.cpp b/ydb/core/kqp/ut/kqp_scan_ut.cpp
index 347ca354412..f35b783bb9f 100644
--- a/ydb/core/kqp/ut/kqp_scan_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_scan_ut.cpp
@@ -1271,104 +1271,104 @@ Y_UNIT_TEST_SUITE(KqpScan) {
UNIT_ASSERT_EQUAL_C(part.GetStatus(), EStatus::PRECONDITION_FAILED, part.GetIssues().ToString());
}
- Y_UNIT_TEST(PureExpr) {
- TKikimrRunner kikimr(AppCfg());
+ Y_UNIT_TEST(PureExpr) {
+ TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
-
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT '42', 42, 5*5;
- )").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- auto res = StreamResultToYson(it);
- CompareYson(R"([["42";42;25]])", res);
- }
-
- Y_UNIT_TEST(UnionWithPureExpr) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ SELECT '42', 42, 5*5;
+ )").GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ auto res = StreamResultToYson(it);
+ CompareYson(R"([["42";42;25]])", res);
+ }
+
+ Y_UNIT_TEST(UnionWithPureExpr) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT '42', 42, 5*5
- UNION ALL
- SELECT 'forty-two';
- )").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- auto res = StreamResultToYson(it);
- CompareYson(R"([["42";[42];[25]];["forty-two";#;#]])", res);
- }
-
- Y_UNIT_TEST(MiltiExprWithPure) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ SELECT '42', 42, 5*5
+ UNION ALL
+ SELECT 'forty-two';
+ )").GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ auto res = StreamResultToYson(it);
+ CompareYson(R"([["42";[42];[25]];["forty-two";#;#]])", res);
+ }
+
+ Y_UNIT_TEST(MiltiExprWithPure) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
SELECT Key FROM `/Root/KeyValue` ORDER BY Key LIMIT 1;
- SELECT 2;
- )").GetValueSync();
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- auto part = it.ReadNext().GetValueSync();
- part.GetIssues().PrintTo(Cerr);
- UNIT_ASSERT_EQUAL_C(part.GetStatus(), EStatus::PRECONDITION_FAILED, part.GetIssues().ToString());
- }
-
- Y_UNIT_TEST(UnionBasic) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ SELECT 2;
+ )").GetValueSync();
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ auto part = it.ReadNext().GetValueSync();
+ part.GetIssues().PrintTo(Cerr);
+ UNIT_ASSERT_EQUAL_C(part.GetStatus(), EStatus::PRECONDITION_FAILED, part.GetIssues().ToString());
+ }
+
+ Y_UNIT_TEST(UnionBasic) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
(SELECT Key FROM `/Root/KeyValue` ORDER BY Key LIMIT 1)
- UNION ALL
+ UNION ALL
(SELECT Key FROM `/Root/EightShard` ORDER BY Key LIMIT 1);
- )").GetValueSync();
- auto res = StreamResultToYson(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[[1u]];[[101u]]])", res);
- }
-
- Y_UNIT_TEST(UnionMixed) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ )").GetValueSync();
+ auto res = StreamResultToYson(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[[1u]];[[101u]]])", res);
+ }
+
+ Y_UNIT_TEST(UnionMixed) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT 42
- UNION ALL
+ SELECT 42
+ UNION ALL
(SELECT Key FROM `/Root/EightShard` ORDER BY Key LIMIT 1);
- )").GetValueSync();
- auto res = StreamResultToYson(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[[42];#];[#;[101u]]])", res);
- }
-
- Y_UNIT_TEST(UnionThree) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ )").GetValueSync();
+ auto res = StreamResultToYson(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[[42];#];[#;[101u]]])", res);
+ }
+
+ Y_UNIT_TEST(UnionThree) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
(SELECT Key FROM `/Root/KeyValue` ORDER BY Key LIMIT 1)
- UNION ALL
+ UNION ALL
(SELECT Key FROM `/Root/EightShard` ORDER BY Key LIMIT 1)
- UNION ALL
+ UNION ALL
(SELECT Key FROM `/Root/TwoShard` ORDER BY Key DESC LIMIT 1);
- )").GetValueSync();
- auto res = StreamResultToYson(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[[1u]];[[101u]];[[4000000003u]]])", res);
- }
-
- Y_UNIT_TEST(UnionSameTable) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ )").GetValueSync();
+ auto res = StreamResultToYson(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[[1u]];[[101u]];[[4000000003u]]])", res);
+ }
+
+ Y_UNIT_TEST(UnionSameTable) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
(SELECT Key FROM `/Root/KeyValue` ORDER BY Key LIMIT 1)
- UNION ALL
+ UNION ALL
(SELECT Key FROM `/Root/KeyValue` ORDER BY Key LIMIT 1);
- )").GetValueSync();
- auto res = StreamResultToYson(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[[1u]];[[1u]]])", res);
- }
-
+ )").GetValueSync();
+ auto res = StreamResultToYson(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[[1u]];[[1u]]])", res);
+ }
+
Y_UNIT_TEST(UnionAggregate) {
TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
@@ -1385,142 +1385,142 @@ Y_UNIT_TEST_SUITE(KqpScan) {
CompareYson(R"([[[2u]];[[24u]];[[11000u]]])", res);
}
- Y_UNIT_TEST(CountDistinct) {
- TKikimrRunner kikimr(AppCfg());
+ Y_UNIT_TEST(CountDistinct) {
+ TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
-
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT count(*) AS c1, count(DISTINCT Value) AS c2
+ SELECT count(*) AS c1, count(DISTINCT Value) AS c2
FROM `/Root/KeyValue` GROUP BY Key;
- )").GetValueSync();
- auto res = StreamResultToYson(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[1u;1u];[1u;1u]])", res);
- }
-
- Y_UNIT_TEST(FullFrameWindow) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ )").GetValueSync();
+ auto res = StreamResultToYson(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[1u;1u];[1u;1u]])", res);
+ }
+
+ Y_UNIT_TEST(FullFrameWindow) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT group_total, count(*) FROM (
- SELECT Key, Text, SUM(Data) OVER w1 AS group_total
+ SELECT group_total, count(*) FROM (
+ SELECT Key, Text, SUM(Data) OVER w1 AS group_total
FROM `/Root/EightShard` WINDOW w1 AS (partition by Text)
- ) GROUP BY group_total ORDER BY group_total;
- )").GetValueSync();
- auto res = StreamResultToYson(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[[15];8u];[[16];8u];[[17];8u]])", res);
- }
-
- Y_UNIT_TEST(SimpleWindow) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ ) GROUP BY group_total ORDER BY group_total;
+ )").GetValueSync();
+ auto res = StreamResultToYson(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[[15];8u];[[16];8u];[[17];8u]])", res);
+ }
+
+ Y_UNIT_TEST(SimpleWindow) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT Text, running_total FROM (
- SELECT Key, Text, SUM(Data) OVER w1 AS running_total
+ SELECT Text, running_total FROM (
+ SELECT Key, Text, SUM(Data) OVER w1 AS running_total
FROM `/Root/EightShard`
- WHERE Text = 'Value2'
- WINDOW w1 AS (partition by Text order by Key)
- ) ORDER BY running_total;
- )").GetValueSync();
- auto res = StreamResultToYson(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- CompareYson(R"([
- [["Value2"];[3]];
- [["Value2"];[4]];
- [["Value2"];[6]];
- [["Value2"];[9]];
- [["Value2"];[10]];
- [["Value2"];[12]];
- [["Value2"];[15]];
- [["Value2"];[16]]
- ])", res);
- }
-
- Y_UNIT_TEST(TwoAggregatesOneFullFrameWindow) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ WHERE Text = 'Value2'
+ WINDOW w1 AS (partition by Text order by Key)
+ ) ORDER BY running_total;
+ )").GetValueSync();
+ auto res = StreamResultToYson(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ CompareYson(R"([
+ [["Value2"];[3]];
+ [["Value2"];[4]];
+ [["Value2"];[6]];
+ [["Value2"];[9]];
+ [["Value2"];[10]];
+ [["Value2"];[12]];
+ [["Value2"];[15]];
+ [["Value2"];[16]]
+ ])", res);
+ }
+
+ Y_UNIT_TEST(TwoAggregatesOneFullFrameWindow) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT tot, avg, count(*) FROM (
- SELECT Key, Text, SUM(Data) OVER w1 AS tot, avg(Data) OVER w1 AS avg
+ SELECT tot, avg, count(*) FROM (
+ SELECT Key, Text, SUM(Data) OVER w1 AS tot, avg(Data) OVER w1 AS avg
FROM `/Root/EightShard`
- WHERE Text = 'Value3'
- WINDOW w1 AS (partition by Text)
- ) GROUP BY tot, avg
- )").GetValueSync();
- auto res = StreamResultToYson(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[[17];[2.125];8u]])", res);
- }
-
- Y_UNIT_TEST(TwoAggregatesTwoWindows) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ WHERE Text = 'Value3'
+ WINDOW w1 AS (partition by Text)
+ ) GROUP BY tot, avg
+ )").GetValueSync();
+ auto res = StreamResultToYson(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[[17];[2.125];8u]])", res);
+ }
+
+ Y_UNIT_TEST(TwoAggregatesTwoWindows) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT Key, Text, sum(Data) OVER w1 AS tot, sum(Data) OVER w2 AS avg
+ SELECT Key, Text, sum(Data) OVER w1 AS tot, sum(Data) OVER w2 AS avg
FROM `/Root/EightShard`
- WHERE Text = 'Value2'
- WINDOW w1 AS (partition by Text),
- w2 AS (partition by Text order by Key)
- ORDER BY Key;
- )").GetValueSync();
- auto res = StreamResultToYson(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([
- [[102u];["Value2"];[16];[3]];
- [[202u];["Value2"];[16];[4]];
- [[302u];["Value2"];[16];[6]];
- [[402u];["Value2"];[16];[9]];
- [[502u];["Value2"];[16];[10]];
- [[602u];["Value2"];[16];[12]];
- [[702u];["Value2"];[16];[15]];
- [[802u];["Value2"];[16];[16]]
- ])", res);
- }
-
- Y_UNIT_TEST(CustomWindow) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ WHERE Text = 'Value2'
+ WINDOW w1 AS (partition by Text),
+ w2 AS (partition by Text order by Key)
+ ORDER BY Key;
+ )").GetValueSync();
+ auto res = StreamResultToYson(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([
+ [[102u];["Value2"];[16];[3]];
+ [[202u];["Value2"];[16];[4]];
+ [[302u];["Value2"];[16];[6]];
+ [[402u];["Value2"];[16];[9]];
+ [[502u];["Value2"];[16];[10]];
+ [[602u];["Value2"];[16];[12]];
+ [[702u];["Value2"];[16];[15]];
+ [[802u];["Value2"];[16];[16]]
+ ])", res);
+ }
+
+ Y_UNIT_TEST(CustomWindow) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
- SELECT Key, sum_short_win FROM (
- SELECT Key, Text, SUM(Data) OVER w1 AS sum_short_win
+ SELECT Key, sum_short_win FROM (
+ SELECT Key, Text, SUM(Data) OVER w1 AS sum_short_win
FROM `/Root/EightShard`
- WHERE Text = 'Value2'
- WINDOW w1 AS (partition by Text order by Key ROWS BETWEEN CURRENT ROW and 2 FOLLOWING)
- ) ORDER BY Key;
- )").GetValueSync();
- auto res = StreamResultToYson(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([
- [[102u];[6]];
- [[202u];[6]];
- [[302u];[6]];
- [[402u];[6]];
- [[502u];[6]];
- [[602u];[6]];
- [[702u];[4]];
- [[802u];[1]]
- ])", res);
- }
-
- Y_UNIT_TEST(EmptySet) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
+ WHERE Text = 'Value2'
+ WINDOW w1 AS (partition by Text order by Key ROWS BETWEEN CURRENT ROW and 2 FOLLOWING)
+ ) ORDER BY Key;
+ )").GetValueSync();
+ auto res = StreamResultToYson(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([
+ [[102u];[6]];
+ [[202u];[6]];
+ [[302u];[6]];
+ [[402u];[6]];
+ [[502u];[6]];
+ [[602u];[6]];
+ [[702u];[4]];
+ [[802u];[1]]
+ ])", res);
+ }
+
+ Y_UNIT_TEST(EmptySet) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
auto it = db.StreamExecuteScanQuery(R"(
SELECT Key FROM `/Root/EightShard` WHERE false;
- )").GetValueSync();
- auto res = StreamResultToYson(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson("[]",res);
- }
-
+ )").GetValueSync();
+ auto res = StreamResultToYson(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson("[]",res);
+ }
+
Y_UNIT_TEST(RestrictSqlV0) {
TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
@@ -1536,18 +1536,18 @@ Y_UNIT_TEST_SUITE(KqpScan) {
UNIT_ASSERT_EQUAL_C(part.GetStatus(), EStatus::GENERIC_ERROR, part.GetStatus());
}
- Y_UNIT_TEST(LongStringCombiner) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
- auto it = db.StreamExecuteScanQuery(R"(
+ Y_UNIT_TEST(LongStringCombiner) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
+ auto it = db.StreamExecuteScanQuery(R"(
SELECT min(Message), max(Message) FROM `/Root/Logs`;
- )").GetValueSync();
- auto res = StreamResultToYson(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[[" GET /index.html HTTP/1.1"];["some very very very very long string"]]])",res);
- }
-
+ )").GetValueSync();
+ auto res = StreamResultToYson(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[[" GET /index.html HTTP/1.1"];["some very very very very long string"]]])",res);
+ }
+
Y_UNIT_TEST(SqlInParameter) {
TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
@@ -1638,46 +1638,46 @@ Y_UNIT_TEST_SUITE(KqpScan) {
testFn(false);
testFn(true);
}
-
- Y_UNIT_TEST(SecondaryIndex) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- CreateSampleTablesWithIndex(session);
-
- // should work with ScanQuery
- auto itOk = db.StreamExecuteScanQuery(R"(
- PRAGMA AnsiInForEmptyOrNullableItemsCollections;
-
- SELECT Value
- FROM `/Root/SecondaryComplexKeys`
- WHERE (Fk1, Fk2) IN AsList((1, "Fk1"), (2, "Fk2"), (42, "Fk5"), (Null, "FkNull"))
- ORDER BY Value;
- )").GetValueSync();
- UNIT_ASSERT_C(itOk.IsSuccess(), itOk.GetIssues().ToString());
- CompareYson(R"([[["Payload1"]];[["Payload2"]]])", StreamResultToYson(itOk));
-
- // should NOT work with explicit view
- auto itIndex = db.StreamExecuteScanQuery(R"(
- PRAGMA AnsiInForEmptyOrNullableItemsCollections;
- SELECT Value
- FROM `/Root/SecondaryComplexKeys` VIEW Index
- WHERE (Fk1, Fk2) IN AsList((1, "Fk1"), (2, "Fk2"), (42, "Fk5"), (Null, "FkNull"))
- ORDER BY Value;
- )").GetValueSync();
-
- NYdb::EStatus status = NYdb::EStatus::STATUS_UNDEFINED;
- for (;;) {
- auto streamPart = itIndex.ReadNext().GetValueSync();
- if (!streamPart.IsSuccess()) {
- status = streamPart.GetStatus();
- break;
- }
- }
+
+ Y_UNIT_TEST(SecondaryIndex) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ CreateSampleTablesWithIndex(session);
+
+ // should work with ScanQuery
+ auto itOk = db.StreamExecuteScanQuery(R"(
+ PRAGMA AnsiInForEmptyOrNullableItemsCollections;
+
+ SELECT Value
+ FROM `/Root/SecondaryComplexKeys`
+ WHERE (Fk1, Fk2) IN AsList((1, "Fk1"), (2, "Fk2"), (42, "Fk5"), (Null, "FkNull"))
+ ORDER BY Value;
+ )").GetValueSync();
+ UNIT_ASSERT_C(itOk.IsSuccess(), itOk.GetIssues().ToString());
+ CompareYson(R"([[["Payload1"]];[["Payload2"]]])", StreamResultToYson(itOk));
+
+ // should NOT work with explicit view
+ auto itIndex = db.StreamExecuteScanQuery(R"(
+ PRAGMA AnsiInForEmptyOrNullableItemsCollections;
+ SELECT Value
+ FROM `/Root/SecondaryComplexKeys` VIEW Index
+ WHERE (Fk1, Fk2) IN AsList((1, "Fk1"), (2, "Fk2"), (42, "Fk5"), (Null, "FkNull"))
+ ORDER BY Value;
+ )").GetValueSync();
+
+ NYdb::EStatus status = NYdb::EStatus::STATUS_UNDEFINED;
+ for (;;) {
+ auto streamPart = itIndex.ReadNext().GetValueSync();
+ if (!streamPart.IsSuccess()) {
+ status = streamPart.GetStatus();
+ break;
+ }
+ }
UNIT_ASSERT_VALUES_EQUAL_C(status, EStatus::BAD_REQUEST, "ScanQuery with explicit index should fail");
- }
-
+ }
+
Y_UNIT_TEST(BoolFlag) {
TKikimrRunner kikimr(AppCfg());
@@ -1712,21 +1712,21 @@ Y_UNIT_TEST_SUITE(KqpScan) {
[[4u];[400u];[%false]]
])", StreamResultToYson(it));
}
-
- Y_UNIT_TEST(Counters) {
- TKikimrRunner kikimr(AppCfg());
- auto db = kikimr.GetTableClient();
-
- auto it = db.StreamExecuteScanQuery(R"(
+
+ Y_UNIT_TEST(Counters) {
+ TKikimrRunner kikimr(AppCfg());
+ auto db = kikimr.GetTableClient();
+
+ auto it = db.StreamExecuteScanQuery(R"(
SELECT * FROM `/Root/KeyValue` WHERE Key IN (1,2,3) LIMIT 10;
- )").GetValueSync();
-
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", StreamResultToYson(it));
- TKqpCounters counters(kikimr.GetTestServer().GetRuntime()->GetAppData().Counters);
+ )").GetValueSync();
+
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ CompareYson(R"([[[1u];["One"]];[[2u];["Two"]]])", StreamResultToYson(it));
+ TKqpCounters counters(kikimr.GetTestServer().GetRuntime()->GetAppData().Counters);
UNIT_ASSERT_VALUES_EQUAL(1, counters.GetQueryTypeCounter(NKikimrKqp::EQueryType::QUERY_TYPE_SQL_SCAN)->Val());
- }
-
+ }
+
Y_UNIT_TEST(DropRedundantSortByPk) {
TKikimrRunner kikimr(AppCfg());
auto db = kikimr.GetTableClient();
diff --git a/ydb/core/kqp/ut/kqp_scripting_ut.cpp b/ydb/core/kqp/ut/kqp_scripting_ut.cpp
index 65e75ac303e..b76d06057f8 100644
--- a/ydb/core/kqp/ut/kqp_scripting_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_scripting_ut.cpp
@@ -266,85 +266,85 @@ Y_UNIT_TEST_SUITE(KqpScripting) {
)").GetValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
}
-
- Y_UNIT_TEST(ScriptExplain) {
- TKikimrRunner kikimr;
- TScriptingClient client(kikimr.GetDriver());
-
- TExplainYqlRequestSettings settings;
- settings.Mode(ExplainYqlRequestMode::Plan);
-
- auto result = client.ExplainYqlScript(R"(
- PRAGMA Kikimr.UseNewEngine = "true";
- CREATE TABLE `/Root/ScriptingTest` (
- Key Uint64,
- Value String,
- PRIMARY KEY (Key)
- );
- COMMIT;
-
- PRAGMA Kikimr.UseNewEngine = "false";
- REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES
- (1, "One"),
- (2, "Two");
- COMMIT;
-
- PRAGMA Kikimr.UseNewEngine = "true";
- REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES
- (3, "Three"),
- (4, "Four");
- COMMIT;
-
- PRAGMA kikimr.ScanQuery = "true";
- PRAGMA Kikimr.UseNewEngine = "true";
- SELECT count(*) FROM `/Root/ScriptingTest`;
- COMMIT;
-
- PRAGMA kikimr.ScanQuery = "false";
- PRAGMA Kikimr.UseNewEngine = "true";
- SELECT count(*) FROM `/Root/ScriptingTest`;
- COMMIT;
-
- PRAGMA kikimr.ScanQuery = "true";
- PRAGMA Kikimr.UseNewEngine = "false";
- SELECT count(*) FROM `/Root/ScriptingTest`;
- COMMIT;
-
- PRAGMA kikimr.ScanQuery = "false";
- PRAGMA Kikimr.UseNewEngine = "false";
- SELECT count(*) FROM `/Root/ScriptingTest`;
- COMMIT;
-
- PRAGMA kikimr.ScanQuery = "true";
- SELECT 1*2*3*4*5;
- COMMIT;
-
- PRAGMA kikimr.ScanQuery = "false";
- SELECT 1*2*3*4*5;
- COMMIT;
- )", settings).GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- auto planJson = result.GetPlan();
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(planJson, &plan, true);
- UNIT_ASSERT_EQUAL(plan.GetMapSafe().at("queries").GetArraySafe().size(), 8);
- }
-
- Y_UNIT_TEST(ScriptValidate) {
- TKikimrRunner kikimr;
- TScriptingClient client(kikimr.GetDriver());
-
- TExplainYqlRequestSettings settings;
- settings.Mode(ExplainYqlRequestMode::Validate);
-
- auto result = client.ExplainYqlScript(R"(
+
+ Y_UNIT_TEST(ScriptExplain) {
+ TKikimrRunner kikimr;
+ TScriptingClient client(kikimr.GetDriver());
+
+ TExplainYqlRequestSettings settings;
+ settings.Mode(ExplainYqlRequestMode::Plan);
+
+ auto result = client.ExplainYqlScript(R"(
+ PRAGMA Kikimr.UseNewEngine = "true";
+ CREATE TABLE `/Root/ScriptingTest` (
+ Key Uint64,
+ Value String,
+ PRIMARY KEY (Key)
+ );
+ COMMIT;
+
+ PRAGMA Kikimr.UseNewEngine = "false";
+ REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES
+ (1, "One"),
+ (2, "Two");
+ COMMIT;
+
+ PRAGMA Kikimr.UseNewEngine = "true";
+ REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES
+ (3, "Three"),
+ (4, "Four");
+ COMMIT;
+
+ PRAGMA kikimr.ScanQuery = "true";
+ PRAGMA Kikimr.UseNewEngine = "true";
+ SELECT count(*) FROM `/Root/ScriptingTest`;
+ COMMIT;
+
+ PRAGMA kikimr.ScanQuery = "false";
+ PRAGMA Kikimr.UseNewEngine = "true";
+ SELECT count(*) FROM `/Root/ScriptingTest`;
+ COMMIT;
+
+ PRAGMA kikimr.ScanQuery = "true";
+ PRAGMA Kikimr.UseNewEngine = "false";
+ SELECT count(*) FROM `/Root/ScriptingTest`;
+ COMMIT;
+
+ PRAGMA kikimr.ScanQuery = "false";
+ PRAGMA Kikimr.UseNewEngine = "false";
+ SELECT count(*) FROM `/Root/ScriptingTest`;
+ COMMIT;
+
+ PRAGMA kikimr.ScanQuery = "true";
+ SELECT 1*2*3*4*5;
+ COMMIT;
+
+ PRAGMA kikimr.ScanQuery = "false";
+ SELECT 1*2*3*4*5;
+ COMMIT;
+ )", settings).GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+ auto planJson = result.GetPlan();
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(planJson, &plan, true);
+ UNIT_ASSERT_EQUAL(plan.GetMapSafe().at("queries").GetArraySafe().size(), 8);
+ }
+
+ Y_UNIT_TEST(ScriptValidate) {
+ TKikimrRunner kikimr;
+ TScriptingClient client(kikimr.GetDriver());
+
+ TExplainYqlRequestSettings settings;
+ settings.Mode(ExplainYqlRequestMode::Validate);
+
+ auto result = client.ExplainYqlScript(R"(
DECLARE $value1 as Utf8;
DECLARE $value2 as UInt32;
SELECT $value1 as value1, $value2 as value2;
- )", settings).GetValueSync();
-
+ )", settings).GetValueSync();
+
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
auto paramTypes = result.GetParameterTypes();
UNIT_ASSERT_VALUES_EQUAL(paramTypes.size(), 2);
@@ -361,76 +361,76 @@ Y_UNIT_TEST_SUITE(KqpScripting) {
UNIT_ASSERT_EQUAL(parser2.GetKind(), TTypeParser::ETypeKind::Primitive);
UNIT_ASSERT_EQUAL(parser2.GetPrimitive(), EPrimitiveType::Uint32);
- UNIT_ASSERT_EQUAL(result.GetPlan(), "");
- }
-
- Y_UNIT_TEST(ScriptStats) {
- TKikimrRunner kikimr;
- TScriptingClient client(kikimr.GetDriver());
-
- TExecuteYqlRequestSettings settings;
+ UNIT_ASSERT_EQUAL(result.GetPlan(), "");
+ }
+
+ Y_UNIT_TEST(ScriptStats) {
+ TKikimrRunner kikimr;
+ TScriptingClient client(kikimr.GetDriver());
+
+ TExecuteYqlRequestSettings settings;
settings.CollectQueryStats(NYdb::NTable::ECollectQueryStatsMode::Full);
-
- auto result = client.ExecuteYqlScript(R"(
- PRAGMA Kikimr.UseNewEngine = "true";
- CREATE TABLE `/Root/ScriptingTest` (
- Key Uint64,
- Value String,
- PRIMARY KEY (Key)
- );
- COMMIT;
-
- PRAGMA Kikimr.UseNewEngine = "false";
- REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES
- (1, "One"),
- (2, "Two");
- COMMIT;
-
- PRAGMA Kikimr.UseNewEngine = "true";
- REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES
- (3, "Three"),
- (4, "Four");
- COMMIT;
-
- PRAGMA kikimr.ScanQuery = "true";
- PRAGMA Kikimr.UseNewEngine = "true";
- SELECT count(*) FROM `/Root/ScriptingTest`;
- COMMIT;
-
- PRAGMA kikimr.ScanQuery = "false";
- PRAGMA Kikimr.UseNewEngine = "true";
- SELECT count(*) FROM `/Root/ScriptingTest`;
- COMMIT;
-
- PRAGMA kikimr.ScanQuery = "true";
- PRAGMA Kikimr.UseNewEngine = "false";
- SELECT count(*) FROM `/Root/ScriptingTest`;
- COMMIT;
-
- PRAGMA kikimr.ScanQuery = "false";
- PRAGMA Kikimr.UseNewEngine = "false";
- SELECT count(*) FROM `/Root/ScriptingTest`;
- COMMIT;
-
- PRAGMA kikimr.ScanQuery = "true";
- SELECT 1*2*3*4*5;
- COMMIT;
-
- PRAGMA kikimr.ScanQuery = "false";
- SELECT 1*2*3*4*5;
- COMMIT;
- )", settings).GetValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 6);
-
- auto stats = result.GetStats().Get();
- auto planJson = NYdb::TProtoAccessor::GetProto(*stats).query_plan();
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(planJson, &plan, true);
+
+ auto result = client.ExecuteYqlScript(R"(
+ PRAGMA Kikimr.UseNewEngine = "true";
+ CREATE TABLE `/Root/ScriptingTest` (
+ Key Uint64,
+ Value String,
+ PRIMARY KEY (Key)
+ );
+ COMMIT;
+
+ PRAGMA Kikimr.UseNewEngine = "false";
+ REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES
+ (1, "One"),
+ (2, "Two");
+ COMMIT;
+
+ PRAGMA Kikimr.UseNewEngine = "true";
+ REPLACE INTO `/Root/ScriptingTest` (Key, Value) VALUES
+ (3, "Three"),
+ (4, "Four");
+ COMMIT;
+
+ PRAGMA kikimr.ScanQuery = "true";
+ PRAGMA Kikimr.UseNewEngine = "true";
+ SELECT count(*) FROM `/Root/ScriptingTest`;
+ COMMIT;
+
+ PRAGMA kikimr.ScanQuery = "false";
+ PRAGMA Kikimr.UseNewEngine = "true";
+ SELECT count(*) FROM `/Root/ScriptingTest`;
+ COMMIT;
+
+ PRAGMA kikimr.ScanQuery = "true";
+ PRAGMA Kikimr.UseNewEngine = "false";
+ SELECT count(*) FROM `/Root/ScriptingTest`;
+ COMMIT;
+
+ PRAGMA kikimr.ScanQuery = "false";
+ PRAGMA Kikimr.UseNewEngine = "false";
+ SELECT count(*) FROM `/Root/ScriptingTest`;
+ COMMIT;
+
+ PRAGMA kikimr.ScanQuery = "true";
+ SELECT 1*2*3*4*5;
+ COMMIT;
+
+ PRAGMA kikimr.ScanQuery = "false";
+ SELECT 1*2*3*4*5;
+ COMMIT;
+ )", settings).GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL(result.GetResultSets().size(), 6);
+
+ auto stats = result.GetStats().Get();
+ auto planJson = NYdb::TProtoAccessor::GetProto(*stats).query_plan();
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(planJson, &plan, true);
auto node = FindPlanNodeByKv(plan.GetMap().at("queries").GetArray()[2], "Node Type", "Aggregate-TableFullScan");
UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 1);
- }
+ }
Y_UNIT_TEST(StreamExecuteYqlScriptScan) {
TKikimrRunner kikimr;
diff --git a/ydb/core/kqp/ut/kqp_stats_ut.cpp b/ydb/core/kqp/ut/kqp_stats_ut.cpp
index f7815c016ee..fb7a56e7c0f 100644
--- a/ydb/core/kqp/ut/kqp_stats_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_stats_ut.cpp
@@ -1,73 +1,73 @@
#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
#include <ydb/public/sdk/cpp/client/resources/ydb_resources.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor.h>
#include <cstdlib>
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-
-Y_UNIT_TEST_SUITE(KqpStats) {
-
-Y_UNIT_TEST(MultiTxStatsFullExp) {
- TKikimrRunner kikimr;
- NExperimental::TStreamQueryClient db{kikimr.GetDriver()};
- auto settings = NExperimental::TExecuteStreamQuerySettings();
- settings.ProfileMode(NYdb::NExperimental::EStreamQueryProfileMode::Full);
-
- auto it = db.ExecuteStreamQuery(R"(
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+
+Y_UNIT_TEST_SUITE(KqpStats) {
+
+Y_UNIT_TEST(MultiTxStatsFullExp) {
+ TKikimrRunner kikimr;
+ NExperimental::TStreamQueryClient db{kikimr.GetDriver()};
+ auto settings = NExperimental::TExecuteStreamQuerySettings();
+ settings.ProfileMode(NYdb::NExperimental::EStreamQueryProfileMode::Full);
+
+ auto it = db.ExecuteStreamQuery(R"(
SELECT * FROM `/Root/EightShard` WHERE Key BETWEEN 150 AND 266 ORDER BY Data LIMIT 4;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
- UNIT_ASSERT_VALUES_EQUAL(res.ResultSetYson, R"([[[1];[202u];["Value2"]];[[2];[201u];["Value1"]];[[3];[203u];["Value3"]]])");
-
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL(res.ResultSetYson, R"([[[1];[202u];["Value2"]];[[2];[201u];["Value1"]];[[3];[203u];["Value3"]]])");
+
UNIT_ASSERT(res.PlanJson);
- NJson::TJsonValue plan;
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangesScan");
UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 2);
-}
-
-Y_UNIT_TEST(JoinNoStats) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- TStreamExecScanQuerySettings settings;
- settings.CollectQueryStats(ECollectQueryStatsMode::None);
-
- auto it = db.StreamExecuteScanQuery(R"(
+}
+
+Y_UNIT_TEST(JoinNoStats) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ TStreamExecScanQuerySettings settings;
+ settings.CollectQueryStats(ECollectQueryStatsMode::None);
+
+ auto it = db.StreamExecuteScanQuery(R"(
SELECT count(*) FROM `/Root/EightShard` AS t JOIN `/Root/KeyValue` AS kv ON t.Data = kv.Key;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT_VALUES_EQUAL(res.ResultSetYson, "[[16u]]");
-
+
UNIT_ASSERT(!res.QueryStats);
UNIT_ASSERT(!res.PlanJson);
-}
-
-Y_UNIT_TEST(JoinStatsBasic) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- TStreamExecScanQuerySettings settings;
- settings.CollectQueryStats(ECollectQueryStatsMode::Basic);
-
- auto it = db.StreamExecuteScanQuery(R"(
- SELECT count(*) FROM `/Root/EightShard` AS t JOIN `/Root/KeyValue` AS kv ON t.Data = kv.Key;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
-
- UNIT_ASSERT_VALUES_EQUAL(res.ResultSetYson, "[[16u]]");
-
+}
+
+Y_UNIT_TEST(JoinStatsBasic) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ TStreamExecScanQuerySettings settings;
+ settings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+ auto it = db.StreamExecuteScanQuery(R"(
+ SELECT count(*) FROM `/Root/EightShard` AS t JOIN `/Root/KeyValue` AS kv ON t.Data = kv.Key;
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+
+ UNIT_ASSERT_VALUES_EQUAL(res.ResultSetYson, "[[16u]]");
+
UNIT_ASSERT(res.QueryStats);
UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases().size(), 2);
if (res.QueryStats->query_phases(0).table_access(0).name() == "/Root/KeyValue") {
@@ -83,170 +83,170 @@ Y_UNIT_TEST(JoinStatsBasic) {
}
UNIT_ASSERT(!res.PlanJson);
-}
-
-Y_UNIT_TEST(MultiTxStatsFull) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- TStreamExecScanQuerySettings settings;
- settings.CollectQueryStats(ECollectQueryStatsMode::Full);
-
- auto it = db.StreamExecuteScanQuery(R"(
- SELECT * FROM `/Root/EightShard` WHERE Key BETWEEN 150 AND 266 ORDER BY Data LIMIT 4;
- )", settings).GetValueSync();
-
- auto res = CollectStreamResult(it);
- UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
+}
+
+Y_UNIT_TEST(MultiTxStatsFull) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ TStreamExecScanQuerySettings settings;
+ settings.CollectQueryStats(ECollectQueryStatsMode::Full);
+
+ auto it = db.StreamExecuteScanQuery(R"(
+ SELECT * FROM `/Root/EightShard` WHERE Key BETWEEN 150 AND 266 ORDER BY Data LIMIT 4;
+ )", settings).GetValueSync();
+
+ auto res = CollectStreamResult(it);
+ UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
UNIT_ASSERT_VALUES_EQUAL(
res.ResultSetYson,
R"([[[1];[202u];["Value2"]];[[2];[201u];["Value1"]];[[3];[203u];["Value3"]]])"
);
-
+
UNIT_ASSERT(res.QueryStats);
UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases().size(), 2);
UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases(1).table_access(0).name(), "/Root/EightShard");
UNIT_ASSERT_VALUES_EQUAL(res.QueryStats->query_phases(1).table_access(0).partitions_count(), 2);
UNIT_ASSERT(res.PlanJson);
- NJson::TJsonValue plan;
+ NJson::TJsonValue plan;
NJson::ReadJsonTree(*res.PlanJson, &plan, true);
auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangesScan");
UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 2);
-}
-
-Y_UNIT_TEST(DeferredEffects) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
- TString planJson;
- NJson::TJsonValue plan;
-
- TExecDataQuerySettings settings;
- settings.CollectQueryStats(ECollectQueryStatsMode::Full);
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
+}
+
+Y_UNIT_TEST(DeferredEffects) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+ TString planJson;
+ NJson::TJsonValue plan;
+
+ TExecDataQuerySettings settings;
+ settings.CollectQueryStats(ECollectQueryStatsMode::Full);
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
UPSERT INTO `/Root/TwoShard`
SELECT Key + 100u AS Key, Value1 FROM `/Root/TwoShard` WHERE Key in (1,2,3,4,5);
- )", TTxControl::BeginTx(), settings).ExtractValueSync();
+ )", TTxControl::BeginTx(), settings).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- // TODO(sk): do proper phase dependency tracking
- //
- // NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true);
- // auto node = FindPlanNodeByKv(plan, "Node Type", "TablePointLookup");
+
+ // TODO(sk): do proper phase dependency tracking
+ //
+ // NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true);
+ // auto node = FindPlanNodeByKv(plan, "Node Type", "TablePointLookup");
// UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 1);
-
- auto tx = result.GetTransaction();
- UNIT_ASSERT(tx);
-
- auto params = db.GetParamsBuilder()
- .AddParam("$key")
- .Uint32(100)
- .Build()
- .AddParam("$value")
- .String("New")
- .Build()
- .Build();
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
- DECLARE $key AS Uint32;
- DECLARE $value AS String;
-
+
+ auto tx = result.GetTransaction();
+ UNIT_ASSERT(tx);
+
+ auto params = db.GetParamsBuilder()
+ .AddParam("$key")
+ .Uint32(100)
+ .Build()
+ .AddParam("$value")
+ .String("New")
+ .Build()
+ .Build();
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
+ DECLARE $key AS Uint32;
+ DECLARE $value AS String;
+
UPSERT INTO `/Root/TwoShard` (Key, Value1) VALUES
- ($key, $value);
+ ($key, $value);
)", TTxControl::Tx(*tx).CommitTx(), std::move(params), settings).ExtractValueSync();
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true);
+
+ NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true);
UNIT_ASSERT_VALUES_EQUAL(plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans").GetArraySafe().size(), 3);
-
- result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
+
+ result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
SELECT * FROM `/Root/TwoShard`;
UPDATE `/Root/TwoShard` SET Value1 = "XXX" WHERE Key in (3,600);
)", TTxControl::BeginTx().CommitTx(), settings).ExtractValueSync();
- result.GetIssues().PrintTo(Cerr);
+ result.GetIssues().PrintTo(Cerr);
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
-
- NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true);
+
+ NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true);
UNIT_ASSERT_VALUES_EQUAL(plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans").GetArraySafe().size(), 4);
auto ru = result.GetResponseMetadata().find(NYdb::YDB_CONSUMED_UNITS_HEADER);
UNIT_ASSERT(ru != result.GetResponseMetadata().end());
UNIT_ASSERT(std::atoi(ru->second.c_str()) > 1);
-}
-
-Y_UNIT_TEST(DataQueryWithEffects) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- TExecDataQuerySettings settings;
- settings.CollectQueryStats(ECollectQueryStatsMode::Full);
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
+}
+
+Y_UNIT_TEST(DataQueryWithEffects) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ TExecDataQuerySettings settings;
+ settings.CollectQueryStats(ECollectQueryStatsMode::Full);
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
UPSERT INTO `/Root/TwoShard`
SELECT Key + 1u AS Key, Value1 FROM `/Root/TwoShard`;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync();
- result.GetIssues().PrintTo(Cerr);
- AssertSuccessResult(result);
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true);
-
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ AssertSuccessResult(result);
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true);
+
auto node = FindPlanNodeByKv(plan, "Node Type", "Upsert-ConstantExpr");
UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 2);
-}
-
-Y_UNIT_TEST(DataQueryOldEngine) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- TExecDataQuerySettings settings;
- settings.CollectQueryStats(ECollectQueryStatsMode::Full);
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "false";
-
+}
+
+Y_UNIT_TEST(DataQueryOldEngine) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ TExecDataQuerySettings settings;
+ settings.CollectQueryStats(ECollectQueryStatsMode::Full);
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "false";
+
UPSERT INTO `/Root/TwoShard`
SELECT Key + 1u AS Key, Value1 FROM `/Root/TwoShard`;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync();
- result.GetIssues().PrintTo(Cerr);
- AssertSuccessResult(result);
-}
-
-Y_UNIT_TEST(DataQueryMulti) {
- TKikimrRunner kikimr;
- auto db = kikimr.GetTableClient();
- auto session = db.CreateSession().GetValueSync().GetSession();
-
- TExecDataQuerySettings settings;
- settings.CollectQueryStats(ECollectQueryStatsMode::Full);
-
- auto result = session.ExecuteDataQuery(R"(
- PRAGMA kikimr.UseNewEngine = "true";
-
- SELECT 1;
- SELECT 2;
- SELECT 3;
- SELECT 4;
- )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync();
- result.GetIssues().PrintTo(Cerr);
- AssertSuccessResult(result);
-
- NJson::TJsonValue plan;
- NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true);
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ AssertSuccessResult(result);
+}
+
+Y_UNIT_TEST(DataQueryMulti) {
+ TKikimrRunner kikimr;
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ TExecDataQuerySettings settings;
+ settings.CollectQueryStats(ECollectQueryStatsMode::Full);
+
+ auto result = session.ExecuteDataQuery(R"(
+ PRAGMA kikimr.UseNewEngine = "true";
+
+ SELECT 1;
+ SELECT 2;
+ SELECT 3;
+ SELECT 4;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), settings).ExtractValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ AssertSuccessResult(result);
+
+ NJson::TJsonValue plan;
+ NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true);
UNIT_ASSERT_EQUAL_C(plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans").GetArraySafe().size(), 0, result.GetQueryPlan());
-}
-
+}
+
Y_UNIT_TEST_NEW_ENGINE(RequestUnitForBadRequestExecute) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
@@ -300,7 +300,7 @@ Y_UNIT_TEST_NEW_ENGINE(RequestUnitForSuccessExplicitPrepare) {
UNIT_ASSERT(result.GetConsumedRu() > 1);
}
-} // suite
-
-} // namespace NKqp
-} // namespace NKikimr
+} // suite
+
+} // namespace NKqp
+} // namespace NKikimr
diff --git a/ydb/core/kqp/ut/ya.make b/ydb/core/kqp/ut/ya.make
index 609ec3237f8..c7cfb9eb4dd 100644
--- a/ydb/core/kqp/ut/ya.make
+++ b/ydb/core/kqp/ut/ya.make
@@ -23,7 +23,7 @@ SRCS(
kqp_arrow_in_channels_ut.cpp
kqp_document_api_ut.cpp
kqp_effects_perf_ut.cpp
- kqp_explain_ut.cpp
+ kqp_explain_ut.cpp
kqp_flip_join_ut.cpp
kqp_index_lookup_join_ut.cpp
kqp_indexes_ut.cpp
@@ -47,7 +47,7 @@ SRCS(
kqp_scripting_ut.cpp
kqp_service_ut.cpp
kqp_sort_ut.cpp
- kqp_stats_ut.cpp
+ kqp_stats_ut.cpp
kqp_sqlin_ut.cpp
kqp_sys_view_ut.cpp
kqp_sys_col_ut.cpp
diff --git a/ydb/core/protos/kqp.proto b/ydb/core/protos/kqp.proto
index 9b46b720e11..98e9b0797c6 100644
--- a/ydb/core/protos/kqp.proto
+++ b/ydb/core/protos/kqp.proto
@@ -46,7 +46,7 @@ enum EQueryAction {
QUERY_ACTION_BEGIN_TX = 6;
QUERY_ACTION_COMMIT_TX = 7;
QUERY_ACTION_ROLLBACK_TX = 8;
- QUERY_ACTION_PARSE = 9;
+ QUERY_ACTION_PARSE = 9;
};
enum EIsolationLevel {
@@ -386,7 +386,7 @@ message TEvExecuterStreamData {
message TEvExecuterStreamDataAck {
optional uint64 SeqNo = 1;
optional int64 FreeSpace = 2;
- optional bool Enough = 3 [default = false];
+ optional bool Enough = 3 [default = false];
};
message TEvExecuterStreamProfile {
@@ -525,28 +525,28 @@ message TKqpNodeResources {
repeated TMemory Memory = 8;
optional uint32 ExecutionUnits = 9;
}
-
+
/// Scans
-message TEvScanInitActor {
- optional uint64 ScanId = 1;
- optional NActorsProto.TActorId ScanActorId = 2;
+message TEvScanInitActor {
+ optional uint64 ScanId = 1;
+ optional NActorsProto.TActorId ScanActorId = 2;
optional uint32 Generation = 3;
-};
-
-message TEvScanError {
- optional Ydb.StatusIds.StatusCode Status = 1;
- repeated Ydb.Issue.IssueMessage Issues = 2;
+};
+
+message TEvScanError {
+ optional Ydb.StatusIds.StatusCode Status = 1;
+ repeated Ydb.Issue.IssueMessage Issues = 2;
optional uint32 Generation = 3;
-}
-
-message TEvRemoteScanData {
- optional uint32 ScanId = 1;
+}
+
+message TEvRemoteScanData {
+ optional uint32 ScanId = 1;
optional uint64 CpuTimeUs = 2;
- optional uint64 WaitTimeMs = 3;
- optional uint32 PageFaults = 4;
- optional bool PageFault = 5;
- optional bool Finished = 6;
- optional bytes LastKey = 7;
+ optional uint64 WaitTimeMs = 3;
+ optional uint32 PageFaults = 4;
+ optional bool PageFault = 5;
+ optional bool Finished = 6;
+ optional bytes LastKey = 7;
optional uint32 Generation = 9;
message TArrowBatch {
@@ -557,14 +557,14 @@ message TEvRemoteScanData {
// Only one of the fields Rows and ArrowBatch must be filled. However, we can not use oneof feature because Rows
// field is repeated. Moving it into oneof is impossible. We may wrap it into a message but this would break
// backwards comparability.
- repeated bytes Rows = 8;
+ repeated bytes Rows = 8;
optional TArrowBatch ArrowBatch = 10;
-}
-
-message TEvRemoteScanDataAck {
- optional uint64 FreeSpace = 1;
+}
+
+message TEvRemoteScanDataAck {
+ optional uint64 FreeSpace = 1;
optional uint32 Generation = 2;
-}
+}
message TEvKillScanTablet {
}
diff --git a/ydb/core/protos/kqp_physical.proto b/ydb/core/protos/kqp_physical.proto
index 44d8f587166..922d6f6c7c9 100644
--- a/ydb/core/protos/kqp_physical.proto
+++ b/ydb/core/protos/kqp_physical.proto
@@ -230,7 +230,7 @@ message TKqpPhyStage {
uint32 OutputsCount = 5;
repeated TKqpPhyTableOperation TableOps = 6;
bool IsEffectsStage = 7;
- string StageGuid = 8;
+ string StageGuid = 8;
}
message TKqpPhyResult {
@@ -252,7 +252,7 @@ message TKqpPhyTx {
repeated TKqpPhyStage Stages = 2;
repeated TKqpPhyResult Results = 3;
repeated TKqpPhyParamBinding ParamBindings = 4;
- string Plan = 5;
+ string Plan = 5;
bool HasEffects = 6; // at least one stage has flag TKqpPhyStage::IsEffectStage set
}
@@ -290,8 +290,8 @@ message TKqpPhyQuery {
repeated TKqpPhyTx Transactions = 2;
reserved 3; // Deprecated, (repeated TKqpPhyTx Effects = 3);
repeated TKqpPhyResultBinding ResultBindings = 4;
- string QueryPlan = 5;
- string QueryAst = 6;
+ string QueryPlan = 5;
+ string QueryAst = 6;
// StrictDml constraints info
repeated TKqpTableOp TableOps = 7;
diff --git a/ydb/core/protos/tx_datashard.proto b/ydb/core/protos/tx_datashard.proto
index 89863b25400..1549fe524f5 100644
--- a/ydb/core/protos/tx_datashard.proto
+++ b/ydb/core/protos/tx_datashard.proto
@@ -221,8 +221,8 @@ message TKqpTransaction {
optional NYql.NDqProto.TComputeRuntimeSettings RuntimeSettings = 4;
reserved 5;
-
- optional NKikimrKqp.TKqpSnapshot Snapshot = 6;
+
+ optional NKikimrKqp.TKqpSnapshot Snapshot = 6;
}
message TKqpTaskInfo {
@@ -1373,20 +1373,20 @@ message TEvBuildIndexProgressResponse {
optional uint64 RequestSeqNoGeneration = 12;
optional uint64 RequestSeqNoRound = 13;
}
-
-message TEvKqpScan {
- optional uint64 TxId = 1;
- optional uint64 ScanId = 2;
- optional uint64 LocalPathId = 3;
- optional string TablePath = 4;
- optional uint64 SchemaVersion = 5;
- repeated uint32 ColumnTags = 6;
- repeated uint32 ColumnTypes = 7;
- repeated bool SkipNullKeys = 8;
+
+message TEvKqpScan {
+ optional uint64 TxId = 1;
+ optional uint64 ScanId = 2;
+ optional uint64 LocalPathId = 3;
+ optional string TablePath = 4;
+ optional uint64 SchemaVersion = 5;
+ repeated uint32 ColumnTags = 6;
+ repeated uint32 ColumnTypes = 7;
+ repeated bool SkipNullKeys = 8;
repeated NKikimrTx.TKeyRange Ranges = 9;
- optional NKikimrKqp.TKqpSnapshot Snapshot = 10;
+ optional NKikimrKqp.TKqpSnapshot Snapshot = 10;
reserved 11; // optional NKqpProto.EKqpStatsMode StatsMode = 11;
- optional uint64 TimeoutMs = 12;
+ optional uint64 TimeoutMs = 12;
optional uint32 Generation = 13;
// Seralized OlapProgram in old format without parameters
reserved 14;
@@ -1396,7 +1396,7 @@ message TEvKqpScan {
optional NYql.NDqProto.EDqStatsMode StatsMode = 18;
optional bytes OlapProgram = 19;
optional NKikimrSchemeOp.EOlapProgramType OlapProgramType = 20;
-}
+}
message TEvCompactTable {
optional NKikimrProto.TPathID PathId = 1;
diff --git a/ydb/core/sys_view/common/scan_actor_base_impl.h b/ydb/core/sys_view/common/scan_actor_base_impl.h
index 298a8efbc03..2c8a3f3fcd6 100644
--- a/ydb/core/sys_view/common/scan_actor_base_impl.h
+++ b/ydb/core/sys_view/common/scan_actor_base_impl.h
@@ -88,8 +88,8 @@ protected:
<< ", error: " << message);
auto error = MakeHolder<NKqp::TEvKqpCompute::TEvScanError>();
- error->Record.SetStatus(status);
- IssueToMessage(NYql::TIssue(message), error->Record.MutableIssues()->Add());
+ error->Record.SetStatus(status);
+ IssueToMessage(NYql::TIssue(message), error->Record.MutableIssues()->Add());
TBase::Send(OwnerActorId, error.Release());
diff --git a/ydb/core/tx/columnshard/columnshard.h b/ydb/core/tx/columnshard/columnshard.h
index 62486e3c33e..9f1da5d78c3 100644
--- a/ydb/core/tx/columnshard/columnshard.h
+++ b/ydb/core/tx/columnshard/columnshard.h
@@ -235,7 +235,7 @@ struct TEvColumnShard {
}
};
- using TEvScan = TEvDataShard::TEvKqpScan;
+ using TEvScan = TEvDataShard::TEvKqpScan;
};
inline auto& Proto(TEvColumnShard::TEvProposeTransaction* ev) {
diff --git a/ydb/core/tx/datashard/datashard.cpp b/ydb/core/tx/datashard/datashard.cpp
index d86568b4a15..e44b05fa25a 100644
--- a/ydb/core/tx/datashard/datashard.cpp
+++ b/ydb/core/tx/datashard/datashard.cpp
@@ -1360,11 +1360,11 @@ void TDataShard::Handle(TEvDataShard::TEvStateChangedResult::TPtr& ev, const TAc
bool TDataShard::CheckDataTxReject(const TString& opDescr,
const TActorContext &ctx,
NKikimrTxDataShard::TEvProposeTransactionResult::EStatus &rejectStatus,
- TString &reason)
+ TString &reason)
{
bool reject = false;
rejectStatus = NKikimrTxDataShard::TEvProposeTransactionResult::OVERLOADED;
- TVector<TString> rejectReasons;
+ TVector<TString> rejectReasons;
// In v0.5 reject all transactions on split Src after receiving EvSplit
if (State == TShardState::SplitSrcWaitForNoTxInFlight ||
@@ -1372,19 +1372,19 @@ bool TDataShard::CheckDataTxReject(const TString& opDescr,
State == TShardState::SplitSrcSendingSnapshot ||
State == TShardState::SplitSrcWaitForPartitioningChanged) {
reject = true;
- rejectReasons.push_back(TStringBuilder()
- << "is in process of split opId " << SrcSplitOpId
- << " state " << DatashardStateName(State)
- << " (wrong shard state)");
+ rejectReasons.push_back(TStringBuilder()
+ << "is in process of split opId " << SrcSplitOpId
+ << " state " << DatashardStateName(State)
+ << " (wrong shard state)");
} else if (State == TShardState::SplitDstReceivingSnapshot) {
reject = true;
- rejectReasons.push_back(TStringBuilder()
- << "is in process of split opId " << DstSplitOpId
- << " state " << DatashardStateName(State));
+ rejectReasons.push_back(TStringBuilder()
+ << "is in process of split opId " << DstSplitOpId
+ << " state " << DatashardStateName(State));
} else if (State == TShardState::PreOffline || State == TShardState::Offline) {
reject = true;
rejectStatus = NKikimrTxDataShard::TEvProposeTransactionResult::ERROR;
- rejectReasons.push_back("is in a pre/offline state assuming this is due to a finished split (wrong shard state)");
+ rejectReasons.push_back("is in a pre/offline state assuming this is due to a finished split (wrong shard state)");
} else if (MvccSwitchState == TSwitchState::SWITCHING) {
reject = true;
rejectReasons.push_back(TStringBuilder()
@@ -1394,7 +1394,7 @@ bool TDataShard::CheckDataTxReject(const TString& opDescr,
if (Pipeline.HasDrop()) {
reject = true;
- rejectReasons.push_back("is in process of drop");
+ rejectReasons.push_back("is in process of drop");
rejectStatus = NKikimrTxDataShard::TEvProposeTransactionResult::ERROR;
}
@@ -1402,24 +1402,24 @@ bool TDataShard::CheckDataTxReject(const TString& opDescr,
TDuration lag = GetDataTxCompleteLag();
if (txInfly > 1 && lag > TDuration::MilliSeconds(MaxTxLagMilliseconds)) {
reject = true;
- rejectReasons.push_back(TStringBuilder()
- << "lags behind, lag: " << lag
- << " in-flight tx count: " << txInfly);
+ rejectReasons.push_back(TStringBuilder()
+ << "lags behind, lag: " << lag
+ << " in-flight tx count: " << txInfly);
}
const float rejectProbabilty = Executor()->GetRejectProbability();
if (!reject && rejectProbabilty > 0) {
float rnd = AppData(ctx)->RandomProvider->GenRandReal2();
reject |= (rnd < rejectProbabilty);
- if (reject)
- rejectReasons.push_back("decided to reject due to given RejectProbability");
+ if (reject)
+ rejectReasons.push_back("decided to reject due to given RejectProbability");
}
size_t totalInFly = (TxInFly() + ImmediateInFly() + ProposeQueue.Size() + TxWaiting());
if (totalInFly > GetMaxTxInFly()) {
- reject = true;
- rejectReasons.push_back("MaxTxInFly was exceeded");
- }
+ reject = true;
+ rejectReasons.push_back("MaxTxInFly was exceeded");
+ }
if (!reject && Stopping) {
reject = true;
@@ -1437,13 +1437,13 @@ bool TDataShard::CheckDataTxReject(const TString& opDescr,
}
}
- if (reject) {
- reason = TStringBuilder()
- << "Rejecting " << opDescr
- << " because datashard " << TabletID() << ": "
- << JoinSeq("; ", rejectReasons);
- }
-
+ if (reject) {
+ reason = TStringBuilder()
+ << "Rejecting " << opDescr
+ << " because datashard " << TabletID() << ": "
+ << JoinSeq("; ", rejectReasons);
+ }
+
return reject;
}
@@ -1463,8 +1463,8 @@ bool TDataShard::CheckDataTxRejectAndReply(TEvDataShard::TEvProposeTransaction*
TString txDescr = TStringBuilder() << "data TxId " << msg->GetTxId();
NKikimrTxDataShard::TEvProposeTransactionResult::EStatus rejectStatus;
- TString rejectReason;
- bool reject = CheckDataTxReject(txDescr, ctx, rejectStatus, rejectReason);
+ TString rejectReason;
+ bool reject = CheckDataTxReject(txDescr, ctx, rejectStatus, rejectReason);
if (reject) {
THolder<TEvDataShard::TEvProposeTransactionResult> result =
@@ -1473,8 +1473,8 @@ bool TDataShard::CheckDataTxRejectAndReply(TEvDataShard::TEvProposeTransaction*
msg->GetTxId(),
rejectStatus));
- result->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, rejectReason);
- LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, rejectReason);
+ result->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, rejectReason);
+ LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, rejectReason);
ctx.Send(msg->GetSource(), result.Release());
IncCounter(COUNTER_PREPARE_OVERLOADED);
diff --git a/ydb/core/tx/datashard/datashard.h b/ydb/core/tx/datashard/datashard.h
index 5a93c035026..33cc99535f0 100644
--- a/ydb/core/tx/datashard/datashard.h
+++ b/ydb/core/tx/datashard/datashard.h
@@ -277,8 +277,8 @@ struct TEvDataShard {
EvUnsafeUploadRowsRequest,
EvUnsafeUploadRowsResponse,
- EvKqpScan,
-
+ EvKqpScan,
+
EvChangeS3UploadStatus,
EvGetRemovedRowVersions, /* for tests */
@@ -1346,13 +1346,13 @@ struct TEvDataShard {
TEvDataShard::EvBuildIndexProgressResponse>
{
};
-
- struct TEvKqpScan
- : public TEventPB<TEvKqpScan,
- NKikimrTxDataShard::TEvKqpScan,
- TEvDataShard::EvKqpScan>
- {
- };
+
+ struct TEvKqpScan
+ : public TEventPB<TEvKqpScan,
+ NKikimrTxDataShard::TEvKqpScan,
+ TEvDataShard::EvKqpScan>
+ {
+ };
struct TEvGetRemovedRowVersions : public TEventLocal<TEvGetRemovedRowVersions, EvGetRemovedRowVersions> {
TPathId PathId;
diff --git a/ydb/core/tx/datashard/datashard__kqp_scan.cpp b/ydb/core/tx/datashard/datashard__kqp_scan.cpp
index b6b6b7e337d..e110f6be0ba 100644
--- a/ydb/core/tx/datashard/datashard__kqp_scan.cpp
+++ b/ydb/core/tx/datashard/datashard__kqp_scan.cpp
@@ -1,61 +1,61 @@
-#include "datashard_impl.h"
+#include "datashard_impl.h"
#include "range_ops.h"
-#include <util/string/vector.h>
-
+#include <util/string/vector.h>
+
#include <ydb/core/actorlib_impl/long_timer.h>
#include <ydb/core/formats/arrow_batch_builder.h>
#include <ydb/core/kqp/kqp_compute.h>
#include <ydb/core/tablet_flat/flat_row_celled.h>
-
+
#include <ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h>
-
-namespace NKikimr {
+
+namespace NKikimr {
namespace NDataShard {
-
-// using namespace NTabletFlatExecutor;
-using namespace NKqp;
-using namespace NMiniKQL;
-
-constexpr ui64 MAX_BATCH_ROWS = 10'000;
-constexpr ui64 INIT_BATCH_ROWS = 1'000;
-constexpr ui64 MIN_BATCH_ROWS_ON_PAGEFAULT = 1'000;
-constexpr ui64 MIN_BATCH_SIZE_ON_PAGEFAULT = 256_KB;
-constexpr ui64 READAHEAD_LO = 32_KB;
-constexpr ui64 READAHEAD_HI = 512_KB;
-constexpr TDuration SCAN_HARD_TIMEOUT = TDuration::Minutes(10);
-constexpr TDuration SCAN_HARD_TIMEOUT_GAP = TDuration::Seconds(5);
-
+
+// using namespace NTabletFlatExecutor;
+using namespace NKqp;
+using namespace NMiniKQL;
+
+constexpr ui64 MAX_BATCH_ROWS = 10'000;
+constexpr ui64 INIT_BATCH_ROWS = 1'000;
+constexpr ui64 MIN_BATCH_ROWS_ON_PAGEFAULT = 1'000;
+constexpr ui64 MIN_BATCH_SIZE_ON_PAGEFAULT = 256_KB;
+constexpr ui64 READAHEAD_LO = 32_KB;
+constexpr ui64 READAHEAD_HI = 512_KB;
+constexpr TDuration SCAN_HARD_TIMEOUT = TDuration::Minutes(10);
+constexpr TDuration SCAN_HARD_TIMEOUT_GAP = TDuration::Seconds(5);
+
class TKqpScanResult : public IDestructable {};
-
-class TKqpScan : public TActor<TKqpScan>, public NTable::IScan {
-public:
+
+class TKqpScan : public TActor<TKqpScan>, public NTable::IScan {
+public:
static constexpr auto ActorActivityType() {
- return NKikimrServices::TActivity::KQP_TABLE_SCAN;
- }
-
-public:
- TKqpScan(const TActorId& computeActorId, const TActorId& datashardActorId, ui32 scanId,
+ return NKikimrServices::TActivity::KQP_TABLE_SCAN;
+ }
+
+public:
+ TKqpScan(const TActorId& computeActorId, const TActorId& datashardActorId, ui32 scanId,
NDataShard::TUserTable::TCPtr tableInfo, const TSmallVec<TSerializedTableRange>&& tableRanges,
const TSmallVec<NTable::TTag>&& columnTags, const TSmallVec<bool>&& skipNullKeys,
const NYql::NDqProto::EDqStatsMode& statsMode, ui64 timeoutMs, ui32 generation,
NKikimrTxDataShard::EScanDataFormat dataFormat)
- : TActor(&TKqpScan::StateScan)
- , ComputeActorId(computeActorId)
- , DatashardActorId(datashardActorId)
- , ScanId(scanId)
- , TableInfo(tableInfo)
+ : TActor(&TKqpScan::StateScan)
+ , ComputeActorId(computeActorId)
+ , DatashardActorId(datashardActorId)
+ , ScanId(scanId)
+ , TableInfo(tableInfo)
, TablePath(TableInfo->Path)
, TableRanges(std::move(tableRanges))
, CurrentRange(0)
, Tags(std::move(columnTags))
, SkipNullKeys(std::move(skipNullKeys))
- , StatsMode(statsMode)
- , Deadline(TInstant::Now() + (timeoutMs ? TDuration::MilliSeconds(timeoutMs) + SCAN_HARD_TIMEOUT_GAP : SCAN_HARD_TIMEOUT))
+ , StatsMode(statsMode)
+ , Deadline(TInstant::Now() + (timeoutMs ? TDuration::MilliSeconds(timeoutMs) + SCAN_HARD_TIMEOUT_GAP : SCAN_HARD_TIMEOUT))
, Generation(generation)
, DataFormat(dataFormat)
- , PeerFreeSpace(0)
- , Sleep(true)
- , IsLocal(computeActorId.NodeId() == datashardActorId.NodeId())
+ , PeerFreeSpace(0)
+ , Sleep(true)
+ , IsLocal(computeActorId.NodeId() == datashardActorId.NodeId())
{
if (DataFormat == NKikimrTxDataShard::EScanDataFormat::ARROW) {
BatchBuilder = MakeHolder<NArrow::TArrowBatchBuilder>();
@@ -79,141 +79,141 @@ public:
<< DebugPrintRange(TableInfo->KeyColumnTypes, range.ToTableRange(), *AppData()->TypeRegistry));
}
}
-
-private:
- STATEFN(StateScan) {
- switch (ev->GetTypeRewrite()) {
- hFunc(TEvKqpCompute::TEvScanDataAck, HandleScan);
+
+private:
+ STATEFN(StateScan) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvKqpCompute::TEvScanDataAck, HandleScan);
hFunc(TEvKqpCompute::TEvKillScanTablet, HandleScan);
- hFunc(TEvKqp::TEvAbortExecution, HandleScan);
- hFunc(TEvents::TEvUndelivered, HandleScan);
- hFunc(TEvents::TEvWakeup, HandleScan);
- default:
- Y_FAIL("TKqpScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
- }
- }
-
- void HandleScan(TEvKqpCompute::TEvScanDataAck::TPtr& ev) {
- if (!Driver) {
- LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got ScanDataAck while driver not set");
- PassAway();
- return;
- }
-
- LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got ScanDataAck"
+ hFunc(TEvKqp::TEvAbortExecution, HandleScan);
+ hFunc(TEvents::TEvUndelivered, HandleScan);
+ hFunc(TEvents::TEvWakeup, HandleScan);
+ default:
+ Y_FAIL("TKqpScan: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite());
+ }
+ }
+
+ void HandleScan(TEvKqpCompute::TEvScanDataAck::TPtr& ev) {
+ if (!Driver) {
+ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got ScanDataAck while driver not set");
+ PassAway();
+ return;
+ }
+
+ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got ScanDataAck"
<< ", at: " << ScanActorId << ", scanId: " << ScanId << ", table: " << TablePath
<< ", gen: " << ev->Get()->Generation << ", tablet: " << DatashardActorId
- << ", freeSpace: " << ev->Get()->FreeSpace << ", prevFreeSpace: " << PeerFreeSpace);
-
+ << ", freeSpace: " << ev->Get()->FreeSpace << ", prevFreeSpace: " << PeerFreeSpace);
+
YQL_ENSURE(ev->Get()->Generation == Generation, "expected: " << Generation << ", got: " << ev->Get()->Generation);
- if (!ComputeActorId) {
- ComputeActorId = ev->Sender;
- }
-
- PeerFreeSpace = ev->Get()->FreeSpace;
- if (PeerFreeSpace > 0) {
- if (Y_UNLIKELY(IsProfile())) {
+ if (!ComputeActorId) {
+ ComputeActorId = ev->Sender;
+ }
+
+ PeerFreeSpace = ev->Get()->FreeSpace;
+ if (PeerFreeSpace > 0) {
+ if (Y_UNLIKELY(IsProfile())) {
StartWaitTime = TInstant::Now();
- }
- if (Sleep) {
- Sleep = false;
- LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Wakeup driver at: " << ScanActorId);
- Driver->Touch(EScan::Feed);
- }
- }
- }
-
+ }
+ if (Sleep) {
+ Sleep = false;
+ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Wakeup driver at: " << ScanActorId);
+ Driver->Touch(EScan::Feed);
+ }
+ }
+ }
+
void HandleScan(TEvKqpCompute::TEvKillScanTablet::TPtr&) {
LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Kill self tablet " << DatashardActorId);
Send(DatashardActorId, new TEvents::TEvPoison);
}
- void HandleScan(TEvKqp::TEvAbortExecution::TPtr& ev) {
- if (!Driver) {
- LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got AbortExecution while driver not set");
- PassAway();
- return;
- }
-
- auto& msg = ev->Get()->Record;
-
- auto prio = msg.GetStatusCode() == Ydb::StatusIds::SUCCESS ? NActors::NLog::PRI_DEBUG : NActors::NLog::PRI_WARN;
- LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::TX_DATASHARD, "Got AbortExecution"
+ void HandleScan(TEvKqp::TEvAbortExecution::TPtr& ev) {
+ if (!Driver) {
+ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got AbortExecution while driver not set");
+ PassAway();
+ return;
+ }
+
+ auto& msg = ev->Get()->Record;
+
+ auto prio = msg.GetStatusCode() == Ydb::StatusIds::SUCCESS ? NActors::NLog::PRI_DEBUG : NActors::NLog::PRI_WARN;
+ LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::TX_DATASHARD, "Got AbortExecution"
<< ", at: " << ScanActorId << ", tablet: " << DatashardActorId
- << ", scanId: " << ScanId << ", table: " << TablePath
- << ", code: " << Ydb::StatusIds_StatusCode_Name(msg.GetStatusCode())
- << ", reason: " << msg.GetMessage());
-
+ << ", scanId: " << ScanId << ", table: " << TablePath
+ << ", code: " << Ydb::StatusIds_StatusCode_Name(msg.GetStatusCode())
+ << ", reason: " << msg.GetMessage());
+
AbortEvent = ev->Release();
- Driver->Touch(EScan::Final);
- }
-
- void HandleScan(TEvents::TEvUndelivered::TPtr& ev) {
- if (!Driver) {
- PassAway();
- return;
- }
-
+ Driver->Touch(EScan::Final);
+ }
+
+ void HandleScan(TEvents::TEvUndelivered::TPtr& ev) {
+ if (!Driver) {
+ PassAway();
+ return;
+ }
+
LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Undelivered event: " << ev->GetTypeRewrite()
<< ", at: " << ScanActorId << ", tablet: " << DatashardActorId
- << ", scanId: " << ScanId << ", table: " << TablePath);
-
- switch (ev->GetTypeRewrite()) {
+ << ", scanId: " << ScanId << ", table: " << TablePath);
+
+ switch (ev->GetTypeRewrite()) {
case TEvKqpCompute::TEvScanInitActor::EventType:
case TEvKqpCompute::TEvScanData::EventType:
- Driver->Touch(EScan::Final);
- }
- }
-
- void HandleScan(TEvents::TEvWakeup::TPtr&) {
- LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Guard execution timeout at: " << ScanActorId
- << ", scanId: " << ScanId << ", table: " << TablePath);
-
- TimeoutActorId = {};
-
- if (Driver) {
- Driver->Touch(EScan::Final);
- } else {
- PassAway();
- }
- }
-
-private:
+ Driver->Touch(EScan::Final);
+ }
+ }
+
+ void HandleScan(TEvents::TEvWakeup::TPtr&) {
+ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Guard execution timeout at: " << ScanActorId
+ << ", scanId: " << ScanId << ", table: " << TablePath);
+
+ TimeoutActorId = {};
+
+ if (Driver) {
+ Driver->Touch(EScan::Final);
+ } else {
+ PassAway();
+ }
+ }
+
+private:
THello Prepare(IDriver* driver, TIntrusiveConstPtr<TScheme> scheme) noexcept final {
- Y_VERIFY(scheme);
- Y_VERIFY(driver);
-
- Driver = driver;
- ScanActorId = TActivationContext::AsActorContext().RegisterWithSameMailbox(this);
-
- // propagate self actor id
+ Y_VERIFY(scheme);
+ Y_VERIFY(driver);
+
+ Driver = driver;
+ ScanActorId = TActivationContext::AsActorContext().RegisterWithSameMailbox(this);
+
+ // propagate self actor id
Send(ComputeActorId, new TEvKqpCompute::TEvScanInitActor(ScanId, ScanActorId, Generation),
IEventHandle::FlagTrackDelivery);
-
- Sleep = true;
-
- THello startConfig;
- startConfig.Scan = EScan::Sleep;
- startConfig.Conf.ReadAheadLo = READAHEAD_LO;
- startConfig.Conf.ReadAheadHi = READAHEAD_HI;
-
- TimeoutActorId = CreateLongTimer(TlsActivationContext->AsActorContext(), Deadline - TInstant::Now(),
- new IEventHandle(SelfId(), SelfId(), new TEvents::TEvWakeup));
-
- if (Y_UNLIKELY(IsProfile())) {
+
+ Sleep = true;
+
+ THello startConfig;
+ startConfig.Scan = EScan::Sleep;
+ startConfig.Conf.ReadAheadLo = READAHEAD_LO;
+ startConfig.Conf.ReadAheadHi = READAHEAD_HI;
+
+ TimeoutActorId = CreateLongTimer(TlsActivationContext->AsActorContext(), Deadline - TInstant::Now(),
+ new IEventHandle(SelfId(), SelfId(), new TEvents::TEvWakeup));
+
+ if (Y_UNLIKELY(IsProfile())) {
StartWaitTime = TInstant::Now();
- }
-
+ }
+
LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Start scan"
<< ", at: " << ScanActorId << ", tablet: " << DatashardActorId
<< ", scanId: " << ScanId << ", table: " << TablePath << ", gen: " << Generation
- << ", deadline: " << Deadline);
-
- return startConfig;
- }
-
- EScan Seek(TLead& lead, ui64 seq) noexcept final {
+ << ", deadline: " << Deadline);
+
+ return startConfig;
+ }
+
+ EScan Seek(TLead& lead, ui64 seq) noexcept final {
YQL_ENSURE(seq == CurrentRange);
if (CurrentRange == TableRanges.size()) {
@@ -221,80 +221,80 @@ private:
"TableRanges is over"
<< ", at: " << ScanActorId << ", scanId: " << ScanId
<< ", table: " << TablePath);
- return EScan::Final;
- }
-
+ return EScan::Final;
+ }
+
auto& range = TableRanges[CurrentRange];
- int cmpFrom;
- int cmpTo;
- cmpFrom = CompareBorders<false, false>(
+ int cmpFrom;
+ int cmpTo;
+ cmpFrom = CompareBorders<false, false>(
range.From.GetCells(),
TableInfo->Range.From.GetCells(),
range.FromInclusive,
TableInfo->Range.FromInclusive,
TableInfo->KeyColumnTypes);
-
- cmpTo = CompareBorders<true, true>(
+
+ cmpTo = CompareBorders<true, true>(
range.To.GetCells(),
TableInfo->Range.To.GetCells(),
range.ToInclusive,
TableInfo->Range.ToInclusive,
TableInfo->KeyColumnTypes);
-
- if (cmpFrom > 0) {
+
+ if (cmpFrom > 0) {
auto seek = range.FromInclusive ? NTable::ESeek::Lower : NTable::ESeek::Upper;
lead.To(Tags, range.From.GetCells(), seek);
- } else {
- lead.To(Tags, {}, NTable::ESeek::Lower);
- }
-
+ } else {
+ lead.To(Tags, {}, NTable::ESeek::Lower);
+ }
+
if (cmpTo < 0) {
lead.Until(range.To.GetCells(), range.ToInclusive);
}
- return EScan::Feed;
- }
-
- EScan Feed(TArrayRef<const TCell> key, const TRow& row) noexcept final {
- LastKey = TOwnedCellVec(key);
-
- Y_VERIFY(SkipNullKeys.size() <= key.size());
- for (ui32 i = 0; i < SkipNullKeys.size(); ++i) {
- if (SkipNullKeys[i] && key[i].IsNull()) {
- return EScan::Feed;
- }
- }
-
+ return EScan::Feed;
+ }
+
+ EScan Feed(TArrayRef<const TCell> key, const TRow& row) noexcept final {
+ LastKey = TOwnedCellVec(key);
+
+ Y_VERIFY(SkipNullKeys.size() <= key.size());
+ for (ui32 i = 0; i < SkipNullKeys.size(); ++i) {
+ if (SkipNullKeys[i] && key[i].IsNull()) {
+ return EScan::Feed;
+ }
+ }
+
MakeResult();
-
- if (Y_UNLIKELY(IsProfile())) {
- Result->WaitTime += TInstant::Now() - StartWaitTime;
- }
-
- Y_DEFER {
- if (Y_UNLIKELY(IsProfile())) {
- StartWaitTime = TInstant::Now();
- }
- };
-
+
+ if (Y_UNLIKELY(IsProfile())) {
+ Result->WaitTime += TInstant::Now() - StartWaitTime;
+ }
+
+ Y_DEFER {
+ if (Y_UNLIKELY(IsProfile())) {
+ StartWaitTime = TInstant::Now();
+ }
+ };
+
AddRow(row);
-
- auto sent = SendResult(/* pageFault */ false);
-
+
+ auto sent = SendResult(/* pageFault */ false);
+
if (!sent) {
// There is free space in memory and results are not sent to caller
return EScan::Feed;
}
-
+
if (PeerFreeSpace <= 0) {
- Sleep = true;
- return EScan::Sleep;
- }
-
+ Sleep = true;
+ return EScan::Sleep;
+ }
+
return EScan::Feed; // sent by rows limit, can send one more batch
- }
-
+ }
+
EScan Exhausted() noexcept override {
LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD,
"Range " << CurrentRange << " of " << TableRanges.size() << " exhausted: try next one."
@@ -311,34 +311,34 @@ private:
return EScan::Reset;
}
- EScan PageFault() noexcept override final {
- ++PageFaults;
- if (Result && !Result->Rows.empty()) {
- bool sent = SendResult(/* pageFault */ true);
-
- if (sent && PeerFreeSpace == 0) {
- Sleep = true;
- return EScan::Sleep;
- }
-
- if (sent && PeerFreeSpace > 0 && Y_UNLIKELY(IsProfile())) {
+ EScan PageFault() noexcept override final {
+ ++PageFaults;
+ if (Result && !Result->Rows.empty()) {
+ bool sent = SendResult(/* pageFault */ true);
+
+ if (sent && PeerFreeSpace == 0) {
+ Sleep = true;
+ return EScan::Sleep;
+ }
+
+ if (sent && PeerFreeSpace > 0 && Y_UNLIKELY(IsProfile())) {
StartWaitTime = TInstant::Now();
- }
- }
- return EScan::Feed;
- }
-
-private:
+ }
+ }
+ return EScan::Feed;
+ }
+
+private:
TAutoPtr<IDestructable> Finish(EAbort abort) noexcept final {
- auto prio = abort == EAbort::None ? NActors::NLog::PRI_DEBUG : NActors::NLog::PRI_ERROR;
- LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::TX_DATASHARD, "Finish scan"
- << ", at: " << ScanActorId << ", scanId: " << ScanId
+ auto prio = abort == EAbort::None ? NActors::NLog::PRI_DEBUG : NActors::NLog::PRI_ERROR;
+ LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::TX_DATASHARD, "Finish scan"
+ << ", at: " << ScanActorId << ", scanId: " << ScanId
<< ", table: " << TablePath << ", reason: " << (int) abort
<< ", abortEvent: " << (AbortEvent ? AbortEvent->Record.ShortDebugString() : TString("<none>")));
-
+
if (abort != EAbort::None || AbortEvent) {
auto ev = MakeHolder<TEvKqpCompute::TEvScanError>(Generation);
-
+
if (AbortEvent) {
ev->Record.SetStatus(AbortEvent->Record.GetStatusCode());
auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_OPERATION_ABORTED, TStringBuilder()
@@ -352,30 +352,30 @@ private:
}
Send(ComputeActorId, ev.Release());
- } else {
- if (Result) {
+ } else {
+ if (Result) {
// TODO:
// Result->CpuTime = CpuTime;
- } else {
+ } else {
Result = MakeHolder<TEvKqpCompute::TEvScanData>(ScanId, Generation);
- }
+ }
auto send = SendResult(Result->PageFault, true);
Y_VERIFY_DEBUG(send);
- }
-
- Driver = nullptr;
- if (TimeoutActorId) {
- Send(TimeoutActorId, new TEvents::TEvPoison);
- }
- PassAway();
-
- return new TKqpScanResult();
- }
-
- void Describe(IOutputStream& out) const noexcept final {
- out << "TExecuteKqpScanTxUnit, TKqpScan";
- }
-
+ }
+
+ Driver = nullptr;
+ if (TimeoutActorId) {
+ Send(TimeoutActorId, new TEvents::TEvPoison);
+ }
+ PassAway();
+
+ return new TKqpScanResult();
+ }
+
+ void Describe(IOutputStream& out) const noexcept final {
+ out << "TExecuteKqpScanTxUnit, TKqpScan";
+ }
+
void MakeResult() {
if (!Result) {
Result = MakeHolder<TEvKqpCompute::TEvScanData>(ScanId, Generation);
@@ -419,16 +419,16 @@ private:
bool SendResult(bool pageFault, bool finish = false) noexcept {
if (Rows >= MAX_BATCH_ROWS || CellvecBytes >= PeerFreeSpace ||
(pageFault && (Rows >= MIN_BATCH_ROWS_ON_PAGEFAULT || CellvecBytes >= MIN_BATCH_SIZE_ON_PAGEFAULT)) || finish)
- {
- Result->PageFault = pageFault;
- Result->PageFaults = PageFaults;
+ {
+ Result->PageFault = pageFault;
+ Result->PageFaults = PageFaults;
if (finish) {
Result->Finished = true;
} else {
Result->LastKey = LastKey;
}
auto sendBytes = CellvecBytes;
-
+
if (DataFormat == NKikimrTxDataShard::EScanDataFormat::ARROW) {
FlushBatchToResult();
sendBytes = NArrow::GetBatchDataSize(Result->ArrowBatch);
@@ -438,20 +438,20 @@ private:
YQL_ENSURE(Result->Rows.size() == Rows);
}
- PageFaults = 0;
-
- LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Send ScanData"
- << ", from: " << ScanActorId << ", to: " << ComputeActorId
- << ", scanId: " << ScanId << ", table: " << TablePath
+ PageFaults = 0;
+
+ LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Send ScanData"
+ << ", from: " << ScanActorId << ", to: " << ComputeActorId
+ << ", scanId: " << ScanId << ", table: " << TablePath
<< ", bytes: " << sendBytes << ", rows: " << Rows << ", page faults: " << Result->PageFaults
- << ", finished: " << Result->Finished << ", pageFault: " << Result->PageFault);
-
+ << ", finished: " << Result->Finished << ", pageFault: " << Result->PageFault);
+
if (PeerFreeSpace < sendBytes) {
- PeerFreeSpace = 0;
- } else {
+ PeerFreeSpace = 0;
+ } else {
PeerFreeSpace -= sendBytes;
- }
-
+ }
+
if (sendBytes >= 48_MB) {
LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Query size limit exceeded.");
if (finish) {
@@ -471,14 +471,14 @@ private:
}
}
- Send(ComputeActorId, Result.Release(), IEventHandle::FlagTrackDelivery);
- ReportDatashardStats();
-
- return true;
- }
- return false;
- }
-
+ Send(ComputeActorId, Result.Release(), IEventHandle::FlagTrackDelivery);
+ ReportDatashardStats();
+
+ return true;
+ }
+ return false;
+ }
+
// Call only after MakeResult method.
void FlushBatchToResult() {
// FlushBatch reset Batch pointer in BatchBuilder only if some rows were added after. So we if we have already
@@ -489,107 +489,107 @@ private:
}
}
- void ReportDatashardStats() {
+ void ReportDatashardStats() {
Send(DatashardActorId, new TDataShard::TEvPrivate::TEvScanStats(Rows, CellvecBytes));
- Rows = 0;
+ Rows = 0;
CellvecBytes = 0;
- }
-
- bool IsProfile() const {
+ }
+
+ bool IsProfile() const {
return StatsMode >= NYql::NDqProto::DQ_STATS_MODE_PROFILE;
- }
-
-private:
- TActorId ComputeActorId;
- const TActorId DatashardActorId;
- const ui32 ScanId;
+ }
+
+private:
+ TActorId ComputeActorId;
+ const TActorId DatashardActorId;
+ const ui32 ScanId;
const NDataShard::TUserTable::TCPtr TableInfo;
- const TString TablePath;
+ const TString TablePath;
const TSmallVec<TSerializedTableRange> TableRanges;
ui32 CurrentRange;
- const TSmallVec<NTable::TTag> Tags;
+ const TSmallVec<NTable::TTag> Tags;
TSmallVec<NScheme::TTypeId> Types;
- const TSmallVec<bool> SkipNullKeys;
+ const TSmallVec<bool> SkipNullKeys;
const NYql::NDqProto::EDqStatsMode StatsMode;
- const TInstant Deadline;
+ const TInstant Deadline;
const ui32 Generation;
const NKikimrTxDataShard::EScanDataFormat DataFormat;
- ui64 PeerFreeSpace = 0;
- bool Sleep;
- const bool IsLocal;
-
- IDriver* Driver = nullptr;
- TActorId ScanActorId;
- TActorId TimeoutActorId;
+ ui64 PeerFreeSpace = 0;
+ bool Sleep;
+ const bool IsLocal;
+
+ IDriver* Driver = nullptr;
+ TActorId ScanActorId;
+ TActorId TimeoutActorId;
TAutoPtr<TEvKqp::TEvAbortExecution> AbortEvent;
-
+
THolder<NArrow::TArrowBatchBuilder> BatchBuilder;
- THolder<TEvKqpCompute::TEvScanData> Result;
- ui64 Rows = 0;
+ THolder<TEvKqpCompute::TEvScanData> Result;
+ ui64 Rows = 0;
ui64 CellvecBytes = 0;
- ui32 PageFaults = 0;
- TInstant StartWaitTime;
-
- // Key corresponding to the latest row added to RowsBuffer. Used to avoid row duplication on
- // scan restarts.
- TOwnedCellVec LastKey;
-};
-
+ ui32 PageFaults = 0;
+ TInstant StartWaitTime;
+
+ // Key corresponding to the latest row added to RowsBuffer. Used to avoid row duplication on
+ // scan restarts.
+ TOwnedCellVec LastKey;
+};
+
void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext&) {
- auto& request = ev->Get()->Record;
- auto scanComputeActor = ev->Sender;
+ auto& request = ev->Get()->Record;
+ auto scanComputeActor = ev->Sender;
auto generation = request.GetGeneration();
-
- auto infoIt = TableInfos.find(request.GetLocalPathId());
-
+
+ auto infoIt = TableInfos.find(request.GetLocalPathId());
+
auto reportError = [this, scanComputeActor, generation] (const TString& table, const TString& detailedReason) {
auto ev = MakeHolder<TEvKqpCompute::TEvScanError>(generation);
- ev->Record.SetStatus(Ydb::StatusIds::ABORTED);
+ ev->Record.SetStatus(Ydb::StatusIds::ABORTED);
auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_SCHEME_MISMATCH, TStringBuilder() <<
"Table '" << table << "' scheme changed.");
- IssueToMessage(issue, ev->Record.MutableIssues()->Add());
- Send(scanComputeActor, ev.Release(), IEventHandle::FlagTrackDelivery);
- LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, detailedReason);
- };
-
- if (infoIt == TableInfos.end()) {
- reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "."
- << " Can not find table '" << request.GetTablePath() << "'"
- << " by LocalPathId " << request.GetLocalPathId() << " at " << TabletID());
- return;
- }
-
+ IssueToMessage(issue, ev->Record.MutableIssues()->Add());
+ Send(scanComputeActor, ev.Release(), IEventHandle::FlagTrackDelivery);
+ LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, detailedReason);
+ };
+
+ if (infoIt == TableInfos.end()) {
+ reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "."
+ << " Can not find table '" << request.GetTablePath() << "'"
+ << " by LocalPathId " << request.GetLocalPathId() << " at " << TabletID());
+ return;
+ }
+
auto tableInfo = infoIt->second; // copy table info ptr here
auto& tableColumns = tableInfo->Columns;
- Y_VERIFY(request.GetColumnTags().size() == request.GetColumnTypes().size());
-
+ Y_VERIFY(request.GetColumnTags().size() == request.GetColumnTypes().size());
+
if (tableInfo->GetTableSchemaVersion() != 0 &&
request.GetSchemaVersion() != tableInfo->GetTableSchemaVersion())
- {
- reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "."
- << " Table '" << request.GetTablePath() << "'"
- << " schema version changed at " << TabletID());
- return;
- }
-
-
- for (int i = 0; i < request.GetColumnTags().size(); ++i) {
- auto* column = tableColumns.FindPtr(request.GetColumnTags(i));
- if (!column) {
- reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "."
- << " Cant find table '" << request.GetTablePath() << "'"
- << " column " << request.GetColumnTags(i) << " at " << TabletID());
- return;
- }
-
- if (column->Type != request.GetColumnTypes(i)) {
- reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "."
- << " Table '" << request.GetTablePath() << "'"
- << " column " << request.GetColumnTags(i) << " type mismatch at " << TabletID());
- return;
- }
- }
-
+ {
+ reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "."
+ << " Table '" << request.GetTablePath() << "'"
+ << " schema version changed at " << TabletID());
+ return;
+ }
+
+
+ for (int i = 0; i < request.GetColumnTags().size(); ++i) {
+ auto* column = tableColumns.FindPtr(request.GetColumnTags(i));
+ if (!column) {
+ reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "."
+ << " Cant find table '" << request.GetTablePath() << "'"
+ << " column " << request.GetColumnTags(i) << " at " << TabletID());
+ return;
+ }
+
+ if (column->Type != request.GetColumnTypes(i)) {
+ reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "."
+ << " Table '" << request.GetTablePath() << "'"
+ << " column " << request.GetColumnTags(i) << " type mismatch at " << TabletID());
+ return;
+ }
+ }
+
if (request.HasOlapProgram()) {
auto msg = TStringBuilder() << "TxId: " << request.GetTxId() << "."
<< " Unexpected process program in datashard scan at " << TabletID();
@@ -603,8 +603,8 @@ void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext&
return;
}
- auto& snapshot = request.GetSnapshot();
-
+ auto& snapshot = request.GetSnapshot();
+
auto snapshotKey = TSnapshotKey(PathOwnerId, request.GetLocalPathId(), snapshot.GetStep(), snapshot.GetTxId());
if (!SnapshotManager.FindAvailable(snapshotKey)) {
reportError(request.GetTablePath(), TStringBuilder() << "TxId: " << request.GetTxId() << "."
@@ -612,8 +612,8 @@ void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext&
return;
}
- Pipeline.StartStreamingTx(snapshot.GetTxId(), 1);
-
+ Pipeline.StartStreamingTx(snapshot.GetTxId(), 1);
+
TSmallVec<TSerializedTableRange> ranges;
ranges.reserve(request.RangesSize());
@@ -621,28 +621,28 @@ void TDataShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext&
ranges.emplace_back(std::move(TSerializedTableRange(range)));
}
- auto* tableScan = new TKqpScan(
- scanComputeActor,
- SelfId(),
- request.GetScanId(),
- tableInfo,
+ auto* tableScan = new TKqpScan(
+ scanComputeActor,
+ SelfId(),
+ request.GetScanId(),
+ tableInfo,
std::move(ranges),
std::move(TSmallVec<NTable::TTag>(request.GetColumnTags().begin(), request.GetColumnTags().end())),
std::move(TSmallVec<bool>(request.GetSkipNullKeys().begin(), request.GetSkipNullKeys().end())),
- request.GetStatsMode(),
+ request.GetStatsMode(),
request.GetTimeoutMs(),
generation,
request.GetDataFormat()
- );
-
- auto scanOptions = TScanOptions()
- .DisableResourceBroker()
- .SetReadPrio(TScanOptions::EReadPrio::Low)
- .SetReadAhead(READAHEAD_LO, READAHEAD_HI)
- .SetSnapshotRowVersion(TRowVersion(snapshot.GetStep(), snapshot.GetTxId()));
-
+ );
+
+ auto scanOptions = TScanOptions()
+ .DisableResourceBroker()
+ .SetReadPrio(TScanOptions::EReadPrio::Low)
+ .SetReadAhead(READAHEAD_LO, READAHEAD_HI)
+ .SetSnapshotRowVersion(TRowVersion(snapshot.GetStep(), snapshot.GetTxId()));
+
Executor()->QueueScan(tableInfo->LocalTid, tableScan, snapshot.GetTxId(), scanOptions);
-}
-
-}
-}
+}
+
+}
+}
diff --git a/ydb/core/tx/datashard/datashard__op_rows.cpp b/ydb/core/tx/datashard/datashard__op_rows.cpp
index 3b3c9d9d6f9..fc77402f2e8 100644
--- a/ydb/core/tx/datashard/datashard__op_rows.cpp
+++ b/ydb/core/tx/datashard/datashard__op_rows.cpp
@@ -92,8 +92,8 @@ static void WrongShardState(NKikimrTxDataShard::TEvEraseRowsResponse& response)
template <typename TEvResponse, typename TEvRequest>
static bool MaybeReject(TDataShard* self, TEvRequest& ev, const TActorContext& ctx, const TString& txDesc, bool isWrite) {
NKikimrTxDataShard::TEvProposeTransactionResult::EStatus rejectStatus;
- TString rejectReason;
- bool reject = self->CheckDataTxReject(txDesc, ctx, rejectStatus, rejectReason);
+ TString rejectReason;
+ bool reject = self->CheckDataTxReject(txDesc, ctx, rejectStatus, rejectReason);
bool outOfSpace = false;
if (!reject && isWrite) {
@@ -116,7 +116,7 @@ static bool MaybeReject(TDataShard* self, TEvRequest& ev, const TActorContext& c
LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, "Rejecting " << txDesc << " request on datashard"
<< ": tablet# " << self->TabletID()
- << ", error# " << rejectReason);
+ << ", error# " << rejectReason);
auto response = MakeHolder<TEvResponse>();
response->Record.SetTabletID(self->TabletID());
@@ -125,7 +125,7 @@ static bool MaybeReject(TDataShard* self, TEvRequest& ev, const TActorContext& c
} else {
WrongShardState(response->Record);
}
- response->Record.SetErrorDescription(rejectReason);
+ response->Record.SetErrorDescription(rejectReason);
ctx.Send(ev->Sender, std::move(response));
return true;
diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h
index 11313474f80..d44eef218b8 100644
--- a/ydb/core/tx/datashard/datashard_impl.h
+++ b/ydb/core/tx/datashard/datashard_impl.h
@@ -229,7 +229,7 @@ class TDataShard
friend class TTxS3Listing;
friend class TExecuteKqpScanTxUnit;
friend class TTableScan;
- friend class TKqpScan;
+ friend class TKqpScan;
friend class TTransQueue;
friend class TOutReadSets;
@@ -917,7 +917,7 @@ class TDataShard
void Handle(TEvDataShard::TEvGetTableStats::TPtr& ev, const TActorContext& ctx);
void Handle(TEvPrivate::TEvAsyncTableStats::TPtr& ev, const TActorContext& ctx);
void Handle(TEvDataShard::TEvS3ListingRequest::TPtr& ev, const TActorContext& ctx);
- void Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext& ctx);
+ void Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext& ctx);
void Handle(TEvDataShard::TEvUploadRowsRequest::TPtr& ev, const TActorContext& ctx);
void Handle(TEvDataShard::TEvEraseRowsRequest::TPtr& ev, const TActorContext& ctx);
void Handle(TEvDataShard::TEvConditionalEraseRowsRequest::TPtr& ev, const TActorContext& ctx);
@@ -1157,7 +1157,7 @@ public:
bool CheckDataTxReject(const TString& opDescr,
const TActorContext &ctx,
NKikimrTxDataShard::TEvProposeTransactionResult::EStatus& rejectStatus,
- TString &reason);
+ TString &reason);
bool CheckDataTxRejectAndReply(TEvDataShard::TEvProposeTransaction* msg, const TActorContext& ctx);
TSysLocks& SysLocksTable() { return SysLocks; }
@@ -2179,7 +2179,7 @@ protected:
HFunc(TEvDataShard::TEvGetTableStats, Handle);
HFunc(TEvPrivate::TEvAsyncTableStats, Handle);
HFunc(TEvDataShard::TEvS3ListingRequest, Handle);
- HFunc(TEvDataShard::TEvKqpScan, Handle);
+ HFunc(TEvDataShard::TEvKqpScan, Handle);
HFunc(TEvDataShard::TEvUploadRowsRequest, Handle);
HFunc(TEvDataShard::TEvEraseRowsRequest, Handle);
HFunc(TEvDataShard::TEvConditionalEraseRowsRequest, Handle);
diff --git a/ydb/core/tx/datashard/datashard_kqp_compute.cpp b/ydb/core/tx/datashard/datashard_kqp_compute.cpp
index edd6ca8e194..31df96e4a11 100644
--- a/ydb/core/tx/datashard/datashard_kqp_compute.cpp
+++ b/ydb/core/tx/datashard/datashard_kqp_compute.cpp
@@ -27,7 +27,7 @@ typedef IComputationNode* (*TCallableDatashardBuilderFunc)(TCallable& callable,
struct TKqpDatashardComputationMap {
TKqpDatashardComputationMap() {
- Map["KqpWideReadTable"] = &WrapKqpWideReadTable;
+ Map["KqpWideReadTable"] = &WrapKqpWideReadTable;
Map["KqpWideReadTableRanges"] = &WrapKqpWideReadTableRanges;
Map["KqpLookupTable"] = &WrapKqpLookupTable;
Map["KqpUpsertRows"] = &WrapKqpUpsertRows;
@@ -65,7 +65,7 @@ typedef IComputationNode* (*TCallableScanBuilderFunc)(TCallable& callable,
struct TKqpScanComputationMap {
TKqpScanComputationMap() {
- Map["KqpWideReadTable"] = &WrapKqpScanWideReadTable;
+ Map["KqpWideReadTable"] = &WrapKqpScanWideReadTable;
Map["KqpWideReadTableRanges"] = &WrapKqpScanWideReadTableRanges;
}
diff --git a/ydb/core/tx/datashard/datashard_kqp_compute.h b/ydb/core/tx/datashard/datashard_kqp_compute.h
index 4bbb8fa0de7..141ae0a1a92 100644
--- a/ydb/core/tx/datashard/datashard_kqp_compute.h
+++ b/ydb/core/tx/datashard/datashard_kqp_compute.h
@@ -102,11 +102,11 @@ IComputationNode* WrapKqpDeleteRows(TCallable& callable, const TComputationNodeF
TKqpDatashardComputeContext& computeCtx);
IComputationNode* WrapKqpEffects(TCallable& callable, const TComputationNodeFactoryContext& ctx,
TKqpDatashardComputeContext& computeCtx);
-IComputationNode* WrapKqpWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx,
- TKqpDatashardComputeContext& computeCtx);
+IComputationNode* WrapKqpWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx,
+ TKqpDatashardComputeContext& computeCtx);
TComputationNodeFactory GetKqpDatashardComputeFactory(TKqpDatashardComputeContext* computeCtx);
TComputationNodeFactory GetKqpScanComputeFactory(TKqpScanComputeContext* computeCtx);
-} // namespace NMiniKQL
+} // namespace NMiniKQL
} // namespace NKikimr
diff --git a/ydb/core/tx/datashard/datashard_kqp_read_table.cpp b/ydb/core/tx/datashard/datashard_kqp_read_table.cpp
index 9fbd8ef0ad2..9cc1bed0fd1 100644
--- a/ydb/core/tx/datashard/datashard_kqp_read_table.cpp
+++ b/ydb/core/tx/datashard/datashard_kqp_read_table.cpp
@@ -74,18 +74,18 @@ TMaybe<TKeyRangesType> ParseKeyRangesType(const TTupleType* rangeTupleType) {
};
}
-TSerializedTableRange CreateTableRange(const TParseReadTableResult& parseResult, const IComputationNode* fromNode,
- const IComputationNode* toNode, const TTypeEnvironment& typeEnv, TComputationContext& ctx)
-{
- TVector<TCell> fromCells;
- BuildKeyTupleCells(parseResult.FromTuple->GetType(), fromNode->GetValue(ctx), fromCells, typeEnv);
-
- TVector<TCell> toCells;
- BuildKeyTupleCells(parseResult.ToTuple->GetType(), toNode->GetValue(ctx), toCells, typeEnv);
-
- return TSerializedTableRange(fromCells, parseResult.FromInclusive, toCells, parseResult.ToInclusive);
-}
-
+TSerializedTableRange CreateTableRange(const TParseReadTableResult& parseResult, const IComputationNode* fromNode,
+ const IComputationNode* toNode, const TTypeEnvironment& typeEnv, TComputationContext& ctx)
+{
+ TVector<TCell> fromCells;
+ BuildKeyTupleCells(parseResult.FromTuple->GetType(), fromNode->GetValue(ctx), fromCells, typeEnv);
+
+ TVector<TCell> toCells;
+ BuildKeyTupleCells(parseResult.ToTuple->GetType(), toNode->GetValue(ctx), toCells, typeEnv);
+
+ return TSerializedTableRange(fromCells, parseResult.FromInclusive, toCells, parseResult.ToInclusive);
+}
+
TSerializedTableRange BuildFullRange(ui32 keyColumnsSize) {
/* Build range from NULL, ... NULL to +inf, ... +inf */
TVector<TCell> fromKeyValues(keyColumnsSize);
@@ -183,51 +183,51 @@ TVector<TSerializedTableRange> CreateTableRanges(const TParseReadTableRangesResu
return ranges;
}
-void CreateRangePoints(ui64 localTid, const TSerializedTableRange& serializedTableRange, TSmallVec<TRawTypeValue>& from,
- TSmallVec<TRawTypeValue>& to, TKqpDatashardComputeContext& computeCtx)
-{
- const auto* tableInfo = computeCtx.Database->GetScheme().GetTableInfo(localTid);
- auto tableRange = serializedTableRange.ToTableRange();
- ConvertTableKeys(computeCtx.Database->GetScheme(), tableInfo, tableRange.From, from, nullptr);
- ConvertTableKeys(computeCtx.Database->GetScheme(), tableInfo, tableRange.To, to, nullptr);
-}
-
+void CreateRangePoints(ui64 localTid, const TSerializedTableRange& serializedTableRange, TSmallVec<TRawTypeValue>& from,
+ TSmallVec<TRawTypeValue>& to, TKqpDatashardComputeContext& computeCtx)
+{
+ const auto* tableInfo = computeCtx.Database->GetScheme().GetTableInfo(localTid);
+ auto tableRange = serializedTableRange.ToTableRange();
+ ConvertTableKeys(computeCtx.Database->GetScheme(), tableInfo, tableRange.From, from, nullptr);
+ ConvertTableKeys(computeCtx.Database->GetScheme(), tableInfo, tableRange.To, to, nullptr);
+}
+
template <bool IsReverse>
class TKqpWideReadTableWrapperBase : public TStatelessWideFlowCodegeneratorNode<TKqpWideReadTableWrapperBase<IsReverse>> {
-public:
+public:
TKqpWideReadTableWrapperBase(TKqpDatashardComputeContext& computeCtx, const TTypeEnvironment& typeEnv,
const TSmallVec<TTag>& systemColumnTags, const TSmallVec<bool>& skipNullKeys)
: TStatelessWideFlowCodegeneratorNode<TKqpWideReadTableWrapperBase<IsReverse>>(this)
, ComputeCtx(computeCtx)
- , TypeEnv(typeEnv)
+ , TypeEnv(typeEnv)
, SystemColumnTags(systemColumnTags)
, SkipNullKeys(skipNullKeys)
, ShardTableStats(ComputeCtx.GetDatashardCounters())
, TaskTableStats(ComputeCtx.GetTaskCounters(ComputeCtx.GetCurrentTaskId())) {
}
-
- EFetchResult DoCalculate(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const {
- return this->ReadValue(ctx, output);
- }
-
-#ifndef MKQL_DISABLE_CODEGEN
- ICodegeneratorInlineWideNode::TGenerateResult DoGenGetValues(const TCodegenContext& ctx, BasicBlock*& block) const {
- Y_UNUSED(ctx, block);
- Y_FAIL("LLVM compilation is not implemented for OLTP-workload");
- }
-#endif
-
+
+ EFetchResult DoCalculate(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const {
+ return this->ReadValue(ctx, output);
+ }
+
+#ifndef MKQL_DISABLE_CODEGEN
+ ICodegeneratorInlineWideNode::TGenerateResult DoGenGetValues(const TCodegenContext& ctx, BasicBlock*& block) const {
+ Y_UNUSED(ctx, block);
+ Y_FAIL("LLVM compilation is not implemented for OLTP-workload");
+ }
+#endif
+
protected:
virtual EFetchResult ReadValue(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const = 0;
-
+
EFetchResult ReadNext(NUdf::TUnboxedValue* const* output) const {
bool breakLocks = false;
- while (Iterator->Next(NTable::ENext::Data) == NTable::EReady::Data) {
+ while (Iterator->Next(NTable::ENext::Data) == NTable::EReady::Data) {
if (!breakLocks && (breakLocks = bool(Iterator->Stats.InvisibleRowSkips))) {
ComputeCtx.BreakSetLocks();
}
- TDbTupleRef rowKey = Iterator->GetKey();
-
+ TDbTupleRef rowKey = Iterator->GetKey();
+
ui64 deletedRowSkips = std::exchange(Iterator->Stats.DeletedRowSkips, 0);
ui64 invisibleRowSkips = std::exchange(Iterator->Stats.InvisibleRowSkips, 0);
@@ -238,47 +238,47 @@ protected:
TaskTableStats.InvisibleRowSkips += invisibleRowSkips;
Y_VERIFY(SkipNullKeys.size() <= rowKey.ColumnCount);
- bool skipRow = false;
+ bool skipRow = false;
for (ui32 i = 0; i < SkipNullKeys.size(); ++i) {
if (SkipNullKeys[i] && rowKey.Columns[i].IsNull()) {
- skipRow = true;
- break;
- }
- }
- if (skipRow) {
- continue;
- }
-
- TDbTupleRef rowValues = Iterator->GetValues();
-
- size_t columnsCount = rowValues.ColumnCount + SystemColumnTags.size();
-
- ui64 rowSize = 0;
- for (ui32 i = 0; i < rowValues.ColumnCount; ++i) {
- rowSize += rowValues.Columns[i].IsNull() ? 1 : rowValues.Columns[i].Size();
- if (auto out = *output++) {
- *out = GetCellValue(rowValues.Cells()[i], rowValues.Types[i]);
- }
- }
-
- // Some per-row overhead to deal with the case when no columns were requested
- rowSize = std::max(rowSize, (ui64) 8);
-
- for (ui32 i = rowValues.ColumnCount, j = 0; i < columnsCount; ++i, ++j) {
- auto out = *output++;
- if (!out) {
- continue;
- }
-
- switch (SystemColumnTags[j]) {
- case TKeyDesc::EColumnIdDataShard:
- *out = TUnboxedValue(TUnboxedValuePod(ComputeCtx.GetShardId()));
- break;
- default:
- throw TSchemeErrorTabletException();
- }
- }
-
+ skipRow = true;
+ break;
+ }
+ }
+ if (skipRow) {
+ continue;
+ }
+
+ TDbTupleRef rowValues = Iterator->GetValues();
+
+ size_t columnsCount = rowValues.ColumnCount + SystemColumnTags.size();
+
+ ui64 rowSize = 0;
+ for (ui32 i = 0; i < rowValues.ColumnCount; ++i) {
+ rowSize += rowValues.Columns[i].IsNull() ? 1 : rowValues.Columns[i].Size();
+ if (auto out = *output++) {
+ *out = GetCellValue(rowValues.Cells()[i], rowValues.Types[i]);
+ }
+ }
+
+ // Some per-row overhead to deal with the case when no columns were requested
+ rowSize = std::max(rowSize, (ui64) 8);
+
+ for (ui32 i = rowValues.ColumnCount, j = 0; i < columnsCount; ++i, ++j) {
+ auto out = *output++;
+ if (!out) {
+ continue;
+ }
+
+ switch (SystemColumnTags[j]) {
+ case TKeyDesc::EColumnIdDataShard:
+ *out = TUnboxedValue(TUnboxedValuePod(ComputeCtx.GetShardId()));
+ break;
+ default:
+ throw TSchemeErrorTabletException();
+ }
+ }
+
if (Remains) {
Remains = *Remains - 1;
}
@@ -289,9 +289,9 @@ protected:
TaskTableStats.SelectRangeRows++;
TaskTableStats.SelectRangeBytes += rowSize;
- return EFetchResult::One;
- }
-
+ return EFetchResult::One;
+ }
+
if (!breakLocks && bool(Iterator->Stats.InvisibleRowSkips)) {
ComputeCtx.BreakSetLocks();
}
@@ -308,11 +308,11 @@ protected:
if (Iterator->Last() == NTable::EReady::Page) {
ComputeCtx.SetTabletNotReady();
return EFetchResult::Yield;
- }
-
- return EFetchResult::Finish;
- }
-
+ }
+
+ return EFetchResult::Finish;
+ }
+
protected:
TKqpDatashardComputeContext& ComputeCtx;
const TTypeEnvironment& TypeEnv;
@@ -343,7 +343,7 @@ public:
this->TaskTableStats.NSelectRange++;
}
-private:
+private:
EFetchResult ReadValue(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const final {
if (!this->Iterator) {
auto serializedTableRange = CreateTableRange(ParseResult, FromNode, ToNode, this->TypeEnv, ctx);
@@ -378,21 +378,21 @@ private:
}
private:
- void RegisterDependencies() const final {
+ void RegisterDependencies() const final {
this->FlowDependsOn(FromNode);
this->FlowDependsOn(ToNode);
- }
-
-private:
- TParseReadTableResult ParseResult;
- IComputationNode* FromNode;
- IComputationNode* ToNode;
+ }
+
+private:
+ TParseReadTableResult ParseResult;
+ IComputationNode* FromNode;
+ IComputationNode* ToNode;
IComputationNode* ItemsLimit;
- ui64 LocalTid;
- TSmallVec<TTag> ColumnTags;
+ ui64 LocalTid;
+ TSmallVec<TTag> ColumnTags;
ui64 TaskId;
-};
-
+};
+
template <bool IsReverse>
class TKqpWideReadTableRangesWrapper : public TKqpWideReadTableWrapperBase<IsReverse> {
public:
@@ -582,13 +582,13 @@ IComputationNode* WrapKqpWideReadTableRanges(TCallable& callable, const TComputa
return new TKqpWideReadTableRangesWrapper<false>(computeCtx, ctx.Env, parseResult, rangesNode, itemsLimit);
}
-IComputationNode* WrapKqpWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx,
- TKqpDatashardComputeContext& computeCtx)
-{
- auto parseResult = ParseWideReadTable(callable);
- auto fromNode = LocateNode(ctx.NodeLocator, *parseResult.FromTuple);
- auto toNode = LocateNode(ctx.NodeLocator, *parseResult.ToTuple);
-
+IComputationNode* WrapKqpWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx,
+ TKqpDatashardComputeContext& computeCtx)
+{
+ auto parseResult = ParseWideReadTable(callable);
+ auto fromNode = LocateNode(ctx.NodeLocator, *parseResult.FromTuple);
+ auto toNode = LocateNode(ctx.NodeLocator, *parseResult.ToTuple);
+
auto keyColumns = computeCtx.GetKeyColumnsInfo(parseResult.TableId);
ValidateKeyTuple(parseResult.FromTuple->GetType(), keyColumns);
ValidateKeyTuple(parseResult.ToTuple->GetType(), keyColumns);
@@ -603,7 +603,7 @@ IComputationNode* WrapKqpWideReadTable(TCallable& callable, const TComputationNo
}
return new TKqpWideReadTableWrapper<false>(computeCtx, ctx.Env, parseResult, fromNode, toNode, itemsLimit);
-}
-
-} // namespace NMiniKQL
+}
+
+} // namespace NMiniKQL
} // namespace NKikimr
diff --git a/ydb/core/tx/datashard/datashard_pipeline.h b/ydb/core/tx/datashard/datashard_pipeline.h
index 67c35260b28..9f265580377 100644
--- a/ydb/core/tx/datashard/datashard_pipeline.h
+++ b/ydb/core/tx/datashard/datashard_pipeline.h
@@ -305,8 +305,8 @@ public:
void HoldExecutionProfile(TOperation::TPtr op);
void FillStoredExecutionProfiles(NKikimrTxDataShard::TEvGetSlowOpProfilesResponse &rec) const;
- void StartStreamingTx(ui64 txId, ui32 count) {
- ActiveStreamingTxs[txId] += count;
+ void StartStreamingTx(ui64 txId, ui32 count) {
+ ActiveStreamingTxs[txId] += count;
}
bool FinishStreamingTx(ui64 txId) {
diff --git a/ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp b/ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp
index b2b14de1904..db45e2820e1 100644
--- a/ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp
+++ b/ydb/core/tx/datashard/datashard_ut_kqp_scan.cpp
@@ -3,29 +3,29 @@
#include <ydb/core/tx/datashard/datashard.h>
#include <ydb/core/tx/datashard/datashard_ut_common_kqp.h>
#include <ydb/core/tx/tx_proxy/proxy.h>
-
-namespace NKikimr {
-namespace NKqp {
-
-using namespace NYdb;
-using namespace NYdb::NTable;
-using namespace NYdb::NScripting;
-using namespace Tests;
+
+namespace NKikimr {
+namespace NKqp {
+
+using namespace NYdb;
+using namespace NYdb::NTable;
+using namespace NYdb::NScripting;
+using namespace Tests;
using namespace NKikimr::NDataShard::NKqpHelpers;
-
-namespace {
-
- /* sum(value) == 596400 */
- TString FillTableQuery() {
- TStringBuilder sql;
- sql << "UPSERT INTO [/Root/table-1] (key, value) VALUES ";
- for (int i = 1; i < 100; ++i) {
- sql << " (" << i << ", " << i << i << "),";
- }
- sql << " (100500, 100500);";
- return sql;
- }
-
+
+namespace {
+
+ /* sum(value) == 596400 */
+ TString FillTableQuery() {
+ TStringBuilder sql;
+ sql << "UPSERT INTO [/Root/table-1] (key, value) VALUES ";
+ for (int i = 1; i < 100; ++i) {
+ sql << " (" << i << ", " << i << i << "),";
+ }
+ sql << " (100500, 100500);";
+ return sql;
+ }
+
void EnableLogging(TTestActorRuntime& runtime) {
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_DEBUG);
//runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG);
@@ -36,344 +36,344 @@ namespace {
runtime.SetLogPriority(NKikimrServices::KQP_COMPUTE, NActors::NLog::PRI_TRACE);
}
-}
-
-Y_UNIT_TEST_SUITE(KqpScan) {
-
-
- /*
+}
+
+Y_UNIT_TEST_SUITE(KqpScan) {
+
+
+ /*
* Start scan on shard with amount of rows bigger then ScanData capacity (~ 10 ScanData's to
- * complete scan). Event capture filter will pass only first ScanData from one table actor and will
- * kill tablet after that. So in order to complete scan ComputeActor need to handle scan restart after
- * each ScanData.
- */
+ * complete scan). Event capture filter will pass only first ScanData from one table actor and will
+ * kill tablet after that. So in order to complete scan ComputeActor need to handle scan restart after
+ * each ScanData.
+ */
Y_UNIT_TEST_WITH_MVCC(ScanRetryRead) {
- NKikimrConfig::TAppConfig appCfg;
-
- auto* rm = appCfg.MutableTableServiceConfig()->MutableResourceManager();
- rm->SetChannelBufferSize(100);
- rm->SetMinChannelBufferSize(100);
- rm->SetScanBufferSize(100);
-
- TPortManager pm;
- TServerSettings serverSettings(pm.GetPort(2134));
- serverSettings.SetDomainName("Root")
+ NKikimrConfig::TAppConfig appCfg;
+
+ auto* rm = appCfg.MutableTableServiceConfig()->MutableResourceManager();
+ rm->SetChannelBufferSize(100);
+ rm->SetMinChannelBufferSize(100);
+ rm->SetScanBufferSize(100);
+
+ TPortManager pm;
+ TServerSettings serverSettings(pm.GetPort(2134));
+ serverSettings.SetDomainName("Root")
.SetEnableMvcc(WithMvcc)
- .SetNodeCount(2)
- .SetAppConfig(appCfg)
- .SetUseRealThreads(false);
-
- Tests::TServer::TPtr server = new TServer(serverSettings);
- auto &runtime = *server->GetRuntime();
- auto sender = runtime.AllocateEdgeActor();
-
+ .SetNodeCount(2)
+ .SetAppConfig(appCfg)
+ .SetUseRealThreads(false);
+
+ Tests::TServer::TPtr server = new TServer(serverSettings);
+ auto &runtime = *server->GetRuntime();
+ auto sender = runtime.AllocateEdgeActor();
+
// EnableLogging(runtime);
-
- InitRoot(server, sender);
+
+ InitRoot(server, sender);
CreateShardedTable(server, sender, "/Root", "table-1", 1);
- ExecSQL(server, sender, FillTableQuery());
-
- TSet<TActorId> scans;
+ ExecSQL(server, sender, FillTableQuery());
+
+ TSet<TActorId> scans;
TSet<TActorId> killedTablets;
-
- ui64 result = 0;
-
- auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> auto {
-
- switch (ev->GetTypeRewrite()) {
- /*
- * Trick executor to think that all datashard are located on node 1.
- */
- case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: {
- auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>();
- for (auto& [shardId, nodeId]: msg->ShardNodes) {
+
+ ui64 result = 0;
+
+ auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) -> auto {
+
+ switch (ev->GetTypeRewrite()) {
+ /*
+ * Trick executor to think that all datashard are located on node 1.
+ */
+ case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: {
+ auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>();
+ for (auto& [shardId, nodeId]: msg->ShardNodes) {
Cerr << "-- nodeId: " << nodeId << Endl;
Cerr.Flush();
nodeId = runtime.GetNodeId(0);
- }
- break;
- }
-
- case TEvDataShard::EvKqpScan: {
- Cerr << (TStringBuilder() << "-- EvScan " << ev->Sender << " -> " << ev->Recipient << Endl);
+ }
+ break;
+ }
+
+ case TEvDataShard::EvKqpScan: {
+ Cerr << (TStringBuilder() << "-- EvScan " << ev->Sender << " -> " << ev->Recipient << Endl);
Cerr.Flush();
- break;
- }
-
- /*
- * Respond to streamData with acks. Without that execution pipeline will stop
- * producing new tuples.
- */
- case NKqp::TKqpExecuterEvents::EvStreamData: {
- auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record;
-
- Cerr << (TStringBuilder() << "-- EvStreamData: " << record.AsJSON() << Endl);
+ break;
+ }
+
+ /*
+ * Respond to streamData with acks. Without that execution pipeline will stop
+ * producing new tuples.
+ */
+ case NKqp::TKqpExecuterEvents::EvStreamData: {
+ auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record;
+
+ Cerr << (TStringBuilder() << "-- EvStreamData: " << record.AsJSON() << Endl);
Cerr.Flush();
-
- Y_ASSERT(record.GetResultSet().rows().size() == 1);
- Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1);
- result = record.GetResultSet().rows().at(0).items().at(0).uint64_value();
-
- auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>();
- resp->Record.SetEnough(false);
- resp->Record.SetSeqNo(ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record.GetSeqNo());
- resp->Record.SetFreeSpace(100);
- runtime.Send(new IEventHandle(ev->Sender, sender, resp.Release()));
- return TTestActorRuntime::EEventAction::DROP;
- }
-
- /* Drop message and kill tablet if we already had seen this tablet */
- case NKqp::TKqpComputeEvents::EvScanData: {
- if (scans.contains(ev->Sender)) {
+
+ Y_ASSERT(record.GetResultSet().rows().size() == 1);
+ Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1);
+ result = record.GetResultSet().rows().at(0).items().at(0).uint64_value();
+
+ auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>();
+ resp->Record.SetEnough(false);
+ resp->Record.SetSeqNo(ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record.GetSeqNo());
+ resp->Record.SetFreeSpace(100);
+ runtime.Send(new IEventHandle(ev->Sender, sender, resp.Release()));
+ return TTestActorRuntime::EEventAction::DROP;
+ }
+
+ /* Drop message and kill tablet if we already had seen this tablet */
+ case NKqp::TKqpComputeEvents::EvScanData: {
+ if (scans.contains(ev->Sender)) {
if (killedTablets.empty()) { // do only 1 kill per test
runtime.Send(new IEventHandle(ev->Sender, ev->Sender, new NKqp::TEvKqpCompute::TEvKillScanTablet));
Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": hijack event, kill tablet " << ev->Sender << Endl);
Cerr.Flush();
}
- } else {
- scans.insert(ev->Sender);
+ } else {
+ scans.insert(ev->Sender);
runtime.EnableScheduleForActor(ev->Sender);
- Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": pass" << Endl);
+ Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": pass" << Endl);
Cerr.Flush();
- }
-
- break;
- }
-
- default:
- break;
- }
- return TTestActorRuntime::EEventAction::PROCESS;
- };
- runtime.SetObserverFunc(captureEvents);
-
- auto streamSender = runtime.AllocateEdgeActor();
- SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, "SELECT sum(value) FROM `/Root/table-1`;", false));
- auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender);
-
- UNIT_ASSERT_VALUES_EQUAL(result, 596400);
- }
-
- /*
- * Force remote scans by meddling with EvShardsResolveStatus. Check that remote scan actually took place.
- */
+ }
+
+ break;
+ }
+
+ default:
+ break;
+ }
+ return TTestActorRuntime::EEventAction::PROCESS;
+ };
+ runtime.SetObserverFunc(captureEvents);
+
+ auto streamSender = runtime.AllocateEdgeActor();
+ SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, "SELECT sum(value) FROM `/Root/table-1`;", false));
+ auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender);
+
+ UNIT_ASSERT_VALUES_EQUAL(result, 596400);
+ }
+
+ /*
+ * Force remote scans by meddling with EvShardsResolveStatus. Check that remote scan actually took place.
+ */
Y_UNIT_TEST_WITH_MVCC(RemoteShardScan) {
- NKikimrConfig::TAppConfig appCfg;
-
- TPortManager pm;
- TServerSettings serverSettings(pm.GetPort(2134));
- serverSettings.SetDomainName("Root")
+ NKikimrConfig::TAppConfig appCfg;
+
+ TPortManager pm;
+ TServerSettings serverSettings(pm.GetPort(2134));
+ serverSettings.SetDomainName("Root")
.SetEnableMvcc(WithMvcc)
- .SetNodeCount(2)
- .SetAppConfig(appCfg)
- .SetUseRealThreads(false);
-
- Tests::TServer::TPtr server = new TServer(serverSettings);
- auto &runtime = *server->GetRuntime();
- auto sender = runtime.AllocateEdgeActor();
-
+ .SetNodeCount(2)
+ .SetAppConfig(appCfg)
+ .SetUseRealThreads(false);
+
+ Tests::TServer::TPtr server = new TServer(serverSettings);
+ auto &runtime = *server->GetRuntime();
+ auto sender = runtime.AllocateEdgeActor();
+
// EnableLogging(runtime);
-
- InitRoot(server, sender);
+
+ InitRoot(server, sender);
CreateShardedTable(server, sender, "/Root", "table-1", 7);
- ExecSQL(server, sender, FillTableQuery());
-
+ ExecSQL(server, sender, FillTableQuery());
+
ui64 firstNodeId = server->GetRuntime()->GetNodeId(0);
// ui64 secondNodeId = server->GetRuntime()->GetNodeId(1);
- bool remoteScanDetected = false;
-
- ui64 result = 0;
-
- auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) {
-
- switch (ev->GetTypeRewrite()) {
- /*
- * Trick executor to think that all datashard are located on node 1.
- */
- case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: {
- auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>();
- for (auto& [shardId, nodeId]: msg->ShardNodes) {
+ bool remoteScanDetected = false;
+
+ ui64 result = 0;
+
+ auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) {
+
+ switch (ev->GetTypeRewrite()) {
+ /*
+ * Trick executor to think that all datashard are located on node 1.
+ */
+ case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: {
+ auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>();
+ for (auto& [shardId, nodeId]: msg->ShardNodes) {
nodeId = firstNodeId;
- }
- break;
- }
-
- /*
- * Respond to streamData with acks. Without that execution pipeline will stop
- * producing new tuples.
- */
- case NKqp::TKqpExecuterEvents::EvStreamData: {
- auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record;
-
- Cerr << (TStringBuilder() << "-- EvStreamData(from: " << ev->Sender << ", to: " << ev->Recipient << "): " << record.AsJSON() << Endl);
-
- Y_ASSERT(record.GetResultSet().rows().size() == 1);
- Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1);
- result = record.GetResultSet().rows().at(0).items().at(0).uint64_value();
-
- auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>();
- resp->Record.SetEnough(false);
- resp->Record.SetSeqNo(ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record.GetSeqNo());
- resp->Record.SetFreeSpace(100);
- runtime.Send(new IEventHandle(ev->Sender, sender, resp.Release()));
- return TTestActorRuntime::EEventAction::DROP;
- }
-
- /*
- * Check that remote scan actually happend.
- */
- case NKqp::TKqpComputeEvents::EvScanData: {
- remoteScanDetected = remoteScanDetected || ev->Sender.NodeId() != ev->Recipient.NodeId();
- break;
- }
-
- default:
- break;
- }
- return TTestActorRuntime::EEventAction::PROCESS;
- };
- runtime.SetObserverFunc(captureEvents);
-
- auto streamSender = runtime.AllocateEdgeActor();
- SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, "SELECT sum(value) FROM `/Root/table-1`;", false));
- auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender);
-
- UNIT_ASSERT(remoteScanDetected);
- UNIT_ASSERT_VALUES_EQUAL(result, 596400);
- }
-
+ }
+ break;
+ }
+
+ /*
+ * Respond to streamData with acks. Without that execution pipeline will stop
+ * producing new tuples.
+ */
+ case NKqp::TKqpExecuterEvents::EvStreamData: {
+ auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record;
+
+ Cerr << (TStringBuilder() << "-- EvStreamData(from: " << ev->Sender << ", to: " << ev->Recipient << "): " << record.AsJSON() << Endl);
+
+ Y_ASSERT(record.GetResultSet().rows().size() == 1);
+ Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1);
+ result = record.GetResultSet().rows().at(0).items().at(0).uint64_value();
+
+ auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>();
+ resp->Record.SetEnough(false);
+ resp->Record.SetSeqNo(ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record.GetSeqNo());
+ resp->Record.SetFreeSpace(100);
+ runtime.Send(new IEventHandle(ev->Sender, sender, resp.Release()));
+ return TTestActorRuntime::EEventAction::DROP;
+ }
+
+ /*
+ * Check that remote scan actually happend.
+ */
+ case NKqp::TKqpComputeEvents::EvScanData: {
+ remoteScanDetected = remoteScanDetected || ev->Sender.NodeId() != ev->Recipient.NodeId();
+ break;
+ }
+
+ default:
+ break;
+ }
+ return TTestActorRuntime::EEventAction::PROCESS;
+ };
+ runtime.SetObserverFunc(captureEvents);
+
+ auto streamSender = runtime.AllocateEdgeActor();
+ SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, "SELECT sum(value) FROM `/Root/table-1`;", false));
+ auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender);
+
+ UNIT_ASSERT(remoteScanDetected);
+ UNIT_ASSERT_VALUES_EQUAL(result, 596400);
+ }
+
Y_UNIT_TEST_WITH_MVCC(ScanDuringSplit) {
- NKikimrConfig::TAppConfig appCfg;
-
- auto* rm = appCfg.MutableTableServiceConfig()->MutableResourceManager();
- rm->SetChannelBufferSize(100);
- rm->SetMinChannelBufferSize(100);
- rm->SetScanBufferSize(100);
-
- TPortManager pm;
- TServerSettings serverSettings(pm.GetPort(2134));
- serverSettings.SetDomainName("Root")
+ NKikimrConfig::TAppConfig appCfg;
+
+ auto* rm = appCfg.MutableTableServiceConfig()->MutableResourceManager();
+ rm->SetChannelBufferSize(100);
+ rm->SetMinChannelBufferSize(100);
+ rm->SetScanBufferSize(100);
+
+ TPortManager pm;
+ TServerSettings serverSettings(pm.GetPort(2134));
+ serverSettings.SetDomainName("Root")
.SetEnableMvcc(WithMvcc)
- .SetNodeCount(2)
- .SetAppConfig(appCfg)
- .SetUseRealThreads(false);
-
- Tests::TServer::TPtr server = new TServer(serverSettings);
- auto &runtime = *server->GetRuntime();
- auto sender = runtime.AllocateEdgeActor();
- auto senderSplit = runtime.AllocateEdgeActor();
-
+ .SetNodeCount(2)
+ .SetAppConfig(appCfg)
+ .SetUseRealThreads(false);
+
+ Tests::TServer::TPtr server = new TServer(serverSettings);
+ auto &runtime = *server->GetRuntime();
+ auto sender = runtime.AllocateEdgeActor();
+ auto senderSplit = runtime.AllocateEdgeActor();
+
// EnableLogging(runtime);
-
+
SetSplitMergePartCountLimit(&runtime, -1);
- InitRoot(server, sender);
+ InitRoot(server, sender);
CreateShardedTable(server, sender, "/Root", "table-1", 1);
- ExecSQL(server, sender, FillTableQuery());
-
- auto shards = GetTableShards(server, sender, "/Root/table-1");
+ ExecSQL(server, sender, FillTableQuery());
+
+ auto shards = GetTableShards(server, sender, "/Root/table-1");
for (const auto& shard: shards) {
Cerr << (TStringBuilder() << "-- shardId=" << shard << Endl);
Cerr.Flush();
}
-
- TSet<TActorId> scans;
- TActorId firstScanActor;
- ui64 tabletId = 0;
-
- ui64 result = 0;
-
+
+ TSet<TActorId> scans;
+ TActorId firstScanActor;
+ ui64 tabletId = 0;
+
+ ui64 result = 0;
+
auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr<IEventHandle> &ev) {
- switch (ev->GetTypeRewrite()) {
- case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: {
- auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>();
- for (auto& [shardId, nodeId]: msg->ShardNodes) {
- tabletId = shardId;
- Cerr << (TStringBuilder() << "-- tabletId= " << tabletId << Endl);
+ switch (ev->GetTypeRewrite()) {
+ case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: {
+ auto* msg = ev->Get<NKqp::TEvKqpExecuter::TEvShardsResolveStatus>();
+ for (auto& [shardId, nodeId]: msg->ShardNodes) {
+ tabletId = shardId;
+ Cerr << (TStringBuilder() << "-- tabletId= " << tabletId << Endl);
Cerr.Flush();
- }
- break;
- }
-
- case TEvDataShard::EvKqpScan: {
- Cerr << (TStringBuilder() << "-- EvScan " << ev->Sender << " -> " << ev->Recipient << Endl);
+ }
+ break;
+ }
+
+ case TEvDataShard::EvKqpScan: {
+ Cerr << (TStringBuilder() << "-- EvScan " << ev->Sender << " -> " << ev->Recipient << Endl);
Cerr.Flush();
- break;
- }
-
- /*
- * Respond to streamData with acks. Without that execution pipeline will stop
- * producing new tuples.
- */
- case NKqp::TKqpExecuterEvents::EvStreamData: {
- auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record;
-
- Cerr << (TStringBuilder() << "-- EvStreamData: " << record.AsJSON() << Endl);
+ break;
+ }
+
+ /*
+ * Respond to streamData with acks. Without that execution pipeline will stop
+ * producing new tuples.
+ */
+ case NKqp::TKqpExecuterEvents::EvStreamData: {
+ auto& record = ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record;
+
+ Cerr << (TStringBuilder() << "-- EvStreamData: " << record.AsJSON() << Endl);
Cerr.Flush();
-
- Y_ASSERT(record.GetResultSet().rows().size() == 1);
- Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1);
- result = record.GetResultSet().rows().at(0).items().at(0).uint64_value();
-
- auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>();
- resp->Record.SetEnough(false);
- resp->Record.SetSeqNo(ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record.GetSeqNo());
- resp->Record.SetFreeSpace(100);
-
- runtime.Send(new IEventHandle(ev->Sender, sender, resp.Release()));
-
- return TTestActorRuntime::EEventAction::DROP;
- }
-
- /* Drop message and kill tablet if we already had seen this tablet */
- case NKqp::TKqpComputeEvents::EvScanData: {
- if (!firstScanActor) {
- firstScanActor = ev->Sender;
-
- // send split
- ui32 splitKey = 55;
- auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
- request->Record.SetExecTimeoutPeriod(Max<ui64>());
- auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme();
+
+ Y_ASSERT(record.GetResultSet().rows().size() == 1);
+ Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1);
+ result = record.GetResultSet().rows().at(0).items().at(0).uint64_value();
+
+ auto resp = MakeHolder<NKqp::TEvKqpExecuter::TEvStreamDataAck>();
+ resp->Record.SetEnough(false);
+ resp->Record.SetSeqNo(ev->Get<NKqp::TEvKqpExecuter::TEvStreamData>()->Record.GetSeqNo());
+ resp->Record.SetFreeSpace(100);
+
+ runtime.Send(new IEventHandle(ev->Sender, sender, resp.Release()));
+
+ return TTestActorRuntime::EEventAction::DROP;
+ }
+
+ /* Drop message and kill tablet if we already had seen this tablet */
+ case NKqp::TKqpComputeEvents::EvScanData: {
+ if (!firstScanActor) {
+ firstScanActor = ev->Sender;
+
+ // send split
+ ui32 splitKey = 55;
+ auto request = MakeHolder<TEvTxUserProxy::TEvProposeTransaction>();
+ request->Record.SetExecTimeoutPeriod(Max<ui64>());
+ auto &tx = *request->Record.MutableTransaction()->MutableModifyScheme();
tx.SetOperationType(NKikimrSchemeOp::ESchemeOpSplitMergeTablePartitions);
- auto &desc = *tx.MutableSplitMergeTablePartitions();
- desc.SetTablePath("/Root/table-1");
- desc.AddSourceTabletId(tabletId);
- desc.AddSplitBoundary()->MutableKeyPrefix()->AddTuple()->MutableOptional()->SetUint32(splitKey);
-
- runtime.Send(new IEventHandle(MakeTxProxyID(), senderSplit, request.Release()));
-
- Cerr << (TStringBuilder() << "-- EvScanData from old tablet " << ev->Sender << ": pass and split" << Endl);
+ auto &desc = *tx.MutableSplitMergeTablePartitions();
+ desc.SetTablePath("/Root/table-1");
+ desc.AddSourceTabletId(tabletId);
+ desc.AddSplitBoundary()->MutableKeyPrefix()->AddTuple()->MutableOptional()->SetUint32(splitKey);
+
+ runtime.Send(new IEventHandle(MakeTxProxyID(), senderSplit, request.Release()));
+
+ Cerr << (TStringBuilder() << "-- EvScanData from old tablet " << ev->Sender << ": pass and split" << Endl);
Cerr.Flush();
- } else if (firstScanActor == ev->Sender) {
- // data from old table scan, drop it
- Cerr << (TStringBuilder() << "-- EvScanData from old tablet " << ev->Sender << ": drop" << Endl);
+ } else if (firstScanActor == ev->Sender) {
+ // data from old table scan, drop it
+ Cerr << (TStringBuilder() << "-- EvScanData from old tablet " << ev->Sender << ": drop" << Endl);
Cerr.Flush();
return TTestActorRuntime::EEventAction::DROP;
- } else {
- // data from new tablet scan, pass it
- Cerr << (TStringBuilder() << "-- EvScanData from new tablet" << ev->Sender << ": pass" << Endl);
+ } else {
+ // data from new tablet scan, pass it
+ Cerr << (TStringBuilder() << "-- EvScanData from new tablet" << ev->Sender << ": pass" << Endl);
Cerr.Flush();
- }
-
- break;
- }
-
- default:
- break;
- }
- return TTestActorRuntime::EEventAction::PROCESS;
- };
- runtime.SetObserverFunc(captureEvents);
-
- auto streamSender = runtime.AllocateEdgeActor();
- SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, "SELECT sum(value) FROM `/Root/table-1`;", false));
- auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender);
-
- UNIT_ASSERT_VALUES_EQUAL(result, 596400);
- }
-
+ }
+
+ break;
+ }
+
+ default:
+ break;
+ }
+ return TTestActorRuntime::EEventAction::PROCESS;
+ };
+ runtime.SetObserverFunc(captureEvents);
+
+ auto streamSender = runtime.AllocateEdgeActor();
+ SendRequest(runtime, streamSender, MakeStreamRequest(streamSender, "SELECT sum(value) FROM `/Root/table-1`;", false));
+ auto ev = runtime.GrabEdgeEventRethrow<NKqp::TEvKqp::TEvQueryResponse>(streamSender);
+
+ UNIT_ASSERT_VALUES_EQUAL(result, 596400);
+ }
+
Y_UNIT_TEST_WITH_MVCC(ScanRetryReadRanges) {
Y_UNUSED(EnableLogging);
@@ -514,8 +514,8 @@ Y_UNIT_TEST_SUITE(KqpScan) {
UNIT_ASSERT_VALUES_EQUAL(incomingRangesSize, 3);
}
-}
-
-} // namespace NKqp
-} // namespace NKikimr
-
+}
+
+} // namespace NKqp
+} // namespace NKikimr
+
diff --git a/ydb/core/tx/datashard/datashard_ut_order.cpp b/ydb/core/tx/datashard/datashard_ut_order.cpp
index 32472dce59e..b819d6e6074 100644
--- a/ydb/core/tx/datashard/datashard_ut_order.cpp
+++ b/ydb/core/tx/datashard/datashard_ut_order.cpp
@@ -3686,18 +3686,18 @@ void TestLateKqpQueryAfterColumnDrop(bool dataQuery, const TString& query, bool
// }
switch (ev->GetTypeRewrite()) {
case TEvDataShard::EvProposeTransaction: {
- auto &rec = ev->Get<TEvDataShard::TEvProposeTransaction>()->Record;
- if (capturePropose && rec.GetTxKind() != NKikimrTxDataShard::TX_KIND_SNAPSHOT) {
- Cerr << "---- capture EvProposeTransaction ---- type=" << rec.GetTxKind() << Endl;
- eventsPropose.emplace_back(ev.Release());
- return TTestActorRuntime::EEventAction::DROP;
- }
- break;
- }
-
- case TEvDataShard::EvKqpScan: {
+ auto &rec = ev->Get<TEvDataShard::TEvProposeTransaction>()->Record;
+ if (capturePropose && rec.GetTxKind() != NKikimrTxDataShard::TX_KIND_SNAPSHOT) {
+ Cerr << "---- capture EvProposeTransaction ---- type=" << rec.GetTxKind() << Endl;
+ eventsPropose.emplace_back(ev.Release());
+ return TTestActorRuntime::EEventAction::DROP;
+ }
+ break;
+ }
+
+ case TEvDataShard::EvKqpScan: {
if (capturePropose) {
- Cerr << "---- capture EvKqpScan ----" << Endl;
+ Cerr << "---- capture EvKqpScan ----" << Endl;
eventsPropose.emplace_back(ev.Release());
return TTestActorRuntime::EEventAction::DROP;
}
diff --git a/ydb/core/tx/datashard/ya.make b/ydb/core/tx/datashard/ya.make
index 9c98e8e79b0..b245e681ff8 100644
--- a/ydb/core/tx/datashard/ya.make
+++ b/ydb/core/tx/datashard/ya.make
@@ -64,7 +64,7 @@ SRCS(
datashard__s3.cpp
datashard__s3_download_txs.cpp
datashard__s3_upload_txs.cpp
- datashard__kqp_scan.cpp
+ datashard__kqp_scan.cpp
datashard__snapshot_txs.cpp
datashard__stats.cpp
datashard__store_table_path.cpp
diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp
index 82f6c5812ac..72403fe9756 100644
--- a/ydb/library/yql/ast/yql_expr.cpp
+++ b/ydb/library/yql/ast/yql_expr.cpp
@@ -1910,8 +1910,8 @@ namespace {
}
}
- template <class TPMap>
- bool GatherParentsImpl(const TExprNode& node, TPMap& parentsMap, TNodeSet& visited, bool withLeaves) {
+ template <class TPMap>
+ bool GatherParentsImpl(const TExprNode& node, TPMap& parentsMap, TNodeSet& visited, bool withLeaves) {
if (node.Type() == TExprNode::Arguments) {
return false;
}
@@ -1927,7 +1927,7 @@ namespace {
}
node.ForEachChild([&](const TExprNode& child) {
- if (GatherParentsImpl<TPMap>(child, parentsMap, visited, withLeaves)) {
+ if (GatherParentsImpl<TPMap>(child, parentsMap, visited, withLeaves)) {
parentsMap[&child].emplace(&node);
}
});
@@ -3180,15 +3180,15 @@ bool CompareExprTreeParts(const TExprNode& one, const TExprNode& two, const TNod
void GatherParents(const TExprNode& node, TParentsMap& parentsMap, bool withLeaves) {
parentsMap.clear();
TNodeSet visisted;
- GatherParentsImpl<TParentsMap>(node, parentsMap, visisted, withLeaves);
-}
-
-void GatherParentsMulti(const TExprNode& node, TParentsMultiMap& parentsMap, bool withLeaves) {
- parentsMap.clear();
- TNodeSet visisted;
- GatherParentsImpl<TParentsMultiMap>(node, parentsMap, visisted, withLeaves);
+ GatherParentsImpl<TParentsMap>(node, parentsMap, visisted, withLeaves);
}
+void GatherParentsMulti(const TExprNode& node, TParentsMultiMap& parentsMap, bool withLeaves) {
+ parentsMap.clear();
+ TNodeSet visisted;
+ GatherParentsImpl<TParentsMultiMap>(node, parentsMap, visisted, withLeaves);
+}
+
void CheckCounts(const TExprNode& root) {
TRefCountsMap refCounts;
CalculateReferences(root, refCounts);
diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h
index d6eb544f9af..9d65574e919 100644
--- a/ydb/library/yql/ast/yql_expr.h
+++ b/ydb/library/yql/ast/yql_expr.h
@@ -2074,9 +2074,9 @@ using TNodeSet = std::unordered_set<const TExprNode*>;
using TNodeOnNodeOwnedMap = TNodeMap<TExprNode::TPtr>;
using TParentsMap = TNodeMap<TNodeSet>;
-using TNodeMultiSet = std::unordered_multiset<const TExprNode*>;
-using TParentsMultiMap = TNodeMap<TNodeMultiSet>;
-
+using TNodeMultiSet = std::unordered_multiset<const TExprNode*>;
+using TParentsMultiMap = TNodeMap<TNodeMultiSet>;
+
template <>
struct TMakeTypeImpl<TVoidExprType> {
static const TVoidExprType* Make(TExprContext& ctx);
@@ -2549,7 +2549,7 @@ bool CompareExprTrees(const TExprNode*& one, const TExprNode*& two);
bool CompareExprTreeParts(const TExprNode& one, const TExprNode& two, const TNodeMap<ui32>& argsMap);
void GatherParents(const TExprNode& node, TParentsMap& parentsMap, bool withLeaves = false);
-void GatherParentsMulti(const TExprNode& node, TParentsMultiMap& parentsMap, bool withLeaves = false);
+void GatherParentsMulti(const TExprNode& node, TParentsMultiMap& parentsMap, bool withLeaves = false);
struct TConvertToAstSettings {
ui32 AnnotationFlags = 0;
diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
index edfcaa15302..aee1590d573 100644
--- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
+++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
@@ -198,11 +198,11 @@
"Match": {"Type": "Callable", "Name": "OrderedLMap"}
},
{
- "Name": "TCoNarrowMap",
- "Base": "TCoMapBase",
- "Match": {"Type": "Callable", "Name": "NarrowMap"}
- },
- {
+ "Name": "TCoNarrowMap",
+ "Base": "TCoMapBase",
+ "Match": {"Type": "Callable", "Name": "NarrowMap"}
+ },
+ {
"Name": "TCoNarrowFlatMap",
"Base": "TCoMapBase",
"Match": {"Type": "Callable", "Name": "NarrowFlatMap"}
@@ -213,11 +213,11 @@
"Match": {"Type": "Callable", "Name": "NarrowMultiMap"}
},
{
- "Name": "TCoWideMap",
- "Base": "TCoMapBase",
- "Match": {"Type": "Callable", "Name": "WideMap"}
- },
- {
+ "Name": "TCoWideMap",
+ "Base": "TCoMapBase",
+ "Match": {"Type": "Callable", "Name": "WideMap"}
+ },
+ {
"Name": "TCoFilterBase",
"Base": "TCoInputBase",
"Match": {"Type": "CallableBase"},
@@ -237,11 +237,11 @@
"Match": {"Type": "Callable", "Name": "OrderedFilter"}
},
{
- "Name": "TCoWideFilter",
- "Base": "TCoFilterBase",
- "Match": {"Type": "Callable", "Name": "WideFilter"}
- },
- {
+ "Name": "TCoWideFilter",
+ "Base": "TCoFilterBase",
+ "Match": {"Type": "Callable", "Name": "WideFilter"}
+ },
+ {
"Name": "TCoFilterNullMembersBase",
"Base": "TCoInputBase",
"Match": {"Type": "CallableBase"},
@@ -1365,19 +1365,19 @@
"Match": {"Type": "Callable", "Name": "EquiJoin"}
},
{
- "Name": "TCoMapJoinCore",
+ "Name": "TCoMapJoinCore",
"Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "MapJoinCore"},
- "Children": [
- {"Index": 0, "Name": "LeftInput", "Type": "TExprBase"},
- {"Index": 1, "Name": "RightDict", "Type": "TExprBase"},
- {"Index": 2, "Name": "JoinKind", "Type": "TCoAtom"},
- {"Index": 3, "Name": "LeftKeysColumns", "Type": "TCoAtomList"},
+ "Match": {"Type": "Callable", "Name": "MapJoinCore"},
+ "Children": [
+ {"Index": 0, "Name": "LeftInput", "Type": "TExprBase"},
+ {"Index": 1, "Name": "RightDict", "Type": "TExprBase"},
+ {"Index": 2, "Name": "JoinKind", "Type": "TCoAtom"},
+ {"Index": 3, "Name": "LeftKeysColumns", "Type": "TCoAtomList"},
{"Index": 4, "Name": "LeftRenames", "Type": "TCoAtomList"},
{"Index": 5, "Name": "RightRenames", "Type": "TCoAtomList"}
- ]
- },
- {
+ ]
+ },
+ {
"Name": "TCoJoinDict",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "JoinDict"},
@@ -1645,18 +1645,18 @@
]
},
{
- "Name": "TCoWideCombiner",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "WideCombiner"},
- "Children": [
- {"Index": 1, "Name": "MemLimit", "Type": "TCoAtom"},
- {"Index": 2, "Name": "KeyExtractor", "Type": "TCoLambda"},
- {"Index": 3, "Name": "InitHandler", "Type": "TCoLambda"},
- {"Index": 4, "Name": "UpdateHandler", "Type": "TCoLambda"},
- {"Index": 5, "Name": "FinishHandler", "Type": "TCoLambda"}
- ]
- },
- {
+ "Name": "TCoWideCombiner",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "WideCombiner"},
+ "Children": [
+ {"Index": 1, "Name": "MemLimit", "Type": "TCoAtom"},
+ {"Index": 2, "Name": "KeyExtractor", "Type": "TCoLambda"},
+ {"Index": 3, "Name": "InitHandler", "Type": "TCoLambda"},
+ {"Index": 4, "Name": "UpdateHandler", "Type": "TCoLambda"},
+ {"Index": 5, "Name": "FinishHandler", "Type": "TCoLambda"}
+ ]
+ },
+ {
"Name": "TCoGroupingCore",
"Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "GroupingCore"},
diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h
index deaadefb42c..4625b45f9f0 100644
--- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h
+++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h
@@ -743,14 +743,14 @@ protected:
protected:
// virtual methods (TODO: replace with static_cast<TDerived*>(this)->Foo()
-
+
virtual THolder<NKikimr::IDestructable> GetSourcesState() {
return nullptr;
- }
-
+ }
+
virtual void PollSources(THolder<NKikimr::IDestructable> /* state */) {
- }
-
+ }
+
virtual void TerminateSources(const TString& /* message */, bool /* success */) {
}
@@ -1358,7 +1358,7 @@ private:
if (TaskRunner) {
TaskRunner->UpdateStats();
-
+
if (auto* taskStats = TaskRunner->GetStats()) {
auto* protoTask = dst->AddTasks();
FillTaskRunnerStats(Task.GetId(), Task.GetStageId(), *taskStats, protoTask, (bool) ProfileStats);
diff --git a/ydb/library/yql/dq/opt/dq_opt.cpp b/ydb/library/yql/dq/opt/dq_opt.cpp
index f60f29dd460..1b05bca4dab 100644
--- a/ydb/library/yql/dq/opt/dq_opt.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt.cpp
@@ -123,13 +123,13 @@ const TNodeSet& GetConsumers(const TExprBase& node, const TParentsMap& parentsMa
return consumersIt->second;
}
-const TNodeMultiSet& GetConsumers(const TExprBase& node, const TParentsMultiMap& parentsMap) {
- auto consumersIt = parentsMap.find(node.Raw());
- YQL_ENSURE(consumersIt != parentsMap.end());
-
- return consumersIt->second;
-}
-
+const TNodeMultiSet& GetConsumers(const TExprBase& node, const TParentsMultiMap& parentsMap) {
+ auto consumersIt = parentsMap.find(node.Raw());
+ YQL_ENSURE(consumersIt != parentsMap.end());
+
+ return consumersIt->second;
+}
+
ui32 GetConsumersCount(const TExprBase& node, const TParentsMap& parentsMap) {
return GetConsumers(node, parentsMap).size();
}
diff --git a/ydb/library/yql/dq/opt/dq_opt.h b/ydb/library/yql/dq/opt/dq_opt.h
index 5b3146d0f42..8974d4b5db8 100644
--- a/ydb/library/yql/dq/opt/dq_opt.h
+++ b/ydb/library/yql/dq/opt/dq_opt.h
@@ -47,8 +47,8 @@ NNodes::TCoLambda BuildIdentityLambda(TPositionHandle pos, TExprContext& ctx);
bool EnsureDqUnion(const NNodes::TExprBase& node, TExprContext& ctx);
const TNodeSet& GetConsumers(const NNodes::TExprBase& node, const TParentsMap& parentsMap);
-const TNodeMultiSet& GetConsumers(const NNodes::TExprBase& node, const TParentsMultiMap& parentsMap);
-
+const TNodeMultiSet& GetConsumers(const NNodes::TExprBase& node, const TParentsMultiMap& parentsMap);
+
ui32 GetConsumersCount(const NNodes::TExprBase& node, const TParentsMap& parentsMap);
bool IsSingleConsumer(const NNodes::TExprBase& node, const TParentsMap& parentsMap);
diff --git a/ydb/library/yql/dq/opt/dq_opt_join.cpp b/ydb/library/yql/dq/opt/dq_opt_join.cpp
index 4097b49e24a..069530054f7 100644
--- a/ydb/library/yql/dq/opt/dq_opt_join.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_join.cpp
@@ -495,7 +495,7 @@ TExprBase DqBuildPhyJoin(const TDqJoin& join, bool pushLeftStage, TExprContext&
TMaybeNode<TDqCnBroadcast> rightBroadcast;
TNodeOnNodeOwnedMap rightPrecomputes;
-
+
if (rightCn) {
auto collectRightStage = Build<TDqStage>(ctx, join.Pos())
.Inputs()
@@ -575,7 +575,7 @@ TExprBase DqBuildPhyJoin(const TDqJoin& join, bool pushLeftStage, TExprContext&
TCoArgument leftInputArg{ctx.NewArgument(join.Pos(), "_dq_join_left")};
TCoArgument rightInputArg{ctx.NewArgument(join.Pos(), "_dq_join_right")};
-
+
bool buildNewStage = !pushLeftStage;
if (!rightCn) {
// right input is DqPure expression, try to push down the join...
@@ -594,7 +594,7 @@ TExprBase DqBuildPhyJoin(const TDqJoin& join, bool pushLeftStage, TExprContext&
buildNewStage = GetStageOutputsCount(leftCn.Output().Stage(), true) > 1;
}
}
-
+
TExprBase joinRightInput = buildNewStage
? (TExprBase) rightInputArg
: (rightBroadcast
diff --git a/ydb/library/yql/dq/opt/dq_opt_log.cpp b/ydb/library/yql/dq/opt/dq_opt_log.cpp
index fed0f09f0f0..839a1f90c32 100644
--- a/ydb/library/yql/dq/opt/dq_opt_log.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_log.cpp
@@ -84,69 +84,69 @@ TExprBase DqRewriteTakeSortToTopSort(TExprBase node, TExprContext& ctx, const TP
.Done();
}
-/*
- * Enforce PARTITION COMPACT BY as it avoids generating join in favour of Fold1Map.
- */
-TExprBase DqEnforceCompactPartition(TExprBase node, TExprList frames, TExprContext& ctx) {
-
- for (const auto &frameNode : frames.Ref().Children()) {
- YQL_ENSURE(frameNode->IsCallable("WinOnRows"));
-
- auto frameSpec = frameNode->Child(0);
- if (frameSpec->Type() == TExprNode::List) {
- TVector<TExprBase> values;
- bool compact = false;
-
- for (const auto& setting : frameSpec->Children()) {
- const auto settingName = setting->Head().Content();
- if (settingName == "compact") {
- compact = true;
- break;
- }
- values.push_back(TExprBase(setting));
- }
-
- if (!compact) {
- auto newFrameSpec = Build<TExprList>(ctx, frameNode->Pos())
- .Add(values)
- .Add<TExprList>()
- .Add<TCoAtom>()
- .Value("compact")
- .Build()
- .Build()
- .Done();
-
- TNodeOnNodeOwnedMap replaces;
- replaces[frameNode->Child(0)] = newFrameSpec.Ptr();
- node = TExprBase(ctx.ReplaceNodes(node.Ptr(), replaces));
- }
- }
- }
-
- return node;
+/*
+ * Enforce PARTITION COMPACT BY as it avoids generating join in favour of Fold1Map.
+ */
+TExprBase DqEnforceCompactPartition(TExprBase node, TExprList frames, TExprContext& ctx) {
+
+ for (const auto &frameNode : frames.Ref().Children()) {
+ YQL_ENSURE(frameNode->IsCallable("WinOnRows"));
+
+ auto frameSpec = frameNode->Child(0);
+ if (frameSpec->Type() == TExprNode::List) {
+ TVector<TExprBase> values;
+ bool compact = false;
+
+ for (const auto& setting : frameSpec->Children()) {
+ const auto settingName = setting->Head().Content();
+ if (settingName == "compact") {
+ compact = true;
+ break;
+ }
+ values.push_back(TExprBase(setting));
+ }
+
+ if (!compact) {
+ auto newFrameSpec = Build<TExprList>(ctx, frameNode->Pos())
+ .Add(values)
+ .Add<TExprList>()
+ .Add<TCoAtom>()
+ .Value("compact")
+ .Build()
+ .Build()
+ .Done();
+
+ TNodeOnNodeOwnedMap replaces;
+ replaces[frameNode->Child(0)] = newFrameSpec.Ptr();
+ node = TExprBase(ctx.ReplaceNodes(node.Ptr(), replaces));
+ }
+ }
+ }
+
+ return node;
}
-
-TExprBase DqExpandWindowFunctions(TExprBase node, TExprContext& ctx, bool enforceCompact) {
+
+TExprBase DqExpandWindowFunctions(TExprBase node, TExprContext& ctx, bool enforceCompact) {
if (node.Maybe<TCoCalcOverWindowBase>() || node.Maybe<TCoCalcOverWindowGroup>()) {
- if (enforceCompact) {
+ if (enforceCompact) {
auto calcs = ExtractCalcsOverWindow(node.Ptr(), ctx);
for (auto& c : calcs) {
TCoCalcOverWindowTuple win(c);
- node = DqEnforceCompactPartition(node, win.Frames(), ctx);
- }
- }
+ node = DqEnforceCompactPartition(node, win.Frames(), ctx);
+ }
+ }
return TExprBase(ExpandCalcOverWindow(node.Ptr(), ctx));
- } else {
- return node;
- }
-}
-
+ } else {
+ return node;
+ }
+}
+
static void CollectSinkStages(const NNodes::TDqQuery& dqQuery, THashSet<TExprNode::TPtr, TExprNode::TPtrHash>& sinkStages) {
for (const auto& stage : dqQuery.SinkStages()) {
sinkStages.insert(stage.Ptr());
}
-}
+}
NNodes::TExprBase DqMergeQueriesWithSinks(NNodes::TExprBase dqQueryNode, TExprContext& ctx) {
NNodes::TDqQuery dqQuery = dqQueryNode.Cast<NNodes::TDqQuery>();
diff --git a/ydb/library/yql/dq/opt/dq_opt_log.h b/ydb/library/yql/dq/opt/dq_opt_log.h
index 8e9ed97b063..b42b492dd9e 100644
--- a/ydb/library/yql/dq/opt/dq_opt_log.h
+++ b/ydb/library/yql/dq/opt/dq_opt_log.h
@@ -18,10 +18,10 @@ NNodes::TExprBase DqRewriteTakeSortToTopSort(NNodes::TExprBase node, TExprContex
NNodes::TExprBase DqRewriteEquiJoin(const NNodes::TExprBase& node, TExprContext& ctx);
-NNodes::TExprBase DqEnforceCompactPartition(NNodes::TExprBase node, NNodes::TExprList frames, TExprContext& ctx);
-
-NNodes::TExprBase DqExpandWindowFunctions(NNodes::TExprBase node, TExprContext& ctx, bool enforceCompact);
-
+NNodes::TExprBase DqEnforceCompactPartition(NNodes::TExprBase node, NNodes::TExprList frames, TExprContext& ctx);
+
+NNodes::TExprBase DqExpandWindowFunctions(NNodes::TExprBase node, TExprContext& ctx, bool enforceCompact);
+
NNodes::TExprBase DqMergeQueriesWithSinks(NNodes::TExprBase dqQueryNode, TExprContext& ctx);
NNodes::TExprBase DqFlatMapOverExtend(NNodes::TExprBase node, TExprContext& ctx);
diff --git a/ydb/library/yql/dq/opt/dq_opt_phy.cpp b/ydb/library/yql/dq/opt/dq_opt_phy.cpp
index cbe871059d7..0c15ea2dceb 100644
--- a/ydb/library/yql/dq/opt/dq_opt_phy.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_phy.cpp
@@ -1338,94 +1338,94 @@ TExprBase DqRewriteLengthOfStageOutput(TExprBase node, TExprContext& ctx, IOptim
.Done();
}
-TExprBase DqBuildPureExprStage(TExprBase node, TExprContext& ctx) {
- if (!IsDqPureExpr(node)) {
- return node;
- }
-
- auto stage = Build<TDqStage>(ctx, node.Pos())
- .Inputs()
- .Build()
- .Program()
- .Args({})
- .Body<TCoToStream>()
- .Input(node)
- .Build()
- .Build()
+TExprBase DqBuildPureExprStage(TExprBase node, TExprContext& ctx) {
+ if (!IsDqPureExpr(node)) {
+ return node;
+ }
+
+ auto stage = Build<TDqStage>(ctx, node.Pos())
+ .Inputs()
+ .Build()
+ .Program()
+ .Args({})
+ .Body<TCoToStream>()
+ .Input(node)
+ .Build()
+ .Build()
.Settings(TDqStageSettings().BuildNode(ctx, node.Pos()))
- .Done();
-
- return Build<TDqCnUnionAll>(ctx, node.Pos())
- .Output()
- .Stage(stage)
- .Index().Build("0")
- .Build()
- .Done();
-}
-
-/*
- * Move (Extend ...) into a separate stage.
- *
- * If overConnsOnly=true stage will be built only if all arguments have TDqConnection type.
- *
- * With overConnsOnly=false non-TDqConnection arguments are left in the same stage. This
- * is needed for handling UNION ALL case, which generates top-level Extend where some arguments
- * can be pure expressions not wrapped in DqStage (e.g. ... UNION ALL SELECT 1).
- */
+ .Done();
+
+ return Build<TDqCnUnionAll>(ctx, node.Pos())
+ .Output()
+ .Stage(stage)
+ .Index().Build("0")
+ .Build()
+ .Done();
+}
+
+/*
+ * Move (Extend ...) into a separate stage.
+ *
+ * If overConnsOnly=true stage will be built only if all arguments have TDqConnection type.
+ *
+ * With overConnsOnly=false non-TDqConnection arguments are left in the same stage. This
+ * is needed for handling UNION ALL case, which generates top-level Extend where some arguments
+ * can be pure expressions not wrapped in DqStage (e.g. ... UNION ALL SELECT 1).
+ */
TExprBase DqBuildExtendStage(TExprBase node, TExprContext& ctx) {
if (!node.Maybe<TCoExtendBase>()) {
- return node;
- }
-
+ return node;
+ }
+
auto extend = node.Cast<TCoExtendBase>();
- TVector<TCoArgument> inputArgs;
- TVector<TExprBase> inputConns;
- TVector<TExprBase> extendArgs;
-
- for (const auto& arg: extend) {
- if (arg.Maybe<TDqConnection>()) {
- auto conn = arg.Cast<TDqConnection>();
- TCoArgument programArg = Build<TCoArgument>(ctx, conn.Pos())
- .Name("arg")
- .Done();
- inputConns.push_back(conn);
- inputArgs.push_back(programArg);
- extendArgs.push_back(programArg);
- } else if (IsDqPureExpr(arg)) {
- // arg is deemed to be a pure expression so leave it inside (Extend ...)
- extendArgs.push_back(Build<TCoToFlow>(ctx, arg.Pos())
- .Input(arg)
- .Done());
- } else {
- return node;
- }
- }
-
- if (inputConns.empty()) {
- return node;
- }
-
- auto stage = Build<TDqStage>(ctx, node.Pos())
- .Inputs()
- .Add(inputConns)
- .Build()
- .Program()
- .Args(inputArgs)
- .Body<TCoExtend>() // TODO: check Extend effectiveness
- .Add(extendArgs)
- .Build()
- .Build()
+ TVector<TCoArgument> inputArgs;
+ TVector<TExprBase> inputConns;
+ TVector<TExprBase> extendArgs;
+
+ for (const auto& arg: extend) {
+ if (arg.Maybe<TDqConnection>()) {
+ auto conn = arg.Cast<TDqConnection>();
+ TCoArgument programArg = Build<TCoArgument>(ctx, conn.Pos())
+ .Name("arg")
+ .Done();
+ inputConns.push_back(conn);
+ inputArgs.push_back(programArg);
+ extendArgs.push_back(programArg);
+ } else if (IsDqPureExpr(arg)) {
+ // arg is deemed to be a pure expression so leave it inside (Extend ...)
+ extendArgs.push_back(Build<TCoToFlow>(ctx, arg.Pos())
+ .Input(arg)
+ .Done());
+ } else {
+ return node;
+ }
+ }
+
+ if (inputConns.empty()) {
+ return node;
+ }
+
+ auto stage = Build<TDqStage>(ctx, node.Pos())
+ .Inputs()
+ .Add(inputConns)
+ .Build()
+ .Program()
+ .Args(inputArgs)
+ .Body<TCoExtend>() // TODO: check Extend effectiveness
+ .Add(extendArgs)
+ .Build()
+ .Build()
.Settings(TDqStageSettings().BuildNode(ctx, node.Pos()))
- .Done();
-
- return Build<TDqCnUnionAll>(ctx, node.Pos())
- .Output()
- .Stage(stage)
- .Index().Build("0")
- .Build()
- .Done();
-}
-
+ .Done();
+
+ return Build<TDqCnUnionAll>(ctx, node.Pos())
+ .Output()
+ .Stage(stage)
+ .Index().Build("0")
+ .Build()
+ .Done();
+}
+
/*
* Precompute input value in a separate stage.
*/
diff --git a/ydb/library/yql/dq/opt/dq_opt_phy.h b/ydb/library/yql/dq/opt/dq_opt_phy.h
index 70a9d70082b..51809112b46 100644
--- a/ydb/library/yql/dq/opt/dq_opt_phy.h
+++ b/ydb/library/yql/dq/opt/dq_opt_phy.h
@@ -74,11 +74,11 @@ NNodes::TExprBase DqBuildJoinDict(const NNodes::TDqJoin& join, TExprContext& ctx
TMaybe<std::pair<NNodes::TExprBase, NNodes::TDqConnection>> ExtractPureExprStage(TExprNode::TPtr input,
TExprContext& ctx);
-
-NNodes::TExprBase DqBuildPureExprStage(NNodes::TExprBase node, TExprContext& ctx);
-
+
+NNodes::TExprBase DqBuildPureExprStage(NNodes::TExprBase node, TExprContext& ctx);
+
NNodes::TExprBase DqBuildExtendStage(NNodes::TExprBase node, TExprContext& ctx);
-
+
NNodes::TExprBase DqBuildPrecomputeStage(NNodes::TExprBase node, TExprContext& ctx);
NYql::NNodes::TExprBase DqBuildHasItems(NYql::NNodes::TExprBase node, NYql::TExprContext& ctx,
diff --git a/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp b/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp
index a1c8cf10c2c..24ed88afb8e 100644
--- a/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp
@@ -123,22 +123,22 @@ std::pair<TDqStage, TVector<TCoAtom>> ReplicateStageOutput(const TDqStage& stage
struct TMultiUsedOutput {
TDqOutput Output;
- const TNodeMultiSet& Consumers;
+ const TNodeMultiSet& Consumers;
- TMultiUsedOutput(const TDqOutput& output, const TNodeMultiSet& consumers)
+ TMultiUsedOutput(const TDqOutput& output, const TNodeMultiSet& consumers)
: Output(output)
, Consumers(consumers) {}
};
struct TMultiUsedConnection {
TDqConnection Connection;
- const TNodeMultiSet& Consumers;
+ const TNodeMultiSet& Consumers;
// set if Connection has single consumer, but it's Output has many consumers
// in that case, Connection - is on of connections, references to this Output
TMaybe<TMultiUsedOutput> Output;
- TMultiUsedConnection(const TDqConnection& connection, const TNodeMultiSet& consumers)
+ TMultiUsedConnection(const TDqConnection& connection, const TNodeMultiSet& consumers)
: Connection(connection)
, Consumers(consumers) {}
diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp
index 10900352c3b..28aadfd7420 100644
--- a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp
+++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp
@@ -333,10 +333,10 @@ public:
LOG(TStringBuilder() << "task: " << TaskId << ", program size: " << programSize
<< ", llvm: `" << Settings.OptLLVM << "`.");
- if (Y_UNLIKELY(CollectProfileStats)) {
- ProgramParsed.StatsRegistry = NMiniKQL::CreateDefaultStatsRegistry();
- }
-
+ if (Y_UNLIKELY(CollectProfileStats)) {
+ ProgramParsed.StatsRegistry = NMiniKQL::CreateDefaultStatsRegistry();
+ }
+
auto validatePolicy = Settings.TerminateOnError ? NUdf::EValidatePolicy::Fail : NUdf::EValidatePolicy::Exception;
TComputationPatternOpts opts(Alloc().Ref(), typeEnv, Context.ComputationFactory, Context.FuncRegistry,
NUdf::EValidateMode::None, validatePolicy, Settings.OptLLVM,
diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.h b/ydb/library/yql/dq/runtime/dq_tasks_runner.h
index a0cc816e91e..50cf0325f1b 100644
--- a/ydb/library/yql/dq/runtime/dq_tasks_runner.h
+++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.h
@@ -77,7 +77,7 @@ struct TDqTaskRunnerStats {
// profile stats
TDuration WaitTime; // wall time of waiting for input, scans & output
TDuration WaitOutputTime;
-
+
NMonitoring::IHistogramCollectorPtr ComputeCpuTimeByRun; // in millis
THashMap<ui64, const TDqInputChannelStats*> InputChannels; // Channel id -> Channel stats
diff --git a/ydb/library/yql/minikql/codegen/codegen.cpp b/ydb/library/yql/minikql/codegen/codegen.cpp
index 6e4096af228..f37c356605a 100644
--- a/ydb/library/yql/minikql/codegen/codegen.cpp
+++ b/ydb/library/yql/minikql/codegen/codegen.cpp
@@ -337,18 +337,18 @@ public:
}
void TogglePerfJITEventListener() override {
-#ifdef __linux__
- PerfListener_ = llvm::JITEventListener::createPerfJITEventListener();
- Engine_->RegisterJITEventListener(PerfListener_);
-#endif
- }
-
+#ifdef __linux__
+ PerfListener_ = llvm::JITEventListener::createPerfJITEventListener();
+ Engine_->RegisterJITEventListener(PerfListener_);
+#endif
+ }
+
~TCodegen() {
-#ifdef __linux__
- if (PerfListener_) {
- Engine_->UnregisterJITEventListener(PerfListener_);
- }
-#endif
+#ifdef __linux__
+ if (PerfListener_) {
+ Engine_->UnregisterJITEventListener(PerfListener_);
+ }
+#endif
Engine_->UnregisterJITEventListener(this);
}
@@ -404,9 +404,9 @@ public:
void Compile(const TStringBuf compileOpts, TCompileStats* compileStats) override {
- bool dumpTimers = compileOpts.Contains("time-passes");
- bool disableOpt = compileOpts.Contains("disable-opt");
-
+ bool dumpTimers = compileOpts.Contains("time-passes");
+ bool disableOpt = compileOpts.Contains("disable-opt");
+
#if defined(_msan_enabled_)
ReverseGlobalMapping_[(const void*)&__emutls_get_address] = "__emutls_get_address";
#endif
@@ -500,9 +500,9 @@ public:
if (compileStats) {
compileStats->ModulePassTime = (Now() - modulePassStart).MilliSeconds();
}
-
+
AllocateTls();
-
+
auto finalizeStart = Now();
Engine_->finalizeObject();
if (compileStats) {
@@ -729,7 +729,7 @@ private:
std::string Diagnostic_;
std::string Triple_;
llvm::Module* Module_;
- llvm::JITEventListener* PerfListener_ = nullptr;
+ llvm::JITEventListener* PerfListener_ = nullptr;
std::unique_ptr<llvm::ExecutionEngine> Engine_;
std::vector<std::pair<llvm::object::SectionRef, ui64>> CodeSections_;
std::vector<std::pair<ui64, llvm::Function*>> SortedFuncs_;
diff --git a/ydb/library/yql/minikql/codegen/codegen.h b/ydb/library/yql/minikql/codegen/codegen.h
index 41326b6e0fe..b1ce8c50b11 100644
--- a/ydb/library/yql/minikql/codegen/codegen.h
+++ b/ydb/library/yql/minikql/codegen/codegen.h
@@ -1,5 +1,5 @@
#pragma once
-
+
#include <memory>
#include <util/generic/fwd.h>
#include <util/generic/strbuf.h>
@@ -58,7 +58,7 @@ public:
virtual void ShowGeneratedFunctions(IOutputStream* out) = 0;
virtual void LoadBitCode(TStringBuf bitcode, TStringBuf uniqId) = 0;
virtual void AddGlobalMapping(TStringBuf name, const void* address) = 0;
- virtual void TogglePerfJITEventListener() = 0;
+ virtual void TogglePerfJITEventListener() = 0;
typedef std::unique_ptr<ICodegen> TPtr;
static TPtr Make(ETarget target, ESanitize sanitize = ESanitize::Auto);
diff --git a/ydb/library/yql/minikql/codegen/ya.make b/ydb/library/yql/minikql/codegen/ya.make
index cdc45b7d261..b371147d309 100644
--- a/ydb/library/yql/minikql/codegen/ya.make
+++ b/ydb/library/yql/minikql/codegen/ya.make
@@ -33,12 +33,12 @@ PEERDIR(
contrib/libs/llvm12/lib/Transforms/ObjCARC
)
-IF (OS_LINUX)
+IF (OS_LINUX)
PEERDIR(
contrib/libs/llvm12/lib/ExecutionEngine/PerfJITEvents
)
-ENDIF()
-
+ENDIF()
+
END()
RECURSE_FOR_TESTS(
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
index f2b188ae47c..8730f201c88 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
@@ -714,10 +714,10 @@ public:
Cerr << "MaxFunctionInstructions: " << codegenStats.MaxFunctionInstructions << Endl;
}
- if (opts.OptLLVM.Contains("--dump-perf-map")) {
- Codegen->TogglePerfJITEventListener();
- }
-
+ if (opts.OptLLVM.Contains("--dump-perf-map")) {
+ Codegen->TogglePerfJITEventListener();
+ }
+
if (codegenStats.TotalFunctions >= TotalFunctionsLimit ||
codegenStats.TotalInstructions >= TotalInstructionsLimit ||
codegenStats.MaxFunctionInstructions >= MaxFunctionInstructionsLimit) {
@@ -782,19 +782,19 @@ public:
TypeEnv = typeEnv;
}
- TStringBuf GetCompileOptions(const TString& s) {
- const TString flag = "--compile-options";
- auto lpos = s.rfind(flag);
- if (lpos == TString::npos)
- return TStringBuf();
- lpos += flag.Size();
- auto rpos = s.find(" --", lpos);
- if (rpos == TString::npos)
- return TStringBuf(s, lpos);
- else
- return TStringBuf(s, lpos, rpos - lpos);
- };
-
+ TStringBuf GetCompileOptions(const TString& s) {
+ const TString flag = "--compile-options";
+ auto lpos = s.rfind(flag);
+ if (lpos == TString::npos)
+ return TStringBuf();
+ lpos += flag.Size();
+ auto rpos = s.find(" --", lpos);
+ if (rpos == TString::npos)
+ return TStringBuf(s, lpos);
+ else
+ return TStringBuf(s, lpos, rpos - lpos);
+ };
+
THolder<IComputationGraph> Clone(const TComputationOptsFull& compOpts) final {
return MakeHolder<TComputationGraph>(PatternNodes, compOpts);
}
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h b/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h
index 0a63ca9d349..11e19a11f51 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h
@@ -567,7 +567,7 @@ private:
THROW yexception() << "Failed to get representation kind.";
}
- NUdf::TUnboxedValue GetValue(TComputationContext&) const {
+ NUdf::TUnboxedValue GetValue(TComputationContext&) const {
THROW yexception() << "Failed to get value from wide flow node.";
}
};
diff --git a/ydb/public/api/grpc/ydb_scripting_v1.proto b/ydb/public/api/grpc/ydb_scripting_v1.proto
index c8743a07de3..cbabd54ef5c 100644
--- a/ydb/public/api/grpc/ydb_scripting_v1.proto
+++ b/ydb/public/api/grpc/ydb_scripting_v1.proto
@@ -11,5 +11,5 @@ service ScriptingService {
// Executes yql request with streaming result.
rpc StreamExecuteYql(Scripting.ExecuteYqlRequest) returns (stream Scripting.ExecuteYqlPartialResponse);
- rpc ExplainYql(Scripting.ExplainYqlRequest) returns (Scripting.ExplainYqlResponse);
+ rpc ExplainYql(Scripting.ExplainYqlRequest) returns (Scripting.ExplainYqlResponse);
}
diff --git a/ydb/public/api/protos/ydb_experimental.proto b/ydb/public/api/protos/ydb_experimental.proto
index 94eed9671b7..62bee205434 100644
--- a/ydb/public/api/protos/ydb_experimental.proto
+++ b/ydb/public/api/protos/ydb_experimental.proto
@@ -22,7 +22,7 @@ message ExecuteStreamQueryRequest {
}
ProfileMode profile_mode = 3;
- bool explain = 4;
+ bool explain = 4;
}
message ExecuteStreamQueryResponse {
@@ -39,6 +39,6 @@ message ExecuteStreamQueryResult {
Ydb.ResultSet result_set = 1;
string profile = 2;
StreamQueryProgress progress = 3;
- string query_plan = 4;
+ string query_plan = 4;
}
}
diff --git a/ydb/public/api/protos/ydb_query_stats.proto b/ydb/public/api/protos/ydb_query_stats.proto
index 300d5d9837c..5a7a4c107e9 100644
--- a/ydb/public/api/protos/ydb_query_stats.proto
+++ b/ydb/public/api/protos/ydb_query_stats.proto
@@ -39,8 +39,8 @@ message QueryStats {
repeated QueryPhaseStats query_phases = 1;
CompilationStats compilation = 2;
uint64 process_cpu_time_us = 3;
- string query_plan = 4;
- string query_ast = 5;
+ string query_plan = 4;
+ string query_ast = 5;
uint64 total_duration_us = 6;
uint64 total_cpu_time_us = 7;
}
diff --git a/ydb/public/api/protos/ydb_scripting.proto b/ydb/public/api/protos/ydb_scripting.proto
index 4c1a2954185..c2197e79e7c 100644
--- a/ydb/public/api/protos/ydb_scripting.proto
+++ b/ydb/public/api/protos/ydb_scripting.proto
@@ -29,7 +29,7 @@ message ExecuteYqlResult {
repeated Ydb.ResultSet result_sets = 1;
Ydb.TableStats.QueryStats query_stats = 2;
}
-
+
// Response for StreamExecuteYql is a stream of ExecuteYqlPartialResponse messages.
// These responses can contain ExecuteYqlPartialResult messages with
// results (or result parts) for data or scan queries in the script.
@@ -52,24 +52,24 @@ message ExecuteYqlPartialResult {
Ydb.TableStats.QueryStats query_stats = 3;
}
-message ExplainYqlRequest {
- enum Mode {
+message ExplainYqlRequest {
+ enum Mode {
MODE_UNSPECIFIED = 0;
// PARSE = 1;
VALIDATE = 2;
PLAN = 3;
- }
-
- Ydb.Operations.OperationParams operation_params = 1;
- string script = 2;
- Mode mode = 3;
-}
-
-message ExplainYqlResponse {
- Ydb.Operations.Operation operation = 1;
-}
-
-message ExplainYqlResult {
- map<string, Type> parameters_types = 1;
- string plan = 2;
-}
+ }
+
+ Ydb.Operations.OperationParams operation_params = 1;
+ string script = 2;
+ Mode mode = 3;
+}
+
+message ExplainYqlResponse {
+ Ydb.Operations.Operation operation = 1;
+}
+
+message ExplainYqlResult {
+ map<string, Type> parameters_types = 1;
+ string plan = 2;
+}
diff --git a/ydb/public/api/protos/ydb_table.proto b/ydb/public/api/protos/ydb_table.proto
index c8063d9475c..bae30553cac 100644
--- a/ydb/public/api/protos/ydb_table.proto
+++ b/ydb/public/api/protos/ydb_table.proto
@@ -653,7 +653,7 @@ message QueryStatsCollection {
STATS_COLLECTION_UNSPECIFIED = 0;
STATS_COLLECTION_NONE = 1; // Stats collection is disabled
STATS_COLLECTION_BASIC = 2; // Aggregated stats of reads, updates and deletes per table
- STATS_COLLECTION_FULL = 3; // Add execution stats and plan on top of STATS_COLLECTION_BASIC
+ STATS_COLLECTION_FULL = 3; // Add execution stats and plan on top of STATS_COLLECTION_BASIC
}
}
@@ -968,7 +968,7 @@ message BulkUpsertResult {
message ExecuteScanQueryRequest {
enum Mode {
MODE_UNSPECIFIED = 0;
- MODE_EXPLAIN = 1;
+ MODE_EXPLAIN = 1;
// MODE_PREPARE = 2;
MODE_EXEC = 3;
}
@@ -980,7 +980,7 @@ message ExecuteScanQueryRequest {
reserved 5; // query_cache_policy
Mode mode = 6;
reserved 7; // report_progress
- QueryStatsCollection.Mode collect_stats = 8;
+ QueryStatsCollection.Mode collect_stats = 8;
}
message ExecuteScanQueryPartialResponse {
@@ -995,6 +995,6 @@ message ExecuteScanQueryPartialResult {
reserved 3; // query_meta
reserved 4; // query_progress
reserved 5; // query_plan
- Ydb.TableStats.QueryStats query_stats = 6;
+ Ydb.TableStats.QueryStats query_stats = 6;
}
diff --git a/ydb/public/lib/experimental/ydb_experimental.cpp b/ydb/public/lib/experimental/ydb_experimental.cpp
index 8765c0c750a..4241ae73ee8 100644
--- a/ydb/public/lib/experimental/ydb_experimental.cpp
+++ b/ydb/public/lib/experimental/ydb_experimental.cpp
@@ -59,16 +59,16 @@ public:
EStatus clientStatus = static_cast<EStatus>(self->Response_.status());
TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}};
TStatus status{std::move(plainStatus)};
-
+
if (self->Response_.result().has_result_set()) {
promise.SetValue({TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())),
std::move(status)});
} else if (!self->Response_.result().profile().empty()) {
promise.SetValue({std::move(*self->Response_.mutable_result()->mutable_profile()),
std::move(status)});
- } else if (!self->Response_.result().query_plan().Empty()) {
- TMaybe<TString> queryPlan = self->Response_.result().query_plan();
- promise.SetValue({queryPlan, std::move(status)});
+ } else if (!self->Response_.result().query_plan().Empty()) {
+ TMaybe<TString> queryPlan = self->Response_.result().query_plan();
+ promise.SetValue({queryPlan, std::move(status)});
} else if (self->Response_.result().has_progress()) {
// skip, not supported yet
} else {
@@ -127,8 +127,8 @@ public:
break;
}
- request.set_explain(settings.Explain_);
-
+ request.set_explain(settings.Explain_);
+
auto promise = NewPromise<std::pair<TPlainStatus, TStreamProcessorPtr>>();
Connections_->StartReadStream<
diff --git a/ydb/public/lib/experimental/ydb_experimental.h b/ydb/public/lib/experimental/ydb_experimental.h
index 3cdd8f6dc8f..4e166427203 100644
--- a/ydb/public/lib/experimental/ydb_experimental.h
+++ b/ydb/public/lib/experimental/ydb_experimental.h
@@ -27,10 +27,10 @@ public:
const TString& GetProfile() const { return *Profile_; }
TString ExtractProfile() { return std::move(*Profile_); }
- bool HasPlan() const { return Plan_.Defined(); }
- const TString& GetPlan() const { return *Plan_; }
- TString ExtractPlan() { return std::move(*Plan_); }
-
+ bool HasPlan() const { return Plan_.Defined(); }
+ const TString& GetPlan() const { return *Plan_; }
+ TString ExtractPlan() { return std::move(*Plan_); }
+
TStreamPart(TStatus&& status)
: TStreamPartStatus(std::move(status))
{}
@@ -45,15 +45,15 @@ public:
, Profile_(std::move(profile))
{}
- TStreamPart(const TMaybe<TString>& plan, TStatus&& status)
- : TStreamPartStatus(std::move(status))
- , Plan_(plan)
- {}
-
+ TStreamPart(const TMaybe<TString>& plan, TStatus&& status)
+ : TStreamPartStatus(std::move(status))
+ , Plan_(plan)
+ {}
+
private:
TMaybe<TResultSet> ResultSet_;
TMaybe<TString> Profile_;
- TMaybe<TString> Plan_;
+ TMaybe<TString> Plan_;
};
using TAsyncStreamPart = NThreading::TFuture<TStreamPart>;
@@ -81,7 +81,7 @@ struct TExecuteStreamQuerySettings : public TRequestSettings<TExecuteStreamQuery
using TSelf = TExecuteStreamQuerySettings;
FLUENT_SETTING_DEFAULT(EStreamQueryProfileMode, ProfileMode, EStreamQueryProfileMode::None);
- FLUENT_SETTING_DEFAULT(bool, Explain, false);
+ FLUENT_SETTING_DEFAULT(bool, Explain, false);
};
class TStreamQueryClient {
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp
index 2f94b2fc033..f1a510172f9 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp
+++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp
@@ -553,12 +553,12 @@ void TCommandExplain::Config(TConfig& config) {
.RequiredArgument("PATH").StoreResult(&QueryFile);
config.Opts->AddLongOption("ast", "Print query AST")
.StoreTrue(&PrintAst);
-
- config.Opts->AddLongOption('t', "type", "Query type [data, scan]")
- .RequiredArgument("[String]").DefaultValue("data").StoreResult(&QueryType);
- config.Opts->AddLongOption("analyze", "Run query and collect execution statistics")
- .NoArgument().SetFlag(&Analyze);
-
+
+ config.Opts->AddLongOption('t', "type", "Query type [data, scan]")
+ .RequiredArgument("[String]").DefaultValue("data").StoreResult(&QueryType);
+ config.Opts->AddLongOption("analyze", "Run query and collect execution statistics")
+ .NoArgument().SetFlag(&Analyze);
+
AddFormats(config, {
EOutputFormat::Pretty,
EOutputFormat::JsonUnicode,
@@ -577,77 +577,77 @@ void TCommandExplain::Parse(TConfig& config) {
int TCommandExplain::Run(TConfig& config) {
CheckQueryFile();
- TString planJson;
- TString ast;
- if (QueryType == "scan") {
- NTable::TTableClient client(CreateDriver(config));
- NTable::TStreamExecScanQuerySettings settings;
-
- if (Analyze) {
- settings.CollectQueryStats(NTable::ECollectQueryStatsMode::Full);
- } else {
- settings.Explain(true);
- }
-
- auto result = client.StreamExecuteScanQuery(Query, settings).GetValueSync();
- ThrowOnError(result);
-
- SetInterruptHandlers();
+ TString planJson;
+ TString ast;
+ if (QueryType == "scan") {
+ NTable::TTableClient client(CreateDriver(config));
+ NTable::TStreamExecScanQuerySettings settings;
+
+ if (Analyze) {
+ settings.CollectQueryStats(NTable::ECollectQueryStatsMode::Full);
+ } else {
+ settings.Explain(true);
+ }
+
+ auto result = client.StreamExecuteScanQuery(Query, settings).GetValueSync();
+ ThrowOnError(result);
+
+ SetInterruptHandlers();
while (!IsInterrupted()) {
- auto tablePart = result.ReadNext().GetValueSync();
- if (!tablePart.IsSuccess()) {
- if (tablePart.EOS()) {
- break;
- }
- ThrowOnError(tablePart);
- }
- if (tablePart.HasQueryStats() ) {
+ auto tablePart = result.ReadNext().GetValueSync();
+ if (!tablePart.IsSuccess()) {
+ if (tablePart.EOS()) {
+ break;
+ }
+ ThrowOnError(tablePart);
+ }
+ if (tablePart.HasQueryStats() ) {
auto proto = NYdb::TProtoAccessor::GetProto(tablePart.GetQueryStats());
- planJson = proto.query_plan();
- ast = proto.query_ast();
- }
- }
-
+ planJson = proto.query_plan();
+ ast = proto.query_ast();
+ }
+ }
+
if (IsInterrupted()) {
Cerr << "<INTERRUPTED>" << Endl;
}
- } else if (QueryType == "data" && Analyze) {
- NTable::TExecDataQuerySettings settings;
- settings.CollectQueryStats(NTable::ECollectQueryStatsMode::Full);
-
- auto result = GetSession(config).ExecuteDataQuery(
- Query,
- NTable::TTxControl::BeginTx(NTable::TTxSettings::SerializableRW()).CommitTx(),
- settings
- ).ExtractValueSync();
- ThrowOnError(result);
- planJson = result.GetQueryPlan();
+ } else if (QueryType == "data" && Analyze) {
+ NTable::TExecDataQuerySettings settings;
+ settings.CollectQueryStats(NTable::ECollectQueryStatsMode::Full);
+
+ auto result = GetSession(config).ExecuteDataQuery(
+ Query,
+ NTable::TTxControl::BeginTx(NTable::TTxSettings::SerializableRW()).CommitTx(),
+ settings
+ ).ExtractValueSync();
+ ThrowOnError(result);
+ planJson = result.GetQueryPlan();
if (auto stats = result.GetStats()) {
- auto proto = NYdb::TProtoAccessor::GetProto(*stats);
- ast = proto.query_ast();
- }
- } else if (QueryType == "data" && !Analyze) {
- NTable::TExplainQueryResult result = GetSession(config).ExplainDataQuery(
- Query,
- FillSettings(NTable::TExplainDataQuerySettings())
- ).GetValueSync();
- ThrowOnError(result);
- planJson = result.GetPlan();
- ast = result.GetAst();
-
- } else {
- throw TMissUseException() << "Unknown query type for explain.";
- }
-
+ auto proto = NYdb::TProtoAccessor::GetProto(*stats);
+ ast = proto.query_ast();
+ }
+ } else if (QueryType == "data" && !Analyze) {
+ NTable::TExplainQueryResult result = GetSession(config).ExplainDataQuery(
+ Query,
+ FillSettings(NTable::TExplainDataQuerySettings())
+ ).GetValueSync();
+ ThrowOnError(result);
+ planJson = result.GetPlan();
+ ast = result.GetAst();
+
+ } else {
+ throw TMissUseException() << "Unknown query type for explain.";
+ }
+
if (PrintAst) {
Cout << "Query AST:" << Endl << ast << Endl;
- } else {
+ } else {
Cout << "Query Plan:" << Endl;
TQueryPlanPrinter queryPlanPrinter(OutputFormat, Analyze);
queryPlanPrinter.Print(planJson);
- }
-
+ }
+
return EXIT_SUCCESS;
}
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h
index 5cd2514aa02..d706fb8f65a 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h
+++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h
@@ -123,10 +123,10 @@ public:
virtual int Run(TConfig& config) override;
private:
-
+
bool PrintAst = false;
- TString QueryType;
- bool Analyze = false;
+ TString QueryType;
+ bool Analyze = false;
};
class TCommandReadTable : public TYdbCommand, public TCommandWithPath,
diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp
index 38b8c8123dd..905d236a42c 100644
--- a/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp
+++ b/ydb/public/sdk/cpp/client/draft/ydb_scripting.cpp
@@ -40,8 +40,8 @@ const TMaybe<NTable::TQueryStats>& TExecuteYqlResult::GetStats() const {
return QueryStats_;
}
-////////////////////////////////////////////////////////////////////////////////
-
+////////////////////////////////////////////////////////////////////////////////
+
class TYqlResultPartIterator::TReaderImpl {
public:
using TSelf = TYqlResultPartIterator::TReaderImpl;
@@ -124,10 +124,10 @@ TAsyncYqlResultPart TYqlResultPartIterator::ReadNext() {
////////////////////////////////////////////////////////////////////////////////
TExplainYqlResult::TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map<TString, Ydb::Type>&& types, TString&& plan)
- : TStatus(std::move(status))
+ : TStatus(std::move(status))
, ParameterTypes_(std::move(types))
- , Plan_(plan) {}
-
+ , Plan_(plan) {}
+
std::map<TString, TType> TExplainYqlResult::GetParameterTypes() const {
std::map<TString, TType> typesMap;
for (const auto& param : ParameterTypes_) {
@@ -136,12 +136,12 @@ std::map<TString, TType> TExplainYqlResult::GetParameterTypes() const {
return typesMap;
}
-const TString& TExplainYqlResult::GetPlan() const {
- return Plan_;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
+const TString& TExplainYqlResult::GetPlan() const {
+ return Plan_;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
class TScriptingClient::TImpl : public TClientImplCommon<TScriptingClient::TImpl> {
public:
using TYqlScriptProcessorPtr = TYqlResultPartIterator::TReaderImpl::TStreamProcessorPtr;
@@ -245,57 +245,57 @@ public:
return promise.GetFuture();
}
- TAsyncExplainYqlResult ExplainYqlScript(const TString& script,
- const TExplainYqlRequestSettings& settings)
- {
- auto request = MakeOperationRequest<Ydb::Scripting::ExplainYqlRequest>(settings);
- request.set_script(script);
-
- switch (settings.Mode_) {
+ TAsyncExplainYqlResult ExplainYqlScript(const TString& script,
+ const TExplainYqlRequestSettings& settings)
+ {
+ auto request = MakeOperationRequest<Ydb::Scripting::ExplainYqlRequest>(settings);
+ request.set_script(script);
+
+ switch (settings.Mode_) {
// KIKIMR-10990
//case ExplainYqlRequestMode::Parse:
// request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_PARSE);
// break;
- case ExplainYqlRequestMode::Validate:
- request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_VALIDATE);
- break;
- case ExplainYqlRequestMode::Plan:
- request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_PLAN);
- break;
- }
-
- auto promise = NewPromise<TExplainYqlResult>();
-
+ case ExplainYqlRequestMode::Validate:
+ request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_VALIDATE);
+ break;
+ case ExplainYqlRequestMode::Plan:
+ request.set_mode(::Ydb::Scripting::ExplainYqlRequest_Mode::ExplainYqlRequest_Mode_PLAN);
+ break;
+ }
+
+ auto promise = NewPromise<TExplainYqlResult>();
+
auto extractor = [promise]
- (google::protobuf::Any* any, TPlainStatus status) mutable {
- TString plan;
+ (google::protobuf::Any* any, TPlainStatus status) mutable {
+ TString plan;
::google::protobuf::Map<TString, Ydb::Type> types;
- if (any) {
- Ydb::Scripting::ExplainYqlResult result;
- any->UnpackTo(&result);
-
- plan = result.plan();
+ if (any) {
+ Ydb::Scripting::ExplainYqlResult result;
+ any->UnpackTo(&result);
+
+ plan = result.plan();
types = result.parameters_types();
- }
-
+ }
+
TExplainYqlResult explainResult(TStatus(std::move(status)),
std::move(types), std::move(plan));
- promise.SetValue(std::move(explainResult));
- };
-
- Connections_->RunDeferred<Ydb::Scripting::V1::ScriptingService, Ydb::Scripting::ExplainYqlRequest,
- Ydb::Scripting::ExplainYqlResponse>(
- std::move(request),
- extractor,
- &Ydb::Scripting::V1::ScriptingService::Stub::AsyncExplainYql,
- DbDriverState_,
- INITIAL_DEFERRED_CALL_DELAY,
- TRpcRequestSettings::Make(settings),
- settings.ClientTimeout_);
-
- return promise.GetFuture();
- }
-
+ promise.SetValue(std::move(explainResult));
+ };
+
+ Connections_->RunDeferred<Ydb::Scripting::V1::ScriptingService, Ydb::Scripting::ExplainYqlRequest,
+ Ydb::Scripting::ExplainYqlResponse>(
+ std::move(request),
+ extractor,
+ &Ydb::Scripting::V1::ScriptingService::Stub::AsyncExplainYql,
+ DbDriverState_,
+ INITIAL_DEFERRED_CALL_DELAY,
+ TRpcRequestSettings::Make(settings),
+ settings.ClientTimeout_);
+
+ return promise.GetFuture();
+ }
+
private:
template<typename TRequest>
static void SetParams(::google::protobuf::Map<TString, Ydb::TypedValue>* params, TRequest* request) {
@@ -373,11 +373,11 @@ TAsyncYqlResultPartIterator TScriptingClient::StreamExecuteYqlScript(const TStri
return Impl_->StreamExecuteYqlScript(script, paramsPtr, settings);
}
-TAsyncExplainYqlResult TScriptingClient::ExplainYqlScript(const TString& script,
- const TExplainYqlRequestSettings& settings)
-{
- return Impl_->ExplainYqlScript(script, settings);
-}
-
+TAsyncExplainYqlResult TScriptingClient::ExplainYqlScript(const TString& script,
+ const TExplainYqlRequestSettings& settings)
+{
+ return Impl_->ExplainYqlScript(script, settings);
+}
+
} // namespace NScheme
} // namespace NYdb
diff --git a/ydb/public/sdk/cpp/client/draft/ydb_scripting.h b/ydb/public/sdk/cpp/client/draft/ydb_scripting.h
index e99f05daa24..d84290b8e20 100644
--- a/ydb/public/sdk/cpp/client/draft/ydb_scripting.h
+++ b/ydb/public/sdk/cpp/client/draft/ydb_scripting.h
@@ -82,21 +82,21 @@ private:
std::shared_ptr<TReaderImpl> ReaderImpl_;
};
-class TExplainYqlResult : public TStatus {
-public:
+class TExplainYqlResult : public TStatus {
+public:
TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map<TString, Ydb::Type>&& types, TString&& plan);
-
+
std::map<TString, TType> GetParameterTypes() const;
- const TString& GetPlan() const;
-
-private:
+ const TString& GetPlan() const;
+
+private:
::google::protobuf::Map<TString, Ydb::Type> ParameterTypes_;
- TString Plan_;
-};
-
+ TString Plan_;
+};
+
using TAsyncExecuteYqlResult = NThreading::TFuture<TExecuteYqlResult>;
using TAsyncYqlResultPartIterator = NThreading::TFuture<TYqlResultPartIterator>;
-using TAsyncExplainYqlResult = NThreading::TFuture<TExplainYqlResult>;
+using TAsyncExplainYqlResult = NThreading::TFuture<TExplainYqlResult>;
////////////////////////////////////////////////////////////////////////////////
@@ -104,18 +104,18 @@ struct TExecuteYqlRequestSettings : public TOperationRequestSettings<TExecuteYql
FLUENT_SETTING_DEFAULT(NTable::ECollectQueryStatsMode, CollectQueryStats, NTable::ECollectQueryStatsMode::None);
};
-enum class ExplainYqlRequestMode {
+enum class ExplainYqlRequestMode {
// Parse = 1,
- Validate = 2,
- Plan = 3,
-};
-
-struct TExplainYqlRequestSettings : public TOperationRequestSettings<TExplainYqlRequestSettings> {
+ Validate = 2,
+ Plan = 3,
+};
+
+struct TExplainYqlRequestSettings : public TOperationRequestSettings<TExplainYqlRequestSettings> {
FLUENT_SETTING_DEFAULT(ExplainYqlRequestMode, Mode, ExplainYqlRequestMode::Validate);
-};
-
-////////////////////////////////////////////////////////////////////////////////
-
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
class TScriptingClient {
class TImpl;
@@ -143,9 +143,9 @@ public:
TAsyncYqlResultPartIterator StreamExecuteYqlScript(const TString& script, TParams&& params,
const TExecuteYqlRequestSettings& settings = TExecuteYqlRequestSettings());
- TAsyncExplainYqlResult ExplainYqlScript(const TString& script,
- const TExplainYqlRequestSettings& settings = TExplainYqlRequestSettings());
-
+ TAsyncExplainYqlResult ExplainYqlScript(const TString& script,
+ const TExplainYqlRequestSettings& settings = TExplainYqlRequestSettings());
+
private:
std::shared_ptr<TImpl> Impl_;
};
diff --git a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h
index 355c271dc24..fcd371df126 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h
+++ b/ydb/public/sdk/cpp/client/ydb_table/query_stats/stats.h
@@ -31,9 +31,9 @@ class TYqlResultPartIterator;
namespace NTable {
enum class ECollectQueryStatsMode {
- None = 0, // Stats collection is disabled
- Basic = 1, // Aggregated stats of reads, updates and deletes per table
- Full = 2 // Add per-stage execution profile and query plan on top of Basic mode
+ None = 0, // Stats collection is disabled
+ Basic = 1, // Aggregated stats of reads, updates and deletes per table
+ Full = 2 // Add per-stage execution profile and query plan on top of Basic mode
};
class TQueryStats {
@@ -41,7 +41,7 @@ class TQueryStats {
friend class NYdb::TProtoAccessor;
friend class NYdb::NScripting::TScriptingClient;
friend class NYdb::NScripting::TYqlResultPartIterator;
- friend class TScanQueryPartIterator;
+ friend class TScanQueryPartIterator;
public:
TString ToString(bool withPlan = false) const;
diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp
index 610fec38847..f30e7402c09 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp
@@ -1274,17 +1274,17 @@ public:
// TODO: Add headers for streaming calls.
TPlainStatus plainStatus{clientStatus, std::move(issues), self->Endpoint_, {}};
TStatus status{std::move(plainStatus)};
- TMaybe<TQueryStats> queryStats;
-
- if (self->Response_.result().has_query_stats()) {
- queryStats = TQueryStats(self->Response_.result().query_stats());
- }
-
+ TMaybe<TQueryStats> queryStats;
+
+ if (self->Response_.result().has_query_stats()) {
+ queryStats = TQueryStats(self->Response_.result().query_stats());
+ }
+
if (self->Response_.result().has_result_set()) {
- promise.SetValue({std::move(status),
- TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), queryStats});
+ promise.SetValue({std::move(status),
+ TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), queryStats});
} else {
- promise.SetValue({std::move(status), queryStats});
+ promise.SetValue({std::move(status), queryStats});
}
}
};
@@ -2602,14 +2602,14 @@ public:
*request.mutable_parameters() = *params;
}
- if (settings.Explain_) {
- request.set_mode(Ydb::Table::ExecuteScanQueryRequest::MODE_EXPLAIN);
- } else {
- request.set_mode(Ydb::Table::ExecuteScanQueryRequest::MODE_EXEC);
- }
-
- request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_));
+ if (settings.Explain_) {
+ request.set_mode(Ydb::Table::ExecuteScanQueryRequest::MODE_EXPLAIN);
+ } else {
+ request.set_mode(Ydb::Table::ExecuteScanQueryRequest::MODE_EXEC);
+ }
+ request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_));
+
auto promise = NewPromise<std::pair<TPlainStatus, TScanQueryProcessorPtr>>();
Connections_->StartReadStream<
@@ -4192,14 +4192,14 @@ const TMaybe<TQueryStats>& TDataQueryResult::GetStats() const {
return QueryStats_;
}
-const TString TDataQueryResult::GetQueryPlan() const {
- if (QueryStats_.Defined()) {
- return NYdb::TProtoAccessor::GetProto(*QueryStats_.Get()).query_plan();
- } else {
- return "";
- }
-}
-
+const TString TDataQueryResult::GetQueryPlan() const {
+ if (QueryStats_.Defined()) {
+ return NYdb::TProtoAccessor::GetProto(*QueryStats_.Get()).query_plan();
+ } else {
+ return "";
+ }
+}
+
////////////////////////////////////////////////////////////////////////////////
TBeginTransactionResult::TBeginTransactionResult(TStatus&& status, TTransaction transaction)
diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h
index 42d21d2fec7..aead5f6014e 100644
--- a/ydb/public/sdk/cpp/client/ydb_table/table.h
+++ b/ydb/public/sdk/cpp/client/ydb_table/table.h
@@ -891,14 +891,14 @@ struct TBulkUpsertSettings : public TOperationRequestSettings<TBulkUpsertSetting
FLUENT_SETTING_DEFAULT(TString, FormatSettings, "");
};
-struct TStreamExecScanQuerySettings : public TRequestSettings<TStreamExecScanQuerySettings> {
- // Return query plan without actual query execution
- FLUENT_SETTING_DEFAULT(bool, Explain, false);
-
- // Collect runtime statistics with a given detalization mode
- FLUENT_SETTING_DEFAULT(ECollectQueryStatsMode, CollectQueryStats, ECollectQueryStatsMode::None);
-};
-
+struct TStreamExecScanQuerySettings : public TRequestSettings<TStreamExecScanQuerySettings> {
+ // Return query plan without actual query execution
+ FLUENT_SETTING_DEFAULT(bool, Explain, false);
+
+ // Collect runtime statistics with a given detalization mode
+ FLUENT_SETTING_DEFAULT(ECollectQueryStatsMode, CollectQueryStats, ECollectQueryStatsMode::None);
+};
+
class TSession;
struct TRetryState;
@@ -1728,8 +1728,8 @@ public:
const TMaybe<TQueryStats>& GetStats() const;
- const TString GetQueryPlan() const;
-
+ const TString GetQueryPlan() const;
+
private:
TMaybe<TTransaction> Transaction_;
TVector<TResultSet> ResultSets_;
@@ -1778,28 +1778,28 @@ public:
const TResultSet& GetResultSet() const { return *ResultSet_; }
TResultSet ExtractResultSet() { return std::move(*ResultSet_); }
- bool HasQueryStats() const { return QueryStats_.Defined(); }
- const TQueryStats& GetQueryStats() const { return *QueryStats_; }
- TQueryStats ExtractQueryStats() { return std::move(*QueryStats_); }
-
+ bool HasQueryStats() const { return QueryStats_.Defined(); }
+ const TQueryStats& GetQueryStats() const { return *QueryStats_; }
+ TQueryStats ExtractQueryStats() { return std::move(*QueryStats_); }
+
TScanQueryPart(TStatus&& status)
: TStreamPartStatus(std::move(status))
{}
- TScanQueryPart(TStatus&& status, const TMaybe<TQueryStats> &queryStats)
- : TStreamPartStatus(std::move(status))
- , QueryStats_(queryStats)
- {}
-
- TScanQueryPart(TStatus&& status, TResultSet&& resultSet, const TMaybe<TQueryStats> &queryStats)
+ TScanQueryPart(TStatus&& status, const TMaybe<TQueryStats> &queryStats)
: TStreamPartStatus(std::move(status))
+ , QueryStats_(queryStats)
+ {}
+
+ TScanQueryPart(TStatus&& status, TResultSet&& resultSet, const TMaybe<TQueryStats> &queryStats)
+ : TStreamPartStatus(std::move(status))
, ResultSet_(std::move(resultSet))
- , QueryStats_(queryStats)
+ , QueryStats_(queryStats)
{}
private:
TMaybe<TResultSet> ResultSet_;
- TMaybe<TQueryStats> QueryStats_;
+ TMaybe<TQueryStats> QueryStats_;
};
using TAsyncScanQueryPart = NThreading::TFuture<TScanQueryPart>;
diff --git a/ydb/public/sdk/python/ydb/scripting.py b/ydb/public/sdk/python/ydb/scripting.py
index 587630f3245..f4ea53b6f7a 100644
--- a/ydb/public/sdk/python/ydb/scripting.py
+++ b/ydb/public/sdk/python/ydb/scripting.py
@@ -1,6 +1,6 @@
from ydb.public.api.protos import ydb_scripting_pb2
from ydb.public.api.grpc import ydb_scripting_v1_pb2_grpc
-from . import issues, convert, settings
+from . import issues, convert, settings
class TypedParameters(object):
@@ -23,21 +23,21 @@ class ScriptingClientSettings(object):
return self
-class ExplainYqlScriptSettings(settings.BaseRequestSettings):
+class ExplainYqlScriptSettings(settings.BaseRequestSettings):
MODE_UNSPECIFIED = 0
MODE_PARSE = 1
MODE_VALIDATE = 2
MODE_EXPLAIN = 3
-
- def __init__(self):
- super(ExplainYqlScriptSettings, self).__init__()
- self.mode = False
-
- def with_mode(self, val):
- self.mode = val
- return self
-
-
+
+ def __init__(self):
+ super(ExplainYqlScriptSettings, self).__init__()
+ self.mode = False
+
+ def with_mode(self, val):
+ self.mode = val
+ return self
+
+
def _execute_yql_query_request_factory(script, tp=None, settings=None):
params = (
None
@@ -54,11 +54,11 @@ class YqlQueryResult(object):
)
-class YqlExplainResult(object):
- def __init__(self, result):
- self.plan = result.plan
-
-
+class YqlExplainResult(object):
+ def __init__(self, result):
+ self.plan = result.plan
+
+
def _wrap_response(rpc_state, response, scripting_client_settings):
issues._process_response(response.operation)
message = ydb_scripting_pb2.ExecuteYqlResult()
@@ -66,13 +66,13 @@ def _wrap_response(rpc_state, response, scripting_client_settings):
return YqlQueryResult(message)
-def _wrap_explain_response(rpc_state, response):
- issues._process_response(response.operation)
- message = ydb_scripting_pb2.ExplainYqlResult()
- response.operation.result.Unpack(message)
- return YqlExplainResult(message)
-
-
+def _wrap_explain_response(rpc_state, response):
+ issues._process_response(response.operation)
+ message = ydb_scripting_pb2.ExplainYqlResult()
+ response.operation.result.Unpack(message)
+ return YqlExplainResult(message)
+
+
class ScriptingClient(object):
def __init__(self, driver, scripting_client_settings=None):
self.driver = driver
@@ -92,12 +92,12 @@ class ScriptingClient(object):
settings=settings,
wrap_args=(self.scripting_client_settings,),
)
-
- def explain_yql(self, script, settings=None):
- return self.driver(
- ydb_scripting_pb2.ExplainYqlRequest(script=script, mode=settings.mode),
- ydb_scripting_v1_pb2_grpc.ScriptingServiceStub,
+
+ def explain_yql(self, script, settings=None):
+ return self.driver(
+ ydb_scripting_pb2.ExplainYqlRequest(script=script, mode=settings.mode),
+ ydb_scripting_v1_pb2_grpc.ScriptingServiceStub,
"ExplainYql",
- _wrap_explain_response,
+ _wrap_explain_response,
settings=settings,
- )
+ )
diff --git a/ydb/services/ydb/ydb_scripting.cpp b/ydb/services/ydb/ydb_scripting.cpp
index 3596b17fd94..7c14425da08 100644
--- a/ydb/services/ydb/ydb_scripting.cpp
+++ b/ydb/services/ydb/ydb_scripting.cpp
@@ -57,18 +57,18 @@ void TGRpcYdbScriptingService::SetupIncomingRequests(NGrpc::TLoggerPtr logger) {
new TGrpcRequestOperationCall<ExecuteYqlRequest, ExecuteYqlResponse>
(ctx, &DoExecuteYqlScript, TRequestAuxSettings{TRateLimiterMode::Ru, nullptr}));
})
-
+
ADD_REQUEST(StreamExecuteYql, ExecuteYqlRequest, ExecuteYqlPartialResponse, {
ActorSystem_->Send(GRpcRequestProxyId_,
new TGrpcRequestNoOperationCall<ExecuteYqlRequest, ExecuteYqlPartialResponse>
(ctx, &DoStreamExecuteYqlScript, TRequestAuxSettings{TRateLimiterMode::Rps, nullptr}));
})
- ADD_REQUEST(ExplainYql, ExplainYqlRequest, ExplainYqlResponse, {
+ ADD_REQUEST(ExplainYql, ExplainYqlRequest, ExplainYqlResponse, {
ActorSystem_->Send(GRpcRequestProxyId_,
new TGrpcRequestOperationCall<ExplainYqlRequest, ExplainYqlResponse>
(ctx, &DoExplainYqlScript, TRequestAuxSettings{TRateLimiterMode::Rps, nullptr}));
- })
+ })
#undef ADD_REQUEST
}
diff --git a/ydb/tests/functional/canonical/canondata/result.json b/ydb/tests/functional/canonical/canondata/result.json
index d927ff0f908..567332cdf7c 100644
--- a/ydb/tests/functional/canonical/canondata/result.json
+++ b/ydb/tests/functional/canonical/canondata/result.json
@@ -91,14 +91,14 @@
"uri": "file://test_sql.TestCanonicalFolder1.test_case_dynumber_select_params.sql-result_sets_/dynumber_select_params.sql_scan_query.results"
}
},
- "test_sql.TestCanonicalFolder1.test_case[explain.script-script]": {
- "script": {
+ "test_sql.TestCanonicalFolder1.test_case[explain.script-script]": {
+ "script": {
"uri": "file://test_sql.TestCanonicalFolder1.test_case_explain.script-script_/explain.script.results"
- },
- "script_plan": {
+ },
+ "script_plan": {
"uri": "file://test_sql.TestCanonicalFolder1.test_case_explain.script-script_/explain.script.plan"
- }
- },
+ }
+ },
"test_sql.TestCanonicalFolder1.test_case[index/select_using_index.sql-plan]": {
"plan": {
"uri": "file://test_sql.TestCanonicalFolder1.test_case_index_select_using_index.sql-plan_/index_select_using_index.sql.plan"
@@ -913,9 +913,9 @@
"script": {
"uri": "file://test_sql.TestCanonicalFolder1.test_case_simple_ct.script-script_/simple_ct.script.results"
},
- "script_plan": {
+ "script_plan": {
"uri": "file://test_sql.TestCanonicalFolder1.test_case_simple_ct.script-script_/simple_ct.script.plan"
- },
+ },
"table_data_Questions": {
"uri": "file://test_sql.TestCanonicalFolder1.test_case_simple_ct.script-script_/simple_ct.script_Questions.results"
}
@@ -1163,16 +1163,16 @@
"test_sql.TestCanonicalFolder1.test_case[simple/script_params.script-script]": {
"script": {
"uri": "file://test_sql.TestCanonicalFolder1.test_case_simple_script_params.script-script_/simple_script_params.script.results"
- },
- "script_plan": {
+ },
+ "script_plan": {
"uri": "file://test_sql.TestCanonicalFolder1.test_case_simple_script_params.script-script_/simple_script_params.script.plan"
}
},
"test_sql.TestCanonicalFolder1.test_case[table_types.script-script]": {
"script": {
"uri": "file://test_sql.TestCanonicalFolder1.test_case_table_types.script-script_/table_types.script.results"
- },
- "script_plan": {
+ },
+ "script_plan": {
"uri": "file://test_sql.TestCanonicalFolder1.test_case_table_types.script-script_/table_types.script.plan"
}
},
diff --git a/ydb/tests/functional/canonical/sql/explain.script b/ydb/tests/functional/canonical/sql/explain.script
index 74d5722adb6..7b349cae29b 100644
--- a/ydb/tests/functional/canonical/sql/explain.script
+++ b/ydb/tests/functional/canonical/sql/explain.script
@@ -1,48 +1,48 @@
---!syntax_v1
-PRAGMA Kikimr.UseNewEngine = "true";
-CREATE TABLE ScriptingTest (
- Key Uint64,
- Value String,
- PRIMARY KEY (Key)
-);
-COMMIT;
-
-PRAGMA Kikimr.UseNewEngine = "false";
-REPLACE INTO ScriptingTest (Key, Value) VALUES
- (1, "One"),
- (2, "Two");
-COMMIT;
-
-PRAGMA Kikimr.UseNewEngine = "true";
-REPLACE INTO ScriptingTest (Key, Value) VALUES
- (3, "Three"),
- (4, "Four");
-COMMIT;
-
-PRAGMA kikimr.ScanQuery = "true";
-PRAGMA Kikimr.UseNewEngine = "true";
-SELECT count(*) FROM ScriptingTest;
-COMMIT;
-
-PRAGMA kikimr.ScanQuery = "false";
-PRAGMA Kikimr.UseNewEngine = "true";
-SELECT count(*) FROM ScriptingTest;
-COMMIT;
-
-PRAGMA kikimr.ScanQuery = "true";
-PRAGMA Kikimr.UseNewEngine = "false";
-SELECT count(*) FROM ScriptingTest;
-COMMIT;
-
-PRAGMA kikimr.ScanQuery = "false";
-PRAGMA Kikimr.UseNewEngine = "false";
-SELECT count(*) FROM ScriptingTest;
-COMMIT;
-
-PRAGMA kikimr.ScanQuery = "true";
-SELECT 1*2*3*4*5;
-COMMIT;
-
-PRAGMA kikimr.ScanQuery = "false";
-SELECT 1*2*3*4*5;
-COMMIT; \ No newline at end of file
+--!syntax_v1
+PRAGMA Kikimr.UseNewEngine = "true";
+CREATE TABLE ScriptingTest (
+ Key Uint64,
+ Value String,
+ PRIMARY KEY (Key)
+);
+COMMIT;
+
+PRAGMA Kikimr.UseNewEngine = "false";
+REPLACE INTO ScriptingTest (Key, Value) VALUES
+ (1, "One"),
+ (2, "Two");
+COMMIT;
+
+PRAGMA Kikimr.UseNewEngine = "true";
+REPLACE INTO ScriptingTest (Key, Value) VALUES
+ (3, "Three"),
+ (4, "Four");
+COMMIT;
+
+PRAGMA kikimr.ScanQuery = "true";
+PRAGMA Kikimr.UseNewEngine = "true";
+SELECT count(*) FROM ScriptingTest;
+COMMIT;
+
+PRAGMA kikimr.ScanQuery = "false";
+PRAGMA Kikimr.UseNewEngine = "true";
+SELECT count(*) FROM ScriptingTest;
+COMMIT;
+
+PRAGMA kikimr.ScanQuery = "true";
+PRAGMA Kikimr.UseNewEngine = "false";
+SELECT count(*) FROM ScriptingTest;
+COMMIT;
+
+PRAGMA kikimr.ScanQuery = "false";
+PRAGMA Kikimr.UseNewEngine = "false";
+SELECT count(*) FROM ScriptingTest;
+COMMIT;
+
+PRAGMA kikimr.ScanQuery = "true";
+SELECT 1*2*3*4*5;
+COMMIT;
+
+PRAGMA kikimr.ScanQuery = "false";
+SELECT 1*2*3*4*5;
+COMMIT; \ No newline at end of file
diff --git a/ydb/tests/functional/canonical/test_sql.py b/ydb/tests/functional/canonical/test_sql.py
index af5cdc2b9c7..fcad5b399ac 100644
--- a/ydb/tests/functional/canonical/test_sql.py
+++ b/ydb/tests/functional/canonical/test_sql.py
@@ -425,12 +425,12 @@ class BaseCanonicalTest(object):
rp = script_client.execute_yql(query, params)
return self.wrap_result_sets(rp.result_sets)
- def script_explain(self, query):
- script_client = ydb.ScriptingClient(self.driver)
- settings = ydb.ExplainYqlScriptSettings().with_mode(ydb.ExplainYqlScriptSettings.MODE_EXPLAIN)
- rp = script_client.explain_yql(query, settings)
- return rp.plan
-
+ def script_explain(self, query):
+ script_client = ydb.ScriptingClient(self.driver)
+ settings = ydb.ExplainYqlScriptSettings().with_mode(ydb.ExplainYqlScriptSettings.MODE_EXPLAIN)
+ rp = script_client.explain_yql(query, settings)
+ return rp.plan
+
def compare_tables_test(self, canons, config, query_name):
for table_name in config.get('compare_tables', []):
canons['table_data_%s' % table_name] = self.canonical_results(
@@ -486,7 +486,7 @@ class BaseCanonicalTest(object):
pt = config.get('parameters_types', {})
result_sets = self.script_query(query, pv, pt)
canons['script'] = self.canonical_results(query_name, self.pretty_json(result_sets))
- canons['script_plan'] = self.canonical_plan(query_name, self.script_explain(query))
+ canons['script_plan'] = self.canonical_plan(query_name, self.script_explain(query))
self.compare_tables_test(canons, config, query_name)
elif kind == 'plan':
plan = self.explain(query)
diff --git a/ydb/tests/functional/suite_tests/canondata/result.json b/ydb/tests/functional/suite_tests/canondata/result.json
index ebfb6f81449..06e3464f701 100644
--- a/ydb/tests/functional/suite_tests/canondata/result.json
+++ b/ydb/tests/functional/suite_tests/canondata/result.json
@@ -19064,11 +19064,11 @@
"query_99": {
"uri": "file://test_sql_logic.TestSQLLogic.test_sql_suite_results-select3-9.test_/query_99"
}
- },
- "test_stream_query.TestStreamQuery.test_sql_suite[plan-window.test]": {},
- "test_stream_query.TestStreamQuery.test_sql_suite[results-window.test]": {
- "window.test.out": {
+ },
+ "test_stream_query.TestStreamQuery.test_sql_suite[plan-window.test]": {},
+ "test_stream_query.TestStreamQuery.test_sql_suite[results-window.test]": {
+ "window.test.out": {
"uri": "file://test_stream_query.TestStreamQuery.test_sql_suite_results-window.test_/window.test.out"
- }
+ }
}
}
diff --git a/ydb/tests/functional/suite_tests/stream_query/window.test b/ydb/tests/functional/suite_tests/stream_query/window.test
index 9704855ef81..4a21fc859a4 100644
--- a/ydb/tests/functional/suite_tests/stream_query/window.test
+++ b/ydb/tests/functional/suite_tests/stream_query/window.test
@@ -1,294 +1,294 @@
-statement ok
-CREATE TABLE empsalary (
- depname String,
- empno Int64,
- salary Int64,
- enroll_date Date,
- primary key(empno)
-);
-
-statement ok
-INSERT INTO empsalary (depname, empno, salary, enroll_date) VALUES
-('develop', 10, 5200, Date('2007-08-01')),
-('sales', 1, 5000, Date('2006-10-01')),
-('personnel', 5, 3500, Date('2007-12-10')),
-('sales', 4, 4800, Date('2007-08-08')),
-('personnel', 2, 3900, Date('2006-12-23')),
-('develop', 7, 4200, Date('2008-01-01')),
-('develop', 9, 4500, Date('2008-01-01')),
-('sales', 3, 4800, Date('2007-08-01')),
-('develop', 8, 6000, Date('2006-10-01')),
-('develop', 11, 5200, Date('2007-08-15'));
-
-statement ok
-CREATE TABLE tenk1 (
- unique1 Int64,
- unique2 Int64,
- two Int64,
- four Int64,
- ten Int64,
- twenty Int64,
- hundred Int64,
- thousand Int64,
- twothousand Int64,
- fivethous Int64,
- tenthous Int64,
- odd Int64,
- even Int64,
- stringu1 String,
- stringu2 String,
- string4 String,
- primary key (unique1, unique2)
-);
-
-statement ok
-CREATE TABLE int4_tbl(f1 Int32, primary key (f1));
-
-statement ok
-INSERT INTO int4_tbl(f1) VALUES (123456), (0);
-
-statement import table data
-insert into tenk1 (unique1, unique2, two, four, ten, twenty, hundred, thousand, twothousand, fivethous, tenthous, odd, even, stringu1, stringu2, string4) values postgres/data/tenk.data
-
-statement stream query
-SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname) ORDER BY depname, salary;
-
-statement stream query
-SELECT depname, empno, salary, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary);
-
--- with GROUP BY
-statement stream query
-SELECT four, ten, SUM(SUM(four)) OVER w, AVG(ten) FROM tenk1
-GROUP BY four, ten WINDOW w AS (PARTITION BY four) ORDER BY four, ten;
-
-statement stream query
-SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname);
-
-statement stream query
-SELECT depname, empno, salary, rank() OVER w AS r FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary) ORDER BY r;
-
--- empty window specification
-statement stream query
-SELECT COUNT(*) OVER w FROM tenk1 WHERE unique2 < 10 WINDOW w AS ();
-
--- no window operation
-statement stream query
-SELECT four FROM tenk1 WHERE FALSE WINDOW w AS (PARTITION BY ten);
-
--- cumulative aggregate
-statement stream query
-SELECT sum(four) OVER w AS sum_1, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY ten ORDER BY unique2);
-
-statement stream query
-SELECT row_number() OVER w FROM tenk1 WHERE unique2 < 10 WINDOW w AS (ORDER BY unique2);
-
-statement stream query
-SELECT rank() OVER w AS rank_1, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
-
-statement stream query
-SELECT dense_rank() OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
-
-statement error
-SELECT percent_rank() OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
-
-statement error
-SELECT cume_dist() OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
-
-statement error
-SELECT ntile(3) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (ORDER BY ten, four);
-
-statement error
-SELECT ntile(NULL) OVER w, ten, four FROM tenk1 WINDOW w AS (ORDER BY ten, four) LIMIT 2;
-
-statement stream query
-SELECT lag(ten) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
-
-statement error
-SELECT lag(ten, four) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
-
-statement error
-SELECT lag(ten, four, 0) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
-
-statement stream query
-SELECT lead(ten) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
-
-statement stream query
-SELECT lead(ten * 2, 1) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
-
-statement error
-SELECT lead(ten * 2, 1, -1) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
-
-statement stream query
-SELECT first_value(ten) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
-
--- last_value returns the last row of the frame, which is CURRENT ROW in ORDER BY window.
--- XXX: different behavior w.r.t postgres
-statement stream query
-SELECT last_value(four) OVER w, last_value(four) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (ORDER BY ten);
-
-statement stream query
-SELECT last_value(ten) OVER w, ten, four FROM
- (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten LIMIT 1000)
-WINDOW w AS (PARTITION BY four)
-ORDER BY four, ten;
-
-statement error
-SELECT nth_value(ten, four + 1) OVER w, ten, four FROM
- (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten)
-WINDOW w AS (PARTITION BY four);
-
-statement stream query
-SELECT ten, two, sum(hundred) AS gsum, sum(sum(hundred)) OVER w AS wsum
-FROM tenk1 GROUP BY ten, two WINDOW w AS (PARTITION BY two ORDER BY ten);
-
-statement stream query
-SELECT count(*) OVER w, four FROM (SELECT * FROM tenk1 WHERE two = 1) WHERE unique2 < 10 WINDOW w AS (PARTITION BY four);
-
--- WRONG!
-statement stream query
-SELECT (count(*) OVER w + sum(hundred) OVER w) AS cntsum
-FROM tenk1 WHERE unique2 < 10
-WINDOW w AS (PARTITION BY four ORDER BY ten);
-
--- opexpr with different windows evaluation.
-statement stream query
-SELECT * FROM(
- SELECT count(*) OVER w1 +
- sum(hundred) OVER w2 AS total,
- count(*) OVER w1 AS fourcount,
- sum(hundred) OVER w2 AS twosum
- FROM tenk1
- WINDOW w1 AS (PARTITION BY four ORDER BY ten), w2 AS (PARTITION BY two ORDER BY ten)
-)
-WHERE total <> fourcount + twosum;
-
-statement stream query
-SELECT avg(four) OVER w FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY thousand / 100);
-
-statement stream query
-SELECT ten, two, sum(hundred) AS gsum, sum(sum(hundred)) OVER win AS wsum
-FROM tenk1 GROUP BY ten, two WINDOW win AS (PARTITION BY two ORDER BY ten);
-
--- more than one window with GROUP BY
-statement stream query
-SELECT sum(salary),
- row_number() OVER w1,
- sum(sum(salary)) OVER w2
-FROM empsalary GROUP BY depname
-WINDOW w1 AS (ORDER BY depname), w2 AS (ORDER BY depname DESC);
-
--- identical windows with different names
-statement stream query
-SELECT sum(salary) OVER w1, count(*) OVER w2
-FROM empsalary WINDOW w1 AS (ORDER BY salary), w2 AS (ORDER BY salary);
-
--- subplan
-statement error
-SELECT lead(ten, (SELECT two FROM tenk1 WHERE s.unique2 = unique2)) OVER w
-FROM tenk1 s WHERE unique2 < 10
-WINDOW w AS (PARTITION BY four ORDER BY ten);
-
--- empty table
-statement stream query
-SELECT count(*) OVER w FROM (SELECT * FROM tenk1 WHERE FALSE) WINDOW w AS (PARTITION BY four);
-
--- mixture of agg/wfunc in the same window
-statement stream query
-SELECT sum(salary) OVER w, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary DESC);
-
--- strict aggs
-statement stream query
-SELECT empno, depname, salary, bonus, depadj, MIN(bonus) OVER w, MAX(depadj) OVER w FROM(
- SELECT empsalary.*,
- CASE WHEN enroll_date < DateTime::MakeDatetime(DateTime::Parse("%Y-%m-%d")('2007-08-01'))
- THEN 2008 - DateTime::GetYear(enroll_date)
- ELSE 42 END * 500 AS bonus,
- CASE WHEN AVG(salary) OVER wi < salary
- THEN 200
- ELSE 42 END AS depadj
- FROM empsalary
- WINDOW wi AS (PARTITION BY depname)
-)
-WINDOW w AS (ORDER BY empno);
-
--- window function over ungrouped agg over empty row set (bug before 9.1)
-statement stream query
-SELECT SUM(COUNT(f1)) OVER w FROM int4_tbl WHERE f1=42 WINDOW w AS ();
-
--- window function with ORDER BY an expression involving aggregates (9.1 bug)
-statement stream query
-select ten,
- sum(unique1) + sum(unique2) as res,
- rank() over w as rank
-from tenk1
-group by ten
-window w AS (order by sum(unique1) + sum(unique2))
-order by ten;
-
--- window and aggregate with GROUP BY expression (9.2 bug)
-statement stream query
-select first_value(max(x)) over w, y
- from (select unique1 as x, ten+four as y from tenk1)
- group by y
- window w AS ();
-
--- test non-default frame specifications
-statement stream query
-SELECT four, ten,
- sum(ten) over w,
- last_value(ten) over w
-FROM (select distinct ten, four from tenk1)
-WINDOW w AS (partition by four order by ten);
-
-statement error
-SELECT four, ten,
- sum(ten) over w,
- last_value(ten) over w
-FROM (select distinct ten, four from tenk1)
-WINDOW w AS (partition by four order by ten range between unbounded preceding and current row);
-
-statement error
-SELECT four, ten,
- sum(ten) over w,
- last_value(ten) over w
-FROM (select distinct ten, four from tenk1)
-WINDOW w AS (partition by four order by ten range between unbounded preceding and unbounded following);
-
-statement error
-SELECT four, ten/4 as two,
- sum(ten/4) over w,
- last_value(ten/4) over w
-FROM (select distinct ten, four from tenk1)
-WINDOW w AS (partition by four order by ten/4 range between unbounded preceding and current row);
-
-statement stream query
-SELECT four, ten/4 as two,
- sum(ten/4) over w,
- last_value(ten/4) over w
-FROM (select distinct ten, four from tenk1)
-WINDOW w AS (partition by four order by ten/4 rows between unbounded preceding and current row)
-
-statement error
-SELECT sum(unique1) over w,
- unique1, four
-FROM tenk1 WHERE unique1 < 10
-WINDOW w AS (order by four range between current row and unbounded following);
-
-statement stream query
-SELECT sum(unique1) over w,
- unique1, four
-FROM tenk1 WHERE unique1 < 10
-WINDOW w AS (rows between current row and unbounded following);
-
-statement stream query
-SELECT sum(unique1) over w,
- unique1, four
-FROM tenk1 WHERE unique1 < 10
-WINDOW w AS (rows between 2 preceding and 2 following);
-
-statement stream query
-SELECT sum(unique1) over w,
- unique1, four
-FROM tenk1 WHERE unique1 < 10
-WINDOW w AS (rows between 2 preceding and 2 following exclude no others);
-
+statement ok
+CREATE TABLE empsalary (
+ depname String,
+ empno Int64,
+ salary Int64,
+ enroll_date Date,
+ primary key(empno)
+);
+
+statement ok
+INSERT INTO empsalary (depname, empno, salary, enroll_date) VALUES
+('develop', 10, 5200, Date('2007-08-01')),
+('sales', 1, 5000, Date('2006-10-01')),
+('personnel', 5, 3500, Date('2007-12-10')),
+('sales', 4, 4800, Date('2007-08-08')),
+('personnel', 2, 3900, Date('2006-12-23')),
+('develop', 7, 4200, Date('2008-01-01')),
+('develop', 9, 4500, Date('2008-01-01')),
+('sales', 3, 4800, Date('2007-08-01')),
+('develop', 8, 6000, Date('2006-10-01')),
+('develop', 11, 5200, Date('2007-08-15'));
+
+statement ok
+CREATE TABLE tenk1 (
+ unique1 Int64,
+ unique2 Int64,
+ two Int64,
+ four Int64,
+ ten Int64,
+ twenty Int64,
+ hundred Int64,
+ thousand Int64,
+ twothousand Int64,
+ fivethous Int64,
+ tenthous Int64,
+ odd Int64,
+ even Int64,
+ stringu1 String,
+ stringu2 String,
+ string4 String,
+ primary key (unique1, unique2)
+);
+
+statement ok
+CREATE TABLE int4_tbl(f1 Int32, primary key (f1));
+
+statement ok
+INSERT INTO int4_tbl(f1) VALUES (123456), (0);
+
+statement import table data
+insert into tenk1 (unique1, unique2, two, four, ten, twenty, hundred, thousand, twothousand, fivethous, tenthous, odd, even, stringu1, stringu2, string4) values postgres/data/tenk.data
+
+statement stream query
+SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname) ORDER BY depname, salary;
+
+statement stream query
+SELECT depname, empno, salary, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary);
+
+-- with GROUP BY
+statement stream query
+SELECT four, ten, SUM(SUM(four)) OVER w, AVG(ten) FROM tenk1
+GROUP BY four, ten WINDOW w AS (PARTITION BY four) ORDER BY four, ten;
+
+statement stream query
+SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname);
+
+statement stream query
+SELECT depname, empno, salary, rank() OVER w AS r FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary) ORDER BY r;
+
+-- empty window specification
+statement stream query
+SELECT COUNT(*) OVER w FROM tenk1 WHERE unique2 < 10 WINDOW w AS ();
+
+-- no window operation
+statement stream query
+SELECT four FROM tenk1 WHERE FALSE WINDOW w AS (PARTITION BY ten);
+
+-- cumulative aggregate
+statement stream query
+SELECT sum(four) OVER w AS sum_1, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY ten ORDER BY unique2);
+
+statement stream query
+SELECT row_number() OVER w FROM tenk1 WHERE unique2 < 10 WINDOW w AS (ORDER BY unique2);
+
+statement stream query
+SELECT rank() OVER w AS rank_1, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
+
+statement stream query
+SELECT dense_rank() OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
+
+statement error
+SELECT percent_rank() OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
+
+statement error
+SELECT cume_dist() OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
+
+statement error
+SELECT ntile(3) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (ORDER BY ten, four);
+
+statement error
+SELECT ntile(NULL) OVER w, ten, four FROM tenk1 WINDOW w AS (ORDER BY ten, four) LIMIT 2;
+
+statement stream query
+SELECT lag(ten) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
+
+statement error
+SELECT lag(ten, four) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
+
+statement error
+SELECT lag(ten, four, 0) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
+
+statement stream query
+SELECT lead(ten) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
+
+statement stream query
+SELECT lead(ten * 2, 1) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
+
+statement error
+SELECT lead(ten * 2, 1, -1) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
+
+statement stream query
+SELECT first_value(ten) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY ten);
+
+-- last_value returns the last row of the frame, which is CURRENT ROW in ORDER BY window.
+-- XXX: different behavior w.r.t postgres
+statement stream query
+SELECT last_value(four) OVER w, last_value(four) OVER w, ten, four FROM tenk1 WHERE unique2 < 10 WINDOW w AS (ORDER BY ten);
+
+statement stream query
+SELECT last_value(ten) OVER w, ten, four FROM
+ (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten LIMIT 1000)
+WINDOW w AS (PARTITION BY four)
+ORDER BY four, ten;
+
+statement error
+SELECT nth_value(ten, four + 1) OVER w, ten, four FROM
+ (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten)
+WINDOW w AS (PARTITION BY four);
+
+statement stream query
+SELECT ten, two, sum(hundred) AS gsum, sum(sum(hundred)) OVER w AS wsum
+FROM tenk1 GROUP BY ten, two WINDOW w AS (PARTITION BY two ORDER BY ten);
+
+statement stream query
+SELECT count(*) OVER w, four FROM (SELECT * FROM tenk1 WHERE two = 1) WHERE unique2 < 10 WINDOW w AS (PARTITION BY four);
+
+-- WRONG!
+statement stream query
+SELECT (count(*) OVER w + sum(hundred) OVER w) AS cntsum
+FROM tenk1 WHERE unique2 < 10
+WINDOW w AS (PARTITION BY four ORDER BY ten);
+
+-- opexpr with different windows evaluation.
+statement stream query
+SELECT * FROM(
+ SELECT count(*) OVER w1 +
+ sum(hundred) OVER w2 AS total,
+ count(*) OVER w1 AS fourcount,
+ sum(hundred) OVER w2 AS twosum
+ FROM tenk1
+ WINDOW w1 AS (PARTITION BY four ORDER BY ten), w2 AS (PARTITION BY two ORDER BY ten)
+)
+WHERE total <> fourcount + twosum;
+
+statement stream query
+SELECT avg(four) OVER w FROM tenk1 WHERE unique2 < 10 WINDOW w AS (PARTITION BY four ORDER BY thousand / 100);
+
+statement stream query
+SELECT ten, two, sum(hundred) AS gsum, sum(sum(hundred)) OVER win AS wsum
+FROM tenk1 GROUP BY ten, two WINDOW win AS (PARTITION BY two ORDER BY ten);
+
+-- more than one window with GROUP BY
+statement stream query
+SELECT sum(salary),
+ row_number() OVER w1,
+ sum(sum(salary)) OVER w2
+FROM empsalary GROUP BY depname
+WINDOW w1 AS (ORDER BY depname), w2 AS (ORDER BY depname DESC);
+
+-- identical windows with different names
+statement stream query
+SELECT sum(salary) OVER w1, count(*) OVER w2
+FROM empsalary WINDOW w1 AS (ORDER BY salary), w2 AS (ORDER BY salary);
+
+-- subplan
+statement error
+SELECT lead(ten, (SELECT two FROM tenk1 WHERE s.unique2 = unique2)) OVER w
+FROM tenk1 s WHERE unique2 < 10
+WINDOW w AS (PARTITION BY four ORDER BY ten);
+
+-- empty table
+statement stream query
+SELECT count(*) OVER w FROM (SELECT * FROM tenk1 WHERE FALSE) WINDOW w AS (PARTITION BY four);
+
+-- mixture of agg/wfunc in the same window
+statement stream query
+SELECT sum(salary) OVER w, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary DESC);
+
+-- strict aggs
+statement stream query
+SELECT empno, depname, salary, bonus, depadj, MIN(bonus) OVER w, MAX(depadj) OVER w FROM(
+ SELECT empsalary.*,
+ CASE WHEN enroll_date < DateTime::MakeDatetime(DateTime::Parse("%Y-%m-%d")('2007-08-01'))
+ THEN 2008 - DateTime::GetYear(enroll_date)
+ ELSE 42 END * 500 AS bonus,
+ CASE WHEN AVG(salary) OVER wi < salary
+ THEN 200
+ ELSE 42 END AS depadj
+ FROM empsalary
+ WINDOW wi AS (PARTITION BY depname)
+)
+WINDOW w AS (ORDER BY empno);
+
+-- window function over ungrouped agg over empty row set (bug before 9.1)
+statement stream query
+SELECT SUM(COUNT(f1)) OVER w FROM int4_tbl WHERE f1=42 WINDOW w AS ();
+
+-- window function with ORDER BY an expression involving aggregates (9.1 bug)
+statement stream query
+select ten,
+ sum(unique1) + sum(unique2) as res,
+ rank() over w as rank
+from tenk1
+group by ten
+window w AS (order by sum(unique1) + sum(unique2))
+order by ten;
+
+-- window and aggregate with GROUP BY expression (9.2 bug)
+statement stream query
+select first_value(max(x)) over w, y
+ from (select unique1 as x, ten+four as y from tenk1)
+ group by y
+ window w AS ();
+
+-- test non-default frame specifications
+statement stream query
+SELECT four, ten,
+ sum(ten) over w,
+ last_value(ten) over w
+FROM (select distinct ten, four from tenk1)
+WINDOW w AS (partition by four order by ten);
+
+statement error
+SELECT four, ten,
+ sum(ten) over w,
+ last_value(ten) over w
+FROM (select distinct ten, four from tenk1)
+WINDOW w AS (partition by four order by ten range between unbounded preceding and current row);
+
+statement error
+SELECT four, ten,
+ sum(ten) over w,
+ last_value(ten) over w
+FROM (select distinct ten, four from tenk1)
+WINDOW w AS (partition by four order by ten range between unbounded preceding and unbounded following);
+
+statement error
+SELECT four, ten/4 as two,
+ sum(ten/4) over w,
+ last_value(ten/4) over w
+FROM (select distinct ten, four from tenk1)
+WINDOW w AS (partition by four order by ten/4 range between unbounded preceding and current row);
+
+statement stream query
+SELECT four, ten/4 as two,
+ sum(ten/4) over w,
+ last_value(ten/4) over w
+FROM (select distinct ten, four from tenk1)
+WINDOW w AS (partition by four order by ten/4 rows between unbounded preceding and current row)
+
+statement error
+SELECT sum(unique1) over w,
+ unique1, four
+FROM tenk1 WHERE unique1 < 10
+WINDOW w AS (order by four range between current row and unbounded following);
+
+statement stream query
+SELECT sum(unique1) over w,
+ unique1, four
+FROM tenk1 WHERE unique1 < 10
+WINDOW w AS (rows between current row and unbounded following);
+
+statement stream query
+SELECT sum(unique1) over w,
+ unique1, four
+FROM tenk1 WHERE unique1 < 10
+WINDOW w AS (rows between 2 preceding and 2 following);
+
+statement stream query
+SELECT sum(unique1) over w,
+ unique1, four
+FROM tenk1 WHERE unique1 < 10
+WINDOW w AS (rows between 2 preceding and 2 following exclude no others);
+
diff --git a/ydb/tests/functional/suite_tests/test_base.py b/ydb/tests/functional/suite_tests/test_base.py
index 0e6e9a35d99..cddbcae6c9f 100644
--- a/ydb/tests/functional/suite_tests/test_base.py
+++ b/ydb/tests/functional/suite_tests/test_base.py
@@ -45,7 +45,7 @@ class StatementTypes(enum.Enum):
Ok = 'statement ok'
Error = 'statement error'
Query = 'statement query'
- StreamQuery = 'statement stream query'
+ StreamQuery = 'statement stream query'
ImportTableData = 'statement import table data'
@@ -101,7 +101,7 @@ def get_single_statement(lines):
statement = "\n".join(statement_lines)
return statement
statement_lines.append(line)
- return "\n".join(statement_lines)
+ return "\n".join(statement_lines)
class ParsedStatement(collections.namedtuple('ParsedStatement', ["at_line", "s_type", "suite_name", "text"])):
@@ -206,36 +206,36 @@ def write_canonical_response(response, file):
)
-def format_as_table(data):
- nrows = len(data)
-
- if nrows == 0:
- return "(0 rows)\n"
-
- keys = data[0].keys()
- widths = [len(k) + 2 for k in keys]
- for row in data:
- for i, val in enumerate(row.values()):
- widths[i] = max(widths[i], len(str(val)) + 2)
-
- col_names = ['{: ^{w}}'.format(str(k), w=widths[i]) for i, k in enumerate(keys)]
-
- table = ""
- table += "|".join(col_names)
- table += "\n"
- table += "+".join(['-'*w for w in widths])
- table += "\n"
-
- for row in data:
- vals = ['{: >{w}}'.format(str(val) + ' ', w=widths[i]) for i, val in enumerate(row.values())]
- table += "|".join(vals)
- table += "\n"
-
- table += "(%d row%s)\n" % (nrows, 's' if nrows != 1 else 0)
-
- return table
-
-
+def format_as_table(data):
+ nrows = len(data)
+
+ if nrows == 0:
+ return "(0 rows)\n"
+
+ keys = data[0].keys()
+ widths = [len(k) + 2 for k in keys]
+ for row in data:
+ for i, val in enumerate(row.values()):
+ widths[i] = max(widths[i], len(str(val)) + 2)
+
+ col_names = ['{: ^{w}}'.format(str(k), w=widths[i]) for i, k in enumerate(keys)]
+
+ table = ""
+ table += "|".join(col_names)
+ table += "\n"
+ table += "+".join(['-'*w for w in widths])
+ table += "\n"
+
+ for row in data:
+ vals = ['{: >{w}}'.format(str(val) + ' ', w=widths[i]) for i, val in enumerate(row.values())]
+ table += "|".join(vals)
+ table += "\n"
+
+ table += "(%d row%s)\n" % (nrows, 's' if nrows != 1 else 0)
+
+ return table
+
+
@six.add_metaclass(abc.ABCMeta)
class BaseSuiteRunner(object):
check_new_engine_plan = True
@@ -317,7 +317,7 @@ class BaseSuiteRunner(object):
from_type = {
StatementTypes.Ok: self.assert_statement_ok,
StatementTypes.Query: self.assert_statement_query,
- StatementTypes.StreamQuery: self.assert_statement_stream_query,
+ StatementTypes.StreamQuery: self.assert_statement_stream_query,
StatementTypes.Error: (lambda x: x),
StatementTypes.ImportTableData: self.assert_statement_import_table_data,
StatementTypes.Skipped: lambda x: x
@@ -458,26 +458,26 @@ class BaseSuiteRunner(object):
if retries == 0:
raise
- def assert_statement_stream_query(self, statement):
- if self.plan:
- return
-
- yql_text = format_yql_statement(statement.text, self.table_path_prefix)
- yql_text = "--!syntax_v1\n" + yql_text + "\n\n"
+ def assert_statement_stream_query(self, statement):
+ if self.plan:
+ return
+
+ yql_text = format_yql_statement(statement.text, self.table_path_prefix)
+ yql_text = "--!syntax_v1\n" + yql_text + "\n\n"
result = self.execute_scan_query(yql_text)
- file_name = statement.suite_name.split('/')[1] + '.out'
- output_path = os.path.join(yatest_common.output_path(), file_name)
- with open(output_path, 'a+') as w:
- w.write(yql_text)
+ file_name = statement.suite_name.split('/')[1] + '.out'
+ output_path = os.path.join(yatest_common.output_path(), file_name)
+ with open(output_path, 'a+') as w:
+ w.write(yql_text)
w.write(format_as_table(result))
- w.write("\n\n")
-
+ w.write("\n\n")
+
self.files[file_name] = yatest_common.canonical_file(
path=output_path,
local=True,
universal_lines=True,
)
-
+
def is_probably_scheme(self, yql_text):
lwr = yql_text.lower()
return 'create table' in lwr or 'drop table' in lwr
diff --git a/ydb/tests/functional/suite_tests/test_stream_query.py b/ydb/tests/functional/suite_tests/test_stream_query.py
index dbc81c363a5..7e1a6f03cab 100644
--- a/ydb/tests/functional/suite_tests/test_stream_query.py
+++ b/ydb/tests/functional/suite_tests/test_stream_query.py
@@ -1,12 +1,12 @@
-# -*- coding: utf-8 -*-
-import pytest
+# -*- coding: utf-8 -*-
+import pytest
from test_base import BaseSuiteRunner, get_test_suites
-
-
-class TestStreamQuery(BaseSuiteRunner):
+
+
+class TestStreamQuery(BaseSuiteRunner):
def execute_assert(self, left, right, message):
assert left == right, message
- @pytest.mark.parametrize(['kind', 'suite'], get_test_suites("stream_query"))
- def test_sql_suite(self, kind, suite):
- return self.run_sql_suite(kind, "stream_query", suite)
+ @pytest.mark.parametrize(['kind', 'suite'], get_test_suites("stream_query"))
+ def test_sql_suite(self, kind, suite):
+ return self.run_sql_suite(kind, "stream_query", suite)