diff options
| author | ulya-sidorina <ulya-sidorina@yandex-team.ru> | 2022-02-10 16:52:15 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:52:15 +0300 |
| commit | 0a454693ecf632bec33e57986ffd2e24c42fe50c (patch) | |
| tree | ab7fbbf3253d4c0e2793218f09378908beb025fb | |
| parent | 321e44853055bb6bd4eb97694168bcce1322224c (diff) | |
| download | ydb-0a454693ecf632bec33e57986ffd2e24c42fe50c.tar.gz | |
Restoring authorship annotation for <ulya-sidorina@yandex-team.ru>. Commit 2 of 2.
135 files changed, 4591 insertions, 4591 deletions
diff --git a/ydb/core/engine/minikql/minikql_engine_host.cpp b/ydb/core/engine/minikql/minikql_engine_host.cpp index 5e6e94acfa5..36c35f32cc4 100644 --- a/ydb/core/engine/minikql/minikql_engine_host.cpp +++ b/ydb/core/engine/minikql/minikql_engine_host.cpp @@ -46,10 +46,10 @@ ui64 TEngineHost::GetShardId() const { return Settings.ShardId; } -const TScheme::TTableInfo* TEngineHost::GetTableInfo(const TTableId& tableId) const { - return Scheme.GetTableInfo(LocalTableId(tableId)); -} - +const TScheme::TTableInfo* TEngineHost::GetTableInfo(const TTableId& tableId) const { + return Scheme.GetTableInfo(LocalTableId(tableId)); +} + bool TEngineHost::IsReadonly() const { return Settings.IsReadonly; } diff --git a/ydb/core/engine/minikql/minikql_engine_host.h b/ydb/core/engine/minikql/minikql_engine_host.h index 3ec16f5dc45..012ee6891bc 100644 --- a/ydb/core/engine/minikql/minikql_engine_host.h +++ b/ydb/core/engine/minikql/minikql_engine_host.h @@ -96,7 +96,7 @@ public: explicit TEngineHost(NTable::TDatabase& db, TEngineHostCounters& counters, const TEngineHostSettings& settings = TEngineHostSettings()); ui64 GetShardId() const override; - const TScheme::TTableInfo* GetTableInfo(const TTableId& tableId) const override; + const TScheme::TTableInfo* GetTableInfo(const TTableId& tableId) const override; bool IsReadonly() const override; bool IsValidKey(TKeyDesc& key, std::pair<ui64, ui64>& maxSnapshotTime) const override; ui64 CalculateReadSize(const TVector<const TKeyDesc*>& keys) const override; diff --git a/ydb/core/engine/mkql_engine_flat_extfunc.cpp b/ydb/core/engine/mkql_engine_flat_extfunc.cpp index 7a3345ce22a..414d3aeeb45 100644 --- a/ydb/core/engine/mkql_engine_flat_extfunc.cpp +++ b/ydb/core/engine/mkql_engine_flat_extfunc.cpp @@ -745,45 +745,45 @@ namespace { LocateNode(ctx.NodeLocator, callable, 1)); } - IComputationNode* WrapUpdateRow(TCallable& callable, const TComputationNodeFactoryContext& ctx, IEngineFlatHost* host) { + IComputationNode* WrapUpdateRow(TCallable& callable, const TComputationNodeFactoryContext& ctx, IEngineFlatHost* host) { MKQL_ENSURE(callable.GetInputsCount() == 3, "Expected 3 arg"); auto tableNode = callable.GetInput(0); const auto tableId = ExtractTableId(tableNode); - - auto tableInfo = host->GetTableInfo(tableId); - MKQL_ENSURE(tableInfo, "Table not found: " << tableId.PathId.ToString()); - + + auto tableInfo = host->GetTableInfo(tableId); + MKQL_ENSURE(tableInfo, "Table not found: " << tableId.PathId.ToString()); + auto tupleType = AS_TYPE(TTupleType, callable.GetInput(1)); - for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) { - auto columnId = tableInfo->KeyColumns[i]; - const auto& column = tableInfo->Columns.at(columnId); - if (column.NotNull) { - MKQL_ENSURE(!tupleType->GetElementType(i)->IsOptional(), - "Not null column " << column.Name << " can't be optional"); - } - } - + for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) { + auto columnId = tableInfo->KeyColumns[i]; + const auto& column = tableInfo->Columns.at(columnId); + if (column.NotNull) { + MKQL_ENSURE(!tupleType->GetElementType(i)->IsOptional(), + "Not null column " << column.Name << " can't be optional"); + } + } + auto structUpdate = AS_VALUE(TStructLiteral, callable.GetInput(2)); - for (ui32 i = 0; i < structUpdate->GetValuesCount(); ++i) { - auto columnId = FromString<ui32>(structUpdate->GetType()->GetMemberName(i)); - const auto& column = tableInfo->Columns.at(columnId); - auto type = structUpdate->GetType()->GetMemberType(i); - - if (type->IsTuple()) { - auto itemType = AS_TYPE(TTupleType, type); - if (column.NotNull) { - MKQL_ENSURE(!itemType->GetElementType(1)->IsOptional(), - "Not null column " << column.Name << " can't be optional"); - } - } else if (!type->IsVoid()) { - if (column.NotNull) { - MKQL_ENSURE(!type->IsOptional(), - "Not null column " << column.Name << " can't be optional"); - } - } - } - + for (ui32 i = 0; i < structUpdate->GetValuesCount(); ++i) { + auto columnId = FromString<ui32>(structUpdate->GetType()->GetMemberName(i)); + const auto& column = tableInfo->Columns.at(columnId); + auto type = structUpdate->GetType()->GetMemberType(i); + + if (type->IsTuple()) { + auto itemType = AS_TYPE(TTupleType, type); + if (column.NotNull) { + MKQL_ENSURE(!itemType->GetElementType(1)->IsOptional(), + "Not null column " << column.Name << " can't be optional"); + } + } else if (!type->IsVoid()) { + if (column.NotNull) { + MKQL_ENSURE(!type->IsOptional(), + "Not null column " << column.Name << " can't be optional"); + } + } + } + return new TUpdateRowWrapper(ctx.Mutables, tableId, tupleType, LocateNode(ctx.NodeLocator, callable, 1), structUpdate, LocateNode(ctx.NodeLocator, callable, 2)); } @@ -874,7 +874,7 @@ TComputationNodeFactory GetFlatShardExecutionFactory(TShardExecData& execData, b } if (nameStr == strings.UpdateRow) { - return WrapUpdateRow(callable, ctx, settings.Host); + return WrapUpdateRow(callable, ctx, settings.Host); } if (nameStr == strings.AcquireLocks) { diff --git a/ydb/core/engine/mkql_engine_flat_host.h b/ydb/core/engine/mkql_engine_flat_host.h index b90f5c38957..0593fb3a9b8 100644 --- a/ydb/core/engine/mkql_engine_flat_host.h +++ b/ydb/core/engine/mkql_engine_flat_host.h @@ -23,9 +23,9 @@ public: // Returns shard id of the host. virtual ui64 GetShardId() const = 0; - // Returns table info - virtual const NTable::TScheme::TTableInfo* GetTableInfo(const TTableId& tableId) const = 0; - + // Returns table info + virtual const NTable::TScheme::TTableInfo* GetTableInfo(const TTableId& tableId) const = 0; + // Returns whether shard is read only. virtual bool IsReadonly() const = 0; diff --git a/ydb/core/grpc_services/rpc_load_rows.cpp b/ydb/core/grpc_services/rpc_load_rows.cpp index f0aa30f8ffc..127347e067e 100644 --- a/ydb/core/grpc_services/rpc_load_rows.cpp +++ b/ydb/core/grpc_services/rpc_load_rows.cpp @@ -282,11 +282,11 @@ private: if (!CellFromProtoVal(fd.Type, &proto.Getitems(fd.PositionInStruct), cells.back(), err, valueDataPool)) { return false; } - - if (fd.NotNull && cells.back().IsNull()) { - err = TStringBuilder() << "Received NULL value for not null column: " << fd.ColName; - return false; - } + + if (fd.NotNull && cells.back().IsNull()) { + err = TStringBuilder() << "Received NULL value for not null column: " << fd.ColName; + return false; + } } return true; diff --git a/ydb/core/keyvalue/keyvalue_flat_impl.h b/ydb/core/keyvalue/keyvalue_flat_impl.h index e7bbb5df89a..292f2feff93 100644 --- a/ydb/core/keyvalue/keyvalue_flat_impl.h +++ b/ydb/core/keyvalue/keyvalue_flat_impl.h @@ -58,9 +58,9 @@ protected: // Init the scheme auto &alter = txc.DB.Alter(); alter.AddTable("kvtable", TABLE_ID); - alter.AddColumn(TABLE_ID, "key", KEY_TAG, NScheme::TSmallBoundedString::TypeId, false); + alter.AddColumn(TABLE_ID, "key", KEY_TAG, NScheme::TSmallBoundedString::TypeId, false); alter.AddColumnToKey(TABLE_ID, KEY_TAG); - alter.AddColumn(TABLE_ID, "value", VALUE_TAG, NScheme::TString::TypeId, false); + alter.AddColumn(TABLE_ID, "value", VALUE_TAG, NScheme::TString::TypeId, false); // Init log batching settings alter.SetExecutorAllowLogBatching(true); alter.SetExecutorLogFlushPeriod(TDuration::MicroSeconds(500)); diff --git a/ydb/core/kqp/common/kqp_common.h b/ydb/core/kqp/common/kqp_common.h index f329bef640e..f9379ad1291 100644 --- a/ydb/core/kqp/common/kqp_common.h +++ b/ydb/core/kqp/common/kqp_common.h @@ -29,7 +29,7 @@ struct TKqpEvents { EvContinueShutdown, EvDataQueryStreamPart, EvDataQueryStreamPartAck, - EvRecompileRequest, + EvRecompileRequest, }; static_assert (EvCompileInvalidateRequest + 1 == EvAbortExecution); diff --git a/ydb/core/kqp/common/kqp_gateway.h b/ydb/core/kqp/common/kqp_gateway.h index e8bbb10dac7..1e06beca393 100644 --- a/ydb/core/kqp/common/kqp_gateway.h +++ b/ydb/core/kqp/common/kqp_gateway.h @@ -158,9 +158,9 @@ public: virtual NThreading::TFuture<TExecPhysicalResult> ExecuteScanQuery(TExecPhysicalRequest&& request, const NActors::TActorId& target) = 0; - virtual NThreading::TFuture<TExecPhysicalResult> ExecutePure(TExecPhysicalRequest&& request, - const NActors::TActorId& target) = 0; - + virtual NThreading::TFuture<TExecPhysicalResult> ExecutePure(TExecPhysicalRequest&& request, + const NActors::TActorId& target) = 0; + /* Scripting */ virtual NThreading::TFuture<TQueryResult> ExplainDataQueryAst(const TString& cluster, const TString& query) = 0; diff --git a/ydb/core/kqp/common/kqp_transform.h b/ydb/core/kqp/common/kqp_transform.h index 7692238708b..457c4248011 100644 --- a/ydb/core/kqp/common/kqp_transform.h +++ b/ydb/core/kqp/common/kqp_transform.h @@ -18,15 +18,15 @@ const TStringBuf LocksTxIdParamName = "%kqp%locks_txid"; const TStringBuf LocksListParamName = "%kqp%locks_list"; const TStringBuf ReadTargetParamName = "%kqp%read_target"; -/* Non-deterministic internal params */ -const std::string_view NowParamName = "%kqp%now"; -const std::string_view CurrentDateParamName = "%kqp%current_utc_date"; -const std::string_view CurrentDatetimeParamName = "%kqp%current_utc_datetime"; -const std::string_view CurrentTimestampParamName = "%kqp%current_utc_timestamp"; -const std::string_view RandomParamName = "%kqp%random"; -const std::string_view RandomNumberParamName = "%kqp%random_number"; -const std::string_view RandomUuidParamName = "%kqp%random_uuid"; - +/* Non-deterministic internal params */ +const std::string_view NowParamName = "%kqp%now"; +const std::string_view CurrentDateParamName = "%kqp%current_utc_date"; +const std::string_view CurrentDatetimeParamName = "%kqp%current_utc_datetime"; +const std::string_view CurrentTimestampParamName = "%kqp%current_utc_timestamp"; +const std::string_view RandomParamName = "%kqp%random"; +const std::string_view RandomNumberParamName = "%kqp%random_number"; +const std::string_view RandomUuidParamName = "%kqp%random_uuid"; + class TKqpTxLock { public: using TKey = std::tuple<ui64, ui64, ui64, ui64>; diff --git a/ydb/core/kqp/compile/kqp_compile.cpp b/ydb/core/kqp/compile/kqp_compile.cpp index 5cdd495e1ba..c7d305ab469 100644 --- a/ydb/core/kqp/compile/kqp_compile.cpp +++ b/ydb/core/kqp/compile/kqp_compile.cpp @@ -47,30 +47,30 @@ NKqpProto::TKqpPhyQuery::EType GetPhyQueryType(const EPhysicalQueryType& type) { YQL_ENSURE(false, "Unexpected physical query type: " << type); } -NKqpProto::TKqpPhyInternalBinding::EType GetPhyInternalBindingType(const std::string_view type) { - NKqpProto::TKqpPhyInternalBinding::EType bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_UNSPECIFIED; - - if (type == "Now"sv) { - bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_NOW; - } else if (type == "CurrentUtcDate"sv) { - bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_DATE; - } else if (type == "CurrentUtcDatetime"sv) { - bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_DATETIME; - } else if (type == "CurrentUtcTimestamp"sv) { - bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_TIMESTAMP; - } else if (type == "Random"sv) { - bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM; - } else if (type == "RandomNumber"sv) { - bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM_NUMBER; - } else if (type == "RandomUuid"sv) { - bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM_UUID; - } - - YQL_ENSURE(bindingType != NKqpProto::TKqpPhyInternalBinding::PARAM_UNSPECIFIED, - "Unexpected internal binding type: " << type); - return bindingType; -} - +NKqpProto::TKqpPhyInternalBinding::EType GetPhyInternalBindingType(const std::string_view type) { + NKqpProto::TKqpPhyInternalBinding::EType bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_UNSPECIFIED; + + if (type == "Now"sv) { + bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_NOW; + } else if (type == "CurrentUtcDate"sv) { + bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_DATE; + } else if (type == "CurrentUtcDatetime"sv) { + bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_DATETIME; + } else if (type == "CurrentUtcTimestamp"sv) { + bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_TIMESTAMP; + } else if (type == "Random"sv) { + bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM; + } else if (type == "RandomNumber"sv) { + bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM_NUMBER; + } else if (type == "RandomUuid"sv) { + bindingType = NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM_UUID; + } + + YQL_ENSURE(bindingType != NKqpProto::TKqpPhyInternalBinding::PARAM_UNSPECIFIED, + "Unexpected internal binding type: " << type); + return bindingType; +} + void FillTable(const TKqpTable& table, NKqpProto::TKqpPhyTable& tableProto) { auto pathId = TKikimrPathId::Parse(table.PathId()); @@ -608,10 +608,10 @@ private: auto& txResultProto = *bindingProto.MutableTxResultBinding(); txResultProto.SetTxIndex(txIndex); txResultProto.SetResultIndex(resultIndex); - } else if (auto maybeInternalBinding = binding.Maybe<TKqpTxInternalBinding>()) { - auto internalBinding = maybeInternalBinding.Cast(); - auto& internalBindingProto = *bindingProto.MutableInternalBinding(); - internalBindingProto.SetType(GetPhyInternalBindingType(internalBinding.Kind().Value())); + } else if (auto maybeInternalBinding = binding.Maybe<TKqpTxInternalBinding>()) { + auto internalBinding = maybeInternalBinding.Cast(); + auto& internalBindingProto = *bindingProto.MutableInternalBinding(); + internalBindingProto.SetType(GetPhyInternalBindingType(internalBinding.Kind().Value())); } else { YQL_ENSURE(false, "Unknown parameter binding type: " << binding.Cast().CallableName()); } diff --git a/ydb/core/kqp/compile/kqp_mkql_compiler.cpp b/ydb/core/kqp/compile/kqp_mkql_compiler.cpp index f3533b5b452..04130446986 100644 --- a/ydb/core/kqp/compile/kqp_mkql_compiler.cpp +++ b/ydb/core/kqp/compile/kqp_mkql_compiler.cpp @@ -22,13 +22,13 @@ TVector<TKqpTableColumn> GetKqpColumns(const TKikimrTableMetadata& table, const for (const auto& name : columnNames) { ui32 columnId = 0; ui32 columnType = 0; - bool notNull = false; + bool notNull = false; auto columnData = table.Columns.FindPtr(name); if (columnData) { columnId = columnData->Id; columnType = columnData->TypeId; - notNull = columnData->NotNull; + notNull = columnData->NotNull; } else if (allowSystemColumns) { auto systemColumn = GetSystemColumns().find(name); YQL_ENSURE(systemColumn != GetSystemColumns().end()); @@ -37,7 +37,7 @@ TVector<TKqpTableColumn> GetKqpColumns(const TKikimrTableMetadata& table, const } YQL_ENSURE(columnId, "Unknown column: " << name); - pgmColumns.emplace_back(columnId, name, columnType, notNull); + pgmColumns.emplace_back(columnId, name, columnType, notNull); } return pgmColumns; diff --git a/ydb/core/kqp/counters/kqp_counters.cpp b/ydb/core/kqp/counters/kqp_counters.cpp index a15b3ea0746..26a28a0249a 100644 --- a/ydb/core/kqp/counters/kqp_counters.cpp +++ b/ydb/core/kqp/counters/kqp_counters.cpp @@ -8,7 +8,7 @@ #include <library/cpp/actors/core/log.h> -#include <util/generic/size_literals.h> +#include <util/generic/size_literals.h> #include <ydb/library/yql/core/issue/protos/issue_id.pb.h> @@ -104,22 +104,22 @@ void TKqpCountersBase::Init() { KqpGroup->GetCounter("Request/QueryTypeAstScan", true); OtherQueryTypes = KqpGroup->GetCounter("Requests/QueryTypeOther", true); - QueriesWithRangeScan = KqpGroup->GetCounter("Query/WithRangeScan", true); - QueriesWithFullScan = KqpGroup->GetCounter("Query/WithFullScan", true); - - QueryAffectedShardsCount = KqpGroup->GetHistogram("Query/AffectedShards", - NMonitoring::ExplicitHistogram({1, 9, 49, 99, 499, 999})); - QueryReadSetsCount = KqpGroup->GetHistogram("Query/ReadSets", - NMonitoring::ExplicitHistogram({99, 999, 9999, 99999})); - QueryReadBytes = KqpGroup->GetHistogram("Query/ReadBytes", - NMonitoring::ExplicitHistogram({1_MB, 9_MB, 99_MB, 999_MB})); - QueryReadRows = KqpGroup->GetHistogram("Query/ReadRows", - NMonitoring::ExplicitHistogram({1, 99, 999, 9999, 99999, 999999})); - QueryMaxShardReplySize = KqpGroup->GetHistogram("Query/MaxShardReplySize", - NMonitoring::ExplicitHistogram({1_MB, 9_MB, 29_MB})); - QueryMaxShardProgramSize = KqpGroup->GetHistogram("Query/MaxShardProgramSize", - NMonitoring::ExplicitHistogram({1_MB, 9_MB, 29_MB})); - + QueriesWithRangeScan = KqpGroup->GetCounter("Query/WithRangeScan", true); + QueriesWithFullScan = KqpGroup->GetCounter("Query/WithFullScan", true); + + QueryAffectedShardsCount = KqpGroup->GetHistogram("Query/AffectedShards", + NMonitoring::ExplicitHistogram({1, 9, 49, 99, 499, 999})); + QueryReadSetsCount = KqpGroup->GetHistogram("Query/ReadSets", + NMonitoring::ExplicitHistogram({99, 999, 9999, 99999})); + QueryReadBytes = KqpGroup->GetHistogram("Query/ReadBytes", + NMonitoring::ExplicitHistogram({1_MB, 9_MB, 99_MB, 999_MB})); + QueryReadRows = KqpGroup->GetHistogram("Query/ReadRows", + NMonitoring::ExplicitHistogram({1, 99, 999, 9999, 99999, 999999})); + QueryMaxShardReplySize = KqpGroup->GetHistogram("Query/MaxShardReplySize", + NMonitoring::ExplicitHistogram({1_MB, 9_MB, 29_MB})); + QueryMaxShardProgramSize = KqpGroup->GetHistogram("Query/MaxShardProgramSize", + NMonitoring::ExplicitHistogram({1_MB, 9_MB, 29_MB})); + /* Request latency */ QueryLatencies[NKikimrKqp::QUERY_ACTION_EXECUTE] = KqpGroup->GetHistogram( "Query/ExecuteLatencyMs", NMonitoring::ExponentialHistogram(20, 2, 1)); @@ -210,7 +210,7 @@ void TKqpCountersBase::Init() { CompileRequestsInvalidate = KqpGroup->GetCounter("Compilation/Requests/Invalidate", true); CompileRequestsRejected = KqpGroup->GetCounter("Compilation/Requests/Rejected", true); CompileRequestsTimeout = KqpGroup->GetCounter("Compilation/Requests/Timeout", true); - CompileRequestsRecompile = KqpGroup->GetCounter("Compilation/Requests/Recompile", true); + CompileRequestsRecompile = KqpGroup->GetCounter("Compilation/Requests/Recompile", true); CompileCpuTime = KqpGroup->GetHistogram( "Compilation/CPUTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1)); @@ -289,38 +289,38 @@ void TKqpCountersBase::ReportQueryRequest(const NKikimrKqp::TQueryRequest& reque *YdbParametersBytes += parametersBytes; } -void TKqpCountersBase::ReportQueryWithRangeScan() { - QueriesWithRangeScan->Inc(); -} - -void TKqpCountersBase::ReportQueryWithFullScan() { - QueriesWithFullScan->Inc(); -} - -void TKqpCountersBase::ReportQueryAffectedShards(ui64 shardsCount) { - QueryAffectedShardsCount->Collect(shardsCount > Max<i64>() ? Max<i64>() : static_cast<i64>(shardsCount)); -} - -void TKqpCountersBase::ReportQueryReadSets(ui64 readSetsCount) { - QueryReadSetsCount->Collect(readSetsCount > Max<i64>() ? Max<i64>() : static_cast<i64>(readSetsCount)); -} - -void TKqpCountersBase::ReportQueryReadBytes(ui64 bytesCount) { - QueryReadBytes->Collect(bytesCount > Max<i64>() ? Max<i64>() : static_cast<i64>(bytesCount)); -} - -void TKqpCountersBase::ReportQueryReadRows(ui64 rowsCount) { - QueryReadRows->Collect(rowsCount > Max<i64>() ? Max<i64>() : static_cast<i64>(rowsCount)); -} - -void TKqpCountersBase::ReportQueryMaxShardReplySize(ui64 replySize) { - QueryMaxShardReplySize->Collect(replySize > Max<i64>() ? Max<i64>() : static_cast<i64>(replySize)); -} - -void TKqpCountersBase::ReportQueryMaxShardProgramSize(ui64 programSize) { - QueryMaxShardProgramSize->Collect(programSize > Max<i64>() ? Max<i64>() : static_cast<i64>(programSize)); -} - +void TKqpCountersBase::ReportQueryWithRangeScan() { + QueriesWithRangeScan->Inc(); +} + +void TKqpCountersBase::ReportQueryWithFullScan() { + QueriesWithFullScan->Inc(); +} + +void TKqpCountersBase::ReportQueryAffectedShards(ui64 shardsCount) { + QueryAffectedShardsCount->Collect(shardsCount > Max<i64>() ? Max<i64>() : static_cast<i64>(shardsCount)); +} + +void TKqpCountersBase::ReportQueryReadSets(ui64 readSetsCount) { + QueryReadSetsCount->Collect(readSetsCount > Max<i64>() ? Max<i64>() : static_cast<i64>(readSetsCount)); +} + +void TKqpCountersBase::ReportQueryReadBytes(ui64 bytesCount) { + QueryReadBytes->Collect(bytesCount > Max<i64>() ? Max<i64>() : static_cast<i64>(bytesCount)); +} + +void TKqpCountersBase::ReportQueryReadRows(ui64 rowsCount) { + QueryReadRows->Collect(rowsCount > Max<i64>() ? Max<i64>() : static_cast<i64>(rowsCount)); +} + +void TKqpCountersBase::ReportQueryMaxShardReplySize(ui64 replySize) { + QueryMaxShardReplySize->Collect(replySize > Max<i64>() ? Max<i64>() : static_cast<i64>(replySize)); +} + +void TKqpCountersBase::ReportQueryMaxShardProgramSize(ui64 programSize) { + QueryMaxShardProgramSize->Collect(programSize > Max<i64>() ? Max<i64>() : static_cast<i64>(programSize)); +} + void TKqpCountersBase::ReportResponseStatus(ui64 responseSize, Ydb::StatusIds::StatusCode ydbStatus) { *ResponseBytes += responseSize; *YdbResponseBytes += responseSize; @@ -525,9 +525,9 @@ void TKqpCountersBase::ReportCompileDurations(TDuration duration, TDuration cpuT CompileCpuTime->Collect(cpuTime.MilliSeconds()); } -void TKqpCountersBase::ReportRecompileRequestGet() { - CompileRequestsRecompile->Inc(); -} +void TKqpCountersBase::ReportRecompileRequestGet() { + CompileRequestsRecompile->Inc(); +} TKqpDbCounters::TKqpDbCounters() { @@ -820,61 +820,61 @@ void TKqpCounters::ReportQueryRequest(TKqpDbCountersPtr dbCounters, const NKikim } void TKqpCounters::ReportQueryWithRangeScan(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportQueryWithRangeScan(); + TKqpCountersBase::ReportQueryWithRangeScan(); if (dbCounters) { dbCounters->ReportQueryWithRangeScan(); - } -} - + } +} + void TKqpCounters::ReportQueryWithFullScan(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportQueryWithFullScan(); + TKqpCountersBase::ReportQueryWithFullScan(); if (dbCounters) { dbCounters->ReportQueryWithFullScan(); - } -} - + } +} + void TKqpCounters::ReportQueryAffectedShards(TKqpDbCountersPtr dbCounters, ui64 shardsCount) { - TKqpCountersBase::ReportQueryAffectedShards(shardsCount); + TKqpCountersBase::ReportQueryAffectedShards(shardsCount); if (dbCounters) { dbCounters->ReportQueryAffectedShards(shardsCount); - } -} - + } +} + void TKqpCounters::ReportQueryReadSets(TKqpDbCountersPtr dbCounters, ui64 readSetsCount) { - TKqpCountersBase::ReportQueryReadSets(readSetsCount); + TKqpCountersBase::ReportQueryReadSets(readSetsCount); if (dbCounters) { dbCounters->ReportQueryReadSets(readSetsCount); - } -} - + } +} + void TKqpCounters::ReportQueryReadBytes(TKqpDbCountersPtr dbCounters, ui64 bytesCount) { - TKqpCountersBase::ReportQueryReadBytes(bytesCount); + TKqpCountersBase::ReportQueryReadBytes(bytesCount); if (dbCounters) { dbCounters->ReportQueryReadBytes(bytesCount); - } -} - + } +} + void TKqpCounters::ReportQueryReadRows(TKqpDbCountersPtr dbCounters, ui64 rowsCount) { - TKqpCountersBase::ReportQueryReadRows(rowsCount); + TKqpCountersBase::ReportQueryReadRows(rowsCount); if (dbCounters) { dbCounters->ReportQueryReadRows(rowsCount); - } -} - + } +} + void TKqpCounters::ReportQueryMaxShardReplySize(TKqpDbCountersPtr dbCounters, ui64 replySize) { - TKqpCountersBase::ReportQueryMaxShardReplySize(replySize); + TKqpCountersBase::ReportQueryMaxShardReplySize(replySize); if (dbCounters) { dbCounters->ReportQueryMaxShardReplySize(replySize); - } -} - + } +} + void TKqpCounters::ReportQueryMaxShardProgramSize(TKqpDbCountersPtr dbCounters, ui64 programSize) { - TKqpCountersBase::ReportQueryMaxShardProgramSize(programSize); + TKqpCountersBase::ReportQueryMaxShardProgramSize(programSize); if (dbCounters) { dbCounters->ReportQueryMaxShardProgramSize(programSize); - } -} - + } +} + void TKqpCounters::ReportResponseStatus(TKqpDbCountersPtr dbCounters, ui64 responseSize, Ydb::StatusIds::StatusCode ydbStatus) { @@ -1067,12 +1067,12 @@ void TKqpCounters::ReportCompileDurations(TKqpDbCountersPtr dbCounters, TDuratio } void TKqpCounters::ReportRecompileRequestGet(TKqpDbCountersPtr dbCounters) { - TKqpCountersBase::ReportRecompileRequestGet(); + TKqpCountersBase::ReportRecompileRequestGet(); if (dbCounters) { dbCounters->ReportRecompileRequestGet(); - } -} - + } +} + void TKqpCounters::ReportNewEngineForcedQueryStats(NKikimrKqp::EQueryAction action, TDuration duration, ui64 computeCpuTime) { @@ -1095,10 +1095,10 @@ void TKqpCounters::ReportNewEngineCompatibleQueryStats(NKikimrKqp::EQueryAction } } -const NMonitoring::TDynamicCounters::TCounterPtr TKqpCounters::RecompileRequestGet() const { - return TKqpCountersBase::CompileRequestsRecompile; -} - +const NMonitoring::TDynamicCounters::TCounterPtr TKqpCounters::RecompileRequestGet() const { + return TKqpCountersBase::CompileRequestsRecompile; +} + NMonitoring::TDynamicCounters::TCounterPtr TKqpCounters::GetQueryTypeCounter( NKikimrKqp::EQueryType queryType) { diff --git a/ydb/core/kqp/counters/kqp_counters.h b/ydb/core/kqp/counters/kqp_counters.h index 47924f4c953..53a8f7d2c23 100644 --- a/ydb/core/kqp/counters/kqp_counters.h +++ b/ydb/core/kqp/counters/kqp_counters.h @@ -43,15 +43,15 @@ protected: void ReportCloseSession(ui64 requestSize); void ReportQueryRequest(const NKikimrKqp::TQueryRequest& request); - void ReportQueryWithRangeScan(); - void ReportQueryWithFullScan(); - void ReportQueryAffectedShards(ui64 shardsCount); - void ReportQueryReadSets(ui64 readSetsCount); - void ReportQueryReadBytes(ui64 bytesCount); - void ReportQueryReadRows(ui64 rowsCount); - void ReportQueryMaxShardReplySize(ui64 replySize); - void ReportQueryMaxShardProgramSize(ui64 programSize); - + void ReportQueryWithRangeScan(); + void ReportQueryWithFullScan(); + void ReportQueryAffectedShards(ui64 shardsCount); + void ReportQueryReadSets(ui64 readSetsCount); + void ReportQueryReadBytes(ui64 bytesCount); + void ReportQueryReadRows(ui64 rowsCount); + void ReportQueryMaxShardReplySize(ui64 replySize); + void ReportQueryMaxShardProgramSize(ui64 programSize); + void ReportResponseStatus(ui64 responseSize, Ydb::StatusIds::StatusCode ydbStatus); void ReportResultsBytes(ui64 resultsSize); @@ -90,7 +90,7 @@ protected: void ReportCompileRequestRejected(); void ReportCompileRequestTimeout(); void ReportCompileDurations(TDuration duration, TDuration cpuTime); - void ReportRecompileRequestGet(); + void ReportRecompileRequestGet(); NMonitoring::TDynamicCounterPtr GetQueryReplayCounters() const; protected: @@ -119,15 +119,15 @@ protected: THashMap<NKikimrKqp::EQueryType, NMonitoring::TDynamicCounters::TCounterPtr> QueryTypes; NMonitoring::TDynamicCounters::TCounterPtr OtherQueryTypes; - NMonitoring::TDynamicCounters::TCounterPtr QueriesWithRangeScan; - NMonitoring::TDynamicCounters::TCounterPtr QueriesWithFullScan; - NMonitoring::THistogramPtr QueryAffectedShardsCount; - NMonitoring::THistogramPtr QueryReadSetsCount; - NMonitoring::THistogramPtr QueryReadBytes; - NMonitoring::THistogramPtr QueryReadRows; - NMonitoring::THistogramPtr QueryMaxShardReplySize; - NMonitoring::THistogramPtr QueryMaxShardProgramSize; - + NMonitoring::TDynamicCounters::TCounterPtr QueriesWithRangeScan; + NMonitoring::TDynamicCounters::TCounterPtr QueriesWithFullScan; + NMonitoring::THistogramPtr QueryAffectedShardsCount; + NMonitoring::THistogramPtr QueryReadSetsCount; + NMonitoring::THistogramPtr QueryReadBytes; + NMonitoring::THistogramPtr QueryReadRows; + NMonitoring::THistogramPtr QueryMaxShardReplySize; + NMonitoring::THistogramPtr QueryMaxShardProgramSize; + // Request latency THashMap<NKikimrKqp::EQueryAction, NMonitoring::THistogramPtr> QueryLatencies; NMonitoring::THistogramPtr YdbQueryExecuteLatency; @@ -178,7 +178,7 @@ protected: NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsInvalidate; NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsRejected; NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsTimeout; - NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsRecompile; + NMonitoring::TDynamicCounters::TCounterPtr CompileRequestsRecompile; NMonitoring::TDynamicCounters::TCounterPtr CompileTotal; NMonitoring::TDynamicCounters::TCounterPtr CompileErrors; NMonitoring::TDynamicCounters::TCounterPtr CompileActive; @@ -253,7 +253,7 @@ public: void ReportResponseStatus(TKqpDbCountersPtr dbCounters, ui64 responseSize, Ydb::StatusIds::StatusCode ydbStatus); void ReportResultsBytes(TKqpDbCountersPtr dbCounters, ui64 resultsSize); void ReportIssues(TKqpDbCountersPtr dbCounters, const Ydb::Issue::IssueMessage& issue); - + void ReportQueryWithRangeScan(TKqpDbCountersPtr dbCounters); void ReportQueryWithFullScan(TKqpDbCountersPtr dbCounters); void ReportQueryAffectedShards(TKqpDbCountersPtr dbCounters, ui64 shardsCount); @@ -300,9 +300,9 @@ public: void ReportCompileDurations(TKqpDbCountersPtr dbCounters, TDuration duration, TDuration cpuTime); void ReportRecompileRequestGet(TKqpDbCountersPtr dbCounters); - const NMonitoring::TDynamicCounters::TCounterPtr RecompileRequestGet() const; + const NMonitoring::TDynamicCounters::TCounterPtr RecompileRequestGet() const; NMonitoring::TDynamicCounterPtr GetQueryReplayCounters() const; - + NMonitoring::TDynamicCounters::TCounterPtr GetQueryTypeCounter(NKikimrKqp::EQueryType queryType); TKqpDbCountersPtr GetDbCounters(const TString& database); diff --git a/ydb/core/kqp/counters/kqp_db_counters.h b/ydb/core/kqp/counters/kqp_db_counters.h index 8b89f0214ab..814c382d4c2 100644 --- a/ydb/core/kqp/counters/kqp_db_counters.h +++ b/ydb/core/kqp/counters/kqp_db_counters.h @@ -82,8 +82,8 @@ namespace NKqp { XX(DB_KQP_YDB_WORKERS_CLOSED_IDLE, YdbWorkersClosedIdle) \ XX(DB_KQP_YDB_REQUEST_BYTES, YdbRequestBytes) \ XX(DB_KQP_YDB_PARAMS_BYTES, YdbParametersBytes) \ - XX(DB_KQP_YDB_RSP_LOCKS_INVALIDATED, YdbResponsesLocksInvalidated) \ - XX(DB_KQP_QUERIES_WITH_RANGE_SCAN, QueriesWithRangeScan) \ + XX(DB_KQP_YDB_RSP_LOCKS_INVALIDATED, YdbResponsesLocksInvalidated) \ + XX(DB_KQP_QUERIES_WITH_RANGE_SCAN, QueriesWithRangeScan) \ XX(DB_KQP_QUERIES_WITH_FULL_SCAN, QueriesWithFullScan) \ XX(DB_KQP_YDB_RESPONSE_BYTES, YdbResponseBytes) @@ -115,12 +115,12 @@ namespace NKqp { XX(DB_KQP_RW_CLIENT_DURATION, YdbTxByKind[TKqpTransactionInfo::EKind::ReadWrite].ClientDuration) \ XX(DB_KQP_YDB_COMPILE_DURATION, YdbCompileDuration) \ XX(DB_KQP_COMPILE_CPU_TIME, CompileCpuTime) \ - XX(DB_KQP_YDB_QUERY_LATENCY_EXECUTE, YdbQueryExecuteLatency) \ - XX(DB_KQP_QUERY_AFFECTED_SHARDS, QueryAffectedShardsCount) \ - XX(DB_KQP_QUERY_READ_SETS, QueryReadSetsCount) \ - XX(DB_KQP_QUERY_READ_BYTES, QueryReadBytes) \ - XX(DB_KQP_QUERY_READ_ROWS, QueryReadRows) \ - XX(DB_KQP_QUERY_MAX_SHARD_REPLY, QueryMaxShardReplySize) \ - XX(DB_KQP_QUERY_MAX_SHARD_PROGRAM, QueryMaxShardProgramSize) + XX(DB_KQP_YDB_QUERY_LATENCY_EXECUTE, YdbQueryExecuteLatency) \ + XX(DB_KQP_QUERY_AFFECTED_SHARDS, QueryAffectedShardsCount) \ + XX(DB_KQP_QUERY_READ_SETS, QueryReadSetsCount) \ + XX(DB_KQP_QUERY_READ_BYTES, QueryReadBytes) \ + XX(DB_KQP_QUERY_READ_ROWS, QueryReadRows) \ + XX(DB_KQP_QUERY_MAX_SHARD_REPLY, QueryMaxShardReplySize) \ + XX(DB_KQP_QUERY_MAX_SHARD_PROGRAM, QueryMaxShardProgramSize) } } diff --git a/ydb/core/kqp/executer/kqp_data_executer.cpp b/ydb/core/kqp/executer/kqp_data_executer.cpp index cdaa83d6052..003d4ffa206 100644 --- a/ydb/core/kqp/executer/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer/kqp_data_executer.cpp @@ -968,7 +968,7 @@ private: Y_VERIFY_DEBUG(stageInfo.Meta.TablePath == op.GetTable().GetPath()); auto columns = BuildKqpColumns(op, table); - THashMap<ui64, TShardInfo> partitions; + THashMap<ui64, TShardInfo> partitions; switch (op.GetTypeCase()) { case NKqpProto::TKqpPhyTableOperation::kReadRanges: @@ -980,21 +980,21 @@ private: NDqProto::TData itemsLimitBytes; NKikimr::NMiniKQL::TType* itemsLimitType = nullptr; - if (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kReadRanges) { - partitions = PrunePartitions(TableKeys, op.GetReadRanges(), stageInfo, holderFactory, typeEnv); - ExtractItemsLimit(stageInfo, op.GetReadRanges().GetItemsLimit(), holderFactory, typeEnv, - itemsLimit, itemsLimitParamName, itemsLimitBytes, itemsLimitType); - reverse = op.GetReadRanges().GetReverse(); - } else if (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kReadRange) { - partitions = PrunePartitions(TableKeys, op.GetReadRange(), stageInfo, holderFactory, typeEnv); + if (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kReadRanges) { + partitions = PrunePartitions(TableKeys, op.GetReadRanges(), stageInfo, holderFactory, typeEnv); + ExtractItemsLimit(stageInfo, op.GetReadRanges().GetItemsLimit(), holderFactory, typeEnv, + itemsLimit, itemsLimitParamName, itemsLimitBytes, itemsLimitType); + reverse = op.GetReadRanges().GetReverse(); + } else if (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kReadRange) { + partitions = PrunePartitions(TableKeys, op.GetReadRange(), stageInfo, holderFactory, typeEnv); ExtractItemsLimit(stageInfo, op.GetReadRange().GetItemsLimit(), holderFactory, typeEnv, itemsLimit, itemsLimitParamName, itemsLimitBytes, itemsLimitType); reverse = op.GetReadRange().GetReverse(); - } else if (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kLookup) { - partitions = PrunePartitions(TableKeys, op.GetLookup(), stageInfo, holderFactory, typeEnv); + } else if (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kLookup) { + partitions = PrunePartitions(TableKeys, op.GetLookup(), stageInfo, holderFactory, typeEnv); } - for (auto& [shardId, shardInfo] : partitions) { + for (auto& [shardId, shardInfo] : partitions) { YQL_ENSURE(!shardInfo.KeyWriteRanges); auto& task = getShardTask(shardId); diff --git a/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp b/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp index 88b509d84f0..3148a2ec59b 100644 --- a/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp +++ b/ydb/core/kqp/executer/ut/kqp_executer_ut.cpp @@ -29,7 +29,7 @@ NKqpProto::TKqpPhyTx BuildTxPlan(const TString& sql, TIntrusivePtr<IKqpGateway> IModuleResolver::TPtr moduleResolver; UNIT_ASSERT(GetYqlDefaultModuleResolver(moduleCtx, moduleResolver)); - auto qp = CreateKqpHost(gateway, cluster, "/Root", config, moduleResolver); + auto qp = CreateKqpHost(gateway, cluster, "/Root", config, moduleResolver); auto result = qp->SyncPrepareDataQuery(sql, IKqpHost::TPrepareSettings()); result.Issues().PrintTo(Cerr); UNIT_ASSERT(result.Success()); diff --git a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json index be00c5574aa..2faa33d5e1d 100644 --- a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json +++ b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json @@ -247,15 +247,15 @@ ] }, { - "Name": "TKqpTxInternalBinding", - "Base": "TCallable", - "Match": {"Type": "Callable", "Name": "KqpTxInternalBinding"}, - "Children": [ - {"Index": 0, "Name": "Type", "Type": "TExprBase"}, - {"Index": 1, "Name": "Kind", "Type": "TCoAtom"} - ] - }, - { + "Name": "TKqpTxInternalBinding", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "KqpTxInternalBinding"}, + "Children": [ + {"Index": 0, "Name": "Type", "Type": "TExprBase"}, + {"Index": 1, "Name": "Kind", "Type": "TCoAtom"} + ] + }, + { "Name": "TKqpPhysicalTx", "Base": "TCallable", "Match": {"Type": "Callable", "Name": "KqpPhysicalTx"}, diff --git a/ydb/core/kqp/host/kqp_explain_prepared.cpp b/ydb/core/kqp/host/kqp_explain_prepared.cpp index cbdbe515430..f022c9f8059 100644 --- a/ydb/core/kqp/host/kqp_explain_prepared.cpp +++ b/ydb/core/kqp/host/kqp_explain_prepared.cpp @@ -1,208 +1,208 @@ -#include "kqp_host_impl.h" - -#include <ydb/core/kqp/common/kqp_gateway.h> -#include <ydb/core/kqp/common/kqp_yql.h> -#include <ydb/core/kqp/prepare/kqp_query_plan.h> - -namespace NKikimr { -namespace NKqp { - -using namespace NYql; -using namespace NYql::NNodes; -using namespace NThreading; - -class TKqpExplainPreparedTransformer : public NYql::TGraphTransformerBase { -public: - TKqpExplainPreparedTransformer(TIntrusivePtr<IKqpGateway> gateway, const TString& cluster, - TIntrusivePtr<TKqlTransformContext> transformCtx) - : Gateway(gateway) - , Cluster(cluster) - , TransformCtx(transformCtx) - , CurrentTxIndex(0) {} - - TStatus DoTransform(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext& ctx) override { - output = input; - - auto pure = [](const auto& tx) { - if (tx.GetType() != NKqpProto::TKqpPhyTx::TYPE_COMPUTE) { - return false; - } - - for (const auto& stage : tx.GetStages()) { - if (stage.InputsSize() != 0) { - return false; - } - } - - return true; - }; - - auto& query = *TransformCtx->QueryCtx->PreparingQuery->MutablePhysicalQuery(); - while (CurrentTxIndex < query.TransactionsSize()) { - const auto& tx = query.GetTransactions(CurrentTxIndex); - auto params = PrepareParameters(tx); - - if (pure(tx) && params) { - IKqpGateway::TExecPhysicalRequest request; - request.Transactions.emplace_back(tx, std::move(*params)); - - ExecuteFuture = Gateway->ExecutePure(std::move(request), {}); - - Promise = NewPromise(); - ExecuteFuture.Apply([promise = Promise](const TFuture<IKqpGateway::TExecPhysicalResult> future) mutable { - YQL_ENSURE(future.HasValue()); - promise.SetValue(); - }); - - return TStatus::Async; - } - - ++CurrentTxIndex; - } - - PhyQuerySetTxPlans(query, TKqpPhysicalQuery(input), std::move(TxResults), - ctx, Cluster, TransformCtx->Tables, TransformCtx->Config); - query.SetQueryAst(KqpExprToPrettyString(*input, ctx)); - - return TStatus::Ok; - } - - NThreading::TFuture<void> DoGetAsyncFuture(const NYql::TExprNode& /*input*/) override { - return Promise.GetFuture(); - } - - TStatus DoApplyAsyncChanges(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, - NYql::TExprContext& ctx) override - { - output = input; - - auto result = ExecuteFuture.ExtractValue(); - result.ReportIssues(ctx.IssueManager); - if (!result.Success()) { - return TStatus::Error; - } - - auto& txResults = result.ExecuterResult.GetResults(); - TxResults[CurrentTxIndex] = {txResults.begin(), txResults.end()}; - - ++CurrentTxIndex; - return TStatus::Repeat; - } - - void Rewind() override { - CurrentTxIndex = 0; - TxResults.clear(); - } - -private: - - TMaybe<NDq::TMkqlValueRef> GetParamValue(const NKqpProto::TKqpPhyParamBinding& paramBinding) - { - switch (paramBinding.GetTypeCase()) { - case NKqpProto::TKqpPhyParamBinding::kExternalBinding: { - auto clientParam = TransformCtx->QueryCtx->Parameters.FindPtr(paramBinding.GetName()); - if (clientParam) { - return TMaybe<NDq::TMkqlValueRef>(NDq::TMkqlValueRef(*clientParam)); - } - return {}; - } - case NKqpProto::TKqpPhyParamBinding::kTxResultBinding: { - auto& txResultBinding = paramBinding.GetTxResultBinding(); - auto txIndex = txResultBinding.GetTxIndex(); - auto resultIndex = txResultBinding.GetResultIndex(); - - if (TxResults.contains(txIndex) && resultIndex < TxResults[txIndex].size()) { - return TMaybe<NDq::TMkqlValueRef>(NDq::TMkqlValueRef(TxResults[txIndex][resultIndex])); - } - return {}; - } - case NKqpProto::TKqpPhyParamBinding::kInternalBinding: { - auto& internalBinding = paramBinding.GetInternalBinding(); - auto& param = TransformCtx->QueryCtx->Parameters[paramBinding.GetName()]; - - switch (internalBinding.GetType()) { - case NKqpProto::TKqpPhyInternalBinding::PARAM_NOW: - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui64>::Id); - param.MutableValue()->SetUint64(TransformCtx->QueryCtx->GetCachedNow()); - break; - case NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_DATE: { - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TDate>::Id); - ui64 date = TransformCtx->QueryCtx->GetCachedDate(); - YQL_ENSURE(date <= Max<ui32>()); - param.MutableValue()->SetUint32(static_cast<ui32>(date)); - break; - } - case NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_DATETIME: { - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TDatetime>::Id); - ui64 datetime = TransformCtx->QueryCtx->GetCachedDatetime(); - YQL_ENSURE(datetime <= Max<ui32>()); - param.MutableValue()->SetUint32(static_cast<ui32>(datetime)); - break; - } - case NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_TIMESTAMP: - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TTimestamp>::Id); - param.MutableValue()->SetUint64(TransformCtx->QueryCtx->GetCachedTimestamp()); - break; - case NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM_NUMBER: - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui64>::Id); - param.MutableValue()->SetUint64(TransformCtx->QueryCtx->GetCachedRandom<ui64>()); - break; - case NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM: - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<double>::Id); - param.MutableValue()->SetDouble(TransformCtx->QueryCtx->GetCachedRandom<double>()); - break; - case NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM_UUID: { - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TUuid>::Id); - auto uuid = TransformCtx->QueryCtx->GetCachedRandom<TGUID>(); - param.MutableValue()->SetBytes(uuid.dw, sizeof(TGUID)); - break; - } - default: - YQL_ENSURE(false, "Unexpected internal parameter type: " << (ui32)internalBinding.GetType()); - } - - return NDq::TMkqlValueRef(param); - } - default: - YQL_ENSURE(false, "Unexpected parameter binding type: " << (ui32)paramBinding.GetTypeCase()); - } - } - - TMaybe<TKqpParamsMap> PrepareParameters(const NKqpProto::TKqpPhyTx& tx) { - TKqpParamsMap params; - for (const auto& paramBinding : tx.GetParamBindings()) { - if (auto paramValue = GetParamValue(paramBinding)) { - params.Values.emplace(std::make_pair(paramBinding.GetName(), *paramValue)); - } else { - return {}; - } - } - - return TMaybe<TKqpParamsMap>(params); - } - - TIntrusivePtr<IKqpGateway> Gateway; - TString Cluster; - THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> TxResults; - TIntrusivePtr<TKqlTransformContext> TransformCtx; - ui32 CurrentTxIndex; - NThreading::TFuture<IKqpGateway::TExecPhysicalResult> ExecuteFuture; - NThreading::TPromise<void> Promise; -}; - - -TAutoPtr<IGraphTransformer> CreateKqpExplainPreparedTransformer(TIntrusivePtr<IKqpGateway> gateway, - const TString& cluster, TIntrusivePtr<TKqlTransformContext> transformCtx) -{ - return new TKqpExplainPreparedTransformer(gateway, cluster, transformCtx); -} - -} // namespace NKqp -} // namespace NKikimr +#include "kqp_host_impl.h" + +#include <ydb/core/kqp/common/kqp_gateway.h> +#include <ydb/core/kqp/common/kqp_yql.h> +#include <ydb/core/kqp/prepare/kqp_query_plan.h> + +namespace NKikimr { +namespace NKqp { + +using namespace NYql; +using namespace NYql::NNodes; +using namespace NThreading; + +class TKqpExplainPreparedTransformer : public NYql::TGraphTransformerBase { +public: + TKqpExplainPreparedTransformer(TIntrusivePtr<IKqpGateway> gateway, const TString& cluster, + TIntrusivePtr<TKqlTransformContext> transformCtx) + : Gateway(gateway) + , Cluster(cluster) + , TransformCtx(transformCtx) + , CurrentTxIndex(0) {} + + TStatus DoTransform(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, NYql::TExprContext& ctx) override { + output = input; + + auto pure = [](const auto& tx) { + if (tx.GetType() != NKqpProto::TKqpPhyTx::TYPE_COMPUTE) { + return false; + } + + for (const auto& stage : tx.GetStages()) { + if (stage.InputsSize() != 0) { + return false; + } + } + + return true; + }; + + auto& query = *TransformCtx->QueryCtx->PreparingQuery->MutablePhysicalQuery(); + while (CurrentTxIndex < query.TransactionsSize()) { + const auto& tx = query.GetTransactions(CurrentTxIndex); + auto params = PrepareParameters(tx); + + if (pure(tx) && params) { + IKqpGateway::TExecPhysicalRequest request; + request.Transactions.emplace_back(tx, std::move(*params)); + + ExecuteFuture = Gateway->ExecutePure(std::move(request), {}); + + Promise = NewPromise(); + ExecuteFuture.Apply([promise = Promise](const TFuture<IKqpGateway::TExecPhysicalResult> future) mutable { + YQL_ENSURE(future.HasValue()); + promise.SetValue(); + }); + + return TStatus::Async; + } + + ++CurrentTxIndex; + } + + PhyQuerySetTxPlans(query, TKqpPhysicalQuery(input), std::move(TxResults), + ctx, Cluster, TransformCtx->Tables, TransformCtx->Config); + query.SetQueryAst(KqpExprToPrettyString(*input, ctx)); + + return TStatus::Ok; + } + + NThreading::TFuture<void> DoGetAsyncFuture(const NYql::TExprNode& /*input*/) override { + return Promise.GetFuture(); + } + + TStatus DoApplyAsyncChanges(NYql::TExprNode::TPtr input, NYql::TExprNode::TPtr& output, + NYql::TExprContext& ctx) override + { + output = input; + + auto result = ExecuteFuture.ExtractValue(); + result.ReportIssues(ctx.IssueManager); + if (!result.Success()) { + return TStatus::Error; + } + + auto& txResults = result.ExecuterResult.GetResults(); + TxResults[CurrentTxIndex] = {txResults.begin(), txResults.end()}; + + ++CurrentTxIndex; + return TStatus::Repeat; + } + + void Rewind() override { + CurrentTxIndex = 0; + TxResults.clear(); + } + +private: + + TMaybe<NDq::TMkqlValueRef> GetParamValue(const NKqpProto::TKqpPhyParamBinding& paramBinding) + { + switch (paramBinding.GetTypeCase()) { + case NKqpProto::TKqpPhyParamBinding::kExternalBinding: { + auto clientParam = TransformCtx->QueryCtx->Parameters.FindPtr(paramBinding.GetName()); + if (clientParam) { + return TMaybe<NDq::TMkqlValueRef>(NDq::TMkqlValueRef(*clientParam)); + } + return {}; + } + case NKqpProto::TKqpPhyParamBinding::kTxResultBinding: { + auto& txResultBinding = paramBinding.GetTxResultBinding(); + auto txIndex = txResultBinding.GetTxIndex(); + auto resultIndex = txResultBinding.GetResultIndex(); + + if (TxResults.contains(txIndex) && resultIndex < TxResults[txIndex].size()) { + return TMaybe<NDq::TMkqlValueRef>(NDq::TMkqlValueRef(TxResults[txIndex][resultIndex])); + } + return {}; + } + case NKqpProto::TKqpPhyParamBinding::kInternalBinding: { + auto& internalBinding = paramBinding.GetInternalBinding(); + auto& param = TransformCtx->QueryCtx->Parameters[paramBinding.GetName()]; + + switch (internalBinding.GetType()) { + case NKqpProto::TKqpPhyInternalBinding::PARAM_NOW: + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui64>::Id); + param.MutableValue()->SetUint64(TransformCtx->QueryCtx->GetCachedNow()); + break; + case NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_DATE: { + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TDate>::Id); + ui64 date = TransformCtx->QueryCtx->GetCachedDate(); + YQL_ENSURE(date <= Max<ui32>()); + param.MutableValue()->SetUint32(static_cast<ui32>(date)); + break; + } + case NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_DATETIME: { + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TDatetime>::Id); + ui64 datetime = TransformCtx->QueryCtx->GetCachedDatetime(); + YQL_ENSURE(datetime <= Max<ui32>()); + param.MutableValue()->SetUint32(static_cast<ui32>(datetime)); + break; + } + case NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_TIMESTAMP: + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TTimestamp>::Id); + param.MutableValue()->SetUint64(TransformCtx->QueryCtx->GetCachedTimestamp()); + break; + case NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM_NUMBER: + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui64>::Id); + param.MutableValue()->SetUint64(TransformCtx->QueryCtx->GetCachedRandom<ui64>()); + break; + case NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM: + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<double>::Id); + param.MutableValue()->SetDouble(TransformCtx->QueryCtx->GetCachedRandom<double>()); + break; + case NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM_UUID: { + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TUuid>::Id); + auto uuid = TransformCtx->QueryCtx->GetCachedRandom<TGUID>(); + param.MutableValue()->SetBytes(uuid.dw, sizeof(TGUID)); + break; + } + default: + YQL_ENSURE(false, "Unexpected internal parameter type: " << (ui32)internalBinding.GetType()); + } + + return NDq::TMkqlValueRef(param); + } + default: + YQL_ENSURE(false, "Unexpected parameter binding type: " << (ui32)paramBinding.GetTypeCase()); + } + } + + TMaybe<TKqpParamsMap> PrepareParameters(const NKqpProto::TKqpPhyTx& tx) { + TKqpParamsMap params; + for (const auto& paramBinding : tx.GetParamBindings()) { + if (auto paramValue = GetParamValue(paramBinding)) { + params.Values.emplace(std::make_pair(paramBinding.GetName(), *paramValue)); + } else { + return {}; + } + } + + return TMaybe<TKqpParamsMap>(params); + } + + TIntrusivePtr<IKqpGateway> Gateway; + TString Cluster; + THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> TxResults; + TIntrusivePtr<TKqlTransformContext> TransformCtx; + ui32 CurrentTxIndex; + NThreading::TFuture<IKqpGateway::TExecPhysicalResult> ExecuteFuture; + NThreading::TPromise<void> Promise; +}; + + +TAutoPtr<IGraphTransformer> CreateKqpExplainPreparedTransformer(TIntrusivePtr<IKqpGateway> gateway, + const TString& cluster, TIntrusivePtr<TKqlTransformContext> transformCtx) +{ + return new TKqpExplainPreparedTransformer(gateway, cluster, transformCtx); +} + +} // namespace NKqp +} // namespace NKikimr diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp index 670f6331160..44af6d11682 100644 --- a/ydb/core/kqp/host/kqp_host.cpp +++ b/ydb/core/kqp/host/kqp_host.cpp @@ -1002,7 +1002,7 @@ private: class TKqpHost : public IKqpHost { public: - TKqpHost(TIntrusivePtr<IKqpGateway> gateway, const TString& cluster, const TString& database, + TKqpHost(TIntrusivePtr<IKqpGateway> gateway, const TString& cluster, const TString& database, TKikimrConfiguration::TPtr config, IModuleResolver::TPtr moduleResolver, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry, bool keepConfigChanges) : Gateway(gateway) @@ -1026,10 +1026,10 @@ public: FuncRegistry = FuncRegistryHolder.Get(); } - SessionCtx->SetDatabase(database); - SessionCtx->QueryPtr()->TimeProvider = TAppData::TimeProvider; - SessionCtx->QueryPtr()->RandomProvider = TAppData::RandomProvider; - + SessionCtx->SetDatabase(database); + SessionCtx->QueryPtr()->TimeProvider = TAppData::TimeProvider; + SessionCtx->QueryPtr()->RandomProvider = TAppData::RandomProvider; + KqpRunner = CreateKqpRunner(Gateway, Cluster, TypesCtx, SessionCtx, *FuncRegistry); ExprCtx->NodesAllocationLimit = SessionCtx->Config()._KqpExprNodesAllocationLimit.Get().GetRef(); @@ -2255,10 +2255,10 @@ TKqpTransactionInfo TKqpTransactionContext::GetInfo() const { } TIntrusivePtr<IKqpHost> CreateKqpHost(TIntrusivePtr<IKqpGateway> gateway, - const TString& cluster, const TString& database, TKikimrConfiguration::TPtr config, IModuleResolver::TPtr moduleResolver, + const TString& cluster, const TString& database, TKikimrConfiguration::TPtr config, IModuleResolver::TPtr moduleResolver, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry, bool keepConfigChanges) { - return MakeIntrusive<TKqpHost>(gateway, cluster, database, config, moduleResolver, funcRegistry, + return MakeIntrusive<TKqpHost>(gateway, cluster, database, config, moduleResolver, funcRegistry, keepConfigChanges); } diff --git a/ydb/core/kqp/host/kqp_host.h b/ydb/core/kqp/host/kqp_host.h index 169e0f2becc..11282d0ebe1 100644 --- a/ydb/core/kqp/host/kqp_host.h +++ b/ydb/core/kqp/host/kqp_host.h @@ -117,7 +117,7 @@ public: }; TIntrusivePtr<IKqpHost> CreateKqpHost(TIntrusivePtr<IKqpGateway> gateway, - const TString& cluster, const TString& database, NYql::TKikimrConfiguration::TPtr config, NYql::IModuleResolver::TPtr moduleResolver, + const TString& cluster, const TString& database, NYql::TKikimrConfiguration::TPtr config, NYql::IModuleResolver::TPtr moduleResolver, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry = nullptr, bool keepConfigChanges = false); } // namespace NKqp diff --git a/ydb/core/kqp/host/kqp_host_impl.h b/ydb/core/kqp/host/kqp_host_impl.h index 250dc8ea42f..fb1fb139cd5 100644 --- a/ydb/core/kqp/host/kqp_host_impl.h +++ b/ydb/core/kqp/host/kqp_host_impl.h @@ -285,9 +285,9 @@ TAutoPtr<NYql::IGraphTransformer> CreateKqpExecutePhysicalDataTransformer(TIntru const TString& cluster, TIntrusivePtr<TKqpTransactionState> txState, TIntrusivePtr<TKqlTransformContext> transformCtx); -TAutoPtr<NYql::IGraphTransformer> CreateKqpExplainPreparedTransformer(TIntrusivePtr<IKqpGateway> gateway, - const TString& cluster, TIntrusivePtr<TKqlTransformContext> transformCtx); - +TAutoPtr<NYql::IGraphTransformer> CreateKqpExplainPreparedTransformer(TIntrusivePtr<IKqpGateway> gateway, + const TString& cluster, TIntrusivePtr<TKqlTransformContext> transformCtx); + TAutoPtr<NYql::IGraphTransformer> CreateKqpExecuteScanTransformer(TIntrusivePtr<IKqpGateway> gateway, const TString& cluster, TIntrusivePtr<TKqpTransactionState> txState, TIntrusivePtr<TKqlTransformContext> transformCtx); diff --git a/ydb/core/kqp/host/kqp_run_physical.cpp b/ydb/core/kqp/host/kqp_run_physical.cpp index 77be281c8c1..e6bd601eef6 100644 --- a/ydb/core/kqp/host/kqp_run_physical.cpp +++ b/ydb/core/kqp/host/kqp_run_physical.cpp @@ -197,60 +197,60 @@ NDq::TMkqlValueRef TKqpExecutePhysicalTransformerBase::GetParamValue( YQL_ENSURE(resultIndex < txResults[txIndex].size()); return NDq::TMkqlValueRef(txResults[txIndex][resultIndex]); } - case NKqpProto::TKqpPhyParamBinding::kInternalBinding: { - auto& internalBinding = paramBinding.GetInternalBinding(); - auto& param = TransformCtx->QueryCtx->Parameters[paramBinding.GetName()]; - - switch (internalBinding.GetType()) { - case NKqpProto::TKqpPhyInternalBinding::PARAM_NOW: - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui64>::Id); - param.MutableValue()->SetUint64(TransformCtx->QueryCtx->GetCachedNow()); - break; - case NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_DATE: { - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TDate>::Id); - ui64 date = TransformCtx->QueryCtx->GetCachedDate(); - YQL_ENSURE(date <= Max<ui32>()); - param.MutableValue()->SetUint32(static_cast<ui32>(date)); - break; - } - case NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_DATETIME: { - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TDatetime>::Id); - ui64 datetime = TransformCtx->QueryCtx->GetCachedDatetime(); - YQL_ENSURE(datetime <= Max<ui32>()); - param.MutableValue()->SetUint32(static_cast<ui32>(datetime)); - break; - } - case NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_TIMESTAMP: - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TTimestamp>::Id); - param.MutableValue()->SetUint64(TransformCtx->QueryCtx->GetCachedTimestamp()); - break; - case NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM_NUMBER: - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui64>::Id); - param.MutableValue()->SetUint64(TransformCtx->QueryCtx->GetCachedRandom<ui64>()); - break; - case NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM: - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<double>::Id); - param.MutableValue()->SetDouble(TransformCtx->QueryCtx->GetCachedRandom<double>()); - break; - case NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM_UUID: { - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TUuid>::Id); - auto uuid = TransformCtx->QueryCtx->GetCachedRandom<TGUID>(); - param.MutableValue()->SetBytes(uuid.dw, sizeof(TGUID)); - break; - } - default: - YQL_ENSURE(false, "Unexpected internal parameter type: " << (ui32)internalBinding.GetType()); - } - - return NDq::TMkqlValueRef(param); - } + case NKqpProto::TKqpPhyParamBinding::kInternalBinding: { + auto& internalBinding = paramBinding.GetInternalBinding(); + auto& param = TransformCtx->QueryCtx->Parameters[paramBinding.GetName()]; + + switch (internalBinding.GetType()) { + case NKqpProto::TKqpPhyInternalBinding::PARAM_NOW: + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui64>::Id); + param.MutableValue()->SetUint64(TransformCtx->QueryCtx->GetCachedNow()); + break; + case NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_DATE: { + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TDate>::Id); + ui64 date = TransformCtx->QueryCtx->GetCachedDate(); + YQL_ENSURE(date <= Max<ui32>()); + param.MutableValue()->SetUint32(static_cast<ui32>(date)); + break; + } + case NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_DATETIME: { + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TDatetime>::Id); + ui64 datetime = TransformCtx->QueryCtx->GetCachedDatetime(); + YQL_ENSURE(datetime <= Max<ui32>()); + param.MutableValue()->SetUint32(static_cast<ui32>(datetime)); + break; + } + case NKqpProto::TKqpPhyInternalBinding::PARAM_CURRENT_TIMESTAMP: + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TTimestamp>::Id); + param.MutableValue()->SetUint64(TransformCtx->QueryCtx->GetCachedTimestamp()); + break; + case NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM_NUMBER: + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui64>::Id); + param.MutableValue()->SetUint64(TransformCtx->QueryCtx->GetCachedRandom<ui64>()); + break; + case NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM: + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<double>::Id); + param.MutableValue()->SetDouble(TransformCtx->QueryCtx->GetCachedRandom<double>()); + break; + case NKqpProto::TKqpPhyInternalBinding::PARAM_RANDOM_UUID: { + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TUuid>::Id); + auto uuid = TransformCtx->QueryCtx->GetCachedRandom<TGUID>(); + param.MutableValue()->SetBytes(uuid.dw, sizeof(TGUID)); + break; + } + default: + YQL_ENSURE(false, "Unexpected internal parameter type: " << (ui32)internalBinding.GetType()); + } + + return NDq::TMkqlValueRef(param); + } default: YQL_ENSURE(false, "Unexpected parameter binding type: " << (ui32)paramBinding.GetTypeCase()); } diff --git a/ydb/core/kqp/host/kqp_runner.cpp b/ydb/core/kqp/host/kqp_runner.cpp index 4122bca04d7..42b4eca7c66 100644 --- a/ydb/core/kqp/host/kqp_runner.cpp +++ b/ydb/core/kqp/host/kqp_runner.cpp @@ -188,10 +188,10 @@ public: .Add(CreateKqpFinalizeTransformer(Gateway, Cluster, TxState, TransformCtx), "Finalize") .Build(false); - PreparedExplainTransformer = TTransformationPipeline(typesCtx) - .Add(CreateKqpExplainPreparedTransformer(Gateway, Cluster, TransformCtx), "ExplainQuery") - .Build(false); - + PreparedExplainTransformer = TTransformationPipeline(typesCtx) + .Add(CreateKqpExplainPreparedTransformer(Gateway, Cluster, TransformCtx), "ExplainQuery") + .Build(false); + PhysicalOptimizeTransformer = TTransformationPipeline(typesCtx) .AddServiceTransformers() .Add(TLogExprTransformer::Sync("PhysicalOptimizeTransformer", logComp, logLevel), "LogPhysicalOptimize") @@ -225,18 +225,18 @@ public: "BuildPhysicalTxs") .Build(false); - PhysicalPeepholeTransformer = TTransformationPipeline(typesCtx) - .AddServiceTransformers() - .Add(TLogExprTransformer::Sync("PhysicalPeepholeTransformer", logComp, logLevel), "LogPhysicalPeephole") - .AddTypeAnnotationTransformer(CreateKqpTypeAnnotationTransformer(Cluster, sessionCtx->TablesPtr(), *typesCtx, Config)) - .AddPostTypeAnnotation() - .Add( - CreateKqpTxsPeepholeTransformer( - CreateTypeAnnotationTransformer( - CreateKqpTypeAnnotationTransformer(Cluster, sessionCtx->TablesPtr(), *typesCtx, Config), - *typesCtx), *typesCtx, Config), "Peephole") - .Build(false); - + PhysicalPeepholeTransformer = TTransformationPipeline(typesCtx) + .AddServiceTransformers() + .Add(TLogExprTransformer::Sync("PhysicalPeepholeTransformer", logComp, logLevel), "LogPhysicalPeephole") + .AddTypeAnnotationTransformer(CreateKqpTypeAnnotationTransformer(Cluster, sessionCtx->TablesPtr(), *typesCtx, Config)) + .AddPostTypeAnnotation() + .Add( + CreateKqpTxsPeepholeTransformer( + CreateTypeAnnotationTransformer( + CreateKqpTypeAnnotationTransformer(Cluster, sessionCtx->TablesPtr(), *typesCtx, Config), + *typesCtx), *typesCtx, Config), "Peephole") + .Build(false); + PhysicalRunQueryTransformer = TTransformationPipeline(typesCtx) .Add(CreateKqpAcquireMvccSnapshotTransformer(Gateway, TxState, TransformCtx, true), "AcquireMvccSnapshot") .Add(CreateKqpExecutePhysicalDataTransformer(Gateway, Cluster, TxState, TransformCtx), "ExecutePhysical") @@ -438,22 +438,22 @@ private: TExprNode::TPtr finalProgram; bool hasNonDeterministicFunctions; - TPeepholeSettings peepholeSettings; - peepholeSettings.WithNonDeterministicRules = false; + TPeepholeSettings peepholeSettings; + peepholeSettings.WithNonDeterministicRules = false; status = PeepHoleOptimizeNode<false>(optimizedProgram, finalProgram, ctx, TypesCtx, KqlTypeAnnTransformer.Get(), - hasNonDeterministicFunctions, peepholeSettings); + hasNonDeterministicFunctions, peepholeSettings); if (status != IGraphTransformer::TStatus::Ok) { ctx.AddError(TIssue(ctx.GetPosition(dataQuery.Pos()), "Failed to peephole optimize KQL query.")); return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues())); } - status = ReplaceNonDetFunctionsWithParams(finalProgram, ctx); - if (status != IGraphTransformer::TStatus::Ok) { - ctx.AddError(TIssue(ctx.GetPosition(dataQuery.Pos()), - "Failed to replace non deterministic functions with params for KQL query.")); - return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues())); - } - + status = ReplaceNonDetFunctionsWithParams(finalProgram, ctx); + if (status != IGraphTransformer::TStatus::Ok) { + ctx.AddError(TIssue(ctx.GetPosition(dataQuery.Pos()), + "Failed to replace non deterministic functions with params for KQL query.")); + return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues())); + } + KqlPrepareTransformer->Rewind(); NKikimrKqp::TKqlSettings kqlSettings; @@ -540,19 +540,19 @@ private: return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues())); } - PhysicalPeepholeTransformer->Rewind(); - auto transformedQuery = builtQuery; - status = InstantTransform(*PhysicalPeepholeTransformer, transformedQuery, ctx); - if (status != IGraphTransformer::TStatus::Ok) { - ctx.AddError(TIssue(ctx.GetPosition(query->Pos()), "Failed peephole.")); - return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>( - ctx.IssueManager.GetIssues())); - } - + PhysicalPeepholeTransformer->Rewind(); + auto transformedQuery = builtQuery; + status = InstantTransform(*PhysicalPeepholeTransformer, transformedQuery, ctx); + if (status != IGraphTransformer::TStatus::Ok) { + ctx.AddError(TIssue(ctx.GetPosition(query->Pos()), "Failed peephole.")); + return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>( + ctx.IssueManager.GetIssues())); + } + YQL_CLOG(INFO, ProviderKqp) << "Physical KQL query: " << KqpExprToPrettyString(*builtQuery, ctx); auto& preparedQuery = *TransformCtx->QueryCtx->PreparingQuery; - TKqpPhysicalQuery physicalQuery(transformedQuery); + TKqpPhysicalQuery physicalQuery(transformedQuery); auto compiler = CreateKqpQueryCompiler(Cluster, OptimizeCtx->Tables, FuncRegistry); auto ret = compiler->CompilePhysicalQuery(physicalQuery, dataQuery.Operations(), *preparedQuery.MutablePhysicalQuery(), ctx); @@ -562,8 +562,8 @@ private: } preparedQuery.SetVersion(NKikimrKqp::TPreparedQuery::VERSION_PHYSICAL_V1); // TODO(sk): only on stats mode or if explain-only - PreparedExplainTransformer->Rewind(); - return MakeIntrusive<TPhysicalAsyncRunResult>(builtQuery, ctx, *PreparedExplainTransformer, *TransformCtx); + PreparedExplainTransformer->Rewind(); + return MakeIntrusive<TPhysicalAsyncRunResult>(builtQuery, ctx, *PreparedExplainTransformer, *TransformCtx); } TIntrusivePtr<TAsyncQueryResult> ExecutePhysicalDataQuery(const TExprNode::TPtr& world, @@ -631,11 +631,11 @@ private: TAutoPtr<IGraphTransformer> KqlOptimizeTransformer; TAutoPtr<IGraphTransformer> KqlPrepareTransformer; TAutoPtr<IGraphTransformer> PreparedRunTransformer; - TAutoPtr<IGraphTransformer> PreparedExplainTransformer; + TAutoPtr<IGraphTransformer> PreparedExplainTransformer; TAutoPtr<IGraphTransformer> PhysicalOptimizeTransformer; TAutoPtr<IGraphTransformer> PhysicalBuildQueryTransformer; - TAutoPtr<IGraphTransformer> PhysicalPeepholeTransformer; + TAutoPtr<IGraphTransformer> PhysicalPeepholeTransformer; TAutoPtr<IGraphTransformer> PhysicalRunQueryTransformer; TAutoPtr<IGraphTransformer> ScanRunQueryTransformer; }; diff --git a/ydb/core/kqp/host/ya.make b/ydb/core/kqp/host/ya.make index 6976de4fb8d..e0f595515c4 100644 --- a/ydb/core/kqp/host/ya.make +++ b/ydb/core/kqp/host/ya.make @@ -9,7 +9,7 @@ SRCS( kqp_host.cpp kqp_ne_helper.cpp kqp_run_data.cpp - kqp_explain_prepared.cpp + kqp_explain_prepared.cpp kqp_run_physical.cpp kqp_run_prepared.cpp kqp_run_scan.cpp diff --git a/ydb/core/kqp/kqp.h b/ydb/core/kqp/kqp.h index 28433939cf2..09f794591d3 100644 --- a/ydb/core/kqp/kqp.h +++ b/ydb/core/kqp/kqp.h @@ -169,12 +169,12 @@ using TPreparedQueryConstPtr = std::shared_ptr<const NKikimrKqp::TPreparedQuery> struct TKqpCompileResult { using TConstPtr = std::shared_ptr<const TKqpCompileResult>; - TKqpCompileResult(const TString& uid, TKqpQueryId&& query, const Ydb::StatusIds::StatusCode& status, - const NYql::TIssues& issues) + TKqpCompileResult(const TString& uid, TKqpQueryId&& query, const Ydb::StatusIds::StatusCode& status, + const NYql::TIssues& issues) : Status(status) , Issues(issues) , Query(std::move(query)) - , Uid(uid) {} + , Uid(uid) {} TKqpCompileResult(const TString& uid, const Ydb::StatusIds::StatusCode& status, const NYql::TIssues& issues) : Status(status) @@ -182,7 +182,7 @@ struct TKqpCompileResult { , Uid(uid) {} static std::shared_ptr<TKqpCompileResult> Make(const TString& uid, TKqpQueryId&& query, - const Ydb::StatusIds::StatusCode& status, const NYql::TIssues& issues) + const Ydb::StatusIds::StatusCode& status, const NYql::TIssues& issues) { return std::make_shared<TKqpCompileResult>(uid, std::move(query), status, issues); } @@ -385,23 +385,23 @@ struct TEvKqp { TMaybe<bool> DocumentApiRestricted; }; - struct TEvRecompileRequest : public TEventLocal<TEvRecompileRequest, TKqpEvents::EvRecompileRequest> { - TEvRecompileRequest(const TString& userToken, const TString& uid, const TMaybe<TKqpQueryId>& query, + struct TEvRecompileRequest : public TEventLocal<TEvRecompileRequest, TKqpEvents::EvRecompileRequest> { + TEvRecompileRequest(const TString& userToken, const TString& uid, const TMaybe<TKqpQueryId>& query, TInstant deadline, TKqpDbCountersPtr dbCounters) - : UserToken(userToken) - , Uid(uid) - , Query(query) - , Deadline(deadline) + : UserToken(userToken) + , Uid(uid) + , Query(query) + , Deadline(deadline) , DbCounters(dbCounters) {} - - TString UserToken; - TString Uid; - TMaybe<TKqpQueryId> Query; - - TInstant Deadline; + + TString UserToken; + TString Uid; + TMaybe<TKqpQueryId> Query; + + TInstant Deadline; TKqpDbCountersPtr DbCounters; - }; - + }; + struct TEvCompileResponse : public TEventLocal<TEvCompileResponse, TKqpEvents::EvCompileResponse> { TEvCompileResponse(const TKqpCompileResult::TConstPtr& compileResult) : CompileResult(compileResult) {} diff --git a/ydb/core/kqp/kqp_compile_actor.cpp b/ydb/core/kqp/kqp_compile_actor.cpp index 4eb65b79129..27cf296a03d 100644 --- a/ydb/core/kqp/kqp_compile_actor.cpp +++ b/ydb/core/kqp/kqp_compile_actor.cpp @@ -50,7 +50,7 @@ public: : Owner(owner) , ModuleResolverState(moduleResolverState) , Counters(counters) - , Uid(uid) + , Uid(uid) , Query(query) , UserToken(userToken) , DbCounters(dbCounters) @@ -96,7 +96,7 @@ public: Config->FeatureFlags = AppData(ctx)->FeatureFlags; - KqpHost = CreateKqpHost(Gateway, Query.Cluster, Query.Database, Config, ModuleResolverState->ModuleResolver, + KqpHost = CreateKqpHost(Gateway, Query.Cluster, Query.Database, Config, ModuleResolverState->ModuleResolver, AppData(ctx)->FunctionRegistry, false); IKqpHost::TPrepareSettings prepareSettings; @@ -217,7 +217,7 @@ private: } void ReplyError(Ydb::StatusIds::StatusCode status, const TIssues& issues, const TActorContext& ctx) { - Reply(TKqpCompileResult::Make(Uid, std::move(Query), status, issues), ctx); + Reply(TKqpCompileResult::Make(Uid, std::move(Query), status, issues), ctx); } void InternalError(const TString message, const TActorContext &ctx) { @@ -260,7 +260,7 @@ private: AddMessageToReplayLog(kqpResult.QueryPlan); } - KqpCompileResult = TKqpCompileResult::Make(Uid, std::move(Query), status, kqpResult.Issues()); + KqpCompileResult = TKqpCompileResult::Make(Uid, std::move(Query), status, kqpResult.Issues()); if (status == Ydb::StatusIds::SUCCESS) { YQL_ENSURE(kqpResult.PreparingQuery); @@ -372,7 +372,7 @@ private: TActorId Owner; TIntrusivePtr<TModuleResolverState> ModuleResolverState; TIntrusivePtr<TKqpCounters> Counters; - TString Uid; + TString Uid; TKqpQueryId Query; TString UserToken; TKqpDbCountersPtr DbCounters; @@ -405,7 +405,7 @@ IActor* CreateKqpCompileActor(const TActorId& owner, const TKqpSettings::TConstP TIntrusivePtr<TKqpCounters> counters, const TString& uid, const TKqpQueryId& query, const TString& userToken, TKqpDbCountersPtr dbCounters, bool recompileWithNewEngine) { - return new TKqpCompileActor(owner, kqpSettings, serviceConfig, moduleResolverState, counters, uid, + return new TKqpCompileActor(owner, kqpSettings, serviceConfig, moduleResolverState, counters, uid, std::move(query), userToken, dbCounters, recompileWithNewEngine); } diff --git a/ydb/core/kqp/kqp_compile_request.cpp b/ydb/core/kqp/kqp_compile_request.cpp index 75ab9db32de..2c18af159bc 100644 --- a/ydb/core/kqp/kqp_compile_request.cpp +++ b/ydb/core/kqp/kqp_compile_request.cpp @@ -75,13 +75,13 @@ public: auto& compileResult = *DeferredResponse->CompileResult; - LOG_INFO_S(ctx, NKikimrServices::KQP_COMPILE_REQUEST, "Recompiling query due to scheme error" + LOG_INFO_S(ctx, NKikimrServices::KQP_COMPILE_REQUEST, "Recompiling query due to scheme error" << ", self: " << ctx.SelfID << ", queryUid: " << compileResult.Uid); - auto recompileEv = MakeHolder<TEvKqp::TEvRecompileRequest>(UserToken, compileResult.Uid, compileResult.Query, + auto recompileEv = MakeHolder<TEvKqp::TEvRecompileRequest>(UserToken, compileResult.Uid, compileResult.Query, Deadline, DbCounters); - ctx.Send(MakeKqpCompileServiceID(ctx.SelfID.NodeId()), recompileEv.Release()); + ctx.Send(MakeKqpCompileServiceID(ctx.SelfID.NodeId()), recompileEv.Release()); DeferredResponse.Reset(); } diff --git a/ydb/core/kqp/kqp_compile_service.cpp b/ydb/core/kqp/kqp_compile_service.cpp index acd62b60b95..da886224a2a 100644 --- a/ydb/core/kqp/kqp_compile_service.cpp +++ b/ydb/core/kqp/kqp_compile_service.cpp @@ -23,9 +23,9 @@ using namespace NYql; class TKqpQueryCache { public: - TKqpQueryCache(size_t size, TDuration ttl) - : List(size) - , Ttl(ttl) {} + TKqpQueryCache(size_t size, TDuration ttl) + : List(size) + , Ttl(ttl) {} bool Insert(const TKqpCompileResult::TConstPtr& compileResult) { Y_ENSURE(compileResult->Query); @@ -34,26 +34,26 @@ public: auto queryIt = QueryIndex.emplace(query, compileResult->Uid); Y_ENSURE(queryIt.second); - auto it = Index.emplace(compileResult->Uid, TCacheEntry{compileResult, - TAppData::TimeProvider->Now() + Ttl}); + auto it = Index.emplace(compileResult->Uid, TCacheEntry{compileResult, + TAppData::TimeProvider->Now() + Ttl}); Y_VERIFY(it.second); TItem* item = &const_cast<TItem&>(*it.first); auto removedItem = List.Insert(item); IncBytes(item->Value.CompileResult->PreparedQuery->ByteSize()); - if (item->Value.CompileResult->PreparedQueryNewEngine) { - IncBytes(item->Value.CompileResult->PreparedQueryNewEngine->ByteSize()); + if (item->Value.CompileResult->PreparedQueryNewEngine) { + IncBytes(item->Value.CompileResult->PreparedQueryNewEngine->ByteSize()); } if (removedItem) { DecBytes(removedItem->Value.CompileResult->PreparedQuery->ByteSize()); - if (removedItem->Value.CompileResult->PreparedQueryNewEngine) { - DecBytes(removedItem->Value.CompileResult->PreparedQueryNewEngine->ByteSize()); + if (removedItem->Value.CompileResult->PreparedQueryNewEngine) { + DecBytes(removedItem->Value.CompileResult->PreparedQueryNewEngine->ByteSize()); } - QueryIndex.erase(*removedItem->Value.CompileResult->Query); + QueryIndex.erase(*removedItem->Value.CompileResult->Query); Index.erase(removedItem->Key); } @@ -68,24 +68,24 @@ public: if (it != Index.end()) { TItem* item = &const_cast<TItem&>(*it); if (promote) { - item->Value.ExpiredAt = TAppData::TimeProvider->Now() + Ttl; + item->Value.ExpiredAt = TAppData::TimeProvider->Now() + Ttl; List.Promote(item); } - return item->Value.CompileResult; + return item->Value.CompileResult; } return nullptr; } - void Replace(const TKqpCompileResult::TConstPtr& compileResult) { - auto it = Index.find(TItem(compileResult->Uid)); - if (it != Index.end()) { - TItem& item = const_cast<TItem&>(*it); + void Replace(const TKqpCompileResult::TConstPtr& compileResult) { + auto it = Index.find(TItem(compileResult->Uid)); + if (it != Index.end()) { + TItem& item = const_cast<TItem&>(*it); item.Value.CompileResult = compileResult; - } - } - + } + } + TKqpCompileResult::TConstPtr FindByQuery(const TKqpQueryId& query, bool promote) { auto uid = QueryIndex.FindPtr(query); if (!uid) { @@ -105,13 +105,13 @@ public: List.Erase(item); DecBytes(item->Value.CompileResult->PreparedQuery->ByteSize()); - if (item->Value.CompileResult->PreparedQueryNewEngine) { - DecBytes(item->Value.CompileResult->PreparedQueryNewEngine->ByteSize()); + if (item->Value.CompileResult->PreparedQueryNewEngine) { + DecBytes(item->Value.CompileResult->PreparedQueryNewEngine->ByteSize()); } - Y_VERIFY(item->Value.CompileResult); - Y_VERIFY(item->Value.CompileResult->Query); - QueryIndex.erase(*item->Value.CompileResult->Query); + Y_VERIFY(item->Value.CompileResult); + Y_VERIFY(item->Value.CompileResult->Query); + QueryIndex.erase(*item->Value.CompileResult->Query); Index.erase(it); @@ -128,19 +128,19 @@ public: return ByteSize; } - size_t EraseExpiredQueries() { - auto prevSize = Size(); - - auto now = TAppData::TimeProvider->Now(); - while (List.GetSize() && List.GetOldest()->Value.ExpiredAt <= now) { - EraseByUid(List.GetOldest()->Key); - } - - Y_VERIFY(List.GetSize() == Index.size()); - Y_VERIFY(List.GetSize() == QueryIndex.size()); - return prevSize - Size(); - } - + size_t EraseExpiredQueries() { + auto prevSize = Size(); + + auto now = TAppData::TimeProvider->Now(); + while (List.GetSize() && List.GetOldest()->Value.ExpiredAt <= now) { + EraseByUid(List.GetOldest()->Key); + } + + Y_VERIFY(List.GetSize() == Index.size()); + Y_VERIFY(List.GetSize() == QueryIndex.size()); + return prevSize - Size(); + } + void Clear() { List = TList(List.GetMaxSize()); Index.clear(); @@ -162,12 +162,12 @@ private: } private: - struct TCacheEntry { - TKqpCompileResult::TConstPtr CompileResult; - TInstant ExpiredAt; - }; - - using TList = TLRUList<TString, TCacheEntry>; + struct TCacheEntry { + TKqpCompileResult::TConstPtr CompileResult; + TInstant ExpiredAt; + }; + + using TList = TLRUList<TString, TCacheEntry>; using TItem = TList::TItem; private: @@ -175,15 +175,15 @@ private: THashSet<TItem, TItem::THash> Index; THashMap<TKqpQueryId, TString, THash<TKqpQueryId>> QueryIndex; ui64 ByteSize = 0; - TDuration Ttl; + TDuration Ttl; }; struct TKqpCompileRequest { - TKqpCompileRequest(const TActorId& sender, const TString& uid, TKqpQueryId query, bool keepInCache, + TKqpCompileRequest(const TActorId& sender, const TString& uid, TKqpQueryId query, bool keepInCache, const TString& userToken, const TInstant& deadline, TKqpDbCountersPtr dbCounters) : Sender(sender) , Query(std::move(query)) - , Uid(uid) + , Uid(uid) , KeepInCache(keepInCache) , UserToken(userToken) , Deadline(deadline) @@ -191,7 +191,7 @@ struct TKqpCompileRequest { TActorId Sender; TKqpQueryId Query; - TString Uid; + TString Uid; bool KeepInCache = false; TString UserToken; TInstant Deadline; @@ -305,7 +305,7 @@ public: , KqpSettings(kqpSettings) , ModuleResolverState(moduleResolverState) , Counters(counters) - , QueryCache(Config.GetCompileQueryCacheSize(), TDuration::Seconds(Config.GetCompileQueryCacheTTLSec())) + , QueryCache(Config.GetCompileQueryCacheSize(), TDuration::Seconds(Config.GetCompileQueryCacheTTLSec())) , RequestsQueue(Config.GetCompileRequestQueueSize()) , QueryReplayFactory(std::move(queryReplayFactory)) {} @@ -321,9 +321,9 @@ public: IEventHandle::FlagTrackDelivery); Become(&TKqpCompileService::MainState); - if (Config.GetCompileQueryCacheTTLSec()) { - StartCheckQueriesTtlTimer(ctx); - } + if (Config.GetCompileQueryCacheTTLSec()) { + StartCheckQueriesTtlTimer(ctx); + } } private: @@ -332,13 +332,13 @@ private: HFunc(TEvKqp::TEvCompileRequest, Handle); HFunc(TEvKqp::TEvCompileResponse, Handle); HFunc(TEvKqp::TEvCompileInvalidateRequest, Handle); - HFunc(TEvKqp::TEvRecompileRequest, Handle); + HFunc(TEvKqp::TEvRecompileRequest, Handle); hFunc(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse, HandleConfig); hFunc(NConsole::TEvConsole::TEvConfigNotificationRequest, HandleConfig); hFunc(TEvents::TEvUndelivered, HandleUndelivery); - CFunc(TEvents::TSystem::Wakeup, HandleTimeout); + CFunc(TEvents::TSystem::Wakeup, HandleTimeout); cFunc(TEvents::TEvPoison::EventType, PassAway); default: Y_FAIL("TKqpCompileService: unexpected event 0x%08" PRIx32, ev->GetTypeRewrite()); @@ -480,7 +480,7 @@ private: Counters->ReportQueryCacheHit(dbCounters, false); - TKqpCompileRequest compileRequest(ev->Sender, CreateGuidAsString(), std::move(*request.Query), + TKqpCompileRequest compileRequest(ev->Sender, CreateGuidAsString(), std::move(*request.Query), request.KeepInCache, request.UserToken, request.Deadline, dbCounters); if (!RequestsQueue.Enqueue(std::move(compileRequest))) { @@ -503,61 +503,61 @@ private: ProcessQueue(ctx); } - void Handle(TEvKqp::TEvRecompileRequest::TPtr& ev, const TActorContext& ctx) { - try { - PerformRequest(ev, ctx); - } - catch (const std::exception& e) { - LogException("TEvRecompileRequest", ev->Sender, e, ctx); - ReplyInternalError(ev->Sender, "", e.what(), ctx); - } - } - - void PerformRequest(TEvKqp::TEvRecompileRequest::TPtr& ev, const TActorContext& ctx) { - auto& request = *ev->Get(); - - LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Received recompile request" - << ", sender: " << ev->Sender); - + void Handle(TEvKqp::TEvRecompileRequest::TPtr& ev, const TActorContext& ctx) { + try { + PerformRequest(ev, ctx); + } + catch (const std::exception& e) { + LogException("TEvRecompileRequest", ev->Sender, e, ctx); + ReplyInternalError(ev->Sender, "", e.what(), ctx); + } + } + + void PerformRequest(TEvKqp::TEvRecompileRequest::TPtr& ev, const TActorContext& ctx) { + auto& request = *ev->Get(); + + LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Received recompile request" + << ", sender: " << ev->Sender); + auto dbCounters = request.DbCounters; Counters->ReportRecompileRequestGet(dbCounters); - - auto compileResult = QueryCache.FindByUid(request.Uid, false); - if (compileResult || request.Query) { + + auto compileResult = QueryCache.FindByUid(request.Uid, false); + if (compileResult || request.Query) { Counters->ReportCompileRequestCompile(dbCounters); - - TKqpCompileRequest compileRequest(ev->Sender, request.Uid, compileResult ? *compileResult->Query : *request.Query, + + TKqpCompileRequest compileRequest(ev->Sender, request.Uid, compileResult ? *compileResult->Query : *request.Query, true, request.UserToken, request.Deadline, dbCounters); - - if (!RequestsQueue.Enqueue(std::move(compileRequest))) { + + if (!RequestsQueue.Enqueue(std::move(compileRequest))) { Counters->ReportCompileRequestRejected(dbCounters); - - LOG_WARN_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Requests queue size limit exceeded" - << ", sender: " << ev->Sender - << ", queueSize: " << RequestsQueue.Size()); - - NYql::TIssue issue(NYql::TPosition(), TStringBuilder() << - "Exceeded maximum number of requests in compile service queue."); - ReplyError(ev->Sender, "", Ydb::StatusIds::OVERLOADED, {issue}, ctx); - return; - } - } else { - LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Query not found" - << ", sender: " << ev->Sender - << ", queryUid: " << request.Uid); - - NYql::TIssue issue(NYql::TPosition(), TStringBuilder() << "Query not found: " << request.Uid); - ReplyError(ev->Sender, request.Uid, Ydb::StatusIds::NOT_FOUND, {issue}, ctx); - return; - } - - LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Added request to queue" - << ", sender: " << ev->Sender - << ", queueSize: " << RequestsQueue.Size()); - - ProcessQueue(ctx); - } - + + LOG_WARN_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Requests queue size limit exceeded" + << ", sender: " << ev->Sender + << ", queueSize: " << RequestsQueue.Size()); + + NYql::TIssue issue(NYql::TPosition(), TStringBuilder() << + "Exceeded maximum number of requests in compile service queue."); + ReplyError(ev->Sender, "", Ydb::StatusIds::OVERLOADED, {issue}, ctx); + return; + } + } else { + LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Query not found" + << ", sender: " << ev->Sender + << ", queryUid: " << request.Uid); + + NYql::TIssue issue(NYql::TPosition(), TStringBuilder() << "Query not found: " << request.Uid); + ReplyError(ev->Sender, request.Uid, Ydb::StatusIds::NOT_FOUND, {issue}, ctx); + return; + } + + LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Added request to queue" + << ", sender: " << ev->Sender + << ", queueSize: " << RequestsQueue.Size()); + + ProcessQueue(ctx); + } + void Handle(TEvKqp::TEvCompileResponse::TPtr& ev, const TActorContext& ctx) { auto compileActorId = ev->Sender; auto& compileResult = ev->Get()->CompileResult; @@ -567,7 +567,7 @@ private: auto compileRequest = RequestsQueue.FinishActiveRequest(*compileResult->Query); Y_VERIFY(compileRequest.CompileActor == compileActorId); - Y_VERIFY(compileRequest.Uid == compileResult->Uid); + Y_VERIFY(compileRequest.Uid == compileResult->Uid); LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Received response" << ", sender: " << compileRequest.Sender @@ -576,9 +576,9 @@ private: try { if (compileResult->Status == Ydb::StatusIds::SUCCESS) { - if (QueryCache.FindByUid(compileResult->Uid, false)) { - QueryCache.Replace(compileResult); - } else if (compileRequest.KeepInCache) { + if (QueryCache.FindByUid(compileResult->Uid, false)) { + QueryCache.Replace(compileResult); + } else if (compileRequest.KeepInCache) { if (QueryCache.Insert(compileResult)) { Counters->CompileQueryCacheEvicted->Inc(); } @@ -592,10 +592,10 @@ private: for (auto& request : requests) { Reply(request.Sender, compileResult, compileStats, ctx); } - } else { - if (QueryCache.FindByUid(compileResult->Uid, false)) { - QueryCache.EraseByUid(compileResult->Uid); - } + } else { + if (QueryCache.FindByUid(compileResult->Uid, false)) { + QueryCache.EraseByUid(compileResult->Uid); + } } Reply(compileRequest.Sender, compileResult, compileStats, ctx); @@ -631,17 +631,17 @@ private: QueryCache.EraseByUid(request.Uid); } - void HandleTimeout(const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Received check queries TTL timeout"); - - auto evicted = QueryCache.EraseExpiredQueries(); - if (evicted != 0) { - Counters->CompileQueryCacheEvicted->Add(evicted); - } - - StartCheckQueriesTtlTimer(ctx); - } - + void HandleTimeout(const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::KQP_COMPILE_SERVICE, "Received check queries TTL timeout"); + + auto evicted = QueryCache.EraseExpiredQueries(); + if (evicted != 0) { + Counters->CompileQueryCacheEvicted->Add(evicted); + } + + StartCheckQueriesTtlTimer(ctx); + } + private: void ProcessQueue(const TActorContext& ctx) { auto maxActiveRequests = Config.GetCompileMaxActiveRequests(); @@ -686,11 +686,11 @@ private: RequestsQueue.AddActiveRequest(std::move(request)); } - void StartCheckQueriesTtlTimer(const TActorContext& ctx) { - CheckQueriesTtlTimer = CreateLongTimer(ctx, TDuration::Seconds(Config.GetCompileQueryCacheTTLSec()), - new IEventHandle(ctx.SelfID, ctx.SelfID, new TEvents::TEvWakeup())); - } - + void StartCheckQueriesTtlTimer(const TActorContext& ctx) { + CheckQueriesTtlTimer = CreateLongTimer(ctx, TDuration::Seconds(Config.GetCompileQueryCacheTTLSec()), + new IEventHandle(ctx.SelfID, ctx.SelfID, new TEvents::TEvWakeup())); + } + void Reply(const TActorId& sender, const TKqpCompileResult::TConstPtr& compileResult, const NKqpProto::TKqpStatsCompile& compileStats, const TActorContext& ctx) { @@ -752,7 +752,7 @@ private: TKqpQueryCache QueryCache; TKqpRequestsQueue RequestsQueue; - TActorId CheckQueriesTtlTimer; + TActorId CheckQueriesTtlTimer; std::shared_ptr<IQueryReplayBackendFactory> QueryReplayFactory; }; diff --git a/ydb/core/kqp/kqp_ic_gateway.cpp b/ydb/core/kqp/kqp_ic_gateway.cpp index 23cb3c339b5..47489a9bd54 100644 --- a/ydb/core/kqp/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/kqp_ic_gateway.cpp @@ -1614,30 +1614,30 @@ public: return ExecutePhysicalQueryInternal(std::move(request), target, false); } - TFuture<TExecPhysicalResult> ExecutePure(TExecPhysicalRequest&& request, const NActors::TActorId& target) override { - YQL_ENSURE(!request.Transactions.empty()); - YQL_ENSURE(request.Locks.empty()); - - auto containOnlyPureStages = [](const auto& request) { - for (const auto& tx : request.Transactions) { - if (tx.Body.GetType() != NKqpProto::TKqpPhyTx::TYPE_COMPUTE) { - return false; - } - - for (const auto& stage : tx.Body.GetStages()) { - if (stage.InputsSize() != 0) { - return false; - } - } - } - - return true; - }; - - YQL_ENSURE(containOnlyPureStages(request)); - return ExecutePhysicalQueryInternal(std::move(request), target, false); - } - + TFuture<TExecPhysicalResult> ExecutePure(TExecPhysicalRequest&& request, const NActors::TActorId& target) override { + YQL_ENSURE(!request.Transactions.empty()); + YQL_ENSURE(request.Locks.empty()); + + auto containOnlyPureStages = [](const auto& request) { + for (const auto& tx : request.Transactions) { + if (tx.Body.GetType() != NKqpProto::TKqpPhyTx::TYPE_COMPUTE) { + return false; + } + + for (const auto& stage : tx.Body.GetStages()) { + if (stage.InputsSize() != 0) { + return false; + } + } + } + + return true; + }; + + YQL_ENSURE(containOnlyPureStages(request)); + return ExecutePhysicalQueryInternal(std::move(request), target, false); + } + TFuture<TQueryResult> ExecScanQueryAst(const TString& cluster, const TString& query, TKqpParamsMap&& params, const TAstQuerySettings& settings, ui64 rowsLimit) override { @@ -2210,17 +2210,17 @@ private: { tableDesc.SetName(name); - Y_ENSURE(metadata->ColumnOrder.size() == metadata->Columns.size()); - for (const auto& name : metadata->ColumnOrder) { - auto columnIt = metadata->Columns.find(name); - Y_ENSURE(columnIt != metadata->Columns.end()); + Y_ENSURE(metadata->ColumnOrder.size() == metadata->Columns.size()); + for (const auto& name : metadata->ColumnOrder) { + auto columnIt = metadata->Columns.find(name); + Y_ENSURE(columnIt != metadata->Columns.end()); TColumnDescription& columnDesc = *tableDesc.AddColumns(); - columnDesc.SetName(columnIt->second.Name); - columnDesc.SetType(columnIt->second.Type); - columnDesc.SetNotNull(columnIt->second.NotNull); - if (columnIt->second.Families) { - columnDesc.SetFamilyName(*columnIt->second.Families.begin()); + columnDesc.SetName(columnIt->second.Name); + columnDesc.SetType(columnIt->second.Type); + columnDesc.SetNotNull(columnIt->second.NotNull); + if (columnIt->second.Families) { + columnDesc.SetFamilyName(*columnIt->second.Families.begin()); } } diff --git a/ydb/core/kqp/kqp_metadata_loader.cpp b/ydb/core/kqp/kqp_metadata_loader.cpp index 73942b36c49..167b8df3c42 100644 --- a/ydb/core/kqp/kqp_metadata_loader.cpp +++ b/ydb/core/kqp/kqp_metadata_loader.cpp @@ -146,19 +146,19 @@ TTableMetadataResult GetLoadTableMetadataResult(const NSchemeCache::TSchemeCache tableMeta->Attributes = entry.Attributes; std::map<ui32, TString, std::less<ui32>> keyColumns; - std::map<ui32, TString, std::less<ui32>> columnOrder; + std::map<ui32, TString, std::less<ui32>> columnOrder; for (auto& pair : entry.Columns) { const auto& columnDesc = pair.second; TString typeName; YQL_ENSURE(NScheme::TryGetTypeName(columnDesc.PType, typeName)); - auto notNull = entry.NotNullColumns.contains(columnDesc.Name); - tableMeta->Columns.emplace(columnDesc.Name, NYql::TKikimrColumnMetadata( - columnDesc.Name, columnDesc.Id, typeName, notNull, columnDesc.PType)); + auto notNull = entry.NotNullColumns.contains(columnDesc.Name); + tableMeta->Columns.emplace(columnDesc.Name, NYql::TKikimrColumnMetadata( + columnDesc.Name, columnDesc.Id, typeName, notNull, columnDesc.PType)); if (columnDesc.KeyOrder >= 0) { keyColumns[columnDesc.KeyOrder] = columnDesc.Name; } - columnOrder[columnDesc.Id] = columnDesc.Name; + columnOrder[columnDesc.Id] = columnDesc.Name; } tableMeta->KeyColumnNames.reserve(keyColumns.size()); @@ -166,11 +166,11 @@ TTableMetadataResult GetLoadTableMetadataResult(const NSchemeCache::TSchemeCache tableMeta->KeyColumnNames.push_back(pair.second); } - tableMeta->ColumnOrder.reserve(columnOrder.size()); - for (const auto& [_, column] : columnOrder) { - tableMeta->ColumnOrder.push_back(column); - } - + tableMeta->ColumnOrder.reserve(columnOrder.size()); + for (const auto& [_, column] : columnOrder) { + tableMeta->ColumnOrder.push_back(column); + } + IndexProtoToMetadata(entry.Indexes, tableMeta); return result; diff --git a/ydb/core/kqp/kqp_response.cpp b/ydb/core/kqp/kqp_response.cpp index e47cae3aafc..d3859cd39c7 100644 --- a/ydb/core/kqp/kqp_response.cpp +++ b/ydb/core/kqp/kqp_response.cpp @@ -39,7 +39,7 @@ TMaybe<Ydb::StatusIds::StatusCode> GetYdbStatus(const TIssue& issue) { case TIssuesIds::KIKIMR_BAD_REQUEST: case TIssuesIds::KIKIMR_BAD_COLUMN_TYPE: - case TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE: + case TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE: return Ydb::StatusIds::BAD_REQUEST; case TIssuesIds::CORE_GC_STRINGS_LIMIT_EXCEEDED: diff --git a/ydb/core/kqp/kqp_worker_actor.cpp b/ydb/core/kqp/kqp_worker_actor.cpp index d6776b6bd2f..cccb8a51e8e 100644 --- a/ydb/core/kqp/kqp_worker_actor.cpp +++ b/ydb/core/kqp/kqp_worker_actor.cpp @@ -21,7 +21,7 @@ #include <library/cpp/actors/core/event_pb.h> #include <library/cpp/actors/core/hfunc.h> #include <library/cpp/actors/core/log.h> -#include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_reader.h> #include <util/string/escape.h> @@ -116,12 +116,12 @@ struct TKqpCleanupState { TIntrusivePtr<IKqpHost::IAsyncQueryResult> AsyncResult; }; -enum ETableReadType { - Other = 0, - Scan = 1, - FullScan = 2, -}; - +enum ETableReadType { + Other = 0, + Scan = 1, + FullScan = 2, +}; + EKikimrStatsMode GetStatsMode(const NKikimrKqp::TQueryRequest& queryRequest, EKikimrStatsMode minMode) { if (queryRequest.GetProfile()) { // TODO: Deprecate, StatsMode is the new way to enable stats. @@ -187,7 +187,7 @@ public: Config->FeatureFlags = AppData(ctx)->FeatureFlags; - KqpHost = CreateKqpHost(Gateway, Settings.Cluster, Settings.Database, Config, ModuleResolverState->ModuleResolver, + KqpHost = CreateKqpHost(Gateway, Settings.Cluster, Settings.Database, Config, ModuleResolverState->ModuleResolver, AppData(ctx)->FunctionRegistry, !Settings.LongSession); Become(&TKqpWorkerActor::ReadyState); @@ -1573,38 +1573,38 @@ private: return Reply(std::move(responseEv), ctx); } - ETableReadType ExtractMostHeavyReadType(const TString& queryPlan) { - ETableReadType maxReadType = ETableReadType::Other; - - if (queryPlan.empty()) { - return maxReadType; - } - - NJson::TJsonValue root; - NJson::ReadJsonTree(queryPlan, &root, false); - - if (root.Has("tables")) { - for (const auto& table : root["tables"].GetArray()) { - if (!table.Has("reads")) { - continue; - } - - for (const auto& read : table["reads"].GetArray()) { - Y_VERIFY(read.Has("type")); - const auto& type = read["type"].GetString(); - - if (type == "Scan") { - maxReadType = Max(maxReadType, ETableReadType::Scan); - } else if (type == "FullScan") { - return ETableReadType::FullScan; - } - } - } - } - - return maxReadType; - } - + ETableReadType ExtractMostHeavyReadType(const TString& queryPlan) { + ETableReadType maxReadType = ETableReadType::Other; + + if (queryPlan.empty()) { + return maxReadType; + } + + NJson::TJsonValue root; + NJson::ReadJsonTree(queryPlan, &root, false); + + if (root.Has("tables")) { + for (const auto& table : root["tables"].GetArray()) { + if (!table.Has("reads")) { + continue; + } + + for (const auto& read : table["reads"].GetArray()) { + Y_VERIFY(read.Has("type")); + const auto& type = read["type"].GetString(); + + if (type == "Scan") { + maxReadType = Max(maxReadType, ETableReadType::Scan); + } else if (type == "FullScan") { + return ETableReadType::FullScan; + } + } + } + } + + return maxReadType; + } + bool ReplyQueryResult(const TActorContext& ctx) { Y_VERIFY(QueryState); auto& queryRequest = QueryState->Request; @@ -1622,31 +1622,31 @@ private: if (status == Ydb::StatusIds::SUCCESS) { Counters->ReportQueryLatency(Settings.DbCounters, queryRequest.GetAction(), queryDuration); - auto maxReadType = ExtractMostHeavyReadType(queryResult.QueryPlan); - if (maxReadType == ETableReadType::FullScan) { + auto maxReadType = ExtractMostHeavyReadType(queryResult.QueryPlan); + if (maxReadType == ETableReadType::FullScan) { Counters->ReportQueryWithFullScan(Settings.DbCounters); - } else if (maxReadType == ETableReadType::Scan) { + } else if (maxReadType == ETableReadType::Scan) { Counters->ReportQueryWithRangeScan(Settings.DbCounters); - } + } ui32 affectedShardsCount = 0; - ui64 readBytesCount = 0; - ui64 readRowsCount = 0; - for (const auto& exec : queryResult.QueryStats.GetExecutions()) { - for (const auto& table : exec.GetTables()) { + ui64 readBytesCount = 0; + ui64 readRowsCount = 0; + for (const auto& exec : queryResult.QueryStats.GetExecutions()) { + for (const auto& table : exec.GetTables()) { affectedShardsCount = std::max(affectedShardsCount, table.GetAffectedPartitions()); - readBytesCount += table.GetReadBytes(); - readRowsCount += table.GetReadRows(); - } - } - + readBytesCount += table.GetReadBytes(); + readRowsCount += table.GetReadRows(); + } + } + Counters->ReportQueryAffectedShards(Settings.DbCounters, affectedShardsCount); Counters->ReportQueryReadRows(Settings.DbCounters, readRowsCount); Counters->ReportQueryReadBytes(Settings.DbCounters, readBytesCount); Counters->ReportQueryReadSets(Settings.DbCounters, queryResult.QueryStats.GetReadSetsCount()); Counters->ReportQueryMaxShardReplySize(Settings.DbCounters, queryResult.QueryStats.GetMaxShardReplySize()); Counters->ReportQueryMaxShardProgramSize(Settings.DbCounters, queryResult.QueryStats.GetMaxShardProgramSize()); - + if (QueryState->QueryCompileResult && QueryState->QueryCompileResult->PreparedQueryNewEngine && QueryState->NewEngineCompatibleQuery) { diff --git a/ydb/core/kqp/opt/kqp_opt_build_txs.cpp b/ydb/core/kqp/opt/kqp_opt_build_txs.cpp index 61f7b23e9a3..f35ba23f666 100644 --- a/ydb/core/kqp/opt/kqp_opt_build_txs.cpp +++ b/ydb/core/kqp/opt/kqp_opt_build_txs.cpp @@ -414,7 +414,7 @@ public: .AddPostTypeAnnotation(/* forSubgraph */ true) .Add(CreateKqpBuildPhyStagesTransformer(config->SpillingEnabled()), "BuildPhysicalStages") .Add(*BuildTxTransformer, "BuildPhysicalTx") - .Add(CreateKqpTxPeepholeTransformer(TypeAnnTransformer.Get(), typesCtx, config, /* withFinalStageRules */ false), "Peephole") + .Add(CreateKqpTxPeepholeTransformer(TypeAnnTransformer.Get(), typesCtx, config, /* withFinalStageRules */ false), "Peephole") .Build(false); } diff --git a/ydb/core/kqp/opt/peephole/kqp_opt_peephole.cpp b/ydb/core/kqp/opt/peephole/kqp_opt_peephole.cpp index 157d2db22c9..55aca64df87 100644 --- a/ydb/core/kqp/opt/peephole/kqp_opt_peephole.cpp +++ b/ydb/core/kqp/opt/peephole/kqp_opt_peephole.cpp @@ -100,17 +100,17 @@ private: TStatus PeepHoleOptimize(const TExprBase& program, TExprNode::TPtr& newProgram, TExprContext& ctx, IGraphTransformer& typeAnnTransformer, TTypeAnnotationContext& typesCtx, TKikimrConfiguration::TPtr config, - bool allowNonDeterministicFunctions, bool withFinalStageRules) + bool allowNonDeterministicFunctions, bool withFinalStageRules) { TKqpPeepholePipelineConfigurator kqpPeephole(config); - TPeepholeSettings peepholeSettings; - peepholeSettings.CommonConfig = &kqpPeephole; - peepholeSettings.WithFinalStageRules = withFinalStageRules; - peepholeSettings.WithNonDeterministicRules = false; + TPeepholeSettings peepholeSettings; + peepholeSettings.CommonConfig = &kqpPeephole; + peepholeSettings.WithFinalStageRules = withFinalStageRules; + peepholeSettings.WithNonDeterministicRules = false; bool hasNonDeterministicFunctions; auto status = PeepHoleOptimizeNode<true>(program.Ptr(), newProgram, ctx, typesCtx, &typeAnnTransformer, - hasNonDeterministicFunctions, peepholeSettings); + hasNonDeterministicFunctions, peepholeSettings); if (status == TStatus::Error) { return status; } @@ -125,12 +125,12 @@ TStatus PeepHoleOptimize(const TExprBase& program, TExprNode::TPtr& newProgram, TMaybeNode<TKqpPhysicalTx> PeepholeOptimize(const TKqpPhysicalTx& tx, TExprContext& ctx, IGraphTransformer& typeAnnTransformer, TTypeAnnotationContext& typesCtx, THashSet<ui64>& optimizedStages, - TKikimrConfiguration::TPtr config, bool withFinalStageRules) + TKikimrConfiguration::TPtr config, bool withFinalStageRules) { TVector<TDqPhyStage> stages; stages.reserve(tx.Stages().Size()); TNodeOnNodeOwnedMap stagesMap; - TVector<TKqpParamBinding> bindings(tx.ParamBindings().begin(), tx.ParamBindings().end()); + TVector<TKqpParamBinding> bindings(tx.ParamBindings().begin(), tx.ParamBindings().end()); for (const auto& stage : tx.Stages()) { YQL_ENSURE(!optimizedStages.contains(stage.Ref().UniqueId())); @@ -153,22 +153,22 @@ TMaybeNode<TKqpPhysicalTx> PeepholeOptimize(const TKqpPhysicalTx& tx, TExprConte TExprNode::TPtr newProgram; auto status = PeepHoleOptimize(program, newProgram, ctx, typeAnnTransformer, typesCtx, config, - allowNonDeterministicFunctions, withFinalStageRules); + allowNonDeterministicFunctions, withFinalStageRules); if (status != TStatus::Ok) { ctx.AddError(TIssue(ctx.GetPosition(stage.Pos()), "Peephole optimization failed for KQP transaction")); return {}; } - if (allowNonDeterministicFunctions) { - status = ReplaceNonDetFunctionsWithParams(newProgram, ctx, &bindings); - - if (status != TStatus::Ok) { - ctx.AddError(TIssue(ctx.GetPosition(stage.Pos()), - "Failed to replace non deterministic functions with params for KQP transaction")); - return {}; - } - } - + if (allowNonDeterministicFunctions) { + status = ReplaceNonDetFunctionsWithParams(newProgram, ctx, &bindings); + + if (status != TStatus::Ok) { + ctx.AddError(TIssue(ctx.GetPosition(stage.Pos()), + "Failed to replace non deterministic functions with params for KQP transaction")); + return {}; + } + } + auto newStage = Build<TDqPhyStage>(ctx, stage.Pos()) .Inputs(ctx.ReplaceNodes(stage.Inputs().Ptr(), stagesMap)) .Program(ctx.DeepCopyLambda(TKqpProgram(newProgram).Lambda().Ref())) @@ -186,7 +186,7 @@ TMaybeNode<TKqpPhysicalTx> PeepholeOptimize(const TKqpPhysicalTx& tx, TExprConte .Add(stages) .Build() .Results(ctx.ReplaceNodes(tx.Results().Ptr(), stagesMap)) - .ParamBindings().Add(bindings).Build() + .ParamBindings().Add(bindings).Build() .Settings(tx.Settings()) .Done(); } @@ -194,11 +194,11 @@ TMaybeNode<TKqpPhysicalTx> PeepholeOptimize(const TKqpPhysicalTx& tx, TExprConte class TKqpTxPeepholeTransformer : public TSyncTransformerBase { public: TKqpTxPeepholeTransformer(IGraphTransformer* typeAnnTransformer, - TTypeAnnotationContext& typesCtx, TKikimrConfiguration::TPtr config, bool withFinalStageRules) + TTypeAnnotationContext& typesCtx, TKikimrConfiguration::TPtr config, bool withFinalStageRules) : TypeAnnTransformer(typeAnnTransformer) , TypesCtx(typesCtx) , Config(config) - , WithFinalStageRules(withFinalStageRules) + , WithFinalStageRules(withFinalStageRules) {} TStatus DoTransform(TExprNode::TPtr inputExpr, TExprNode::TPtr& outputExpr, TExprContext& ctx) final { @@ -216,7 +216,7 @@ public: auto tx = input.Cast<TKqpPhysicalTx>(); THashSet<ui64> optimizedStages; - auto optimizedTx = PeepholeOptimize(tx, ctx, *TypeAnnTransformer, TypesCtx, optimizedStages, Config, WithFinalStageRules); + auto optimizedTx = PeepholeOptimize(tx, ctx, *TypeAnnTransformer, TypesCtx, optimizedStages, Config, WithFinalStageRules); if (!optimizedTx) { return TStatus::Error; @@ -237,147 +237,147 @@ private: TTypeAnnotationContext& TypesCtx; TKikimrConfiguration::TPtr Config; bool Optimized = false; - bool WithFinalStageRules = true; + bool WithFinalStageRules = true; +}; + +class TKqpTxsPeepholeTransformer : public TSyncTransformerBase { +public: + TKqpTxsPeepholeTransformer(TAutoPtr<NYql::IGraphTransformer> typeAnnTransformer, + TTypeAnnotationContext& typesCtx, TKikimrConfiguration::TPtr config) + : TypeAnnTransformer(std::move(typeAnnTransformer)) + { + TxTransformer = TTransformationPipeline(&typesCtx) + .AddServiceTransformers() + .Add(TLogExprTransformer::Sync("TxsPeephole", NLog::EComponent::ProviderKqp, NLog::ELevel::TRACE), "TxsPeephole") + .Add(*TypeAnnTransformer, "TypeAnnotation") + .AddPostTypeAnnotation(/* forSubgraph */ true) + .Add(CreateKqpTxPeepholeTransformer(TypeAnnTransformer.Get(), typesCtx, config), "Peephole") + .Build(false); + } + + TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { + + if (!TKqpPhysicalQuery::Match(input.Get())) { + return TStatus::Error; + } + + YQL_CLOG(DEBUG, ProviderKqp) << ">>> TKqpTxsPeepholeTransformer: " << KqpExprToPrettyString(*input, ctx); + + TKqpPhysicalQuery query(input); + + TVector<TKqpPhysicalTx> txs; + txs.reserve(query.Transactions().Size()); + for (const auto& tx : query.Transactions()) { + auto expr = TransformTx(tx, ctx); + txs.push_back(expr.Cast()); + } + + auto phyQuery = Build<TKqpPhysicalQuery>(ctx, query.Pos()) + .Transactions() + .Add(txs) + .Build() + .Results(query.Results()) + .Settings(query.Settings()) + .Done(); + + output = phyQuery.Ptr(); + return TStatus::Ok; + } + + void Rewind() final { + TSyncTransformerBase::Rewind(); + TxTransformer->Rewind(); + } + +private: + TMaybeNode<TKqpPhysicalTx> TransformTx(const TKqpPhysicalTx& tx, TExprContext& ctx) { + TxTransformer->Rewind(); + + auto expr = tx.Ptr(); + + while (true) { + auto status = InstantTransform(*TxTransformer, expr, ctx); + if (status == TStatus::Error) { + return {}; + } + if (status == TStatus::Ok) { + break; + } + } + return TKqpPhysicalTx(expr); + } + + TAutoPtr<IGraphTransformer> TxTransformer; + TAutoPtr<NYql::IGraphTransformer> TypeAnnTransformer; }; -class TKqpTxsPeepholeTransformer : public TSyncTransformerBase { -public: - TKqpTxsPeepholeTransformer(TAutoPtr<NYql::IGraphTransformer> typeAnnTransformer, - TTypeAnnotationContext& typesCtx, TKikimrConfiguration::TPtr config) - : TypeAnnTransformer(std::move(typeAnnTransformer)) - { - TxTransformer = TTransformationPipeline(&typesCtx) - .AddServiceTransformers() - .Add(TLogExprTransformer::Sync("TxsPeephole", NLog::EComponent::ProviderKqp, NLog::ELevel::TRACE), "TxsPeephole") - .Add(*TypeAnnTransformer, "TypeAnnotation") - .AddPostTypeAnnotation(/* forSubgraph */ true) - .Add(CreateKqpTxPeepholeTransformer(TypeAnnTransformer.Get(), typesCtx, config), "Peephole") - .Build(false); - } - - TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - - if (!TKqpPhysicalQuery::Match(input.Get())) { - return TStatus::Error; - } - - YQL_CLOG(DEBUG, ProviderKqp) << ">>> TKqpTxsPeepholeTransformer: " << KqpExprToPrettyString(*input, ctx); - - TKqpPhysicalQuery query(input); - - TVector<TKqpPhysicalTx> txs; - txs.reserve(query.Transactions().Size()); - for (const auto& tx : query.Transactions()) { - auto expr = TransformTx(tx, ctx); - txs.push_back(expr.Cast()); - } - - auto phyQuery = Build<TKqpPhysicalQuery>(ctx, query.Pos()) - .Transactions() - .Add(txs) - .Build() - .Results(query.Results()) - .Settings(query.Settings()) - .Done(); - - output = phyQuery.Ptr(); - return TStatus::Ok; - } - - void Rewind() final { - TSyncTransformerBase::Rewind(); - TxTransformer->Rewind(); - } - -private: - TMaybeNode<TKqpPhysicalTx> TransformTx(const TKqpPhysicalTx& tx, TExprContext& ctx) { - TxTransformer->Rewind(); - - auto expr = tx.Ptr(); - - while (true) { - auto status = InstantTransform(*TxTransformer, expr, ctx); - if (status == TStatus::Error) { - return {}; - } - if (status == TStatus::Ok) { - break; - } - } - return TKqpPhysicalTx(expr); - } - - TAutoPtr<IGraphTransformer> TxTransformer; - TAutoPtr<NYql::IGraphTransformer> TypeAnnTransformer; -}; - } // anonymous namespace -TAutoPtr<IGraphTransformer> CreateKqpTxPeepholeTransformer(NYql::IGraphTransformer* typeAnnTransformer, - TTypeAnnotationContext& typesCtx, const TKikimrConfiguration::TPtr& config, bool withFinalStageRules) -{ - return new TKqpTxPeepholeTransformer(typeAnnTransformer, typesCtx, config, withFinalStageRules); -} - -TAutoPtr<IGraphTransformer> CreateKqpTxsPeepholeTransformer(TAutoPtr<NYql::IGraphTransformer> typeAnnTransformer, +TAutoPtr<IGraphTransformer> CreateKqpTxPeepholeTransformer(NYql::IGraphTransformer* typeAnnTransformer, + TTypeAnnotationContext& typesCtx, const TKikimrConfiguration::TPtr& config, bool withFinalStageRules) +{ + return new TKqpTxPeepholeTransformer(typeAnnTransformer, typesCtx, config, withFinalStageRules); +} + +TAutoPtr<IGraphTransformer> CreateKqpTxsPeepholeTransformer(TAutoPtr<NYql::IGraphTransformer> typeAnnTransformer, TTypeAnnotationContext& typesCtx, const TKikimrConfiguration::TPtr& config) { - return new TKqpTxsPeepholeTransformer(std::move(typeAnnTransformer), typesCtx, config); + return new TKqpTxsPeepholeTransformer(std::move(typeAnnTransformer), typesCtx, config); +} + +TStatus ReplaceNonDetFunctionsWithParams(TExprNode::TPtr& input, TExprContext& ctx, TVector<TKqpParamBinding>* paramBindings) { + static const std::unordered_set<std::string_view> nonDeterministicFunctions = { + "RandomNumber", + "Random", + "RandomUuid", + "Now", + "CurrentUtcDate", + "CurrentUtcDatetime", + "CurrentUtcTimestamp" + }; + + TOptimizeExprSettings settings(nullptr); + settings.VisitChanges = true; + + TExprNode::TPtr output; + auto status = OptimizeExpr(input, output, [paramBindings](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { + if (auto maybeCallable = TMaybeNode<TCallable>(node)) { + auto callable = maybeCallable.Cast(); + if (nonDeterministicFunctions.contains(callable.CallableName()) && callable.Ref().ChildrenSize() == 0) { + const auto paramName = TStringBuilder() << ParamNamePrefix + << NNaming::CamelToSnakeCase(TString(callable.CallableName())); + + auto param = Build<TCoParameter>(ctx, node->Pos()) + .Name().Build(paramName) + .Type(ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)) + .Done(); + + if (paramBindings) { + auto binding = Build<TKqpTxInternalBinding>(ctx, node->Pos()) + .Type(ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)) + .Kind().Build(callable.CallableName()) + .Done(); + + auto paramBinding = Build<TKqpParamBinding>(ctx, param.Pos()) + .Name().Build(paramName) + .Binding(binding) + .Done(); + + paramBindings->emplace_back(std::move(paramBinding)); + } + + return param.Ptr(); + } + } + + return node; + }, ctx, settings); + + if (output) { + input = output; + } + + return status; } -TStatus ReplaceNonDetFunctionsWithParams(TExprNode::TPtr& input, TExprContext& ctx, TVector<TKqpParamBinding>* paramBindings) { - static const std::unordered_set<std::string_view> nonDeterministicFunctions = { - "RandomNumber", - "Random", - "RandomUuid", - "Now", - "CurrentUtcDate", - "CurrentUtcDatetime", - "CurrentUtcTimestamp" - }; - - TOptimizeExprSettings settings(nullptr); - settings.VisitChanges = true; - - TExprNode::TPtr output; - auto status = OptimizeExpr(input, output, [paramBindings](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { - if (auto maybeCallable = TMaybeNode<TCallable>(node)) { - auto callable = maybeCallable.Cast(); - if (nonDeterministicFunctions.contains(callable.CallableName()) && callable.Ref().ChildrenSize() == 0) { - const auto paramName = TStringBuilder() << ParamNamePrefix - << NNaming::CamelToSnakeCase(TString(callable.CallableName())); - - auto param = Build<TCoParameter>(ctx, node->Pos()) - .Name().Build(paramName) - .Type(ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)) - .Done(); - - if (paramBindings) { - auto binding = Build<TKqpTxInternalBinding>(ctx, node->Pos()) - .Type(ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)) - .Kind().Build(callable.CallableName()) - .Done(); - - auto paramBinding = Build<TKqpParamBinding>(ctx, param.Pos()) - .Name().Build(paramName) - .Binding(binding) - .Done(); - - paramBindings->emplace_back(std::move(paramBinding)); - } - - return param.Ptr(); - } - } - - return node; - }, ctx, settings); - - if (output) { - input = output; - } - - return status; -} - } // namespace NKikimr::NKqp::NOpt diff --git a/ydb/core/kqp/opt/peephole/kqp_opt_peephole.h b/ydb/core/kqp/opt/peephole/kqp_opt_peephole.h index 07b3c810f91..a0602bb07da 100644 --- a/ydb/core/kqp/opt/peephole/kqp_opt_peephole.h +++ b/ydb/core/kqp/opt/peephole/kqp_opt_peephole.h @@ -6,12 +6,12 @@ namespace NKikimr::NKqp::NOpt { TAutoPtr<NYql::IGraphTransformer> CreateKqpTxPeepholeTransformer(NYql::IGraphTransformer* typeAnnTransformer, - NYql::TTypeAnnotationContext& typesCtx, const NYql::TKikimrConfiguration::TPtr& config, bool withFinalStageRules = true); - -TAutoPtr<NYql::IGraphTransformer> CreateKqpTxsPeepholeTransformer(TAutoPtr<NYql::IGraphTransformer> typeAnnTransformer, + NYql::TTypeAnnotationContext& typesCtx, const NYql::TKikimrConfiguration::TPtr& config, bool withFinalStageRules = true); + +TAutoPtr<NYql::IGraphTransformer> CreateKqpTxsPeepholeTransformer(TAutoPtr<NYql::IGraphTransformer> typeAnnTransformer, NYql::TTypeAnnotationContext& typesCtx, const NYql::TKikimrConfiguration::TPtr& config); -NYql::IGraphTransformer::TStatus ReplaceNonDetFunctionsWithParams(NYql::TExprNode::TPtr& input, NYql::TExprContext& ctx, - TVector<NYql::NNodes::TKqpParamBinding>* paramBindings = nullptr); - +NYql::IGraphTransformer::TStatus ReplaceNonDetFunctionsWithParams(NYql::TExprNode::TPtr& input, NYql::TExprContext& ctx, + TVector<NYql::NNodes::TKqpParamBinding>* paramBindings = nullptr); + } // namespace NKikimr::NKqp::NOpt diff --git a/ydb/core/kqp/prepare/kqp_query_exec.cpp b/ydb/core/kqp/prepare/kqp_query_exec.cpp index 6b5d37c0115..bc2d4e94ce9 100644 --- a/ydb/core/kqp/prepare/kqp_query_exec.cpp +++ b/ydb/core/kqp/prepare/kqp_query_exec.cpp @@ -402,17 +402,17 @@ void ExtractQueryStats(NKqpProto::TKqpStatsQuery& dst, const NKikimrQueryStats:: ui64 minCpu = Max<ui64>(); ui64 maxCpu = 0; ui64 sumCpu = 0; - ui64 sumReadSets = 0; - ui64 maxProgramSize = 0; - ui64 maxReplySize = 0; + ui64 sumReadSets = 0; + ui64 maxProgramSize = 0; + ui64 maxReplySize = 0; for (const auto& perShard : txStats.GetPerShardStats()) { ui64 cpu = perShard.GetCpuTimeUsec(); minCpu = Min(minCpu, cpu); maxCpu = Max(maxCpu, cpu); sumCpu += cpu; - sumReadSets += perShard.GetOutgoingReadSetsCount(); - maxProgramSize = Max(maxProgramSize, perShard.GetProgramSize()); - maxReplySize = Max(maxReplySize, perShard.GetReplySize()); + sumReadSets += perShard.GetOutgoingReadSetsCount(); + maxProgramSize = Max(maxProgramSize, perShard.GetProgramSize()); + maxReplySize = Max(maxReplySize, perShard.GetReplySize()); ++cnt; } if (cnt) { @@ -421,10 +421,10 @@ void ExtractQueryStats(NKqpProto::TKqpStatsQuery& dst, const NKikimrQueryStats:: dstShardTime.SetMax(maxCpu); dstShardTime.SetSum(sumCpu); dstShardTime.SetCnt(cnt); - - dst.SetReadSetsCount(dst.GetReadSetsCount() + sumReadSets); - dst.SetMaxShardProgramSize(Max(dst.GetMaxShardProgramSize(), maxProgramSize)); - dst.SetMaxShardReplySize(Max(dst.GetMaxShardReplySize(), maxReplySize)); + + dst.SetReadSetsCount(dst.GetReadSetsCount() + sumReadSets); + dst.SetMaxShardProgramSize(Max(dst.GetMaxShardProgramSize(), maxProgramSize)); + dst.SetMaxShardReplySize(Max(dst.GetMaxShardReplySize(), maxReplySize)); dstExec.SetCpuTimeUs(dstExec.GetCpuTimeUs() + sumCpu); } @@ -613,53 +613,53 @@ TKqpParamsMap BuildParamsMap(const TVector<NKikimrKqp::TParameterBinding>& bindi param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui32>::Id); param.MutableValue()->SetUint32(readTarget); paramRef = NDq::TMkqlValueRef(param); - } else if (binding.GetName() == NowParamName) { - auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui64>::Id); - param.MutableValue()->SetUint64(transformCtx->QueryCtx->GetCachedNow()); - paramRef = NDq::TMkqlValueRef(param); - } else if (binding.GetName() == CurrentDateParamName) { - auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TDate>::Id); - ui64 date = transformCtx->QueryCtx->GetCachedDate(); - YQL_ENSURE(date <= Max<ui32>()); - param.MutableValue()->SetUint32(static_cast<ui32>(date)); - paramRef = NDq::TMkqlValueRef(param); - } else if (binding.GetName() == CurrentDatetimeParamName) { - auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TDatetime>::Id); - ui64 datetime = transformCtx->QueryCtx->GetCachedDatetime(); - YQL_ENSURE(datetime <= Max<ui32>()); - param.MutableValue()->SetUint32(static_cast<ui32>(datetime)); - paramRef = NDq::TMkqlValueRef(param); - } else if (binding.GetName() == CurrentTimestampParamName) { - auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TTimestamp>::Id); - param.MutableValue()->SetUint64(transformCtx->QueryCtx->GetCachedTimestamp()); - paramRef = NDq::TMkqlValueRef(param); - } else if (binding.GetName() == RandomNumberParamName) { - auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui64>::Id); - param.MutableValue()->SetUint64(transformCtx->QueryCtx->GetCachedRandom<ui64>()); - paramRef = NDq::TMkqlValueRef(param); - } else if (binding.GetName() == RandomParamName) { - auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<double>::Id); - param.MutableValue()->SetDouble(transformCtx->QueryCtx->GetCachedRandom<double>()); - paramRef = NDq::TMkqlValueRef(param); - } else if (binding.GetName() == RandomUuidParamName) { - auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; - param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); - param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TUuid>::Id); - auto uuid = transformCtx->QueryCtx->GetCachedRandom<TGUID>(); - param.MutableValue()->SetBytes(uuid.dw, sizeof(TGUID)); - paramRef = NDq::TMkqlValueRef(param); + } else if (binding.GetName() == NowParamName) { + auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui64>::Id); + param.MutableValue()->SetUint64(transformCtx->QueryCtx->GetCachedNow()); + paramRef = NDq::TMkqlValueRef(param); + } else if (binding.GetName() == CurrentDateParamName) { + auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TDate>::Id); + ui64 date = transformCtx->QueryCtx->GetCachedDate(); + YQL_ENSURE(date <= Max<ui32>()); + param.MutableValue()->SetUint32(static_cast<ui32>(date)); + paramRef = NDq::TMkqlValueRef(param); + } else if (binding.GetName() == CurrentDatetimeParamName) { + auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TDatetime>::Id); + ui64 datetime = transformCtx->QueryCtx->GetCachedDatetime(); + YQL_ENSURE(datetime <= Max<ui32>()); + param.MutableValue()->SetUint32(static_cast<ui32>(datetime)); + paramRef = NDq::TMkqlValueRef(param); + } else if (binding.GetName() == CurrentTimestampParamName) { + auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TTimestamp>::Id); + param.MutableValue()->SetUint64(transformCtx->QueryCtx->GetCachedTimestamp()); + paramRef = NDq::TMkqlValueRef(param); + } else if (binding.GetName() == RandomNumberParamName) { + auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<ui64>::Id); + param.MutableValue()->SetUint64(transformCtx->QueryCtx->GetCachedRandom<ui64>()); + paramRef = NDq::TMkqlValueRef(param); + } else if (binding.GetName() == RandomParamName) { + auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<double>::Id); + param.MutableValue()->SetDouble(transformCtx->QueryCtx->GetCachedRandom<double>()); + paramRef = NDq::TMkqlValueRef(param); + } else if (binding.GetName() == RandomUuidParamName) { + auto& param = txState->Tx().ParamsState->Values[binding.GetName()]; + param.MutableType()->SetKind(NKikimrMiniKQL::ETypeKind::Data); + param.MutableType()->MutableData()->SetScheme(NKikimr::NUdf::TDataType<NUdf::TUuid>::Id); + auto uuid = transformCtx->QueryCtx->GetCachedRandom<TGUID>(); + param.MutableValue()->SetBytes(uuid.dw, sizeof(TGUID)); + paramRef = NDq::TMkqlValueRef(param); } else if (binding.HasMkqlIndex()) { paramRef = GetParamFromResult(binding, *transformCtx); } else { diff --git a/ydb/core/kqp/prepare/kqp_query_plan.cpp b/ydb/core/kqp/prepare/kqp_query_plan.cpp index d81b8e4bd34..346a8b15cbe 100644 --- a/ydb/core/kqp/prepare/kqp_query_plan.cpp +++ b/ydb/core/kqp/prepare/kqp_query_plan.cpp @@ -2,7 +2,7 @@ #include <ydb/core/kqp/common/kqp_yql.h> #include <ydb/core/kqp/provider/yql_kikimr_provider_impl.h> -#include <ydb/public/lib/value/value.h> +#include <ydb/public/lib/value/value.h> #include <ydb/library/yql/ast/yql_ast_escaping.h> #include <ydb/library/yql/core/yql_expr_optimize.h> @@ -17,15 +17,15 @@ #include <util/string/strip.h> #include <util/string/vector.h> -#include <regex> - +#include <regex> + namespace NKikimr { namespace NKqp { using namespace NYql; using namespace NYql::NNodes; using namespace NYql::NDq; -using namespace NClient; +using namespace NClient; namespace { @@ -52,28 +52,28 @@ struct TTableInfo { struct TSerializerCtx { TSerializerCtx(TExprContext& exprCtx, const TString& cluster, const TIntrusivePtr<NYql::TKikimrTablesData> tablesData, - const TKikimrConfiguration::TPtr config, - THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> pureTxResults) + const TKikimrConfiguration::TPtr config, + THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> pureTxResults) : ExprCtx(exprCtx) , Cluster(cluster) , TablesData(tablesData) , Config(config) - , PureTxResults(std::move(pureTxResults)) + , PureTxResults(std::move(pureTxResults)) {} TMap<TString, TTableInfo> Tables; TMap<TString, TExprNode::TPtr> BindNameToStage; TMap<TString, ui32> StageGuidToId; - THashMap<ui32, TVector<std::pair<ui32, ui32>>> ParamBindings; - THashSet<ui32> PrecomputePhases; + THashMap<ui32, TVector<std::pair<ui32, ui32>>> ParamBindings; + THashSet<ui32> PrecomputePhases; ui32 TxId = 0; - ui32 PlanNodeId = 0; + ui32 PlanNodeId = 0; const TExprContext& ExprCtx; const TString& Cluster; const TIntrusivePtr<NYql::TKikimrTablesData> TablesData; const TKikimrConfiguration::TPtr Config; - THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> PureTxResults; + THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> PureTxResults; }; TString GetExprStr(const TExprBase& scalar) { @@ -282,958 +282,958 @@ void FillTablesInfo(const TExprNode::TPtr& query, TMap<TString, TTableInfo>& tab } }; -class TxPlanSerializer { -public: - TxPlanSerializer(TSerializerCtx& serializerCtx, ui32 txId, const TKqpPhysicalTx& tx) : SerializerCtx(serializerCtx), TxId(txId), Tx(tx) {} - - void Serialize() { - auto& phaseNode = QueryPlanNodes[++SerializerCtx.PlanNodeId]; - phaseNode.TypeName = "Phase"; - - for (ui32 resId = 0; resId < Tx.Results().Size(); ++resId) { - auto res = Tx.Results().Item(resId); - auto& planNode = AddPlanNode(phaseNode); - - if (SerializerCtx.PrecomputePhases.find(TxId) != SerializerCtx.PrecomputePhases.end()) { - planNode.TypeName = TStringBuilder() << "Precompute_" << TxId << "_" << resId; - planNode.CteName = TStringBuilder() << "tx_result_binding_" << TxId << "_" << resId; - } else { - planNode.TypeName = TStringBuilder() << "ResultSet_" << TxId << "_" << resId; - planNode.Type = EPlanNodeType::ResultSet; - } - - if (res.Maybe<TDqCnResult>()) { - Visit(res.Cast<TDqCnResult>().Output().Stage(), planNode); - } else if (res.Maybe<TDqCnValue>()) { - Visit(res.Cast<TDqCnValue>().Output().Stage(), planNode); - } else { - Y_ENSURE(false, res.Ref().Content()); - } - } - - for (const auto& stage: Tx.Stages()) { - if (stage.Cast<TDqStageBase>().Program().Body().Maybe<TKqpEffects>()) { - auto &planNode = AddPlanNode(phaseNode); - planNode.TypeName = "Effect"; - Visit(TExprBase(stage), planNode); - } +class TxPlanSerializer { +public: + TxPlanSerializer(TSerializerCtx& serializerCtx, ui32 txId, const TKqpPhysicalTx& tx) : SerializerCtx(serializerCtx), TxId(txId), Tx(tx) {} + + void Serialize() { + auto& phaseNode = QueryPlanNodes[++SerializerCtx.PlanNodeId]; + phaseNode.TypeName = "Phase"; + + for (ui32 resId = 0; resId < Tx.Results().Size(); ++resId) { + auto res = Tx.Results().Item(resId); + auto& planNode = AddPlanNode(phaseNode); + + if (SerializerCtx.PrecomputePhases.find(TxId) != SerializerCtx.PrecomputePhases.end()) { + planNode.TypeName = TStringBuilder() << "Precompute_" << TxId << "_" << resId; + planNode.CteName = TStringBuilder() << "tx_result_binding_" << TxId << "_" << resId; + } else { + planNode.TypeName = TStringBuilder() << "ResultSet_" << TxId << "_" << resId; + planNode.Type = EPlanNodeType::ResultSet; + } + + if (res.Maybe<TDqCnResult>()) { + Visit(res.Cast<TDqCnResult>().Output().Stage(), planNode); + } else if (res.Maybe<TDqCnValue>()) { + Visit(res.Cast<TDqCnValue>().Output().Stage(), planNode); + } else { + Y_ENSURE(false, res.Ref().Content()); + } + } + + for (const auto& stage: Tx.Stages()) { + if (stage.Cast<TDqStageBase>().Program().Body().Maybe<TKqpEffects>()) { + auto &planNode = AddPlanNode(phaseNode); + planNode.TypeName = "Effect"; + Visit(TExprBase(stage), planNode); + } + } + + /* set NodeId using reverse order */ + for (auto& [id, planNode] : QueryPlanNodes) { + planNode.NodeId = SerializerCtx.PlanNodeId - id + 1; + } + } + + void WriteToJson(NJsonWriter::TBuf& writer) const { + if (!QueryPlanNodes.empty()) { + auto phasePlanNode = QueryPlanNodes.begin(); + WritePlanNodeToJson(phasePlanNode->second, writer); + } + } + +private: + struct TPredicate { + TVector<TString> Args; + TString Body; + }; + + struct TOperator { + TMap<TString, NJson::TJsonValue> Properties; + TSet<ui32> Inputs; + }; + + enum class EPlanNodeType { + Stage, + Connection, + ResultSet + }; + + struct TQueryPlanNode { + ui32 NodeId; + TString Guid; + TString TypeName; + EPlanNodeType Type = EPlanNodeType::Stage; + TMaybe<TString> CteName; + TMaybe<TString> CteRefName; + TMap<TString, NJson::TJsonValue> NodeInfo; + TVector<TOperator> Operators; + THashSet<ui32> Plans; + }; + + void WritePlanNodeToJson(const TQueryPlanNode& planNode, NJsonWriter::TBuf& writer) const { + writer.BeginObject(); + + writer.WriteKey("PlanNodeId").WriteInt(planNode.NodeId); + writer.WriteKey("Node Type").WriteString(planNode.TypeName); + writer.WriteKey("StageGuid").WriteString(planNode.Guid); + + if (auto type = GetPlanNodeType(planNode)) { + writer.WriteKey("PlanNodeType").WriteString(type); + } + + if (planNode.CteName) { + writer.WriteKey("Parent Relationship").WriteString("InitPlan"); + writer.WriteKey("Subplan Name").WriteString("CTE " + *planNode.CteName); + } + + if (planNode.CteRefName) { + writer.WriteKey("CTE Name").WriteString(*planNode.CteRefName); + } + + for (const auto& [key, value] : planNode.NodeInfo) { + writer.WriteKey(key); + writer.WriteJsonValue(&value, true); + } + + if (!planNode.Operators.empty()) { + writer.WriteKey("Operators"); + writer.BeginList(); + + for (auto& op : planNode.Operators) { + writer.BeginObject(); + for (const auto& [key, value] : op.Properties) { + writer.WriteKey(key); + writer.WriteJsonValue(&value, true); + } + + if (op.Inputs.size() > 1) { + writer.WriteKey("Inputs"); + writer.BeginList(); + + for (const auto& input : op.Inputs) { + writer.WriteInt(input); + } + + writer.EndList(); + } + + writer.EndObject(); + } + + writer.EndList(); + } + + if (!planNode.Plans.empty()) { + writer.WriteKey("Plans"); + writer.BeginList(); + + for (auto planId : planNode.Plans) { + Y_ENSURE(QueryPlanNodes.contains(planId)); + WritePlanNodeToJson(QueryPlanNodes.at(planId), writer); + } + + writer.EndList(); + } + + writer.EndObject(); + } + + TString GetPlanNodeType(const TQueryPlanNode& planNode) const { + switch (planNode.Type) { + case EPlanNodeType::Connection: + return "Connection"; + case EPlanNodeType::ResultSet: + return "ResultSet"; + default: + return {}; + } + } + + TMaybe<std::pair<ui32, ui32>> ContainResultBinding(const TString& str) { + static const std::regex regex("tx_result_binding_(\\d+)_(\\d+)"); + std::smatch match; + if (std::regex_search(str.c_str(), match, regex)) { + return TMaybe<std::pair<ui32, ui32>>( + {FromString<ui32>(match[1].str()), FromString<ui32>(match[2].str())}); + } + + return {}; + } + + TMaybe<TValue> GetResult(ui32 txId, ui32 resId) const { + if (SerializerCtx.PureTxResults.contains(txId) && resId < SerializerCtx.PureTxResults[txId].size()) { + auto result = TValue::Create(SerializerCtx.PureTxResults[txId][resId]); + Y_ENSURE(result.HaveValue()); + return TMaybe<TValue>(result); + } + + return {}; + } + + ui32 AddOperator(TQueryPlanNode& planNode, const TString& name, TOperator op) { + if (!planNode.TypeName.Empty()) { + planNode.TypeName += "-" + name; + } else { + planNode.TypeName = name; } - - /* set NodeId using reverse order */ - for (auto& [id, planNode] : QueryPlanNodes) { - planNode.NodeId = SerializerCtx.PlanNodeId - id + 1; - } - } - - void WriteToJson(NJsonWriter::TBuf& writer) const { - if (!QueryPlanNodes.empty()) { - auto phasePlanNode = QueryPlanNodes.begin(); - WritePlanNodeToJson(phasePlanNode->second, writer); - } + + planNode.Operators.push_back(std::move(op)); + return planNode.Operators.size() - 1; + } + + TQueryPlanNode& GetParent(ui32 nodeId) { + auto it = std::find_if(QueryPlanNodes.begin(), QueryPlanNodes.end(), [nodeId](const auto& node) { + return node.second.Plans.contains(nodeId); + }); + + Y_ENSURE(it != QueryPlanNodes.end()); + return it->second; + } + + TQueryPlanNode& AddPlanNode(TQueryPlanNode& planNode) { + planNode.Plans.insert(++SerializerCtx.PlanNodeId); + return QueryPlanNodes[SerializerCtx.PlanNodeId]; } -private: - struct TPredicate { - TVector<TString> Args; - TString Body; - }; - - struct TOperator { - TMap<TString, NJson::TJsonValue> Properties; - TSet<ui32> Inputs; - }; - - enum class EPlanNodeType { - Stage, - Connection, - ResultSet - }; - - struct TQueryPlanNode { - ui32 NodeId; - TString Guid; - TString TypeName; - EPlanNodeType Type = EPlanNodeType::Stage; - TMaybe<TString> CteName; - TMaybe<TString> CteRefName; - TMap<TString, NJson::TJsonValue> NodeInfo; - TVector<TOperator> Operators; - THashSet<ui32> Plans; - }; - - void WritePlanNodeToJson(const TQueryPlanNode& planNode, NJsonWriter::TBuf& writer) const { - writer.BeginObject(); - - writer.WriteKey("PlanNodeId").WriteInt(planNode.NodeId); - writer.WriteKey("Node Type").WriteString(planNode.TypeName); - writer.WriteKey("StageGuid").WriteString(planNode.Guid); - - if (auto type = GetPlanNodeType(planNode)) { - writer.WriteKey("PlanNodeType").WriteString(type); - } - - if (planNode.CteName) { - writer.WriteKey("Parent Relationship").WriteString("InitPlan"); - writer.WriteKey("Subplan Name").WriteString("CTE " + *planNode.CteName); - } - - if (planNode.CteRefName) { - writer.WriteKey("CTE Name").WriteString(*planNode.CteRefName); - } - - for (const auto& [key, value] : planNode.NodeInfo) { - writer.WriteKey(key); - writer.WriteJsonValue(&value, true); - } - - if (!planNode.Operators.empty()) { - writer.WriteKey("Operators"); - writer.BeginList(); - - for (auto& op : planNode.Operators) { - writer.BeginObject(); - for (const auto& [key, value] : op.Properties) { - writer.WriteKey(key); - writer.WriteJsonValue(&value, true); - } - - if (op.Inputs.size() > 1) { - writer.WriteKey("Inputs"); - writer.BeginList(); - - for (const auto& input : op.Inputs) { - writer.WriteInt(input); - } - - writer.EndList(); - } - - writer.EndObject(); - } - - writer.EndList(); - } - - if (!planNode.Plans.empty()) { - writer.WriteKey("Plans"); - writer.BeginList(); - - for (auto planId : planNode.Plans) { - Y_ENSURE(QueryPlanNodes.contains(planId)); - WritePlanNodeToJson(QueryPlanNodes.at(planId), writer); - } - - writer.EndList(); - } - - writer.EndObject(); - } - - TString GetPlanNodeType(const TQueryPlanNode& planNode) const { - switch (planNode.Type) { - case EPlanNodeType::Connection: - return "Connection"; - case EPlanNodeType::ResultSet: - return "ResultSet"; - default: - return {}; - } - } - - TMaybe<std::pair<ui32, ui32>> ContainResultBinding(const TString& str) { - static const std::regex regex("tx_result_binding_(\\d+)_(\\d+)"); - std::smatch match; - if (std::regex_search(str.c_str(), match, regex)) { - return TMaybe<std::pair<ui32, ui32>>( - {FromString<ui32>(match[1].str()), FromString<ui32>(match[2].str())}); - } - - return {}; - } - - TMaybe<TValue> GetResult(ui32 txId, ui32 resId) const { - if (SerializerCtx.PureTxResults.contains(txId) && resId < SerializerCtx.PureTxResults[txId].size()) { - auto result = TValue::Create(SerializerCtx.PureTxResults[txId][resId]); - Y_ENSURE(result.HaveValue()); - return TMaybe<TValue>(result); - } - - return {}; - } - - ui32 AddOperator(TQueryPlanNode& planNode, const TString& name, TOperator op) { - if (!planNode.TypeName.Empty()) { - planNode.TypeName += "-" + name; - } else { - planNode.TypeName = name; - } - - planNode.Operators.push_back(std::move(op)); - return planNode.Operators.size() - 1; - } - - TQueryPlanNode& GetParent(ui32 nodeId) { - auto it = std::find_if(QueryPlanNodes.begin(), QueryPlanNodes.end(), [nodeId](const auto& node) { - return node.second.Plans.contains(nodeId); - }); - - Y_ENSURE(it != QueryPlanNodes.end()); - return it->second; - } - - TQueryPlanNode& AddPlanNode(TQueryPlanNode& planNode) { - planNode.Plans.insert(++SerializerCtx.PlanNodeId); - return QueryPlanNodes[SerializerCtx.PlanNodeId]; - } - - TString ToStr(const TCoDataCtor& data) { - TStringStream out; - EscapeArbitraryAtom(data.Literal().Value(), '"', &out); - return out.Str(); - } - - TString ToStr(const TCoLambda& lambda) { - return PrettyExprStr(lambda.Body()); - } - - TString ToStr(const TCoAsStruct& asStruct) { - TVector<TString> args; - for (const auto& kv : asStruct.Args()) { - auto key = PrettyExprStr(TExprBase(kv->Child(0))); - auto value = PrettyExprStr(TExprBase(kv->Child(1))); - - if (!key.empty() && !value.empty()) { - args.push_back(TStringBuilder() << key << ": " << value); - } - } - - return TStringBuilder() << "{" << JoinStrings(std::move(args), ",") << "}"; - } - - TString ToStr(const TCoAsList& asList) { - TVector<TString> args; - for (const auto& arg : asList.Args()) { - if (auto str = PrettyExprStr(TExprBase(arg))) { - args.push_back(std::move(str)); - } - } - - return TStringBuilder() << "[" << JoinStrings(std::move(args), ",") << "]"; - } - - TString ToStr(const TCoMember& member) { - auto structName = PrettyExprStr(member.Struct()); - auto memberName = PrettyExprStr(member.Name()); - - if (!structName.empty() && !memberName.empty()) { - return TStringBuilder() << structName << "." << memberName; - } - - return {}; - } - - TString ToStr(const TCoIfPresent& ifPresent) { - /* expected IfPresent with 3 children: - * 0-Optional, 1-PresentHandler, 2-MissingValue */ - if (ifPresent.Ref().ChildrenSize() == 3) { - auto arg = PrettyExprStr(ifPresent.Optional()); - auto pred = ExtractPredicate(ifPresent.PresentHandler()); - - Y_ENSURE(!pred.Args.empty()); - return std::regex_replace(pred.Body.c_str(), - std::regex(pred.Args[0].c_str()), arg.c_str()).data(); - } - - return "..."; - } - - TString ToStr(const TCoExists& exist) { - if (auto str = PrettyExprStr(exist.Optional())) { - return TStringBuilder() << "Exist(" << str << ")"; - } - - return {}; - } - - TString AggrOpToStr(const TExprBase& aggr) { - TVector<TString> args; - for (const auto& child : aggr.Ref().Children()) { - if (auto str = PrettyExprStr(TExprBase(child))) { - args.push_back(std::move(str)); - } - } - - return TStringBuilder() << aggr.Ref().Content() << "(" - << JoinStrings(std::move(args), ",") << ")"; - } - - TString BinaryOpToStr(const TExprBase& op) { - auto left = PrettyExprStr(TExprBase(op.Ref().Child(0))); - auto right = PrettyExprStr(TExprBase(op.Ref().Child(1))); - - TStringBuilder str; - str << left; - if (left && right) { - str << " " << op.Ref().Content() << " "; - } - str << right; - - return str; - } - - TString LogicOpToStr(const TExprBase& op) { - TVector<TString> args; - for (const auto& child : op.Ref().Children()) { - if (auto str = PrettyExprStr(TExprBase(child))) { - args.push_back(std::move(str)); - } - } - - return JoinStrings(std::move(args), TStringBuilder() << " " << op.Ref().Content() << " "); - } - - TString PrettyExprStr(const TExprBase& expr) { - static const THashMap<TString, TString> aggregations = { - {"AggrMin", "MIN"}, - {"AggrMax", "MAX"}, - {"AggrCountUpdate", "COUNT"}, - {"AggrAdd", "SUM"} - }; - - TStringBuilder str; - - if (expr.Maybe<TCoIntegralCtor>()) { - str << expr.Ref().Child(0)->Content(); - } else if (auto data = expr.Maybe<TCoDataCtor>()) { - str << ToStr(data.Cast()); - } else if (auto lambda = expr.Maybe<TCoLambda>()) { - str << ToStr(lambda.Cast()); - } else if (auto asStruct = expr.Maybe<TCoAsStruct>()) { - str << ToStr(asStruct.Cast()); - } else if (auto asList = expr.Maybe<TCoAsList>()) { - str << ToStr(asList.Cast()); - } else if (auto member = expr.Maybe<TCoMember>()) { - str << ToStr(member.Cast()); - } else if (auto ifPresent = expr.Maybe<TCoIfPresent>()) { - str << ToStr(ifPresent.Cast()); - } else if (auto exist = expr.Maybe<TCoExists>()) { - str << ToStr(exist.Cast()); - } else if (expr.Maybe<TCoMin>() || expr.Maybe<TCoMax>() || expr.Maybe<TCoInc>()) { - str << AggrOpToStr(expr); - } else if (aggregations.contains(expr.Ref().Content())) { - str << aggregations.at(expr.Ref().Content()) << "(" - << PrettyExprStr(TExprBase(expr.Ref().Child(0))) << ")"; - } else if (expr.Maybe<TCoBinaryArithmetic>() || expr.Maybe<TCoCompare>()) { - str << BinaryOpToStr(expr); - } else if (expr.Maybe<TCoAnd>() || expr.Maybe<TCoOr>() || expr.Maybe<TCoXor>()) { - str << LogicOpToStr(expr); - } else if (expr.Maybe<TCoParameter>() || expr.Maybe<TCoJust>() || expr.Maybe<TCoSafeCast>() - || expr.Maybe<TCoCoalesce>() || expr.Maybe<TCoConvert>()) { - str << PrettyExprStr(TExprBase(expr.Ref().Child(0))); - } else { - str << expr.Ref().Content(); - } - - return str; - } - - void Visit(const TExprBase& expr, TQueryPlanNode& planNode) { - if (expr.Maybe<TDqPhyStage>()) { - auto stageGuid = NDq::TDqStageSettings::Parse(expr.Cast<TDqPhyStage>()).Id; - - if (VisitedStages.find(expr.Raw()) != VisitedStages.end()) { - auto stageId = SerializerCtx.StageGuidToId[stageGuid]; - Y_ENSURE(QueryPlanNodes.contains(stageId)); - auto& commonNode = QueryPlanNodes[stageId]; - - auto& parentNode = GetParent(stageId); - parentNode.Plans.erase(stageId); - - auto& cteNode = AddPlanNode(QueryPlanNodes.begin()->second); - cteNode.Plans.insert(stageId); - cteNode.TypeName = TStringBuilder() << commonNode.TypeName << " (PlanNodeId: " << stageId << ")"; - cteNode.CteName = cteNode.TypeName; - - parentNode.CteRefName = *cteNode.CteName; - planNode.CteRefName = *cteNode.CteName; - - return; + TString ToStr(const TCoDataCtor& data) { + TStringStream out; + EscapeArbitraryAtom(data.Literal().Value(), '"', &out); + return out.Str(); + } + + TString ToStr(const TCoLambda& lambda) { + return PrettyExprStr(lambda.Body()); + } + + TString ToStr(const TCoAsStruct& asStruct) { + TVector<TString> args; + for (const auto& kv : asStruct.Args()) { + auto key = PrettyExprStr(TExprBase(kv->Child(0))); + auto value = PrettyExprStr(TExprBase(kv->Child(1))); + + if (!key.empty() && !value.empty()) { + args.push_back(TStringBuilder() << key << ": " << value); } + } + + return TStringBuilder() << "{" << JoinStrings(std::move(args), ",") << "}"; + } - auto& stagePlanNode = AddPlanNode(planNode); - stagePlanNode.Guid = stageGuid; - SerializerCtx.StageGuidToId[stageGuid] = SerializerCtx.PlanNodeId; - VisitedStages.insert(expr.Raw()); - auto node = expr.Cast<TDqStageBase>().Program().Body().Ptr(); - Visit(node, stagePlanNode); - - /* is that collect stage? */ - if (stagePlanNode.TypeName.Empty()) { - if (expr.Cast<TDqStageBase>().Program().Body().Maybe<TCoArgument>()) { - stagePlanNode.TypeName = "Collect"; - } else { - stagePlanNode.TypeName = "Stage"; - } + TString ToStr(const TCoAsList& asList) { + TVector<TString> args; + for (const auto& arg : asList.Args()) { + if (auto str = PrettyExprStr(TExprBase(arg))) { + args.push_back(std::move(str)); } + } - for (const auto& input : expr.Cast<TDqStageBase>().Inputs()) { - auto inputCn = input.Cast<TDqConnection>(); + return TStringBuilder() << "[" << JoinStrings(std::move(args), ",") << "]"; + } + + TString ToStr(const TCoMember& member) { + auto structName = PrettyExprStr(member.Struct()); + auto memberName = PrettyExprStr(member.Name()); + + if (!structName.empty() && !memberName.empty()) { + return TStringBuilder() << structName << "." << memberName; + } + + return {}; + } + + TString ToStr(const TCoIfPresent& ifPresent) { + /* expected IfPresent with 3 children: + * 0-Optional, 1-PresentHandler, 2-MissingValue */ + if (ifPresent.Ref().ChildrenSize() == 3) { + auto arg = PrettyExprStr(ifPresent.Optional()); + auto pred = ExtractPredicate(ifPresent.PresentHandler()); + + Y_ENSURE(!pred.Args.empty()); + return std::regex_replace(pred.Body.c_str(), + std::regex(pred.Args[0].c_str()), arg.c_str()).data(); + } - auto& inputPlanNode = AddPlanNode(stagePlanNode); - inputPlanNode.Type = EPlanNodeType::Connection; + return "..."; + } + + TString ToStr(const TCoExists& exist) { + if (auto str = PrettyExprStr(exist.Optional())) { + return TStringBuilder() << "Exist(" << str << ")"; + } + + return {}; + } - if (inputCn.Maybe<TDqCnUnionAll>()) { + TString AggrOpToStr(const TExprBase& aggr) { + TVector<TString> args; + for (const auto& child : aggr.Ref().Children()) { + if (auto str = PrettyExprStr(TExprBase(child))) { + args.push_back(std::move(str)); + } + } + + return TStringBuilder() << aggr.Ref().Content() << "(" + << JoinStrings(std::move(args), ",") << ")"; + } + + TString BinaryOpToStr(const TExprBase& op) { + auto left = PrettyExprStr(TExprBase(op.Ref().Child(0))); + auto right = PrettyExprStr(TExprBase(op.Ref().Child(1))); + + TStringBuilder str; + str << left; + if (left && right) { + str << " " << op.Ref().Content() << " "; + } + str << right; + + return str; + } + + TString LogicOpToStr(const TExprBase& op) { + TVector<TString> args; + for (const auto& child : op.Ref().Children()) { + if (auto str = PrettyExprStr(TExprBase(child))) { + args.push_back(std::move(str)); + } + } + + return JoinStrings(std::move(args), TStringBuilder() << " " << op.Ref().Content() << " "); + } + + TString PrettyExprStr(const TExprBase& expr) { + static const THashMap<TString, TString> aggregations = { + {"AggrMin", "MIN"}, + {"AggrMax", "MAX"}, + {"AggrCountUpdate", "COUNT"}, + {"AggrAdd", "SUM"} + }; + + TStringBuilder str; + + if (expr.Maybe<TCoIntegralCtor>()) { + str << expr.Ref().Child(0)->Content(); + } else if (auto data = expr.Maybe<TCoDataCtor>()) { + str << ToStr(data.Cast()); + } else if (auto lambda = expr.Maybe<TCoLambda>()) { + str << ToStr(lambda.Cast()); + } else if (auto asStruct = expr.Maybe<TCoAsStruct>()) { + str << ToStr(asStruct.Cast()); + } else if (auto asList = expr.Maybe<TCoAsList>()) { + str << ToStr(asList.Cast()); + } else if (auto member = expr.Maybe<TCoMember>()) { + str << ToStr(member.Cast()); + } else if (auto ifPresent = expr.Maybe<TCoIfPresent>()) { + str << ToStr(ifPresent.Cast()); + } else if (auto exist = expr.Maybe<TCoExists>()) { + str << ToStr(exist.Cast()); + } else if (expr.Maybe<TCoMin>() || expr.Maybe<TCoMax>() || expr.Maybe<TCoInc>()) { + str << AggrOpToStr(expr); + } else if (aggregations.contains(expr.Ref().Content())) { + str << aggregations.at(expr.Ref().Content()) << "(" + << PrettyExprStr(TExprBase(expr.Ref().Child(0))) << ")"; + } else if (expr.Maybe<TCoBinaryArithmetic>() || expr.Maybe<TCoCompare>()) { + str << BinaryOpToStr(expr); + } else if (expr.Maybe<TCoAnd>() || expr.Maybe<TCoOr>() || expr.Maybe<TCoXor>()) { + str << LogicOpToStr(expr); + } else if (expr.Maybe<TCoParameter>() || expr.Maybe<TCoJust>() || expr.Maybe<TCoSafeCast>() + || expr.Maybe<TCoCoalesce>() || expr.Maybe<TCoConvert>()) { + str << PrettyExprStr(TExprBase(expr.Ref().Child(0))); + } else { + str << expr.Ref().Content(); + } + + return str; + } + + void Visit(const TExprBase& expr, TQueryPlanNode& planNode) { + if (expr.Maybe<TDqPhyStage>()) { + auto stageGuid = NDq::TDqStageSettings::Parse(expr.Cast<TDqPhyStage>()).Id; + + if (VisitedStages.find(expr.Raw()) != VisitedStages.end()) { + auto stageId = SerializerCtx.StageGuidToId[stageGuid]; + Y_ENSURE(QueryPlanNodes.contains(stageId)); + auto& commonNode = QueryPlanNodes[stageId]; + + auto& parentNode = GetParent(stageId); + parentNode.Plans.erase(stageId); + + auto& cteNode = AddPlanNode(QueryPlanNodes.begin()->second); + cteNode.Plans.insert(stageId); + cteNode.TypeName = TStringBuilder() << commonNode.TypeName << " (PlanNodeId: " << stageId << ")"; + cteNode.CteName = cteNode.TypeName; + + parentNode.CteRefName = *cteNode.CteName; + planNode.CteRefName = *cteNode.CteName; + + return; + } + + auto& stagePlanNode = AddPlanNode(planNode); + stagePlanNode.Guid = stageGuid; + SerializerCtx.StageGuidToId[stageGuid] = SerializerCtx.PlanNodeId; + VisitedStages.insert(expr.Raw()); + auto node = expr.Cast<TDqStageBase>().Program().Body().Ptr(); + Visit(node, stagePlanNode); + + /* is that collect stage? */ + if (stagePlanNode.TypeName.Empty()) { + if (expr.Cast<TDqStageBase>().Program().Body().Maybe<TCoArgument>()) { + stagePlanNode.TypeName = "Collect"; + } else { + stagePlanNode.TypeName = "Stage"; + } + } + + for (const auto& input : expr.Cast<TDqStageBase>().Inputs()) { + auto inputCn = input.Cast<TDqConnection>(); + + auto& inputPlanNode = AddPlanNode(stagePlanNode); + inputPlanNode.Type = EPlanNodeType::Connection; + + if (inputCn.Maybe<TDqCnUnionAll>()) { inputPlanNode.TypeName = "UnionAll"; - } else if (inputCn.Maybe<TDqCnBroadcast>()) { + } else if (inputCn.Maybe<TDqCnBroadcast>()) { inputPlanNode.TypeName = "Broadcast"; - } else if (auto hashShuffle = inputCn.Maybe<TDqCnHashShuffle>()) { + } else if (auto hashShuffle = inputCn.Maybe<TDqCnHashShuffle>()) { inputPlanNode.TypeName = "HashShuffle"; - auto& keyColumns = inputPlanNode.NodeInfo["KeyColumns"]; - for (const auto& column : hashShuffle.Cast().KeyColumns()) { - keyColumns.AppendValue(TString(column.Value())); - } - } else { - inputPlanNode.TypeName = inputCn.Ref().Content(); - } - - Visit(inputCn.Output().Stage(), inputPlanNode); + auto& keyColumns = inputPlanNode.NodeInfo["KeyColumns"]; + for (const auto& column : hashShuffle.Cast().KeyColumns()) { + keyColumns.AppendValue(TString(column.Value())); + } + } else { + inputPlanNode.TypeName = inputCn.Ref().Content(); + } + + Visit(inputCn.Output().Stage(), inputPlanNode); } - } else { - Visit(expr.Ptr(), planNode); + } else { + Visit(expr.Ptr(), planNode); - if (planNode.TypeName.Empty()) { - planNode.TypeName = "Stage"; + if (planNode.TypeName.Empty()) { + planNode.TypeName = "Stage"; } - } - } - - TVector<ui32> Visit(TExprNode::TPtr node, TQueryPlanNode& planNode) { - TMaybe<ui32> operatorId; - if (auto maybeRead = TMaybeNode<TKqlReadTableBase>(node)) { - operatorId = Visit(maybeRead.Cast(), planNode); - } else if (auto maybeReadRanges = TMaybeNode<TKqlReadTableRangesBase>(node)) { - operatorId = Visit(maybeReadRanges.Cast(), planNode); - } else if (auto maybeLookup = TMaybeNode<TKqlLookupTableBase>(node)) { - operatorId = Visit(maybeLookup.Cast(), planNode); + } + } + + TVector<ui32> Visit(TExprNode::TPtr node, TQueryPlanNode& planNode) { + TMaybe<ui32> operatorId; + if (auto maybeRead = TMaybeNode<TKqlReadTableBase>(node)) { + operatorId = Visit(maybeRead.Cast(), planNode); + } else if (auto maybeReadRanges = TMaybeNode<TKqlReadTableRangesBase>(node)) { + operatorId = Visit(maybeReadRanges.Cast(), planNode); + } else if (auto maybeLookup = TMaybeNode<TKqlLookupTableBase>(node)) { + operatorId = Visit(maybeLookup.Cast(), planNode); } else if (auto maybeFilter = TMaybeNode<TCoFilterBase>(node)) { - operatorId = Visit(maybeFilter.Cast(), planNode); + operatorId = Visit(maybeFilter.Cast(), planNode); } else if (auto maybeMapJoin = TMaybeNode<TCoMapJoinCore>(node)) { - operatorId = Visit(maybeMapJoin.Cast(), planNode); - } else if (TMaybeNode<TCoFlatMapBase>(node).Lambda().Body().Maybe<TCoMapJoinCore>() || - TMaybeNode<TCoFlatMapBase>(node).Lambda().Body().Maybe<TCoMap>().Input().Maybe<TCoMapJoinCore>()) { + operatorId = Visit(maybeMapJoin.Cast(), planNode); + } else if (TMaybeNode<TCoFlatMapBase>(node).Lambda().Body().Maybe<TCoMapJoinCore>() || + TMaybeNode<TCoFlatMapBase>(node).Lambda().Body().Maybe<TCoMap>().Input().Maybe<TCoMapJoinCore>()) { auto flatMap = TMaybeNode<TCoFlatMapBase>(node).Cast(); auto join = TExprBase(FindNode(node, [](const TExprNode::TPtr& node) { Y_ENSURE(!TMaybeNode<TDqConnection>(node).IsValid()); return TMaybeNode<TCoMapJoinCore>(node).IsValid(); })).Cast<TCoMapJoinCore>(); - operatorId = Visit(flatMap, join, planNode); - node = join.Ptr(); - } else if (auto maybeJoinDict = TMaybeNode<TCoJoinDict>(node)) { - operatorId = Visit(maybeJoinDict.Cast(), planNode); - } else if (TMaybeNode<TCoFlatMapBase>(node).Lambda().Body().Maybe<TCoJoinDict>() || - TMaybeNode<TCoFlatMapBase>(node).Lambda().Body().Maybe<TCoMap>().Input().Maybe<TCoJoinDict>()) { - auto flatMap = TMaybeNode<TCoFlatMapBase>(node).Cast(); - auto join = TExprBase(FindNode(node, [](const TExprNode::TPtr& node) { - Y_ENSURE(!TMaybeNode<TDqConnection>(node).IsValid()); - return TMaybeNode<TCoJoinDict>(node).IsValid(); - })).Cast<TCoJoinDict>(); - operatorId = Visit(flatMap, join, planNode); - node = join.Ptr(); - } else if (auto maybeCondense = TMaybeNode<TCoCondense1>(node)) { - operatorId = Visit(maybeCondense.Cast(), planNode); - } else if (auto maybeCombiner = TMaybeNode<TCoCombineCore>(node)) { - operatorId = Visit(maybeCombiner.Cast(), planNode); - } else if (auto maybeSort = TMaybeNode<TCoSort>(node)) { - operatorId = Visit(maybeSort.Cast(), planNode); - } else if (auto maybeTopSort = TMaybeNode<TCoTopSort>(node)) { - operatorId = Visit(maybeTopSort.Cast(), planNode); - } else if (auto maybeTake = TMaybeNode<TCoTake>(node)) { - operatorId = Visit(maybeTake.Cast(), planNode); - } else if (auto maybeSkip = TMaybeNode<TCoSkip>(node)) { - operatorId = Visit(maybeSkip.Cast(), planNode); - } else if (auto maybeExtend = TMaybeNode<TCoExtendBase>(node)) { - operatorId = Visit(maybeExtend.Cast(), planNode); - } else if (auto maybeIter = TMaybeNode<TCoIterator>(node)) { - operatorId = Visit(maybeIter.Cast(), planNode); - } else if (auto maybeUpsert = TMaybeNode<TKqpUpsertRows>(node)) { - operatorId = Visit(maybeUpsert.Cast(), planNode); - } else if (auto maybeDelete = TMaybeNode<TKqpDeleteRows>(node)) { - operatorId = Visit(maybeDelete.Cast(), planNode); - } - - TVector<ui32> inputIds; - if (auto maybeEffects = TMaybeNode<TKqpEffects>(node)) { - for (const auto& effect : maybeEffects.Cast().Args()) { - auto ids = Visit(effect, planNode); - inputIds.insert(inputIds.end(), ids.begin(), ids.end()); + operatorId = Visit(flatMap, join, planNode); + node = join.Ptr(); + } else if (auto maybeJoinDict = TMaybeNode<TCoJoinDict>(node)) { + operatorId = Visit(maybeJoinDict.Cast(), planNode); + } else if (TMaybeNode<TCoFlatMapBase>(node).Lambda().Body().Maybe<TCoJoinDict>() || + TMaybeNode<TCoFlatMapBase>(node).Lambda().Body().Maybe<TCoMap>().Input().Maybe<TCoJoinDict>()) { + auto flatMap = TMaybeNode<TCoFlatMapBase>(node).Cast(); + auto join = TExprBase(FindNode(node, [](const TExprNode::TPtr& node) { + Y_ENSURE(!TMaybeNode<TDqConnection>(node).IsValid()); + return TMaybeNode<TCoJoinDict>(node).IsValid(); + })).Cast<TCoJoinDict>(); + operatorId = Visit(flatMap, join, planNode); + node = join.Ptr(); + } else if (auto maybeCondense = TMaybeNode<TCoCondense1>(node)) { + operatorId = Visit(maybeCondense.Cast(), planNode); + } else if (auto maybeCombiner = TMaybeNode<TCoCombineCore>(node)) { + operatorId = Visit(maybeCombiner.Cast(), planNode); + } else if (auto maybeSort = TMaybeNode<TCoSort>(node)) { + operatorId = Visit(maybeSort.Cast(), planNode); + } else if (auto maybeTopSort = TMaybeNode<TCoTopSort>(node)) { + operatorId = Visit(maybeTopSort.Cast(), planNode); + } else if (auto maybeTake = TMaybeNode<TCoTake>(node)) { + operatorId = Visit(maybeTake.Cast(), planNode); + } else if (auto maybeSkip = TMaybeNode<TCoSkip>(node)) { + operatorId = Visit(maybeSkip.Cast(), planNode); + } else if (auto maybeExtend = TMaybeNode<TCoExtendBase>(node)) { + operatorId = Visit(maybeExtend.Cast(), planNode); + } else if (auto maybeIter = TMaybeNode<TCoIterator>(node)) { + operatorId = Visit(maybeIter.Cast(), planNode); + } else if (auto maybeUpsert = TMaybeNode<TKqpUpsertRows>(node)) { + operatorId = Visit(maybeUpsert.Cast(), planNode); + } else if (auto maybeDelete = TMaybeNode<TKqpDeleteRows>(node)) { + operatorId = Visit(maybeDelete.Cast(), planNode); + } + + TVector<ui32> inputIds; + if (auto maybeEffects = TMaybeNode<TKqpEffects>(node)) { + for (const auto& effect : maybeEffects.Cast().Args()) { + auto ids = Visit(effect, planNode); + inputIds.insert(inputIds.end(), ids.begin(), ids.end()); + } + } else { + for (const auto& child : node->Children()) { + auto ids = Visit(child, planNode); + inputIds.insert(inputIds.end(), ids.begin(), ids.end()); } - } else { - for (const auto& child : node->Children()) { - auto ids = Visit(child, planNode); - inputIds.insert(inputIds.end(), ids.begin(), ids.end()); - } - } - - if (operatorId) { - planNode.Operators[*operatorId].Inputs.insert(inputIds.begin(), inputIds.end()); - return {*operatorId}; - } - - return inputIds; - } - - ui32 Visit(const TCoCondense1& /*condense*/, TQueryPlanNode& planNode) { - TOperator op; - op.Properties["Name"] = "Aggregate"; - - return AddOperator(planNode, "Aggregate", std::move(op)); - } - - ui32 Visit(const TCoCombineCore& combiner, TQueryPlanNode& planNode) { - TOperator op; - op.Properties["Name"] = "Aggregate"; - op.Properties["GroupBy"] = PrettyExprStr(combiner.KeyExtractor()); - op.Properties["Aggregation"] = PrettyExprStr(combiner.UpdateHandler()); - - return AddOperator(planNode, "Aggregate", std::move(op)); - } - - ui32 Visit(const TCoSort& sort, TQueryPlanNode& planNode) { - TOperator op; - op.Properties["Name"] = "Sort"; - op.Properties["SortBy"] = PrettyExprStr(sort.KeySelectorLambda()); - - return AddOperator(planNode, "Sort", std::move(op)); - } - - ui32 Visit(const TCoTopSort& topSort, TQueryPlanNode& planNode) { - TOperator op; - op.Properties["Name"] = "TopSort"; - op.Properties["TopSortBy"] = PrettyExprStr(topSort.KeySelectorLambda()); - op.Properties["Limit"] = PrettyExprStr(topSort.Count()); - - return AddOperator(planNode, "TopSort", std::move(op)); - } - - ui32 Visit(const TCoTake& take, TQueryPlanNode& planNode) { - TOperator op; - op.Properties["Name"] = "Limit"; - op.Properties["Limit"] = PrettyExprStr(take.Count()); - - return AddOperator(planNode, "Limit", std::move(op)); - } - - ui32 Visit(const TCoSkip& skip, TQueryPlanNode& planNode) { - TOperator op; - op.Properties["Name"] = "Offset"; - op.Properties["Offset"] = PrettyExprStr(skip.Count()); - - return AddOperator(planNode, "Offset", std::move(op)); - } - - ui32 Visit(const TCoExtendBase& /*extend*/, TQueryPlanNode& planNode) { - TOperator op; - op.Properties["Name"] = "Union"; - - return AddOperator(planNode, "Union", std::move(op)); - } - - ui32 Visit(const TCoIterator& iter, TQueryPlanNode& planNode) { - const auto iterValue = PrettyExprStr(iter.List()); - - TOperator op; - op.Properties["Name"] = "Iterator"; - op.Properties["Iterator"] = iterValue; - - if (auto maybeResultBinding = ContainResultBinding(iterValue)) { - auto [txId, resId] = *maybeResultBinding; - planNode.CteRefName = TStringBuilder() << "tx_result_binding_" << TxId << "_" << resId; - } - - return AddOperator(planNode, "ConstantExpr", std::move(op)); - } - - ui32 Visit(const TKqpUpsertRows& upsert, TQueryPlanNode& planNode) { - const auto table = upsert.Table().Path().StringValue(); - - TOperator op; - op.Properties["Name"] = "Upsert"; - auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table); - op.Properties["Table"] = tableData.RelativePath ? *tableData.RelativePath : table; - - TTableWrite writeInfo; - writeInfo.Type = ETableWriteType::MultiUpsert; - for (const auto& column : upsert.Columns()) { - writeInfo.Columns.push_back(TString(column.Value())); - } - - SerializerCtx.Tables[table].Writes.push_back(writeInfo); - planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); - return AddOperator(planNode, "Upsert", std::move(op)); - } - - ui32 Visit(const TKqpDeleteRows& del, TQueryPlanNode& planNode) { - const auto table = del.Table().Path().StringValue(); - - TOperator op; - op.Properties["Name"] = "Delete"; - auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table); - op.Properties["Table"] = tableData.RelativePath ? *tableData.RelativePath : table; - - TTableWrite writeInfo; - writeInfo.Type = ETableWriteType::MultiErase; - - SerializerCtx.Tables[table].Writes.push_back(writeInfo); - planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); - return AddOperator(planNode, "Delete", std::move(op)); - } - - ui32 Visit(const TCoFlatMapBase& flatMap, const TCoMapJoinCore& join, TQueryPlanNode& planNode) { - const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (MapJoin)"; - - TOperator op; - op.Properties["Name"] = name; - auto operatorId = AddOperator(planNode, name, std::move(op)); - - auto inputs = Visit(flatMap.Input().Ptr(), planNode); - planNode.Operators[operatorId].Inputs.insert(inputs.begin(), inputs.end()); - return operatorId; + } + + if (operatorId) { + planNode.Operators[*operatorId].Inputs.insert(inputIds.begin(), inputIds.end()); + return {*operatorId}; + } + + return inputIds; } - ui32 Visit(const TCoMapJoinCore& join, TQueryPlanNode& planNode) { - const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (MapJoin)"; - - TOperator op; - op.Properties["Name"] = name; - return AddOperator(planNode, name, std::move(op)); + ui32 Visit(const TCoCondense1& /*condense*/, TQueryPlanNode& planNode) { + TOperator op; + op.Properties["Name"] = "Aggregate"; + + return AddOperator(planNode, "Aggregate", std::move(op)); + } + + ui32 Visit(const TCoCombineCore& combiner, TQueryPlanNode& planNode) { + TOperator op; + op.Properties["Name"] = "Aggregate"; + op.Properties["GroupBy"] = PrettyExprStr(combiner.KeyExtractor()); + op.Properties["Aggregation"] = PrettyExprStr(combiner.UpdateHandler()); + + return AddOperator(planNode, "Aggregate", std::move(op)); + } + + ui32 Visit(const TCoSort& sort, TQueryPlanNode& planNode) { + TOperator op; + op.Properties["Name"] = "Sort"; + op.Properties["SortBy"] = PrettyExprStr(sort.KeySelectorLambda()); + + return AddOperator(planNode, "Sort", std::move(op)); + } + + ui32 Visit(const TCoTopSort& topSort, TQueryPlanNode& planNode) { + TOperator op; + op.Properties["Name"] = "TopSort"; + op.Properties["TopSortBy"] = PrettyExprStr(topSort.KeySelectorLambda()); + op.Properties["Limit"] = PrettyExprStr(topSort.Count()); + + return AddOperator(planNode, "TopSort", std::move(op)); + } + + ui32 Visit(const TCoTake& take, TQueryPlanNode& planNode) { + TOperator op; + op.Properties["Name"] = "Limit"; + op.Properties["Limit"] = PrettyExprStr(take.Count()); + + return AddOperator(planNode, "Limit", std::move(op)); + } + + ui32 Visit(const TCoSkip& skip, TQueryPlanNode& planNode) { + TOperator op; + op.Properties["Name"] = "Offset"; + op.Properties["Offset"] = PrettyExprStr(skip.Count()); + + return AddOperator(planNode, "Offset", std::move(op)); + } + + ui32 Visit(const TCoExtendBase& /*extend*/, TQueryPlanNode& planNode) { + TOperator op; + op.Properties["Name"] = "Union"; + + return AddOperator(planNode, "Union", std::move(op)); + } + + ui32 Visit(const TCoIterator& iter, TQueryPlanNode& planNode) { + const auto iterValue = PrettyExprStr(iter.List()); + + TOperator op; + op.Properties["Name"] = "Iterator"; + op.Properties["Iterator"] = iterValue; + + if (auto maybeResultBinding = ContainResultBinding(iterValue)) { + auto [txId, resId] = *maybeResultBinding; + planNode.CteRefName = TStringBuilder() << "tx_result_binding_" << TxId << "_" << resId; + } + + return AddOperator(planNode, "ConstantExpr", std::move(op)); + } + + ui32 Visit(const TKqpUpsertRows& upsert, TQueryPlanNode& planNode) { + const auto table = upsert.Table().Path().StringValue(); + + TOperator op; + op.Properties["Name"] = "Upsert"; + auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table); + op.Properties["Table"] = tableData.RelativePath ? *tableData.RelativePath : table; + + TTableWrite writeInfo; + writeInfo.Type = ETableWriteType::MultiUpsert; + for (const auto& column : upsert.Columns()) { + writeInfo.Columns.push_back(TString(column.Value())); + } + + SerializerCtx.Tables[table].Writes.push_back(writeInfo); + planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); + return AddOperator(planNode, "Upsert", std::move(op)); + } + + ui32 Visit(const TKqpDeleteRows& del, TQueryPlanNode& planNode) { + const auto table = del.Table().Path().StringValue(); + + TOperator op; + op.Properties["Name"] = "Delete"; + auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table); + op.Properties["Table"] = tableData.RelativePath ? *tableData.RelativePath : table; + + TTableWrite writeInfo; + writeInfo.Type = ETableWriteType::MultiErase; + + SerializerCtx.Tables[table].Writes.push_back(writeInfo); + planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); + return AddOperator(planNode, "Delete", std::move(op)); + } + + ui32 Visit(const TCoFlatMapBase& flatMap, const TCoMapJoinCore& join, TQueryPlanNode& planNode) { + const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (MapJoin)"; + + TOperator op; + op.Properties["Name"] = name; + auto operatorId = AddOperator(planNode, name, std::move(op)); + + auto inputs = Visit(flatMap.Input().Ptr(), planNode); + planNode.Operators[operatorId].Inputs.insert(inputs.begin(), inputs.end()); + return operatorId; + } + + ui32 Visit(const TCoMapJoinCore& join, TQueryPlanNode& planNode) { + const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (MapJoin)"; + + TOperator op; + op.Properties["Name"] = name; + return AddOperator(planNode, name, std::move(op)); + } + + ui32 Visit(const TCoFlatMapBase& flatMap, const TCoJoinDict& join, TQueryPlanNode& planNode) { + const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (JoinDict)"; + + TOperator op; + op.Properties["Name"] = name; + auto operatorId = AddOperator(planNode, name, std::move(op)); + + auto inputs = Visit(flatMap.Input().Ptr(), planNode); + planNode.Operators[operatorId].Inputs.insert(inputs.begin(), inputs.end()); + return operatorId; + } + + ui32 Visit(const TCoJoinDict& join, TQueryPlanNode& planNode) { + const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (JoinDict)"; + + TOperator op; + op.Properties["Name"] = name; + return AddOperator(planNode, name, std::move(op)); + } + + TPredicate ExtractPredicate(const TCoLambda& expr) { + TPredicate pred; + pred.Args.reserve(expr.Args().Ref().ChildrenSize()); + for (const auto& child : expr.Args().Ref().Children()) { + pred.Args.push_back(PrettyExprStr(TExprBase(child))); + } + + pred.Body = PrettyExprStr(expr.Body()); + return pred; + } + + ui32 Visit(const TCoFilterBase& filter, TQueryPlanNode& planNode) { + TOperator op; + op.Properties["Name"] = "Filter"; + + auto pred = ExtractPredicate(filter.Lambda()); + op.Properties["Predicate"] = pred.Body; + + if (filter.Limit()) { + op.Properties["Limit"] = PrettyExprStr(filter.Limit().Cast()); + } + + return AddOperator(planNode, "Filter", std::move(op)); + } + + ui32 Visit(const TKqlLookupTableBase& lookup, TQueryPlanNode& planNode) { + auto table = TString(lookup.Table().Path().Value()); + TTableRead readInfo; + readInfo.Type = ETableReadType::Lookup; + + TOperator op; + op.Properties["Name"] = "TablePointLookup"; + auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table); + op.Properties["Table"] = tableData.RelativePath ? *tableData.RelativePath : table; + auto& columns = op.Properties["ReadColumns"]; + for (auto const& col : lookup.Columns()) { + readInfo.Columns.push_back(TString(col.Value())); + columns.AppendValue(col.Value()); + } + + SerializerCtx.Tables[table].Reads.push_back(readInfo); + planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); + return AddOperator(planNode, "TablePointLookup", std::move(op)); } - ui32 Visit(const TCoFlatMapBase& flatMap, const TCoJoinDict& join, TQueryPlanNode& planNode) { - const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (JoinDict)"; - - TOperator op; - op.Properties["Name"] = name; - auto operatorId = AddOperator(planNode, name, std::move(op)); - - auto inputs = Visit(flatMap.Input().Ptr(), planNode); - planNode.Operators[operatorId].Inputs.insert(inputs.begin(), inputs.end()); - return operatorId; - } - - ui32 Visit(const TCoJoinDict& join, TQueryPlanNode& planNode) { - const auto name = TStringBuilder() << join.JoinKind().Value() << "Join (JoinDict)"; - - TOperator op; - op.Properties["Name"] = name; - return AddOperator(planNode, name, std::move(op)); - } - - TPredicate ExtractPredicate(const TCoLambda& expr) { - TPredicate pred; - pred.Args.reserve(expr.Args().Ref().ChildrenSize()); - for (const auto& child : expr.Args().Ref().Children()) { - pred.Args.push_back(PrettyExprStr(TExprBase(child))); - } - - pred.Body = PrettyExprStr(expr.Body()); - return pred; - } - - ui32 Visit(const TCoFilterBase& filter, TQueryPlanNode& planNode) { - TOperator op; - op.Properties["Name"] = "Filter"; - - auto pred = ExtractPredicate(filter.Lambda()); - op.Properties["Predicate"] = pred.Body; - - if (filter.Limit()) { - op.Properties["Limit"] = PrettyExprStr(filter.Limit().Cast()); - } - - return AddOperator(planNode, "Filter", std::move(op)); - } - - ui32 Visit(const TKqlLookupTableBase& lookup, TQueryPlanNode& planNode) { - auto table = TString(lookup.Table().Path().Value()); - TTableRead readInfo; - readInfo.Type = ETableReadType::Lookup; - - TOperator op; - op.Properties["Name"] = "TablePointLookup"; - auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table); - op.Properties["Table"] = tableData.RelativePath ? *tableData.RelativePath : table; - auto& columns = op.Properties["ReadColumns"]; - for (auto const& col : lookup.Columns()) { - readInfo.Columns.push_back(TString(col.Value())); - columns.AppendValue(col.Value()); - } - - SerializerCtx.Tables[table].Reads.push_back(readInfo); - planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); - return AddOperator(planNode, "TablePointLookup", std::move(op)); - } - - ui32 Visit(const TKqlReadTableRangesBase& read, TQueryPlanNode& planNode) { - const auto table = TString(read.Table().Path()); - const auto explainPrompt = TKqpReadTableExplainPrompt::Parse(read); - - TTableRead readInfo; - TOperator op; - - auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table); - op.Properties["Table"] = tableData.RelativePath ? *tableData.RelativePath : table; - planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); - - auto rangesDesc = PrettyExprStr(read.Ranges()); + ui32 Visit(const TKqlReadTableRangesBase& read, TQueryPlanNode& planNode) { + const auto table = TString(read.Table().Path()); + const auto explainPrompt = TKqpReadTableExplainPrompt::Parse(read); + + TTableRead readInfo; + TOperator op; + + auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table); + op.Properties["Table"] = tableData.RelativePath ? *tableData.RelativePath : table; + planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); + + auto rangesDesc = PrettyExprStr(read.Ranges()); if (rangesDesc == "Void" || explainPrompt.UsedKeyColumns.empty()) { - readInfo.Type = ETableReadType::FullScan; - - auto& ranges = op.Properties["ReadRanges"]; - for (const auto& col : tableData.Metadata->KeyColumnNames) { - TStringBuilder rangeDesc; - rangeDesc << col << " (-∞, +∞)"; - readInfo.ScanBy.push_back(rangeDesc); - ranges.AppendValue(rangeDesc); - } - } else if (auto maybeResultBinding = ContainResultBinding(rangesDesc)) { - readInfo.Type = ETableReadType::Scan; - - auto [txId, resId] = *maybeResultBinding; - if (auto result = GetResult(txId, resId)) { - auto ranges = (*result)[0]; - const auto& keyColumns = tableData.Metadata->KeyColumnNames; - for (size_t rangeId = 0; rangeId < ranges.Size(); ++rangeId) { - Y_ENSURE(ranges[rangeId].HaveValue() && ranges[rangeId].Size() == 2); - auto from = ranges[rangeId][0]; - auto to = ranges[rangeId][1]; - - for (size_t colId = 0; colId < keyColumns.size(); ++colId) { - if (!from[colId].HaveValue() && !to[colId].HaveValue()) { - continue; - } - - TStringBuilder rangeDesc; - rangeDesc << keyColumns[colId] << " " - << (from[keyColumns.size()].GetDataText() == "1" ? "[" : "(") - << (from[colId].HaveValue() ? from[colId].GetDataText() : "-∞") << ", " - << (to[colId].HaveValue() ? to[colId].GetDataText() : "+∞") - << (to[keyColumns.size()].GetDataText() == "1" ? "]" : ")"); - - readInfo.ScanBy.push_back(rangeDesc); - op.Properties["ReadRanges"].AppendValue(rangeDesc); - } - } - } else { - op.Properties["ReadRanges"] = rangesDesc; - } - } else { - Y_ENSURE(false, rangesDesc); - } - + readInfo.Type = ETableReadType::FullScan; + + auto& ranges = op.Properties["ReadRanges"]; + for (const auto& col : tableData.Metadata->KeyColumnNames) { + TStringBuilder rangeDesc; + rangeDesc << col << " (-∞, +∞)"; + readInfo.ScanBy.push_back(rangeDesc); + ranges.AppendValue(rangeDesc); + } + } else if (auto maybeResultBinding = ContainResultBinding(rangesDesc)) { + readInfo.Type = ETableReadType::Scan; + + auto [txId, resId] = *maybeResultBinding; + if (auto result = GetResult(txId, resId)) { + auto ranges = (*result)[0]; + const auto& keyColumns = tableData.Metadata->KeyColumnNames; + for (size_t rangeId = 0; rangeId < ranges.Size(); ++rangeId) { + Y_ENSURE(ranges[rangeId].HaveValue() && ranges[rangeId].Size() == 2); + auto from = ranges[rangeId][0]; + auto to = ranges[rangeId][1]; + + for (size_t colId = 0; colId < keyColumns.size(); ++colId) { + if (!from[colId].HaveValue() && !to[colId].HaveValue()) { + continue; + } + + TStringBuilder rangeDesc; + rangeDesc << keyColumns[colId] << " " + << (from[keyColumns.size()].GetDataText() == "1" ? "[" : "(") + << (from[colId].HaveValue() ? from[colId].GetDataText() : "-∞") << ", " + << (to[colId].HaveValue() ? to[colId].GetDataText() : "+∞") + << (to[keyColumns.size()].GetDataText() == "1" ? "]" : ")"); + + readInfo.ScanBy.push_back(rangeDesc); + op.Properties["ReadRanges"].AppendValue(rangeDesc); + } + } + } else { + op.Properties["ReadRanges"] = rangesDesc; + } + } else { + Y_ENSURE(false, rangesDesc); + } + if (!explainPrompt.UsedKeyColumns.empty()) { - auto& usedColumns = op.Properties["ReadRangesKeys"]; + auto& usedColumns = op.Properties["ReadRangesKeys"]; for (const auto& col : explainPrompt.UsedKeyColumns) { - usedColumns.AppendValue(col); + usedColumns.AppendValue(col); } } - if (explainPrompt.ExpectedMaxRanges) { - op.Properties["ReadRangesExpectedSize"] = explainPrompt.ExpectedMaxRanges; - } - - auto& columns = op.Properties["ReadColumns"]; - for (const auto& col : read.Columns()) { - readInfo.Columns.emplace_back(TString(col.Value())); - columns.AppendValue(col.Value()); - } - - auto settings = NYql::TKqpReadTableSettings::Parse(read); - if (settings.ItemsLimit && !readInfo.Limit) { - auto limit = GetExprStr(TExprBase(settings.ItemsLimit)); - if (auto maybeResultBinding = ContainResultBinding(limit)) { - const auto [txId, resId] = *maybeResultBinding; - if (auto result = GetResult(txId, resId)) { - limit = result->GetDataText(); - } - } - - readInfo.Limit = limit; - op.Properties["ReadLimit"] = limit; - } - if (settings.Reverse) { - readInfo.Reverse = true; - op.Properties["Reverse"] = true; - } - - if (tableData.Metadata->Kind == EKikimrTableKind::Olap) { - op.Properties["PredicatePushdown"] = SerializerCtx.Config.Get()->PushOlapProcess(); - } - - ui32 operatorId; - if (readInfo.Type == ETableReadType::FullScan) { - op.Properties["Name"] = "TableFullScan"; - operatorId = AddOperator(planNode, "TableFullScan", std::move(op)); - } else { - op.Properties["Name"] = "TableRangesScan"; - operatorId = AddOperator(planNode, "TableRangesScan", std::move(op)); - } - - SerializerCtx.Tables[table].Reads.push_back(std::move(readInfo)); - return operatorId; - } - - ui32 Visit(const TKqlReadTableBase& read, TQueryPlanNode& planNode) { - auto table = TString(read.Table().Path()); - auto range = read.Range(); - - TOperator op; - TTableRead readInfo; - - auto describeBoundary = [this](const TExprBase& key) { - if (auto param = key.Maybe<TCoParameter>()) { - return param.Cast().Name().StringValue(); - } - - if (auto param = key.Maybe<TCoNth>().Tuple().Maybe<TCoParameter>()) { - if (auto maybeResultBinding = ContainResultBinding(param.Cast().Name().StringValue())) { - auto [txId, resId] = *maybeResultBinding; - if (auto result = GetResult(txId, resId)) { - auto index = FromString<ui32>(key.Cast<TCoNth>().Index()); - Y_ENSURE(index < result->Size()); - return (*result)[index].GetDataText(); - } - } + if (explainPrompt.ExpectedMaxRanges) { + op.Properties["ReadRangesExpectedSize"] = explainPrompt.ExpectedMaxRanges; + } + + auto& columns = op.Properties["ReadColumns"]; + for (const auto& col : read.Columns()) { + readInfo.Columns.emplace_back(TString(col.Value())); + columns.AppendValue(col.Value()); + } + + auto settings = NYql::TKqpReadTableSettings::Parse(read); + if (settings.ItemsLimit && !readInfo.Limit) { + auto limit = GetExprStr(TExprBase(settings.ItemsLimit)); + if (auto maybeResultBinding = ContainResultBinding(limit)) { + const auto [txId, resId] = *maybeResultBinding; + if (auto result = GetResult(txId, resId)) { + limit = result->GetDataText(); + } } - return TString("n/a"); - }; - - /* Collect info about scan range */ - struct TKeyPartRange { - TString From; - TString To; - TString ColumnName; - }; - auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table); - op.Properties["Table"] = tableData.RelativePath ? *tableData.RelativePath : table; - planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); - TVector<TKeyPartRange> scanRangeDescr(tableData.Metadata->KeyColumnNames.size()); - - auto maybeFromKey = range.From().Maybe<TKqlKeyTuple>(); - auto maybeToKey = range.To().Maybe<TKqlKeyTuple>(); - if (maybeFromKey && maybeToKey) { - auto fromKey = maybeFromKey.Cast(); - auto toKey = maybeToKey.Cast(); - - for (ui32 i = 0; i < fromKey.ArgCount(); ++i) { - scanRangeDescr[i].From = describeBoundary(fromKey.Arg(i)); + + readInfo.Limit = limit; + op.Properties["ReadLimit"] = limit; + } + if (settings.Reverse) { + readInfo.Reverse = true; + op.Properties["Reverse"] = true; + } + + if (tableData.Metadata->Kind == EKikimrTableKind::Olap) { + op.Properties["PredicatePushdown"] = SerializerCtx.Config.Get()->PushOlapProcess(); + } + + ui32 operatorId; + if (readInfo.Type == ETableReadType::FullScan) { + op.Properties["Name"] = "TableFullScan"; + operatorId = AddOperator(planNode, "TableFullScan", std::move(op)); + } else { + op.Properties["Name"] = "TableRangesScan"; + operatorId = AddOperator(planNode, "TableRangesScan", std::move(op)); + } + + SerializerCtx.Tables[table].Reads.push_back(std::move(readInfo)); + return operatorId; + } + + ui32 Visit(const TKqlReadTableBase& read, TQueryPlanNode& planNode) { + auto table = TString(read.Table().Path()); + auto range = read.Range(); + + TOperator op; + TTableRead readInfo; + + auto describeBoundary = [this](const TExprBase& key) { + if (auto param = key.Maybe<TCoParameter>()) { + return param.Cast().Name().StringValue(); + } + + if (auto param = key.Maybe<TCoNth>().Tuple().Maybe<TCoParameter>()) { + if (auto maybeResultBinding = ContainResultBinding(param.Cast().Name().StringValue())) { + auto [txId, resId] = *maybeResultBinding; + if (auto result = GetResult(txId, resId)) { + auto index = FromString<ui32>(key.Cast<TCoNth>().Index()); + Y_ENSURE(index < result->Size()); + return (*result)[index].GetDataText(); + } + } } - for (ui32 i = 0; i < toKey.ArgCount(); ++i) { - scanRangeDescr[i].To = describeBoundary(toKey.Arg(i)); - } - for (ui32 i = 0; i < scanRangeDescr.size(); ++i) { - scanRangeDescr[i].ColumnName = tableData.Metadata->KeyColumnNames[i]; - } - - TString leftParen = range.From().Maybe<TKqlKeyInc>().IsValid() ? "[" : "("; - TString rightParen = range.To().Maybe<TKqlKeyInc>().IsValid() ? "]" : ")"; - bool hasRangeScans = false; - auto& ranges = op.Properties["ReadRange"]; - for (const auto& keyPartRange : scanRangeDescr) { - TStringBuilder rangeDescr; - - if (keyPartRange.From == keyPartRange.To) { - if (keyPartRange.From.Empty()) { - rangeDescr << keyPartRange.ColumnName << " (-∞, +∞)"; - readInfo.ScanBy.push_back(rangeDescr); - } else { - rangeDescr << keyPartRange.ColumnName - << " (" << keyPartRange.From << ")"; - readInfo.LookupBy.push_back(rangeDescr); - } - } else { - rangeDescr << keyPartRange.ColumnName << " " - << (keyPartRange.From.Empty() ? "(" : leftParen) - << (keyPartRange.From.Empty() ? "-∞" : keyPartRange.From) << ", " - << (keyPartRange.To.Empty() ? "+∞" : keyPartRange.To) - << (keyPartRange.To.Empty() ? ")" : rightParen); - readInfo.ScanBy.push_back(rangeDescr); - hasRangeScans = true; - } - - ranges.AppendValue(rangeDescr); + return TString("n/a"); + }; + + /* Collect info about scan range */ + struct TKeyPartRange { + TString From; + TString To; + TString ColumnName; + }; + auto& tableData = SerializerCtx.TablesData->GetTable(SerializerCtx.Cluster, table); + op.Properties["Table"] = tableData.RelativePath ? *tableData.RelativePath : table; + planNode.NodeInfo["Tables"].AppendValue(op.Properties["Table"]); + TVector<TKeyPartRange> scanRangeDescr(tableData.Metadata->KeyColumnNames.size()); + + auto maybeFromKey = range.From().Maybe<TKqlKeyTuple>(); + auto maybeToKey = range.To().Maybe<TKqlKeyTuple>(); + if (maybeFromKey && maybeToKey) { + auto fromKey = maybeFromKey.Cast(); + auto toKey = maybeToKey.Cast(); + + for (ui32 i = 0; i < fromKey.ArgCount(); ++i) { + scanRangeDescr[i].From = describeBoundary(fromKey.Arg(i)); + } + for (ui32 i = 0; i < toKey.ArgCount(); ++i) { + scanRangeDescr[i].To = describeBoundary(toKey.Arg(i)); + } + for (ui32 i = 0; i < scanRangeDescr.size(); ++i) { + scanRangeDescr[i].ColumnName = tableData.Metadata->KeyColumnNames[i]; } - // Scan which fixes only few first members of compound primary key were called "Lookup" - // by older explain version. We continue to do so. - if (readInfo.LookupBy.size() > 0) { - readInfo.Type = ETableReadType::Lookup; - } else { - readInfo.Type = hasRangeScans ? ETableReadType::Scan : ETableReadType::FullScan; - } + TString leftParen = range.From().Maybe<TKqlKeyInc>().IsValid() ? "[" : "("; + TString rightParen = range.To().Maybe<TKqlKeyInc>().IsValid() ? "]" : ")"; + bool hasRangeScans = false; + auto& ranges = op.Properties["ReadRange"]; + for (const auto& keyPartRange : scanRangeDescr) { + TStringBuilder rangeDescr; + + if (keyPartRange.From == keyPartRange.To) { + if (keyPartRange.From.Empty()) { + rangeDescr << keyPartRange.ColumnName << " (-∞, +∞)"; + readInfo.ScanBy.push_back(rangeDescr); + } else { + rangeDescr << keyPartRange.ColumnName + << " (" << keyPartRange.From << ")"; + readInfo.LookupBy.push_back(rangeDescr); + } + } else { + rangeDescr << keyPartRange.ColumnName << " " + << (keyPartRange.From.Empty() ? "(" : leftParen) + << (keyPartRange.From.Empty() ? "-∞" : keyPartRange.From) << ", " + << (keyPartRange.To.Empty() ? "+∞" : keyPartRange.To) + << (keyPartRange.To.Empty() ? ")" : rightParen); + readInfo.ScanBy.push_back(rangeDescr); + hasRangeScans = true; + } + + ranges.AppendValue(rangeDescr); + } + + // Scan which fixes only few first members of compound primary key were called "Lookup" + // by older explain version. We continue to do so. + if (readInfo.LookupBy.size() > 0) { + readInfo.Type = ETableReadType::Lookup; + } else { + readInfo.Type = hasRangeScans ? ETableReadType::Scan : ETableReadType::FullScan; + } + } + + auto& columns = op.Properties["ReadColumns"]; + for (auto const& col : read.Columns()) { + readInfo.Columns.emplace_back(TString(col.Value())); + columns.AppendValue(col.Value()); + } + + auto settings = NYql::TKqpReadTableSettings::Parse(read); + if (settings.ItemsLimit && !readInfo.Limit) { + auto limit = GetExprStr(TExprBase(settings.ItemsLimit)); + if (auto maybeResultBinding = ContainResultBinding(limit)) { + const auto [txId, resId] = *maybeResultBinding; + if (auto result = GetResult(txId, resId)) { + limit = result->GetDataText(); + } + } + + readInfo.Limit = limit; + op.Properties["ReadLimit"] = limit; } - - auto& columns = op.Properties["ReadColumns"]; - for (auto const& col : read.Columns()) { - readInfo.Columns.emplace_back(TString(col.Value())); - columns.AppendValue(col.Value()); - } - - auto settings = NYql::TKqpReadTableSettings::Parse(read); - if (settings.ItemsLimit && !readInfo.Limit) { - auto limit = GetExprStr(TExprBase(settings.ItemsLimit)); - if (auto maybeResultBinding = ContainResultBinding(limit)) { - const auto [txId, resId] = *maybeResultBinding; - if (auto result = GetResult(txId, resId)) { - limit = result->GetDataText(); - } - } - - readInfo.Limit = limit; - op.Properties["ReadLimit"] = limit; - } - if (settings.Reverse) { - readInfo.Reverse = true; - op.Properties["Reverse"] = true; - } - - SerializerCtx.Tables[table].Reads.push_back(readInfo); - - ui32 operatorId; - if (readInfo.Type == ETableReadType::Scan) { - op.Properties["Name"] = "TableRangeScan"; - operatorId = AddOperator(planNode, "TableRangeScan", std::move(op)); - } else if (readInfo.Type == ETableReadType::FullScan) { - op.Properties["Name"] = "TableFullScan"; - operatorId = AddOperator(planNode, "TableFullScan", std::move(op)); - } else if (readInfo.Type == ETableReadType::Lookup) { - op.Properties["Name"] = "TablePointLookup"; - operatorId = AddOperator(planNode, "TablePointLookup", std::move(op)); - } else { - op.Properties["Name"] = "TableScan"; - operatorId = AddOperator(planNode, "TableScan", std::move(op)); - } - - return operatorId; + if (settings.Reverse) { + readInfo.Reverse = true; + op.Properties["Reverse"] = true; + } + + SerializerCtx.Tables[table].Reads.push_back(readInfo); + + ui32 operatorId; + if (readInfo.Type == ETableReadType::Scan) { + op.Properties["Name"] = "TableRangeScan"; + operatorId = AddOperator(planNode, "TableRangeScan", std::move(op)); + } else if (readInfo.Type == ETableReadType::FullScan) { + op.Properties["Name"] = "TableFullScan"; + operatorId = AddOperator(planNode, "TableFullScan", std::move(op)); + } else if (readInfo.Type == ETableReadType::Lookup) { + op.Properties["Name"] = "TablePointLookup"; + operatorId = AddOperator(planNode, "TablePointLookup", std::move(op)); + } else { + op.Properties["Name"] = "TableScan"; + operatorId = AddOperator(planNode, "TableScan", std::move(op)); + } + + return operatorId; } -private: - TSerializerCtx& SerializerCtx; - const ui32 TxId; - const TKqpPhysicalTx& Tx; +private: + TSerializerCtx& SerializerCtx; + const ui32 TxId; + const TKqpPhysicalTx& Tx; + + TMap<ui32, TQueryPlanNode> QueryPlanNodes; + TNodeSet VisitedStages; +}; - TMap<ui32, TQueryPlanNode> QueryPlanNodes; - TNodeSet VisitedStages; -}; - -using ModifyFunction = std::function<void (NJson::TJsonValue& node)>; +using ModifyFunction = std::function<void (NJson::TJsonValue& node)>; -void ModifyPlan(NJson::TJsonValue& plan, const ModifyFunction& modify) { - modify(plan); +void ModifyPlan(NJson::TJsonValue& plan, const ModifyFunction& modify) { + modify(plan); - auto& map = plan.GetMapSafe(); + auto& map = plan.GetMapSafe(); if (map.contains("Plans")) { - for (auto& subplan : map.at("Plans").GetArraySafe()) { - ModifyPlan(subplan, modify); + for (auto& subplan : map.at("Plans").GetArraySafe()) { + ModifyPlan(subplan, modify); } } } @@ -1372,14 +1372,14 @@ void WriteKqlPlan(NJsonWriter::TBuf& writer, const TExprNode::TPtr& query) { // TODO(sk): check prepared statements params in read ranges // TODO(sk): check params from correlated subqueries // lookup join -void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const TKqpPhysicalQuery& query, - THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> pureTxResults, TExprContext& ctx, const TString& cluster, - const TIntrusivePtr<NYql::TKikimrTablesData> tablesData, TKikimrConfiguration::TPtr config) +void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const TKqpPhysicalQuery& query, + THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> pureTxResults, TExprContext& ctx, const TString& cluster, + const TIntrusivePtr<NYql::TKikimrTablesData> tablesData, TKikimrConfiguration::TPtr config) { - TSerializerCtx serializerCtx(ctx, cluster, tablesData, config, std::move(pureTxResults)); + TSerializerCtx serializerCtx(ctx, cluster, tablesData, config, std::move(pureTxResults)); /* bindingName -> stage */ - auto collectBindings = [&serializerCtx, &query] (auto id, const auto& phase) { + auto collectBindings = [&serializerCtx, &query] (auto id, const auto& phase) { for (const auto& param: phase.ParamBindings()) { if (auto maybeResultBinding = param.Binding().template Maybe<TKqpTxResultBinding>()) { auto resultBinding = maybeResultBinding.Cast(); @@ -1389,34 +1389,34 @@ void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const TKqpPhysicalQ if (auto maybeConnection = result.template Maybe<TDqConnection>()) { auto stage = maybeConnection.Cast().Output().Stage(); serializerCtx.BindNameToStage[param.Name().StringValue()] = stage.Ptr(); - serializerCtx.PrecomputePhases.insert(txId); - serializerCtx.ParamBindings[id].push_back({txId, resultId}); + serializerCtx.PrecomputePhases.insert(txId); + serializerCtx.ParamBindings[id].push_back({txId, resultId}); } } } }; - - ui32 id = 0; + + ui32 id = 0; for (const auto& tx: query.Transactions()) { - collectBindings(id++, tx); + collectBindings(id++, tx); } - auto setPlan = [&serializerCtx](auto txId, const auto& tx, auto& txProto) { + auto setPlan = [&serializerCtx](auto txId, const auto& tx, auto& txProto) { NJsonWriter::TBuf txWriter; txWriter.SetIndentSpaces(2); - TxPlanSerializer txPlanSerializer(serializerCtx, txId, tx); - if (!serializerCtx.PureTxResults.contains(txId)) { - txPlanSerializer.Serialize(); - txPlanSerializer.WriteToJson(txWriter); - txProto.SetPlan(txWriter.Str()); - } - }; - - id = 0; - for (ui32 txId = 0; txId < query.Transactions().Size(); ++txId) { - setPlan(id++, query.Transactions().Item(txId), (*queryProto.MutableTransactions())[txId]); + TxPlanSerializer txPlanSerializer(serializerCtx, txId, tx); + if (!serializerCtx.PureTxResults.contains(txId)) { + txPlanSerializer.Serialize(); + txPlanSerializer.WriteToJson(txWriter); + txProto.SetPlan(txWriter.Str()); + } + }; + + id = 0; + for (ui32 txId = 0; txId < query.Transactions().Size(); ++txId) { + setPlan(id++, query.Transactions().Item(txId), (*queryProto.MutableTransactions())[txId]); } - + NJsonWriter::TBuf writer; writer.SetIndentSpaces(2); WriteCommonTablesInfo(writer, serializerCtx.Tables); @@ -1424,10 +1424,10 @@ void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const TKqpPhysicalQ } TString AddExecStatsToTxPlan(const TString& txPlanJson, const NYql::NDqProto::TDqExecutionStats& stats) { - if (txPlanJson.empty()) { - return {}; - } - + if (txPlanJson.empty()) { + return {}; + } + THashMap<TProtoStringType, const NYql::NDqProto::TDqStageStats*> stages; for (const auto& stage : stats.GetStages()) { stages[stage.GetStageGuid()] = &stage; @@ -1436,26 +1436,26 @@ TString AddExecStatsToTxPlan(const TString& txPlanJson, const NYql::NDqProto::TD NJson::TJsonValue root; NJson::ReadJsonTree(txPlanJson, &root, true); - auto addStatsToPlanNode = [&stages](NJson::TJsonValue& node) { - if (auto stageGuid = node.GetMapSafe().FindPtr("StageGuid")) { - if (auto stat = stages.FindPtr(stageGuid->GetStringSafe())) { - auto& stats = node["Stats"]; - - stats["TotalTasks"] = (*stat)->GetTotalTasksCount(); - stats["TotalDurationMs"] = (*stat)->GetFinishTimeMs().GetMax() - (*stat)->GetFirstRowTimeMs().GetMin(); - stats["TotalCpuTimeUs"] = (*stat)->GetCpuTimeUs().GetSum(); - stats["TotalInputRows"] = (*stat)->GetInputRows().GetSum(); - stats["TotalInputBytes"] = (*stat)->GetInputBytes().GetSum(); - stats["TotalOutputRows"] = (*stat)->GetOutputRows().GetSum(); - stats["TotalOutputBytes"] = (*stat)->GetOutputBytes().GetSum(); + auto addStatsToPlanNode = [&stages](NJson::TJsonValue& node) { + if (auto stageGuid = node.GetMapSafe().FindPtr("StageGuid")) { + if (auto stat = stages.FindPtr(stageGuid->GetStringSafe())) { + auto& stats = node["Stats"]; + + stats["TotalTasks"] = (*stat)->GetTotalTasksCount(); + stats["TotalDurationMs"] = (*stat)->GetFinishTimeMs().GetMax() - (*stat)->GetFirstRowTimeMs().GetMin(); + stats["TotalCpuTimeUs"] = (*stat)->GetCpuTimeUs().GetSum(); + stats["TotalInputRows"] = (*stat)->GetInputRows().GetSum(); + stats["TotalInputBytes"] = (*stat)->GetInputBytes().GetSum(); + stats["TotalOutputRows"] = (*stat)->GetOutputRows().GetSum(); + stats["TotalOutputBytes"] = (*stat)->GetOutputBytes().GetSum(); } } - }; + }; + + ModifyPlan(root, addStatsToPlanNode); - ModifyPlan(root, addStatsToPlanNode); - - NJsonWriter::TBuf txWriter; - txWriter.WriteJsonValue(&root, true); + NJsonWriter::TBuf txWriter; + txWriter.WriteJsonValue(&root, true); return txWriter.Str(); } @@ -1481,27 +1481,27 @@ TString SerializeTxPlans(const TVector<const TString>& txPlans, const TString co writer.WriteKey("Plans"); writer.BeginList(); - auto removeStageGuid = [](NJson::TJsonValue& node) { - auto& map = node.GetMapSafe(); - if (map.contains("StageGuid")) { - map.erase("StageGuid"); + auto removeStageGuid = [](NJson::TJsonValue& node) { + auto& map = node.GetMapSafe(); + if (map.contains("StageGuid")) { + map.erase("StageGuid"); + } + }; + + for (const auto& txPlan : txPlans) { + if (txPlan.empty()) { + continue; + } + + NJson::TJsonValue txPlanJson; + NJson::ReadJsonTree(txPlan, &txPlanJson, true); + + for (auto& subplan : txPlanJson.GetMapSafe().at("Plans").GetArraySafe()) { + ModifyPlan(subplan, removeStageGuid); + writer.WriteJsonValue(&subplan, true); } - }; - - for (const auto& txPlan : txPlans) { - if (txPlan.empty()) { - continue; - } - - NJson::TJsonValue txPlanJson; - NJson::ReadJsonTree(txPlan, &txPlanJson, true); - - for (auto& subplan : txPlanJson.GetMapSafe().at("Plans").GetArraySafe()) { - ModifyPlan(subplan, removeStageGuid); - writer.WriteJsonValue(&subplan, true); - } } - + writer.EndList(); writer.EndObject(); writer.EndObject(); @@ -1542,11 +1542,11 @@ TString SerializeScriptPlan(const TVector<const TString>& queryPlans) { writer.WriteKey("queries"); writer.BeginList(); - for (const auto& plan : queryPlans) { - if (plan.empty()) { - continue; - } - + for (const auto& plan : queryPlans) { + if (plan.empty()) { + continue; + } + NJson::TJsonValue root; NJson::ReadJsonTree(plan, &root, true); auto planMap = root.GetMapSafe(); diff --git a/ydb/core/kqp/prepare/kqp_query_plan.h b/ydb/core/kqp/prepare/kqp_query_plan.h index 9b3170533c4..4e299e14903 100644 --- a/ydb/core/kqp/prepare/kqp_query_plan.h +++ b/ydb/core/kqp/prepare/kqp_query_plan.h @@ -18,9 +18,9 @@ void WriteKqlPlan(NJsonWriter::TBuf& writer, const NYql::TExprNode::TPtr& query) * Set dqPlan in each physical transaction (TKqpPhyQuery.Transactions[].Plan). Common query plan with all * table accesses is stored in top-level TKqpPhyQuery.QueryPlan. */ -void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const NYql::NNodes::TKqpPhysicalQuery& query, - THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> pureTxResults, NYql::TExprContext& ctx, const TString& cluster, - const TIntrusivePtr<NYql::TKikimrTablesData> tablesData, NYql::TKikimrConfiguration::TPtr config); +void PhyQuerySetTxPlans(NKqpProto::TKqpPhyQuery& queryProto, const NYql::NNodes::TKqpPhysicalQuery& query, + THashMap<ui32, TVector<NKikimrMiniKQL::TResult>> pureTxResults, NYql::TExprContext& ctx, const TString& cluster, + const TIntrusivePtr<NYql::TKikimrTablesData> tablesData, NYql::TKikimrConfiguration::TPtr config); /* * Fill stages in given txPlan with ExecutionStats fields. Each plan stage stores StageGuid which is diff --git a/ydb/core/kqp/prepare/kqp_type_ann.cpp b/ydb/core/kqp/prepare/kqp_type_ann.cpp index 4752cf4dc50..ee51c7349c2 100644 --- a/ydb/core/kqp/prepare/kqp_type_ann.cpp +++ b/ydb/core/kqp/prepare/kqp_type_ann.cpp @@ -473,28 +473,28 @@ TStatus AnnotateUpsertRows(const TExprNode::TPtr& node, TExprContext& ctx, const for (auto& keyColumnName : table.second->Metadata->KeyColumnNames) { if (!rowType->FindItem(keyColumnName)) { - ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder() + ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder() << "Missing key column in input type: " << keyColumnName)); return TStatus::Error; } } - for (auto& [name, meta] : table.second->Metadata->Columns) { - if (meta.NotNull && !rowType->FindItem(name)) { - ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE, TStringBuilder() - << "Missing not null column in input: " << name - << ". All not null columns should be initialized")); - return TStatus::Error; - } - - if (meta.NotNull && rowType->FindItemType(name)->HasOptionalOrNull()) { - ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() - << "Can't set optional or NULL value to not null column: " << name - << ". All not null columns should be initialized")); - return TStatus::Error; - } - } - + for (auto& [name, meta] : table.second->Metadata->Columns) { + if (meta.NotNull && !rowType->FindItem(name)) { + ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE, TStringBuilder() + << "Missing not null column in input: " << name + << ". All not null columns should be initialized")); + return TStatus::Error; + } + + if (meta.NotNull && rowType->FindItemType(name)->HasOptionalOrNull()) { + ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() + << "Can't set optional or NULL value to not null column: " << name + << ". All not null columns should be initialized")); + return TStatus::Error; + } + } + if (TKqlUpsertRowsIndex::Match(node.Get())) { Y_ENSURE(!table.second->Metadata->SecondaryGlobalIndexMetadata.empty()); } @@ -546,28 +546,28 @@ TStatus AnnotateInsertRows(const TExprNode::TPtr& node, TExprContext& ctx, const for (auto& keyColumnName : table.second->Metadata->KeyColumnNames) { if (!rowType->FindItem(keyColumnName)) { - ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder() + ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder() << "Missing key column in input type: " << keyColumnName)); return TStatus::Error; } } - for (auto& [name, meta] : table.second->Metadata->Columns) { - if (meta.NotNull && !rowType->FindItem(name)) { - ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE, TStringBuilder() - << "Missing not null column in input: " << name - << ". All not null columns should be initialized")); - return TStatus::Error; - } - - if (meta.NotNull && rowType->FindItemType(name)->HasOptionalOrNull()) { - ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() - << "Can't set optional or NULL value to not null column: " << name - << ". All not null columns should be initialized")); - return TStatus::Error; - } - } - + for (auto& [name, meta] : table.second->Metadata->Columns) { + if (meta.NotNull && !rowType->FindItem(name)) { + ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE, TStringBuilder() + << "Missing not null column in input: " << name + << ". All not null columns should be initialized")); + return TStatus::Error; + } + + if (meta.NotNull && rowType->FindItemType(name)->HasOptionalOrNull()) { + ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() + << "Can't set optional or NULL value to not null column: " << name + << ". All not null columns should be initialized")); + return TStatus::Error; + } + } + if (!EnsureAtom(*node->Child(TKqlInsertRows::idx_OnConflict), ctx)) { return TStatus::Error; } @@ -616,23 +616,23 @@ TStatus AnnotateUpdateRows(const TExprNode::TPtr& node, TExprContext& ctx, const } for (auto& keyColumnName : table.second->Metadata->KeyColumnNames) { - if (!rowType->FindItem(keyColumnName)) { - ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE, TStringBuilder() - << "Missing key column in input type: " << keyColumnName)); - return TStatus::Error; - } - } - - for (const auto& item : rowType->GetItems()) { - auto column = table.second->Metadata->Columns.FindPtr(TString(item->GetName())); - YQL_ENSURE(column); - if (column->NotNull && item->HasOptionalOrNull()) { - ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() - << "Can't set optional or NULL value to not null column: " << column->Name)); - return TStatus::Error; - } - } - + if (!rowType->FindItem(keyColumnName)) { + ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE, TStringBuilder() + << "Missing key column in input type: " << keyColumnName)); + return TStatus::Error; + } + } + + for (const auto& item : rowType->GetItems()) { + auto column = table.second->Metadata->Columns.FindPtr(TString(item->GetName())); + YQL_ENSURE(column); + if (column->NotNull && item->HasOptionalOrNull()) { + ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() + << "Can't set optional or NULL value to not null column: " << column->Name)); + return TStatus::Error; + } + } + node->SetTypeAnn(ctx.MakeType<TListExprType>(MakeKqpEffectType(ctx))); return TStatus::Ok; } @@ -676,7 +676,7 @@ TStatus AnnotateDeleteRows(const TExprNode::TPtr& node, TExprContext& ctx, const auto rowType = itemType->Cast<TStructExprType>(); for (auto& keyColumnName : table.second->Metadata->KeyColumnNames) { if (!rowType->FindItem(keyColumnName)) { - ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder() + ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder() << "Missing key column in input type: " << keyColumnName)); return TStatus::Error; } @@ -836,30 +836,30 @@ TStatus AnnotateOlapFilterExists(const TExprNode::TPtr& node, TExprContext& ctx) return TStatus::Ok; } -TStatus AnnotateKqpTxInternalBinding(const TExprNode::TPtr& node, TExprContext& ctx) { - if (!EnsureArgsCount(*node, 2, ctx)) { - return TStatus::Error; - } - - auto* kind = node->Child(TKqpTxInternalBinding::idx_Kind); - if (!EnsureAtom(*kind, ctx)) { - return TStatus::Error; - } - - if (kind->IsAtom("Unspecified")) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Unspecified internal binding kind")); - return TStatus::Error; - } - - auto* type = node->Child(TKqpTxInternalBinding::idx_Type); - if (!EnsureType(*type, ctx)) { - return TStatus::Error; - } - - node->SetTypeAnn(type->GetTypeAnn()->Cast<TTypeExprType>()->GetType()); - return TStatus::Ok; -} - +TStatus AnnotateKqpTxInternalBinding(const TExprNode::TPtr& node, TExprContext& ctx) { + if (!EnsureArgsCount(*node, 2, ctx)) { + return TStatus::Error; + } + + auto* kind = node->Child(TKqpTxInternalBinding::idx_Kind); + if (!EnsureAtom(*kind, ctx)) { + return TStatus::Error; + } + + if (kind->IsAtom("Unspecified")) { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Unspecified internal binding kind")); + return TStatus::Error; + } + + auto* type = node->Child(TKqpTxInternalBinding::idx_Type); + if (!EnsureType(*type, ctx)) { + return TStatus::Error; + } + + node->SetTypeAnn(type->GetTypeAnn()->Cast<TTypeExprType>()->GetType()); + return TStatus::Ok; +} + TStatus AnnotateKqpTxResultBinding(const TExprNode::TPtr& node, TExprContext& ctx) { if (!EnsureArgsCount(*node, 3, ctx)) { return TStatus::Error; @@ -1098,10 +1098,10 @@ TAutoPtr<IGraphTransformer> CreateKqpTypeAnnotationTransformer(const TString& cl return AnnotateKqpTxResultBinding(input, ctx); } - if (TKqpTxInternalBinding::Match(input.Get())) { - return AnnotateKqpTxInternalBinding(input, ctx); - } - + if (TKqpTxInternalBinding::Match(input.Get())) { + return AnnotateKqpTxInternalBinding(input, ctx); + } + if (TKqpPhysicalTx::Match(input.Get())) { return AnnotateKqpPhysicalTx(input, ctx); } diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp index 806f4e49f00..035d6b77012 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp @@ -13,8 +13,8 @@ using namespace NNodes; class TKiSinkIntentDeterminationTransformer: public TKiSinkVisitorTransformer { public: - TKiSinkIntentDeterminationTransformer(TIntrusivePtr<TKikimrSessionContext> sessionCtx) - : SessionCtx(sessionCtx) {} + TKiSinkIntentDeterminationTransformer(TIntrusivePtr<TKikimrSessionContext> sessionCtx) + : SessionCtx(sessionCtx) {} private: TStatus HandleClusterConfig(TKiClusterConfig node, TExprContext& ctx) override { @@ -30,7 +30,7 @@ private: auto cluster = node.DataSink().Cluster(); auto table = node.Table(); - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); return TStatus::Ok; } @@ -40,7 +40,7 @@ private: auto cluster = node.DataSink().Cluster(); auto table = node.Table(); - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); return TStatus::Ok; } @@ -50,7 +50,7 @@ private: auto cluster = node.DataSink().Cluster(); auto table = node.Table(); - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); return TStatus::Ok; } @@ -60,7 +60,7 @@ private: auto cluster = node.DataSink().Cluster(); auto table = node.Table(); - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); return TStatus::Ok; } @@ -70,7 +70,7 @@ private: auto cluster = node.DataSink().Cluster(); auto table = node.Table(); - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); return TStatus::Ok; } @@ -80,7 +80,7 @@ private: auto cluster = node.DataSink().Cluster(); auto table = node.Table(); - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table)); return TStatus::Ok; } @@ -139,7 +139,7 @@ private: auto mode = settings.Mode.Cast(); if (mode == "drop") { - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); return TStatus::Ok; } else if ( mode == "upsert" || @@ -149,7 +149,7 @@ private: mode == "delete_on" || mode == "update_on") { - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); return TStatus::Ok; } else if (mode == "insert_ignore") { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() @@ -164,14 +164,14 @@ private: ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Update option is required for table update.")); return TStatus::Error; } - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); return TStatus::Ok; } else if (mode == "delete") { if (!settings.Filter) { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Filter option is required for table delete.")); return TStatus::Error; } - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); return TStatus::Ok; } else { ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() @@ -197,7 +197,7 @@ private: return TStatus::Error; } - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); return TStatus::Ok; } else if (mode == "alter") { if (!settings.AlterActions) { @@ -206,7 +206,7 @@ private: return TStatus::Error; } - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); return TStatus::Ok; } @@ -236,7 +236,7 @@ private: TStatus HandleDataQuery(TKiDataQuery node, TExprContext& ctx) override { Y_UNUSED(ctx); for (const auto& op : node.Operations()) { - SessionCtx->Tables().GetOrAddTable(TString(op.Cluster()), SessionCtx->GetDatabase(), TString(op.Table())); + SessionCtx->Tables().GetOrAddTable(TString(op.Cluster()), SessionCtx->GetDatabase(), TString(op.Table())); } return TStatus::Ok; } @@ -262,7 +262,7 @@ private: auto cluster = call.Cast().Cluster().Value(); auto table = call.Cast().Table(); - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table.Path())); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table.Path())); return TStatus::Ok; } @@ -270,7 +270,7 @@ private: auto cluster = call.Cast().Cluster().Value(); auto table = call.Cast().Table(); - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table.Path())); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table.Path())); return TStatus::Ok; } @@ -279,7 +279,7 @@ private: auto cluster = call.Cast().Cluster().Value(); auto table = call.Cast().Table(); - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table.Path())); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table.Path())); return TStatus::Ok; } @@ -287,7 +287,7 @@ private: auto cluster = call.Cast().Cluster().Value(); auto table = call.Cast().Table(); - SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table.Path())); + SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), TString(table.Path())); return TStatus::Ok; } @@ -295,7 +295,7 @@ private: } private: - TIntrusivePtr<TKikimrSessionContext> SessionCtx; + TIntrusivePtr<TKikimrSessionContext> SessionCtx; }; class TKikimrDataSink : public TDataProviderBase @@ -792,7 +792,7 @@ TIntrusivePtr<IDataProvider> CreateKikimrDataSink( TAutoPtr<IGraphTransformer> CreateKiSinkIntentDeterminationTransformer( TIntrusivePtr<TKikimrSessionContext> sessionCtx) { - return new TKiSinkIntentDeterminationTransformer(sessionCtx); + return new TKiSinkIntentDeterminationTransformer(sessionCtx); } } // namespace NYql diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp index 38fb32e54ad..65ef7eb932a 100644 --- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp @@ -55,7 +55,7 @@ private: switch (key.GetKeyType()) { case TKikimrKey::Type::Table: case TKikimrKey::Type::TableScheme: { - auto& table = SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); + auto& table = SessionCtx->Tables().GetOrAddTable(TString(cluster), SessionCtx->GetDatabase(), key.GetTablePath()); if (key.GetKeyType() == TKikimrKey::Type::TableScheme) { table.RequireStats(); @@ -171,7 +171,7 @@ public: YQL_ENSURE(res.Metadata->Indexes.size() == res.Metadata->SecondaryGlobalIndexMetadata.size()); for (const auto& indexMeta : res.Metadata->SecondaryGlobalIndexMetadata) { YQL_ENSURE(indexMeta); - auto& desc = SessionCtx->Tables().GetOrAddTable(indexMeta->Cluster, SessionCtx->GetDatabase(), indexMeta->Name); + auto& desc = SessionCtx->Tables().GetOrAddTable(indexMeta->Cluster, SessionCtx->GetDatabase(), indexMeta->Name); desc.Metadata = indexMeta; desc.Load(ctx, sysColumnsEnabled); } @@ -281,7 +281,7 @@ public: , ConfigurationTransformer(new TKikimrConfigurationTransformer(sessionCtx, types)) , IntentDeterminationTransformer(new TKiSourceIntentDeterminationTransformer(sessionCtx)) , LoadTableMetadataTransformer(CreateKiSourceLoadTableMetadataTransformer(gateway, sessionCtx)) - , TypeAnnotationTransformer(CreateKiSourceTypeAnnotationTransformer(sessionCtx, types)) + , TypeAnnotationTransformer(CreateKiSourceTypeAnnotationTransformer(sessionCtx, types)) , CallableExecutionTransformer(CreateKiSourceCallableExecutionTransformer(gateway, sessionCtx)) { diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index f0d6b6cb2b8..ad360fcaebe 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -551,10 +551,10 @@ public: auto typeNode = columnTuple.Item(1); auto columnType = typeNode.Ref().GetTypeAnn(); auto type = columnType->Cast<TTypeExprType>()->GetType(); - auto notNull = type->GetKind() != ETypeAnnotationKind::Optional; - auto actualType = notNull ? type : type->Cast<TOptionalExprType>()->GetItemType(); + auto notNull = type->GetKind() != ETypeAnnotationKind::Optional; + auto actualType = notNull ? type : type->Cast<TOptionalExprType>()->GetItemType(); auto dataType = actualType->Cast<TDataExprType>(); - SetColumnType(*add_column->mutable_type(), TString(dataType->GetName()), notNull); + SetColumnType(*add_column->mutable_type(), TString(dataType->GetName()), notNull); if (columnTuple.Size() > 2) { auto families = columnTuple.Item(2).Cast<TCoAtomList>(); if (families.Size() > 1) { diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp index b94c04f383f..e49f84b5e1e 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp @@ -273,19 +273,19 @@ EYqlIssueCode YqlStatusFromYdbStatus(ui32 ydbStatus) { } } -void SetColumnType(Ydb::Type& protoType, const TString& typeName, bool notNull) { +void SetColumnType(Ydb::Type& protoType, const TString& typeName, bool notNull) { NUdf::EDataSlot dataSlot = NUdf::GetDataSlot(typeName); if (dataSlot == NUdf::EDataSlot::Decimal) { - auto decimal = notNull ? protoType.mutable_decimal_type() : - protoType.mutable_optional_type()->mutable_item()->mutable_decimal_type(); + auto decimal = notNull ? protoType.mutable_decimal_type() : + protoType.mutable_optional_type()->mutable_item()->mutable_decimal_type(); // We have no params right now // TODO: Fix decimal params support for kikimr decimal->set_precision(22); decimal->set_scale(9); } else { - auto& primitive = notNull ? protoType : *protoType.mutable_optional_type()->mutable_item(); + auto& primitive = notNull ? protoType : *protoType.mutable_optional_type()->mutable_item(); auto id = NUdf::GetDataTypeInfo(dataSlot).TypeId; - primitive.set_type_id(static_cast<Ydb::Type::PrimitiveTypeId>(id)); + primitive.set_type_id(static_cast<Ydb::Type::PrimitiveTypeId>(id)); } } diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 4da1a8066b6..7598494a2b9 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -181,17 +181,17 @@ struct TKikimrColumnMetadata { TString Name; ui32 Id = 0; TString Type; - bool NotNull = false; + bool NotNull = false; ui32 TypeId = 0; TVector<TString> Families; TKikimrColumnMetadata() = default; - TKikimrColumnMetadata(const TString& name, ui32 id, const TString& type, bool notNull, ui32 typeId = 0) + TKikimrColumnMetadata(const TString& name, ui32 id, const TString& type, bool notNull, ui32 typeId = 0) : Name(name) , Id(id) , Type(type) - , NotNull(notNull) + , NotNull(notNull) , TypeId(typeId) {} @@ -199,7 +199,7 @@ struct TKikimrColumnMetadata { : Name(message->GetName()) , Id(message->GetId()) , Type(message->GetType()) - , NotNull(message->GetNotNull()) + , NotNull(message->GetNotNull()) , TypeId(message->GetTypeId()) , Families(message->GetFamily().begin(), message->GetFamily().end()) {} @@ -208,7 +208,7 @@ struct TKikimrColumnMetadata { message->SetName(Name); message->SetId(Id); message->SetType(Type); - message->SetNotNull(NotNull); + message->SetNotNull(NotNull); message->SetTypeId(TypeId); for(auto& family: Families) { message->AddFamily(family); @@ -216,12 +216,12 @@ struct TKikimrColumnMetadata { } bool IsSameScheme(const TKikimrColumnMetadata& other) const { - return Name == other.Name && Type == other.Type && NotNull == other.NotNull; + return Name == other.Name && Type == other.Type && NotNull == other.NotNull; + } + + void SetNotNull() { + NotNull = true; } - - void SetNotNull() { - NotNull = true; - } }; struct TKikimrPathId { @@ -290,7 +290,7 @@ struct TKikimrTableMetadata : public TThrRefBase { TMap<TString, TKikimrColumnMetadata> Columns; TVector<TString> KeyColumnNames; - TVector<TString> ColumnOrder; + TVector<TString> ColumnOrder; // Indexes and SecondaryGlobalIndexMetadata must be in same order TVector<TIndexDescription> Indexes; @@ -644,7 +644,7 @@ public: EYqlIssueCode YqlStatusFromYdbStatus(ui32 ydbStatus); Ydb::FeatureFlag::Status GetFlagValue(const TMaybe<bool>& value); -void SetColumnType(Ydb::Type& protoType, const TString& typeName, bool notNull); +void SetColumnType(Ydb::Type& protoType, const TString& typeName, bool notNull); bool ConvertReadReplicasSettingsToProto(const TString settings, Ydb::Table::ReadReplicasSettings& proto, Ydb::StatusIds::StatusCode& code, TString& error); void ConvertTtlSettingsToProto(const NYql::TTtlSettings& settings, Ydb::Table::TtlSettings& proto); diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp index c459d82d8af..791a617c083 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp @@ -193,13 +193,13 @@ void TestCreateTableCommon(TIntrusivePtr<IKikimrGateway> gateway, Tests::TClient metadata->Cluster = TestCluster; metadata->Name = "/Root/f1/f2/table"; - UNIT_ASSERT(metadata->ColumnOrder.size() == metadata->Columns.size()); - - metadata->Columns.insert(std::make_pair("Column1", TKikimrColumnMetadata{"Column1", 0, "Uint32", false})); - metadata->ColumnOrder.push_back("Column1"); - - metadata->Columns.insert(std::make_pair("Column2", TKikimrColumnMetadata{"Column2", 0, "String", false})); - metadata->ColumnOrder.push_back("Column2"); + UNIT_ASSERT(metadata->ColumnOrder.size() == metadata->Columns.size()); + + metadata->Columns.insert(std::make_pair("Column1", TKikimrColumnMetadata{"Column1", 0, "Uint32", false})); + metadata->ColumnOrder.push_back("Column1"); + + metadata->Columns.insert(std::make_pair("Column2", TKikimrColumnMetadata{"Column2", 0, "String", false})); + metadata->ColumnOrder.push_back("Column2"); if (withExtendedDdl) { metadata->Columns["Column2"].Families.push_back("Family2"); @@ -210,8 +210,8 @@ void TestCreateTableCommon(TIntrusivePtr<IKikimrGateway> gateway, Tests::TClient if (withIndex) { TVector<TString> dataColumns; if (withIndex->WithDataColumns) { - metadata->Columns.insert(std::make_pair("Column3", TKikimrColumnMetadata{"Column3", 0, "String", false})); - metadata->ColumnOrder.push_back("Column3"); + metadata->Columns.insert(std::make_pair("Column3", TKikimrColumnMetadata{"Column3", 0, "String", false})); + metadata->ColumnOrder.push_back("Column3"); dataColumns.push_back("Column3"); } TIndexDescription indexDesc{ diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp index ed468c248fe..cce81efc425 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp @@ -743,10 +743,10 @@ TExprList TKikimrKeyRange::ToRangeExpr(TExprBase owner, TExprContext& ctx) { }; auto nInf = [owner, &ctx] (const TTypeAnnotationNode* type) -> TExprBase { - /* required optional type for TCoNothing */ - if (type->GetKind() != ETypeAnnotationKind::Optional) { - type = ctx.MakeType<TOptionalExprType>(type); - } + /* required optional type for TCoNothing */ + if (type->GetKind() != ETypeAnnotationKind::Optional) { + type = ctx.MakeType<TOptionalExprType>(type); + } return Build<TCoNothing>(ctx, owner.Pos()) .OptionalType(BuildTypeExpr(owner.Pos(), *type, ctx)) .Done(); diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp index 247142819a3..635d1648279 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp @@ -132,20 +132,20 @@ const TKikimrTableDescription* TKikimrTablesData::EnsureTableExists(const TStrin return nullptr; } -TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster, const TString& database, const TString& table) { - if (!Tables.FindPtr(std::make_pair(cluster, table))) { - auto& desc = Tables[std::make_pair(cluster, table)]; - - TString error; - std::pair<TString, TString> pathPair; - if (NKikimr::TrySplitPathByDb(table, database, pathPair, error)) { - desc.RelativePath = pathPair.second; - } - - return desc; - } - - return Tables[std::make_pair(cluster, table)]; +TKikimrTableDescription& TKikimrTablesData::GetOrAddTable(const TString& cluster, const TString& database, const TString& table) { + if (!Tables.FindPtr(std::make_pair(cluster, table))) { + auto& desc = Tables[std::make_pair(cluster, table)]; + + TString error; + std::pair<TString, TString> pathPair; + if (NKikimr::TrySplitPathByDb(table, database, pathPair, error)) { + desc.RelativePath = pathPair.second; + } + + return desc; + } + + return Tables[std::make_pair(cluster, table)]; } TKikimrTableDescription& TKikimrTablesData::GetTable(const TString& cluster, const TString& table) { @@ -175,7 +175,7 @@ bool TKikimrTableDescription::Load(TExprContext& ctx, bool withSystemColumns) { // Currently Kikimr doesn't have parametrized types and Decimal type // is passed with no params. It's known to always be Decimal(22,9), // so we transform Decimal type here. - const TTypeAnnotationNode *type; + const TTypeAnnotationNode *type; if (to_lower(column.Type) == "decimal") type = ctx.MakeType<TDataExprParamsType>( NKikimr::NUdf::GetDataSlot(column.Type), @@ -184,13 +184,13 @@ bool TKikimrTableDescription::Load(TExprContext& ctx, bool withSystemColumns) { else type = ctx.MakeType<TDataExprType>(NKikimr::NUdf::GetDataSlot(column.Type)); - if (!column.NotNull) { - type = ctx.MakeType<TOptionalExprType>(type); - } + if (!column.NotNull) { + type = ctx.MakeType<TOptionalExprType>(type); + } + + items.push_back(ctx.MakeType<TItemExprType>(column.Name, type)); - items.push_back(ctx.MakeType<TItemExprType>(column.Name, type)); - - auto insertResult = ColumnTypes.insert(std::make_pair(column.Name, type)); + auto insertResult = ColumnTypes.insert(std::make_pair(column.Name, type)); YQL_ENSURE(insertResult.second); } @@ -788,19 +788,19 @@ TCoNameValueTupleList TKiExecDataQuerySettings::BuildNode(TExprContext& ctx, TPo .Done(); } -template <> -double TKikimrQueryContext::GetRandom<double>() const { - return RandomProvider->GenRandReal2(); -} - -template <> -ui64 TKikimrQueryContext::GetRandom<ui64>() const { - return RandomProvider->GenRand64(); -} - -template <> -TGUID TKikimrQueryContext::GetRandom<TGUID>() const { - return RandomProvider->GenUuid4(); -} - +template <> +double TKikimrQueryContext::GetRandom<double>() const { + return RandomProvider->GenRandReal2(); +} + +template <> +ui64 TKikimrQueryContext::GetRandom<ui64>() const { + return RandomProvider->GenRand64(); +} + +template <> +TGUID TKikimrQueryContext::GetRandom<TGUID>() const { + return RandomProvider->GenUuid4(); +} + } // namespace NYql diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.h b/ydb/core/kqp/provider/yql_kikimr_provider.h index 15e67257630..6903e6f1932 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider.h @@ -111,45 +111,45 @@ struct TKikimrQueryContext : TThrRefBase { NActors::TActorId ReplyTarget; TMaybe<NKikimrKqp::TRlPath> RlPath; - TIntrusivePtr<ITimeProvider> TimeProvider; - TIntrusivePtr<IRandomProvider> RandomProvider; - - std::optional<ui64> CachedNow; - std::tuple<std::optional<ui64>, std::optional<double>, std::optional<TGUID>> CachedRandom; - - ui64 GetCachedNow() { - if (!CachedNow) { - CachedNow = TimeProvider->Now().GetValue(); - } - - return *CachedNow; - } - - ui64 GetCachedDate() { - return std::min<ui64>(NUdf::MAX_DATE - 1u, GetCachedNow() / 86400000000ul); - } - - ui64 GetCachedDatetime() { - return std::min<ui64>(NUdf::MAX_DATETIME - 1u, GetCachedNow() / 1000000ul); - } - - ui64 GetCachedTimestamp() { - return std::min<ui64>(NUdf::MAX_TIMESTAMP - 1u, GetCachedNow()); - } - - template <typename T> - T GetRandom() const; - - template <typename T> - T GetCachedRandom() { - auto& cached = std::get<std::optional<T>>(CachedRandom); - if (!cached) { - cached = GetRandom<T>(); - } - - return *cached; - } - + TIntrusivePtr<ITimeProvider> TimeProvider; + TIntrusivePtr<IRandomProvider> RandomProvider; + + std::optional<ui64> CachedNow; + std::tuple<std::optional<ui64>, std::optional<double>, std::optional<TGUID>> CachedRandom; + + ui64 GetCachedNow() { + if (!CachedNow) { + CachedNow = TimeProvider->Now().GetValue(); + } + + return *CachedNow; + } + + ui64 GetCachedDate() { + return std::min<ui64>(NUdf::MAX_DATE - 1u, GetCachedNow() / 86400000000ul); + } + + ui64 GetCachedDatetime() { + return std::min<ui64>(NUdf::MAX_DATETIME - 1u, GetCachedNow() / 1000000ul); + } + + ui64 GetCachedTimestamp() { + return std::min<ui64>(NUdf::MAX_TIMESTAMP - 1u, GetCachedNow()); + } + + template <typename T> + T GetRandom() const; + + template <typename T> + T GetCachedRandom() { + auto& cached = std::get<std::optional<T>>(CachedRandom); + if (!cached) { + cached = GetRandom<T>(); + } + + return *cached; + } + void Reset() { PrepareOnly = false; SuppressDdlChecks = false; @@ -168,11 +168,11 @@ struct TKikimrQueryContext : TThrRefBase { ExecutionOrder.clear(); RlPath.Clear(); - - CachedNow.reset(); - std::get<0>(CachedRandom).reset(); - std::get<1>(CachedRandom).reset(); - std::get<2>(CachedRandom).reset(); + + CachedNow.reset(); + std::get<0>(CachedRandom).reset(); + std::get<1>(CachedRandom).reset(); + std::get<2>(CachedRandom).reset(); } }; @@ -185,7 +185,7 @@ public: TKikimrTableMetadataPtr Metadata = nullptr; const TStructExprType* SchemeNode = nullptr; - TMaybe<TString> RelativePath; + TMaybe<TString> RelativePath; bool Load(TExprContext& ctx, bool withVirtualColumns = false); void ToYson(NYson::TYsonWriter& writer) const; @@ -211,7 +211,7 @@ public: TKikimrTablesData(const TKikimrTablesData&) = delete; TKikimrTablesData& operator=(const TKikimrTablesData&) = delete; - TKikimrTableDescription& GetOrAddTable(const TString& cluster, const TString& database, const TString& table); + TKikimrTableDescription& GetOrAddTable(const TString& cluster, const TString& database, const TString& table); TKikimrTableDescription& GetTable(const TString& cluster, const TString& table); const TKikimrTableDescription* EnsureTableExists(const TString& cluster, const TString& table, @@ -348,14 +348,14 @@ public: UserName = userName; } - TString GetDatabase() const { - return Database; - } - - void SetDatabase(const TString& database) { - Database = database; - } - + TString GetDatabase() const { + return Database; + } + + void SetDatabase(const TString& database) { + Database = database; + } + void Reset(bool keepConfigChanges) { TablesData->Reset(); QueryCtx->Reset(); @@ -368,7 +368,7 @@ public: private: TString UserName; - TString Database; + TString Database; TKikimrConfiguration::TPtr Configuration; TIntrusivePtr<TKikimrTablesData> TablesData; TIntrusivePtr<TKikimrQueryContext> QueryCtx; diff --git a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h index f8397b7e0d9..e1aa3cda2b4 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h +++ b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h @@ -177,8 +177,8 @@ private: bool Completed; }; -TAutoPtr<IGraphTransformer> CreateKiSourceTypeAnnotationTransformer(TIntrusivePtr<TKikimrSessionContext> sessionCtx, - TTypeAnnotationContext& types); +TAutoPtr<IGraphTransformer> CreateKiSourceTypeAnnotationTransformer(TIntrusivePtr<TKikimrSessionContext> sessionCtx, + TTypeAnnotationContext& types); TAutoPtr<IGraphTransformer> CreateKiSinkTypeAnnotationTransformer(TIntrusivePtr<IKikimrGateway> gateway, TIntrusivePtr<TKikimrSessionContext> sessionCtx); TAutoPtr<IGraphTransformer> CreateKiLogicalOptProposalTransformer(TIntrusivePtr<TKikimrSessionContext> sessionCtx); diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index 4d4f3ba20d8..692165204ae 100644 --- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -99,9 +99,9 @@ IGraphTransformer::TStatus ConvertTableRowType(TExprNode::TPtr& input, const TKi class TKiSourceTypeAnnotationTransformer : public TKiSourceVisitorTransformer { public: - TKiSourceTypeAnnotationTransformer(TIntrusivePtr<TKikimrSessionContext> sessionCtx, TTypeAnnotationContext& types) - : SessionCtx(sessionCtx) - , Types(types) {} + TKiSourceTypeAnnotationTransformer(TIntrusivePtr<TKikimrSessionContext> sessionCtx, TTypeAnnotationContext& types) + : SessionCtx(sessionCtx) + , Types(types) {} private: TStatus HandleKiRead(TKiReadBase node, TExprContext& ctx) override { @@ -162,9 +162,9 @@ private: children.push_back(listSelectType); auto tupleAnn = ctx.MakeType<TTupleExprType>(children); node.Ptr()->SetTypeAnn(tupleAnn); - - YQL_ENSURE(tableDesc->Metadata->ColumnOrder.size() == tableDesc->Metadata->Columns.size()); - return Types.SetColumnOrder(node.Ref(), tableDesc->Metadata->ColumnOrder, ctx); + + YQL_ENSURE(tableDesc->Metadata->ColumnOrder.size() == tableDesc->Metadata->Columns.size()); + return Types.SetColumnOrder(node.Ref(), tableDesc->Metadata->ColumnOrder, ctx); } case TKikimrKey::Type::TableList: @@ -222,7 +222,7 @@ private: private: TIntrusivePtr<TKikimrSessionContext> SessionCtx; - TTypeAnnotationContext& Types; + TTypeAnnotationContext& Types; }; namespace { @@ -322,8 +322,8 @@ private: TExprNode::TPtr node = value.Ptr(); if (TryConvertTo(node, *expectedType, ctx) == TStatus::Error) { - ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() - << "Failed to convert input columns types to scheme types")); + ctx.AddError(YqlIssue(ctx.GetPosition(node->Pos()), TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() + << "Failed to convert input columns types to scheme types")); return TStatus::Error; } @@ -363,43 +363,43 @@ private: for (auto& keyColumnName : table->Metadata->KeyColumnNames) { if (!rowType->FindItem(keyColumnName)) { - ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder() + ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder() << "Missing key column in input: " << keyColumnName << " for table: " << table->Metadata->Name)); return TStatus::Error; } } - auto op = GetTableOp(node); + auto op = GetTableOp(node); if (op == TYdbOperation::InsertAbort || op == TYdbOperation::InsertRevert || op == TYdbOperation::Upsert || op == TYdbOperation::Replace) { - for (const auto& [name, meta] : table->Metadata->Columns) { - if (meta.NotNull && !rowType->FindItem(name)) { - ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE, TStringBuilder() - << "Missing not null column in input: " << name - << ". All not null columns should be initialized")); - return TStatus::Error; - } - - if (meta.NotNull && rowType->FindItemType(name)->HasOptionalOrNull()) { - ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() - << "Can't set NULL or optional value to not null column: " << name - << ". All not null columns should be initialized")); - return TStatus::Error; - } - } + for (const auto& [name, meta] : table->Metadata->Columns) { + if (meta.NotNull && !rowType->FindItem(name)) { + ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE, TStringBuilder() + << "Missing not null column in input: " << name + << ". All not null columns should be initialized")); + return TStatus::Error; + } + + if (meta.NotNull && rowType->FindItemType(name)->HasOptionalOrNull()) { + ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() + << "Can't set NULL or optional value to not null column: " << name + << ". All not null columns should be initialized")); + return TStatus::Error; + } + } } else if (op == TYdbOperation::UpdateOn) { - for (const auto& item : rowType->GetItems()) { - auto column = table->Metadata->Columns.FindPtr(TString(item->GetName())); - YQL_ENSURE(column); - if (column->NotNull && item->HasOptionalOrNull()) { - ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() - << "Can't set NULL or optional value to not null column: " << column->Name)); - return TStatus::Error; - } - } - } - + for (const auto& item : rowType->GetItems()) { + auto column = table->Metadata->Columns.FindPtr(TString(item->GetName())); + YQL_ENSURE(column); + if (column->NotNull && item->HasOptionalOrNull()) { + ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() + << "Can't set NULL or optional value to not null column: " << column->Name)); + return TStatus::Error; + } + } + } + auto inputColumns = GetSetting(node.Settings().Ref(), "input_columns"); if (!inputColumns) { TExprNode::TListType columns; @@ -486,16 +486,16 @@ private: } } - for (const auto& item : updateResultType->GetItems()) { - auto column = table->Metadata->Columns.FindPtr(TString(item->GetName())); - YQL_ENSURE(column); - if (column->NotNull && item->HasOptionalOrNull()) { - ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() - << "Can't set NULL or optional value to not null column: " << column->Name)); - return TStatus::Error; - } - } - + for (const auto& item : updateResultType->GetItems()) { + auto column = table->Metadata->Columns.FindPtr(TString(item->GetName())); + YQL_ENSURE(column); + if (column->NotNull && item->HasOptionalOrNull()) { + ctx.AddError(YqlIssue(ctx.GetPosition(node.Pos()), TIssuesIds::KIKIMR_BAD_COLUMN_TYPE, TStringBuilder() + << "Can't set NULL or optional value to not null column: " << column->Name)); + return TStatus::Error; + } + } + auto updateBody = node.Update().Body().Ptr(); auto status = ConvertTableRowType(updateBody, *table, ctx); if (status != IGraphTransformer::TStatus::Ok) { @@ -558,7 +558,7 @@ private: TKikimrTableMetadataPtr meta = new TKikimrTableMetadata(cluster, table); meta->DoesExist = true; - meta->ColumnOrder.reserve(create.Columns().Size()); + meta->ColumnOrder.reserve(create.Columns().Size()); for (auto atom : create.PrimaryKey()) { meta->KeyColumnNames.emplace_back(atom.Value()); @@ -578,8 +578,8 @@ private: YQL_ENSURE(columnType && columnType->GetKind() == ETypeAnnotationKind::Type); auto type = columnType->Cast<TTypeExprType>()->GetType(); - auto notNull = type->GetKind() != ETypeAnnotationKind::Optional; - auto actualType = notNull ? type : type->Cast<TOptionalExprType>()->GetItemType(); + auto notNull = type->GetKind() != ETypeAnnotationKind::Optional; + auto actualType = notNull ? type : type->Cast<TOptionalExprType>()->GetItemType(); if (actualType->GetKind() != ETypeAnnotationKind::Data) { columnTypeError(typeNode.Pos(), columnName, "Only core YQL data types are currently supported"); return TStatus::Error; @@ -594,7 +594,7 @@ private: TKikimrColumnMetadata columnMeta; columnMeta.Name = columnName; columnMeta.Type = dataType->GetName(); - columnMeta.NotNull = notNull; + columnMeta.NotNull = notNull; if (columnTuple.Size() > 2) { auto families = columnTuple.Item(2).Cast<TCoAtomList>(); @@ -603,7 +603,7 @@ private: } } - meta->ColumnOrder.push_back(columnName); + meta->ColumnOrder.push_back(columnName); auto insertRes = meta->Columns.insert(std::make_pair(columnName, columnMeta)); if (!insertRes.second) { ctx.AddError(TIssue(ctx.GetPosition(create.Pos()), TStringBuilder() @@ -903,9 +903,9 @@ private: auto columnType = typeNode.Ref().GetTypeAnn(); YQL_ENSURE(columnType && columnType->GetKind() == ETypeAnnotationKind::Type); auto type = columnType->Cast<TTypeExprType>()->GetType(); - auto actualType = (type->GetKind() == ETypeAnnotationKind::Optional) ? - type->Cast<TOptionalExprType>()->GetItemType() : type; - + auto actualType = (type->GetKind() == ETypeAnnotationKind::Optional) ? + type->Cast<TOptionalExprType>()->GetItemType() : type; + if (actualType->GetKind() != ETypeAnnotationKind::Data) { columnTypeError(typeNode.Pos(), name, "Only core YQL data types are currently supported"); return TStatus::Error; @@ -1435,10 +1435,10 @@ private: } // namespace -TAutoPtr<IGraphTransformer> CreateKiSourceTypeAnnotationTransformer(TIntrusivePtr<TKikimrSessionContext> sessionCtx, - TTypeAnnotationContext& types) -{ - return new TKiSourceTypeAnnotationTransformer(sessionCtx, types); +TAutoPtr<IGraphTransformer> CreateKiSourceTypeAnnotationTransformer(TIntrusivePtr<TKikimrSessionContext> sessionCtx, + TTypeAnnotationContext& types) +{ + return new TKiSourceTypeAnnotationTransformer(sessionCtx, types); } TAutoPtr<IGraphTransformer> CreateKiSinkTypeAnnotationTransformer(TIntrusivePtr<IKikimrGateway> gateway, diff --git a/ydb/core/kqp/runtime/kqp_program_builder.cpp b/ydb/core/kqp/runtime/kqp_program_builder.cpp index 95afeeb39e2..b5169c19d82 100644 --- a/ydb/core/kqp/runtime/kqp_program_builder.cpp +++ b/ydb/core/kqp/runtime/kqp_program_builder.cpp @@ -14,18 +14,18 @@ namespace { TType* GetRowType(const TProgramBuilder& builder, const TArrayRef<TKqpTableColumn>& columns) { TStructTypeBuilder rowTypeBuilder(builder.GetTypeEnvironment()); for (auto& column : columns) { - TType* type = nullptr; + TType* type = nullptr; if (column.Type == NUdf::TDataType<NUdf::TDecimal>::Id) { - type = TDataDecimalType::Create(NScheme::DECIMAL_PRECISION, NScheme::DECIMAL_SCALE, builder.GetTypeEnvironment()); + type = TDataDecimalType::Create(NScheme::DECIMAL_PRECISION, NScheme::DECIMAL_SCALE, builder.GetTypeEnvironment()); } else { - type = TDataType::Create(column.Type, builder.GetTypeEnvironment()); + type = TDataType::Create(column.Type, builder.GetTypeEnvironment()); } - - if (!column.NotNull) { - type = TOptionalType::Create(type, builder.GetTypeEnvironment()); - } - - rowTypeBuilder.Add(column.Name, type); + + if (!column.NotNull) { + type = TOptionalType::Create(type, builder.GetTypeEnvironment()); + } + + rowTypeBuilder.Add(column.Name, type); } return rowTypeBuilder.Build(); diff --git a/ydb/core/kqp/runtime/kqp_program_builder.h b/ydb/core/kqp/runtime/kqp_program_builder.h index f65ba22294b..348d25eeea3 100644 --- a/ydb/core/kqp/runtime/kqp_program_builder.h +++ b/ydb/core/kqp/runtime/kqp_program_builder.h @@ -12,13 +12,13 @@ struct TKqpTableColumn { ui32 Id; TString Name; NUdf::TDataTypeId Type; - bool NotNull; + bool NotNull; - TKqpTableColumn(ui32 id, const TStringBuf& name, NUdf::TDataTypeId type, bool notNull) + TKqpTableColumn(ui32 id, const TStringBuf& name, NUdf::TDataTypeId type, bool notNull) : Id(id) , Name(name) - , Type(type) - , NotNull(notNull) {} + , Type(type) + , NotNull(notNull) {} }; using TKqpKeyTuple = TVector<TRuntimeNode>; diff --git a/ydb/core/kqp/runtime/kqp_read_table.h b/ydb/core/kqp/runtime/kqp_read_table.h index d19e1e5f42a..99c9cc3f050 100644 --- a/ydb/core/kqp/runtime/kqp_read_table.h +++ b/ydb/core/kqp/runtime/kqp_read_table.h @@ -39,7 +39,7 @@ void ParseReadColumns(const TType* readType, const TRuntimeNode& tagsNode, TSmallVec<TKqpComputeContextBase::TColumn>& columns, TSmallVec<TKqpComputeContextBase::TColumn>& systemColumns); TParseReadTableResult ParseWideReadTable(TCallable& callable); -TParseReadTableRangesResult ParseWideReadTableRanges(TCallable& callable); +TParseReadTableRangesResult ParseWideReadTableRanges(TCallable& callable); IComputationNode* WrapKqpScanWideReadTableRanges(TCallable& callable, const TComputationNodeFactoryContext& ctx, TKqpScanComputeContext& computeCtx); diff --git a/ydb/core/kqp/ut/common/kqp_ut_common.cpp b/ydb/core/kqp/ut/common/kqp_ut_common.cpp index a31d705e8db..9bc7889f92f 100644 --- a/ydb/core/kqp/ut/common/kqp_ut_common.cpp +++ b/ydb/core/kqp/ut/common/kqp_ut_common.cpp @@ -108,7 +108,7 @@ TKikimrRunner::TKikimrRunner(const TKikimrSettings& settings) { ServerSettings->SetKeepSnapshotTimeout(settings.KeepSnapshotTimeout); ServerSettings->SetFrFactory(&UdfFrFactory); ServerSettings->SetEnableSchemeTransactionsAtSchemeShard(true); - ServerSettings->SetEnableNotNullColumns(true); + ServerSettings->SetEnableNotNullColumns(true); if (settings.LogStream) ServerSettings->SetLogBackend(new TStreamLogBackend(settings.LogStream)); @@ -770,15 +770,15 @@ NJson::TJsonValue FindPlanNodeByKv(const NJson::TJsonValue& plan, const TString& return stage; } } - - if (map.contains("Operators")) { - for (const auto &node : map["Operators"].GetArraySafe()) { - auto op = FindPlanNodeByKv(node, key, value); - if (op.IsDefined()) { - return op; - } - } - } + + if (map.contains("Operators")) { + for (const auto &node : map["Operators"].GetArraySafe()) { + auto op = FindPlanNodeByKv(node, key, value); + if (op.IsDefined()) { + return op; + } + } + } } else { Y_ASSERT(false); } diff --git a/ydb/core/kqp/ut/kqp_explain_ut.cpp b/ydb/core/kqp/ut/kqp_explain_ut.cpp index 8b12e0be725..c3a06836367 100644 --- a/ydb/core/kqp/ut/kqp_explain_ut.cpp +++ b/ydb/core/kqp/ut/kqp_explain_ut.cpp @@ -67,9 +67,9 @@ Y_UNIT_TEST_SUITE(KqpExplain) { auto join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan"); UNIT_ASSERT(join.IsDefined()); - auto left = FindPlanNodeByKv(join, "Table", "EightShard"); + auto left = FindPlanNodeByKv(join, "Table", "EightShard"); UNIT_ASSERT(left.IsDefined()); - auto right = FindPlanNodeByKv(join, "Table", "KeyValue"); + auto right = FindPlanNodeByKv(join, "Table", "KeyValue"); UNIT_ASSERT(right.IsDefined()); } @@ -93,9 +93,9 @@ Y_UNIT_TEST_SUITE(KqpExplain) { auto join = FindPlanNodeByKv(plan, "Node Type", "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan"); UNIT_ASSERT(join.IsDefined()); - auto left = FindPlanNodeByKv(join, "Table", "EightShard"); + auto left = FindPlanNodeByKv(join, "Table", "EightShard"); UNIT_ASSERT(left.IsDefined()); - auto right = FindPlanNodeByKv(join, "Table", "KeyValue"); + auto right = FindPlanNodeByKv(join, "Table", "KeyValue"); UNIT_ASSERT(right.IsDefined()); } @@ -134,19 +134,19 @@ Y_UNIT_TEST_SUITE(KqpExplain) { NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - auto read = FindPlanNodeByKv(plan, "Node Type", "Aggregate-Filter-TableFullScan"); + auto read = FindPlanNodeByKv(plan, "Node Type", "Aggregate-Filter-TableFullScan"); UNIT_ASSERT(read.IsDefined()); - auto tables = read.GetMapSafe().at("Tables").GetArraySafe(); - UNIT_ASSERT(tables[0].GetStringSafe() == "Logs"); - auto shuffle = FindPlanNodeByKv(plan, "Node Type", "HashShuffle"); - UNIT_ASSERT(shuffle.IsDefined()); - auto& columns = shuffle.GetMapSafe().at("KeyColumns").GetArraySafe(); - UNIT_ASSERT(!columns.empty() && columns[0] == "App"); - auto aggregate = FindPlanNodeByKv(read, "Name", "Aggregate"); - UNIT_ASSERT(aggregate.IsDefined()); - UNIT_ASSERT(aggregate.GetMapSafe().at("GroupBy").GetStringSafe() == "item.App"); - UNIT_ASSERT(aggregate.GetMapSafe().at("Aggregation").GetStringSafe() == - "{_yql_agg_0: MIN(item.Message),_yql_agg_1: MAX(item.Message)}"); + auto tables = read.GetMapSafe().at("Tables").GetArraySafe(); + UNIT_ASSERT(tables[0].GetStringSafe() == "Logs"); + auto shuffle = FindPlanNodeByKv(plan, "Node Type", "HashShuffle"); + UNIT_ASSERT(shuffle.IsDefined()); + auto& columns = shuffle.GetMapSafe().at("KeyColumns").GetArraySafe(); + UNIT_ASSERT(!columns.empty() && columns[0] == "App"); + auto aggregate = FindPlanNodeByKv(read, "Name", "Aggregate"); + UNIT_ASSERT(aggregate.IsDefined()); + UNIT_ASSERT(aggregate.GetMapSafe().at("GroupBy").GetStringSafe() == "item.App"); + UNIT_ASSERT(aggregate.GetMapSafe().at("Aggregation").GetStringSafe() == + "{_yql_agg_0: MIN(item.Message),_yql_agg_1: MAX(item.Message)}"); } Y_UNIT_TEST(ComplexJoin) { @@ -182,9 +182,9 @@ Y_UNIT_TEST_SUITE(KqpExplain) { "Aggregate-InnerJoin (MapJoin)-Filter-TableFullScan" ); UNIT_ASSERT(join.IsDefined()); - auto left = FindPlanNodeByKv(join, "Table", "EightShard"); + auto left = FindPlanNodeByKv(join, "Table", "EightShard"); UNIT_ASSERT(left.IsDefined()); - auto right = FindPlanNodeByKv(join, "Table", "FourShard"); + auto right = FindPlanNodeByKv(join, "Table", "FourShard"); UNIT_ASSERT(right.IsDefined()); } @@ -206,14 +206,14 @@ Y_UNIT_TEST_SUITE(KqpExplain) { NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangesScan"); + auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangesScan"); UNIT_ASSERT(node.IsDefined()); - - auto operators = node.GetMapSafe().at("Operators").GetArraySafe(); - UNIT_ASSERT(operators[1].GetMapSafe().at("Name") == "TableRangesScan"); - - auto& readRanges = operators[1].GetMapSafe().at("ReadRanges").GetArraySafe(); - UNIT_ASSERT(readRanges[0] == "Key [150, 266]"); + + auto operators = node.GetMapSafe().at("Operators").GetArraySafe(); + UNIT_ASSERT(operators[1].GetMapSafe().at("Name") == "TableRangesScan"); + + auto& readRanges = operators[1].GetMapSafe().at("ReadRanges").GetArraySafe(); + UNIT_ASSERT(readRanges[0] == "Key [150, 266]"); } Y_UNIT_TEST(CompoundKeyRange) { @@ -235,13 +235,13 @@ Y_UNIT_TEST_SUITE(KqpExplain) { NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - auto read = FindPlanNodeByKv(plan, "Node Type", "Limit-TablePointLookup"); - auto& operators = read.GetMapSafe().at("Operators").GetArraySafe(); - UNIT_ASSERT(operators.size() == 2); - - auto& lookup = operators[1].GetMapSafe(); - UNIT_ASSERT(lookup.at("Name") == "TablePointLookup"); - UNIT_ASSERT(lookup.at("ReadRange").GetArraySafe()[0] == "App (new_app_1)"); + auto read = FindPlanNodeByKv(plan, "Node Type", "Limit-TablePointLookup"); + auto& operators = read.GetMapSafe().at("Operators").GetArraySafe(); + UNIT_ASSERT(operators.size() == 2); + + auto& lookup = operators[1].GetMapSafe(); + UNIT_ASSERT(lookup.at("Name") == "TablePointLookup"); + UNIT_ASSERT(lookup.at("ReadRange").GetArraySafe()[0] == "App (new_app_1)"); } Y_UNIT_TEST(SortStage) { @@ -262,7 +262,7 @@ Y_UNIT_TEST_SUITE(KqpExplain) { NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - auto scanSort = FindPlanNodeByKv(plan, "Node Type", "Sort-TableRangesScan"); + auto scanSort = FindPlanNodeByKv(plan, "Node Type", "Sort-TableRangesScan"); UNIT_ASSERT(scanSort.IsDefined()); } @@ -370,17 +370,17 @@ Y_UNIT_TEST_SUITE(KqpExplain) { NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - bool containCte = false; - auto& plans = plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans"); - for (auto& planNode : plans.GetArraySafe()) { - auto& planMap = planNode.GetMapSafe(); - if (planMap.contains("Subplan Name") && planMap["Subplan Name"].GetStringSafe().Contains("CTE")) { - containCte = true; - break; - } - } - - UNIT_ASSERT(containCte); + bool containCte = false; + auto& plans = plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans"); + for (auto& planNode : plans.GetArraySafe()) { + auto& planMap = planNode.GetMapSafe(); + if (planMap.contains("Subplan Name") && planMap["Subplan Name"].GetStringSafe().Contains("CTE")) { + containCte = true; + break; + } + } + + UNIT_ASSERT(containCte); } Y_UNIT_TEST(SqlIn) { @@ -406,7 +406,7 @@ Y_UNIT_TEST_SUITE(KqpExplain) { NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - auto unionNode = FindPlanNodeByKv(plan, "Node Type", "Sort-Union"); + auto unionNode = FindPlanNodeByKv(plan, "Node Type", "Sort-Union"); UNIT_ASSERT_EQUAL(unionNode.GetMap().at("Plans").GetArraySafe().size(), 4); } @@ -442,12 +442,12 @@ Y_UNIT_TEST_SUITE(KqpExplain) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Name", "TableRangeScan"); - UNIT_ASSERT_EQUAL(node.GetMapSafe().at("Table").GetStringSafe(), "KeyValue"); - node = FindPlanNodeByKv(plan, "Name", "TableFullScan"); - UNIT_ASSERT_EQUAL(node.GetMapSafe().at("Table").GetStringSafe(), "KeyValue"); - node = FindPlanNodeByKv(plan, "Name", "TablePointLookup"); - UNIT_ASSERT_EQUAL(node.GetMapSafe().at("Table").GetStringSafe(), "KeyValue"); + auto node = FindPlanNodeByKv(plan, "Name", "TableRangeScan"); + UNIT_ASSERT_EQUAL(node.GetMapSafe().at("Table").GetStringSafe(), "KeyValue"); + node = FindPlanNodeByKv(plan, "Name", "TableFullScan"); + UNIT_ASSERT_EQUAL(node.GetMapSafe().at("Table").GetStringSafe(), "KeyValue"); + node = FindPlanNodeByKv(plan, "Name", "TablePointLookup"); + UNIT_ASSERT_EQUAL(node.GetMapSafe().at("Table").GetStringSafe(), "KeyValue"); } Y_UNIT_TEST(FewEffects) { @@ -471,37 +471,37 @@ Y_UNIT_TEST_SUITE(KqpExplain) { auto upsertsCount = CountPlanNodesByKv(plan, "Node Type", "Upsert-ConstantExpr"); UNIT_ASSERT_VALUES_EQUAL(upsertsCount, 2); - auto deletesCount = CountPlanNodesByKv(plan, "Node Type", "Delete-ConstantExpr"); - UNIT_ASSERT_VALUES_EQUAL(deletesCount, 1); - - auto fullScansCount = CountPlanNodesByKv(plan, "Node Type", "TableFullScan"); - UNIT_ASSERT_VALUES_EQUAL(fullScansCount, 1); - - auto rangeScansCount = CountPlanNodesByKv(plan, "Node Type", "TableRangeScan"); - UNIT_ASSERT_VALUES_EQUAL(rangeScansCount, 1); - - auto lookupsCount = CountPlanNodesByKv(plan, "Node Type", "TablePointLookup-ConstantExpr"); - UNIT_ASSERT_VALUES_EQUAL(lookupsCount, 3); - - /* check tables section */ - const auto& tableInfo = plan.GetMapSafe().at("tables").GetArraySafe()[0].GetMapSafe(); - UNIT_ASSERT_VALUES_EQUAL(tableInfo.at("name"), "/Root/EightShard"); - - THashMap<TString, int> counter; - auto countOperationsByType = [&tableInfo, &counter](const auto& type) { - for (const auto& op : tableInfo.at(type).GetArraySafe()) { - ++counter[op.GetMapSafe().at("type").GetStringSafe()]; - } - }; - - countOperationsByType("reads"); - countOperationsByType("writes"); - - UNIT_ASSERT_VALUES_EQUAL(counter["MultiUpsert"], upsertsCount); - UNIT_ASSERT_VALUES_EQUAL(counter["MultiErase"], deletesCount); - UNIT_ASSERT_VALUES_EQUAL(counter["FullScan"], fullScansCount); - UNIT_ASSERT_VALUES_EQUAL(counter["Scan"], rangeScansCount); - UNIT_ASSERT_VALUES_EQUAL(counter["Lookup"], lookupsCount); + auto deletesCount = CountPlanNodesByKv(plan, "Node Type", "Delete-ConstantExpr"); + UNIT_ASSERT_VALUES_EQUAL(deletesCount, 1); + + auto fullScansCount = CountPlanNodesByKv(plan, "Node Type", "TableFullScan"); + UNIT_ASSERT_VALUES_EQUAL(fullScansCount, 1); + + auto rangeScansCount = CountPlanNodesByKv(plan, "Node Type", "TableRangeScan"); + UNIT_ASSERT_VALUES_EQUAL(rangeScansCount, 1); + + auto lookupsCount = CountPlanNodesByKv(plan, "Node Type", "TablePointLookup-ConstantExpr"); + UNIT_ASSERT_VALUES_EQUAL(lookupsCount, 3); + + /* check tables section */ + const auto& tableInfo = plan.GetMapSafe().at("tables").GetArraySafe()[0].GetMapSafe(); + UNIT_ASSERT_VALUES_EQUAL(tableInfo.at("name"), "/Root/EightShard"); + + THashMap<TString, int> counter; + auto countOperationsByType = [&tableInfo, &counter](const auto& type) { + for (const auto& op : tableInfo.at(type).GetArraySafe()) { + ++counter[op.GetMapSafe().at("type").GetStringSafe()]; + } + }; + + countOperationsByType("reads"); + countOperationsByType("writes"); + + UNIT_ASSERT_VALUES_EQUAL(counter["MultiUpsert"], upsertsCount); + UNIT_ASSERT_VALUES_EQUAL(counter["MultiErase"], deletesCount); + UNIT_ASSERT_VALUES_EQUAL(counter["FullScan"], fullScansCount); + UNIT_ASSERT_VALUES_EQUAL(counter["Scan"], rangeScansCount); + UNIT_ASSERT_VALUES_EQUAL(counter["Lookup"], lookupsCount); } Y_UNIT_TEST(ExplainDataQueryWithParams) { @@ -524,43 +524,43 @@ Y_UNIT_TEST_SUITE(KqpExplain) { UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), EStatus::SUCCESS, result2.GetIssues().ToString()); } - Y_UNIT_TEST(FullOuterJoin) { - TKikimrRunner kikimr; - CreateSampleTables(kikimr); + Y_UNIT_TEST(FullOuterJoin) { + TKikimrRunner kikimr; + CreateSampleTables(kikimr); TStreamExecScanQuerySettings settings; - settings.Explain(true); + settings.Explain(true); auto db = kikimr.GetTableClient(); - + auto it = db.StreamExecuteScanQuery(R"( - SELECT l.Key, l.Text, l.Data, r.Value1, r.Value2 - FROM `/Root/EightShard` AS l FULL OUTER JOIN `/Root/FourShard` AS r - ON l.Key = r.Key - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - UNIT_ASSERT(res.PlanJson); - - NJson::TJsonValue plan; - NJson::ReadJsonTree(*res.PlanJson, &plan, true); - + SELECT l.Key, l.Text, l.Data, r.Value1, r.Value2 + FROM `/Root/EightShard` AS l FULL OUTER JOIN `/Root/FourShard` AS r + ON l.Key = r.Key + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + UNIT_ASSERT(res.PlanJson); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(*res.PlanJson, &plan, true); + auto join = FindPlanNodeByKv(plan, "Node Type", "FullJoin (JoinDict)"); - UNIT_ASSERT(join.IsDefined()); - auto left = FindPlanNodeByKv(join, "Table", "EightShard"); - UNIT_ASSERT(left.IsDefined()); - auto right = FindPlanNodeByKv(join, "Table", "FourShard"); - UNIT_ASSERT(right.IsDefined()); - } - - Y_UNIT_TEST(ReadTableRangesFullScan) { - TKikimrRunner kikimr; + UNIT_ASSERT(join.IsDefined()); + auto left = FindPlanNodeByKv(join, "Table", "EightShard"); + UNIT_ASSERT(left.IsDefined()); + auto right = FindPlanNodeByKv(join, "Table", "FourShard"); + UNIT_ASSERT(right.IsDefined()); + } + + Y_UNIT_TEST(ReadTableRangesFullScan) { + TKikimrRunner kikimr; TStreamExecScanQuerySettings settings; - settings.Explain(true); + settings.Explain(true); auto db = kikimr.GetTableClient(); - + auto session = db.CreateSession().GetValueSync().GetSession(); - + auto res = session.ExecuteSchemeQuery(R"( CREATE TABLE `/Root/TwoKeys` ( Key1 Int32, @@ -570,7 +570,7 @@ Y_UNIT_TEST_SUITE(KqpExplain) { ); )").GetValueSync(); UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); - + auto result = session.ExecuteDataQuery(R"( REPLACE INTO `TwoKeys` (Key1, Key2, Value) VALUES (1, 1, 1), @@ -583,7 +583,7 @@ Y_UNIT_TEST_SUITE(KqpExplain) { (1003, 103, 8); )", TTxControl::BeginTx().CommitTx()).GetValueSync(); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - + TVector<std::pair<TString, TString>> testData = { { "SELECT * FROM `/Root/TwoKeys`;", @@ -591,10 +591,10 @@ Y_UNIT_TEST_SUITE(KqpExplain) { }, { "SELECT * FROM `/Root/TwoKeys` WHERE Key2 > 101;", - "Filter-TableFullScan" + "Filter-TableFullScan" } }; - + for (auto& data: testData) { auto it = db.StreamExecuteScanQuery(data.first, settings).GetValueSync(); @@ -614,104 +614,104 @@ Y_UNIT_TEST_SUITE(KqpExplain) { auto expected = FindPlanNodeByKv(plan, "ReadRangesExpectedSize", ""); UNIT_ASSERT(!expected.IsDefined()); } - } - - Y_UNIT_TEST(ReadTableRanges) { - TKikimrRunner kikimr; - CreateSampleTables(kikimr); + } + + Y_UNIT_TEST(ReadTableRanges) { + TKikimrRunner kikimr; + CreateSampleTables(kikimr); TStreamExecScanQuerySettings settings; - settings.Explain(true); + settings.Explain(true); auto db = kikimr.GetTableClient(); - + auto it = db.StreamExecuteScanQuery(R"( - SELECT * FROM `/Root/KeyValue` - WHERE Key >= 2000 OR Key < 100; - )", settings).GetValueSync(); - - auto res = CollectStreamResult(it); - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - UNIT_ASSERT(res.PlanJson); - - NJson::TJsonValue plan; - NJson::ReadJsonTree(*res.PlanJson, &plan, true); - - auto read = FindPlanNodeByKv(plan, "Node Type", "TableRangesScan"); - UNIT_ASSERT(read.IsDefined()); + SELECT * FROM `/Root/KeyValue` + WHERE Key >= 2000 OR Key < 100; + )", settings).GetValueSync(); + + auto res = CollectStreamResult(it); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + UNIT_ASSERT(res.PlanJson); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(*res.PlanJson, &plan, true); + + auto read = FindPlanNodeByKv(plan, "Node Type", "TableRangesScan"); + UNIT_ASSERT(read.IsDefined()); auto keys = FindPlanNodeByKv(plan, "ReadRangesKeys", "[\"Key\"]"); UNIT_ASSERT(keys.IsDefined()); auto count = FindPlanNodeByKv(plan, "ReadRangesExpectedSize", "2"); UNIT_ASSERT(count.IsDefined()); - } - - Y_UNIT_TEST(Predicates) { - TKikimrRunner kikimr; - TStreamExecScanQuerySettings settings; - settings.Explain(true); - - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto res = session.ExecuteSchemeQuery(R"( - CREATE TABLE `/Root/TwoKeys` ( - Key1 Int32, - Key2 Int32, - Value Int64, - PRIMARY KEY (Key1, Key2) - ); - )").GetValueSync(); - UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); - - auto result = session.ExecuteDataQuery(R"( - REPLACE INTO `TwoKeys` (Key1, Key2, Value) VALUES - (1, 1, 1), - (2, 1, 2), - (3, 2, 3), - (4, 2, 4), - (1000, 100, 5), - (1001, 101, 6), - (1002, 102, 7), - (1003, 103, 8); - )", TTxControl::BeginTx().CommitTx()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - TVector<std::pair<TString, TString>> testData = { - { - "SELECT * FROM `/Root/TwoKeys` WHERE Value > 5 And Value <= 10", - "item.Value > 5 And item.Value <= 10" - }, - { - "SELECT * FROM `/Root/TwoKeys` WHERE Key2 < 100 Or Value == 5", - "item.Key2 < 100 Or item.Value == 5" - }, - { - "SELECT * FROM `/Root/TwoKeys` WHERE Key2 < 100 And Key2 >= 10 And Value != 5", - "item.Key2 < 100 And item.Key2 >= 10 And item.Value != 5" - }, - { - "SELECT * FROM `/Root/TwoKeys` WHERE Key2 < 10 Or Cast(Key2 As Int64) < Value", - "item.Key2 < 10 Or ..." - } - }; - - for (const auto& [query, predicate] : testData) { - auto it = db.StreamExecuteScanQuery(query, settings).GetValueSync(); - auto res = CollectStreamResult(it); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - UNIT_ASSERT(res.PlanJson); - - NJson::TJsonValue plan; - NJson::ReadJsonTree(*res.PlanJson, &plan, true); - - auto filter = FindPlanNodeByKv(plan, "Name", "Filter"); - UNIT_ASSERT(filter.IsDefined()); - UNIT_ASSERT_C(filter.GetMapSafe().at("Predicate") == predicate, - TStringBuilder() << "For query: " << query - << " expected predicate: " << predicate - << " but received: " << filter.GetMapSafe().at("Predicate")); - } - } + } + + Y_UNIT_TEST(Predicates) { + TKikimrRunner kikimr; + TStreamExecScanQuerySettings settings; + settings.Explain(true); + + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto res = session.ExecuteSchemeQuery(R"( + CREATE TABLE `/Root/TwoKeys` ( + Key1 Int32, + Key2 Int32, + Value Int64, + PRIMARY KEY (Key1, Key2) + ); + )").GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + + auto result = session.ExecuteDataQuery(R"( + REPLACE INTO `TwoKeys` (Key1, Key2, Value) VALUES + (1, 1, 1), + (2, 1, 2), + (3, 2, 3), + (4, 2, 4), + (1000, 100, 5), + (1001, 101, 6), + (1002, 102, 7), + (1003, 103, 8); + )", TTxControl::BeginTx().CommitTx()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + TVector<std::pair<TString, TString>> testData = { + { + "SELECT * FROM `/Root/TwoKeys` WHERE Value > 5 And Value <= 10", + "item.Value > 5 And item.Value <= 10" + }, + { + "SELECT * FROM `/Root/TwoKeys` WHERE Key2 < 100 Or Value == 5", + "item.Key2 < 100 Or item.Value == 5" + }, + { + "SELECT * FROM `/Root/TwoKeys` WHERE Key2 < 100 And Key2 >= 10 And Value != 5", + "item.Key2 < 100 And item.Key2 >= 10 And item.Value != 5" + }, + { + "SELECT * FROM `/Root/TwoKeys` WHERE Key2 < 10 Or Cast(Key2 As Int64) < Value", + "item.Key2 < 10 Or ..." + } + }; + + for (const auto& [query, predicate] : testData) { + auto it = db.StreamExecuteScanQuery(query, settings).GetValueSync(); + auto res = CollectStreamResult(it); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + UNIT_ASSERT(res.PlanJson); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(*res.PlanJson, &plan, true); + + auto filter = FindPlanNodeByKv(plan, "Name", "Filter"); + UNIT_ASSERT(filter.IsDefined()); + UNIT_ASSERT_C(filter.GetMapSafe().at("Predicate") == predicate, + TStringBuilder() << "For query: " << query + << " expected predicate: " << predicate + << " but received: " << filter.GetMapSafe().at("Predicate")); + } + } } } // namespace NKqp diff --git a/ydb/core/kqp/ut/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/kqp_indexes_ut.cpp index ec1254dcea8..505cf09e6b8 100644 --- a/ydb/core/kqp/ut/kqp_indexes_ut.cpp +++ b/ydb/core/kqp/ut/kqp_indexes_ut.cpp @@ -11,8 +11,8 @@ #include <ydb/library/yql/core/services/mounts/yql_mounts.h> #include <ydb/library/yql/providers/common/provider/yql_provider.h> -#include <library/cpp/json/json_reader.h> - +#include <library/cpp/json/json_reader.h> + #include <util/string/printf.h> namespace NKikimr { @@ -51,7 +51,7 @@ TIntrusivePtr<IKqpHost> CreateKikimrQueryProcessor(TIntrusivePtr<IKqpGateway> ga UNIT_ASSERT(TryParseFromTextFormat(defaultSettingsStream, defaultSettings)); kikimrConfig->Init(defaultSettings.GetDefaultSettings(), cluster, settings, true); - return NKqp::CreateKqpHost(gateway, cluster, "/Root", kikimrConfig, moduleResolver, funcRegistry, + return NKqp::CreateKqpHost(gateway, cluster, "/Root", kikimrConfig, moduleResolver, funcRegistry, keepConfigChanges); } @@ -74,10 +74,10 @@ void CreateTableWithIndexWithState( using NYql::TKikimrColumnMetadata; TKikimrTableMetadataPtr metadata = new TKikimrTableMetadata(TestCluster, server.GetSettings().DomainName + "/IndexedTableWithState"); - metadata->Columns["key"] = TKikimrColumnMetadata("key", 0, "Uint32", false); - metadata->Columns["value"] = TKikimrColumnMetadata("value", 0, "String", false); - metadata->Columns["value2"] = TKikimrColumnMetadata("value2", 0, "String", false); - metadata->ColumnOrder = {"key", "value", "value2"}; + metadata->Columns["key"] = TKikimrColumnMetadata("key", 0, "Uint32", false); + metadata->Columns["value"] = TKikimrColumnMetadata("value", 0, "String", false); + metadata->Columns["value2"] = TKikimrColumnMetadata("value2", 0, "String", false); + metadata->ColumnOrder = {"key", "value", "value2"}; metadata->Indexes.push_back( NYql::TIndexDescription( indexName, @@ -3160,19 +3160,19 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda ("Primary1", "Val1", "Val2"); )")); - auto explainResult = session.ExplainDataQuery(query1).ExtractValueSync(); - UNIT_ASSERT_C(explainResult.IsSuccess(), explainResult.GetIssues().ToString()); - - NJson::TJsonValue plan; - NJson::ReadJsonTree(explainResult.GetPlan(), &plan, true); - - UNIT_ASSERT(plan.GetMapSafe().contains("tables")); - const auto& tables = plan.GetMapSafe().at("tables").GetArraySafe(); - UNIT_ASSERT(tables.size() == 3); - UNIT_ASSERT(tables.at(0).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable"); - UNIT_ASSERT(tables.at(1).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable/Index1/indexImplTable"); - UNIT_ASSERT(tables.at(2).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable/Index2/indexImplTable"); - + auto explainResult = session.ExplainDataQuery(query1).ExtractValueSync(); + UNIT_ASSERT_C(explainResult.IsSuccess(), explainResult.GetIssues().ToString()); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(explainResult.GetPlan(), &plan, true); + + UNIT_ASSERT(plan.GetMapSafe().contains("tables")); + const auto& tables = plan.GetMapSafe().at("tables").GetArraySafe(); + UNIT_ASSERT(tables.size() == 3); + UNIT_ASSERT(tables.at(0).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable"); + UNIT_ASSERT(tables.at(1).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable/Index1/indexImplTable"); + UNIT_ASSERT(tables.at(2).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable/Index2/indexImplTable"); + auto result = session.ExecuteDataQuery( query1, TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) @@ -4438,18 +4438,18 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda SELECT Key, Index2, Value FROM AS_TABLE($rows); )")); - auto explainResult = session.ExplainDataQuery(query).ExtractValueSync(); - UNIT_ASSERT_C(explainResult.IsSuccess(), explainResult.GetIssues().ToString()); - - NJson::TJsonValue plan; - NJson::ReadJsonTree(explainResult.GetPlan(), &plan, true); - - UNIT_ASSERT(plan.GetMapSafe().contains("tables")); - const auto& tables = plan.GetMapSafe().at("tables").GetArraySafe(); - UNIT_ASSERT(tables.size() == 2); - UNIT_ASSERT(tables.at(0).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable"); - UNIT_ASSERT(tables.at(1).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable/Index/indexImplTable"); - + auto explainResult = session.ExplainDataQuery(query).ExtractValueSync(); + UNIT_ASSERT_C(explainResult.IsSuccess(), explainResult.GetIssues().ToString()); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(explainResult.GetPlan(), &plan, true); + + UNIT_ASSERT(plan.GetMapSafe().contains("tables")); + const auto& tables = plan.GetMapSafe().at("tables").GetArraySafe(); + UNIT_ASSERT(tables.size() == 2); + UNIT_ASSERT(tables.at(0).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable"); + UNIT_ASSERT(tables.at(1).GetMapSafe().at("name").GetStringSafe() == "/Root/TestTable/Index/indexImplTable"); + auto qId = session.PrepareDataQuery(query).ExtractValueSync().GetQuery(); auto params = qId.GetParamsBuilder() diff --git a/ydb/core/kqp/ut/kqp_limits_ut.cpp b/ydb/core/kqp/ut/kqp_limits_ut.cpp index 1406c734d32..0e2478dad79 100644 --- a/ydb/core/kqp/ut/kqp_limits_ut.cpp +++ b/ydb/core/kqp/ut/kqp_limits_ut.cpp @@ -373,9 +373,9 @@ Y_UNIT_TEST_SUITE(KqpLimits) { UNIT_ASSERT_VALUES_EQUAL_C(prepareResult.GetStatus(), EStatus::SUCCESS, prepareResult.GetIssues().ToString()); auto dataQuery = prepareResult.GetQuery(); - auto settings = TExecDataQuerySettings() - .OperationTimeout(TDuration::MilliSeconds(500)); - auto result = dataQuery.Execute(TTxControl::BeginTx().CommitTx(), settings).GetValueSync(); + auto settings = TExecDataQuerySettings() + .OperationTimeout(TDuration::MilliSeconds(500)); + auto result = dataQuery.Execute(TTxControl::BeginTx().CommitTx(), settings).GetValueSync(); result.GetIssues().PrintTo(Cerr); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::TIMEOUT); diff --git a/ydb/core/kqp/ut/kqp_newengine_ut.cpp b/ydb/core/kqp/ut/kqp_newengine_ut.cpp index 12087262b6f..3737687d8d6 100644 --- a/ydb/core/kqp/ut/kqp_newengine_ut.cpp +++ b/ydb/core/kqp/ut/kqp_newengine_ut.cpp @@ -1960,19 +1960,19 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { query << " order by Key"; // redundant sort by PK } - const int limitValue = 5; - const int offsetValue = 1; - + const int limitValue = 5; + const int offsetValue = 1; + switch (takeType) { case None: UNIT_FAIL("Unexpected"); break; - case Literal: query << " limit " << limitValue; break; + case Literal: query << " limit " << limitValue; break; case Parameter: query << " limit $limit"; break; - case Expression: query << " limit ($limit + 1)"; break; + case Expression: query << " limit ($limit + 1)"; break; } switch (skipType) { case None: break; - case Literal: query << " offset " << offsetValue; break; + case Literal: query << " offset " << offsetValue; break; case Parameter: query << " offset $offset"; break; case Expression: query << " offset ($offset + 1)"; break; } @@ -1985,20 +1985,20 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); - UNIT_ASSERT(node.IsDefined()); - - TStringBuilder readLimitValue; - if (takeType == EType::Literal && skipType == EType::None) { - readLimitValue << limitValue; - } else if (takeType == EType::Literal && skipType == EType::Literal) { - readLimitValue << limitValue + offsetValue; - } else { - readLimitValue << "%kqp%tx_result_binding_0_0"; - } - - auto readLimit = FindPlanNodeByKv(node, "ReadLimit", readLimitValue); - UNIT_ASSERT(readLimit.IsDefined()); + auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); + UNIT_ASSERT(node.IsDefined()); + + TStringBuilder readLimitValue; + if (takeType == EType::Literal && skipType == EType::None) { + readLimitValue << limitValue; + } else if (takeType == EType::Literal && skipType == EType::Literal) { + readLimitValue << limitValue + offsetValue; + } else { + readLimitValue << "%kqp%tx_result_binding_0_0"; + } + + auto readLimit = FindPlanNodeByKv(node, "ReadLimit", readLimitValue); + UNIT_ASSERT(readLimit.IsDefined()); } } } @@ -2175,25 +2175,25 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) { ] )", FormatResultSetYson(result.GetResultSet(0))); } - - Y_UNIT_TEST(ReadRangeWithParams) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto result = session.ExplainDataQuery(R"( - PRAGMA Kikimr.UseNewEngine = "true"; - DECLARE $max_key as Uint64; - DECLARE $min_key as Uint64; - SELECT * FROM `/Root/KeyValue` WHERE Key <= $max_key AND Key >= $min_key; - )").ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - NJson::TJsonValue plan; - NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto range = FindPlanNodeByKv(plan, "ReadRange", "[\"Key [$min_key, $max_key]\"]"); - UNIT_ASSERT(range.IsDefined()); - } + + Y_UNIT_TEST(ReadRangeWithParams) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExplainDataQuery(R"( + PRAGMA Kikimr.UseNewEngine = "true"; + DECLARE $max_key as Uint64; + DECLARE $min_key as Uint64; + SELECT * FROM `/Root/KeyValue` WHERE Key <= $max_key AND Key >= $min_key; + )").ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + NJson::TJsonValue plan; + NJson::ReadJsonTree(result.GetPlan(), &plan, true); + auto range = FindPlanNodeByKv(plan, "ReadRange", "[\"Key [$min_key, $max_key]\"]"); + UNIT_ASSERT(range.IsDefined()); + } Y_UNIT_TEST(Nondeterministic) { // TODO: KIKIMR-4759 TKikimrRunner kikimr; diff --git a/ydb/core/kqp/ut/kqp_not_null_columns_ut.cpp b/ydb/core/kqp/ut/kqp_not_null_columns_ut.cpp index 1266673b16d..104c3143cf6 100644 --- a/ydb/core/kqp/ut/kqp_not_null_columns_ut.cpp +++ b/ydb/core/kqp/ut/kqp_not_null_columns_ut.cpp @@ -1,661 +1,661 @@ #include <ydb/core/kqp/ut/common/kqp_ut_common.h> - -namespace NKikimr { -namespace NKqp { - -using namespace NYdb; -using namespace NYdb::NTable; - -Y_UNIT_TEST_SUITE(KqpNotNullColumns) { - Y_UNIT_TEST_NEW_ENGINE(InsertNotNullPk) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q_(R"( - CREATE TABLE `/Root/TestInsertNotNullPk` ( - Key Uint64 NOT NULL, - Value String, - PRIMARY KEY (Key)) - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const auto query = Q_("INSERT INTO `/Root/TestInsertNotNullPk` (Key, Value) VALUES (1, 'Value1')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* missing not null pk column */ - const auto query = Q_("INSERT INTO `/Root/TestInsertNotNullPk` (Value) VALUES ('Value2')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED), result.GetIssues().ToString()); - } - - { /* set NULL to not null pk column */ - const auto query = Q_("INSERT INTO `/Root/TestInsertNotNullPk` (Key, Value) VALUES (NULL, 'Value3')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST_NEW_ENGINE(UpsertNotNullPk) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q_(R"( - CREATE TABLE `/Root/TestUpsertNotNullPk` ( - Key Uint64 NOT NULL, - Value String, - PRIMARY KEY (Key)) - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const auto query = Q_("UPSERT INTO `/Root/TestUpsertNotNullPk` (Key, Value) VALUES (1, 'Value1')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* missing not null pk column */ - const auto query = Q_("UPSERT INTO `/Root/TestUpsertNotNullPk` (Value) VALUES ('Value2')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED), result.GetIssues().ToString()); - } - - { /* set NULL to not null pk column */ - const auto query = Q_("UPSERT INTO `/Root/TestUpsertNotNullPk` (Key, Value) VALUES (NULL, 'Value3')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST_NEW_ENGINE(ReplaceNotNullPk) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q_(R"( - CREATE TABLE `/Root/TestReplaceNotNullPk` ( - Key Uint64 NOT NULL, - Value String, - PRIMARY KEY (Key)) - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const auto query = Q_("REPLACE INTO `/Root/TestReplaceNotNullPk` (Key, Value) VALUES (1, 'Value1')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* missing not null pk column */ - const auto query = Q_("REPLACE INTO `/Root/TestReplaceNotNullPk` (Value) VALUES ('Value2')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED), result.GetIssues().ToString()); - } - - { /* set NULL to not null pk column */ - const auto query = Q_("REPLACE INTO `/Root/TestReplaceNotNullPk` (Key, Value) VALUES (NULL, 'Value3')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST_NEW_ENGINE(UpdateNotNullPk) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q_(R"( - CREATE TABLE `/Root/TestUpdateNotNullPk` ( - Key Uint64 NOT NULL, - Value String, - PRIMARY KEY (Key)) - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* init table */ - const auto query = Q_(R"( - REPLACE INTO `/Root/TestUpdateNotNullPk` (Key, Value) VALUES - (1, 'Value1'), - (2, 'Value2'), - (3, 'Value3'); - )"); - - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* update not null pk column */ - const auto query = Q_("UPDATE `/Root/TestUpdateNotNullPk` SET Key = 10 WHERE Key = 1"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - } - - { /* set NULL to not null pk column */ - const auto query = Q_("UPDATE `/Root/TestUpdateNotNullPk` SET Key = NULL WHERE Key = 1"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST_NEW_ENGINE(InsertNotNull) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q_(R"( - CREATE TABLE `/Root/TestInsertNotNull` ( - Key Uint64, - Value String NOT NULL, - PRIMARY KEY (Key)) - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const auto query = Q_("INSERT INTO `/Root/TestInsertNotNull` (Key, Value) VALUES (1, 'Value1')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* missing not null column */ - const auto query = Q_("INSERT INTO `/Root/TestInsertNotNull` (Key) VALUES (2)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), result.GetIssues().ToString()); - } - - { /* set NULL to not null column */ - const auto query = Q_("INSERT INTO `/Root/TestInsertNotNull` (Key, Value) VALUES (3, NULL)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST_NEW_ENGINE(UpsertNotNull) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q_(R"( - CREATE TABLE `/Root/TestUpsertNotNull` ( - Key Uint64, - Value String NOT NULL, - PRIMARY KEY (Key)) - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const auto query = Q_("UPSERT INTO `/Root/TestUpsertNotNull` (Key, Value) VALUES (1, 'Value1')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* missing not null column */ - const auto query = Q_("UPSERT INTO `/Root/TestUpsertNotNull` (Key) VALUES (2)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), result.GetIssues().ToString()); - } - - { /* set NULL to not null column */ - const auto query = Q_("UPSERT INTO `/Root/TestUpsertNotNull` (Key, Value) VALUES (3, NULL)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST_NEW_ENGINE(ReplaceNotNull) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q_(R"( - CREATE TABLE `/Root/TestReplaceNotNull` ( - Key Uint64, - Value String NOT NULL, - PRIMARY KEY (Key)) - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const auto query = Q_("REPLACE INTO `/Root/TestReplaceNotNull` (Key, Value) VALUES (1, 'Value1')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* missing not null column */ - const auto query = Q_("REPLACE INTO `/Root/TestReplaceNotNull` (Key) VALUES (2)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), result.GetIssues().ToString()); - } - - { /* set NULL to not null column */ - const auto query = Q_("REPLACE INTO `/Root/TestReplaceNotNull` (Key, Value) VALUES (3, NULL)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST_NEW_ENGINE(UpdateNotNull) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q_(R"( - CREATE TABLE `/Root/TestUpdateNotNull` ( - Key Uint64, - Value String NOT NULL, - PRIMARY KEY (Key)) - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* init table */ - const auto query = Q_(R"( - REPLACE INTO `/Root/TestUpdateNotNull` (Key, Value) VALUES - (1, 'Value1'), - (2, 'Value2'), - (3, 'Value3'); - )"); - - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* update not null column */ - const auto query = Q_("UPDATE `/Root/TestUpdateNotNull` SET Value = 'NewValue1' WHERE Key = 1"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* set NULL to not null column */ - const auto query = Q_("UPDATE `/Root/TestUpdateNotNull` SET Value = NULL WHERE Key = 1"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST_NEW_ENGINE(UpdateOnNotNull) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q_(R"( - CREATE TABLE `/Root/TestUpdateOnNotNull` ( - Key Uint64, - Value String NOT NULL, - PRIMARY KEY (Key)) - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* init table */ - const auto query = Q_(R"( - REPLACE INTO `/Root/TestUpdateOnNotNull` (Key, Value) VALUES - (1, 'Value1'), - (2, 'Value2'), - (3, 'Value3'); - )"); - - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* update not null column */ - const auto query = Q_("UPDATE `/Root/TestUpdateOnNotNull` ON (Key, Value) VALUES (2, 'NewValue2')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* set NULL to not null column */ - const auto query = Q_("UPDATE `/Root/TestUpdateOnNotNull` ON (Key, Value) VALUES (2, NULL)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST_NEW_ENGINE(AlterAddNotNullColumn) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q_(R"( - CREATE TABLE `/Root/TestAddNotNullColumn` ( - Key Uint64, - Value1 String, - PRIMARY KEY (Key)) - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const auto query = Q_("ALTER TABLE `/Root/TestAddNotNullColumn` ADD COLUMN Value2 String NOT NULL"); - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { /* set NULL to not null column */ - const auto query = Q_("UPSERT INTO `/Root/TestAddNotNullColumn` (Key, Value1, Value2) VALUES (1, 'Value1', NULL)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST_NEW_ENGINE(AlterDropNotNullColumn) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q_(R"( - CREATE TABLE `/Root/TestDropNotNullColumn` ( - Key Uint64, - Value1 String, - Value2 String NOT NULL, - PRIMARY KEY (Key)) - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const auto query = Q_("ALTER TABLE `/Root/TestDropNotNullColumn` DROP COLUMN Value2"); - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST_NEW_ENGINE(FailedMultiEffects) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q_(R"( - CREATE TABLE `/Root/TestNotNull` ( - Key Uint64 NOT NULL, - Value String NOT NULL, - PRIMARY KEY (Key)) - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const auto query = Q_("REPLACE INTO `/Root/TestNotNull` (Key, Value) VALUES (1, 'Value1')"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const auto query = Q_(R"( - UPDATE `/Root/TestNotNull` SET Value = 'NewValue1' WHERE Key = 1; - UPSERT INTO `/Root/TestNotNull` (Key, Value) VALUES (2, NULL); - )"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); - - auto yson = ReadTablePartToYson(session, "/Root/TestNotNull"); - CompareYson(R"([[[1u];["Value1"]]])", yson); - } - } - - Y_UNIT_TEST_NEW_ENGINE(SecondaryKeyWithNotNullColumn) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q1_(R"( - CREATE TABLE `/Root/TestNotNullSecondaryKey` ( - Key1 Uint64 NOT NULL, - Key2 Uint64 NOT NULL, - Key3 Uint64, - Value String, - PRIMARY KEY (Key1), - INDEX Index GLOBAL ON (Key2, Key3)); - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - /* upsert row and update not null secondary Key2 */ - { - const auto query = Q_(R"( - UPSERT INTO `/Root/TestNotNullSecondaryKey` (Key1, Key2, Key3, Value) VALUES (1, 11, 111, 'Value1') - )"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const auto query = Q_("UPDATE `/Root/TestNotNullSecondaryKey` SET Key2 = NULL WHERE Key1 = 1"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), - result.GetIssues().ToString()); - } - - { - auto yson = ReadTablePartToYson(session, "/Root/TestNotNullSecondaryKey/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(yson, R"([[[11u];[111u];[1u]]])"); - } - - /* missing not null secondary Key2 */ - { - const auto query = Q_("INSERT INTO `/Root/TestNotNullSecondaryKey` (Key1) VALUES (2)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), - result.GetIssues().ToString()); - } - - { - const auto query = Q_("UPSERT INTO `/Root/TestNotNullSecondaryKey` (Key1) VALUES (3)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), - result.GetIssues().ToString()); - } - - { - const auto query = Q_("REPLACE INTO `/Root/TestNotNullSecondaryKey` (Key1) VALUES (4)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), - result.GetIssues().ToString()); - } - - /* set NULL to not null secondary Key2 */ - { - const auto query = Q_(R"( - INSERT INTO `/Root/TestNotNullSecondaryKey` (Key1, Key2, Value) VALUES (5, NULL, 'Value5') - )"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), - result.GetIssues().ToString()); - } - - { - const auto query = Q_(R"( - UPSERT INTO `/Root/TestNotNullSecondaryKey` (Key1, Key2, Value) VALUES (6, NULL, 'Value6') - )"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), - result.GetIssues().ToString()); - } - - { - const auto query = Q_(R"( - REPLACE INTO `/Root/TestNotNullSecondaryKey` (Key1, Key2, Value) VALUES (7, NULL, 'Value7') - )"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), - result.GetIssues().ToString()); - } - } - - Y_UNIT_TEST_NEW_ENGINE(SecondaryIndexWithNotNullDataColumn) { - TKikimrRunner kikimr; - auto client = kikimr.GetTableClient(); - auto session = client.CreateSession().GetValueSync().GetSession(); - - { - const auto query = Q1_(R"( - CREATE TABLE `/Root/TestNotNullSecondaryIndex` ( - Key Uint64 NOT NULL, - Value String, - Index1 String NOT NULL, - Index2 String, - PRIMARY KEY (Key), - INDEX Index GLOBAL ON (Index1, Index2) - COVER (Value)); - )"); - - auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - /* upsert row and update not null index column Index1 */ - { - const auto query = Q_(R"( - UPSERT INTO `/Root/TestNotNullSecondaryIndex` (Key, Index1, Index2) - VALUES (1, 'Secondary1', 'Secondary11') - )"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const auto query = Q_("UPDATE `/Root/TestNotNullSecondaryIndex` SET Index1 = NULL WHERE Key = 1"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), - result.GetIssues().ToString()); - } - - { - auto yson = ReadTablePartToYson(session, "/Root/TestNotNullSecondaryIndex/Index/indexImplTable"); - UNIT_ASSERT_VALUES_EQUAL(yson, R"([[["Secondary1"];["Secondary11"];[1u];#]])"); - } - - /* missing not null index column Index1 */ - { - const auto query = Q_("INSERT INTO `/Root/TestNotNullSecondaryIndex` (Key) VALUES (2)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), - result.GetIssues().ToString()); - } - - { - const auto query = Q_("UPSERT INTO `/Root/TestNotNullSecondaryIndex` (Key) VALUES (3)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), - result.GetIssues().ToString()); - } - - { - const auto query = Q_("REPLACE INTO `/Root/TestNotNullSecondaryIndex` (Key) VALUES (4)"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), - result.GetIssues().ToString()); - } - - /* set NULL to not null index column Index1 */ - { - const auto query = Q_(R"( - INSERT INTO `/Root/TestNotNullSecondaryIndex` (Key, Value, Index1) VALUES (5, 'Value5', NULL) - )"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), - result.GetIssues().ToString()); - } - - { - const auto query = Q_(R"( - UPSERT INTO `/Root/TestNotNullSecondaryIndex` (Key, Value, Index1) VALUES (6, 'Value6', NULL) - )"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), - result.GetIssues().ToString()); - } - - { - const auto query = Q_(R"( - REPLACE INTO `/Root/TestNotNullSecondaryIndex` (Key, Value, Index1) VALUES (7, 'Value7', NULL) - )"); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT(!result.IsSuccess()); - UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), - result.GetIssues().ToString()); - } - } -} - -} // namespace NKqp -} // namespace NKikimr + +namespace NKikimr { +namespace NKqp { + +using namespace NYdb; +using namespace NYdb::NTable; + +Y_UNIT_TEST_SUITE(KqpNotNullColumns) { + Y_UNIT_TEST_NEW_ENGINE(InsertNotNullPk) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q_(R"( + CREATE TABLE `/Root/TestInsertNotNullPk` ( + Key Uint64 NOT NULL, + Value String, + PRIMARY KEY (Key)) + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_("INSERT INTO `/Root/TestInsertNotNullPk` (Key, Value) VALUES (1, 'Value1')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* missing not null pk column */ + const auto query = Q_("INSERT INTO `/Root/TestInsertNotNullPk` (Value) VALUES ('Value2')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED), result.GetIssues().ToString()); + } + + { /* set NULL to not null pk column */ + const auto query = Q_("INSERT INTO `/Root/TestInsertNotNullPk` (Key, Value) VALUES (NULL, 'Value3')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST_NEW_ENGINE(UpsertNotNullPk) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q_(R"( + CREATE TABLE `/Root/TestUpsertNotNullPk` ( + Key Uint64 NOT NULL, + Value String, + PRIMARY KEY (Key)) + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_("UPSERT INTO `/Root/TestUpsertNotNullPk` (Key, Value) VALUES (1, 'Value1')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* missing not null pk column */ + const auto query = Q_("UPSERT INTO `/Root/TestUpsertNotNullPk` (Value) VALUES ('Value2')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED), result.GetIssues().ToString()); + } + + { /* set NULL to not null pk column */ + const auto query = Q_("UPSERT INTO `/Root/TestUpsertNotNullPk` (Key, Value) VALUES (NULL, 'Value3')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST_NEW_ENGINE(ReplaceNotNullPk) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q_(R"( + CREATE TABLE `/Root/TestReplaceNotNullPk` ( + Key Uint64 NOT NULL, + Value String, + PRIMARY KEY (Key)) + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_("REPLACE INTO `/Root/TestReplaceNotNullPk` (Key, Value) VALUES (1, 'Value1')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* missing not null pk column */ + const auto query = Q_("REPLACE INTO `/Root/TestReplaceNotNullPk` (Value) VALUES ('Value2')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED), result.GetIssues().ToString()); + } + + { /* set NULL to not null pk column */ + const auto query = Q_("REPLACE INTO `/Root/TestReplaceNotNullPk` (Key, Value) VALUES (NULL, 'Value3')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST_NEW_ENGINE(UpdateNotNullPk) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q_(R"( + CREATE TABLE `/Root/TestUpdateNotNullPk` ( + Key Uint64 NOT NULL, + Value String, + PRIMARY KEY (Key)) + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* init table */ + const auto query = Q_(R"( + REPLACE INTO `/Root/TestUpdateNotNullPk` (Key, Value) VALUES + (1, 'Value1'), + (2, 'Value2'), + (3, 'Value3'); + )"); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* update not null pk column */ + const auto query = Q_("UPDATE `/Root/TestUpdateNotNullPk` SET Key = 10 WHERE Key = 1"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + } + + { /* set NULL to not null pk column */ + const auto query = Q_("UPDATE `/Root/TestUpdateNotNullPk` SET Key = NULL WHERE Key = 1"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST_NEW_ENGINE(InsertNotNull) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q_(R"( + CREATE TABLE `/Root/TestInsertNotNull` ( + Key Uint64, + Value String NOT NULL, + PRIMARY KEY (Key)) + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_("INSERT INTO `/Root/TestInsertNotNull` (Key, Value) VALUES (1, 'Value1')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* missing not null column */ + const auto query = Q_("INSERT INTO `/Root/TestInsertNotNull` (Key) VALUES (2)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), result.GetIssues().ToString()); + } + + { /* set NULL to not null column */ + const auto query = Q_("INSERT INTO `/Root/TestInsertNotNull` (Key, Value) VALUES (3, NULL)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST_NEW_ENGINE(UpsertNotNull) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q_(R"( + CREATE TABLE `/Root/TestUpsertNotNull` ( + Key Uint64, + Value String NOT NULL, + PRIMARY KEY (Key)) + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_("UPSERT INTO `/Root/TestUpsertNotNull` (Key, Value) VALUES (1, 'Value1')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* missing not null column */ + const auto query = Q_("UPSERT INTO `/Root/TestUpsertNotNull` (Key) VALUES (2)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), result.GetIssues().ToString()); + } + + { /* set NULL to not null column */ + const auto query = Q_("UPSERT INTO `/Root/TestUpsertNotNull` (Key, Value) VALUES (3, NULL)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST_NEW_ENGINE(ReplaceNotNull) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q_(R"( + CREATE TABLE `/Root/TestReplaceNotNull` ( + Key Uint64, + Value String NOT NULL, + PRIMARY KEY (Key)) + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_("REPLACE INTO `/Root/TestReplaceNotNull` (Key, Value) VALUES (1, 'Value1')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* missing not null column */ + const auto query = Q_("REPLACE INTO `/Root/TestReplaceNotNull` (Key) VALUES (2)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), result.GetIssues().ToString()); + } + + { /* set NULL to not null column */ + const auto query = Q_("REPLACE INTO `/Root/TestReplaceNotNull` (Key, Value) VALUES (3, NULL)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST_NEW_ENGINE(UpdateNotNull) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q_(R"( + CREATE TABLE `/Root/TestUpdateNotNull` ( + Key Uint64, + Value String NOT NULL, + PRIMARY KEY (Key)) + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* init table */ + const auto query = Q_(R"( + REPLACE INTO `/Root/TestUpdateNotNull` (Key, Value) VALUES + (1, 'Value1'), + (2, 'Value2'), + (3, 'Value3'); + )"); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* update not null column */ + const auto query = Q_("UPDATE `/Root/TestUpdateNotNull` SET Value = 'NewValue1' WHERE Key = 1"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* set NULL to not null column */ + const auto query = Q_("UPDATE `/Root/TestUpdateNotNull` SET Value = NULL WHERE Key = 1"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST_NEW_ENGINE(UpdateOnNotNull) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q_(R"( + CREATE TABLE `/Root/TestUpdateOnNotNull` ( + Key Uint64, + Value String NOT NULL, + PRIMARY KEY (Key)) + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* init table */ + const auto query = Q_(R"( + REPLACE INTO `/Root/TestUpdateOnNotNull` (Key, Value) VALUES + (1, 'Value1'), + (2, 'Value2'), + (3, 'Value3'); + )"); + + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* update not null column */ + const auto query = Q_("UPDATE `/Root/TestUpdateOnNotNull` ON (Key, Value) VALUES (2, 'NewValue2')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* set NULL to not null column */ + const auto query = Q_("UPDATE `/Root/TestUpdateOnNotNull` ON (Key, Value) VALUES (2, NULL)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST_NEW_ENGINE(AlterAddNotNullColumn) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q_(R"( + CREATE TABLE `/Root/TestAddNotNullColumn` ( + Key Uint64, + Value1 String, + PRIMARY KEY (Key)) + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_("ALTER TABLE `/Root/TestAddNotNullColumn` ADD COLUMN Value2 String NOT NULL"); + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { /* set NULL to not null column */ + const auto query = Q_("UPSERT INTO `/Root/TestAddNotNullColumn` (Key, Value1, Value2) VALUES (1, 'Value1', NULL)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST_NEW_ENGINE(AlterDropNotNullColumn) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q_(R"( + CREATE TABLE `/Root/TestDropNotNullColumn` ( + Key Uint64, + Value1 String, + Value2 String NOT NULL, + PRIMARY KEY (Key)) + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_("ALTER TABLE `/Root/TestDropNotNullColumn` DROP COLUMN Value2"); + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST_NEW_ENGINE(FailedMultiEffects) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q_(R"( + CREATE TABLE `/Root/TestNotNull` ( + Key Uint64 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key)) + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_("REPLACE INTO `/Root/TestNotNull` (Key, Value) VALUES (1, 'Value1')"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_(R"( + UPDATE `/Root/TestNotNull` SET Value = 'NewValue1' WHERE Key = 1; + UPSERT INTO `/Root/TestNotNull` (Key, Value) VALUES (2, NULL); + )"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), result.GetIssues().ToString()); + + auto yson = ReadTablePartToYson(session, "/Root/TestNotNull"); + CompareYson(R"([[[1u];["Value1"]]])", yson); + } + } + + Y_UNIT_TEST_NEW_ENGINE(SecondaryKeyWithNotNullColumn) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q1_(R"( + CREATE TABLE `/Root/TestNotNullSecondaryKey` ( + Key1 Uint64 NOT NULL, + Key2 Uint64 NOT NULL, + Key3 Uint64, + Value String, + PRIMARY KEY (Key1), + INDEX Index GLOBAL ON (Key2, Key3)); + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + /* upsert row and update not null secondary Key2 */ + { + const auto query = Q_(R"( + UPSERT INTO `/Root/TestNotNullSecondaryKey` (Key1, Key2, Key3, Value) VALUES (1, 11, 111, 'Value1') + )"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_("UPDATE `/Root/TestNotNullSecondaryKey` SET Key2 = NULL WHERE Key1 = 1"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), + result.GetIssues().ToString()); + } + + { + auto yson = ReadTablePartToYson(session, "/Root/TestNotNullSecondaryKey/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(yson, R"([[[11u];[111u];[1u]]])"); + } + + /* missing not null secondary Key2 */ + { + const auto query = Q_("INSERT INTO `/Root/TestNotNullSecondaryKey` (Key1) VALUES (2)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), + result.GetIssues().ToString()); + } + + { + const auto query = Q_("UPSERT INTO `/Root/TestNotNullSecondaryKey` (Key1) VALUES (3)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), + result.GetIssues().ToString()); + } + + { + const auto query = Q_("REPLACE INTO `/Root/TestNotNullSecondaryKey` (Key1) VALUES (4)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), + result.GetIssues().ToString()); + } + + /* set NULL to not null secondary Key2 */ + { + const auto query = Q_(R"( + INSERT INTO `/Root/TestNotNullSecondaryKey` (Key1, Key2, Value) VALUES (5, NULL, 'Value5') + )"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), + result.GetIssues().ToString()); + } + + { + const auto query = Q_(R"( + UPSERT INTO `/Root/TestNotNullSecondaryKey` (Key1, Key2, Value) VALUES (6, NULL, 'Value6') + )"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), + result.GetIssues().ToString()); + } + + { + const auto query = Q_(R"( + REPLACE INTO `/Root/TestNotNullSecondaryKey` (Key1, Key2, Value) VALUES (7, NULL, 'Value7') + )"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), + result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST_NEW_ENGINE(SecondaryIndexWithNotNullDataColumn) { + TKikimrRunner kikimr; + auto client = kikimr.GetTableClient(); + auto session = client.CreateSession().GetValueSync().GetSession(); + + { + const auto query = Q1_(R"( + CREATE TABLE `/Root/TestNotNullSecondaryIndex` ( + Key Uint64 NOT NULL, + Value String, + Index1 String NOT NULL, + Index2 String, + PRIMARY KEY (Key), + INDEX Index GLOBAL ON (Index1, Index2) + COVER (Value)); + )"); + + auto result = session.ExecuteSchemeQuery(query).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + /* upsert row and update not null index column Index1 */ + { + const auto query = Q_(R"( + UPSERT INTO `/Root/TestNotNullSecondaryIndex` (Key, Index1, Index2) + VALUES (1, 'Secondary1', 'Secondary11') + )"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto query = Q_("UPDATE `/Root/TestNotNullSecondaryIndex` SET Index1 = NULL WHERE Key = 1"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), + result.GetIssues().ToString()); + } + + { + auto yson = ReadTablePartToYson(session, "/Root/TestNotNullSecondaryIndex/Index/indexImplTable"); + UNIT_ASSERT_VALUES_EQUAL(yson, R"([[["Secondary1"];["Secondary11"];[1u];#]])"); + } + + /* missing not null index column Index1 */ + { + const auto query = Q_("INSERT INTO `/Root/TestNotNullSecondaryIndex` (Key) VALUES (2)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), + result.GetIssues().ToString()); + } + + { + const auto query = Q_("UPSERT INTO `/Root/TestNotNullSecondaryIndex` (Key) VALUES (3)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), + result.GetIssues().ToString()); + } + + { + const auto query = Q_("REPLACE INTO `/Root/TestNotNullSecondaryIndex` (Key) VALUES (4)"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_NO_COLUMN_DEFAULT_VALUE), + result.GetIssues().ToString()); + } + + /* set NULL to not null index column Index1 */ + { + const auto query = Q_(R"( + INSERT INTO `/Root/TestNotNullSecondaryIndex` (Key, Value, Index1) VALUES (5, 'Value5', NULL) + )"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), + result.GetIssues().ToString()); + } + + { + const auto query = Q_(R"( + UPSERT INTO `/Root/TestNotNullSecondaryIndex` (Key, Value, Index1) VALUES (6, 'Value6', NULL) + )"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), + result.GetIssues().ToString()); + } + + { + const auto query = Q_(R"( + REPLACE INTO `/Root/TestNotNullSecondaryIndex` (Key, Value, Index1) VALUES (7, 'Value7', NULL) + )"); + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT(!result.IsSuccess()); + UNIT_ASSERT_C(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_BAD_COLUMN_TYPE), + result.GetIssues().ToString()); + } + } +} + +} // namespace NKqp +} // namespace NKikimr diff --git a/ydb/core/kqp/ut/kqp_olap_ut.cpp b/ydb/core/kqp/ut/kqp_olap_ut.cpp index 943e2861820..908fc34a13d 100644 --- a/ydb/core/kqp/ut/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/kqp_olap_ut.cpp @@ -924,7 +924,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { NJson::ReadJsonTree(*result.PlanJson, &plan, true); Cerr << *result.PlanJson << Endl; - node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); + node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); UNIT_ASSERT(node.IsDefined()); reverse = FindPlanNodeByKv(node, "Reverse", "true"); UNIT_ASSERT(reverse.IsDefined()); diff --git a/ydb/core/kqp/ut/kqp_pragma_ut.cpp b/ydb/core/kqp/ut/kqp_pragma_ut.cpp index 0619c0c6fbd..cdf99e40430 100644 --- a/ydb/core/kqp/ut/kqp_pragma_ut.cpp +++ b/ydb/core/kqp/ut/kqp_pragma_ut.cpp @@ -1,7 +1,7 @@ #include <ydb/core/kqp/ut/common/kqp_ut_common.h> #include <ydb/public/sdk/cpp/client/draft/ydb_scripting.h> - + namespace NKikimr { namespace NKqp { @@ -69,33 +69,33 @@ Y_UNIT_TEST_SUITE(KqpPragma) { UNIT_ASSERT(result.IsSuccess()); CompareYson(R"([[[1u];["One"]]])", FormatResultSetYson(result.GetResultSet(0))); } - - Y_UNIT_TEST(OrderedColumns) { - TKikimrRunner kikimr; - NYdb::NScripting::TScriptingClient client(kikimr.GetDriver()); - - auto result = client.ExecuteYqlScript(R"( - --!syntax_v1 - CREATE TABLE `/Root/NewTable` ( - Column3 Uint32, - Column2 Uint32, - Column1 Uint32, - PRIMARY KEY (Column1) - ); - COMMIT; - - INSERT INTO `/Root/NewTable` (Column1, Column2, Column3) VALUES (1, 2, 3); - COMMIT; - - PRAGMA OrderedColumns; - SELECT * FROM `/Root/NewTable`; - )").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - CompareYson(R"([ - [[3u];[2u];[1u]] - ])", FormatResultSetYson(result.GetResultSet(0))); - } + + Y_UNIT_TEST(OrderedColumns) { + TKikimrRunner kikimr; + NYdb::NScripting::TScriptingClient client(kikimr.GetDriver()); + + auto result = client.ExecuteYqlScript(R"( + --!syntax_v1 + CREATE TABLE `/Root/NewTable` ( + Column3 Uint32, + Column2 Uint32, + Column1 Uint32, + PRIMARY KEY (Column1) + ); + COMMIT; + + INSERT INTO `/Root/NewTable` (Column1, Column2, Column3) VALUES (1, 2, 3); + COMMIT; + + PRAGMA OrderedColumns; + SELECT * FROM `/Root/NewTable`; + )").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + CompareYson(R"([ + [[3u];[2u];[1u]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } } } // namspace NKqp diff --git a/ydb/core/kqp/ut/kqp_query_ut.cpp b/ydb/core/kqp/ut/kqp_query_ut.cpp index 60de01e3926..5ef6d18f381 100644 --- a/ydb/core/kqp/ut/kqp_query_ut.cpp +++ b/ydb/core/kqp/ut/kqp_query_ut.cpp @@ -50,10 +50,10 @@ Y_UNIT_TEST_SUITE(KqpQuery) { } while (result.GetStatus() == EStatus::UNAVAILABLE || result.GetStatus() == EStatus::ABORTED); result.GetIssues().PrintTo(Cerr); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - TKqpCounters counters(kikimr.GetTestServer().GetRuntime()->GetAppData().Counters); - UNIT_ASSERT_VALUES_EQUAL(counters.RecompileRequestGet()->Val(), 1); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + TKqpCounters counters(kikimr.GetTestServer().GetRuntime()->GetAppData().Counters); + UNIT_ASSERT_VALUES_EQUAL(counters.RecompileRequestGet()->Val(), 1); } Y_UNIT_TEST_NEW_ENGINE(QueryCache) { @@ -84,53 +84,53 @@ Y_UNIT_TEST_SUITE(KqpQuery) { } Y_UNIT_TEST_NEW_ENGINE(QueryCacheTtl) { - NKikimrConfig::TAppConfig appConfig; - appConfig.MutableTableServiceConfig()->SetCompileQueryCacheTTLSec(2); - - TKikimrRunner kikimr(appConfig); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetCompileQueryCacheTTLSec(2); + + TKikimrRunner kikimr(appConfig); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + auto query = Q_(R"( SELECT * FROM `/Root/Test`; )"); - - auto txControl = TTxControl::BeginTx().CommitTx(); - - TExecDataQuerySettings execSettings; - execSettings.KeepInQueryCache(true); - execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); - - auto result = session.ExecuteDataQuery(query, txControl, execSettings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), false); - - result = session.ExecuteDataQuery(query, txControl, execSettings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), true); - - execSettings.KeepInQueryCache(false); - - auto delay = TDuration::Seconds(appConfig.GetTableServiceConfig().GetCompileQueryCacheTTLSec()); - for (int i = 0; i < 10; ++i) { - Sleep(delay); - - result = session.ExecuteDataQuery(query, txControl, execSettings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - - stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - if (!stats.compilation().from_cache()) - break; - } - - result = session.ExecuteDataQuery(query, txControl, execSettings).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); - stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), false); - } - + + auto txControl = TTxControl::BeginTx().CommitTx(); + + TExecDataQuerySettings execSettings; + execSettings.KeepInQueryCache(true); + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + + auto result = session.ExecuteDataQuery(query, txControl, execSettings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), false); + + result = session.ExecuteDataQuery(query, txControl, execSettings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), true); + + execSettings.KeepInQueryCache(false); + + auto delay = TDuration::Seconds(appConfig.GetTableServiceConfig().GetCompileQueryCacheTTLSec()); + for (int i = 0; i < 10; ++i) { + Sleep(delay); + + result = session.ExecuteDataQuery(query, txControl, execSettings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + + stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + if (!stats.compilation().from_cache()) + break; + } + + result = session.ExecuteDataQuery(query, txControl, execSettings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(stats.compilation().from_cache(), false); + } + Y_UNIT_TEST_NEW_ENGINE(QueryCacheInvalidate) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); @@ -154,13 +154,13 @@ Y_UNIT_TEST_SUITE(KqpQuery) { UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::SUCCESS, alterResult.GetIssues().ToString()); result = dataQuery->Execute(txControl).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); result = session.ExecuteDataQuery(query, txControl).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - TKqpCounters counters(kikimr.GetTestServer().GetRuntime()->GetAppData().Counters); - UNIT_ASSERT_VALUES_EQUAL(counters.RecompileRequestGet()->Val(), 1); + + TKqpCounters counters(kikimr.GetTestServer().GetRuntime()->GetAppData().Counters); + UNIT_ASSERT_VALUES_EQUAL(counters.RecompileRequestGet()->Val(), 1); } Y_UNIT_TEST_NEW_ENGINE(QueryCachePermissionsLoss) { @@ -613,22 +613,22 @@ Y_UNIT_TEST_SUITE(KqpQuery) { } Y_UNIT_TEST_NEW_ENGINE(YqlTableSample) { - auto setting = NKikimrKqp::TKqpSetting(); - setting.SetName("_KqpYqlSyntaxVersion"); - setting.SetValue("1"); - - TKikimrRunner kikimr({setting}); - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - + auto setting = NKikimrKqp::TKqpSetting(); + setting.SetName("_KqpYqlSyntaxVersion"); + setting.SetValue("1"); + + TKikimrRunner kikimr({setting}); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + const TString query(Q_(R"(SELECT * FROM `/Root/Test` TABLESAMPLE SYSTEM(1.0);)")); - auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const NYql::TIssue& issue) { - return issue.Message.Contains("ATOM evaluation is not supported in YDB queries."); - })); - } - + auto result = session.ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const NYql::TIssue& issue) { + return issue.Message.Contains("ATOM evaluation is not supported in YDB queries."); + })); + } + Y_UNIT_TEST(YqlSyntaxServiceOverride) { auto setting = NKikimrKqp::TKqpSetting(); setting.SetName("_KqpYqlSyntaxVersion"); @@ -737,14 +737,14 @@ Y_UNIT_TEST_SUITE(KqpQuery) { FormatResultSetYson(result.GetResultSet(0))); } - Y_UNIT_TEST_NEW_ENGINE(Now) { + Y_UNIT_TEST_NEW_ENGINE(Now) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - auto query = Q_(R"( - SELECT YQL::Now(), YQL::Now(); - )"); + auto query = Q_(R"( + SELECT YQL::Now(), YQL::Now(); + )"); const ui32 QueriesCount = 5; @@ -761,7 +761,7 @@ Y_UNIT_TEST_SUITE(KqpQuery) { UNIT_ASSERT(parser.TryNextRow()); auto value = parser.ColumnParser(0).GetUint64(); - UNIT_ASSERT(value == parser.ColumnParser(1).GetUint64()); + UNIT_ASSERT(value == parser.ColumnParser(1).GetUint64()); timestamps.insert(value); } @@ -781,82 +781,82 @@ Y_UNIT_TEST_SUITE(KqpQuery) { UNIT_ASSERT(parser.TryNextRow()); auto value = parser.ColumnParser(0).GetUint64(); - UNIT_ASSERT(value == parser.ColumnParser(1).GetUint64()); + UNIT_ASSERT(value == parser.ColumnParser(1).GetUint64()); timestamps.insert(value); } - UNIT_ASSERT_VALUES_EQUAL(timestamps.size(), QueriesCount); + UNIT_ASSERT_VALUES_EQUAL(timestamps.size(), QueriesCount); + } + + Y_UNIT_TEST_NEW_ENGINE(RandomNumber) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto query = Q_(R"( + SELECT YQL::RandomNumber(), YQL::RandomNumber(), RandomNumber(1); + )"); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx().CommitTx() + ).ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + TResultSetParser parser(result.GetResultSet(0)); + UNIT_ASSERT(parser.TryNextRow()); + + auto value = parser.ColumnParser(0).GetUint64(); + UNIT_ASSERT(value == parser.ColumnParser(1).GetUint64()); + UNIT_ASSERT(value != parser.ColumnParser(2).GetUint64()); + } + + Y_UNIT_TEST_NEW_ENGINE(CurrentUtcTimestamp) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto query = Q_(R"( + SELECT YQL::CurrentUtcTimestamp(), YQL::CurrentUtcTimestamp(); + )"); + + TInstant timestamp; + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx().CommitTx(), + TExecDataQuerySettings().KeepInQueryCache(true) + ).ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + TResultSetParser parser(result.GetResultSet(0)); + UNIT_ASSERT(parser.TryNextRow()); + + timestamp = parser.ColumnParser(0).GetTimestamp(); + UNIT_ASSERT(timestamp == parser.ColumnParser(1).GetTimestamp()); + } + + { + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx().CommitTx(), + TExecDataQuerySettings().KeepInQueryCache(true) + ).ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + + TResultSetParser parser(result.GetResultSet(0)); + UNIT_ASSERT(parser.TryNextRow()); + + auto value = parser.ColumnParser(0).GetTimestamp(); + UNIT_ASSERT(value == parser.ColumnParser(1).GetTimestamp()); + UNIT_ASSERT(timestamp != value); + } } - Y_UNIT_TEST_NEW_ENGINE(RandomNumber) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto query = Q_(R"( - SELECT YQL::RandomNumber(), YQL::RandomNumber(), RandomNumber(1); - )"); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx().CommitTx() - ).ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - TResultSetParser parser(result.GetResultSet(0)); - UNIT_ASSERT(parser.TryNextRow()); - - auto value = parser.ColumnParser(0).GetUint64(); - UNIT_ASSERT(value == parser.ColumnParser(1).GetUint64()); - UNIT_ASSERT(value != parser.ColumnParser(2).GetUint64()); - } - - Y_UNIT_TEST_NEW_ENGINE(CurrentUtcTimestamp) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - auto query = Q_(R"( - SELECT YQL::CurrentUtcTimestamp(), YQL::CurrentUtcTimestamp(); - )"); - - TInstant timestamp; - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx().CommitTx(), - TExecDataQuerySettings().KeepInQueryCache(true) - ).ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - TResultSetParser parser(result.GetResultSet(0)); - UNIT_ASSERT(parser.TryNextRow()); - - timestamp = parser.ColumnParser(0).GetTimestamp(); - UNIT_ASSERT(timestamp == parser.ColumnParser(1).GetTimestamp()); - } - - { - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx().CommitTx(), - TExecDataQuerySettings().KeepInQueryCache(true) - ).ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - - TResultSetParser parser(result.GetResultSet(0)); - UNIT_ASSERT(parser.TryNextRow()); - - auto value = parser.ColumnParser(0).GetTimestamp(); - UNIT_ASSERT(value == parser.ColumnParser(1).GetTimestamp()); - UNIT_ASSERT(timestamp != value); - } - } - Y_UNIT_TEST(UnsafeTimestampCastV0) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); diff --git a/ydb/core/kqp/ut/kqp_scan_ut.cpp b/ydb/core/kqp/ut/kqp_scan_ut.cpp index a34ff99b731..347ca354412 100644 --- a/ydb/core/kqp/ut/kqp_scan_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scan_ut.cpp @@ -1734,7 +1734,7 @@ Y_UNIT_TEST_SUITE(KqpScan) { auto settings = TStreamExecScanQuerySettings() .Explain(true); - auto test = [&](const TString& table, const TString& tableRlPath, const TVector<TString>& keys, bool top, bool expectSort) { + auto test = [&](const TString& table, const TString& tableRlPath, const TVector<TString>& keys, bool top, bool expectSort) { auto query = TStringBuilder() << "SELECT * FROM `" << table << "` " << "ORDER BY " << JoinSeq(", ", keys) @@ -1749,24 +1749,24 @@ Y_UNIT_TEST_SUITE(KqpScan) { NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, /* throwOnError */ true); - auto node = FindPlanNodeByKv(plan, "Tables", NJson::TJsonArray({tableRlPath}).GetStringRobust()); - UNIT_ASSERT_C(node.IsDefined(), query); - UNIT_ASSERT_EQUAL_C(node.GetMapSafe().at("Node Type").GetStringSafe().Contains("Sort"), expectSort, query); + auto node = FindPlanNodeByKv(plan, "Tables", NJson::TJsonArray({tableRlPath}).GetStringRobust()); + UNIT_ASSERT_C(node.IsDefined(), query); + UNIT_ASSERT_EQUAL_C(node.GetMapSafe().at("Node Type").GetStringSafe().Contains("Sort"), expectSort, query); }; // simple key for (bool top : {false, true}) { - test("/Root/KeyValue", "KeyValue", {"Key"}, top, false); // key - test("/Root/KeyValue", "KeyValue", {"Value"}, top, true); // not key + test("/Root/KeyValue", "KeyValue", {"Key"}, top, false); // key + test("/Root/KeyValue", "KeyValue", {"Value"}, top, true); // not key } // complex key for (bool top : {false, true}) { - test("/Root/Logs", "Logs", {"App", "Ts", "Host"}, top, false); // full key - test("/Root/Logs", "Logs", {"App", "Ts"}, top, false); // key prefix - test("/Root/Logs", "Logs", {"App"}, top, false); // key prefix - test("/Root/Logs", "Logs", {"Ts", "Host"}, top, true); // not key prefix - test("/Root/Logs", "Logs", {"Message"}, top, true); // not key + test("/Root/Logs", "Logs", {"App", "Ts", "Host"}, top, false); // full key + test("/Root/Logs", "Logs", {"App", "Ts"}, top, false); // key prefix + test("/Root/Logs", "Logs", {"App"}, top, false); // key prefix + test("/Root/Logs", "Logs", {"Ts", "Host"}, top, true); // not key prefix + test("/Root/Logs", "Logs", {"Message"}, top, true); // not key } } @@ -1814,25 +1814,25 @@ Y_UNIT_TEST_SUITE(KqpScan) { std::cerr << result << std::endl; CompareYson(result, R"([[[2];[1000];["Dogecoin"]];[[4];[1];["XTC"]];[[5];[2];["Cardano"]];[[6];[3];["Tether"]]])"); } - - Y_UNIT_TEST(YqlTableSample) { - auto setting = NKikimrKqp::TKqpSetting(); - setting.SetName("_KqpYqlSyntaxVersion"); - setting.SetValue("1"); - - TKikimrRunner kikimr({setting}); - auto db = kikimr.GetTableClient(); - - const TString query(R"(SELECT * FROM `/Root/Test` TABLESAMPLE SYSTEM(1.0);)"); - auto it = db.StreamExecuteScanQuery(query).ExtractValueSync(); - - UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); - auto result = it.ReadNext().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); - UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const NYql::TIssue& issue) { - return issue.Message.Contains("ATOM evaluation is not supported in YDB queries."); - })); - } + + Y_UNIT_TEST(YqlTableSample) { + auto setting = NKikimrKqp::TKqpSetting(); + setting.SetName("_KqpYqlSyntaxVersion"); + setting.SetValue("1"); + + TKikimrRunner kikimr({setting}); + auto db = kikimr.GetTableClient(); + + const TString query(R"(SELECT * FROM `/Root/Test` TABLESAMPLE SYSTEM(1.0);)"); + auto it = db.StreamExecuteScanQuery(query).ExtractValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + auto result = it.ReadNext().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::UNSUPPORTED); + UNIT_ASSERT(HasIssue(result.GetIssues(), NYql::TIssuesIds::KIKIMR_UNSUPPORTED, [](const NYql::TIssue& issue) { + return issue.Message.Contains("ATOM evaluation is not supported in YDB queries."); + })); + } Y_UNIT_TEST(CrossJoinOneColumn) { TKikimrRunner kikimr; diff --git a/ydb/core/kqp/ut/kqp_scripting_ut.cpp b/ydb/core/kqp/ut/kqp_scripting_ut.cpp index 8de9fe04bf9..65e75ac303e 100644 --- a/ydb/core/kqp/ut/kqp_scripting_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scripting_ut.cpp @@ -428,8 +428,8 @@ Y_UNIT_TEST_SUITE(KqpScripting) { NJson::TJsonValue plan; NJson::ReadJsonTree(planJson, &plan, true); - auto node = FindPlanNodeByKv(plan.GetMap().at("queries").GetArray()[2], "Node Type", "Aggregate-TableFullScan"); - UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 1); + auto node = FindPlanNodeByKv(plan.GetMap().at("queries").GetArray()[2], "Node Type", "Aggregate-TableFullScan"); + UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 1); } Y_UNIT_TEST(StreamExecuteYqlScriptScan) { diff --git a/ydb/core/kqp/ut/kqp_sort_ut.cpp b/ydb/core/kqp/ut/kqp_sort_ut.cpp index 63939bd3d47..0b7c65a3d5c 100644 --- a/ydb/core/kqp/ut/kqp_sort_ut.cpp +++ b/ydb/core/kqp/ut/kqp_sort_ut.cpp @@ -53,11 +53,11 @@ Y_UNIT_TEST_SUITE(KqpSort) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); // without `Sort` + auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); // without `Sort` UNIT_ASSERT(node.IsDefined()); - auto read = FindPlanNodeByKv(node, "Name", "TableFullScan"); - UNIT_ASSERT(read.IsDefined()); - UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); + auto read = FindPlanNodeByKv(node, "Name", "TableFullScan"); + UNIT_ASSERT(read.IsDefined()); + UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); } else { UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); UNIT_ASSERT_C(!result.GetAst().Contains("Sort"), result.GetAst()); @@ -98,11 +98,11 @@ Y_UNIT_TEST_SUITE(KqpSort) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableRangeScan"); // without `Sort` + auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableRangeScan"); // without `Sort` UNIT_ASSERT_C(node.IsDefined(), result.GetPlan()); - auto read = FindPlanNodeByKv(node, "Name", "TableRangeScan"); - UNIT_ASSERT(read.IsDefined()); - UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); + auto read = FindPlanNodeByKv(node, "Name", "TableRangeScan"); + UNIT_ASSERT(read.IsDefined()); + UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); } else { UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); } @@ -141,11 +141,11 @@ Y_UNIT_TEST_SUITE(KqpSort) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); // without `Sort` + auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); // without `Sort` UNIT_ASSERT_C(node.IsDefined(), result.GetPlan()); - auto read = FindPlanNodeByKv(node, "Name", "TableFullScan"); - UNIT_ASSERT(read.IsDefined()); - UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); + auto read = FindPlanNodeByKv(node, "Name", "TableFullScan"); + UNIT_ASSERT(read.IsDefined()); + UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); } else { UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); } @@ -186,7 +186,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableFullScan"); + auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableFullScan"); UNIT_ASSERT_C(node.IsDefined(), result.GetPlan()); UNIT_ASSERT(!node.GetMapSafe().contains("Reverse")); } else { @@ -230,11 +230,11 @@ Y_UNIT_TEST_SUITE(KqpSort) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableRangeScan"); + auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableRangeScan"); UNIT_ASSERT_C(node.IsDefined(), result.GetPlan()); - auto read = FindPlanNodeByKv(node, "Name", "TableRangeScan"); - UNIT_ASSERT(read.IsDefined()); - UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); + auto read = FindPlanNodeByKv(node, "Name", "TableRangeScan"); + UNIT_ASSERT(read.IsDefined()); + UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); } else { UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); } @@ -277,14 +277,14 @@ Y_UNIT_TEST_SUITE(KqpSort) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); + auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); UNIT_ASSERT_C(node.IsDefined(), result.GetPlan()); - auto read = FindPlanNodeByKv(node, "Name", "TableFullScan"); - UNIT_ASSERT(read.IsDefined()); - UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); - auto limit = FindPlanNodeByKv(node, "Name", "Limit"); - UNIT_ASSERT(limit.IsDefined()); - UNIT_ASSERT(limit.GetMapSafe().contains("Limit")); + auto read = FindPlanNodeByKv(node, "Name", "TableFullScan"); + UNIT_ASSERT(read.IsDefined()); + UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); + auto limit = FindPlanNodeByKv(node, "Name", "Limit"); + UNIT_ASSERT(limit.IsDefined()); + UNIT_ASSERT(limit.GetMapSafe().contains("Limit")); } else { UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); } @@ -325,14 +325,14 @@ Y_UNIT_TEST_SUITE(KqpSort) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableRangeScan"); + auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableRangeScan"); UNIT_ASSERT_C(node.IsDefined(), result.GetPlan()); - auto read = FindPlanNodeByKv(node, "Name", "TableRangeScan"); - UNIT_ASSERT(read.IsDefined()); - UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); - auto limit = FindPlanNodeByKv(node, "Name", "Limit"); - UNIT_ASSERT(limit.IsDefined()); - UNIT_ASSERT(limit.GetMapSafe().contains("Limit")); + auto read = FindPlanNodeByKv(node, "Name", "TableRangeScan"); + UNIT_ASSERT(read.IsDefined()); + UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); + auto limit = FindPlanNodeByKv(node, "Name", "Limit"); + UNIT_ASSERT(limit.IsDefined()); + UNIT_ASSERT(limit.GetMapSafe().contains("Limit")); } else { UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); } @@ -376,11 +376,11 @@ Y_UNIT_TEST_SUITE(KqpSort) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); + auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); UNIT_ASSERT_C(node.IsDefined(), result.GetPlan()); - auto read = FindPlanNodeByKv(node, "Name", "TableFullScan"); - UNIT_ASSERT(read.IsDefined()); - UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); + auto read = FindPlanNodeByKv(node, "Name", "TableFullScan"); + UNIT_ASSERT(read.IsDefined()); + UNIT_ASSERT(read.GetMapSafe().contains("Reverse")); } else { UNIT_ASSERT_C(result.GetAst().Contains("'\"Reverse\" (Bool '\"true\")"), result.GetAst()); } @@ -427,7 +427,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { if (UseNewEngine) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangeScan"); + auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangeScan"); UNIT_ASSERT_C(node.IsDefined(), result.GetPlan()); } else { UNIT_ASSERT_C(result.GetAst().Contains("KiPartialTake"), result.GetAst()); @@ -485,7 +485,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { if (UseNewEngine) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangeScan"); + auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangeScan"); UNIT_ASSERT_C(node.IsDefined(), result.GetPlan()); } else { UNIT_ASSERT_C(result.GetAst().Contains("KiPartialTake"), result.GetAst()); @@ -991,7 +991,7 @@ Y_UNIT_TEST_SUITE(KqpSort) { if (UseNewEngine) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); + auto node = FindPlanNodeByKv(plan, "Node Type", "Limit-TableFullScan"); UNIT_ASSERT_C(node.IsDefined(), result.GetPlan()); } else { UNIT_ASSERT_C(!result.GetAst().Contains("KiPartialTake"), result.GetAst()); @@ -1041,10 +1041,10 @@ Y_UNIT_TEST_SUITE(KqpSort) { if (UseNewEngine) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "Filter-TableFullScan"); + auto node = FindPlanNodeByKv(plan, "Node Type", "Filter-TableFullScan"); UNIT_ASSERT_C(node.IsDefined(), result.GetPlan()); - auto limit = FindPlanNodeByKv(node, "Limit", "Min(1001,$limit)"); - UNIT_ASSERT(limit.IsDefined()); + auto limit = FindPlanNodeByKv(node, "Limit", "Min(1001,$limit)"); + UNIT_ASSERT(limit.IsDefined()); } else { UNIT_ASSERT_C(result.GetAst().Contains("KiPartialTake"), result.GetAst()); } diff --git a/ydb/core/kqp/ut/kqp_stats_ut.cpp b/ydb/core/kqp/ut/kqp_stats_ut.cpp index 7b8fd927298..f7815c016ee 100644 --- a/ydb/core/kqp/ut/kqp_stats_ut.cpp +++ b/ydb/core/kqp/ut/kqp_stats_ut.cpp @@ -31,8 +31,8 @@ Y_UNIT_TEST(MultiTxStatsFullExp) { UNIT_ASSERT(res.PlanJson); NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangesScan"); - UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 2); + auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangesScan"); + UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 2); } Y_UNIT_TEST(JoinNoStats) { @@ -110,8 +110,8 @@ Y_UNIT_TEST(MultiTxStatsFull) { UNIT_ASSERT(res.PlanJson); NJson::TJsonValue plan; NJson::ReadJsonTree(*res.PlanJson, &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangesScan"); - UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 2); + auto node = FindPlanNodeByKv(plan, "Node Type", "TopSort-TableRangesScan"); + UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 2); } Y_UNIT_TEST(DeferredEffects) { @@ -136,7 +136,7 @@ Y_UNIT_TEST(DeferredEffects) { // // NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); // auto node = FindPlanNodeByKv(plan, "Node Type", "TablePointLookup"); - // UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 1); + // UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 1); auto tx = result.GetTransaction(); UNIT_ASSERT(tx); @@ -201,8 +201,8 @@ Y_UNIT_TEST(DataQueryWithEffects) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); - auto node = FindPlanNodeByKv(plan, "Node Type", "Upsert-ConstantExpr"); - UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 2); + auto node = FindPlanNodeByKv(plan, "Node Type", "Upsert-ConstantExpr"); + UNIT_ASSERT_EQUAL(node.GetMap().at("Stats").GetMapSafe().at("TotalTasks").GetIntegerSafe(), 2); } Y_UNIT_TEST(DataQueryOldEngine) { @@ -244,7 +244,7 @@ Y_UNIT_TEST(DataQueryMulti) { NJson::TJsonValue plan; NJson::ReadJsonTree(result.GetQueryPlan(), &plan, true); - UNIT_ASSERT_EQUAL_C(plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans").GetArraySafe().size(), 0, result.GetQueryPlan()); + UNIT_ASSERT_EQUAL_C(plan.GetMapSafe().at("Plan").GetMapSafe().at("Plans").GetArraySafe().size(), 0, result.GetQueryPlan()); } Y_UNIT_TEST_NEW_ENGINE(RequestUnitForBadRequestExecute) { diff --git a/ydb/core/kqp/ut/kqp_table_predicate_ut.cpp b/ydb/core/kqp/ut/kqp_table_predicate_ut.cpp index 36cb2cedab2..5ab5108968f 100644 --- a/ydb/core/kqp/ut/kqp_table_predicate_ut.cpp +++ b/ydb/core/kqp/ut/kqp_table_predicate_ut.cpp @@ -1391,90 +1391,90 @@ Y_UNIT_TEST_SUITE(KqpTablePredicate) { // TODO: Need to get real ranges from explain, no anything in JSON } - Y_UNIT_TEST(ValidatePredicatesDataQuery) { - TKikimrRunner kikimr; - auto db = kikimr.GetTableClient(); - auto session = db.CreateSession().GetValueSync().GetSession(); - - { - UNIT_ASSERT(session.ExecuteSchemeQuery(R"( - CREATE TABLE [/Root/TestTable] ( - Key1 Uint32, - Key2 Uint32, - Key3 String, - Value Uint32, - PRIMARY KEY (Key1, Key2, Key3) - ); - )").GetValueSync().IsSuccess()); - - auto result = session.ExecuteDataQuery(R"( - REPLACE INTO [/Root/TestTable] (Key1, Key2, Key3, Value) VALUES - (1u, 10u, 'SomeString1', 100), - (2u, 20u, 'SomeString2', 200), - (3u, 30u, 'SomeString3', 300), - (4u, 40u, 'SomeString4', 400), - (5u, 50u, 'SomeString5', 500), - (6u, 60u, 'SomeString6', 600), - (NULL, 70u, 'SomeString7', 700), - (8u, NULL, 'SomeString8', 800), - (9u, 90u, NULL, 900), - (10u, 100u, 'SomeString10', NULL); - )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } - - std::vector<TString> predicates = { - "Key1 < 2", - "Key1 > 1", - "Key1 = 1", - "Key1 = 1 AND Key2 > 0", - "Key1 >= 1 AND Key2 > 8", - "Key1 >= 1 AND Key2 = 20", - "Key1 >= 8 AND Key2 >= 20", - "Key1 < 2 AND Key2 < 80", - "Key1 <= 8 AND Key2 < 80", - "Key1 <= 9 AND Key2 <= 200", - "Key1 > 4 AND Key2 > 40 AND Key3 > \"SomeString\"", - "Key1 >= 4000 AND Key2 >= 4 AND Key3 >= \"SomeString3\"", - "Key2 > 80", - "Key2 < 90", - "Key2 <= 20 AND Key3 <= \"SomeString3\"", - "Key1 IS NULL", - "Key2 IS NULL", - "Key1 IS NOT NULL", - "Key1 > 1 AND Key2 IS NULL", - "Key1 > 1 OR Key2 IS NULL", - "Key1 >= 1 OR Key2 IS NOT NULL", - "Key1 < 9 OR Key3 IS NOT NULL", - "Key1 < 9 OR Key3 IS NULL", - "Value = 200", - "(Key1 <= 10) OR (Key1 > 2 AND Key1 < 5) OR (Key1 >= 8)", - "Key1 < NULL" - }; - - TString enablePredicateExtractor = R"( - PRAGMA Kikimr.OptEnablePredicateExtract = "true"; - )"; - - TString query = R"( - PRAGMA kikimr.UseNewEngine = "true"; - SELECT * FROM [/Root/TestTable] WHERE <PREDICATE> ORDER BY `Value`; - )"; - - for (const auto& predicate : predicates) { - SubstGlobal(query, "<PREDICATE>", predicate); - auto expectedResult = session.ExecuteDataQuery(query, TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(expectedResult.IsSuccess(), expectedResult.GetIssues().ToString()); - const auto expectedYson = FormatResultSetYson(expectedResult.GetResultSet(0)); - - auto result = session.ExecuteDataQuery(enablePredicateExtractor + query, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - - CompareYson(expectedYson, FormatResultSetYson(result.GetResultSet(0))); - } - } + Y_UNIT_TEST(ValidatePredicatesDataQuery) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + UNIT_ASSERT(session.ExecuteSchemeQuery(R"( + CREATE TABLE [/Root/TestTable] ( + Key1 Uint32, + Key2 Uint32, + Key3 String, + Value Uint32, + PRIMARY KEY (Key1, Key2, Key3) + ); + )").GetValueSync().IsSuccess()); + + auto result = session.ExecuteDataQuery(R"( + REPLACE INTO [/Root/TestTable] (Key1, Key2, Key3, Value) VALUES + (1u, 10u, 'SomeString1', 100), + (2u, 20u, 'SomeString2', 200), + (3u, 30u, 'SomeString3', 300), + (4u, 40u, 'SomeString4', 400), + (5u, 50u, 'SomeString5', 500), + (6u, 60u, 'SomeString6', 600), + (NULL, 70u, 'SomeString7', 700), + (8u, NULL, 'SomeString8', 800), + (9u, 90u, NULL, 900), + (10u, 100u, 'SomeString10', NULL); + )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + std::vector<TString> predicates = { + "Key1 < 2", + "Key1 > 1", + "Key1 = 1", + "Key1 = 1 AND Key2 > 0", + "Key1 >= 1 AND Key2 > 8", + "Key1 >= 1 AND Key2 = 20", + "Key1 >= 8 AND Key2 >= 20", + "Key1 < 2 AND Key2 < 80", + "Key1 <= 8 AND Key2 < 80", + "Key1 <= 9 AND Key2 <= 200", + "Key1 > 4 AND Key2 > 40 AND Key3 > \"SomeString\"", + "Key1 >= 4000 AND Key2 >= 4 AND Key3 >= \"SomeString3\"", + "Key2 > 80", + "Key2 < 90", + "Key2 <= 20 AND Key3 <= \"SomeString3\"", + "Key1 IS NULL", + "Key2 IS NULL", + "Key1 IS NOT NULL", + "Key1 > 1 AND Key2 IS NULL", + "Key1 > 1 OR Key2 IS NULL", + "Key1 >= 1 OR Key2 IS NOT NULL", + "Key1 < 9 OR Key3 IS NOT NULL", + "Key1 < 9 OR Key3 IS NULL", + "Value = 200", + "(Key1 <= 10) OR (Key1 > 2 AND Key1 < 5) OR (Key1 >= 8)", + "Key1 < NULL" + }; + + TString enablePredicateExtractor = R"( + PRAGMA Kikimr.OptEnablePredicateExtract = "true"; + )"; + + TString query = R"( + PRAGMA kikimr.UseNewEngine = "true"; + SELECT * FROM [/Root/TestTable] WHERE <PREDICATE> ORDER BY `Value`; + )"; + + for (const auto& predicate : predicates) { + SubstGlobal(query, "<PREDICATE>", predicate); + auto expectedResult = session.ExecuteDataQuery(query, TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(expectedResult.IsSuccess(), expectedResult.GetIssues().ToString()); + const auto expectedYson = FormatResultSetYson(expectedResult.GetResultSet(0)); + + auto result = session.ExecuteDataQuery(enablePredicateExtractor + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + CompareYson(expectedYson, FormatResultSetYson(result.GetResultSet(0))); + } + } Y_UNIT_TEST_NEW_ENGINE(CastKeyBounds) { TKikimrRunner kikimr; diff --git a/ydb/core/kqp/ut/ya.make b/ydb/core/kqp/ut/ya.make index ae9924f131e..609ec3237f8 100644 --- a/ydb/core/kqp/ut/ya.make +++ b/ydb/core/kqp/ut/ya.make @@ -36,7 +36,7 @@ SRCS( kqp_newengine_effects_ut.cpp kqp_newengine_flowcontrol_ut.cpp kqp_newengine_ut.cpp - kqp_not_null_columns_ut.cpp + kqp_not_null_columns_ut.cpp kqp_olap_ut.cpp kqp_params_ut.cpp kqp_pragma_ut.cpp diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 478459e8670..d64169d4fc0 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -666,7 +666,7 @@ message TFeatureFlags { optional bool EnableBackgroundCompaction = 51 [default = true]; optional bool EnableArrowFormatInChannels = 52 [default = false]; optional bool EnableBackgroundCompactionServerless = 53 [default = false]; - optional bool EnableNotNullColumns = 54 [default = false]; + optional bool EnableNotNullColumns = 54 [default = false]; optional bool EnableTtlOnAsyncIndexedTables = 55 [default = false]; optional bool EnableBulkUpsertToAsyncIndexedTables = 56 [default = false]; optional bool EnableNodeBrokerSingleDomainMode = 57 [default = false]; @@ -1080,7 +1080,7 @@ message TTableServiceConfig { // 2 - All read-only queries. Without fallback to new engine (in case of rw tx, read queries will be executed with new engine, write queries - with old one). // 3 - New engine for all queries. optional uint32 ForceNewEngineLevel = 22 [default = 0]; - optional uint32 CompileQueryCacheTTLSec = 20 [default = 0]; + optional uint32 CompileQueryCacheTTLSec = 20 [default = 0]; optional TQueryReplayConfig QueryReplayConfig = 21; }; diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index b37e0473a76..33e598c1c25 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -80,7 +80,7 @@ message TColumnDescription { oneof DefaultValue { string DefaultFromSequence = 7; // Path to sequence for default values } - optional bool NotNull = 8; + optional bool NotNull = 8; } message TStorageSettings { diff --git a/ydb/core/protos/flat_tx_scheme.proto b/ydb/core/protos/flat_tx_scheme.proto index bd3a9b944ac..dc06523e5cc 100644 --- a/ydb/core/protos/flat_tx_scheme.proto +++ b/ydb/core/protos/flat_tx_scheme.proto @@ -216,7 +216,7 @@ message TMigrateColumn { optional uint32 Family = 7; optional uint32 DefaultKind = 8; optional bytes DefaultValue = 9; - optional bool NotNull = 10; + optional bool NotNull = 10; } message TMigratePartition { diff --git a/ydb/core/protos/kqp.proto b/ydb/core/protos/kqp.proto index 2fcc6388960..9b46b720e11 100644 --- a/ydb/core/protos/kqp.proto +++ b/ydb/core/protos/kqp.proto @@ -116,7 +116,7 @@ message TKqpColumnMetadataProto { optional string Type = 3; optional uint32 TypeId = 4; repeated string Family = 5; - optional bool NotNull = 6 [default = false]; + optional bool NotNull = 6 [default = false]; }; message TKqpTableMetadataProto { diff --git a/ydb/core/protos/kqp_physical.proto b/ydb/core/protos/kqp_physical.proto index 6d63a020a3f..44d8f587166 100644 --- a/ydb/core/protos/kqp_physical.proto +++ b/ydb/core/protos/kqp_physical.proto @@ -10,21 +10,21 @@ import "ydb/library/yql/dq/proto/dq_tasks.proto"; message TKqpPhyExternalBinding { } -message TKqpPhyInternalBinding { - enum EType { - PARAM_UNSPECIFIED = 0; - PARAM_NOW = 1; - PARAM_CURRENT_DATE = 2; - PARAM_CURRENT_DATETIME = 3; - PARAM_CURRENT_TIMESTAMP = 4; - PARAM_RANDOM_NUMBER = 5; - PARAM_RANDOM = 6; - PARAM_RANDOM_UUID = 7; - }; - - EType Type = 1; -} - +message TKqpPhyInternalBinding { + enum EType { + PARAM_UNSPECIFIED = 0; + PARAM_NOW = 1; + PARAM_CURRENT_DATE = 2; + PARAM_CURRENT_DATETIME = 3; + PARAM_CURRENT_TIMESTAMP = 4; + PARAM_RANDOM_NUMBER = 5; + PARAM_RANDOM = 6; + PARAM_RANDOM_UUID = 7; + }; + + EType Type = 1; +} + message TKqpPhyTxResultBinding { uint32 TxIndex = 1; uint32 ResultIndex = 2; @@ -36,7 +36,7 @@ message TKqpPhyParamBinding { oneof Type { TKqpPhyExternalBinding ExternalBinding = 2; TKqpPhyTxResultBinding TxResultBinding = 3; - TKqpPhyInternalBinding InternalBinding = 4; + TKqpPhyInternalBinding InternalBinding = 4; } } diff --git a/ydb/core/protos/kqp_stats.proto b/ydb/core/protos/kqp_stats.proto index 1d0c09b5345..c2579c24ff6 100644 --- a/ydb/core/protos/kqp_stats.proto +++ b/ydb/core/protos/kqp_stats.proto @@ -66,9 +66,9 @@ message TKqpStatsQuery { reserved 3; // repeated TKqpStatsExecution Executions = 3; uint64 WorkerCpuTimeUs = 4; - uint64 ReadSetsCount = 5; - uint64 MaxShardProgramSize = 6; - uint64 MaxShardReplySize = 7; + uint64 ReadSetsCount = 5; + uint64 MaxShardProgramSize = 6; + uint64 MaxShardReplySize = 7; repeated NYql.NDqProto.TDqExecutionStats Executions = 8; } diff --git a/ydb/core/protos/query_stats.proto b/ydb/core/protos/query_stats.proto index b222c1365ac..453b3034013 100644 --- a/ydb/core/protos/query_stats.proto +++ b/ydb/core/protos/query_stats.proto @@ -34,9 +34,9 @@ message TTableAccessStats { message TPerShardStats { optional uint64 ShardId = 1; optional uint64 CpuTimeUsec = 2; - optional uint64 OutgoingReadSetsCount = 3; - optional uint64 ProgramSize = 4; - optional uint64 ReplySize = 5; + optional uint64 OutgoingReadSetsCount = 3; + optional uint64 ProgramSize = 4; + optional uint64 ReplySize = 5; } message TTxStats { diff --git a/ydb/core/protos/scheme_log.proto b/ydb/core/protos/scheme_log.proto index 43417d539a5..fcf4ba52b94 100644 --- a/ydb/core/protos/scheme_log.proto +++ b/ydb/core/protos/scheme_log.proto @@ -58,7 +58,7 @@ message TAlterRecord { optional string ColumnName = 5; optional uint32 ColumnType = 6; optional bytes Default = 10; // Serialized default value for cell - optional bool NotNull = 24 [default = false]; + optional bool NotNull = 24 [default = false]; optional uint64 ExecutorCacheSize = 100; optional NKikimrSchemeOp.TCompactionPolicy CompactionPolicy = 101; diff --git a/ydb/core/tablet_flat/flat_cxx_database.h b/ydb/core/tablet_flat/flat_cxx_database.h index 22939f5abda..27a260bcb27 100644 --- a/ydb/core/tablet_flat/flat_cxx_database.h +++ b/ydb/core/tablet_flat/flat_cxx_database.h @@ -640,11 +640,11 @@ struct Schema { using Precharge = AutoPrecharge; - template <TColumnId _ColumnId, NScheme::TTypeId _ColumnType, bool _IsNotNull = false> + template <TColumnId _ColumnId, NScheme::TTypeId _ColumnType, bool _IsNotNull = false> struct Column { constexpr static TColumnId ColumnId = _ColumnId; constexpr static NScheme::TTypeId ColumnType = _ColumnType; - constexpr static bool IsNotNull = _IsNotNull; + constexpr static bool IsNotNull = _IsNotNull; using Type = typename NSchemeTypeMapper<_ColumnType>::Type; static TString GetColumnName(const TString& typeName) { @@ -670,7 +670,7 @@ struct Schema { } static void Materialize(TToughDb& database) { - database.Alter().AddColumn(TableId, GetColumnName(), T::ColumnId, T::ColumnType, T::IsNotNull); + database.Alter().AddColumn(TableId, GetColumnName(), T::ColumnId, T::ColumnType, T::IsNotNull); } static constexpr bool HaveColumn(ui32 columnId) { diff --git a/ydb/core/tablet_flat/flat_dbase_apply.cpp b/ydb/core/tablet_flat/flat_dbase_apply.cpp index e8e8251f07d..f87b315d332 100644 --- a/ydb/core/tablet_flat/flat_dbase_apply.cpp +++ b/ydb/core/tablet_flat/flat_dbase_apply.cpp @@ -32,7 +32,7 @@ bool TSchemeModifier::Apply(const TAlterRecord &delta) } changes = AddColumn(table, delta.GetColumnName(), delta.GetColumnId(), - delta.GetColumnType(), delta.GetNotNull(), null); + delta.GetColumnType(), delta.GetNotNull(), null); } else if (action == TAlterRecord::DropColumn) { changes = DropColumn(table, delta.GetColumnId()); } else if (action == TAlterRecord::AddColumnToKey) { @@ -175,12 +175,12 @@ bool TSchemeModifier::DropTable(ui32 id) return false; } -bool TSchemeModifier::AddColumn(ui32 tid, const TString &name, ui32 id, ui32 type, bool notNull, TCell null) +bool TSchemeModifier::AddColumn(ui32 tid, const TString &name, ui32 id, ui32 type, bool notNull, TCell null) { auto *table = Table(tid); auto itName = table->ColumnNames.find(name); bool haveName = itName != table->ColumnNames.end(); - auto it = table->Columns.emplace(id, TColumn(name, id, type, notNull)); + auto it = table->Columns.emplace(id, TColumn(name, id, type, notNull)); if (it.second) it.first->second.SetDefault(null); diff --git a/ydb/core/tablet_flat/flat_dbase_apply.h b/ydb/core/tablet_flat/flat_dbase_apply.h index 646151d813e..3502030b71b 100644 --- a/ydb/core/tablet_flat/flat_dbase_apply.h +++ b/ydb/core/tablet_flat/flat_dbase_apply.h @@ -32,7 +32,7 @@ namespace NTable { bool AddTable(const TString& name, ui32 id); bool DropTable(ui32 id); - bool AddColumn(ui32 table, const TString& name, ui32 id, ui32 type, bool notNull, TCell null = { }); + bool AddColumn(ui32 table, const TString& name, ui32 id, ui32 type, bool notNull, TCell null = { }); bool DropColumn(ui32 table, ui32 id); bool AddColumnToFamily(ui32 table, ui32 column, ui32 family); bool AddColumnToKey(ui32 table, ui32 column); diff --git a/ydb/core/tablet_flat/flat_dbase_scheme.cpp b/ydb/core/tablet_flat/flat_dbase_scheme.cpp index 0e635156b35..fa12005009b 100644 --- a/ydb/core/tablet_flat/flat_dbase_scheme.cpp +++ b/ydb/core/tablet_flat/flat_dbase_scheme.cpp @@ -28,7 +28,7 @@ TAutoPtr<TSchemeChanges> TScheme::GetSnapshot() const { for(const auto& it : itTable.second.Columns) { const auto &col = it.second; - delta.AddColumn(table, col.Name, it.first, col.PType, col.NotNull, col.Null); + delta.AddColumn(table, col.Name, it.first, col.PType, col.NotNull, col.Null); delta.AddColumnToFamily(table, it.first, col.Family); } @@ -85,7 +85,7 @@ TAlter& TAlter::DropTable(ui32 id) return *this; } -TAlter& TAlter::AddColumn(ui32 table, const TString& name, ui32 id, ui32 type, bool notNull, TCell null) +TAlter& TAlter::AddColumn(ui32 table, const TString& name, ui32 id, ui32 type, bool notNull, TCell null) { TAlterRecord& delta = *Log.AddDelta(); delta.SetDeltaType(TAlterRecord::AddColumn); @@ -93,7 +93,7 @@ TAlter& TAlter::AddColumn(ui32 table, const TString& name, ui32 id, ui32 type, b delta.SetTableId(table); delta.SetColumnId(id); delta.SetColumnType(type); - delta.SetNotNull(notNull); + delta.SetNotNull(notNull); if (!null.IsNull()) delta.SetDefault(null.Data(), null.Size()); diff --git a/ydb/core/tablet_flat/flat_dbase_scheme.h b/ydb/core/tablet_flat/flat_dbase_scheme.h index f69e61785d8..1fd5ed2b480 100644 --- a/ydb/core/tablet_flat/flat_dbase_scheme.h +++ b/ydb/core/tablet_flat/flat_dbase_scheme.h @@ -214,7 +214,7 @@ public: TAlter& Merge(const TSchemeChanges &delta); TAlter& AddTable(const TString& name, ui32 id); TAlter& DropTable(ui32 id); - TAlter& AddColumn(ui32 table, const TString& name, ui32 id, ui32 type, bool notNull, TCell null = { }); + TAlter& AddColumn(ui32 table, const TString& name, ui32 id, ui32 type, bool notNull, TCell null = { }); TAlter& DropColumn(ui32 table, ui32 id); TAlter& AddColumnToFamily(ui32 table, ui32 column, ui32 family); TAlter& AddFamily(ui32 table, ui32 family, ui32 room); diff --git a/ydb/core/tablet_flat/flat_executor_database_ut.cpp b/ydb/core/tablet_flat/flat_executor_database_ut.cpp index 2f66a0733c7..0168615c55b 100644 --- a/ydb/core/tablet_flat/flat_executor_database_ut.cpp +++ b/ydb/core/tablet_flat/flat_executor_database_ut.cpp @@ -143,14 +143,14 @@ private: NTable::TAlter delta; delta.AddTable("TxTable", TxTable); - delta.AddColumn(TxTable, "TxId", 1, NScheme::TUint64::TypeId, false); - delta.AddColumn(TxTable, "Plan", 2, NScheme::TUint64::TypeId, false); + delta.AddColumn(TxTable, "TxId", 1, NScheme::TUint64::TypeId, false); + delta.AddColumn(TxTable, "Plan", 2, NScheme::TUint64::TypeId, false); delta.AddColumnToKey(TxTable, 1); delta.AddTable("AffectedTable", AffectedTable); - delta.AddColumn(AffectedTable, "TxId", 1, NScheme::TUint64::TypeId, false); - delta.AddColumn(AffectedTable, "Datashard", 2, NScheme::TUint64::TypeId, false); - delta.AddColumn(AffectedTable, "Plan", 3, NScheme::TUint64::TypeId, false); + delta.AddColumn(AffectedTable, "TxId", 1, NScheme::TUint64::TypeId, false); + delta.AddColumn(AffectedTable, "Datashard", 2, NScheme::TUint64::TypeId, false); + delta.AddColumn(AffectedTable, "Plan", 3, NScheme::TUint64::TypeId, false); delta.AddColumnToKey(AffectedTable, 1); delta.AddColumnToKey(AffectedTable, 2); @@ -277,9 +277,9 @@ private: delta.AddTable("table", Table); delta.SetFamily(Table, AltFamily, NTable::NPage::ECache::None, NTable::NPage::ECodec::Plain); - delta.AddColumn(Table, "Id", 1, NScheme::TUint32::TypeId, false); - delta.AddColumn(Table, "value", 2, NScheme::TUint64::TypeId, false); - delta.AddColumn(Table, "large", 3, NScheme::TString::TypeId, false); + delta.AddColumn(Table, "Id", 1, NScheme::TUint32::TypeId, false); + delta.AddColumn(Table, "value", 2, NScheme::TUint64::TypeId, false); + delta.AddColumn(Table, "large", 3, NScheme::TString::TypeId, false); delta.AddColumnToKey(Table, 1); delta.AddColumnToFamily(Table, 2, AltFamily); delta.AddColumnToFamily(Table, 3, AltFamily); diff --git a/ydb/core/tablet_flat/flat_executor_ut.cpp b/ydb/core/tablet_flat/flat_executor_ut.cpp index e4e43149bb9..c9cf5289711 100644 --- a/ydb/core/tablet_flat/flat_executor_ut.cpp +++ b/ydb/core/tablet_flat/flat_executor_ut.cpp @@ -25,8 +25,8 @@ namespace NTabletFlatExecutor { txc.DB.Alter() .AddTable("test" + ToString(ui32(TableId)), TableId) - .AddColumn(TableId, "key", ColumnKeyId, NScheme::TInt64::TypeId, false) - .AddColumn(TableId, "value", ColumnValueId, NScheme::TString::TypeId, false) + .AddColumn(TableId, "key", ColumnKeyId, NScheme::TInt64::TypeId, false) + .AddColumn(TableId, "value", ColumnValueId, NScheme::TString::TypeId, false) .AddColumnToKey(TableId, ColumnKeyId) .SetCompactionPolicy(TableId, *Policy); @@ -2475,9 +2475,9 @@ Y_UNIT_TEST_SUITE(TFlatTableExecutorVersionedLargeBlobs) { // Values bigger than 128 bytes stored in large blobs txc.DB.Alter() .AddTable("test", TableId) - .AddColumn(TableId, "key", KeyColumnId, NScheme::TInt64::TypeId, false) - .AddColumn(TableId, "value", ValueColumnId, NScheme::TString::TypeId, false) - .AddColumn(TableId, "counter", CounterColumnId, NScheme::TInt64::TypeId, false) + .AddColumn(TableId, "key", KeyColumnId, NScheme::TInt64::TypeId, false) + .AddColumn(TableId, "value", ValueColumnId, NScheme::TString::TypeId, false) + .AddColumn(TableId, "counter", CounterColumnId, NScheme::TInt64::TypeId, false) .AddColumnToKey(TableId, KeyColumnId) .SetCompactionPolicy(TableId, *policy) .SetFamilyBlobs(TableId, 0, -1, 128); @@ -2819,8 +2819,8 @@ Y_UNIT_TEST_SUITE(TFlatTableExecutorMoveTableData) { txc.DB.Alter() .AddTable("test" + ToString(ui32(tableId)), tableId) - .AddColumn(tableId, "key", TRowsModel::ColumnKeyId, NScheme::TInt64::TypeId, false) - .AddColumn(tableId, "value", TRowsModel::ColumnValueId, NScheme::TString::TypeId, false) + .AddColumn(tableId, "key", TRowsModel::ColumnKeyId, NScheme::TInt64::TypeId, false) + .AddColumn(tableId, "value", TRowsModel::ColumnValueId, NScheme::TString::TypeId, false) .AddColumnToKey(tableId, TRowsModel::ColumnKeyId) .SetCompactionPolicy(tableId, policy); } @@ -3557,8 +3557,8 @@ Y_UNIT_TEST_SUITE(TFlatTableExecutorRejectProbability) { for (ui32 tableId = 201; tableId <= 208; ++tableId) { txc.DB.Alter() .AddTable("test" + ToString(ui32(tableId)), tableId) - .AddColumn(tableId, "key", TRowsModel::ColumnKeyId, NScheme::TInt64::TypeId, false) - .AddColumn(tableId, "value", TRowsModel::ColumnValueId, NScheme::TString::TypeId, false) + .AddColumn(tableId, "key", TRowsModel::ColumnKeyId, NScheme::TInt64::TypeId, false) + .AddColumn(tableId, "value", TRowsModel::ColumnValueId, NScheme::TString::TypeId, false) .AddColumnToKey(tableId, TRowsModel::ColumnKeyId) .SetCompactionPolicy(tableId, *Policy); } @@ -3764,8 +3764,8 @@ Y_UNIT_TEST_SUITE(TFlatTableCold) { txc.DB.Alter() .AddTable("test" + ToString(ui32(TRowsModel::TableId)), TRowsModel::TableId) - .AddColumn(TRowsModel::TableId, "key", TRowsModel::ColumnKeyId, NScheme::TInt64::TypeId, false) - .AddColumn(TRowsModel::TableId, "value", TRowsModel::ColumnValueId, NScheme::TString::TypeId, false) + .AddColumn(TRowsModel::TableId, "key", TRowsModel::ColumnKeyId, NScheme::TInt64::TypeId, false) + .AddColumn(TRowsModel::TableId, "value", TRowsModel::ColumnValueId, NScheme::TString::TypeId, false) .AddColumnToKey(TRowsModel::TableId, TRowsModel::ColumnKeyId) .SetCompactionPolicy(TRowsModel::TableId, *Policy) .SetColdBorrow(TRowsModel::TableId, true); diff --git a/ydb/core/tablet_flat/flat_table_column.h b/ydb/core/tablet_flat/flat_table_column.h index a53fafebd30..2f9791a6363 100644 --- a/ydb/core/tablet_flat/flat_table_column.h +++ b/ydb/core/tablet_flat/flat_table_column.h @@ -15,11 +15,11 @@ namespace NTable { TColumn() = default; - TColumn(const TString& name, TTag tag, NScheme::TTypeId type, bool notNull = false) + TColumn(const TString& name, TTag tag, NScheme::TTypeId type, bool notNull = false) : Id(tag) , PType(type) , Name(name) - , NotNull(notNull) + , NotNull(notNull) { } @@ -31,8 +31,8 @@ namespace NTable { && PType == col.PType && KeyOrder == col.KeyOrder && Name == col.Name - && Family == col.Family - && NotNull == col.NotNull; + && Family == col.Family + && NotNull == col.NotNull; } void SetDefault(const TCell &null) @@ -53,7 +53,7 @@ namespace NTable { NTable::TPos KeyOrder = Max<TPos>(); TCell Null; TString Storage; - bool NotNull = false; + bool NotNull = false; }; } } diff --git a/ydb/core/tablet_flat/test/libs/exec/fuzzy.h b/ydb/core/tablet_flat/test/libs/exec/fuzzy.h index b884a683737..1523dc0b39e 100644 --- a/ydb/core/tablet_flat/test/libs/exec/fuzzy.h +++ b/ydb/core/tablet_flat/test/libs/exec/fuzzy.h @@ -35,8 +35,8 @@ namespace NFake { delta.SetFamilyBlobs(table, 0, 6500, 7000); } - delta.AddColumn(table, "key", KeyId, NScheme::TInt64::TypeId, false); - delta.AddColumn(table, "value", 200, NScheme::TString::TypeId, false); + delta.AddColumn(table, "key", KeyId, NScheme::TInt64::TypeId, false); + delta.AddColumn(table, "value", 200, NScheme::TString::TypeId, false); delta.AddColumnToKey(table, KeyId); if (table >= 2) { /* used { 0, 1, 2, 3 } tables */ diff --git a/ydb/core/tablet_flat/ut/ut_bloom.cpp b/ydb/core/tablet_flat/ut/ut_bloom.cpp index dc7be1a3d09..0ba3f5b6629 100644 --- a/ydb/core/tablet_flat/ut/ut_bloom.cpp +++ b/ydb/core/tablet_flat/ut/ut_bloom.cpp @@ -96,8 +96,8 @@ Y_UNIT_TEST_SUITE(Bloom) { alter .AddTable("me", table) - .AddColumn(1, "name", 1, ETypes::String, false) - .AddColumn(1, "salt", 2, ETypes::Uint64, false) + .AddColumn(1, "name", 1, ETypes::String, false) + .AddColumn(1, "salt", 2, ETypes::Uint64, false) .AddColumnToKey(1, 1) .AddColumnToKey(1, 2); @@ -261,7 +261,7 @@ Y_UNIT_TEST_SUITE(Bloom) { TAlter alter; auto name = Sprintf("sub_%04u", col); - alter.AddColumn(1, name, col, ETypes::String, false); + alter.AddColumn(1, name, col, ETypes::String, false); alter.AddColumnToKey(1, col); alter.SetByKeyFilter(1, col > 10); /* first 8 parts w/o filter */ diff --git a/ydb/core/tablet_flat/ut/ut_db_iface.cpp b/ydb/core/tablet_flat/ut/ut_db_iface.cpp index dbd9929cec9..9843e5a6033 100644 --- a/ydb/core/tablet_flat/ut/ut_db_iface.cpp +++ b/ydb/core/tablet_flat/ut/ut_db_iface.cpp @@ -17,9 +17,9 @@ Y_UNIT_TEST_SUITE(DBase) { alter .AddTable(Sprintf("me_%02u", table), table) - .AddColumn(table, "key", 1, ETypes::String, false) - .AddColumn(table, "arg1", 4, ETypes::Uint64, false, Cimple(77_u64)) - .AddColumn(table, "arg2", 5, ETypes::String, false) + .AddColumn(table, "key", 1, ETypes::String, false) + .AddColumn(table, "arg1", 4, ETypes::Uint64, false, Cimple(77_u64)) + .AddColumn(table, "arg2", 5, ETypes::String, false) .AddColumnToKey(table, 1); return alter; @@ -144,10 +144,10 @@ Y_UNIT_TEST_SUITE(DBase) { me.To(12).Begin().Apply( *TAlter() - .AddColumn(1, "sub", 2, ETypes::Uint32, false, Cimple(77_u32)) - .AddColumn(1, "en0", 6, ETypes::String, false, TCell("", 0)) - .AddColumn(1, "en1", 7, ETypes::String, false, TCell(large30)) - .AddColumn(1, "en2", 8, ETypes::String, false, TCell(large35)) + .AddColumn(1, "sub", 2, ETypes::Uint32, false, Cimple(77_u32)) + .AddColumn(1, "en0", 6, ETypes::String, false, TCell("", 0)) + .AddColumn(1, "en1", 7, ETypes::String, false, TCell(large30)) + .AddColumn(1, "en2", 8, ETypes::String, false, TCell(large35)) .DropColumn(1, 5) .AddColumnToKey(1, 2)); @@ -243,7 +243,7 @@ Y_UNIT_TEST_SUITE(DBase) { me.To(14).Compact(1, false).Iter(1).Has(ro3); { /*_ Ressurect column, ** hack allowed only in this UT ** */ - auto raw = TAlter().AddColumn(1, "new", 4, ETypes::Uint64, false).Flush(); + auto raw = TAlter().AddColumn(1, "new", 4, ETypes::Uint64, false).Flush(); me.Begin().Apply(*raw).Add(1, ro4).Commit(); } @@ -330,7 +330,7 @@ Y_UNIT_TEST_SUITE(DBase) { for (auto sub : xrange(keys)) { auto name = Sprintf("sub_%04u", sub); - alter.AddColumn(1, name, 6 + sub, ETypes::Uint32, false); + alter.AddColumn(1, name, 6 + sub, ETypes::Uint32, false); alter.AddColumnToKey(1, 6 + sub); } @@ -592,9 +592,9 @@ Y_UNIT_TEST_SUITE(DBase) { .Begin() .Apply(*TAlter() .AddTable("me_1", table) - .AddColumn(table, "key", 1, ETypes::Uint64, false) - .AddColumn(table, "arg1", 4, ETypes::Uint64, false, Cimple(10004_u64)) - .AddColumn(table, "arg2", 5, ETypes::Uint64, false, Cimple(10005_u64)) + .AddColumn(table, "key", 1, ETypes::Uint64, false) + .AddColumn(table, "arg1", 4, ETypes::Uint64, false, Cimple(10004_u64)) + .AddColumn(table, "arg2", 5, ETypes::Uint64, false, Cimple(10005_u64)) .AddColumnToKey(table, 1)) .Commit(); diff --git a/ydb/core/tx/datashard/datashard_kqp_compute.cpp b/ydb/core/tx/datashard/datashard_kqp_compute.cpp index 60c1b5f0d11..edd6ca8e194 100644 --- a/ydb/core/tx/datashard/datashard_kqp_compute.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_compute.cpp @@ -141,13 +141,13 @@ TString TKqpDatashardComputeContext::GetTablePath(const TTableId &tableId) const return (*table)->Path; } -const NDataShard::TUserTable* TKqpDatashardComputeContext::GetTable(const TTableId& tableId) const { - MKQL_ENSURE_S(Shard); - auto ptr = Shard->GetUserTables().FindPtr(tableId.PathId.LocalPathId); - MKQL_ENSURE_S(ptr); - return ptr->Get(); -} - +const NDataShard::TUserTable* TKqpDatashardComputeContext::GetTable(const TTableId& tableId) const { + MKQL_ENSURE_S(Shard); + auto ptr = Shard->GetUserTables().FindPtr(tableId.PathId.LocalPathId); + MKQL_ENSURE_S(ptr); + return ptr->Get(); +} + void TKqpDatashardComputeContext::ReadTable(const TTableId& tableId, const TTableRange& range) const { MKQL_ENSURE_S(Shard); Shard->SysLocksTable().SetLock(tableId, range, LockTxId); diff --git a/ydb/core/tx/datashard/datashard_kqp_compute.h b/ydb/core/tx/datashard/datashard_kqp_compute.h index f3d9e89efb6..4bbb8fa0de7 100644 --- a/ydb/core/tx/datashard/datashard_kqp_compute.h +++ b/ydb/core/tx/datashard/datashard_kqp_compute.h @@ -9,7 +9,7 @@ namespace NKikimr { namespace NDataShard { class TExecuteKqpScanTxUnit; class TDataShard; - struct TUserTable; + struct TUserTable; } } @@ -29,7 +29,7 @@ public: ui64 GetLocalTableId(const TTableId& tableId) const; TString GetTablePath(const TTableId& tableId) const; - const NDataShard::TUserTable* GetTable(const TTableId& tableId) const; + const NDataShard::TUserTable* GetTable(const TTableId& tableId) const; void ReadTable(const TTableId& tableId, const TTableRange& range) const; void ReadTable(const TTableId& tableId, const TArrayRef<const TCell>& key) const; void BreakSetLocks() const; diff --git a/ydb/core/tx/datashard/datashard_kqp_read_table.cpp b/ydb/core/tx/datashard/datashard_kqp_read_table.cpp index 3081637f0d6..9fbd8ef0ad2 100644 --- a/ydb/core/tx/datashard/datashard_kqp_read_table.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_read_table.cpp @@ -86,103 +86,103 @@ TSerializedTableRange CreateTableRange(const TParseReadTableResult& parseResult, return TSerializedTableRange(fromCells, parseResult.FromInclusive, toCells, parseResult.ToInclusive); } -TSerializedTableRange BuildFullRange(ui32 keyColumnsSize) { - /* Build range from NULL, ... NULL to +inf, ... +inf */ - TVector<TCell> fromKeyValues(keyColumnsSize); - return TSerializedTableRange(fromKeyValues, true, TVector<TCell>(), false); -} - -TSerializedTableRange BuildRange(const TTupleType* fromType, const NUdf::TUnboxedValue& fromValue, - const TTupleType* toType, const NUdf::TUnboxedValue& toValue, const TTypeEnvironment& typeEnv, ui32 keyColumnsSize) { - - auto fillTupleCells = [&typeEnv, keyColumnsSize](const auto tupleType, const auto& tupleValue) { - TVector<TCell> cells; - cells.reserve(keyColumnsSize); - - for (ui32 i = 0; i < keyColumnsSize; ++i) { - auto type = tupleType->GetElementType(i); - auto value = tupleValue.GetElement(i); - - if (type->IsOptional()) { - if (!value) { - return cells; - } - - type = AS_TYPE(TOptionalType, type)->GetItemType(); - value = value.GetOptionalValue(); - - if (type->IsOptional()) { - if (!value) { - cells.emplace_back(TCell()); - continue; - } - - type = AS_TYPE(TOptionalType, type)->GetItemType(); - value = value.GetOptionalValue(); - } - } - - cells.emplace_back(MakeCell(AS_TYPE(TDataType, type)->GetSchemeType(), value, typeEnv, /* copy */ true)); - } - - return cells; - }; - - Y_ENSURE(fromType->GetElementsCount() == keyColumnsSize + 1); - bool fromInclusive = !!fromValue.GetElement(keyColumnsSize).Get<int>(); - auto fromCells = fillTupleCells(fromType, fromValue); - - if (fromCells.empty()) { - fromInclusive = true; - } - - if (fromInclusive) { - while (fromCells.size() != keyColumnsSize) { - fromCells.emplace_back(TCell()); - } - } - - Y_ENSURE(toType->GetElementsCount() == keyColumnsSize + 1); - bool toInclusive = !!toValue.GetElement(keyColumnsSize).Get<int>(); - auto toCells = fillTupleCells(toType, toValue); - - if (!toInclusive && !toCells.empty()) { - while (toCells.size() != keyColumnsSize) { - toCells.emplace_back(TCell()); - } - } - - return TSerializedTableRange(fromCells, fromInclusive, toCells, toInclusive); -} - -template <bool IsReverse> -TVector<TSerializedTableRange> CreateTableRanges(const TParseReadTableRangesResult& parseResult, - const IComputationNode* rangesNode, const TTypeEnvironment& typeEnv, TComputationContext& ctx, ui32 keyColumnsSize) -{ +TSerializedTableRange BuildFullRange(ui32 keyColumnsSize) { + /* Build range from NULL, ... NULL to +inf, ... +inf */ + TVector<TCell> fromKeyValues(keyColumnsSize); + return TSerializedTableRange(fromKeyValues, true, TVector<TCell>(), false); +} + +TSerializedTableRange BuildRange(const TTupleType* fromType, const NUdf::TUnboxedValue& fromValue, + const TTupleType* toType, const NUdf::TUnboxedValue& toValue, const TTypeEnvironment& typeEnv, ui32 keyColumnsSize) { + + auto fillTupleCells = [&typeEnv, keyColumnsSize](const auto tupleType, const auto& tupleValue) { + TVector<TCell> cells; + cells.reserve(keyColumnsSize); + + for (ui32 i = 0; i < keyColumnsSize; ++i) { + auto type = tupleType->GetElementType(i); + auto value = tupleValue.GetElement(i); + + if (type->IsOptional()) { + if (!value) { + return cells; + } + + type = AS_TYPE(TOptionalType, type)->GetItemType(); + value = value.GetOptionalValue(); + + if (type->IsOptional()) { + if (!value) { + cells.emplace_back(TCell()); + continue; + } + + type = AS_TYPE(TOptionalType, type)->GetItemType(); + value = value.GetOptionalValue(); + } + } + + cells.emplace_back(MakeCell(AS_TYPE(TDataType, type)->GetSchemeType(), value, typeEnv, /* copy */ true)); + } + + return cells; + }; + + Y_ENSURE(fromType->GetElementsCount() == keyColumnsSize + 1); + bool fromInclusive = !!fromValue.GetElement(keyColumnsSize).Get<int>(); + auto fromCells = fillTupleCells(fromType, fromValue); + + if (fromCells.empty()) { + fromInclusive = true; + } + + if (fromInclusive) { + while (fromCells.size() != keyColumnsSize) { + fromCells.emplace_back(TCell()); + } + } + + Y_ENSURE(toType->GetElementsCount() == keyColumnsSize + 1); + bool toInclusive = !!toValue.GetElement(keyColumnsSize).Get<int>(); + auto toCells = fillTupleCells(toType, toValue); + + if (!toInclusive && !toCells.empty()) { + while (toCells.size() != keyColumnsSize) { + toCells.emplace_back(TCell()); + } + } + + return TSerializedTableRange(fromCells, fromInclusive, toCells, toInclusive); +} + +template <bool IsReverse> +TVector<TSerializedTableRange> CreateTableRanges(const TParseReadTableRangesResult& parseResult, + const IComputationNode* rangesNode, const TTypeEnvironment& typeEnv, TComputationContext& ctx, ui32 keyColumnsSize) +{ auto keyRangesType = ParseKeyRangesType(parseResult.Ranges->GetType()); if (!keyRangesType) { - return {BuildFullRange(keyColumnsSize)}; - } - - auto list = rangesNode->GetValue(ctx).GetElement(0).GetElement(0); - - TVector<TSerializedTableRange> ranges; - auto listIt = list.GetListIterator(); - TUnboxedValue tuple; - while (listIt.Next(tuple)) { - const auto& from = tuple.GetElement(0); - const auto& to = tuple.GetElement(1); - + return {BuildFullRange(keyColumnsSize)}; + } + + auto list = rangesNode->GetValue(ctx).GetElement(0).GetElement(0); + + TVector<TSerializedTableRange> ranges; + auto listIt = list.GetListIterator(); + TUnboxedValue tuple; + while (listIt.Next(tuple)) { + const auto& from = tuple.GetElement(0); + const auto& to = tuple.GetElement(1); + ranges.emplace_back(BuildRange(keyRangesType->From, from, keyRangesType->To, to, typeEnv, keyColumnsSize)); - } - - if constexpr (IsReverse) { - Reverse(ranges.begin(), ranges.end()); - } - - return ranges; -} - + } + + if constexpr (IsReverse) { + Reverse(ranges.begin(), ranges.end()); + } + + return ranges; +} + void CreateRangePoints(ui64 localTid, const TSerializedTableRange& serializedTableRange, TSmallVec<TRawTypeValue>& from, TSmallVec<TRawTypeValue>& to, TKqpDatashardComputeContext& computeCtx) { @@ -193,17 +193,17 @@ void CreateRangePoints(ui64 localTid, const TSerializedTableRange& serializedTab } template <bool IsReverse> -class TKqpWideReadTableWrapperBase : public TStatelessWideFlowCodegeneratorNode<TKqpWideReadTableWrapperBase<IsReverse>> { +class TKqpWideReadTableWrapperBase : public TStatelessWideFlowCodegeneratorNode<TKqpWideReadTableWrapperBase<IsReverse>> { public: - TKqpWideReadTableWrapperBase(TKqpDatashardComputeContext& computeCtx, const TTypeEnvironment& typeEnv, - const TSmallVec<TTag>& systemColumnTags, const TSmallVec<bool>& skipNullKeys) - : TStatelessWideFlowCodegeneratorNode<TKqpWideReadTableWrapperBase<IsReverse>>(this) + TKqpWideReadTableWrapperBase(TKqpDatashardComputeContext& computeCtx, const TTypeEnvironment& typeEnv, + const TSmallVec<TTag>& systemColumnTags, const TSmallVec<bool>& skipNullKeys) + : TStatelessWideFlowCodegeneratorNode<TKqpWideReadTableWrapperBase<IsReverse>>(this) , ComputeCtx(computeCtx) , TypeEnv(typeEnv) - , SystemColumnTags(systemColumnTags) - , SkipNullKeys(skipNullKeys) + , SystemColumnTags(systemColumnTags) + , SkipNullKeys(skipNullKeys) , ShardTableStats(ComputeCtx.GetDatashardCounters()) - , TaskTableStats(ComputeCtx.GetTaskCounters(ComputeCtx.GetCurrentTaskId())) { + , TaskTableStats(ComputeCtx.GetTaskCounters(ComputeCtx.GetCurrentTaskId())) { } EFetchResult DoCalculate(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const { @@ -217,10 +217,10 @@ public: } #endif -protected: - virtual EFetchResult ReadValue(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const = 0; +protected: + virtual EFetchResult ReadValue(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const = 0; - EFetchResult ReadNext(NUdf::TUnboxedValue* const* output) const { + EFetchResult ReadNext(NUdf::TUnboxedValue* const* output) const { bool breakLocks = false; while (Iterator->Next(NTable::ENext::Data) == NTable::EReady::Data) { if (!breakLocks && (breakLocks = bool(Iterator->Stats.InvisibleRowSkips))) { @@ -237,10 +237,10 @@ protected: TaskTableStats.SelectRangeDeletedRowSkips += deletedRowSkips; TaskTableStats.InvisibleRowSkips += invisibleRowSkips; - Y_VERIFY(SkipNullKeys.size() <= rowKey.ColumnCount); + Y_VERIFY(SkipNullKeys.size() <= rowKey.ColumnCount); bool skipRow = false; - for (ui32 i = 0; i < SkipNullKeys.size(); ++i) { - if (SkipNullKeys[i] && rowKey.Columns[i].IsNull()) { + for (ui32 i = 0; i < SkipNullKeys.size(); ++i) { + if (SkipNullKeys[i] && rowKey.Columns[i].IsNull()) { skipRow = true; break; } @@ -305,7 +305,7 @@ protected: TaskTableStats.SelectRangeDeletedRowSkips += deletedRowSkips; TaskTableStats.InvisibleRowSkips += invisibleRowSkips; - if (Iterator->Last() == NTable::EReady::Page) { + if (Iterator->Last() == NTable::EReady::Page) { ComputeCtx.SetTabletNotReady(); return EFetchResult::Yield; } @@ -313,71 +313,71 @@ protected: return EFetchResult::Finish; } -protected: - TKqpDatashardComputeContext& ComputeCtx; - const TTypeEnvironment& TypeEnv; - TSmallVec<TTag> SystemColumnTags; - TSmallVec<bool> SkipNullKeys; - TKqpTableStats& ShardTableStats; - TKqpTableStats& TaskTableStats; - using TTableIterator = std::conditional_t<IsReverse, NTable::TTableReverseIt, NTable::TTableIt>; - mutable TAutoPtr<TTableIterator> Iterator; - mutable std::optional<ui64> Remains; -}; - -template <bool IsReverse> -class TKqpWideReadTableWrapper : public TKqpWideReadTableWrapperBase<IsReverse> { -public: - TKqpWideReadTableWrapper(TKqpDatashardComputeContext& computeCtx, const TTypeEnvironment& typeEnv, - const TParseReadTableResult& parseResult, IComputationNode* fromNode, IComputationNode* toNode, - IComputationNode* itemsLimit) - : TKqpWideReadTableWrapperBase<IsReverse>(computeCtx, typeEnv, ExtractTags(parseResult.SystemColumns), parseResult.SkipNullKeys) - , ParseResult(parseResult) - , FromNode(fromNode) - , ToNode(toNode) - , ItemsLimit(itemsLimit) - , LocalTid(computeCtx.GetLocalTableId(parseResult.TableId)) - , ColumnTags(ExtractTags(parseResult.Columns)) - { - this->ShardTableStats.NSelectRange++; - this->TaskTableStats.NSelectRange++; - } - +protected: + TKqpDatashardComputeContext& ComputeCtx; + const TTypeEnvironment& TypeEnv; + TSmallVec<TTag> SystemColumnTags; + TSmallVec<bool> SkipNullKeys; + TKqpTableStats& ShardTableStats; + TKqpTableStats& TaskTableStats; + using TTableIterator = std::conditional_t<IsReverse, NTable::TTableReverseIt, NTable::TTableIt>; + mutable TAutoPtr<TTableIterator> Iterator; + mutable std::optional<ui64> Remains; +}; + +template <bool IsReverse> +class TKqpWideReadTableWrapper : public TKqpWideReadTableWrapperBase<IsReverse> { +public: + TKqpWideReadTableWrapper(TKqpDatashardComputeContext& computeCtx, const TTypeEnvironment& typeEnv, + const TParseReadTableResult& parseResult, IComputationNode* fromNode, IComputationNode* toNode, + IComputationNode* itemsLimit) + : TKqpWideReadTableWrapperBase<IsReverse>(computeCtx, typeEnv, ExtractTags(parseResult.SystemColumns), parseResult.SkipNullKeys) + , ParseResult(parseResult) + , FromNode(fromNode) + , ToNode(toNode) + , ItemsLimit(itemsLimit) + , LocalTid(computeCtx.GetLocalTableId(parseResult.TableId)) + , ColumnTags(ExtractTags(parseResult.Columns)) + { + this->ShardTableStats.NSelectRange++; + this->TaskTableStats.NSelectRange++; + } + +private: + EFetchResult ReadValue(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const final { + if (!this->Iterator) { + auto serializedTableRange = CreateTableRange(ParseResult, FromNode, ToNode, this->TypeEnv, ctx); + auto tableRange = serializedTableRange.ToTableRange(); + this->ComputeCtx.ReadTable(ParseResult.TableId, tableRange); + + TSmallVec<TRawTypeValue> from, to; + CreateRangePoints(LocalTid, serializedTableRange, from, to, this->ComputeCtx); + + NTable::TKeyRange keyRange; + keyRange.MinKey = from; + keyRange.MaxKey = to; + keyRange.MinInclusive = tableRange.InclusiveFrom; + keyRange.MaxInclusive = tableRange.InclusiveTo; + + if (ItemsLimit) { + this->Remains = ItemsLimit->GetValue(ctx).Get<ui64>(); + } + + if constexpr (IsReverse) { + this->Iterator = this->ComputeCtx.Database->IterateRangeReverse(LocalTid, keyRange, ColumnTags, this->ComputeCtx.GetReadVersion()); + } else { + this->Iterator = this->ComputeCtx.Database->IterateRange(LocalTid, keyRange, ColumnTags, this->ComputeCtx.GetReadVersion()); + } + } + + if (this->Remains && *this->Remains == 0) { + return EFetchResult::Finish; + } + + return this->ReadNext(output); + } + private: - EFetchResult ReadValue(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const final { - if (!this->Iterator) { - auto serializedTableRange = CreateTableRange(ParseResult, FromNode, ToNode, this->TypeEnv, ctx); - auto tableRange = serializedTableRange.ToTableRange(); - this->ComputeCtx.ReadTable(ParseResult.TableId, tableRange); - - TSmallVec<TRawTypeValue> from, to; - CreateRangePoints(LocalTid, serializedTableRange, from, to, this->ComputeCtx); - - NTable::TKeyRange keyRange; - keyRange.MinKey = from; - keyRange.MaxKey = to; - keyRange.MinInclusive = tableRange.InclusiveFrom; - keyRange.MaxInclusive = tableRange.InclusiveTo; - - if (ItemsLimit) { - this->Remains = ItemsLimit->GetValue(ctx).Get<ui64>(); - } - - if constexpr (IsReverse) { - this->Iterator = this->ComputeCtx.Database->IterateRangeReverse(LocalTid, keyRange, ColumnTags, this->ComputeCtx.GetReadVersion()); - } else { - this->Iterator = this->ComputeCtx.Database->IterateRange(LocalTid, keyRange, ColumnTags, this->ComputeCtx.GetReadVersion()); - } - } - - if (this->Remains && *this->Remains == 0) { - return EFetchResult::Finish; - } - - return this->ReadNext(output); - } - -private: void RegisterDependencies() const final { this->FlowDependsOn(FromNode); this->FlowDependsOn(ToNode); @@ -393,87 +393,87 @@ private: ui64 TaskId; }; -template <bool IsReverse> -class TKqpWideReadTableRangesWrapper : public TKqpWideReadTableWrapperBase<IsReverse> { -public: - TKqpWideReadTableRangesWrapper(TKqpDatashardComputeContext& computeCtx, const TTypeEnvironment& typeEnv, - const TParseReadTableRangesResult& parseResult, IComputationNode* rangesNode, IComputationNode* itemsLimit) - : TKqpWideReadTableWrapperBase<IsReverse>(computeCtx, typeEnv, ExtractTags(parseResult.SystemColumns), parseResult.SkipNullKeys) - , ParseResult(parseResult) - , RangesNode(rangesNode) - , ItemsLimit(itemsLimit) - , LocalTid(computeCtx.GetLocalTableId(parseResult.TableId)) - , ColumnTags(ExtractTags(parseResult.Columns)) { - } - -private: - EFetchResult ReadValue(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const final { - if (!RangeId) { - const auto* tableInfo = this->ComputeCtx.Database->GetScheme().GetTableInfo(LocalTid); - Ranges = CreateTableRanges<IsReverse>(ParseResult, RangesNode, this->TypeEnv, ctx, tableInfo->KeyColumns.size()); - RangeId = 0; - - if (ItemsLimit) { - this->Remains = ItemsLimit->GetValue(ctx).Get<ui64>(); - } - } - - Y_ENSURE(RangeId); - while (*RangeId < Ranges.size()) { - this->ShardTableStats.NSelectRange++; - this->TaskTableStats.NSelectRange++; - - if (!this->Iterator) { - auto& range = Ranges[*RangeId]; - auto tableRange = range.ToTableRange(); - this->ComputeCtx.ReadTable(ParseResult.TableId, tableRange); - - TSmallVec<TRawTypeValue> from, to; - CreateRangePoints(LocalTid, range, from, to, this->ComputeCtx); - - NTable::TKeyRange keyRange; - keyRange.MinKey = from; - keyRange.MaxKey = to; - keyRange.MinInclusive = tableRange.InclusiveFrom; - keyRange.MaxInclusive = tableRange.InclusiveTo; - - if constexpr (IsReverse) { - this->Iterator = this->ComputeCtx.Database->IterateRangeReverse(LocalTid, keyRange, ColumnTags, this->ComputeCtx.GetReadVersion()); - } else { - this->Iterator = this->ComputeCtx.Database->IterateRange(LocalTid, keyRange, ColumnTags, this->ComputeCtx.GetReadVersion()); - } - } - - if (this->Remains && *this->Remains == 0) { - return EFetchResult::Finish; - } - - auto status = this->ReadNext(output); - if (status != EFetchResult::Finish) { - return status; - } - - this->Iterator.Reset(); - ++(*RangeId); - } - - return EFetchResult::Finish; - } - - void RegisterDependencies() const final { - this->FlowDependsOn(RangesNode); - } - - TParseReadTableRangesResult ParseResult; - IComputationNode* RangesNode; - IComputationNode* ItemsLimit; - ui64 LocalTid; - TSmallVec<TTag> ColumnTags; - ui64 TaskId; - mutable TVector<TSerializedTableRange> Ranges; - mutable std::optional<ui32> RangeId; -}; - +template <bool IsReverse> +class TKqpWideReadTableRangesWrapper : public TKqpWideReadTableWrapperBase<IsReverse> { +public: + TKqpWideReadTableRangesWrapper(TKqpDatashardComputeContext& computeCtx, const TTypeEnvironment& typeEnv, + const TParseReadTableRangesResult& parseResult, IComputationNode* rangesNode, IComputationNode* itemsLimit) + : TKqpWideReadTableWrapperBase<IsReverse>(computeCtx, typeEnv, ExtractTags(parseResult.SystemColumns), parseResult.SkipNullKeys) + , ParseResult(parseResult) + , RangesNode(rangesNode) + , ItemsLimit(itemsLimit) + , LocalTid(computeCtx.GetLocalTableId(parseResult.TableId)) + , ColumnTags(ExtractTags(parseResult.Columns)) { + } + +private: + EFetchResult ReadValue(TComputationContext& ctx, NUdf::TUnboxedValue* const* output) const final { + if (!RangeId) { + const auto* tableInfo = this->ComputeCtx.Database->GetScheme().GetTableInfo(LocalTid); + Ranges = CreateTableRanges<IsReverse>(ParseResult, RangesNode, this->TypeEnv, ctx, tableInfo->KeyColumns.size()); + RangeId = 0; + + if (ItemsLimit) { + this->Remains = ItemsLimit->GetValue(ctx).Get<ui64>(); + } + } + + Y_ENSURE(RangeId); + while (*RangeId < Ranges.size()) { + this->ShardTableStats.NSelectRange++; + this->TaskTableStats.NSelectRange++; + + if (!this->Iterator) { + auto& range = Ranges[*RangeId]; + auto tableRange = range.ToTableRange(); + this->ComputeCtx.ReadTable(ParseResult.TableId, tableRange); + + TSmallVec<TRawTypeValue> from, to; + CreateRangePoints(LocalTid, range, from, to, this->ComputeCtx); + + NTable::TKeyRange keyRange; + keyRange.MinKey = from; + keyRange.MaxKey = to; + keyRange.MinInclusive = tableRange.InclusiveFrom; + keyRange.MaxInclusive = tableRange.InclusiveTo; + + if constexpr (IsReverse) { + this->Iterator = this->ComputeCtx.Database->IterateRangeReverse(LocalTid, keyRange, ColumnTags, this->ComputeCtx.GetReadVersion()); + } else { + this->Iterator = this->ComputeCtx.Database->IterateRange(LocalTid, keyRange, ColumnTags, this->ComputeCtx.GetReadVersion()); + } + } + + if (this->Remains && *this->Remains == 0) { + return EFetchResult::Finish; + } + + auto status = this->ReadNext(output); + if (status != EFetchResult::Finish) { + return status; + } + + this->Iterator.Reset(); + ++(*RangeId); + } + + return EFetchResult::Finish; + } + + void RegisterDependencies() const final { + this->FlowDependsOn(RangesNode); + } + + TParseReadTableRangesResult ParseResult; + IComputationNode* RangesNode; + IComputationNode* ItemsLimit; + ui64 LocalTid; + TSmallVec<TTag> ColumnTags; + ui64 TaskId; + mutable TVector<TSerializedTableRange> Ranges; + mutable std::optional<ui32> RangeId; +}; + void FetchRowImpl(const TDbTupleRef& dbTuple, TUnboxedValue& row, TComputationContext& ctx, TKqpTableStats& tableStats, const TKqpDatashardComputeContext& computeCtx, const TSmallVec<TTag>& systemColumnTags) { @@ -560,9 +560,9 @@ void FetchRow(const TDbTupleRef& dbTuple, TUnboxedValue& row, TComputationContex IComputationNode* WrapKqpWideReadTableRanges(TCallable& callable, const TComputationNodeFactoryContext& ctx, TKqpDatashardComputeContext& computeCtx) { - auto parseResult = ParseWideReadTableRanges(callable); - auto rangesNode = LocateNode(ctx.NodeLocator, *parseResult.Ranges); - + auto parseResult = ParseWideReadTableRanges(callable); + auto rangesNode = LocateNode(ctx.NodeLocator, *parseResult.Ranges); + auto keyColumns = computeCtx.GetKeyColumnsInfo(parseResult.TableId); auto keyRangesType = ParseKeyRangesType(parseResult.Ranges->GetType()); if (keyRangesType) { @@ -570,16 +570,16 @@ IComputationNode* WrapKqpWideReadTableRanges(TCallable& callable, const TComputa ValidateRangeBound(keyRangesType->To, keyColumns); } - IComputationNode* itemsLimit = nullptr; - if (parseResult.ItemsLimit) { - itemsLimit = LocateNode(ctx.NodeLocator, *parseResult.ItemsLimit); - } - - if (parseResult.Reverse) { - return new TKqpWideReadTableRangesWrapper<true>(computeCtx, ctx.Env, parseResult, rangesNode, itemsLimit); - } - - return new TKqpWideReadTableRangesWrapper<false>(computeCtx, ctx.Env, parseResult, rangesNode, itemsLimit); + IComputationNode* itemsLimit = nullptr; + if (parseResult.ItemsLimit) { + itemsLimit = LocateNode(ctx.NodeLocator, *parseResult.ItemsLimit); + } + + if (parseResult.Reverse) { + return new TKqpWideReadTableRangesWrapper<true>(computeCtx, ctx.Env, parseResult, rangesNode, itemsLimit); + } + + return new TKqpWideReadTableRangesWrapper<false>(computeCtx, ctx.Env, parseResult, rangesNode, itemsLimit); } IComputationNode* WrapKqpWideReadTable(TCallable& callable, const TComputationNodeFactoryContext& ctx, diff --git a/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp b/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp index a9edd6b5304..a2dfa61bbfc 100644 --- a/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp +++ b/ydb/core/tx/datashard/datashard_kqp_upsert_rows.cpp @@ -1,5 +1,5 @@ #include "datashard_kqp_compute.h" -#include "datashard_user_table.h" +#include "datashard_user_table.h" #include <ydb/core/engine/mkql_keys.h> #include <ydb/core/engine/mkql_engine_flat_host.h> @@ -171,11 +171,11 @@ IComputationNode* WrapKqpUpsertRows(TCallable& callable, const TComputationNodeF MKQL_ENSURE_S(localTableId); auto tableKeyTypes = computeCtx.GetKeyColumnsInfo(tableId); - auto tableInfo = computeCtx.GetTable(tableId); - MKQL_ENSURE(tableInfo, "Table not found: " << tableId.PathId.ToString()); - + auto tableInfo = computeCtx.GetTable(tableId); + MKQL_ENSURE(tableInfo, "Table not found: " << tableId.PathId.ToString()); + auto rowType = AS_TYPE(TStructType, AS_TYPE(TStreamType, rowsNode.GetStaticType())->GetItemType()); - + MKQL_ENSURE_S(tableKeyTypes.size() <= rowType->GetMembersCount(), "not enough columns in the runtime node"); THashMap<TString, ui32> inputIndex; diff --git a/ydb/core/tx/datashard/datashard_user_table.cpp b/ydb/core/tx/datashard/datashard_user_table.cpp index 1f28e729d65..70bc9400733 100644 --- a/ydb/core/tx/datashard/datashard_user_table.cpp +++ b/ydb/core/tx/datashard/datashard_user_table.cpp @@ -220,7 +220,7 @@ void TUserTable::ParseProto(const NKikimrSchemeOp::TTableDescription& descr) column = TUserColumn(col.GetTypeId(), col.GetName()); } column.Family = col.GetFamily(); - column.NotNull = col.GetNotNull(); + column.NotNull = col.GetNotNull(); } for (const auto& col : descr.GetDropColumns()) { @@ -331,7 +331,7 @@ void TUserTable::AlterSchema() { descr->SetId(col.first); descr->SetTypeId(column.Type); descr->SetFamily(column.Family); - descr->SetNotNull(column.NotNull); + descr->SetNotNull(column.NotNull); } schema.SetPartitionRangeBegin(Range.From.GetBuffer()); @@ -388,7 +388,7 @@ void TUserTable::DoApplyCreate( ui32 columnId = col.first; const TUserColumn& column = col.second; - alter.AddColumn(tid, column.Name, columnId, column.Type, column.NotNull); + alter.AddColumn(tid, column.Name, columnId, column.Type, column.NotNull); alter.AddColumnToFamily(tid, columnId, column.Family); } @@ -490,7 +490,7 @@ void TUserTable::ApplyAlter( if (!oldTable.Columns.contains(colId)) { for (ui32 tid : tids) { - alter.AddColumn(tid, column.Name, colId, column.Type, column.NotNull); + alter.AddColumn(tid, column.Name, colId, column.Type, column.NotNull); } } diff --git a/ydb/core/tx/datashard/datashard_user_table.h b/ydb/core/tx/datashard/datashard_user_table.h index 196ef9e9bfc..0dab30cb84f 100644 --- a/ydb/core/tx/datashard/datashard_user_table.h +++ b/ydb/core/tx/datashard/datashard_user_table.h @@ -235,7 +235,7 @@ struct TUserTable : public TThrRefBase { TString Name; bool IsKey; ui32 Family = 0; - bool NotNull = false; + bool NotNull = false; TUserColumn(NScheme::TTypeId type, TString name, bool isKey = false) : Type(type) diff --git a/ydb/core/tx/datashard/datashard_ut_build_index.cpp b/ydb/core/tx/datashard/datashard_ut_build_index.cpp index 461a89472cb..2b9e3f0a3fc 100644 --- a/ydb/core/tx/datashard/datashard_ut_build_index.cpp +++ b/ydb/core/tx/datashard/datashard_ut_build_index.cpp @@ -90,7 +90,7 @@ Y_UNIT_TEST_SUITE(TTxDataShardBuildIndexScan) { .EnableOutOfOrder(enableOutOfOrder) .Policy(policy.Get()) .ShadowData(EShadowDataMode::Enabled) - .Columns({{"value", "Uint32", true, false}, {"key", "Uint32", true, false}}); + .Columns({{"value", "Uint32", true, false}, {"key", "Uint32", true, false}}); CreateShardedTable(server, sender, root, name, opts); } diff --git a/ydb/core/tx/datashard/datashard_ut_change_collector.cpp b/ydb/core/tx/datashard/datashard_ut_change_collector.cpp index bb5d2c0dc70..cdce49ecdb0 100644 --- a/ydb/core/tx/datashard/datashard_ut_change_collector.cpp +++ b/ydb/core/tx/datashard/datashard_ut_change_collector.cpp @@ -344,8 +344,8 @@ Y_UNIT_TEST_SUITE(AsyncIndexChangeCollector) { TShardedTableOptions SimpleTable() { return TShardedTableOptions() .Columns({ - {"pkey", "Uint32", true, false}, - {"ikey", "Uint32", false, false}, + {"pkey", "Uint32", true, false}, + {"ikey", "Uint32", false, false}, }) .Indexes({ {"by_ikey", {"ikey"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, @@ -411,9 +411,9 @@ Y_UNIT_TEST_SUITE(AsyncIndexChangeCollector) { TShardedTableOptions MultiIndexedTable() { return TShardedTableOptions() .Columns({ - {"pkey", "Uint32", true, false}, - {"ikey1", "Uint32", false, false}, - {"ikey2", "Uint32", false, false}, + {"pkey", "Uint32", true, false}, + {"ikey1", "Uint32", false, false}, + {"ikey2", "Uint32", false, false}, }) .Indexes({ {"by_ikey1", {"ikey1"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, @@ -465,8 +465,8 @@ Y_UNIT_TEST_SUITE(AsyncIndexChangeCollector) { TShardedTableOptions IndexedPrimaryKey() { return TShardedTableOptions() .Columns({ - {"pkey", "Uint32", true, false}, - {"ikey", "Uint32", false, false}, + {"pkey", "Uint32", true, false}, + {"ikey", "Uint32", false, false}, }) .Indexes({ {"by_ikey_pkey", {"ikey", "pkey"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, @@ -494,9 +494,9 @@ Y_UNIT_TEST_SUITE(AsyncIndexChangeCollector) { TShardedTableOptions CoveredIndex() { return TShardedTableOptions() .Columns({ - {"pkey", "Uint32", true, false}, - {"ikey", "Uint32", false, false}, - {"value", "Uint32", false, false}, + {"pkey", "Uint32", true, false}, + {"ikey", "Uint32", false, false}, + {"value", "Uint32", false, false}, }) .Indexes({ {"by_ikey", {"ikey"}, {"value"}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, @@ -637,8 +637,8 @@ Y_UNIT_TEST_SUITE(CdcStreamChangeCollector) { TShardedTableOptions SimpleTable() { return TShardedTableOptions() .Columns({ - {"key", "Uint32", true, false}, - {"value", "Uint32", false, false}, + {"key", "Uint32", true, false}, + {"value", "Uint32", false, false}, }); } @@ -743,8 +743,8 @@ Y_UNIT_TEST_SUITE(CdcStreamChangeCollector) { TShardedTableOptions IndexedTable() { return TShardedTableOptions() .Columns({ - {"pkey", "Uint32", true, false}, - {"ikey", "Uint32", false, false}, + {"pkey", "Uint32", true, false}, + {"ikey", "Uint32", false, false}, }) .Indexes({ {"by_ikey", {"ikey"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, diff --git a/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp b/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp index 31b2c236035..5e312f673c4 100644 --- a/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp +++ b/ydb/core/tx/datashard/datashard_ut_change_exchange.cpp @@ -55,8 +55,8 @@ Y_UNIT_TEST_SUITE(AsyncIndexChangeExchange) { TShardedTableOptions TableWoIndexes() { return TShardedTableOptions() .Columns({ - {"pkey", "Uint32", true, false}, - {"ikey", "Uint32", false, false}, + {"pkey", "Uint32", true, false}, + {"ikey", "Uint32", false, false}, }); } @@ -75,8 +75,8 @@ Y_UNIT_TEST_SUITE(AsyncIndexChangeExchange) { TShardedTableOptions TableWithIndex(const TShardedTableOptions::TIndex& index) { return TShardedTableOptions() .Columns({ - {"pkey", "Uint32", true, false}, - {"ikey", "Uint32", false, false}, + {"pkey", "Uint32", true, false}, + {"ikey", "Uint32", false, false}, }) .Indexes({ index @@ -151,9 +151,9 @@ Y_UNIT_TEST_SUITE(AsyncIndexChangeExchange) { Y_UNIT_TEST(SenderShouldShakeHandsTwice) { SenderShouldShakeHands("/Root/Table", 2, TShardedTableOptions() .Columns({ - {"pkey", "Uint32", true, false}, - {"i1key", "Uint32", false, false}, - {"i2key", "Uint32", false, false}, + {"pkey", "Uint32", true, false}, + {"i1key", "Uint32", false, false}, + {"i2key", "Uint32", false, false}, }) .Indexes({ {"by_i1key", {"i1key"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync}, diff --git a/ydb/core/tx/datashard/datashard_ut_common.cpp b/ydb/core/tx/datashard/datashard_ut_common.cpp index cd2efc2073e..d754f7ad0e2 100644 --- a/ydb/core/tx/datashard/datashard_ut_common.cpp +++ b/ydb/core/tx/datashard/datashard_ut_common.cpp @@ -1120,7 +1120,7 @@ void CreateShardedTable(Tests::TServer::TPtr server, auto col = desc->AddColumns(); col->SetName(column.Name); col->SetType(column.Type); - col->SetNotNull(column.NotNull); + col->SetNotNull(column.NotNull); if (column.IsKey) { desc->AddKeyColumnNames(column.Name); } diff --git a/ydb/core/tx/datashard/datashard_ut_common.h b/ydb/core/tx/datashard/datashard_ut_common.h index 6870c535dfc..4b5d7fe4ea0 100644 --- a/ydb/core/tx/datashard/datashard_ut_common.h +++ b/ydb/core/tx/datashard/datashard_ut_common.h @@ -371,16 +371,16 @@ struct TShardedTableOptions { using TSelf = TShardedTableOptions; struct TColumn { - TColumn(const TString& name, const TString& type, bool isKey, bool notNull) - : Name(name) - , Type(type) - , IsKey(isKey) - , NotNull(notNull) {} - + TColumn(const TString& name, const TString& type, bool isKey, bool notNull) + : Name(name) + , Type(type) + , IsKey(isKey) + , NotNull(notNull) {} + TString Name; TString Type; bool IsKey; - bool NotNull; + bool NotNull; }; struct TIndex { @@ -412,7 +412,7 @@ struct TShardedTableOptions { TABLE_OPTION(bool, EnableOutOfOrder, true); TABLE_OPTION(const NLocalDb::TCompactionPolicy*, Policy, nullptr); TABLE_OPTION(EShadowDataMode, ShadowData, EShadowDataMode::Default); - TABLE_OPTION(TVector<TColumn>, Columns, (TVector<TColumn>{{"key", "Uint32", true, false}, {"value", "Uint32", false, false}})); + TABLE_OPTION(TVector<TColumn>, Columns, (TVector<TColumn>{{"key", "Uint32", true, false}, {"value", "Uint32", false, false}})); TABLE_OPTION(TVector<TIndex>, Indexes, {}); TABLE_OPTION(ui64, Followers, 0); TABLE_OPTION(bool, FollowerPromotion, false); diff --git a/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp b/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp index 62d5ce61deb..87a51ef80ab 100644 --- a/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp +++ b/ydb/core/tx/datashard/datashard_ut_erase_rows.cpp @@ -48,8 +48,8 @@ void CreateTable(TServer::TPtr server, const TActorId& sender, const TString& ro auto opts = TShardedTableOptions() .EnableOutOfOrder(false) .Columns({ - {"key", "Uint32", true, false}, - {"value", ttlColType, false, false} + {"key", "Uint32", true, false}, + {"value", ttlColType, false, false} }); CreateShardedTable(server, sender, root, name, opts); } @@ -59,10 +59,10 @@ void CreateIndexedTable(TServer::TPtr server, const TActorId& sender, const TStr auto opts = TShardedTableOptions() .EnableOutOfOrder(false) .Columns({ - {"key", "Uint32", true, false}, - {"skey", "Uint32", false, false}, - {"tkey", "Uint32", false, false}, - {"value", ttlColType, false, false} + {"key", "Uint32", true, false}, + {"skey", "Uint32", false, false}, + {"tkey", "Uint32", false, false}, + {"value", ttlColType, false, false} }) .Indexes({ {"by_skey", {"skey"}}, diff --git a/ydb/core/tx/datashard/datashard_ut_locks.cpp b/ydb/core/tx/datashard/datashard_ut_locks.cpp index ed37de8eb75..1d25c39f99d 100644 --- a/ydb/core/tx/datashard/datashard_ut_locks.cpp +++ b/ydb/core/tx/datashard/datashard_ut_locks.cpp @@ -61,12 +61,12 @@ namespace NTest { for (ui32 tableId = 0; tableId < NumSysTables(); ++tableId) { delta.AddTable(TString("Table") + ('A'+tableId), tableId); - delta.AddColumn(tableId, "key", 0, NScheme::NTypeIds::Uint32, false); + delta.AddColumn(tableId, "key", 0, NScheme::NTypeIds::Uint32, false); delta.AddColumnToKey(tableId, 0); } delta.AddTable("user____Table", EUserTableId); - delta.AddColumn(EUserTableId, "key", 0, NScheme::NTypeIds::Uint32, false); + delta.AddColumn(EUserTableId, "key", 0, NScheme::NTypeIds::Uint32, false); delta.AddColumnToKey(EUserTableId, 0); diff --git a/ydb/core/tx/datashard/datashard_ut_order.cpp b/ydb/core/tx/datashard/datashard_ut_order.cpp index b7dfb171102..32472dce59e 100644 --- a/ydb/core/tx/datashard/datashard_ut_order.cpp +++ b/ydb/core/tx/datashard/datashard_ut_order.cpp @@ -3672,9 +3672,9 @@ void TestLateKqpQueryAfterColumnDrop(bool dataQuery, const TString& query, bool CreateShardedTable(server, sender, "/Root", "table-1", TShardedTableOptions() .Columns({ - {"key", "Uint32", true, false}, - {"value1", "Uint32", false, false}, - {"value2", "Uint32", false, false}})); + {"key", "Uint32", true, false}, + {"value1", "Uint32", false, false}, + {"value2", "Uint32", false, false}})); ExecSQL(server, sender, "UPSERT INTO `/Root/table-1` (key, value1, value2) VALUES (1, 1, 10), (2, 2, 20);"); diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index d910aea1789..8e778a4a1d8 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -198,7 +198,7 @@ namespace { entry.ListNodeEntry.Drop(); entry.DomainDescription.Drop(); entry.Columns.clear(); - entry.NotNullColumns.clear(); + entry.NotNullColumns.clear(); entry.Indexes.clear(); entry.CdcStreams.clear(); entry.RTMRVolumeInfo.Drop(); @@ -692,7 +692,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { Columns.clear(); KeyColumnTypes.clear(); - NotNullColumns.clear(); + NotNullColumns.clear(); Indexes.clear(); CdcStreams.clear(); Partitioning.clear(); @@ -719,10 +719,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { column.Id = columnDesc.GetId(); column.Name = columnDesc.GetName(); column.PType = columnDesc.GetTypeId(); - - if (columnDesc.GetNotNull()) { - NotNullColumns.insert(columnDesc.GetName()); - } + + if (columnDesc.GetNotNull()) { + NotNullColumns.insert(columnDesc.GetName()); + } } KeyColumnTypes.resize(tableDesc.KeyColumnIdsSize()); @@ -1667,7 +1667,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { // specific (it's safe to fill them all) entry.Self = Self; entry.Columns = Columns; - entry.NotNullColumns = NotNullColumns; + entry.NotNullColumns = NotNullColumns; entry.Indexes = Indexes; entry.CdcStreams = CdcStreams; entry.DomainDescription = DomainDescription; @@ -1911,7 +1911,7 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { // table specific THashMap<ui32, TSysTables::TTableColumnInfo> Columns; TVector<NScheme::TTypeId> KeyColumnTypes; - THashSet<TString> NotNullColumns; + THashSet<TString> NotNullColumns; TVector<NKikimrSchemeOp::TIndexDescription> Indexes; TVector<NKikimrSchemeOp::TCdcStreamDescription> CdcStreams; TVector<TKeyDesc::TPartitionInfo> Partitioning; diff --git a/ydb/core/tx/scheme_cache/scheme_cache.h b/ydb/core/tx/scheme_cache/scheme_cache.h index 88e3b109620..26a07897a11 100644 --- a/ydb/core/tx/scheme_cache/scheme_cache.h +++ b/ydb/core/tx/scheme_cache/scheme_cache.h @@ -230,7 +230,7 @@ struct TSchemeCacheNavigate { // table specific THashMap<TString, TString> Attributes; THashMap<ui32, TSysTables::TTableColumnInfo> Columns; - THashSet<TString> NotNullColumns; + THashSet<TString> NotNullColumns; TVector<NKikimrSchemeOp::TIndexDescription> Indexes; TVector<NKikimrSchemeOp::TCdcStreamDescription> CdcStreams; diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index a65a317fe7e..650506549e0 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -374,7 +374,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return true; } - typedef std::tuple<TPathId, ui32, TString, NScheme::TTypeId, ui32, ui64, ui64, ui32, ETableColumnDefaultKind, TString, bool> TColumnRec; + typedef std::tuple<TPathId, ui32, TString, NScheme::TTypeId, ui32, ui64, ui64, ui32, ETableColumnDefaultKind, TString, bool> TColumnRec; typedef TDeque<TColumnRec> TColumnRows; bool LoadColumns(NIceDb::TNiceDb& db, TColumnRows& columnRows) const { @@ -396,11 +396,11 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui32 family = rowSet.GetValueOrDefault<Schema::Columns::Family>(0); auto defaultKind = rowSet.GetValue<Schema::Columns::DefaultKind>(); auto defaultValue = rowSet.GetValue<Schema::Columns::DefaultValue>(); - auto notNull = rowSet.GetValueOrDefault<Schema::Columns::NotNull>(false); + auto notNull = rowSet.GetValueOrDefault<Schema::Columns::NotNull>(false); columnRows.emplace_back(pathId, colId, colName, typeId, keyOrder, createVersion, deleteVersion, - family, defaultKind, defaultValue, notNull); + family, defaultKind, defaultValue, notNull); if (!rowSet.Next()) { return false; @@ -428,11 +428,11 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui32 family = rowSet.GetValueOrDefault<Schema::MigratedColumns::Family>(0); auto defaultKind = rowSet.GetValue<Schema::MigratedColumns::DefaultKind>(); auto defaultValue = rowSet.GetValue<Schema::MigratedColumns::DefaultValue>(); - auto notNull = rowSet.GetValueOrDefault<Schema::MigratedColumns::NotNull>(false); + auto notNull = rowSet.GetValueOrDefault<Schema::MigratedColumns::NotNull>(false); columnRows.emplace_back(pathId, colId, colName, typeId, keyOrder, createVersion, deleteVersion, - family, defaultKind, defaultValue, notNull); + family, defaultKind, defaultValue, notNull); if (!rowSet.Next()) { return false; @@ -462,11 +462,11 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui32 family = rowSet.GetValueOrDefault<Schema::ColumnAlters::Family>(0); auto defaultKind = rowSet.GetValue<Schema::ColumnAlters::DefaultKind>(); auto defaultValue = rowSet.GetValue<Schema::ColumnAlters::DefaultValue>(); - auto notNull = rowSet.GetValueOrDefault<Schema::ColumnAlters::NotNull>(false); + auto notNull = rowSet.GetValueOrDefault<Schema::ColumnAlters::NotNull>(false); columnRows.emplace_back(pathId, colId, colName, typeId, keyOrder, createVersion, deleteVersion, - family, defaultKind, defaultValue, notNull); + family, defaultKind, defaultValue, notNull); if (!rowSet.Next()) { return false; @@ -494,11 +494,11 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui32 family = rowSet.GetValueOrDefault<Schema::MigratedColumnAlters::Family>(0); auto defaultKind = rowSet.GetValue<Schema::MigratedColumnAlters::DefaultKind>(); auto defaultValue = rowSet.GetValue<Schema::MigratedColumnAlters::DefaultValue>(); - auto notNull = rowSet.GetValueOrDefault<Schema::MigratedColumnAlters::NotNull>(false); + auto notNull = rowSet.GetValueOrDefault<Schema::MigratedColumnAlters::NotNull>(false); columnRows.emplace_back(pathId, colId, colName, typeId, keyOrder, createVersion, deleteVersion, - family, defaultKind, defaultValue, notNull); + family, defaultKind, defaultValue, notNull); if (!rowSet.Next()) { return false; @@ -1831,7 +1831,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui32 family = std::get<7>(rec); auto defaultKind = std::get<8>(rec); auto defaultValue = std::get<9>(rec); - auto notNull = std::get<10>(rec); + auto notNull = std::get<10>(rec); Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); Y_VERIFY_S(Self->PathsById.at(pathId)->IsTable(), "Path is not a table, pathId: " << pathId); @@ -1849,7 +1849,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { colInfo.Family = family; colInfo.DefaultKind = defaultKind; colInfo.DefaultValue = defaultValue; - colInfo.NotNull = notNull; + colInfo.NotNull = notNull; tableInfo->Columns[colId] = colInfo; @@ -1883,7 +1883,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui32 family = std::get<7>(rec); auto defaultKind = std::get<8>(rec); auto defaultValue = std::get<9>(rec); - auto notNull = std::get<10>(rec); + auto notNull = std::get<10>(rec); Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); Y_VERIFY_S(Self->PathsById.at(pathId)->IsTable(), "Path is not a table, pathId: " << pathId); @@ -1903,7 +1903,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { colInfo.Family = family; colInfo.DefaultKind = defaultKind; colInfo.DefaultValue = defaultValue; - colInfo.NotNull = notNull; + colInfo.NotNull = notNull; tableInfo->AlterData->Columns[colId] = colInfo; } } diff --git a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp index ea857ef20ba..74399680cda 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init_root.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init_root.cpp @@ -619,8 +619,8 @@ struct TSchemeShard::TTxMigrate : public TSchemeShard::TRwTxBase { NIceDb::TUpdate<Schema::MigratedColumns::DeleteVersion>(colDescr.GetDeleteVersion()), NIceDb::TUpdate<Schema::MigratedColumns::Family>(colDescr.GetFamily()), NIceDb::TUpdate<Schema::MigratedColumns::DefaultKind>(ETableColumnDefaultKind(colDescr.GetDefaultKind())), - NIceDb::TUpdate<Schema::MigratedColumns::DefaultValue>(colDescr.GetDefaultValue()), - NIceDb::TUpdate<Schema::MigratedColumns::NotNull>(colDescr.GetNotNull())); + NIceDb::TUpdate<Schema::MigratedColumns::DefaultValue>(colDescr.GetDefaultValue()), + NIceDb::TUpdate<Schema::MigratedColumns::NotNull>(colDescr.GetNotNull())); } for (const NKikimrScheme::TMigratePartition& partDescr: tableDescr.GetPartitions()) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp index e958e3011f9..5beb0a88488 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp @@ -217,10 +217,10 @@ TVector<ISubOperationBase::TPtr> CreateIndexedTable(TOperationId nextId, const T for (auto& column : baseTableDescription.GetColumns()) { if (column.GetNotNull() && !AppData()->FeatureFlags.GetEnableNotNullColumns()) { - TString msg = TStringBuilder() << "It is not allowed to create not null column"; - return {CreateReject(nextId, NKikimrScheme::EStatus::StatusPreconditionFailed, msg)}; - } - + TString msg = TStringBuilder() << "It is not allowed to create not null column"; + return {CreateReject(nextId, NKikimrScheme::EStatus::StatusPreconditionFailed, msg)}; + } + if (column.HasDefaultFromSequence()) { const TString& sequenceName = column.GetDefaultFromSequence(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp index 2b24f955651..ad9c6808fca 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp @@ -224,7 +224,7 @@ public: colDescr->SetCreateVersion(column.CreateVersion); colDescr->SetDeleteVersion(column.DeleteVersion); colDescr->SetFamily(column.Family); - colDescr->SetNotNull(column.NotNull); + colDescr->SetNotNull(column.NotNull); if (column.DefaultKind != ETableColumnDefaultKind::None) { colDescr->SetDefaultKind(ui32(column.DefaultKind)); colDescr->SetDefaultValue(column.DefaultValue); diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 5db0827508c..0715543a229 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -2208,8 +2208,8 @@ void TSchemeShard::PersistTableAltered(NIceDb::TNiceDb& db, const TPathId pathId NIceDb::TUpdate<Schema::Columns::DeleteVersion>(cinfo.DeleteVersion), NIceDb::TUpdate<Schema::Columns::Family>(cinfo.Family), NIceDb::TUpdate<Schema::Columns::DefaultKind>(cinfo.DefaultKind), - NIceDb::TUpdate<Schema::Columns::DefaultValue>(cinfo.DefaultValue), - NIceDb::TUpdate<Schema::Columns::NotNull>(cinfo.NotNull)); + NIceDb::TUpdate<Schema::Columns::DefaultValue>(cinfo.DefaultValue), + NIceDb::TUpdate<Schema::Columns::NotNull>(cinfo.NotNull)); db.Table<Schema::ColumnAlters>().Key(pathId.LocalPathId, colId).Delete(); } else { @@ -2221,8 +2221,8 @@ void TSchemeShard::PersistTableAltered(NIceDb::TNiceDb& db, const TPathId pathId NIceDb::TUpdate<Schema::MigratedColumns::DeleteVersion>(cinfo.DeleteVersion), NIceDb::TUpdate<Schema::MigratedColumns::Family>(cinfo.Family), NIceDb::TUpdate<Schema::MigratedColumns::DefaultKind>(cinfo.DefaultKind), - NIceDb::TUpdate<Schema::MigratedColumns::DefaultValue>(cinfo.DefaultValue), - NIceDb::TUpdate<Schema::MigratedColumns::NotNull>(cinfo.NotNull)); + NIceDb::TUpdate<Schema::MigratedColumns::DefaultValue>(cinfo.DefaultValue), + NIceDb::TUpdate<Schema::MigratedColumns::NotNull>(cinfo.NotNull)); } db.Table<Schema::MigratedColumnAlters>().Key(pathId.OwnerId, pathId.LocalPathId, colId).Delete(); } @@ -2259,8 +2259,8 @@ void TSchemeShard::PersistAddAlterTable(NIceDb::TNiceDb& db, TPathId pathId, con NIceDb::TUpdate<Schema::ColumnAlters::DeleteVersion>(cinfo.DeleteVersion), NIceDb::TUpdate<Schema::ColumnAlters::Family>(cinfo.Family), NIceDb::TUpdate<Schema::ColumnAlters::DefaultKind>(cinfo.DefaultKind), - NIceDb::TUpdate<Schema::ColumnAlters::DefaultValue>(cinfo.DefaultValue), - NIceDb::TUpdate<Schema::ColumnAlters::NotNull>(cinfo.NotNull)); + NIceDb::TUpdate<Schema::ColumnAlters::DefaultValue>(cinfo.DefaultValue), + NIceDb::TUpdate<Schema::ColumnAlters::NotNull>(cinfo.NotNull)); } else { db.Table<Schema::MigratedColumnAlters>().Key(pathId.OwnerId, pathId.LocalPathId, colId).Update( NIceDb::TUpdate<Schema::MigratedColumnAlters::ColName>(cinfo.Name), @@ -2270,8 +2270,8 @@ void TSchemeShard::PersistAddAlterTable(NIceDb::TNiceDb& db, TPathId pathId, con NIceDb::TUpdate<Schema::MigratedColumnAlters::DeleteVersion>(cinfo.DeleteVersion), NIceDb::TUpdate<Schema::MigratedColumnAlters::Family>(cinfo.Family), NIceDb::TUpdate<Schema::MigratedColumnAlters::DefaultKind>(cinfo.DefaultKind), - NIceDb::TUpdate<Schema::MigratedColumnAlters::DefaultValue>(cinfo.DefaultValue), - NIceDb::TUpdate<Schema::MigratedColumnAlters::NotNull>(cinfo.NotNull)); + NIceDb::TUpdate<Schema::MigratedColumnAlters::DefaultValue>(cinfo.DefaultValue), + NIceDb::TUpdate<Schema::MigratedColumnAlters::NotNull>(cinfo.NotNull)); } } } diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index a01ca8ce4d6..fcb564b6065 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -148,7 +148,7 @@ TTableInfo::TAlterDataPtr TTableInfo::CreateAlterData( TTableInfo::TColumn& column = alterData->Columns[colId]; column = TTableInfo::TColumn(colName, colId, type->GetTypeId()); column.Family = columnFamily ? columnFamily->GetId() : 0; - column.NotNull = col.GetNotNull(); + column.NotNull = col.GetNotNull(); if (source) column.CreateVersion = alterData->AlterVersion; if (col.HasDefaultFromSequence()) { diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 5cd330d8142..2e5710c3d3e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -176,7 +176,7 @@ struct TTableInfo : public TSimpleRefCount<TTableInfo> { ui64 DeleteVersion; ETableColumnDefaultKind DefaultKind = ETableColumnDefaultKind::None; TString DefaultValue; - bool NotNull = false; + bool NotNull = false; TColumn(const TString& name, ui32 id, NScheme::TTypeId type) : NTable::TScheme::TColumn(name, id, type) diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index 717ff93024f..3789d3af1b8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -911,7 +911,7 @@ void TSchemeShard::DescribeTable(const TTableInfo::TPtr tableInfo, const NScheme colDescr->SetType(typeRegistry->GetTypeName(cinfo.PType)); colDescr->SetTypeId(cinfo.PType); colDescr->SetId(cinfo.Id); - colDescr->SetNotNull(cinfo.NotNull); + colDescr->SetNotNull(cinfo.NotNull); if (cinfo.Family != 0) { colDescr->SetFamily(cinfo.Family); diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index a69c8a0cb6d..f293136fe85 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -173,12 +173,12 @@ struct Schema : NIceDb::Schema { struct Family : Column<8, NScheme::NTypeIds::Uint32> {}; struct DefaultKind : Column<9, NScheme::NTypeIds::Uint32> { using Type = ETableColumnDefaultKind; static constexpr Type Default = Type::None; }; struct DefaultValue : Column<10, NScheme::NTypeIds::String> {}; - struct NotNull: Column<11, NScheme::NTypeIds::Bool> {}; + struct NotNull: Column<11, NScheme::NTypeIds::Bool> {}; using TKey = TableKey<TabId, ColId>; using TColumns = TableColumns<TabId, ColId, ColName, ColType, ColKeyOrder, CreateVersion, DeleteVersion, - Family, DefaultKind, DefaultValue, NotNull>; + Family, DefaultKind, DefaultValue, NotNull>; }; struct MigratedColumns : Table<55> { @@ -194,12 +194,12 @@ struct Schema : NIceDb::Schema { struct Family : Column<9, NScheme::NTypeIds::Uint32> {}; struct DefaultKind : Column<10, NScheme::NTypeIds::Uint32> { using Type = ETableColumnDefaultKind; static constexpr Type Default = Type::None; }; struct DefaultValue : Column<11, NScheme::NTypeIds::String> {}; - struct NotNull: Column<12, NScheme::NTypeIds::Bool> {}; + struct NotNull: Column<12, NScheme::NTypeIds::Bool> {}; using TKey = TableKey<OwnerPathId, LocalPathId, ColId>; using TColumns = TableColumns<OwnerPathId, LocalPathId, ColId, ColName, ColType, ColKeyOrder, CreateVersion, DeleteVersion, - Family, DefaultKind, DefaultValue, NotNull>; + Family, DefaultKind, DefaultValue, NotNull>; }; struct ColumnAlters : Table<13> { @@ -213,11 +213,11 @@ struct Schema : NIceDb::Schema { struct Family : Column<8, NScheme::NTypeIds::Uint32> {}; struct DefaultKind : Column<9, NScheme::NTypeIds::Uint32> { using Type = ETableColumnDefaultKind; static constexpr Type Default = Type::None; }; struct DefaultValue : Column<10, NScheme::NTypeIds::String> {}; - struct NotNull: Column<11, NScheme::NTypeIds::Bool> {}; + struct NotNull: Column<11, NScheme::NTypeIds::Bool> {}; using TKey = TableKey<TabId, ColId>; using TColumns = TableColumns<TabId, ColId, ColName, ColType, ColKeyOrder, CreateVersion, DeleteVersion, - Family, DefaultKind, DefaultValue, NotNull>; + Family, DefaultKind, DefaultValue, NotNull>; }; struct MigratedColumnAlters : Table<63> { @@ -233,11 +233,11 @@ struct Schema : NIceDb::Schema { struct Family : Column<9, NScheme::NTypeIds::Uint32> {}; struct DefaultKind : Column<10, NScheme::NTypeIds::Uint32> { using Type = ETableColumnDefaultKind; static constexpr Type Default = Type::None; }; struct DefaultValue : Column<11, NScheme::NTypeIds::String> {}; - struct NotNull: Column<12, NScheme::NTypeIds::Bool> {}; + struct NotNull: Column<12, NScheme::NTypeIds::Bool> {}; using TKey = TableKey<OwnerPathId, LocalPathId, ColId>; using TColumns = TableColumns<OwnerPathId, LocalPathId, ColId, ColName, ColType, ColKeyOrder, CreateVersion, DeleteVersion, - Family, DefaultKind, DefaultValue, NotNull>; + Family, DefaultKind, DefaultValue, NotNull>; }; struct Shards : Table<7> { diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp index b164b6c0ff7..9cfd8037b7e 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp @@ -500,7 +500,7 @@ NSchemeShardUT_Private::TTestEnv::TTestEnv(TTestActorRuntime& runtime, const TTe app.SetEnableTtlOnAsyncIndexedTables(opts.EnableTtlOnAsyncIndexedTables_); app.SetAllowUpdateChannelsBindingOfSolomonPartitions(opts.AllowUpdateChannelsBindingOfSolomonPartitions_); app.SetEnableAsyncIndexes(opts.EnableAsyncIndexes_); - app.SetEnableNotNullColumns(opts.EnableNotNullColumns_); + app.SetEnableNotNullColumns(opts.EnableNotNullColumns_); app.SetEnableSchemeTransactionsAtSchemeShard(opts.EnableSchemeTransactionsAtSchemeShard_); app.SetEnableOlapSchemaOperations(opts.EnableOlapSchemaOperations_); app.SetEnableProtoSourceIdInfo(opts.EnableProtoSourceIdInfo_); diff --git a/ydb/core/tx/schemeshard/ut_reboots.cpp b/ydb/core/tx/schemeshard/ut_reboots.cpp index f39ab5c60c2..dce2c127122 100644 --- a/ydb/core/tx/schemeshard/ut_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_reboots.cpp @@ -599,61 +599,61 @@ Y_UNIT_TEST_SUITE(TConsistentOpsWithReboots) { } }); } - - Y_UNIT_TEST(CreateNotNullColumnTableWithReboots) { - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - TestMkDir(runtime, ++t.TxId, "/MyRoot", "DirB"); - TestCreateTable(runtime, ++t.TxId, "/MyRoot/DirB", R"( - Name: "TestNotNullTable" - Columns { Name: "key" Type: "Uint64" NotNull: true} - Columns { Name: "value" Type: "Utf8" NotNull: true} - KeyColumnNames: ["key"] - )"); - - t.TestEnv->TestWaitNotification(runtime, {t.TxId-1, t.TxId}); - - { - TInactiveZone inactive(activeZone); - TestDescribeResult(DescribePath(runtime, "/MyRoot/DirB"), - { NLs::Finished, - NLs::ChildrenCount(1) }); - TestDescribeResult(DescribePath(runtime, "/MyRoot/DirB/TestNotNullTable"), - { NLs::Finished, - NLs::PathExist }); - } - }); - } - - Y_UNIT_TEST(DropNotNullColumnTableWithReboots) { - TTestWithReboots t; - t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { - { - TInactiveZone inactive(activeZone); - TestMkDir(runtime, ++t.TxId, "/MyRoot", "DirB"); - TestCreateTable(runtime, ++t.TxId, "/MyRoot/DirB", R"( - Name: "TestNotNullTable" - Columns { Name: "key" Type: "Uint64" NotNull: true} - Columns { Name: "value" Type: "Utf8" NotNull: true} - KeyColumnNames: ["key"] - )"); - t.TestEnv->TestWaitNotification(runtime, {t.TxId-1, t.TxId}); - } - - TestDropTable(runtime, ++t.TxId, "/MyRoot/DirB", "TestNotNullTable"); - t.TestEnv->TestWaitNotification(runtime, t.TxId); + + Y_UNIT_TEST(CreateNotNullColumnTableWithReboots) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + TestMkDir(runtime, ++t.TxId, "/MyRoot", "DirB"); + TestCreateTable(runtime, ++t.TxId, "/MyRoot/DirB", R"( + Name: "TestNotNullTable" + Columns { Name: "key" Type: "Uint64" NotNull: true} + Columns { Name: "value" Type: "Utf8" NotNull: true} + KeyColumnNames: ["key"] + )"); + + t.TestEnv->TestWaitNotification(runtime, {t.TxId-1, t.TxId}); + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/DirB"), + { NLs::Finished, + NLs::ChildrenCount(1) }); + TestDescribeResult(DescribePath(runtime, "/MyRoot/DirB/TestNotNullTable"), + { NLs::Finished, + NLs::PathExist }); + } + }); + } + + Y_UNIT_TEST(DropNotNullColumnTableWithReboots) { + TTestWithReboots t; + t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { + { + TInactiveZone inactive(activeZone); + TestMkDir(runtime, ++t.TxId, "/MyRoot", "DirB"); + TestCreateTable(runtime, ++t.TxId, "/MyRoot/DirB", R"( + Name: "TestNotNullTable" + Columns { Name: "key" Type: "Uint64" NotNull: true} + Columns { Name: "value" Type: "Utf8" NotNull: true} + KeyColumnNames: ["key"] + )"); + t.TestEnv->TestWaitNotification(runtime, {t.TxId-1, t.TxId}); + } + + TestDropTable(runtime, ++t.TxId, "/MyRoot/DirB", "TestNotNullTable"); + t.TestEnv->TestWaitNotification(runtime, t.TxId); t.TestEnv->TestWaitTabletDeletion(runtime, {TTestTxConfig::FakeHiveTablets, TTestTxConfig::FakeHiveTablets + 10}); - - { - TInactiveZone inactive(activeZone); - TestDescribeResult(DescribePath(runtime, "/MyRoot/DirB"), - { NLs::Finished, - NLs::ChildrenCount(0) }); - TestDescribeResult(DescribePath(runtime, "/MyRoot/DirB/TestNotNullTable"), - { NLs::PathNotExist }); - } - }); - } + + { + TInactiveZone inactive(activeZone); + TestDescribeResult(DescribePath(runtime, "/MyRoot/DirB"), + { NLs::Finished, + NLs::ChildrenCount(0) }); + TestDescribeResult(DescribePath(runtime, "/MyRoot/DirB/TestNotNullTable"), + { NLs::PathNotExist }); + } + }); + } } Y_UNIT_TEST_SUITE(TSolomonReboots) { diff --git a/ydb/core/tx/tx_proxy/datareq.cpp b/ydb/core/tx/tx_proxy/datareq.cpp index 488821ed1aa..af48b42d60f 100644 --- a/ydb/core/tx/tx_proxy/datareq.cpp +++ b/ydb/core/tx/tx_proxy/datareq.cpp @@ -262,9 +262,9 @@ public: ETabletStatus TabletStatus = ETabletStatus::StatusUnknown; ui64 ReadSize = 0; ui64 ReplySize = 0; - ui64 ProgramSize = 0; + ui64 ProgramSize = 0; ui64 IncomingReadSetsSize = 0; - ui64 OutgoingReadSetsSize = 0; + ui64 OutgoingReadSetsSize = 0; bool StreamCleared = false; bool Restarting = false; size_t RestartCount = 0; @@ -914,11 +914,11 @@ void TDataReq::BuildTxStats(NKikimrQueryStats::TTxStats& stats) { tableStats.SetShardCount(tableStats.GetShardCount() + 1); } if (shard.second.Stats->PerShardStatsSize() == 1) { - auto shardStats = stats.AddPerShardStats(); - shardStats->CopyFrom(shard.second.Stats->GetPerShardStats(0)); - shardStats->SetOutgoingReadSetsCount(shard.second.OutgoingReadSetsSize); - shardStats->SetProgramSize(shard.second.ProgramSize); - shardStats->SetReplySize(shard.second.ReplySize); + auto shardStats = stats.AddPerShardStats(); + shardStats->CopyFrom(shard.second.Stats->GetPerShardStats(0)); + shardStats->SetOutgoingReadSetsCount(shard.second.OutgoingReadSetsSize); + shardStats->SetProgramSize(shard.second.ProgramSize); + shardStats->SetReplySize(shard.second.ReplySize); } } @@ -1023,7 +1023,7 @@ void TDataReq::ProcessFlatMKQLResolve(NSchemeCache::TSchemeCacheRequest *cacheRe TPerTablet &perTablet = PerTablet[shardData.ShardId]; Y_VERIFY(perTablet.TabletStatus == TPerTablet::ETabletStatus::StatusUnknown); perTablet.TabletStatus = TPerTablet::ETabletStatus::StatusWait; - perTablet.ProgramSize = transactionBuffer.size(); + perTablet.ProgramSize = transactionBuffer.size(); ++TabletsLeft; perTablet.AffectedFlags |= affectedType; @@ -1808,7 +1808,7 @@ void TDataReq::HandlePrepare(TEvDataShard::TEvProposeTransactionResult::TPtr &ev perTablet->MaxStep = record.GetMaxStep(); perTablet->ReadSize = record.GetReadSize(); perTablet->ReplySize = record.GetReplySize(); - perTablet->OutgoingReadSetsSize = record.OutgoingReadSetInfoSize(); + perTablet->OutgoingReadSetsSize = record.OutgoingReadSetInfoSize(); for (size_t i = 0; i < record.OutgoingReadSetInfoSize(); ++i) { auto& rs = record.GetOutgoingReadSetInfo(i); ui64 targetTabletId = rs.GetShardId(); diff --git a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h index 67d56bc37ea..0aaee7e7f59 100644 --- a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h +++ b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h @@ -133,10 +133,10 @@ protected: // Positions of key and value fields in the request proto struct struct TFieldDescription { ui32 ColId; - TString ColName; + TString ColName; ui32 PositionInStruct; NScheme::TTypeId Type; - bool NotNull = false; + bool NotNull = false; }; TVector<TString> KeyColumnNames; TVector<TFieldDescription> KeyColumnPositions; @@ -298,7 +298,7 @@ private: TVector<ui32> keyColumnIds; THashMap<TString, ui32> columnByName; THashSet<TString> keyColumnsLeft; - THashSet<TString> notNullColumnsLeft = entry.NotNullColumns; + THashSet<TString> notNullColumnsLeft = entry.NotNullColumns; SrcColumns.reserve(entry.Columns.size()); for (const auto& [_, colInfo] : entry.Columns) { @@ -371,18 +371,18 @@ private: return false; } - bool notNull = entry.NotNullColumns.contains(ci.Name); - if (notNull) { - notNullColumnsLeft.erase(ci.Name); - } - + bool notNull = entry.NotNullColumns.contains(ci.Name); + if (notNull) { + notNullColumnsLeft.erase(ci.Name); + } + NScheme::TTypeId typeId = (NScheme::TTypeId)ci.PType; if (ci.KeyOrder != -1) { - KeyColumnPositions[ci.KeyOrder] = TFieldDescription{ci.Id, ci.Name, (ui32)pos, typeId, notNull}; + KeyColumnPositions[ci.KeyOrder] = TFieldDescription{ci.Id, ci.Name, (ui32)pos, typeId, notNull}; keyColumnsLeft.erase(ci.Name); KeyColumnNames[ci.KeyOrder] = ci.Name; } else { - ValueColumnPositions.emplace_back(TFieldDescription{ci.Id, ci.Name, (ui32)pos, typeId, notNull}); + ValueColumnPositions.emplace_back(TFieldDescription{ci.Id, ci.Name, (ui32)pos, typeId, notNull}); ValueColumnNames.emplace_back(ci.Name); ValueColumnTypes.emplace_back(ci.PType); } @@ -410,11 +410,11 @@ private: return false; } - if (!notNullColumnsLeft.empty()) { - errorMessage = Sprintf("Missing not null columns: %s", JoinSeq(", ", notNullColumnsLeft).c_str()); - return false; - } - + if (!notNullColumnsLeft.empty()) { + errorMessage = Sprintf("Missing not null columns: %s", JoinSeq(", ", notNullColumnsLeft).c_str()); + return false; + } + return true; } diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp index 0a680bc2e2e..0d10bb8450b 100644 --- a/ydb/core/ydb_convert/table_description.cpp +++ b/ydb/core/ydb_convert/table_description.cpp @@ -103,7 +103,7 @@ void FillColumnDescription(Ydb::Table::CreateTableRequest& out, bool ExtractColumnTypeId(ui32& outTypeId, const Ydb::Type& inType, Ydb::StatusIds::StatusCode& status, TString& error) { ui32 typeId; - auto itemType = inType.has_optional_type() ? inType.optional_type().item() : inType; + auto itemType = inType.has_optional_type() ? inType.optional_type().item() : inType; switch (itemType.type_case()) { case Ydb::Type::kTypeId: typeId = (ui32)itemType.type_id(); @@ -154,12 +154,12 @@ bool FillColumnDescription(NKikimrSchemeOp::TTableDescription& out, cd->SetName(column.name()); if (!column.type().has_optional_type()) { if (!AppData()->FeatureFlags.GetEnableNotNullColumns()) { - status = Ydb::StatusIds::UNSUPPORTED; - error = "Not null columns feature is not supported yet"; - return false; - } - - cd->SetNotNull(true); + status = Ydb::StatusIds::UNSUPPORTED; + error = "Not null columns feature is not supported yet"; + return false; + } + + cd->SetNotNull(true); } ui32 typeId; diff --git a/ydb/library/yql/core/issue/protos/issue_id.proto b/ydb/library/yql/core/issue/protos/issue_id.proto index 3f0df12c5c0..a2ed91d6401 100644 --- a/ydb/library/yql/core/issue/protos/issue_id.proto +++ b/ydb/library/yql/core/issue/protos/issue_id.proto @@ -73,7 +73,7 @@ message TIssuesIds { KIKIMR_PRECONDITION_FAILED = 2029; KIKIMR_UNSUPPORTED = 2030; KIKIMR_BAD_COLUMN_TYPE = 2031; - KIKIMR_NO_COLUMN_DEFAULT_VALUE = 2032; + KIKIMR_NO_COLUMN_DEFAULT_VALUE = 2032; // kikimr warnings KIKIMR_READ_MODIFIED_TABLE = 2500; diff --git a/ydb/library/yql/core/issue/yql_issue.txt b/ydb/library/yql/core/issue/yql_issue.txt index 654519bfd36..897d5d6f9a5 100644 --- a/ydb/library/yql/core/issue/yql_issue.txt +++ b/ydb/library/yql/core/issue/yql_issue.txt @@ -289,10 +289,10 @@ ids { severity: S_ERROR } ids { - code: KIKIMR_NO_COLUMN_DEFAULT_VALUE - severity: S_ERROR -} -ids { + code: KIKIMR_NO_COLUMN_DEFAULT_VALUE + severity: S_ERROR +} +ids { code: YQL_PRAGMA_WARNING_MSG severity: S_WARNING } diff --git a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp index 477441cf0a9..ce22b125714 100644 --- a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp +++ b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp @@ -5758,19 +5758,19 @@ struct TPeepHoleRules { {"PgCall", &ExpandPgCall}, }; - static constexpr std::initializer_list<TPeepHoleOptimizerMap::value_type> SimplifyStageRulesInit = { + static constexpr std::initializer_list<TPeepHoleOptimizerMap::value_type> SimplifyStageRulesInit = { {"Map", &OptimizeMap<false, EnableNewOptimizers>}, {"OrderedMap", &OptimizeMap<true, EnableNewOptimizers>}, {"FlatMap", &ExpandFlatMap<false, EnableNewOptimizers>}, {"OrderedFlatMap", &ExpandFlatMap<true, EnableNewOptimizers>}, - {"ListIf", &ExpandContainerIf<false, true>}, - {"OptionalIf", &ExpandContainerIf<false, false>}, - {"FlatListIf", &ExpandContainerIf<true, true>}, - {"FlatOptionalIf", &ExpandContainerIf<true, false>}, - {"IfPresent", &OptimizeIfPresent<false, EnableNewOptimizers>} - }; - - static constexpr std::initializer_list<TPeepHoleOptimizerMap::value_type> FinalStageRulesInit = { + {"ListIf", &ExpandContainerIf<false, true>}, + {"OptionalIf", &ExpandContainerIf<false, false>}, + {"FlatListIf", &ExpandContainerIf<true, true>}, + {"FlatOptionalIf", &ExpandContainerIf<true, false>}, + {"IfPresent", &OptimizeIfPresent<false, EnableNewOptimizers>} + }; + + static constexpr std::initializer_list<TPeepHoleOptimizerMap::value_type> FinalStageRulesInit = { {"Take", &OptimizeTake<EnableNewOptimizers>}, {"Skip", &OptimizeSkip}, {"Likely", &LikelyExclude}, @@ -5830,7 +5830,7 @@ struct TPeepHoleRules { TPeepHoleRules() : CommonStageRules(CommonStageRulesInit) , FinalStageRules(FinalStageRulesInit) - , SimplifyStageRules(SimplifyStageRulesInit) + , SimplifyStageRules(SimplifyStageRulesInit) , FinalStageNonDetRules(FinalStageNonDetRulesInit) {} @@ -5840,22 +5840,22 @@ struct TPeepHoleRules { const TPeepHoleOptimizerMap CommonStageRules; const TPeepHoleOptimizerMap FinalStageRules; - const TPeepHoleOptimizerMap SimplifyStageRules; + const TPeepHoleOptimizerMap SimplifyStageRules; const TNonDeterministicOptimizerMap FinalStageNonDetRules; }; template <bool EnableNewOptimizers> THolder<IGraphTransformer> CreatePeepHoleCommonStageTransformer(TTypeAnnotationContext& types, - IGraphTransformer* typeAnnotator, const TPeepholeSettings& peepholeSettings) + IGraphTransformer* typeAnnotator, const TPeepholeSettings& peepholeSettings) { TTransformationPipeline pipeline(&types); - if (peepholeSettings.CommonConfig) { - peepholeSettings.CommonConfig->AfterCreate(&pipeline); + if (peepholeSettings.CommonConfig) { + peepholeSettings.CommonConfig->AfterCreate(&pipeline); } AddStandardTransformers(pipeline, typeAnnotator); - if (peepholeSettings.CommonConfig) { - peepholeSettings.CommonConfig->AfterTypeAnnotation(&pipeline); + if (peepholeSettings.CommonConfig) { + peepholeSettings.CommonConfig->AfterTypeAnnotation(&pipeline); } auto issueCode = TIssuesIds::CORE_EXEC; @@ -5869,8 +5869,8 @@ THolder<IGraphTransformer> CreatePeepHoleCommonStageTransformer(TTypeAnnotationC "PeepHoleCommon", issueCode); - if (peepholeSettings.CommonConfig) { - peepholeSettings.CommonConfig->AfterOptimize(&pipeline); + if (peepholeSettings.CommonConfig) { + peepholeSettings.CommonConfig->AfterOptimize(&pipeline); } return pipeline.BuildWithNoArgChecks(false); @@ -5878,43 +5878,43 @@ THolder<IGraphTransformer> CreatePeepHoleCommonStageTransformer(TTypeAnnotationC template <bool EnableNewOptimizers> THolder<IGraphTransformer> CreatePeepHoleFinalStageTransformer(TTypeAnnotationContext& types, - IGraphTransformer* typeAnnotator, - bool* hasNonDeterministicFunctions, - const TPeepholeSettings& peepholeSettings) + IGraphTransformer* typeAnnotator, + bool* hasNonDeterministicFunctions, + const TPeepholeSettings& peepholeSettings) { TTransformationPipeline pipeline(&types); - if (peepholeSettings.FinalConfig) { - peepholeSettings.FinalConfig->AfterCreate(&pipeline); + if (peepholeSettings.FinalConfig) { + peepholeSettings.FinalConfig->AfterCreate(&pipeline); } AddStandardTransformers(pipeline, typeAnnotator); - if (peepholeSettings.FinalConfig) { - peepholeSettings.FinalConfig->AfterTypeAnnotation(&pipeline); + if (peepholeSettings.FinalConfig) { + peepholeSettings.FinalConfig->AfterTypeAnnotation(&pipeline); } auto issueCode = TIssuesIds::CORE_EXEC; pipeline.Add( CreateFunctorTransformer( - [&types, hasNonDeterministicFunctions, withFinalRules = peepholeSettings.WithFinalStageRules, - withNonDeterministicRules = peepholeSettings.WithNonDeterministicRules](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - auto stageRules = TPeepHoleRules<EnableNewOptimizers>::Instance().SimplifyStageRules; - if (withFinalRules) { - const auto& finalRules = TPeepHoleRules<EnableNewOptimizers>::Instance().FinalStageRules; - stageRules.insert(finalRules.begin(), finalRules.end()); - } - - const auto& nonDetStageRules = withNonDeterministicRules ? - TPeepHoleRules<EnableNewOptimizers>::Instance().FinalStageNonDetRules : TNonDeterministicOptimizerMap{}; - - return PeepHoleFinalStage(input, output, ctx, types, hasNonDeterministicFunctions, stageRules, nonDetStageRules); + [&types, hasNonDeterministicFunctions, withFinalRules = peepholeSettings.WithFinalStageRules, + withNonDeterministicRules = peepholeSettings.WithNonDeterministicRules](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + auto stageRules = TPeepHoleRules<EnableNewOptimizers>::Instance().SimplifyStageRules; + if (withFinalRules) { + const auto& finalRules = TPeepHoleRules<EnableNewOptimizers>::Instance().FinalStageRules; + stageRules.insert(finalRules.begin(), finalRules.end()); + } + + const auto& nonDetStageRules = withNonDeterministicRules ? + TPeepHoleRules<EnableNewOptimizers>::Instance().FinalStageNonDetRules : TNonDeterministicOptimizerMap{}; + + return PeepHoleFinalStage(input, output, ctx, types, hasNonDeterministicFunctions, stageRules, nonDetStageRules); } ), "PeepHoleFinal", issueCode); - if (peepholeSettings.FinalConfig) { - peepholeSettings.FinalConfig->AfterOptimize(&pipeline); + if (peepholeSettings.FinalConfig) { + peepholeSettings.FinalConfig->AfterOptimize(&pipeline); } return pipeline.BuildWithNoArgChecks(false); @@ -5946,20 +5946,20 @@ IGraphTransformer::TStatus DoPeepHoleOptimizeNode(const TExprNode::TPtr& input, template <bool EnableNewOptimizers> IGraphTransformer::TStatus PeepHoleOptimizeNode(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator, - bool& hasNonDeterministicFunctions, const TPeepholeSettings& peepholeSettings) + bool& hasNonDeterministicFunctions, const TPeepholeSettings& peepholeSettings) { hasNonDeterministicFunctions = false; - const auto commonTransformer = CreatePeepHoleCommonStageTransformer<EnableNewOptimizers>(types, typeAnnotator, peepholeSettings); - const auto finalTransformer = CreatePeepHoleFinalStageTransformer<EnableNewOptimizers>(types, typeAnnotator, - &hasNonDeterministicFunctions, peepholeSettings); + const auto commonTransformer = CreatePeepHoleCommonStageTransformer<EnableNewOptimizers>(types, typeAnnotator, peepholeSettings); + const auto finalTransformer = CreatePeepHoleFinalStageTransformer<EnableNewOptimizers>(types, typeAnnotator, + &hasNonDeterministicFunctions, peepholeSettings); return DoPeepHoleOptimizeNode(input, output, ctx, *commonTransformer, *finalTransformer); } THolder<IGraphTransformer> MakePeepholeOptimization(TTypeAnnotationContextPtr typeAnnotationContext, const IPipelineConfigurator* config) { - TPeepholeSettings peepholeSettings; - peepholeSettings.CommonConfig = peepholeSettings.FinalConfig = config; - auto commonTransformer = CreatePeepHoleCommonStageTransformer<true>(*typeAnnotationContext, nullptr, peepholeSettings); - auto finalTransformer = CreatePeepHoleFinalStageTransformer<true>(*typeAnnotationContext, nullptr, nullptr, peepholeSettings); + TPeepholeSettings peepholeSettings; + peepholeSettings.CommonConfig = peepholeSettings.FinalConfig = config; + auto commonTransformer = CreatePeepHoleCommonStageTransformer<true>(*typeAnnotationContext, nullptr, peepholeSettings); + auto finalTransformer = CreatePeepHoleFinalStageTransformer<true>(*typeAnnotationContext, nullptr, nullptr, peepholeSettings); return CreateFunctorTransformer( [common = std::move(commonTransformer), final = std::move(finalTransformer)](TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) -> IGraphTransformer::TStatus { return DoPeepHoleOptimizeNode(input, output, ctx, *common, *final); @@ -5968,9 +5968,9 @@ THolder<IGraphTransformer> MakePeepholeOptimization(TTypeAnnotationContextPtr ty template IGraphTransformer::TStatus PeepHoleOptimizeNode<true>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator, - bool& hasNonDeterministicFunctions, const TPeepholeSettings& peepholeSettings); + bool& hasNonDeterministicFunctions, const TPeepholeSettings& peepholeSettings); template IGraphTransformer::TStatus PeepHoleOptimizeNode<false>(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator, - bool& hasNonDeterministicFunctions, const TPeepholeSettings& peepholeSettings); + bool& hasNonDeterministicFunctions, const TPeepholeSettings& peepholeSettings); } diff --git a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h index 6e95e1687f4..a357c5b3952 100644 --- a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h +++ b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h @@ -7,17 +7,17 @@ namespace NYql { struct IPipelineConfigurator; -struct TPeepholeSettings { - const IPipelineConfigurator* CommonConfig = nullptr; - const IPipelineConfigurator* FinalConfig = nullptr; - bool WithFinalStageRules = true; - bool WithNonDeterministicRules = true; -}; - +struct TPeepholeSettings { + const IPipelineConfigurator* CommonConfig = nullptr; + const IPipelineConfigurator* FinalConfig = nullptr; + bool WithFinalStageRules = true; + bool WithNonDeterministicRules = true; +}; + template <bool EnableNewOptimizers> IGraphTransformer::TStatus PeepHoleOptimizeNode(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx, TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator, - bool& hasNonDeterministicFunctions, const TPeepholeSettings& peepholeSettings = {}); + bool& hasNonDeterministicFunctions, const TPeepholeSettings& peepholeSettings = {}); THolder<IGraphTransformer> MakePeepholeOptimization(TTypeAnnotationContextPtr typeAnnotationContext, const IPipelineConfigurator* config = nullptr); diff --git a/ydb/library/yql/dq/connections.md b/ydb/library/yql/dq/connections.md index 14b468f55f8..09da3d1d255 100644 --- a/ydb/library/yql/dq/connections.md +++ b/ydb/library/yql/dq/connections.md @@ -74,7 +74,7 @@ DQ-граф состоит из стадий (_DqStage_), соединённых ## DqCnHashShuffle * _NodeType_: **HashShuffle** * _PlanNodeType_: **Connection** -* _KeyColumns_: список колонок, по которым идёт перемешивание +* _KeyColumns_: список колонок, по которым идёт перемешивание Рассылает результаты с тасков одной стадии (producer stage) на таски другой (consumer stage) по некоторому правилу для заданных колонок. Правило жётско описано в коде, но вот список колонок указывается в каждом коннекшине независимо. diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp index 23f12886447..10900352c3b 100644 --- a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp +++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp @@ -385,9 +385,9 @@ public: } auto buildTime = TInstant::Now() - startTime; - if (Stats) { - Stats->BuildCpuTime = buildTime; - } + if (Stats) { + Stats->BuildCpuTime = buildTime; + } LOG(TStringBuilder() << "Build task: " << TaskId << " takes " << buildTime.MicroSeconds() << " us"); } @@ -496,18 +496,18 @@ public: } LOG(TStringBuilder() << "Prepare task: " << TaskId << ", takes " << prepareTime.MicroSeconds() << " us"); - if (Stats) { - Stats->BuildCpuTime += prepareTime; - - for (auto&[channelId, inputChannel] : InputChannels) { - Stats->InputChannels.emplace(channelId, inputChannel->GetStats()); - } - for (auto&[inputIndex, source] : Sources) { - Stats->Sources.emplace(inputIndex, source->GetStats()); - } - for (auto&[channelId, outputChannel] : OutputChannels) { - Stats->OutputChannels.emplace(channelId, outputChannel->GetStats()); - } + if (Stats) { + Stats->BuildCpuTime += prepareTime; + + for (auto&[channelId, inputChannel] : InputChannels) { + Stats->InputChannels.emplace(channelId, inputChannel->GetStats()); + } + for (auto&[inputIndex, source] : Sources) { + Stats->Sources.emplace(inputIndex, source->GetStats()); + } + for (auto&[channelId, outputChannel] : OutputChannels) { + Stats->OutputChannels.emplace(channelId, outputChannel->GetStats()); + } } } @@ -517,9 +517,9 @@ public: RunComputeTime = TDuration::Zero(); auto runStatus = FetchAndDispatch(); - if (Stats) { - Stats->RunStatusTimeMetrics.SetCurrentStatus(runStatus, RunComputeTime); - } + if (Stats) { + Stats->RunStatusTimeMetrics.SetCurrentStatus(runStatus, RunComputeTime); + } if (Y_UNLIKELY(CollectProfileStats)) { Stats->ComputeCpuTimeByRun->Collect(RunComputeTime.MilliSeconds()); @@ -533,9 +533,9 @@ public: } if (runStatus == ERunStatus::Finished) { - if (Stats) { - Stats->FinishTs = TInstant::Now(); - } + if (Stats) { + Stats->FinishTs = TInstant::Now(); + } if (Y_UNLIKELY(CollectProfileStats)) { StopWaiting(Stats->FinishTs); } diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp index 62413587725..8759f442219 100644 --- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp +++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp @@ -889,11 +889,11 @@ private: optimizedInput->CopyConstraints(pull.Input().Ref()); TDqsPipelineConfigurator peepholeConfig; - TPeepholeSettings peepholeSettings; - peepholeSettings.CommonConfig = &peepholeConfig; + TPeepholeSettings peepholeSettings; + peepholeSettings.CommonConfig = &peepholeConfig; bool hasNonDeterministicFunctions; // TODO: do it per stage - auto status = PeepHoleOptimizeNode<true>(optimizedInput, optimizedInput, ctx, *State->TypeCtx, nullptr, hasNonDeterministicFunctions, peepholeSettings); + auto status = PeepHoleOptimizeNode<true>(optimizedInput, optimizedInput, ctx, *State->TypeCtx, nullptr, hasNonDeterministicFunctions, peepholeSettings); if (status != TStatus::Ok) { ctx.AddError(TIssue(ctx.GetPosition(optimizedInput->Pos()), TString("Peephole optimization failed for Dq stage"))); return SyncStatus(status); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp index 98596c5fa3a..4a25c372ce4 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.cpp @@ -26,7 +26,7 @@ void TCommandExecuteYqlScript::Config(TConfig& config) { .RequiredArgument("[String]").StoreResult(&CollectStatsMode); config.Opts->AddLongOption('s', "script", "Text of script to execute").RequiredArgument("[String]").StoreResult(&Script); config.Opts->AddLongOption('f', "file", "[Required] Script file").RequiredArgument("PATH").StoreResult(&ScriptFile); - config.Opts->AddLongOption("explain", "Explain query").Optional().StoreTrue(&Explain); + config.Opts->AddLongOption("explain", "Explain query").Optional().StoreTrue(&Explain); config.Opts->AddLongOption("show-response-metadata", ResponseHeadersHelp).Optional().StoreTrue(&ShowHeaders); AddParametersOption(config); @@ -82,37 +82,37 @@ int TCommandExecuteYqlScript::Run(TConfig& config) { } NScripting::TScriptingClient client(CreateDriver(config)); - if (Explain) { - NScripting::TExplainYqlRequestSettings settings; - settings.Mode(NScripting::ExplainYqlRequestMode::Plan); + if (Explain) { + NScripting::TExplainYqlRequestSettings settings; + settings.Mode(NScripting::ExplainYqlRequestMode::Plan); - auto result = client.ExplainYqlScript(Script, settings).GetValueSync(); - - ThrowOnError(result); - PrintExplainResult(result); + auto result = client.ExplainYqlScript(Script, settings).GetValueSync(); + + ThrowOnError(result); + PrintExplainResult(result); } else { - NScripting::TExecuteYqlRequestSettings settings; - settings.CollectQueryStats(ParseQueryStatsMode(CollectStatsMode, NTable::ECollectQueryStatsMode::None)); - - NScripting::TAsyncExecuteYqlResult asyncResult; - if (Parameters.size()) { - auto validateResult = ExplainQuery(config, Script, NScripting::ExplainYqlRequestMode::Validate); - asyncResult = client.ExecuteYqlScript( - Script, - BuildParams(validateResult.GetParameterTypes(), InputFormat), - FillSettings(settings) - ); - } else { - asyncResult = client.ExecuteYqlScript( - Script, - FillSettings(settings) - ); - } - auto result = asyncResult.GetValueSync(); - - ThrowOnError(result); - PrintResponseHeader(result); - PrintResponse(result); + NScripting::TExecuteYqlRequestSettings settings; + settings.CollectQueryStats(ParseQueryStatsMode(CollectStatsMode, NTable::ECollectQueryStatsMode::None)); + + NScripting::TAsyncExecuteYqlResult asyncResult; + if (Parameters.size()) { + auto validateResult = ExplainQuery(config, Script, NScripting::ExplainYqlRequestMode::Validate); + asyncResult = client.ExecuteYqlScript( + Script, + BuildParams(validateResult.GetParameterTypes(), InputFormat), + FillSettings(settings) + ); + } else { + asyncResult = client.ExecuteYqlScript( + Script, + FillSettings(settings) + ); + } + auto result = asyncResult.GetValueSync(); + + ThrowOnError(result); + PrintResponseHeader(result); + PrintResponse(result); } return EXIT_SUCCESS; @@ -136,10 +136,10 @@ void TCommandExecuteYqlScript::PrintResponse(NScripting::TExecuteYqlResult& resu } } -void TCommandExecuteYqlScript::PrintExplainResult(NScripting::TExplainYqlResult& result) { - TQueryPlanPrinter queryPlanPrinter(OutputFormat); - queryPlanPrinter.Print(result.GetPlan()); +void TCommandExecuteYqlScript::PrintExplainResult(NScripting::TExplainYqlResult& result) { + TQueryPlanPrinter queryPlanPrinter(OutputFormat); + queryPlanPrinter.Print(result.GetPlan()); +} + } - } -} diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.h b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.h index b85571473db..9911c6a7443 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scripting.h @@ -24,13 +24,13 @@ public: virtual void Parse(TConfig& config) override; virtual int Run(TConfig& config) override; void PrintResponse(NScripting::TExecuteYqlResult& result); - void PrintExplainResult(NScripting::TExplainYqlResult& result); + void PrintExplainResult(NScripting::TExplainYqlResult& result); private: TString CollectStatsMode; TString Script; TString ScriptFile; - bool Explain = false; + bool Explain = false; }; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp index 2f6d7fd51df..2f94b2fc033 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.cpp @@ -559,18 +559,18 @@ void TCommandExplain::Config(TConfig& config) { config.Opts->AddLongOption("analyze", "Run query and collect execution statistics") .NoArgument().SetFlag(&Analyze); - AddFormats(config, { - EOutputFormat::Pretty, - EOutputFormat::JsonUnicode, - EOutputFormat::JsonBase64 - }); - + AddFormats(config, { + EOutputFormat::Pretty, + EOutputFormat::JsonUnicode, + EOutputFormat::JsonBase64 + }); + config.SetFreeArgsNum(0); } void TCommandExplain::Parse(TConfig& config) { TClientCommand::Parse(config); - ParseFormats(); + ParseFormats(); CheckQueryOptions(); } @@ -602,7 +602,7 @@ int TCommandExplain::Run(TConfig& config) { ThrowOnError(tablePart); } if (tablePart.HasQueryStats() ) { - auto proto = NYdb::TProtoAccessor::GetProto(tablePart.GetQueryStats()); + auto proto = NYdb::TProtoAccessor::GetProto(tablePart.GetQueryStats()); planJson = proto.query_plan(); ast = proto.query_ast(); } @@ -623,7 +623,7 @@ int TCommandExplain::Run(TConfig& config) { ).ExtractValueSync(); ThrowOnError(result); planJson = result.GetQueryPlan(); - if (auto stats = result.GetStats()) { + if (auto stats = result.GetStats()) { auto proto = NYdb::TProtoAccessor::GetProto(*stats); ast = proto.query_ast(); } @@ -641,11 +641,11 @@ int TCommandExplain::Run(TConfig& config) { } if (PrintAst) { - Cout << "Query AST:" << Endl << ast << Endl; + Cout << "Query AST:" << Endl << ast << Endl; } else { - Cout << "Query Plan:" << Endl; - TQueryPlanPrinter queryPlanPrinter(OutputFormat, Analyze); - queryPlanPrinter.Print(planJson); + Cout << "Query Plan:" << Endl; + TQueryPlanPrinter queryPlanPrinter(OutputFormat, Analyze); + queryPlanPrinter.Print(planJson); } return EXIT_SUCCESS; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h index a85e8d33295..5cd2514aa02 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_table.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_table.h @@ -115,7 +115,7 @@ private: bool BasicStats = false; }; -class TCommandExplain : public TTableCommand, public TCommandWithFormat, TCommandQueryBase, TInterruptibleCommand { +class TCommandExplain : public TTableCommand, public TCommandWithFormat, TCommandQueryBase, TInterruptibleCommand { public: TCommandExplain(); virtual void Config(TConfig& config) override; diff --git a/ydb/public/lib/ydb_cli/common/format.cpp b/ydb/public/lib/ydb_cli/common/format.cpp index a97f9bb32c8..7184ab41d1f 100644 --- a/ydb/public/lib/ydb_cli/common/format.cpp +++ b/ydb/public/lib/ydb_cli/common/format.cpp @@ -1,9 +1,9 @@ #include "format.h" #include "pretty_table.h" -#include <util/string/vector.h> -#include <library/cpp/json/json_prettifier.h> - +#include <util/string/vector.h> +#include <library/cpp/json/json_prettifier.h> + #include <ydb/public/lib/json_value/ydb_json_value.h> namespace NYdb { @@ -110,155 +110,155 @@ void TCommandWithFormat::ParseFormats() { } } -void TQueryPlanPrinter::Print(const TString& plan) { - switch (Format) { - case EOutputFormat::Default: - case EOutputFormat::Pretty: { - NJson::TJsonValue planJson; - NJson::ReadJsonTree(plan, &planJson, true); - - Y_ENSURE(planJson.GetMapSafe().contains("meta")); - const auto& meta = planJson.GetMapSafe().at("meta"); - - Y_ENSURE(meta.GetMapSafe().contains("type")); - if (meta.GetMapSafe().at("type").GetStringSafe() == "script") { - Y_ENSURE(planJson.GetMapSafe().contains("queries")); - const auto& queries = planJson.GetMapSafe().at("queries").GetArraySafe(); - for (size_t queryId = 0; queryId < queries.size(); ++queryId) { - const auto& query = queries[queryId]; - Cout << "Query " << queryId << ":" << Endl; - PrintPretty(query); - } - } else { - PrintPretty(planJson); - } - - break; - } - case EOutputFormat::JsonUnicode: - case EOutputFormat::JsonBase64: - PrintJson(plan); - break; - default: - throw TMissUseException() << "This command doesn't support " << Format << " output format"; - } +void TQueryPlanPrinter::Print(const TString& plan) { + switch (Format) { + case EOutputFormat::Default: + case EOutputFormat::Pretty: { + NJson::TJsonValue planJson; + NJson::ReadJsonTree(plan, &planJson, true); + + Y_ENSURE(planJson.GetMapSafe().contains("meta")); + const auto& meta = planJson.GetMapSafe().at("meta"); + + Y_ENSURE(meta.GetMapSafe().contains("type")); + if (meta.GetMapSafe().at("type").GetStringSafe() == "script") { + Y_ENSURE(planJson.GetMapSafe().contains("queries")); + const auto& queries = planJson.GetMapSafe().at("queries").GetArraySafe(); + for (size_t queryId = 0; queryId < queries.size(); ++queryId) { + const auto& query = queries[queryId]; + Cout << "Query " << queryId << ":" << Endl; + PrintPretty(query); + } + } else { + PrintPretty(planJson); + } + + break; + } + case EOutputFormat::JsonUnicode: + case EOutputFormat::JsonBase64: + PrintJson(plan); + break; + default: + throw TMissUseException() << "This command doesn't support " << Format << " output format"; + } +} + +void TQueryPlanPrinter::PrintJson(const TString& plan) { + Cout << NJson::PrettifyJson(plan, true) << Endl; } - -void TQueryPlanPrinter::PrintJson(const TString& plan) { - Cout << NJson::PrettifyJson(plan, true) << Endl; + +void TQueryPlanPrinter::PrintPretty(const NJson::TJsonValue& plan) { + if (plan.GetMapSafe().contains("Plan")) { + const auto& queryPlan = plan.GetMapSafe().at("Plan").GetMapSafe(); + Y_ENSURE(queryPlan.contains("Plans")); + + TVector<TString> offsets; + for (const auto& subplan : queryPlan.at("Plans").GetArraySafe()) { + PrintPrettyImpl(subplan, offsets); + } + } else { /* old format plan */ + PrintJson(plan.GetStringRobust()); + } } - -void TQueryPlanPrinter::PrintPretty(const NJson::TJsonValue& plan) { - if (plan.GetMapSafe().contains("Plan")) { - const auto& queryPlan = plan.GetMapSafe().at("Plan").GetMapSafe(); - Y_ENSURE(queryPlan.contains("Plans")); - - TVector<TString> offsets; - for (const auto& subplan : queryPlan.at("Plans").GetArraySafe()) { - PrintPrettyImpl(subplan, offsets); - } - } else { /* old format plan */ - PrintJson(plan.GetStringRobust()); - } -} - -void TQueryPlanPrinter::PrintPrettyImpl(const NJson::TJsonValue& plan, TVector<TString>& offsets) { - static const TString edge = "| "; - static const TString noEdge = " "; - static const TString edgeBranch = "├──"; - static const TString edgeBranchLast = "└──"; - - TStringBuilder prefix; - TStringBuilder headerPrefix; - for (const auto& offset : offsets) { - if (&offset != &offsets.back()) { - prefix << offset; - } - headerPrefix << offset; - } - - if (!offsets.empty()) { - bool last = (offsets.back() == edge); - prefix << (last ? edgeBranch : edgeBranchLast); - } - - const auto& node = plan.GetMapSafe(); - - if (node.contains("Operators")) { - for (const auto& op : node.at("Operators").GetArraySafe()) { - TVector<TString> info; - for (const auto& [key, value] : op.GetMapSafe()) { - if (key != "Name") { - info.emplace_back(TStringBuilder() << key << ": " << JsonToString(value)); - } - } - - if (info.empty()) { - Cout << prefix << op.GetMapSafe().at("Name").GetString() << Endl; - } else { - Cout << prefix << op.GetMapSafe().at("Name").GetString() - << " (" << JoinStrings(info, ", ") << ")" << Endl; - } - } - } else if (node.contains("PlanNodeType") && node.at("PlanNodeType").GetString() == "Connection") { - Cout << prefix << "<" << node.at("Node Type").GetString() << ">" << Endl; - } else { - Cout << prefix << node.at("Node Type").GetString() << Endl; - } - - static const THashSet<TString> requiredFields = {"CTE Name", "Tables"}; - for (const auto& [key, value] : node) { - if (requiredFields.contains(key)) { - Cout << headerPrefix << key << ": " << JsonToString(value) << Endl; - } - } - - if (AnalyzeMode && node.contains("Stats")) { - NColorizer::TColors colors = NColorizer::AutoColors(Cout); - for (const auto& [key, value] : node.at("Stats").GetMapSafe()) { - Cout << headerPrefix << colors.Yellow() << key << ": " << colors.Cyan() - << JsonToString(value) << colors.Default() << Endl; - } - } - - if (node.contains("Plans")) { - const auto& plans = node.at("Plans").GetArraySafe(); - for (const auto& subplan : plans) { - offsets.push_back(&subplan != &plans.back() ? edge : noEdge); - PrintPrettyImpl(subplan, offsets); - offsets.pop_back(); - } - } -} - -TString TQueryPlanPrinter::JsonToString(const NJson::TJsonValue& jsonValue) { - TStringBuilder str; - - if (jsonValue.IsString()) { - str << jsonValue.GetString(); - } else if (jsonValue.IsArray()) { - str << "["; - const auto& array = jsonValue.GetArraySafe(); - for (auto it = array.begin(); it != array.end(); ++it) { - str << (it != array.begin() ? ", " : "") - << JsonToString(*it); - } - str << "]"; - } else if (jsonValue.IsMap()) { - str << "{"; - const auto& map = jsonValue.GetMapSafe(); - for (auto it = map.begin(); it != map.end(); ++it) { - str << (it != map.begin() ? ", " : "") - << it->first << ": " << JsonToString(it->second); - } - str << "}"; - } else { - str << jsonValue; - } - - return str; -} - + +void TQueryPlanPrinter::PrintPrettyImpl(const NJson::TJsonValue& plan, TVector<TString>& offsets) { + static const TString edge = "| "; + static const TString noEdge = " "; + static const TString edgeBranch = "├──"; + static const TString edgeBranchLast = "└──"; + + TStringBuilder prefix; + TStringBuilder headerPrefix; + for (const auto& offset : offsets) { + if (&offset != &offsets.back()) { + prefix << offset; + } + headerPrefix << offset; + } + + if (!offsets.empty()) { + bool last = (offsets.back() == edge); + prefix << (last ? edgeBranch : edgeBranchLast); + } + + const auto& node = plan.GetMapSafe(); + + if (node.contains("Operators")) { + for (const auto& op : node.at("Operators").GetArraySafe()) { + TVector<TString> info; + for (const auto& [key, value] : op.GetMapSafe()) { + if (key != "Name") { + info.emplace_back(TStringBuilder() << key << ": " << JsonToString(value)); + } + } + + if (info.empty()) { + Cout << prefix << op.GetMapSafe().at("Name").GetString() << Endl; + } else { + Cout << prefix << op.GetMapSafe().at("Name").GetString() + << " (" << JoinStrings(info, ", ") << ")" << Endl; + } + } + } else if (node.contains("PlanNodeType") && node.at("PlanNodeType").GetString() == "Connection") { + Cout << prefix << "<" << node.at("Node Type").GetString() << ">" << Endl; + } else { + Cout << prefix << node.at("Node Type").GetString() << Endl; + } + + static const THashSet<TString> requiredFields = {"CTE Name", "Tables"}; + for (const auto& [key, value] : node) { + if (requiredFields.contains(key)) { + Cout << headerPrefix << key << ": " << JsonToString(value) << Endl; + } + } + + if (AnalyzeMode && node.contains("Stats")) { + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + for (const auto& [key, value] : node.at("Stats").GetMapSafe()) { + Cout << headerPrefix << colors.Yellow() << key << ": " << colors.Cyan() + << JsonToString(value) << colors.Default() << Endl; + } + } + + if (node.contains("Plans")) { + const auto& plans = node.at("Plans").GetArraySafe(); + for (const auto& subplan : plans) { + offsets.push_back(&subplan != &plans.back() ? edge : noEdge); + PrintPrettyImpl(subplan, offsets); + offsets.pop_back(); + } + } +} + +TString TQueryPlanPrinter::JsonToString(const NJson::TJsonValue& jsonValue) { + TStringBuilder str; + + if (jsonValue.IsString()) { + str << jsonValue.GetString(); + } else if (jsonValue.IsArray()) { + str << "["; + const auto& array = jsonValue.GetArraySafe(); + for (auto it = array.begin(); it != array.end(); ++it) { + str << (it != array.begin() ? ", " : "") + << JsonToString(*it); + } + str << "]"; + } else if (jsonValue.IsMap()) { + str << "{"; + const auto& map = jsonValue.GetMapSafe(); + for (auto it = map.begin(); it != map.end(); ++it) { + str << (it != map.begin() ? ", " : "") + << it->first << ": " << JsonToString(it->second); + } + str << "}"; + } else { + str << jsonValue; + } + + return str; +} + TResultSetPrinter::TResultSetPrinter(EOutputFormat format, std::function<bool()> isInterrupted) : Format(format) @@ -269,7 +269,7 @@ TResultSetPrinter::~TResultSetPrinter() { if (PrintedSomething && !IsInterrupted()) { EndResultSet(); } -} +} void TResultSetPrinter::Print(const TResultSet& resultSet) { if (FirstPart) { @@ -302,7 +302,7 @@ void TResultSetPrinter::Print(const TResultSet& resultSet) { default: throw TMissUseException() << "This command doesn't support " << Format << " output format"; } -} +} void TResultSetPrinter::Reset() { if (PrintedSomething) { diff --git a/ydb/public/lib/ydb_cli/common/format.h b/ydb/public/lib/ydb_cli/common/format.h index 0c756942116..2d8d6a2e38b 100644 --- a/ydb/public/lib/ydb_cli/common/format.h +++ b/ydb/public/lib/ydb_cli/common/format.h @@ -65,23 +65,23 @@ private: std::function<bool()> IsInterrupted; }; -class TQueryPlanPrinter { -public: - TQueryPlanPrinter(EOutputFormat format, bool analyzeMode = false) - : Format(format) - , AnalyzeMode(analyzeMode) {} - - void Print(const TString& plan); - -private: - void PrintPretty(const NJson::TJsonValue& plan); - void PrintPrettyImpl(const NJson::TJsonValue& plan, TVector<TString>& offsets); - void PrintJson(const TString& plan); - TString JsonToString(const NJson::TJsonValue& jsonValue); - - EOutputFormat Format; - bool AnalyzeMode; -}; - +class TQueryPlanPrinter { +public: + TQueryPlanPrinter(EOutputFormat format, bool analyzeMode = false) + : Format(format) + , AnalyzeMode(analyzeMode) {} + + void Print(const TString& plan); + +private: + void PrintPretty(const NJson::TJsonValue& plan); + void PrintPrettyImpl(const NJson::TJsonValue& plan, TVector<TString>& offsets); + void PrintJson(const TString& plan); + TString JsonToString(const NJson::TJsonValue& jsonValue); + + EOutputFormat Format; + bool AnalyzeMode; +}; + } } diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp index 238868cfee5..610fec38847 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.cpp +++ b/ydb/public/sdk/cpp/client/ydb_table/table.cpp @@ -1015,24 +1015,24 @@ TTableBuilder& TTableBuilder::AddNullableColumn(const TString& name, const TDeci return *this; } -TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const EPrimitiveType& type, const TString& family) { - auto columnType = TTypeBuilder() - .Primitive(type) - .Build(); - - TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family); - return *this; -} - -TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const TDecimalType& type, const TString& family) { - auto columnType = TTypeBuilder() - .Decimal(type) - .Build(); - - TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family); - return *this; -} - +TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const EPrimitiveType& type, const TString& family) { + auto columnType = TTypeBuilder() + .Primitive(type) + .Build(); + + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family); + return *this; +} + +TTableBuilder& TTableBuilder::AddNonNullableColumn(const TString& name, const TDecimalType& type, const TString& family) { + auto columnType = TTypeBuilder() + .Decimal(type) + .Build(); + + TableDescription_.AddColumn(name, TProtoAccessor::GetProto(columnType), family); + return *this; +} + TTableBuilder& TTableBuilder::SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns) { TableDescription_.SetPrimaryKeyColumns(primaryKeyColumns); return *this; diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h index 3553b721e6c..42d21d2fec7 100644 --- a/ydb/public/sdk/cpp/client/ydb_table/table.h +++ b/ydb/public/sdk/cpp/client/ydb_table/table.h @@ -648,8 +648,8 @@ public: TTableBuilder& AddNullableColumn(const TString& name, const EPrimitiveType& type, const TString& family = TString()); TTableBuilder& AddNullableColumn(const TString& name, const TDecimalType& type, const TString& family = TString()); - TTableBuilder& AddNonNullableColumn(const TString& name, const EPrimitiveType& type, const TString& family = TString()); - TTableBuilder& AddNonNullableColumn(const TString& name, const TDecimalType& type, const TString& family = TString()); + TTableBuilder& AddNonNullableColumn(const TString& name, const EPrimitiveType& type, const TString& family = TString()); + TTableBuilder& AddNonNullableColumn(const TString& name, const TDecimalType& type, const TString& family = TString()); TTableBuilder& SetPrimaryKeyColumns(const TVector<TString>& primaryKeyColumns); TTableBuilder& SetPrimaryKeyColumn(const TString& primaryKeyColumn); diff --git a/ydb/services/ydb/ydb_bulk_upsert_ut.cpp b/ydb/services/ydb/ydb_bulk_upsert_ut.cpp index 5873c314ada..0665008ee49 100644 --- a/ydb/services/ydb/ydb_bulk_upsert_ut.cpp +++ b/ydb/services/ydb/ydb_bulk_upsert_ut.cpp @@ -218,117 +218,117 @@ Y_UNIT_TEST_SUITE(YdbTableBulkUpsert) { } } - Y_UNIT_TEST(NotNulls) { - TKikimrWithGrpcAndRootSchema server; - ui16 grpc = server.GetPort(); - - TString location = TStringBuilder() << "localhost:" << grpc; - - auto connection = NYdb::TDriver(TDriverConfig().SetEndpoint(location)); - NYdb::NTable::TTableClient client(connection); - auto session = client.GetSession().ExtractValueSync().GetSession(); - - TString tableName = "/Root/TestNotNullColumns"; - - { - auto tableBuilder = client.GetTableBuilder(); - tableBuilder - .AddNonNullableColumn("Key", EPrimitiveType::Uint64) - .AddNonNullableColumn("Value", EPrimitiveType::Uint64) - .SetPrimaryKeyColumns({"Key"}); - auto result = session.CreateTable(tableName, tableBuilder.Build()).ExtractValueSync(); - - Cerr << result.GetIssues().ToString(); - UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - - { - TValueBuilder rows; - rows.BeginList(); - rows.AddListItem() - .BeginStruct() - .AddMember("Key").Uint64(1) - .AddMember("Value").Uint64(10) - .EndStruct(); - rows.EndList(); - - auto res = client.BulkUpsert(tableName, rows.Build()).GetValueSync(); - Cerr << res.GetIssues().ToString(); - UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::SUCCESS); - } - - { /* missing not null primary key column */ - TValueBuilder rows; - rows.BeginList(); - rows.AddListItem() - .BeginStruct() - .AddMember("Value").Uint64(20) - .EndStruct(); - rows.EndList(); - - auto res = client.BulkUpsert(tableName, rows.Build()).GetValueSync(); - Cerr << res.GetIssues().ToString(); - Cerr << res.GetStatus(); - UNIT_ASSERT_STRING_CONTAINS(res.GetIssues().ToString(), "Missing key columns: Key"); - UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::SCHEME_ERROR); - } - - { /* missing not null value column */ - TValueBuilder rows; - rows.BeginList(); - rows.AddListItem() - .BeginStruct() - .AddMember("Key").Uint64(2) - .EndStruct(); - rows.EndList(); - - auto res = client.BulkUpsert(tableName, rows.Build()).GetValueSync(); - Cerr << res.GetIssues().ToString(); - UNIT_ASSERT_STRING_CONTAINS(res.GetIssues().ToString(), "Missing not null columns: Value"); - UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::SCHEME_ERROR); - } - - { /* set null to not null primary key column */ - TValueBuilder rows; - rows.BeginList(); - rows.AddListItem() - .BeginStruct() - .AddMember("Key").EmptyOptional(EPrimitiveType::Uint64) - .AddMember("Value").Uint64(20) - .EndStruct(); - rows.EndList(); - - auto res = client.BulkUpsert(tableName, rows.Build()).GetValueSync(); - Cerr << res.GetIssues().ToString(); - UNIT_ASSERT_STRING_CONTAINS(res.GetIssues().ToString(), "Received NULL value for not null column"); - UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::BAD_REQUEST); - } - - { /* set null to not null value column */ - TValueBuilder rows; - rows.BeginList(); - rows.AddListItem() - .BeginStruct() - .AddMember("Key").Uint64(2) - .AddMember("Value").EmptyOptional(EPrimitiveType::Uint64) - .EndStruct(); - rows.EndList(); - - auto res = client.BulkUpsert(tableName, rows.Build()).GetValueSync(); - Cerr << res.GetIssues().ToString(); - UNIT_ASSERT_STRING_CONTAINS(res.GetIssues().ToString(), "Received NULL value for not null column"); - UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::BAD_REQUEST); - } - - { - auto result = session.DropTable(tableName).ExtractValueSync(); - - UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); - } - } - + Y_UNIT_TEST(NotNulls) { + TKikimrWithGrpcAndRootSchema server; + ui16 grpc = server.GetPort(); + + TString location = TStringBuilder() << "localhost:" << grpc; + + auto connection = NYdb::TDriver(TDriverConfig().SetEndpoint(location)); + NYdb::NTable::TTableClient client(connection); + auto session = client.GetSession().ExtractValueSync().GetSession(); + + TString tableName = "/Root/TestNotNullColumns"; + + { + auto tableBuilder = client.GetTableBuilder(); + tableBuilder + .AddNonNullableColumn("Key", EPrimitiveType::Uint64) + .AddNonNullableColumn("Value", EPrimitiveType::Uint64) + .SetPrimaryKeyColumns({"Key"}); + auto result = session.CreateTable(tableName, tableBuilder.Build()).ExtractValueSync(); + + Cerr << result.GetIssues().ToString(); + UNIT_ASSERT_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + TValueBuilder rows; + rows.BeginList(); + rows.AddListItem() + .BeginStruct() + .AddMember("Key").Uint64(1) + .AddMember("Value").Uint64(10) + .EndStruct(); + rows.EndList(); + + auto res = client.BulkUpsert(tableName, rows.Build()).GetValueSync(); + Cerr << res.GetIssues().ToString(); + UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::SUCCESS); + } + + { /* missing not null primary key column */ + TValueBuilder rows; + rows.BeginList(); + rows.AddListItem() + .BeginStruct() + .AddMember("Value").Uint64(20) + .EndStruct(); + rows.EndList(); + + auto res = client.BulkUpsert(tableName, rows.Build()).GetValueSync(); + Cerr << res.GetIssues().ToString(); + Cerr << res.GetStatus(); + UNIT_ASSERT_STRING_CONTAINS(res.GetIssues().ToString(), "Missing key columns: Key"); + UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::SCHEME_ERROR); + } + + { /* missing not null value column */ + TValueBuilder rows; + rows.BeginList(); + rows.AddListItem() + .BeginStruct() + .AddMember("Key").Uint64(2) + .EndStruct(); + rows.EndList(); + + auto res = client.BulkUpsert(tableName, rows.Build()).GetValueSync(); + Cerr << res.GetIssues().ToString(); + UNIT_ASSERT_STRING_CONTAINS(res.GetIssues().ToString(), "Missing not null columns: Value"); + UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::SCHEME_ERROR); + } + + { /* set null to not null primary key column */ + TValueBuilder rows; + rows.BeginList(); + rows.AddListItem() + .BeginStruct() + .AddMember("Key").EmptyOptional(EPrimitiveType::Uint64) + .AddMember("Value").Uint64(20) + .EndStruct(); + rows.EndList(); + + auto res = client.BulkUpsert(tableName, rows.Build()).GetValueSync(); + Cerr << res.GetIssues().ToString(); + UNIT_ASSERT_STRING_CONTAINS(res.GetIssues().ToString(), "Received NULL value for not null column"); + UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::BAD_REQUEST); + } + + { /* set null to not null value column */ + TValueBuilder rows; + rows.BeginList(); + rows.AddListItem() + .BeginStruct() + .AddMember("Key").Uint64(2) + .AddMember("Value").EmptyOptional(EPrimitiveType::Uint64) + .EndStruct(); + rows.EndList(); + + auto res = client.BulkUpsert(tableName, rows.Build()).GetValueSync(); + Cerr << res.GetIssues().ToString(); + UNIT_ASSERT_STRING_CONTAINS(res.GetIssues().ToString(), "Received NULL value for not null column"); + UNIT_ASSERT_EQUAL(res.GetStatus(), EStatus::BAD_REQUEST); + } + + { + auto result = session.DropTable(tableName).ExtractValueSync(); + + UNIT_ASSERT_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + } + Y_UNIT_TEST(Errors) { TKikimrWithGrpcAndRootSchema server; ui16 grpc = server.GetPort(); diff --git a/ydb/services/ydb/ydb_common_ut.h b/ydb/services/ydb/ydb_common_ut.h index 22bd3e1fbc6..c067928c785 100644 --- a/ydb/services/ydb/ydb_common_ut.h +++ b/ydb/services/ydb/ydb_common_ut.h @@ -74,7 +74,7 @@ public: ServerSettings->SetKqpSettings(kqpSettings); ServerSettings->SetEnableAsyncIndexes(true); ServerSettings->SetEnableDataColumnForIndexTable(true); - ServerSettings->SetEnableNotNullColumns(true); + ServerSettings->SetEnableNotNullColumns(true); ServerSettings->SetEnableSystemViews(TestSettings::EnableSystemViews); ServerSettings->SetEnableSchemeTransactionsAtSchemeShard(true); ServerSettings->SetEnableYq(enableYq); diff --git a/ydb/services/ydb/ydb_table_ut.cpp b/ydb/services/ydb/ydb_table_ut.cpp index 9a41a59478b..4861c06b92e 100644 --- a/ydb/services/ydb/ydb_table_ut.cpp +++ b/ydb/services/ydb/ydb_table_ut.cpp @@ -180,7 +180,7 @@ Y_UNIT_TEST_SUITE(YdbYqlClient) { auto ref = R"___(<main>: Error: Type annotation, code: 1030 <main>:2:25: Error: At function: KiWriteTable! <main>:2:43: Error: Failed to convert type: Struct<'Key':String,'Value':String> to Struct<'Key':Uint32?,'Value':String?> - <main>:2:43: Error: Failed to convert input columns types to scheme types, code: 2031 + <main>:2:43: Error: Failed to convert input columns types to scheme types, code: 2031 )___"; UNIT_ASSERT_EQUAL(result.GetIssues().Size(), 1); UNIT_ASSERT_NO_DIFF(result.GetIssues().ToString(), ref); |