aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMikhail Surin <surinmike@gmail.com>2024-02-27 14:26:56 +0300
committerGitHub <noreply@github.com>2024-02-27 14:26:56 +0300
commitcc1ad2eb9c6809780c5f35e5f296d11fe25a85db (patch)
treeeb1c8678affa6d8959c9004b82e373689ab39711
parentf554ad1181c20eddccefd475d3214ad384ce7635 (diff)
downloadydb-cc1ad2eb9c6809780c5f35e5f296d11fe25a85db.tar.gz
add fullscan metric (#2192)
-rw-r--r--ydb/core/kqp/counters/kqp_counters.cpp2
-rw-r--r--ydb/core/kqp/counters/kqp_counters.h2
-rw-r--r--ydb/core/kqp/executer_actor/kqp_data_executer.cpp13
-rw-r--r--ydb/core/kqp/executer_actor/kqp_executer_impl.h15
-rw-r--r--ydb/core/kqp/executer_actor/kqp_partition_helper.cpp80
-rw-r--r--ydb/core/kqp/executer_actor/kqp_partition_helper.h12
-rw-r--r--ydb/core/kqp/ut/opt/kqp_ne_ut.cpp76
7 files changed, 174 insertions, 26 deletions
diff --git a/ydb/core/kqp/counters/kqp_counters.cpp b/ydb/core/kqp/counters/kqp_counters.cpp
index c3ad52dcb6b..7b7df007625 100644
--- a/ydb/core/kqp/counters/kqp_counters.cpp
+++ b/ydb/core/kqp/counters/kqp_counters.cpp
@@ -822,6 +822,8 @@ TKqpCounters::TKqpCounters(const ::NMonitoring::TDynamicCounterPtr& counters, co
"PhyTx/DataTxTotalTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1));
ScanTxTotalTimeHistogram = KqpGroup->GetHistogram(
"PhyTx/ScanTxTotalTimeMs", NMonitoring::ExponentialHistogram(20, 2, 1));
+
+ FullScansExecuted = KqpGroup->GetCounter("FullScans", true);
}
::NMonitoring::TDynamicCounterPtr TKqpCounters::GetKqpCounters() const {
diff --git a/ydb/core/kqp/counters/kqp_counters.h b/ydb/core/kqp/counters/kqp_counters.h
index f6297434457..d1e6bc2ae73 100644
--- a/ydb/core/kqp/counters/kqp_counters.h
+++ b/ydb/core/kqp/counters/kqp_counters.h
@@ -345,6 +345,8 @@ public:
void RemoveDbCounters(const TString& database);
public:
+ ::NMonitoring::TDynamicCounters::TCounterPtr FullScansExecuted;
+
// Lease updates counters
::NMonitoring::THistogramPtr LeaseUpdateLatency;
::NMonitoring::THistogramPtr RunActorLeaseUpdateBacklog;
diff --git a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp
index 6ff10294b4c..c448248e653 100644
--- a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp
+++ b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp
@@ -1352,9 +1352,14 @@ private:
case NKqpProto::TKqpPhyTableOperation::kReadRanges:
case NKqpProto::TKqpPhyTableOperation::kReadRange:
case NKqpProto::TKqpPhyTableOperation::kLookup: {
- auto partitions = PrunePartitions(op, stageInfo, HolderFactory(), TypeEnv());
+ bool isFullScan = false;
+ auto partitions = PrunePartitions(op, stageInfo, HolderFactory(), TypeEnv(), isFullScan);
auto readSettings = ExtractReadSettings(op, stageInfo, HolderFactory(), TypeEnv());
+ if (!readSettings.ItemsLimit && isFullScan) {
+ Counters->Counters->FullScansExecuted->Inc();
+ }
+
for (auto& [shardId, shardInfo] : partitions) {
YQL_ENSURE(!shardInfo.KeyWriteRanges);
@@ -1890,7 +1895,11 @@ private:
const auto& stage = stageInfo.Meta.GetStage(stageInfo.Id);
if (stage.SourcesSize() > 0 && stage.GetSources(0).GetTypeCase() == NKqpProto::TKqpSource::kReadRangesSource) {
const auto& source = stage.GetSources(0).GetReadRangesSource();
- SourceScanStageIdToParititions[stageInfo.Id] = PrunePartitions(source, stageInfo, HolderFactory(), TypeEnv());
+ bool isFullScan;
+ SourceScanStageIdToParititions[stageInfo.Id] = PrunePartitions(source, stageInfo, HolderFactory(), TypeEnv(), isFullScan);
+ if (isFullScan && !source.HasItemsLimit()) {
+ Counters->Counters->FullScansExecuted->Inc();
+ }
for (const auto& [shardId, _] : SourceScanStageIdToParititions.at(stageId)) {
shardIds.insert(shardId);
}
diff --git a/ydb/core/kqp/executer_actor/kqp_executer_impl.h b/ydb/core/kqp/executer_actor/kqp_executer_impl.h
index 3f83142232f..2d5802bbda7 100644
--- a/ydb/core/kqp/executer_actor/kqp_executer_impl.h
+++ b/ydb/core/kqp/executer_actor/kqp_executer_impl.h
@@ -1202,9 +1202,15 @@ protected:
}
};
+ bool isFullScan = false;
const THashMap<ui64, TShardInfo> partitions = SourceScanStageIdToParititions.empty()
- ? PrunePartitions(source, stageInfo, HolderFactory(), TypeEnv())
+ ? PrunePartitions(source, stageInfo, HolderFactory(), TypeEnv(), isFullScan)
: SourceScanStageIdToParititions.at(stageInfo.Id);
+
+ if (isFullScan && !source.HasItemsLimit()) {
+ Counters->Counters->FullScansExecuted->Inc();
+ }
+
if (partitions.size() > 0 && source.GetSequentialInFlightShards() > 0 && partitions.size() > source.GetSequentialInFlightShards()) {
auto [startShard, shardInfo] = MakeVirtualTablePartition(source, stageInfo, HolderFactory(), TypeEnv());
if (Stats) {
@@ -1476,10 +1482,15 @@ protected:
Y_DEBUG_ABORT_UNLESS(stageInfo.Meta.TablePath == op.GetTable().GetPath());
auto columns = BuildKqpColumns(op, tableInfo);
- auto partitions = PrunePartitions(op, stageInfo, HolderFactory(), TypeEnv());
+ bool isFullScan;
+ auto partitions = PrunePartitions(op, stageInfo, HolderFactory(), TypeEnv(), isFullScan);
const bool isOlapScan = (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kReadOlapRange);
auto readSettings = ExtractReadSettings(op, stageInfo, HolderFactory(), TypeEnv());
+ if (isFullScan && readSettings.ItemsLimit) {
+ Counters->Counters->FullScansExecuted->Inc();
+ }
+
if (op.GetTypeCase() == NKqpProto::TKqpPhyTableOperation::kReadRange) {
stageInfo.Meta.SkipNullKeys.assign(op.GetReadRange().GetSkipNullKeys().begin(),
op.GetReadRange().GetSkipNullKeys().end());
diff --git a/ydb/core/kqp/executer_actor/kqp_partition_helper.cpp b/ydb/core/kqp/executer_actor/kqp_partition_helper.cpp
index 8c9b3e9a075..84fbd82ac5c 100644
--- a/ydb/core/kqp/executer_actor/kqp_partition_helper.cpp
+++ b/ydb/core/kqp/executer_actor/kqp_partition_helper.cpp
@@ -112,7 +112,7 @@ THashMap<ui64, TShardParamValuesAndRanges> PartitionParamByKey(
THashMap<ui64, TShardParamValuesAndRanges> PartitionParamByKeyPrefix(
const NUdf::TUnboxedValue& value, NKikimr::NMiniKQL::TType* type,
const TTableId& tableId, const TIntrusiveConstPtr<TTableConstInfo>& tableInfo, const TKeyDesc& key,
- const NMiniKQL::THolderFactory&, const NMiniKQL::TTypeEnvironment& typeEnv)
+ const NMiniKQL::THolderFactory&, const NMiniKQL::TTypeEnvironment& typeEnv, bool isFullScan)
{
YQL_ENSURE(tableInfo);
@@ -127,6 +127,7 @@ THashMap<ui64, TShardParamValuesAndRanges> PartitionParamByKeyPrefix(
auto& structType = static_cast<NMiniKQL::TStructType&>(*itemType);
const ui64 keyLen = tableInfo->KeyColumns.size();
+ isFullScan = keyLen == 0;
TVector<NScheme::TTypeInfo> keyFullType{Reserve(keyLen)};
TVector<NScheme::TTypeInfo> keyPrefixType{Reserve(keyLen)};
@@ -338,6 +339,40 @@ TSerializedPointOrRange FillOneRange(NUdf::TUnboxedValue& begin, NUdf::TUnboxedV
return range;
}
+bool IsFullRange(const TVector<NScheme::TTypeInfo>& keyColumnTypes, const TSerializedTableRange& range) {
+ if (!range.To.GetCells().empty() || range.ToInclusive) {
+ return false;
+ }
+ if (range.From.GetCells().size() != keyColumnTypes.size() || !range.FromInclusive) {
+ return false;
+ }
+ for (auto& cell : range.From.GetCells()) {
+ if (!cell.IsNull()) {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool IsFullRange(const TVector<NScheme::TTypeInfo>&, const TSerializedCellVec& vec) {
+ return vec.GetCells().empty();
+}
+
+bool IsFullRange(const TVector<NScheme::TTypeInfo>& keyColumnTypes, const TSerializedPointOrRange& one) {
+ if (std::holds_alternative<TSerializedCellVec>(one)) {
+ return IsFullRange(keyColumnTypes, std::get<TSerializedCellVec>(one));
+ } else {
+ auto& range = std::get<TSerializedTableRange>(one);
+ return IsFullRange(keyColumnTypes, range);
+ }
+}
+
+bool IsFullRange(const TVector<NScheme::TTypeInfo>& keyColumnTypes, const TVector<TSerializedPointOrRange>& ranges) {
+ if (!ranges) return true;
+ if (IsFullRange(keyColumnTypes, ranges[0])) return true;
+ return false;
+}
+
TVector<TSerializedPointOrRange> BuildFullRange(const TVector<NScheme::TTypeInfo>& keyColumnTypes) {
// Build range from NULL, NULL ... NULL to +inf, +inf ... +inf
TVector<TCell> fromKeyValues(keyColumnTypes.size());
@@ -487,7 +522,7 @@ TString TShardInfo::ToString(const TVector<NScheme::TTypeInfo>& keyTypes, const
}
THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyOpReadRange& readRange, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv, bool& isFullScan)
{
auto guard = typeEnv.BindAllocator();
const auto& tableInfo = stageInfo.Meta.TableConstInfo;
@@ -496,6 +531,8 @@ THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyOpReadRange&
YQL_ENSURE(readRange.HasKeyRange());
auto range = MakeKeyRange(keyColumnTypes, readRange.GetKeyRange(), stageInfo, holderFactory, typeEnv);
+ isFullScan = IsFullRange(keyColumnTypes, range);
+
auto readPartitions = GetKeyRangePartitions(range.ToTableRange(), stageInfo.Meta.ShardKey->GetPartitions(),
keyColumnTypes);
@@ -517,13 +554,14 @@ THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyOpReadRange&
}
THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyOpReadRanges& readRanges, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv, bool& isFullScan)
{
Y_UNUSED(holderFactory);
const auto& tableInfo = stageInfo.Meta.TableConstInfo;
const auto& keyColumnTypes = tableInfo->KeyColumnTypes;
auto ranges = FillReadRangesInternal(keyColumnTypes, readRanges, stageInfo, typeEnv);
+ isFullScan = IsFullRange(keyColumnTypes, ranges);
THashMap<ui64, TShardInfo> shardInfoMap;
@@ -621,13 +659,14 @@ std::pair<ui64, TShardInfo> MakeVirtualTablePartition(const NKqpProto::TKqpReadR
THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpReadRangesSource& source, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv, bool& isFullScan)
{
auto guard = typeEnv.BindAllocator();
const auto& tableInfo = stageInfo.Meta.TableConstInfo;
const auto& keyColumnTypes = tableInfo->KeyColumnTypes;
auto ranges = ExtractRanges(source, stageInfo, holderFactory, typeEnv, guard);
+ isFullScan = IsFullRange(keyColumnTypes, ranges);
THashMap<ui64, TShardInfo> shardInfoMap;
@@ -661,7 +700,7 @@ THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpReadRangesSource
THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyOpReadOlapRanges& readRanges, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv, bool& isFullScan)
{
Y_UNUSED(holderFactory);
auto guard = typeEnv.BindAllocator();
@@ -672,6 +711,7 @@ THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyOpReadOlapRan
const auto& keyColumnTypes = tableInfo->KeyColumnTypes;
auto ranges = FillReadRanges(keyColumnTypes, readRanges, stageInfo, typeEnv);
+ isFullScan = IsFullRange(keyColumnTypes, ranges);
THashMap<ui64, TShardInfo> shardInfoMap;
@@ -690,17 +730,17 @@ THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyOpReadOlapRan
}
THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyTableOperation& operation, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv, bool& isFullScan)
{
switch (operation.GetTypeCase()) {
case NKqpProto::TKqpPhyTableOperation::kReadRanges:
- return PrunePartitions(operation.GetReadRanges(), stageInfo, holderFactory, typeEnv);
+ return PrunePartitions(operation.GetReadRanges(), stageInfo, holderFactory, typeEnv, isFullScan);
case NKqpProto::TKqpPhyTableOperation::kReadRange:
- return PrunePartitions(operation.GetReadRange(), stageInfo, holderFactory, typeEnv);
+ return PrunePartitions(operation.GetReadRange(), stageInfo, holderFactory, typeEnv, isFullScan);
case NKqpProto::TKqpPhyTableOperation::kLookup:
- return PrunePartitions(operation.GetLookup(), stageInfo, holderFactory, typeEnv);
+ return PrunePartitions(operation.GetLookup(), stageInfo, holderFactory, typeEnv, isFullScan);
case NKqpProto::TKqpPhyTableOperation::kReadOlapRange:
- return PrunePartitions(operation.GetReadOlapRange(), stageInfo, holderFactory, typeEnv);
+ return PrunePartitions(operation.GetReadOlapRange(), stageInfo, holderFactory, typeEnv, isFullScan);
default:
YQL_ENSURE(false, "Unexpected table scan operation: " << static_cast<ui32>(operation.GetTypeCase()));
break;
@@ -714,12 +754,12 @@ using namespace NMiniKQL;
THashMap<ui64, TShardInfo> PartitionLookupByParameterValue(const NKqpProto::TKqpPhyParamValue& proto,
const TStageInfo& stageInfo, const THolderFactory& holderFactory,
- const TTypeEnvironment& typeEnv)
+ const TTypeEnvironment& typeEnv, bool isFullScan)
{
const auto& name = proto.GetParamName();
auto [type, value] = stageInfo.Meta.Tx.Params->GetParameterUnboxedValue(name);
auto shardsMap = PartitionParamByKeyPrefix(value, type, stageInfo.Meta.TableId, stageInfo.Meta.TableConstInfo, *stageInfo.Meta.ShardKey,
- holderFactory, typeEnv);
+ holderFactory, typeEnv, isFullScan);
THashMap<ui64, TShardInfo> shardInfoMap;
@@ -744,7 +784,7 @@ THashMap<ui64, TShardInfo> PartitionLookupByParameterValue(const NKqpProto::TKqp
THashMap<ui64, TShardInfo> PartitionLookupByRowsList(const NKqpProto::TKqpPhyRowsList& proto,
const TStageInfo& stageInfo, const THolderFactory& holderFactory,
- const TTypeEnvironment& typeEnv)
+ const TTypeEnvironment& typeEnv, bool isFullScan)
{
std::unordered_map<ui64, THashSet<TString>> shardParams; // shardId -> paramNames
std::unordered_map<ui64, TShardParamValuesAndRanges> ret;
@@ -753,12 +793,18 @@ THashMap<ui64, TShardInfo> PartitionLookupByRowsList(const NKqpProto::TKqpPhyRow
const auto& tableInfo = stageInfo.Meta.TableConstInfo;
+ isFullScan = false;
+
for (const auto& row : proto.GetRows()) {
TVector<TCell> keyFrom, keyTo;
keyFrom.resize(tableInfo->KeyColumns.size());
keyTo.resize(row.GetColumns().size());
NUdf::TUnboxedValue mkqlValue;
+ if (row.ColumnsSize() == 0) {
+ isFullScan = true;
+ }
+
NMiniKQL::TType* mkqlType = nullptr;
for (const auto& [columnName, columnValue]: row.GetColumns()) {
switch (columnValue.GetKindCase()) {
@@ -834,7 +880,7 @@ THashMap<ui64, TShardInfo> PartitionLookupByRowsList(const NKqpProto::TKqpPhyRow
} // namespace
THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyOpLookup& lookup,
- const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv)
+ const TStageInfo& stageInfo, const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv, bool& isFullScan)
{
auto guard = typeEnv.BindAllocator();
YQL_CLOG(TRACE, ProviderKqp) << "PrunePartitions: " << lookup.DebugString();
@@ -842,18 +888,20 @@ THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyOpLookup& loo
if (!lookup.HasKeysValue()) {
THashMap<ui64, TShardInfo> shardInfoMap;
FillFullRange(stageInfo, shardInfoMap, /* read */ true);
+ isFullScan = true;
return shardInfoMap;
}
+ isFullScan = false;
switch (auto kind = lookup.GetKeysValue().GetKindCase()) {
case NKqpProto::TKqpPhyValue::kParamValue: {
return PartitionLookupByParameterValue(lookup.GetKeysValue().GetParamValue(), stageInfo,
- holderFactory, typeEnv);
+ holderFactory, typeEnv, isFullScan);
}
case NKqpProto::TKqpPhyValue::kRowsList: {
return PartitionLookupByRowsList(lookup.GetKeysValue().GetRowsList(), stageInfo,
- holderFactory, typeEnv);
+ holderFactory, typeEnv, isFullScan);
}
case NKqpProto::TKqpPhyValue::kParamElementValue:
diff --git a/ydb/core/kqp/executer_actor/kqp_partition_helper.h b/ydb/core/kqp/executer_actor/kqp_partition_helper.h
index de2ae9069f1..8892c20e72d 100644
--- a/ydb/core/kqp/executer_actor/kqp_partition_helper.h
+++ b/ydb/core/kqp/executer_actor/kqp_partition_helper.h
@@ -52,19 +52,19 @@ TVector<TSerializedPointOrRange> FillReadRanges(const TVector<NScheme::TTypeInfo
const NMiniKQL::TTypeEnvironment& typeEnv);
THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyOpReadRange& readRange, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv, bool& isFullScan);
THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyOpReadRanges& readRanges, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv, bool& isFullScan);
THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyOpLookup& lookup, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv, bool& isFullScan);
std::pair<ui64, TShardInfo> MakeVirtualTablePartition(const NKqpProto::TKqpReadRangesSource& source, const TStageInfo& stageInfo,
const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpReadRangesSource& source, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv, bool& isFullScan);
ui64 ExtractItemsLimit(const TStageInfo& stageInfo, const NKqpProto::TKqpPhyValue& protoItemsLimit,
const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
@@ -73,10 +73,10 @@ ui64 ExtractItemsLimit(const TStageInfo& stageInfo, const NKqpProto::TKqpPhyValu
// NOTE: Unlike OLTP tables that store data in DataShards, data in OLAP tables is not range
// partitioned and multiple ColumnShards store data from the same key range
THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyOpReadOlapRanges& readRanges, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv, bool& isFullScan);
THashMap<ui64, TShardInfo> PrunePartitions(const NKqpProto::TKqpPhyTableOperation& operation, const TStageInfo& stageInfo,
- const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
+ const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv, bool& isFullScan);
THashMap<ui64, TShardInfo> PruneEffectPartitions(const NKqpProto::TKqpPhyOpUpsertRows& effect, const TStageInfo& stageInfo,
const NMiniKQL::THolderFactory& holderFactory, const NMiniKQL::TTypeEnvironment& typeEnv);
diff --git a/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp b/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp
index 29ca801dce6..7eb4e290e98 100644
--- a/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp
+++ b/ydb/core/kqp/ut/opt/kqp_ne_ut.cpp
@@ -3976,6 +3976,82 @@ Y_UNIT_TEST_SUITE(KqpNewEngine) {
CompareYson(R"([[[1u];[2u]];[[2u];[2u]]])", FormatResultSetYson(result.GetResultSet(0)));
}
+
+ Y_UNIT_TEST(FullScanCount) {
+ TKikimrSettings settings;
+ NKikimrConfig::TAppConfig appConfig;
+ appConfig.MutableTableServiceConfig()->SetExtractPredicateRangesLimit(4);
+ settings.SetAppConfig(appConfig);
+
+ TKikimrRunner kikimr(settings);
+ auto db = kikimr.GetTableClient();
+ auto session = db.CreateSession().GetValueSync().GetSession();
+
+ NKqp::TKqpCounters counters(kikimr.GetTestServer().GetRuntime()->GetAppData().Counters);
+
+ {
+ TAtomicBase before = counters.FullScansExecuted->GetAtomic();
+ auto result = session.ExecuteDataQuery(R"(
+ SELECT * FROM `/Root/EightShard` WHERE Key > 202 AND Key < 404 ORDER BY Key;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ AssertSuccessResult(result);
+ UNIT_ASSERT_EQUAL(before, counters.FullScansExecuted->GetAtomic());
+ }
+
+ {
+ TAtomicBase before = counters.FullScansExecuted->GetAtomic();
+ auto result = session.ExecuteDataQuery(R"(
+ SELECT COUNT(*) FROM `/Root/EightShard`;
+ )", TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ AssertSuccessResult(result);
+ UNIT_ASSERT_GT(counters.FullScansExecuted->GetAtomic(), before);
+ }
+
+
+ {
+ auto req = R"(
+ DECLARE $items AS List<Uint64>;
+ SELECT Key FROM `/Root/EightShard` where Key in $items;
+ )";
+
+ auto params1 = TParamsBuilder()
+ .AddParam("$items")
+ .BeginList()
+ .AddListItem().Uint64(0)
+ .EndList()
+ .Build()
+ .Build();
+
+ TAtomicBase before = counters.FullScansExecuted->GetAtomic();
+ auto result = session.ExecuteDataQuery(req, TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params1).ExtractValueSync();
+ result.GetIssues().PrintTo(Cerr);
+ AssertSuccessResult(result);
+ UNIT_ASSERT_EQUAL(counters.FullScansExecuted->GetAtomic(), before);
+
+ auto params2 = TParamsBuilder()
+ .AddParam("$items")
+ .BeginList()
+ .AddListItem().Uint64(0)
+ .AddListItem().Uint64(1)
+ .AddListItem().Uint64(2)
+ .AddListItem().Uint64(3)
+ .AddListItem().Uint64(4)
+ .AddListItem().Uint64(5)
+ .EndList()
+ .Build()
+ .Build();
+ before = counters.FullScansExecuted->GetAtomic();
+ auto result2 = session.ExecuteDataQuery(req, TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), params2).ExtractValueSync();
+ result2.GetIssues().PrintTo(Cerr);
+ AssertSuccessResult(result);
+ UNIT_ASSERT_GT(counters.FullScansExecuted->GetAtomic(), before);
+ }
+ }
+
+
+
}
} // namespace NKikimr::NKqp