diff options
author | kungasc <kungasc@yandex-team.com> | 2023-06-22 17:11:25 +0300 |
---|---|---|
committer | kungasc <kungasc@yandex-team.com> | 2023-06-22 17:11:25 +0300 |
commit | 6d818f0d96fee2df1b7c7e31d12442a39f81f8eb (patch) | |
tree | f8fd277a7503a8796dabb6b5feeccb930acc340e | |
parent | c3748121232aaab487cd1623b9ce7e225f904c22 (diff) | |
download | ydb-6d818f0d96fee2df1b7c7e31d12442a39f81f8eb.tar.gz |
Calculate stats by channel
-rw-r--r-- | ydb/core/protos/table_stats.proto | 8 | ||||
-rw-r--r-- | ydb/core/tablet_flat/flat_part_store.h | 22 | ||||
-rw-r--r-- | ydb/core/tablet_flat/flat_stat_part.h | 179 | ||||
-rw-r--r-- | ydb/core/tablet_flat/flat_stat_table.cpp | 23 | ||||
-rw-r--r-- | ydb/core/tablet_flat/flat_stat_table.h | 79 | ||||
-rw-r--r-- | ydb/core/tablet_flat/flat_table_part.h | 2 | ||||
-rw-r--r-- | ydb/core/tablet_flat/flat_table_part_ut.cpp | 49 | ||||
-rw-r--r-- | ydb/core/tablet_flat/test/libs/table/test_part.h | 14 | ||||
-rw-r--r-- | ydb/core/tx/datashard/datashard__monitoring.cpp | 2 | ||||
-rw-r--r-- | ydb/core/tx/datashard/datashard__stats.cpp | 7 | ||||
-rw-r--r-- | ydb/core/tx/datashard/datashard_impl.h | 26 | ||||
-rw-r--r-- | ydb/core/tx/datashard/datashard_ut_common.cpp | 14 | ||||
-rw-r--r-- | ydb/core/tx/datashard/datashard_ut_common.h | 14 | ||||
-rw-r--r-- | ydb/core/tx/datashard/datashard_ut_stats.cpp | 165 |
14 files changed, 399 insertions, 205 deletions
diff --git a/ydb/core/protos/table_stats.proto b/ydb/core/protos/table_stats.proto index 4a44614a6ca..6e45078f3d3 100644 --- a/ydb/core/protos/table_stats.proto +++ b/ydb/core/protos/table_stats.proto @@ -10,6 +10,12 @@ message THistogram { repeated THistogramBucket Buckets = 1; } +message TChannelStats { + optional uint32 Channel = 1; + optional uint64 DataSize = 2; + optional uint64 IndexSize = 3; +} + message TTableStats { optional uint64 DataSize = 1; // both inMem and ondisk optional uint64 RowCount = 2; // both inMem and ondisk @@ -47,4 +53,6 @@ message TTableStats { // i.e. this shard lent to other shards optional bool HasLoanedParts = 29; + + repeated TChannelStats Channels = 30; } diff --git a/ydb/core/tablet_flat/flat_part_store.h b/ydb/core/tablet_flat/flat_part_store.h index 2a5335d1ffe..46806e812a2 100644 --- a/ydb/core/tablet_flat/flat_part_store.h +++ b/ydb/core/tablet_flat/flat_part_store.h @@ -79,6 +79,28 @@ public: return PageCollections[groupId.Index]->PageCollection->Page(id).Size; } + ui8 GetPageChannel(NPage::TPageId id, NPage::TGroupId groupId) const override + { + Y_UNUSED(id); + Y_VERIFY(groupId.Index < PageCollections.size()); + return PageCollections[groupId.Index]->Id.Channel(); + } + + ui8 GetPageChannel(ELargeObj lob, ui64 ref) const override + { + if ((lob != ELargeObj::Extern && lob != ELargeObj::Outer) || (ref >> 32)) { + Y_Fail("Invalid ref ELargeObj{" << int(lob) << ", " << ref << "}"); + } + + if (lob == ELargeObj::Extern) { + auto bounds = Pseudo.Get()->PageCollection->Bounds(ref); + auto glob = Pseudo.Get()->PageCollection->Glob(bounds.Lo.Blob); + return glob.Logo.Channel(); + } else { + return PageCollections.at(Groups).Get()->Id.Channel(); + } + } + TIntrusiveConstPtr<TPart> CloneWithEpoch(TEpoch epoch) const override { return new TPartStore(*this, epoch); diff --git a/ydb/core/tablet_flat/flat_stat_part.h b/ydb/core/tablet_flat/flat_stat_part.h index 14a07477cec..3dd285c5cbf 100644 --- a/ydb/core/tablet_flat/flat_stat_part.h +++ b/ydb/core/tablet_flat/flat_stat_part.h @@ -11,6 +11,24 @@ namespace NKikimr { namespace NTable { +struct TPartDataSize { + ui64 Size = 0; + TVector<ui64> ByChannel = { }; + + void Add(ui64 size, ui8 channel) { + Size += size; + if (!(channel < ByChannel.size())) { + ByChannel.resize(channel + 1); + } + ByChannel[channel] += size; + } +}; + +struct TPartDataStats { + ui64 RowCount = 0; + TPartDataSize DataSize = { }; +}; + /** * Helper for calculating small blobs size between a pair of rows */ @@ -95,11 +113,17 @@ private: // Iterates over part index and calculates total row count and data size // NOTE: we don't know row count for the last page so we also ignore its size // This shouldn't be a problem for big parts with many pages -class TPartIndexIterator { +// This iterator skipps pages that are screened. Currently the logic is simple: +// if page start key is screened then we assume that the whole previous page is screened +// if page start key is not screened then the whole previous page is added to stats +class TScreenedPartIndexIterator { public: - TPartIndexIterator(TIntrusiveConstPtr<TPart> part, TIntrusiveConstPtr<TKeyCellDefaults> keys) - : Part(std::move(part)) - , KeyColumns(std::move(keys)) + TScreenedPartIndexIterator(TPartView partView, TIntrusiveConstPtr<TKeyCellDefaults> keyColumns, TIntrusiveConstPtr<NPage::TFrames> small) + : Part(std::move(partView.Part)) + , KeyColumns(std::move(keyColumns)) + , Screen(std::move(partView.Screen)) + , Small(std::move(small)) + , CurrentHole(TScreen::Iter(Screen, CurrentHoleIdx, 0, 1)) { Pos = Part->Index->Begin(); End = Part->Index->End(); @@ -117,22 +141,25 @@ public: return Pos != End; } - void Next() { + void Next(TPartDataStats& stats) { Y_VERIFY(IsValid()); auto curPageId = Pos->GetPageId(); - LastSize = CurrentSize; LastRowId = Pos->GetRowId(); - CurrentSize += GetPageSize(curPageId); ++Pos; + ui64 rowCount = IncludedRows(GetLastRowId(), GetCurrentRowId()); + stats.RowCount += rowCount; + + if (rowCount) AddPageSize(stats.DataSize, curPageId); TRowId nextRowId = Pos ? Pos->GetRowId() : Max<TRowId>(); for (auto& g : AltGroups) { while (g.Pos && g.Pos->GetRowId() < nextRowId) { // eagerly include all data up to the next row id - CurrentSize += GetPageSize(g.Pos->GetPageId(), g.GroupId); + if (rowCount) AddPageSize(stats.DataSize, g.Pos->GetPageId(), g.GroupId); ++g.Pos; } } + // Include mvcc data if (!HistoryGroups.empty()) { auto& h = HistoryGroups[0]; @@ -140,7 +167,7 @@ public: Y_VERIFY_DEBUG(hscheme.ColsKeyIdx.size() == 3); while (h.Pos && h.Pos->Cell(hscheme.ColsKeyIdx[0]).AsValue<TRowId>() < nextRowId) { // eagerly include all history up to the next row id - CurrentSize += GetPageSize(h.Pos->GetPageId(), h.GroupId); + if (rowCount) AddPageSize(stats.DataSize, h.Pos->GetPageId(), h.GroupId); ++h.Pos; } TRowId nextHistoryRowId = h.Pos ? h.Pos->GetRowId() : Max<TRowId>(); @@ -148,11 +175,16 @@ public: auto& g = HistoryGroups[index]; while (g.Pos && g.Pos->GetRowId() < nextHistoryRowId) { // eagerly include all data up to the next row id - CurrentSize += GetPageSize(g.Pos->GetPageId(), g.GroupId); + if (rowCount) AddPageSize(stats.DataSize, g.Pos->GetPageId(), g.GroupId); ++g.Pos; } } } + + if (rowCount && Small) { + AddSmallSize(stats.DataSize); + } + FillKey(); } @@ -161,6 +193,7 @@ public: return TDbTupleRef(KeyColumns->BasicTypes().data(), CurrentKey.data(), CurrentKey.size()); } +private: ui64 GetLastRowId() const { return LastRowId; } @@ -176,19 +209,14 @@ public: return LastRowId; } - ui64 GetLastDataSize() const { - return LastSize; - } - - ui64 GetCurrentDataSize() const { - return CurrentSize; - } - private: - ui64 GetPageSize(TPageId pageId, NPage::TGroupId groupId = { }) const { - return Part->GetPageSize(pageId, groupId); + void AddPageSize(TPartDataSize& stats, TPageId pageId, NPage::TGroupId groupId = { }) const { + ui64 size = Part->GetPageSize(pageId, groupId); + ui8 channel = Part->GetPageChannel(pageId, groupId); + stats.Add(size, channel); } +private: void FillKey() { CurrentKey.clear(); @@ -208,78 +236,6 @@ private: } private: - struct TGroupState { - NPage::TIndex::TIter Pos; - const NPage::TGroupId GroupId; - - TGroupState(const TPart* part, NPage::TGroupId groupId) - : Pos(part->GetGroupIndex(groupId)->Begin()) - , GroupId(groupId) - { } - }; - -private: - TIntrusiveConstPtr<TPart> Part; - TIntrusiveConstPtr<TKeyCellDefaults> KeyColumns; - NPage::TIndex::TIter Pos; - NPage::TIndex::TIter End; - TSmallVec<TCell> CurrentKey; - ui64 CurrentSize = 0; - ui64 LastRowId = 0; - ui64 LastSize = 0; - TSmallVec<TGroupState> AltGroups; - TSmallVec<TGroupState> HistoryGroups; -}; - -// This iterator skipps pages that are screened. Currently the logic is simple: -// if page start key is screened then we assume that the whole previous page is screened -// if page start key is not screened then the whole previous page is added to stats -class TScreenedPartIndexIterator { -public: - TScreenedPartIndexIterator(TPartView partView, TIntrusiveConstPtr<TKeyCellDefaults> keyColumns, - TIntrusiveConstPtr<NPage::TFrames> small) - : PartIter(partView.Part, keyColumns) - , Screen(std::move(partView.Screen)) - , Small(std::move(small)) - , CurrentHole(TScreen::Iter(Screen, CurrentHoleIdx, 0, 1)) - { - } - - bool IsValid() const { - return PartIter.IsValid(); - } - - void Next() { - Y_VERIFY(IsValid()); - - PrevRowCount = CurrentRowCount; - PrevSize = CurrentSize; - PartIter.Next(); - - ui64 rowCount = IncludedRows(PartIter.GetLastRowId(), PartIter.GetCurrentRowId()); - if (rowCount > 0) { - // We try to count rows precisely, but data is only counted per-page - CurrentRowCount += rowCount; - CurrentSize += PartIter.GetCurrentDataSize() - PartIter.GetLastDataSize(); - if (Small) { - CurrentSize += CalcSmallBytes(); - } - } - } - - TDbTupleRef GetCurrentKey() const { - return PartIter.GetCurrentKey(); - } - - ui64 GetRowCountDelta() const { - return CurrentRowCount - PrevRowCount; - } - - ui64 GetDataSizeDelta() const { - return CurrentSize - PrevSize; - } - -private: ui64 IncludedRows(TRowId beginRowId, TRowId endRowId) noexcept { if (!Screen) { // Include all rows @@ -308,37 +264,50 @@ private: return rowCount; } - ui64 CalcSmallBytes() noexcept { - ui64 bytes = 0; - - const auto row = PartIter.GetLastRowId(); - const auto end = PartIter.GetCurrentRowId(); +private: + void AddSmallSize(TPartDataSize& stats) noexcept { + const auto row = GetLastRowId(); + const auto end = GetCurrentRowId(); PrevSmallPage = Small->Lower(row, PrevSmallPage, Max<ui32>()); while (auto &rel = Small->Relation(PrevSmallPage)) { if (rel.Row < end) { - bytes += rel.Size, ++PrevSmallPage; + auto channel = Part->GetPageChannel(ELargeObj::Outer, PrevSmallPage); + stats.Add(rel.Size, channel); + ++PrevSmallPage; } else if (!rel.IsHead()) { Y_FAIL("Got unaligned NPage::TFrames head record"); } else { break; } } - - return bytes; } private: - TPartIndexIterator PartIter; + struct TGroupState { + NPage::TIndex::TIter Pos; + const NPage::TGroupId GroupId; + + TGroupState(const TPart* part, NPage::TGroupId groupId) + : Pos(part->GetGroupIndex(groupId)->Begin()) + , GroupId(groupId) + { } + }; + +private: + TIntrusiveConstPtr<TPart> Part; + TIntrusiveConstPtr<TKeyCellDefaults> KeyColumns; + NPage::TIndex::TIter Pos; + NPage::TIndex::TIter End; + TSmallVec<TCell> CurrentKey; + ui64 LastRowId = 0; + TSmallVec<TGroupState> AltGroups; + TSmallVec<TGroupState> HistoryGroups; TIntrusiveConstPtr<TScreen> Screen; TIntrusiveConstPtr<NPage::TFrames> Small; /* Inverted index for small blobs */ size_t CurrentHoleIdx = 0; TScreen::THole CurrentHole; - ui64 CurrentRowCount = 0; - ui64 PrevRowCount = 0; - ui64 CurrentSize = 0; - ui64 PrevSize = 0; ui32 PrevSmallPage = 0; }; diff --git a/ydb/core/tablet_flat/flat_stat_table.cpp b/ydb/core/tablet_flat/flat_stat_table.cpp index dfbaab9da75..b79568fa207 100644 --- a/ydb/core/tablet_flat/flat_stat_table.cpp +++ b/ydb/core/tablet_flat/flat_stat_table.cpp @@ -11,10 +11,12 @@ void BuildStats(const TSubset& subset, TStats& stats, ui64 rowCountResolution, u stats.Clear(); + TPartDataStats stIterStats = { }; TStatsIterator stIter(subset.Scheme->Keys); // Make index iterators for all parts for (auto& pi : subset.Flatten) { + stats.IndexSize.Add(pi->IndexesRawSize, pi->Label.Channel()); TAutoPtr<TScreenedPartIndexIterator> iter = new TScreenedPartIndexIterator(pi, subset.Scheme->Keys, pi->Small); if (iter->IsValid()) { stIter.Add(iter); @@ -23,12 +25,9 @@ void BuildStats(const TSubset& subset, TStats& stats, ui64 rowCountResolution, u ui64 prevRows = 0; ui64 prevSize = 0; - for (; stIter.IsValid(); stIter.Next()) { - stats.RowCount = stIter.GetCurrentRowCount(); - stats.DataSize = stIter.GetCurrentDataSize(); - - const bool nextRowsBucket = (stats.RowCount >= prevRows + rowCountResolution); - const bool nextSizeBucket = (stats.DataSize >= prevSize + dataSizeResolution); + while (stIter.Next(stIterStats)) { + const bool nextRowsBucket = (stIterStats.RowCount >= prevRows + rowCountResolution); + const bool nextSizeBucket = (stIterStats.DataSize.Size >= prevSize + dataSizeResolution); if (!nextRowsBucket && !nextSizeBucket) continue; @@ -37,18 +36,18 @@ void BuildStats(const TSubset& subset, TStats& stats, ui64 rowCountResolution, u TString serializedKey = TSerializedCellVec::Serialize(TConstArrayRef<TCell>(currentKey.Columns, currentKey.ColumnCount)); if (nextRowsBucket) { - stats.RowCountHistogram.push_back({serializedKey, stats.RowCount}); - prevRows = stats.RowCount; + prevRows = stIterStats.RowCount; + stats.RowCountHistogram.push_back({serializedKey, prevRows}); } if (nextSizeBucket) { - stats.DataSizeHistogram.push_back({serializedKey, stats.DataSize}); - prevSize = stats.DataSize; + prevSize = stIterStats.DataSize.Size; + stats.DataSizeHistogram.push_back({serializedKey, prevSize}); } } - stats.RowCount = stIter.GetCurrentRowCount(); - stats.DataSize = stIter.GetCurrentDataSize(); + stats.RowCount = stIterStats.RowCount; + stats.DataSize = std::move(stIterStats.DataSize); } void GetPartOwners(const TSubset& subset, THashSet<ui64>& partOwners) { diff --git a/ydb/core/tablet_flat/flat_stat_table.h b/ydb/core/tablet_flat/flat_stat_table.h index ce4cb408926..a8d1af2c17b 100644 --- a/ydb/core/tablet_flat/flat_stat_table.h +++ b/ydb/core/tablet_flat/flat_stat_table.h @@ -19,66 +19,51 @@ public: {} void Add(THolder<TScreenedPartIndexIterator> pi) { + Y_VERIFY(pi->IsValid()); Iterators.PushBack(std::move(pi)); TScreenedPartIndexIterator* it = Iterators.back(); - if (it->IsValid()) { - NextRowCount += it->GetRowCountDelta(); - NextDataSize += it->GetDataSizeDelta(); - Heap.push(it); - } - } - - bool IsValid() const { - return !Heap.empty() || CurrentKeyValid; + Heap.push(it); } - void Next() { - ui64 lastRowCount = RowCount; - ui64 lastDataSize = DataSize; - Y_VERIFY(IsValid()); + /** + * @return true when we haven't reached the end and have current key + * @return false when we have reached the end and don't have current key + */ + bool Next(TPartDataStats& stats) { + ui64 lastRowCount = stats.RowCount; + ui64 lastDataSize = stats.DataSize.Size; while (!Heap.empty()) { - RowCount = NextRowCount; - DataSize = NextDataSize; TScreenedPartIndexIterator* it = Heap.top(); Heap.pop(); - TDbTupleRef key = it->GetCurrentKey(); - TString serialized = TSerializedCellVec::Serialize({key.Columns, key.ColumnCount}); - CurrentKey = TSerializedCellVec(serialized); - CurrentKeyValid = true; - TDbTupleRef currentKeyTuple(KeyColumns->BasicTypes().data(), CurrentKey.GetCells().data(), CurrentKey.GetCells().size()); - if (MoveIterator(it)) + // makes key copy + TSerializedCellVec serialized = TSerializedCellVec(TSerializedCellVec::Serialize({it->GetCurrentKey().Columns, it->GetCurrentKey().ColumnCount})); + TDbTupleRef key(KeyColumns->BasicTypes().data(), serialized.GetCells().data(), serialized.GetCells().size()); + + if (MoveIterator(it, stats)) Heap.push(it); - while (!Heap.empty() && CompareKeys(currentKeyTuple, Heap.top()->GetCurrentKey()) == 0) { + // guarantees that all results will be different + while (!Heap.empty() && CompareKeys(key, Heap.top()->GetCurrentKey()) == 0) { it = Heap.top(); Heap.pop(); - if (MoveIterator(it)) + if (MoveIterator(it, stats)) Heap.push(it); } - if (RowCount != lastRowCount && DataSize != lastDataSize) { - return; + if (stats.RowCount != lastRowCount && stats.DataSize.Size != lastDataSize) { + break; } } - RowCount = NextRowCount; - DataSize = NextDataSize; - CurrentKeyValid = false; + return !Heap.empty(); } TDbTupleRef GetCurrentKey() const { - return TDbTupleRef(KeyColumns->BasicTypes().data(), CurrentKey.GetCells().data(), CurrentKey.GetCells().size()); - } - - ui64 GetCurrentRowCount() const { - return RowCount; - } - - ui64 GetCurrentDataSize() const { - return DataSize; + Y_VERIFY(!Heap.empty()); + return Heap.top()->GetCurrentKey(); } private: @@ -94,23 +79,14 @@ private: } }; - bool MoveIterator(TScreenedPartIndexIterator* it) { - it->Next(); - NextRowCount += it->GetRowCountDelta(); - NextDataSize += it->GetDataSizeDelta(); - + bool MoveIterator(TScreenedPartIndexIterator* it, TPartDataStats& stats) { + it->Next(stats); return it->IsValid(); } TIntrusiveConstPtr<TKeyCellDefaults> KeyColumns; THolderVector<TScreenedPartIndexIterator> Iterators; TPriorityQueue<TScreenedPartIndexIterator*, TSmallVec<TScreenedPartIndexIterator*>, TIterKeyGreater> Heap; - TSerializedCellVec CurrentKey; - ui64 RowCount = 0; - ui64 DataSize = 0; - ui64 NextRowCount = 0; - ui64 NextDataSize = 0; - bool CurrentKeyValid = false; }; struct TBucket { @@ -122,13 +98,15 @@ using THistogram = TVector<TBucket>; struct TStats { ui64 RowCount = 0; - ui64 DataSize = 0; + TPartDataSize DataSize = { }; + TPartDataSize IndexSize = { }; THistogram RowCountHistogram; THistogram DataSizeHistogram; void Clear() { RowCount = 0; - DataSize = 0; + DataSize = { }; + IndexSize = { }; RowCountHistogram.clear(); DataSizeHistogram.clear(); } @@ -136,6 +114,7 @@ struct TStats { void Swap(TStats& other) { std::swap(RowCount, other.RowCount); std::swap(DataSize, other.DataSize); + std::swap(IndexSize, other.IndexSize); RowCountHistogram.swap(other.RowCountHistogram); DataSizeHistogram.swap(other.DataSizeHistogram); } diff --git a/ydb/core/tablet_flat/flat_table_part.h b/ydb/core/tablet_flat/flat_table_part.h index 2d8b407cc45..589d43fb6b4 100644 --- a/ydb/core/tablet_flat/flat_table_part.h +++ b/ydb/core/tablet_flat/flat_table_part.h @@ -124,6 +124,8 @@ namespace NTable { virtual ui64 DataSize() const = 0; virtual ui64 BackingSize() const = 0; virtual ui64 GetPageSize(NPage::TPageId id, NPage::TGroupId groupId = { }) const = 0; + virtual ui8 GetPageChannel(NPage::TPageId id, NPage::TGroupId groupId = { }) const = 0; + virtual ui8 GetPageChannel(ELargeObj lob, ui64 ref) const = 0; const NPage::TIndex& GetGroupIndex(NPage::TGroupId groupId) const noexcept { if (!groupId.Historic) { diff --git a/ydb/core/tablet_flat/flat_table_part_ut.cpp b/ydb/core/tablet_flat/flat_table_part_ut.cpp index 4b83632d383..9ebcd5c8077 100644 --- a/ydb/core/tablet_flat/flat_table_part_ut.cpp +++ b/ydb/core/tablet_flat/flat_table_part_ut.cpp @@ -75,14 +75,16 @@ Y_UNIT_TEST_SUITE(TLegacy) { TIntrusiveConstPtr<TRowScheme> scheme, std::vector<ui64>& sizes) { - TPartIndexIterator idxIter(part, scheme->Keys); + TPartDataStats stats = { }; + // TScreenedPartIndexIterator without screen previously was TPartIndexIterator + TScreenedPartIndexIterator idxIter(TPartView{part, nullptr, nullptr}, scheme->Keys, nullptr); sizes.clear(); while (idxIter.IsValid()) { TDbTupleRef key = idxIter.GetCurrentKey(); - dbgOut << DbgPrintTuple(key, typeRegistry) << " " << idxIter.GetCurrentRowId() << " " << idxIter.GetCurrentDataSize() << Endl; - sizes.push_back(idxIter.GetCurrentDataSize()); - idxIter.Next(); + dbgOut << DbgPrintTuple(key, typeRegistry) << " " << stats.RowCount << " " << stats.DataSize.Size << Endl; + sizes.push_back(stats.DataSize.Size); + idxIter.Next(stats); } }; @@ -141,24 +143,17 @@ Y_UNIT_TEST_SUITE(TLegacy) { auto fnIterate = [&dbgOut, &typeRegistry] (TIntrusiveConstPtr<TPartStore> part, TIntrusiveConstPtr<TScreen> screen, TIntrusiveConstPtr<TRowScheme> scheme, TIntrusiveConstPtr<NPage::TFrames> frames) -> std::pair<ui64, ui64> { + TPartDataStats stats = { }; TScreenedPartIndexIterator idxIter(TPartView{part, screen, nullptr}, scheme->Keys, std::move(frames)); - ui64 rowCount = 0; - ui64 size = 0; while (idxIter.IsValid()) { TDbTupleRef key = idxIter.GetCurrentKey(); - rowCount += idxIter.GetRowCountDelta(); - size += idxIter.GetDataSizeDelta(); dbgOut << DbgPrintTuple(key, typeRegistry) - << " " << idxIter.GetRowCountDelta() << " " << idxIter.GetDataSizeDelta() - << " " << rowCount << " " << size << Endl; - idxIter.Next(); + << " " << stats.RowCount << " " << stats.DataSize.Size << Endl; + idxIter.Next(stats); } - rowCount += idxIter.GetRowCountDelta(); - size += idxIter.GetDataSizeDelta(); - - return {rowCount, size}; + return {stats.RowCount, stats.DataSize.Size}; }; dbgOut << "Hide all" << Endl; @@ -305,33 +300,31 @@ Y_UNIT_TEST_SUITE(TLegacy) { TScreen::THole(4200, 100000) }); + TPartDataStats stats = { }; TStatsIterator stIter(lay2.RowScheme()->Keys); stIter.Add(MakeHolder<TScreenedPartIndexIterator>(TPartView{eggs2.At(0), screen2, nullptr}, lay2.RowScheme()->Keys, nullptr)); stIter.Add(MakeHolder<TScreenedPartIndexIterator>(TPartView{eggs1.At(0), screen1, nullptr}, lay2.RowScheme()->Keys, nullptr)); - UNIT_ASSERT(stIter.IsValid()); - UNIT_ASSERT(stIter.GetCurrentRowCount() == 0); - UNIT_ASSERT(stIter.GetCurrentDataSize() == 0); - stIter.Next(); + TSerializedCellVec prevKey; ui64 prevRowCount = 0; ui64 prevDataSize = 0; - while (stIter.IsValid()) { + while (stIter.Next(stats)) { TDbTupleRef key = stIter.GetCurrentKey(); + + dbgOut << DbgPrintTuple(key, typeRegistry) + << " " << stats.RowCount << " " << stats.DataSize.Size << Endl; + UNIT_ASSERT_C(CompareTypedCellVectors(key.Columns, prevKey.GetCells().data(), key.Types, key.ColumnCount, prevKey.GetCells().size()) > 0, "Keys must be sorted"); - UNIT_ASSERT(prevRowCount < stIter.GetCurrentRowCount()); - UNIT_ASSERT(prevDataSize < stIter.GetCurrentDataSize()); - - dbgOut << DbgPrintTuple(key, typeRegistry) - << " " << stIter.GetCurrentRowCount() << " " << stIter.GetCurrentDataSize() << Endl; + UNIT_ASSERT(prevRowCount < stats.RowCount); + UNIT_ASSERT(prevDataSize < stats.DataSize.Size); prevKey = TSerializedCellVec(TSerializedCellVec::Serialize(TConstArrayRef<TCell>(key.Columns, key.ColumnCount))); - prevRowCount = stIter.GetCurrentRowCount(); - prevDataSize = stIter.GetCurrentDataSize(); - stIter.Next(); + prevRowCount = stats.RowCount; + prevDataSize = stats.DataSize.Size; } } diff --git a/ydb/core/tablet_flat/test/libs/table/test_part.h b/ydb/core/tablet_flat/test/libs/table/test_part.h index 064f17d3772..2fd42844c4d 100644 --- a/ydb/core/tablet_flat/test/libs/table/test_part.h +++ b/ydb/core/tablet_flat/test/libs/table/test_part.h @@ -51,6 +51,20 @@ namespace NTest { return Store->GetPage(groupId.Index, id)->size(); } + ui8 GetPageChannel(NPage::TPageId id, NPage::TGroupId groupId) const override + { + Y_UNUSED(id); + Y_UNUSED(groupId); + return 0; + } + + ui8 GetPageChannel(ELargeObj lob, ui64 ref) const override + { + Y_UNUSED(lob); + Y_UNUSED(ref); + return 0; + } + TIntrusiveConstPtr<NTable::TPart> CloneWithEpoch(NTable::TEpoch epoch) const override { return new TPartStore(*this, epoch); diff --git a/ydb/core/tx/datashard/datashard__monitoring.cpp b/ydb/core/tx/datashard/datashard__monitoring.cpp index a75357d7382..c3277c4122c 100644 --- a/ydb/core/tx/datashard/datashard__monitoring.cpp +++ b/ydb/core/tx/datashard/datashard__monitoring.cpp @@ -83,7 +83,7 @@ public: if (pr.second->Stats.StatsUpdateTime) { auto &stats = *rec.MutableStats(); stats.SetRowCount(pr.second->Stats.DataStats.RowCount); - stats.SetDataSize(pr.second->Stats.DataStats.DataSize); + stats.SetDataSize(pr.second->Stats.DataStats.DataSize.Size); stats.SetLastAccessTime(pr.second->Stats.AccessTime.ToStringLocalUpToSeconds()); stats.SetLastUpdateTime(pr.second->Stats.UpdateTime.ToStringLocalUpToSeconds()); stats.SetLastStatsUpdateTime(Self->LastDbStatsUpdateTime.ToStringLocalUpToSeconds()); diff --git a/ydb/core/tx/datashard/datashard__stats.cpp b/ydb/core/tx/datashard/datashard__stats.cpp index b1e3b94fc3e..4b439de0505 100644 --- a/ydb/core/tx/datashard/datashard__stats.cpp +++ b/ydb/core/tx/datashard/datashard__stats.cpp @@ -86,6 +86,7 @@ private: NTable::TSizeEnv szEnv; Subset->ColdParts.clear(); // stats won't include cold parts, if any NTable::BuildStats(*Subset, ev->Stats, RowCountResolution, DataSizeResolution, &szEnv); + Y_VERIFY_DEBUG(IndexSize == ev->Stats.IndexSize.Size); ctx.Send(ReplyTo, ev.Release()); @@ -163,7 +164,7 @@ public: return true; const NTable::TStats& stats = tableInfo.Stats.DataStats; - Result->Record.MutableTableStats()->SetDataSize(stats.DataSize + memSize); + Result->Record.MutableTableStats()->SetDataSize(stats.DataSize.Size + memSize); Result->Record.MutableTableStats()->SetRowCount(stats.RowCount + memRowCount); FillHistogram(stats.DataSizeHistogram, *Result->Record.MutableTableStats()->MutableDataSizeHistogram()); FillHistogram(stats.RowCountHistogram, *Result->Record.MutableTableStats()->MutableRowCountHistogram()); @@ -246,7 +247,7 @@ void TDataShard::Handle(TEvPrivate::TEvAsyncTableStats::TPtr& ev, const TActorCo tableInfo.Stats.MemRowCount = ev->Get()->MemRowCount; tableInfo.Stats.MemDataSize = ev->Get()->MemDataSize; - dataSize += tableInfo.Stats.DataStats.DataSize; + dataSize += tableInfo.Stats.DataStats.DataSize.Size; tableInfo.Stats.SearchHeight = ev->Get()->SearchHeight; @@ -355,7 +356,7 @@ public: const ui64 MaxBuckets = 500; if (ti.second->Stats.DataSizeResolution && - ti.second->Stats.DataStats.DataSize / ti.second->Stats.DataSizeResolution <= MaxBuckets) + ti.second->Stats.DataStats.DataSize.Size / ti.second->Stats.DataSizeResolution <= MaxBuckets) { dataSizeResolution = ti.second->Stats.DataSizeResolution; } diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h index 1940eb769b5..79d691d35ae 100644 --- a/ydb/core/tx/datashard/datashard_impl.h +++ b/ydb/core/tx/datashard/datashard_impl.h @@ -2994,9 +2994,29 @@ protected: ev->Record.SetGeneration(Executor()->Generation()); ev->Record.SetRound(StatsReportRound++); ev->Record.MutableTableStats()->SetRowCount(ti.Stats.DataStats.RowCount + ti.Stats.MemRowCount); - ev->Record.MutableTableStats()->SetDataSize(ti.Stats.DataStats.DataSize + ti.Stats.MemDataSize); - ev->Record.MutableTableStats()->SetInMemSize(ti.Stats.MemDataSize); - ev->Record.MutableTableStats()->SetIndexSize(ti.Stats.IndexSize); + + ev->Record.MutableTableStats()->SetDataSize(ti.Stats.DataStats.DataSize.Size + ti.Stats.MemDataSize); + ev->Record.MutableTableStats()->SetIndexSize(ti.Stats.DataStats.IndexSize.Size); + ev->Record.MutableTableStats()->SetInMemSize(ti.Stats.MemDataSize); + + TMap<ui8, std::tuple<ui64, ui64>> channels; // Channel -> (DataSize, IndexSize) + for (size_t channel = 0; channel < ti.Stats.DataStats.DataSize.ByChannel.size(); channel++) { + if (ti.Stats.DataStats.DataSize.ByChannel[channel]) { + std::get<0>(channels[channel]) = ti.Stats.DataStats.DataSize.ByChannel[channel]; + } + } + for (size_t channel = 0; channel < ti.Stats.DataStats.IndexSize.ByChannel.size(); channel++) { + if (ti.Stats.DataStats.IndexSize.ByChannel[channel]) { + std::get<1>(channels[channel]) = ti.Stats.DataStats.IndexSize.ByChannel[channel]; + } + } + for (auto p : channels) { + auto item = ev->Record.MutableTableStats()->AddChannels(); + item->SetChannel(p.first); + item->SetDataSize(std::get<0>(p.second)); + item->SetIndexSize(std::get<1>(p.second)); + } + ev->Record.MutableTableStats()->SetLastAccessTime(ti.Stats.AccessTime.MilliSeconds()); ev->Record.MutableTableStats()->SetLastUpdateTime(ti.Stats.UpdateTime.MilliSeconds()); diff --git a/ydb/core/tx/datashard/datashard_ut_common.cpp b/ydb/core/tx/datashard/datashard_ut_common.cpp index bc446d86443..7ae46deea40 100644 --- a/ydb/core/tx/datashard/datashard_ut_common.cpp +++ b/ydb/core/tx/datashard/datashard_ut_common.cpp @@ -1087,6 +1087,11 @@ static ui64 RunSchemeTx( runtime.Send(new IEventHandle(MakeTxProxyID(), sender, request.Release()), 0, viaActorSystem); auto ev = runtime.GrabEdgeEventRethrow<TEvTxUserProxy::TEvProposeTransactionStatus>(sender); + + for (auto i : ev->Get()->Record.GetIssues()) { + Cerr << "Issue: " << i.AsJSON() << Endl; + } + UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetStatus(), expectedStatus); return ev->Get()->Record.GetTxId(); @@ -1123,6 +1128,15 @@ void CreateShardedTable( if (column.IsKey) { desc->AddKeyColumnNames(column.Name); } + col->SetFamilyName(column.Family); + } + + for (const auto& family : opts.Families_) { + auto fam = desc->MutablePartitionConfig()->AddColumnFamilies(); + if (family.Name) fam->SetName(family.Name); + if (family.LogPoolKind) fam->MutableStorageConfig()->MutableLog()->SetPreferredPoolKind(family.LogPoolKind); + if (family.SysLogPoolKind) fam->MutableStorageConfig()->MutableSysLog()->SetPreferredPoolKind(family.SysLogPoolKind); + if (family.DataPoolKind) fam->MutableStorageConfig()->MutableData()->SetPreferredPoolKind(family.DataPoolKind); } for (const auto& index : opts.Indexes_) { diff --git a/ydb/core/tx/datashard/datashard_ut_common.h b/ydb/core/tx/datashard/datashard_ut_common.h index 7b407f87e7a..1aad79ce936 100644 --- a/ydb/core/tx/datashard/datashard_ut_common.h +++ b/ydb/core/tx/datashard/datashard_ut_common.h @@ -405,16 +405,18 @@ struct TShardedTableOptions { using TSelf = TShardedTableOptions; struct TColumn { - TColumn(const TString& name, const TString& type, bool isKey, bool notNull) + TColumn(const TString& name, const TString& type, bool isKey, bool notNull, TString family = {}) : Name(name) , Type(type) , IsKey(isKey) - , NotNull(notNull) {} + , NotNull(notNull) + , Family(family) {} TString Name; TString Type; bool IsKey; bool NotNull; + TString Family; }; struct TIndex { @@ -439,6 +441,13 @@ struct TShardedTableOptions { TMaybe<TString> AwsRegion; }; + struct TFamily { + TString Name; + TString LogPoolKind; + TString SysLogPoolKind; + TString DataPoolKind; + }; + using TAttributes = THashMap<TString, TString>; #define TABLE_OPTION_IMPL(type, name, defaultValue) \ @@ -456,6 +465,7 @@ struct TShardedTableOptions { TABLE_OPTION(EShadowDataMode, ShadowData, EShadowDataMode::Default); TABLE_OPTION(TVector<TColumn>, Columns, (TVector<TColumn>{{"key", "Uint32", true, false}, {"value", "Uint32", false, false}})); TABLE_OPTION(TVector<TIndex>, Indexes, {}); + TABLE_OPTION(TVector<TFamily>, Families, {}); TABLE_OPTION(ui64, Followers, 0); TABLE_OPTION(bool, FollowerPromotion, false); TABLE_OPTION(bool, ExternalStorage, false); diff --git a/ydb/core/tx/datashard/datashard_ut_stats.cpp b/ydb/core/tx/datashard/datashard_ut_stats.cpp index c12ddd7fe4e..e55dacc2d91 100644 --- a/ydb/core/tx/datashard/datashard_ut_stats.cpp +++ b/ydb/core/tx/datashard/datashard_ut_stats.cpp @@ -53,7 +53,26 @@ Y_UNIT_TEST_SUITE(DataShardStats) { return ev->Get()->Record; } - Y_UNIT_TEST(SmallStatsNotLostOnCompaction) { + NKikimrTableStats::TTableStats GetTableStats(TTestActorRuntime& runtime, ui64 tabletId, ui64 tableId) { + auto sender = runtime.AllocateEdgeActor(); + auto request = MakeHolder<TEvDataShard::TEvGetTableStats>(tableId); + runtime.SendToPipe(tabletId, sender, request.Release(), 0, GetPipeConfigWithRetries()); + + auto ev = runtime.GrabEdgeEventRethrow<TEvDataShard::TEvGetTableStatsResult>(sender); + return ev->Get()->Record.GetTableStats(); + } + + TVector<std::pair<ui64, ui64>> ReadHistogram(NKikimrTableStats::THistogram histogram) { + TVector<std::pair<ui64, ui64>> result; + for (auto b : histogram.GetBuckets()) { + TSerializedCellVec key(b.GetKey()); + auto keyValue = key.GetCells()[0].AsValue<ui32>(); + result.push_back({keyValue, b.GetValue()}); + } + return result; + } + + Y_UNIT_TEST(OneChannelStatsCorrect) { TPortManager pm; TServerSettings serverSettings(pm.GetPort(2134)); serverSettings.SetDomainName("Root") @@ -93,9 +112,153 @@ Y_UNIT_TEST_SUITE(DataShardStats) { UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetRowCount(), 3u); UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetPartCount(), 1u); UNIT_ASSERT_GT(stats.GetTableStats().GetDataSize(), 0u); + + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetDataSize(), + stats.GetTableStats().GetDataSize() - stats.GetTableStats().GetInMemSize()); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetIndexSize(), + stats.GetTableStats().GetIndexSize()); + } + } + + Y_UNIT_TEST(MultipleChannelsStatsCorrect) { + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings.SetDomainName("Root") + .SetUseRealThreads(false) + .AddStoragePool("ssd") + .AddStoragePool("hdd"); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); + + InitRoot(server, sender); + + auto opts = TShardedTableOptions() + .Shards(1) + .Columns({{"key", "Uint32", true, false}, {"value", "Uint32", false, false}, {"value2", "Uint32", false, false, "hdd"}}) + .Families({{.Name = "default", .LogPoolKind = "ssd", .SysLogPoolKind = "ssd", .DataPoolKind = "ssd"}, {.Name = "hdd", .DataPoolKind = "hdd"}}); + CreateShardedTable(server, sender, "/Root", "table-1", opts); + + auto shards = GetTableShards(server, sender, "/Root/table-1"); + UNIT_ASSERT_VALUES_EQUAL(shards.size(), 1u); + + ExecSQL(server, sender, "UPSERT INTO `/Root/table-1` (key, value, value2) VALUES (1, 1, 1), (2, 2, 2), (3, 3, 3)"); + + TPathId pathId; + { + Cerr << "... waiting for stats after upsert" << Endl; + auto stats = WaitTableStats(runtime); + UNIT_ASSERT_VALUES_EQUAL(stats.GetDatashardId(), shards.at(0)); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetRowCount(), 3u); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetPartCount(), 0u); + UNIT_ASSERT_GT(stats.GetTableStats().GetDataSize(), 0u); + pathId = TPathId(stats.GetTableOwnerId(), stats.GetTableLocalId()); + } + + CompactTable(runtime, shards.at(0), pathId); + + { + Cerr << "... waiting for stats after compaction" << Endl; + auto stats = WaitTableStats(runtime, /* minPartCount */ 1); + UNIT_ASSERT_VALUES_EQUAL(stats.GetDatashardId(), shards.at(0)); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetRowCount(), 3u); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetPartCount(), 1u); + UNIT_ASSERT_GT(stats.GetTableStats().GetDataSize(), 0u); + + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels().size(), 2); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetChannel(), 1u); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[1].GetChannel(), 2u); + + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetDataSize() + stats.GetTableStats().GetChannels()[1].GetDataSize(), + stats.GetTableStats().GetDataSize() - stats.GetTableStats().GetInMemSize()); + UNIT_ASSERT_GT(stats.GetTableStats().GetChannels()[0].GetDataSize(), 0u); + UNIT_ASSERT_GT(stats.GetTableStats().GetChannels()[1].GetDataSize(), 0u); + + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetIndexSize(), stats.GetTableStats().GetIndexSize()); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[1].GetIndexSize(), 0); } } + Y_UNIT_TEST(HistogramStatsCorrect) { + const auto gDbStatsDataSizeResolutionBefore = NDataShard::gDbStatsDataSizeResolution; + const auto gDbStatsRowCountResolutionBefore = NDataShard::gDbStatsRowCountResolution; + NDataShard::gDbStatsDataSizeResolution = 1; // by page stats + NDataShard::gDbStatsRowCountResolution = 1; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings.SetDomainName("Root") + .SetUseRealThreads(false); + + TServer::TPtr server = new TServer(serverSettings); + auto& runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); + + InitRoot(server, sender); + + CreateShardedTable(server, sender, "/Root", "table-1", 1); + auto shards = GetTableShards(server, sender, "/Root/table-1"); + UNIT_ASSERT_VALUES_EQUAL(shards.size(), 1u); + + const int count = 2000; + TString query = "UPSERT INTO `/Root/table-1` (key, value) VALUES "; + for (auto times = 0; times < count; times++) { + if (times != 0) + query += ", "; + query += "(" + ToString(times) + ", " + ToString(times) + ") "; + } + ExecSQL(server, sender, query); + + TPathId pathId; + { + Cerr << "... waiting for stats after upsert" << Endl; + auto stats = WaitTableStats(runtime); + UNIT_ASSERT_VALUES_EQUAL(stats.GetDatashardId(), shards.at(0)); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetRowCount(), count); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetPartCount(), 0u); + UNIT_ASSERT_GT(stats.GetTableStats().GetDataSize(), 0u); + pathId = TPathId(stats.GetTableOwnerId(), stats.GetTableLocalId()); + } + + CompactTable(runtime, shards.at(0), pathId); + + { + Cerr << "... waiting for stats after compaction" << Endl; + auto stats = WaitTableStats(runtime, /* minPartCount */ 1); + UNIT_ASSERT_VALUES_EQUAL(stats.GetDatashardId(), shards.at(0)); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetRowCount(), count); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetPartCount(), 1u); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetDataSize(), 30100); + + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetDataSize(), + stats.GetTableStats().GetDataSize() - stats.GetTableStats().GetInMemSize()); + UNIT_ASSERT_VALUES_EQUAL(stats.GetTableStats().GetChannels()[0].GetIndexSize(), + stats.GetTableStats().GetIndexSize()); + } + + { + auto stats = GetTableStats(runtime, shards.at(0), pathId.LocalPathId); + + auto dataSizeHistogram = ReadHistogram(stats.GetDataSizeHistogram()); + TVector<std::pair<ui64, ui64>> expectedDataSizeHistogram = {{475, 7145}, {950, 14290}, {1425, 21435}, {1900, 28580}}; + UNIT_ASSERT_VALUES_EQUAL(expectedDataSizeHistogram, dataSizeHistogram); + + auto rowCountHistogram = ReadHistogram(stats.GetRowCountHistogram()); + TVector<std::pair<ui64, ui64>> expectedRowCountHistogram = {{475, 475}, {950, 950}, {1425, 1425}, {1900, 1900}}; + UNIT_ASSERT_VALUES_EQUAL(expectedRowCountHistogram, rowCountHistogram); + } + + NDataShard::gDbStatsDataSizeResolution = gDbStatsDataSizeResolutionBefore; + NDataShard::gDbStatsRowCountResolution = gDbStatsRowCountResolutionBefore; + } + } // Y_UNIT_TEST_SUITE(DataShardStats) } // namespace NKikimr |