aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorkungasc <kungasc@yandex-team.com>2023-09-14 16:28:10 +0300
committerkungasc <kungasc@yandex-team.com>2023-09-14 16:49:53 +0300
commitaab08a6fe3bcdbe14fa7e61c38825fecd152b7a0 (patch)
tree491e7dd291de6006601631df15eb40822629c182
parent9cf080f80a962a389ec2a7f260b85a888674d5a5 (diff)
downloadydb-aab08a6fe3bcdbe14fa7e61c38825fecd152b7a0.tar.gz
KIKIMR-19139 Get index pages from Env
-rw-r--r--ydb/core/tablet_flat/flat_fwd_blobs.h2
-rw-r--r--ydb/core/tablet_flat/flat_fwd_cache.h2
-rw-r--r--ydb/core/tablet_flat/flat_fwd_env.h62
-rw-r--r--ydb/core/tablet_flat/flat_fwd_iface.h5
-rw-r--r--ydb/core/tablet_flat/flat_part_index_iter.h14
-rw-r--r--ydb/core/tablet_flat/flat_part_iter_multi.h126
-rw-r--r--ydb/core/tablet_flat/flat_part_loader.cpp7
-rw-r--r--ydb/core/tablet_flat/flat_table_part.h33
-rw-r--r--ydb/core/tablet_flat/test/libs/table/test_envs.h50
-rw-r--r--ydb/core/tablet_flat/test/libs/table/test_store.h5
-rw-r--r--ydb/core/tablet_flat/test/libs/table/test_writer.h10
-rw-r--r--ydb/core/tablet_flat/ut/ut_charge.cpp5
-rw-r--r--ydb/core/tablet_flat/ut/ut_comp_shard.cpp39
-rw-r--r--ydb/core/tablet_flat/ut/ut_forward.cpp2
-rw-r--r--ydb/core/tablet_flat/ut/ut_part.cpp5
15 files changed, 266 insertions, 101 deletions
diff --git a/ydb/core/tablet_flat/flat_fwd_blobs.h b/ydb/core/tablet_flat/flat_fwd_blobs.h
index 583ba48ee8..4794e8eb15 100644
--- a/ydb/core/tablet_flat/flat_fwd_blobs.h
+++ b/ydb/core/tablet_flat/flat_fwd_blobs.h
@@ -149,7 +149,7 @@ namespace NFwd {
if (!Tags.at(page.Tag) || page.Size >= Edge.at(page.Tag) || !Filter.Has(rel.Row)) {
/* Page doesn't fits to load criteria */
} else if (page.Fetch == EFetch::None) {
- auto size = head->AddToQueue(Grow, ui16(EPage::Opaque));
+ auto size = head->AddToQueue(Grow, EPage::Opaque);
Y_VERIFY(size == page.Size, "Inconsistent page sizez");
diff --git a/ydb/core/tablet_flat/flat_fwd_cache.h b/ydb/core/tablet_flat/flat_fwd_cache.h
index f5ff3f5033..25bf7c74bb 100644
--- a/ydb/core/tablet_flat/flat_fwd_cache.h
+++ b/ydb/core/tablet_flat/flat_fwd_cache.h
@@ -120,7 +120,7 @@ namespace NFwd {
};
while (auto more = Index.More(until())) {
- auto size = head->AddToQueue(more, ui16(EPage::DataPage));
+ auto size = head->AddToQueue(more, EPage::DataPage);
Stat.Fetch += size;
OnFetch += size;
diff --git a/ydb/core/tablet_flat/flat_fwd_env.h b/ydb/core/tablet_flat/flat_fwd_env.h
index 93c17490f6..5f92861f67 100644
--- a/ydb/core/tablet_flat/flat_fwd_env.h
+++ b/ydb/core/tablet_flat/flat_fwd_env.h
@@ -40,7 +40,7 @@ namespace NFwd {
return PageLoadingLogic.Get();
}
- ui64 AddToQueue(TPageId pageId, ui16 type) noexcept override
+ ui64 AddToQueue(TPageId pageId, EPage type) noexcept override
{
if (!Fetch) {
Fetch.Reset(new TFetch(Cookie, PageCollection, { }));
@@ -49,7 +49,7 @@ namespace NFwd {
const auto meta = PageCollection->Page(pageId);
- if (meta.Type != type || meta.Size == 0)
+ if (meta.Type != ui16(type) || meta.Size == 0)
Y_FAIL("Got a non-data page while part index traverse");
Fetch->Pages.emplace_back(pageId);
@@ -75,9 +75,11 @@ namespace NFwd {
TIntrusiveConstPtr<IPageCollection> PageCollection;
};
- public:
- using TData = const TSharedData*;
+ struct TSimpleEnv {
+ TMap<TPageId, NPageCollection::TLoadedPage> Pages;
+ };
+ public:
TEnv(const TConf &conf, const TSubset &subset)
: Salt(RandomNumber<ui32>())
, Conf(conf)
@@ -106,10 +108,16 @@ namespace NFwd {
return Pending == 0;
}
- TData TryGetPage(const TPart* part, TPageId ref, TGroupId groupId) override
+ const TSharedData* TryGetPage(const TPart* part, TPageId ref, TGroupId groupId) override
{
ui16 room = (groupId.Historic ? part->Groups + 2 : 0) + groupId.Index;
- return Handle(GetQueue(part, room), ref).Page;
+ TSlot slot = GetQueueSlot(part, room);
+
+ if (part->IndexPages.Has(groupId, ref)) {
+ return TryGetIndexPage(slot, ref);
+ }
+
+ return Handle(Queues.at(slot), ref).Page;
}
TResult Locate(const TMemTable *memTable, ui64 ref, ui32 tag) noexcept override
@@ -150,7 +158,18 @@ namespace NFwd {
"Page fwd cache got more pages than was requested");
Pending -= pages.size();
- Queues.at(part)->Apply(pages);
+
+ TVector<NPageCollection::TLoadedPage> queuePages(Reserve(pages.size()));
+ for (auto& page : pages) {
+ const auto &meta = pageCollection->Page(page.PageId);
+ if (NTable::EPage(meta.Type) == EPage::Index) {
+ IndexPages.at(part).Pages[page.PageId] = page;
+ } else {
+ queuePages.push_back(page);
+ }
+ }
+
+ Queues.at(part)->Apply(queuePages);
}
IPages* Reset() noexcept
@@ -168,6 +187,7 @@ namespace NFwd {
Total = Stats();
Parts.clear();
Queues.clear();
+ IndexPages.clear();
ColdParts.clear();
for (auto &one : Subset.Flatten)
@@ -239,6 +259,23 @@ namespace NFwd {
}
private:
+ const TSharedData* TryGetIndexPage(TSlot slot, TPageId pageId) noexcept
+ {
+ // TODO: count index pages in Stats later
+
+ auto &env = IndexPages.at(slot);
+ auto pageIt = env.Pages.find(pageId);
+
+ if (pageIt != env.Pages.end()) {
+ return &pageIt->second.Data;
+ } else {
+ auto &queue = Queues.at(slot);
+ queue.AddToQueue(pageId, EPage::Index);
+ Queue.PushBack(&queue);
+ return nullptr;
+ }
+ }
+
TResult Handle(TPageLoadingQueue &q, TPageId ref) noexcept
{
auto got = q->Handle(&q, ref, Conf.AheadLo);
@@ -277,7 +314,7 @@ namespace NFwd {
"Cannot handle multiple parts on the same page collection");
}
- TPageLoadingQueue& GetQueue(const TPart *part, ui16 room) noexcept
+ TSlot GetQueueSlot(const TPart *part, ui16 room) noexcept
{
auto it = Parts.find(part);
Y_VERIFY(it != Parts.end(),
@@ -289,7 +326,12 @@ namespace NFwd {
Y_VERIFY(Queues.at(it->second[0]).PageCollection->Label() == part->Label,
"Cannot handle multiple parts on the same page collection");
- return Queues.at(it->second[room]);
+ return it->second[room];
+ }
+
+ TPageLoadingQueue& GetQueue(const TPart *part, ui16 room) noexcept
+ {
+ return Queues.at(GetQueueSlot(part, room));
}
TSlot Settle(TEgg egg, ui32 slot) noexcept
@@ -298,6 +340,7 @@ namespace NFwd {
const ui64 cookie = (ui64(Queues.size()) << 32) | ui32(Salt + Epoch);
Queues.emplace_back(slot, cookie, std::move(egg.PageCollection), egg.PageLoadingLogic);
+ IndexPages.emplace_back();
return Queues.size() - 1;
} else {
@@ -373,6 +416,7 @@ namespace NFwd {
const TVector<ui32> Keys; /* Tags to expand ELargeObj references */
TDeque<TPageLoadingQueue> Queues;
+ TDeque<TSimpleEnv> IndexPages;
THashMap<const TPart*, TSlotVec> Parts;
THashSet<const TPart*> ColdParts;
// Wrapper for memable blobs
diff --git a/ydb/core/tablet_flat/flat_fwd_iface.h b/ydb/core/tablet_flat/flat_fwd_iface.h
index 460e0412b5..541c987843 100644
--- a/ydb/core/tablet_flat/flat_fwd_iface.h
+++ b/ydb/core/tablet_flat/flat_fwd_iface.h
@@ -1,10 +1,13 @@
#pragma once
+#include "flat_page_iface.h"
#include "flat_sausage_fetch.h"
#include "flat_fwd_misc.h"
namespace NKikimr {
namespace NTable {
+ using EPage = NPage::EPage;
+
namespace NFwd {
struct TPage;
@@ -13,7 +16,7 @@ namespace NFwd {
public:
virtual ~IPageLoadingQueue() = default;
- virtual ui64 AddToQueue(ui32 page, ui16 type) noexcept = 0;
+ virtual ui64 AddToQueue(ui32 page, EPage type) noexcept = 0;
};
class IPageLoadingLogic {
diff --git a/ydb/core/tablet_flat/flat_part_index_iter.h b/ydb/core/tablet_flat/flat_part_index_iter.h
index eeedefb6fe..979763cf45 100644
--- a/ydb/core/tablet_flat/flat_part_index_iter.h
+++ b/ydb/core/tablet_flat/flat_part_index_iter.h
@@ -15,8 +15,9 @@ public:
using TIter = NPage::TIndex::TIter;
using TGroupId = NPage::TGroupId;
- TPartIndexIt(const TPart* part, TGroupId groupId)
+ TPartIndexIt(const TPart* part, IPages* env, TGroupId groupId)
: Part(part)
+ , Env(env)
, GroupId(groupId)
, EndRowId(part->GetGroupIndex(groupId).GetEndRowId())
{ }
@@ -112,13 +113,18 @@ private:
if (Index) {
return &*Index;
}
- // TODO: get index from Env
- Index = Part->GetGroupIndex(GroupId);
- return &*Index;
+ auto pageId = GroupId.IsHistoric() ? Part->IndexPages.Historic[GroupId.Index] : Part->IndexPages.Groups[GroupId.Index];
+ auto page = Env->TryGetPage(Part, pageId);
+ if (page) {
+ Index = TIndex(*page);
+ return &*Index;
+ }
+ return { };
}
private:
const TPart* const Part;
+ IPages* const Env;
const TGroupId GroupId;
std::optional<TIndex> Index;
TIter Iter;
diff --git a/ydb/core/tablet_flat/flat_part_iter_multi.h b/ydb/core/tablet_flat/flat_part_iter_multi.h
index 3b733124dc..0e8c92ff85 100644
--- a/ydb/core/tablet_flat/flat_part_iter_multi.h
+++ b/ydb/core/tablet_flat/flat_part_iter_multi.h
@@ -17,13 +17,15 @@ namespace NTable {
*/
class TPartGroupRowIt {
public:
- TPartGroupRowIt(const TPart* part, NPage::TGroupId groupId)
- : GroupId(groupId)
- , Index(part, groupId)
+ TPartGroupRowIt(const TPart* part, IPages* env, NPage::TGroupId groupId)
+ : Part(part)
+ , Env(env)
+ , GroupId(groupId)
+ , Index(part, env, groupId)
{
}
- EReady Seek(TRowId rowId, const TPart* part, IPages* env) noexcept {
+ EReady Seek(TRowId rowId) noexcept {
// Fast path, check if we already have the needed data page
if (Data) {
TRowId minRowId = Page.BaseRow();
@@ -49,7 +51,7 @@ namespace NTable {
}
// Make sure we have the correct data page loaded
- if (!LoadPage(Index.GetPageId(), Index.GetRowId(), part, env)) {
+ if (!LoadPage(Index.GetPageId(), Index.GetRowId())) {
return EReady::Page;
}
Y_VERIFY_DEBUG(Page.BaseRow() <= rowId, "Index and row have an unexpected relation");
@@ -73,11 +75,11 @@ namespace NTable {
}
protected:
- bool LoadPage(TPageId pageId, TRowId baseRow, const TPart* part, IPages* env) noexcept
+ bool LoadPage(TPageId pageId, TRowId baseRow) noexcept
{
if (PageId != pageId) {
Data = { };
- if (!Page.Set(env->TryGetPage(part, pageId, GroupId))) {
+ if (!Page.Set(Env->TryGetPage(Part, pageId, GroupId))) {
PageId = Max<TPageId>();
return false;
}
@@ -88,10 +90,13 @@ namespace NTable {
}
protected:
+ const TPart* const Part;
+ IPages* const Env;
const NPage::TGroupId GroupId;
+
TPageId PageId = Max<TPageId>();
- TPartIndexIt Index;
+ TPartIndexIt Index;
NPage::TDataPage Page;
NPage::TDataPage::TIter Data;
};
@@ -105,8 +110,8 @@ namespace NTable {
public:
using TCells = NPage::TCells;
- TPartGroupKeyIt(const TPart* part, NPage::TGroupId groupId)
- : TPartGroupRowIt(part, groupId)
+ TPartGroupKeyIt(const TPart* part, IPages* env, NPage::TGroupId groupId)
+ : TPartGroupRowIt(part, env, groupId)
{
ClearBounds();
}
@@ -129,7 +134,6 @@ namespace NTable {
EReady Seek(
const TCells key, ESeek seek,
- const TPart* part, IPages* env,
const TPartScheme::TGroupInfo& scheme, const TKeyCellDefaults* keyDefaults) noexcept
{
Y_VERIFY_DEBUG(seek == ESeek::Exact || seek == ESeek::Lower || seek == ESeek::Upper,
@@ -158,7 +162,7 @@ namespace NTable {
}
}
- if (!LoadPage(Index.GetPageId(), Index.GetRowId(), part, env)) {
+ if (!LoadPage(Index.GetPageId(), Index.GetRowId())) {
// Exact RowId unknown, don't allow Next until another Seek
return Terminate(EReady::Page);
}
@@ -199,7 +203,7 @@ namespace NTable {
RowId, BeginRowId);
if (RowId < EndRowId) {
- return Next(part, env);
+ return Next();
}
}
@@ -208,7 +212,6 @@ namespace NTable {
EReady SeekReverse(
const TCells key, ESeek seek,
- const TPart* part, IPages* env,
const TPartScheme::TGroupInfo& scheme, const TKeyCellDefaults* keyDefaults) noexcept
{
Y_VERIFY_DEBUG(seek == ESeek::Exact || seek == ESeek::Lower || seek == ESeek::Upper,
@@ -233,7 +236,7 @@ namespace NTable {
}
}
- if (!LoadPage(Index.GetPageId(), Index.GetRowId(), part, env)) {
+ if (!LoadPage(Index.GetPageId(), Index.GetRowId())) {
// Exact RowId unknown, don't allow Next until another Seek
return Terminate(EReady::Page);
}
@@ -269,14 +272,14 @@ namespace NTable {
"Unexpected prev page RowId=%lu (EndRowId=%lu)",
RowId, EndRowId);
if (RowId >= BeginRowId) {
- return Prev(part, env);
+ return Prev();
}
}
return Exhausted();
}
- EReady Seek(TRowId rowId, const TPart* part, IPages* env) noexcept
+ EReady Seek(TRowId rowId) noexcept
{
if (Y_UNLIKELY(rowId < BeginRowId || rowId >= EndRowId)) {
return Exhausted();
@@ -289,15 +292,15 @@ namespace NTable {
RowId = rowId;
Data = { };
- return Next(part, env);
+ return Next();
}
- EReady SeekToStart(const TPart* part, IPages* env) noexcept
+ EReady SeekToStart() noexcept
{
- return Seek(BeginRowId, part, env);
+ return Seek(BeginRowId);
}
- EReady SeekReverse(TRowId rowId, const TPart* part, IPages* env) noexcept
+ EReady SeekReverse(TRowId rowId) noexcept
{
if (Y_UNLIKELY(rowId < BeginRowId || rowId >= EndRowId)) {
return Exhausted();
@@ -310,15 +313,15 @@ namespace NTable {
RowId = rowId;
Data = { };
- return Prev(part, env);
+ return Prev();
}
- EReady SeekToEnd(const TPart* part, IPages* env) noexcept
+ EReady SeekToEnd() noexcept
{
- return SeekReverse(EndRowId - 1, part, env);
+ return SeekReverse(EndRowId - 1);
}
- EReady Next(const TPart* part, IPages* env) noexcept
+ EReady Next() noexcept
{
if (Y_UNLIKELY(RowId == Max<TRowId>())) {
return EReady::Gone;
@@ -345,7 +348,7 @@ namespace NTable {
Y_VERIFY_DEBUG(Index.IsValid() && Index.GetRowId() <= RowId,
"Next called without a valid index record");
- if (!LoadPage(Index.GetPageId(), Index.GetRowId(), part, env)) {
+ if (!LoadPage(Index.GetPageId(), Index.GetRowId())) {
return EReady::Page;
}
@@ -357,7 +360,7 @@ namespace NTable {
return Exhausted();
}
- EReady Prev(const TPart* part, IPages* env) noexcept
+ EReady Prev() noexcept
{
if (Y_UNLIKELY(RowId == Max<TRowId>())) {
return EReady::Gone;
@@ -386,7 +389,7 @@ namespace NTable {
Y_VERIFY_DEBUG(Index.IsValid() && Index.GetRowId() <= RowId,
"Prev called without a valid index record");
- if (!LoadPage(Index.GetPageId(), Index.GetRowId(), part, env)) {
+ if (!LoadPage(Index.GetPageId(), Index.GetRowId())) {
return EReady::Page;
}
@@ -443,13 +446,12 @@ namespace NTable {
public:
using TCells = NPage::TCells;
- explicit TPartGroupHistoryIt(const TPart* part)
- : TPartGroupRowIt(part, NPage::TGroupId(0, /* historic */ true))
+ explicit TPartGroupHistoryIt(const TPart* part, IPages* env)
+ : TPartGroupRowIt(part, env, NPage::TGroupId(0, /* historic */ true))
{ }
EReady Seek(
- TRowId rowId, const TRowVersion& rowVersion,
- const TPart* part, IPages* env) noexcept
+ TRowId rowId, const TRowVersion& rowVersion) noexcept
{
Y_VERIFY_DEBUG(rowId != Max<TRowId>());
@@ -466,12 +468,12 @@ namespace NTable {
TCells key{ keyCells, 3 };
// Directly use the history group scheme
- const auto& scheme = part->Scheme->HistoryGroup;
+ const auto& scheme = Part->Scheme->HistoryGroup;
Y_VERIFY_DEBUG(scheme.ColsKeyIdx.size() == 3);
Y_VERIFY_DEBUG(scheme.ColsKeyData.size() == 3);
// Directly use the histroy key keyDefaults with correct sort order
- const TKeyCellDefaults* keyDefaults = part->Scheme->HistoryKeys.Get();
+ const TKeyCellDefaults* keyDefaults = Part->Scheme->HistoryKeys.Get();
// Helper for loading row id and row version from the index
auto checkIndex = [&]() -> bool {
@@ -570,7 +572,7 @@ namespace NTable {
Y_VERIFY_DEBUG(Index.IsValid());
Y_VERIFY_DEBUG(!Data);
- if (!LoadPage(Index.GetPageId(), Index.GetRowId(), part, env)) {
+ if (!LoadPage(Index.GetPageId(), Index.GetRowId())) {
return EReady::Page;
}
@@ -593,7 +595,7 @@ namespace NTable {
return Exhausted();
}
- if (!LoadPage(Index.GetPageId(), Index.GetRowId(), part, env)) {
+ if (!LoadPage(Index.GetPageId(), Index.GetRowId())) {
// It's ok to repeat binary search on the next iteration,
// since page faults take a long time and optimizing it
// wouldn't be worth it.
@@ -632,7 +634,7 @@ namespace NTable {
// an index search the first row must be the one we want.
Y_VERIFY(RowVersion <= rowVersion, "Index binary search bug");
- if (!LoadPage(Index.GetPageId(), Index.GetRowId(), part, env)) {
+ if (!LoadPage(Index.GetPageId(), Index.GetRowId())) {
// We don't want to repeat binary search on the next
// iteration, as we already know the row is not on a
// previous page, but index search would point to it
@@ -701,7 +703,7 @@ namespace NTable {
, Env(env)
, Pinout(Part->Scheme->MakePinout(tags))
, KeyCellDefaults(std::move(keyDefaults))
- , Main(Part, TGroupId(0))
+ , Main(Part, Env, TGroupId(0))
, SkipMainDeltas(0)
, SkipMainVersion(false)
, SkipEraseVersion(false)
@@ -712,7 +714,7 @@ namespace NTable {
for (size_t idx : xrange(Pinout.AltGroups().size())) {
ui32 group = Pinout.AltGroups()[idx];
TGroupId groupId(group);
- Groups.emplace_back(Part, groupId);
+ Groups.emplace_back(Part, Env, groupId);
GroupRemap[group] = idx;
}
@@ -743,49 +745,49 @@ namespace NTable {
EReady Seek(const TCells key, ESeek seek) noexcept
{
ClearKey();
- return Main.Seek(key, seek, Part, Env, Part->Scheme->Groups[0], &*KeyCellDefaults);
+ return Main.Seek(key, seek, Part->Scheme->Groups[0], &*KeyCellDefaults);
}
EReady SeekReverse(const TCells key, ESeek seek) noexcept
{
ClearKey();
- return Main.SeekReverse(key, seek, Part, Env, Part->Scheme->Groups[0], &*KeyCellDefaults);
+ return Main.SeekReverse(key, seek, Part->Scheme->Groups[0], &*KeyCellDefaults);
}
EReady Seek(TRowId rowId) noexcept
{
ClearKey();
- return Main.Seek(rowId, Part, Env);
+ return Main.Seek(rowId);
}
EReady SeekToStart() noexcept
{
ClearKey();
- return Main.SeekToStart(Part, Env);
+ return Main.SeekToStart();
}
EReady SeekReverse(TRowId rowId) noexcept
{
ClearKey();
- return Main.SeekReverse(rowId, Part, Env);
+ return Main.SeekReverse(rowId);
}
EReady SeekToEnd() noexcept
{
ClearKey();
- return Main.SeekToEnd(Part, Env);
+ return Main.SeekToEnd();
}
EReady Next() noexcept
{
ClearKey();
- return Main.Next(Part, Env);
+ return Main.Next();
}
EReady Prev() noexcept
{
ClearKey();
- return Main.Prev(Part, Env);
+ return Main.Prev();
}
Y_FORCE_INLINE bool IsValid() const noexcept
@@ -940,7 +942,7 @@ namespace NTable {
if (!HistoryState) {
// Initialize history state once per iterator
- HistoryState.ConstructInPlace(Part, Pinout);
+ HistoryState.ConstructInPlace(Part, Env, Pinout);
}
trustHistory = false;
@@ -969,7 +971,7 @@ namespace NTable {
SkipEraseVersion = false;
// Find the first row that is as old as rowVersion
- ready = HistoryState->History.Seek(Main.GetRowId(), rowVersion, Part, Env);
+ ready = HistoryState->History.Seek(Main.GetRowId(), rowVersion);
}
switch (ready) {
@@ -1200,7 +1202,7 @@ namespace NTable {
auto& g = SkipMainVersion ? HistoryState->Groups.at(altIdx) : Groups.at(altIdx);
TRowId altRowId = SkipMainVersion ? HistoryState->History.GetHistoryRowId() : Main.GetRowId();
- switch (g.Seek(altRowId, Part, Env)) {
+ switch (g.Seek(altRowId)) {
case EReady::Data:
data = g.GetRecord()->GetAltRecord(altIndex);
break;
@@ -1287,15 +1289,15 @@ namespace NTable {
TPartGroupHistoryIt History;
mutable TSmallVec<TPartGroupRowIt> Groups;
- THistoryState(const TPart* part, const TPinout& pinout)
- : History(part)
+ THistoryState(const TPart* part, IPages* env, const TPinout& pinout)
+ : History(part, env)
{
Groups.reserve(pinout.AltGroups().size());
for (size_t idx : xrange(pinout.AltGroups().size())) {
ui32 group = pinout.AltGroups()[idx];
TGroupId groupId(group, /* historic */ true);
- Groups.emplace_back(part, groupId);
+ Groups.emplace_back(part, env, groupId);
}
}
};
@@ -1521,7 +1523,14 @@ namespace NTable {
UpdateCurrent();
- return SeekToStart();
+ ready = SeekToStart();
+ if (ready == EReady::Page) {
+ // we haven't seeked start, will do it again later
+ Current--;
+ UpdateCurrent();
+ }
+
+ return ready;
}
EReady Prev() noexcept
@@ -1546,7 +1555,14 @@ namespace NTable {
--Current;
UpdateCurrent();
- return SeekToEnd();
+ ready = SeekToEnd();
+ if (ready == EReady::Page) {
+ // we haven't seeked end, will do it again later
+ Current++;
+ UpdateCurrent();
+ }
+
+ return ready;
}
bool IsValid() const noexcept
diff --git a/ydb/core/tablet_flat/flat_part_loader.cpp b/ydb/core/tablet_flat/flat_part_loader.cpp
index 1e09175596..48b6f979ef 100644
--- a/ydb/core/tablet_flat/flat_part_loader.cpp
+++ b/ydb/core/tablet_flat/flat_part_loader.cpp
@@ -188,11 +188,18 @@ TAutoPtr<NPageCollection::TFetch> TLoader::StageCreatePartView() noexcept
// Use epoch from metadata unless it has been provided to loader externally
TEpoch epoch = Epoch != TEpoch::Max() ? Epoch : TEpoch(Root.GetEpoch());
+ TVector<TPageId> groupIndexesIds(Reserve(GroupIndexesIds.size() + 1));
+ groupIndexesIds.push_back(IndexId);
+ for (auto pageId : GroupIndexesIds) {
+ groupIndexesIds.push_back(pageId);
+ }
+
auto *partStore = new TPartStore(
Packs.front()->PageCollection->Label(),
{
epoch,
TPartScheme::Parse(*scheme, Rooted),
+ { groupIndexesIds, HistoricIndexesIds },
*index,
blobs ? new NPage::TExtBlobs(*blobs, extra) : nullptr,
byKey ? new NPage::TBloom(*byKey) : nullptr,
diff --git a/ydb/core/tablet_flat/flat_table_part.h b/ydb/core/tablet_flat/flat_table_part.h
index 589d43fb6b..618530a8b8 100644
--- a/ydb/core/tablet_flat/flat_table_part.h
+++ b/ydb/core/tablet_flat/flat_table_part.h
@@ -46,9 +46,39 @@ namespace NTable {
Trace = 2, /* how many last data pages to keep while seq scans */
};
+ class TIndexPages {
+ public:
+ using TPageId = NPage::TPageId;
+ using TGroupId = NPage::TGroupId;
+
+ TIndexPages(TVector<TPageId> groups, TVector<TPageId> historic)
+ : Groups(std::move(groups))
+ , Historic(std::move(historic))
+ , All(Groups.size() + Historic.size())
+ {
+ for (auto p : Groups) {
+ All.insert(p);
+ }
+ for (auto p : Historic) {
+ All.insert(p);
+ }
+ }
+
+ const TVector<TPageId> Groups;
+ const TVector<TPageId> Historic;
+
+ bool Has(NPage::TGroupId groupId, TPageId pageId) const {
+ return groupId.IsMain() && All.contains(pageId);
+ }
+
+ private:
+ THashSet<TPageId> All;
+ };
+
struct TParams {
TEpoch Epoch;
TIntrusiveConstPtr<TPartScheme> Scheme;
+ TIndexPages IndexPages;
TSharedData Index;
TIntrusiveConstPtr<NPage::TExtBlobs> Blobs;
TIntrusiveConstPtr<NPage::TBloom> ByKey;
@@ -78,6 +108,7 @@ namespace NTable {
, Blobs(std::move(params.Blobs))
, Large(std::move(params.Large))
, Small(std::move(params.Small))
+ , IndexPages(std::move(params.IndexPages))
, Index(std::move(params.Index))
, GroupIndexes(
std::make_move_iterator(params.GroupIndexes.begin()),
@@ -154,6 +185,7 @@ namespace NTable {
, Blobs(src.Blobs)
, Large(src.Large)
, Small(src.Small)
+ , IndexPages(src.IndexPages)
, Index(src.Index)
, GroupIndexes(src.GroupIndexes)
, HistoricIndexes(src.HistoricIndexes)
@@ -182,6 +214,7 @@ namespace NTable {
const TIntrusiveConstPtr<NPage::TExtBlobs> Blobs;
const TIntrusiveConstPtr<NPage::TFrames> Large;
const TIntrusiveConstPtr<NPage::TFrames> Small;
+ const TIndexPages IndexPages;
const NPage::TIndex Index;
const TVector<NPage::TIndex> GroupIndexes;
const TVector<NPage::TIndex> HistoricIndexes;
diff --git a/ydb/core/tablet_flat/test/libs/table/test_envs.h b/ydb/core/tablet_flat/test/libs/table/test_envs.h
index 52066ddd33..476f0fdd5f 100644
--- a/ydb/core/tablet_flat/test/libs/table/test_envs.h
+++ b/ydb/core/tablet_flat/test/libs/table/test_envs.h
@@ -61,6 +61,11 @@ namespace NTest {
{
return Token == seen.Token && Ref == seen.Ref;
}
+
+ bool operator<(const TSeen &seen) const noexcept
+ {
+ return Token < seen.Token || Token == seen.Token && Ref < seen.Ref;
+ }
const void *Token = nullptr;
ui64 Ref = Max<ui64>();
@@ -89,7 +94,7 @@ namespace NTest {
TResult Locate(const TPart *part, ui64 ref, ELargeObj lob) noexcept override
{
- if (ShouldPass((const void*)part->Large.Get(), ref)) {
+ if (ShouldPass((const void*)part->Large.Get(), ref, false)) {
return TTestEnv::Locate(part, ref, lob);
} else {
return { true, nullptr };
@@ -98,29 +103,45 @@ namespace NTest {
const TSharedData* TryGetPage(const TPart* part, TPageId ref, TGroupId groupId) override
{
- auto pass = ShouldPass((const void*)part, ref | (ui64(groupId.Raw()) << 32));
+ auto pass = ShouldPass((const void*)part, ref | (ui64(groupId.Raw()) << 32), part->IndexPages.Has(groupId, ref));
return pass ? TTestEnv::TryGetPage(part, ref, groupId) : nullptr;
}
- bool ShouldPass(const void *token, ui64 id)
+ bool ShouldPass(const void *token, ui64 id, bool isIndex)
{
- const auto pass = IsRecent({ token, id }) || AmILucky();
+ const TSeen seen(token, id);
+ const auto pass = IsRecent(seen, isIndex) || AmILucky();
Touches++, Success += pass ? 1 : 0;
if (!pass) {
- Offset = (Offset + 1) % Trace.size();
-
- Trace[Offset] = { token, id };
+ if (isIndex) {
+ // allow to pass on index page next ttl times
+ IndexTraceTtl[seen] = Rnd.Uniform(5, 10);
+ } else {
+ Offset = (Offset + 1) % Trace.size();
+ Trace[Offset] = seen;
+ }
}
return pass;
}
- bool IsRecent(TSeen seen) const noexcept
+ bool IsRecent(TSeen seen, bool isIndex) noexcept
{
- return std::find(Trace.begin(), Trace.end(), seen) != Trace.end();
+ if (isIndex) {
+ auto it = IndexTraceTtl.find(seen);
+ if (it == IndexTraceTtl.end()) {
+ return false;
+ }
+ if (it->second-- <= 1) {
+ IndexTraceTtl.erase(it);
+ }
+ return true;
+ } else {
+ return std::find(Trace.begin(), Trace.end(), seen) != Trace.end();
+ }
}
bool AmILucky() noexcept
@@ -133,6 +154,7 @@ namespace NTest {
ui64 Touches = 0;
ui64 Success = 0;
TVector<TSeen> Trace;
+ TMap<TSeen, ui64> IndexTraceTtl;
ui32 Offset = Max<ui32>();
};
@@ -170,7 +192,7 @@ namespace NTest {
}
private:
- ui64 AddToQueue(TPageId pageId, ui16 /* type */) noexcept override
+ ui64 AddToQueue(TPageId pageId, EPage) noexcept override
{
Fetch.push_back(pageId);
@@ -233,8 +255,12 @@ namespace NTest {
Y_VERIFY(groupId.Index < partStore->Store->GetGroupCount());
- ui32 room = (groupId.Historic ? partStore->Store->GetRoomCount() : 0) + groupId.Index;
- return Get(part, room).DoLoad(pageId, AheadLo, AheadHi).Page;
+ if (part->IndexPages.Has(groupId, pageId)) {
+ return partStore->Store->GetPage(groupId.Index, pageId);
+ } else {
+ ui32 room = (groupId.Historic ? partStore->Store->GetRoomCount() : 0) + groupId.Index;
+ return Get(part, room).DoLoad(pageId, AheadLo, AheadHi).Page;
+ }
}
private:
diff --git a/ydb/core/tablet_flat/test/libs/table/test_store.h b/ydb/core/tablet_flat/test/libs/table/test_store.h
index 2847ecbbd5..b76b0a95c9 100644
--- a/ydb/core/tablet_flat/test/libs/table/test_store.h
+++ b/ydb/core/tablet_flat/test/libs/table/test_store.h
@@ -1,5 +1,6 @@
#pragma once
+#include "ydb/core/tablet_flat/flat_table_part.h"
#include <ydb/core/tablet_flat/flat_page_label.h>
#include <ydb/core/tablet_flat/flat_part_iface.h>
#include <ydb/core/tablet_flat/flat_sausage_misc.h>
@@ -23,6 +24,8 @@ namespace NTest {
struct TEggs {
bool Rooted;
+ TVector<TPageId> IndexGroupsPages;
+ TVector<TPageId> IndexHistoricPages;
TData *Index;
TData *Scheme;
TData *Blobs;
@@ -111,6 +114,8 @@ namespace NTest {
return {
Rooted,
+ { Indexes.back() },
+ { },
GetPage(MainPageCollection, Indexes.back()),
GetPage(MainPageCollection, Scheme),
GetPage(MainPageCollection, Globs),
diff --git a/ydb/core/tablet_flat/test/libs/table/test_writer.h b/ydb/core/tablet_flat/test/libs/table/test_writer.h
index 6f36649636..e5d6171fae 100644
--- a/ydb/core/tablet_flat/test/libs/table/test_writer.h
+++ b/ydb/core/tablet_flat/test/libs/table/test_writer.h
@@ -80,6 +80,7 @@ namespace NTest {
{
epoch,
TPartScheme::Parse(*eggs.Scheme, eggs.Rooted),
+ { eggs.IndexGroupsPages, eggs.IndexHistoricPages },
*eggs.Index,
eggs.Blobs ? new TExtBlobs(*eggs.Blobs, { }) : nullptr,
eggs.ByKey ? new TBloom(*eggs.ByKey) : nullptr,
@@ -109,18 +110,27 @@ namespace NTest {
{
const auto undef = Max<NPage::TPageId>();
+ TVector<TPageId> indexGroupsPages, indexHistoricPages;
+ if (lay.HasIndex()) {
+ indexGroupsPages.push_back(lay.GetIndex());
+ }
+
TVector<TSharedData> groupIndexes;
for (ui32 pageId : lay.GetGroupIndexes()) {
+ indexGroupsPages.push_back(pageId);
groupIndexes.emplace_back(*Store->GetPage(0, pageId));
}
TVector<TSharedData> historicIndexes;
for (ui32 pageId : lay.GetHistoricIndexes()) {
+ indexHistoricPages.push_back(pageId);
historicIndexes.emplace_back(*Store->GetPage(0, pageId));
}
return {
true /* rooted page collection */,
+ indexGroupsPages,
+ indexHistoricPages,
Store->GetPage(0, lay.HasIndex() ? lay.GetIndex() : undef),
Store->GetPage(0, lay.HasScheme() ? lay.GetScheme() : undef),
Store->GetPage(0, lay.HasGlobs() ? lay.GetGlobs() : undef),
diff --git a/ydb/core/tablet_flat/ut/ut_charge.cpp b/ydb/core/tablet_flat/ut/ut_charge.cpp
index 79c2f8116e..c7eb5e7a80 100644
--- a/ydb/core/tablet_flat/ut/ut_charge.cpp
+++ b/ydb/core/tablet_flat/ut/ut_charge.cpp
@@ -50,6 +50,11 @@ namespace {
const TSharedData* TryGetPage(const TPart *part, TPageId id, TGroupId groupId) override
{
+ if (part->IndexPages.Has(groupId, id)) {
+ // TODO: delete after index precharge
+ return NTest::TTestEnv::TryGetPage(part, id, groupId);
+ }
+
Touched[groupId].insert(id);
return Fail ? nullptr : NTest::TTestEnv::TryGetPage(part, id, groupId);
}
diff --git a/ydb/core/tablet_flat/ut/ut_comp_shard.cpp b/ydb/core/tablet_flat/ut/ut_comp_shard.cpp
index 19beb16775..afc6642d0f 100644
--- a/ydb/core/tablet_flat/ut/ut_comp_shard.cpp
+++ b/ydb/core/tablet_flat/ut/ut_comp_shard.cpp
@@ -33,36 +33,41 @@ namespace {
/**
* A special kind of TTestEnv that will fail loading every page until
- * Lock() call, and then would only pass previously requested pages
+ * Load() call, and then would only pass previously requested pages
*/
class TStrictEnv : public TTestEnv {
public:
const TSharedData *TryGetPage(const TPart *part, TPageId ref, TGroupId groupId) override {
ui64 token = ref | (ui64(groupId.Raw()) << 32);
- if (Locked) {
- if (auto* info = Parts.FindPtr(part)) {
- if (info->Seen.contains(token)) {
- return TTestEnv::TryGetPage(part, ref, groupId);
- }
- }
- } else {
- Parts[part].Seen.insert(token);
+ auto& info = Parts[part];
+
+ info.Touched.insert(token);
+
+ if (part->IndexPages.Has(groupId, ref)) {
+ // TODO: delete after index precharge
+ return NTest::TTestEnv::TryGetPage(part, ref, groupId);
+ }
+
+ if (info.Loaded.contains(token)) {
+ return TTestEnv::TryGetPage(part, ref, groupId);
}
return nullptr;
}
- void Lock() {
- Locked = true;
+ void Load() {
+ for (auto &p: Parts) {
+ p.second.Loaded = std::move(p.second.Touched);
+ }
}
private:
struct TPartInfo {
- THashSet<ui64> Seen;
+ THashSet<ui64> Touched;
+ THashSet<ui64> Loaded;
};
private:
- bool Locked = false;
THashMap<const TPart*, TPartInfo> Parts;
};
@@ -284,7 +289,7 @@ Y_UNIT_TEST_SUITE(TShardedCompaction) {
bool ok1 = op.Execute(&env);
UNIT_ASSERT_VALUES_EQUAL(ok1, false);
- env.Lock();
+ env.Load();
bool ok2 = op.Execute(&env);
UNIT_ASSERT_VALUES_EQUAL(ok2, true);
@@ -368,7 +373,7 @@ Y_UNIT_TEST_SUITE(TShardedCompaction) {
bool ok1 = op.Execute(&env);
UNIT_ASSERT_VALUES_EQUAL(ok1, false);
- env.Lock();
+ env.Load();
bool ok2 = op.Execute(&env);
UNIT_ASSERT_VALUES_EQUAL(ok2, true);
@@ -434,7 +439,7 @@ Y_UNIT_TEST_SUITE(TShardedCompaction) {
bool ok1 = op.Execute(&env);
UNIT_ASSERT_VALUES_EQUAL(ok1, false);
- env.Lock();
+ env.Load();
bool ok2 = op.Execute(&env);
UNIT_ASSERT_VALUES_EQUAL(ok2, true);
@@ -1236,7 +1241,7 @@ Y_UNIT_TEST_SUITE(TShardedCompactionScenarios) {
TStrictEnv env;
auto first = backend.RunRead(&env);
UNIT_ASSERT(!first.Completed);
- env.Lock();
+ env.Load();
auto second = backend.RunRead(first.ReadId, &env);
UNIT_ASSERT(second.Completed);
}
diff --git a/ydb/core/tablet_flat/ut/ut_forward.cpp b/ydb/core/tablet_flat/ut/ut_forward.cpp
index 246d3c70b3..741245adf7 100644
--- a/ydb/core/tablet_flat/ut/ut_forward.cpp
+++ b/ydb/core/tablet_flat/ut/ut_forward.cpp
@@ -34,7 +34,7 @@ namespace {
{
}
- ui64 AddToQueue(ui32 page, ui16) noexcept override
+ ui64 AddToQueue(ui32 page, EPage) noexcept override
{
Pages.push_back(page);
diff --git a/ydb/core/tablet_flat/ut/ut_part.cpp b/ydb/core/tablet_flat/ut/ut_part.cpp
index 83d8eb8f43..d532f9e7ee 100644
--- a/ydb/core/tablet_flat/ut/ut_part.cpp
+++ b/ydb/core/tablet_flat/ut/ut_part.cpp
@@ -59,6 +59,11 @@ namespace {
struct TTouchEnv : public NTest::TTestEnv {
const TSharedData* TryGetPage(const TPart *part, TPageId id, TGroupId groupId) override
{
+ if (part->IndexPages.Has(groupId, id)) {
+ // TODO: delete after index precharge
+ return NTest::TTestEnv::TryGetPage(part, id, groupId);
+ }
+
if (PrechargePhase) {
Precharged[groupId].insert(id);
return NTest::TTestEnv::TryGetPage(part, id, groupId);