diff options
author | ivanmorozov333 <ivanmorozov@hotmail.com> | 2024-04-19 22:09:17 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-04-19 22:09:17 +0300 |
commit | 32415d9ecd85e93bf2490b103002e9bf1b1841d6 (patch) | |
tree | a37380f7bde8361f2a04ae61590908a25af7f9fa | |
parent | 7147f14eafadc1df4151d684336579a4c29ce7be (diff) | |
download | ydb-32415d9ecd85e93bf2490b103002e9bf1b1841d6.tar.gz |
separate library for olap modification schemas (#3928)
21 files changed, 620 insertions, 530 deletions
diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 84e9a1c037..d25c953c43 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -621,7 +621,6 @@ message TColumnStorageConfig { // This is used internally by schemeshard message TColumnStoreSharding { - repeated TShardIdx MetaShards = 1; repeated TShardIdx ColumnShards = 2; } diff --git a/ydb/core/tx/schemeshard/olap/layout/layout.cpp b/ydb/core/tx/schemeshard/olap/layout/layout.cpp new file mode 100644 index 0000000000..40791d3b59 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/layout/layout.cpp @@ -0,0 +1,25 @@ +#include "layout.h" +#include <ydb/core/tx/schemeshard/schemeshard_impl.h> + +namespace NKikimr::NSchemeShard { + +std::vector<ui64> TColumnTablesLayout::ShardIdxToTabletId(const std::vector<TShardIdx>& shards, const TSchemeShard& ss) { + std::vector<ui64> result; + for (const auto& shardIdx : shards) { + auto* shardInfo = ss.ShardInfos.FindPtr(shardIdx); + Y_ABORT_UNLESS(shardInfo, "ColumnShard not found"); + result.emplace_back(shardInfo->TabletID.GetValue()); + } + return result; +} + +TColumnTablesLayout TColumnTablesLayout::BuildTrivial(const std::vector<ui64>& tabletIds) { + TTableIdsGroup emptyGroup; + TShardIdsGroup shardIdsGroup; + for (const auto& tabletId : tabletIds) { + shardIdsGroup.AddId(tabletId); + } + return TColumnTablesLayout({ TTablesGroup(std::move(emptyGroup), std::move(shardIdsGroup)) }); +} + +} diff --git a/ydb/core/tx/schemeshard/schemeshard_tables_storage.h b/ydb/core/tx/schemeshard/olap/layout/layout.h index 0360512959..3f01733204 100644 --- a/ydb/core/tx/schemeshard/schemeshard_tables_storage.h +++ b/ydb/core/tx/schemeshard/olap/layout/layout.h @@ -1,10 +1,12 @@ #pragma once -#include "schemeshard_info_types.h" - #include <ydb/core/scheme/scheme_pathid.h> +#include <ydb/core/tx/schemeshard/schemeshard_identificators.h> #include <ydb/library/accessor/accessor.h> -#include <util/digest/numeric.h> + +#include <util/system/types.h> + +#include <set> namespace NKikimr::NSchemeShard { @@ -149,118 +151,4 @@ public: static TColumnTablesLayout BuildTrivial(const std::vector<ui64>& tabletIds); }; -class TTablesStorage { -private: - THashMap<TPathId, TColumnTableInfo::TPtr> Tables; - THashMap<TString, std::set<TPathId>> PathsByTieringId; - THashMap<ui64, TColumnTablesLayout::TTableIdsGroup> TablesByShard; - - void OnAddObject(const TPathId& pathId, TColumnTableInfo::TPtr object); - void OnRemoveObject(const TPathId& pathId, TColumnTableInfo::TPtr object); - TColumnTableInfo::TPtr ExtractPtr(const TPathId& id); -public: - std::unordered_set<TPathId> GetAllPathIds() const; - - TColumnTablesLayout GetTablesLayout(const std::vector<ui64>& tabletIds) const; - - const std::set<TPathId>& GetTablesWithTiering(const TString& tieringId) const; - - class TTableReadGuard { - protected: - TColumnTableInfo::TPtr Object; - public: - bool operator!() const { - return !Object; - } - - TTableReadGuard(TColumnTableInfo::TPtr object) - : Object(object) { - } - const TColumnTableInfo* operator->() const { - Y_DEBUG_ABORT_UNLESS(Object); - return Object.Get(); - } - const TColumnTableInfo& operator*() const { - Y_DEBUG_ABORT_UNLESS(Object); - return *Object; - } - const TColumnTableInfo* operator->() { - Y_DEBUG_ABORT_UNLESS(Object); - return Object.Get(); - } - }; - - class TTableCreateOperator: public TTableReadGuard { - private: - using TBase = TTableReadGuard; - public: - using TBase::TBase; - bool InitShardingTablets(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, TOlapStoreInfo::ILayoutPolicy::TPtr layoutPolicy, bool& isNewGroup) const; - }; - - class TTableCreatedGuard: public TTableCreateOperator, TMoveOnly { - protected: - const TPathId PathId; - TTablesStorage& Owner; - public: - TTableCreatedGuard(TTablesStorage& owner, const TPathId& id, TColumnTableInfo::TPtr object) - : TTableCreateOperator(object) - , PathId(id) - , Owner(owner) - - { - Y_ABORT_UNLESS(!Owner.contains(id)); - } - - TColumnTableInfo::TPtr GetPtr() const { - return Object; - } - - TTableCreatedGuard(TTablesStorage& owner, const TPathId& id) - : TTableCreateOperator(new TColumnTableInfo) - , PathId(id) - , Owner(owner) { - Y_ABORT_UNLESS(!Owner.contains(id)); - } - TColumnTableInfo* operator->() { - return Object.Get(); - } - const TColumnTableInfo* operator->() const { - return Object.Get(); - } - ~TTableCreatedGuard() { - Y_ABORT_UNLESS(Owner.Tables.emplace(PathId, Object).second); - Owner.OnAddObject(PathId, Object); - } - }; - - class TTableExtractedGuard: public TTableCreatedGuard { - private: - void UseAlterDataVerified(); - public: - TTableExtractedGuard(TTablesStorage& owner, const TPathId& id, TColumnTableInfo::TPtr object, const bool extractAlter) - : TTableCreatedGuard(owner, id, object) - { - Owner.OnRemoveObject(PathId, object); - if (extractAlter) { - UseAlterDataVerified(); - } - } - }; - - TTableCreatedGuard BuildNew(const TPathId& id); - TTableCreatedGuard BuildNew(const TPathId& id, TColumnTableInfo::TPtr object); - TTableExtractedGuard TakeVerified(const TPathId& id); - TTableExtractedGuard TakeAlterVerified(const TPathId& id); - - bool contains(const TPathId& id) const { - return Tables.contains(id); - } - TTableReadGuard GetVerified(const TPathId& id) const; - TTableReadGuard at(const TPathId& id) const { - return TTableReadGuard(Tables.at(id)); - } - size_t Drop(const TPathId& id); -}; - } diff --git a/ydb/core/tx/schemeshard/olap/layout/ya.make b/ydb/core/tx/schemeshard/olap/layout/ya.make new file mode 100644 index 0000000000..f22a412813 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/layout/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +SRCS( + layout.cpp +) + +PEERDIR( + ydb/core/scheme + ydb/core/tablet_flat +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/schemeshard/schemeshard_tables_storage.cpp b/ydb/core/tx/schemeshard/olap/manager/manager.cpp index 5961d458fa..c4c24e31f3 100644 --- a/ydb/core/tx/schemeshard/schemeshard_tables_storage.cpp +++ b/ydb/core/tx/schemeshard/olap/manager/manager.cpp @@ -1,5 +1,4 @@ -#include "schemeshard_tables_storage.h" -#include "schemeshard_impl.h" +#include "manager.h" namespace NKikimr::NSchemeShard { @@ -113,25 +112,6 @@ void TTablesStorage::TTableExtractedGuard::UseAlterDataVerified() { Object = alterInfo; } -std::vector<ui64> TColumnTablesLayout::ShardIdxToTabletId(const std::vector<TShardIdx>& shards, const TSchemeShard& ss) { - std::vector<ui64> result; - for (const auto& shardIdx : shards) { - auto* shardInfo = ss.ShardInfos.FindPtr(shardIdx); - Y_ABORT_UNLESS(shardInfo, "ColumnShard not found"); - result.emplace_back(shardInfo->TabletID.GetValue()); - } - return result; -} - -TColumnTablesLayout TColumnTablesLayout::BuildTrivial(const std::vector<ui64>& tabletIds) { - TTableIdsGroup emptyGroup; - TShardIdsGroup shardIdsGroup; - for (const auto& tabletId : tabletIds) { - shardIdsGroup.AddId(tabletId); - } - return TColumnTablesLayout({ TTablesGroup(std::move(emptyGroup), std::move(shardIdsGroup)) }); -} - bool TTablesStorage::TTableCreateOperator::InitShardingTablets(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, TOlapStoreInfo::ILayoutPolicy::TPtr layoutPolicy, bool& isNewGroup) const { if (!layoutPolicy->Layout(currentLayout, shardsCount, Object->ColumnShards, isNewGroup)) { ALS_ERROR(NKikimrServices::FLAT_TX_SCHEMESHARD) << "cannot layout new table with " << shardsCount << " shards"; diff --git a/ydb/core/tx/schemeshard/olap/manager/manager.h b/ydb/core/tx/schemeshard/olap/manager/manager.h new file mode 100644 index 0000000000..4ed09e97bc --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/manager/manager.h @@ -0,0 +1,119 @@ +#pragma once +#include <ydb/core/scheme/scheme_pathid.h> +#include <ydb/core/tx/schemeshard/olap/layout/layout.h> +#include <ydb/core/tx/schemeshard/olap/store/store.h> +#include <ydb/core/tx/schemeshard/olap/table/table.h> + +namespace NKikimr::NSchemeShard { + +class TTablesStorage { +private: + THashMap<TPathId, TColumnTableInfo::TPtr> Tables; + THashMap<TString, std::set<TPathId>> PathsByTieringId; + THashMap<ui64, TColumnTablesLayout::TTableIdsGroup> TablesByShard; + + void OnAddObject(const TPathId& pathId, TColumnTableInfo::TPtr object); + void OnRemoveObject(const TPathId& pathId, TColumnTableInfo::TPtr object); + TColumnTableInfo::TPtr ExtractPtr(const TPathId& id); +public: + std::unordered_set<TPathId> GetAllPathIds() const; + + TColumnTablesLayout GetTablesLayout(const std::vector<ui64>& tabletIds) const; + + const std::set<TPathId>& GetTablesWithTiering(const TString& tieringId) const; + + class TTableReadGuard { + protected: + TColumnTableInfo::TPtr Object; + public: + bool operator!() const { + return !Object; + } + + TTableReadGuard(TColumnTableInfo::TPtr object) + : Object(object) { + } + const TColumnTableInfo* operator->() const { + Y_DEBUG_ABORT_UNLESS(Object); + return Object.get(); + } + const TColumnTableInfo& operator*() const { + Y_DEBUG_ABORT_UNLESS(Object); + return *Object; + } + }; + + class TTableCreateOperator: public TTableReadGuard { + private: + using TBase = TTableReadGuard; + public: + using TBase::TBase; + bool InitShardingTablets(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, TOlapStoreInfo::ILayoutPolicy::TPtr layoutPolicy, bool& isNewGroup) const; + }; + + class TTableCreatedGuard: public TTableCreateOperator, TMoveOnly { + protected: + const TPathId PathId; + TTablesStorage& Owner; + public: + TTableCreatedGuard(TTablesStorage& owner, const TPathId& id, TColumnTableInfo::TPtr object) + : TTableCreateOperator(object) + , PathId(id) + , Owner(owner) + + { + Y_ABORT_UNLESS(!Owner.contains(id)); + } + + TColumnTableInfo::TPtr GetPtr() const { + return Object; + } + + TTableCreatedGuard(TTablesStorage& owner, const TPathId& id) + : TTableCreateOperator(std::make_shared<TColumnTableInfo>()) + , PathId(id) + , Owner(owner) { + Y_ABORT_UNLESS(!Owner.contains(id)); + } + TColumnTableInfo* operator->() { + return Object.get(); + } + const TColumnTableInfo* operator->() const { + return Object.get(); + } + ~TTableCreatedGuard() { + Y_ABORT_UNLESS(Owner.Tables.emplace(PathId, Object).second); + Owner.OnAddObject(PathId, Object); + } + }; + + class TTableExtractedGuard: public TTableCreatedGuard { + private: + void UseAlterDataVerified(); + public: + TTableExtractedGuard(TTablesStorage& owner, const TPathId& id, TColumnTableInfo::TPtr object, const bool extractAlter) + : TTableCreatedGuard(owner, id, object) + { + Owner.OnRemoveObject(PathId, object); + if (extractAlter) { + UseAlterDataVerified(); + } + } + }; + + TTableCreatedGuard BuildNew(const TPathId& id); + TTableCreatedGuard BuildNew(const TPathId& id, TColumnTableInfo::TPtr object); + TTableExtractedGuard TakeVerified(const TPathId& id); + TTableExtractedGuard TakeAlterVerified(const TPathId& id); + + bool contains(const TPathId& id) const { + return Tables.contains(id); + } + TTableReadGuard GetVerified(const TPathId& id) const; + TTableReadGuard at(const TPathId& id) const { + return TTableReadGuard(Tables.at(id)); + } + size_t Drop(const TPathId& id); +}; + +} diff --git a/ydb/core/tx/schemeshard/olap/manager/ya.make b/ydb/core/tx/schemeshard/olap/manager/ya.make new file mode 100644 index 0000000000..a5b6e0e4c2 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/manager/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +SRCS( + manager.cpp +) + +PEERDIR( + ydb/core/scheme + ydb/core/tx/schemeshard/olap/table + ydb/core/tx/schemeshard/olap/layout +) + +END() diff --git a/ydb/core/tx/schemeshard/olap/operations/create_store.cpp b/ydb/core/tx/schemeshard/olap/operations/create_store.cpp index 776b8d2737..bc93fcb88b 100644 --- a/ydb/core/tx/schemeshard/olap/operations/create_store.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/create_store.cpp @@ -397,7 +397,7 @@ public: } TProposeErrorCollector errors(*result); - TOlapStoreInfo::TPtr storeInfo = new TOlapStoreInfo; + TOlapStoreInfo::TPtr storeInfo = std::make_shared<TOlapStoreInfo>(); if (!storeInfo->ParseFromRequest(createDescription, errors)) { return result; } @@ -438,7 +438,7 @@ public: NIceDb::TNiceDb db(context.GetDB()); - TOlapStoreInfo::TPtr pending = new TOlapStoreInfo; + TOlapStoreInfo::TPtr pending = std::make_shared<TOlapStoreInfo>(); pending->AlterData = storeInfo; context.SS->OlapStores[pathId] = pending; context.SS->PersistOlapStore(db, pathId, *pending); diff --git a/ydb/core/tx/schemeshard/olap/operations/create_table.cpp b/ydb/core/tx/schemeshard/olap/operations/create_table.cpp index db77e7491a..d1ef7e9cf6 100644 --- a/ydb/core/tx/schemeshard/olap/operations/create_table.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/create_table.cpp @@ -58,7 +58,7 @@ public: } TColumnTableInfo::TPtr BuildTableInfo(IErrorCollector& errors) const { - TColumnTableInfo::TPtr tableInfo = new TColumnTableInfo; + TColumnTableInfo::TPtr tableInfo = std::make_shared<TColumnTableInfo>(); tableInfo->AlterVersion = 1; BuildDescription(tableInfo->Description); diff --git a/ydb/core/tx/schemeshard/olap/store/store.cpp b/ydb/core/tx/schemeshard/olap/store/store.cpp new file mode 100644 index 0000000000..46153949a5 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/store/store.cpp @@ -0,0 +1,192 @@ +#include "store.h" + +namespace NKikimr::NSchemeShard { + +bool TOlapStoreInfo::ILayoutPolicy::Layout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const { + if (!DoLayout(currentLayout, shardsCount, result, isNewGroup)) { + return false; + } + Y_ABORT_UNLESS(result.size() == shardsCount); + return true; +} + +bool TOlapStoreInfo::TIdentityGroupsLayout::DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const { + isNewGroup = false; + for (auto&& i : currentLayout.GetGroups()) { + if (i.GetTableIds().Size() == 0 && i.GetShardIds().Size() >= shardsCount) { + result = i.GetShardIds().GetIdsVector(shardsCount); + isNewGroup = true; + return true; + } + if (i.GetShardIds().Size() != shardsCount) { + continue; + } + result = i.GetShardIds().GetIdsVector(); + return true; + } + return false; +} + +bool TOlapStoreInfo::TMinimalTablesCountLayout::DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const { + isNewGroup = true; + std::vector<ui64> resultLocal; + for (auto&& i : currentLayout.GetGroups()) { + if (i.GetTableIds().Size() > 0) { + isNewGroup = false; + } + for (auto&& s : i.GetShardIds()) { + resultLocal.emplace_back(s); + if (resultLocal.size() == shardsCount) { + std::swap(result, resultLocal); + return true; + } + } + } + return false; +} + +TOlapStoreInfo::TOlapStoreInfo( + ui64 alterVersion, + NKikimrSchemeOp::TColumnStoreSharding&& sharding, + TMaybe<NKikimrSchemeOp::TAlterColumnStore>&& alterBody) + : AlterVersion(alterVersion) + , Sharding(std::move(sharding)) + , AlterBody(std::move(alterBody)) { + for (const auto& shardIdx : Sharding.GetColumnShards()) { + ColumnShards.push_back(TShardIdx( + TOwnerId(shardIdx.GetOwnerId()), + TLocalShardIdx(shardIdx.GetLocalId()))); + } +} + +TOlapStoreInfo::TPtr TOlapStoreInfo::BuildStoreWithAlter(const TOlapStoreInfo& initialStore, const NKikimrSchemeOp::TAlterColumnStore& alterBody) { + TOlapStoreInfo::TPtr alterData = std::make_shared<TOlapStoreInfo>(initialStore); + alterData->AlterVersion++; + alterData->AlterBody.ConstructInPlace(alterBody); + return alterData; +} + +void TOlapStoreInfo::SerializeDescription(NKikimrSchemeOp::TColumnStoreDescription& descriptionProto) const { + descriptionProto.SetName(Name); + descriptionProto.SetColumnShardCount(ColumnShards.size()); + descriptionProto.SetNextSchemaPresetId(NextSchemaPresetId); + descriptionProto.SetNextTtlSettingsPresetId(NextTtlSettingsPresetId); + + for (const auto& [name, preset] : SchemaPresets) { + Y_UNUSED(name); + auto presetProto = descriptionProto.AddSchemaPresets(); + preset.Serialize(*presetProto); + } +} + +void TOlapStoreInfo::ParseFromLocalDB(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto) { + StorageConfig = descriptionProto.GetStorageConfig(); + NextTtlSettingsPresetId = descriptionProto.GetNextTtlSettingsPresetId(); + NextSchemaPresetId = descriptionProto.GetNextSchemaPresetId(); + Name = descriptionProto.GetName(); + + size_t schemaPresetIndex = 0; + for (const auto& presetProto : descriptionProto.GetSchemaPresets()) { + Y_ABORT_UNLESS(!SchemaPresets.contains(presetProto.GetId())); + auto& preset = SchemaPresets[presetProto.GetId()]; + preset.ParseFromLocalDB(presetProto); + preset.SetProtoIndex(schemaPresetIndex++); + SchemaPresetByName[preset.GetName()] = preset.GetId(); + } + SerializeDescription(Description); +} + +bool TOlapStoreInfo::UpdatePreset(const TString& presetName, const TOlapSchemaUpdate& schemaUpdate, IErrorCollector& errors) { + const ui32 presetId = SchemaPresetByName.at(presetName); + auto& currentPreset = SchemaPresets.at(presetId); + if (!currentPreset.Update(schemaUpdate, errors)) { + return false; + } + + NKikimrSchemeOp::TColumnTableSchemaPreset schemaUpdateProto; + currentPreset.Serialize(schemaUpdateProto); + + auto mutablePresetProto = Description.MutableSchemaPresets(currentPreset.GetProtoIndex()); + *mutablePresetProto = schemaUpdateProto; + return true; +} + +bool TOlapStoreInfo::ParseFromRequest(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto, IErrorCollector& errors) { + AlterVersion = 1; + if (descriptionProto.GetRESERVED_MetaShardCount() != 0) { + errors.AddError("trying to create OLAP store with meta shards (not supported yet)"); + return false; + } + + if (!descriptionProto.HasColumnShardCount()) { + errors.AddError("trying to create OLAP store without shards number specified"); + return false; + } + + if (descriptionProto.GetColumnShardCount() == 0) { + errors.AddError("trying to create OLAP store without zero shards"); + return false; + } + + for (auto& presetProto : descriptionProto.GetRESERVED_TtlSettingsPresets()) { + Y_UNUSED(presetProto); + errors.AddError("TTL presets are not supported"); + return false; + } + + Name = descriptionProto.GetName(); + StorageConfig = descriptionProto.GetStorageConfig(); + // Make it easier by having data channel count always specified internally + if (!StorageConfig.HasDataChannelCount()) { + StorageConfig.SetDataChannelCount(1); + } + + size_t protoIndex = 0; + for (const auto& presetProto : descriptionProto.GetSchemaPresets()) { + TOlapStoreSchemaPreset preset; + if (!preset.ParseFromRequest(presetProto, errors)) { + return false; + } + if (SchemaPresets.contains(NextSchemaPresetId) || SchemaPresetByName.contains(preset.GetName())) { + errors.AddError(TStringBuilder() << "Duplicate schema preset " << NextSchemaPresetId << " with name '" << preset.GetName() << "'"); + return false; + } + preset.SetId(NextSchemaPresetId++); + preset.SetProtoIndex(protoIndex++); + + TOlapSchemaUpdate schemaDiff; + if (!schemaDiff.Parse(presetProto.GetSchema(), errors)) { + return false; + } + + if (!preset.Update(schemaDiff, errors)) { + return false; + } + + SchemaPresetByName[preset.GetName()] = preset.GetId(); + SchemaPresets[preset.GetId()] = std::move(preset); + } + + if (!SchemaPresetByName.contains("default") || SchemaPresets.size() > 1) { + errors.AddError("A single schema preset named 'default' is required"); + return false; + } + + ColumnShards.resize(descriptionProto.GetColumnShardCount()); + SerializeDescription(Description); + return true; +} + +TOlapStoreInfo::ILayoutPolicy::TPtr TOlapStoreInfo::GetTablesLayoutPolicy() const { + ILayoutPolicy::TPtr result; + if (AppData()->ColumnShardConfig.GetTablesStorageLayoutPolicy().HasMinimalTables()) { + result = std::make_shared<TMinimalTablesCountLayout>(); + } else if (AppData()->ColumnShardConfig.GetTablesStorageLayoutPolicy().HasIdentityGroups()) { + result = std::make_shared<TIdentityGroupsLayout>(); + } else { + result = std::make_shared<TMinimalTablesCountLayout>(); + } + return result; +} + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/store/store.h b/ydb/core/tx/schemeshard/olap/store/store.h new file mode 100644 index 0000000000..3e777dc798 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/store/store.h @@ -0,0 +1,107 @@ +#pragma once +#include <ydb/core/protos/flat_scheme_op.pb.h> +#include <ydb/core/tx/schemeshard/olap/layout/layout.h> +#include <ydb/core/tx/schemeshard/olap/schema/schema.h> +#include <ydb/core/tx/schemeshard/schemeshard_info_types.h> + +#include <util/system/types.h> + +namespace NKikimr::NSchemeShard { + +struct TOlapStoreInfo { +private: + TString Name; + ui64 NextSchemaPresetId = 1; + ui64 NextTtlSettingsPresetId = 1; + NKikimrSchemeOp::TColumnStorageConfig StorageConfig; + NKikimrSchemeOp::TColumnStoreDescription Description; + ui64 AlterVersion = 0; +public: + using TPtr = std::shared_ptr<TOlapStoreInfo>; + + class ILayoutPolicy { + protected: + virtual bool DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const = 0; + public: + using TPtr = std::shared_ptr<ILayoutPolicy>; + virtual ~ILayoutPolicy() = default; + bool Layout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const; + }; + + class TMinimalTablesCountLayout: public ILayoutPolicy { + protected: + virtual bool DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const override; + }; + + class TIdentityGroupsLayout: public ILayoutPolicy { + protected: + virtual bool DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const override; + }; + + TPtr AlterData; + + const NKikimrSchemeOp::TColumnStoreDescription& GetDescription() const { + return Description; + } + + NKikimrSchemeOp::TColumnStoreSharding Sharding; + TMaybe<NKikimrSchemeOp::TAlterColumnStore> AlterBody; + + TVector<TShardIdx> ColumnShards; + + THashMap<ui32, TOlapStoreSchemaPreset> SchemaPresets; + THashMap<TString, ui32> SchemaPresetByName; + + THashSet<TPathId> ColumnTables; + THashSet<TPathId> ColumnTablesUnderOperation; + TAggregatedStats Stats; + + TOlapStoreInfo() = default; + TOlapStoreInfo(ui64 alterVersion, + NKikimrSchemeOp::TColumnStoreSharding&& sharding, + TMaybe<NKikimrSchemeOp::TAlterColumnStore>&& alterBody = Nothing()); + + static TOlapStoreInfo::TPtr BuildStoreWithAlter(const TOlapStoreInfo& initialStore, const NKikimrSchemeOp::TAlterColumnStore& alterBody); + + const NKikimrSchemeOp::TColumnStorageConfig& GetStorageConfig() const { + return StorageConfig; + } + + const TVector<TShardIdx>& GetColumnShards() const { + return ColumnShards; + } + + ui64 GetAlterVersion() const { + return AlterVersion; + } + + void ApplySharding(const TVector<TShardIdx>& shardsIndexes) { + Y_ABORT_UNLESS(ColumnShards.size() == shardsIndexes.size()); + Sharding.ClearColumnShards(); + for (ui64 i = 0; i < ColumnShards.size(); ++i) { + const auto& idx = shardsIndexes[i]; + ColumnShards[i] = idx; + auto* shardInfoProto = Sharding.AddColumnShards(); + shardInfoProto->SetOwnerId(idx.GetOwnerId()); + shardInfoProto->SetLocalId(idx.GetLocalId().GetValue()); + } + } + void SerializeDescription(NKikimrSchemeOp::TColumnStoreDescription& descriptionProto) const; + void ParseFromLocalDB(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto); + bool ParseFromRequest(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto, IErrorCollector& errors); + bool UpdatePreset(const TString& presetName, const TOlapSchemaUpdate& schemaUpdate, IErrorCollector& errors); + + const TAggregatedStats& GetStats() const { + return Stats; + } + + ILayoutPolicy::TPtr GetTablesLayoutPolicy() const; + + void UpdateShardStats(TShardIdx shardIdx, const TPartitionStats& newStats) { + Stats.Aggregated.PartCount = ColumnShards.size(); + Stats.PartitionStats[shardIdx]; // insert if none + Stats.UpdateShardStats(shardIdx, newStats); + } +}; + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/store/ya.make b/ydb/core/tx/schemeshard/olap/store/ya.make new file mode 100644 index 0000000000..3213054753 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/store/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +SRCS( + store.cpp +) + +PEERDIR( + ydb/core/tx/schemeshard/olap/layout + ydb/core/protos +) + +END() diff --git a/ydb/core/tx/schemeshard/olap/table/table.cpp b/ydb/core/tx/schemeshard/olap/table/table.cpp new file mode 100644 index 0000000000..04b4b1f510 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/table/table.cpp @@ -0,0 +1,49 @@ +#include "table.h" + +namespace NKikimr::NSchemeShard { + +TColumnTableInfo::TColumnTableInfo( + ui64 alterVersion, + NKikimrSchemeOp::TColumnTableDescription&& description, + NKikimrSchemeOp::TColumnTableSharding&& sharding, + TMaybe<NKikimrSchemeOp::TColumnStoreSharding>&& standaloneSharding, + TMaybe<NKikimrSchemeOp::TAlterColumnTable>&& alterBody) + : AlterVersion(alterVersion) + , Description(std::move(description)) + , Sharding(std::move(sharding)) + , StandaloneSharding(std::move(standaloneSharding)) + , AlterBody(std::move(alterBody)) { + if (Description.HasColumnStorePathId()) { + OlapStorePathId = TPathId( + TOwnerId(Description.GetColumnStorePathId().GetOwnerId()), + TLocalPathId(Description.GetColumnStorePathId().GetLocalId())); + } + + if (Description.HasSchema()) { + TOlapSchema schema; + schema.ParseFromLocalDB(Description.GetSchema()); + } + + ColumnShards.reserve(Sharding.GetColumnShards().size()); + for (ui64 columnShard : Sharding.GetColumnShards()) { + ColumnShards.push_back(columnShard); + } + + if (StandaloneSharding) { + OwnedColumnShards.reserve(StandaloneSharding->GetColumnShards().size()); + for (const auto& shardIdx : StandaloneSharding->GetColumnShards()) { + OwnedColumnShards.push_back(TShardIdx( + TOwnerId(shardIdx.GetOwnerId()), + TLocalShardIdx(shardIdx.GetLocalId()))); + } + } +} + +TColumnTableInfo::TPtr TColumnTableInfo::BuildTableWithAlter(const TColumnTableInfo& initialTable, const NKikimrSchemeOp::TAlterColumnTable& alterBody) { + TColumnTableInfo::TPtr alterData = std::make_shared<TColumnTableInfo>(initialTable); + alterData->AlterBody.ConstructInPlace(alterBody); + ++alterData->AlterVersion; + return alterData; +} + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/table/table.h b/ydb/core/tx/schemeshard/olap/table/table.h new file mode 100644 index 0000000000..6e10fecf72 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/table/table.h @@ -0,0 +1,58 @@ +#pragma once +#include <ydb/core/protos/flat_scheme_op.pb.h> +#include <ydb/core/tx/schemeshard/schemeshard_identificators.h> +#include <ydb/core/tx/schemeshard/schemeshard_info_types.h> + +namespace NKikimr::NSchemeShard { + +struct TColumnTableInfo { + using TPtr = std::shared_ptr<TColumnTableInfo>; + + ui64 AlterVersion = 0; + TPtr AlterData; + + NKikimrSchemeOp::TColumnTableDescription Description; + NKikimrSchemeOp::TColumnTableSharding Sharding; + TMaybe<NKikimrSchemeOp::TColumnStoreSharding> StandaloneSharding; + TMaybe<NKikimrSchemeOp::TAlterColumnTable> AlterBody; + + TMaybe<TPathId> OlapStorePathId; // PathId of the table store + + TVector<ui64> ColumnShards; // Current list of column shards + TVector<TShardIdx> OwnedColumnShards; + TAggregatedStats Stats; + + TColumnTableInfo() = default; + TColumnTableInfo(ui64 alterVersion, NKikimrSchemeOp::TColumnTableDescription&& description, + NKikimrSchemeOp::TColumnTableSharding&& sharding, + TMaybe<NKikimrSchemeOp::TColumnStoreSharding>&& standaloneSharding, + TMaybe<NKikimrSchemeOp::TAlterColumnTable>&& alterBody = Nothing()); + + void SetOlapStorePathId(const TPathId& pathId) { + OlapStorePathId = pathId; + Description.MutableColumnStorePathId()->SetOwnerId(pathId.OwnerId); + Description.MutableColumnStorePathId()->SetLocalId(pathId.LocalPathId); + } + + static TColumnTableInfo::TPtr BuildTableWithAlter(const TColumnTableInfo& initialTable, const NKikimrSchemeOp::TAlterColumnTable& alterBody); + + bool IsStandalone() const { + return !OwnedColumnShards.empty(); + } + + const TAggregatedStats& GetStats() const { + return Stats; + } + + void UpdateShardStats(const TShardIdx shardIdx, const TPartitionStats& newStats) { + Stats.Aggregated.PartCount = ColumnShards.size(); + Stats.PartitionStats[shardIdx]; // insert if none + Stats.UpdateShardStats(shardIdx, newStats); + } + + void UpdateTableStats(const TPathId& pathId, const TPartitionStats& newStats) { + Stats.UpdateTableStats(pathId, newStats); + } +}; + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/table/ya.make b/ydb/core/tx/schemeshard/olap/table/ya.make new file mode 100644 index 0000000000..6032784bee --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/table/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +SRCS( + table.cpp +) + +PEERDIR( + ydb/core/protos + ydb/core/tablet_flat +) + +END() diff --git a/ydb/core/tx/schemeshard/olap/ya.make b/ydb/core/tx/schemeshard/olap/ya.make index 224d54e7e4..be40fc04d2 100644 --- a/ydb/core/tx/schemeshard/olap/ya.make +++ b/ydb/core/tx/schemeshard/olap/ya.make @@ -8,6 +8,10 @@ PEERDIR( ydb/core/tx/schemeshard/olap/operations ydb/core/tx/schemeshard/olap/statistics ydb/core/tx/schemeshard/olap/options + ydb/core/tx/schemeshard/olap/layout + ydb/core/tx/schemeshard/olap/manager + ydb/core/tx/schemeshard/olap/store + ydb/core/tx/schemeshard/olap/table ) END() diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 18543a2122..486b4c4829 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4538,7 +4538,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { NKikimrSchemeOp::TColumnStoreSharding sharding; Y_ABORT_UNLESS(sharding.ParseFromString(rowset.GetValue<Schema::OlapStores::Sharding>())); - TOlapStoreInfo::TPtr storeInfo = new TOlapStoreInfo(alterVersion, std::move(sharding)); + TOlapStoreInfo::TPtr storeInfo = std::make_shared<TOlapStoreInfo>(alterVersion, std::move(sharding)); storeInfo->ParseFromLocalDB(description); Self->OlapStores[pathId] = storeInfo; Self->IncrementPathDbRefCount(pathId); @@ -4572,7 +4572,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { Y_VERIFY_S(Self->OlapStores.contains(pathId), "Cannot load alter for olap store " << pathId); - TOlapStoreInfo::TPtr storeInfo = new TOlapStoreInfo(alterVersion, std::move(sharding), std::move(alterBody)); + TOlapStoreInfo::TPtr storeInfo = std::make_shared<TOlapStoreInfo>(alterVersion, std::move(sharding), std::move(alterBody)); storeInfo->ParseFromLocalDB(description); Self->OlapStores[pathId]->AlterData = storeInfo; @@ -4602,7 +4602,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { rowset.GetValue<Schema::ColumnTables::StandaloneSharding>())); } - auto tableInfo = Self->ColumnTables.BuildNew(pathId, new TColumnTableInfo(alterVersion, + auto tableInfo = Self->ColumnTables.BuildNew(pathId, std::make_shared<TColumnTableInfo>(alterVersion, std::move(description), std::move(sharding), std::move(storeSharding))); Self->IncrementPathDbRefCount(pathId); @@ -4649,7 +4649,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { Y_VERIFY_S(Self->ColumnTables.contains(pathId), "Cannot load alter for olap table " << pathId); - TColumnTableInfo::TPtr alterData = new TColumnTableInfo(alterVersion, + TColumnTableInfo::TPtr alterData = std::make_shared<TColumnTableInfo>(alterVersion, std::move(description), std::move(sharding), std::move(storeSharding), std::move(alterBody)); auto ctInfo = Self->ColumnTables.TakeVerified(pathId); ctInfo->AlterData = alterData; diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 77a5260827..3cfd21feed 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -12,13 +12,14 @@ #include "schemeshard_path.h" #include "schemeshard_domain_links.h" #include "schemeshard_info_types.h" -#include "schemeshard_tables_storage.h" #include "schemeshard_tx_infly.h" #include "schemeshard_utils.h" #include "schemeshard_schema.h" #include "schemeshard__operation.h" #include "schemeshard__stats.h" +#include "olap/manager/manager.h" + #include <ydb/core/base/hive.h> #include <ydb/core/base/storage_pools.h> #include <ydb/core/base/subdomain.h> diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index 3de1ae1d70..37424932b2 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -1,5 +1,4 @@ #include "schemeshard_info_types.h" -#include "schemeshard_tables_storage.h" #include "schemeshard_path.h" #include "schemeshard_utils.h" @@ -584,49 +583,6 @@ inline THashMap<ui32, size_t> DeduplicateRepeatedById( } -bool TOlapStoreInfo::ILayoutPolicy::Layout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const { - if (!DoLayout(currentLayout, shardsCount, result, isNewGroup)) { - return false; - } - Y_ABORT_UNLESS(result.size() == shardsCount); - return true; -} - -bool TOlapStoreInfo::TIdentityGroupsLayout::DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const { - isNewGroup = false; - for (auto&& i : currentLayout.GetGroups()) { - if (i.GetTableIds().Size() == 0 && i.GetShardIds().Size() >= shardsCount) { - result = i.GetShardIds().GetIdsVector(shardsCount); - isNewGroup = true; - return true; - } - if (i.GetShardIds().Size() != shardsCount) { - continue; - } - result = i.GetShardIds().GetIdsVector(); - return true; - } - return false; -} - -bool TOlapStoreInfo::TMinimalTablesCountLayout::DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const { - isNewGroup = true; - std::vector<ui64> resultLocal; - for (auto&& i : currentLayout.GetGroups()) { - if (i.GetTableIds().Size() > 0) { - isNewGroup = false; - } - for (auto&& s : i.GetShardIds()) { - resultLocal.emplace_back(s); - if (resultLocal.size() == shardsCount) { - std::swap(result, resultLocal); - return true; - } - } - } - return false; -} - NKikimrSchemeOp::TPartitionConfig TPartitionConfigMerger::DefaultConfig(const TAppData* appData) { NKikimrSchemeOp::TPartitionConfig cfg; @@ -2315,196 +2271,6 @@ NKikimr::NSchemeShard::TBillingStats::operator bool() const { return Rows || Bytes; } -TOlapStoreInfo::TOlapStoreInfo( - ui64 alterVersion, - NKikimrSchemeOp::TColumnStoreSharding&& sharding, - TMaybe<NKikimrSchemeOp::TAlterColumnStore>&& alterBody) - : AlterVersion(alterVersion) - , Sharding(std::move(sharding)) - , AlterBody(std::move(alterBody)) -{ - for (const auto& shardIdx : Sharding.GetColumnShards()) { - ColumnShards.push_back(TShardIdx( - TOwnerId(shardIdx.GetOwnerId()), - TLocalShardIdx(shardIdx.GetLocalId()))); - } -} - -TOlapStoreInfo::TPtr TOlapStoreInfo::BuildStoreWithAlter(const TOlapStoreInfo& initialStore, const NKikimrSchemeOp::TAlterColumnStore& alterBody) { - TOlapStoreInfo::TPtr alterData = new TOlapStoreInfo(initialStore); - alterData->AlterVersion++; - alterData->AlterBody.ConstructInPlace(alterBody); - return alterData; -} - -void TOlapStoreInfo::SerializeDescription(NKikimrSchemeOp::TColumnStoreDescription& descriptionProto) const { - descriptionProto.SetName(Name); - descriptionProto.SetColumnShardCount(ColumnShards.size()); - descriptionProto.SetNextSchemaPresetId(NextSchemaPresetId); - descriptionProto.SetNextTtlSettingsPresetId(NextTtlSettingsPresetId); - - for (const auto& [name, preset] : SchemaPresets) { - Y_UNUSED(name); - auto presetProto = descriptionProto.AddSchemaPresets(); - preset.Serialize(*presetProto); - } -} - -void TOlapStoreInfo::ParseFromLocalDB(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto) { - StorageConfig = descriptionProto.GetStorageConfig(); - NextTtlSettingsPresetId = descriptionProto.GetNextTtlSettingsPresetId(); - NextSchemaPresetId = descriptionProto.GetNextSchemaPresetId(); - Name = descriptionProto.GetName(); - - size_t schemaPresetIndex = 0; - for (const auto& presetProto : descriptionProto.GetSchemaPresets()) { - Y_ABORT_UNLESS(!SchemaPresets.contains(presetProto.GetId())); - auto& preset = SchemaPresets[presetProto.GetId()]; - preset.ParseFromLocalDB(presetProto); - preset.SetProtoIndex(schemaPresetIndex++); - SchemaPresetByName[preset.GetName()] = preset.GetId(); - } - SerializeDescription(Description); -} - -bool TOlapStoreInfo::UpdatePreset(const TString& presetName, const TOlapSchemaUpdate& schemaUpdate, IErrorCollector& errors) { - const ui32 presetId = SchemaPresetByName.at(presetName); - auto& currentPreset = SchemaPresets.at(presetId); - if (!currentPreset.Update(schemaUpdate, errors)) { - return false; - } - - NKikimrSchemeOp::TColumnTableSchemaPreset schemaUpdateProto; - currentPreset.Serialize(schemaUpdateProto); - - auto mutablePresetProto = Description.MutableSchemaPresets(currentPreset.GetProtoIndex()); - *mutablePresetProto = schemaUpdateProto; - return true; -} - -bool TOlapStoreInfo::ParseFromRequest(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto, IErrorCollector& errors) { - AlterVersion = 1; - if (descriptionProto.GetRESERVED_MetaShardCount() != 0) { - errors.AddError("trying to create OLAP store with meta shards (not supported yet)"); - return false; - } - - if (!descriptionProto.HasColumnShardCount()) { - errors.AddError("trying to create OLAP store without shards number specified"); - return false; - } - - if (descriptionProto.GetColumnShardCount() == 0) { - errors.AddError("trying to create OLAP store without zero shards"); - return false; - } - - for (auto& presetProto : descriptionProto.GetRESERVED_TtlSettingsPresets()) { - Y_UNUSED(presetProto); - errors.AddError("TTL presets are not supported"); - return false; - } - - Name = descriptionProto.GetName(); - StorageConfig = descriptionProto.GetStorageConfig(); - // Make it easier by having data channel count always specified internally - if (!StorageConfig.HasDataChannelCount()) { - StorageConfig.SetDataChannelCount(1); - } - - size_t protoIndex = 0; - for (const auto& presetProto : descriptionProto.GetSchemaPresets()) { - TOlapStoreSchemaPreset preset; - if (!preset.ParseFromRequest(presetProto, errors)) { - return false; - } - if (SchemaPresets.contains(NextSchemaPresetId) || SchemaPresetByName.contains(preset.GetName())) { - errors.AddError(TStringBuilder() << "Duplicate schema preset " << NextSchemaPresetId << " with name '" << preset.GetName() << "'"); - return false; - } - preset.SetId(NextSchemaPresetId++); - preset.SetProtoIndex(protoIndex++); - - TOlapSchemaUpdate schemaDiff; - if (!schemaDiff.Parse(presetProto.GetSchema(), errors)) { - return false; - } - - if (!preset.Update(schemaDiff, errors)) { - return false; - } - - SchemaPresetByName[preset.GetName()] = preset.GetId(); - SchemaPresets[preset.GetId()] = std::move(preset); - } - - if (!SchemaPresetByName.contains("default") || SchemaPresets.size() > 1) { - errors.AddError("A single schema preset named 'default' is required"); - return false; - } - - ColumnShards.resize(descriptionProto.GetColumnShardCount()); - SerializeDescription(Description); - return true; -} - -TOlapStoreInfo::ILayoutPolicy::TPtr TOlapStoreInfo::GetTablesLayoutPolicy() const { - ILayoutPolicy::TPtr result; - if (AppData()->ColumnShardConfig.GetTablesStorageLayoutPolicy().HasMinimalTables()) { - result = std::make_shared<TMinimalTablesCountLayout>(); - } else if (AppData()->ColumnShardConfig.GetTablesStorageLayoutPolicy().HasIdentityGroups()) { - result = std::make_shared<TIdentityGroupsLayout>(); - } else { - result = std::make_shared<TMinimalTablesCountLayout>(); - } - return result; -} - -TColumnTableInfo::TColumnTableInfo( - ui64 alterVersion, - NKikimrSchemeOp::TColumnTableDescription&& description, - NKikimrSchemeOp::TColumnTableSharding&& sharding, - TMaybe<NKikimrSchemeOp::TColumnStoreSharding>&& standaloneSharding, - TMaybe<NKikimrSchemeOp::TAlterColumnTable>&& alterBody) - : AlterVersion(alterVersion) - , Description(std::move(description)) - , Sharding(std::move(sharding)) - , StandaloneSharding(std::move(standaloneSharding)) - , AlterBody(std::move(alterBody)) -{ - if (Description.HasColumnStorePathId()) { - OlapStorePathId = TPathId( - TOwnerId(Description.GetColumnStorePathId().GetOwnerId()), - TLocalPathId(Description.GetColumnStorePathId().GetLocalId())); - } - - if (Description.HasSchema()) { - TOlapSchema schema; - schema.ParseFromLocalDB(Description.GetSchema()); - } - - ColumnShards.reserve(Sharding.GetColumnShards().size()); - for (ui64 columnShard : Sharding.GetColumnShards()) { - ColumnShards.push_back(columnShard); - } - - if (StandaloneSharding) { - OwnedColumnShards.reserve(StandaloneSharding->GetColumnShards().size()); - for (const auto& shardIdx : StandaloneSharding->GetColumnShards()) { - OwnedColumnShards.push_back(TShardIdx( - TOwnerId(shardIdx.GetOwnerId()), - TLocalShardIdx(shardIdx.GetLocalId()))); - } - } -} - -TColumnTableInfo::TPtr TColumnTableInfo::BuildTableWithAlter(const TColumnTableInfo& initialTable, const NKikimrSchemeOp::TAlterColumnTable& alterBody) { - TColumnTableInfo::TPtr alterData = new TColumnTableInfo(initialTable); - alterData->AlterBody.ConstructInPlace(alterBody); - ++alterData->AlterVersion; - return alterData; -} - TSequenceInfo::TSequenceInfo( ui64 alterVersion, NKikimrSchemeOp::TSequenceDescription&& description, diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index f698f00829..3863e627b1 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -884,154 +884,6 @@ public: } }; -class TColumnTablesLayout; - -struct TOlapStoreInfo : TSimpleRefCount<TOlapStoreInfo> { -private: - TString Name; - ui64 NextSchemaPresetId = 1; - ui64 NextTtlSettingsPresetId = 1; - NKikimrSchemeOp::TColumnStorageConfig StorageConfig; - NKikimrSchemeOp::TColumnStoreDescription Description; - ui64 AlterVersion = 0; -public: - using TPtr = TIntrusivePtr<TOlapStoreInfo>; - - class ILayoutPolicy { - protected: - virtual bool DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const = 0; - public: - using TPtr = std::shared_ptr<ILayoutPolicy>; - virtual ~ILayoutPolicy() = default; - bool Layout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const; - }; - - class TMinimalTablesCountLayout: public ILayoutPolicy { - protected: - virtual bool DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const override; - }; - - class TIdentityGroupsLayout: public ILayoutPolicy { - protected: - virtual bool DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const override; - }; - - TPtr AlterData; - - const NKikimrSchemeOp::TColumnStoreDescription& GetDescription() const { - return Description; - } - - NKikimrSchemeOp::TColumnStoreSharding Sharding; - TMaybe<NKikimrSchemeOp::TAlterColumnStore> AlterBody; - - TVector<TShardIdx> ColumnShards; - - THashMap<ui32, TOlapStoreSchemaPreset> SchemaPresets; - THashMap<TString, ui32> SchemaPresetByName; - - THashSet<TPathId> ColumnTables; - THashSet<TPathId> ColumnTablesUnderOperation; - TAggregatedStats Stats; - - TOlapStoreInfo() = default; - TOlapStoreInfo(ui64 alterVersion, - NKikimrSchemeOp::TColumnStoreSharding&& sharding, - TMaybe<NKikimrSchemeOp::TAlterColumnStore>&& alterBody = Nothing()); - - static TOlapStoreInfo::TPtr BuildStoreWithAlter(const TOlapStoreInfo& initialStore, const NKikimrSchemeOp::TAlterColumnStore& alterBody); - - const NKikimrSchemeOp::TColumnStorageConfig& GetStorageConfig() const { - return StorageConfig; - } - - const TVector<TShardIdx>& GetColumnShards() const { - return ColumnShards; - } - - ui64 GetAlterVersion() const { - return AlterVersion; - } - - void ApplySharding(const TVector<TShardIdx>& shardsIndexes) { - Y_ABORT_UNLESS(ColumnShards.size() == shardsIndexes.size()); - Sharding.ClearColumnShards(); - for (ui64 i = 0; i < ColumnShards.size(); ++i) { - const auto& idx = shardsIndexes[i]; - ColumnShards[i] = idx; - auto* shardInfoProto = Sharding.AddColumnShards(); - shardInfoProto->SetOwnerId(idx.GetOwnerId()); - shardInfoProto->SetLocalId(idx.GetLocalId().GetValue()); - } - } - void SerializeDescription(NKikimrSchemeOp::TColumnStoreDescription& descriptionProto) const; - void ParseFromLocalDB(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto); - bool ParseFromRequest(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto, IErrorCollector& errors); - bool UpdatePreset(const TString& presetName, const TOlapSchemaUpdate& schemaUpdate, IErrorCollector& errors); - - const TAggregatedStats& GetStats() const { - return Stats; - } - - ILayoutPolicy::TPtr GetTablesLayoutPolicy() const; - - void UpdateShardStats(TShardIdx shardIdx, const TPartitionStats& newStats) { - Stats.Aggregated.PartCount = ColumnShards.size(); - Stats.PartitionStats[shardIdx]; // insert if none - Stats.UpdateShardStats(shardIdx, newStats); - } -}; - -struct TColumnTableInfo : TSimpleRefCount<TColumnTableInfo> { - using TPtr = TIntrusivePtr<TColumnTableInfo>; - - ui64 AlterVersion = 0; - TPtr AlterData; - - NKikimrSchemeOp::TColumnTableDescription Description; - NKikimrSchemeOp::TColumnTableSharding Sharding; - TMaybe<NKikimrSchemeOp::TColumnStoreSharding> StandaloneSharding; - TMaybe<NKikimrSchemeOp::TAlterColumnTable> AlterBody; - - TMaybe<TPathId> OlapStorePathId; // PathId of the table store - - TVector<ui64> ColumnShards; // Current list of column shards - TVector<TShardIdx> OwnedColumnShards; - TAggregatedStats Stats; - - TColumnTableInfo() = default; - TColumnTableInfo(ui64 alterVersion, NKikimrSchemeOp::TColumnTableDescription&& description, - NKikimrSchemeOp::TColumnTableSharding&& sharding, - TMaybe<NKikimrSchemeOp::TColumnStoreSharding>&& standaloneSharding, - TMaybe<NKikimrSchemeOp::TAlterColumnTable>&& alterBody = Nothing()); - - void SetOlapStorePathId(const TPathId& pathId) { - OlapStorePathId = pathId; - Description.MutableColumnStorePathId()->SetOwnerId(pathId.OwnerId); - Description.MutableColumnStorePathId()->SetLocalId(pathId.LocalPathId); - } - - static TColumnTableInfo::TPtr BuildTableWithAlter(const TColumnTableInfo& initialTable, const NKikimrSchemeOp::TAlterColumnTable& alterBody); - - bool IsStandalone() const { - return !OwnedColumnShards.empty(); - } - - const TAggregatedStats& GetStats() const { - return Stats; - } - - void UpdateShardStats(const TShardIdx shardIdx, const TPartitionStats& newStats) { - Stats.Aggregated.PartCount = ColumnShards.size(); - Stats.PartitionStats[shardIdx]; // insert if none - Stats.UpdateShardStats(shardIdx, newStats); - } - - void UpdateTableStats(const TPathId& pathId, const TPartitionStats& newStats) { - Stats.UpdateTableStats(pathId, newStats); - } -}; - struct TTopicStats { TMessageSeqNo SeqNo; diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make index 32a4329c07..bc4f84fa87 100644 --- a/ydb/core/tx/schemeshard/ya.make +++ b/ydb/core/tx/schemeshard/ya.make @@ -193,7 +193,6 @@ SRCS( schemeshard_svp_migration.h schemeshard_svp_migration.cpp schemeshard_tx_infly.h - schemeshard_tables_storage.cpp schemeshard_types.cpp schemeshard_types.h schemeshard_user_attr_limits.h |