aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorivanmorozov333 <ivanmorozov@hotmail.com>2024-04-19 22:09:17 +0300
committerGitHub <noreply@github.com>2024-04-19 22:09:17 +0300
commit32415d9ecd85e93bf2490b103002e9bf1b1841d6 (patch)
treea37380f7bde8361f2a04ae61590908a25af7f9fa
parent7147f14eafadc1df4151d684336579a4c29ce7be (diff)
downloadydb-32415d9ecd85e93bf2490b103002e9bf1b1841d6.tar.gz
separate library for olap modification schemas (#3928)
-rw-r--r--ydb/core/protos/flat_scheme_op.proto1
-rw-r--r--ydb/core/tx/schemeshard/olap/layout/layout.cpp25
-rw-r--r--ydb/core/tx/schemeshard/olap/layout/layout.h (renamed from ydb/core/tx/schemeshard/schemeshard_tables_storage.h)122
-rw-r--r--ydb/core/tx/schemeshard/olap/layout/ya.make14
-rw-r--r--ydb/core/tx/schemeshard/olap/manager/manager.cpp (renamed from ydb/core/tx/schemeshard/schemeshard_tables_storage.cpp)22
-rw-r--r--ydb/core/tx/schemeshard/olap/manager/manager.h119
-rw-r--r--ydb/core/tx/schemeshard/olap/manager/ya.make13
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/create_store.cpp4
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/create_table.cpp2
-rw-r--r--ydb/core/tx/schemeshard/olap/store/store.cpp192
-rw-r--r--ydb/core/tx/schemeshard/olap/store/store.h107
-rw-r--r--ydb/core/tx/schemeshard/olap/store/ya.make12
-rw-r--r--ydb/core/tx/schemeshard/olap/table/table.cpp49
-rw-r--r--ydb/core/tx/schemeshard/olap/table/table.h58
-rw-r--r--ydb/core/tx/schemeshard/olap/table/ya.make12
-rw-r--r--ydb/core/tx/schemeshard/olap/ya.make4
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__init.cpp8
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_impl.h3
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_info_types.cpp234
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_info_types.h148
-rw-r--r--ydb/core/tx/schemeshard/ya.make1
21 files changed, 620 insertions, 530 deletions
diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
index 84e9a1c037..d25c953c43 100644
--- a/ydb/core/protos/flat_scheme_op.proto
+++ b/ydb/core/protos/flat_scheme_op.proto
@@ -621,7 +621,6 @@ message TColumnStorageConfig {
// This is used internally by schemeshard
message TColumnStoreSharding {
- repeated TShardIdx MetaShards = 1;
repeated TShardIdx ColumnShards = 2;
}
diff --git a/ydb/core/tx/schemeshard/olap/layout/layout.cpp b/ydb/core/tx/schemeshard/olap/layout/layout.cpp
new file mode 100644
index 0000000000..40791d3b59
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/layout/layout.cpp
@@ -0,0 +1,25 @@
+#include "layout.h"
+#include <ydb/core/tx/schemeshard/schemeshard_impl.h>
+
+namespace NKikimr::NSchemeShard {
+
+std::vector<ui64> TColumnTablesLayout::ShardIdxToTabletId(const std::vector<TShardIdx>& shards, const TSchemeShard& ss) {
+ std::vector<ui64> result;
+ for (const auto& shardIdx : shards) {
+ auto* shardInfo = ss.ShardInfos.FindPtr(shardIdx);
+ Y_ABORT_UNLESS(shardInfo, "ColumnShard not found");
+ result.emplace_back(shardInfo->TabletID.GetValue());
+ }
+ return result;
+}
+
+TColumnTablesLayout TColumnTablesLayout::BuildTrivial(const std::vector<ui64>& tabletIds) {
+ TTableIdsGroup emptyGroup;
+ TShardIdsGroup shardIdsGroup;
+ for (const auto& tabletId : tabletIds) {
+ shardIdsGroup.AddId(tabletId);
+ }
+ return TColumnTablesLayout({ TTablesGroup(std::move(emptyGroup), std::move(shardIdsGroup)) });
+}
+
+}
diff --git a/ydb/core/tx/schemeshard/schemeshard_tables_storage.h b/ydb/core/tx/schemeshard/olap/layout/layout.h
index 0360512959..3f01733204 100644
--- a/ydb/core/tx/schemeshard/schemeshard_tables_storage.h
+++ b/ydb/core/tx/schemeshard/olap/layout/layout.h
@@ -1,10 +1,12 @@
#pragma once
-#include "schemeshard_info_types.h"
-
#include <ydb/core/scheme/scheme_pathid.h>
+#include <ydb/core/tx/schemeshard/schemeshard_identificators.h>
#include <ydb/library/accessor/accessor.h>
-#include <util/digest/numeric.h>
+
+#include <util/system/types.h>
+
+#include <set>
namespace NKikimr::NSchemeShard {
@@ -149,118 +151,4 @@ public:
static TColumnTablesLayout BuildTrivial(const std::vector<ui64>& tabletIds);
};
-class TTablesStorage {
-private:
- THashMap<TPathId, TColumnTableInfo::TPtr> Tables;
- THashMap<TString, std::set<TPathId>> PathsByTieringId;
- THashMap<ui64, TColumnTablesLayout::TTableIdsGroup> TablesByShard;
-
- void OnAddObject(const TPathId& pathId, TColumnTableInfo::TPtr object);
- void OnRemoveObject(const TPathId& pathId, TColumnTableInfo::TPtr object);
- TColumnTableInfo::TPtr ExtractPtr(const TPathId& id);
-public:
- std::unordered_set<TPathId> GetAllPathIds() const;
-
- TColumnTablesLayout GetTablesLayout(const std::vector<ui64>& tabletIds) const;
-
- const std::set<TPathId>& GetTablesWithTiering(const TString& tieringId) const;
-
- class TTableReadGuard {
- protected:
- TColumnTableInfo::TPtr Object;
- public:
- bool operator!() const {
- return !Object;
- }
-
- TTableReadGuard(TColumnTableInfo::TPtr object)
- : Object(object) {
- }
- const TColumnTableInfo* operator->() const {
- Y_DEBUG_ABORT_UNLESS(Object);
- return Object.Get();
- }
- const TColumnTableInfo& operator*() const {
- Y_DEBUG_ABORT_UNLESS(Object);
- return *Object;
- }
- const TColumnTableInfo* operator->() {
- Y_DEBUG_ABORT_UNLESS(Object);
- return Object.Get();
- }
- };
-
- class TTableCreateOperator: public TTableReadGuard {
- private:
- using TBase = TTableReadGuard;
- public:
- using TBase::TBase;
- bool InitShardingTablets(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, TOlapStoreInfo::ILayoutPolicy::TPtr layoutPolicy, bool& isNewGroup) const;
- };
-
- class TTableCreatedGuard: public TTableCreateOperator, TMoveOnly {
- protected:
- const TPathId PathId;
- TTablesStorage& Owner;
- public:
- TTableCreatedGuard(TTablesStorage& owner, const TPathId& id, TColumnTableInfo::TPtr object)
- : TTableCreateOperator(object)
- , PathId(id)
- , Owner(owner)
-
- {
- Y_ABORT_UNLESS(!Owner.contains(id));
- }
-
- TColumnTableInfo::TPtr GetPtr() const {
- return Object;
- }
-
- TTableCreatedGuard(TTablesStorage& owner, const TPathId& id)
- : TTableCreateOperator(new TColumnTableInfo)
- , PathId(id)
- , Owner(owner) {
- Y_ABORT_UNLESS(!Owner.contains(id));
- }
- TColumnTableInfo* operator->() {
- return Object.Get();
- }
- const TColumnTableInfo* operator->() const {
- return Object.Get();
- }
- ~TTableCreatedGuard() {
- Y_ABORT_UNLESS(Owner.Tables.emplace(PathId, Object).second);
- Owner.OnAddObject(PathId, Object);
- }
- };
-
- class TTableExtractedGuard: public TTableCreatedGuard {
- private:
- void UseAlterDataVerified();
- public:
- TTableExtractedGuard(TTablesStorage& owner, const TPathId& id, TColumnTableInfo::TPtr object, const bool extractAlter)
- : TTableCreatedGuard(owner, id, object)
- {
- Owner.OnRemoveObject(PathId, object);
- if (extractAlter) {
- UseAlterDataVerified();
- }
- }
- };
-
- TTableCreatedGuard BuildNew(const TPathId& id);
- TTableCreatedGuard BuildNew(const TPathId& id, TColumnTableInfo::TPtr object);
- TTableExtractedGuard TakeVerified(const TPathId& id);
- TTableExtractedGuard TakeAlterVerified(const TPathId& id);
-
- bool contains(const TPathId& id) const {
- return Tables.contains(id);
- }
- TTableReadGuard GetVerified(const TPathId& id) const;
- TTableReadGuard at(const TPathId& id) const {
- return TTableReadGuard(Tables.at(id));
- }
- size_t Drop(const TPathId& id);
-};
-
}
diff --git a/ydb/core/tx/schemeshard/olap/layout/ya.make b/ydb/core/tx/schemeshard/olap/layout/ya.make
new file mode 100644
index 0000000000..f22a412813
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/layout/ya.make
@@ -0,0 +1,14 @@
+LIBRARY()
+
+SRCS(
+ layout.cpp
+)
+
+PEERDIR(
+ ydb/core/scheme
+ ydb/core/tablet_flat
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/core/tx/schemeshard/schemeshard_tables_storage.cpp b/ydb/core/tx/schemeshard/olap/manager/manager.cpp
index 5961d458fa..c4c24e31f3 100644
--- a/ydb/core/tx/schemeshard/schemeshard_tables_storage.cpp
+++ b/ydb/core/tx/schemeshard/olap/manager/manager.cpp
@@ -1,5 +1,4 @@
-#include "schemeshard_tables_storage.h"
-#include "schemeshard_impl.h"
+#include "manager.h"
namespace NKikimr::NSchemeShard {
@@ -113,25 +112,6 @@ void TTablesStorage::TTableExtractedGuard::UseAlterDataVerified() {
Object = alterInfo;
}
-std::vector<ui64> TColumnTablesLayout::ShardIdxToTabletId(const std::vector<TShardIdx>& shards, const TSchemeShard& ss) {
- std::vector<ui64> result;
- for (const auto& shardIdx : shards) {
- auto* shardInfo = ss.ShardInfos.FindPtr(shardIdx);
- Y_ABORT_UNLESS(shardInfo, "ColumnShard not found");
- result.emplace_back(shardInfo->TabletID.GetValue());
- }
- return result;
-}
-
-TColumnTablesLayout TColumnTablesLayout::BuildTrivial(const std::vector<ui64>& tabletIds) {
- TTableIdsGroup emptyGroup;
- TShardIdsGroup shardIdsGroup;
- for (const auto& tabletId : tabletIds) {
- shardIdsGroup.AddId(tabletId);
- }
- return TColumnTablesLayout({ TTablesGroup(std::move(emptyGroup), std::move(shardIdsGroup)) });
-}
-
bool TTablesStorage::TTableCreateOperator::InitShardingTablets(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, TOlapStoreInfo::ILayoutPolicy::TPtr layoutPolicy, bool& isNewGroup) const {
if (!layoutPolicy->Layout(currentLayout, shardsCount, Object->ColumnShards, isNewGroup)) {
ALS_ERROR(NKikimrServices::FLAT_TX_SCHEMESHARD) << "cannot layout new table with " << shardsCount << " shards";
diff --git a/ydb/core/tx/schemeshard/olap/manager/manager.h b/ydb/core/tx/schemeshard/olap/manager/manager.h
new file mode 100644
index 0000000000..4ed09e97bc
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/manager/manager.h
@@ -0,0 +1,119 @@
+#pragma once
+#include <ydb/core/scheme/scheme_pathid.h>
+#include <ydb/core/tx/schemeshard/olap/layout/layout.h>
+#include <ydb/core/tx/schemeshard/olap/store/store.h>
+#include <ydb/core/tx/schemeshard/olap/table/table.h>
+
+namespace NKikimr::NSchemeShard {
+
+class TTablesStorage {
+private:
+ THashMap<TPathId, TColumnTableInfo::TPtr> Tables;
+ THashMap<TString, std::set<TPathId>> PathsByTieringId;
+ THashMap<ui64, TColumnTablesLayout::TTableIdsGroup> TablesByShard;
+
+ void OnAddObject(const TPathId& pathId, TColumnTableInfo::TPtr object);
+ void OnRemoveObject(const TPathId& pathId, TColumnTableInfo::TPtr object);
+ TColumnTableInfo::TPtr ExtractPtr(const TPathId& id);
+public:
+ std::unordered_set<TPathId> GetAllPathIds() const;
+
+ TColumnTablesLayout GetTablesLayout(const std::vector<ui64>& tabletIds) const;
+
+ const std::set<TPathId>& GetTablesWithTiering(const TString& tieringId) const;
+
+ class TTableReadGuard {
+ protected:
+ TColumnTableInfo::TPtr Object;
+ public:
+ bool operator!() const {
+ return !Object;
+ }
+
+ TTableReadGuard(TColumnTableInfo::TPtr object)
+ : Object(object) {
+ }
+ const TColumnTableInfo* operator->() const {
+ Y_DEBUG_ABORT_UNLESS(Object);
+ return Object.get();
+ }
+ const TColumnTableInfo& operator*() const {
+ Y_DEBUG_ABORT_UNLESS(Object);
+ return *Object;
+ }
+ };
+
+ class TTableCreateOperator: public TTableReadGuard {
+ private:
+ using TBase = TTableReadGuard;
+ public:
+ using TBase::TBase;
+ bool InitShardingTablets(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, TOlapStoreInfo::ILayoutPolicy::TPtr layoutPolicy, bool& isNewGroup) const;
+ };
+
+ class TTableCreatedGuard: public TTableCreateOperator, TMoveOnly {
+ protected:
+ const TPathId PathId;
+ TTablesStorage& Owner;
+ public:
+ TTableCreatedGuard(TTablesStorage& owner, const TPathId& id, TColumnTableInfo::TPtr object)
+ : TTableCreateOperator(object)
+ , PathId(id)
+ , Owner(owner)
+
+ {
+ Y_ABORT_UNLESS(!Owner.contains(id));
+ }
+
+ TColumnTableInfo::TPtr GetPtr() const {
+ return Object;
+ }
+
+ TTableCreatedGuard(TTablesStorage& owner, const TPathId& id)
+ : TTableCreateOperator(std::make_shared<TColumnTableInfo>())
+ , PathId(id)
+ , Owner(owner) {
+ Y_ABORT_UNLESS(!Owner.contains(id));
+ }
+ TColumnTableInfo* operator->() {
+ return Object.get();
+ }
+ const TColumnTableInfo* operator->() const {
+ return Object.get();
+ }
+ ~TTableCreatedGuard() {
+ Y_ABORT_UNLESS(Owner.Tables.emplace(PathId, Object).second);
+ Owner.OnAddObject(PathId, Object);
+ }
+ };
+
+ class TTableExtractedGuard: public TTableCreatedGuard {
+ private:
+ void UseAlterDataVerified();
+ public:
+ TTableExtractedGuard(TTablesStorage& owner, const TPathId& id, TColumnTableInfo::TPtr object, const bool extractAlter)
+ : TTableCreatedGuard(owner, id, object)
+ {
+ Owner.OnRemoveObject(PathId, object);
+ if (extractAlter) {
+ UseAlterDataVerified();
+ }
+ }
+ };
+
+ TTableCreatedGuard BuildNew(const TPathId& id);
+ TTableCreatedGuard BuildNew(const TPathId& id, TColumnTableInfo::TPtr object);
+ TTableExtractedGuard TakeVerified(const TPathId& id);
+ TTableExtractedGuard TakeAlterVerified(const TPathId& id);
+
+ bool contains(const TPathId& id) const {
+ return Tables.contains(id);
+ }
+ TTableReadGuard GetVerified(const TPathId& id) const;
+ TTableReadGuard at(const TPathId& id) const {
+ return TTableReadGuard(Tables.at(id));
+ }
+ size_t Drop(const TPathId& id);
+};
+
+}
diff --git a/ydb/core/tx/schemeshard/olap/manager/ya.make b/ydb/core/tx/schemeshard/olap/manager/ya.make
new file mode 100644
index 0000000000..a5b6e0e4c2
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/manager/ya.make
@@ -0,0 +1,13 @@
+LIBRARY()
+
+SRCS(
+ manager.cpp
+)
+
+PEERDIR(
+ ydb/core/scheme
+ ydb/core/tx/schemeshard/olap/table
+ ydb/core/tx/schemeshard/olap/layout
+)
+
+END()
diff --git a/ydb/core/tx/schemeshard/olap/operations/create_store.cpp b/ydb/core/tx/schemeshard/olap/operations/create_store.cpp
index 776b8d2737..bc93fcb88b 100644
--- a/ydb/core/tx/schemeshard/olap/operations/create_store.cpp
+++ b/ydb/core/tx/schemeshard/olap/operations/create_store.cpp
@@ -397,7 +397,7 @@ public:
}
TProposeErrorCollector errors(*result);
- TOlapStoreInfo::TPtr storeInfo = new TOlapStoreInfo;
+ TOlapStoreInfo::TPtr storeInfo = std::make_shared<TOlapStoreInfo>();
if (!storeInfo->ParseFromRequest(createDescription, errors)) {
return result;
}
@@ -438,7 +438,7 @@ public:
NIceDb::TNiceDb db(context.GetDB());
- TOlapStoreInfo::TPtr pending = new TOlapStoreInfo;
+ TOlapStoreInfo::TPtr pending = std::make_shared<TOlapStoreInfo>();
pending->AlterData = storeInfo;
context.SS->OlapStores[pathId] = pending;
context.SS->PersistOlapStore(db, pathId, *pending);
diff --git a/ydb/core/tx/schemeshard/olap/operations/create_table.cpp b/ydb/core/tx/schemeshard/olap/operations/create_table.cpp
index db77e7491a..d1ef7e9cf6 100644
--- a/ydb/core/tx/schemeshard/olap/operations/create_table.cpp
+++ b/ydb/core/tx/schemeshard/olap/operations/create_table.cpp
@@ -58,7 +58,7 @@ public:
}
TColumnTableInfo::TPtr BuildTableInfo(IErrorCollector& errors) const {
- TColumnTableInfo::TPtr tableInfo = new TColumnTableInfo;
+ TColumnTableInfo::TPtr tableInfo = std::make_shared<TColumnTableInfo>();
tableInfo->AlterVersion = 1;
BuildDescription(tableInfo->Description);
diff --git a/ydb/core/tx/schemeshard/olap/store/store.cpp b/ydb/core/tx/schemeshard/olap/store/store.cpp
new file mode 100644
index 0000000000..46153949a5
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/store/store.cpp
@@ -0,0 +1,192 @@
+#include "store.h"
+
+namespace NKikimr::NSchemeShard {
+
+bool TOlapStoreInfo::ILayoutPolicy::Layout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const {
+ if (!DoLayout(currentLayout, shardsCount, result, isNewGroup)) {
+ return false;
+ }
+ Y_ABORT_UNLESS(result.size() == shardsCount);
+ return true;
+}
+
+bool TOlapStoreInfo::TIdentityGroupsLayout::DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const {
+ isNewGroup = false;
+ for (auto&& i : currentLayout.GetGroups()) {
+ if (i.GetTableIds().Size() == 0 && i.GetShardIds().Size() >= shardsCount) {
+ result = i.GetShardIds().GetIdsVector(shardsCount);
+ isNewGroup = true;
+ return true;
+ }
+ if (i.GetShardIds().Size() != shardsCount) {
+ continue;
+ }
+ result = i.GetShardIds().GetIdsVector();
+ return true;
+ }
+ return false;
+}
+
+bool TOlapStoreInfo::TMinimalTablesCountLayout::DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const {
+ isNewGroup = true;
+ std::vector<ui64> resultLocal;
+ for (auto&& i : currentLayout.GetGroups()) {
+ if (i.GetTableIds().Size() > 0) {
+ isNewGroup = false;
+ }
+ for (auto&& s : i.GetShardIds()) {
+ resultLocal.emplace_back(s);
+ if (resultLocal.size() == shardsCount) {
+ std::swap(result, resultLocal);
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+TOlapStoreInfo::TOlapStoreInfo(
+ ui64 alterVersion,
+ NKikimrSchemeOp::TColumnStoreSharding&& sharding,
+ TMaybe<NKikimrSchemeOp::TAlterColumnStore>&& alterBody)
+ : AlterVersion(alterVersion)
+ , Sharding(std::move(sharding))
+ , AlterBody(std::move(alterBody)) {
+ for (const auto& shardIdx : Sharding.GetColumnShards()) {
+ ColumnShards.push_back(TShardIdx(
+ TOwnerId(shardIdx.GetOwnerId()),
+ TLocalShardIdx(shardIdx.GetLocalId())));
+ }
+}
+
+TOlapStoreInfo::TPtr TOlapStoreInfo::BuildStoreWithAlter(const TOlapStoreInfo& initialStore, const NKikimrSchemeOp::TAlterColumnStore& alterBody) {
+ TOlapStoreInfo::TPtr alterData = std::make_shared<TOlapStoreInfo>(initialStore);
+ alterData->AlterVersion++;
+ alterData->AlterBody.ConstructInPlace(alterBody);
+ return alterData;
+}
+
+void TOlapStoreInfo::SerializeDescription(NKikimrSchemeOp::TColumnStoreDescription& descriptionProto) const {
+ descriptionProto.SetName(Name);
+ descriptionProto.SetColumnShardCount(ColumnShards.size());
+ descriptionProto.SetNextSchemaPresetId(NextSchemaPresetId);
+ descriptionProto.SetNextTtlSettingsPresetId(NextTtlSettingsPresetId);
+
+ for (const auto& [name, preset] : SchemaPresets) {
+ Y_UNUSED(name);
+ auto presetProto = descriptionProto.AddSchemaPresets();
+ preset.Serialize(*presetProto);
+ }
+}
+
+void TOlapStoreInfo::ParseFromLocalDB(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto) {
+ StorageConfig = descriptionProto.GetStorageConfig();
+ NextTtlSettingsPresetId = descriptionProto.GetNextTtlSettingsPresetId();
+ NextSchemaPresetId = descriptionProto.GetNextSchemaPresetId();
+ Name = descriptionProto.GetName();
+
+ size_t schemaPresetIndex = 0;
+ for (const auto& presetProto : descriptionProto.GetSchemaPresets()) {
+ Y_ABORT_UNLESS(!SchemaPresets.contains(presetProto.GetId()));
+ auto& preset = SchemaPresets[presetProto.GetId()];
+ preset.ParseFromLocalDB(presetProto);
+ preset.SetProtoIndex(schemaPresetIndex++);
+ SchemaPresetByName[preset.GetName()] = preset.GetId();
+ }
+ SerializeDescription(Description);
+}
+
+bool TOlapStoreInfo::UpdatePreset(const TString& presetName, const TOlapSchemaUpdate& schemaUpdate, IErrorCollector& errors) {
+ const ui32 presetId = SchemaPresetByName.at(presetName);
+ auto& currentPreset = SchemaPresets.at(presetId);
+ if (!currentPreset.Update(schemaUpdate, errors)) {
+ return false;
+ }
+
+ NKikimrSchemeOp::TColumnTableSchemaPreset schemaUpdateProto;
+ currentPreset.Serialize(schemaUpdateProto);
+
+ auto mutablePresetProto = Description.MutableSchemaPresets(currentPreset.GetProtoIndex());
+ *mutablePresetProto = schemaUpdateProto;
+ return true;
+}
+
+bool TOlapStoreInfo::ParseFromRequest(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto, IErrorCollector& errors) {
+ AlterVersion = 1;
+ if (descriptionProto.GetRESERVED_MetaShardCount() != 0) {
+ errors.AddError("trying to create OLAP store with meta shards (not supported yet)");
+ return false;
+ }
+
+ if (!descriptionProto.HasColumnShardCount()) {
+ errors.AddError("trying to create OLAP store without shards number specified");
+ return false;
+ }
+
+ if (descriptionProto.GetColumnShardCount() == 0) {
+ errors.AddError("trying to create OLAP store without zero shards");
+ return false;
+ }
+
+ for (auto& presetProto : descriptionProto.GetRESERVED_TtlSettingsPresets()) {
+ Y_UNUSED(presetProto);
+ errors.AddError("TTL presets are not supported");
+ return false;
+ }
+
+ Name = descriptionProto.GetName();
+ StorageConfig = descriptionProto.GetStorageConfig();
+ // Make it easier by having data channel count always specified internally
+ if (!StorageConfig.HasDataChannelCount()) {
+ StorageConfig.SetDataChannelCount(1);
+ }
+
+ size_t protoIndex = 0;
+ for (const auto& presetProto : descriptionProto.GetSchemaPresets()) {
+ TOlapStoreSchemaPreset preset;
+ if (!preset.ParseFromRequest(presetProto, errors)) {
+ return false;
+ }
+ if (SchemaPresets.contains(NextSchemaPresetId) || SchemaPresetByName.contains(preset.GetName())) {
+ errors.AddError(TStringBuilder() << "Duplicate schema preset " << NextSchemaPresetId << " with name '" << preset.GetName() << "'");
+ return false;
+ }
+ preset.SetId(NextSchemaPresetId++);
+ preset.SetProtoIndex(protoIndex++);
+
+ TOlapSchemaUpdate schemaDiff;
+ if (!schemaDiff.Parse(presetProto.GetSchema(), errors)) {
+ return false;
+ }
+
+ if (!preset.Update(schemaDiff, errors)) {
+ return false;
+ }
+
+ SchemaPresetByName[preset.GetName()] = preset.GetId();
+ SchemaPresets[preset.GetId()] = std::move(preset);
+ }
+
+ if (!SchemaPresetByName.contains("default") || SchemaPresets.size() > 1) {
+ errors.AddError("A single schema preset named 'default' is required");
+ return false;
+ }
+
+ ColumnShards.resize(descriptionProto.GetColumnShardCount());
+ SerializeDescription(Description);
+ return true;
+}
+
+TOlapStoreInfo::ILayoutPolicy::TPtr TOlapStoreInfo::GetTablesLayoutPolicy() const {
+ ILayoutPolicy::TPtr result;
+ if (AppData()->ColumnShardConfig.GetTablesStorageLayoutPolicy().HasMinimalTables()) {
+ result = std::make_shared<TMinimalTablesCountLayout>();
+ } else if (AppData()->ColumnShardConfig.GetTablesStorageLayoutPolicy().HasIdentityGroups()) {
+ result = std::make_shared<TIdentityGroupsLayout>();
+ } else {
+ result = std::make_shared<TMinimalTablesCountLayout>();
+ }
+ return result;
+}
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/store/store.h b/ydb/core/tx/schemeshard/olap/store/store.h
new file mode 100644
index 0000000000..3e777dc798
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/store/store.h
@@ -0,0 +1,107 @@
+#pragma once
+#include <ydb/core/protos/flat_scheme_op.pb.h>
+#include <ydb/core/tx/schemeshard/olap/layout/layout.h>
+#include <ydb/core/tx/schemeshard/olap/schema/schema.h>
+#include <ydb/core/tx/schemeshard/schemeshard_info_types.h>
+
+#include <util/system/types.h>
+
+namespace NKikimr::NSchemeShard {
+
+struct TOlapStoreInfo {
+private:
+ TString Name;
+ ui64 NextSchemaPresetId = 1;
+ ui64 NextTtlSettingsPresetId = 1;
+ NKikimrSchemeOp::TColumnStorageConfig StorageConfig;
+ NKikimrSchemeOp::TColumnStoreDescription Description;
+ ui64 AlterVersion = 0;
+public:
+ using TPtr = std::shared_ptr<TOlapStoreInfo>;
+
+ class ILayoutPolicy {
+ protected:
+ virtual bool DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const = 0;
+ public:
+ using TPtr = std::shared_ptr<ILayoutPolicy>;
+ virtual ~ILayoutPolicy() = default;
+ bool Layout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const;
+ };
+
+ class TMinimalTablesCountLayout: public ILayoutPolicy {
+ protected:
+ virtual bool DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const override;
+ };
+
+ class TIdentityGroupsLayout: public ILayoutPolicy {
+ protected:
+ virtual bool DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const override;
+ };
+
+ TPtr AlterData;
+
+ const NKikimrSchemeOp::TColumnStoreDescription& GetDescription() const {
+ return Description;
+ }
+
+ NKikimrSchemeOp::TColumnStoreSharding Sharding;
+ TMaybe<NKikimrSchemeOp::TAlterColumnStore> AlterBody;
+
+ TVector<TShardIdx> ColumnShards;
+
+ THashMap<ui32, TOlapStoreSchemaPreset> SchemaPresets;
+ THashMap<TString, ui32> SchemaPresetByName;
+
+ THashSet<TPathId> ColumnTables;
+ THashSet<TPathId> ColumnTablesUnderOperation;
+ TAggregatedStats Stats;
+
+ TOlapStoreInfo() = default;
+ TOlapStoreInfo(ui64 alterVersion,
+ NKikimrSchemeOp::TColumnStoreSharding&& sharding,
+ TMaybe<NKikimrSchemeOp::TAlterColumnStore>&& alterBody = Nothing());
+
+ static TOlapStoreInfo::TPtr BuildStoreWithAlter(const TOlapStoreInfo& initialStore, const NKikimrSchemeOp::TAlterColumnStore& alterBody);
+
+ const NKikimrSchemeOp::TColumnStorageConfig& GetStorageConfig() const {
+ return StorageConfig;
+ }
+
+ const TVector<TShardIdx>& GetColumnShards() const {
+ return ColumnShards;
+ }
+
+ ui64 GetAlterVersion() const {
+ return AlterVersion;
+ }
+
+ void ApplySharding(const TVector<TShardIdx>& shardsIndexes) {
+ Y_ABORT_UNLESS(ColumnShards.size() == shardsIndexes.size());
+ Sharding.ClearColumnShards();
+ for (ui64 i = 0; i < ColumnShards.size(); ++i) {
+ const auto& idx = shardsIndexes[i];
+ ColumnShards[i] = idx;
+ auto* shardInfoProto = Sharding.AddColumnShards();
+ shardInfoProto->SetOwnerId(idx.GetOwnerId());
+ shardInfoProto->SetLocalId(idx.GetLocalId().GetValue());
+ }
+ }
+ void SerializeDescription(NKikimrSchemeOp::TColumnStoreDescription& descriptionProto) const;
+ void ParseFromLocalDB(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto);
+ bool ParseFromRequest(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto, IErrorCollector& errors);
+ bool UpdatePreset(const TString& presetName, const TOlapSchemaUpdate& schemaUpdate, IErrorCollector& errors);
+
+ const TAggregatedStats& GetStats() const {
+ return Stats;
+ }
+
+ ILayoutPolicy::TPtr GetTablesLayoutPolicy() const;
+
+ void UpdateShardStats(TShardIdx shardIdx, const TPartitionStats& newStats) {
+ Stats.Aggregated.PartCount = ColumnShards.size();
+ Stats.PartitionStats[shardIdx]; // insert if none
+ Stats.UpdateShardStats(shardIdx, newStats);
+ }
+};
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/store/ya.make b/ydb/core/tx/schemeshard/olap/store/ya.make
new file mode 100644
index 0000000000..3213054753
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/store/ya.make
@@ -0,0 +1,12 @@
+LIBRARY()
+
+SRCS(
+ store.cpp
+)
+
+PEERDIR(
+ ydb/core/tx/schemeshard/olap/layout
+ ydb/core/protos
+)
+
+END()
diff --git a/ydb/core/tx/schemeshard/olap/table/table.cpp b/ydb/core/tx/schemeshard/olap/table/table.cpp
new file mode 100644
index 0000000000..04b4b1f510
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/table/table.cpp
@@ -0,0 +1,49 @@
+#include "table.h"
+
+namespace NKikimr::NSchemeShard {
+
+TColumnTableInfo::TColumnTableInfo(
+ ui64 alterVersion,
+ NKikimrSchemeOp::TColumnTableDescription&& description,
+ NKikimrSchemeOp::TColumnTableSharding&& sharding,
+ TMaybe<NKikimrSchemeOp::TColumnStoreSharding>&& standaloneSharding,
+ TMaybe<NKikimrSchemeOp::TAlterColumnTable>&& alterBody)
+ : AlterVersion(alterVersion)
+ , Description(std::move(description))
+ , Sharding(std::move(sharding))
+ , StandaloneSharding(std::move(standaloneSharding))
+ , AlterBody(std::move(alterBody)) {
+ if (Description.HasColumnStorePathId()) {
+ OlapStorePathId = TPathId(
+ TOwnerId(Description.GetColumnStorePathId().GetOwnerId()),
+ TLocalPathId(Description.GetColumnStorePathId().GetLocalId()));
+ }
+
+ if (Description.HasSchema()) {
+ TOlapSchema schema;
+ schema.ParseFromLocalDB(Description.GetSchema());
+ }
+
+ ColumnShards.reserve(Sharding.GetColumnShards().size());
+ for (ui64 columnShard : Sharding.GetColumnShards()) {
+ ColumnShards.push_back(columnShard);
+ }
+
+ if (StandaloneSharding) {
+ OwnedColumnShards.reserve(StandaloneSharding->GetColumnShards().size());
+ for (const auto& shardIdx : StandaloneSharding->GetColumnShards()) {
+ OwnedColumnShards.push_back(TShardIdx(
+ TOwnerId(shardIdx.GetOwnerId()),
+ TLocalShardIdx(shardIdx.GetLocalId())));
+ }
+ }
+}
+
+TColumnTableInfo::TPtr TColumnTableInfo::BuildTableWithAlter(const TColumnTableInfo& initialTable, const NKikimrSchemeOp::TAlterColumnTable& alterBody) {
+ TColumnTableInfo::TPtr alterData = std::make_shared<TColumnTableInfo>(initialTable);
+ alterData->AlterBody.ConstructInPlace(alterBody);
+ ++alterData->AlterVersion;
+ return alterData;
+}
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/table/table.h b/ydb/core/tx/schemeshard/olap/table/table.h
new file mode 100644
index 0000000000..6e10fecf72
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/table/table.h
@@ -0,0 +1,58 @@
+#pragma once
+#include <ydb/core/protos/flat_scheme_op.pb.h>
+#include <ydb/core/tx/schemeshard/schemeshard_identificators.h>
+#include <ydb/core/tx/schemeshard/schemeshard_info_types.h>
+
+namespace NKikimr::NSchemeShard {
+
+struct TColumnTableInfo {
+ using TPtr = std::shared_ptr<TColumnTableInfo>;
+
+ ui64 AlterVersion = 0;
+ TPtr AlterData;
+
+ NKikimrSchemeOp::TColumnTableDescription Description;
+ NKikimrSchemeOp::TColumnTableSharding Sharding;
+ TMaybe<NKikimrSchemeOp::TColumnStoreSharding> StandaloneSharding;
+ TMaybe<NKikimrSchemeOp::TAlterColumnTable> AlterBody;
+
+ TMaybe<TPathId> OlapStorePathId; // PathId of the table store
+
+ TVector<ui64> ColumnShards; // Current list of column shards
+ TVector<TShardIdx> OwnedColumnShards;
+ TAggregatedStats Stats;
+
+ TColumnTableInfo() = default;
+ TColumnTableInfo(ui64 alterVersion, NKikimrSchemeOp::TColumnTableDescription&& description,
+ NKikimrSchemeOp::TColumnTableSharding&& sharding,
+ TMaybe<NKikimrSchemeOp::TColumnStoreSharding>&& standaloneSharding,
+ TMaybe<NKikimrSchemeOp::TAlterColumnTable>&& alterBody = Nothing());
+
+ void SetOlapStorePathId(const TPathId& pathId) {
+ OlapStorePathId = pathId;
+ Description.MutableColumnStorePathId()->SetOwnerId(pathId.OwnerId);
+ Description.MutableColumnStorePathId()->SetLocalId(pathId.LocalPathId);
+ }
+
+ static TColumnTableInfo::TPtr BuildTableWithAlter(const TColumnTableInfo& initialTable, const NKikimrSchemeOp::TAlterColumnTable& alterBody);
+
+ bool IsStandalone() const {
+ return !OwnedColumnShards.empty();
+ }
+
+ const TAggregatedStats& GetStats() const {
+ return Stats;
+ }
+
+ void UpdateShardStats(const TShardIdx shardIdx, const TPartitionStats& newStats) {
+ Stats.Aggregated.PartCount = ColumnShards.size();
+ Stats.PartitionStats[shardIdx]; // insert if none
+ Stats.UpdateShardStats(shardIdx, newStats);
+ }
+
+ void UpdateTableStats(const TPathId& pathId, const TPartitionStats& newStats) {
+ Stats.UpdateTableStats(pathId, newStats);
+ }
+};
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/table/ya.make b/ydb/core/tx/schemeshard/olap/table/ya.make
new file mode 100644
index 0000000000..6032784bee
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/table/ya.make
@@ -0,0 +1,12 @@
+LIBRARY()
+
+SRCS(
+ table.cpp
+)
+
+PEERDIR(
+ ydb/core/protos
+ ydb/core/tablet_flat
+)
+
+END()
diff --git a/ydb/core/tx/schemeshard/olap/ya.make b/ydb/core/tx/schemeshard/olap/ya.make
index 224d54e7e4..be40fc04d2 100644
--- a/ydb/core/tx/schemeshard/olap/ya.make
+++ b/ydb/core/tx/schemeshard/olap/ya.make
@@ -8,6 +8,10 @@ PEERDIR(
ydb/core/tx/schemeshard/olap/operations
ydb/core/tx/schemeshard/olap/statistics
ydb/core/tx/schemeshard/olap/options
+ ydb/core/tx/schemeshard/olap/layout
+ ydb/core/tx/schemeshard/olap/manager
+ ydb/core/tx/schemeshard/olap/store
+ ydb/core/tx/schemeshard/olap/table
)
END()
diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp
index 18543a2122..486b4c4829 100644
--- a/ydb/core/tx/schemeshard/schemeshard__init.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp
@@ -4538,7 +4538,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
NKikimrSchemeOp::TColumnStoreSharding sharding;
Y_ABORT_UNLESS(sharding.ParseFromString(rowset.GetValue<Schema::OlapStores::Sharding>()));
- TOlapStoreInfo::TPtr storeInfo = new TOlapStoreInfo(alterVersion, std::move(sharding));
+ TOlapStoreInfo::TPtr storeInfo = std::make_shared<TOlapStoreInfo>(alterVersion, std::move(sharding));
storeInfo->ParseFromLocalDB(description);
Self->OlapStores[pathId] = storeInfo;
Self->IncrementPathDbRefCount(pathId);
@@ -4572,7 +4572,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
Y_VERIFY_S(Self->OlapStores.contains(pathId),
"Cannot load alter for olap store " << pathId);
- TOlapStoreInfo::TPtr storeInfo = new TOlapStoreInfo(alterVersion, std::move(sharding), std::move(alterBody));
+ TOlapStoreInfo::TPtr storeInfo = std::make_shared<TOlapStoreInfo>(alterVersion, std::move(sharding), std::move(alterBody));
storeInfo->ParseFromLocalDB(description);
Self->OlapStores[pathId]->AlterData = storeInfo;
@@ -4602,7 +4602,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
rowset.GetValue<Schema::ColumnTables::StandaloneSharding>()));
}
- auto tableInfo = Self->ColumnTables.BuildNew(pathId, new TColumnTableInfo(alterVersion,
+ auto tableInfo = Self->ColumnTables.BuildNew(pathId, std::make_shared<TColumnTableInfo>(alterVersion,
std::move(description), std::move(sharding), std::move(storeSharding)));
Self->IncrementPathDbRefCount(pathId);
@@ -4649,7 +4649,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
Y_VERIFY_S(Self->ColumnTables.contains(pathId),
"Cannot load alter for olap table " << pathId);
- TColumnTableInfo::TPtr alterData = new TColumnTableInfo(alterVersion,
+ TColumnTableInfo::TPtr alterData = std::make_shared<TColumnTableInfo>(alterVersion,
std::move(description), std::move(sharding), std::move(storeSharding), std::move(alterBody));
auto ctInfo = Self->ColumnTables.TakeVerified(pathId);
ctInfo->AlterData = alterData;
diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h
index 77a5260827..3cfd21feed 100644
--- a/ydb/core/tx/schemeshard/schemeshard_impl.h
+++ b/ydb/core/tx/schemeshard/schemeshard_impl.h
@@ -12,13 +12,14 @@
#include "schemeshard_path.h"
#include "schemeshard_domain_links.h"
#include "schemeshard_info_types.h"
-#include "schemeshard_tables_storage.h"
#include "schemeshard_tx_infly.h"
#include "schemeshard_utils.h"
#include "schemeshard_schema.h"
#include "schemeshard__operation.h"
#include "schemeshard__stats.h"
+#include "olap/manager/manager.h"
+
#include <ydb/core/base/hive.h>
#include <ydb/core/base/storage_pools.h>
#include <ydb/core/base/subdomain.h>
diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp
index 3de1ae1d70..37424932b2 100644
--- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp
@@ -1,5 +1,4 @@
#include "schemeshard_info_types.h"
-#include "schemeshard_tables_storage.h"
#include "schemeshard_path.h"
#include "schemeshard_utils.h"
@@ -584,49 +583,6 @@ inline THashMap<ui32, size_t> DeduplicateRepeatedById(
}
-bool TOlapStoreInfo::ILayoutPolicy::Layout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const {
- if (!DoLayout(currentLayout, shardsCount, result, isNewGroup)) {
- return false;
- }
- Y_ABORT_UNLESS(result.size() == shardsCount);
- return true;
-}
-
-bool TOlapStoreInfo::TIdentityGroupsLayout::DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const {
- isNewGroup = false;
- for (auto&& i : currentLayout.GetGroups()) {
- if (i.GetTableIds().Size() == 0 && i.GetShardIds().Size() >= shardsCount) {
- result = i.GetShardIds().GetIdsVector(shardsCount);
- isNewGroup = true;
- return true;
- }
- if (i.GetShardIds().Size() != shardsCount) {
- continue;
- }
- result = i.GetShardIds().GetIdsVector();
- return true;
- }
- return false;
-}
-
-bool TOlapStoreInfo::TMinimalTablesCountLayout::DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const {
- isNewGroup = true;
- std::vector<ui64> resultLocal;
- for (auto&& i : currentLayout.GetGroups()) {
- if (i.GetTableIds().Size() > 0) {
- isNewGroup = false;
- }
- for (auto&& s : i.GetShardIds()) {
- resultLocal.emplace_back(s);
- if (resultLocal.size() == shardsCount) {
- std::swap(result, resultLocal);
- return true;
- }
- }
- }
- return false;
-}
-
NKikimrSchemeOp::TPartitionConfig TPartitionConfigMerger::DefaultConfig(const TAppData* appData) {
NKikimrSchemeOp::TPartitionConfig cfg;
@@ -2315,196 +2271,6 @@ NKikimr::NSchemeShard::TBillingStats::operator bool() const {
return Rows || Bytes;
}
-TOlapStoreInfo::TOlapStoreInfo(
- ui64 alterVersion,
- NKikimrSchemeOp::TColumnStoreSharding&& sharding,
- TMaybe<NKikimrSchemeOp::TAlterColumnStore>&& alterBody)
- : AlterVersion(alterVersion)
- , Sharding(std::move(sharding))
- , AlterBody(std::move(alterBody))
-{
- for (const auto& shardIdx : Sharding.GetColumnShards()) {
- ColumnShards.push_back(TShardIdx(
- TOwnerId(shardIdx.GetOwnerId()),
- TLocalShardIdx(shardIdx.GetLocalId())));
- }
-}
-
-TOlapStoreInfo::TPtr TOlapStoreInfo::BuildStoreWithAlter(const TOlapStoreInfo& initialStore, const NKikimrSchemeOp::TAlterColumnStore& alterBody) {
- TOlapStoreInfo::TPtr alterData = new TOlapStoreInfo(initialStore);
- alterData->AlterVersion++;
- alterData->AlterBody.ConstructInPlace(alterBody);
- return alterData;
-}
-
-void TOlapStoreInfo::SerializeDescription(NKikimrSchemeOp::TColumnStoreDescription& descriptionProto) const {
- descriptionProto.SetName(Name);
- descriptionProto.SetColumnShardCount(ColumnShards.size());
- descriptionProto.SetNextSchemaPresetId(NextSchemaPresetId);
- descriptionProto.SetNextTtlSettingsPresetId(NextTtlSettingsPresetId);
-
- for (const auto& [name, preset] : SchemaPresets) {
- Y_UNUSED(name);
- auto presetProto = descriptionProto.AddSchemaPresets();
- preset.Serialize(*presetProto);
- }
-}
-
-void TOlapStoreInfo::ParseFromLocalDB(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto) {
- StorageConfig = descriptionProto.GetStorageConfig();
- NextTtlSettingsPresetId = descriptionProto.GetNextTtlSettingsPresetId();
- NextSchemaPresetId = descriptionProto.GetNextSchemaPresetId();
- Name = descriptionProto.GetName();
-
- size_t schemaPresetIndex = 0;
- for (const auto& presetProto : descriptionProto.GetSchemaPresets()) {
- Y_ABORT_UNLESS(!SchemaPresets.contains(presetProto.GetId()));
- auto& preset = SchemaPresets[presetProto.GetId()];
- preset.ParseFromLocalDB(presetProto);
- preset.SetProtoIndex(schemaPresetIndex++);
- SchemaPresetByName[preset.GetName()] = preset.GetId();
- }
- SerializeDescription(Description);
-}
-
-bool TOlapStoreInfo::UpdatePreset(const TString& presetName, const TOlapSchemaUpdate& schemaUpdate, IErrorCollector& errors) {
- const ui32 presetId = SchemaPresetByName.at(presetName);
- auto& currentPreset = SchemaPresets.at(presetId);
- if (!currentPreset.Update(schemaUpdate, errors)) {
- return false;
- }
-
- NKikimrSchemeOp::TColumnTableSchemaPreset schemaUpdateProto;
- currentPreset.Serialize(schemaUpdateProto);
-
- auto mutablePresetProto = Description.MutableSchemaPresets(currentPreset.GetProtoIndex());
- *mutablePresetProto = schemaUpdateProto;
- return true;
-}
-
-bool TOlapStoreInfo::ParseFromRequest(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto, IErrorCollector& errors) {
- AlterVersion = 1;
- if (descriptionProto.GetRESERVED_MetaShardCount() != 0) {
- errors.AddError("trying to create OLAP store with meta shards (not supported yet)");
- return false;
- }
-
- if (!descriptionProto.HasColumnShardCount()) {
- errors.AddError("trying to create OLAP store without shards number specified");
- return false;
- }
-
- if (descriptionProto.GetColumnShardCount() == 0) {
- errors.AddError("trying to create OLAP store without zero shards");
- return false;
- }
-
- for (auto& presetProto : descriptionProto.GetRESERVED_TtlSettingsPresets()) {
- Y_UNUSED(presetProto);
- errors.AddError("TTL presets are not supported");
- return false;
- }
-
- Name = descriptionProto.GetName();
- StorageConfig = descriptionProto.GetStorageConfig();
- // Make it easier by having data channel count always specified internally
- if (!StorageConfig.HasDataChannelCount()) {
- StorageConfig.SetDataChannelCount(1);
- }
-
- size_t protoIndex = 0;
- for (const auto& presetProto : descriptionProto.GetSchemaPresets()) {
- TOlapStoreSchemaPreset preset;
- if (!preset.ParseFromRequest(presetProto, errors)) {
- return false;
- }
- if (SchemaPresets.contains(NextSchemaPresetId) || SchemaPresetByName.contains(preset.GetName())) {
- errors.AddError(TStringBuilder() << "Duplicate schema preset " << NextSchemaPresetId << " with name '" << preset.GetName() << "'");
- return false;
- }
- preset.SetId(NextSchemaPresetId++);
- preset.SetProtoIndex(protoIndex++);
-
- TOlapSchemaUpdate schemaDiff;
- if (!schemaDiff.Parse(presetProto.GetSchema(), errors)) {
- return false;
- }
-
- if (!preset.Update(schemaDiff, errors)) {
- return false;
- }
-
- SchemaPresetByName[preset.GetName()] = preset.GetId();
- SchemaPresets[preset.GetId()] = std::move(preset);
- }
-
- if (!SchemaPresetByName.contains("default") || SchemaPresets.size() > 1) {
- errors.AddError("A single schema preset named 'default' is required");
- return false;
- }
-
- ColumnShards.resize(descriptionProto.GetColumnShardCount());
- SerializeDescription(Description);
- return true;
-}
-
-TOlapStoreInfo::ILayoutPolicy::TPtr TOlapStoreInfo::GetTablesLayoutPolicy() const {
- ILayoutPolicy::TPtr result;
- if (AppData()->ColumnShardConfig.GetTablesStorageLayoutPolicy().HasMinimalTables()) {
- result = std::make_shared<TMinimalTablesCountLayout>();
- } else if (AppData()->ColumnShardConfig.GetTablesStorageLayoutPolicy().HasIdentityGroups()) {
- result = std::make_shared<TIdentityGroupsLayout>();
- } else {
- result = std::make_shared<TMinimalTablesCountLayout>();
- }
- return result;
-}
-
-TColumnTableInfo::TColumnTableInfo(
- ui64 alterVersion,
- NKikimrSchemeOp::TColumnTableDescription&& description,
- NKikimrSchemeOp::TColumnTableSharding&& sharding,
- TMaybe<NKikimrSchemeOp::TColumnStoreSharding>&& standaloneSharding,
- TMaybe<NKikimrSchemeOp::TAlterColumnTable>&& alterBody)
- : AlterVersion(alterVersion)
- , Description(std::move(description))
- , Sharding(std::move(sharding))
- , StandaloneSharding(std::move(standaloneSharding))
- , AlterBody(std::move(alterBody))
-{
- if (Description.HasColumnStorePathId()) {
- OlapStorePathId = TPathId(
- TOwnerId(Description.GetColumnStorePathId().GetOwnerId()),
- TLocalPathId(Description.GetColumnStorePathId().GetLocalId()));
- }
-
- if (Description.HasSchema()) {
- TOlapSchema schema;
- schema.ParseFromLocalDB(Description.GetSchema());
- }
-
- ColumnShards.reserve(Sharding.GetColumnShards().size());
- for (ui64 columnShard : Sharding.GetColumnShards()) {
- ColumnShards.push_back(columnShard);
- }
-
- if (StandaloneSharding) {
- OwnedColumnShards.reserve(StandaloneSharding->GetColumnShards().size());
- for (const auto& shardIdx : StandaloneSharding->GetColumnShards()) {
- OwnedColumnShards.push_back(TShardIdx(
- TOwnerId(shardIdx.GetOwnerId()),
- TLocalShardIdx(shardIdx.GetLocalId())));
- }
- }
-}
-
-TColumnTableInfo::TPtr TColumnTableInfo::BuildTableWithAlter(const TColumnTableInfo& initialTable, const NKikimrSchemeOp::TAlterColumnTable& alterBody) {
- TColumnTableInfo::TPtr alterData = new TColumnTableInfo(initialTable);
- alterData->AlterBody.ConstructInPlace(alterBody);
- ++alterData->AlterVersion;
- return alterData;
-}
-
TSequenceInfo::TSequenceInfo(
ui64 alterVersion,
NKikimrSchemeOp::TSequenceDescription&& description,
diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h
index f698f00829..3863e627b1 100644
--- a/ydb/core/tx/schemeshard/schemeshard_info_types.h
+++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h
@@ -884,154 +884,6 @@ public:
}
};
-class TColumnTablesLayout;
-
-struct TOlapStoreInfo : TSimpleRefCount<TOlapStoreInfo> {
-private:
- TString Name;
- ui64 NextSchemaPresetId = 1;
- ui64 NextTtlSettingsPresetId = 1;
- NKikimrSchemeOp::TColumnStorageConfig StorageConfig;
- NKikimrSchemeOp::TColumnStoreDescription Description;
- ui64 AlterVersion = 0;
-public:
- using TPtr = TIntrusivePtr<TOlapStoreInfo>;
-
- class ILayoutPolicy {
- protected:
- virtual bool DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const = 0;
- public:
- using TPtr = std::shared_ptr<ILayoutPolicy>;
- virtual ~ILayoutPolicy() = default;
- bool Layout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const;
- };
-
- class TMinimalTablesCountLayout: public ILayoutPolicy {
- protected:
- virtual bool DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const override;
- };
-
- class TIdentityGroupsLayout: public ILayoutPolicy {
- protected:
- virtual bool DoLayout(const TColumnTablesLayout& currentLayout, const ui32 shardsCount, std::vector<ui64>& result, bool& isNewGroup) const override;
- };
-
- TPtr AlterData;
-
- const NKikimrSchemeOp::TColumnStoreDescription& GetDescription() const {
- return Description;
- }
-
- NKikimrSchemeOp::TColumnStoreSharding Sharding;
- TMaybe<NKikimrSchemeOp::TAlterColumnStore> AlterBody;
-
- TVector<TShardIdx> ColumnShards;
-
- THashMap<ui32, TOlapStoreSchemaPreset> SchemaPresets;
- THashMap<TString, ui32> SchemaPresetByName;
-
- THashSet<TPathId> ColumnTables;
- THashSet<TPathId> ColumnTablesUnderOperation;
- TAggregatedStats Stats;
-
- TOlapStoreInfo() = default;
- TOlapStoreInfo(ui64 alterVersion,
- NKikimrSchemeOp::TColumnStoreSharding&& sharding,
- TMaybe<NKikimrSchemeOp::TAlterColumnStore>&& alterBody = Nothing());
-
- static TOlapStoreInfo::TPtr BuildStoreWithAlter(const TOlapStoreInfo& initialStore, const NKikimrSchemeOp::TAlterColumnStore& alterBody);
-
- const NKikimrSchemeOp::TColumnStorageConfig& GetStorageConfig() const {
- return StorageConfig;
- }
-
- const TVector<TShardIdx>& GetColumnShards() const {
- return ColumnShards;
- }
-
- ui64 GetAlterVersion() const {
- return AlterVersion;
- }
-
- void ApplySharding(const TVector<TShardIdx>& shardsIndexes) {
- Y_ABORT_UNLESS(ColumnShards.size() == shardsIndexes.size());
- Sharding.ClearColumnShards();
- for (ui64 i = 0; i < ColumnShards.size(); ++i) {
- const auto& idx = shardsIndexes[i];
- ColumnShards[i] = idx;
- auto* shardInfoProto = Sharding.AddColumnShards();
- shardInfoProto->SetOwnerId(idx.GetOwnerId());
- shardInfoProto->SetLocalId(idx.GetLocalId().GetValue());
- }
- }
- void SerializeDescription(NKikimrSchemeOp::TColumnStoreDescription& descriptionProto) const;
- void ParseFromLocalDB(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto);
- bool ParseFromRequest(const NKikimrSchemeOp::TColumnStoreDescription& descriptionProto, IErrorCollector& errors);
- bool UpdatePreset(const TString& presetName, const TOlapSchemaUpdate& schemaUpdate, IErrorCollector& errors);
-
- const TAggregatedStats& GetStats() const {
- return Stats;
- }
-
- ILayoutPolicy::TPtr GetTablesLayoutPolicy() const;
-
- void UpdateShardStats(TShardIdx shardIdx, const TPartitionStats& newStats) {
- Stats.Aggregated.PartCount = ColumnShards.size();
- Stats.PartitionStats[shardIdx]; // insert if none
- Stats.UpdateShardStats(shardIdx, newStats);
- }
-};
-
-struct TColumnTableInfo : TSimpleRefCount<TColumnTableInfo> {
- using TPtr = TIntrusivePtr<TColumnTableInfo>;
-
- ui64 AlterVersion = 0;
- TPtr AlterData;
-
- NKikimrSchemeOp::TColumnTableDescription Description;
- NKikimrSchemeOp::TColumnTableSharding Sharding;
- TMaybe<NKikimrSchemeOp::TColumnStoreSharding> StandaloneSharding;
- TMaybe<NKikimrSchemeOp::TAlterColumnTable> AlterBody;
-
- TMaybe<TPathId> OlapStorePathId; // PathId of the table store
-
- TVector<ui64> ColumnShards; // Current list of column shards
- TVector<TShardIdx> OwnedColumnShards;
- TAggregatedStats Stats;
-
- TColumnTableInfo() = default;
- TColumnTableInfo(ui64 alterVersion, NKikimrSchemeOp::TColumnTableDescription&& description,
- NKikimrSchemeOp::TColumnTableSharding&& sharding,
- TMaybe<NKikimrSchemeOp::TColumnStoreSharding>&& standaloneSharding,
- TMaybe<NKikimrSchemeOp::TAlterColumnTable>&& alterBody = Nothing());
-
- void SetOlapStorePathId(const TPathId& pathId) {
- OlapStorePathId = pathId;
- Description.MutableColumnStorePathId()->SetOwnerId(pathId.OwnerId);
- Description.MutableColumnStorePathId()->SetLocalId(pathId.LocalPathId);
- }
-
- static TColumnTableInfo::TPtr BuildTableWithAlter(const TColumnTableInfo& initialTable, const NKikimrSchemeOp::TAlterColumnTable& alterBody);
-
- bool IsStandalone() const {
- return !OwnedColumnShards.empty();
- }
-
- const TAggregatedStats& GetStats() const {
- return Stats;
- }
-
- void UpdateShardStats(const TShardIdx shardIdx, const TPartitionStats& newStats) {
- Stats.Aggregated.PartCount = ColumnShards.size();
- Stats.PartitionStats[shardIdx]; // insert if none
- Stats.UpdateShardStats(shardIdx, newStats);
- }
-
- void UpdateTableStats(const TPathId& pathId, const TPartitionStats& newStats) {
- Stats.UpdateTableStats(pathId, newStats);
- }
-};
-
struct TTopicStats {
TMessageSeqNo SeqNo;
diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make
index 32a4329c07..bc4f84fa87 100644
--- a/ydb/core/tx/schemeshard/ya.make
+++ b/ydb/core/tx/schemeshard/ya.make
@@ -193,7 +193,6 @@ SRCS(
schemeshard_svp_migration.h
schemeshard_svp_migration.cpp
schemeshard_tx_infly.h
- schemeshard_tables_storage.cpp
schemeshard_types.cpp
schemeshard_types.h
schemeshard_user_attr_limits.h