diff options
author | ivanmorozov333 <ivanmorozov@ydb.tech> | 2024-05-23 12:03:59 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-05-23 12:03:59 +0300 |
commit | 0107b7dd2f1f0160a767951122c65e87af629374 (patch) | |
tree | b031ca3dbfd3aeb40ca06836da724ec8781e6aad | |
parent | ae0ee005720bd4d3678b0b0bdf92250ff3e822ea (diff) | |
download | ydb-0107b7dd2f1f0160a767951122c65e87af629374.tar.gz |
columntables resharding draft (#4630)
148 files changed, 4157 insertions, 1147 deletions
diff --git a/ydb/core/formats/arrow/arrow_filter.h b/ydb/core/formats/arrow/arrow_filter.h index a83b37a783..347baf9f02 100644 --- a/ydb/core/formats/arrow/arrow_filter.h +++ b/ydb/core/formats/arrow/arrow_filter.h @@ -154,6 +154,27 @@ public: Add(currentValue, sameValueCount); } + template <class TGetterLambda> + struct TAdapterLambda { + private: + TGetterLambda Getter; + public: + TAdapterLambda(const TGetterLambda& getter) + : Getter(getter) + { + + } + + bool operator[](const ui32 index) const { + return Getter(index); + } + }; + + template <class TGetterLambda> + void ResetWithLambda(const ui32 count, const TGetterLambda getter) { + return Reset(count, TAdapterLambda<TGetterLambda>(getter)); + } + ui32 Size() const { return Count; } diff --git a/ydb/core/formats/arrow/arrow_helpers.cpp b/ydb/core/formats/arrow/arrow_helpers.cpp index a6c7f8c742..ffc1edebb6 100644 --- a/ydb/core/formats/arrow/arrow_helpers.cpp +++ b/ydb/core/formats/arrow/arrow_helpers.cpp @@ -348,21 +348,44 @@ std::shared_ptr<arrow::RecordBatch> Reorder(const std::shared_ptr<arrow::RecordB return (*res).record_batch(); } -std::vector<std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch, - const std::vector<ui32>& sharding, ui32 numShards) { - Y_ABORT_UNLESS((size_t)batch->num_rows() == sharding.size()); +THashMap<ui64, std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch, const THashMap<ui64, std::vector<ui32>>& shardRows) { + AFL_VERIFY(batch); + std::shared_ptr<arrow::UInt64Array> permutation; + { + arrow::UInt64Builder builder; + Y_VERIFY_OK(builder.Reserve(batch->num_rows())); - std::vector<std::vector<ui32>> shardRows(numShards); - for (size_t row = 0; row < sharding.size(); ++row) { - ui32 shardNo = sharding[row]; - Y_ABORT_UNLESS(shardNo < numShards); - shardRows[shardNo].push_back(row); + for (auto&& [shardId, rowIdxs]: shardRows) { + for (auto& row : rowIdxs) { + Y_VERIFY_OK(builder.Append(row)); + } + } + Y_VERIFY_OK(builder.Finish(&permutation)); } + auto reorderedBatch = Reorder(batch, permutation, false); + + THashMap<ui64, std::shared_ptr<arrow::RecordBatch>> out; + + int offset = 0; + for (auto&& [shardId, shardRowIdxs] : shardRows) { + if (shardRowIdxs.empty()) { + continue; + } + out.emplace(shardId, reorderedBatch->Slice(offset, shardRowIdxs.size())); + offset += shardRowIdxs.size(); + } + + Y_ABORT_UNLESS(offset == batch->num_rows()); + return out; +} + +std::vector<std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch, const std::vector<std::vector<ui32>>& shardRows, const ui32 numShards) { + AFL_VERIFY(batch); std::shared_ptr<arrow::UInt64Array> permutation; { arrow::UInt64Builder builder; - Y_VERIFY_OK(builder.Reserve(sharding.size())); + Y_VERIFY_OK(builder.Reserve(batch->num_rows())); for (ui32 shardNo = 0; shardNo < numShards; ++shardNo) { for (auto& row : shardRows[shardNo]) { @@ -389,6 +412,20 @@ std::vector<std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared return out; } +std::vector<std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch, + const std::vector<ui32>& sharding, ui32 numShards) { + AFL_VERIFY(batch); + Y_ABORT_UNLESS((size_t)batch->num_rows() == sharding.size()); + + std::vector<std::vector<ui32>> shardRows(numShards); + for (size_t row = 0; row < sharding.size(); ++row) { + ui32 shardNo = sharding[row]; + Y_ABORT_UNLESS(shardNo < numShards); + shardRows[shardNo].push_back(row); + } + return ShardingSplit(batch, shardRows, numShards); +} + void DedupSortedBatch(const std::shared_ptr<arrow::RecordBatch>& batch, const std::shared_ptr<arrow::Schema>& sortingKey, std::vector<std::shared_ptr<arrow::RecordBatch>>& out) { diff --git a/ydb/core/formats/arrow/arrow_helpers.h b/ydb/core/formats/arrow/arrow_helpers.h index 4a5acb22ea..60944ae8e8 100644 --- a/ydb/core/formats/arrow/arrow_helpers.h +++ b/ydb/core/formats/arrow/arrow_helpers.h @@ -83,6 +83,8 @@ std::shared_ptr<arrow::RecordBatch> ToBatch(const std::shared_ptr<arrow::Table>& std::shared_ptr<arrow::RecordBatch> CombineBatches(const std::vector<std::shared_ptr<arrow::RecordBatch>>& batches); std::shared_ptr<arrow::RecordBatch> MergeColumns(const std::vector<std::shared_ptr<arrow::RecordBatch>>& rb); std::vector<std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch, const std::vector<ui32>& sharding, ui32 numShards); +std::vector<std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch, const std::vector<std::vector<ui32>>& shardRows, const ui32 numShards); +THashMap<ui64, std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch, const THashMap<ui64, std::vector<ui32>>& shardRows); std::unique_ptr<arrow::ArrayBuilder> MakeBuilder(const std::shared_ptr<arrow::Field>& field); std::unique_ptr<arrow::ArrayBuilder> MakeBuilder(const std::shared_ptr<arrow::DataType>& type); diff --git a/ydb/core/formats/arrow/hash/calcer.h b/ydb/core/formats/arrow/hash/calcer.h index 34e9585669..d82f669fbe 100644 --- a/ydb/core/formats/arrow/hash/calcer.h +++ b/ydb/core/formats/arrow/hash/calcer.h @@ -61,16 +61,20 @@ public: TXX64(const std::vector<TString>& columnNames, const ENoColumnPolicy noColumnPolicy, const ui64 seed = 0); TXX64(const std::vector<std::string>& columnNames, const ENoColumnPolicy noColumnPolicy, const ui64 seed = 0); + const std::vector<TString>& GetColumnNames() const { + return ColumnNames; + } + static void AppendField(const std::shared_ptr<arrow::Array>& array, const int row, NXX64::TStreamStringHashCalcer& hashCalcer); static void AppendField(const std::shared_ptr<arrow::Scalar>& scalar, NXX64::TStreamStringHashCalcer& hashCalcer); static ui64 CalcHash(const std::shared_ptr<arrow::Scalar>& scalar); std::optional<std::vector<ui64>> Execute(const std::shared_ptr<arrow::RecordBatch>& batch) const; - template <class TDataContainer> - std::shared_ptr<arrow::Array> ExecuteToArray(const std::shared_ptr<TDataContainer>& batch) const { + template <class TDataContainer, class TAcceptor> + [[nodiscard]] bool ExecuteToArrayImpl(const std::shared_ptr<TDataContainer>& batch, const TAcceptor& acceptor) const { std::vector<std::shared_ptr<typename NAdapter::TDataBuilderPolicy<TDataContainer>::TColumn>> columns = GetColumns(batch); if (columns.empty()) { - return nullptr; + return false; } std::vector<NAccessor::IChunkedArray::TReader> columnScanners; @@ -79,9 +83,6 @@ public: } - auto builder = NArrow::MakeBuilder(arrow::TypeTraits<arrow::UInt64Type>::type_singleton()); - auto& intBuilder = static_cast<arrow::UInt64Builder&>(*builder); - TStatusValidator::Validate(intBuilder.Reserve(batch->num_rows())); { NXX64::TStreamStringHashCalcer hashCalcer(Seed); for (int row = 0; row < batch->num_rows(); ++row) { @@ -90,12 +91,42 @@ public: auto address = column.GetReadChunk(row); AppendField(address.GetArray(), address.GetPosition(), hashCalcer); } - intBuilder.UnsafeAppend(hashCalcer.Finish()); + acceptor(hashCalcer.Finish()); } } + return true; + } + + template <class TDataContainer> + std::shared_ptr<arrow::Array> ExecuteToArray(const std::shared_ptr<TDataContainer>& batch) const { + auto builder = NArrow::MakeBuilder(arrow::TypeTraits<arrow::UInt64Type>::type_singleton()); + auto& intBuilder = static_cast<arrow::UInt64Builder&>(*builder); + TStatusValidator::Validate(intBuilder.Reserve(batch->num_rows())); + + const auto acceptor = [&](const ui64 hash) { + intBuilder.UnsafeAppend(hash); + }; + + if (!ExecuteToArrayImpl(batch, acceptor)) { + return nullptr; + } + return NArrow::TStatusValidator::GetValid(builder->Finish()); } + template <class TDataContainer> + std::vector<ui64> ExecuteToVector(const std::shared_ptr<TDataContainer>& batch) const { + std::vector<ui64> result; + result.reserve(batch->num_rows()); + + const auto acceptor = [&](const ui64 hash) { + result.emplace_back(hash); + }; + + AFL_VERIFY(ExecuteToArrayImpl(batch, acceptor)); + return result; + } + }; } diff --git a/ydb/core/kqp/common/kqp_resolve.h b/ydb/core/kqp/common/kqp_resolve.h index 9cd41e4387..7a079f3fa4 100644 --- a/ydb/core/kqp/common/kqp_resolve.h +++ b/ydb/core/kqp/common/kqp_resolve.h @@ -27,7 +27,7 @@ enum class ETableKind { struct TTableConstInfo : public TAtomicRefCount<TTableConstInfo> { TString Path; - TMap<TString, NSharding::TShardingBase::TColumn> Columns; + TMap<TString, NSharding::IShardingBase::TColumn> Columns; TVector<TString> KeyColumns; TVector<NScheme::TTypeInfo> KeyColumnTypes; ETableKind TableKind = ETableKind::Unknown; @@ -43,7 +43,7 @@ struct TTableConstInfo : public TAtomicRefCount<TTableConstInfo> { return; } - NSharding::TShardingBase::TColumn column; + NSharding::IShardingBase::TColumn column; column.Id = phyColumn.GetId().GetId(); if (phyColumn.GetTypeId() != NScheme::NTypeIds::Pg) { @@ -83,7 +83,7 @@ struct TTableConstInfo : public TAtomicRefCount<TTableConstInfo> { << ", table: " << Path << ", column: " << columnName); - NSharding::TShardingBase::TColumn column; + NSharding::IShardingBase::TColumn column; column.Id = systemColumn->ColumnId; column.Type = NScheme::TTypeInfo(systemColumn->TypeId); column.NotNull = false; @@ -143,7 +143,7 @@ public: return TableConstInfo->Path; } - const TMap<TString, NSharding::TShardingBase::TColumn>& GetColumns() const { + const TMap<TString, NSharding::IShardingBase::TColumn>& GetColumns() const { return TableConstInfo->Columns; } diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.cpp index 76f9eea6d4..6e01379176 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.cpp +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.cpp @@ -30,8 +30,7 @@ TConclusionStatus ITableStoreOperation::Deserialize(const NYql::TObjectSettingsI return TConclusionStatus::Success(); } -void ITableStoreOperation::SerializeScheme(NKikimrSchemeOp::TModifyScheme& scheme, const bool isStandalone) const { - scheme.SetWorkingDir(WorkingDir); +void ITableStoreOperation::DoSerializeScheme(NKikimrSchemeOp::TModifyScheme& scheme, const bool isStandalone) const { if (isStandalone) { scheme.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterColumnTable); NKikimrSchemeOp::TAlterColumnTable* alter = scheme.MutableAlterColumnTable(); @@ -48,4 +47,9 @@ void ITableStoreOperation::SerializeScheme(NKikimrSchemeOp::TModifyScheme& schem } } +void ITableStoreOperation::SerializeScheme(NKikimrSchemeOp::TModifyScheme& scheme, const bool isStandalone) const { + scheme.SetWorkingDir(WorkingDir); + DoSerializeScheme(scheme, isStandalone); +} + } diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.h b/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.h index aee052d7a2..0eda9de572 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.h +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.h @@ -10,7 +10,7 @@ public: private: TString PresetName = "default"; TString WorkingDir; - TString StoreName; + YDB_READONLY_DEF(TString, StoreName); public: virtual ~ITableStoreOperation() {}; @@ -20,6 +20,7 @@ public: private: virtual TConclusionStatus DoDeserialize(NYql::TObjectSettingsImpl::TFeaturesExtractor& features) = 0; virtual void DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchema& scheme) const = 0; + virtual void DoSerializeScheme(NKikimrSchemeOp::TModifyScheme& scheme, const bool isStandalone) const; }; } diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.cpp new file mode 100644 index 0000000000..1bfc18c9d9 --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.cpp @@ -0,0 +1,28 @@ +#include "alter_sharding.h" +#include <util/string/type.h> + +namespace NKikimr::NKqp { + +TConclusionStatus TAlterShardingOperation::DoDeserialize(NYql::TObjectSettingsImpl::TFeaturesExtractor& features) { + const std::optional<TString> modification = features.Extract<TString>("MODIFICATION"); + if (!modification) { + return TConclusionStatus::Fail("modification type not specified in request"); + } + if (*modification == "SPLIT") { + Increase = true; + } else if (*modification == "MERGE") { + return TConclusionStatus::Fail("modification is impossible yet"); + } else { + return TConclusionStatus::Fail("undefined modification: \"" + *modification + "\""); + } + return TConclusionStatus::Success(); +} + +void TAlterShardingOperation::DoSerializeScheme(NKikimrSchemeOp::TModifyScheme& scheme, const bool isStandalone) const { + AFL_VERIFY(!isStandalone); + scheme.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterColumnTable); + scheme.MutableAlterColumnTable()->SetName(GetStoreName()); + *scheme.MutableAlterColumnTable()->MutableReshardColumnTable() = NKikimrSchemeOp::TReshardColumnTable(); +} + +} diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.h b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.h new file mode 100644 index 0000000000..52f58e14d7 --- /dev/null +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.h @@ -0,0 +1,26 @@ +#include "abstract.h" +#include <ydb/core/tx/columnshard/engines/scheme/statistics/abstract/constructor.h> + +namespace NKikimr::NKqp { + +class TAlterShardingOperation: public ITableStoreOperation { +private: + static TString GetTypeName() { + return "ALTER_SHARDING"; + } + + static inline const auto Registrator = TFactory::TRegistrator<TAlterShardingOperation>(GetTypeName()); +private: + std::optional<bool> Increase; + virtual void DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchema& /*scheme*/) const override { + AFL_VERIFY(false); + } + virtual void DoSerializeScheme(NKikimrSchemeOp::TModifyScheme& scheme, const bool isStandalone) const override; + +public: + TConclusionStatus DoDeserialize(NYql::TObjectSettingsImpl::TFeaturesExtractor& features) override; + +}; + +} + diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_opt.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_opt.cpp index 5cdfd696e2..72e9616fba 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_opt.cpp +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_opt.cpp @@ -7,7 +7,7 @@ namespace NKikimr::NKqp { TConclusionStatus TUpsertOptionsOperation::DoDeserialize(NYql::TObjectSettingsImpl::TFeaturesExtractor& features) { auto value = features.Extract<bool>("SCHEME_NEED_ACTUALIZATION", false); if (!value) { - TConclusionStatus::Fail("Incorrect value for SCHEME_NEED_ACTUALIZATION: cannot parse as boolean"); + return TConclusionStatus::Fail("Incorrect value for SCHEME_NEED_ACTUALIZATION: cannot parse as boolean"); } SchemeNeedActualization = *value; ExternalGuaranteeExclusivePK = features.Extract<bool>("EXTERNAL_GUARANTEE_EXCLUSIVE_PK"); diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/ya.make b/ydb/core/kqp/gateway/behaviour/tablestore/operations/ya.make index 17d70abf65..d13c57aa70 100644 --- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/ya.make +++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/ya.make @@ -10,6 +10,7 @@ SRCS( GLOBAL upsert_stat.cpp GLOBAL drop_stat.cpp GLOBAL upsert_opt.cpp + GLOBAL alter_sharding.cpp ) PEERDIR( diff --git a/ydb/core/protos/data_events.proto b/ydb/core/protos/data_events.proto index 80b0662b2b..627296a769 100644 --- a/ydb/core/protos/data_events.proto +++ b/ydb/core/protos/data_events.proto @@ -111,6 +111,7 @@ message TEvWrite { // Writes are performed "over" the specified snapshot when specified // This mostly affects the minimum MVCC version of the resulting write optional TMvccSnapshot MvccSnapshot = 8; + optional uint32 GranuleShardingVersionId = 9; } message TEvWriteResult { @@ -121,7 +122,7 @@ message TEvWriteResult { STATUS_ABORTED = 3; STATUS_INTERNAL_ERROR = 4; STATUS_OVERLOADED = 5; - STATUS_CANCELLED = 6; + STATUS_CANCELLED = 6; STATUS_BAD_REQUEST = 7; STATUS_SCHEME_CHANGED = 8; STATUS_LOCKS_BROKEN = 9; diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index e4b73013fb..0c2ca24845 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -18,6 +18,7 @@ import "ydb/library/actors/protos/actors.proto"; import "ydb/library/mkql_proto/protos/minikql.proto"; import "ydb/core/protos/index_builder.proto"; import "ydb/core/tx/columnshard/engines/scheme/statistics/protos/data.proto"; +import "ydb/core/tx/columnshard/common/protos/snapshot.proto"; import "google/protobuf/empty.proto"; @@ -649,11 +650,56 @@ message TAlterColumnStore { repeated TRemoveColumnTableTtlSettingsPreset RESERVED_RemoveTtlSettingsPresets = 7; } -message TColumnTableSharding { - // Version is incremented each time table sharding is changed - // This field is managed by the system. - optional uint64 Version = 1; +message TConsistencyShardingTablet { + optional uint64 TabletId = 1; + optional uint64 HashIntervalLeftClosed = 2; + optional uint64 HashIntervalRightOpened = 3; +} + +message TModuloShardingTablet { + optional uint64 TabletId = 1; + repeated uint32 AppropriateMods = 2; +} + +message THashShardingInfo { + repeated string ColumnNames = 1; +} + +message TGranuleConsistencyShardingInfo { + optional THashShardingInfo Hashing = 1; + optional TConsistencyShardingTablet ShardInfo = 2; +} + +message TGranuleModuloShardingInfo { + optional THashShardingInfo Hashing = 1; + optional TModuloShardingTablet ShardInfo = 2; + optional uint64 ModuloPartsCount = 3; +} +message TGranuleShardingLogicContainer { + optional string ClassName = 1; + oneof Implementation { + TGranuleConsistencyShardingInfo Consistency = 20; + TGranuleModuloShardingInfo Modulo = 21; + } +} + +message TGranuleShardingInfo { + optional uint64 PathId = 1; + optional uint64 VersionId = 2; + optional TGranuleShardingLogicContainer Container = 3; +} + +message TGranuleShardInfo { + optional uint64 TabletId = 1; + optional uint32 SequenceIdx = 2; + optional bool IsOpenForRead = 3; + optional bool IsOpenForWrite = 4; + optional NKikimrColumnShardProto.TSnapshot OpenForWriteSnapshot = 5; + optional uint64 ShardingVersion = 6; +} + +message TColumnTableSharding { // An ordered list of currently used column shards // This field is managed by the system. repeated uint64 ColumnShards = 2; @@ -680,12 +726,18 @@ message TColumnTableSharding { // Argument for HASH_FUNCTION_CLOUD_LOGS optional uint32 ActiveShardsCount = 4; + repeated TConsistencyShardingTablet TabletsForConsistency = 5; + + optional uint32 ModuloPartsCount = 6; + repeated TModuloShardingTablet TabletsForModulo = 7; } oneof Method { TRandomSharding RandomSharding = 5; THashSharding HashSharding = 6; } + + repeated TGranuleShardInfo ShardsInfo = 7; } message TColumnTableDescription { @@ -729,6 +781,13 @@ message TAlterColumnTable { optional string AlterSchemaPresetName = 4; optional string RESERVED_AlterTtlSettingsPresetName = 5; + + optional TAlterShards AlterShards = 6; + optional TReshardColumnTable ReshardColumnTable = 7; +} + +message TReshardColumnTable { + optional bool Increase = 1; } message TLoginCreateUser { @@ -1540,6 +1599,46 @@ message TDropIndex { optional string IndexName = 2; } +message TConsistencyShardsModification { + repeated TConsistencyShardingTablet Shards = 1; +} + +message TModuloShardsModification { + repeated TModuloShardingTablet Shards = 1; + optional uint32 PartsCount = 2; +} + +message TShardingModification { + repeated uint64 NewShardIds = 1; + repeated uint64 DeleteShardIds = 2; + + repeated uint64 CloseWriteIds = 3; + repeated uint64 CloseReadIds = 4; + repeated uint64 OpenWriteIds = 5; + repeated uint64 OpenReadIds = 6; + + oneof Implementation { + TConsistencyShardsModification Consistency = 20; + TModuloShardsModification Modulo = 21; + } +} + +message TShardingTransfer { + optional uint64 DestinationTabletId = 1; + repeated uint64 SourceTabletIds = 2; +} + +message TShardingTransfers { + repeated TShardingTransfer Transfers = 1; +} + +message TAlterShards { + oneof Implementation { + TShardingModification Modification = 20; + TShardingTransfers Transfer = 21; + } +} + // Request for scheme modification // Has only one of the operations message TModifyScheme { @@ -1731,7 +1830,6 @@ message TPathVersion { optional uint64 FileStoreVersion = 15; optional uint64 ColumnStoreVersion = 16; optional uint64 ColumnTableVersion = 17; - optional uint64 ColumnTableShardingVersion = 18; optional uint64 SubDomainStateVersion = 19; optional uint64 ColumnTableSchemaVersion = 20; optional uint64 ColumnTableTtlSettingsVersion = 21; diff --git a/ydb/core/protos/tx_columnshard.proto b/ydb/core/protos/tx_columnshard.proto index 17a143f754..c254df1f64 100644 --- a/ydb/core/protos/tx_columnshard.proto +++ b/ydb/core/protos/tx_columnshard.proto @@ -79,6 +79,7 @@ message TEvWrite { optional TMetadata Meta = 7; optional NKikimrLongTxService.TLongTxId LongTxId = 8; optional uint32 WritePartId = 9 [default = 0]; + optional uint32 GranuleShardingVersion = 10; } message TEvWriteResult { @@ -284,6 +285,8 @@ message TSchemaTxBody { TAlterTable AlterTable = 5; TAlterStore AlterStore = 6; } + + optional NKikimrSchemeOp.TGranuleShardingInfo GranuleShardingInfo = 20; } message TTtlTxBody { diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index 53ec70b60b..1f3012f1ca 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -168,6 +168,7 @@ PEERDIR( ydb/core/tx/columnshard/engines/scheme/statistics/protos ydb/core/tx/columnshard/engines/protos ydb/core/formats/arrow/protos + ydb/core/tx/columnshard/common/protos ) CPP_PROTO_PLUGIN0(config_proto_plugin ydb/core/config/tools/protobuf_plugin) diff --git a/ydb/core/tx/columnshard/bg_tasks/abstract/session.cpp b/ydb/core/tx/columnshard/bg_tasks/abstract/session.cpp index d6fad196cf..2722d80017 100644 --- a/ydb/core/tx/columnshard/bg_tasks/abstract/session.cpp +++ b/ydb/core/tx/columnshard/bg_tasks/abstract/session.cpp @@ -1,12 +1,13 @@ #include "session.h" #include "adapter.h" #include <ydb/public/api/protos/ydb_operation.pb.h> +#include <ydb/public/lib/operation_id/protos/operation_id.pb.h> namespace NKikimr::NOlap::NBackground { Ydb::Operations::Operation TSessionInfoReport::SerializeToProto() const { Ydb::Operations::Operation result; - result.set_id(ClassName + "::" + Identifier); + result.set_id("/" + ::ToString((int)Ydb::TOperationId::SS_BG_TASKS) + "?type=" + ClassName + "&id=" + Identifier); result.set_ready(IsFinished); return result; } diff --git a/ydb/core/tx/columnshard/bg_tasks/abstract/session.h b/ydb/core/tx/columnshard/bg_tasks/abstract/session.h index 027d80a32f..5f48bfed54 100644 --- a/ydb/core/tx/columnshard/bg_tasks/abstract/session.h +++ b/ydb/core/tx/columnshard/bg_tasks/abstract/session.h @@ -87,7 +87,7 @@ public: virtual bool IsReadyForStart() const = 0; virtual bool IsFinished() const = 0; - virtual bool IsReadyForRemove() const = 0; + virtual bool IsReadyForRemoveOnFinished() const = 0; }; template <class TProtoLogicExt, class TProtoProgressExt, class TProtoStateExt> diff --git a/ydb/core/tx/columnshard/bg_tasks/abstract/ya.make b/ydb/core/tx/columnshard/bg_tasks/abstract/ya.make index 4a25babe82..4cdec62648 100644 --- a/ydb/core/tx/columnshard/bg_tasks/abstract/ya.make +++ b/ydb/core/tx/columnshard/bg_tasks/abstract/ya.make @@ -13,6 +13,7 @@ PEERDIR( ydb/library/accessor ydb/library/services ydb/core/tx/columnshard/bg_tasks/protos + ydb/public/lib/operation_id/protos ydb/public/api/protos ) diff --git a/ydb/core/tx/columnshard/bg_tasks/transactions/tx_remove.cpp b/ydb/core/tx/columnshard/bg_tasks/transactions/tx_remove.cpp index 611f786de6..ae3a49dd71 100644 --- a/ydb/core/tx/columnshard/bg_tasks/transactions/tx_remove.cpp +++ b/ydb/core/tx/columnshard/bg_tasks/transactions/tx_remove.cpp @@ -9,7 +9,7 @@ bool TTxRemoveSession::Execute(NTabletFlatExecutor::TTransactionContext& txc, co } void TTxRemoveSession::Complete(const TActorContext& /*ctx*/) { - AFL_VERIFY(Sessions->RemoveSession(ClassName, Identifier)); + AFL_VERIFY(Sessions->RemoveSession(ClassName, Identifier))("class_name", ClassName)("id", Identifier); } } diff --git a/ydb/core/tx/columnshard/bg_tasks/transactions/tx_save_state.cpp b/ydb/core/tx/columnshard/bg_tasks/transactions/tx_save_state.cpp index 1eda1b3eb9..d47dfd0f1a 100644 --- a/ydb/core/tx/columnshard/bg_tasks/transactions/tx_save_state.cpp +++ b/ydb/core/tx/columnshard/bg_tasks/transactions/tx_save_state.cpp @@ -9,7 +9,7 @@ bool TTxSaveSessionState::Execute(NTabletFlatExecutor::TTransactionContext& txc, } void TTxSaveSessionState::DoComplete(const TActorContext& ctx) { - if (Session->GetLogicContainer()->IsReadyForRemove()) { + if (Session->GetLogicContainer()->IsFinished() && Session->GetLogicContainer()->IsReadyForRemoveOnFinished()) { ctx.Send(Adapter->GetTabletActorId(), new TEvRemoveSession(Session->GetLogicClassName(), Session->GetIdentifier())); } else if (!Session->IsRunning() && Session->GetLogicContainer()->IsReadyForStart()) { TStartContext context(Session, Adapter); diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp index 6e1ea436fb..7e92f9a5e4 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp @@ -50,7 +50,7 @@ bool TTxWrite::Execute(TTransactionContext& txc, const TActorContext&) { auto writeId = TWriteId(writeMeta.GetWriteId()); if (!operation) { NIceDb::TNiceDb db(txc.DB); - writeId = Self->GetLongTxWrite(db, writeMeta.GetLongTxIdUnsafe(), writeMeta.GetWritePartId()); + writeId = Self->GetLongTxWrite(db, writeMeta.GetLongTxIdUnsafe(), writeMeta.GetWritePartId(), writeMeta.GetGranuleShardingVersion()); aggr->AddWriteId(writeId); } diff --git a/ydb/core/tx/columnshard/columnshard__init.cpp b/ydb/core/tx/columnshard/columnshard__init.cpp index 82f1b114ec..3ff5452696 100644 --- a/ydb/core/tx/columnshard/columnshard__init.cpp +++ b/ydb/core/tx/columnshard/columnshard__init.cpp @@ -185,7 +185,12 @@ bool TTxInit::ReadEverything(TTransactionContext& txc, const TActorContext& ctx) Y_ABORT_UNLESS(proto.ParseFromString(rowset.GetValue<Schema::LongTxWrites::LongTxId>())); const auto longTxId = NLongTxService::TLongTxId::FromProto(proto); - Self->LoadLongTxWrite(writeId, writePartId, longTxId); + std::optional<ui32> granuleShardingVersion; + if (rowset.HaveValue<Schema::LongTxWrites::GranuleShardingVersion>() && rowset.GetValue<Schema::LongTxWrites::GranuleShardingVersion>()) { + granuleShardingVersion = rowset.GetValue<Schema::LongTxWrites::GranuleShardingVersion>(); + } + + Self->LoadLongTxWrite(writeId, writePartId, longTxId, granuleShardingVersion); if (!rowset.Next()) { return false; diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp index 06e92ad0a8..0113746521 100644 --- a/ydb/core/tx/columnshard/columnshard__write.cpp +++ b/ydb/core/tx/columnshard/columnshard__write.cpp @@ -162,7 +162,12 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex const TString dedupId = record.GetDedupId(); const auto source = ev->Sender; - NEvWrite::TWriteMeta writeMeta(writeId, tableId, source); + std::optional<ui32> granuleShardingVersion; + if (record.HasGranuleShardingVersion()) { + granuleShardingVersion = record.GetGranuleShardingVersion(); + } + + NEvWrite::TWriteMeta writeMeta(writeId, tableId, source, granuleShardingVersion); writeMeta.SetDedupId(dedupId); Y_ABORT_UNLESS(record.HasLongTxId()); writeMeta.SetLongTxId(NLongTxService::TLongTxId::FromProto(record.GetLongTxId())); @@ -362,7 +367,7 @@ void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActor auto overloadStatus = CheckOverloaded(tableId); if (overloadStatus != EOverloadStatus::None) { - NEvWrite::TWriteData writeData(NEvWrite::TWriteMeta(0, tableId, source), arrowData, nullptr, nullptr); + NEvWrite::TWriteData writeData(NEvWrite::TWriteMeta(0, tableId, source, {}), arrowData, nullptr, nullptr); std::unique_ptr<NActors::IEventBase> result = NEvents::TDataEvents::TEvWriteResult::BuildError(TabletID(), 0, NKikimrDataEvents::TEvWriteResult::STATUS_OVERLOADED, "overload data error"); OverloadWriteFail(overloadStatus, writeData, cookie, std::move(result), ctx); return; @@ -370,7 +375,12 @@ void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActor auto wg = WritesMonitor.RegisterWrite(arrowData->GetSize()); - auto writeOperation = OperationsManager->RegisterOperation(lockId, cookie); + std::optional<ui32> granuleShardingVersionId; + if (record.HasGranuleShardingVersionId()) { + granuleShardingVersionId = record.GetGranuleShardingVersionId(); + } + + auto writeOperation = OperationsManager->RegisterOperation(lockId, cookie, granuleShardingVersionId); Y_ABORT_UNLESS(writeOperation); writeOperation->SetBehaviour(behaviour); writeOperation->Start(*this, tableId, arrowData, source, ctx); diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 5bc2946deb..4addc50c4d 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -200,7 +200,7 @@ ui64 TColumnShard::GetMinReadStep() const { return minReadStep; } -TWriteId TColumnShard::HasLongTxWrite(const NLongTxService::TLongTxId& longTxId, const ui32 partId) { +TWriteId TColumnShard::HasLongTxWrite(const NLongTxService::TLongTxId& longTxId, const ui32 partId) const { auto it = LongTxWritesByUniqueId.find(longTxId.UniqueId); if (it != LongTxWritesByUniqueId.end()) { auto itPart = it->second.find(partId); @@ -211,7 +211,7 @@ TWriteId TColumnShard::HasLongTxWrite(const NLongTxService::TLongTxId& longTxId, return (TWriteId)0; } -TWriteId TColumnShard::GetLongTxWrite(NIceDb::TNiceDb& db, const NLongTxService::TLongTxId& longTxId, const ui32 partId) { +TWriteId TColumnShard::GetLongTxWrite(NIceDb::TNiceDb& db, const NLongTxService::TLongTxId& longTxId, const ui32 partId, const std::optional<ui32> granuleShardingVersionId) { auto it = LongTxWritesByUniqueId.find(longTxId.UniqueId); if (it != LongTxWritesByUniqueId.end()) { auto itPart = it->second.find(partId); @@ -227,9 +227,10 @@ TWriteId TColumnShard::GetLongTxWrite(NIceDb::TNiceDb& db, const NLongTxService: lw.WriteId = (ui64)writeId; lw.WritePartId = partId; lw.LongTxId = longTxId; + lw.GranuleShardingVersionId = granuleShardingVersionId; it->second[partId] = &lw; - Schema::SaveLongTxWrite(db, writeId, partId, longTxId); + Schema::SaveLongTxWrite(db, writeId, partId, longTxId, granuleShardingVersionId); return writeId; } @@ -249,11 +250,12 @@ void TColumnShard::AddLongTxWrite(TWriteId writeId, ui64 txId) { lw.PreparedTxId = txId; } -void TColumnShard::LoadLongTxWrite(TWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId) { +void TColumnShard::LoadLongTxWrite(TWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId, const std::optional<ui32> granuleShardingVersion) { auto& lw = LongTxWrites[writeId]; lw.WritePartId = writePartId; lw.WriteId = (ui64)writeId; lw.LongTxId = longTxId; + lw.GranuleShardingVersionId = granuleShardingVersion; LongTxWritesByUniqueId[longTxId.UniqueId][writePartId] = &lw; } diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index f059faed10..1a451d36b7 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -409,6 +409,7 @@ private: ui32 WritePartId; NLongTxService::TLongTxId LongTxId; ui64 PreparedTxId = 0; + std::optional<ui32> GranuleShardingVersionId; }; class TWritesMonitor { @@ -531,10 +532,10 @@ private: return ProgressTxController->GetTxCompleteLag(mediatorTime); } - TWriteId HasLongTxWrite(const NLongTxService::TLongTxId& longTxId, const ui32 partId); - TWriteId GetLongTxWrite(NIceDb::TNiceDb& db, const NLongTxService::TLongTxId& longTxId, const ui32 partId); + TWriteId HasLongTxWrite(const NLongTxService::TLongTxId& longTxId, const ui32 partId) const; + TWriteId GetLongTxWrite(NIceDb::TNiceDb& db, const NLongTxService::TLongTxId& longTxId, const ui32 partId, const std::optional<ui32> granuleShardingVersionId); void AddLongTxWrite(TWriteId writeId, ui64 txId); - void LoadLongTxWrite(TWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId); + void LoadLongTxWrite(TWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId, const std::optional<ui32> granuleShardingVersion); bool RemoveLongTxWrite(NIceDb::TNiceDb& db, TWriteId writeId, ui64 txId = 0); void TryAbortWrites(NIceDb::TNiceDb& db, NOlap::TDbWrapper& dbTable, THashSet<TWriteId>&& writesToAbort); diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h index a9cf777673..8bba15f162 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.h +++ b/ydb/core/tx/columnshard/columnshard_schema.h @@ -54,7 +54,8 @@ struct Schema : NIceDb::Schema { BackupIdsDeprecated, ExportSessionsId, PortionsTableId, - BackgroundSessionsTableId + BackgroundSessionsTableId, + ShardingInfoTabletId }; enum class ETierTables: ui32 { @@ -194,9 +195,10 @@ struct Schema : NIceDb::Schema { struct WriteId: Column<1, NScheme::NTypeIds::Uint64> {}; struct LongTxId : Column<2, NScheme::NTypeIds::String> {}; struct WritePartId: Column<3, NScheme::NTypeIds::Uint32> {}; + struct GranuleShardingVersion: Column<4, NScheme::NTypeIds::Uint32> {}; using TKey = TableKey<WriteId>; - using TColumns = TableColumns<WriteId, LongTxId, WritePartId>; + using TColumns = TableColumns<WriteId, LongTxId, WritePartId, GranuleShardingVersion>; }; struct BlobsToKeep : Table<(ui32)ECommonTables::BlobsToKeep> { @@ -315,10 +317,11 @@ struct Schema : NIceDb::Schema { struct CreatedAt : Column<4, NScheme::NTypeIds::Uint64> {}; struct GlobalWriteId : Column<5, NScheme::NTypeIds::Uint64> {}; struct Metadata : Column<6, NScheme::NTypeIds::String> {}; - struct Cookie : Column<7, NScheme::NTypeIds::Uint64> {}; + struct Cookie: Column<7, NScheme::NTypeIds::Uint64> {}; + struct GranuleShardingVersionId: Column<8, NScheme::NTypeIds::Uint32> {}; using TKey = TableKey<WriteId>; - using TColumns = TableColumns<LockId, WriteId, Status, CreatedAt, GlobalWriteId, Metadata, Cookie>; + using TColumns = TableColumns<LockId, WriteId, Status, CreatedAt, GlobalWriteId, Metadata, Cookie, GranuleShardingVersionId>; }; struct OperationTxIds : NIceDb::Schema::Table<OperationTxIdsId> { @@ -453,9 +456,10 @@ struct Schema : NIceDb::Schema { struct XPlanStep: Column<4, NScheme::NTypeIds::Uint64> {}; struct XTxId: Column<5, NScheme::NTypeIds::Uint64> {}; struct Metadata: Column<6, NScheme::NTypeIds::String> {}; // NKikimrTxColumnShard.TIndexColumnMeta + struct ShardingVersion: Column<7, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey<PathId, PortionId>; - using TColumns = TableColumns<PathId, PortionId, SchemaVersion, XPlanStep, XTxId, Metadata>; + using TColumns = TableColumns<PathId, PortionId, SchemaVersion, XPlanStep, XTxId, Metadata, ShardingVersion>; }; struct BackgroundSessions: Table<BackgroundSessionsTableId> { @@ -470,6 +474,16 @@ struct Schema : NIceDb::Schema { using TColumns = TableColumns<ClassName, Identifier, StatusChannel, LogicDescription, Progress, State>; }; + struct ShardingInfo : Table<ShardingInfoTabletId> { + struct PathId : Column<1, NScheme::NTypeIds::Uint64> {}; + struct VersionId : Column<2, NScheme::NTypeIds::Uint64> {}; + struct Snapshot : Column<3, NScheme::NTypeIds::String> {}; + struct Logic : Column<4, NScheme::NTypeIds::String> {}; + + using TKey = TableKey<PathId, VersionId>; + using TColumns = TableColumns<PathId, VersionId, Snapshot, Logic>; + }; + using TTables = SchemaTables< Value, TxInfo, @@ -500,7 +514,8 @@ struct Schema : NIceDb::Schema { DestinationSessions, OperationTxIds, IndexPortions, - BackgroundSessions + BackgroundSessions, + ShardingInfo >; // @@ -654,14 +669,16 @@ struct Schema : NIceDb::Schema { db.Table<TableInfo>().Key(pathId).Delete(); } - static void SaveLongTxWrite(NIceDb::TNiceDb& db, TWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId) { + static void SaveLongTxWrite(NIceDb::TNiceDb& db, TWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId, const std::optional<ui32> granuleShardingVersion) { NKikimrLongTxService::TLongTxId proto; longTxId.ToProto(&proto); TString serialized; Y_ABORT_UNLESS(proto.SerializeToString(&serialized)); db.Table<LongTxWrites>().Key((ui64)writeId).Update( - NIceDb::TUpdate<LongTxWrites::LongTxId>(serialized), - NIceDb::TUpdate<LongTxWrites::WritePartId>(writePartId)); + NIceDb::TUpdate<LongTxWrites::LongTxId>(serialized), + NIceDb::TUpdate<LongTxWrites::WritePartId>(writePartId), + NIceDb::TUpdate<LongTxWrites::GranuleShardingVersion>(granuleShardingVersion.value_or(0)) + ); } static void EraseLongTxWrite(NIceDb::TNiceDb& db, TWriteId writeId) { diff --git a/ydb/core/tx/columnshard/common/snapshot.cpp b/ydb/core/tx/columnshard/common/snapshot.cpp index a38565ee34..d76b847d8e 100644 --- a/ydb/core/tx/columnshard/common/snapshot.cpp +++ b/ydb/core/tx/columnshard/common/snapshot.cpp @@ -14,4 +14,16 @@ NKikimrColumnShardProto::TSnapshot TSnapshot::SerializeToProto() const { return result; } +TConclusionStatus TSnapshot::DeserializeFromString(const TString& data) { + NKikimrColumnShardProto::TSnapshot proto; + if (!proto.ParseFromArray(data.data(), data.size())) { + return TConclusionStatus::Fail("cannot parse string as snapshot proto"); + } + return DeserializeFromProto(proto); +} + +TString TSnapshot::SerializeToString() const { + return SerializeToProto().SerializeAsString(); +} + }; diff --git a/ydb/core/tx/columnshard/common/snapshot.h b/ydb/core/tx/columnshard/common/snapshot.h index 22466cb555..a9b7e89a26 100644 --- a/ydb/core/tx/columnshard/common/snapshot.h +++ b/ydb/core/tx/columnshard/common/snapshot.h @@ -79,6 +79,10 @@ public: return TConclusionStatus::Success(); } + TConclusionStatus DeserializeFromString(const TString& data); + + TString SerializeToString() const; + TString DebugString() const; }; diff --git a/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h b/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h index d4205a0db6..f69e08334d 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h +++ b/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h @@ -20,7 +20,7 @@ namespace NKikimr::NOlap::NDataSharing::NEvents { class TPathIdData { private: YDB_READONLY(ui64, PathId, 0); - YDB_READONLY_DEF(std::vector<TPortionInfo>, Portions); + YDB_ACCESSOR_DEF(std::vector<TPortionInfo>, Portions); TPathIdData() = default; diff --git a/ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp b/ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp index e09e894f8f..96b329ad5a 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp +++ b/ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp @@ -1,10 +1,11 @@ #include "destination.h" + #include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h> #include <ydb/core/tx/columnshard/data_locks/locks/list.h> #include <ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h> +#include <ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.h> #include <ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_finish_ack_from_initiator.h> #include <ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_finish_from_source.h> -#include <ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.h> #include <ydb/core/tx/columnshard/engines/column_engine_logs.h> #include <ydb/core/tx/columnshard/hooks/abstract/abstract.h> @@ -16,27 +17,9 @@ NKikimr::TConclusionStatus TDestinationSession::DataReceived(THashMap<ui64, NEve auto it = PathIds.find(i.first); AFL_VERIFY(it != PathIds.end())("path_id_undefined", i.first); for (auto&& portion : i.second.DetachPortions()) { - ui32 contains = 0; - ui32 notContains = 0; - THashMap<TString, THashSet<TUnifiedBlobId>> blobIds; - portion.FillBlobIdsByStorage(blobIds, index.GetVersionedIndex()); - for (auto&& s : blobIds) { - auto it = CurrentBlobIds.find(s.first); - if (it == CurrentBlobIds.end()) { - notContains += s.second.size(); - continue; - } - for (auto&& b : s.second) { - if (it->second.contains(b)) { - ++contains; - } - } - } - AFL_VERIFY(!contains || !notContains); - if (!contains) { - portion.SetPathId(it->second); - index.UpsertPortion(std::move(portion)); - } + portion.ResetShardingVersion(); + portion.SetPathId(it->second); + index.UpsertPortion(std::move(portion)); } } return TConclusionStatus::Success(); @@ -186,4 +169,23 @@ bool TDestinationSession::DoStart(const NColumnShard::TColumnShard& shard, const return true; } -}
\ No newline at end of file +bool TDestinationSession::TryTakePortionBlobs(const TVersionedIndex& vIndex, const TPortionInfo& portion) { + THashMap<TString, THashSet<TUnifiedBlobId>> blobIds; + portion.FillBlobIdsByStorage(blobIds, vIndex); + ui32 containsCounter = 0; + ui32 newCounter = 0; + for (auto&& i : blobIds) { + auto& storageBlobIds = CurrentBlobIds[i.first]; + for (auto&& b : i.second) { + if (storageBlobIds.emplace(b).second) { + ++newCounter; + } else { + ++containsCounter; + } + } + } + AFL_VERIFY((containsCounter == 0) ^ (newCounter == 0)); + return newCounter; +} + +} // namespace NKikimr::NOlap::NDataSharing diff --git a/ydb/core/tx/columnshard/data_sharing/destination/session/destination.h b/ydb/core/tx/columnshard/data_sharing/destination/session/destination.h index 1c702e62e1..7207f058a2 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/session/destination.h +++ b/ydb/core/tx/columnshard/data_sharing/destination/session/destination.h @@ -1,10 +1,11 @@ #pragma once -#include <ydb/core/tx/columnshard/common/tablet_id.h> - #include <ydb/core/tx/columnshard/blob.h> +#include <ydb/core/tx/columnshard/common/tablet_id.h> #include <ydb/core/tx/columnshard/data_sharing/common/session/common.h> #include <ydb/core/tx/columnshard/data_sharing/initiator/controller/abstract.h> #include <ydb/core/tx/columnshard/data_sharing/protos/sessions.pb.h> +#include <ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h> + #include <ydb/library/conclusion/result.h> namespace NKikimr::NColumnShard { @@ -14,7 +15,7 @@ class TColumnShard; namespace NKikimr::NOlap { class TColumnEngineForLogs; class IStoragesManager; -} +} // namespace NKikimr::NOlap namespace NKikimr::NOlap::NDataSharing { @@ -27,12 +28,11 @@ private: YDB_READONLY(TTabletId, TabletId, (TTabletId)0); YDB_READONLY(ui32, PackIdx, 0); YDB_READONLY(bool, DataFinished, false); + public: TSourceCursorForDestination() = default; TSourceCursorForDestination(const TTabletId tabletId) - : TabletId(tabletId) - { - + : TabletId(tabletId) { } TConclusionStatus ReceiveData(const ui32 packIdxReceived) { @@ -65,7 +65,6 @@ public: result.SetFinished(DataFinished); return result; } - }; class TDestinationSession: public TCommonSession { @@ -77,6 +76,7 @@ private: YDB_READONLY_FLAG(Confirmed, false); THashMap<TTabletId, TSourceCursorForDestination> Cursors; THashMap<TString, THashSet<TUnifiedBlobId>> CurrentBlobIds; + protected: virtual bool DoStart(const NColumnShard::TColumnShard& shard, const THashMap<ui64, std::vector<std::shared_ptr<TPortionInfo>>>& portions) override; virtual THashSet<ui64> GetPathIdsForStart() const override { @@ -86,7 +86,10 @@ protected: } return result; } + public: + bool TryTakePortionBlobs(const TVersionedIndex& vIndex, const TPortionInfo& portion); + void SetBarrierSnapshot(const TSnapshot& value) { TransferContext.SetSnapshotBarrier(value); } @@ -100,15 +103,11 @@ public: TDestinationSession(const TInitiatorControllerContainer& controller, const TPathIdsRemapper& remapper, const TString& sessionId, const TTransferContext& context) : TBase(sessionId, "destination_base", context) , InitiatorController(controller) - , PathIds(remapper) - { - + , PathIds(remapper) { } TDestinationSession() - : TBase("dest_proto") - { - + : TBase("dest_proto") { } void Confirm(const bool allowRepeat = false) { @@ -119,7 +118,7 @@ public: [[nodiscard]] TConclusionStatus DataReceived(THashMap<ui64, NEvents::TPathIdData>&& data, TColumnEngineForLogs& index, const std::shared_ptr<IStoragesManager>& manager); ui32 GetSourcesInProgressCount() const; - void SendCurrentCursorAck(const NColumnShard::TColumnShard & shard, const std::optional<TTabletId> tabletId); + void SendCurrentCursorAck(const NColumnShard::TColumnShard& shard, const std::optional<TTabletId> tabletId); NKikimrColumnShardDataSharingProto::TDestinationSession SerializeDataToProto() const; @@ -136,4 +135,4 @@ public: [[nodiscard]] TConclusionStatus DeserializeDataFromProto(const NKikimrColumnShardDataSharingProto::TDestinationSession& proto, const TColumnEngineForLogs& index); }; -}
\ No newline at end of file +} // namespace NKikimr::NOlap::NDataSharing diff --git a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp index a3b6b32646..9d3b10743b 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp +++ b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp @@ -41,6 +41,16 @@ TTxDataFromSource::TTxDataFromSource(NColumnShard::TColumnShard* self, const std , PortionsByPathId(portionsByPathId) , SourceTabletId(sourceTabletId) { + for (auto&& i : PortionsByPathId) { + for (ui32 p = 0; p < i.second.GetPortions().size();) { + if (Session->TryTakePortionBlobs(Self->GetIndexAs<TColumnEngineForLogs>().GetVersionedIndex(), i.second.GetPortions()[p])) { + ++p; + } else { + i.second.MutablePortions()[p] = std::move(i.second.MutablePortions().back()); + i.second.MutablePortions().pop_back(); + } + } + } } }
\ No newline at end of file diff --git a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_start_from_initiator.cpp b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_start_from_initiator.cpp index 2c80eca952..c07a010e6f 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_start_from_initiator.cpp +++ b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_start_from_initiator.cpp @@ -11,7 +11,7 @@ bool TTxProposeFromInitiator::DoExecute(NTabletFlatExecutor::TTransactionContext } void TTxProposeFromInitiator::DoComplete(const TActorContext& /*ctx*/) { - AFL_VERIFY(!Session->IsConfirmed()); + AFL_VERIFY(!Session->IsConfirmed() || Session->GetTransferContext().GetTxId()); AFL_VERIFY(Sessions->emplace(Session->GetSessionId(), Session).second); Session->GetInitiatorController().ProposeSuccess(Session->GetSessionId()); } diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp index 5e763b9164..17a7d10475 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp +++ b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp @@ -66,6 +66,7 @@ TColumnEngineChanges::~TColumnEngineChanges() { } void TColumnEngineChanges::Abort(NColumnShard::TColumnShard& self, TChangesFinishContext& context) { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("problem", "changes_aborted")("reason", context.ErrorMessage); AFL_VERIFY(Stage != EStage::Finished && Stage != EStage::Created && Stage != EStage::Aborted)("stage", Stage)("reason", context.ErrorMessage); Stage = EStage::Aborted; OnFinish(self, context); diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp index 3f5e692df4..e183060da2 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp @@ -7,27 +7,32 @@ #include "compaction/merged_column.h" #include "counters/general.h" +#include <ydb/core/formats/arrow/reader/merger.h> #include <ydb/core/formats/arrow/simple_builder/array.h> #include <ydb/core/formats/arrow/simple_builder/filler.h> +#include <ydb/core/tx/columnshard/columnshard_impl.h> #include <ydb/core/tx/columnshard/engines/portions/read_with_blobs.h> #include <ydb/core/tx/columnshard/engines/portions/write_with_blobs.h> -#include <ydb/core/tx/columnshard/columnshard_impl.h> #include <ydb/core/tx/columnshard/engines/storage/chunks/null_column.h> #include <ydb/core/tx/columnshard/splitter/batch_slice.h> #include <ydb/core/tx/columnshard/splitter/settings.h> -#include <ydb/core/formats/arrow/reader/merger.h> namespace NKikimr::NOlap::NCompaction { void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByFullBatches(TConstructionContext& context, std::vector<TReadPortionInfoWithBlobs>&& portions) noexcept { std::vector<std::shared_ptr<arrow::RecordBatch>> batchResults; auto resultSchema = context.SchemaVersions.GetLastSchema(); + auto shardingActual = context.SchemaVersions.GetShardingInfoActual(GranuleMeta->GetPathId()); { auto resultDataSchema = resultSchema->GetIndexInfo().ArrowSchemaWithSpecials(); NArrow::NMerger::TMergePartialStream mergeStream(resultSchema->GetIndexInfo().GetReplaceKey(), resultDataSchema, false, IIndexInfo::GetSpecialColumnNames()); + for (auto&& i : portions) { auto dataSchema = i.GetPortionInfo().GetSchema(context.SchemaVersions); auto batch = i.GetBatch(dataSchema, *resultSchema); + if (shardingActual && i.GetPortionInfo().NeedShardingFilter(*shardingActual)) { + shardingActual->GetShardingInfo()->GetFilter(batch)->Apply(batch); + } batch = resultSchema->NormalizeBatch(*dataSchema, batch); Y_DEBUG_ABORT_UNLESS(NArrow::IsSortedAndUnique(batch, resultSchema->GetIndexInfo().GetReplaceKey())); mergeStream.AddSource(batch, nullptr); @@ -39,6 +44,9 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByFullBatches(TCon auto portions = MakeAppendedPortions(b, GranuleMeta->GetPathId(), resultSchema->GetSnapshot(), GranuleMeta.get(), context, {}); Y_ABORT_UNLESS(portions.size()); for (auto& portion : portions) { + if (shardingActual) { + portion.GetPortionConstructor().SetShardingVersion(shardingActual->GetSnapshotVersion()); + } AppendedPortions.emplace_back(std::move(portion)); } } @@ -51,6 +59,7 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstruc static const std::shared_ptr<arrow::Field> portionRecordIndexField = std::make_shared<arrow::Field>(portionRecordIndexFieldName, std::make_shared<arrow::UInt32Type>()); auto resultSchema = context.SchemaVersions.GetLastSchema(); + auto shardingActual = context.SchemaVersions.GetShardingInfoActual(GranuleMeta->GetPathId()); std::vector<std::string> pkFieldNames = resultSchema->GetIndexInfo().GetReplaceKey()->field_names(); std::set<std::string> pkFieldNamesSet(pkFieldNames.begin(), pkFieldNames.end()); @@ -81,7 +90,12 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstruc batch = NArrow::TStatusValidator::GetValid(batch->AddColumn(batch->num_columns(), portionRecordIndexField, column->BuildArray(batch->num_rows()))); } Y_DEBUG_ABORT_UNLESS(NArrow::IsSortedAndUnique(batch, resultSchema->GetIndexInfo().GetReplaceKey())); - mergeStream.AddSource(batch, nullptr); + std::shared_ptr<NArrow::TColumnFilter> filter; + if (shardingActual && i.GetPortionInfo().NeedShardingFilter(*shardingActual)) { + filter = shardingActual->GetShardingInfo()->GetFilter(batch); + } + + mergeStream.AddSource(batch, filter); } batchResults = mergeStream.DrainAllParts(CheckPoints, indexFields); } @@ -158,7 +172,6 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstruc ++batchIdx; } AFL_VERIFY(columnRecordsCount == batchesRecordsCount)("mCount", columnRecordsCount)("bCount", batchesRecordsCount); - } ui32 batchIdx = 0; @@ -202,6 +215,9 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstruc NArrow::TMinMaxSpecialKeys snapshotKeys(b, TIndexInfo::ArrowSchemaSnapshot()); AppendedPortions.back().GetPortionConstructor().AddMetadata(*resultSchema, primaryKeys, snapshotKeys); AppendedPortions.back().GetPortionConstructor().MutableMeta().SetTierName(IStoragesManager::DefaultStorageId); + if (shardingActual) { + AppendedPortions.back().GetPortionConstructor().SetShardingVersion(shardingActual->GetSnapshotVersion()); + } recordIdx += slice.GetRecordsCount(); } } @@ -312,4 +328,4 @@ ui64 TGeneralCompactColumnEngineChanges::TMemoryPredictorChunkedPolicy::AddPorti return SumMemoryFix + SumMemoryDelta; } -} +} // namespace NKikimr::NOlap::NCompaction diff --git a/ydb/core/tx/columnshard/engines/changes/indexation.cpp b/ydb/core/tx/columnshard/engines/changes/indexation.cpp index ffbac159cc..123e61ef0f 100644 --- a/ydb/core/tx/columnshard/engines/changes/indexation.cpp +++ b/ydb/core/tx/columnshard/engines/changes/indexation.cpp @@ -50,6 +50,70 @@ void TInsertColumnEngineChanges::DoOnFinish(NColumnShard::TColumnShard& self, TC self.BackgroundController.FinishIndexing(*this); } +namespace { +class TPathData { +private: + std::vector<std::shared_ptr<arrow::RecordBatch>> Batches; + YDB_READONLY_DEF(std::optional<TGranuleShardingInfo>, ShardingInfo); + +public: + TPathData(const std::optional<TGranuleShardingInfo>& shardingInfo) + : ShardingInfo(shardingInfo) + { + + } + void AddBatch(const std::shared_ptr<arrow::RecordBatch>& batch) { + Batches.emplace_back(batch); + } + + void AddShardingInfo(const std::optional<TGranuleShardingInfo>& info) { + if (!info) { + ShardingInfo.reset(); + } else if (ShardingInfo && info->GetSnapshotVersion() < ShardingInfo->GetSnapshotVersion()) { + ShardingInfo = info; + } + } + + std::shared_ptr<arrow::RecordBatch> Merge(const TIndexInfo& indexInfo) const { + NArrow::NMerger::TMergePartialStream stream(indexInfo.GetReplaceKey(), indexInfo.ArrowSchemaWithSpecials(), false, IIndexInfo::GetSpecialColumnNames()); + THashMap<std::string, ui64> fieldSizes; + ui64 rowsCount = 0; + for (auto&& batch : Batches) { + stream.AddSource(batch, nullptr); + for (ui32 cIdx = 0; cIdx < (ui32)batch->num_columns(); ++cIdx) { + fieldSizes[batch->column_name(cIdx)] += NArrow::GetArrayDataSize(batch->column(cIdx)); + } + rowsCount += batch->num_rows(); + } + + NArrow::NMerger::TRecordBatchBuilder builder(indexInfo.ArrowSchemaWithSpecials()->fields(), rowsCount, fieldSizes); + stream.SetPossibleSameVersion(true); + stream.DrainAll(builder); + return builder.Finalize(); + } +}; + +class TPathesData { +private: + THashMap<ui64, TPathData> Data; + +public: + const THashMap<ui64, TPathData>& GetData() const { + return Data; + } + + void Add(const ui64 pathId, const std::optional<TGranuleShardingInfo>& info, const std::shared_ptr<arrow::RecordBatch>& batch) { + auto it = Data.find(pathId); + if (it == Data.end()) { + it = Data.emplace(pathId, info).first; + } + it->second.AddShardingInfo(info); + it->second.AddBatch(batch); + + } +}; +} + TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionContext& context) noexcept { Y_ABORT_UNLESS(!DataToIndex.empty()); Y_ABORT_UNLESS(AppendedPortions.empty()); @@ -67,10 +131,12 @@ TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionCont auto resultSchema = context.SchemaVersions.GetSchema(maxSnapshot); Y_ABORT_UNLESS(resultSchema->GetIndexInfo().IsSorted()); - THashMap<ui64, std::vector<std::shared_ptr<arrow::RecordBatch>>> pathBatches; + TPathesData pathBatches; for (auto& inserted : DataToIndex) { const TBlobRange& blobRange = inserted.GetBlobRange(); + auto shardingFilterCommit = context.SchemaVersions.GetShardingInfoOptional(inserted.PathId, inserted.GetSnapshot()); + auto blobSchema = context.SchemaVersions.GetSchemaVerified(inserted.GetSchemaVersion()); auto& indexInfo = blobSchema->GetIndexInfo(); Y_ABORT_UNLESS(indexInfo.IsSorted()); @@ -89,31 +155,20 @@ TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionCont batch = AddSpecials(batch, indexInfo, inserted); batch = resultSchema->NormalizeBatch(*blobSchema, batch); - pathBatches[inserted.PathId].push_back(batch); - Y_DEBUG_ABORT_UNLESS(NArrow::IsSorted(pathBatches[inserted.PathId].back(), resultSchema->GetIndexInfo().GetReplaceKey())); + pathBatches.Add(inserted.PathId, shardingFilterCommit, batch); + Y_DEBUG_ABORT_UNLESS(NArrow::IsSorted(batch, resultSchema->GetIndexInfo().GetReplaceKey())); } Y_ABORT_UNLESS(Blobs.IsEmpty()); const std::vector<std::string> comparableColumns = resultSchema->GetIndexInfo().GetReplaceKey()->field_names(); - for (auto& [pathId, batches] : pathBatches) { - NArrow::NMerger::TMergePartialStream stream(resultSchema->GetIndexInfo().GetReplaceKey(), resultSchema->GetIndexInfo().ArrowSchemaWithSpecials(), false, IIndexInfo::GetSpecialColumnNames()); - THashMap<std::string, ui64> fieldSizes; - ui64 rowsCount = 0; - for (auto&& batch : batches) { - stream.AddSource(batch, nullptr); - for (ui32 cIdx = 0; cIdx < (ui32)batch->num_columns(); ++cIdx) { - fieldSizes[batch->column_name(cIdx)] += NArrow::GetArrayDataSize(batch->column(cIdx)); - } - rowsCount += batch->num_rows(); - } - - NArrow::NMerger::TRecordBatchBuilder builder(resultSchema->GetIndexInfo().ArrowSchemaWithSpecials()->fields(), rowsCount, fieldSizes); - stream.SetPossibleSameVersion(true); - stream.DrainAll(builder); + for (auto& [pathId, pathInfo] : pathBatches.GetData()) { + auto shardingFilter = context.SchemaVersions.GetShardingInfoActual(pathId); + auto mergedBatch = pathInfo.Merge(resultSchema->GetIndexInfo()); auto itGranule = PathToGranule.find(pathId); AFL_VERIFY(itGranule != PathToGranule.end()); - std::vector<std::shared_ptr<arrow::RecordBatch>> result = NArrow::NMerger::TSortableBatchPosition::SplitByBordersInSequentialContainer(builder.Finalize(), comparableColumns, itGranule->second); + std::vector<std::shared_ptr<arrow::RecordBatch>> result = NArrow::NMerger::TSortableBatchPosition:: + SplitByBordersInSequentialContainer(mergedBatch, comparableColumns, itGranule->second); for (auto&& b : result) { if (!b) { continue; @@ -127,12 +182,15 @@ TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionCont auto portions = MakeAppendedPortions(b, pathId, maxSnapshot, nullptr, context, externalSaver); Y_ABORT_UNLESS(portions.size()); for (auto& portion : portions) { + if (pathInfo.GetShardingInfo()) { + portion.GetPortionConstructor().SetShardingVersion(pathInfo.GetShardingInfo()->GetSnapshotVersion()); + } AppendedPortions.emplace_back(std::move(portion)); } } } - Y_ABORT_UNLESS(PathToGranule.size() == pathBatches.size()); + Y_ABORT_UNLESS(PathToGranule.size() == pathBatches.GetData().size()); return TConclusionStatus::Success(); } diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp index 787f34ac5a..f57f6660de 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp @@ -135,8 +135,6 @@ std::vector<TWritePortionInfoWithBlobs> TChangesWithAppend::MakeAppendedPortions auto b = batch->Slice(recordIdx, slice.GetRecordsCount()); out.emplace_back(TWritePortionInfoWithBlobs::BuildByBlobs(slice.GroupChunksByBlobs(groups), pathId, resultSchema->GetVersion(), snapshot, SaverContext.GetStoragesManager())); out.back().FillStatistics(resultSchema->GetIndexInfo()); - NArrow::TFirstLastSpecialKeys primaryKeys(slice.GetFirstLastPKBatch(resultSchema->GetIndexInfo().GetReplaceKey())); - NArrow::TMinMaxSpecialKeys snapshotKeys(b, TIndexInfo::ArrowSchemaSnapshot()); out.back().GetPortionConstructor().AddMetadata(*resultSchema, b); out.back().GetPortionConstructor().MutableMeta().SetTierName(IStoragesManager::DefaultStorageId); recordIdx += slice.GetRecordsCount(); diff --git a/ydb/core/tx/columnshard/engines/column_engine.h b/ydb/core/tx/columnshard/engines/column_engine.h index 58a2d8eb31..2487d8d9cf 100644 --- a/ydb/core/tx/columnshard/engines/column_engine.h +++ b/ydb/core/tx/columnshard/engines/column_engine.h @@ -1,7 +1,8 @@ #pragma once #include "db_wrapper.h" -#include "changes/abstract/settings.h" + #include "changes/abstract/compaction_info.h" +#include "changes/abstract/settings.h" #include "predicate/filter.h" #include "scheme/snapshot_scheme.h" #include "scheme/versions/versioned_index.h" @@ -11,7 +12,7 @@ namespace NKikimr::NColumnShard { class TTiersManager; class TTtl; -} +} // namespace NKikimr::NColumnShard namespace NKikimr::NOlap { class TInsertColumnEngineChanges; @@ -33,7 +34,7 @@ struct TSelectInfo { size_t Rows{}; size_t Bytes{}; - const TStats& operator += (const TStats& stats) { + const TStats& operator+=(const TStats& stats) { Portions += stats.Portions; Records += stats.Records; Blobs += stats.Blobs; @@ -60,6 +61,7 @@ class TColumnEngineStats { private: static constexpr const ui64 NUM_KINDS = 5; static_assert(NUM_KINDS == NOlap::TPortionMeta::EProduced::EVICTED, "NUM_KINDS must match NOlap::TPortionMeta::EProduced enum"); + public: class TPortionsStats { private: @@ -69,8 +71,8 @@ public: Y_ABORT_UNLESS(result >= 0); return result; } - public: + public: i64 Portions = 0; i64 Blobs = 0; i64 Rows = 0; @@ -263,8 +265,8 @@ public: class IColumnEngine { protected: virtual void DoRegisterTable(const ui64 pathId) = 0; -public: +public: static ui64 GetMetadataLimit(); virtual ~IColumnEngine() = default; @@ -280,24 +282,25 @@ public: return DoRegisterTable(pathId); } virtual bool IsOverloadedByMetadata(const ui64 limit) const = 0; - virtual std::shared_ptr<TSelectInfo> Select(ui64 pathId, TSnapshot snapshot, - const TPKRangesFilter& pkRangesFilter) const = 0; + virtual std::shared_ptr<TSelectInfo> Select(ui64 pathId, TSnapshot snapshot, const TPKRangesFilter& pkRangesFilter) const = 0; virtual std::shared_ptr<TInsertColumnEngineChanges> StartInsert(std::vector<TInsertedData>&& dataToIndex) noexcept = 0; virtual std::shared_ptr<TColumnEngineChanges> StartCompaction(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) noexcept = 0; - virtual std::shared_ptr<TCleanupPortionsColumnEngineChanges> StartCleanupPortions(const TSnapshot& snapshot, const THashSet<ui64>& pathsToDrop, - const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) noexcept = 0; + virtual std::shared_ptr<TCleanupPortionsColumnEngineChanges> StartCleanupPortions(const TSnapshot& snapshot, const THashSet<ui64>& pathsToDrop, const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) noexcept = 0; virtual std::shared_ptr<TCleanupTablesColumnEngineChanges> StartCleanupTables(THashSet<ui64>& pathsToDrop) noexcept = 0; - virtual std::vector<std::shared_ptr<TTTLColumnEngineChanges>> StartTtl(const THashMap<ui64, TTiering>& pathEviction, - const std::shared_ptr<NDataLocks::TManager>& dataLocksManager, const ui64 memoryUsageLimit) noexcept = 0; + virtual std::vector<std::shared_ptr<TTTLColumnEngineChanges>> StartTtl(const THashMap<ui64, TTiering>& pathEviction, const std::shared_ptr<NDataLocks::TManager>& dataLocksManager, const ui64 memoryUsageLimit) noexcept = 0; virtual bool ApplyChangesOnTxCreate(std::shared_ptr<TColumnEngineChanges> changes, const TSnapshot& snapshot) noexcept = 0; virtual bool ApplyChangesOnExecute(IDbWrapper& db, std::shared_ptr<TColumnEngineChanges> changes, const TSnapshot& snapshot) noexcept = 0; virtual void RegisterSchemaVersion(const TSnapshot& snapshot, TIndexInfo&& info) = 0; virtual void RegisterSchemaVersion(const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema) = 0; virtual const TMap<ui64, std::shared_ptr<TColumnEngineStats>>& GetStats() const = 0; virtual const TColumnEngineStats& GetTotalStats() = 0; - virtual ui64 MemoryUsage() const { return 0; } - virtual TSnapshot LastUpdate() const { return TSnapshot::Zero(); } + virtual ui64 MemoryUsage() const { + return 0; + } + virtual TSnapshot LastUpdate() const { + return TSnapshot::Zero(); + } virtual void OnTieringModified(const std::shared_ptr<NColumnShard::TTiersManager>& manager, const NColumnShard::TTtl& ttl, const std::optional<ui64> pathId) = 0; }; -} +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index 31ae9866ac..50dfe3bb73 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -162,6 +162,13 @@ bool TColumnEngineForLogs::Load(IDbWrapper& db) { Loaded = true; THashMap<ui64, ui64> granuleToPathIdDecoder; { + TMemoryProfileGuard g("TTxInit/LoadShardingInfo"); + if (!VersionedIndex.LoadShardingInfo(db)) { + return false; + } + } + + { TMemoryProfileGuard g("TTxInit/LoadColumns"); auto guard = GranulesStorage->GetStats()->StartPackModification(); if (!LoadColumns(db)) { diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index bdad1f4ba4..5628ef979b 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -45,6 +45,7 @@ class TColumnEngineForLogs : public IColumnEngine { friend class TCleanupPortionsColumnEngineChanges; friend class TCleanupTablesColumnEngineChanges; friend class NDataSharing::TDestinationSession; + private: bool ActualizationStarted = false; const NColumnShard::TEngineLogsCounters SignalCounters; @@ -56,6 +57,7 @@ private: static TDuration GetRemovedPortionLivetime(); const TDuration RemovedPortionLivetime = GetRemovedPortionLivetime(); + public: const std::shared_ptr<NActualizer::TController>& GetActualizationController() const { return ActualizationController; @@ -97,9 +99,12 @@ public: const TMap<ui64, std::shared_ptr<TColumnEngineStats>>& GetStats() const override; const TColumnEngineStats& GetTotalStats() override; - TSnapshot LastUpdate() const override { return LastSnapshot; } + TSnapshot LastUpdate() const override { + return LastSnapshot; + } virtual void DoRegisterTable(const ui64 pathId) override; + public: bool Load(IDbWrapper& db) override; @@ -111,8 +116,7 @@ public: std::shared_ptr<TColumnEngineChanges> StartCompaction(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) noexcept override; std::shared_ptr<TCleanupPortionsColumnEngineChanges> StartCleanupPortions(const TSnapshot& snapshot, const THashSet<ui64>& pathsToDrop, const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) noexcept override; std::shared_ptr<TCleanupTablesColumnEngineChanges> StartCleanupTables(THashSet<ui64>& pathsToDrop) noexcept override; - std::vector<std::shared_ptr<TTTLColumnEngineChanges>> StartTtl(const THashMap<ui64, TTiering>& pathEviction, - const std::shared_ptr<NDataLocks::TManager>& locksManager, const ui64 memoryUsageLimit) noexcept override; + std::vector<std::shared_ptr<TTTLColumnEngineChanges>> StartTtl(const THashMap<ui64, TTiering>& pathEviction, const std::shared_ptr<NDataLocks::TManager>& locksManager, const ui64 memoryUsageLimit) noexcept override; void ReturnToIndexes(const THashMap<ui64, THashSet<ui64>>& portions) const { return GranulesStorage->ReturnToIndexes(portions); @@ -123,8 +127,7 @@ public: void RegisterSchemaVersion(const TSnapshot& snapshot, TIndexInfo&& info) override; void RegisterSchemaVersion(const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema) override; - std::shared_ptr<TSelectInfo> Select(ui64 pathId, TSnapshot snapshot, - const TPKRangesFilter& pkRangesFilter) const override; + std::shared_ptr<TSelectInfo> Select(ui64 pathId, TSnapshot snapshot, const TPKRangesFilter& pkRangesFilter) const override; bool IsPortionExists(const ui64 pathId, const ui64 portionId) const { return !!GranulesStorage->GetPortionOptional(pathId, portionId); @@ -164,30 +167,32 @@ public: void AddCleanupPortion(const TPortionInfo& info) { CleanupPortions[info.GetRemoveSnapshotVerified().GetPlanInstant() + RemovedPortionLivetime].emplace_back(info); } + void AddShardingInfo(const TGranuleShardingInfo& shardingInfo) { + VersionedIndex.AddShardingInfo(shardingInfo); + } private: TVersionedIndex VersionedIndex; ui64 TabletId; - TMap<ui64, std::shared_ptr<TColumnEngineStats>> PathStats; // per path_id stats sorted by path_id + TMap<ui64, std::shared_ptr<TColumnEngineStats>> PathStats; // per path_id stats sorted by path_id std::map<TInstant, std::vector<TPortionInfo>> CleanupPortions; TColumnEngineStats Counters; ui64 LastPortion; ui64 LastGranule; TSnapshot LastSnapshot = TSnapshot::Zero(); bool Loaded = false; + private: bool LoadColumns(IDbWrapper& db); + bool LoadShardingInfo(IDbWrapper& db); bool LoadCounters(IDbWrapper& db); void EraseTable(const ui64 pathId); void UpsertPortion(const TPortionInfo& portionInfo, const TPortionInfo* exInfo = nullptr); bool ErasePortion(const TPortionInfo& portionInfo, bool updateStats = true); - void UpdatePortionStats(const TPortionInfo& portionInfo, EStatsUpdateType updateType = EStatsUpdateType::DEFAULT, - const TPortionInfo* exPortionInfo = nullptr); - void UpdatePortionStats(TColumnEngineStats& engineStats, const TPortionInfo& portionInfo, - EStatsUpdateType updateType, - const TPortionInfo* exPortionInfo = nullptr) const; + void UpdatePortionStats(const TPortionInfo& portionInfo, EStatsUpdateType updateType = EStatsUpdateType::DEFAULT, const TPortionInfo* exPortionInfo = nullptr); + void UpdatePortionStats(TColumnEngineStats& engineStats, const TPortionInfo& portionInfo, EStatsUpdateType updateType, const TPortionInfo* exPortionInfo = nullptr) const; }; -} // namespace NKikimr::NOlap +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp index 4bc8f9bf09..302c3b6451 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp +++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp @@ -1,7 +1,9 @@ #include "defs.h" #include "db_wrapper.h" #include "portions/constructor.h" +#include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/tx/columnshard/columnshard_schema.h> +#include <ydb/core/tx/sharding/sharding.h> namespace NKikimr::NOlap { @@ -68,6 +70,7 @@ void TDbWrapper::WritePortion(const NOlap::TPortionInfo& portion) { auto removeSnapshot = portion.GetRemoveSnapshotOptional(); db.Table<IndexPortions>().Key(portion.GetPathId(), portion.GetPortion()).Update( NIceDb::TUpdate<IndexPortions::SchemaVersion>(portion.GetSchemaVersionVerified()), + NIceDb::TUpdate<IndexPortions::ShardingVersion>(portion.GetShardingVersionDef(0)), NIceDb::TUpdate<IndexPortions::XPlanStep>(removeSnapshot ? removeSnapshot->GetPlanStep() : 0), NIceDb::TUpdate<IndexPortions::XTxId>(removeSnapshot ? removeSnapshot->GetTxId() : 0), NIceDb::TUpdate<IndexPortions::Metadata>(metaProto.SerializeAsString())); @@ -123,6 +126,9 @@ bool TDbWrapper::LoadPortions(const std::function<void(NOlap::TPortionInfoConstr while (!rowset.EndOfSet()) { NOlap::TPortionInfoConstructor portion(rowset.GetValue<IndexPortions::PathId>(), rowset.GetValue<IndexPortions::PortionId>()); portion.SetSchemaVersion(rowset.GetValue<IndexPortions::SchemaVersion>()); + if (rowset.HaveValue<IndexPortions::ShardingVersion>() && rowset.GetValue<IndexPortions::ShardingVersion>()) { + portion.SetShardingVersion(rowset.GetValue<IndexPortions::ShardingVersion>()); + } portion.SetRemoveSnapshot(rowset.GetValue<IndexPortions::XPlanStep>(), rowset.GetValue<IndexPortions::XTxId>()); NKikimrTxColumnShard::TIndexPortionMeta metaProto; @@ -185,4 +191,27 @@ bool TDbWrapper::LoadCounters(const std::function<void(ui32 id, ui64 value)>& ca return NColumnShard::Schema::IndexCounters_Load(db, callback); } +TConclusion<THashMap<ui64, std::map<NOlap::TSnapshot, TGranuleShardingInfo>>> TDbWrapper::LoadGranulesShardingInfo() { + using Schema = NColumnShard::Schema; + NIceDb::TNiceDb db(Database); + auto rowset = db.Table<Schema::ShardingInfo>().Select(); + if (!rowset.IsReady()) { + return TConclusionStatus::Fail("cannot read rowset"); + } + THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>> result; + while (!rowset.EndOfSet()) { + NOlap::TSnapshot snapshot = NOlap::TSnapshot::Zero(); + snapshot.DeserializeFromString(rowset.GetValue<Schema::ShardingInfo::Snapshot>()).Validate(); + NSharding::TGranuleShardingLogicContainer logic; + logic.DeserializeFromString(rowset.GetValue<Schema::ShardingInfo::Logic>()).Validate(); + TGranuleShardingInfo gShardingInfo(logic, snapshot, rowset.GetValue<Schema::ShardingInfo::VersionId>(), rowset.GetValue<Schema::ShardingInfo::PathId>()); + AFL_VERIFY(result[gShardingInfo.GetPathId()].emplace(gShardingInfo.GetSinceSnapshot(), gShardingInfo).second); + + if (!rowset.Next()) { + return TConclusionStatus::Fail("cannot read rowset"); + } + } + return result; +} + } diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.h b/ydb/core/tx/columnshard/engines/db_wrapper.h index d9435913bd..39536cb9c9 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.h +++ b/ydb/core/tx/columnshard/engines/db_wrapper.h @@ -1,6 +1,8 @@ #pragma once #include "defs.h" +#include "scheme/versions/versioned_index.h" #include <ydb/core/tx/columnshard/common/blob.h> +#include <ydb/core/tx/columnshard/common/snapshot.h> namespace NKikimrTxColumnShard { class TIndexPortionMeta; @@ -34,8 +36,7 @@ public: virtual void EraseCommitted(const TInsertedData& data) = 0; virtual void EraseAborted(const TInsertedData& data) = 0; - virtual bool Load(TInsertTableAccessor& insertTable, - const TInstant& loadTime) = 0; + virtual bool Load(TInsertTableAccessor& insertTable, const TInstant& loadTime) = 0; virtual void WriteColumn(const TPortionInfo& portion, const TColumnRecord& row, const ui32 firstPKColumnId) = 0; virtual void EraseColumn(const TPortionInfo& portion, const TColumnRecord& row) = 0; @@ -51,6 +52,7 @@ public: virtual void WriteCounter(ui32 counterId, ui64 value) = 0; virtual bool LoadCounters(const std::function<void(ui32 id, ui64 value)>& callback) = 0; + virtual TConclusion<THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>>> LoadGranulesShardingInfo() = 0; }; class TDbWrapper : public IDbWrapper { @@ -84,6 +86,8 @@ public: void WriteCounter(ui32 counterId, ui64 value) override; bool LoadCounters(const std::function<void(ui32 id, ui64 value)>& callback) override; + virtual TConclusion<THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>>> LoadGranulesShardingInfo() override; + private: NTable::TDatabase& Database; const IBlobGroupSelector* DsGroupSelector; diff --git a/ydb/core/tx/columnshard/engines/portions/constructor.cpp b/ydb/core/tx/columnshard/engines/portions/constructor.cpp index f3de740f4f..9ed5046dad 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor.cpp +++ b/ydb/core/tx/columnshard/engines/portions/constructor.cpp @@ -1,8 +1,11 @@ #include "constructor.h" + #include <ydb/core/tx/columnshard/columnshard_schema.h> +#include <ydb/core/tx/columnshard/common/limits.h> #include <ydb/core/tx/columnshard/engines/scheme/index_info.h> #include <ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h> #include <ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h> +#include <ydb/core/tx/columnshard/hooks/abstract/abstract.h> namespace NKikimr::NOlap { @@ -20,6 +23,7 @@ TPortionInfo TPortionInfoConstructor::Build(const bool needChunksNormalization) result.RemoveSnapshot = *RemoveSnapshot; } result.SchemaVersion = SchemaVersion; + result.ShardingVersion = ShardingVersion; if (needChunksNormalization) { ReorderChunks(); @@ -84,4 +88,4 @@ void TPortionInfoConstructor::AddMetadata(const ISnapshotSchema& snapshotSchema, NArrow::TMinMaxSpecialKeys(batch, TIndexInfo::ArrowSchemaSnapshot()), snapshotSchema.GetIndexInfo()); } -}
\ No newline at end of file +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/portions/constructor.h b/ydb/core/tx/columnshard/engines/portions/constructor.h index 23229c6c9b..13369a2e08 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor.h +++ b/ydb/core/tx/columnshard/engines/portions/constructor.h @@ -1,6 +1,6 @@ #pragma once -#include "constructor_meta.h" #include "column_record.h" +#include "constructor_meta.h" #include "index_chunk.h" #include "portion_info.h" @@ -11,6 +11,7 @@ class TPortionInfo; class TVersionedIndex; class ISnapshotSchema; class TIndexChunkLoadContext; +class TGranuleShardingInfo; class TPortionInfoConstructor { private: @@ -22,10 +23,12 @@ private: std::optional<TSnapshot> MinSnapshotDeprecated; std::optional<TSnapshot> RemoveSnapshot; std::optional<ui64> SchemaVersion; + std::optional<ui64> ShardingVersion; std::vector<TIndexChunk> Indexes; YDB_ACCESSOR_DEF(std::vector<TColumnRecord>, Records); std::vector<TUnifiedBlobId> BlobIds; + public: void SetPortionId(const ui64 value) { AFL_VERIFY(value); @@ -58,7 +61,7 @@ public: , MinSnapshotDeprecated(portion.GetMinSnapshotDeprecated()) , RemoveSnapshot(portion.GetRemoveSnapshotOptional()) , SchemaVersion(portion.GetSchemaVersionOptional()) - { + , ShardingVersion(portion.GetShardingVersionOptional()) { if (withMetadata) { MetaConstructor = TPortionMetaConstructor(portion.Meta); } @@ -75,7 +78,7 @@ public: , MinSnapshotDeprecated(portion.GetMinSnapshotDeprecated()) , RemoveSnapshot(portion.GetRemoveSnapshotOptional()) , SchemaVersion(portion.GetSchemaVersionOptional()) - { + , ShardingVersion(portion.GetShardingVersionOptional()) { MetaConstructor = TPortionMetaConstructor(portion.Meta); Indexes = std::move(portion.Indexes); Records = std::move(portion.Records); @@ -152,10 +155,15 @@ public: } void SetSchemaVersion(const ui64 version) { -// AFL_VERIFY(version); engines/ut +// AFL_VERIFY(version); SchemaVersion = version; } + void SetShardingVersion(const ui64 version) { +// AFL_VERIFY(version); + ShardingVersion = version; + } + void SetRemoveSnapshot(const TSnapshot& snap) { AFL_VERIFY(!RemoveSnapshot); if (snap.Valid()) { @@ -287,6 +295,7 @@ public: class TPortionConstructors { private: THashMap<ui64, THashMap<ui64, TPortionInfoConstructor>> Constructors; + public: THashMap<ui64, THashMap<ui64, TPortionInfoConstructor>>::iterator begin() { return Constructors.begin(); @@ -326,7 +335,6 @@ public: itPortionId->second.Merge(std::move(constructor)); return &itPortionId->second; } - }; -} +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index 39dfa6af20..065f9768fb 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -672,6 +672,13 @@ TPortionInfo::TPreparedBatchData TPortionInfo::PrepareForAssemble(const ISnapsho return PrepareForAssembleImpl(*this, dataSchema, resultSchema, blobsData); } +bool TPortionInfo::NeedShardingFilter(const TGranuleShardingInfo& shardingInfo) const { + if (ShardingVersion && shardingInfo.GetSnapshotVersion() <= *ShardingVersion) { + return false; + } + return true; +} + std::shared_ptr<TDeserializeChunkedArray> TPortionInfo::TPreparedColumn::AssembleForSeqAccess() const { Y_ABORT_UNLESS(!Blobs.empty()); diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h index 966d636394..8232839dc4 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.h +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.h @@ -101,6 +101,7 @@ public: }; class TPortionInfoConstructor; +class TGranuleShardingInfo; class TPortionInfo { public: @@ -120,6 +121,7 @@ private: TSnapshot MinSnapshotDeprecated = TSnapshot::Zero(); // {PlanStep, TxId} is min snapshot for {Granule, Portion} TSnapshot RemoveSnapshot = TSnapshot::Zero(); // {XPlanStep, XTxId} is snapshot where the blob has been removed (i.e. compacted into another one) std::optional<ui64> SchemaVersion; + std::optional<ui64> ShardingVersion; TPortionMeta Meta; YDB_READONLY_DEF(std::vector<TIndexChunk>, Indexes); @@ -172,6 +174,16 @@ private: public: ui64 GetMinMemoryForReadColumns(const std::optional<std::set<ui32>>& columnIds) const; + bool NeedShardingFilter(const TGranuleShardingInfo& shardingInfo) const; + + const std::optional<ui64>& GetShardingVersionOptional() const { + return ShardingVersion; + } + + ui64 GetShardingVersionDef(const ui64 verDefault) const { + return ShardingVersion.value_or(verDefault); + } + void SetRemoveSnapshot(const TSnapshot& snap) { AFL_VERIFY(!RemoveSnapshot.Valid()); RemoveSnapshot = snap; @@ -438,6 +450,10 @@ public: return TPortionAddress(PathId, Portion); } + void ResetShardingVersion() { + ShardingVersion.reset(); + } + void SetPathId(const ui64 pathId) { PathId = pathId; } diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h index 19ff369833..7770674caa 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h +++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h @@ -38,6 +38,8 @@ private: TProgramContainer Program; std::shared_ptr<TVersionedIndex> IndexVersionsPointer; TSnapshot RequestSnapshot; + std::optional<TGranuleShardingInfo> RequestShardingInfo; + protected: std::shared_ptr<ISnapshotSchema> ResultIndexSchema; const TVersionedIndex& GetIndexVersions() const { @@ -47,6 +49,10 @@ protected: public: using TConstPtr = std::shared_ptr<const TReadMetadataBase>; + const std::optional<TGranuleShardingInfo>& GetRequestShardingInfo() const { + return RequestShardingInfo; + } + void SetPKRangesFilter(const TPKRangesFilter& value) { Y_ABORT_UNLESS(IsSorted() && value.IsReverse() == IsDescSorted()); Y_ABORT_UNLESS(!PKRangesFilter); @@ -79,6 +85,11 @@ public: return ResultIndexSchema->GetIndexInfo(); } + void InitShardingInfo(const ui64 pathId) { + AFL_VERIFY(!RequestShardingInfo); + RequestShardingInfo = IndexVersionsPointer->GetShardingInfoOptional(pathId, RequestSnapshot); + } + TReadMetadataBase(const std::shared_ptr<TVersionedIndex> index, const ESorting sorting, const TProgramContainer& ssaProgram, const std::shared_ptr<ISnapshotSchema>& schema, const TSnapshot& requestSnapshot) : Sorting(sorting) , Program(ssaProgram) diff --git a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp index 878153f52b..8b655d899a 100644 --- a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp +++ b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp @@ -121,7 +121,7 @@ void TColumnShardScan::HandleScan(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr& ev) Y_ABORT_UNLESS(!AckReceivedInstant); AckReceivedInstant = TMonotonic::Now(); - Y_ABORT_UNLESS(ev->Get()->Generation == ScanGen); + AFL_VERIFY(ev->Get()->Generation == ScanGen)("ev_gen", ev->Get()->Generation)("scan_gen", ScanGen); ChunksLimiter = TChunksLimiter(ev->Get()->FreeSpace, ev->Get()->MaxChunksCount); Y_ABORT_UNLESS(ev->Get()->MaxChunksCount == 1); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp index a664f71756..076b69c7f4 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp @@ -10,6 +10,7 @@ std::unique_ptr<TScanIteratorBase> TReadMetadata::StartScan(const std::shared_pt TConclusionStatus TReadMetadata::Init(const TReadDescription& readDescription, const TDataStorageAccessor& dataAccessor) { SetPKRangesFilter(readDescription.PKRangesFilter); + InitShardingInfo(readDescription.PathId); /// @note We could have column name changes between schema versions: /// Add '1:foo', Drop '1:foo', Add '2:foo'. Drop should hide '1:foo' from reads. diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp index 06cbc877a3..f5415a4acf 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp @@ -24,7 +24,7 @@ ui64 TSpecialReadContext::GetMemoryForSources(const THashMap<ui32, std::shared_p result += ReadSequentiallyBufferSize; } else { if (!isExclusive && !CommonContext->IsReverse()) { - result = 2 * result; // due to in time we will have data in original portion + data in merged(or reversed) interval + result = 2 * result; // due to in time we will have data in original portion + data in merged(or reversed) interval } } return result; @@ -35,8 +35,16 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::GetColumnsFetchingPlan(con const bool partialUsageByPK = ReadMetadata->GetPKRangesFilter().IsPortionInPartialUsage(source->GetStartReplaceKey(), source->GetFinishReplaceKey(), ReadMetadata->GetIndexInfo()); const bool useIndexes = (IndexChecker ? source->HasIndexes(IndexChecker->GetIndexIds()) : false); const bool isWholeExclusiveSource = source->GetExclusiveIntervalOnly() && source->IsSourceInMemory(); - if (auto result = CacheFetchingScripts[needSnapshots ? 1 : 0][isWholeExclusiveSource ? 1 : 0][partialUsageByPK ? 1 : 0][useIndexes ? 1 : 0]) { -// AFL_WARN(NKikimrServices::TX_COLUMNSHARD_SCAN)("SS", needSnapshots)("PK", partialUsageByPK)("IDX", useIndexes)("EXCL", source->GetExclusiveIntervalOnly())("MEM", source->IsSourceInMemory())("result", result->DebugString()); + bool needShardingFilter = false; + if (!!ReadMetadata->GetRequestShardingInfo()) { + auto ver = source->GetShardingVersionOptional(); + if (!ver || *ver < ReadMetadata->GetRequestShardingInfo()->GetSnapshotVersion()) { + needShardingFilter = true; + } + } + if (auto result = CacheFetchingScripts[needSnapshots ? 1 : 0][isWholeExclusiveSource ? 1 : 0][partialUsageByPK ? 1 : 0][useIndexes ? 1 : 0][needShardingFilter ? 1 : 0]) { +// AFL_WARN(NKikimrServices::TX_COLUMNSHARD_SCAN)("SS", needSnapshots)("PK", partialUsageByPK)("IDX", useIndexes)("SHARDING", needShardingFilter) +// ("EXCL", source->GetExclusiveIntervalOnly())("MEM", source->IsSourceInMemory())("result", result->DebugString()); return result; } { @@ -47,28 +55,35 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::GetColumnsFetchingPlan(con } } -std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(const bool needSnapshots, const bool exclusiveSource, - const bool partialUsageByPredicateExt, const bool useIndexes) const { +std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(const bool needSnapshots, const bool exclusiveSource, const bool partialUsageByPredicateExt, const bool useIndexes, + const bool needFilterSharding) const { std::shared_ptr<TFetchingScript> result = std::make_shared<TFetchingScript>(); const bool partialUsageByPredicate = partialUsageByPredicateExt && PredicateColumns->GetColumnsCount(); if (!!IndexChecker && useIndexes && exclusiveSource) { result->AddStep(std::make_shared<TIndexBlobsFetchingStep>(std::make_shared<TIndexesSet>(IndexChecker->GetIndexIds()))); result->AddStep(std::make_shared<TApplyIndexStep>(IndexChecker)); } + bool hasFilterSharding = false; + if (needFilterSharding && !ShardingColumns->IsEmpty()) { + hasFilterSharding = true; + result->AddStep(std::make_shared<TColumnBlobsFetchingStep>(ShardingColumns)); + result->AddStep(std::make_shared<TAssemblerStep>(ShardingColumns, "SPEC_SHARDING")); + result->AddStep(std::make_shared<TShardingFilter>()); + } if (!EFColumns->GetColumnsCount() && !partialUsageByPredicate) { result->SetBranchName("simple"); - TColumnsSet columnsFetch = *FFColumns; + TColumnsSet columnsFetch = *FFColumns - *ShardingColumns; if (needSnapshots) { columnsFetch = columnsFetch + *SpecColumns; } if (!exclusiveSource) { columnsFetch = columnsFetch + *PKColumns + *SpecColumns; } else { - if (columnsFetch.GetColumnsCount() == 1 && SpecColumns->Contains(columnsFetch)) { + if (columnsFetch.GetColumnsCount() == 1 && SpecColumns->Contains(columnsFetch) && !hasFilterSharding) { return nullptr; } } - if (columnsFetch.GetColumnsCount()) { + if (columnsFetch.GetColumnsCount() || hasFilterSharding) { result->AddStep(std::make_shared<TColumnBlobsFetchingStep>(std::make_shared<TColumnsSet>(columnsFetch))); if (!exclusiveSource) { result->AddStep(std::make_shared<TAssemblerStep>(std::make_shared<TColumnsSet>(*PKColumns + *SpecColumns), "LAST")); @@ -84,7 +99,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(c } } else if (exclusiveSource) { result->SetBranchName("exclusive"); - TColumnsSet columnsFetch = *EFColumns; + TColumnsSet columnsFetch = *EFColumns - *ShardingColumns; if (needSnapshots || FFColumns->Cross(*SpecColumns)) { columnsFetch = columnsFetch + *SpecColumns; } @@ -120,7 +135,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(c result->AddStep(std::make_shared<TFilterProgramStep>(i)); } AFL_VERIFY(columnsFetch.IsEmpty()); - TColumnsSet columnsAdditionalFetch = *FFColumns - *EFColumns - *SpecColumns; + TColumnsSet columnsAdditionalFetch = *FFColumns - *EFColumns - *SpecColumns - *ShardingColumns; if (partialUsageByPredicate) { columnsAdditionalFetch = columnsAdditionalFetch - *PredicateColumns; } @@ -130,7 +145,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(c } } else { result->SetBranchName("merge"); - TColumnsSet columnsFetch = *MergeColumns + *EFColumns; + TColumnsSet columnsFetch = *MergeColumns + *EFColumns - *ShardingColumns; AFL_VERIFY(columnsFetch.GetColumnsCount()); result->AddStep(std::make_shared<TColumnBlobsFetchingStep>(std::make_shared<TColumnsSet>(columnsFetch))); result->AddStep(std::make_shared<TAssemblerStep>(SpecColumns, "SPEC")); @@ -158,7 +173,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(c result->AddStep(std::make_shared<TFilterProgramStep>(i)); } AFL_VERIFY(columnsFetchEF.IsEmpty()); - const TColumnsSet columnsAdditionalFetch = *FFColumns - *EFColumns - *SpecColumns - *PKColumns - *PredicateColumns; + const TColumnsSet columnsAdditionalFetch = *FFColumns - *EFColumns - *SpecColumns - *PKColumns - *PredicateColumns - *ShardingColumns; if (columnsAdditionalFetch.GetColumnsCount()) { result->AddStep(std::make_shared<TColumnBlobsFetchingStep>(std::make_shared<TColumnsSet>(columnsAdditionalFetch))); result->AddStep(std::make_shared<TOptionalAssemblerStep>(std::make_shared<TColumnsSet>(columnsAdditionalFetch), "LAST")); @@ -168,8 +183,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(c } TSpecialReadContext::TSpecialReadContext(const std::shared_ptr<TReadContext>& commonContext) - : CommonContext(commonContext) -{ + : CommonContext(commonContext) { ReadMetadata = dynamic_pointer_cast<const TReadMetadata>(CommonContext->GetReadMetadata()); Y_ABORT_UNLESS(ReadMetadata); Y_ABORT_UNLESS(ReadMetadata->SelectInfo); @@ -185,6 +199,12 @@ TSpecialReadContext::TSpecialReadContext(const std::shared_ptr<TReadContext>& co PredicateColumns = std::make_shared<TColumnsSet>(); } } + if (!!ReadMetadata->GetRequestShardingInfo()) { + auto shardingColumnIds = ReadMetadata->GetIndexInfo().GetColumnIdsVerified(ReadMetadata->GetRequestShardingInfo()->GetShardingInfo()->GetColumnNames()); + ShardingColumns = std::make_shared<TColumnsSet>(shardingColumnIds, ReadMetadata->GetIndexInfo(), ReadMetadata->GetResultSchema()); + } else { + ShardingColumns = std::make_shared<TColumnsSet>(); + } { auto efColumns = ReadMetadata->GetEarlyFilterColumnIds(); if (efColumns.size()) { @@ -214,24 +234,14 @@ TSpecialReadContext::TSpecialReadContext(const std::shared_ptr<TReadContext>& co PKColumns = std::make_shared<TColumnsSet>(ReadMetadata->GetPKColumnIds(), ReadMetadata->GetIndexInfo(), readSchema); MergeColumns = std::make_shared<TColumnsSet>(*PKColumns + *SpecColumns); - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("columns_context_info", DebugString()); - CacheFetchingScripts[0][0][0][0] = BuildColumnsFetchingPlan(false, false, false, false); - CacheFetchingScripts[0][1][0][0] = BuildColumnsFetchingPlan(false, true, false, false); - CacheFetchingScripts[1][0][0][0] = BuildColumnsFetchingPlan(true, false, false, false); - CacheFetchingScripts[1][1][0][0] = BuildColumnsFetchingPlan(true, true, false, false); - CacheFetchingScripts[0][0][1][0] = BuildColumnsFetchingPlan(false, false, true, false); - CacheFetchingScripts[0][1][1][0] = BuildColumnsFetchingPlan(false, true, true, false); - CacheFetchingScripts[1][0][1][0] = BuildColumnsFetchingPlan(true, false, true, false); - CacheFetchingScripts[1][1][1][0] = BuildColumnsFetchingPlan(true, true, true, false); + const auto GetBit = [](const ui32 val, const ui32 pos) -> ui32 { + return (val & (1 << pos)) ? 1 : 0; + }; - CacheFetchingScripts[0][0][0][1] = BuildColumnsFetchingPlan(false, false, false, true); - CacheFetchingScripts[0][1][0][1] = BuildColumnsFetchingPlan(false, true, false, true); - CacheFetchingScripts[1][0][0][1] = BuildColumnsFetchingPlan(true, false, false, true); - CacheFetchingScripts[1][1][0][1] = BuildColumnsFetchingPlan(true, true, false, true); - CacheFetchingScripts[0][0][1][1] = BuildColumnsFetchingPlan(false, false, true, true); - CacheFetchingScripts[0][1][1][1] = BuildColumnsFetchingPlan(false, true, true, true); - CacheFetchingScripts[1][0][1][1] = BuildColumnsFetchingPlan(true, false, true, true); - CacheFetchingScripts[1][1][1][1] = BuildColumnsFetchingPlan(true, true, true, true); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("columns_context_info", DebugString()); + for (ui32 i = 0; i < (1 << 6); ++i) { + CacheFetchingScripts[GetBit(i, 0)][GetBit(i, 1)][GetBit(i, 2)][GetBit(i, 3)][GetBit(i, 4)] = BuildColumnsFetchingPlan(GetBit(i, 0), GetBit(i, 1), GetBit(i, 2), GetBit(i, 3), GetBit(i, 4)); + } } -} +} // namespace NKikimr::NOlap::NReader::NPlain diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h index dc46c4b70b..2a08d3c062 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h @@ -16,6 +16,7 @@ private: YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, SpecColumns); YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, MergeColumns); + YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, ShardingColumns); YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, EFColumns); YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, PredicateColumns); YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, PKColumns); @@ -25,8 +26,9 @@ private: NIndexes::TIndexCheckerContainer IndexChecker; TReadMetadata::TConstPtr ReadMetadata; std::shared_ptr<TColumnsSet> EmptyColumns = std::make_shared<TColumnsSet>(); - std::shared_ptr<TFetchingScript> BuildColumnsFetchingPlan(const bool needSnapshotsFilter, const bool exclusiveSource, const bool partialUsageByPredicate, const bool useIndexes) const; - std::array<std::array<std::array<std::array<std::shared_ptr<TFetchingScript>, 2>, 2>, 2>, 2> CacheFetchingScripts; + std::shared_ptr<TFetchingScript> BuildColumnsFetchingPlan(const bool needSnapshotsFilter, const bool exclusiveSource, const bool partialUsageByPredicate, const bool useIndexes, const bool needFilterSharding) const; + std::array<std::array<std::array<std::array<std::array<std::shared_ptr<TFetchingScript>, 2>, 2>, 2>, 2>, 2> CacheFetchingScripts; + public: static const inline ui64 DefaultRejectMemoryIntervalLimit = ((ui64)3) << 30; static const inline ui64 DefaultReduceMemoryIntervalLimit = DefaultRejectMemoryIntervalLimit; @@ -45,12 +47,11 @@ public: std::unique_ptr<NArrow::NMerger::TMergePartialStream> BuildMerger() const; TString DebugString() const { - return TStringBuilder() << - "ef=" << EFColumns->DebugString() << ";" << - "pk=" << PKColumns->DebugString() << ";" << - "ff=" << FFColumns->DebugString() << ";" << - "program_input=" << ProgramInputColumns->DebugString() - ; + return TStringBuilder() << "ef=" << EFColumns->DebugString() << ";" + << "sharding=" << ShardingColumns->DebugString() << ";" + << "pk=" << PKColumns->DebugString() << ";" + << "ff=" << FFColumns->DebugString() << ";" + << "program_input=" << ProgramInputColumns->DebugString(); } TSpecialReadContext(const std::shared_ptr<TReadContext>& commonContext); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp index daef434331..c557f2064b 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp @@ -101,6 +101,13 @@ TConclusion<bool> TSnapshotFilter::DoExecuteInplace(const std::shared_ptr<IDataS return true; } +TConclusion<bool> TShardingFilter::DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& /*step*/) const { + NYDBTest::TControllers::GetColumnShardController()->OnSelectShardingFilter(); + auto filter = source->GetContext()->GetReadMetadata()->GetRequestShardingInfo()->GetShardingInfo()->GetFilter(source->GetStageData().GetTable()->BuildTable()); + source->MutableStageData().AddFilter(filter); + return true; +} + TConclusion<bool> TBuildFakeSpec::DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& /*step*/) const { std::vector<std::shared_ptr<arrow::Array>> columns; for (auto&& f : TIndexInfo::ArrowSchemaSnapshot()->fields()) { diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.h index 1d0a1ceee8..ac8220f50c 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.h @@ -280,15 +280,27 @@ public: } }; -class TSnapshotFilter: public IFetchingStep { +class TSnapshotFilter : public IFetchingStep { private: using TBase = IFetchingStep; + public: virtual TConclusion<bool> DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& step) const override; TSnapshotFilter() : TBase("SNAPSHOT") { + } +}; +class TShardingFilter : public IFetchingStep { +private: + using TBase = IFetchingStep; + +public: + virtual TConclusion<bool> DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& step) const override; + TShardingFilter() + : TBase("SHARDING") { } }; + } diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp index 723ba175f5..2436f2ebd7 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp @@ -1,13 +1,15 @@ #include "source.h" -#include "interval.h" + +#include "constructor.h" #include "fetched_data.h" +#include "interval.h" #include "plain_read_data.h" -#include "constructor.h" + +#include <ydb/core/formats/arrow/simple_arrays_cache.h> #include <ydb/core/tx/columnshard/blobs_reader/actor.h> #include <ydb/core/tx/columnshard/blobs_reader/events.h> -#include <ydb/core/tx/conveyor/usage/service.h> -#include <ydb/core/formats/arrow/simple_arrays_cache.h> #include <ydb/core/tx/columnshard/hooks/abstract/abstract.h> +#include <ydb/core/tx/conveyor/usage/service.h> namespace NKikimr::NOlap::NReader::NPlain { @@ -50,9 +52,8 @@ void IDataSource::OnInitResourcesGuard(const std::shared_ptr<IDataSource>& sourc } } -void TPortionDataSource::NeedFetchColumns(const std::set<ui32>& columnIds, - TBlobsAction& blobsAction, THashMap<TChunkAddress, ui32>& nullBlocks, - const std::shared_ptr<NArrow::TColumnFilter>& filter) { +void TPortionDataSource::NeedFetchColumns(const std::set<ui32>& columnIds, TBlobsAction& blobsAction, + THashMap<TChunkAddress, ui32>& nullBlocks, const std::shared_ptr<NArrow::TColumnFilter>& filter) { const NArrow::TColumnFilter& cFilter = filter ? *filter : NArrow::TColumnFilter::BuildAllowFilter(); ui32 fetchedChunks = 0; ui32 nullChunks = 0; @@ -78,7 +79,8 @@ void TPortionDataSource::NeedFetchColumns(const std::set<ui32>& columnIds, } AFL_VERIFY(itFinished)("filter", itFilter.DebugString())("count", Portion->NumRows(i)); } - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "chunks_stats")("fetch", fetchedChunks)("null", nullChunks)("reading_actions", blobsAction.GetStorageIds())("columns", columnIds.size()); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "chunks_stats")("fetch", fetchedChunks)("null", nullChunks) + ("reading_actions", blobsAction.GetStorageIds())("columns", columnIds.size()); } bool TPortionDataSource::DoStartFetchingColumns(const std::shared_ptr<IDataSource>& sourcePtr, const TFetchingScriptCursor& step, const std::shared_ptr<TColumnsSet>& columns) { @@ -143,7 +145,7 @@ void TPortionDataSource::DoAbort() { void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& indexChecker) { THashMap<ui32, std::vector<TString>> indexBlobs; std::set<ui32> indexIds = indexChecker->GetIndexIds(); -// NActors::TLogContextGuard gLog = NActors::TLogContextBuilder::Build()("records_count", GetRecordsCount())("portion_id", Portion->GetAddress().DebugString()); + // NActors::TLogContextGuard gLog = NActors::TLogContextBuilder::Build()("records_count", GetRecordsCount())("portion_id", Portion->GetAddress().DebugString()); std::vector<TPortionInfo::TPage> pages = Portion->BuildPages(); NArrow::TColumnFilter constructor = NArrow::TColumnFilter::BuildAllowFilter(); for (auto&& p : pages) { @@ -238,4 +240,4 @@ void TCommittedDataSource::DoAssembleColumns(const std::shared_ptr<TColumnsSet>& MutableStageData().SyncTableColumns(columns->GetSchema()->fields()); } -} +} // namespace NKikimr::NOlap::NReader::NPlain diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h index 8e95c20d79..9230c9940c 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h @@ -33,8 +33,10 @@ private: NArrow::TReplaceKey StartReplaceKey; NArrow::TReplaceKey FinishReplaceKey; YDB_READONLY_DEF(std::shared_ptr<TSpecialReadContext>, Context); + YDB_READONLY(TSnapshot, RecordSnapshotMin, TSnapshot::Zero()); YDB_READONLY(TSnapshot, RecordSnapshotMax, TSnapshot::Zero()); YDB_READONLY(ui32, RecordsCount, 0); + YDB_READONLY_DEF(std::optional<ui64>, ShardingVersionOptional); YDB_READONLY(ui32, IntervalsCount, 0); virtual NJson::TJsonValue DoDebugJson() const = 0; bool MergingStartedFlag = false; @@ -182,16 +184,18 @@ public: void RegisterInterval(TFetchingInterval& interval); IDataSource(const ui32 sourceIdx, const std::shared_ptr<TSpecialReadContext>& context, - const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& finish, - const TSnapshot& recordSnapshotMax, const ui32 recordsCount) + const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& finish, const TSnapshot& recordSnapshotMin, const TSnapshot& recordSnapshotMax, + const ui32 recordsCount, const std::optional<ui64> shardingVersion) : SourceIdx(sourceIdx) , Start(context->GetReadMetadata()->BuildSortedPosition(start)) , Finish(context->GetReadMetadata()->BuildSortedPosition(finish)) , StartReplaceKey(start) , FinishReplaceKey(finish) , Context(context) + , RecordSnapshotMin(recordSnapshotMin) , RecordSnapshotMax(recordSnapshotMax) , RecordsCount(recordsCount) + , ShardingVersionOptional(shardingVersion) { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "portions_for_merge")("start", Start.DebugJson())("finish", Finish.DebugJson()); if (Start.IsReverseSort()) { @@ -305,7 +309,7 @@ public: TPortionDataSource(const ui32 sourceIdx, const std::shared_ptr<TPortionInfo>& portion, const std::shared_ptr<TSpecialReadContext>& context, const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& finish) - : TBase(sourceIdx, context, start, finish, portion->RecordSnapshotMax(), portion->GetRecordsCount()) + : TBase(sourceIdx, context, start, finish, portion->RecordSnapshotMin(), portion->RecordSnapshotMax(), portion->GetRecordsCount(), portion->GetShardingVersionOptional()) , Portion(portion) , Schema(GetContext()->GetReadMetadata()->GetLoadSchemaVerified(*Portion)) { @@ -377,7 +381,7 @@ public: TCommittedDataSource(const ui32 sourceIdx, const TCommittedBlob& committed, const std::shared_ptr<TSpecialReadContext>& context, const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& finish) - : TBase(sourceIdx, context, start, finish, committed.GetSnapshot(), committed.GetRecordsCount()) + : TBase(sourceIdx, context, start, finish, committed.GetSnapshot(), committed.GetSnapshot(), committed.GetRecordsCount(), {}) , CommittedBlob(committed) { } diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp index 8db0b5e2d0..59dccac5ee 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp @@ -73,7 +73,7 @@ bool TIndexInfo::IsSpecialColumn(const ui32 fieldId) { || fieldId == (ui32)ESpecialColumn::TX_ID; } -ui32 TIndexInfo::GetColumnId(const std::string& name) const { +ui32 TIndexInfo::GetColumnIdVerified(const std::string& name) const { auto id = GetColumnIdOptional(name); Y_ABORT_UNLESS(!!id, "undefined column %s", name.data()); return *id; diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.h b/ydb/core/tx/columnshard/engines/scheme/index_info.h index bd1dfec353..b0e274a3a2 100644 --- a/ydb/core/tx/columnshard/engines/scheme/index_info.h +++ b/ydb/core/tx/columnshard/engines/scheme/index_info.h @@ -233,7 +233,17 @@ public: } /// Returns an id of the column located by name. The name should exists in the schema. - ui32 GetColumnId(const std::string& name) const; + ui32 GetColumnIdVerified(const std::string& name) const; + ui32 GetColumnId(const std::string& name) const { + return GetColumnIdVerified(name); + } + std::set<ui32> GetColumnIdsVerified(const std::set<TString>& names) const { + std::set<ui32> result; + for (auto&& i : names) { + AFL_VERIFY(result.emplace(GetColumnIdVerified(i)).second); + } + return result; + } std::optional<ui32> GetColumnIdOptional(const std::string& name) const; /// Returns a name of the column located by id. diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp index e7cf6f30ed..115cdc16dd 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp @@ -2,6 +2,7 @@ #include "snapshot_scheme.h" #include <ydb/core/tx/columnshard/engines/scheme/index_info.h> +#include <ydb/core/tx/columnshard/engines/db_wrapper.h> namespace NKikimr::NOlap { @@ -28,4 +29,22 @@ void TVersionedIndex::AddIndex(const TSnapshot& snapshot, TIndexInfo&& indexInfo LastSchemaVersion = std::max(newVersion, LastSchemaVersion); } +bool TVersionedIndex::LoadShardingInfo(IDbWrapper& db) { + TConclusion<THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>>> shardingLocal = db.LoadGranulesShardingInfo(); + if (shardingLocal.IsFail()) { + return false; + } + ShardingInfo = std::move(shardingLocal.DetachResult()); + return true; +} + +std::optional<NKikimr::NOlap::TGranuleShardingInfo> TVersionedIndex::GetShardingInfoActual(const ui64 pathId) const { + auto it = ShardingInfo.find(pathId); + if (it == ShardingInfo.end() || it->second.empty()) { + return std::nullopt; + } else { + return it->second.rbegin()->second; + } +} + } diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h index d83d338350..ae2dca6eff 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h @@ -1,15 +1,37 @@ #pragma once #include "abstract_scheme.h" +#include <ydb/core/tx/sharding/sharding.h> namespace NKikimr::NOlap { +class IDbWrapper; + +class TGranuleShardingInfo { +private: + YDB_READONLY_DEF(NSharding::TGranuleShardingLogicContainer, ShardingInfo); + YDB_READONLY(TSnapshot, SinceSnapshot, TSnapshot::Zero()); + YDB_READONLY(ui64, SnapshotVersion, 0); + YDB_READONLY(ui64, PathId, 0); + +public: + TGranuleShardingInfo(const NSharding::TGranuleShardingLogicContainer& shardingInfo, const TSnapshot& sinceSnapshot, const ui64 version, const ui64 pathId) + : ShardingInfo(shardingInfo) + , SinceSnapshot(sinceSnapshot) + , SnapshotVersion(version) + , PathId(pathId) { + AFL_VERIFY(!!ShardingInfo); + } +}; + class TVersionedIndex { + THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>> ShardingInfo; std::map<TSnapshot, ISnapshotSchema::TPtr> Snapshots; std::shared_ptr<arrow::Schema> PrimaryKey; std::map<ui64, ISnapshotSchema::TPtr> SnapshotByVersion; ui64 LastSchemaVersion = 0; std::optional<ui64> SchemeVersionForActualization; ISnapshotSchema::TPtr SchemeForActualization; + public: ISnapshotSchema::TPtr GetLastCriticalSchema() const { return SchemeForActualization; @@ -20,6 +42,29 @@ public: return result ? result : defaultSchema; } + std::optional<TGranuleShardingInfo> GetShardingInfoOptional(const ui64 pathId, const TSnapshot& ss) const { + auto it = ShardingInfo.find(pathId); + if (it == ShardingInfo.end() || it->second.empty()) { + return std::nullopt; + } else { + auto itSS = it->second.upper_bound(ss); + if (itSS == it->second.end()) { + return it->second.rbegin()->second; + } else if (itSS == it->second.begin()) { + return std::nullopt; + } else { + --itSS; + return itSS->second; + } + } + } + + std::optional<TGranuleShardingInfo> GetShardingInfoActual(const ui64 pathId) const; + + void AddShardingInfo(const TGranuleShardingInfo& shardingInfo) { + AFL_VERIFY(ShardingInfo[shardingInfo.GetPathId()].emplace(shardingInfo.GetSinceSnapshot(), shardingInfo).second); + } + TString DebugString() const { TStringBuilder sb; for (auto&& i : Snapshots) { @@ -47,7 +92,7 @@ public: } Y_ABORT_UNLESS(!Snapshots.empty()); Y_ABORT_UNLESS(version.IsZero()); - return Snapshots.begin()->second; // For old compaction logic compatibility + return Snapshots.begin()->second; } ISnapshotSchema::TPtr GetLastSchema() const { @@ -64,5 +109,7 @@ public: } void AddIndex(const TSnapshot& snapshot, TIndexInfo&& indexInfo); + + bool LoadShardingInfo(IDbWrapper& db); }; -} +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp b/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp index 34a570d2de..946b657400 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp @@ -1,8 +1,8 @@ #include "helper.h" -#include <ydb/core/tx/columnshard/engines/db_wrapper.h> -#include <ydb/core/tx/columnshard/engines/insert_table/insert_table.h> #include <ydb/core/tx/columnshard/columnshard_schema.h> +#include <ydb/core/tx/columnshard/engines/db_wrapper.h> +#include <ydb/core/tx/columnshard/engines/insert_table/insert_table.h> #include <library/cpp/testing/unittest/registar.h> #include <util/string/printf.h> @@ -14,43 +14,62 @@ using namespace NKikimr::NOlap::NEngines::NTest; namespace { -class TTestInsertTableDB: public IDbWrapper { +class TTestInsertTableDB : public IDbWrapper { public: - void Insert(const TInsertedData&) override {} - void Commit(const TInsertedData&) override {} - void Abort(const TInsertedData&) override {} - void EraseInserted(const TInsertedData&) override {} - void EraseCommitted(const TInsertedData&) override {} - void EraseAborted(const TInsertedData&) override {} - - bool Load(TInsertTableAccessor&, - const TInstant&) override { + void Insert(const TInsertedData&) override { + } + void Commit(const TInsertedData&) override { + } + void Abort(const TInsertedData&) override { + } + void EraseInserted(const TInsertedData&) override { + } + void EraseCommitted(const TInsertedData&) override { + } + void EraseAborted(const TInsertedData&) override { + } + + virtual TConclusion<THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>>> LoadGranulesShardingInfo() override { + THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>> result; + return result; + } + + bool Load(TInsertTableAccessor&, const TInstant&) override { return true; } virtual void WritePortion(const NOlap::TPortionInfo& /*portion*/) override { - } virtual void ErasePortion(const NOlap::TPortionInfo& /*portion*/) override { - } virtual bool LoadPortions(const std::function<void(NOlap::TPortionInfoConstructor&&, const NKikimrTxColumnShard::TIndexPortionMeta&)>& /*callback*/) override { return true; } - void WriteColumn(const TPortionInfo&, const TColumnRecord&, const ui32 /*firstPKColumnId*/) override {} - void EraseColumn(const TPortionInfo&, const TColumnRecord&) override {} - bool LoadColumns(const std::function<void(NOlap::TPortionInfoConstructor&&, const TColumnChunkLoadContext&)>&) override { return true; } + void WriteColumn(const TPortionInfo&, const TColumnRecord&, const ui32 /*firstPKColumnId*/) override { + } + void EraseColumn(const TPortionInfo&, const TColumnRecord&) override { + } + bool LoadColumns(const std::function<void(NOlap::TPortionInfoConstructor&&, const TColumnChunkLoadContext&)>&) override { + return true; + } - virtual void WriteIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override {} - virtual void EraseIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override {} - virtual bool LoadIndexes(const std::function<void(const ui64 /*pathId*/, const ui64 /*portionId*/, const TIndexChunkLoadContext&)>& /*callback*/) override { return true; } + virtual void WriteIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override { + } + virtual void EraseIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override { + } + virtual bool LoadIndexes(const std::function<void(const ui64 /*pathId*/, const ui64 /*portionId*/, const TIndexChunkLoadContext&)>& /*callback*/) override { + return true; + } - void WriteCounter(ui32, ui64) override {} - bool LoadCounters(const std::function<void(ui32 id, ui64 value)>&) override { return true; } + void WriteCounter(ui32, ui64) override { + } + bool LoadCounters(const std::function<void(ui32 id, ui64 value)>&) override { + return true; + } }; -} +} // namespace Y_UNIT_TEST_SUITE(TColumnEngineTestInsertTable) { Y_UNIT_TEST(TestInsertCommit) { @@ -79,13 +98,15 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestInsertTable) { // read nothing auto blobs = insertTable.Read(tableId, TSnapshot::Zero(), nullptr); UNIT_ASSERT_EQUAL(blobs.size(), 0); - blobs = insertTable.Read(tableId+1, TSnapshot::Zero(), nullptr); + blobs = insertTable.Read(tableId + 1, TSnapshot::Zero(), nullptr); UNIT_ASSERT_EQUAL(blobs.size(), 0); // commit ui64 planStep = 100; ui64 txId = 42; - insertTable.Commit(dbTable, planStep, txId, {TWriteId{writeId}}, [](ui64){ return true; }); + insertTable.Commit(dbTable, planStep, txId, {TWriteId{writeId}}, [](ui64) { + return true; + }); UNIT_ASSERT_EQUAL(insertTable.GetPathPriorities().size(), 1); UNIT_ASSERT_EQUAL(insertTable.GetPathPriorities().begin()->second.size(), 1); @@ -94,15 +115,15 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestInsertTable) { // read old snapshot blobs = insertTable.Read(tableId, TSnapshot::Zero(), nullptr); UNIT_ASSERT_EQUAL(blobs.size(), 0); - blobs = insertTable.Read(tableId+1, TSnapshot::Zero(), nullptr); + blobs = insertTable.Read(tableId + 1, TSnapshot::Zero(), nullptr); UNIT_ASSERT_EQUAL(blobs.size(), 0); // read new snapshot blobs = insertTable.Read(tableId, TSnapshot(planStep, txId), nullptr); UNIT_ASSERT_EQUAL(blobs.size(), 1); - blobs = insertTable.Read(tableId+1, TSnapshot::Zero(), nullptr); + blobs = insertTable.Read(tableId + 1, TSnapshot::Zero(), nullptr); UNIT_ASSERT_EQUAL(blobs.size(), 0); } } -} +} // namespace NKikimr diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp index fc4607fb62..9b5bec6ff3 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp @@ -41,6 +41,11 @@ public: THashMap<ui32, ui64> Counters; }; + virtual TConclusion<THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>>> LoadGranulesShardingInfo() override { + THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>> result; + return result; + } + void Insert(const TInsertedData& data) override { Inserted.emplace(TWriteId{data.WriteTxId}, data); } diff --git a/ydb/core/tx/columnshard/export/session/session.h b/ydb/core/tx/columnshard/export/session/session.h index ce98f56a8d..7d73595a0e 100644 --- a/ydb/core/tx/columnshard/export/session/session.h +++ b/ydb/core/tx/columnshard/export/session/session.h @@ -86,7 +86,7 @@ public: virtual bool IsFinished() const override { return Status == EStatus::Finished; } - virtual bool IsReadyForRemove() const override { + virtual bool IsReadyForRemoveOnFinished() const override { return Status == EStatus::Aborted; } diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h index bf90233492..7ffa19144a 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h @@ -88,6 +88,10 @@ public: using TPtr = std::shared_ptr<ICSController>; virtual ~ICSController() = default; + virtual void OnSelectShardingFilter() { + + } + virtual TDuration GetCompactionActualizationLag(const TDuration def) const { return def; } diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.cpp b/ydb/core/tx/columnshard/hooks/testing/controller.cpp index e711a72d4c..4943addc75 100644 --- a/ydb/core/tx/columnshard/hooks/testing/controller.cpp +++ b/ydb/core/tx/columnshard/hooks/testing/controller.cpp @@ -112,9 +112,11 @@ bool TController::IsTrivialLinks() const { TGuard<TMutex> g(Mutex); for (auto&& i : ShardActuals) { if (!i.second->GetStoragesManager()->GetSharedBlobsManager()->IsTrivialLinks()) { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("reason", "non_trivial"); return false; } if (i.second->GetStoragesManager()->HasBlobsToDelete()) { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("reason", "has_delete"); return false; } } diff --git a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h index 8d52e8400f..191e7b7b34 100644 --- a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h @@ -27,10 +27,15 @@ private: YDB_READONLY(TAtomicCounter, ActualizationsCount, 0); YDB_READONLY(TAtomicCounter, ActualizationRefreshSchemeCount, 0); YDB_READONLY(TAtomicCounter, ActualizationRefreshTieringCount, 0); + YDB_READONLY(TAtomicCounter, ShardingFiltersCount, 0); YDB_ACCESSOR(TAtomicCounter, CompactionsLimit, 10000000); protected: + virtual void OnSelectShardingFilter() override { + ShardingFiltersCount.Inc(); + } + virtual void AddPortionForActualizer(const i32 portionsCount) override { NeedActualizationCount.Add(portionsCount); } diff --git a/ydb/core/tx/columnshard/operations/write.cpp b/ydb/core/tx/columnshard/operations/write.cpp index 27daff6a31..119a29f02c 100644 --- a/ydb/core/tx/columnshard/operations/write.cpp +++ b/ydb/core/tx/columnshard/operations/write.cpp @@ -1,31 +1,29 @@ -#include "write.h" #include "slice_builder.h" +#include "write.h" -#include <ydb/core/tx/columnshard/columnshard_schema.h> +#include <ydb/core/tablet_flat/tablet_flat_executor.h> +#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h> #include <ydb/core/tx/columnshard/blobs_action/blob_manager_db.h> #include <ydb/core/tx/columnshard/columnshard_impl.h> -#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h> +#include <ydb/core/tx/columnshard/columnshard_schema.h> #include <ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h> #include <ydb/core/tx/conveyor/usage/service.h> -#include <ydb/core/tablet_flat/tablet_flat_executor.h> - - namespace NKikimr::NColumnShard { - TWriteOperation::TWriteOperation(const TWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, const TInstant createdAt) + TWriteOperation::TWriteOperation(const TWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, const TInstant createdAt, const std::optional<ui32> granuleShardingVersionId) : Status(status) , CreatedAt(createdAt) , WriteId(writeId) , LockId(lockId) , Cookie(cookie) - { + , GranuleShardingVersionId(granuleShardingVersionId) { } void TWriteOperation::Start(TColumnShard& owner, const ui64 tableId, const NEvWrite::IDataContainer::TPtr& data, const NActors::TActorId& source, const TActorContext& ctx) { Y_ABORT_UNLESS(Status == EOperationStatus::Draft); - NEvWrite::TWriteMeta writeMeta((ui64)WriteId, tableId, source); + NEvWrite::TWriteMeta writeMeta((ui64)WriteId, tableId, source, GranuleShardingVersionId); std::shared_ptr<NConveyor::ITask> task = std::make_shared<NOlap::TBuildSlicesTask>(owner.TabletID(), ctx.SelfID, owner.BufferizationWriteActorId, NEvWrite::TWriteData(writeMeta, data, owner.TablesManager.GetPrimaryIndex()->GetReplaceKey(), owner.StoragesManager->GetInsertOperator()->StartWritingAction(NOlap::NBlobOperations::EConsumer::WRITING_OPERATOR))); @@ -46,7 +44,7 @@ namespace NKikimr::NColumnShard { }; auto counters = owner.InsertTable->Commit(dbTable, snapshot.GetPlanStep(), snapshot.GetTxId(), { gWriteId }, - pathExists); + pathExists); owner.IncCounter(COUNTER_BLOBS_COMMITTED, counters.Rows); owner.IncCounter(COUNTER_BYTES_COMMITTED, counters.Bytes); @@ -72,11 +70,11 @@ namespace NKikimr::NColumnShard { NIceDb::TUpdate<Schema::Operations::CreatedAt>(CreatedAt.Seconds()), NIceDb::TUpdate<Schema::Operations::Metadata>(metadata), NIceDb::TUpdate<Schema::Operations::LockId>(LockId), - NIceDb::TUpdate<Schema::Operations::Cookie>(Cookie) - ); + NIceDb::TUpdate<Schema::Operations::Cookie>(Cookie), + NIceDb::TUpdate<Schema::Operations::GranuleShardingVersionId>(GranuleShardingVersionId.value_or(0))); } - void TWriteOperation::ToProto(NKikimrTxColumnShard::TInternalOperationData& proto) const { + void TWriteOperation::ToProto(NKikimrTxColumnShard::TInternalOperationData& proto) const { for (auto&& writeId : GlobalWriteIds) { proto.AddInternalWriteIds((ui64)writeId); } @@ -108,17 +106,21 @@ namespace NKikimr::NColumnShard { } while (!rowset.EndOfSet()) { - const TWriteId writeId = (TWriteId) rowset.GetValue<Schema::Operations::WriteId>(); + const TWriteId writeId = (TWriteId)rowset.GetValue<Schema::Operations::WriteId>(); const ui64 createdAtSec = rowset.GetValue<Schema::Operations::CreatedAt>(); const ui64 lockId = rowset.GetValue<Schema::Operations::LockId>(); const ui64 cookie = rowset.GetValueOrDefault<Schema::Operations::Cookie>(0); const TString metadata = rowset.GetValue<Schema::Operations::Metadata>(); - const EOperationStatus status = (EOperationStatus) rowset.GetValue<Schema::Operations::Status>(); + const EOperationStatus status = (EOperationStatus)rowset.GetValue<Schema::Operations::Status>(); + std::optional<ui32> granuleShardingVersionId; + if (rowset.HaveValue<Schema::Operations::GranuleShardingVersionId>() && rowset.GetValue<Schema::Operations::GranuleShardingVersionId>()) { + granuleShardingVersionId = rowset.GetValue<Schema::Operations::GranuleShardingVersionId>(); + } NKikimrTxColumnShard::TInternalOperationData metaProto; Y_ABORT_UNLESS(metaProto.ParseFromString(metadata)); - auto operation = std::make_shared<TWriteOperation>(writeId, lockId, cookie, status, TInstant::Seconds(createdAtSec)); + auto operation = std::make_shared<TWriteOperation>(writeId, lockId, cookie, status, TInstant::Seconds(createdAtSec), granuleShardingVersionId); operation->FromProto(metaProto); AFL_VERIFY(operation->GetStatus() != EOperationStatus::Draft); @@ -145,7 +147,7 @@ namespace NKikimr::NColumnShard { const ui64 txId = rowset.GetValue<Schema::OperationTxIds::TxId>(); AFL_VERIFY(Locks.contains(lockId))("lock_id", lockId); Tx2Lock[txId] = lockId; - if (!rowset.Next()) { + if (!rowset.Next()) { return false; } } @@ -208,7 +210,7 @@ namespace NKikimr::NColumnShard { AFL_VERIFY(!!lockId)("tx_id", txId); Locks.erase(*lockId); Tx2Lock.erase(txId); - for (auto&& op: operations) { + for (auto&& op : operations) { RemoveOperation(op, txc); } NIceDb::TNiceDb db(txc.DB); @@ -239,9 +241,9 @@ namespace NKikimr::NColumnShard { db.Table<Schema::OperationTxIds>().Key(txId, lockId).Update(); } - TWriteOperation::TPtr TOperationsManager::RegisterOperation(const ui64 lockId, const ui64 cookie) { + TWriteOperation::TPtr TOperationsManager::RegisterOperation(const ui64 lockId, const ui64 cookie, const std::optional<ui32> granuleShardingVersionId) { auto writeId = BuildNextWriteId(); - auto operation = std::make_shared<TWriteOperation>(writeId, lockId, cookie, EOperationStatus::Draft, AppData()->TimeProvider->Now()); + auto operation = std::make_shared<TWriteOperation>(writeId, lockId, cookie, EOperationStatus::Draft, AppData()->TimeProvider->Now(), granuleShardingVersionId); Y_ABORT_UNLESS(Operations.emplace(operation->GetWriteId(), operation).second); Locks[operation->GetLockId()].push_back(operation->GetWriteId()); return operation; @@ -265,4 +267,4 @@ namespace NKikimr::NColumnShard { } return EOperationBehaviour::Undefined; } -} +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/operations/write.h b/ydb/core/tx/columnshard/operations/write.h index 08fe7724b3..f9719dd79b 100644 --- a/ydb/core/tx/columnshard/operations/write.h +++ b/ydb/core/tx/columnshard/operations/write.h @@ -44,11 +44,11 @@ namespace NKikimr::NColumnShard { YDB_READONLY(ui64, Cookie, 0); YDB_READONLY_DEF(TVector<TWriteId>, GlobalWriteIds); YDB_ACCESSOR(EOperationBehaviour, Behaviour, EOperationBehaviour::Undefined); - + YDB_READONLY_DEF(std::optional<ui32>, GranuleShardingVersionId); public: using TPtr = std::shared_ptr<TWriteOperation>; - TWriteOperation(const TWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, const TInstant createdAt); + TWriteOperation(const TWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, const TInstant createdAt, const std::optional<ui32> granuleShardingVersionId); void Start(TColumnShard& owner, const ui64 tableId, const NEvWrite::IDataContainer::TPtr& data, const NActors::TActorId& source, const TActorContext& ctx); void OnWriteFinish(NTabletFlatExecutor::TTransactionContext& txc, const TVector<TWriteId>& globalWriteIds); @@ -78,7 +78,7 @@ namespace NKikimr::NColumnShard { void LinkTransaction(const ui64 lockId, const ui64 txId, NTabletFlatExecutor::TTransactionContext& txc); std::optional<ui64> GetLockForTx(const ui64 lockId) const; - TWriteOperation::TPtr RegisterOperation(const ui64 lockId, const ui64 cookie); + TWriteOperation::TPtr RegisterOperation(const ui64 lockId, const ui64 cookie, const std::optional<ui32> granuleShardingVersionId); static EOperationBehaviour GetBehaviour(const NEvents::TDataEvents::TEvWrite& evWrite); private: diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index e7d3ca6ba0..d1e89d0da2 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -1,14 +1,15 @@ #include "tables_manager.h" + #include "columnshard_schema.h" #include "engines/column_engine_logs.h" -#include <ydb/core/tx/columnshard/blobs_action/blob_manager_db.h> +#include "transactions/transactions/tx_add_sharding_info.h" + #include <ydb/core/scheme/scheme_types_proto.h> -#include <ydb/core/tx/tiering/manager.h> #include <ydb/core/tablet_flat/tablet_flat_executor.h> +#include <ydb/core/tx/columnshard/blobs_action/blob_manager_db.h> +#include <ydb/core/tx/tiering/manager.h> #include <library/cpp/protobuf/json/proto2json.h> -#include <ydb/core/tablet_flat/tablet_flat_executor.h> - namespace NKikimr::NColumnShard { @@ -293,6 +294,10 @@ void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapsho } } +std::unique_ptr<NTabletFlatExecutor::ITransaction> TTablesManager::CreateAddShardingInfoTx(TColumnShard& owner, const ui64 pathId, const ui64 versionId, const NSharding::TGranuleShardingLogicContainer& tabletShardingLogic) const { + return std::make_unique<TTxAddShardingInfo>(owner, tabletShardingLogic, pathId, versionId); +} + void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, NIceDb::TNiceDb& db, std::shared_ptr<TTiersManager>& manager) { auto it = Tables.find(pathId); AFL_VERIFY(it != Tables.end()); @@ -329,8 +334,7 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& TTablesManager::TTablesManager(const std::shared_ptr<NOlap::IStoragesManager>& storagesManager, const ui64 tabletId) : StoragesManager(storagesManager) - , TabletId(tabletId) -{ + , TabletId(tabletId) { } bool TTablesManager::TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, const ui64 pathId) const { @@ -358,4 +362,4 @@ bool TTablesManager::TryFinalizeDropPathOnComplete(const ui64 pathId) { return true; } -} +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 60e1a5f66d..72606173c5 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -242,6 +242,8 @@ public: void AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db); void AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, NIceDb::TNiceDb& db, std::shared_ptr<TTiersManager>& manager); bool FillMonitoringReport(NTabletFlatExecutor::TTransactionContext& txc, NJson::TJsonValue& json); + + [[nodiscard]] std::unique_ptr<NTabletFlatExecutor::ITransaction> CreateAddShardingInfoTx(TColumnShard& owner, const ui64 pathId, const ui64 versionId, const NSharding::TGranuleShardingLogicContainer& tabletShardingLogic) const; }; } diff --git a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.cpp b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.cpp index 784ebc3eda..4d934c3f00 100644 --- a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.cpp +++ b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.cpp @@ -1,4 +1,5 @@ #include "long_tx_write.h" +#include <ydb/core/tx/columnshard/engines/column_engine_logs.h> namespace NKikimr::NColumnShard { @@ -18,6 +19,15 @@ TLongTxTransactionOperator::TProposeResult TLongTxTransactionOperator::DoStartPr return TProposeResult(NKikimrTxColumnShard::EResultStatus::ERROR, TStringBuilder() << "Commit TxId# " << GetTxId() << " references WriteId# " << (ui64)writeId << " that is already locked by TxId# " << lw.PreparedTxId); } + + auto it = owner.InsertTable->GetInserted().find(writeId); + AFL_VERIFY(it != owner.InsertTable->GetInserted().end()); + + auto granuleShardingInfo = owner.GetIndexAs<NOlap::TColumnEngineForLogs>().GetVersionedIndex().GetShardingInfoActual(it->second.PathId); + if (granuleShardingInfo && lw.GranuleShardingVersionId && *lw.GranuleShardingVersionId != granuleShardingInfo->GetSnapshotVersion()) { + return TProposeResult(NKikimrTxColumnShard::EResultStatus::ERROR, + TStringBuilder() << "Commit TxId# " << GetTxId() << " references WriteId# " << (ui64)writeId << " declined through sharding deprecated"); + } } for (auto&& writeId : WriteIds) { diff --git a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h index ba952c4551..f164c06baf 100644 --- a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h +++ b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h @@ -1,14 +1,16 @@ #pragma once #include "propose_tx.h" + #include <ydb/core/tx/columnshard/columnshard_impl.h> namespace NKikimr::NColumnShard { - class TLongTxTransactionOperator : public IProposeTxOperator { + class TLongTxTransactionOperator: public IProposeTxOperator { using TBase = IProposeTxOperator; using TProposeResult = TTxController::TProposeResult; static inline auto Registrator = TFactory::TRegistrator<TLongTxTransactionOperator>(NKikimrTxColumnShard::TX_KIND_COMMIT); + private: virtual TProposeResult DoStartProposeOnExecute(TColumnShard& owner, NTabletFlatExecutor::TTransactionContext& txc) override; virtual void DoStartProposeOnComplete(TColumnShard& /*owner*/, const TActorContext& /*ctx*/) override { @@ -25,11 +27,12 @@ namespace NKikimr::NColumnShard { virtual bool DoCheckAllowUpdate(const TFullTxInfo& currentTxInfo) const override { return (currentTxInfo.Source == GetTxInfo().Source && currentTxInfo.Cookie == GetTxInfo().Cookie); } + public: using TBase::TBase; void OnTabletInit(TColumnShard& owner) override { - for (auto&& writeId : WriteIds) { + for (auto&& writeId : WriteIds) { Y_ABORT_UNLESS(owner.LongTxWrites.contains(writeId), "TTxInit at %" PRIu64 " : Commit %" PRIu64 " references local write %" PRIu64 " that doesn't exist", owner.TabletID(), GetTxId(), (ui64)writeId); owner.AddLongTxWrite(writeId, GetTxId()); @@ -44,8 +47,7 @@ namespace NKikimr::NColumnShard { return owner.TablesManager.HasTable(pathId); }; - auto counters = owner.InsertTable->Commit(dbTable, version.GetPlanStep(), version.GetTxId(), WriteIds, - pathExists); + auto counters = owner.InsertTable->Commit(dbTable, version.GetPlanStep(), version.GetTxId(), WriteIds, pathExists); owner.IncCounter(COUNTER_BLOBS_COMMITTED, counters.Rows); owner.IncCounter(COUNTER_BYTES_COMMITTED, counters.Bytes); @@ -60,9 +62,8 @@ namespace NKikimr::NColumnShard { } bool CompleteOnProgress(TColumnShard& owner, const TActorContext& ctx) override { - auto result = std::make_unique<TEvColumnShard::TEvProposeTransactionResult>( - owner.TabletID(), TxInfo.TxKind, GetTxId(), NKikimrTxColumnShard::SUCCESS); - result->Record.SetStep(TxInfo.PlanStep); + auto result = std::make_unique<TEvColumnShard::TEvProposeTransactionResult>(owner.TabletID(), TxInfo.TxKind, GetTxId(), NKikimrTxColumnShard::SUCCESS); + result->Record.SetStep(TxInfo.PlanStep); ctx.Send(TxInfo.Source, result.release(), 0, TxInfo.Cookie); return true; } @@ -84,4 +85,5 @@ namespace NKikimr::NColumnShard { private: THashSet<TWriteId> WriteIds; }; -} + +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/transactions/operators/schema.h b/ydb/core/tx/columnshard/transactions/operators/schema.h index 2cc36018ff..84faeab92b 100644 --- a/ydb/core/tx/columnshard/transactions/operators/schema.h +++ b/ydb/core/tx/columnshard/transactions/operators/schema.h @@ -1,7 +1,9 @@ #pragma once #include "propose_tx.h" + #include <ydb/core/tx/columnshard/columnshard_impl.h> +#include <ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.h> namespace NKikimr::NColumnShard { @@ -11,6 +13,7 @@ namespace NKikimr::NColumnShard { using TProposeResult = TTxController::TProposeResult; static inline auto Registrator = TFactory::TRegistrator<TSchemaTransactionOperator>(NKikimrTxColumnShard::TX_KIND_SCHEMA); + std::unique_ptr<NTabletFlatExecutor::ITransaction> TxAddSharding; virtual TTxController::TProposeResult DoStartProposeOnExecute(TColumnShard& owner, NTabletFlatExecutor::TTransactionContext& txc) override; virtual void DoStartProposeOnComplete(TColumnShard& /*owner*/, const TActorContext& /*ctx*/) override { @@ -24,30 +27,53 @@ namespace NKikimr::NColumnShard { virtual bool DoIsAsync() const override { return false; } - virtual bool DoParse(TColumnShard& /*owner*/, const TString& data) override { - return SchemaTxBody.ParseFromString(data); + virtual bool DoParse(TColumnShard& owner, const TString& data) override { + if (!SchemaTxBody.ParseFromString(data)) { + return false; + } + if (SchemaTxBody.HasGranuleShardingInfo()) { + NSharding::TGranuleShardingLogicContainer infoContainer; + if (!infoContainer.DeserializeFromProto(SchemaTxBody.GetGranuleShardingInfo().GetContainer())) { + AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("problem", "cannot parse incoming tx message"); + return false; + } + TxAddSharding = owner.TablesManager.CreateAddShardingInfoTx(owner, SchemaTxBody.GetGranuleShardingInfo().GetPathId(), + SchemaTxBody.GetGranuleShardingInfo().GetVersionId(), infoContainer); + } + return true; } + public: using TBase::TBase; - bool TxWithDeadline() const override { - return false; - } + bool TxWithDeadline() const override { + return false; + } virtual bool ExecuteOnProgress(TColumnShard& owner, const NOlap::TSnapshot& version, NTabletFlatExecutor::TTransactionContext& txc) override { - owner.RunSchemaTx(SchemaTxBody, version, txc); - owner.ProtectSchemaSeqNo(SchemaTxBody.GetSeqNo(), txc); + if (!!TxAddSharding) { + auto* tx = dynamic_cast<TTxAddShardingInfo*>(TxAddSharding.get()); + AFL_VERIFY(tx); + tx->SetSnapshotVersion(version); + TxAddSharding->Execute(txc, NActors::TActivationContext::AsActorContext()); + } + if (SchemaTxBody.TxBody_case() != NKikimrTxColumnShard::TSchemaTxBody::TXBODY_NOT_SET) { + owner.RunSchemaTx(SchemaTxBody, version, txc); + owner.ProtectSchemaSeqNo(SchemaTxBody.GetSeqNo(), txc); + } return true; } virtual bool CompleteOnProgress(TColumnShard& owner, const TActorContext& ctx) override { + if (!!TxAddSharding) { + TxAddSharding->Complete(ctx); + } for (TActorId subscriber : NotifySubscribers) { auto event = MakeHolder<TEvColumnShard::TEvNotifyTxCompletionResult>(owner.TabletID(), GetTxId()); ctx.Send(subscriber, event.Release(), 0, 0); } - auto result = std::make_unique<TEvColumnShard::TEvProposeTransactionResult>( - owner.TabletID(), TxInfo.TxKind, TxInfo.TxId, NKikimrTxColumnShard::SUCCESS); + auto result = std::make_unique<TEvColumnShard::TEvProposeTransactionResult>(owner.TabletID(), TxInfo.TxKind, TxInfo.TxId, NKikimrTxColumnShard::SUCCESS); result->Record.SetStep(TxInfo.PlanStep); ctx.Send(TxInfo.Source, result.release(), 0, TxInfo.Cookie); return true; @@ -81,4 +107,4 @@ namespace NKikimr::NColumnShard { THashSet<TActorId> NotifySubscribers; }; -} +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/transactions/operators/ya.make b/ydb/core/tx/columnshard/transactions/operators/ya.make index b7d63d38b1..579b0d0767 100644 --- a/ydb/core/tx/columnshard/transactions/operators/ya.make +++ b/ydb/core/tx/columnshard/transactions/operators/ya.make @@ -10,7 +10,6 @@ SRCS( ) PEERDIR( - ydb/core/tx/columnshard/transactions ydb/core/tx/columnshard/data_sharing/destination/events ydb/core/tx/columnshard/export/session ) diff --git a/ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.cpp b/ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.cpp new file mode 100644 index 0000000000..500f67a2b3 --- /dev/null +++ b/ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.cpp @@ -0,0 +1,23 @@ +#include "tx_add_sharding_info.h" +#include <ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h> +#include <ydb/core/tx/columnshard/engines/column_engine_logs.h> + +namespace NKikimr::NColumnShard { + +bool TTxAddShardingInfo::Execute(TTransactionContext& txc, const TActorContext& /*ctx*/) { + AFL_VERIFY(!!SnapshotVersion); + NIceDb::TNiceDb db(txc.DB); + db.Table<Schema::ShardingInfo>() + .Key(PathId, ShardingVersion) + .Update(NIceDb::TUpdate<Schema::ShardingInfo::Snapshot>(SnapshotVersion->SerializeToString()), + NIceDb::TUpdate<Schema::ShardingInfo::Logic>(GranuleShardingLogic.SerializeToString())); + return true; +} + +void TTxAddShardingInfo::Complete(const TActorContext& /*ctx*/) { + AFL_VERIFY(!!SnapshotVersion); + NOlap::TGranuleShardingInfo info(GranuleShardingLogic, *SnapshotVersion, ShardingVersion, PathId); + Self->MutableIndexAs<NOlap::TColumnEngineForLogs>().AddShardingInfo(info); +} + +} diff --git a/ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.h b/ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.h new file mode 100644 index 0000000000..dcec1eda40 --- /dev/null +++ b/ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.h @@ -0,0 +1,32 @@ +#pragma once +#include <ydb/core/tx/columnshard/columnshard_impl.h> + +namespace NKikimr::NColumnShard { + +class TTxAddShardingInfo : public NTabletFlatExecutor::TTransactionBase<TColumnShard> { +private: + using TBase = NTabletFlatExecutor::TTransactionBase<TColumnShard>; + NSharding::TGranuleShardingLogicContainer GranuleShardingLogic; + const ui64 PathId; + const ui64 ShardingVersion; + std::optional<NOlap::TSnapshot> SnapshotVersion; + +public: + void SetSnapshotVersion(const NOlap::TSnapshot& ss) { + SnapshotVersion = ss; + } + + TTxAddShardingInfo(TColumnShard& owner, const NSharding::TGranuleShardingLogicContainer& granuleShardingLogic, const ui64 pathId, const ui64 version) + : TBase(&owner) + , GranuleShardingLogic(granuleShardingLogic) + , PathId(pathId) + , ShardingVersion(version) + { + AFL_VERIFY(!!GranuleShardingLogic); + } + + virtual bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; + virtual void Complete(const TActorContext& ctx) override; +}; + +} diff --git a/ydb/core/tx/columnshard/transactions/transactions/ya.make b/ydb/core/tx/columnshard/transactions/transactions/ya.make new file mode 100644 index 0000000000..6ef4144bb7 --- /dev/null +++ b/ydb/core/tx/columnshard/transactions/transactions/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +SRCS( + tx_add_sharding_info.cpp +) + +PEERDIR( + ydb/services/metadata/abstract + ydb/core/tx/columnshard/blobs_action/protos + ydb/core/tx/columnshard/data_sharing/protos +) + +END() diff --git a/ydb/core/tx/columnshard/transactions/ya.make b/ydb/core/tx/columnshard/transactions/ya.make index a6767fb385..8479005c4d 100644 --- a/ydb/core/tx/columnshard/transactions/ya.make +++ b/ydb/core/tx/columnshard/transactions/ya.make @@ -9,6 +9,8 @@ PEERDIR( ydb/core/tablet_flat ydb/core/tx/data_events ydb/core/tx/columnshard/data_sharing/destination/events + ydb/core/tx/columnshard/transactions/operators + ydb/core/tx/columnshard/transactions/transactions ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/data_events/columnshard_splitter.cpp b/ydb/core/tx/data_events/columnshard_splitter.cpp index 098beaa44a..5b435a0f91 100644 --- a/ydb/core/tx/data_events/columnshard_splitter.cpp +++ b/ydb/core/tx/data_events/columnshard_splitter.cpp @@ -47,7 +47,7 @@ NKikimr::NEvWrite::IShardsSplitter::TYdbConclusionStatus TColumnShardShardsSplit NSchemeShard::TOlapSchema olapSchema; olapSchema.ParseFromLocalDB(scheme); - auto shardingConclusion = NSharding::TShardingBase::BuildFromProto(olapSchema, sharding); + auto shardingConclusion = NSharding::IShardingBase::BuildFromProto(olapSchema, sharding); if (shardingConclusion.IsFail()) { return TYdbConclusionStatus::Fail(Ydb::StatusIds::SCHEME_ERROR, shardingConclusion.GetErrorMessage()); } @@ -56,7 +56,7 @@ NKikimr::NEvWrite::IShardsSplitter::TYdbConclusionStatus TColumnShardShardsSplit } NKikimr::NEvWrite::IShardsSplitter::TYdbConclusionStatus TColumnShardShardsSplitter::SplitImpl(const std::shared_ptr<arrow::RecordBatch>& batch, - const std::shared_ptr<NSharding::TShardingBase>& sharding) + const std::shared_ptr<NSharding::IShardingBase>& sharding) { Y_ABORT_UNLESS(batch); @@ -68,7 +68,7 @@ NKikimr::NEvWrite::IShardsSplitter::TYdbConclusionStatus TColumnShardShardsSplit TFullSplitData result(sharding->GetShardsCount()); for (auto&& [shardId, chunks] : split.GetResult()) { for (auto&& c : chunks) { - result.AddShardInfo(shardId, std::make_shared<TShardInfo>(c.GetSchemaData(), c.GetData(), c.GetRowsCount())); + result.AddShardInfo(shardId, std::make_shared<TShardInfo>(c.GetSchemaData(), c.GetData(), c.GetRowsCount(), sharding->GetShardInfoVerified(shardId).GetShardingVersion())); } } diff --git a/ydb/core/tx/data_events/columnshard_splitter.h b/ydb/core/tx/data_events/columnshard_splitter.h index 2abdd25ed4..5225ac6c88 100644 --- a/ydb/core/tx/data_events/columnshard_splitter.h +++ b/ydb/core/tx/data_events/columnshard_splitter.h @@ -17,11 +17,13 @@ class TColumnShardShardsSplitter : public IShardsSplitter { const TString SchemaData; const TString Data; const ui32 RowsCount; + const ui32 GranuleShardingVersion; public: - TShardInfo(const TString& schemaData, const TString& data, const ui32 rowsCount) + TShardInfo(const TString& schemaData, const TString& data, const ui32 rowsCount, const ui32 granuleShardingVersion) : SchemaData(schemaData) , Data(data) , RowsCount(rowsCount) + , GranuleShardingVersion(granuleShardingVersion) {} ui64 GetBytes() const override { @@ -38,6 +40,7 @@ class TColumnShardShardsSplitter : public IShardsSplitter { void Serialize(TEvWrite& evWrite) const override { evWrite.SetArrowData(SchemaData, Data); + evWrite.Record.SetGranuleShardingVersion(GranuleShardingVersion); } }; @@ -45,7 +48,7 @@ private: TYdbConclusionStatus DoSplitData(const NSchemeCache::TSchemeCacheNavigate::TEntry& schemeEntry, const IEvWriteDataAccessor& data) override; private: - TYdbConclusionStatus SplitImpl(const std::shared_ptr<arrow::RecordBatch>& batch, const std::shared_ptr<NSharding::TShardingBase>& sharding); + TYdbConclusionStatus SplitImpl(const std::shared_ptr<arrow::RecordBatch>& batch, const std::shared_ptr<NSharding::IShardingBase>& sharding); std::shared_ptr<arrow::Schema> ExtractArrowSchema(const NKikimrSchemeOp::TColumnTableSchema& schema); }; diff --git a/ydb/core/tx/data_events/write_data.h b/ydb/core/tx/data_events/write_data.h index d25c65e65c..0ad33cf9bf 100644 --- a/ydb/core/tx/data_events/write_data.h +++ b/ydb/core/tx/data_events/write_data.h @@ -26,6 +26,7 @@ class TWriteMeta { YDB_ACCESSOR(ui64, WriteId, 0); YDB_READONLY(ui64, TableId, 0); YDB_ACCESSOR_DEF(NActors::TActorId, Source); + YDB_ACCESSOR_DEF(std::optional<ui32>, GranuleShardingVersion); // Long Tx logic YDB_OPT(NLongTxService::TLongTxId, LongTxId); @@ -41,10 +42,11 @@ class TWriteMeta { YDB_ACCESSOR(TMonotonic, WriteMiddle5StartInstant, TMonotonic::Now()); YDB_ACCESSOR(TMonotonic, WriteMiddle6StartInstant, TMonotonic::Now()); public: - TWriteMeta(const ui64 writeId, const ui64 tableId, const NActors::TActorId& source) + TWriteMeta(const ui64 writeId, const ui64 tableId, const NActors::TActorId& source, const std::optional<ui32> granuleShardingVersion) : WriteId(writeId) , TableId(tableId) , Source(source) + , GranuleShardingVersion(granuleShardingVersion) {} }; diff --git a/ydb/core/tx/schemeshard/olap/bg_tasks/transactions/tasks_list.cpp b/ydb/core/tx/schemeshard/olap/bg_tasks/transactions/tasks_list.cpp index 27f45deb1d..44c555451a 100644 --- a/ydb/core/tx/schemeshard/olap/bg_tasks/transactions/tasks_list.cpp +++ b/ydb/core/tx/schemeshard/olap/bg_tasks/transactions/tasks_list.cpp @@ -54,7 +54,7 @@ TTxTasksList::TTxTasksList(TSelf* self, TEvListRequest::TPtr& ev) , RequestCookie(ev->Cookie) , DatabaseName(ev->Get()->Record.GetDatabaseName()) { - if (ev->Get()->Record.HasPageSize()) { + if (ev->Get()->Record.HasPageSize() && ev->Get()->Record.GetPageSize()) { PageSize = ev->Get()->Record.GetPageSize(); } if (ev->Get()->Record.HasPageToken()) { diff --git a/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.cpp b/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.cpp index 44e1a9f804..6cdcc8749b 100644 --- a/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.cpp +++ b/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.cpp @@ -40,10 +40,10 @@ void TTxChainActor::Handle(TEvSchemeShard::TEvModifySchemeTransactionResult::TPt } void TTxChainActor::OnSessionProgressSaved() { - if (SessionLogic->GetStepForExecute() < SessionLogic->GetTxData().GetTransactions().size()) { - NActors::TActivationContext::AsActorContext().Send(TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(1)); + if (SessionLogic->IsFinished()) { + SaveSessionState(); } else { - Session->FinishActor(); + NActors::TActivationContext::AsActorContext().Send(TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(1)); } } diff --git a/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.h b/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.h index 89b04aa673..26ae57aa5e 100644 --- a/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.h +++ b/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.h @@ -31,7 +31,11 @@ protected: } virtual void OnSessionProgressSaved() override; virtual void OnSessionStateSaved() override { - SendCurrentTxToSS(); + if (SessionLogic->IsFinished()) { + Session->FinishActor(); + } else { + SendCurrentTxToSS(); + } } virtual void OnBootstrap(const TActorContext& /*ctx*/) override; diff --git a/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/session.h b/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/session.h index f93b1d4267..1a56abe419 100644 --- a/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/session.h +++ b/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/session.h @@ -87,7 +87,7 @@ public: AFL_VERIFY(StepForExecute <= GetTxData().GetTransactions().size()); return StepForExecute == GetTxData().GetTransactions().size(); } - virtual bool IsReadyForRemove() const override { + virtual bool IsReadyForRemoveOnFinished() const override { return true; } }; diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/context.h b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/context.h index 65623ae802..f5423ce938 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/context.h +++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/context.h @@ -3,9 +3,12 @@ #include <ydb/core/scheme/scheme_pathid.h> #include <ydb/core/tx/schemeshard/schemeshard__operation_part.h> #include <ydb/core/tx/schemeshard/schemeshard_path.h> +#include <ydb/core/tx/columnshard/common/snapshot.h> namespace NKikimr::NSchemeShard::NOlap::NAlter { +class ISSEntity; + class TEntityInitializationContext { private: const TOperationContext* SSOperationContext = nullptr; @@ -19,21 +22,51 @@ public: } }; -class TUpdateInitializationContext { +class TUpdateRestoreContext { private: + const ISSEntity* OriginalEntity; const TOperationContext* SSOperationContext = nullptr; - const NKikimrSchemeOp::TModifyScheme* Modification = nullptr; + const ui64 TxId; public: + ui64 GetTxId() const { + return TxId; + } + + const ISSEntity& GetOriginalEntity() const { + return *OriginalEntity; + } + + template <class T> + const T& GetOriginalEntityAsVerified() const { + auto* result = dynamic_cast<const T*>(OriginalEntity); + AFL_VERIFY(!!result); + return *result; + } + const TOperationContext* GetSSOperationContext() const { return SSOperationContext; } + TUpdateRestoreContext(const ISSEntity* originalEntity, const TOperationContext* ssOperationContext, const ui64 txId) + : OriginalEntity(originalEntity) + , SSOperationContext(ssOperationContext) + , TxId(txId) { + AFL_VERIFY(OriginalEntity); + AFL_VERIFY(SSOperationContext); + AFL_VERIFY(TxId); + } +}; + +class TUpdateInitializationContext: public TUpdateRestoreContext { +private: + using TBase = TUpdateRestoreContext; + const NKikimrSchemeOp::TModifyScheme* Modification = nullptr; +public: const NKikimrSchemeOp::TModifyScheme* GetModification() const { return Modification; } - TUpdateInitializationContext(const TOperationContext* ssOperationContext, const NKikimrSchemeOp::TModifyScheme* modification) - : SSOperationContext(ssOperationContext) + TUpdateInitializationContext(const ISSEntity* originalEntity, const TOperationContext* ssOperationContext, const NKikimrSchemeOp::TModifyScheme* modification, const ui64 txId) + : TBase(originalEntity, ssOperationContext, txId) , Modification(modification) { - AFL_VERIFY(SSOperationContext); AFL_VERIFY(Modification); } }; @@ -51,7 +84,7 @@ public: } }; -class TEvolutionStartContext { +class TUpdateStartContext { private: const TPath* ObjectPath = nullptr; TOperationContext* SSOperationContext = nullptr; @@ -67,14 +100,33 @@ public: return SSOperationContext; } - TEvolutionStartContext(const TPath* objectPath, TOperationContext* ssOperationContext, NIceDb::TNiceDb* db) + TUpdateStartContext(const TPath* objectPath, TOperationContext* ssOperationContext, NIceDb::TNiceDb* db) : ObjectPath(objectPath) , SSOperationContext(ssOperationContext) - , DB(db) { + , DB(db) + { AFL_VERIFY(DB); AFL_VERIFY(ObjectPath); AFL_VERIFY(SSOperationContext); } }; +class TUpdateFinishContext: public TUpdateStartContext { +private: + using TBase = TUpdateStartContext; + YDB_READONLY_DEF(std::optional<NKikimr::NOlap::TSnapshot>, Snapshot); +public: + + const NKikimr::NOlap::TSnapshot& GetSnapshotVerified() const { + AFL_VERIFY(Snapshot); + return *Snapshot; + } + + TUpdateFinishContext(const TPath* objectPath, TOperationContext* ssOperationContext, NIceDb::TNiceDb* db, const std::optional<NKikimr::NOlap::TSnapshot>& ss) + : TBase(objectPath, ssOperationContext, db) + , Snapshot(ss) + { + } +}; + }
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h index 4163f82fb7..7a45468961 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h +++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h @@ -103,10 +103,6 @@ public: return TConclusionStatus::Fail("Changing table schema is not supported"); } - if (result.HasRESERVED_AlterTtlSettingsPresetName()) { - return TConclusionStatus::Fail("TTL presets are not supported"); - } - return result; } }; diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.cpp deleted file mode 100644 index 4ec6d89d9b..0000000000 --- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "evolution.h" - -namespace NKikimr::NSchemeShard::NOlap::NAlter { - -}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.h b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.h deleted file mode 100644 index 7884fbd4fd..0000000000 --- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once -#include "context.h" -#include <ydb/library/accessor/accessor.h> -#include <ydb/core/scheme/scheme_pathid.h> -#include <ydb/core/tablet_flat/flat_cxx_database.h> -#include <ydb/core/tx/schemeshard/schemeshard__operation_part.h> -#include <ydb/core/tx/schemeshard/schemeshard_path.h> - -namespace NKikimr::NSchemeShard::NOlap::NAlter { - -class ISSEntityEvolution { -private: - YDB_READONLY_DEF(TPathId, PathId); - bool Initialized = false; -protected: - virtual TConclusionStatus DoInitialize(const TEvolutionInitializationContext& context) = 0; - virtual TConclusionStatus DoStartEvolution(const TEvolutionStartContext& context) = 0; - virtual NKikimrTxColumnShard::TSchemaTxBody DoGetShardTxBody(const NKikimrTxColumnShard::TSchemaTxBody& original, const ui64 tabletId) const = 0; -public: - virtual ~ISSEntityEvolution() = default; - TConclusionStatus Initialize(const TEvolutionInitializationContext& context) { - AFL_VERIFY(!Initialized); - Initialized = true; - return DoInitialize(context); - } - - TConclusionStatus StartEvolution(const TEvolutionStartContext& context) { - return DoStartEvolution(context); - } - - virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const = 0; - NKikimrTxColumnShard::TSchemaTxBody GetShardTxBody(const NKikimrTxColumnShard::TSchemaTxBody& original, const ui64 tabletId) const { - return DoGetShardTxBody(original, tabletId); - } - - ISSEntityEvolution(const TPathId& pathId) - : PathId(pathId) { - - } -}; - -}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.cpp index 0a1c3272f3..372991cdc3 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.cpp @@ -1,5 +1,16 @@ #include "object.h" +#include <ydb/core/tx/schemeshard/schemeshard_impl.h> namespace NKikimr::NSchemeShard::NOlap::NAlter { +std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEntity> ISSEntity::GetEntityVerified(TOperationContext& context, const TPath& path) { + if (path->IsColumnTable()) { + auto readGuard = context.SS->ColumnTables.GetVerified(path.Base()->PathId); + TEntityInitializationContext iContext(&context); + return readGuard->BuildEntity(path.Base()->PathId, iContext).DetachResult(); + } + AFL_VERIFY(false); + return nullptr; +} + }
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h index 4b24ef4a76..fe6d5db9f6 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h +++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h @@ -14,13 +14,23 @@ private: bool Initialized = false; protected: [[nodiscard]] virtual TConclusionStatus DoInitialize(const TEntityInitializationContext& context) = 0; - virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdate(const TUpdateInitializationContext& context, const std::shared_ptr<ISSEntity>& selfPtr) const = 0; + virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdate(const TUpdateInitializationContext& context) const = 0; + virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoRestoreUpdate(const TUpdateRestoreContext& context) const = 0; + public: virtual ~ISSEntity() = default; virtual TString GetClassName() const = 0; - virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> CreateUpdate(const TUpdateInitializationContext& context, const std::shared_ptr<ISSEntity>& selfPtr) const { - return DoCreateUpdate(context, selfPtr); + TConclusion<std::shared_ptr<ISSEntityUpdate>> CreateUpdate(const TUpdateInitializationContext& context) const { + return DoCreateUpdate(context); + } + + TConclusion<std::shared_ptr<ISSEntityUpdate>> RestoreUpdate(const TUpdateRestoreContext& context) const { + return DoRestoreUpdate(context); + } + + std::shared_ptr<ISSEntityUpdate> RestoreUpdateVerified(const TUpdateRestoreContext& context) const { + return RestoreUpdate(context).DetachResult(); } [[nodiscard]] TConclusionStatus Initialize(const TEntityInitializationContext& context) { @@ -34,6 +44,8 @@ public: { } + + static std::shared_ptr<ISSEntity> GetEntityVerified(TOperationContext& context, const TPath& path); }; }
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h index 521432745f..61953ec45a 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h +++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h @@ -8,31 +8,47 @@ namespace NKikimr::NSchemeShard::NOlap::NAlter { -class ISSEntityEvolution; - class ISSEntityUpdate { private: - YDB_READONLY_DEF(std::shared_ptr<ISSEntity>, OriginalEntity); bool Initialized = false; protected: virtual TConclusionStatus DoInitialize(const TUpdateInitializationContext& context) = 0; - virtual TConclusion<std::shared_ptr<ISSEntityEvolution>> DoBuildEvolution(const std::shared_ptr<ISSEntityUpdate>& selfPtr) const = 0; + virtual TConclusionStatus DoStart(const TUpdateStartContext& context) = 0; + virtual TConclusionStatus DoFinish(const TUpdateFinishContext& context) = 0; + virtual TString DoGetShardTxBodyString(const ui64 tabletId, const TMessageSeqNo& seqNo) const = 0; + virtual std::set<ui64> DoGetShardIds() const = 0; public: + ISSEntityUpdate() = default; virtual ~ISSEntityUpdate() = default; - ISSEntityUpdate(const std::shared_ptr<ISSEntity>& originalEntity) - : OriginalEntity(originalEntity) { - AFL_VERIFY(OriginalEntity); + virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const = 0; + + std::set<ui64> GetShardIds() const { + AFL_VERIFY(Initialized); + return DoGetShardIds(); } - TConclusionStatus Initialize(const TUpdateInitializationContext& context) { - AFL_VERIFY(!Initialized); - Initialized = true; - return DoInitialize(context); + TString GetShardTxBodyString(const ui64 tabletId, const TMessageSeqNo& seqNo) const { + AFL_VERIFY(Initialized); + return DoGetShardTxBodyString(tabletId, seqNo); + } + + TConclusionStatus Start(const TUpdateStartContext& context) { + AFL_VERIFY(Initialized); + return DoStart(context); + } + + TConclusionStatus Finish(const TUpdateFinishContext& context) { + AFL_VERIFY(Initialized); + return DoFinish(context); } - TConclusion<std::shared_ptr<ISSEntityEvolution>> BuildEvolution(const std::shared_ptr<ISSEntityUpdate>& selfPtr) const { - AFL_VERIFY(!!selfPtr); - return DoBuildEvolution(selfPtr); + TConclusionStatus Initialize(const TUpdateInitializationContext& context) { + AFL_VERIFY(!Initialized); + auto result = DoInitialize(context); + if (result.IsSuccess()) { + Initialized = true; + } + return result; } }; diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/ya.make index 63b5dc1a63..66a40695a6 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/ya.make +++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/ya.make @@ -3,7 +3,6 @@ LIBRARY() SRCS( object.cpp update.cpp - evolution.cpp converter.cpp context.cpp ) diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/common/object.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/common/object.cpp new file mode 100644 index 0000000000..f8987b734d --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/common/object.cpp @@ -0,0 +1,40 @@ +#include "object.h" +#include <ydb/core/tx/schemeshard/schemeshard_impl.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +TConclusion<std::shared_ptr<ISSEntityUpdate>> TColumnTableEntity::DoRestoreUpdate(const TUpdateRestoreContext& context) const { + auto& ssContext = *context.GetSSOperationContext(); + auto tableInfo = ssContext.SS->ColumnTables.GetVerifiedPtr(GetPathId()); + if (!tableInfo) { + return TConclusionStatus::Fail("object not exists"); + } + if (!tableInfo->AlterData) { + return TConclusionStatus::Fail("object not in update"); + } + if (!tableInfo->AlterData->AlterBody) { + return TConclusionStatus::Fail("object is incorrect in update"); + } + NKikimrSchemeOp::TModifyScheme mScheme; + *mScheme.MutableAlterColumnTable() = *tableInfo->AlterData->AlterBody; + mScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterColumnTable); + TUpdateInitializationContext uContext(&context.GetOriginalEntity(), context.GetSSOperationContext(), &mScheme, context.GetTxId()); + return DoCreateUpdateImpl(uContext); +} + +TConclusionStatus TColumnTableEntity::DoInitialize(const TEntityInitializationContext& context) { + TableInfo = context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(GetPathId()); + return DoInitializeImpl(context); +} + +TConclusion<std::shared_ptr<ISSEntityUpdate>> TColumnTableEntity::DoCreateUpdate(const TUpdateInitializationContext& context) const { + if (GetTableInfo()->AlterVersion == 0) { + return NKikimr::TConclusionStatus::Fail("Table is not created yet"); + } + if (GetTableInfo()->AlterData) { + return NKikimr::TConclusionStatus::Fail("There's another Alter in flight"); + } + return DoCreateUpdateImpl(context); +} + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/common/object.h b/ydb/core/tx/schemeshard/olap/operations/alter/common/object.h new file mode 100644 index 0000000000..215cf77b95 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/common/object.h @@ -0,0 +1,45 @@ +#pragma once +#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h> +#include <ydb/core/tx/schemeshard/olap/table/table.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +class TColumnTableEntity: public ISSEntity { +private: + using TBase = ISSEntity; + YDB_READONLY_DEF(TColumnTableInfo::TPtr, TableInfo); + virtual TConclusionStatus DoInitializeImpl(const TEntityInitializationContext& context) = 0; + virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdateImpl(const TUpdateInitializationContext& context) const = 0; +protected: + TConclusion<std::shared_ptr<ISSEntityUpdate>> DoRestoreUpdate(const TUpdateRestoreContext& ssContext) const override; + virtual TConclusionStatus DoInitialize(const TEntityInitializationContext& context) override final; + virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdate(const TUpdateInitializationContext& context) const override final;; + + TColumnTableEntity(const TPathId& pathId, const TColumnTableInfo::TPtr& tabletInfo) + : TBase(pathId) + , TableInfo(tabletInfo) { + } + +public: + + TColumnTableEntity(const TPathId& pathId) + : TBase(pathId) { + } + + const TColumnTableInfo& GetTableInfoVerified() const { + AFL_VERIFY(!!TableInfo); + return *TableInfo; + } + + TColumnTableInfo::TPtr GetTableInfoPtrVerified() const { + AFL_VERIFY(!!TableInfo); + return TableInfo; + } + + const NKikimrSchemeOp::TColumnDataLifeCycle& GetTableTTLProto() { + AFL_VERIFY(TableInfo); + return TableInfo->Description.GetTtlSettings(); + } +}; + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/common/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/common/update.cpp new file mode 100644 index 0000000000..442c678335 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/common/update.cpp @@ -0,0 +1,31 @@ +#include "update.h" +#include <ydb/core/tx/schemeshard/schemeshard_impl.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +TConclusionStatus TColumnTableUpdate::DoStart(const TUpdateStartContext& context) { + auto conclusion = DoStartImpl(context); + if (conclusion.IsFail()) { + return conclusion; + } + const auto pathId = context.GetObjectPath()->Base()->PathId; + auto tableInfo = context.GetSSOperationContext()->SS->ColumnTables.TakeVerified(pathId); + context.GetSSOperationContext()->SS->PersistColumnTableAlter(*context.GetDB(), pathId, *GetTargetTableInfoVerified()); + tableInfo->AlterData = GetTargetTableInfoVerified(); + return TConclusionStatus::Success(); +} + +TConclusionStatus TColumnTableUpdate::DoFinish(const TUpdateFinishContext& context) { + auto conclusion = DoFinishImpl(context); + if (conclusion.IsFail()) { + return conclusion; + } + + const auto pathId = context.GetObjectPath()->Base()->PathId; + auto tableInfo = context.GetSSOperationContext()->SS->ColumnTables.TakeAlterVerified(pathId); + context.GetSSOperationContext()->SS->PersistColumnTableAlterRemove(*context.GetDB(), pathId); + context.GetSSOperationContext()->SS->PersistColumnTable(*context.GetDB(), pathId, *tableInfo); + return TConclusionStatus::Success(); +} + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/common/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/common/update.h new file mode 100644 index 0000000000..c33f18eb03 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/common/update.h @@ -0,0 +1,58 @@ +#pragma once +#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/context.h> +#include <ydb/core/tx/schemeshard/olap/table/table.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +class TColumnTableUpdate: public ISSEntityUpdate { +private: + using TBase = ISSEntityUpdate; + + virtual std::shared_ptr<TColumnTableInfo> GetTargetTableInfo() const = 0; + virtual std::shared_ptr<ISSEntity> GetTargetSSEntity() const = 0; + + virtual TConclusionStatus DoStart(const TUpdateStartContext& context) override final; + virtual TConclusionStatus DoFinish(const TUpdateFinishContext& context) override final; + + virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override { + return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SCHEMA; + } + virtual TConclusionStatus DoInitializeImpl(const TUpdateInitializationContext& context) = 0; +protected: + virtual TConclusionStatus DoStartImpl(const TUpdateStartContext& /*context*/) { + return TConclusionStatus::Success(); + } + virtual TConclusionStatus DoFinishImpl(const TUpdateFinishContext& /*context*/) { + return TConclusionStatus::Success(); + } + virtual TConclusionStatus DoInitialize(const TUpdateInitializationContext& context) override final { + if (!context.GetModification()->HasAlterColumnTable() && !context.GetModification()->HasAlterTable()) { + return TConclusionStatus::Fail("no update data"); + } + return DoInitializeImpl(context); + } + + std::shared_ptr<TColumnTableInfo> GetTargetTableInfoVerified() const { + auto result = GetTargetTableInfo(); + AFL_VERIFY(!!result); + return result; + } + + template <class T> + const T& GetTargetEntityAsVerified() const { + auto resultPtr = dynamic_pointer_cast<T>(GetTargetEntityVerified()); + AFL_VERIFY(!!resultPtr); + return *resultPtr; + } + + std::shared_ptr<ISSEntity> GetTargetEntityVerified() const { + auto result = GetTargetSSEntity(); + AFL_VERIFY(!!result); + return result; + } + +public: +}; + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/common/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/common/ya.make new file mode 100644 index 0000000000..24a9ac14d1 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/common/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +SRCS( + update.cpp + object.cpp +) + +PEERDIR( + ydb/core/tx/schemeshard/olap/operations/alter/abstract +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.cpp new file mode 100644 index 0000000000..62c52b4b56 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.cpp @@ -0,0 +1,26 @@ +#include "update.h" +#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h> +#include <ydb/core/tx/schemeshard/schemeshard_impl.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +NKikimr::TConclusionStatus TInStoreTableUpdate::DoStartImpl(const TUpdateStartContext& context) { + const auto& inStoreTable = GetTargetEntityAsVerified<TInStoreTable>(); + + auto tableInfo = GetTargetTableInfoVerified(); + const auto storePathId = tableInfo->GetOlapStorePathIdVerified(); + TPath storePath = TPath::Init(storePathId, context.GetSSOperationContext()->SS); + + Y_ABORT_UNLESS(inStoreTable.GetStoreInfo()->ColumnTables.contains((*context.GetObjectPath())->PathId)); + inStoreTable.GetStoreInfo()->ColumnTablesUnderOperation.insert((*context.GetObjectPath())->PathId); + + // Sequentially chain operations in the same olap store + if (context.GetSSOperationContext()->SS->Operations.contains(storePath.Base()->LastTxId)) { + context.GetSSOperationContext()->OnComplete.Dependence(storePath.Base()->LastTxId, (*context.GetObjectPath())->LastTxId); + } + storePath.Base()->LastTxId = (*context.GetObjectPath())->LastTxId; + context.GetSSOperationContext()->SS->PersistLastTxId(*context.GetDB(), storePath.Base()); + return DoStartInStoreImpl(context); +} + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.h new file mode 100644 index 0000000000..415945c182 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.h @@ -0,0 +1,19 @@ +#pragma once +#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/common/update.h> +#include <ydb/core/tx/schemeshard/olap/ttl/update.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +class TInStoreTableUpdate: public TColumnTableUpdate { +private: + using TBase = TColumnTableUpdate; + + virtual TConclusionStatus DoStartImpl(const TUpdateStartContext& context) override final; + virtual TConclusionStatus DoStartInStoreImpl(const TUpdateStartContext& /*context*/) { + return TConclusionStatus::Success(); + } +public: +}; + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/ya.make new file mode 100644 index 0000000000..13fb75778f --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +SRCS( + update.cpp +) + +PEERDIR( + ydb/core/tx/schemeshard/olap/operations/alter/abstract +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.cpp new file mode 100644 index 0000000000..6dbcbfb1d7 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.cpp @@ -0,0 +1,82 @@ +#include "update.h" +#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h> +#include <ydb/core/tx/schemeshard/schemeshard_impl.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +TConclusionStatus TInStoreShardsUpdate::DoInitializeImpl(const TUpdateInitializationContext& context) { + const auto& original = context.GetOriginalEntityAsVerified<TInStoreTable>(); + if (!context.GetModification()->GetAlterColumnTable().HasAlterShards()) { + return TConclusionStatus::Fail("no data about shards altering"); + } + auto& alterShards = context.GetModification()->GetAlterColumnTable().GetAlterShards(); + Alter = alterShards; + AFL_VERIFY(Alter.HasModification()); + + for (auto&& i : Alter.GetModification().GetNewShardIds()) { + AFL_VERIFY(ShardIds.emplace(i).second); + AFL_VERIFY(NewShardIds.emplace(i).second); + } + + for (auto&& i : Alter.GetModification().GetDeleteShardIds()) { + AFL_VERIFY(ShardIds.emplace(i).second); + AFL_VERIFY(DeleteShardIds.emplace(i).second); + } + + auto tableInfo = original.GetTableInfoPtrVerified(); + + auto description = tableInfo->Description; + + Sharding = tableInfo->GetShardingVerified(original.GetTableSchemaVerified()); + Sharding->ApplyModification(Alter.GetModification()).Validate(); + *description.MutableSharding() = Sharding->SerializeToProto(); + for (auto&& i : Sharding->GetModifiedShardIds(Alter.GetModification())) { + ShardIds.emplace(i); + AFL_VERIFY(ModifiedShardIds.emplace(i).second); + } + + auto targetInfo = std::make_shared<TColumnTableInfo>(tableInfo->AlterVersion + 1, std::move(description), + TMaybe<NKikimrSchemeOp::TColumnStoreSharding>(), context.GetModification()->GetAlterColumnTable()); + + TEntityInitializationContext eContext(context.GetSSOperationContext()); + TargetInStoreTable = std::make_shared<TInStoreTable>(original.GetPathId(), targetInfo, eContext); + + return TConclusionStatus::Success(); +} + +void TInStoreShardsUpdate::FillToShardTx(NKikimrTxColumnShard::TCreateTable& info) const { + info.SetPathId(TargetInStoreTable->GetPathId().LocalPathId); + auto& alterBody = TargetInStoreTable->GetTableInfoVerified(); + + AFL_VERIFY(alterBody.Description.HasSchemaPresetId()); + const ui32 presetId = alterBody.Description.GetSchemaPresetId(); + Y_ABORT_UNLESS(!!TargetInStoreTable->GetStoreInfo(), "Unexpected schema preset without olap store"); + Y_ABORT_UNLESS(TargetInStoreTable->GetStoreInfo()->SchemaPresets.contains(presetId), "Failed to find schema preset %" PRIu32 " in an olap store", presetId); + auto& preset = TargetInStoreTable->GetStoreInfo()->SchemaPresets.at(presetId); + size_t presetIndex = preset.GetProtoIndex(); + *info.MutableSchemaPreset() = TargetInStoreTable->GetStoreInfo()->GetDescription().GetSchemaPresets(presetIndex); + + if (alterBody.Description.HasSchemaPresetVersionAdj()) { + info.SetSchemaPresetVersionAdj(alterBody.Description.GetSchemaPresetVersionAdj()); + } + if (alterBody.Description.HasTtlSettings()) { + *info.MutableTtlSettings() = alterBody.Description.GetTtlSettings(); + } +} + +NKikimr::TConclusionStatus TInStoreShardsUpdate::DoFinishImpl(const TUpdateFinishContext& context) { + auto conclusion = TBase::DoFinishImpl(context); + if (conclusion.IsFail()) { + return conclusion; + } + auto alter = context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(TargetInStoreTable->GetPathId())->AlterData; + for (auto&& i : Alter.GetModification().GetOpenWriteIds()) { + AFL_VERIFY(!!alter); + Sharding->SetShardingOpenSnapshotVerified(i, context.GetSnapshotVerified()); + } + AFL_VERIFY(!!alter->AlterBody); + *alter->Description.MutableSharding() = Sharding->SerializeToProto(); + return TConclusionStatus::Success(); +} + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.h new file mode 100644 index 0000000000..87d1bc7f18 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.h @@ -0,0 +1,63 @@ +#pragma once +#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.h> +#include <ydb/core/tx/schemeshard/olap/ttl/update.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +class TInStoreShardsUpdate: public TInStoreTableUpdate { +private: + using TBase = TInStoreTableUpdate; + NKikimrSchemeOp::TAlterShards Alter; + std::shared_ptr<TInStoreTable> TargetInStoreTable; + std::set<ui64> ShardIds; + std::set<ui64> NewShardIds; + std::set<ui64> DeleteShardIds; + std::set<ui64> ModifiedShardIds; + std::shared_ptr<NSharding::IShardingBase> Sharding; + virtual TConclusionStatus DoInitializeImpl(const TUpdateInitializationContext& context) override; + void FillToShardTx(NKikimrTxColumnShard::TCreateTable& shardAlter) const; + + virtual TConclusionStatus DoFinishImpl(const TUpdateFinishContext& context) override; + virtual std::shared_ptr<TColumnTableInfo> GetTargetTableInfo() const override { + return TargetInStoreTable->GetTableInfoPtrVerified(); + } + virtual std::shared_ptr<ISSEntity> GetTargetSSEntity() const override { + return TargetInStoreTable; + } + + virtual TString DoGetShardTxBodyString(const ui64 tabletId, const TMessageSeqNo& seqNo) const override { + NKikimrTxColumnShard::TSchemaTxBody result; + result.MutableSeqNo()->SetGeneration(seqNo.Generation); + result.MutableSeqNo()->SetRound(seqNo.Round); + AFL_VERIFY(NewShardIds.contains(tabletId) || ModifiedShardIds.contains(tabletId) || DeleteShardIds.contains(tabletId)); + if (NewShardIds.contains(tabletId)) { + auto& alter = *result.MutableEnsureTables(); + auto& create = *alter.AddTables(); + FillToShardTx(create); + create.SetPathId(TargetInStoreTable->GetPathId().LocalPathId); + } + if (DeleteShardIds.contains(tabletId)) { + result.MutableDropTable()->SetPathId(TargetInStoreTable->GetPathId().LocalPathId); + } + auto container = Sharding->GetTabletShardingInfoOptional(tabletId); + if (!!container) { + auto& shardingInfo = *result.MutableGranuleShardingInfo(); + shardingInfo.SetPathId(TargetInStoreTable->GetPathId().LocalPathId); + shardingInfo.SetVersionId(Sharding->GetShardInfoVerified(tabletId).GetShardingVersion()); + *shardingInfo.MutableContainer() = container.SerializeToProto(); + AFL_VERIFY(ModifiedShardIds.contains(tabletId)); + } + return result.SerializeAsString(); + } + + virtual std::set<ui64> DoGetShardIds() const override { + return ShardIds; + } + +public: + +}; + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/ya.make new file mode 100644 index 0000000000..13fb75778f --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +SRCS( + update.cpp +) + +PEERDIR( + ydb/core/tx/schemeshard/olap/operations/alter/abstract +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.cpp deleted file mode 100644 index 7fbdcebcc5..0000000000 --- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "evolution.h" -#include <ydb/core/tx/schemeshard/schemeshard_impl.h> - -namespace NKikimr::NSchemeShard::NOlap::NAlter { - -NKikimr::TConclusionStatus TInStoreSchemaEvolution::DoInitialize(const TEvolutionInitializationContext& context) { - TColumnTableInfo::TPtr tableInfo = context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(GetPathId()); - AFL_VERIFY(!tableInfo->IsStandalone()); - TEntityInitializationContext eContext(context.GetSSOperationContext()); - Original = std::make_shared<TInStoreTable>(GetPathId(), tableInfo, eContext); - Target = std::make_shared<TInStoreTable>(GetPathId(), tableInfo->AlterData, eContext); - Update = std::make_shared<TInStoreSchemaUpdate>(Original, Target); - return TConclusionStatus::Success(); -} - -NKikimr::TConclusionStatus TInStoreSchemaEvolution::DoStartEvolution(const TEvolutionStartContext& context) { - const auto storePathId = Original->GetTableInfo()->GetOlapStorePathIdVerified(); - TPath storePath = TPath::Init(storePathId, context.GetSSOperationContext()->SS); - - Y_ABORT_UNLESS(Original->GetStoreInfo()->ColumnTables.contains((*context.GetObjectPath())->PathId)); - Original->GetStoreInfo()->ColumnTablesUnderOperation.insert((*context.GetObjectPath())->PathId); - - // Sequentially chain operations in the same olap store - if (context.GetSSOperationContext()->SS->Operations.contains(storePath.Base()->LastTxId)) { - context.GetSSOperationContext()->OnComplete.Dependence(storePath.Base()->LastTxId, (*context.GetObjectPath())->LastTxId); - } - storePath.Base()->LastTxId = (*context.GetObjectPath())->LastTxId; - context.GetSSOperationContext()->SS->PersistLastTxId(*context.GetDB(), storePath.Base()); - context.GetSSOperationContext()->SS->PersistColumnTableAlter(*context.GetDB(), GetPathId(), *Target->GetTableInfo()); - auto table = context.GetSSOperationContext()->SS->ColumnTables.TakeVerified(GetPathId()); - table->AlterData = Target->GetTableInfoPtrVerified(); - return TConclusionStatus::Success(); -} - -}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.h deleted file mode 100644 index a26a43d94c..0000000000 --- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once -#include "object.h" -#include "update.h" - -#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.h> - -namespace NKikimr::NSchemeShard::NOlap::NAlter { - -class TInStoreSchemaEvolution: public ISSEntityEvolution { -private: - using TBase = ISSEntityEvolution; - std::shared_ptr<TInStoreTable> Original; - std::shared_ptr<TInStoreTable> Target; - std::shared_ptr<TInStoreSchemaUpdate> Update; -protected: - virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override { - return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SCHEMA; - } - - virtual NKikimrTxColumnShard::TSchemaTxBody DoGetShardTxBody(const NKikimrTxColumnShard::TSchemaTxBody& original, const ui64 /*tabletId*/) const override { - NKikimrTxColumnShard::TSchemaTxBody copy = original; - - auto& alter = *copy.MutableAlterTable(); - Update->FillToShardTx(alter, Target); - alter.SetPathId(GetPathId().LocalPathId); - return copy; - } - - virtual TConclusionStatus DoInitialize(const TEvolutionInitializationContext& context) override; - - virtual TConclusionStatus DoStartEvolution(const TEvolutionStartContext& context) override; -public: - TInStoreSchemaEvolution(const TPathId& pathId) - : TBase(pathId) - { - - } - TInStoreSchemaEvolution(const std::shared_ptr<TInStoreTable>& original, const std::shared_ptr<TInStoreTable>& target, const std::shared_ptr<TInStoreSchemaUpdate>& update) - : TBase(original ? original->GetPathId() : target->GetPathId()) - , Original(original) - , Target(target) - , Update(update) - { - AFL_VERIFY(!!Original); - AFL_VERIFY(!!Target); - AFL_VERIFY(!!Update); - } -}; - -}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.cpp index de08a2396b..867849583e 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.cpp @@ -1,14 +1,15 @@ #include "object.h" -#include "update.h" +#include "config_shards/update.h" +#include "resharding/update.h" +#include "schema/update.h" +#include "transfer/update.h" #include <ydb/core/tx/schemeshard/schemeshard_impl.h> namespace NKikimr::NSchemeShard::NOlap::NAlter { -TConclusionStatus TInStoreTable::InitializeWithTableInfo(const TColumnTableInfo::TPtr& tInfo, const TEntityInitializationContext& context) { - TableInfo = tInfo; - AFL_VERIFY(!!TableInfo); - AFL_VERIFY(!TableInfo->IsStandalone()); - const auto storePathId = TableInfo->GetOlapStorePathIdVerified(); +TConclusionStatus TInStoreTable::InitializeWithTableInfo(const TEntityInitializationContext& context) { + AFL_VERIFY(!GetTableInfoPtrVerified()->IsStandalone()); + const auto storePathId = GetTableInfoPtrVerified()->GetOlapStorePathIdVerified(); TPath storePath = TPath::Init(storePathId, context.GetSSOperationContext()->SS); { TPath::TChecker checks = storePath.Check(); @@ -28,22 +29,41 @@ TConclusionStatus TInStoreTable::InitializeWithTableInfo(const TColumnTableInfo: schema.ParseFromLocalDB(GetTableSchemaProto().DetachResult()); TableSchema = std::move(schema); - if (TableInfo->Description.HasTtlSettings()) { + if (GetTableInfoPtrVerified()->Description.HasTtlSettings()) { TOlapTTL ttl; - ttl.DeserializeFromProto(TableInfo->Description.GetTtlSettings()).Validate(); + ttl.DeserializeFromProto(GetTableInfoPtrVerified()->Description.GetTtlSettings()).Validate(); TableTTL = std::move(ttl); } return TConclusionStatus::Success(); } -TConclusionStatus TInStoreTable::DoInitialize(const TEntityInitializationContext& context) { - TableInfo = context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(GetPathId()); - return InitializeWithTableInfo(TableInfo, context); +TConclusionStatus TInStoreTable::DoInitializeImpl(const TEntityInitializationContext& context) { + return InitializeWithTableInfo(context); } -NKikimr::TConclusion<std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEntityUpdate>> TInStoreTable::DoCreateUpdate(const TUpdateInitializationContext& context, const std::shared_ptr<ISSEntity>& selfPtr) const { - std::shared_ptr<ISSEntityUpdate> result = std::make_shared<TInStoreSchemaUpdate>(selfPtr); +TConclusion<std::shared_ptr<ISSEntityUpdate>> TInStoreTable::DoCreateUpdateImpl(const TUpdateInitializationContext& context) const { + std::shared_ptr<ISSEntityUpdate> result; + if (context.GetModification()->HasAlterTable()) { + result = std::make_shared<TInStoreSchemaUpdate>(); + } else if (context.GetModification()->HasAlterColumnTable()) { + auto& alter = context.GetModification()->GetAlterColumnTable(); + + if (alter.HasAlterShards()) { + if (alter.GetAlterShards().HasTransfer()) { + result = std::make_shared<TInStoreShardsTransfer>(); + } else if (alter.GetAlterShards().HasModification()) { + result = std::make_shared<TInStoreShardsUpdate>(); + } + } else if (alter.HasReshardColumnTable()) { + result = std::make_shared<TInStoreShardingUpdate>(); + } else if (alter.HasAlterSchema() || alter.HasAlterTtlSettings()) { + result = std::make_shared<TInStoreSchemaUpdate>(); + } + } + if (!result) { + return NKikimr::TConclusionStatus::Fail("Undefined modification type"); + } auto initConclusion = result->Initialize(context); if (initConclusion.IsFail()) { return initConclusion; @@ -51,4 +71,18 @@ NKikimr::TConclusion<std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEn return result; } +NKikimr::TConclusion<NKikimrSchemeOp::TColumnTableSchema> TInStoreTable::GetTableSchemaProto() const { + AFL_VERIFY(!!StoreInfo); + if (!StoreInfo->SchemaPresets.count(GetTableInfoPtrVerified()->Description.GetSchemaPresetId())) { + return TConclusionStatus::Fail("No preset for in-store column table"); + } + + auto& preset = StoreInfo->SchemaPresets.at(GetTableInfoPtrVerified()->Description.GetSchemaPresetId()); + auto& presetProto = StoreInfo->GetDescription().GetSchemaPresets(preset.GetProtoIndex()); + if (!presetProto.HasSchema()) { + return TConclusionStatus::Fail("No schema in preset for in-store column table"); + } + return presetProto.GetSchema(); +} + }
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h index 5c6b9b0c02..e0fb2bd502 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h @@ -1,5 +1,5 @@ #pragma once -#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/common/object.h> #include <ydb/core/tx/schemeshard/olap/table/table.h> #include <ydb/core/tx/schemeshard/olap/store/store.h> #include <ydb/core/tx/schemeshard/olap/schema/schema.h> @@ -7,58 +7,32 @@ namespace NKikimr::NSchemeShard::NOlap::NAlter { -class TInStoreTable: public ISSEntity { +class TInStoreTable: public TColumnTableEntity { private: - using TBase = ISSEntity; - YDB_READONLY_DEF(TColumnTableInfo::TPtr, TableInfo); + using TBase = TColumnTableEntity; YDB_READONLY_DEF(TOlapStoreInfo::TPtr, StoreInfo); std::optional<TOlapSchema> TableSchema; std::optional<TOlapTTL> TableTTL; - virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdate(const TUpdateInitializationContext& context, const std::shared_ptr<ISSEntity>& selfPtr) const override; - virtual TConclusionStatus DoInitialize(const TEntityInitializationContext& context) override; - [[nodiscard]] TConclusionStatus InitializeWithTableInfo(const TColumnTableInfo::TPtr& tInfo, const TEntityInitializationContext& context); + virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdateImpl(const TUpdateInitializationContext& context) const override; + virtual TConclusionStatus DoInitializeImpl(const TEntityInitializationContext& context) override; + [[nodiscard]] TConclusionStatus InitializeWithTableInfo(const TEntityInitializationContext& context); public: virtual TString GetClassName() const override { return "IN_STORE_TABLE"; } - TInStoreTable(const TPathId& pathId) - : TBase(pathId) { + using TBase::TBase; + TInStoreTable(const TPathId& pathId, const std::shared_ptr<TColumnTableInfo>& tableInfo, const TEntityInitializationContext& context) + : TBase(pathId, tableInfo) + { + InitializeWithTableInfo(context).Validate(); } - TInStoreTable(const TPathId& pathId, const TColumnTableInfo::TPtr& tInfo, const TEntityInitializationContext& context) - : TBase(pathId) { - InitializeWithTableInfo(tInfo, context).Validate(); - } - - const TColumnTableInfo& GetTableInfoVerified() const { - AFL_VERIFY(!!TableInfo); - return *TableInfo; - } - - TColumnTableInfo::TPtr GetTableInfoPtrVerified() const { - AFL_VERIFY(!!TableInfo); - return TableInfo; - } - - TConclusion<NKikimrSchemeOp::TColumnTableSchema> GetTableSchemaProto() const { - AFL_VERIFY(!!StoreInfo); - if (!StoreInfo->SchemaPresets.count(TableInfo->Description.GetSchemaPresetId())) { - return TConclusionStatus::Fail("No preset for in-store column table"); - } - - auto& preset = StoreInfo->SchemaPresets.at(TableInfo->Description.GetSchemaPresetId()); - auto& presetProto = StoreInfo->GetDescription().GetSchemaPresets(preset.GetProtoIndex()); - if (!presetProto.HasSchema()) { - return TConclusionStatus::Fail("No schema in preset for in-store column table"); - } - return presetProto.GetSchema(); - } + TConclusion<NKikimrSchemeOp::TColumnTableSchema> GetTableSchemaProto() const; const NKikimrSchemeOp::TColumnDataLifeCycle& GetTableTTLProto() { - AFL_VERIFY(TableInfo); - return TableInfo->Description.GetTtlSettings(); + return GetTableInfoPtrVerified()->Description.GetTtlSettings(); } const TOlapSchema& GetTableSchemaVerified() const { diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.cpp new file mode 100644 index 0000000000..1aef72056e --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.cpp @@ -0,0 +1,62 @@ +#include "update.h" +#include <ydb/core/tx/columnshard/bg_tasks/abstract/task.h> +#include <ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/task.h> +#include <ydb/core/tx/schemeshard/schemeshard_impl.h> +#include <ydb/core/tx/sharding/sharding.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +TConclusionStatus TInStoreShardingUpdate::DoStart(const TUpdateStartContext& context) { + NKikimr::NOlap::NBackground::TTask task("SPLIT_SHARDS::" + context.GetObjectPath()->PathString(), std::make_shared<NKikimr::NOlap::NBackground::TFakeStatusChannel>(), + std::make_shared<NOlap::NBackground::TTxChainTask>(TxChainData)); + auto tx = context.GetSSOperationContext()->SS->BackgroundSessionsManager->TxAddTask(task); + if (!tx->Execute(context.GetSSOperationContext()->GetTxc(), context.GetSSOperationContext()->Ctx)) { + return TConclusionStatus::Fail("cannot execute transaction for write task"); + } + tx->Complete(context.GetSSOperationContext()->Ctx); + return TConclusionStatus::Success(); +} + +TConclusionStatus TInStoreShardingUpdate::DoInitialize(const TUpdateInitializationContext& context) { + auto& inStoreTable = context.GetOriginalEntityAsVerified<TInStoreTable>(); + AFL_VERIFY(context.GetModification()->GetAlterColumnTable().HasReshardColumnTable()); + std::shared_ptr<NSharding::IShardingBase> sharding = inStoreTable.GetTableInfoPtrVerified()->GetShardingVerified(inStoreTable.GetTableSchemaVerified()); + TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> alters = std::vector<NKikimrSchemeOp::TAlterShards>(); + auto& storeInfo = *inStoreTable.GetStoreInfo(); + auto layoutPolicy = storeInfo.GetTablesLayoutPolicy(); + auto currentLayout = context.GetSSOperationContext()->SS->ColumnTables.GetTablesLayout(TColumnTablesLayout::ShardIdxToTabletId( + storeInfo.GetColumnShards(), *context.GetSSOperationContext()->SS)); + auto tablePtr = context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(context.GetOriginalEntity().GetPathId()); + if (context.GetModification()->GetAlterColumnTable().GetReshardColumnTable().GetIncrease()) { + const ui32 shardsCount = inStoreTable.GetTableInfoPtrVerified()->GetColumnShards().size(); + auto layoutConclusion = layoutPolicy->Layout(currentLayout, shardsCount); + if (layoutConclusion.IsFail()) { + return layoutConclusion; + } + alters = sharding->BuildAddShardsModifiers(layoutConclusion->GetTabletIds()); + } else { + if (inStoreTable.GetTableInfoPtrVerified()->GetColumnShards().size() % 2) { + return TConclusionStatus::Fail("cannot reduce shards count (possible for even shards count only)"); + } + const ui32 newShardsCount = inStoreTable.GetTableInfoPtrVerified()->GetColumnShards().size() / 2; + auto layoutConclusion = layoutPolicy->Layout(currentLayout, newShardsCount); + if (layoutConclusion.IsFail()) { + return layoutConclusion; + } + alters = sharding->BuildReduceShardsModifiers(layoutConclusion->GetTabletIds()); + } + if (alters.IsFail()) { + return alters; + } + for (auto&& i : *alters) { + NKikimrSchemeOp::TModifyScheme modification; + modification.SetWorkingDir(context.GetModification()->GetWorkingDir()); + modification.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterColumnTable); + *modification.MutableAlterColumnTable()->MutableAlterShards() = std::move(i); + modification.MutableAlterColumnTable()->SetName(context.GetModification()->GetAlterColumnTable().GetName()); + TxChainData.MutableTransactions().emplace_back(std::move(modification)); + } + return TConclusionStatus::Success(); +} + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.h new file mode 100644 index 0000000000..8b78c3ddb5 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.h @@ -0,0 +1,36 @@ +#pragma once +#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h> +#include <ydb/core/tx/schemeshard/olap/ttl/update.h> +#include <ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/common.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +class TInStoreShardingUpdate: public ISSEntityUpdate { +private: + using TBase = ISSEntityUpdate; + NOlap::NBackground::TTxChainData TxChainData; + virtual TString DoGetShardTxBodyString(const ui64 /*tabletId*/, const TMessageSeqNo& /*seqNo*/) const override { + AFL_VERIFY(false); + return ""; + } + + virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override { + return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SHARING; + } + + virtual TConclusionStatus DoFinish(const TUpdateFinishContext& /*context*/) override { + return TConclusionStatus::Success(); + } + + virtual TConclusionStatus DoInitialize(const TUpdateInitializationContext& context) override; + virtual TConclusionStatus DoStart(const TUpdateStartContext& context) override; + + virtual std::set<ui64> DoGetShardIds() const override { + return {}; + } + +public: +}; + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/ya.make new file mode 100644 index 0000000000..87d19eb784 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +SRCS( + update.cpp +) + +PEERDIR( + ydb/core/tx/schemeshard/olap/operations/alter/abstract + ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.cpp new file mode 100644 index 0000000000..bbf1845ac1 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.cpp @@ -0,0 +1,69 @@ +#include "update.h" +#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +NKikimr::TConclusionStatus TInStoreSchemaUpdate::DoInitializeImpl(const TUpdateInitializationContext& context) { + auto alter = TConverterModifyToAlter().Convert(*context.GetModification()); + if (alter.IsFail()) { + return alter; + } + auto alterCS = alter.DetachResult(); + if (alterCS.HasAlterSchema()) { + return TConclusionStatus::Fail("cannot modify scheme for table in store"); + } + if (!alterCS.HasAlterTtlSettings()) { + return TConclusionStatus::Fail("no data for update"); + } + AlterTTL.emplace(alterCS.GetAlterTtlSettings()); + const auto& originalTable = context.GetOriginalEntityAsVerified<TInStoreTable>(); + auto description = originalTable.GetTableInfoVerified().Description; + + { + const auto& originalTableInfo = originalTable.GetTableInfoVerified(); + const auto& storeInfo = originalTable.GetStoreInfo(); + AFL_VERIFY(!!storeInfo)("problem", "Unexpected schema preset without olap store"); + AFL_VERIFY(originalTableInfo.Description.HasSchemaPresetId()); + const ui32 presetId = originalTableInfo.Description.GetSchemaPresetId(); + AFL_VERIFY(storeInfo->SchemaPresets.contains(presetId))("problem", "Failed to find schema preset in an olap store")("id", presetId); + auto& preset = storeInfo->SchemaPresets.at(presetId); + size_t presetIndex = preset.GetProtoIndex(); + SchemaPreset = storeInfo->GetDescription().GetSchemaPresets(presetIndex); + } + + if (AlterTTL) { + TOlapSchema originalSchema; + originalSchema.ParseFromLocalDB(SchemaPreset->GetSchema()); + + auto ttl = originalTable.GetTableTTLOptional() ? *originalTable.GetTableTTLOptional() : TOlapTTL(); + auto patch = ttl.Update(*AlterTTL); + if (patch.IsFail()) { + return patch; + } + TSimpleErrorCollector collector; + if (!originalSchema.ValidateTtlSettings(ttl.GetData(), collector)) { + return TConclusionStatus::Fail("ttl update error: " + collector->GetErrorMessage() + ". in alter constructor STANDALONE_UPDATE"); + } + *description.MutableTtlSettings() = ttl.SerializeToProto(); + } + + auto targetInfo = std::make_shared<TColumnTableInfo>(context.GetOriginalEntityAsVerified<TInStoreTable>().GetTableInfoVerified().AlterVersion + 1, + std::move(description), TMaybe<NKikimrSchemeOp::TColumnStoreSharding>(), std::move(alterCS)); + TEntityInitializationContext eContext(context.GetSSOperationContext()); + TargetInStoreTable = std::make_shared<TInStoreTable>(context.GetOriginalEntity().GetPathId(), targetInfo, eContext); + return TConclusionStatus::Success(); +} + +void TInStoreSchemaUpdate::FillToShardTx(NKikimrTxColumnShard::TAlterTable& shardAlter) const { + auto& alterBody = TargetInStoreTable->GetTableInfoVerified(); + *shardAlter.MutableTtlSettings() = alterBody.Description.GetTtlSettings(); + + AFL_VERIFY(!!SchemaPreset); + *shardAlter.MutableSchemaPreset() = *SchemaPreset; + + if (alterBody.Description.HasSchemaPresetVersionAdj()) { + shardAlter.SetSchemaPresetVersionAdj(alterBody.Description.GetSchemaPresetVersionAdj()); + } +} + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.h new file mode 100644 index 0000000000..4ec15b86ac --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.h @@ -0,0 +1,50 @@ +#pragma once +#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h> +#include <ydb/core/protos/flat_scheme_op.pb.h> +#include <ydb/core/tx/schemeshard/olap/ttl/update.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +class TInStoreSchemaUpdate: public TInStoreTableUpdate { +private: + using TBase = TInStoreTableUpdate; + std::optional<TOlapTTLUpdate> AlterTTL; + std::optional<NKikimrSchemeOp::TColumnTableSchemaPreset> SchemaPreset; + std::shared_ptr<TInStoreTable> TargetInStoreTable; + virtual TConclusionStatus DoInitializeImpl(const TUpdateInitializationContext& context) override; + + virtual std::shared_ptr<ISSEntity> GetTargetSSEntity() const override { + return TargetInStoreTable; + } + + virtual std::shared_ptr<TColumnTableInfo> GetTargetTableInfo() const override { + return TargetInStoreTable->GetTableInfoPtrVerified(); + } + + virtual TString DoGetShardTxBodyString(const ui64 /*tabletId*/, const TMessageSeqNo& seqNo) const override { + NKikimrTxColumnShard::TSchemaTxBody result; + result.MutableSeqNo()->SetGeneration(seqNo.Generation); + result.MutableSeqNo()->SetRound(seqNo.Round); + + auto& alter = *result.MutableAlterTable(); + FillToShardTx(alter); + alter.SetPathId(TargetInStoreTable->GetPathId().LocalPathId); + return result.SerializeAsString(); + } + + void FillToShardTx(NKikimrTxColumnShard::TAlterTable& shardAlter) const; + + virtual std::set<ui64> DoGetShardIds() const override { + return TargetInStoreTable->GetTableInfoVerified().GetShardIdsSet(); + } + +public: + + virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override { + return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SCHEMA; + } +}; + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/ya.make new file mode 100644 index 0000000000..1c221f9bf5 --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( + update.cpp +) + +PEERDIR( + ydb/core/tx/schemeshard/olap/operations/alter/abstract +) + +END() diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.cpp new file mode 100644 index 0000000000..73bbd10a3d --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.cpp @@ -0,0 +1,44 @@ +#include "update.h" +#include <ydb/core/tx/columnshard/data_sharing/initiator/controller/schemeshard.h> +#include <ydb/core/tx/schemeshard/schemeshard_impl.h> +#include <ydb/core/tx/columnshard/common/snapshot.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +NKikimr::TConclusionStatus TInStoreShardsTransfer::DoInitializeImpl(const TUpdateInitializationContext& context) { + if (!context.GetModification()->GetAlterColumnTable().GetAlterShards().GetTransfer().GetTransfers().size()) { + return TConclusionStatus::Fail("hasn't data about shards transfer"); + } + auto& table = context.GetOriginalEntityAsVerified<TInStoreTable>(); + auto sharding = table.GetTableInfo()->GetShardingVerified(table.GetTableSchemaVerified()); + for (auto&& alter : context.GetModification()->GetAlterColumnTable().GetAlterShards().GetTransfer().GetTransfers()) { + NKikimrColumnShardDataSharingProto::TDestinationSession destinationSession; + destinationSession.SetSessionId("SHARE_TO_SHARD::" + ::ToString(alter.GetDestinationTabletId())); + *destinationSession.MutableInitiatorController() = NKikimr::NOlap::NDataSharing::TInitiatorControllerContainer( + std::make_shared<NKikimr::NOlap::NDataSharing::TSSInitiatorController>(context.GetSSOperationContext()->SS->TabletID(), 0)).SerializeToProto(); + { + auto& pathIdRemap = *destinationSession.AddPathIds(); + pathIdRemap.SetSourcePathId(context.GetOriginalEntity().GetPathId().LocalPathId); + pathIdRemap.SetDestPathId(context.GetOriginalEntity().GetPathId().LocalPathId); + } + ::NKikimr::NOlap::TSnapshot ssOpen = sharding->GetShardingOpenSnapshotVerified(alter.GetDestinationTabletId()); + + destinationSession.MutableTransferContext()->SetDestinationTabletId(alter.GetDestinationTabletId()); + destinationSession.MutableTransferContext()->SetTxId(context.GetTxId()); + *destinationSession.MutableTransferContext()->MutableSnapshotBarrier() = ssOpen.SerializeToProto(); + for (auto&& i : alter.GetSourceTabletIds()) { + destinationSession.MutableTransferContext()->AddSourceTabletIds(i); + } + DestinationSessions.emplace_back(destinationSession); + AFL_VERIFY(ShardIdsUsage.emplace(alter.GetDestinationTabletId()).second); + } + const auto& inStoreOriginal = context.GetOriginalEntityAsVerified<TInStoreTable>(); + auto targetInfo = std::make_shared<TColumnTableInfo>(inStoreOriginal.GetTableInfoVerified().AlterVersion, + inStoreOriginal.GetTableInfoVerified().Description, TMaybe<NKikimrSchemeOp::TColumnStoreSharding>(), context.GetModification()->GetAlterColumnTable()); + TEntityInitializationContext eContext(context.GetSSOperationContext()); + TargetInStoreTable = std::make_shared<TInStoreTable>(context.GetOriginalEntity().GetPathId(), targetInfo, eContext); + + return TConclusionStatus::Success(); +} + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.h new file mode 100644 index 0000000000..5b19aa69fb --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.h @@ -0,0 +1,48 @@ +#pragma once +#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h> +#include <ydb/core/tx/columnshard/data_sharing/protos/sessions.pb.h> +#include <ydb/core/tx/schemeshard/olap/ttl/update.h> + +namespace NKikimr::NSchemeShard::NOlap::NAlter { + +class TInStoreShardsTransfer: public TInStoreTableUpdate { +private: + using TBase = TInStoreTableUpdate; + std::vector<NKikimrColumnShardDataSharingProto::TDestinationSession> DestinationSessions; + std::shared_ptr<TInStoreTable> TargetInStoreTable; + std::set<ui64> ShardIdsUsage; + + virtual std::shared_ptr<TColumnTableInfo> GetTargetTableInfo() const override { + AFL_VERIFY(TargetInStoreTable); + return TargetInStoreTable->GetTableInfoPtrVerified(); + } + virtual std::shared_ptr<ISSEntity> GetTargetSSEntity() const override { + return TargetInStoreTable; + } + + virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override { + return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SHARING; + } + + virtual TConclusionStatus DoInitializeImpl(const TUpdateInitializationContext& context) override; + + virtual TString DoGetShardTxBodyString(const ui64 tabletId, const TMessageSeqNo& /*seqNo*/) const override { + for (auto&& i : DestinationSessions) { + if (i.GetTransferContext().GetDestinationTabletId() == tabletId) { + return i.SerializeAsString(); + } + } + AFL_VERIFY(false); + return ""; + } + + virtual std::set<ui64> DoGetShardIds() const override { + return ShardIdsUsage; + } + +public: +}; + +}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/ya.make new file mode 100644 index 0000000000..8ad440b4ea --- /dev/null +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +SRCS( + update.cpp +) + +PEERDIR( + ydb/core/tx/schemeshard/olap/operations/alter/abstract + ydb/core/tx/columnshard/data_sharing/protos +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.cpp deleted file mode 100644 index e10fdb0b5f..0000000000 --- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.cpp +++ /dev/null @@ -1,94 +0,0 @@ -#include "evolution.h" -#include "update.h" -#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h> - -namespace NKikimr::NSchemeShard::NOlap::NAlter { - -NKikimr::TConclusionStatus TInStoreSchemaUpdate::DoInitialize(const TUpdateInitializationContext& context) { - auto alter = TConverterModifyToAlter().Convert(*context.GetModification()); - if (alter.IsFail()) { - return alter; - } - auto alterCS = alter.DetachResult(); - { - auto conclusion = InitializeDiff(alterCS); - if (conclusion.IsFail()) { - return conclusion; - } - } - auto description = OriginalInStoreTable->GetTableInfoVerified().Description; - { - auto ttlConclusion = GetTtlPatched(); - if (ttlConclusion.IsFail()) { - return ttlConclusion; - } - *description.MutableTtlSettings() = ttlConclusion->SerializeToProto(); - } - - auto targetInfo = std::make_shared<TColumnTableInfo>(OriginalInStoreTable->GetTableInfoVerified().AlterVersion + 1, std::move(description), TMaybe<NKikimrSchemeOp::TColumnStoreSharding>(), std::move(alterCS)); - TEntityInitializationContext eContext(context.GetSSOperationContext()); - TargetInStoreTable = std::make_shared<TInStoreTable>(GetOriginalEntity()->GetPathId(), targetInfo, eContext); - - return TConclusionStatus::Success(); -} - -NKikimr::TConclusion<std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEntityEvolution>> TInStoreSchemaUpdate::DoBuildEvolution(const std::shared_ptr<ISSEntityUpdate>& updateSelfPtr) const { - auto update = dynamic_pointer_cast<TInStoreSchemaUpdate>(updateSelfPtr); - AFL_VERIFY(!!update); - return std::make_shared<TInStoreSchemaEvolution>(OriginalInStoreTable, TargetInStoreTable, update); -} - -NKikimr::TConclusionStatus TInStoreSchemaUpdate::InitializeDiff(const NKikimrSchemeOp::TAlterColumnTable& alterCS) { - if (alterCS.HasAlterSchema()) { - return TConclusionStatus::Fail("cannot modify scheme for table in store"); - } - AlterTTL = alterCS.GetAlterTtlSettings(); - if (!AlterTTL) { - return TConclusionStatus::Fail("no data for update"); - } - auto ttlConclusion = GetTtlPatched(); - if (ttlConclusion.IsFail()) { - return ttlConclusion; - } - return TConclusionStatus::Success(); -} - -NKikimr::TConclusion<NKikimr::NSchemeShard::NOlap::NAlter::TOlapTTL> TInStoreSchemaUpdate::GetTtlPatched() const { - const TOlapSchema& originalSchema = OriginalInStoreTable->GetTableSchemaVerified(); - TOlapTTL ttl = OriginalInStoreTable->GetTableTTLOptional() ? *OriginalInStoreTable->GetTableTTLOptional() : TOlapTTL(); - auto patch = ttl.Update(*AlterTTL); - if (patch.IsFail()) { - return patch; - } - TSimpleErrorCollector collector; - if (!originalSchema.ValidateTtlSettings(ttl.GetData(), collector)) { - return TConclusionStatus::Fail("ttl update error: " + collector->GetErrorMessage() + ". in alter constructor IN_STORE_TABLE"); - } - return ttl; -} - -void TInStoreSchemaUpdate::FillToShardTx(NKikimrTxColumnShard::TAlterTable& shardAlter, const std::shared_ptr<TInStoreTable>& target) const { - if (AlterTTL) { - *shardAlter.MutableTtlSettings() = target->GetTableTTLProto(); - } - auto& alterBody = OriginalInStoreTable->GetTableInfoVerified(); - - AFL_VERIFY(alterBody.Description.HasSchemaPresetId()); - const ui32 presetId = alterBody.Description.GetSchemaPresetId(); - Y_ABORT_UNLESS(!!TargetInStoreTable->GetStoreInfo(), - "Unexpected schema preset without olap store"); - Y_ABORT_UNLESS(TargetInStoreTable->GetStoreInfo()->SchemaPresets.contains(presetId), - "Failed to find schema preset %" PRIu32 - " in an olap store", - presetId); - auto& preset = TargetInStoreTable->GetStoreInfo()->SchemaPresets.at(presetId); - size_t presetIndex = preset.GetProtoIndex(); - *shardAlter.MutableSchemaPreset() = - TargetInStoreTable->GetStoreInfo()->GetDescription().GetSchemaPresets(presetIndex); - - if (alterBody.Description.HasSchemaPresetVersionAdj()) { - shardAlter.SetSchemaPresetVersionAdj(alterBody.Description.GetSchemaPresetVersionAdj()); - } -} - -}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.h deleted file mode 100644 index 37b6f2c95c..0000000000 --- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.h +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once -#include "object.h" -#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h> -#include <ydb/core/tx/schemeshard/olap/ttl/update.h> - -namespace NKikimr::NSchemeShard::NOlap::NAlter { - -class TInStoreSchemaUpdate: public ISSEntityUpdate { -private: - using TBase = ISSEntityUpdate; - std::optional<TOlapTTLUpdate> AlterTTL; - std::shared_ptr<TInStoreTable> OriginalInStoreTable; - std::shared_ptr<TInStoreTable> TargetInStoreTable; - virtual TConclusionStatus DoInitialize(const TUpdateInitializationContext& context) override; - TConclusionStatus InitializeDiff(const NKikimrSchemeOp::TAlterColumnTable& alterCS); - TConclusion<TOlapTTL> GetTtlPatched() const; -public: - TInStoreSchemaUpdate(const std::shared_ptr<ISSEntity>& originalEntity) - : TBase(originalEntity) { - OriginalInStoreTable = dynamic_pointer_cast<TInStoreTable>(originalEntity); - AFL_VERIFY(OriginalInStoreTable); - } - - TInStoreSchemaUpdate(const std::shared_ptr<ISSEntity>& originalEntity, const std::shared_ptr<ISSEntity>& targetEntity) - : TBase(originalEntity) { - OriginalInStoreTable = dynamic_pointer_cast<TInStoreTable>(originalEntity); - AFL_VERIFY(OriginalInStoreTable); - TargetInStoreTable = dynamic_pointer_cast<TInStoreTable>(targetEntity); - AFL_VERIFY(TargetInStoreTable); - AFL_VERIFY(!!TargetInStoreTable->GetTableInfoVerified().AlterBody); - InitializeDiff(*TargetInStoreTable->GetTableInfoVerified().AlterBody).Validate(); - } - - void FillToShardTx(NKikimrTxColumnShard::TAlterTable& shardAlter, const std::shared_ptr<TInStoreTable>& target) const; - - const TOlapTTLUpdate* GetAlterTTLOptional() const { - return AlterTTL ? &*AlterTTL : nullptr; - } - - virtual TConclusion<std::shared_ptr<ISSEntityEvolution>> DoBuildEvolution(const std::shared_ptr<ISSEntityUpdate>& updateSelfPtr) const override; -}; - -}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/ya.make index 4106bd5a63..6f2b79b6e0 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/ya.make +++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/ya.make @@ -2,12 +2,15 @@ LIBRARY() SRCS( object.cpp - update.cpp - evolution.cpp ) PEERDIR( ydb/core/tx/schemeshard/olap/operations/alter/abstract + ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards + ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding + ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema + ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer + ydb/core/tx/schemeshard/olap/operations/alter/in_store/common ydb/core/tx/schemeshard/olap/ttl ydb/core/tx/schemeshard/olap/table ydb/core/tx/schemeshard/olap/store diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.cpp deleted file mode 100644 index 7dfe50652d..0000000000 --- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "evolution.h" -#include <ydb/core/tx/schemeshard/schemeshard_impl.h> - -namespace NKikimr::NSchemeShard::NOlap::NAlter { - -NKikimr::TConclusionStatus TStandaloneSchemaEvolution::DoInitialize(const TEvolutionInitializationContext& context) { - TColumnTableInfo::TPtr tableInfo = context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(GetPathId()); - if (!tableInfo->IsStandalone()) { - return TConclusionStatus::Fail("incorrect path object - not standalone"); - } - Original = std::make_shared<TStandaloneTable>(GetPathId(), tableInfo); - Target = std::make_shared<TStandaloneTable>(GetPathId(), tableInfo->AlterData); - { - NKikimrSchemeOp::TModifyScheme txSchema; - AFL_VERIFY(!!tableInfo->AlterData->AlterBody); - *txSchema.MutableAlterColumnTable() = *tableInfo->AlterData->AlterBody; - txSchema.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterColumnTable); - Update = std::make_shared<TStandaloneSchemaUpdate>(Original); - TUpdateInitializationContext uContext(context.GetSSOperationContext(), &txSchema); - auto conclusion = Update->Initialize(uContext); - if (conclusion.IsFail()) { - return conclusion; - } - } - return TConclusionStatus::Success(); -} - -NKikimr::TConclusionStatus TStandaloneSchemaEvolution::DoStartEvolution(const TEvolutionStartContext& context) { - auto table = context.GetSSOperationContext()->SS->ColumnTables.TakeVerified(GetPathId()); - table->AlterData = Target->GetTableInfoPtrVerified(); - context.GetSSOperationContext()->SS->PersistColumnTableAlter(*context.GetDB(), GetPathId(), Target->GetTableInfoVerified()); - return TConclusionStatus::Success(); -} - -}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.h b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.h deleted file mode 100644 index 86bbc1e9e0..0000000000 --- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once -#include "object.h" -#include "update.h" -#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.h> - -namespace NKikimr::NSchemeShard::NOlap::NAlter { - -class TStandaloneSchemaEvolution: public ISSEntityEvolution { -private: - using TBase = ISSEntityEvolution; - std::shared_ptr<TStandaloneTable> Original; - std::shared_ptr<TStandaloneTable> Target; - std::shared_ptr<TStandaloneSchemaUpdate> Update; -protected: - virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override { - return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SCHEMA; - } - - virtual NKikimrTxColumnShard::TSchemaTxBody DoGetShardTxBody(const NKikimrTxColumnShard::TSchemaTxBody& original, const ui64 /*tabletId*/) const override { - NKikimrTxColumnShard::TSchemaTxBody copy = original; - - auto& alter = *copy.MutableAlterTable(); - Update->FillToShardTx(alter, Target); - alter.SetPathId(GetPathId().LocalPathId); - return copy; - } - - virtual TConclusionStatus DoInitialize(const TEvolutionInitializationContext& context) override; - - virtual TConclusionStatus DoStartEvolution(const TEvolutionStartContext& context) override; -public: - TStandaloneSchemaEvolution(const TPathId& pathId) - : TBase(pathId) - { - - } - TStandaloneSchemaEvolution(const std::shared_ptr<TStandaloneTable>& original, const std::shared_ptr<TStandaloneTable>& target, const std::shared_ptr<TStandaloneSchemaUpdate>& update) - : TBase(original ? original->GetPathId() : target->GetPathId()) - , Original(original) - , Target(target) - , Update(update) - { - AFL_VERIFY(!!Original); - AFL_VERIFY(!!Target); - AFL_VERIFY(!!Update); - } -}; - -}
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.cpp index e6d8dc3abb..0f4e693274 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.cpp @@ -4,8 +4,14 @@ namespace NKikimr::NSchemeShard::NOlap::NAlter { -NKikimr::TConclusion<std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEntityUpdate>> TStandaloneTable::DoCreateUpdate(const TUpdateInitializationContext& context, const std::shared_ptr<ISSEntity>& selfPtr) const { - auto result = std::make_shared<TStandaloneSchemaUpdate>(selfPtr); +NKikimr::TConclusion<std::shared_ptr<ISSEntityUpdate>> TStandaloneTable::DoCreateUpdateImpl(const TUpdateInitializationContext& context) const { + std::shared_ptr<ISSEntityUpdate> result; + if (context.GetModification()->HasAlterTable() || context.GetModification()->HasAlterColumnTable()) { + result = std::make_shared<TStandaloneSchemaUpdate>(); + } + if (!result) { + return TConclusionStatus::Fail("alter data not found"); + } auto conclusion = result->Initialize(context); if (conclusion.IsFail()) { return conclusion; @@ -13,23 +19,21 @@ NKikimr::TConclusion<std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEn return result; } -NKikimr::TConclusionStatus TStandaloneTable::DoInitialize(const TEntityInitializationContext& context) { - return InitializeFromTableInfo(context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(GetPathId())); +NKikimr::TConclusionStatus TStandaloneTable::DoInitializeImpl(const TEntityInitializationContext& /*context*/) { + return InitializeFromTableInfo(); } -NKikimr::TConclusionStatus TStandaloneTable::InitializeFromTableInfo(const TColumnTableInfo::TPtr& tableInfo) { - AFL_VERIFY(!!tableInfo); - TableInfo = tableInfo; - if (!TableInfo->Description.HasSchema()) { +NKikimr::TConclusionStatus TStandaloneTable::InitializeFromTableInfo() { + if (!GetTableInfoPtrVerified()->Description.HasSchema()) { return TConclusionStatus::Fail("path id object has no schema owned for " + GetPathId().ToString()); } TOlapSchema schema; - schema.ParseFromLocalDB(TableInfo->Description.GetSchema()); + schema.ParseFromLocalDB(GetTableInfoPtrVerified()->Description.GetSchema()); TableSchema = std::move(schema); - if (TableInfo->Description.HasTtlSettings()) { + if (GetTableInfoPtrVerified()->Description.HasTtlSettings()) { TOlapTTL ttl; - ttl.DeserializeFromProto(TableInfo->Description.GetTtlSettings()).Validate(); + ttl.DeserializeFromProto(GetTableInfoPtrVerified()->Description.GetTtlSettings()).Validate(); TableTTL = std::move(ttl); } diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.h b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.h index 929d61590c..bd3c445254 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.h +++ b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.h @@ -1,52 +1,48 @@ #pragma once -#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/common/object.h> #include <ydb/core/tx/schemeshard/olap/table/table.h> #include <ydb/core/tx/schemeshard/olap/ttl/schema.h> namespace NKikimr::NSchemeShard::NOlap::NAlter { -class TStandaloneTable: public ISSEntity { +class TStandaloneTable: public TColumnTableEntity { private: - using TBase = ISSEntity; - TColumnTableInfo::TPtr TableInfo; + using TBase = TColumnTableEntity; std::optional<TOlapSchema> TableSchema; std::optional<TOlapTTL> TableTTL; - virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdate(const TUpdateInitializationContext& context, const std::shared_ptr<ISSEntity>& selfPtr) const override; - virtual TConclusionStatus DoInitialize(const TEntityInitializationContext& context) override; + virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdateImpl(const TUpdateInitializationContext& context) const override; + virtual TConclusionStatus DoInitializeImpl(const TEntityInitializationContext& context) override; + TConclusionStatus InitializeFromTableInfo(); public: TStandaloneTable(const TPathId& pathId) : TBase(pathId) { } - TStandaloneTable(const TPathId& pathId, const TColumnTableInfo::TPtr& tableInfo) - : TBase(pathId) - { - InitializeFromTableInfo(tableInfo).Validate(); - } - TColumnTableInfo::TPtr GetTableInfoPtrVerified() const { - AFL_VERIFY(!!TableInfo); - return TableInfo; + TStandaloneTable(const TPathId& pathId, const std::shared_ptr<TColumnTableInfo>& tableInfo) + : TBase(pathId, tableInfo) + { + InitializeFromTableInfo(); } virtual TString GetClassName() const override { return "STANDALONE_TABLE"; } - const NKikimrSchemeOp::TColumnTableSchema& GetTableSchemaProto() { - AFL_VERIFY(TableInfo); - return TableInfo->Description.GetSchema(); + const NKikimrSchemeOp::TColumnTableSchema& GetTableSchemaProto() const { + AFL_VERIFY(GetTableInfo()); + return GetTableInfo()->Description.GetSchema(); } - const TColumnTableInfo& GetTableInfoVerified() { - AFL_VERIFY(TableInfo); - return *TableInfo; + const TColumnTableInfo& GetTableInfoVerified() const { + AFL_VERIFY(GetTableInfo()); + return *GetTableInfo(); } - const NKikimrSchemeOp::TColumnDataLifeCycle& GetTableTTLProto() { - AFL_VERIFY(TableInfo); - return TableInfo->Description.GetTtlSettings(); + const NKikimrSchemeOp::TColumnDataLifeCycle& GetTableTTLProto() const { + AFL_VERIFY(GetTableInfo()); + return GetTableInfo()->Description.GetTtlSettings(); } const TOlapSchema& GetTableSchemaVerified() const { @@ -58,9 +54,6 @@ public: return TableTTL ? &*TableTTL : nullptr; } - TConclusionStatus InitializeFromTableInfo(const TColumnTableInfo::TPtr& tableInfo); - - }; }
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp index 172d0694b4..b94ff1888a 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp @@ -1,22 +1,11 @@ #include "update.h" -#include "evolution.h" #include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h> #include <ydb/core/tx/schemeshard/olap/common/common.h> namespace NKikimr::NSchemeShard::NOlap::NAlter { -NKikimr::TConclusion<std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEntityEvolution>> TStandaloneSchemaUpdate::DoBuildEvolution(const std::shared_ptr<ISSEntityUpdate>& updateSelfPtr) const { - auto originalInfo = dynamic_pointer_cast<TStandaloneTable>(GetOriginalEntity()); - if (!originalInfo) { - return TConclusionStatus::Fail("incorrect incoming type for patch: " + GetOriginalEntity()->GetClassName() + " in STANDALONE_UPDATE"); - } - - auto updateSelfCast = dynamic_pointer_cast<TStandaloneSchemaUpdate>(updateSelfPtr); - AFL_VERIFY(!!updateSelfCast); - return std::make_shared<TStandaloneSchemaEvolution>(OriginalStandalone, TargetStandalone, updateSelfCast); -} - -NKikimr::TConclusionStatus TStandaloneSchemaUpdate::DoInitialize(const TUpdateInitializationContext& context) { +NKikimr::TConclusionStatus TStandaloneSchemaUpdate::DoInitializeImpl(const TUpdateInitializationContext& context) { + const auto& originalTable = context.GetOriginalEntityAsVerified<TStandaloneTable>(); auto alter = TConverterModifyToAlter().Convert(*context.GetModification()); if (alter.IsFail()) { return alter; @@ -30,13 +19,15 @@ NKikimr::TConclusionStatus TStandaloneSchemaUpdate::DoInitialize(const TUpdateIn } AlterSchema = std::move(schemaUpdate); } - AlterTTL = alterCS.GetAlterTtlSettings(); + if (alterCS.HasAlterTtlSettings()) { + AlterTTL = alterCS.GetAlterTtlSettings(); + } if (!AlterSchema && !AlterTTL) { return TConclusionStatus::Fail("no data for update"); } TOlapSchema originalSchema; - originalSchema.ParseFromLocalDB(OriginalStandalone->GetTableInfoVerified().Description.GetSchema()); + originalSchema.ParseFromLocalDB(originalTable.GetTableInfoVerified().Description.GetSchema()); TSimpleErrorCollector collector; TOlapSchema targetSchema = originalSchema; @@ -45,24 +36,23 @@ NKikimr::TConclusionStatus TStandaloneSchemaUpdate::DoInitialize(const TUpdateIn return TConclusionStatus::Fail("schema update error: " + collector->GetErrorMessage() + ". in alter constructor STANDALONE_UPDATE"); } } - auto description = OriginalStandalone->GetTableInfoVerified().Description; + auto description = originalTable.GetTableInfoVerified().Description; targetSchema.Serialize(*description.MutableSchema()); - std::optional<TOlapTTL> ttl; + auto ttl = originalTable.GetTableTTLOptional() ? *originalTable.GetTableTTLOptional() : TOlapTTL(); if (AlterTTL) { - ttl = OriginalStandalone->GetTableTTLOptional() ? *OriginalStandalone->GetTableTTLOptional() : TOlapTTL(); - auto patch = ttl->Update(*AlterTTL); + auto patch = ttl.Update(*AlterTTL); if (patch.IsFail()) { return patch; } - if (!targetSchema.ValidateTtlSettings(ttl->GetData(), collector)) { - return TConclusionStatus::Fail("ttl update error: " + collector->GetErrorMessage() + ". in alter constructor STANDALONE_UPDATE"); - } - *description.MutableTtlSettings() = ttl->SerializeToProto(); + *description.MutableTtlSettings() = ttl.SerializeToProto(); + } + if (!targetSchema.ValidateTtlSettings(ttl.GetData(), collector)) { + return TConclusionStatus::Fail("ttl update error: " + collector->GetErrorMessage() + ". in alter constructor STANDALONE_UPDATE"); } - auto saSharding = OriginalStandalone->GetTableInfoVerified().GetStandaloneShardingVerified(); + auto saSharding = originalTable.GetTableInfoVerified().GetStandaloneShardingVerified(); - auto targetInfo = std::make_shared<TColumnTableInfo>(OriginalStandalone->GetTableInfoVerified().AlterVersion + 1, std::move(description), std::move(saSharding), alterCS); - TargetStandalone = std::make_shared<TStandaloneTable>(GetOriginalEntity()->GetPathId(), targetInfo); + auto targetInfo = std::make_shared<TColumnTableInfo>(originalTable.GetTableInfoVerified().AlterVersion + 1, std::move(description), std::move(saSharding), alterCS); + TargetStandalone = std::make_shared<TStandaloneTable>(context.GetOriginalEntity().GetPathId(), targetInfo); return TConclusionStatus::Success(); } diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.h index 99168cc11f..3f2b62cc93 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.h +++ b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.h @@ -1,46 +1,56 @@ #pragma once #include "object.h" #include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/common/update.h> #include <ydb/core/tx/schemeshard/olap/schema/update.h> #include <ydb/core/tx/schemeshard/olap/ttl/update.h> namespace NKikimr::NSchemeShard::NOlap::NAlter { -class TStandaloneSchemaUpdate: public ISSEntityUpdate { +class TStandaloneSchemaUpdate: public TColumnTableUpdate { private: using TBase = ISSEntityUpdate; std::optional<TOlapSchemaUpdate> AlterSchema; std::optional<TOlapTTLUpdate> AlterTTL; std::shared_ptr<TStandaloneTable> TargetStandalone; - std::shared_ptr<TStandaloneTable> OriginalStandalone; - virtual TConclusionStatus DoInitialize(const TUpdateInitializationContext& context) override; + virtual TConclusionStatus DoInitializeImpl(const TUpdateInitializationContext& context) override; -public: - TStandaloneSchemaUpdate(const std::shared_ptr<ISSEntity>& original) - : TBase(original) - , OriginalStandalone(dynamic_pointer_cast<TStandaloneTable>(GetOriginalEntity())) - { - AFL_VERIFY(!!OriginalStandalone); + virtual std::shared_ptr<TColumnTableInfo> GetTargetTableInfo() const override { + return TargetStandalone->GetTableInfoPtrVerified(); } - void FillToShardTx(NKikimrTxColumnShard::TAlterTable& shardAlter, const std::shared_ptr<TStandaloneTable>& schema) const { - if (AlterSchema) { - *shardAlter.MutableSchema() = schema->GetTableSchemaProto(); - } - if (AlterTTL) { - *shardAlter.MutableTtlSettings() = schema->GetTableTTLProto(); - } + virtual std::shared_ptr<ISSEntity> GetTargetSSEntity() const override { + return TargetStandalone; + } + + virtual std::set<ui64> DoGetShardIds() const override { + return TargetStandalone->GetTableInfoVerified().GetShardIdsSet(); } - const TOlapSchemaUpdate* GetAlterSchemaOptional() const { - return AlterSchema ? &*AlterSchema : nullptr; + virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override { + return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SCHEMA; } - const TOlapTTLUpdate* GetAlterTTLOptional() const { - return AlterTTL ? &*AlterTTL : nullptr; + virtual TString DoGetShardTxBodyString(const ui64 /*tabletId*/, const TMessageSeqNo& seqNo) const override { + NKikimrTxColumnShard::TSchemaTxBody result; + result.MutableSeqNo()->SetGeneration(seqNo.Generation); + result.MutableSeqNo()->SetRound(seqNo.Round); + + auto& alter = *result.MutableAlterTable(); + FillToShardTx(alter); + alter.SetPathId(TargetStandalone->GetPathId().LocalPathId); + return result.SerializeAsString(); } - virtual TConclusion<std::shared_ptr<ISSEntityEvolution>> DoBuildEvolution(const std::shared_ptr<ISSEntityUpdate>& updateSelfPtr) const override; + void FillToShardTx(NKikimrTxColumnShard::TAlterTable& shardAlter) const { + if (AlterSchema) { + *shardAlter.MutableSchema() = TargetStandalone->GetTableSchemaProto(); + } + if (AlterTTL) { + *shardAlter.MutableTtlSettings() = TargetStandalone->GetTableTTLProto(); + } + } +public: }; }
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/ya.make index be763ef9e5..808037e757 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/ya.make +++ b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/ya.make @@ -3,7 +3,6 @@ LIBRARY() SRCS( object.cpp update.cpp - evolution.cpp ) PEERDIR( diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/ya.make index c4cb68a39c..9408ce658e 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter/ya.make +++ b/ydb/core/tx/schemeshard/olap/operations/alter/ya.make @@ -2,6 +2,7 @@ LIBRARY() PEERDIR( ydb/core/tx/schemeshard/olap/operations/alter/abstract + ydb/core/tx/schemeshard/olap/operations/alter/common ydb/core/tx/schemeshard/olap/operations/alter/in_store ydb/core/tx/schemeshard/olap/operations/alter/standalone ) diff --git a/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp b/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp index cd1eb3f973..fa7eb0f7b1 100644 --- a/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp @@ -1,6 +1,5 @@ #include "alter/abstract/object.h" #include "alter/abstract/update.h" -#include "alter/abstract/evolution.h" #include <ydb/core/tx/schemeshard/schemeshard__operation_part.h> #include <ydb/core/tx/schemeshard/schemeshard__operation_common.h> #include <ydb/core/tx/schemeshard/schemeshard_impl.h> @@ -44,34 +43,27 @@ public: TPath path = TPath::Init(pathId, context.SS); TString pathString = path.PathString(); - auto tableInfo = context.SS->ColumnTables.GetVerifiedPtr(pathId); - std::shared_ptr<ISSEntityEvolution> evolution; - { - TEvolutionInitializationContext eContext(&context); - evolution = tableInfo->BuildEvolution(pathId, eContext).DetachResult(); - } - TColumnTableInfo::TPtr alterData = tableInfo->AlterData; - Y_ABORT_UNLESS(alterData); + std::shared_ptr<ISSEntity> originalEntity = ISSEntity::GetEntityVerified(context, path); + TUpdateRestoreContext urContext(originalEntity.get(), &context, (ui64)OperationId.GetTxId()); + std::shared_ptr<ISSEntityUpdate> update = originalEntity->RestoreUpdateVerified(urContext); TSimpleErrorCollector errors; TEntityInitializationContext iContext(&context); txState->ClearShardsInProgress(); - NKikimrTxColumnShard::TSchemaTxBody tx; auto seqNo = context.SS->StartRound(*txState); - context.SS->FillSeqNo(tx, seqNo); for (auto& shard : txState->Shards) { const TTabletId tabletId = context.SS->ShardInfos[shard.Idx].TabletID; AFL_VERIFY(shard.TabletType == ETabletType::ColumnShard); - auto txShard = evolution->GetShardTxBody(tx, (ui64)tabletId); + auto txShardString = update->GetShardTxBodyString((ui64)tabletId, seqNo); auto event = std::make_unique<TEvColumnShard::TEvProposeTransaction>( - evolution->GetShardTransactionKind(), + update->GetShardTransactionKind(), context.SS->TabletID(), context.Ctx.SelfID, ui64(OperationId.GetTxId()), - txShard.SerializeAsString(), - context.SS->SelectProcessingParams(txState->TargetPathId)); + txShardString, + context.SS->SelectProcessingParams(txState->TargetPathId), seqNo); context.OnComplete.BindMsgToPipe(OperationId, tabletId, shard.Idx, event.release()); @@ -118,12 +110,16 @@ public: TPathId pathId = txState->TargetPathId; TPathElement::TPtr path = context.SS->PathsById.at(pathId); + TPath objPath = TPath::Init(pathId, context.SS); NIceDb::TNiceDb db(context.GetDB()); - auto tableInfo = context.SS->ColumnTables.TakeAlterVerified(pathId); - context.SS->PersistColumnTableAlterRemove(db, pathId); - context.SS->PersistColumnTable(db, pathId, *tableInfo); + std::shared_ptr<ISSEntity> originalEntity = ISSEntity::GetEntityVerified(context, objPath); + TUpdateRestoreContext urContext(originalEntity.get(), &context, (ui64)OperationId.GetTxId()); + std::shared_ptr<ISSEntityUpdate> update = originalEntity->RestoreUpdateVerified(urContext); + + TUpdateFinishContext fContext(&objPath, &context, &db, NKikimr::NOlap::TSnapshot(ev->Get()->StepId, ev->Get()->TxId)); + update->Finish(fContext).Validate(); auto parentDir = context.SS->PathsById.at(path->ParentPathId); if (parentDir->IsLikeDirectory()) { @@ -184,7 +180,7 @@ public: } bool HandleReply(TEvColumnShard::TEvNotifyTxCompletionResult::TPtr& ev, TOperationContext& context) override { - TTxState* txState = context.SS->FindTxSafe(OperationId, TTxState::TxAlterColumnTable); + TTxState* txState = context.SS->FindTxSafe(OperationId, TTxState::TxAlterColumnTable); auto shardId = TTabletId(ev->Get()->Record.GetOrigin()); auto shardIdx = context.SS->MustGetShardIdx(shardId); Y_ABORT_UNLESS(context.SS->ShardInfos.contains(shardIdx)); @@ -295,32 +291,14 @@ public: } } - auto tableInfo = context.SS->ColumnTables.GetVerifiedPtr(path.Base()->PathId); - - if (tableInfo->AlterVersion == 0) { - result->SetError(NKikimrScheme::StatusMultipleModifications, "Table is not created yet"); - return result; - } - if (tableInfo->AlterData) { - result->SetError(NKikimrScheme::StatusMultipleModifications, "There's another Alter in flight"); - return result; - } TProposeErrorCollector errors(*result); - std::shared_ptr<ISSEntity> originalEntity; - { - TEntityInitializationContext iContext(&context); - auto conclusion = tableInfo->BuildEntity(path.Base()->PathId, iContext); - if (conclusion.IsFail()) { - errors.AddError(conclusion.GetErrorMessage()); - return result; - } - originalEntity = conclusion.DetachResult(); - } + TEntityInitializationContext iContext(&context); + std::shared_ptr<ISSEntity> originalEntity = ISSEntity::GetEntityVerified(context, path); std::shared_ptr<ISSEntityUpdate> update; { - TUpdateInitializationContext uContext(&context, &Transaction); - TConclusion<std::shared_ptr<ISSEntityUpdate>> conclusion = originalEntity->CreateUpdate(uContext, originalEntity); + TUpdateInitializationContext uContext(&*originalEntity, &context, &Transaction, (ui64)OperationId.GetTxId()); + TConclusion<std::shared_ptr<ISSEntityUpdate>> conclusion = originalEntity->CreateUpdate(uContext); if (conclusion.IsFail()) { errors.AddError(conclusion.GetErrorMessage()); return result; @@ -328,16 +306,6 @@ public: update = conclusion.DetachResult(); } - std::shared_ptr<ISSEntityEvolution> evolution; - { - auto conclusion = update->BuildEvolution(update); - if (conclusion.IsFail()) { - errors.AddError(conclusion.GetErrorMessage()); - return result; - } - evolution = conclusion.DetachResult(); - } - TString errStr; if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); @@ -346,39 +314,61 @@ public: NIceDb::TNiceDb db(context.GetDB()); - TTxState& txState = context.SS->CreateTx(OperationId, TTxState::TxAlterColumnTable, path->PathId); - txState.State = TTxState::ConfigureParts; + if (update->GetShardIds().size()) { + TTxState& txState = context.SS->CreateTx(OperationId, TTxState::TxAlterColumnTable, path->PathId); + txState.State = TTxState::ConfigureParts; - // TODO: we need to know all shards where this table is currently active - for (ui64 columnShardId : tableInfo->GetColumnShards()) { - auto tabletId = TTabletId(columnShardId); - auto shardIdx = context.SS->TabletIdToShardIdx.at(tabletId); + for (ui64 columnShardId : update->GetShardIds()) { + auto tabletId = TTabletId(columnShardId); + auto shardIdx = context.SS->TabletIdToShardIdx.at(tabletId); - Y_VERIFY_S(context.SS->ShardInfos.contains(shardIdx), "Unknown shardIdx " << shardIdx); - txState.Shards.emplace_back(shardIdx, context.SS->ShardInfos[shardIdx].TabletType, TTxState::ConfigureParts); + Y_VERIFY_S(context.SS->ShardInfos.contains(shardIdx), "Unknown shardIdx " << shardIdx); + txState.Shards.emplace_back(shardIdx, context.SS->ShardInfos[shardIdx].TabletType, TTxState::ConfigureParts); - context.SS->ShardInfos[shardIdx].CurrentTxId = OperationId.GetTxId(); - context.SS->PersistShardTx(db, shardIdx, OperationId.GetTxId()); - } + context.SS->ShardInfos[shardIdx].CurrentTxId = OperationId.GetTxId(); + context.SS->PersistShardTx(db, shardIdx, OperationId.GetTxId()); + } - path->LastTxId = OperationId.GetTxId(); - path->PathState = TPathElement::EPathState::EPathStateAlter; - context.SS->PersistLastTxId(db, path.Base()); + path->LastTxId = OperationId.GetTxId(); + path->PathState = TPathElement::EPathState::EPathStateAlter; + context.SS->PersistLastTxId(db, path.Base()); - { - TEvolutionStartContext startContext(&path, &context, &db); - auto status = evolution->StartEvolution(startContext); - if (status.IsFail()) { - errors.AddError(status.GetErrorMessage()); - return result; + { + TUpdateStartContext startContext(&path, &context, &db); + auto status = update->Start(startContext); + if (status.IsFail()) { + errors.AddError(status.GetErrorMessage()); + return result; + } + } + context.SS->PersistTxState(db, OperationId); + + context.OnComplete.ActivateTx(OperationId); + + SetState(NextState()); + } else { + { + { + TUpdateStartContext startContext(&path, &context, &db); + auto status = update->Start(startContext); + if (status.IsFail()) { + errors.AddError(status.GetErrorMessage()); + return result; + } + } + { + TUpdateFinishContext fContext(&path, &context, &db, {}); + auto status = update->Finish(fContext); + if (status.IsFail()) { + errors.AddError(status.GetErrorMessage()); + return result; + } + } } + result->SetStatus(NKikimrScheme::StatusSuccess); + SetState(TTxState::Done); } - context.SS->PersistTxState(db, OperationId); - - context.OnComplete.ActivateTx(OperationId); - - SetState(NextState()); return result; } diff --git a/ydb/core/tx/schemeshard/olap/operations/create_table.cpp b/ydb/core/tx/schemeshard/olap/operations/create_table.cpp index b23286ef4d..908474451e 100644 --- a/ydb/core/tx/schemeshard/olap/operations/create_table.cpp +++ b/ydb/core/tx/schemeshard/olap/operations/create_table.cpp @@ -71,7 +71,7 @@ public: } } tableInfo->AlterVersion = 1; - auto shardingValidation = NSharding::TShardingBase::ValidateBehaviour(GetSchema(), tableInfo->Description.GetSharding()); + auto shardingValidation = NSharding::IShardingBase::ValidateBehaviour(GetSchema(), tableInfo->Description.GetSharding()); if (shardingValidation.IsFail()) { errors.AddError(shardingValidation.GetErrorMessage()); return nullptr; @@ -170,7 +170,7 @@ private: for (auto&& i : layoutConclusion->MutableTabletIds()) { description.MutableSharding()->AddColumnShards(i); } - auto shardingObject = NSharding::TShardingBase::BuildFromProto(GetSchema(), description.GetSharding()); + auto shardingObject = NSharding::IShardingBase::BuildFromProto(GetSchema(), description.GetSharding()); if (shardingObject.IsFail()) { return shardingObject; } diff --git a/ydb/core/tx/schemeshard/olap/table/table.cpp b/ydb/core/tx/schemeshard/olap/table/table.cpp index dfb54dcb18..9c90e758e0 100644 --- a/ydb/core/tx/schemeshard/olap/table/table.cpp +++ b/ydb/core/tx/schemeshard/olap/table/table.cpp @@ -1,17 +1,18 @@ #include "table.h" #include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h> -#include <ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.h> -#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.h> +#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h> +#include <ydb/core/tx/columnshard/common/protos/snapshot.pb.h> namespace NKikimr::NSchemeShard { TColumnTableInfo::TColumnTableInfo( ui64 alterVersion, - NKikimrSchemeOp::TColumnTableDescription&& description, + const NKikimrSchemeOp::TColumnTableDescription& description, TMaybe<NKikimrSchemeOp::TColumnStoreSharding>&& standaloneSharding, TMaybe<NKikimrSchemeOp::TAlterColumnTable>&& alterBody) : AlterVersion(alterVersion) - , Description(std::move(description)) + , Description(description) , StandaloneSharding(std::move(standaloneSharding)) , AlterBody(std::move(alterBody)) { @@ -38,18 +39,4 @@ TConclusion<std::shared_ptr<NOlap::NAlter::ISSEntity>> TColumnTableInfo::BuildEn return result; } -TConclusion<std::shared_ptr<NOlap::NAlter::ISSEntityEvolution>> TColumnTableInfo::BuildEvolution(const TPathId& pathId, const NOlap::NAlter::TEvolutionInitializationContext& iContext) const { - std::shared_ptr<NOlap::NAlter::ISSEntityEvolution> result; - if (IsStandalone()) { - result = std::make_shared<NOlap::NAlter::TStandaloneSchemaEvolution>(pathId); - } else { - result = std::make_shared<NOlap::NAlter::TInStoreSchemaEvolution>(pathId); - } - auto initConclusion = result->Initialize(iContext); - if (initConclusion.IsFail()) { - return initConclusion; - } - return result; -} - }
\ No newline at end of file diff --git a/ydb/core/tx/schemeshard/olap/table/table.h b/ydb/core/tx/schemeshard/olap/table/table.h index 5f9b426e81..e37ede6c3e 100644 --- a/ydb/core/tx/schemeshard/olap/table/table.h +++ b/ydb/core/tx/schemeshard/olap/table/table.h @@ -2,6 +2,8 @@ #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/tx/schemeshard/schemeshard_identificators.h> #include <ydb/core/tx/schemeshard/schemeshard_info_types.h> +#include <ydb/core/tx/sharding/sharding.h> +#include <ydb/core/tx/columnshard/common/snapshot.h> namespace NKikimr::NSchemeShard::NOlap::NAlter { class ISSEntity; @@ -24,6 +26,14 @@ public: return PathIdFromPathId(Description.GetColumnStorePathId()); } + std::shared_ptr<NSharding::IShardingBase> GetShardingVerified(const TOlapSchema& olapSchema) const { + return NSharding::IShardingBase::BuildFromProto(olapSchema, Description.GetSharding()).DetachResult(); + } + + std::set<ui64> GetShardIdsSet() const { + return std::set<ui64>(Description.GetSharding().GetColumnShards().begin(), Description.GetSharding().GetColumnShards().end()); + } + const auto& GetColumnShards() const { return Description.GetSharding().GetColumnShards(); } @@ -31,7 +41,6 @@ public: void SetColumnShards(const std::vector<ui64>& columnShards) { AFL_VERIFY(GetColumnShards().empty())("original", Description.DebugString()); AFL_VERIFY(columnShards.size()); - Description.MutableSharding()->SetVersion(1); Description.MutableSharding()->MutableColumnShards()->Clear(); Description.MutableSharding()->MutableColumnShards()->Reserve(columnShards.size()); @@ -47,7 +56,7 @@ public: TAggregatedStats Stats; TColumnTableInfo() = default; - TColumnTableInfo(ui64 alterVersion, NKikimrSchemeOp::TColumnTableDescription&& description, + TColumnTableInfo(ui64 alterVersion, const NKikimrSchemeOp::TColumnTableDescription& description, TMaybe<NKikimrSchemeOp::TColumnStoreSharding>&& standaloneSharding, TMaybe<NKikimrSchemeOp::TAlterColumnTable>&& alterBody = Nothing()); diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index b6f905f163..a9d8f20bfe 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -4067,9 +4067,6 @@ NKikimrSchemeOp::TPathVersion TSchemeShard::GetPathVersion(const TPath& path) co result.SetColumnTableVersion(tableInfo->AlterVersion); generalVersion += result.GetColumnTableVersion(); - result.SetColumnTableShardingVersion(tableInfo->Description.GetSharding().GetVersion()); - generalVersion += result.GetColumnTableShardingVersion(); - if (tableInfo->Description.HasSchema()) { result.SetColumnTableSchemaVersion(tableInfo->Description.GetSchema().GetVersion()); } else if (tableInfo->Description.HasSchemaPresetId() && tableInfo->GetOlapStorePathIdVerified()) { @@ -4428,6 +4425,7 @@ void TSchemeShard::StateConfigure(STFUNC_SIG) { HFuncTraced(TEvents::TEvUndelivered, Handle); HFuncTraced(NKikimr::NOlap::NBackground::TEvExecuteGeneralLocalTransaction, Handle); + HFuncTraced(NKikimr::NOlap::NBackground::TEvRemoveSession, Handle); HFuncTraced(TEvSchemeShard::TEvInitRootShard, Handle); HFuncTraced(TEvSchemeShard::TEvInitTenantSchemeShard, Handle); HFuncTraced(TEvSchemeShard::TEvMigrateSchemeShard, Handle); @@ -4471,6 +4469,7 @@ void TSchemeShard::StateWork(STFUNC_SIG) { HFuncTraced(TEvents::TEvUndelivered, Handle); HFuncTraced(TEvSchemeShard::TEvInitRootShard, Handle); HFuncTraced(NKikimr::NOlap::NBackground::TEvExecuteGeneralLocalTransaction, Handle); + HFuncTraced(NKikimr::NOlap::NBackground::TEvRemoveSession, Handle); HFuncTraced(TEvSchemeShard::TEvMeasureSelfResponseTime, SelfPinger->Handle); HFuncTraced(TEvSchemeShard::TEvWakeupToMeasureSelfResponseTime, SelfPinger->Handle); @@ -5164,6 +5163,12 @@ void TSchemeShard::Handle(TEvSchemeShard::TEvInitRootShard::TPtr& ev, const TAct Execute(CreateTxInitRootCompatibility(ev), ctx); } +void TSchemeShard::Handle(NKikimr::NOlap::NBackground::TEvRemoveSession::TPtr& ev, const TActorContext& ctx) { + auto txRemove = BackgroundSessionsManager->TxRemove(ev->Get()->GetClassName(), ev->Get()->GetIdentifier()); + AFL_VERIFY(!!txRemove); + Execute(txRemove.release(), ctx); +} + void TSchemeShard::Handle(NKikimr::NOlap::NBackground::TEvExecuteGeneralLocalTransaction::TPtr& ev, const TActorContext& ctx) { Execute(ev->Get()->ExtractTransaction().release(), ctx); } diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 82544ed169..1dcd1b994e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -1011,6 +1011,8 @@ public: static void FillTableBoundaries(const TTableInfo::TPtr tableInfo, google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TSplitBoundary>& boundaries); void Handle(NKikimr::NOlap::NBackground::TEvExecuteGeneralLocalTransaction::TPtr& ev, const TActorContext& ctx); + void Handle(NKikimr::NOlap::NBackground::TEvRemoveSession::TPtr& ev, const TActorContext& ctx); + void Handle(TEvSchemeShard::TEvInitRootShard::TPtr &ev, const TActorContext &ctx); void Handle(TEvSchemeShard::TEvInitTenantSchemeShard::TPtr &ev, const TActorContext &ctx); diff --git a/ydb/core/tx/sharding/hash_intervals.cpp b/ydb/core/tx/sharding/hash_intervals.cpp new file mode 100644 index 0000000000..7d972a00af --- /dev/null +++ b/ydb/core/tx/sharding/hash_intervals.cpp @@ -0,0 +1,187 @@ +#include "hash_intervals.h" + +namespace NKikimr::NSharding::NConsistency { + +NKikimr::TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> TConsistencySharding64::DoBuildSplitShardsModifiers(const std::vector<ui64>& newTabletIds) const { + if (newTabletIds.size() != GetOrderedShardIds().size()) { + return TConclusionStatus::Fail("can multiple 2 only for add shards count"); + } + if (!!SpecialShardingInfo) { + return TConclusionStatus::Fail("not unified shards distribution for consistency intervals modification"); + } + const TSpecificShardingInfo shardingInfo = SpecialShardingInfo ? *SpecialShardingInfo : TSpecificShardingInfo(GetOrderedShardIds()); + std::vector<NKikimrSchemeOp::TAlterShards> result; + { + ui32 idx = 0; + for (auto&& i : GetOrderedShardIds()) { + { + NKikimrSchemeOp::TAlterShards alter; + auto specSharding = shardingInfo.GetShardingTabletVerified(i); + specSharding.SetTabletId(newTabletIds[idx]); + specSharding.CutHalfIntervalFromStart(); + alter.MutableModification()->AddOpenWriteIds(newTabletIds[idx]); + *alter.MutableModification()->MutableConsistency()->AddShards() = specSharding.SerializeToProto(); + result.emplace_back(alter); + } + { + NKikimrSchemeOp::TAlterShards alter; + auto& transfer = *alter.MutableTransfer()->AddTransfers(); + transfer.SetDestinationTabletId(newTabletIds[idx]); + transfer.AddSourceTabletIds(i); + result.emplace_back(alter); + } + { + NKikimrSchemeOp::TAlterShards alter; + alter.MutableModification()->AddOpenReadIds(newTabletIds[idx]); + auto specSharding = shardingInfo.GetShardingTabletVerified(i); + specSharding.CutHalfIntervalToEnd(); + *alter.MutableModification()->MutableConsistency()->AddShards() = specSharding.SerializeToProto(); + result.emplace_back(alter); + } + ++idx; + } + } + return result; +} + +NKikimr::TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> TConsistencySharding64::DoBuildMergeShardsModifiers(const std::vector<ui64>& newTabletIds) const { + if (newTabletIds.size() * 2 != GetOrderedShardIds().size()) { + return TConclusionStatus::Fail("can div 2 only for reduce shards count"); + } + if (!!SpecialShardingInfo) { + return TConclusionStatus::Fail("not unified shards distribution for consistency intervals modification"); + } + const TSpecificShardingInfo shardingInfo = SpecialShardingInfo ? *SpecialShardingInfo : TSpecificShardingInfo(GetOrderedShardIds()); + std::vector<NKikimrSchemeOp::TAlterShards> result; + { + ui32 idx = 0; + for (auto&& i : newTabletIds) { + const ui64 from1 = GetOrderedShardIds()[2 * idx]; + const ui64 from2 = GetOrderedShardIds()[2 * idx + 1]; + auto source1 = shardingInfo.GetShardingTabletVerified(from1); + auto source2 = shardingInfo.GetShardingTabletVerified(from2); + AFL_VERIFY(source1.GetHashIntervalRightOpened() == source2.GetHashIntervalLeftClosed()); + { + NKikimrSchemeOp::TAlterShards alter; + TSpecificShardingInfo::TConsistencyShardingTablet newInterval(i, source1.GetHashIntervalLeftClosed(), source2.GetHashIntervalRightOpened()); + alter.MutableModification()->AddOpenWriteIds(i); + *alter.MutableModification()->MutableConsistency()->AddShards() = newInterval.SerializeToProto(); + result.emplace_back(alter); + } + { + NKikimrSchemeOp::TAlterShards alter; + auto& transfer = *alter.MutableTransfer()->AddTransfers(); + transfer.SetDestinationTabletId(newTabletIds[idx]); + transfer.AddSourceTabletIds(from1); + transfer.AddSourceTabletIds(from2); + result.emplace_back(alter); + } + { + NKikimrSchemeOp::TAlterShards alter; + alter.MutableModification()->AddOpenReadIds(i); + alter.MutableModification()->AddCloseWriteIds(from1); + alter.MutableModification()->AddCloseWriteIds(from2); + alter.MutableModification()->AddCloseReadIds(from1); + alter.MutableModification()->AddCloseReadIds(from2); + result.emplace_back(alter); + } + ++idx; + } + } + return result; +} + +NKikimr::TConclusionStatus TConsistencySharding64::DoApplyModification(const NKikimrSchemeOp::TShardingModification& proto) { + if (!proto.HasConsistency()) { + return TConclusionStatus::Success(); + } + + for (auto&& i : proto.GetConsistency().GetShards()) { + TSpecificShardingInfo::TConsistencyShardingTablet info; + { + auto conclusion = info.DeserializeFromProto(i); + if (conclusion.IsFail()) { + return conclusion; + } + } + if (!UpdateShardInfo(info)) { + return TConclusionStatus::Fail("no shard with same id on update consistency interval"); + } + } + return TConclusionStatus::Success(); +} + +NKikimr::TConclusionStatus TConsistencySharding64::DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) { + auto conclusion = TBase::DoDeserializeFromProto(proto); + if (conclusion.IsFail()) { + return conclusion; + } + if (!proto.HasHashSharding()) { + return TConclusionStatus::Fail("no data for consistency sharding"); + } + AFL_VERIFY(proto.GetHashSharding().GetFunction() == NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CONSISTENCY_64); + { + TSpecificShardingInfo specSharding; + auto parseResult = specSharding.DeserializeFromProto(proto, GetClosedWritingShardIds(), GetClosedReadingShardIds()); + if (parseResult.IsFail()) { + return parseResult; + } + if (!specSharding.IsEmpty()) { + SpecialShardingInfo = std::move(specSharding); + } + } + return TConclusionStatus::Success(); +} + +NKikimr::TConclusionStatus TConsistencySharding64::DoOnAfterModification() { + AFL_VERIFY(!!SpecialShardingInfo); + auto result = SpecialShardingInfo->BuildActivityIndex(GetClosedWritingShardIds(), GetClosedReadingShardIds()); + if (result.IsFail()) { + return result; + } + + std::vector<ui64> shardIds; + if (SpecialShardingInfo->CheckUnifiedDistribution(GetOrderedShardIds().size(), shardIds)) { + SetOrderedShardIds(shardIds); + SpecialShardingInfo.reset(); + } + + return TConclusionStatus::Success(); +} + +THashMap<ui64, std::vector<ui32>> TConsistencySharding64::MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const { + std::vector<ui64> hashes = MakeHashes(batch); + + if (!SpecialShardingInfo) { + std::vector<std::vector<ui32>> result; + result.resize(GetShardsCount()); + for (auto&& i : result) { + i.reserve(hashes.size()); + } + ui32 idx = 0; + for (auto&& hash : hashes) { + result[std::min<ui32>(hash / (Max<ui64>() / result.size()), result.size() - 1)].emplace_back(idx); + ++idx; + } + THashMap<ui64, std::vector<ui32>> resultHash; + for (ui32 i = 0; i < result.size(); ++i) { + if (result[i].size()) { + resultHash.emplace(GetShardIdByOrderIdx(i), std::move(result[i])); + } + } + return resultHash; + } else { + return SpecialShardingInfo->MakeShardingWrite(hashes); + } +} + +std::shared_ptr<NKikimr::NSharding::IGranuleShardingLogic> TConsistencySharding64::DoGetTabletShardingInfoOptional(const ui64 tabletId) const { + if (SpecialShardingInfo) { + return std::make_shared<TGranuleSharding>(GetShardingColumns(), SpecialShardingInfo->GetShardingTabletVerified(tabletId)); + } else { + TSpecificShardingInfo info(GetOrderedShardIds()); + return std::make_shared<TGranuleSharding>(GetShardingColumns(), info.GetShardingTabletVerified(tabletId)); + } +} + +} diff --git a/ydb/core/tx/sharding/hash_intervals.h b/ydb/core/tx/sharding/hash_intervals.h new file mode 100644 index 0000000000..0f11e36628 --- /dev/null +++ b/ydb/core/tx/sharding/hash_intervals.h @@ -0,0 +1,385 @@ +#pragma once +#include "hash_sharding.h" +#include <ydb/core/protos/flat_scheme_op.pb.h> + +namespace NKikimr::NSharding::NConsistency { + +class TSpecificShardingInfo { +public: + class TConsistencyShardingTablet { + private: + YDB_ACCESSOR(ui64, TabletId, 0); + YDB_READONLY(ui64, HashIntervalLeftClosed, 0); + YDB_READONLY(ui64, HashIntervalRightOpened, Max<ui64>()); + public: + TConsistencyShardingTablet() = default; + TConsistencyShardingTablet(const ui64 tabletId, const ui64 hashIntervalLeftClosed, const ui64 hashIntervalRightOpened) + : TabletId(tabletId) + , HashIntervalLeftClosed(hashIntervalLeftClosed) + , HashIntervalRightOpened(hashIntervalRightOpened) { + AFL_VERIFY(HashIntervalLeftClosed < HashIntervalRightOpened); + } + + void CutHalfIntervalFromStart() { + const ui64 toHalf = 0.5 * HashIntervalRightOpened; + const ui64 fromHalf = 0.5 * HashIntervalLeftClosed; + HashIntervalRightOpened = toHalf + fromHalf; + } + + void CutHalfIntervalToEnd() { + const ui64 toHalf = 0.5 * HashIntervalRightOpened; + const ui64 fromHalf = 0.5 * HashIntervalLeftClosed; + HashIntervalLeftClosed = toHalf + fromHalf; + } + + NKikimrSchemeOp::TConsistencyShardingTablet SerializeToProto() const { + NKikimrSchemeOp::TConsistencyShardingTablet result; + result.SetTabletId(TabletId); + result.SetHashIntervalLeftClosed(HashIntervalLeftClosed); + result.SetHashIntervalRightOpened(HashIntervalRightOpened); + return result; + } + + TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TConsistencyShardingTablet& proto) { + TabletId = proto.GetTabletId(); + HashIntervalLeftClosed = proto.GetHashIntervalLeftClosed(); + HashIntervalRightOpened = proto.GetHashIntervalRightOpened(); + AFL_VERIFY(HashIntervalLeftClosed < HashIntervalRightOpened); + return TConclusionStatus::Success(); + } + + bool operator<(const TConsistencyShardingTablet& item) const { + if (HashIntervalLeftClosed == item.HashIntervalLeftClosed) { + return HashIntervalRightOpened < item.HashIntervalRightOpened; + } else { + return HashIntervalLeftClosed < item.HashIntervalLeftClosed; + } + } + }; + +private: + bool IndexConstructed = false; + std::vector<TConsistencyShardingTablet> SpecialSharding; + std::vector<TConsistencyShardingTablet*> ActiveWriteSpecialSharding; + std::vector<TConsistencyShardingTablet*> ActiveReadSpecialSharding; + ui64 GetUnifiedDistributionBorder(const ui32 idx, const ui64 shardsCount) const { + AFL_VERIFY(idx <= shardsCount); + if (idx == shardsCount) { + return Max<ui64>(); + } + return Max<ui64>() * (1.0 * idx / shardsCount); + } + + TConclusionStatus CheckIntervalsFilling() const { + { + ui64 currentPos = 0; + for (auto&& i : ActiveReadSpecialSharding) { + if (currentPos < i->GetHashIntervalLeftClosed()) { + return TConclusionStatus::Fail("sharding special intervals not covered (reading) full ui64 line"); + } else if (currentPos > i->GetHashIntervalLeftClosed()) { + return TConclusionStatus::Fail("sharding intervals covered twice for reading full ui64 line"); + } + currentPos = i->GetHashIntervalRightOpened(); + } + if (currentPos != Max<ui64>()) { + return TConclusionStatus::Fail("sharding special intervals not covered (reading) full ui64 line (final segment)"); + } + } + { + ui64 currentPos = 0; + for (auto&& i : ActiveWriteSpecialSharding) { + if (currentPos < i->GetHashIntervalLeftClosed()) { + return TConclusionStatus::Fail("sharding special intervals not covered (writing) full ui64 line"); + } + currentPos = std::max<ui64>(currentPos, i->GetHashIntervalRightOpened()); + } + if (currentPos != Max<ui64>()) { + return TConclusionStatus::Fail("sharding special intervals not covered (writing) full ui64 line (final segment)"); + } + } + return TConclusionStatus::Success(); + } + +public: + bool IsEmpty() const { + return SpecialSharding.empty(); + } + + TSpecificShardingInfo() = default; + + TSpecificShardingInfo(const std::vector<ui64>& shardIds) { + for (ui32 i = 0; i < shardIds.size(); ++i) { + const ui64 start = GetUnifiedDistributionBorder(i, shardIds.size()); + const ui64 finish = GetUnifiedDistributionBorder(i + 1, shardIds.size()); + TConsistencyShardingTablet info(shardIds[i], start, finish); + SpecialSharding.emplace_back(info); + } + BuildActivityIndex(Default<std::set<ui64>>(), Default<std::set<ui64>>()).Validate(); + } + + THashMap<ui64, std::vector<ui32>> MakeShardingWrite(const std::vector<ui64> hashes) const { + AFL_VERIFY(IndexConstructed); + std::vector<std::vector<ui32>> result; + result.resize(ActiveWriteSpecialSharding.size()); + for (auto&& i : result) { + i.reserve(hashes.size()); + } + ui32 idxRecord = 0; + for (auto&& i : hashes) { + ui32 idxShard = 0; + bool found = false; + for (auto&& s : ActiveWriteSpecialSharding) { + if (s->GetHashIntervalLeftClosed() > i || i >= s->GetHashIntervalRightOpened()) { + break; + } + result[idxShard].emplace_back(idxRecord); + found = true; + ++idxShard; + } + AFL_VERIFY(found); + ++idxRecord; + } + THashMap<ui64, std::vector<ui32>> resultHash; + for (ui32 i = 0; i < result.size(); ++i) { + if (result[i].size()) { + resultHash.emplace(ActiveWriteSpecialSharding[i]->GetTabletId(), std::move(result[i])); + } + } + return resultHash; + } + + void SerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const { + AFL_VERIFY(IndexConstructed); + for (auto&& i : SpecialSharding) { + *proto.MutableHashSharding()->AddTabletsForConsistency() = i.SerializeToProto(); + } + } + + TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto, const std::set<ui64>& closedForWrite, const std::set<ui64>& closedForRead) { + for (auto&& i : proto.GetHashSharding().GetTabletsForConsistency()) { + TConsistencyShardingTablet info; + auto conclusion = info.DeserializeFromProto(i); + if (conclusion.IsFail()) { + return conclusion; + } + SpecialSharding.emplace_back(std::move(info)); + } + IndexConstructed = false; + return BuildActivityIndex(closedForWrite, closedForRead); + } + + [[nodiscard]] TConclusionStatus BuildActivityIndex(const std::set<ui64>& closedForWrite, const std::set<ui64>& closedForRead) { + std::sort(SpecialSharding.begin(), SpecialSharding.end()); + ActiveWriteSpecialSharding.clear(); + ActiveReadSpecialSharding.clear(); + if (SpecialSharding.empty()) { + return TConclusionStatus::Success(); + } + for (auto&& i : SpecialSharding) { + if (!closedForWrite.contains(i.GetTabletId())) { + ActiveWriteSpecialSharding.emplace_back(&i); + } + if (!closedForRead.contains(i.GetTabletId())) { + ActiveReadSpecialSharding.emplace_back(&i); + } + } + auto result = CheckIntervalsFilling(); + IndexConstructed = result.IsSuccess(); + return result; + } + + const TConsistencyShardingTablet& GetShardingTabletVerified(const ui64 tabletId) const { + for (auto&& i : SpecialSharding) { + if (i.GetTabletId() == tabletId) { + return i; + } + } + AFL_VERIFY(false); + return Default<TConsistencyShardingTablet>(); + } + + bool CheckUnifiedDistribution(const ui32 originalShardsCount, std::vector<ui64>& orderedShardIds) { + if (originalShardsCount != SpecialSharding.size()) { + return false; + } + AFL_VERIFY(IndexConstructed); + std::set<ui64> activeReadTabletIds; + for (auto&& i : ActiveReadSpecialSharding) { + activeReadTabletIds.emplace(i->GetTabletId()); + } + std::set<ui64> activeWriteTabletIds; + for (auto&& i : ActiveWriteSpecialSharding) { + activeWriteTabletIds.emplace(i->GetTabletId()); + } + std::set<ui64> tabletIds; + for (auto&& i : SpecialSharding) { + tabletIds.emplace(i.GetTabletId()); + } + if (activeReadTabletIds != tabletIds || activeWriteTabletIds != tabletIds) { + return false; + } + ui32 idx = 0; + std::vector<ui64> shardIdsCorrectOrder; + for (auto&& i : SpecialSharding) { + const ui64 start = GetUnifiedDistributionBorder(idx, SpecialSharding.size()); + const ui64 finish = GetUnifiedDistributionBorder(idx + 1, SpecialSharding.size()); + if (i.GetHashIntervalLeftClosed() != start || i.GetHashIntervalRightOpened() != finish) { + return false; + } + shardIdsCorrectOrder.emplace_back(i.GetTabletId()); + ++idx; + } + orderedShardIds = shardIdsCorrectOrder; + return true; + } + + bool UpdateShardInfo(const TConsistencyShardingTablet& info) { + for (auto&& i : SpecialSharding) { + if (i.GetTabletId() == info.GetTabletId()) { + i = info; + IndexConstructed = false; + return true; + } + } + return false; + } + + void AddShardInfo(const TConsistencyShardingTablet& info) { + for (auto&& i : SpecialSharding) { + AFL_VERIFY(i.GetTabletId() != info.GetTabletId()); + } + SpecialSharding.emplace_back(info); + IndexConstructed = false; + } + +}; + +class TConsistencySharding64: public THashShardingImpl { +public: + static TString GetClassNameStatic() { + return "CONSISTENCY"; + } +private: + using TBase = THashShardingImpl; + + std::optional<TSpecificShardingInfo> SpecialShardingInfo; + + virtual TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> DoBuildSplitShardsModifiers(const std::vector<ui64>& newTabletIds) const override; + virtual TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> DoBuildMergeShardsModifiers(const std::vector<ui64>& newTabletIds) const override; + + bool UpdateShardInfo(const TSpecificShardingInfo::TConsistencyShardingTablet& info) { + GetShardInfoVerified(info.GetTabletId()).IncrementVersion(); + AFL_VERIFY(!!SpecialShardingInfo); + if (SpecialShardingInfo->UpdateShardInfo(info)) { + return true; + } + for (auto&& i : GetOrderedShardIds()) { + if (i == info.GetTabletId()) { + SpecialShardingInfo->AddShardInfo(info); + return true; + } + } + return false; + } + + virtual std::shared_ptr<IGranuleShardingLogic> DoGetTabletShardingInfoOptional(const ui64 tabletId) const override; + + virtual TConclusionStatus DoOnBeforeModification() override { + if (!SpecialShardingInfo) { + AFL_VERIFY(!HasReadClosedShards() && !HasWriteClosedShards()); + SpecialShardingInfo = TSpecificShardingInfo(GetOrderedShardIds()); + } + return TConclusionStatus::Success(); + } + + virtual TConclusionStatus DoApplyModification(const NKikimrSchemeOp::TShardingModification& proto) override; + virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override { + TBase::DoSerializeToProto(proto); + if (SpecialShardingInfo) { + SpecialShardingInfo->SerializeToProto(proto); + } + proto.MutableHashSharding()->SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CONSISTENCY_64); + } + virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) override; + + virtual TConclusionStatus DoOnAfterModification() override; + + virtual std::set<ui64> DoGetModifiedShardIds(const NKikimrSchemeOp::TShardingModification& proto) const override { + std::set<ui64> result; + for (auto&& i : proto.GetConsistency().GetShards()) { + result.emplace(i.GetTabletId()); + } + return result; + } +public: + using TBase::TBase; + + TConsistencySharding64() = default; + + TConsistencySharding64(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui64 seed = 0) + : TBase(shardIds, columnNames, seed) { + } + + virtual THashMap<ui64, std::vector<ui32>> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override; + + virtual TString GetClassName() const override { + return GetClassNameStatic(); + } +}; + +class TGranuleSharding: public THashGranuleSharding { +public: + static TString GetClassNameStatic() { + return "CONSISTENCY"; + } +private: + using TBase = THashGranuleSharding; + TSpecificShardingInfo::TConsistencyShardingTablet Interval; + static const inline TFactory::TRegistrator<TGranuleSharding> Registrator = TFactory::TRegistrator<TGranuleSharding>(GetClassNameStatic()); +protected: + virtual std::shared_ptr<NArrow::TColumnFilter> DoGetFilter(const std::shared_ptr<arrow::Table>& table) const override { + const std::vector<ui64> hashes = CalcHashes(table); + auto result = std::make_shared<NArrow::TColumnFilter>(NArrow::TColumnFilter::BuildAllowFilter()); + const auto getter = [&](const ui64 index) { + const ui64 hash = hashes[index]; + return Interval.GetHashIntervalLeftClosed() <= hash && hash < Interval.GetHashIntervalRightOpened(); + }; + result->ResetWithLambda(hashes.size(), getter); + return result; + + } + virtual void DoSerializeToProto(TProto& proto) const override { + *proto.MutableConsistency()->MutableHashing() = TBase::SerializeHashingToProto(); + *proto.MutableConsistency()->MutableShardInfo() = Interval.SerializeToProto(); + } + virtual TConclusionStatus DoDeserializeFromProto(const TProto& proto) override { + { + auto conclusion = TBase::DeserializeHashingFromProto(proto.GetConsistency().GetHashing()); + if (conclusion.IsFail()) { + return conclusion; + } + } + { + auto conclusion = Interval.DeserializeFromProto(proto.GetConsistency().GetShardInfo()); + if (conclusion.IsFail()) { + return conclusion; + } + } + + return TConclusionStatus::Success(); + } +public: + TGranuleSharding() = default; + + TGranuleSharding(const std::vector<TString>& columnNames, const TSpecificShardingInfo::TConsistencyShardingTablet& interval) + : TBase(columnNames) + , Interval(interval) { + + } + + virtual TString GetClassName() const override { + return GetClassNameStatic(); + } +}; + +} diff --git a/ydb/core/tx/sharding/hash_modulo.cpp b/ydb/core/tx/sharding/hash_modulo.cpp new file mode 100644 index 0000000000..09dc844e65 --- /dev/null +++ b/ydb/core/tx/sharding/hash_modulo.cpp @@ -0,0 +1,208 @@ +#include "hash_modulo.h" + +namespace NKikimr::NSharding::NModulo { + +THashMap<ui64, std::vector<ui32>> THashShardingModuloN::MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const { + const std::vector<ui64> hashes = MakeHashes(batch); + if (!SpecialShardingInfo) { + THashMap<ui64, std::vector<ui32>> resultHash; + std::vector<std::vector<ui32>> result; + result.resize(GetShardsCount()); + for (auto&& i : result) { + i.reserve(hashes.size()); + } + ui32 idx = 0; + for (auto&& i : hashes) { + result[i % GetShardsCount()].emplace_back(idx++); + } + for (ui32 i = 0; i < result.size(); ++i) { + if (result[i].size()) { + resultHash[GetOrderedShardIds()[i]] = std::move(result[i]); + } + } + return resultHash; + } else { + return SpecialShardingInfo->MakeShardingWrite(hashes); + } +} + +NKikimr::TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> THashShardingModuloN::DoBuildSplitShardsModifiers(const std::vector<ui64>& newTabletIds) const { + if (newTabletIds.size() != GetOrderedShardIds().size()) { + return TConclusionStatus::Fail("can multiple 2 only for add shards count"); + } + if (!!SpecialShardingInfo) { + return TConclusionStatus::Fail("not unified shards distribution for module"); + } + TSpecificShardingInfo info(GetOrderedShardIds()); + std::vector<NKikimrSchemeOp::TAlterShards> result; + { + NKikimrSchemeOp::TAlterShards alter; + alter.MutableModification()->MutableModulo()->SetPartsCount(2 * GetOrderedShardIds().size()); + for (auto&& i : GetOrderedShardIds()) { + auto specSharding = info.GetShardingTabletVerified(i); + AFL_VERIFY(specSharding.MutableAppropriateMods().size() == 1); + AFL_VERIFY(specSharding.MutableAppropriateMods().emplace(*specSharding.MutableAppropriateMods().begin() + GetOrderedShardIds().size()).second); + *alter.MutableModification()->MutableModulo()->AddShards() = specSharding.SerializeToProto(); + } + + result.emplace_back(std::move(alter)); + } + { + ui32 idx = 0; + for (auto&& i : GetOrderedShardIds()) { + { + NKikimrSchemeOp::TAlterShards alter; + alter.MutableModification()->AddOpenWriteIds(newTabletIds[idx]); + auto specSharding = info.GetShardingTabletVerified(i); + specSharding.SetTabletId(newTabletIds[idx]); + AFL_VERIFY(specSharding.MutableAppropriateMods().size() == 1); + *alter.MutableModification()->MutableModulo()->AddShards() = specSharding.SerializeToProto(); + result.emplace_back(alter); + } + { + NKikimrSchemeOp::TAlterShards alter; + auto& transfer = *alter.MutableTransfer()->AddTransfers(); + transfer.SetDestinationTabletId(newTabletIds[idx]); + transfer.AddSourceTabletIds(i); + result.emplace_back(alter); + } + { + NKikimrSchemeOp::TAlterShards alter; + alter.MutableModification()->AddOpenReadIds(newTabletIds[idx]); + auto specSharding = info.GetShardingTabletVerified(i); + AFL_VERIFY(specSharding.MutableAppropriateMods().size() == 1); + const ui32 original = *specSharding.MutableAppropriateMods().begin(); + specSharding.MutableAppropriateMods().erase(original); + specSharding.MutableAppropriateMods().emplace(original + GetOrderedShardIds().size()); + *alter.MutableModification()->MutableModulo()->AddShards() = specSharding.SerializeToProto(); + result.emplace_back(alter); + } + ++idx; + } + } + return result; +} + +NKikimr::TConclusionStatus THashShardingModuloN::DoApplyModification(const NKikimrSchemeOp::TShardingModification& proto) { + AFL_VERIFY(!!SpecialShardingInfo); + if (!proto.HasModulo()) { + return TConclusionStatus::Success(); + } + + if (proto.GetModulo().HasPartsCount()) { + SpecialShardingInfo->SetPartsCount(proto.GetModulo().GetPartsCount()); + } + + for (auto&& i : proto.GetModulo().GetShards()) { + TSpecificShardingInfo::TModuloShardingTablet info; + { + auto conclusion = info.DeserializeFromProto(i); + if (conclusion.IsFail()) { + return conclusion; + } + } + if (!UpdateShardInfo(info)) { + return TConclusionStatus::Fail("no shard with same id on update modulo"); + } + } + return TConclusionStatus::Success(); +} + +NKikimr::TConclusionStatus THashShardingModuloN::DoOnAfterModification() { + AFL_VERIFY(!!SpecialShardingInfo); + + auto result = SpecialShardingInfo->BuildActivityIndex(GetClosedWritingShardIds(), GetClosedReadingShardIds()); + if (result.IsFail()) { + return result; + } + + std::vector<ui64> shardIdsOrdered; + if (SpecialShardingInfo->CheckUnifiedDistribution(GetOrderedShardIds().size(), shardIdsOrdered)) { + SetOrderedShardIds(shardIdsOrdered); + SpecialShardingInfo.reset(); + } + + return TConclusionStatus::Success(); +} + +NKikimr::TConclusionStatus THashShardingModuloN::DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) { + auto conclusion = TBase::DoDeserializeFromProto(proto); + if (conclusion.IsFail()) { + return conclusion; + } + if (!proto.HasHashSharding()) { + return TConclusionStatus::Fail("no data for modulo n sharding"); + } + AFL_VERIFY(proto.GetHashSharding().GetFunction() == NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_MODULO_N); + { + TSpecificShardingInfo specialInfo; + auto result = specialInfo.DeserializeFromProto(proto, GetClosedWritingShardIds(), GetClosedReadingShardIds()); + if (result.IsFail()) { + return result; + } + if (!specialInfo.IsEmpty()) { + SpecialShardingInfo = std::move(specialInfo); + } + } + return TConclusionStatus::Success(); +} + +std::shared_ptr<NKikimr::NSharding::IGranuleShardingLogic> THashShardingModuloN::DoGetTabletShardingInfoOptional(const ui64 tabletId) const { + if (SpecialShardingInfo) { + return std::make_shared<TGranuleSharding>(GetShardingColumns(), SpecialShardingInfo->GetShardingTabletVerified(tabletId), SpecialShardingInfo->GetPartsCount()); + } else { + TSpecificShardingInfo info(GetOrderedShardIds()); + return std::make_shared<TGranuleSharding>(GetShardingColumns(), info.GetShardingTabletVerified(tabletId), info.GetPartsCount()); + } +} + +NKikimr::TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> THashShardingModuloN::DoBuildMergeShardsModifiers(const std::vector<ui64>& newTabletIds) const { + if (newTabletIds.size() * 2 != GetOrderedShardIds().size()) { + return TConclusionStatus::Fail("can div 2 only for reduce shards count"); + } + if (!!SpecialShardingInfo) { + return TConclusionStatus::Fail("not unified shards distribution for consistency intervals modification"); + } + const TSpecificShardingInfo shardingInfo = SpecialShardingInfo ? *SpecialShardingInfo : TSpecificShardingInfo(GetOrderedShardIds()); + std::vector<NKikimrSchemeOp::TAlterShards> result; + { + ui32 idx = 0; + for (auto&& i : newTabletIds) { + const ui64 from1 = GetOrderedShardIds()[2 * idx]; + const ui64 from2 = GetOrderedShardIds()[2 * idx + 1]; + auto source1 = shardingInfo.GetShardingTabletVerified(from1); + auto source2 = shardingInfo.GetShardingTabletVerified(from2); + AFL_VERIFY(source1.GetAppropriateMods().size() == 1); + AFL_VERIFY(source2.GetAppropriateMods().size() == 1); + AFL_VERIFY(*source1.GetAppropriateMods().begin() + 1 == *source2.GetAppropriateMods().begin()); + { + NKikimrSchemeOp::TAlterShards alter; + TSpecificShardingInfo::TModuloShardingTablet newInterval(i, { *source1.GetAppropriateMods().begin() ,*source2.GetAppropriateMods().begin() }); + alter.MutableModification()->AddOpenWriteIds(i); + *alter.MutableModification()->MutableModulo()->AddShards() = newInterval.SerializeToProto(); + result.emplace_back(alter); + } + { + NKikimrSchemeOp::TAlterShards alter; + auto& transfer = *alter.MutableTransfer()->AddTransfers(); + transfer.SetDestinationTabletId(newTabletIds[idx]); + transfer.AddSourceTabletIds(from1); + transfer.AddSourceTabletIds(from2); + result.emplace_back(alter); + } + { + NKikimrSchemeOp::TAlterShards alter; + alter.MutableModification()->AddOpenReadIds(i); + alter.MutableModification()->AddCloseWriteIds(from1); + alter.MutableModification()->AddCloseWriteIds(from2); + alter.MutableModification()->AddCloseReadIds(from1); + alter.MutableModification()->AddCloseReadIds(from2); + result.emplace_back(alter); + } + ++idx; + } + } + return result; +} + +} diff --git a/ydb/core/tx/sharding/hash_modulo.h b/ydb/core/tx/sharding/hash_modulo.h new file mode 100644 index 0000000000..769c552bdf --- /dev/null +++ b/ydb/core/tx/sharding/hash_modulo.h @@ -0,0 +1,367 @@ +#pragma once +#include "hash_sharding.h" +#include <ydb/core/protos/flat_scheme_op.pb.h> + +namespace NKikimr::NSharding::NModulo { + +class TSpecificShardingInfo { +public: + class TModuloShardingTablet { + private: + YDB_ACCESSOR(ui64, TabletId, 0); + YDB_ACCESSOR_DEF(std::set<ui32>, AppropriateMods); + public: + TModuloShardingTablet() = default; + TModuloShardingTablet(const ui64 tabletId, const std::set<ui32>& mods) + : TabletId(tabletId) + , AppropriateMods(mods) { + + } + + NKikimrSchemeOp::TModuloShardingTablet SerializeToProto() const { + NKikimrSchemeOp::TModuloShardingTablet result; + result.SetTabletId(TabletId); + for (auto&& i : AppropriateMods) { + result.AddAppropriateMods(i); + } + return result; + } + + TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TModuloShardingTablet& proto) { + TabletId = proto.GetTabletId(); + for (auto&& i : proto.GetAppropriateMods()) { + AppropriateMods.emplace(i); + } + return TConclusionStatus::Success(); + } + }; +private: + bool IndexConstructed = false; + YDB_READONLY(ui32, PartsCount, 0); + std::vector<TModuloShardingTablet> SpecialSharding; + std::vector<TModuloShardingTablet*> ActiveWriteSpecialSharding; + std::vector<TModuloShardingTablet*> ActiveReadSpecialSharding; + +public: + TSpecificShardingInfo() = default; + TSpecificShardingInfo(const std::vector<ui64>& shardIds) + : PartsCount(shardIds.size()) + { + for (ui32 i = 0; i < shardIds.size(); ++i) { + TModuloShardingTablet info(shardIds[i], { i }); + SpecialSharding.emplace_back(info); + } + BuildActivityIndex(Default<std::set<ui64>>(), Default<std::set<ui64>>()).Validate(); + } + + void SetPartsCount(const ui32 partsCount) { + PartsCount = partsCount; + } + + THashMap<ui64, std::vector<ui32>> MakeShardingWrite(const std::vector<ui64>& hashes) const { + AFL_VERIFY(PartsCount); + AFL_VERIFY(IndexConstructed); + std::vector<std::vector<ui32>> result; + result.resize(ActiveWriteSpecialSharding.size()); + THashMap<ui64, std::vector<ui32>> resultHash; + std::vector<std::vector<ui32>> shardsByMod; + ui32 idx = 0; + for (auto&& i : ActiveWriteSpecialSharding) { + for (auto&& m : i->GetAppropriateMods()) { + shardsByMod[m].emplace_back(idx); + } + ++idx; + } + ui32 recordIdx = 0; + for (auto&& i : hashes) { + for (auto&& s : shardsByMod[i % PartsCount]) { + result[s].emplace_back(recordIdx); + } + ++recordIdx; + } + for (ui32 i = 0; i < result.size(); ++i) { + if (result[i].size()) { + resultHash[ActiveWriteSpecialSharding[i]->GetTabletId()] = std::move(result[i]); + } + } + return resultHash; + } + + bool CheckUnifiedDistribution(const ui32 summaryShardsCount, std::vector<ui64>& orderedShardIds) { + AFL_VERIFY(IndexConstructed); + if (summaryShardsCount != PartsCount) { + return false; + } + std::set<ui64> activeReadTabletIds; + for (auto&& i : ActiveReadSpecialSharding) { + activeReadTabletIds.emplace(i->GetTabletId()); + } + std::set<ui64> activeWriteTabletIds; + for (auto&& i : ActiveWriteSpecialSharding) { + activeWriteTabletIds.emplace(i->GetTabletId()); + } + std::set<ui64> tabletIds; + for (auto&& i : SpecialSharding) { + tabletIds.emplace(i.GetTabletId()); + } + if (activeReadTabletIds != tabletIds || activeWriteTabletIds != tabletIds) { + return false; + } + + std::map<ui32, ui64> orderedTabletIds; + for (auto&& i : SpecialSharding) { + if (i.GetAppropriateMods().size() != 1) { + return false; + } + for (auto&& m : i.GetAppropriateMods()) { + AFL_VERIFY(m < PartsCount)("m", m)("parts", PartsCount); + if (!orderedTabletIds.emplace(m, i.GetTabletId()).second) { + return false; + } + } + } + orderedShardIds.clear(); + for (auto&& i : orderedTabletIds) { + orderedShardIds.emplace_back(i.second); + } + AFL_VERIFY(orderedTabletIds.size() == summaryShardsCount); + return true; + } + + bool IsEmpty() const { + return SpecialSharding.empty(); + } + + TConclusionStatus BuildActivityIndex(const std::set<ui64>& closedWriteIds, const std::set<ui64>& closedReadIds) { + if (SpecialSharding.empty()) { + AFL_VERIFY(!PartsCount); + return TConclusionStatus::Success(); + } + ActiveReadSpecialSharding.clear(); + ActiveWriteSpecialSharding.clear(); + std::set<ui32> modsRead; + std::set<ui32> modsWrite; + for (auto&& i : SpecialSharding) { + for (auto&& n : i.GetAppropriateMods()) { + if (PartsCount <= n) { + return TConclusionStatus::Fail("too large parts mod in compare with parts count"); + } + if (!closedWriteIds.contains(i.GetTabletId())) { + modsWrite.emplace(n); + } + if (!closedReadIds.contains(i.GetTabletId())) { + if (!modsRead.emplace(n).second) { + return TConclusionStatus::Fail("read interval twice usage impossible"); + } + } + } + if (!closedReadIds.contains(i.GetTabletId())) { + ActiveReadSpecialSharding.emplace_back(&i); + } + if (!closedWriteIds.contains(i.GetTabletId())) { + ActiveWriteSpecialSharding.emplace_back(&i); + } + } + if (modsRead.size() && modsRead.size() != PartsCount) { + return TConclusionStatus::Fail("incorrect sharding configuration read from proto (read): " + JoinSeq(", ", modsRead) + "; " + ::ToString(PartsCount)); + } + if (modsWrite.size() && modsWrite.size() != PartsCount) { + return TConclusionStatus::Fail("incorrect sharding configuration read from proto (write): " + JoinSeq(", ", modsWrite) + "; " + ::ToString(PartsCount)); + } + IndexConstructed = true; + return TConclusionStatus::Success(); + } + + TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto, const std::set<ui64>& closedWriteIds, const std::set<ui64>& closedReadIds) { + IndexConstructed = false; + + PartsCount = proto.GetHashSharding().GetModuloPartsCount(); + for (auto&& i : proto.GetHashSharding().GetTabletsForModulo()) { + TModuloShardingTablet info; + auto conclusion = info.DeserializeFromProto(i); + if (conclusion.IsFail()) { + return conclusion; + } + SpecialSharding.emplace_back(std::move(info)); + } + AFL_VERIFY(PartsCount || SpecialSharding.empty()); + return BuildActivityIndex(closedWriteIds, closedReadIds); + } + + const TModuloShardingTablet& GetShardingTabletVerified(const ui64 tabletId) const { + for (auto&& i : SpecialSharding) { + if (i.GetTabletId() == tabletId) { + return i; + } + } + AFL_VERIFY(false); + return Default<TModuloShardingTablet>(); + } + + void SerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const { + AFL_VERIFY(PartsCount); + proto.MutableHashSharding()->SetModuloPartsCount(PartsCount); + for (auto&& i : SpecialSharding) { + *proto.MutableHashSharding()->AddTabletsForModulo() = i.SerializeToProto(); + } + } + + bool UpdateShardInfo(const TModuloShardingTablet& info) { + for (auto&& i : SpecialSharding) { + if (i.GetTabletId() == info.GetTabletId()) { + i = info; + IndexConstructed = false; + return true; + } + } + return false; + } + + void AddShardInfo(const TModuloShardingTablet& info) { + IndexConstructed = false; + for (auto&& i : SpecialSharding) { + AFL_VERIFY(i.GetTabletId() != info.GetTabletId()); + } + SpecialSharding.emplace_back(info); + } +}; + +class THashShardingModuloN : public THashShardingImpl { +public: + static TString GetClassNameStatic() { + return "MODULO"; + } +private: + using TBase = THashShardingImpl; +private: + std::optional<TSpecificShardingInfo> SpecialShardingInfo; + + bool UpdateShardInfo(const TSpecificShardingInfo::TModuloShardingTablet& info) { + AFL_VERIFY(SpecialShardingInfo); + GetShardInfoVerified(info.GetTabletId()).IncrementVersion(); + if (SpecialShardingInfo->UpdateShardInfo(info)) { + return true; + } + for (auto&& i : GetOrderedShardIds()) { + if (i == info.GetTabletId()) { + SpecialShardingInfo->AddShardInfo(info); + return true; + } + } + AFL_VERIFY(false); + return false; + } + virtual std::shared_ptr<IGranuleShardingLogic> DoGetTabletShardingInfoOptional(const ui64 tabletId) const override; + +protected: + virtual TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> DoBuildSplitShardsModifiers(const std::vector<ui64>& newTabletIds) const override; + + virtual TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> DoBuildMergeShardsModifiers(const std::vector<ui64>& newTabletIds) const override; + + virtual TConclusionStatus DoOnAfterModification() override; + virtual TConclusionStatus DoOnBeforeModification() override { + if (!SpecialShardingInfo) { + SpecialShardingInfo = TSpecificShardingInfo(GetOrderedShardIds()); + } + return TConclusionStatus::Success(); + } + + virtual TConclusionStatus DoApplyModification(const NKikimrSchemeOp::TShardingModification& proto) override; + virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override { + TBase::DoSerializeToProto(proto); + if (SpecialShardingInfo) { + SpecialShardingInfo->SerializeToProto(proto); + } + proto.MutableHashSharding()->SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_MODULO_N); + } + + + virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) override; + + virtual std::set<ui64> DoGetModifiedShardIds(const NKikimrSchemeOp::TShardingModification& proto) const override { + std::set<ui64> result; + for (auto&& i : proto.GetModulo().GetShards()) { + result.emplace(i.GetTabletId()); + } + return result; + } +public: + using TBase::TBase; + + THashShardingModuloN() = default; + + THashShardingModuloN(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui64 seed = 0) + : TBase(shardIds, columnNames, seed) + {} + + virtual TString GetClassName() const override { + return GetClassNameStatic(); + } + + virtual THashMap<ui64, std::vector<ui32>> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override; +}; + +class TGranuleSharding: public THashGranuleSharding { +public: + static TString GetClassNameStatic() { + return "MODULO"; + } +private: + using TBase = THashGranuleSharding; + ui64 PartsCount = 0; + TSpecificShardingInfo::TModuloShardingTablet Interval; + static const inline TFactory::TRegistrator<TGranuleSharding> Registrator = TFactory::TRegistrator<TGranuleSharding>(GetClassNameStatic()); + +protected: + virtual std::shared_ptr<NArrow::TColumnFilter> DoGetFilter(const std::shared_ptr<arrow::Table>& table) const override { + const std::vector<ui64> hashes = CalcHashes(table); + auto result = std::make_shared<NArrow::TColumnFilter>(NArrow::TColumnFilter::BuildAllowFilter()); + const auto getter = [&](const ui32 index) { + return Interval.GetAppropriateMods().contains(hashes[index] % PartsCount); + }; + result->ResetWithLambda(hashes.size(), getter); + return result; + + } + virtual void DoSerializeToProto(TProto& proto) const override { + AFL_VERIFY(PartsCount); + proto.MutableModulo()->SetModuloPartsCount(PartsCount); + *proto.MutableModulo()->MutableHashing() = TBase::SerializeHashingToProto(); + *proto.MutableModulo()->MutableShardInfo() = Interval.SerializeToProto(); + } + virtual TConclusionStatus DoDeserializeFromProto(const TProto& proto) override { + PartsCount = proto.GetModulo().GetModuloPartsCount(); + if (!PartsCount) { + return TConclusionStatus::Fail("incorrect parts count for modulo info"); + } + { + auto conclusion = TBase::DeserializeHashingFromProto(proto.GetModulo().GetHashing()); + if (conclusion.IsFail()) { + return conclusion; + } + } + { + auto conclusion = Interval.DeserializeFromProto(proto.GetModulo().GetShardInfo()); + if (conclusion.IsFail()) { + return conclusion; + } + } + + return TConclusionStatus::Success(); + } +public: + TGranuleSharding() = default; + + TGranuleSharding(const std::vector<TString>& columnNames, const TSpecificShardingInfo::TModuloShardingTablet& interval, const ui64 partsCount) + : TBase(columnNames) + , PartsCount(partsCount) + , Interval(interval) { + AFL_VERIFY(PartsCount); + } + + virtual TString GetClassName() const override { + return GetClassNameStatic(); + } +}; + +} diff --git a/ydb/core/tx/sharding/hash_sharding.cpp b/ydb/core/tx/sharding/hash_sharding.cpp new file mode 100644 index 0000000000..06db098990 --- /dev/null +++ b/ydb/core/tx/sharding/hash_sharding.cpp @@ -0,0 +1,5 @@ +#include "hash_sharding.h" + +namespace NKikimr::NSharding { + +} diff --git a/ydb/core/tx/sharding/hash_sharding.h b/ydb/core/tx/sharding/hash_sharding.h new file mode 100644 index 0000000000..30111f3fe8 --- /dev/null +++ b/ydb/core/tx/sharding/hash_sharding.h @@ -0,0 +1,104 @@ +#pragma once +#include "sharding.h" +#include <ydb/core/formats/arrow/hash/calcer.h> + +namespace NKikimr::NSharding { + +class THashGranuleSharding: public IGranuleShardingLogic { +private: + std::optional<NArrow::NHash::TXX64> HashCalcer; +protected: + std::vector<ui64> CalcHashes(const std::shared_ptr<arrow::Table>& table) const { + AFL_VERIFY(!!HashCalcer); + return HashCalcer->ExecuteToVector(table); + } + + virtual std::set<TString> DoGetColumnNames() const final { + AFL_VERIFY(!!HashCalcer); + auto columnsVector = HashCalcer->GetColumnNames(); + return std::set<TString>(columnsVector.begin(), columnsVector.end()); + } + NKikimrSchemeOp::THashShardingInfo SerializeHashingToProto() const { + NKikimrSchemeOp::THashShardingInfo proto; + AFL_VERIFY(!!HashCalcer); + for (auto&& i : HashCalcer->GetColumnNames()) { + proto.AddColumnNames(i); + } + return proto; + } + TConclusionStatus DeserializeHashingFromProto(const NKikimrSchemeOp::THashShardingInfo& proto) { + AFL_VERIFY(!HashCalcer); + if (proto.GetColumnNames().empty()) { + return TConclusionStatus::Fail("no column names for THashGranuleSharding in proto"); + } + std::vector<TString> columnNames; + for (auto&& i : proto.GetColumnNames()) { + columnNames.emplace_back(i); + } + HashCalcer.emplace(columnNames, NArrow::NHash::TXX64::ENoColumnPolicy::Verify, 0); + return TConclusionStatus::Success(); + } +public: + THashGranuleSharding() = default; + + THashGranuleSharding(const std::vector<TString>& columnNames) + : HashCalcer(NArrow::NHash::TXX64(columnNames, NArrow::NHash::TXX64::ENoColumnPolicy::Verify, 0)) + { + + } +}; + +class THashShardingImpl: public IShardingBase { +private: + using TBase = IShardingBase; + ui64 Seed = 0; + std::optional<NArrow::NHash::TXX64> HashCalcer; + YDB_READONLY_DEF(std::vector<TString>, ShardingColumns); +protected: + virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override { + for (auto&& i : ShardingColumns) { + proto.MutableHashSharding()->AddColumns(i); + } + } + virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) override { + if (!proto.HasHashSharding()) { + return TConclusionStatus::Fail("no data about hash sharding"); + } + if (!proto.GetHashSharding().GetColumns().size()) { + return TConclusionStatus::Fail("no columns for hash sharding"); + } + for (auto&& i : proto.GetHashSharding().GetColumns()) { + ShardingColumns.emplace_back(i); + } + AFL_VERIFY(!HashCalcer); + HashCalcer.emplace(ShardingColumns, NArrow::NHash::TXX64::ENoColumnPolicy::Verify, Seed); + return TConclusionStatus::Success(); + } +public: + THashShardingImpl() = default; + + THashShardingImpl(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui64 seed = 0) + : TBase(shardIds) + , Seed(seed) + , ShardingColumns(columnNames) { + HashCalcer.emplace(columnNames, NArrow::NHash::TXX64::ENoColumnPolicy::Verify, Seed); + } + + virtual TString DebugString() const override { + return TBase::DebugString() + ";Columns: " + JoinSeq(", ", GetShardingColumns()); + } + + virtual std::vector<ui64> MakeHashes(const std::shared_ptr<arrow::RecordBatch>& batch) const { + AFL_VERIFY(!!HashCalcer); + return HashCalcer->Execute(batch).value_or(Default<std::vector<ui64>>()); + } + + template <typename T> + static ui64 CalcHash(const T value, const ui32 seed = 0) { + static_assert(std::is_arithmetic<T>::value); + return XXH64(&value, sizeof(value), seed); + } + +}; + +} diff --git a/ydb/core/tx/sharding/hash_slider.cpp b/ydb/core/tx/sharding/hash_slider.cpp new file mode 100644 index 0000000000..d6c2f116ae --- /dev/null +++ b/ydb/core/tx/sharding/hash_slider.cpp @@ -0,0 +1,55 @@ +#include "hash_slider.h" + +namespace NKikimr::NSharding { + +THashMap<ui64, std::vector<ui32>> TLogsSharding::MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const { + if (GetShardingColumns().size() < 2) { + return {}; + } + + auto tsArray = batch->GetColumnByName(GetShardingColumns()[0]); + if (!tsArray || tsArray->type_id() != arrow::Type::TIMESTAMP) { + return {}; + } + + const std::vector<ui64> hashes = MakeHashes(batch); + if (hashes.empty()) { + return {}; + } + + auto tsColumn = std::static_pointer_cast<arrow::TimestampArray>(tsArray); + + std::vector<std::vector<ui32>> result; + result.resize(GetOrderedShardIds().size()); + for (auto&& i : result) { + i.reserve(hashes.size()); + } + + for (int row = 0; row < batch->num_rows(); ++row) { + result[ShardNo(tsColumn->Value(row), hashes[row])].emplace_back(row); + } + + THashMap<ui64, std::vector<ui32>> resultHash; + for (ui32 i = 0; i < result.size(); ++i) { + if (result[i].size()) { + resultHash.emplace(GetShardIdByOrderIdx(i), std::move(result[i])); + } + } + + return resultHash; +} + +NKikimr::TConclusionStatus TLogsSharding::DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) { + auto conclusion = TBase::DoDeserializeFromProto(proto); + if (conclusion.IsFail()) { + return conclusion; + } + AFL_VERIFY(proto.GetHashSharding().GetFunction() == NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CLOUD_LOGS); + if (proto.GetHashSharding().HasActiveShardsCount()) { + NumActive = proto.GetHashSharding().GetActiveShardsCount(); + } + + return TConclusionStatus::Success(); +} + +} diff --git a/ydb/core/tx/sharding/hash_slider.h b/ydb/core/tx/sharding/hash_slider.h new file mode 100644 index 0000000000..6cd61efa91 --- /dev/null +++ b/ydb/core/tx/sharding/hash_slider.h @@ -0,0 +1,76 @@ +#pragma once +#include "hash_sharding.h" + +namespace NKikimr::NSharding { + +class TLogsSharding : public THashShardingImpl { +public: + static constexpr ui32 DEFAULT_ACITVE_SHARDS = 10; + static constexpr TDuration DEFAULT_CHANGE_PERIOD = TDuration::Minutes(5); + static TString GetClassNameStatic() { + return "LOGS_SHARDING"; + } +private: + using TBase = THashShardingImpl; + ui32 NumActive = DEFAULT_ACITVE_SHARDS; + ui64 TsMin = 0; + ui64 ChangePeriod = DEFAULT_CHANGE_PERIOD.MicroSeconds(); + + virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override { + TBase::DoSerializeToProto(proto); + proto.MutableHashSharding()->SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CLOUD_LOGS); + proto.MutableHashSharding()->SetActiveShardsCount(NumActive); + } + + virtual std::shared_ptr<IGranuleShardingLogic> DoGetTabletShardingInfoOptional(const ui64 /*tabletId*/) const override { + return nullptr; + } + + virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) override; + virtual TConclusionStatus DoOnAfterModification() override { + return TConclusionStatus::Success(); + } + virtual TConclusionStatus DoOnBeforeModification() override { + return TConclusionStatus::Success(); + } + + virtual std::set<ui64> DoGetModifiedShardIds(const NKikimrSchemeOp::TShardingModification& /*proto*/) const override { + return {}; + } + + virtual TConclusionStatus DoApplyModification(const NKikimrSchemeOp::TShardingModification& /*proto*/) override { + return TConclusionStatus::Fail("its impossible to modify logs sharding"); + } + + virtual TString GetClassName() const override { + return GetClassNameStatic(); + } +public: + TLogsSharding() = default; + + TLogsSharding(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui32 shardsCountActive, TDuration changePeriod = DEFAULT_CHANGE_PERIOD) + : TBase(shardIds, columnNames) + , NumActive(Min<ui32>(shardsCountActive, GetShardsCount())) + , TsMin(0) + , ChangePeriod(changePeriod.MicroSeconds()) + { + AFL_VERIFY(NumActive); + } + + // tsMin = GetTsMin(tabletIdsMap, timestamp); + // tabletIds = GetTableIdsByTs(tabletIdsMap, timestamp); + // numIntervals = tabletIds.size() / nActive; + // tsInterval = (timestamp - tsMin) / changePeriod; + // shardNo = (hash(uid) % nActive) + (tsInterval % numIntervals) * nActive; + // tabletId = tabletIds[shardNo]; + ui32 ShardNo(ui64 timestamp, const ui64 uidHash) const { + const ui32 tsInterval = (timestamp - TsMin) / ChangePeriod; + const ui32 numIntervals = std::max<ui32>(1, GetShardsCount() / NumActive); + return ((uidHash % NumActive) + (tsInterval % numIntervals) * NumActive) % GetShardsCount(); + } + + virtual THashMap<ui64, std::vector<ui32>> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override; + +}; + +} diff --git a/ydb/core/tx/sharding/random.cpp b/ydb/core/tx/sharding/random.cpp new file mode 100644 index 0000000000..fc6ed93c2f --- /dev/null +++ b/ydb/core/tx/sharding/random.cpp @@ -0,0 +1,21 @@ +#include "random.h" + +namespace NKikimr::NSharding { + +THashMap<ui64, std::vector<ui32>> TRandomSharding::MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const { + std::vector<ui64> activeShardIds; + for (auto&& i : GetOrderedShardIds()) { + if (IsActiveForWrite(i)) { + activeShardIds.emplace_back(i); + } + } + AFL_VERIFY(activeShardIds.size()); + THashMap<ui64, std::vector<ui32>> resultHash; + std::vector<ui32>& sVector = resultHash.emplace(activeShardIds[RandomNumber<ui32>(activeShardIds.size())], std::vector<ui32>()).first->second; + for (ui32 i = 0; i < batch->num_rows(); ++i) { + sVector.emplace_back(i); + } + return resultHash; +} + +} diff --git a/ydb/core/tx/sharding/random.h b/ydb/core/tx/sharding/random.h new file mode 100644 index 0000000000..cd0d40ca9f --- /dev/null +++ b/ydb/core/tx/sharding/random.h @@ -0,0 +1,51 @@ +#pragma once +#include "sharding.h" + +namespace NKikimr::NSharding { + +class TRandomSharding: public IShardingBase { +private: + using TBase = IShardingBase; + static TString GetClassNameStatic() { + return "RANDOM"; + } +protected: + virtual std::shared_ptr<IGranuleShardingLogic> DoGetTabletShardingInfoOptional(const ui64 /*tabletId*/) const override { + return nullptr; + } + + virtual TConclusionStatus DoOnAfterModification() override { + return TConclusionStatus::Success(); + } + virtual TConclusionStatus DoOnBeforeModification() override { + return TConclusionStatus::Success(); + } + + virtual std::set<ui64> DoGetModifiedShardIds(const NKikimrSchemeOp::TShardingModification& /*proto*/) const override { + return {}; + } + + virtual TConclusionStatus DoApplyModification(const NKikimrSchemeOp::TShardingModification& /*proto*/) override { + return TConclusionStatus::Fail("its impossible to modify random sharding"); + } + virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override { + proto.MutableRandomSharding(); + } + + virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) override { + if (!proto.HasRandomSharding()) { + return TConclusionStatus::Fail("no random sharding data"); + } + return TConclusionStatus::Success(); + } +public: + using TBase::TBase; + + virtual THashMap<ui64, std::vector<ui32>> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override; + + virtual TString GetClassName() const override { + return GetClassNameStatic(); + } +}; + +} diff --git a/ydb/core/tx/sharding/sharding.cpp b/ydb/core/tx/sharding/sharding.cpp index a2b9f390be..ba34521c21 100644 --- a/ydb/core/tx/sharding/sharding.cpp +++ b/ydb/core/tx/sharding/sharding.cpp @@ -1,10 +1,16 @@ #include "sharding.h" +#include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/library/yql/utils/yql_panic.h> +#include <ydb/core/tx/columnshard/common/protos/snapshot.pb.h> #include <util/string/join.h> +#include "hash_intervals.h" +#include "hash_modulo.h" +#include "hash_slider.h" +#include "random.h" namespace NKikimr::NSharding { -TConclusionStatus TShardingBase::ValidateBehaviour(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo) { +TConclusionStatus IShardingBase::ValidateBehaviour(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo) { auto copy = shardingInfo; if (copy.GetColumnShards().size() == 0) { copy.AddColumnShards(1); @@ -16,15 +22,14 @@ TConclusionStatus TShardingBase::ValidateBehaviour(const NSchemeShard::TOlapSche return TConclusionStatus::Success(); } -TConclusion<std::unique_ptr<TShardingBase>> TShardingBase::BuildFromProto(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingProto) { +TConclusion<std::unique_ptr<IShardingBase>> IShardingBase::BuildFromProto(const NSchemeShard::TOlapSchema* schema, const NKikimrSchemeOp::TColumnTableSharding& shardingProto) { if (!shardingProto.GetColumnShards().size()) { return TConclusionStatus::Fail("config is incorrect for construct sharding behaviour"); } - std::vector<ui64> shardIds(shardingProto.GetColumnShards().begin(), shardingProto.GetColumnShards().end()); + std::unique_ptr<IShardingBase> result; if (shardingProto.HasRandomSharding()) { - return std::make_unique<TRandomSharding>(shardIds); - } - if (shardingProto.HasHashSharding()) { + result = std::make_unique<TRandomSharding>(); + } else if (shardingProto.HasHashSharding()) { auto& hashSharding = shardingProto.GetHashSharding(); std::vector<TString> columnNames; if (hashSharding.GetColumns().empty()) { @@ -32,67 +37,264 @@ TConclusion<std::unique_ptr<TShardingBase>> TShardingBase::BuildFromProto(const } else { for (auto&& i : hashSharding.GetColumns()) { columnNames.emplace_back(i); - if (!schema.GetColumns().GetByName(i)) { - return TConclusionStatus::Fail("incorrect sharding column name: " + i); - } - if (!schema.GetColumns().GetByName(i)->IsKeyColumn()) { - return TConclusionStatus::Fail("sharding column name have to been primary key column: " + i); + if (schema) { + if (!schema->GetColumns().GetByName(i)) { + return TConclusionStatus::Fail("incorrect sharding column name: " + i); + } + if (!schema->GetColumns().GetByName(i)->IsKeyColumn()) { + return TConclusionStatus::Fail("sharding column name have to been primary key column: " + i); + } } } } if (hashSharding.GetFunction() == NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CONSISTENCY_64) { - return std::make_unique<TConsistencySharding64>(shardIds, columnNames, 0); + result = std::make_unique<NConsistency::TConsistencySharding64>(); } else if (hashSharding.GetFunction() == NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_MODULO_N) { - return std::make_unique<THashShardingModuloN>(shardIds, columnNames, 0); + result = std::make_unique<NModulo::THashShardingModuloN>(); } else if (hashSharding.GetFunction() == NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CLOUD_LOGS) { - ui32 activeShards = TLogsSharding::DEFAULT_ACITVE_SHARDS; - if (hashSharding.HasActiveShardsCount()) { - activeShards = hashSharding.GetActiveShardsCount(); - } - return std::make_unique<TLogsSharding>(shardIds, columnNames, activeShards); + result = std::make_unique<TLogsSharding>(); + } + } + if (!result) { + return TConclusionStatus::Fail("not determined type of sharding"); + } + { + auto conclusion = result->DeserializeFromProto(shardingProto); + if (conclusion.IsFail()) { + return conclusion; } + return result; } - return TConclusionStatus::Fail("not determined sharding type"); } -TString TShardingBase::DebugString() const { +TString IShardingBase::DebugString() const { return "SHARDING"; } -std::vector<ui32> THashShardingModuloN::MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const { - std::vector<ui64> hashes = MakeHashes(batch); - std::vector<ui32> result; - result.reserve(hashes.size()); - for (auto&& i : hashes) { - result.emplace_back(i % GetShardsCount()); +NKikimr::TConclusionStatus IShardingBase::DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) { + std::set<ui64> shardIdsSetOriginal; + if (proto.GetShardsInfo().empty()) { + std::vector<ui64> orderedTabletIds(proto.GetColumnShards().begin(), proto.GetColumnShards().end()); + InitializeFromOrdered(orderedTabletIds); + } else { + std::map<ui32, ui64> sortedTabletIds; + for (auto&& i : proto.GetShardsInfo()) { + TConclusion<TShardInfo> shardInfo = TShardInfo::BuildFromProto(i); + if (shardInfo.IsFail()) { + return shardInfo; + } + AFL_VERIFY(sortedTabletIds.emplace(shardInfo->GetSequenceIdx(), shardInfo->GetTabletId()).second); + AFL_VERIFY(Shards.emplace(shardInfo->GetTabletId(), shardInfo.GetResult()).second); + } + for (auto&& i : sortedTabletIds) { + OrderedShardIds.emplace_back(i.second); + } + } + if (shardIdsSetOriginal.size()) { + return TConclusionStatus::Fail("not for all shard in sequence we have info"); + } + { + auto conclusion = DoDeserializeFromProto(proto); + if (conclusion.IsFail()) { + return conclusion; + } + } + return TConclusionStatus::Success(); +} + +NKikimr::TConclusionStatus IShardingBase::ApplyModification(const NKikimrSchemeOp::TShardingModification& proto) { + { + auto conclusion = OnBeforeModification(); + if (conclusion.IsFail()) { + return conclusion; + } + } + for (auto&& i : proto.GetNewShardIds()) { + if (!Shards.emplace(i, TShardInfo(i, OrderedShardIds.size())).second) { + return TConclusionStatus::Fail("shard id from NewShardIds duplication with current full shardIds list"); + } + OrderedShardIds.emplace_back(i); + } + for (auto&& i : proto.GetCloseWriteIds()) { + auto* shardInfo = GetShardInfo(i); + if (!shardInfo) { + return TConclusionStatus::Fail("incorrect shard id from CloseWriteIds - not exists in full scope"); + } + shardInfo->SetIsOpenForWrite(false); + } + for (auto&& i : proto.GetCloseReadIds()) { + auto* shardInfo = GetShardInfo(i); + if (!shardInfo) { + return TConclusionStatus::Fail("incorrect shard id from CloseReadIds - not exists in full scope"); + } + shardInfo->SetIsOpenForRead(false); + } + for (auto&& i : proto.GetOpenWriteIds()) { + auto* shardInfo = GetShardInfo(i); + if (!shardInfo) { + return TConclusionStatus::Fail("incorrect shard id from OpenWriteIds - not exists in full scope"); + } + shardInfo->SetIsOpenForWrite(true); + } + for (auto&& i : proto.GetOpenReadIds()) { + auto* shardInfo = GetShardInfo(i); + if (!shardInfo) { + return TConclusionStatus::Fail("incorrect shard id from OpenReadIds - not exists in full scope"); + } + shardInfo->SetIsOpenForRead(true); + } + { + auto conclusion = DoApplyModification(proto); + if (conclusion.IsFail()) { + return conclusion; + } + } + { + auto conclusion = OnAfterModification(); + if (conclusion.IsFail()) { + return conclusion; + } + } + return TConclusionStatus::Success(); +} + +NKikimr::TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> IShardingBase::BuildAddShardsModifiers(const std::vector<ui64>& newTabletIds) const { + NKikimrSchemeOp::TShardingModification startModification; + for (auto&& i : newTabletIds) { + startModification.AddNewShardIds(i); + startModification.AddCloseReadIds(i); + startModification.AddCloseWriteIds(i); + } + + TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> infosConclusion = BuildSplitShardsModifiers(newTabletIds); + if (infosConclusion.IsFail()) { + return infosConclusion; + } + std::vector<NKikimrSchemeOp::TAlterShards> result; + { + NKikimrSchemeOp::TAlterShards startAlter; + *startAlter.MutableModification() = std::move(startModification); + result.emplace_back(std::move(startAlter)); + } + for (auto&& i : *infosConclusion) { + result.emplace_back(std::move(i)); } return result; } -std::vector<ui32> TLogsSharding::MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const { - if (ShardingColumns.size() < 2) { - return {}; +NKikimr::TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> IShardingBase::BuildReduceShardsModifiers(const std::vector<ui64>& newTabletIds) const { + NKikimrSchemeOp::TShardingModification startModification; + for (auto&& i : newTabletIds) { + startModification.AddNewShardIds(i); + startModification.AddCloseReadIds(i); + startModification.AddCloseWriteIds(i); } - auto tsArray = batch->GetColumnByName(ShardingColumns[0]); - if (!tsArray || tsArray->type_id() != arrow::Type::TIMESTAMP) { - return {}; + TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> infosConclusion = BuildMergeShardsModifiers(newTabletIds); + if (infosConclusion.IsFail()) { + return infosConclusion; } + std::vector<NKikimrSchemeOp::TAlterShards> result; + { + NKikimrSchemeOp::TAlterShards startAlter; + *startAlter.MutableModification() = std::move(startModification); + result.emplace_back(std::move(startAlter)); + } + for (auto&& i : *infosConclusion) { + result.emplace_back(std::move(i)); + } + { + NKikimrSchemeOp::TShardingModification finishModification; + for (auto&& i : GetOrderedShardIds()) { + finishModification.AddDeleteShardIds(i); + } + NKikimrSchemeOp::TAlterShards finishAlter; + *finishAlter.MutableModification() = std::move(finishModification); + result.emplace_back(std::move(finishAlter)); + } + return result; +} - const std::vector<ui64> hashes = MakeHashes(batch); - if (hashes.empty()) { - return {}; +NKikimrSchemeOp::TColumnTableSharding IShardingBase::SerializeToProto() const { + NKikimrSchemeOp::TColumnTableSharding result; + AFL_VERIFY(Shards.size() == OrderedShardIds.size()); + AFL_VERIFY(OrderedShardIds.size()); + for (auto&& i : OrderedShardIds) { + *result.AddShardsInfo() = GetShardInfoVerified(i).SerializeToProto(); + result.AddColumnShards(i); } + DoSerializeToProto(result); + return result; +} - auto tsColumn = std::static_pointer_cast<arrow::TimestampArray>(tsArray); - std::vector<ui32> out; - out.reserve(batch->num_rows()); +NKikimr::TConclusion<THashMap<ui64, std::vector<NKikimr::NArrow::TSerializedBatch>>> IShardingBase::SplitByShards(const std::shared_ptr<arrow::RecordBatch>& batch, const ui64 chunkBytesLimit) { + THashMap<ui64, std::vector<ui32>> sharding = MakeSharding(batch); + THashMap<ui64, std::shared_ptr<arrow::RecordBatch>> chunks; + if (sharding.size() == 1) { + AFL_VERIFY(chunks.emplace(sharding.begin()->first, batch).second); + } else { + chunks = NArrow::ShardingSplit(batch, sharding); + } + AFL_VERIFY(chunks.size() == sharding.size()); + NArrow::TBatchSplitttingContext context(chunkBytesLimit); + THashMap<ui64, std::vector<NArrow::TSerializedBatch>> result; + for (auto&& [tabletId, chunk]: chunks) { + if (!chunk) { + continue; + } + auto blobsSplittedConclusion = NArrow::SplitByBlobSize(chunk, context); + if (blobsSplittedConclusion.IsFail()) { + return TConclusionStatus::Fail("cannot split batch in according to limits: " + blobsSplittedConclusion.GetErrorMessage()); + } + result.emplace(tabletId, blobsSplittedConclusion.DetachResult()); + } + return result; +} - for (int row = 0; row < batch->num_rows(); ++row) { - out.emplace_back(ShardNo(tsColumn->Value(row), hashes[row])); +void IShardingBase::InitializeFromOrdered(const std::vector<ui64>& orderedIds) { + AFL_VERIFY(orderedIds.size()); + std::set<ui64> shardIdsSetOriginal; + OrderedShardIds.clear(); + Shards.clear(); + for (auto&& i : orderedIds) { + AFL_VERIFY(shardIdsSetOriginal.emplace(i).second); + Shards.emplace(i, TShardInfo(i, OrderedShardIds.size())); + OrderedShardIds.emplace_back(i); } +} - return out; +NKikimrSchemeOp::TGranuleShardInfo TShardInfo::SerializeToProto() const { + NKikimrSchemeOp::TGranuleShardInfo result; + result.SetTabletId(TabletId); + result.SetSequenceIdx(SequenceIdx); + result.SetIsOpenForWrite(IsOpenForWrite); + result.SetIsOpenForRead(IsOpenForRead); + if (OpenForWriteSnapshot) { + *result.MutableOpenForWriteSnapshot() = OpenForWriteSnapshot->SerializeToProto(); + } + result.SetShardingVersion(ShardingVersion); + return result; +} + +NKikimr::TConclusionStatus TShardInfo::DeserializeFromProto(const NKikimrSchemeOp::TGranuleShardInfo& proto) { + TabletId = proto.GetTabletId(); + SequenceIdx = proto.GetSequenceIdx(); + IsOpenForRead = proto.GetIsOpenForRead(); + IsOpenForWrite = proto.GetIsOpenForWrite(); + if (proto.HasOpenForWriteSnapshot()) { + NKikimr::NOlap::TSnapshot ss = NKikimr::NOlap::TSnapshot::Zero(); + auto conclusion = ss.DeserializeFromProto(proto.GetOpenForWriteSnapshot()); + if (conclusion.IsFail()) { + return conclusion; + } + OpenForWriteSnapshot = ss; + } + AFL_VERIFY(proto.HasShardingVersion()); + ShardingVersion = proto.GetShardingVersion(); + if (!TabletId) { + return TConclusionStatus::Fail("incorrect TabletId for TShardInfo proto"); + } + return TConclusionStatus::Success(); } } diff --git a/ydb/core/tx/sharding/sharding.h b/ydb/core/tx/sharding/sharding.h index 3b16bf46ce..1b49ba9743 100644 --- a/ydb/core/tx/sharding/sharding.h +++ b/ydb/core/tx/sharding/sharding.h @@ -1,236 +1,320 @@ #pragma once +#include <ydb/library/accessor/accessor.h> +#include <ydb/core/tx/schemeshard/olap/schema/schema.h> +#include <ydb/core/formats/arrow/size_calcer.h> +#include <ydb/core/tx/columnshard/common/snapshot.h> -#include "hash.h" +namespace NKikimrSchemeOp { +class TColumnTableSharding; +class TGranuleShardingLogicContainer; +} -#include <ydb/core/formats/arrow/arrow_helpers.h> -#include <ydb/core/formats/arrow/hash/calcer.h> -#include <ydb/core/formats/arrow/size_calcer.h> -#include <ydb/core/protos/flat_scheme_op.pb.h> -#include <ydb/core/tx/schemeshard/olap/schema/schema.h> +namespace NKikimr::NSharding { -#include <ydb/library/accessor/accessor.h> -#include <ydb/library/conclusion/result.h> -#include <ydb/library/conclusion/status.h> +struct TExternalTableColumn; -#include <contrib/libs/apache/arrow/cpp/src/arrow/compute/api.h> -#include <contrib/libs/xxhash/xxhash.h> +class IGranuleShardingLogic { +public: + using TProto = NKikimrSchemeOp::TGranuleShardingLogicContainer; + using TFactory = NObjectFactory::TObjectFactory<IGranuleShardingLogic, TString>; -#include <util/random/random.h> +private: + virtual std::shared_ptr<NArrow::TColumnFilter> DoGetFilter(const std::shared_ptr<arrow::Table>& table) const = 0; + virtual std::set<TString> DoGetColumnNames() const = 0; + virtual void DoSerializeToProto(TProto& proto) const = 0; + virtual TConclusionStatus DoDeserializeFromProto(const TProto& proto) = 0; -#include <type_traits> +public: + IGranuleShardingLogic() = default; + virtual ~IGranuleShardingLogic() = default; -namespace NKikimr::NSharding { + std::shared_ptr<NArrow::TColumnFilter> GetFilter(const std::shared_ptr<arrow::Table>& table) const { + return DoGetFilter(table); + } -struct TExternalTableColumn; + std::shared_ptr<NArrow::TColumnFilter> GetFilter(const std::shared_ptr<arrow::RecordBatch>& rb) const { + return DoGetFilter(NArrow::TStatusValidator::GetValid(arrow::Table::FromRecordBatches({ rb }))); + } + std::set<TString> GetColumnNames() const { + return DoGetColumnNames(); + } + + virtual TString GetClassName() const = 0; -class TShardingBase { + void SerializeToProto(TProto& proto) const { + DoSerializeToProto(proto); + } + TConclusionStatus DeserializeFromProto(const TProto& proto) { + return DoDeserializeFromProto(proto); + } +}; + +class TGranuleShardingLogicContainer: public NBackgroundTasks::TInterfaceProtoContainer<IGranuleShardingLogic> { private: - YDB_READONLY_DEF(std::vector<ui64>, ShardIds); - YDB_READONLY(ui64, Version, 1); - TShardingBase() = default; -protected: - virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const = 0; + using TBase = NBackgroundTasks::TInterfaceProtoContainer<IGranuleShardingLogic>; + using TProto = IGranuleShardingLogic::TProto; public: - using TColumn = TExternalTableColumn; + using TBase::TBase; +}; + +class TShardInfo { +private: + YDB_READONLY(ui64, TabletId, 0); + YDB_ACCESSOR(ui32, SequenceIdx, 0); + YDB_ACCESSOR(bool, IsOpenForRead, true); + YDB_ACCESSOR(bool, IsOpenForWrite, true); + YDB_ACCESSOR_DEF(std::optional<NKikimr::NOlap::TSnapshot>, OpenForWriteSnapshot); + YDB_READONLY(ui32, ShardingVersion, 0); + + TShardInfo() = default; + + TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TGranuleShardInfo& proto); public: - static TConclusionStatus ValidateBehaviour(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo); - static TConclusion<std::unique_ptr<TShardingBase>> BuildFromProto(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo); + TShardInfo(const ui64 tabletId, const ui32 seqIdx) + : TabletId(tabletId) + , SequenceIdx(seqIdx) + { - TShardingBase(const std::vector<ui64>& shardIds) - : ShardIds(shardIds) { + } + const NKikimr::NOlap::TSnapshot& GetOpenForWriteSnapshotVerified() const { + AFL_VERIFY(OpenForWriteSnapshot); + return *OpenForWriteSnapshot; } - ui32 GetShardsCount() const { - return ShardIds.size(); + void IncrementVersion() { + ++ShardingVersion; } - NKikimrSchemeOp::TColumnTableSharding SerializeToProto() const { - NKikimrSchemeOp::TColumnTableSharding result; - result.SetVersion(1); - AFL_VERIFY(ShardIds.size()); - for (auto&& i : ShardIds) { - result.AddColumnShards(i); + NKikimrSchemeOp::TGranuleShardInfo SerializeToProto() const; + + static TConclusion<TShardInfo> BuildFromProto(const NKikimrSchemeOp::TGranuleShardInfo& proto) { + TShardInfo result; + auto conclusion = result.DeserializeFromProto(proto); + if (conclusion.IsFail()) { + return conclusion; } - DoSerializeToProto(result); return result; } +}; - virtual std::vector<ui32> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const = 0; +class IShardingBase { +private: + std::vector<ui64> OrderedShardIds; + THashMap<ui64, TShardInfo> Shards; + TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto); - TConclusion<THashMap<ui64, std::vector<NArrow::TSerializedBatch>>> SplitByShards(const std::shared_ptr<arrow::RecordBatch>& batch, const ui64 chunkBytesLimit) { - auto sharding = MakeSharding(batch); - std::vector<std::shared_ptr<arrow::RecordBatch>> chunks; - if (ShardIds.size() == 1) { - chunks = {batch}; - } else { - chunks = NArrow::ShardingSplit(batch, sharding, ShardIds.size()); - } - AFL_VERIFY(chunks.size() == ShardIds.size()); - NArrow::TBatchSplitttingContext context(chunkBytesLimit); - THashMap<ui64, std::vector<NArrow::TSerializedBatch>> result; - for (ui32 i = 0; i < chunks.size(); ++i) { - if (!chunks[i]) { - continue; + void InitializeFromOrdered(const std::vector<ui64>& orderedIds); + +protected: + std::set<ui64> GetClosedWritingShardIds() const { + std::set<ui64> result; + for (auto&& i : Shards) { + if (!i.second.GetIsOpenForWrite()) { + result.emplace(i.first); } - auto blobsSplittedConclusion = NArrow::SplitByBlobSize(chunks[i], context); - if (blobsSplittedConclusion.IsFail()) { - return TConclusionStatus::Fail("cannot split batch in according to limits: " + blobsSplittedConclusion.GetErrorMessage()); + } + return result; + } + + std::set<ui64> GetClosedReadingShardIds() const { + std::set<ui64> result; + for (auto&& i : Shards) { + if (!i.second.GetIsOpenForRead()) { + result.emplace(i.first); } - result.emplace(ShardIds[i], blobsSplittedConclusion.DetachResult()); } return result; } - virtual TString DebugString() const; + void SetOrderedShardIds(const std::vector<ui64>& ids) { + ui32 idx = 0; + for (auto&& i : ids) { + auto* shardInfo = GetShardInfo(i); + AFL_VERIFY(shardInfo); + shardInfo->SetSequenceIdx(idx++); + } + OrderedShardIds = ids; + AFL_VERIFY(OrderedShardIds.size() == Shards.size()); + } - virtual ~TShardingBase() = default; -}; + const std::vector<ui64>& GetOrderedShardIds() const { + return OrderedShardIds; + } -class THashShardingImpl: public TShardingBase { -private: - using TBase = TShardingBase; - const ui64 Seed; - const NArrow::NHash::TXX64 HashCalcer; -protected: - const std::vector<TString> ShardingColumns; - virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override { - for (auto&& i : ShardingColumns) { - proto.MutableHashSharding()->AddColumns(i); + bool HasReadClosedShards() const { + for (auto&& [_, i] : Shards) { + if (!i.GetIsOpenForRead()) { + return true; + } + } + return false; + } + + bool HasWriteClosedShards() const { + for (auto&& [_, i] : Shards) { + if (!i.GetIsOpenForWrite()) { + return true; + } } + return false; + } + virtual TString GetClassName() const = 0; + virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const = 0; + virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) = 0; + virtual TConclusionStatus DoApplyModification(const NKikimrSchemeOp::TShardingModification& proto) = 0; + virtual std::set<ui64> DoGetModifiedShardIds(const NKikimrSchemeOp::TShardingModification& proto) const = 0; + virtual TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> DoBuildSplitShardsModifiers(const std::vector<ui64>& /*newTabletIds*/) const { + return TConclusionStatus::Fail("shards splitting not implemented for " + GetClassName()); } + virtual TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> DoBuildMergeShardsModifiers(const std::vector<ui64>& /*newTabletIds*/) const { + return TConclusionStatus::Fail("shards merging not implemented for " + GetClassName()); + } + virtual TConclusionStatus DoOnAfterModification() = 0; + virtual TConclusionStatus DoOnBeforeModification() = 0; + virtual std::shared_ptr<IGranuleShardingLogic> DoGetTabletShardingInfoOptional(const ui64 tabletId) const = 0; + +public: + using TColumn = TExternalTableColumn; public: - THashShardingImpl(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui64 seed = 0) - : TBase(shardIds) - , Seed(seed) - , HashCalcer(columnNames, NArrow::NHash::TXX64::ENoColumnPolicy::Verify, Seed) - , ShardingColumns(columnNames) { + IShardingBase() = default; + + TShardInfo& GetShardInfoVerified(const ui64 tabletId) { + auto it = Shards.find(tabletId); + AFL_VERIFY(it != Shards.end()); + return it->second; } - virtual TString DebugString() const override { - return TBase::DebugString() + ";Columns: " + JoinSeq(", ", GetShardingColumns()); + const TShardInfo& GetShardInfoVerified(const ui64 tabletId) const { + auto it = Shards.find(tabletId); + AFL_VERIFY(it != Shards.end()); + return it->second; } - virtual std::vector<ui64> MakeHashes(const std::shared_ptr<arrow::RecordBatch>& batch) const { - return HashCalcer.Execute(batch).value_or(Default<std::vector<ui64>>()); + TShardInfo* GetShardInfo(const ui64 tabletId) { + auto it = Shards.find(tabletId); + if (it == Shards.end()) { + return nullptr; + } + return &it->second; } - template <typename T> - static ui64 CalcHash(const T value, const ui32 seed = 0) { - static_assert(std::is_arithmetic<T>::value); - return XXH64(&value, sizeof(value), seed); + const TShardInfo* GetShardInfo(const ui64 tabletId) const { + auto it = Shards.find(tabletId); + if (it == Shards.end()) { + return nullptr; + } + return &it->second; } - virtual const std::vector<TString>& GetShardingColumns() const { - return ShardingColumns; + void SetShardingOpenSnapshotVerified(const ui64 tabletId, const NKikimr::NOlap::TSnapshot& ss) { + GetShardInfoVerified(tabletId).SetOpenForWriteSnapshot(ss); } -}; -class THashShardingModuloN : public THashShardingImpl { -private: - using TBase = THashShardingImpl; -protected: - virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override { - TBase::DoSerializeToProto(proto); - proto.MutableHashSharding()->SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_MODULO_N); + NKikimr::NOlap::TSnapshot GetShardingOpenSnapshotVerified(const ui64 tabletId) const { + return GetShardInfoVerified(tabletId).GetOpenForWriteSnapshotVerified(); } -public: - THashShardingModuloN(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui64 seed = 0) - : TBase(shardIds, columnNames, seed) - {} - virtual std::vector<ui32> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override; -}; + TConclusionStatus OnAfterModification() { + return DoOnAfterModification(); + } -class TRandomSharding: public TShardingBase { -private: - using TBase = TShardingBase; -protected: - virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override { - proto.MutableRandomSharding(); + TConclusionStatus OnBeforeModification() { + return DoOnBeforeModification(); } -public: - using TBase::TBase; - virtual std::vector<ui32> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override { - return std::vector<ui32>(batch->num_rows(), RandomNumber<ui32>(GetShardsCount())); + TGranuleShardingLogicContainer GetTabletShardingInfoOptional(const ui64 tabletId) const { + if (IsShardClosedForWrite(tabletId)) { + return TGranuleShardingLogicContainer(); + } + return TGranuleShardingLogicContainer(DoGetTabletShardingInfoOptional(tabletId)); } -}; + TConclusionStatus ApplyModification(const NKikimrSchemeOp::TShardingModification& proto); + std::set<ui64> GetModifiedShardIds(const NKikimrSchemeOp::TShardingModification& proto) const { + return DoGetModifiedShardIds(proto); + } -class TConsistencySharding64: public THashShardingImpl { -private: - using TBase = THashShardingImpl; + TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> BuildSplitShardsModifiers(const std::vector<ui64>& newTabletIds) const { + return DoBuildSplitShardsModifiers(newTabletIds); + } + + TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> BuildMergeShardsModifiers(const std::vector<ui64>& newTabletIds) const { + return DoBuildMergeShardsModifiers(newTabletIds); + } + + TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> BuildAddShardsModifiers(const std::vector<ui64>& newTabletIds) const; + TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> BuildReduceShardsModifiers(const std::vector<ui64>& newTabletIds) const; - static ui32 CalcShardIdImpl(const ui64 hash, const ui32 shardsCount) { - AFL_VERIFY(shardsCount); - return std::min<ui32>(hash / (Max<ui64>() / shardsCount), shardsCount - 1); + void CloseShardWriting(const ui64 shardId) { + GetShardInfoVerified(shardId).SetIsOpenForWrite(false); } - virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override { - TBase::DoSerializeToProto(proto); - proto.MutableHashSharding()->SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CONSISTENCY_64); + void CloseShardReading(const ui64 shardId) { + GetShardInfoVerified(shardId).SetIsOpenForRead(false); } -public: - TConsistencySharding64(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui64 seed = 0) - : TBase(shardIds, columnNames, seed){ + + bool IsActiveForWrite(const ui64 tabletId) const { + return GetShardInfoVerified(tabletId).GetIsOpenForWrite(); } - virtual std::vector<ui32> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override { - auto hashes = MakeHashes(batch); - std::vector<ui32> result; - result.reserve(hashes.size()); - for (auto&& i : hashes) { - result.emplace_back(CalcShardIdImpl(i, GetShardsCount())); + bool IsActiveForRead(const ui64 tabletId) const { + return GetShardInfoVerified(tabletId).GetIsOpenForRead(); + } + + std::vector<ui64> GetActiveReadShardIds() const { + std::vector<ui64> result; + for (auto&& [_, i] : Shards) { + if (i.GetIsOpenForRead()) { + result.emplace_back(i.GetTabletId()); + } } return result; } - template <typename T> - static ui32 ShardNo(const T value, const ui32 shardsCount, const ui32 seed = 0) { - Y_ASSERT(shardsCount); - return CalcShardIdImpl(CalcHash(value, seed), shardsCount); + ui64 GetShardIdByOrderIdx(const ui32 shardIdx) const { + AFL_VERIFY(shardIdx < OrderedShardIds.size()); + return OrderedShardIds[shardIdx]; } -}; -// KIKIMR-11529 -class TLogsSharding : public THashShardingImpl { -private: - using TBase = THashShardingImpl; - ui32 NumActive; - ui64 TsMin; - ui64 ChangePeriod; + bool IsShardExists(const ui64 shardId) const { + return Shards.contains(shardId); + } - virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override { - TBase::DoSerializeToProto(proto); - proto.MutableHashSharding()->SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CLOUD_LOGS); - proto.MutableHashSharding()->SetActiveShardsCount(NumActive); + bool IsShardClosedForRead(const ui64 shardId) const { + return !IsActiveForRead(shardId); } -public: - static constexpr ui32 DEFAULT_ACITVE_SHARDS = 10; - static constexpr TDuration DEFAULT_CHANGE_PERIOD = TDuration::Minutes(5); - - TLogsSharding(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui32 shardsCountActive, TDuration changePeriod = DEFAULT_CHANGE_PERIOD) - : TBase(shardIds, columnNames) - , NumActive(Min<ui32>(shardsCountActive, GetShardsCount())) - , TsMin(0) - , ChangePeriod(changePeriod.MicroSeconds()) - {} - - // tsMin = GetTsMin(tabletIdsMap, timestamp); - // tabletIds = GetTableIdsByTs(tabletIdsMap, timestamp); - // numIntervals = tabletIds.size() / nActive; - // tsInterval = (timestamp - tsMin) / changePeriod; - // shardNo = (hash(uid) % nActive) + (tsInterval % numIntervals) * nActive; - // tabletId = tabletIds[shardNo]; - ui32 ShardNo(ui64 timestamp, const ui64 uidHash) const { - ui32 tsInterval = (timestamp - TsMin) / ChangePeriod; - ui32 numIntervals = GetShardsCount() / NumActive; - return ((uidHash % NumActive) + (tsInterval % numIntervals) * NumActive) % GetShardsCount(); - } - - virtual std::vector<ui32> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override; + bool IsShardClosedForWrite(const ui64 shardId) const { + return !IsActiveForWrite(shardId); + } + + static TConclusionStatus ValidateBehaviour(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo); + static TConclusion<std::unique_ptr<IShardingBase>> BuildFromProto(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo) { + return BuildFromProto(&schema, shardingInfo); + } + static TConclusion<std::unique_ptr<IShardingBase>> BuildFromProto(const NSchemeShard::TOlapSchema* schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo); + static TConclusion<std::unique_ptr<IShardingBase>> BuildFromProto(const NKikimrSchemeOp::TColumnTableSharding& shardingInfo) { + return BuildFromProto(nullptr, shardingInfo); + } + + IShardingBase(const std::vector<ui64>& shardIds) { + InitializeFromOrdered(shardIds); + } + + ui32 GetShardsCount() const { + return Shards.size(); + } + + NKikimrSchemeOp::TColumnTableSharding SerializeToProto() const; + + virtual THashMap<ui64, std::vector<ui32>> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const = 0; + + TConclusion<THashMap<ui64, std::vector<NArrow::TSerializedBatch>>> SplitByShards(const std::shared_ptr<arrow::RecordBatch>& batch, const ui64 chunkBytesLimit); + + virtual TString DebugString() const; + + virtual ~IShardingBase() = default; }; } diff --git a/ydb/core/tx/sharding/ya.make b/ydb/core/tx/sharding/ya.make index d0ae61d41a..9f6d68f5fb 100644 --- a/ydb/core/tx/sharding/ya.make +++ b/ydb/core/tx/sharding/ya.make @@ -6,6 +6,7 @@ PEERDIR( ydb/library/yql/public/udf ydb/core/formats/arrow/hash ydb/core/tx/schemeshard/olap/schema + ydb/core/tx/columnshard/common ydb/core/formats ydb/core/protos ) @@ -16,6 +17,10 @@ SRCS( sharding.cpp hash.cpp unboxed_reader.cpp + hash_slider.cpp + GLOBAL hash_modulo.cpp + GLOBAL hash_intervals.cpp + random.cpp ) END() diff --git a/ydb/library/conclusion/status.h b/ydb/library/conclusion/status.h index 8e798f0583..8af77479de 100644 --- a/ydb/library/conclusion/status.h +++ b/ydb/library/conclusion/status.h @@ -32,43 +32,43 @@ private: public: void Validate(const TString& processInfo = Default<TString>()) const; - const TString& GetErrorMessage() const { + [[nodiscard]] const TString& GetErrorMessage() const { return ErrorMessage ? *ErrorMessage : Default<TString>(); } - Ydb::StatusIds::StatusCode GetStatus() const { + [[nodiscard]] Ydb::StatusIds::StatusCode GetStatus() const { return Status; } - static TConclusionStatus Fail(const char* errorMessage) { + [[nodiscard]] static TConclusionStatus Fail(const char* errorMessage) { return TConclusionStatus(errorMessage); } - static TConclusionStatus Fail(const TString& errorMessage) { + [[nodiscard]] static TConclusionStatus Fail(const TString& errorMessage) { return TConclusionStatus(errorMessage); } - static TConclusionStatus Fail(const std::string& errorMessage) { + [[nodiscard]] static TConclusionStatus Fail(const std::string& errorMessage) { return TConclusionStatus(errorMessage); } - bool IsFail() const { + [[nodiscard]] bool IsFail() const { return !Ok(); } - bool IsSuccess() const { + [[nodiscard]] bool IsSuccess() const { return Ok(); } - bool Ok() const { + [[nodiscard]] bool Ok() const { return !ErrorMessage; } - bool operator!() const { + [[nodiscard]] bool operator!() const { return !!ErrorMessage; } - static TConclusionStatus Success() { + [[nodiscard]] static TConclusionStatus Success() { return TConclusionStatus(); } }; diff --git a/ydb/services/bg_tasks/abstract/interface.h b/ydb/services/bg_tasks/abstract/interface.h index ad249c2c08..68c9ab9a23 100644 --- a/ydb/services/bg_tasks/abstract/interface.h +++ b/ydb/services/bg_tasks/abstract/interface.h @@ -381,6 +381,21 @@ public: Object->SerializeToProto(result); TOperatorPolicy::SetClassName(result, Object->GetClassName()); } + + TString SerializeToString() const { + return SerializeToProto().SerializeAsString(); + } + + TConclusionStatus DeserializeFromString(const TString& data) { + TProto proto; + if (!proto.ParseFromArray(data.data(), data.size())) { + return TConclusionStatus::Fail("cannot parse string as proto"); + } + if (!DeserializeFromProto(proto)) { + return TConclusionStatus::Fail("cannot parse proto in container"); + } + return TConclusionStatus::Success(); + } }; } |