aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorivanmorozov333 <ivanmorozov@ydb.tech>2024-05-23 12:03:59 +0300
committerGitHub <noreply@github.com>2024-05-23 12:03:59 +0300
commit0107b7dd2f1f0160a767951122c65e87af629374 (patch)
treeb031ca3dbfd3aeb40ca06836da724ec8781e6aad
parentae0ee005720bd4d3678b0b0bdf92250ff3e822ea (diff)
downloadydb-0107b7dd2f1f0160a767951122c65e87af629374.tar.gz
columntables resharding draft (#4630)
-rw-r--r--ydb/core/formats/arrow/arrow_filter.h21
-rw-r--r--ydb/core/formats/arrow/arrow_helpers.cpp55
-rw-r--r--ydb/core/formats/arrow/arrow_helpers.h2
-rw-r--r--ydb/core/formats/arrow/hash/calcer.h45
-rw-r--r--ydb/core/kqp/common/kqp_resolve.h8
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.cpp8
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.h3
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.cpp28
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.h26
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_opt.cpp2
-rw-r--r--ydb/core/kqp/gateway/behaviour/tablestore/operations/ya.make1
-rw-r--r--ydb/core/protos/data_events.proto3
-rw-r--r--ydb/core/protos/flat_scheme_op.proto108
-rw-r--r--ydb/core/protos/tx_columnshard.proto3
-rw-r--r--ydb/core/protos/ya.make1
-rw-r--r--ydb/core/tx/columnshard/bg_tasks/abstract/session.cpp3
-rw-r--r--ydb/core/tx/columnshard/bg_tasks/abstract/session.h2
-rw-r--r--ydb/core/tx/columnshard/bg_tasks/abstract/ya.make1
-rw-r--r--ydb/core/tx/columnshard/bg_tasks/transactions/tx_remove.cpp2
-rw-r--r--ydb/core/tx/columnshard/bg_tasks/transactions/tx_save_state.cpp2
-rw-r--r--ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp2
-rw-r--r--ydb/core/tx/columnshard/columnshard__init.cpp7
-rw-r--r--ydb/core/tx/columnshard/columnshard__write.cpp16
-rw-r--r--ydb/core/tx/columnshard/columnshard_impl.cpp10
-rw-r--r--ydb/core/tx/columnshard/columnshard_impl.h7
-rw-r--r--ydb/core/tx/columnshard/columnshard_schema.h35
-rw-r--r--ydb/core/tx/columnshard/common/snapshot.cpp12
-rw-r--r--ydb/core/tx/columnshard/common/snapshot.h4
-rw-r--r--ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h2
-rw-r--r--ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp48
-rw-r--r--ydb/core/tx/columnshard/data_sharing/destination/session/destination.h29
-rw-r--r--ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp10
-rw-r--r--ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_start_from_initiator.cpp2
-rw-r--r--ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp1
-rw-r--r--ydb/core/tx/columnshard/engines/changes/general_compaction.cpp26
-rw-r--r--ydb/core/tx/columnshard/engines/changes/indexation.cpp98
-rw-r--r--ydb/core/tx/columnshard/engines/changes/with_appended.cpp2
-rw-r--r--ydb/core/tx/columnshard/engines/column_engine.h31
-rw-r--r--ydb/core/tx/columnshard/engines/column_engine_logs.cpp7
-rw-r--r--ydb/core/tx/columnshard/engines/column_engine_logs.h29
-rw-r--r--ydb/core/tx/columnshard/engines/db_wrapper.cpp29
-rw-r--r--ydb/core/tx/columnshard/engines/db_wrapper.h8
-rw-r--r--ydb/core/tx/columnshard/engines/portions/constructor.cpp6
-rw-r--r--ydb/core/tx/columnshard/engines/portions/constructor.h20
-rw-r--r--ydb/core/tx/columnshard/engines/portions/portion_info.cpp7
-rw-r--r--ydb/core/tx/columnshard/engines/portions/portion_info.h16
-rw-r--r--ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h11
-rw-r--r--ydb/core/tx/columnshard/engines/reader/actor/actor.cpp2
-rw-r--r--ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp1
-rw-r--r--ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp74
-rw-r--r--ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h17
-rw-r--r--ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp7
-rw-r--r--ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.h14
-rw-r--r--ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp22
-rw-r--r--ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h12
-rw-r--r--ydb/core/tx/columnshard/engines/scheme/index_info.cpp2
-rw-r--r--ydb/core/tx/columnshard/engines/scheme/index_info.h12
-rw-r--r--ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp19
-rw-r--r--ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h51
-rw-r--r--ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp77
-rw-r--r--ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp5
-rw-r--r--ydb/core/tx/columnshard/export/session/session.h2
-rw-r--r--ydb/core/tx/columnshard/hooks/abstract/abstract.h4
-rw-r--r--ydb/core/tx/columnshard/hooks/testing/controller.cpp2
-rw-r--r--ydb/core/tx/columnshard/hooks/testing/ro_controller.h5
-rw-r--r--ydb/core/tx/columnshard/operations/write.cpp44
-rw-r--r--ydb/core/tx/columnshard/operations/write.h6
-rw-r--r--ydb/core/tx/columnshard/tables_manager.cpp18
-rw-r--r--ydb/core/tx/columnshard/tables_manager.h2
-rw-r--r--ydb/core/tx/columnshard/transactions/operators/long_tx_write.cpp10
-rw-r--r--ydb/core/tx/columnshard/transactions/operators/long_tx_write.h18
-rw-r--r--ydb/core/tx/columnshard/transactions/operators/schema.h46
-rw-r--r--ydb/core/tx/columnshard/transactions/operators/ya.make1
-rw-r--r--ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.cpp23
-rw-r--r--ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.h32
-rw-r--r--ydb/core/tx/columnshard/transactions/transactions/ya.make13
-rw-r--r--ydb/core/tx/columnshard/transactions/ya.make2
-rw-r--r--ydb/core/tx/data_events/columnshard_splitter.cpp6
-rw-r--r--ydb/core/tx/data_events/columnshard_splitter.h7
-rw-r--r--ydb/core/tx/data_events/write_data.h4
-rw-r--r--ydb/core/tx/schemeshard/olap/bg_tasks/transactions/tasks_list.cpp2
-rw-r--r--ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.cpp6
-rw-r--r--ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.h6
-rw-r--r--ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/session.h2
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/abstract/context.h68
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h4
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.cpp5
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.h42
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.cpp11
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h18
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h44
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/abstract/ya.make1
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/common/object.cpp40
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/common/object.h45
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/common/update.cpp31
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/common/update.h58
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/common/ya.make14
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.cpp26
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.h19
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/ya.make13
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.cpp82
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.h63
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/ya.make13
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.cpp35
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.h50
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.cpp60
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h52
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.cpp62
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.h36
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/ya.make14
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.cpp69
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.h50
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/ya.make11
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.cpp44
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.h48
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/ya.make14
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.cpp94
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.h43
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/in_store/ya.make7
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.cpp35
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.h49
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.cpp26
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.h45
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp42
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.h52
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/standalone/ya.make1
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter/ya.make1
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/alter_table.cpp142
-rw-r--r--ydb/core/tx/schemeshard/olap/operations/create_table.cpp4
-rw-r--r--ydb/core/tx/schemeshard/olap/table/table.cpp23
-rw-r--r--ydb/core/tx/schemeshard/olap/table/table.h13
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_impl.cpp11
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_impl.h2
-rw-r--r--ydb/core/tx/sharding/hash_intervals.cpp187
-rw-r--r--ydb/core/tx/sharding/hash_intervals.h385
-rw-r--r--ydb/core/tx/sharding/hash_modulo.cpp208
-rw-r--r--ydb/core/tx/sharding/hash_modulo.h367
-rw-r--r--ydb/core/tx/sharding/hash_sharding.cpp5
-rw-r--r--ydb/core/tx/sharding/hash_sharding.h104
-rw-r--r--ydb/core/tx/sharding/hash_slider.cpp55
-rw-r--r--ydb/core/tx/sharding/hash_slider.h76
-rw-r--r--ydb/core/tx/sharding/random.cpp21
-rw-r--r--ydb/core/tx/sharding/random.h51
-rw-r--r--ydb/core/tx/sharding/sharding.cpp284
-rw-r--r--ydb/core/tx/sharding/sharding.h414
-rw-r--r--ydb/core/tx/sharding/ya.make5
-rw-r--r--ydb/library/conclusion/status.h20
-rw-r--r--ydb/services/bg_tasks/abstract/interface.h15
148 files changed, 4157 insertions, 1147 deletions
diff --git a/ydb/core/formats/arrow/arrow_filter.h b/ydb/core/formats/arrow/arrow_filter.h
index a83b37a783..347baf9f02 100644
--- a/ydb/core/formats/arrow/arrow_filter.h
+++ b/ydb/core/formats/arrow/arrow_filter.h
@@ -154,6 +154,27 @@ public:
Add(currentValue, sameValueCount);
}
+ template <class TGetterLambda>
+ struct TAdapterLambda {
+ private:
+ TGetterLambda Getter;
+ public:
+ TAdapterLambda(const TGetterLambda& getter)
+ : Getter(getter)
+ {
+
+ }
+
+ bool operator[](const ui32 index) const {
+ return Getter(index);
+ }
+ };
+
+ template <class TGetterLambda>
+ void ResetWithLambda(const ui32 count, const TGetterLambda getter) {
+ return Reset(count, TAdapterLambda<TGetterLambda>(getter));
+ }
+
ui32 Size() const {
return Count;
}
diff --git a/ydb/core/formats/arrow/arrow_helpers.cpp b/ydb/core/formats/arrow/arrow_helpers.cpp
index a6c7f8c742..ffc1edebb6 100644
--- a/ydb/core/formats/arrow/arrow_helpers.cpp
+++ b/ydb/core/formats/arrow/arrow_helpers.cpp
@@ -348,21 +348,44 @@ std::shared_ptr<arrow::RecordBatch> Reorder(const std::shared_ptr<arrow::RecordB
return (*res).record_batch();
}
-std::vector<std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch,
- const std::vector<ui32>& sharding, ui32 numShards) {
- Y_ABORT_UNLESS((size_t)batch->num_rows() == sharding.size());
+THashMap<ui64, std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch, const THashMap<ui64, std::vector<ui32>>& shardRows) {
+ AFL_VERIFY(batch);
+ std::shared_ptr<arrow::UInt64Array> permutation;
+ {
+ arrow::UInt64Builder builder;
+ Y_VERIFY_OK(builder.Reserve(batch->num_rows()));
- std::vector<std::vector<ui32>> shardRows(numShards);
- for (size_t row = 0; row < sharding.size(); ++row) {
- ui32 shardNo = sharding[row];
- Y_ABORT_UNLESS(shardNo < numShards);
- shardRows[shardNo].push_back(row);
+ for (auto&& [shardId, rowIdxs]: shardRows) {
+ for (auto& row : rowIdxs) {
+ Y_VERIFY_OK(builder.Append(row));
+ }
+ }
+ Y_VERIFY_OK(builder.Finish(&permutation));
}
+ auto reorderedBatch = Reorder(batch, permutation, false);
+
+ THashMap<ui64, std::shared_ptr<arrow::RecordBatch>> out;
+
+ int offset = 0;
+ for (auto&& [shardId, shardRowIdxs] : shardRows) {
+ if (shardRowIdxs.empty()) {
+ continue;
+ }
+ out.emplace(shardId, reorderedBatch->Slice(offset, shardRowIdxs.size()));
+ offset += shardRowIdxs.size();
+ }
+
+ Y_ABORT_UNLESS(offset == batch->num_rows());
+ return out;
+}
+
+std::vector<std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch, const std::vector<std::vector<ui32>>& shardRows, const ui32 numShards) {
+ AFL_VERIFY(batch);
std::shared_ptr<arrow::UInt64Array> permutation;
{
arrow::UInt64Builder builder;
- Y_VERIFY_OK(builder.Reserve(sharding.size()));
+ Y_VERIFY_OK(builder.Reserve(batch->num_rows()));
for (ui32 shardNo = 0; shardNo < numShards; ++shardNo) {
for (auto& row : shardRows[shardNo]) {
@@ -389,6 +412,20 @@ std::vector<std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared
return out;
}
+std::vector<std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch,
+ const std::vector<ui32>& sharding, ui32 numShards) {
+ AFL_VERIFY(batch);
+ Y_ABORT_UNLESS((size_t)batch->num_rows() == sharding.size());
+
+ std::vector<std::vector<ui32>> shardRows(numShards);
+ for (size_t row = 0; row < sharding.size(); ++row) {
+ ui32 shardNo = sharding[row];
+ Y_ABORT_UNLESS(shardNo < numShards);
+ shardRows[shardNo].push_back(row);
+ }
+ return ShardingSplit(batch, shardRows, numShards);
+}
+
void DedupSortedBatch(const std::shared_ptr<arrow::RecordBatch>& batch,
const std::shared_ptr<arrow::Schema>& sortingKey,
std::vector<std::shared_ptr<arrow::RecordBatch>>& out) {
diff --git a/ydb/core/formats/arrow/arrow_helpers.h b/ydb/core/formats/arrow/arrow_helpers.h
index 4a5acb22ea..60944ae8e8 100644
--- a/ydb/core/formats/arrow/arrow_helpers.h
+++ b/ydb/core/formats/arrow/arrow_helpers.h
@@ -83,6 +83,8 @@ std::shared_ptr<arrow::RecordBatch> ToBatch(const std::shared_ptr<arrow::Table>&
std::shared_ptr<arrow::RecordBatch> CombineBatches(const std::vector<std::shared_ptr<arrow::RecordBatch>>& batches);
std::shared_ptr<arrow::RecordBatch> MergeColumns(const std::vector<std::shared_ptr<arrow::RecordBatch>>& rb);
std::vector<std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch, const std::vector<ui32>& sharding, ui32 numShards);
+std::vector<std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch, const std::vector<std::vector<ui32>>& shardRows, const ui32 numShards);
+THashMap<ui64, std::shared_ptr<arrow::RecordBatch>> ShardingSplit(const std::shared_ptr<arrow::RecordBatch>& batch, const THashMap<ui64, std::vector<ui32>>& shardRows);
std::unique_ptr<arrow::ArrayBuilder> MakeBuilder(const std::shared_ptr<arrow::Field>& field);
std::unique_ptr<arrow::ArrayBuilder> MakeBuilder(const std::shared_ptr<arrow::DataType>& type);
diff --git a/ydb/core/formats/arrow/hash/calcer.h b/ydb/core/formats/arrow/hash/calcer.h
index 34e9585669..d82f669fbe 100644
--- a/ydb/core/formats/arrow/hash/calcer.h
+++ b/ydb/core/formats/arrow/hash/calcer.h
@@ -61,16 +61,20 @@ public:
TXX64(const std::vector<TString>& columnNames, const ENoColumnPolicy noColumnPolicy, const ui64 seed = 0);
TXX64(const std::vector<std::string>& columnNames, const ENoColumnPolicy noColumnPolicy, const ui64 seed = 0);
+ const std::vector<TString>& GetColumnNames() const {
+ return ColumnNames;
+ }
+
static void AppendField(const std::shared_ptr<arrow::Array>& array, const int row, NXX64::TStreamStringHashCalcer& hashCalcer);
static void AppendField(const std::shared_ptr<arrow::Scalar>& scalar, NXX64::TStreamStringHashCalcer& hashCalcer);
static ui64 CalcHash(const std::shared_ptr<arrow::Scalar>& scalar);
std::optional<std::vector<ui64>> Execute(const std::shared_ptr<arrow::RecordBatch>& batch) const;
- template <class TDataContainer>
- std::shared_ptr<arrow::Array> ExecuteToArray(const std::shared_ptr<TDataContainer>& batch) const {
+ template <class TDataContainer, class TAcceptor>
+ [[nodiscard]] bool ExecuteToArrayImpl(const std::shared_ptr<TDataContainer>& batch, const TAcceptor& acceptor) const {
std::vector<std::shared_ptr<typename NAdapter::TDataBuilderPolicy<TDataContainer>::TColumn>> columns = GetColumns(batch);
if (columns.empty()) {
- return nullptr;
+ return false;
}
std::vector<NAccessor::IChunkedArray::TReader> columnScanners;
@@ -79,9 +83,6 @@ public:
}
- auto builder = NArrow::MakeBuilder(arrow::TypeTraits<arrow::UInt64Type>::type_singleton());
- auto& intBuilder = static_cast<arrow::UInt64Builder&>(*builder);
- TStatusValidator::Validate(intBuilder.Reserve(batch->num_rows()));
{
NXX64::TStreamStringHashCalcer hashCalcer(Seed);
for (int row = 0; row < batch->num_rows(); ++row) {
@@ -90,12 +91,42 @@ public:
auto address = column.GetReadChunk(row);
AppendField(address.GetArray(), address.GetPosition(), hashCalcer);
}
- intBuilder.UnsafeAppend(hashCalcer.Finish());
+ acceptor(hashCalcer.Finish());
}
}
+ return true;
+ }
+
+ template <class TDataContainer>
+ std::shared_ptr<arrow::Array> ExecuteToArray(const std::shared_ptr<TDataContainer>& batch) const {
+ auto builder = NArrow::MakeBuilder(arrow::TypeTraits<arrow::UInt64Type>::type_singleton());
+ auto& intBuilder = static_cast<arrow::UInt64Builder&>(*builder);
+ TStatusValidator::Validate(intBuilder.Reserve(batch->num_rows()));
+
+ const auto acceptor = [&](const ui64 hash) {
+ intBuilder.UnsafeAppend(hash);
+ };
+
+ if (!ExecuteToArrayImpl(batch, acceptor)) {
+ return nullptr;
+ }
+
return NArrow::TStatusValidator::GetValid(builder->Finish());
}
+ template <class TDataContainer>
+ std::vector<ui64> ExecuteToVector(const std::shared_ptr<TDataContainer>& batch) const {
+ std::vector<ui64> result;
+ result.reserve(batch->num_rows());
+
+ const auto acceptor = [&](const ui64 hash) {
+ result.emplace_back(hash);
+ };
+
+ AFL_VERIFY(ExecuteToArrayImpl(batch, acceptor));
+ return result;
+ }
+
};
}
diff --git a/ydb/core/kqp/common/kqp_resolve.h b/ydb/core/kqp/common/kqp_resolve.h
index 9cd41e4387..7a079f3fa4 100644
--- a/ydb/core/kqp/common/kqp_resolve.h
+++ b/ydb/core/kqp/common/kqp_resolve.h
@@ -27,7 +27,7 @@ enum class ETableKind {
struct TTableConstInfo : public TAtomicRefCount<TTableConstInfo> {
TString Path;
- TMap<TString, NSharding::TShardingBase::TColumn> Columns;
+ TMap<TString, NSharding::IShardingBase::TColumn> Columns;
TVector<TString> KeyColumns;
TVector<NScheme::TTypeInfo> KeyColumnTypes;
ETableKind TableKind = ETableKind::Unknown;
@@ -43,7 +43,7 @@ struct TTableConstInfo : public TAtomicRefCount<TTableConstInfo> {
return;
}
- NSharding::TShardingBase::TColumn column;
+ NSharding::IShardingBase::TColumn column;
column.Id = phyColumn.GetId().GetId();
if (phyColumn.GetTypeId() != NScheme::NTypeIds::Pg) {
@@ -83,7 +83,7 @@ struct TTableConstInfo : public TAtomicRefCount<TTableConstInfo> {
<< ", table: " << Path
<< ", column: " << columnName);
- NSharding::TShardingBase::TColumn column;
+ NSharding::IShardingBase::TColumn column;
column.Id = systemColumn->ColumnId;
column.Type = NScheme::TTypeInfo(systemColumn->TypeId);
column.NotNull = false;
@@ -143,7 +143,7 @@ public:
return TableConstInfo->Path;
}
- const TMap<TString, NSharding::TShardingBase::TColumn>& GetColumns() const {
+ const TMap<TString, NSharding::IShardingBase::TColumn>& GetColumns() const {
return TableConstInfo->Columns;
}
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.cpp
index 76f9eea6d4..6e01379176 100644
--- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.cpp
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.cpp
@@ -30,8 +30,7 @@ TConclusionStatus ITableStoreOperation::Deserialize(const NYql::TObjectSettingsI
return TConclusionStatus::Success();
}
-void ITableStoreOperation::SerializeScheme(NKikimrSchemeOp::TModifyScheme& scheme, const bool isStandalone) const {
- scheme.SetWorkingDir(WorkingDir);
+void ITableStoreOperation::DoSerializeScheme(NKikimrSchemeOp::TModifyScheme& scheme, const bool isStandalone) const {
if (isStandalone) {
scheme.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterColumnTable);
NKikimrSchemeOp::TAlterColumnTable* alter = scheme.MutableAlterColumnTable();
@@ -48,4 +47,9 @@ void ITableStoreOperation::SerializeScheme(NKikimrSchemeOp::TModifyScheme& schem
}
}
+void ITableStoreOperation::SerializeScheme(NKikimrSchemeOp::TModifyScheme& scheme, const bool isStandalone) const {
+ scheme.SetWorkingDir(WorkingDir);
+ DoSerializeScheme(scheme, isStandalone);
+}
+
}
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.h b/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.h
index aee052d7a2..0eda9de572 100644
--- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.h
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/abstract.h
@@ -10,7 +10,7 @@ public:
private:
TString PresetName = "default";
TString WorkingDir;
- TString StoreName;
+ YDB_READONLY_DEF(TString, StoreName);
public:
virtual ~ITableStoreOperation() {};
@@ -20,6 +20,7 @@ public:
private:
virtual TConclusionStatus DoDeserialize(NYql::TObjectSettingsImpl::TFeaturesExtractor& features) = 0;
virtual void DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchema& scheme) const = 0;
+ virtual void DoSerializeScheme(NKikimrSchemeOp::TModifyScheme& scheme, const bool isStandalone) const;
};
}
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.cpp
new file mode 100644
index 0000000000..1bfc18c9d9
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.cpp
@@ -0,0 +1,28 @@
+#include "alter_sharding.h"
+#include <util/string/type.h>
+
+namespace NKikimr::NKqp {
+
+TConclusionStatus TAlterShardingOperation::DoDeserialize(NYql::TObjectSettingsImpl::TFeaturesExtractor& features) {
+ const std::optional<TString> modification = features.Extract<TString>("MODIFICATION");
+ if (!modification) {
+ return TConclusionStatus::Fail("modification type not specified in request");
+ }
+ if (*modification == "SPLIT") {
+ Increase = true;
+ } else if (*modification == "MERGE") {
+ return TConclusionStatus::Fail("modification is impossible yet");
+ } else {
+ return TConclusionStatus::Fail("undefined modification: \"" + *modification + "\"");
+ }
+ return TConclusionStatus::Success();
+}
+
+void TAlterShardingOperation::DoSerializeScheme(NKikimrSchemeOp::TModifyScheme& scheme, const bool isStandalone) const {
+ AFL_VERIFY(!isStandalone);
+ scheme.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterColumnTable);
+ scheme.MutableAlterColumnTable()->SetName(GetStoreName());
+ *scheme.MutableAlterColumnTable()->MutableReshardColumnTable() = NKikimrSchemeOp::TReshardColumnTable();
+}
+
+}
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.h b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.h
new file mode 100644
index 0000000000..52f58e14d7
--- /dev/null
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/alter_sharding.h
@@ -0,0 +1,26 @@
+#include "abstract.h"
+#include <ydb/core/tx/columnshard/engines/scheme/statistics/abstract/constructor.h>
+
+namespace NKikimr::NKqp {
+
+class TAlterShardingOperation: public ITableStoreOperation {
+private:
+ static TString GetTypeName() {
+ return "ALTER_SHARDING";
+ }
+
+ static inline const auto Registrator = TFactory::TRegistrator<TAlterShardingOperation>(GetTypeName());
+private:
+ std::optional<bool> Increase;
+ virtual void DoSerializeScheme(NKikimrSchemeOp::TAlterColumnTableSchema& /*scheme*/) const override {
+ AFL_VERIFY(false);
+ }
+ virtual void DoSerializeScheme(NKikimrSchemeOp::TModifyScheme& scheme, const bool isStandalone) const override;
+
+public:
+ TConclusionStatus DoDeserialize(NYql::TObjectSettingsImpl::TFeaturesExtractor& features) override;
+
+};
+
+}
+
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_opt.cpp b/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_opt.cpp
index 5cdfd696e2..72e9616fba 100644
--- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_opt.cpp
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/upsert_opt.cpp
@@ -7,7 +7,7 @@ namespace NKikimr::NKqp {
TConclusionStatus TUpsertOptionsOperation::DoDeserialize(NYql::TObjectSettingsImpl::TFeaturesExtractor& features) {
auto value = features.Extract<bool>("SCHEME_NEED_ACTUALIZATION", false);
if (!value) {
- TConclusionStatus::Fail("Incorrect value for SCHEME_NEED_ACTUALIZATION: cannot parse as boolean");
+ return TConclusionStatus::Fail("Incorrect value for SCHEME_NEED_ACTUALIZATION: cannot parse as boolean");
}
SchemeNeedActualization = *value;
ExternalGuaranteeExclusivePK = features.Extract<bool>("EXTERNAL_GUARANTEE_EXCLUSIVE_PK");
diff --git a/ydb/core/kqp/gateway/behaviour/tablestore/operations/ya.make b/ydb/core/kqp/gateway/behaviour/tablestore/operations/ya.make
index 17d70abf65..d13c57aa70 100644
--- a/ydb/core/kqp/gateway/behaviour/tablestore/operations/ya.make
+++ b/ydb/core/kqp/gateway/behaviour/tablestore/operations/ya.make
@@ -10,6 +10,7 @@ SRCS(
GLOBAL upsert_stat.cpp
GLOBAL drop_stat.cpp
GLOBAL upsert_opt.cpp
+ GLOBAL alter_sharding.cpp
)
PEERDIR(
diff --git a/ydb/core/protos/data_events.proto b/ydb/core/protos/data_events.proto
index 80b0662b2b..627296a769 100644
--- a/ydb/core/protos/data_events.proto
+++ b/ydb/core/protos/data_events.proto
@@ -111,6 +111,7 @@ message TEvWrite {
// Writes are performed "over" the specified snapshot when specified
// This mostly affects the minimum MVCC version of the resulting write
optional TMvccSnapshot MvccSnapshot = 8;
+ optional uint32 GranuleShardingVersionId = 9;
}
message TEvWriteResult {
@@ -121,7 +122,7 @@ message TEvWriteResult {
STATUS_ABORTED = 3;
STATUS_INTERNAL_ERROR = 4;
STATUS_OVERLOADED = 5;
- STATUS_CANCELLED = 6;
+ STATUS_CANCELLED = 6;
STATUS_BAD_REQUEST = 7;
STATUS_SCHEME_CHANGED = 8;
STATUS_LOCKS_BROKEN = 9;
diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
index e4b73013fb..0c2ca24845 100644
--- a/ydb/core/protos/flat_scheme_op.proto
+++ b/ydb/core/protos/flat_scheme_op.proto
@@ -18,6 +18,7 @@ import "ydb/library/actors/protos/actors.proto";
import "ydb/library/mkql_proto/protos/minikql.proto";
import "ydb/core/protos/index_builder.proto";
import "ydb/core/tx/columnshard/engines/scheme/statistics/protos/data.proto";
+import "ydb/core/tx/columnshard/common/protos/snapshot.proto";
import "google/protobuf/empty.proto";
@@ -649,11 +650,56 @@ message TAlterColumnStore {
repeated TRemoveColumnTableTtlSettingsPreset RESERVED_RemoveTtlSettingsPresets = 7;
}
-message TColumnTableSharding {
- // Version is incremented each time table sharding is changed
- // This field is managed by the system.
- optional uint64 Version = 1;
+message TConsistencyShardingTablet {
+ optional uint64 TabletId = 1;
+ optional uint64 HashIntervalLeftClosed = 2;
+ optional uint64 HashIntervalRightOpened = 3;
+}
+
+message TModuloShardingTablet {
+ optional uint64 TabletId = 1;
+ repeated uint32 AppropriateMods = 2;
+}
+
+message THashShardingInfo {
+ repeated string ColumnNames = 1;
+}
+
+message TGranuleConsistencyShardingInfo {
+ optional THashShardingInfo Hashing = 1;
+ optional TConsistencyShardingTablet ShardInfo = 2;
+}
+
+message TGranuleModuloShardingInfo {
+ optional THashShardingInfo Hashing = 1;
+ optional TModuloShardingTablet ShardInfo = 2;
+ optional uint64 ModuloPartsCount = 3;
+}
+message TGranuleShardingLogicContainer {
+ optional string ClassName = 1;
+ oneof Implementation {
+ TGranuleConsistencyShardingInfo Consistency = 20;
+ TGranuleModuloShardingInfo Modulo = 21;
+ }
+}
+
+message TGranuleShardingInfo {
+ optional uint64 PathId = 1;
+ optional uint64 VersionId = 2;
+ optional TGranuleShardingLogicContainer Container = 3;
+}
+
+message TGranuleShardInfo {
+ optional uint64 TabletId = 1;
+ optional uint32 SequenceIdx = 2;
+ optional bool IsOpenForRead = 3;
+ optional bool IsOpenForWrite = 4;
+ optional NKikimrColumnShardProto.TSnapshot OpenForWriteSnapshot = 5;
+ optional uint64 ShardingVersion = 6;
+}
+
+message TColumnTableSharding {
// An ordered list of currently used column shards
// This field is managed by the system.
repeated uint64 ColumnShards = 2;
@@ -680,12 +726,18 @@ message TColumnTableSharding {
// Argument for HASH_FUNCTION_CLOUD_LOGS
optional uint32 ActiveShardsCount = 4;
+ repeated TConsistencyShardingTablet TabletsForConsistency = 5;
+
+ optional uint32 ModuloPartsCount = 6;
+ repeated TModuloShardingTablet TabletsForModulo = 7;
}
oneof Method {
TRandomSharding RandomSharding = 5;
THashSharding HashSharding = 6;
}
+
+ repeated TGranuleShardInfo ShardsInfo = 7;
}
message TColumnTableDescription {
@@ -729,6 +781,13 @@ message TAlterColumnTable {
optional string AlterSchemaPresetName = 4;
optional string RESERVED_AlterTtlSettingsPresetName = 5;
+
+ optional TAlterShards AlterShards = 6;
+ optional TReshardColumnTable ReshardColumnTable = 7;
+}
+
+message TReshardColumnTable {
+ optional bool Increase = 1;
}
message TLoginCreateUser {
@@ -1540,6 +1599,46 @@ message TDropIndex {
optional string IndexName = 2;
}
+message TConsistencyShardsModification {
+ repeated TConsistencyShardingTablet Shards = 1;
+}
+
+message TModuloShardsModification {
+ repeated TModuloShardingTablet Shards = 1;
+ optional uint32 PartsCount = 2;
+}
+
+message TShardingModification {
+ repeated uint64 NewShardIds = 1;
+ repeated uint64 DeleteShardIds = 2;
+
+ repeated uint64 CloseWriteIds = 3;
+ repeated uint64 CloseReadIds = 4;
+ repeated uint64 OpenWriteIds = 5;
+ repeated uint64 OpenReadIds = 6;
+
+ oneof Implementation {
+ TConsistencyShardsModification Consistency = 20;
+ TModuloShardsModification Modulo = 21;
+ }
+}
+
+message TShardingTransfer {
+ optional uint64 DestinationTabletId = 1;
+ repeated uint64 SourceTabletIds = 2;
+}
+
+message TShardingTransfers {
+ repeated TShardingTransfer Transfers = 1;
+}
+
+message TAlterShards {
+ oneof Implementation {
+ TShardingModification Modification = 20;
+ TShardingTransfers Transfer = 21;
+ }
+}
+
// Request for scheme modification
// Has only one of the operations
message TModifyScheme {
@@ -1731,7 +1830,6 @@ message TPathVersion {
optional uint64 FileStoreVersion = 15;
optional uint64 ColumnStoreVersion = 16;
optional uint64 ColumnTableVersion = 17;
- optional uint64 ColumnTableShardingVersion = 18;
optional uint64 SubDomainStateVersion = 19;
optional uint64 ColumnTableSchemaVersion = 20;
optional uint64 ColumnTableTtlSettingsVersion = 21;
diff --git a/ydb/core/protos/tx_columnshard.proto b/ydb/core/protos/tx_columnshard.proto
index 17a143f754..c254df1f64 100644
--- a/ydb/core/protos/tx_columnshard.proto
+++ b/ydb/core/protos/tx_columnshard.proto
@@ -79,6 +79,7 @@ message TEvWrite {
optional TMetadata Meta = 7;
optional NKikimrLongTxService.TLongTxId LongTxId = 8;
optional uint32 WritePartId = 9 [default = 0];
+ optional uint32 GranuleShardingVersion = 10;
}
message TEvWriteResult {
@@ -284,6 +285,8 @@ message TSchemaTxBody {
TAlterTable AlterTable = 5;
TAlterStore AlterStore = 6;
}
+
+ optional NKikimrSchemeOp.TGranuleShardingInfo GranuleShardingInfo = 20;
}
message TTtlTxBody {
diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make
index 53ec70b60b..1f3012f1ca 100644
--- a/ydb/core/protos/ya.make
+++ b/ydb/core/protos/ya.make
@@ -168,6 +168,7 @@ PEERDIR(
ydb/core/tx/columnshard/engines/scheme/statistics/protos
ydb/core/tx/columnshard/engines/protos
ydb/core/formats/arrow/protos
+ ydb/core/tx/columnshard/common/protos
)
CPP_PROTO_PLUGIN0(config_proto_plugin ydb/core/config/tools/protobuf_plugin)
diff --git a/ydb/core/tx/columnshard/bg_tasks/abstract/session.cpp b/ydb/core/tx/columnshard/bg_tasks/abstract/session.cpp
index d6fad196cf..2722d80017 100644
--- a/ydb/core/tx/columnshard/bg_tasks/abstract/session.cpp
+++ b/ydb/core/tx/columnshard/bg_tasks/abstract/session.cpp
@@ -1,12 +1,13 @@
#include "session.h"
#include "adapter.h"
#include <ydb/public/api/protos/ydb_operation.pb.h>
+#include <ydb/public/lib/operation_id/protos/operation_id.pb.h>
namespace NKikimr::NOlap::NBackground {
Ydb::Operations::Operation TSessionInfoReport::SerializeToProto() const {
Ydb::Operations::Operation result;
- result.set_id(ClassName + "::" + Identifier);
+ result.set_id("/" + ::ToString((int)Ydb::TOperationId::SS_BG_TASKS) + "?type=" + ClassName + "&id=" + Identifier);
result.set_ready(IsFinished);
return result;
}
diff --git a/ydb/core/tx/columnshard/bg_tasks/abstract/session.h b/ydb/core/tx/columnshard/bg_tasks/abstract/session.h
index 027d80a32f..5f48bfed54 100644
--- a/ydb/core/tx/columnshard/bg_tasks/abstract/session.h
+++ b/ydb/core/tx/columnshard/bg_tasks/abstract/session.h
@@ -87,7 +87,7 @@ public:
virtual bool IsReadyForStart() const = 0;
virtual bool IsFinished() const = 0;
- virtual bool IsReadyForRemove() const = 0;
+ virtual bool IsReadyForRemoveOnFinished() const = 0;
};
template <class TProtoLogicExt, class TProtoProgressExt, class TProtoStateExt>
diff --git a/ydb/core/tx/columnshard/bg_tasks/abstract/ya.make b/ydb/core/tx/columnshard/bg_tasks/abstract/ya.make
index 4a25babe82..4cdec62648 100644
--- a/ydb/core/tx/columnshard/bg_tasks/abstract/ya.make
+++ b/ydb/core/tx/columnshard/bg_tasks/abstract/ya.make
@@ -13,6 +13,7 @@ PEERDIR(
ydb/library/accessor
ydb/library/services
ydb/core/tx/columnshard/bg_tasks/protos
+ ydb/public/lib/operation_id/protos
ydb/public/api/protos
)
diff --git a/ydb/core/tx/columnshard/bg_tasks/transactions/tx_remove.cpp b/ydb/core/tx/columnshard/bg_tasks/transactions/tx_remove.cpp
index 611f786de6..ae3a49dd71 100644
--- a/ydb/core/tx/columnshard/bg_tasks/transactions/tx_remove.cpp
+++ b/ydb/core/tx/columnshard/bg_tasks/transactions/tx_remove.cpp
@@ -9,7 +9,7 @@ bool TTxRemoveSession::Execute(NTabletFlatExecutor::TTransactionContext& txc, co
}
void TTxRemoveSession::Complete(const TActorContext& /*ctx*/) {
- AFL_VERIFY(Sessions->RemoveSession(ClassName, Identifier));
+ AFL_VERIFY(Sessions->RemoveSession(ClassName, Identifier))("class_name", ClassName)("id", Identifier);
}
}
diff --git a/ydb/core/tx/columnshard/bg_tasks/transactions/tx_save_state.cpp b/ydb/core/tx/columnshard/bg_tasks/transactions/tx_save_state.cpp
index 1eda1b3eb9..d47dfd0f1a 100644
--- a/ydb/core/tx/columnshard/bg_tasks/transactions/tx_save_state.cpp
+++ b/ydb/core/tx/columnshard/bg_tasks/transactions/tx_save_state.cpp
@@ -9,7 +9,7 @@ bool TTxSaveSessionState::Execute(NTabletFlatExecutor::TTransactionContext& txc,
}
void TTxSaveSessionState::DoComplete(const TActorContext& ctx) {
- if (Session->GetLogicContainer()->IsReadyForRemove()) {
+ if (Session->GetLogicContainer()->IsFinished() && Session->GetLogicContainer()->IsReadyForRemoveOnFinished()) {
ctx.Send(Adapter->GetTabletActorId(), new TEvRemoveSession(Session->GetLogicClassName(), Session->GetIdentifier()));
} else if (!Session->IsRunning() && Session->GetLogicContainer()->IsReadyForStart()) {
TStartContext context(Session, Adapter);
diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp
index 6e1ea436fb..7e92f9a5e4 100644
--- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp
+++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp
@@ -50,7 +50,7 @@ bool TTxWrite::Execute(TTransactionContext& txc, const TActorContext&) {
auto writeId = TWriteId(writeMeta.GetWriteId());
if (!operation) {
NIceDb::TNiceDb db(txc.DB);
- writeId = Self->GetLongTxWrite(db, writeMeta.GetLongTxIdUnsafe(), writeMeta.GetWritePartId());
+ writeId = Self->GetLongTxWrite(db, writeMeta.GetLongTxIdUnsafe(), writeMeta.GetWritePartId(), writeMeta.GetGranuleShardingVersion());
aggr->AddWriteId(writeId);
}
diff --git a/ydb/core/tx/columnshard/columnshard__init.cpp b/ydb/core/tx/columnshard/columnshard__init.cpp
index 82f1b114ec..3ff5452696 100644
--- a/ydb/core/tx/columnshard/columnshard__init.cpp
+++ b/ydb/core/tx/columnshard/columnshard__init.cpp
@@ -185,7 +185,12 @@ bool TTxInit::ReadEverything(TTransactionContext& txc, const TActorContext& ctx)
Y_ABORT_UNLESS(proto.ParseFromString(rowset.GetValue<Schema::LongTxWrites::LongTxId>()));
const auto longTxId = NLongTxService::TLongTxId::FromProto(proto);
- Self->LoadLongTxWrite(writeId, writePartId, longTxId);
+ std::optional<ui32> granuleShardingVersion;
+ if (rowset.HaveValue<Schema::LongTxWrites::GranuleShardingVersion>() && rowset.GetValue<Schema::LongTxWrites::GranuleShardingVersion>()) {
+ granuleShardingVersion = rowset.GetValue<Schema::LongTxWrites::GranuleShardingVersion>();
+ }
+
+ Self->LoadLongTxWrite(writeId, writePartId, longTxId, granuleShardingVersion);
if (!rowset.Next()) {
return false;
diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp
index 06e92ad0a8..0113746521 100644
--- a/ydb/core/tx/columnshard/columnshard__write.cpp
+++ b/ydb/core/tx/columnshard/columnshard__write.cpp
@@ -162,7 +162,12 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex
const TString dedupId = record.GetDedupId();
const auto source = ev->Sender;
- NEvWrite::TWriteMeta writeMeta(writeId, tableId, source);
+ std::optional<ui32> granuleShardingVersion;
+ if (record.HasGranuleShardingVersion()) {
+ granuleShardingVersion = record.GetGranuleShardingVersion();
+ }
+
+ NEvWrite::TWriteMeta writeMeta(writeId, tableId, source, granuleShardingVersion);
writeMeta.SetDedupId(dedupId);
Y_ABORT_UNLESS(record.HasLongTxId());
writeMeta.SetLongTxId(NLongTxService::TLongTxId::FromProto(record.GetLongTxId()));
@@ -362,7 +367,7 @@ void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActor
auto overloadStatus = CheckOverloaded(tableId);
if (overloadStatus != EOverloadStatus::None) {
- NEvWrite::TWriteData writeData(NEvWrite::TWriteMeta(0, tableId, source), arrowData, nullptr, nullptr);
+ NEvWrite::TWriteData writeData(NEvWrite::TWriteMeta(0, tableId, source, {}), arrowData, nullptr, nullptr);
std::unique_ptr<NActors::IEventBase> result = NEvents::TDataEvents::TEvWriteResult::BuildError(TabletID(), 0, NKikimrDataEvents::TEvWriteResult::STATUS_OVERLOADED, "overload data error");
OverloadWriteFail(overloadStatus, writeData, cookie, std::move(result), ctx);
return;
@@ -370,7 +375,12 @@ void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActor
auto wg = WritesMonitor.RegisterWrite(arrowData->GetSize());
- auto writeOperation = OperationsManager->RegisterOperation(lockId, cookie);
+ std::optional<ui32> granuleShardingVersionId;
+ if (record.HasGranuleShardingVersionId()) {
+ granuleShardingVersionId = record.GetGranuleShardingVersionId();
+ }
+
+ auto writeOperation = OperationsManager->RegisterOperation(lockId, cookie, granuleShardingVersionId);
Y_ABORT_UNLESS(writeOperation);
writeOperation->SetBehaviour(behaviour);
writeOperation->Start(*this, tableId, arrowData, source, ctx);
diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp
index 5bc2946deb..4addc50c4d 100644
--- a/ydb/core/tx/columnshard/columnshard_impl.cpp
+++ b/ydb/core/tx/columnshard/columnshard_impl.cpp
@@ -200,7 +200,7 @@ ui64 TColumnShard::GetMinReadStep() const {
return minReadStep;
}
-TWriteId TColumnShard::HasLongTxWrite(const NLongTxService::TLongTxId& longTxId, const ui32 partId) {
+TWriteId TColumnShard::HasLongTxWrite(const NLongTxService::TLongTxId& longTxId, const ui32 partId) const {
auto it = LongTxWritesByUniqueId.find(longTxId.UniqueId);
if (it != LongTxWritesByUniqueId.end()) {
auto itPart = it->second.find(partId);
@@ -211,7 +211,7 @@ TWriteId TColumnShard::HasLongTxWrite(const NLongTxService::TLongTxId& longTxId,
return (TWriteId)0;
}
-TWriteId TColumnShard::GetLongTxWrite(NIceDb::TNiceDb& db, const NLongTxService::TLongTxId& longTxId, const ui32 partId) {
+TWriteId TColumnShard::GetLongTxWrite(NIceDb::TNiceDb& db, const NLongTxService::TLongTxId& longTxId, const ui32 partId, const std::optional<ui32> granuleShardingVersionId) {
auto it = LongTxWritesByUniqueId.find(longTxId.UniqueId);
if (it != LongTxWritesByUniqueId.end()) {
auto itPart = it->second.find(partId);
@@ -227,9 +227,10 @@ TWriteId TColumnShard::GetLongTxWrite(NIceDb::TNiceDb& db, const NLongTxService:
lw.WriteId = (ui64)writeId;
lw.WritePartId = partId;
lw.LongTxId = longTxId;
+ lw.GranuleShardingVersionId = granuleShardingVersionId;
it->second[partId] = &lw;
- Schema::SaveLongTxWrite(db, writeId, partId, longTxId);
+ Schema::SaveLongTxWrite(db, writeId, partId, longTxId, granuleShardingVersionId);
return writeId;
}
@@ -249,11 +250,12 @@ void TColumnShard::AddLongTxWrite(TWriteId writeId, ui64 txId) {
lw.PreparedTxId = txId;
}
-void TColumnShard::LoadLongTxWrite(TWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId) {
+void TColumnShard::LoadLongTxWrite(TWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId, const std::optional<ui32> granuleShardingVersion) {
auto& lw = LongTxWrites[writeId];
lw.WritePartId = writePartId;
lw.WriteId = (ui64)writeId;
lw.LongTxId = longTxId;
+ lw.GranuleShardingVersionId = granuleShardingVersion;
LongTxWritesByUniqueId[longTxId.UniqueId][writePartId] = &lw;
}
diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h
index f059faed10..1a451d36b7 100644
--- a/ydb/core/tx/columnshard/columnshard_impl.h
+++ b/ydb/core/tx/columnshard/columnshard_impl.h
@@ -409,6 +409,7 @@ private:
ui32 WritePartId;
NLongTxService::TLongTxId LongTxId;
ui64 PreparedTxId = 0;
+ std::optional<ui32> GranuleShardingVersionId;
};
class TWritesMonitor {
@@ -531,10 +532,10 @@ private:
return ProgressTxController->GetTxCompleteLag(mediatorTime);
}
- TWriteId HasLongTxWrite(const NLongTxService::TLongTxId& longTxId, const ui32 partId);
- TWriteId GetLongTxWrite(NIceDb::TNiceDb& db, const NLongTxService::TLongTxId& longTxId, const ui32 partId);
+ TWriteId HasLongTxWrite(const NLongTxService::TLongTxId& longTxId, const ui32 partId) const;
+ TWriteId GetLongTxWrite(NIceDb::TNiceDb& db, const NLongTxService::TLongTxId& longTxId, const ui32 partId, const std::optional<ui32> granuleShardingVersionId);
void AddLongTxWrite(TWriteId writeId, ui64 txId);
- void LoadLongTxWrite(TWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId);
+ void LoadLongTxWrite(TWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId, const std::optional<ui32> granuleShardingVersion);
bool RemoveLongTxWrite(NIceDb::TNiceDb& db, TWriteId writeId, ui64 txId = 0);
void TryAbortWrites(NIceDb::TNiceDb& db, NOlap::TDbWrapper& dbTable, THashSet<TWriteId>&& writesToAbort);
diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h
index a9cf777673..8bba15f162 100644
--- a/ydb/core/tx/columnshard/columnshard_schema.h
+++ b/ydb/core/tx/columnshard/columnshard_schema.h
@@ -54,7 +54,8 @@ struct Schema : NIceDb::Schema {
BackupIdsDeprecated,
ExportSessionsId,
PortionsTableId,
- BackgroundSessionsTableId
+ BackgroundSessionsTableId,
+ ShardingInfoTabletId
};
enum class ETierTables: ui32 {
@@ -194,9 +195,10 @@ struct Schema : NIceDb::Schema {
struct WriteId: Column<1, NScheme::NTypeIds::Uint64> {};
struct LongTxId : Column<2, NScheme::NTypeIds::String> {};
struct WritePartId: Column<3, NScheme::NTypeIds::Uint32> {};
+ struct GranuleShardingVersion: Column<4, NScheme::NTypeIds::Uint32> {};
using TKey = TableKey<WriteId>;
- using TColumns = TableColumns<WriteId, LongTxId, WritePartId>;
+ using TColumns = TableColumns<WriteId, LongTxId, WritePartId, GranuleShardingVersion>;
};
struct BlobsToKeep : Table<(ui32)ECommonTables::BlobsToKeep> {
@@ -315,10 +317,11 @@ struct Schema : NIceDb::Schema {
struct CreatedAt : Column<4, NScheme::NTypeIds::Uint64> {};
struct GlobalWriteId : Column<5, NScheme::NTypeIds::Uint64> {};
struct Metadata : Column<6, NScheme::NTypeIds::String> {};
- struct Cookie : Column<7, NScheme::NTypeIds::Uint64> {};
+ struct Cookie: Column<7, NScheme::NTypeIds::Uint64> {};
+ struct GranuleShardingVersionId: Column<8, NScheme::NTypeIds::Uint32> {};
using TKey = TableKey<WriteId>;
- using TColumns = TableColumns<LockId, WriteId, Status, CreatedAt, GlobalWriteId, Metadata, Cookie>;
+ using TColumns = TableColumns<LockId, WriteId, Status, CreatedAt, GlobalWriteId, Metadata, Cookie, GranuleShardingVersionId>;
};
struct OperationTxIds : NIceDb::Schema::Table<OperationTxIdsId> {
@@ -453,9 +456,10 @@ struct Schema : NIceDb::Schema {
struct XPlanStep: Column<4, NScheme::NTypeIds::Uint64> {};
struct XTxId: Column<5, NScheme::NTypeIds::Uint64> {};
struct Metadata: Column<6, NScheme::NTypeIds::String> {}; // NKikimrTxColumnShard.TIndexColumnMeta
+ struct ShardingVersion: Column<7, NScheme::NTypeIds::Uint64> {};
using TKey = TableKey<PathId, PortionId>;
- using TColumns = TableColumns<PathId, PortionId, SchemaVersion, XPlanStep, XTxId, Metadata>;
+ using TColumns = TableColumns<PathId, PortionId, SchemaVersion, XPlanStep, XTxId, Metadata, ShardingVersion>;
};
struct BackgroundSessions: Table<BackgroundSessionsTableId> {
@@ -470,6 +474,16 @@ struct Schema : NIceDb::Schema {
using TColumns = TableColumns<ClassName, Identifier, StatusChannel, LogicDescription, Progress, State>;
};
+ struct ShardingInfo : Table<ShardingInfoTabletId> {
+ struct PathId : Column<1, NScheme::NTypeIds::Uint64> {};
+ struct VersionId : Column<2, NScheme::NTypeIds::Uint64> {};
+ struct Snapshot : Column<3, NScheme::NTypeIds::String> {};
+ struct Logic : Column<4, NScheme::NTypeIds::String> {};
+
+ using TKey = TableKey<PathId, VersionId>;
+ using TColumns = TableColumns<PathId, VersionId, Snapshot, Logic>;
+ };
+
using TTables = SchemaTables<
Value,
TxInfo,
@@ -500,7 +514,8 @@ struct Schema : NIceDb::Schema {
DestinationSessions,
OperationTxIds,
IndexPortions,
- BackgroundSessions
+ BackgroundSessions,
+ ShardingInfo
>;
//
@@ -654,14 +669,16 @@ struct Schema : NIceDb::Schema {
db.Table<TableInfo>().Key(pathId).Delete();
}
- static void SaveLongTxWrite(NIceDb::TNiceDb& db, TWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId) {
+ static void SaveLongTxWrite(NIceDb::TNiceDb& db, TWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId, const std::optional<ui32> granuleShardingVersion) {
NKikimrLongTxService::TLongTxId proto;
longTxId.ToProto(&proto);
TString serialized;
Y_ABORT_UNLESS(proto.SerializeToString(&serialized));
db.Table<LongTxWrites>().Key((ui64)writeId).Update(
- NIceDb::TUpdate<LongTxWrites::LongTxId>(serialized),
- NIceDb::TUpdate<LongTxWrites::WritePartId>(writePartId));
+ NIceDb::TUpdate<LongTxWrites::LongTxId>(serialized),
+ NIceDb::TUpdate<LongTxWrites::WritePartId>(writePartId),
+ NIceDb::TUpdate<LongTxWrites::GranuleShardingVersion>(granuleShardingVersion.value_or(0))
+ );
}
static void EraseLongTxWrite(NIceDb::TNiceDb& db, TWriteId writeId) {
diff --git a/ydb/core/tx/columnshard/common/snapshot.cpp b/ydb/core/tx/columnshard/common/snapshot.cpp
index a38565ee34..d76b847d8e 100644
--- a/ydb/core/tx/columnshard/common/snapshot.cpp
+++ b/ydb/core/tx/columnshard/common/snapshot.cpp
@@ -14,4 +14,16 @@ NKikimrColumnShardProto::TSnapshot TSnapshot::SerializeToProto() const {
return result;
}
+TConclusionStatus TSnapshot::DeserializeFromString(const TString& data) {
+ NKikimrColumnShardProto::TSnapshot proto;
+ if (!proto.ParseFromArray(data.data(), data.size())) {
+ return TConclusionStatus::Fail("cannot parse string as snapshot proto");
+ }
+ return DeserializeFromProto(proto);
+}
+
+TString TSnapshot::SerializeToString() const {
+ return SerializeToProto().SerializeAsString();
+}
+
};
diff --git a/ydb/core/tx/columnshard/common/snapshot.h b/ydb/core/tx/columnshard/common/snapshot.h
index 22466cb555..a9b7e89a26 100644
--- a/ydb/core/tx/columnshard/common/snapshot.h
+++ b/ydb/core/tx/columnshard/common/snapshot.h
@@ -79,6 +79,10 @@ public:
return TConclusionStatus::Success();
}
+ TConclusionStatus DeserializeFromString(const TString& data);
+
+ TString SerializeToString() const;
+
TString DebugString() const;
};
diff --git a/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h b/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h
index d4205a0db6..f69e08334d 100644
--- a/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h
+++ b/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h
@@ -20,7 +20,7 @@ namespace NKikimr::NOlap::NDataSharing::NEvents {
class TPathIdData {
private:
YDB_READONLY(ui64, PathId, 0);
- YDB_READONLY_DEF(std::vector<TPortionInfo>, Portions);
+ YDB_ACCESSOR_DEF(std::vector<TPortionInfo>, Portions);
TPathIdData() = default;
diff --git a/ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp b/ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp
index e09e894f8f..96b329ad5a 100644
--- a/ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp
+++ b/ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp
@@ -1,10 +1,11 @@
#include "destination.h"
+
#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h>
#include <ydb/core/tx/columnshard/data_locks/locks/list.h>
#include <ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h>
+#include <ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.h>
#include <ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_finish_ack_from_initiator.h>
#include <ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_finish_from_source.h>
-#include <ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.h>
#include <ydb/core/tx/columnshard/engines/column_engine_logs.h>
#include <ydb/core/tx/columnshard/hooks/abstract/abstract.h>
@@ -16,27 +17,9 @@ NKikimr::TConclusionStatus TDestinationSession::DataReceived(THashMap<ui64, NEve
auto it = PathIds.find(i.first);
AFL_VERIFY(it != PathIds.end())("path_id_undefined", i.first);
for (auto&& portion : i.second.DetachPortions()) {
- ui32 contains = 0;
- ui32 notContains = 0;
- THashMap<TString, THashSet<TUnifiedBlobId>> blobIds;
- portion.FillBlobIdsByStorage(blobIds, index.GetVersionedIndex());
- for (auto&& s : blobIds) {
- auto it = CurrentBlobIds.find(s.first);
- if (it == CurrentBlobIds.end()) {
- notContains += s.second.size();
- continue;
- }
- for (auto&& b : s.second) {
- if (it->second.contains(b)) {
- ++contains;
- }
- }
- }
- AFL_VERIFY(!contains || !notContains);
- if (!contains) {
- portion.SetPathId(it->second);
- index.UpsertPortion(std::move(portion));
- }
+ portion.ResetShardingVersion();
+ portion.SetPathId(it->second);
+ index.UpsertPortion(std::move(portion));
}
}
return TConclusionStatus::Success();
@@ -186,4 +169,23 @@ bool TDestinationSession::DoStart(const NColumnShard::TColumnShard& shard, const
return true;
}
-} \ No newline at end of file
+bool TDestinationSession::TryTakePortionBlobs(const TVersionedIndex& vIndex, const TPortionInfo& portion) {
+ THashMap<TString, THashSet<TUnifiedBlobId>> blobIds;
+ portion.FillBlobIdsByStorage(blobIds, vIndex);
+ ui32 containsCounter = 0;
+ ui32 newCounter = 0;
+ for (auto&& i : blobIds) {
+ auto& storageBlobIds = CurrentBlobIds[i.first];
+ for (auto&& b : i.second) {
+ if (storageBlobIds.emplace(b).second) {
+ ++newCounter;
+ } else {
+ ++containsCounter;
+ }
+ }
+ }
+ AFL_VERIFY((containsCounter == 0) ^ (newCounter == 0));
+ return newCounter;
+}
+
+} // namespace NKikimr::NOlap::NDataSharing
diff --git a/ydb/core/tx/columnshard/data_sharing/destination/session/destination.h b/ydb/core/tx/columnshard/data_sharing/destination/session/destination.h
index 1c702e62e1..7207f058a2 100644
--- a/ydb/core/tx/columnshard/data_sharing/destination/session/destination.h
+++ b/ydb/core/tx/columnshard/data_sharing/destination/session/destination.h
@@ -1,10 +1,11 @@
#pragma once
-#include <ydb/core/tx/columnshard/common/tablet_id.h>
-
#include <ydb/core/tx/columnshard/blob.h>
+#include <ydb/core/tx/columnshard/common/tablet_id.h>
#include <ydb/core/tx/columnshard/data_sharing/common/session/common.h>
#include <ydb/core/tx/columnshard/data_sharing/initiator/controller/abstract.h>
#include <ydb/core/tx/columnshard/data_sharing/protos/sessions.pb.h>
+#include <ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h>
+
#include <ydb/library/conclusion/result.h>
namespace NKikimr::NColumnShard {
@@ -14,7 +15,7 @@ class TColumnShard;
namespace NKikimr::NOlap {
class TColumnEngineForLogs;
class IStoragesManager;
-}
+} // namespace NKikimr::NOlap
namespace NKikimr::NOlap::NDataSharing {
@@ -27,12 +28,11 @@ private:
YDB_READONLY(TTabletId, TabletId, (TTabletId)0);
YDB_READONLY(ui32, PackIdx, 0);
YDB_READONLY(bool, DataFinished, false);
+
public:
TSourceCursorForDestination() = default;
TSourceCursorForDestination(const TTabletId tabletId)
- : TabletId(tabletId)
- {
-
+ : TabletId(tabletId) {
}
TConclusionStatus ReceiveData(const ui32 packIdxReceived) {
@@ -65,7 +65,6 @@ public:
result.SetFinished(DataFinished);
return result;
}
-
};
class TDestinationSession: public TCommonSession {
@@ -77,6 +76,7 @@ private:
YDB_READONLY_FLAG(Confirmed, false);
THashMap<TTabletId, TSourceCursorForDestination> Cursors;
THashMap<TString, THashSet<TUnifiedBlobId>> CurrentBlobIds;
+
protected:
virtual bool DoStart(const NColumnShard::TColumnShard& shard, const THashMap<ui64, std::vector<std::shared_ptr<TPortionInfo>>>& portions) override;
virtual THashSet<ui64> GetPathIdsForStart() const override {
@@ -86,7 +86,10 @@ protected:
}
return result;
}
+
public:
+ bool TryTakePortionBlobs(const TVersionedIndex& vIndex, const TPortionInfo& portion);
+
void SetBarrierSnapshot(const TSnapshot& value) {
TransferContext.SetSnapshotBarrier(value);
}
@@ -100,15 +103,11 @@ public:
TDestinationSession(const TInitiatorControllerContainer& controller, const TPathIdsRemapper& remapper, const TString& sessionId, const TTransferContext& context)
: TBase(sessionId, "destination_base", context)
, InitiatorController(controller)
- , PathIds(remapper)
- {
-
+ , PathIds(remapper) {
}
TDestinationSession()
- : TBase("dest_proto")
- {
-
+ : TBase("dest_proto") {
}
void Confirm(const bool allowRepeat = false) {
@@ -119,7 +118,7 @@ public:
[[nodiscard]] TConclusionStatus DataReceived(THashMap<ui64, NEvents::TPathIdData>&& data, TColumnEngineForLogs& index, const std::shared_ptr<IStoragesManager>& manager);
ui32 GetSourcesInProgressCount() const;
- void SendCurrentCursorAck(const NColumnShard::TColumnShard & shard, const std::optional<TTabletId> tabletId);
+ void SendCurrentCursorAck(const NColumnShard::TColumnShard& shard, const std::optional<TTabletId> tabletId);
NKikimrColumnShardDataSharingProto::TDestinationSession SerializeDataToProto() const;
@@ -136,4 +135,4 @@ public:
[[nodiscard]] TConclusionStatus DeserializeDataFromProto(const NKikimrColumnShardDataSharingProto::TDestinationSession& proto, const TColumnEngineForLogs& index);
};
-} \ No newline at end of file
+} // namespace NKikimr::NOlap::NDataSharing
diff --git a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp
index a3b6b32646..9d3b10743b 100644
--- a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp
+++ b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp
@@ -41,6 +41,16 @@ TTxDataFromSource::TTxDataFromSource(NColumnShard::TColumnShard* self, const std
, PortionsByPathId(portionsByPathId)
, SourceTabletId(sourceTabletId)
{
+ for (auto&& i : PortionsByPathId) {
+ for (ui32 p = 0; p < i.second.GetPortions().size();) {
+ if (Session->TryTakePortionBlobs(Self->GetIndexAs<TColumnEngineForLogs>().GetVersionedIndex(), i.second.GetPortions()[p])) {
+ ++p;
+ } else {
+ i.second.MutablePortions()[p] = std::move(i.second.MutablePortions().back());
+ i.second.MutablePortions().pop_back();
+ }
+ }
+ }
}
} \ No newline at end of file
diff --git a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_start_from_initiator.cpp b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_start_from_initiator.cpp
index 2c80eca952..c07a010e6f 100644
--- a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_start_from_initiator.cpp
+++ b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_start_from_initiator.cpp
@@ -11,7 +11,7 @@ bool TTxProposeFromInitiator::DoExecute(NTabletFlatExecutor::TTransactionContext
}
void TTxProposeFromInitiator::DoComplete(const TActorContext& /*ctx*/) {
- AFL_VERIFY(!Session->IsConfirmed());
+ AFL_VERIFY(!Session->IsConfirmed() || Session->GetTransferContext().GetTxId());
AFL_VERIFY(Sessions->emplace(Session->GetSessionId(), Session).second);
Session->GetInitiatorController().ProposeSuccess(Session->GetSessionId());
}
diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp
index 5e763b9164..17a7d10475 100644
--- a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp
+++ b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp
@@ -66,6 +66,7 @@ TColumnEngineChanges::~TColumnEngineChanges() {
}
void TColumnEngineChanges::Abort(NColumnShard::TColumnShard& self, TChangesFinishContext& context) {
+ AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("problem", "changes_aborted")("reason", context.ErrorMessage);
AFL_VERIFY(Stage != EStage::Finished && Stage != EStage::Created && Stage != EStage::Aborted)("stage", Stage)("reason", context.ErrorMessage);
Stage = EStage::Aborted;
OnFinish(self, context);
diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp
index 3f5e692df4..e183060da2 100644
--- a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp
+++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp
@@ -7,27 +7,32 @@
#include "compaction/merged_column.h"
#include "counters/general.h"
+#include <ydb/core/formats/arrow/reader/merger.h>
#include <ydb/core/formats/arrow/simple_builder/array.h>
#include <ydb/core/formats/arrow/simple_builder/filler.h>
+#include <ydb/core/tx/columnshard/columnshard_impl.h>
#include <ydb/core/tx/columnshard/engines/portions/read_with_blobs.h>
#include <ydb/core/tx/columnshard/engines/portions/write_with_blobs.h>
-#include <ydb/core/tx/columnshard/columnshard_impl.h>
#include <ydb/core/tx/columnshard/engines/storage/chunks/null_column.h>
#include <ydb/core/tx/columnshard/splitter/batch_slice.h>
#include <ydb/core/tx/columnshard/splitter/settings.h>
-#include <ydb/core/formats/arrow/reader/merger.h>
namespace NKikimr::NOlap::NCompaction {
void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByFullBatches(TConstructionContext& context, std::vector<TReadPortionInfoWithBlobs>&& portions) noexcept {
std::vector<std::shared_ptr<arrow::RecordBatch>> batchResults;
auto resultSchema = context.SchemaVersions.GetLastSchema();
+ auto shardingActual = context.SchemaVersions.GetShardingInfoActual(GranuleMeta->GetPathId());
{
auto resultDataSchema = resultSchema->GetIndexInfo().ArrowSchemaWithSpecials();
NArrow::NMerger::TMergePartialStream mergeStream(resultSchema->GetIndexInfo().GetReplaceKey(), resultDataSchema, false, IIndexInfo::GetSpecialColumnNames());
+
for (auto&& i : portions) {
auto dataSchema = i.GetPortionInfo().GetSchema(context.SchemaVersions);
auto batch = i.GetBatch(dataSchema, *resultSchema);
+ if (shardingActual && i.GetPortionInfo().NeedShardingFilter(*shardingActual)) {
+ shardingActual->GetShardingInfo()->GetFilter(batch)->Apply(batch);
+ }
batch = resultSchema->NormalizeBatch(*dataSchema, batch);
Y_DEBUG_ABORT_UNLESS(NArrow::IsSortedAndUnique(batch, resultSchema->GetIndexInfo().GetReplaceKey()));
mergeStream.AddSource(batch, nullptr);
@@ -39,6 +44,9 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByFullBatches(TCon
auto portions = MakeAppendedPortions(b, GranuleMeta->GetPathId(), resultSchema->GetSnapshot(), GranuleMeta.get(), context, {});
Y_ABORT_UNLESS(portions.size());
for (auto& portion : portions) {
+ if (shardingActual) {
+ portion.GetPortionConstructor().SetShardingVersion(shardingActual->GetSnapshotVersion());
+ }
AppendedPortions.emplace_back(std::move(portion));
}
}
@@ -51,6 +59,7 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstruc
static const std::shared_ptr<arrow::Field> portionRecordIndexField = std::make_shared<arrow::Field>(portionRecordIndexFieldName, std::make_shared<arrow::UInt32Type>());
auto resultSchema = context.SchemaVersions.GetLastSchema();
+ auto shardingActual = context.SchemaVersions.GetShardingInfoActual(GranuleMeta->GetPathId());
std::vector<std::string> pkFieldNames = resultSchema->GetIndexInfo().GetReplaceKey()->field_names();
std::set<std::string> pkFieldNamesSet(pkFieldNames.begin(), pkFieldNames.end());
@@ -81,7 +90,12 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstruc
batch = NArrow::TStatusValidator::GetValid(batch->AddColumn(batch->num_columns(), portionRecordIndexField, column->BuildArray(batch->num_rows())));
}
Y_DEBUG_ABORT_UNLESS(NArrow::IsSortedAndUnique(batch, resultSchema->GetIndexInfo().GetReplaceKey()));
- mergeStream.AddSource(batch, nullptr);
+ std::shared_ptr<NArrow::TColumnFilter> filter;
+ if (shardingActual && i.GetPortionInfo().NeedShardingFilter(*shardingActual)) {
+ filter = shardingActual->GetShardingInfo()->GetFilter(batch);
+ }
+
+ mergeStream.AddSource(batch, filter);
}
batchResults = mergeStream.DrainAllParts(CheckPoints, indexFields);
}
@@ -158,7 +172,6 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstruc
++batchIdx;
}
AFL_VERIFY(columnRecordsCount == batchesRecordsCount)("mCount", columnRecordsCount)("bCount", batchesRecordsCount);
-
}
ui32 batchIdx = 0;
@@ -202,6 +215,9 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstruc
NArrow::TMinMaxSpecialKeys snapshotKeys(b, TIndexInfo::ArrowSchemaSnapshot());
AppendedPortions.back().GetPortionConstructor().AddMetadata(*resultSchema, primaryKeys, snapshotKeys);
AppendedPortions.back().GetPortionConstructor().MutableMeta().SetTierName(IStoragesManager::DefaultStorageId);
+ if (shardingActual) {
+ AppendedPortions.back().GetPortionConstructor().SetShardingVersion(shardingActual->GetSnapshotVersion());
+ }
recordIdx += slice.GetRecordsCount();
}
}
@@ -312,4 +328,4 @@ ui64 TGeneralCompactColumnEngineChanges::TMemoryPredictorChunkedPolicy::AddPorti
return SumMemoryFix + SumMemoryDelta;
}
-}
+} // namespace NKikimr::NOlap::NCompaction
diff --git a/ydb/core/tx/columnshard/engines/changes/indexation.cpp b/ydb/core/tx/columnshard/engines/changes/indexation.cpp
index ffbac159cc..123e61ef0f 100644
--- a/ydb/core/tx/columnshard/engines/changes/indexation.cpp
+++ b/ydb/core/tx/columnshard/engines/changes/indexation.cpp
@@ -50,6 +50,70 @@ void TInsertColumnEngineChanges::DoOnFinish(NColumnShard::TColumnShard& self, TC
self.BackgroundController.FinishIndexing(*this);
}
+namespace {
+class TPathData {
+private:
+ std::vector<std::shared_ptr<arrow::RecordBatch>> Batches;
+ YDB_READONLY_DEF(std::optional<TGranuleShardingInfo>, ShardingInfo);
+
+public:
+ TPathData(const std::optional<TGranuleShardingInfo>& shardingInfo)
+ : ShardingInfo(shardingInfo)
+ {
+
+ }
+ void AddBatch(const std::shared_ptr<arrow::RecordBatch>& batch) {
+ Batches.emplace_back(batch);
+ }
+
+ void AddShardingInfo(const std::optional<TGranuleShardingInfo>& info) {
+ if (!info) {
+ ShardingInfo.reset();
+ } else if (ShardingInfo && info->GetSnapshotVersion() < ShardingInfo->GetSnapshotVersion()) {
+ ShardingInfo = info;
+ }
+ }
+
+ std::shared_ptr<arrow::RecordBatch> Merge(const TIndexInfo& indexInfo) const {
+ NArrow::NMerger::TMergePartialStream stream(indexInfo.GetReplaceKey(), indexInfo.ArrowSchemaWithSpecials(), false, IIndexInfo::GetSpecialColumnNames());
+ THashMap<std::string, ui64> fieldSizes;
+ ui64 rowsCount = 0;
+ for (auto&& batch : Batches) {
+ stream.AddSource(batch, nullptr);
+ for (ui32 cIdx = 0; cIdx < (ui32)batch->num_columns(); ++cIdx) {
+ fieldSizes[batch->column_name(cIdx)] += NArrow::GetArrayDataSize(batch->column(cIdx));
+ }
+ rowsCount += batch->num_rows();
+ }
+
+ NArrow::NMerger::TRecordBatchBuilder builder(indexInfo.ArrowSchemaWithSpecials()->fields(), rowsCount, fieldSizes);
+ stream.SetPossibleSameVersion(true);
+ stream.DrainAll(builder);
+ return builder.Finalize();
+ }
+};
+
+class TPathesData {
+private:
+ THashMap<ui64, TPathData> Data;
+
+public:
+ const THashMap<ui64, TPathData>& GetData() const {
+ return Data;
+ }
+
+ void Add(const ui64 pathId, const std::optional<TGranuleShardingInfo>& info, const std::shared_ptr<arrow::RecordBatch>& batch) {
+ auto it = Data.find(pathId);
+ if (it == Data.end()) {
+ it = Data.emplace(pathId, info).first;
+ }
+ it->second.AddShardingInfo(info);
+ it->second.AddBatch(batch);
+
+ }
+};
+}
+
TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionContext& context) noexcept {
Y_ABORT_UNLESS(!DataToIndex.empty());
Y_ABORT_UNLESS(AppendedPortions.empty());
@@ -67,10 +131,12 @@ TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionCont
auto resultSchema = context.SchemaVersions.GetSchema(maxSnapshot);
Y_ABORT_UNLESS(resultSchema->GetIndexInfo().IsSorted());
- THashMap<ui64, std::vector<std::shared_ptr<arrow::RecordBatch>>> pathBatches;
+ TPathesData pathBatches;
for (auto& inserted : DataToIndex) {
const TBlobRange& blobRange = inserted.GetBlobRange();
+ auto shardingFilterCommit = context.SchemaVersions.GetShardingInfoOptional(inserted.PathId, inserted.GetSnapshot());
+
auto blobSchema = context.SchemaVersions.GetSchemaVerified(inserted.GetSchemaVersion());
auto& indexInfo = blobSchema->GetIndexInfo();
Y_ABORT_UNLESS(indexInfo.IsSorted());
@@ -89,31 +155,20 @@ TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionCont
batch = AddSpecials(batch, indexInfo, inserted);
batch = resultSchema->NormalizeBatch(*blobSchema, batch);
- pathBatches[inserted.PathId].push_back(batch);
- Y_DEBUG_ABORT_UNLESS(NArrow::IsSorted(pathBatches[inserted.PathId].back(), resultSchema->GetIndexInfo().GetReplaceKey()));
+ pathBatches.Add(inserted.PathId, shardingFilterCommit, batch);
+ Y_DEBUG_ABORT_UNLESS(NArrow::IsSorted(batch, resultSchema->GetIndexInfo().GetReplaceKey()));
}
Y_ABORT_UNLESS(Blobs.IsEmpty());
const std::vector<std::string> comparableColumns = resultSchema->GetIndexInfo().GetReplaceKey()->field_names();
- for (auto& [pathId, batches] : pathBatches) {
- NArrow::NMerger::TMergePartialStream stream(resultSchema->GetIndexInfo().GetReplaceKey(), resultSchema->GetIndexInfo().ArrowSchemaWithSpecials(), false, IIndexInfo::GetSpecialColumnNames());
- THashMap<std::string, ui64> fieldSizes;
- ui64 rowsCount = 0;
- for (auto&& batch : batches) {
- stream.AddSource(batch, nullptr);
- for (ui32 cIdx = 0; cIdx < (ui32)batch->num_columns(); ++cIdx) {
- fieldSizes[batch->column_name(cIdx)] += NArrow::GetArrayDataSize(batch->column(cIdx));
- }
- rowsCount += batch->num_rows();
- }
-
- NArrow::NMerger::TRecordBatchBuilder builder(resultSchema->GetIndexInfo().ArrowSchemaWithSpecials()->fields(), rowsCount, fieldSizes);
- stream.SetPossibleSameVersion(true);
- stream.DrainAll(builder);
+ for (auto& [pathId, pathInfo] : pathBatches.GetData()) {
+ auto shardingFilter = context.SchemaVersions.GetShardingInfoActual(pathId);
+ auto mergedBatch = pathInfo.Merge(resultSchema->GetIndexInfo());
auto itGranule = PathToGranule.find(pathId);
AFL_VERIFY(itGranule != PathToGranule.end());
- std::vector<std::shared_ptr<arrow::RecordBatch>> result = NArrow::NMerger::TSortableBatchPosition::SplitByBordersInSequentialContainer(builder.Finalize(), comparableColumns, itGranule->second);
+ std::vector<std::shared_ptr<arrow::RecordBatch>> result = NArrow::NMerger::TSortableBatchPosition::
+ SplitByBordersInSequentialContainer(mergedBatch, comparableColumns, itGranule->second);
for (auto&& b : result) {
if (!b) {
continue;
@@ -127,12 +182,15 @@ TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionCont
auto portions = MakeAppendedPortions(b, pathId, maxSnapshot, nullptr, context, externalSaver);
Y_ABORT_UNLESS(portions.size());
for (auto& portion : portions) {
+ if (pathInfo.GetShardingInfo()) {
+ portion.GetPortionConstructor().SetShardingVersion(pathInfo.GetShardingInfo()->GetSnapshotVersion());
+ }
AppendedPortions.emplace_back(std::move(portion));
}
}
}
- Y_ABORT_UNLESS(PathToGranule.size() == pathBatches.size());
+ Y_ABORT_UNLESS(PathToGranule.size() == pathBatches.GetData().size());
return TConclusionStatus::Success();
}
diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp
index 787f34ac5a..f57f6660de 100644
--- a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp
+++ b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp
@@ -135,8 +135,6 @@ std::vector<TWritePortionInfoWithBlobs> TChangesWithAppend::MakeAppendedPortions
auto b = batch->Slice(recordIdx, slice.GetRecordsCount());
out.emplace_back(TWritePortionInfoWithBlobs::BuildByBlobs(slice.GroupChunksByBlobs(groups), pathId, resultSchema->GetVersion(), snapshot, SaverContext.GetStoragesManager()));
out.back().FillStatistics(resultSchema->GetIndexInfo());
- NArrow::TFirstLastSpecialKeys primaryKeys(slice.GetFirstLastPKBatch(resultSchema->GetIndexInfo().GetReplaceKey()));
- NArrow::TMinMaxSpecialKeys snapshotKeys(b, TIndexInfo::ArrowSchemaSnapshot());
out.back().GetPortionConstructor().AddMetadata(*resultSchema, b);
out.back().GetPortionConstructor().MutableMeta().SetTierName(IStoragesManager::DefaultStorageId);
recordIdx += slice.GetRecordsCount();
diff --git a/ydb/core/tx/columnshard/engines/column_engine.h b/ydb/core/tx/columnshard/engines/column_engine.h
index 58a2d8eb31..2487d8d9cf 100644
--- a/ydb/core/tx/columnshard/engines/column_engine.h
+++ b/ydb/core/tx/columnshard/engines/column_engine.h
@@ -1,7 +1,8 @@
#pragma once
#include "db_wrapper.h"
-#include "changes/abstract/settings.h"
+
#include "changes/abstract/compaction_info.h"
+#include "changes/abstract/settings.h"
#include "predicate/filter.h"
#include "scheme/snapshot_scheme.h"
#include "scheme/versions/versioned_index.h"
@@ -11,7 +12,7 @@
namespace NKikimr::NColumnShard {
class TTiersManager;
class TTtl;
-}
+} // namespace NKikimr::NColumnShard
namespace NKikimr::NOlap {
class TInsertColumnEngineChanges;
@@ -33,7 +34,7 @@ struct TSelectInfo {
size_t Rows{};
size_t Bytes{};
- const TStats& operator += (const TStats& stats) {
+ const TStats& operator+=(const TStats& stats) {
Portions += stats.Portions;
Records += stats.Records;
Blobs += stats.Blobs;
@@ -60,6 +61,7 @@ class TColumnEngineStats {
private:
static constexpr const ui64 NUM_KINDS = 5;
static_assert(NUM_KINDS == NOlap::TPortionMeta::EProduced::EVICTED, "NUM_KINDS must match NOlap::TPortionMeta::EProduced enum");
+
public:
class TPortionsStats {
private:
@@ -69,8 +71,8 @@ public:
Y_ABORT_UNLESS(result >= 0);
return result;
}
- public:
+ public:
i64 Portions = 0;
i64 Blobs = 0;
i64 Rows = 0;
@@ -263,8 +265,8 @@ public:
class IColumnEngine {
protected:
virtual void DoRegisterTable(const ui64 pathId) = 0;
-public:
+public:
static ui64 GetMetadataLimit();
virtual ~IColumnEngine() = default;
@@ -280,24 +282,25 @@ public:
return DoRegisterTable(pathId);
}
virtual bool IsOverloadedByMetadata(const ui64 limit) const = 0;
- virtual std::shared_ptr<TSelectInfo> Select(ui64 pathId, TSnapshot snapshot,
- const TPKRangesFilter& pkRangesFilter) const = 0;
+ virtual std::shared_ptr<TSelectInfo> Select(ui64 pathId, TSnapshot snapshot, const TPKRangesFilter& pkRangesFilter) const = 0;
virtual std::shared_ptr<TInsertColumnEngineChanges> StartInsert(std::vector<TInsertedData>&& dataToIndex) noexcept = 0;
virtual std::shared_ptr<TColumnEngineChanges> StartCompaction(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) noexcept = 0;
- virtual std::shared_ptr<TCleanupPortionsColumnEngineChanges> StartCleanupPortions(const TSnapshot& snapshot, const THashSet<ui64>& pathsToDrop,
- const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) noexcept = 0;
+ virtual std::shared_ptr<TCleanupPortionsColumnEngineChanges> StartCleanupPortions(const TSnapshot& snapshot, const THashSet<ui64>& pathsToDrop, const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) noexcept = 0;
virtual std::shared_ptr<TCleanupTablesColumnEngineChanges> StartCleanupTables(THashSet<ui64>& pathsToDrop) noexcept = 0;
- virtual std::vector<std::shared_ptr<TTTLColumnEngineChanges>> StartTtl(const THashMap<ui64, TTiering>& pathEviction,
- const std::shared_ptr<NDataLocks::TManager>& dataLocksManager, const ui64 memoryUsageLimit) noexcept = 0;
+ virtual std::vector<std::shared_ptr<TTTLColumnEngineChanges>> StartTtl(const THashMap<ui64, TTiering>& pathEviction, const std::shared_ptr<NDataLocks::TManager>& dataLocksManager, const ui64 memoryUsageLimit) noexcept = 0;
virtual bool ApplyChangesOnTxCreate(std::shared_ptr<TColumnEngineChanges> changes, const TSnapshot& snapshot) noexcept = 0;
virtual bool ApplyChangesOnExecute(IDbWrapper& db, std::shared_ptr<TColumnEngineChanges> changes, const TSnapshot& snapshot) noexcept = 0;
virtual void RegisterSchemaVersion(const TSnapshot& snapshot, TIndexInfo&& info) = 0;
virtual void RegisterSchemaVersion(const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema) = 0;
virtual const TMap<ui64, std::shared_ptr<TColumnEngineStats>>& GetStats() const = 0;
virtual const TColumnEngineStats& GetTotalStats() = 0;
- virtual ui64 MemoryUsage() const { return 0; }
- virtual TSnapshot LastUpdate() const { return TSnapshot::Zero(); }
+ virtual ui64 MemoryUsage() const {
+ return 0;
+ }
+ virtual TSnapshot LastUpdate() const {
+ return TSnapshot::Zero();
+ }
virtual void OnTieringModified(const std::shared_ptr<NColumnShard::TTiersManager>& manager, const NColumnShard::TTtl& ttl, const std::optional<ui64> pathId) = 0;
};
-}
+} // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp
index 31ae9866ac..50dfe3bb73 100644
--- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp
+++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp
@@ -162,6 +162,13 @@ bool TColumnEngineForLogs::Load(IDbWrapper& db) {
Loaded = true;
THashMap<ui64, ui64> granuleToPathIdDecoder;
{
+ TMemoryProfileGuard g("TTxInit/LoadShardingInfo");
+ if (!VersionedIndex.LoadShardingInfo(db)) {
+ return false;
+ }
+ }
+
+ {
TMemoryProfileGuard g("TTxInit/LoadColumns");
auto guard = GranulesStorage->GetStats()->StartPackModification();
if (!LoadColumns(db)) {
diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h
index bdad1f4ba4..5628ef979b 100644
--- a/ydb/core/tx/columnshard/engines/column_engine_logs.h
+++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h
@@ -45,6 +45,7 @@ class TColumnEngineForLogs : public IColumnEngine {
friend class TCleanupPortionsColumnEngineChanges;
friend class TCleanupTablesColumnEngineChanges;
friend class NDataSharing::TDestinationSession;
+
private:
bool ActualizationStarted = false;
const NColumnShard::TEngineLogsCounters SignalCounters;
@@ -56,6 +57,7 @@ private:
static TDuration GetRemovedPortionLivetime();
const TDuration RemovedPortionLivetime = GetRemovedPortionLivetime();
+
public:
const std::shared_ptr<NActualizer::TController>& GetActualizationController() const {
return ActualizationController;
@@ -97,9 +99,12 @@ public:
const TMap<ui64, std::shared_ptr<TColumnEngineStats>>& GetStats() const override;
const TColumnEngineStats& GetTotalStats() override;
- TSnapshot LastUpdate() const override { return LastSnapshot; }
+ TSnapshot LastUpdate() const override {
+ return LastSnapshot;
+ }
virtual void DoRegisterTable(const ui64 pathId) override;
+
public:
bool Load(IDbWrapper& db) override;
@@ -111,8 +116,7 @@ public:
std::shared_ptr<TColumnEngineChanges> StartCompaction(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) noexcept override;
std::shared_ptr<TCleanupPortionsColumnEngineChanges> StartCleanupPortions(const TSnapshot& snapshot, const THashSet<ui64>& pathsToDrop, const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) noexcept override;
std::shared_ptr<TCleanupTablesColumnEngineChanges> StartCleanupTables(THashSet<ui64>& pathsToDrop) noexcept override;
- std::vector<std::shared_ptr<TTTLColumnEngineChanges>> StartTtl(const THashMap<ui64, TTiering>& pathEviction,
- const std::shared_ptr<NDataLocks::TManager>& locksManager, const ui64 memoryUsageLimit) noexcept override;
+ std::vector<std::shared_ptr<TTTLColumnEngineChanges>> StartTtl(const THashMap<ui64, TTiering>& pathEviction, const std::shared_ptr<NDataLocks::TManager>& locksManager, const ui64 memoryUsageLimit) noexcept override;
void ReturnToIndexes(const THashMap<ui64, THashSet<ui64>>& portions) const {
return GranulesStorage->ReturnToIndexes(portions);
@@ -123,8 +127,7 @@ public:
void RegisterSchemaVersion(const TSnapshot& snapshot, TIndexInfo&& info) override;
void RegisterSchemaVersion(const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema) override;
- std::shared_ptr<TSelectInfo> Select(ui64 pathId, TSnapshot snapshot,
- const TPKRangesFilter& pkRangesFilter) const override;
+ std::shared_ptr<TSelectInfo> Select(ui64 pathId, TSnapshot snapshot, const TPKRangesFilter& pkRangesFilter) const override;
bool IsPortionExists(const ui64 pathId, const ui64 portionId) const {
return !!GranulesStorage->GetPortionOptional(pathId, portionId);
@@ -164,30 +167,32 @@ public:
void AddCleanupPortion(const TPortionInfo& info) {
CleanupPortions[info.GetRemoveSnapshotVerified().GetPlanInstant() + RemovedPortionLivetime].emplace_back(info);
}
+ void AddShardingInfo(const TGranuleShardingInfo& shardingInfo) {
+ VersionedIndex.AddShardingInfo(shardingInfo);
+ }
private:
TVersionedIndex VersionedIndex;
ui64 TabletId;
- TMap<ui64, std::shared_ptr<TColumnEngineStats>> PathStats; // per path_id stats sorted by path_id
+ TMap<ui64, std::shared_ptr<TColumnEngineStats>> PathStats; // per path_id stats sorted by path_id
std::map<TInstant, std::vector<TPortionInfo>> CleanupPortions;
TColumnEngineStats Counters;
ui64 LastPortion;
ui64 LastGranule;
TSnapshot LastSnapshot = TSnapshot::Zero();
bool Loaded = false;
+
private:
bool LoadColumns(IDbWrapper& db);
+ bool LoadShardingInfo(IDbWrapper& db);
bool LoadCounters(IDbWrapper& db);
void EraseTable(const ui64 pathId);
void UpsertPortion(const TPortionInfo& portionInfo, const TPortionInfo* exInfo = nullptr);
bool ErasePortion(const TPortionInfo& portionInfo, bool updateStats = true);
- void UpdatePortionStats(const TPortionInfo& portionInfo, EStatsUpdateType updateType = EStatsUpdateType::DEFAULT,
- const TPortionInfo* exPortionInfo = nullptr);
- void UpdatePortionStats(TColumnEngineStats& engineStats, const TPortionInfo& portionInfo,
- EStatsUpdateType updateType,
- const TPortionInfo* exPortionInfo = nullptr) const;
+ void UpdatePortionStats(const TPortionInfo& portionInfo, EStatsUpdateType updateType = EStatsUpdateType::DEFAULT, const TPortionInfo* exPortionInfo = nullptr);
+ void UpdatePortionStats(TColumnEngineStats& engineStats, const TPortionInfo& portionInfo, EStatsUpdateType updateType, const TPortionInfo* exPortionInfo = nullptr) const;
};
-} // namespace NKikimr::NOlap
+} // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp
index 4bc8f9bf09..302c3b6451 100644
--- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp
+++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp
@@ -1,7 +1,9 @@
#include "defs.h"
#include "db_wrapper.h"
#include "portions/constructor.h"
+#include <ydb/core/protos/flat_scheme_op.pb.h>
#include <ydb/core/tx/columnshard/columnshard_schema.h>
+#include <ydb/core/tx/sharding/sharding.h>
namespace NKikimr::NOlap {
@@ -68,6 +70,7 @@ void TDbWrapper::WritePortion(const NOlap::TPortionInfo& portion) {
auto removeSnapshot = portion.GetRemoveSnapshotOptional();
db.Table<IndexPortions>().Key(portion.GetPathId(), portion.GetPortion()).Update(
NIceDb::TUpdate<IndexPortions::SchemaVersion>(portion.GetSchemaVersionVerified()),
+ NIceDb::TUpdate<IndexPortions::ShardingVersion>(portion.GetShardingVersionDef(0)),
NIceDb::TUpdate<IndexPortions::XPlanStep>(removeSnapshot ? removeSnapshot->GetPlanStep() : 0),
NIceDb::TUpdate<IndexPortions::XTxId>(removeSnapshot ? removeSnapshot->GetTxId() : 0),
NIceDb::TUpdate<IndexPortions::Metadata>(metaProto.SerializeAsString()));
@@ -123,6 +126,9 @@ bool TDbWrapper::LoadPortions(const std::function<void(NOlap::TPortionInfoConstr
while (!rowset.EndOfSet()) {
NOlap::TPortionInfoConstructor portion(rowset.GetValue<IndexPortions::PathId>(), rowset.GetValue<IndexPortions::PortionId>());
portion.SetSchemaVersion(rowset.GetValue<IndexPortions::SchemaVersion>());
+ if (rowset.HaveValue<IndexPortions::ShardingVersion>() && rowset.GetValue<IndexPortions::ShardingVersion>()) {
+ portion.SetShardingVersion(rowset.GetValue<IndexPortions::ShardingVersion>());
+ }
portion.SetRemoveSnapshot(rowset.GetValue<IndexPortions::XPlanStep>(), rowset.GetValue<IndexPortions::XTxId>());
NKikimrTxColumnShard::TIndexPortionMeta metaProto;
@@ -185,4 +191,27 @@ bool TDbWrapper::LoadCounters(const std::function<void(ui32 id, ui64 value)>& ca
return NColumnShard::Schema::IndexCounters_Load(db, callback);
}
+TConclusion<THashMap<ui64, std::map<NOlap::TSnapshot, TGranuleShardingInfo>>> TDbWrapper::LoadGranulesShardingInfo() {
+ using Schema = NColumnShard::Schema;
+ NIceDb::TNiceDb db(Database);
+ auto rowset = db.Table<Schema::ShardingInfo>().Select();
+ if (!rowset.IsReady()) {
+ return TConclusionStatus::Fail("cannot read rowset");
+ }
+ THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>> result;
+ while (!rowset.EndOfSet()) {
+ NOlap::TSnapshot snapshot = NOlap::TSnapshot::Zero();
+ snapshot.DeserializeFromString(rowset.GetValue<Schema::ShardingInfo::Snapshot>()).Validate();
+ NSharding::TGranuleShardingLogicContainer logic;
+ logic.DeserializeFromString(rowset.GetValue<Schema::ShardingInfo::Logic>()).Validate();
+ TGranuleShardingInfo gShardingInfo(logic, snapshot, rowset.GetValue<Schema::ShardingInfo::VersionId>(), rowset.GetValue<Schema::ShardingInfo::PathId>());
+ AFL_VERIFY(result[gShardingInfo.GetPathId()].emplace(gShardingInfo.GetSinceSnapshot(), gShardingInfo).second);
+
+ if (!rowset.Next()) {
+ return TConclusionStatus::Fail("cannot read rowset");
+ }
+ }
+ return result;
+}
+
}
diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.h b/ydb/core/tx/columnshard/engines/db_wrapper.h
index d9435913bd..39536cb9c9 100644
--- a/ydb/core/tx/columnshard/engines/db_wrapper.h
+++ b/ydb/core/tx/columnshard/engines/db_wrapper.h
@@ -1,6 +1,8 @@
#pragma once
#include "defs.h"
+#include "scheme/versions/versioned_index.h"
#include <ydb/core/tx/columnshard/common/blob.h>
+#include <ydb/core/tx/columnshard/common/snapshot.h>
namespace NKikimrTxColumnShard {
class TIndexPortionMeta;
@@ -34,8 +36,7 @@ public:
virtual void EraseCommitted(const TInsertedData& data) = 0;
virtual void EraseAborted(const TInsertedData& data) = 0;
- virtual bool Load(TInsertTableAccessor& insertTable,
- const TInstant& loadTime) = 0;
+ virtual bool Load(TInsertTableAccessor& insertTable, const TInstant& loadTime) = 0;
virtual void WriteColumn(const TPortionInfo& portion, const TColumnRecord& row, const ui32 firstPKColumnId) = 0;
virtual void EraseColumn(const TPortionInfo& portion, const TColumnRecord& row) = 0;
@@ -51,6 +52,7 @@ public:
virtual void WriteCounter(ui32 counterId, ui64 value) = 0;
virtual bool LoadCounters(const std::function<void(ui32 id, ui64 value)>& callback) = 0;
+ virtual TConclusion<THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>>> LoadGranulesShardingInfo() = 0;
};
class TDbWrapper : public IDbWrapper {
@@ -84,6 +86,8 @@ public:
void WriteCounter(ui32 counterId, ui64 value) override;
bool LoadCounters(const std::function<void(ui32 id, ui64 value)>& callback) override;
+ virtual TConclusion<THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>>> LoadGranulesShardingInfo() override;
+
private:
NTable::TDatabase& Database;
const IBlobGroupSelector* DsGroupSelector;
diff --git a/ydb/core/tx/columnshard/engines/portions/constructor.cpp b/ydb/core/tx/columnshard/engines/portions/constructor.cpp
index f3de740f4f..9ed5046dad 100644
--- a/ydb/core/tx/columnshard/engines/portions/constructor.cpp
+++ b/ydb/core/tx/columnshard/engines/portions/constructor.cpp
@@ -1,8 +1,11 @@
#include "constructor.h"
+
#include <ydb/core/tx/columnshard/columnshard_schema.h>
+#include <ydb/core/tx/columnshard/common/limits.h>
#include <ydb/core/tx/columnshard/engines/scheme/index_info.h>
#include <ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h>
#include <ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h>
+#include <ydb/core/tx/columnshard/hooks/abstract/abstract.h>
namespace NKikimr::NOlap {
@@ -20,6 +23,7 @@ TPortionInfo TPortionInfoConstructor::Build(const bool needChunksNormalization)
result.RemoveSnapshot = *RemoveSnapshot;
}
result.SchemaVersion = SchemaVersion;
+ result.ShardingVersion = ShardingVersion;
if (needChunksNormalization) {
ReorderChunks();
@@ -84,4 +88,4 @@ void TPortionInfoConstructor::AddMetadata(const ISnapshotSchema& snapshotSchema,
NArrow::TMinMaxSpecialKeys(batch, TIndexInfo::ArrowSchemaSnapshot()), snapshotSchema.GetIndexInfo());
}
-} \ No newline at end of file
+} // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/engines/portions/constructor.h b/ydb/core/tx/columnshard/engines/portions/constructor.h
index 23229c6c9b..13369a2e08 100644
--- a/ydb/core/tx/columnshard/engines/portions/constructor.h
+++ b/ydb/core/tx/columnshard/engines/portions/constructor.h
@@ -1,6 +1,6 @@
#pragma once
-#include "constructor_meta.h"
#include "column_record.h"
+#include "constructor_meta.h"
#include "index_chunk.h"
#include "portion_info.h"
@@ -11,6 +11,7 @@ class TPortionInfo;
class TVersionedIndex;
class ISnapshotSchema;
class TIndexChunkLoadContext;
+class TGranuleShardingInfo;
class TPortionInfoConstructor {
private:
@@ -22,10 +23,12 @@ private:
std::optional<TSnapshot> MinSnapshotDeprecated;
std::optional<TSnapshot> RemoveSnapshot;
std::optional<ui64> SchemaVersion;
+ std::optional<ui64> ShardingVersion;
std::vector<TIndexChunk> Indexes;
YDB_ACCESSOR_DEF(std::vector<TColumnRecord>, Records);
std::vector<TUnifiedBlobId> BlobIds;
+
public:
void SetPortionId(const ui64 value) {
AFL_VERIFY(value);
@@ -58,7 +61,7 @@ public:
, MinSnapshotDeprecated(portion.GetMinSnapshotDeprecated())
, RemoveSnapshot(portion.GetRemoveSnapshotOptional())
, SchemaVersion(portion.GetSchemaVersionOptional())
- {
+ , ShardingVersion(portion.GetShardingVersionOptional()) {
if (withMetadata) {
MetaConstructor = TPortionMetaConstructor(portion.Meta);
}
@@ -75,7 +78,7 @@ public:
, MinSnapshotDeprecated(portion.GetMinSnapshotDeprecated())
, RemoveSnapshot(portion.GetRemoveSnapshotOptional())
, SchemaVersion(portion.GetSchemaVersionOptional())
- {
+ , ShardingVersion(portion.GetShardingVersionOptional()) {
MetaConstructor = TPortionMetaConstructor(portion.Meta);
Indexes = std::move(portion.Indexes);
Records = std::move(portion.Records);
@@ -152,10 +155,15 @@ public:
}
void SetSchemaVersion(const ui64 version) {
-// AFL_VERIFY(version); engines/ut
+// AFL_VERIFY(version);
SchemaVersion = version;
}
+ void SetShardingVersion(const ui64 version) {
+// AFL_VERIFY(version);
+ ShardingVersion = version;
+ }
+
void SetRemoveSnapshot(const TSnapshot& snap) {
AFL_VERIFY(!RemoveSnapshot);
if (snap.Valid()) {
@@ -287,6 +295,7 @@ public:
class TPortionConstructors {
private:
THashMap<ui64, THashMap<ui64, TPortionInfoConstructor>> Constructors;
+
public:
THashMap<ui64, THashMap<ui64, TPortionInfoConstructor>>::iterator begin() {
return Constructors.begin();
@@ -326,7 +335,6 @@ public:
itPortionId->second.Merge(std::move(constructor));
return &itPortionId->second;
}
-
};
-}
+} // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp
index 39dfa6af20..065f9768fb 100644
--- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp
+++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp
@@ -672,6 +672,13 @@ TPortionInfo::TPreparedBatchData TPortionInfo::PrepareForAssemble(const ISnapsho
return PrepareForAssembleImpl(*this, dataSchema, resultSchema, blobsData);
}
+bool TPortionInfo::NeedShardingFilter(const TGranuleShardingInfo& shardingInfo) const {
+ if (ShardingVersion && shardingInfo.GetSnapshotVersion() <= *ShardingVersion) {
+ return false;
+ }
+ return true;
+}
+
std::shared_ptr<TDeserializeChunkedArray> TPortionInfo::TPreparedColumn::AssembleForSeqAccess() const {
Y_ABORT_UNLESS(!Blobs.empty());
diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h
index 966d636394..8232839dc4 100644
--- a/ydb/core/tx/columnshard/engines/portions/portion_info.h
+++ b/ydb/core/tx/columnshard/engines/portions/portion_info.h
@@ -101,6 +101,7 @@ public:
};
class TPortionInfoConstructor;
+class TGranuleShardingInfo;
class TPortionInfo {
public:
@@ -120,6 +121,7 @@ private:
TSnapshot MinSnapshotDeprecated = TSnapshot::Zero(); // {PlanStep, TxId} is min snapshot for {Granule, Portion}
TSnapshot RemoveSnapshot = TSnapshot::Zero(); // {XPlanStep, XTxId} is snapshot where the blob has been removed (i.e. compacted into another one)
std::optional<ui64> SchemaVersion;
+ std::optional<ui64> ShardingVersion;
TPortionMeta Meta;
YDB_READONLY_DEF(std::vector<TIndexChunk>, Indexes);
@@ -172,6 +174,16 @@ private:
public:
ui64 GetMinMemoryForReadColumns(const std::optional<std::set<ui32>>& columnIds) const;
+ bool NeedShardingFilter(const TGranuleShardingInfo& shardingInfo) const;
+
+ const std::optional<ui64>& GetShardingVersionOptional() const {
+ return ShardingVersion;
+ }
+
+ ui64 GetShardingVersionDef(const ui64 verDefault) const {
+ return ShardingVersion.value_or(verDefault);
+ }
+
void SetRemoveSnapshot(const TSnapshot& snap) {
AFL_VERIFY(!RemoveSnapshot.Valid());
RemoveSnapshot = snap;
@@ -438,6 +450,10 @@ public:
return TPortionAddress(PathId, Portion);
}
+ void ResetShardingVersion() {
+ ShardingVersion.reset();
+ }
+
void SetPathId(const ui64 pathId) {
PathId = pathId;
}
diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h
index 19ff369833..7770674caa 100644
--- a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h
+++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h
@@ -38,6 +38,8 @@ private:
TProgramContainer Program;
std::shared_ptr<TVersionedIndex> IndexVersionsPointer;
TSnapshot RequestSnapshot;
+ std::optional<TGranuleShardingInfo> RequestShardingInfo;
+
protected:
std::shared_ptr<ISnapshotSchema> ResultIndexSchema;
const TVersionedIndex& GetIndexVersions() const {
@@ -47,6 +49,10 @@ protected:
public:
using TConstPtr = std::shared_ptr<const TReadMetadataBase>;
+ const std::optional<TGranuleShardingInfo>& GetRequestShardingInfo() const {
+ return RequestShardingInfo;
+ }
+
void SetPKRangesFilter(const TPKRangesFilter& value) {
Y_ABORT_UNLESS(IsSorted() && value.IsReverse() == IsDescSorted());
Y_ABORT_UNLESS(!PKRangesFilter);
@@ -79,6 +85,11 @@ public:
return ResultIndexSchema->GetIndexInfo();
}
+ void InitShardingInfo(const ui64 pathId) {
+ AFL_VERIFY(!RequestShardingInfo);
+ RequestShardingInfo = IndexVersionsPointer->GetShardingInfoOptional(pathId, RequestSnapshot);
+ }
+
TReadMetadataBase(const std::shared_ptr<TVersionedIndex> index, const ESorting sorting, const TProgramContainer& ssaProgram, const std::shared_ptr<ISnapshotSchema>& schema, const TSnapshot& requestSnapshot)
: Sorting(sorting)
, Program(ssaProgram)
diff --git a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp
index 878153f52b..8b655d899a 100644
--- a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp
+++ b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp
@@ -121,7 +121,7 @@ void TColumnShardScan::HandleScan(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr& ev)
Y_ABORT_UNLESS(!AckReceivedInstant);
AckReceivedInstant = TMonotonic::Now();
- Y_ABORT_UNLESS(ev->Get()->Generation == ScanGen);
+ AFL_VERIFY(ev->Get()->Generation == ScanGen)("ev_gen", ev->Get()->Generation)("scan_gen", ScanGen);
ChunksLimiter = TChunksLimiter(ev->Get()->FreeSpace, ev->Get()->MaxChunksCount);
Y_ABORT_UNLESS(ev->Get()->MaxChunksCount == 1);
diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp
index a664f71756..076b69c7f4 100644
--- a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp
+++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp
@@ -10,6 +10,7 @@ std::unique_ptr<TScanIteratorBase> TReadMetadata::StartScan(const std::shared_pt
TConclusionStatus TReadMetadata::Init(const TReadDescription& readDescription, const TDataStorageAccessor& dataAccessor) {
SetPKRangesFilter(readDescription.PKRangesFilter);
+ InitShardingInfo(readDescription.PathId);
/// @note We could have column name changes between schema versions:
/// Add '1:foo', Drop '1:foo', Add '2:foo'. Drop should hide '1:foo' from reads.
diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp
index 06cbc877a3..f5415a4acf 100644
--- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp
+++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp
@@ -24,7 +24,7 @@ ui64 TSpecialReadContext::GetMemoryForSources(const THashMap<ui32, std::shared_p
result += ReadSequentiallyBufferSize;
} else {
if (!isExclusive && !CommonContext->IsReverse()) {
- result = 2 * result; // due to in time we will have data in original portion + data in merged(or reversed) interval
+ result = 2 * result; // due to in time we will have data in original portion + data in merged(or reversed) interval
}
}
return result;
@@ -35,8 +35,16 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::GetColumnsFetchingPlan(con
const bool partialUsageByPK = ReadMetadata->GetPKRangesFilter().IsPortionInPartialUsage(source->GetStartReplaceKey(), source->GetFinishReplaceKey(), ReadMetadata->GetIndexInfo());
const bool useIndexes = (IndexChecker ? source->HasIndexes(IndexChecker->GetIndexIds()) : false);
const bool isWholeExclusiveSource = source->GetExclusiveIntervalOnly() && source->IsSourceInMemory();
- if (auto result = CacheFetchingScripts[needSnapshots ? 1 : 0][isWholeExclusiveSource ? 1 : 0][partialUsageByPK ? 1 : 0][useIndexes ? 1 : 0]) {
-// AFL_WARN(NKikimrServices::TX_COLUMNSHARD_SCAN)("SS", needSnapshots)("PK", partialUsageByPK)("IDX", useIndexes)("EXCL", source->GetExclusiveIntervalOnly())("MEM", source->IsSourceInMemory())("result", result->DebugString());
+ bool needShardingFilter = false;
+ if (!!ReadMetadata->GetRequestShardingInfo()) {
+ auto ver = source->GetShardingVersionOptional();
+ if (!ver || *ver < ReadMetadata->GetRequestShardingInfo()->GetSnapshotVersion()) {
+ needShardingFilter = true;
+ }
+ }
+ if (auto result = CacheFetchingScripts[needSnapshots ? 1 : 0][isWholeExclusiveSource ? 1 : 0][partialUsageByPK ? 1 : 0][useIndexes ? 1 : 0][needShardingFilter ? 1 : 0]) {
+// AFL_WARN(NKikimrServices::TX_COLUMNSHARD_SCAN)("SS", needSnapshots)("PK", partialUsageByPK)("IDX", useIndexes)("SHARDING", needShardingFilter)
+// ("EXCL", source->GetExclusiveIntervalOnly())("MEM", source->IsSourceInMemory())("result", result->DebugString());
return result;
}
{
@@ -47,28 +55,35 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::GetColumnsFetchingPlan(con
}
}
-std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(const bool needSnapshots, const bool exclusiveSource,
- const bool partialUsageByPredicateExt, const bool useIndexes) const {
+std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(const bool needSnapshots, const bool exclusiveSource, const bool partialUsageByPredicateExt, const bool useIndexes,
+ const bool needFilterSharding) const {
std::shared_ptr<TFetchingScript> result = std::make_shared<TFetchingScript>();
const bool partialUsageByPredicate = partialUsageByPredicateExt && PredicateColumns->GetColumnsCount();
if (!!IndexChecker && useIndexes && exclusiveSource) {
result->AddStep(std::make_shared<TIndexBlobsFetchingStep>(std::make_shared<TIndexesSet>(IndexChecker->GetIndexIds())));
result->AddStep(std::make_shared<TApplyIndexStep>(IndexChecker));
}
+ bool hasFilterSharding = false;
+ if (needFilterSharding && !ShardingColumns->IsEmpty()) {
+ hasFilterSharding = true;
+ result->AddStep(std::make_shared<TColumnBlobsFetchingStep>(ShardingColumns));
+ result->AddStep(std::make_shared<TAssemblerStep>(ShardingColumns, "SPEC_SHARDING"));
+ result->AddStep(std::make_shared<TShardingFilter>());
+ }
if (!EFColumns->GetColumnsCount() && !partialUsageByPredicate) {
result->SetBranchName("simple");
- TColumnsSet columnsFetch = *FFColumns;
+ TColumnsSet columnsFetch = *FFColumns - *ShardingColumns;
if (needSnapshots) {
columnsFetch = columnsFetch + *SpecColumns;
}
if (!exclusiveSource) {
columnsFetch = columnsFetch + *PKColumns + *SpecColumns;
} else {
- if (columnsFetch.GetColumnsCount() == 1 && SpecColumns->Contains(columnsFetch)) {
+ if (columnsFetch.GetColumnsCount() == 1 && SpecColumns->Contains(columnsFetch) && !hasFilterSharding) {
return nullptr;
}
}
- if (columnsFetch.GetColumnsCount()) {
+ if (columnsFetch.GetColumnsCount() || hasFilterSharding) {
result->AddStep(std::make_shared<TColumnBlobsFetchingStep>(std::make_shared<TColumnsSet>(columnsFetch)));
if (!exclusiveSource) {
result->AddStep(std::make_shared<TAssemblerStep>(std::make_shared<TColumnsSet>(*PKColumns + *SpecColumns), "LAST"));
@@ -84,7 +99,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(c
}
} else if (exclusiveSource) {
result->SetBranchName("exclusive");
- TColumnsSet columnsFetch = *EFColumns;
+ TColumnsSet columnsFetch = *EFColumns - *ShardingColumns;
if (needSnapshots || FFColumns->Cross(*SpecColumns)) {
columnsFetch = columnsFetch + *SpecColumns;
}
@@ -120,7 +135,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(c
result->AddStep(std::make_shared<TFilterProgramStep>(i));
}
AFL_VERIFY(columnsFetch.IsEmpty());
- TColumnsSet columnsAdditionalFetch = *FFColumns - *EFColumns - *SpecColumns;
+ TColumnsSet columnsAdditionalFetch = *FFColumns - *EFColumns - *SpecColumns - *ShardingColumns;
if (partialUsageByPredicate) {
columnsAdditionalFetch = columnsAdditionalFetch - *PredicateColumns;
}
@@ -130,7 +145,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(c
}
} else {
result->SetBranchName("merge");
- TColumnsSet columnsFetch = *MergeColumns + *EFColumns;
+ TColumnsSet columnsFetch = *MergeColumns + *EFColumns - *ShardingColumns;
AFL_VERIFY(columnsFetch.GetColumnsCount());
result->AddStep(std::make_shared<TColumnBlobsFetchingStep>(std::make_shared<TColumnsSet>(columnsFetch)));
result->AddStep(std::make_shared<TAssemblerStep>(SpecColumns, "SPEC"));
@@ -158,7 +173,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(c
result->AddStep(std::make_shared<TFilterProgramStep>(i));
}
AFL_VERIFY(columnsFetchEF.IsEmpty());
- const TColumnsSet columnsAdditionalFetch = *FFColumns - *EFColumns - *SpecColumns - *PKColumns - *PredicateColumns;
+ const TColumnsSet columnsAdditionalFetch = *FFColumns - *EFColumns - *SpecColumns - *PKColumns - *PredicateColumns - *ShardingColumns;
if (columnsAdditionalFetch.GetColumnsCount()) {
result->AddStep(std::make_shared<TColumnBlobsFetchingStep>(std::make_shared<TColumnsSet>(columnsAdditionalFetch)));
result->AddStep(std::make_shared<TOptionalAssemblerStep>(std::make_shared<TColumnsSet>(columnsAdditionalFetch), "LAST"));
@@ -168,8 +183,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::BuildColumnsFetchingPlan(c
}
TSpecialReadContext::TSpecialReadContext(const std::shared_ptr<TReadContext>& commonContext)
- : CommonContext(commonContext)
-{
+ : CommonContext(commonContext) {
ReadMetadata = dynamic_pointer_cast<const TReadMetadata>(CommonContext->GetReadMetadata());
Y_ABORT_UNLESS(ReadMetadata);
Y_ABORT_UNLESS(ReadMetadata->SelectInfo);
@@ -185,6 +199,12 @@ TSpecialReadContext::TSpecialReadContext(const std::shared_ptr<TReadContext>& co
PredicateColumns = std::make_shared<TColumnsSet>();
}
}
+ if (!!ReadMetadata->GetRequestShardingInfo()) {
+ auto shardingColumnIds = ReadMetadata->GetIndexInfo().GetColumnIdsVerified(ReadMetadata->GetRequestShardingInfo()->GetShardingInfo()->GetColumnNames());
+ ShardingColumns = std::make_shared<TColumnsSet>(shardingColumnIds, ReadMetadata->GetIndexInfo(), ReadMetadata->GetResultSchema());
+ } else {
+ ShardingColumns = std::make_shared<TColumnsSet>();
+ }
{
auto efColumns = ReadMetadata->GetEarlyFilterColumnIds();
if (efColumns.size()) {
@@ -214,24 +234,14 @@ TSpecialReadContext::TSpecialReadContext(const std::shared_ptr<TReadContext>& co
PKColumns = std::make_shared<TColumnsSet>(ReadMetadata->GetPKColumnIds(), ReadMetadata->GetIndexInfo(), readSchema);
MergeColumns = std::make_shared<TColumnsSet>(*PKColumns + *SpecColumns);
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("columns_context_info", DebugString());
- CacheFetchingScripts[0][0][0][0] = BuildColumnsFetchingPlan(false, false, false, false);
- CacheFetchingScripts[0][1][0][0] = BuildColumnsFetchingPlan(false, true, false, false);
- CacheFetchingScripts[1][0][0][0] = BuildColumnsFetchingPlan(true, false, false, false);
- CacheFetchingScripts[1][1][0][0] = BuildColumnsFetchingPlan(true, true, false, false);
- CacheFetchingScripts[0][0][1][0] = BuildColumnsFetchingPlan(false, false, true, false);
- CacheFetchingScripts[0][1][1][0] = BuildColumnsFetchingPlan(false, true, true, false);
- CacheFetchingScripts[1][0][1][0] = BuildColumnsFetchingPlan(true, false, true, false);
- CacheFetchingScripts[1][1][1][0] = BuildColumnsFetchingPlan(true, true, true, false);
+ const auto GetBit = [](const ui32 val, const ui32 pos) -> ui32 {
+ return (val & (1 << pos)) ? 1 : 0;
+ };
- CacheFetchingScripts[0][0][0][1] = BuildColumnsFetchingPlan(false, false, false, true);
- CacheFetchingScripts[0][1][0][1] = BuildColumnsFetchingPlan(false, true, false, true);
- CacheFetchingScripts[1][0][0][1] = BuildColumnsFetchingPlan(true, false, false, true);
- CacheFetchingScripts[1][1][0][1] = BuildColumnsFetchingPlan(true, true, false, true);
- CacheFetchingScripts[0][0][1][1] = BuildColumnsFetchingPlan(false, false, true, true);
- CacheFetchingScripts[0][1][1][1] = BuildColumnsFetchingPlan(false, true, true, true);
- CacheFetchingScripts[1][0][1][1] = BuildColumnsFetchingPlan(true, false, true, true);
- CacheFetchingScripts[1][1][1][1] = BuildColumnsFetchingPlan(true, true, true, true);
+ AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("columns_context_info", DebugString());
+ for (ui32 i = 0; i < (1 << 6); ++i) {
+ CacheFetchingScripts[GetBit(i, 0)][GetBit(i, 1)][GetBit(i, 2)][GetBit(i, 3)][GetBit(i, 4)] = BuildColumnsFetchingPlan(GetBit(i, 0), GetBit(i, 1), GetBit(i, 2), GetBit(i, 3), GetBit(i, 4));
+ }
}
-}
+} // namespace NKikimr::NOlap::NReader::NPlain
diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h
index dc46c4b70b..2a08d3c062 100644
--- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h
+++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h
@@ -16,6 +16,7 @@ private:
YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, SpecColumns);
YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, MergeColumns);
+ YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, ShardingColumns);
YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, EFColumns);
YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, PredicateColumns);
YDB_READONLY_DEF(std::shared_ptr<TColumnsSet>, PKColumns);
@@ -25,8 +26,9 @@ private:
NIndexes::TIndexCheckerContainer IndexChecker;
TReadMetadata::TConstPtr ReadMetadata;
std::shared_ptr<TColumnsSet> EmptyColumns = std::make_shared<TColumnsSet>();
- std::shared_ptr<TFetchingScript> BuildColumnsFetchingPlan(const bool needSnapshotsFilter, const bool exclusiveSource, const bool partialUsageByPredicate, const bool useIndexes) const;
- std::array<std::array<std::array<std::array<std::shared_ptr<TFetchingScript>, 2>, 2>, 2>, 2> CacheFetchingScripts;
+ std::shared_ptr<TFetchingScript> BuildColumnsFetchingPlan(const bool needSnapshotsFilter, const bool exclusiveSource, const bool partialUsageByPredicate, const bool useIndexes, const bool needFilterSharding) const;
+ std::array<std::array<std::array<std::array<std::array<std::shared_ptr<TFetchingScript>, 2>, 2>, 2>, 2>, 2> CacheFetchingScripts;
+
public:
static const inline ui64 DefaultRejectMemoryIntervalLimit = ((ui64)3) << 30;
static const inline ui64 DefaultReduceMemoryIntervalLimit = DefaultRejectMemoryIntervalLimit;
@@ -45,12 +47,11 @@ public:
std::unique_ptr<NArrow::NMerger::TMergePartialStream> BuildMerger() const;
TString DebugString() const {
- return TStringBuilder() <<
- "ef=" << EFColumns->DebugString() << ";" <<
- "pk=" << PKColumns->DebugString() << ";" <<
- "ff=" << FFColumns->DebugString() << ";" <<
- "program_input=" << ProgramInputColumns->DebugString()
- ;
+ return TStringBuilder() << "ef=" << EFColumns->DebugString() << ";"
+ << "sharding=" << ShardingColumns->DebugString() << ";"
+ << "pk=" << PKColumns->DebugString() << ";"
+ << "ff=" << FFColumns->DebugString() << ";"
+ << "program_input=" << ProgramInputColumns->DebugString();
}
TSpecialReadContext(const std::shared_ptr<TReadContext>& commonContext);
diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp
index daef434331..c557f2064b 100644
--- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp
+++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp
@@ -101,6 +101,13 @@ TConclusion<bool> TSnapshotFilter::DoExecuteInplace(const std::shared_ptr<IDataS
return true;
}
+TConclusion<bool> TShardingFilter::DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& /*step*/) const {
+ NYDBTest::TControllers::GetColumnShardController()->OnSelectShardingFilter();
+ auto filter = source->GetContext()->GetReadMetadata()->GetRequestShardingInfo()->GetShardingInfo()->GetFilter(source->GetStageData().GetTable()->BuildTable());
+ source->MutableStageData().AddFilter(filter);
+ return true;
+}
+
TConclusion<bool> TBuildFakeSpec::DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& /*step*/) const {
std::vector<std::shared_ptr<arrow::Array>> columns;
for (auto&& f : TIndexInfo::ArrowSchemaSnapshot()->fields()) {
diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.h
index 1d0a1ceee8..ac8220f50c 100644
--- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.h
+++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.h
@@ -280,15 +280,27 @@ public:
}
};
-class TSnapshotFilter: public IFetchingStep {
+class TSnapshotFilter : public IFetchingStep {
private:
using TBase = IFetchingStep;
+
public:
virtual TConclusion<bool> DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& step) const override;
TSnapshotFilter()
: TBase("SNAPSHOT") {
+ }
+};
+class TShardingFilter : public IFetchingStep {
+private:
+ using TBase = IFetchingStep;
+
+public:
+ virtual TConclusion<bool> DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& step) const override;
+ TShardingFilter()
+ : TBase("SHARDING") {
}
};
+
}
diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp
index 723ba175f5..2436f2ebd7 100644
--- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp
+++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp
@@ -1,13 +1,15 @@
#include "source.h"
-#include "interval.h"
+
+#include "constructor.h"
#include "fetched_data.h"
+#include "interval.h"
#include "plain_read_data.h"
-#include "constructor.h"
+
+#include <ydb/core/formats/arrow/simple_arrays_cache.h>
#include <ydb/core/tx/columnshard/blobs_reader/actor.h>
#include <ydb/core/tx/columnshard/blobs_reader/events.h>
-#include <ydb/core/tx/conveyor/usage/service.h>
-#include <ydb/core/formats/arrow/simple_arrays_cache.h>
#include <ydb/core/tx/columnshard/hooks/abstract/abstract.h>
+#include <ydb/core/tx/conveyor/usage/service.h>
namespace NKikimr::NOlap::NReader::NPlain {
@@ -50,9 +52,8 @@ void IDataSource::OnInitResourcesGuard(const std::shared_ptr<IDataSource>& sourc
}
}
-void TPortionDataSource::NeedFetchColumns(const std::set<ui32>& columnIds,
- TBlobsAction& blobsAction, THashMap<TChunkAddress, ui32>& nullBlocks,
- const std::shared_ptr<NArrow::TColumnFilter>& filter) {
+void TPortionDataSource::NeedFetchColumns(const std::set<ui32>& columnIds, TBlobsAction& blobsAction,
+ THashMap<TChunkAddress, ui32>& nullBlocks, const std::shared_ptr<NArrow::TColumnFilter>& filter) {
const NArrow::TColumnFilter& cFilter = filter ? *filter : NArrow::TColumnFilter::BuildAllowFilter();
ui32 fetchedChunks = 0;
ui32 nullChunks = 0;
@@ -78,7 +79,8 @@ void TPortionDataSource::NeedFetchColumns(const std::set<ui32>& columnIds,
}
AFL_VERIFY(itFinished)("filter", itFilter.DebugString())("count", Portion->NumRows(i));
}
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "chunks_stats")("fetch", fetchedChunks)("null", nullChunks)("reading_actions", blobsAction.GetStorageIds())("columns", columnIds.size());
+ AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "chunks_stats")("fetch", fetchedChunks)("null", nullChunks)
+ ("reading_actions", blobsAction.GetStorageIds())("columns", columnIds.size());
}
bool TPortionDataSource::DoStartFetchingColumns(const std::shared_ptr<IDataSource>& sourcePtr, const TFetchingScriptCursor& step, const std::shared_ptr<TColumnsSet>& columns) {
@@ -143,7 +145,7 @@ void TPortionDataSource::DoAbort() {
void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& indexChecker) {
THashMap<ui32, std::vector<TString>> indexBlobs;
std::set<ui32> indexIds = indexChecker->GetIndexIds();
-// NActors::TLogContextGuard gLog = NActors::TLogContextBuilder::Build()("records_count", GetRecordsCount())("portion_id", Portion->GetAddress().DebugString());
+ // NActors::TLogContextGuard gLog = NActors::TLogContextBuilder::Build()("records_count", GetRecordsCount())("portion_id", Portion->GetAddress().DebugString());
std::vector<TPortionInfo::TPage> pages = Portion->BuildPages();
NArrow::TColumnFilter constructor = NArrow::TColumnFilter::BuildAllowFilter();
for (auto&& p : pages) {
@@ -238,4 +240,4 @@ void TCommittedDataSource::DoAssembleColumns(const std::shared_ptr<TColumnsSet>&
MutableStageData().SyncTableColumns(columns->GetSchema()->fields());
}
-}
+} // namespace NKikimr::NOlap::NReader::NPlain
diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h
index 8e95c20d79..9230c9940c 100644
--- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h
+++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h
@@ -33,8 +33,10 @@ private:
NArrow::TReplaceKey StartReplaceKey;
NArrow::TReplaceKey FinishReplaceKey;
YDB_READONLY_DEF(std::shared_ptr<TSpecialReadContext>, Context);
+ YDB_READONLY(TSnapshot, RecordSnapshotMin, TSnapshot::Zero());
YDB_READONLY(TSnapshot, RecordSnapshotMax, TSnapshot::Zero());
YDB_READONLY(ui32, RecordsCount, 0);
+ YDB_READONLY_DEF(std::optional<ui64>, ShardingVersionOptional);
YDB_READONLY(ui32, IntervalsCount, 0);
virtual NJson::TJsonValue DoDebugJson() const = 0;
bool MergingStartedFlag = false;
@@ -182,16 +184,18 @@ public:
void RegisterInterval(TFetchingInterval& interval);
IDataSource(const ui32 sourceIdx, const std::shared_ptr<TSpecialReadContext>& context,
- const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& finish,
- const TSnapshot& recordSnapshotMax, const ui32 recordsCount)
+ const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& finish, const TSnapshot& recordSnapshotMin, const TSnapshot& recordSnapshotMax,
+ const ui32 recordsCount, const std::optional<ui64> shardingVersion)
: SourceIdx(sourceIdx)
, Start(context->GetReadMetadata()->BuildSortedPosition(start))
, Finish(context->GetReadMetadata()->BuildSortedPosition(finish))
, StartReplaceKey(start)
, FinishReplaceKey(finish)
, Context(context)
+ , RecordSnapshotMin(recordSnapshotMin)
, RecordSnapshotMax(recordSnapshotMax)
, RecordsCount(recordsCount)
+ , ShardingVersionOptional(shardingVersion)
{
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "portions_for_merge")("start", Start.DebugJson())("finish", Finish.DebugJson());
if (Start.IsReverseSort()) {
@@ -305,7 +309,7 @@ public:
TPortionDataSource(const ui32 sourceIdx, const std::shared_ptr<TPortionInfo>& portion, const std::shared_ptr<TSpecialReadContext>& context,
const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& finish)
- : TBase(sourceIdx, context, start, finish, portion->RecordSnapshotMax(), portion->GetRecordsCount())
+ : TBase(sourceIdx, context, start, finish, portion->RecordSnapshotMin(), portion->RecordSnapshotMax(), portion->GetRecordsCount(), portion->GetShardingVersionOptional())
, Portion(portion)
, Schema(GetContext()->GetReadMetadata()->GetLoadSchemaVerified(*Portion))
{
@@ -377,7 +381,7 @@ public:
TCommittedDataSource(const ui32 sourceIdx, const TCommittedBlob& committed, const std::shared_ptr<TSpecialReadContext>& context,
const NArrow::TReplaceKey& start, const NArrow::TReplaceKey& finish)
- : TBase(sourceIdx, context, start, finish, committed.GetSnapshot(), committed.GetRecordsCount())
+ : TBase(sourceIdx, context, start, finish, committed.GetSnapshot(), committed.GetSnapshot(), committed.GetRecordsCount(), {})
, CommittedBlob(committed) {
}
diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp
index 8db0b5e2d0..59dccac5ee 100644
--- a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp
+++ b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp
@@ -73,7 +73,7 @@ bool TIndexInfo::IsSpecialColumn(const ui32 fieldId) {
|| fieldId == (ui32)ESpecialColumn::TX_ID;
}
-ui32 TIndexInfo::GetColumnId(const std::string& name) const {
+ui32 TIndexInfo::GetColumnIdVerified(const std::string& name) const {
auto id = GetColumnIdOptional(name);
Y_ABORT_UNLESS(!!id, "undefined column %s", name.data());
return *id;
diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.h b/ydb/core/tx/columnshard/engines/scheme/index_info.h
index bd1dfec353..b0e274a3a2 100644
--- a/ydb/core/tx/columnshard/engines/scheme/index_info.h
+++ b/ydb/core/tx/columnshard/engines/scheme/index_info.h
@@ -233,7 +233,17 @@ public:
}
/// Returns an id of the column located by name. The name should exists in the schema.
- ui32 GetColumnId(const std::string& name) const;
+ ui32 GetColumnIdVerified(const std::string& name) const;
+ ui32 GetColumnId(const std::string& name) const {
+ return GetColumnIdVerified(name);
+ }
+ std::set<ui32> GetColumnIdsVerified(const std::set<TString>& names) const {
+ std::set<ui32> result;
+ for (auto&& i : names) {
+ AFL_VERIFY(result.emplace(GetColumnIdVerified(i)).second);
+ }
+ return result;
+ }
std::optional<ui32> GetColumnIdOptional(const std::string& name) const;
/// Returns a name of the column located by id.
diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp
index e7cf6f30ed..115cdc16dd 100644
--- a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp
+++ b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp
@@ -2,6 +2,7 @@
#include "snapshot_scheme.h"
#include <ydb/core/tx/columnshard/engines/scheme/index_info.h>
+#include <ydb/core/tx/columnshard/engines/db_wrapper.h>
namespace NKikimr::NOlap {
@@ -28,4 +29,22 @@ void TVersionedIndex::AddIndex(const TSnapshot& snapshot, TIndexInfo&& indexInfo
LastSchemaVersion = std::max(newVersion, LastSchemaVersion);
}
+bool TVersionedIndex::LoadShardingInfo(IDbWrapper& db) {
+ TConclusion<THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>>> shardingLocal = db.LoadGranulesShardingInfo();
+ if (shardingLocal.IsFail()) {
+ return false;
+ }
+ ShardingInfo = std::move(shardingLocal.DetachResult());
+ return true;
+}
+
+std::optional<NKikimr::NOlap::TGranuleShardingInfo> TVersionedIndex::GetShardingInfoActual(const ui64 pathId) const {
+ auto it = ShardingInfo.find(pathId);
+ if (it == ShardingInfo.end() || it->second.empty()) {
+ return std::nullopt;
+ } else {
+ return it->second.rbegin()->second;
+ }
+}
+
}
diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h
index d83d338350..ae2dca6eff 100644
--- a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h
+++ b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h
@@ -1,15 +1,37 @@
#pragma once
#include "abstract_scheme.h"
+#include <ydb/core/tx/sharding/sharding.h>
namespace NKikimr::NOlap {
+class IDbWrapper;
+
+class TGranuleShardingInfo {
+private:
+ YDB_READONLY_DEF(NSharding::TGranuleShardingLogicContainer, ShardingInfo);
+ YDB_READONLY(TSnapshot, SinceSnapshot, TSnapshot::Zero());
+ YDB_READONLY(ui64, SnapshotVersion, 0);
+ YDB_READONLY(ui64, PathId, 0);
+
+public:
+ TGranuleShardingInfo(const NSharding::TGranuleShardingLogicContainer& shardingInfo, const TSnapshot& sinceSnapshot, const ui64 version, const ui64 pathId)
+ : ShardingInfo(shardingInfo)
+ , SinceSnapshot(sinceSnapshot)
+ , SnapshotVersion(version)
+ , PathId(pathId) {
+ AFL_VERIFY(!!ShardingInfo);
+ }
+};
+
class TVersionedIndex {
+ THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>> ShardingInfo;
std::map<TSnapshot, ISnapshotSchema::TPtr> Snapshots;
std::shared_ptr<arrow::Schema> PrimaryKey;
std::map<ui64, ISnapshotSchema::TPtr> SnapshotByVersion;
ui64 LastSchemaVersion = 0;
std::optional<ui64> SchemeVersionForActualization;
ISnapshotSchema::TPtr SchemeForActualization;
+
public:
ISnapshotSchema::TPtr GetLastCriticalSchema() const {
return SchemeForActualization;
@@ -20,6 +42,29 @@ public:
return result ? result : defaultSchema;
}
+ std::optional<TGranuleShardingInfo> GetShardingInfoOptional(const ui64 pathId, const TSnapshot& ss) const {
+ auto it = ShardingInfo.find(pathId);
+ if (it == ShardingInfo.end() || it->second.empty()) {
+ return std::nullopt;
+ } else {
+ auto itSS = it->second.upper_bound(ss);
+ if (itSS == it->second.end()) {
+ return it->second.rbegin()->second;
+ } else if (itSS == it->second.begin()) {
+ return std::nullopt;
+ } else {
+ --itSS;
+ return itSS->second;
+ }
+ }
+ }
+
+ std::optional<TGranuleShardingInfo> GetShardingInfoActual(const ui64 pathId) const;
+
+ void AddShardingInfo(const TGranuleShardingInfo& shardingInfo) {
+ AFL_VERIFY(ShardingInfo[shardingInfo.GetPathId()].emplace(shardingInfo.GetSinceSnapshot(), shardingInfo).second);
+ }
+
TString DebugString() const {
TStringBuilder sb;
for (auto&& i : Snapshots) {
@@ -47,7 +92,7 @@ public:
}
Y_ABORT_UNLESS(!Snapshots.empty());
Y_ABORT_UNLESS(version.IsZero());
- return Snapshots.begin()->second; // For old compaction logic compatibility
+ return Snapshots.begin()->second;
}
ISnapshotSchema::TPtr GetLastSchema() const {
@@ -64,5 +109,7 @@ public:
}
void AddIndex(const TSnapshot& snapshot, TIndexInfo&& indexInfo);
+
+ bool LoadShardingInfo(IDbWrapper& db);
};
-}
+} // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp b/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp
index 34a570d2de..946b657400 100644
--- a/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp
+++ b/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp
@@ -1,8 +1,8 @@
#include "helper.h"
-#include <ydb/core/tx/columnshard/engines/db_wrapper.h>
-#include <ydb/core/tx/columnshard/engines/insert_table/insert_table.h>
#include <ydb/core/tx/columnshard/columnshard_schema.h>
+#include <ydb/core/tx/columnshard/engines/db_wrapper.h>
+#include <ydb/core/tx/columnshard/engines/insert_table/insert_table.h>
#include <library/cpp/testing/unittest/registar.h>
#include <util/string/printf.h>
@@ -14,43 +14,62 @@ using namespace NKikimr::NOlap::NEngines::NTest;
namespace {
-class TTestInsertTableDB: public IDbWrapper {
+class TTestInsertTableDB : public IDbWrapper {
public:
- void Insert(const TInsertedData&) override {}
- void Commit(const TInsertedData&) override {}
- void Abort(const TInsertedData&) override {}
- void EraseInserted(const TInsertedData&) override {}
- void EraseCommitted(const TInsertedData&) override {}
- void EraseAborted(const TInsertedData&) override {}
-
- bool Load(TInsertTableAccessor&,
- const TInstant&) override {
+ void Insert(const TInsertedData&) override {
+ }
+ void Commit(const TInsertedData&) override {
+ }
+ void Abort(const TInsertedData&) override {
+ }
+ void EraseInserted(const TInsertedData&) override {
+ }
+ void EraseCommitted(const TInsertedData&) override {
+ }
+ void EraseAborted(const TInsertedData&) override {
+ }
+
+ virtual TConclusion<THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>>> LoadGranulesShardingInfo() override {
+ THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>> result;
+ return result;
+ }
+
+ bool Load(TInsertTableAccessor&, const TInstant&) override {
return true;
}
virtual void WritePortion(const NOlap::TPortionInfo& /*portion*/) override {
-
}
virtual void ErasePortion(const NOlap::TPortionInfo& /*portion*/) override {
-
}
virtual bool LoadPortions(const std::function<void(NOlap::TPortionInfoConstructor&&, const NKikimrTxColumnShard::TIndexPortionMeta&)>& /*callback*/) override {
return true;
}
- void WriteColumn(const TPortionInfo&, const TColumnRecord&, const ui32 /*firstPKColumnId*/) override {}
- void EraseColumn(const TPortionInfo&, const TColumnRecord&) override {}
- bool LoadColumns(const std::function<void(NOlap::TPortionInfoConstructor&&, const TColumnChunkLoadContext&)>&) override { return true; }
+ void WriteColumn(const TPortionInfo&, const TColumnRecord&, const ui32 /*firstPKColumnId*/) override {
+ }
+ void EraseColumn(const TPortionInfo&, const TColumnRecord&) override {
+ }
+ bool LoadColumns(const std::function<void(NOlap::TPortionInfoConstructor&&, const TColumnChunkLoadContext&)>&) override {
+ return true;
+ }
- virtual void WriteIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override {}
- virtual void EraseIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override {}
- virtual bool LoadIndexes(const std::function<void(const ui64 /*pathId*/, const ui64 /*portionId*/, const TIndexChunkLoadContext&)>& /*callback*/) override { return true; }
+ virtual void WriteIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override {
+ }
+ virtual void EraseIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override {
+ }
+ virtual bool LoadIndexes(const std::function<void(const ui64 /*pathId*/, const ui64 /*portionId*/, const TIndexChunkLoadContext&)>& /*callback*/) override {
+ return true;
+ }
- void WriteCounter(ui32, ui64) override {}
- bool LoadCounters(const std::function<void(ui32 id, ui64 value)>&) override { return true; }
+ void WriteCounter(ui32, ui64) override {
+ }
+ bool LoadCounters(const std::function<void(ui32 id, ui64 value)>&) override {
+ return true;
+ }
};
-}
+} // namespace
Y_UNIT_TEST_SUITE(TColumnEngineTestInsertTable) {
Y_UNIT_TEST(TestInsertCommit) {
@@ -79,13 +98,15 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestInsertTable) {
// read nothing
auto blobs = insertTable.Read(tableId, TSnapshot::Zero(), nullptr);
UNIT_ASSERT_EQUAL(blobs.size(), 0);
- blobs = insertTable.Read(tableId+1, TSnapshot::Zero(), nullptr);
+ blobs = insertTable.Read(tableId + 1, TSnapshot::Zero(), nullptr);
UNIT_ASSERT_EQUAL(blobs.size(), 0);
// commit
ui64 planStep = 100;
ui64 txId = 42;
- insertTable.Commit(dbTable, planStep, txId, {TWriteId{writeId}}, [](ui64){ return true; });
+ insertTable.Commit(dbTable, planStep, txId, {TWriteId{writeId}}, [](ui64) {
+ return true;
+ });
UNIT_ASSERT_EQUAL(insertTable.GetPathPriorities().size(), 1);
UNIT_ASSERT_EQUAL(insertTable.GetPathPriorities().begin()->second.size(), 1);
@@ -94,15 +115,15 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestInsertTable) {
// read old snapshot
blobs = insertTable.Read(tableId, TSnapshot::Zero(), nullptr);
UNIT_ASSERT_EQUAL(blobs.size(), 0);
- blobs = insertTable.Read(tableId+1, TSnapshot::Zero(), nullptr);
+ blobs = insertTable.Read(tableId + 1, TSnapshot::Zero(), nullptr);
UNIT_ASSERT_EQUAL(blobs.size(), 0);
// read new snapshot
blobs = insertTable.Read(tableId, TSnapshot(planStep, txId), nullptr);
UNIT_ASSERT_EQUAL(blobs.size(), 1);
- blobs = insertTable.Read(tableId+1, TSnapshot::Zero(), nullptr);
+ blobs = insertTable.Read(tableId + 1, TSnapshot::Zero(), nullptr);
UNIT_ASSERT_EQUAL(blobs.size(), 0);
}
}
-}
+} // namespace NKikimr
diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp
index fc4607fb62..9b5bec6ff3 100644
--- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp
+++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp
@@ -41,6 +41,11 @@ public:
THashMap<ui32, ui64> Counters;
};
+ virtual TConclusion<THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>>> LoadGranulesShardingInfo() override {
+ THashMap<ui64, std::map<TSnapshot, TGranuleShardingInfo>> result;
+ return result;
+ }
+
void Insert(const TInsertedData& data) override {
Inserted.emplace(TWriteId{data.WriteTxId}, data);
}
diff --git a/ydb/core/tx/columnshard/export/session/session.h b/ydb/core/tx/columnshard/export/session/session.h
index ce98f56a8d..7d73595a0e 100644
--- a/ydb/core/tx/columnshard/export/session/session.h
+++ b/ydb/core/tx/columnshard/export/session/session.h
@@ -86,7 +86,7 @@ public:
virtual bool IsFinished() const override {
return Status == EStatus::Finished;
}
- virtual bool IsReadyForRemove() const override {
+ virtual bool IsReadyForRemoveOnFinished() const override {
return Status == EStatus::Aborted;
}
diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h
index bf90233492..7ffa19144a 100644
--- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h
+++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h
@@ -88,6 +88,10 @@ public:
using TPtr = std::shared_ptr<ICSController>;
virtual ~ICSController() = default;
+ virtual void OnSelectShardingFilter() {
+
+ }
+
virtual TDuration GetCompactionActualizationLag(const TDuration def) const {
return def;
}
diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.cpp b/ydb/core/tx/columnshard/hooks/testing/controller.cpp
index e711a72d4c..4943addc75 100644
--- a/ydb/core/tx/columnshard/hooks/testing/controller.cpp
+++ b/ydb/core/tx/columnshard/hooks/testing/controller.cpp
@@ -112,9 +112,11 @@ bool TController::IsTrivialLinks() const {
TGuard<TMutex> g(Mutex);
for (auto&& i : ShardActuals) {
if (!i.second->GetStoragesManager()->GetSharedBlobsManager()->IsTrivialLinks()) {
+ AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("reason", "non_trivial");
return false;
}
if (i.second->GetStoragesManager()->HasBlobsToDelete()) {
+ AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("reason", "has_delete");
return false;
}
}
diff --git a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h
index 8d52e8400f..191e7b7b34 100644
--- a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h
+++ b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h
@@ -27,10 +27,15 @@ private:
YDB_READONLY(TAtomicCounter, ActualizationsCount, 0);
YDB_READONLY(TAtomicCounter, ActualizationRefreshSchemeCount, 0);
YDB_READONLY(TAtomicCounter, ActualizationRefreshTieringCount, 0);
+ YDB_READONLY(TAtomicCounter, ShardingFiltersCount, 0);
YDB_ACCESSOR(TAtomicCounter, CompactionsLimit, 10000000);
protected:
+ virtual void OnSelectShardingFilter() override {
+ ShardingFiltersCount.Inc();
+ }
+
virtual void AddPortionForActualizer(const i32 portionsCount) override {
NeedActualizationCount.Add(portionsCount);
}
diff --git a/ydb/core/tx/columnshard/operations/write.cpp b/ydb/core/tx/columnshard/operations/write.cpp
index 27daff6a31..119a29f02c 100644
--- a/ydb/core/tx/columnshard/operations/write.cpp
+++ b/ydb/core/tx/columnshard/operations/write.cpp
@@ -1,31 +1,29 @@
-#include "write.h"
#include "slice_builder.h"
+#include "write.h"
-#include <ydb/core/tx/columnshard/columnshard_schema.h>
+#include <ydb/core/tablet_flat/tablet_flat_executor.h>
+#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h>
#include <ydb/core/tx/columnshard/blobs_action/blob_manager_db.h>
#include <ydb/core/tx/columnshard/columnshard_impl.h>
-#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h>
+#include <ydb/core/tx/columnshard/columnshard_schema.h>
#include <ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h>
#include <ydb/core/tx/conveyor/usage/service.h>
-#include <ydb/core/tablet_flat/tablet_flat_executor.h>
-
-
namespace NKikimr::NColumnShard {
- TWriteOperation::TWriteOperation(const TWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, const TInstant createdAt)
+ TWriteOperation::TWriteOperation(const TWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, const TInstant createdAt, const std::optional<ui32> granuleShardingVersionId)
: Status(status)
, CreatedAt(createdAt)
, WriteId(writeId)
, LockId(lockId)
, Cookie(cookie)
- {
+ , GranuleShardingVersionId(granuleShardingVersionId) {
}
void TWriteOperation::Start(TColumnShard& owner, const ui64 tableId, const NEvWrite::IDataContainer::TPtr& data, const NActors::TActorId& source, const TActorContext& ctx) {
Y_ABORT_UNLESS(Status == EOperationStatus::Draft);
- NEvWrite::TWriteMeta writeMeta((ui64)WriteId, tableId, source);
+ NEvWrite::TWriteMeta writeMeta((ui64)WriteId, tableId, source, GranuleShardingVersionId);
std::shared_ptr<NConveyor::ITask> task = std::make_shared<NOlap::TBuildSlicesTask>(owner.TabletID(), ctx.SelfID, owner.BufferizationWriteActorId,
NEvWrite::TWriteData(writeMeta, data, owner.TablesManager.GetPrimaryIndex()->GetReplaceKey(),
owner.StoragesManager->GetInsertOperator()->StartWritingAction(NOlap::NBlobOperations::EConsumer::WRITING_OPERATOR)));
@@ -46,7 +44,7 @@ namespace NKikimr::NColumnShard {
};
auto counters = owner.InsertTable->Commit(dbTable, snapshot.GetPlanStep(), snapshot.GetTxId(), { gWriteId },
- pathExists);
+ pathExists);
owner.IncCounter(COUNTER_BLOBS_COMMITTED, counters.Rows);
owner.IncCounter(COUNTER_BYTES_COMMITTED, counters.Bytes);
@@ -72,11 +70,11 @@ namespace NKikimr::NColumnShard {
NIceDb::TUpdate<Schema::Operations::CreatedAt>(CreatedAt.Seconds()),
NIceDb::TUpdate<Schema::Operations::Metadata>(metadata),
NIceDb::TUpdate<Schema::Operations::LockId>(LockId),
- NIceDb::TUpdate<Schema::Operations::Cookie>(Cookie)
- );
+ NIceDb::TUpdate<Schema::Operations::Cookie>(Cookie),
+ NIceDb::TUpdate<Schema::Operations::GranuleShardingVersionId>(GranuleShardingVersionId.value_or(0)));
}
- void TWriteOperation::ToProto(NKikimrTxColumnShard::TInternalOperationData& proto) const {
+ void TWriteOperation::ToProto(NKikimrTxColumnShard::TInternalOperationData& proto) const {
for (auto&& writeId : GlobalWriteIds) {
proto.AddInternalWriteIds((ui64)writeId);
}
@@ -108,17 +106,21 @@ namespace NKikimr::NColumnShard {
}
while (!rowset.EndOfSet()) {
- const TWriteId writeId = (TWriteId) rowset.GetValue<Schema::Operations::WriteId>();
+ const TWriteId writeId = (TWriteId)rowset.GetValue<Schema::Operations::WriteId>();
const ui64 createdAtSec = rowset.GetValue<Schema::Operations::CreatedAt>();
const ui64 lockId = rowset.GetValue<Schema::Operations::LockId>();
const ui64 cookie = rowset.GetValueOrDefault<Schema::Operations::Cookie>(0);
const TString metadata = rowset.GetValue<Schema::Operations::Metadata>();
- const EOperationStatus status = (EOperationStatus) rowset.GetValue<Schema::Operations::Status>();
+ const EOperationStatus status = (EOperationStatus)rowset.GetValue<Schema::Operations::Status>();
+ std::optional<ui32> granuleShardingVersionId;
+ if (rowset.HaveValue<Schema::Operations::GranuleShardingVersionId>() && rowset.GetValue<Schema::Operations::GranuleShardingVersionId>()) {
+ granuleShardingVersionId = rowset.GetValue<Schema::Operations::GranuleShardingVersionId>();
+ }
NKikimrTxColumnShard::TInternalOperationData metaProto;
Y_ABORT_UNLESS(metaProto.ParseFromString(metadata));
- auto operation = std::make_shared<TWriteOperation>(writeId, lockId, cookie, status, TInstant::Seconds(createdAtSec));
+ auto operation = std::make_shared<TWriteOperation>(writeId, lockId, cookie, status, TInstant::Seconds(createdAtSec), granuleShardingVersionId);
operation->FromProto(metaProto);
AFL_VERIFY(operation->GetStatus() != EOperationStatus::Draft);
@@ -145,7 +147,7 @@ namespace NKikimr::NColumnShard {
const ui64 txId = rowset.GetValue<Schema::OperationTxIds::TxId>();
AFL_VERIFY(Locks.contains(lockId))("lock_id", lockId);
Tx2Lock[txId] = lockId;
- if (!rowset.Next()) {
+ if (!rowset.Next()) {
return false;
}
}
@@ -208,7 +210,7 @@ namespace NKikimr::NColumnShard {
AFL_VERIFY(!!lockId)("tx_id", txId);
Locks.erase(*lockId);
Tx2Lock.erase(txId);
- for (auto&& op: operations) {
+ for (auto&& op : operations) {
RemoveOperation(op, txc);
}
NIceDb::TNiceDb db(txc.DB);
@@ -239,9 +241,9 @@ namespace NKikimr::NColumnShard {
db.Table<Schema::OperationTxIds>().Key(txId, lockId).Update();
}
- TWriteOperation::TPtr TOperationsManager::RegisterOperation(const ui64 lockId, const ui64 cookie) {
+ TWriteOperation::TPtr TOperationsManager::RegisterOperation(const ui64 lockId, const ui64 cookie, const std::optional<ui32> granuleShardingVersionId) {
auto writeId = BuildNextWriteId();
- auto operation = std::make_shared<TWriteOperation>(writeId, lockId, cookie, EOperationStatus::Draft, AppData()->TimeProvider->Now());
+ auto operation = std::make_shared<TWriteOperation>(writeId, lockId, cookie, EOperationStatus::Draft, AppData()->TimeProvider->Now(), granuleShardingVersionId);
Y_ABORT_UNLESS(Operations.emplace(operation->GetWriteId(), operation).second);
Locks[operation->GetLockId()].push_back(operation->GetWriteId());
return operation;
@@ -265,4 +267,4 @@ namespace NKikimr::NColumnShard {
}
return EOperationBehaviour::Undefined;
}
-}
+} // namespace NKikimr::NColumnShard
diff --git a/ydb/core/tx/columnshard/operations/write.h b/ydb/core/tx/columnshard/operations/write.h
index 08fe7724b3..f9719dd79b 100644
--- a/ydb/core/tx/columnshard/operations/write.h
+++ b/ydb/core/tx/columnshard/operations/write.h
@@ -44,11 +44,11 @@ namespace NKikimr::NColumnShard {
YDB_READONLY(ui64, Cookie, 0);
YDB_READONLY_DEF(TVector<TWriteId>, GlobalWriteIds);
YDB_ACCESSOR(EOperationBehaviour, Behaviour, EOperationBehaviour::Undefined);
-
+ YDB_READONLY_DEF(std::optional<ui32>, GranuleShardingVersionId);
public:
using TPtr = std::shared_ptr<TWriteOperation>;
- TWriteOperation(const TWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, const TInstant createdAt);
+ TWriteOperation(const TWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, const TInstant createdAt, const std::optional<ui32> granuleShardingVersionId);
void Start(TColumnShard& owner, const ui64 tableId, const NEvWrite::IDataContainer::TPtr& data, const NActors::TActorId& source, const TActorContext& ctx);
void OnWriteFinish(NTabletFlatExecutor::TTransactionContext& txc, const TVector<TWriteId>& globalWriteIds);
@@ -78,7 +78,7 @@ namespace NKikimr::NColumnShard {
void LinkTransaction(const ui64 lockId, const ui64 txId, NTabletFlatExecutor::TTransactionContext& txc);
std::optional<ui64> GetLockForTx(const ui64 lockId) const;
- TWriteOperation::TPtr RegisterOperation(const ui64 lockId, const ui64 cookie);
+ TWriteOperation::TPtr RegisterOperation(const ui64 lockId, const ui64 cookie, const std::optional<ui32> granuleShardingVersionId);
static EOperationBehaviour GetBehaviour(const NEvents::TDataEvents::TEvWrite& evWrite);
private:
diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp
index e7d3ca6ba0..d1e89d0da2 100644
--- a/ydb/core/tx/columnshard/tables_manager.cpp
+++ b/ydb/core/tx/columnshard/tables_manager.cpp
@@ -1,14 +1,15 @@
#include "tables_manager.h"
+
#include "columnshard_schema.h"
#include "engines/column_engine_logs.h"
-#include <ydb/core/tx/columnshard/blobs_action/blob_manager_db.h>
+#include "transactions/transactions/tx_add_sharding_info.h"
+
#include <ydb/core/scheme/scheme_types_proto.h>
-#include <ydb/core/tx/tiering/manager.h>
#include <ydb/core/tablet_flat/tablet_flat_executor.h>
+#include <ydb/core/tx/columnshard/blobs_action/blob_manager_db.h>
+#include <ydb/core/tx/tiering/manager.h>
#include <library/cpp/protobuf/json/proto2json.h>
-#include <ydb/core/tablet_flat/tablet_flat_executor.h>
-
namespace NKikimr::NColumnShard {
@@ -293,6 +294,10 @@ void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapsho
}
}
+std::unique_ptr<NTabletFlatExecutor::ITransaction> TTablesManager::CreateAddShardingInfoTx(TColumnShard& owner, const ui64 pathId, const ui64 versionId, const NSharding::TGranuleShardingLogicContainer& tabletShardingLogic) const {
+ return std::make_unique<TTxAddShardingInfo>(owner, tabletShardingLogic, pathId, versionId);
+}
+
void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, NIceDb::TNiceDb& db, std::shared_ptr<TTiersManager>& manager) {
auto it = Tables.find(pathId);
AFL_VERIFY(it != Tables.end());
@@ -329,8 +334,7 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot&
TTablesManager::TTablesManager(const std::shared_ptr<NOlap::IStoragesManager>& storagesManager, const ui64 tabletId)
: StoragesManager(storagesManager)
- , TabletId(tabletId)
-{
+ , TabletId(tabletId) {
}
bool TTablesManager::TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, const ui64 pathId) const {
@@ -358,4 +362,4 @@ bool TTablesManager::TryFinalizeDropPathOnComplete(const ui64 pathId) {
return true;
}
-}
+} // namespace NKikimr::NColumnShard
diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h
index 60e1a5f66d..72606173c5 100644
--- a/ydb/core/tx/columnshard/tables_manager.h
+++ b/ydb/core/tx/columnshard/tables_manager.h
@@ -242,6 +242,8 @@ public:
void AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db);
void AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, NIceDb::TNiceDb& db, std::shared_ptr<TTiersManager>& manager);
bool FillMonitoringReport(NTabletFlatExecutor::TTransactionContext& txc, NJson::TJsonValue& json);
+
+ [[nodiscard]] std::unique_ptr<NTabletFlatExecutor::ITransaction> CreateAddShardingInfoTx(TColumnShard& owner, const ui64 pathId, const ui64 versionId, const NSharding::TGranuleShardingLogicContainer& tabletShardingLogic) const;
};
}
diff --git a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.cpp b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.cpp
index 784ebc3eda..4d934c3f00 100644
--- a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.cpp
+++ b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.cpp
@@ -1,4 +1,5 @@
#include "long_tx_write.h"
+#include <ydb/core/tx/columnshard/engines/column_engine_logs.h>
namespace NKikimr::NColumnShard {
@@ -18,6 +19,15 @@ TLongTxTransactionOperator::TProposeResult TLongTxTransactionOperator::DoStartPr
return TProposeResult(NKikimrTxColumnShard::EResultStatus::ERROR,
TStringBuilder() << "Commit TxId# " << GetTxId() << " references WriteId# " << (ui64)writeId << " that is already locked by TxId# " << lw.PreparedTxId);
}
+
+ auto it = owner.InsertTable->GetInserted().find(writeId);
+ AFL_VERIFY(it != owner.InsertTable->GetInserted().end());
+
+ auto granuleShardingInfo = owner.GetIndexAs<NOlap::TColumnEngineForLogs>().GetVersionedIndex().GetShardingInfoActual(it->second.PathId);
+ if (granuleShardingInfo && lw.GranuleShardingVersionId && *lw.GranuleShardingVersionId != granuleShardingInfo->GetSnapshotVersion()) {
+ return TProposeResult(NKikimrTxColumnShard::EResultStatus::ERROR,
+ TStringBuilder() << "Commit TxId# " << GetTxId() << " references WriteId# " << (ui64)writeId << " declined through sharding deprecated");
+ }
}
for (auto&& writeId : WriteIds) {
diff --git a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h
index ba952c4551..f164c06baf 100644
--- a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h
+++ b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h
@@ -1,14 +1,16 @@
#pragma once
#include "propose_tx.h"
+
#include <ydb/core/tx/columnshard/columnshard_impl.h>
namespace NKikimr::NColumnShard {
- class TLongTxTransactionOperator : public IProposeTxOperator {
+ class TLongTxTransactionOperator: public IProposeTxOperator {
using TBase = IProposeTxOperator;
using TProposeResult = TTxController::TProposeResult;
static inline auto Registrator = TFactory::TRegistrator<TLongTxTransactionOperator>(NKikimrTxColumnShard::TX_KIND_COMMIT);
+
private:
virtual TProposeResult DoStartProposeOnExecute(TColumnShard& owner, NTabletFlatExecutor::TTransactionContext& txc) override;
virtual void DoStartProposeOnComplete(TColumnShard& /*owner*/, const TActorContext& /*ctx*/) override {
@@ -25,11 +27,12 @@ namespace NKikimr::NColumnShard {
virtual bool DoCheckAllowUpdate(const TFullTxInfo& currentTxInfo) const override {
return (currentTxInfo.Source == GetTxInfo().Source && currentTxInfo.Cookie == GetTxInfo().Cookie);
}
+
public:
using TBase::TBase;
void OnTabletInit(TColumnShard& owner) override {
- for (auto&& writeId : WriteIds) {
+ for (auto&& writeId : WriteIds) {
Y_ABORT_UNLESS(owner.LongTxWrites.contains(writeId), "TTxInit at %" PRIu64 " : Commit %" PRIu64 " references local write %" PRIu64 " that doesn't exist",
owner.TabletID(), GetTxId(), (ui64)writeId);
owner.AddLongTxWrite(writeId, GetTxId());
@@ -44,8 +47,7 @@ namespace NKikimr::NColumnShard {
return owner.TablesManager.HasTable(pathId);
};
- auto counters = owner.InsertTable->Commit(dbTable, version.GetPlanStep(), version.GetTxId(), WriteIds,
- pathExists);
+ auto counters = owner.InsertTable->Commit(dbTable, version.GetPlanStep(), version.GetTxId(), WriteIds, pathExists);
owner.IncCounter(COUNTER_BLOBS_COMMITTED, counters.Rows);
owner.IncCounter(COUNTER_BYTES_COMMITTED, counters.Bytes);
@@ -60,9 +62,8 @@ namespace NKikimr::NColumnShard {
}
bool CompleteOnProgress(TColumnShard& owner, const TActorContext& ctx) override {
- auto result = std::make_unique<TEvColumnShard::TEvProposeTransactionResult>(
- owner.TabletID(), TxInfo.TxKind, GetTxId(), NKikimrTxColumnShard::SUCCESS);
- result->Record.SetStep(TxInfo.PlanStep);
+ auto result = std::make_unique<TEvColumnShard::TEvProposeTransactionResult>(owner.TabletID(), TxInfo.TxKind, GetTxId(), NKikimrTxColumnShard::SUCCESS);
+ result->Record.SetStep(TxInfo.PlanStep);
ctx.Send(TxInfo.Source, result.release(), 0, TxInfo.Cookie);
return true;
}
@@ -84,4 +85,5 @@ namespace NKikimr::NColumnShard {
private:
THashSet<TWriteId> WriteIds;
};
-}
+
+} // namespace NKikimr::NColumnShard
diff --git a/ydb/core/tx/columnshard/transactions/operators/schema.h b/ydb/core/tx/columnshard/transactions/operators/schema.h
index 2cc36018ff..84faeab92b 100644
--- a/ydb/core/tx/columnshard/transactions/operators/schema.h
+++ b/ydb/core/tx/columnshard/transactions/operators/schema.h
@@ -1,7 +1,9 @@
#pragma once
#include "propose_tx.h"
+
#include <ydb/core/tx/columnshard/columnshard_impl.h>
+#include <ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.h>
namespace NKikimr::NColumnShard {
@@ -11,6 +13,7 @@ namespace NKikimr::NColumnShard {
using TProposeResult = TTxController::TProposeResult;
static inline auto Registrator = TFactory::TRegistrator<TSchemaTransactionOperator>(NKikimrTxColumnShard::TX_KIND_SCHEMA);
+ std::unique_ptr<NTabletFlatExecutor::ITransaction> TxAddSharding;
virtual TTxController::TProposeResult DoStartProposeOnExecute(TColumnShard& owner, NTabletFlatExecutor::TTransactionContext& txc) override;
virtual void DoStartProposeOnComplete(TColumnShard& /*owner*/, const TActorContext& /*ctx*/) override {
@@ -24,30 +27,53 @@ namespace NKikimr::NColumnShard {
virtual bool DoIsAsync() const override {
return false;
}
- virtual bool DoParse(TColumnShard& /*owner*/, const TString& data) override {
- return SchemaTxBody.ParseFromString(data);
+ virtual bool DoParse(TColumnShard& owner, const TString& data) override {
+ if (!SchemaTxBody.ParseFromString(data)) {
+ return false;
+ }
+ if (SchemaTxBody.HasGranuleShardingInfo()) {
+ NSharding::TGranuleShardingLogicContainer infoContainer;
+ if (!infoContainer.DeserializeFromProto(SchemaTxBody.GetGranuleShardingInfo().GetContainer())) {
+ AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("problem", "cannot parse incoming tx message");
+ return false;
+ }
+ TxAddSharding = owner.TablesManager.CreateAddShardingInfoTx(owner, SchemaTxBody.GetGranuleShardingInfo().GetPathId(),
+ SchemaTxBody.GetGranuleShardingInfo().GetVersionId(), infoContainer);
+ }
+ return true;
}
+
public:
using TBase::TBase;
- bool TxWithDeadline() const override {
- return false;
- }
+ bool TxWithDeadline() const override {
+ return false;
+ }
virtual bool ExecuteOnProgress(TColumnShard& owner, const NOlap::TSnapshot& version, NTabletFlatExecutor::TTransactionContext& txc) override {
- owner.RunSchemaTx(SchemaTxBody, version, txc);
- owner.ProtectSchemaSeqNo(SchemaTxBody.GetSeqNo(), txc);
+ if (!!TxAddSharding) {
+ auto* tx = dynamic_cast<TTxAddShardingInfo*>(TxAddSharding.get());
+ AFL_VERIFY(tx);
+ tx->SetSnapshotVersion(version);
+ TxAddSharding->Execute(txc, NActors::TActivationContext::AsActorContext());
+ }
+ if (SchemaTxBody.TxBody_case() != NKikimrTxColumnShard::TSchemaTxBody::TXBODY_NOT_SET) {
+ owner.RunSchemaTx(SchemaTxBody, version, txc);
+ owner.ProtectSchemaSeqNo(SchemaTxBody.GetSeqNo(), txc);
+ }
return true;
}
virtual bool CompleteOnProgress(TColumnShard& owner, const TActorContext& ctx) override {
+ if (!!TxAddSharding) {
+ TxAddSharding->Complete(ctx);
+ }
for (TActorId subscriber : NotifySubscribers) {
auto event = MakeHolder<TEvColumnShard::TEvNotifyTxCompletionResult>(owner.TabletID(), GetTxId());
ctx.Send(subscriber, event.Release(), 0, 0);
}
- auto result = std::make_unique<TEvColumnShard::TEvProposeTransactionResult>(
- owner.TabletID(), TxInfo.TxKind, TxInfo.TxId, NKikimrTxColumnShard::SUCCESS);
+ auto result = std::make_unique<TEvColumnShard::TEvProposeTransactionResult>(owner.TabletID(), TxInfo.TxKind, TxInfo.TxId, NKikimrTxColumnShard::SUCCESS);
result->Record.SetStep(TxInfo.PlanStep);
ctx.Send(TxInfo.Source, result.release(), 0, TxInfo.Cookie);
return true;
@@ -81,4 +107,4 @@ namespace NKikimr::NColumnShard {
THashSet<TActorId> NotifySubscribers;
};
-}
+} // namespace NKikimr::NColumnShard
diff --git a/ydb/core/tx/columnshard/transactions/operators/ya.make b/ydb/core/tx/columnshard/transactions/operators/ya.make
index b7d63d38b1..579b0d0767 100644
--- a/ydb/core/tx/columnshard/transactions/operators/ya.make
+++ b/ydb/core/tx/columnshard/transactions/operators/ya.make
@@ -10,7 +10,6 @@ SRCS(
)
PEERDIR(
- ydb/core/tx/columnshard/transactions
ydb/core/tx/columnshard/data_sharing/destination/events
ydb/core/tx/columnshard/export/session
)
diff --git a/ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.cpp b/ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.cpp
new file mode 100644
index 0000000000..500f67a2b3
--- /dev/null
+++ b/ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.cpp
@@ -0,0 +1,23 @@
+#include "tx_add_sharding_info.h"
+#include <ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h>
+#include <ydb/core/tx/columnshard/engines/column_engine_logs.h>
+
+namespace NKikimr::NColumnShard {
+
+bool TTxAddShardingInfo::Execute(TTransactionContext& txc, const TActorContext& /*ctx*/) {
+ AFL_VERIFY(!!SnapshotVersion);
+ NIceDb::TNiceDb db(txc.DB);
+ db.Table<Schema::ShardingInfo>()
+ .Key(PathId, ShardingVersion)
+ .Update(NIceDb::TUpdate<Schema::ShardingInfo::Snapshot>(SnapshotVersion->SerializeToString()),
+ NIceDb::TUpdate<Schema::ShardingInfo::Logic>(GranuleShardingLogic.SerializeToString()));
+ return true;
+}
+
+void TTxAddShardingInfo::Complete(const TActorContext& /*ctx*/) {
+ AFL_VERIFY(!!SnapshotVersion);
+ NOlap::TGranuleShardingInfo info(GranuleShardingLogic, *SnapshotVersion, ShardingVersion, PathId);
+ Self->MutableIndexAs<NOlap::TColumnEngineForLogs>().AddShardingInfo(info);
+}
+
+}
diff --git a/ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.h b/ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.h
new file mode 100644
index 0000000000..dcec1eda40
--- /dev/null
+++ b/ydb/core/tx/columnshard/transactions/transactions/tx_add_sharding_info.h
@@ -0,0 +1,32 @@
+#pragma once
+#include <ydb/core/tx/columnshard/columnshard_impl.h>
+
+namespace NKikimr::NColumnShard {
+
+class TTxAddShardingInfo : public NTabletFlatExecutor::TTransactionBase<TColumnShard> {
+private:
+ using TBase = NTabletFlatExecutor::TTransactionBase<TColumnShard>;
+ NSharding::TGranuleShardingLogicContainer GranuleShardingLogic;
+ const ui64 PathId;
+ const ui64 ShardingVersion;
+ std::optional<NOlap::TSnapshot> SnapshotVersion;
+
+public:
+ void SetSnapshotVersion(const NOlap::TSnapshot& ss) {
+ SnapshotVersion = ss;
+ }
+
+ TTxAddShardingInfo(TColumnShard& owner, const NSharding::TGranuleShardingLogicContainer& granuleShardingLogic, const ui64 pathId, const ui64 version)
+ : TBase(&owner)
+ , GranuleShardingLogic(granuleShardingLogic)
+ , PathId(pathId)
+ , ShardingVersion(version)
+ {
+ AFL_VERIFY(!!GranuleShardingLogic);
+ }
+
+ virtual bool Execute(TTransactionContext& txc, const TActorContext& ctx) override;
+ virtual void Complete(const TActorContext& ctx) override;
+};
+
+}
diff --git a/ydb/core/tx/columnshard/transactions/transactions/ya.make b/ydb/core/tx/columnshard/transactions/transactions/ya.make
new file mode 100644
index 0000000000..6ef4144bb7
--- /dev/null
+++ b/ydb/core/tx/columnshard/transactions/transactions/ya.make
@@ -0,0 +1,13 @@
+LIBRARY()
+
+SRCS(
+ tx_add_sharding_info.cpp
+)
+
+PEERDIR(
+ ydb/services/metadata/abstract
+ ydb/core/tx/columnshard/blobs_action/protos
+ ydb/core/tx/columnshard/data_sharing/protos
+)
+
+END()
diff --git a/ydb/core/tx/columnshard/transactions/ya.make b/ydb/core/tx/columnshard/transactions/ya.make
index a6767fb385..8479005c4d 100644
--- a/ydb/core/tx/columnshard/transactions/ya.make
+++ b/ydb/core/tx/columnshard/transactions/ya.make
@@ -9,6 +9,8 @@ PEERDIR(
ydb/core/tablet_flat
ydb/core/tx/data_events
ydb/core/tx/columnshard/data_sharing/destination/events
+ ydb/core/tx/columnshard/transactions/operators
+ ydb/core/tx/columnshard/transactions/transactions
)
YQL_LAST_ABI_VERSION()
diff --git a/ydb/core/tx/data_events/columnshard_splitter.cpp b/ydb/core/tx/data_events/columnshard_splitter.cpp
index 098beaa44a..5b435a0f91 100644
--- a/ydb/core/tx/data_events/columnshard_splitter.cpp
+++ b/ydb/core/tx/data_events/columnshard_splitter.cpp
@@ -47,7 +47,7 @@ NKikimr::NEvWrite::IShardsSplitter::TYdbConclusionStatus TColumnShardShardsSplit
NSchemeShard::TOlapSchema olapSchema;
olapSchema.ParseFromLocalDB(scheme);
- auto shardingConclusion = NSharding::TShardingBase::BuildFromProto(olapSchema, sharding);
+ auto shardingConclusion = NSharding::IShardingBase::BuildFromProto(olapSchema, sharding);
if (shardingConclusion.IsFail()) {
return TYdbConclusionStatus::Fail(Ydb::StatusIds::SCHEME_ERROR, shardingConclusion.GetErrorMessage());
}
@@ -56,7 +56,7 @@ NKikimr::NEvWrite::IShardsSplitter::TYdbConclusionStatus TColumnShardShardsSplit
}
NKikimr::NEvWrite::IShardsSplitter::TYdbConclusionStatus TColumnShardShardsSplitter::SplitImpl(const std::shared_ptr<arrow::RecordBatch>& batch,
- const std::shared_ptr<NSharding::TShardingBase>& sharding)
+ const std::shared_ptr<NSharding::IShardingBase>& sharding)
{
Y_ABORT_UNLESS(batch);
@@ -68,7 +68,7 @@ NKikimr::NEvWrite::IShardsSplitter::TYdbConclusionStatus TColumnShardShardsSplit
TFullSplitData result(sharding->GetShardsCount());
for (auto&& [shardId, chunks] : split.GetResult()) {
for (auto&& c : chunks) {
- result.AddShardInfo(shardId, std::make_shared<TShardInfo>(c.GetSchemaData(), c.GetData(), c.GetRowsCount()));
+ result.AddShardInfo(shardId, std::make_shared<TShardInfo>(c.GetSchemaData(), c.GetData(), c.GetRowsCount(), sharding->GetShardInfoVerified(shardId).GetShardingVersion()));
}
}
diff --git a/ydb/core/tx/data_events/columnshard_splitter.h b/ydb/core/tx/data_events/columnshard_splitter.h
index 2abdd25ed4..5225ac6c88 100644
--- a/ydb/core/tx/data_events/columnshard_splitter.h
+++ b/ydb/core/tx/data_events/columnshard_splitter.h
@@ -17,11 +17,13 @@ class TColumnShardShardsSplitter : public IShardsSplitter {
const TString SchemaData;
const TString Data;
const ui32 RowsCount;
+ const ui32 GranuleShardingVersion;
public:
- TShardInfo(const TString& schemaData, const TString& data, const ui32 rowsCount)
+ TShardInfo(const TString& schemaData, const TString& data, const ui32 rowsCount, const ui32 granuleShardingVersion)
: SchemaData(schemaData)
, Data(data)
, RowsCount(rowsCount)
+ , GranuleShardingVersion(granuleShardingVersion)
{}
ui64 GetBytes() const override {
@@ -38,6 +40,7 @@ class TColumnShardShardsSplitter : public IShardsSplitter {
void Serialize(TEvWrite& evWrite) const override {
evWrite.SetArrowData(SchemaData, Data);
+ evWrite.Record.SetGranuleShardingVersion(GranuleShardingVersion);
}
};
@@ -45,7 +48,7 @@ private:
TYdbConclusionStatus DoSplitData(const NSchemeCache::TSchemeCacheNavigate::TEntry& schemeEntry, const IEvWriteDataAccessor& data) override;
private:
- TYdbConclusionStatus SplitImpl(const std::shared_ptr<arrow::RecordBatch>& batch, const std::shared_ptr<NSharding::TShardingBase>& sharding);
+ TYdbConclusionStatus SplitImpl(const std::shared_ptr<arrow::RecordBatch>& batch, const std::shared_ptr<NSharding::IShardingBase>& sharding);
std::shared_ptr<arrow::Schema> ExtractArrowSchema(const NKikimrSchemeOp::TColumnTableSchema& schema);
};
diff --git a/ydb/core/tx/data_events/write_data.h b/ydb/core/tx/data_events/write_data.h
index d25c65e65c..0ad33cf9bf 100644
--- a/ydb/core/tx/data_events/write_data.h
+++ b/ydb/core/tx/data_events/write_data.h
@@ -26,6 +26,7 @@ class TWriteMeta {
YDB_ACCESSOR(ui64, WriteId, 0);
YDB_READONLY(ui64, TableId, 0);
YDB_ACCESSOR_DEF(NActors::TActorId, Source);
+ YDB_ACCESSOR_DEF(std::optional<ui32>, GranuleShardingVersion);
// Long Tx logic
YDB_OPT(NLongTxService::TLongTxId, LongTxId);
@@ -41,10 +42,11 @@ class TWriteMeta {
YDB_ACCESSOR(TMonotonic, WriteMiddle5StartInstant, TMonotonic::Now());
YDB_ACCESSOR(TMonotonic, WriteMiddle6StartInstant, TMonotonic::Now());
public:
- TWriteMeta(const ui64 writeId, const ui64 tableId, const NActors::TActorId& source)
+ TWriteMeta(const ui64 writeId, const ui64 tableId, const NActors::TActorId& source, const std::optional<ui32> granuleShardingVersion)
: WriteId(writeId)
, TableId(tableId)
, Source(source)
+ , GranuleShardingVersion(granuleShardingVersion)
{}
};
diff --git a/ydb/core/tx/schemeshard/olap/bg_tasks/transactions/tasks_list.cpp b/ydb/core/tx/schemeshard/olap/bg_tasks/transactions/tasks_list.cpp
index 27f45deb1d..44c555451a 100644
--- a/ydb/core/tx/schemeshard/olap/bg_tasks/transactions/tasks_list.cpp
+++ b/ydb/core/tx/schemeshard/olap/bg_tasks/transactions/tasks_list.cpp
@@ -54,7 +54,7 @@ TTxTasksList::TTxTasksList(TSelf* self, TEvListRequest::TPtr& ev)
, RequestCookie(ev->Cookie)
, DatabaseName(ev->Get()->Record.GetDatabaseName())
{
- if (ev->Get()->Record.HasPageSize()) {
+ if (ev->Get()->Record.HasPageSize() && ev->Get()->Record.GetPageSize()) {
PageSize = ev->Get()->Record.GetPageSize();
}
if (ev->Get()->Record.HasPageToken()) {
diff --git a/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.cpp b/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.cpp
index 44e1a9f804..6cdcc8749b 100644
--- a/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.cpp
+++ b/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.cpp
@@ -40,10 +40,10 @@ void TTxChainActor::Handle(TEvSchemeShard::TEvModifySchemeTransactionResult::TPt
}
void TTxChainActor::OnSessionProgressSaved() {
- if (SessionLogic->GetStepForExecute() < SessionLogic->GetTxData().GetTransactions().size()) {
- NActors::TActivationContext::AsActorContext().Send(TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(1));
+ if (SessionLogic->IsFinished()) {
+ SaveSessionState();
} else {
- Session->FinishActor();
+ NActors::TActivationContext::AsActorContext().Send(TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(1));
}
}
diff --git a/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.h b/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.h
index 89b04aa673..26ae57aa5e 100644
--- a/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.h
+++ b/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/actor.h
@@ -31,7 +31,11 @@ protected:
}
virtual void OnSessionProgressSaved() override;
virtual void OnSessionStateSaved() override {
- SendCurrentTxToSS();
+ if (SessionLogic->IsFinished()) {
+ Session->FinishActor();
+ } else {
+ SendCurrentTxToSS();
+ }
}
virtual void OnBootstrap(const TActorContext& /*ctx*/) override;
diff --git a/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/session.h b/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/session.h
index f93b1d4267..1a56abe419 100644
--- a/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/session.h
+++ b/ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/session.h
@@ -87,7 +87,7 @@ public:
AFL_VERIFY(StepForExecute <= GetTxData().GetTransactions().size());
return StepForExecute == GetTxData().GetTransactions().size();
}
- virtual bool IsReadyForRemove() const override {
+ virtual bool IsReadyForRemoveOnFinished() const override {
return true;
}
};
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/context.h b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/context.h
index 65623ae802..f5423ce938 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/context.h
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/context.h
@@ -3,9 +3,12 @@
#include <ydb/core/scheme/scheme_pathid.h>
#include <ydb/core/tx/schemeshard/schemeshard__operation_part.h>
#include <ydb/core/tx/schemeshard/schemeshard_path.h>
+#include <ydb/core/tx/columnshard/common/snapshot.h>
namespace NKikimr::NSchemeShard::NOlap::NAlter {
+class ISSEntity;
+
class TEntityInitializationContext {
private:
const TOperationContext* SSOperationContext = nullptr;
@@ -19,21 +22,51 @@ public:
}
};
-class TUpdateInitializationContext {
+class TUpdateRestoreContext {
private:
+ const ISSEntity* OriginalEntity;
const TOperationContext* SSOperationContext = nullptr;
- const NKikimrSchemeOp::TModifyScheme* Modification = nullptr;
+ const ui64 TxId;
public:
+ ui64 GetTxId() const {
+ return TxId;
+ }
+
+ const ISSEntity& GetOriginalEntity() const {
+ return *OriginalEntity;
+ }
+
+ template <class T>
+ const T& GetOriginalEntityAsVerified() const {
+ auto* result = dynamic_cast<const T*>(OriginalEntity);
+ AFL_VERIFY(!!result);
+ return *result;
+ }
+
const TOperationContext* GetSSOperationContext() const {
return SSOperationContext;
}
+ TUpdateRestoreContext(const ISSEntity* originalEntity, const TOperationContext* ssOperationContext, const ui64 txId)
+ : OriginalEntity(originalEntity)
+ , SSOperationContext(ssOperationContext)
+ , TxId(txId) {
+ AFL_VERIFY(OriginalEntity);
+ AFL_VERIFY(SSOperationContext);
+ AFL_VERIFY(TxId);
+ }
+};
+
+class TUpdateInitializationContext: public TUpdateRestoreContext {
+private:
+ using TBase = TUpdateRestoreContext;
+ const NKikimrSchemeOp::TModifyScheme* Modification = nullptr;
+public:
const NKikimrSchemeOp::TModifyScheme* GetModification() const {
return Modification;
}
- TUpdateInitializationContext(const TOperationContext* ssOperationContext, const NKikimrSchemeOp::TModifyScheme* modification)
- : SSOperationContext(ssOperationContext)
+ TUpdateInitializationContext(const ISSEntity* originalEntity, const TOperationContext* ssOperationContext, const NKikimrSchemeOp::TModifyScheme* modification, const ui64 txId)
+ : TBase(originalEntity, ssOperationContext, txId)
, Modification(modification) {
- AFL_VERIFY(SSOperationContext);
AFL_VERIFY(Modification);
}
};
@@ -51,7 +84,7 @@ public:
}
};
-class TEvolutionStartContext {
+class TUpdateStartContext {
private:
const TPath* ObjectPath = nullptr;
TOperationContext* SSOperationContext = nullptr;
@@ -67,14 +100,33 @@ public:
return SSOperationContext;
}
- TEvolutionStartContext(const TPath* objectPath, TOperationContext* ssOperationContext, NIceDb::TNiceDb* db)
+ TUpdateStartContext(const TPath* objectPath, TOperationContext* ssOperationContext, NIceDb::TNiceDb* db)
: ObjectPath(objectPath)
, SSOperationContext(ssOperationContext)
- , DB(db) {
+ , DB(db)
+ {
AFL_VERIFY(DB);
AFL_VERIFY(ObjectPath);
AFL_VERIFY(SSOperationContext);
}
};
+class TUpdateFinishContext: public TUpdateStartContext {
+private:
+ using TBase = TUpdateStartContext;
+ YDB_READONLY_DEF(std::optional<NKikimr::NOlap::TSnapshot>, Snapshot);
+public:
+
+ const NKikimr::NOlap::TSnapshot& GetSnapshotVerified() const {
+ AFL_VERIFY(Snapshot);
+ return *Snapshot;
+ }
+
+ TUpdateFinishContext(const TPath* objectPath, TOperationContext* ssOperationContext, NIceDb::TNiceDb* db, const std::optional<NKikimr::NOlap::TSnapshot>& ss)
+ : TBase(objectPath, ssOperationContext, db)
+ , Snapshot(ss)
+ {
+ }
+};
+
} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h
index 4163f82fb7..7a45468961 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h
@@ -103,10 +103,6 @@ public:
return TConclusionStatus::Fail("Changing table schema is not supported");
}
- if (result.HasRESERVED_AlterTtlSettingsPresetName()) {
- return TConclusionStatus::Fail("TTL presets are not supported");
- }
-
return result;
}
};
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.cpp
deleted file mode 100644
index 4ec6d89d9b..0000000000
--- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.cpp
+++ /dev/null
@@ -1,5 +0,0 @@
-#include "evolution.h"
-
-namespace NKikimr::NSchemeShard::NOlap::NAlter {
-
-} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.h b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.h
deleted file mode 100644
index 7884fbd4fd..0000000000
--- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.h
+++ /dev/null
@@ -1,42 +0,0 @@
-#pragma once
-#include "context.h"
-#include <ydb/library/accessor/accessor.h>
-#include <ydb/core/scheme/scheme_pathid.h>
-#include <ydb/core/tablet_flat/flat_cxx_database.h>
-#include <ydb/core/tx/schemeshard/schemeshard__operation_part.h>
-#include <ydb/core/tx/schemeshard/schemeshard_path.h>
-
-namespace NKikimr::NSchemeShard::NOlap::NAlter {
-
-class ISSEntityEvolution {
-private:
- YDB_READONLY_DEF(TPathId, PathId);
- bool Initialized = false;
-protected:
- virtual TConclusionStatus DoInitialize(const TEvolutionInitializationContext& context) = 0;
- virtual TConclusionStatus DoStartEvolution(const TEvolutionStartContext& context) = 0;
- virtual NKikimrTxColumnShard::TSchemaTxBody DoGetShardTxBody(const NKikimrTxColumnShard::TSchemaTxBody& original, const ui64 tabletId) const = 0;
-public:
- virtual ~ISSEntityEvolution() = default;
- TConclusionStatus Initialize(const TEvolutionInitializationContext& context) {
- AFL_VERIFY(!Initialized);
- Initialized = true;
- return DoInitialize(context);
- }
-
- TConclusionStatus StartEvolution(const TEvolutionStartContext& context) {
- return DoStartEvolution(context);
- }
-
- virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const = 0;
- NKikimrTxColumnShard::TSchemaTxBody GetShardTxBody(const NKikimrTxColumnShard::TSchemaTxBody& original, const ui64 tabletId) const {
- return DoGetShardTxBody(original, tabletId);
- }
-
- ISSEntityEvolution(const TPathId& pathId)
- : PathId(pathId) {
-
- }
-};
-
-} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.cpp
index 0a1c3272f3..372991cdc3 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.cpp
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.cpp
@@ -1,5 +1,16 @@
#include "object.h"
+#include <ydb/core/tx/schemeshard/schemeshard_impl.h>
namespace NKikimr::NSchemeShard::NOlap::NAlter {
+std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEntity> ISSEntity::GetEntityVerified(TOperationContext& context, const TPath& path) {
+ if (path->IsColumnTable()) {
+ auto readGuard = context.SS->ColumnTables.GetVerified(path.Base()->PathId);
+ TEntityInitializationContext iContext(&context);
+ return readGuard->BuildEntity(path.Base()->PathId, iContext).DetachResult();
+ }
+ AFL_VERIFY(false);
+ return nullptr;
+}
+
} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h
index 4b24ef4a76..fe6d5db9f6 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h
@@ -14,13 +14,23 @@ private:
bool Initialized = false;
protected:
[[nodiscard]] virtual TConclusionStatus DoInitialize(const TEntityInitializationContext& context) = 0;
- virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdate(const TUpdateInitializationContext& context, const std::shared_ptr<ISSEntity>& selfPtr) const = 0;
+ virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdate(const TUpdateInitializationContext& context) const = 0;
+ virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoRestoreUpdate(const TUpdateRestoreContext& context) const = 0;
+
public:
virtual ~ISSEntity() = default;
virtual TString GetClassName() const = 0;
- virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> CreateUpdate(const TUpdateInitializationContext& context, const std::shared_ptr<ISSEntity>& selfPtr) const {
- return DoCreateUpdate(context, selfPtr);
+ TConclusion<std::shared_ptr<ISSEntityUpdate>> CreateUpdate(const TUpdateInitializationContext& context) const {
+ return DoCreateUpdate(context);
+ }
+
+ TConclusion<std::shared_ptr<ISSEntityUpdate>> RestoreUpdate(const TUpdateRestoreContext& context) const {
+ return DoRestoreUpdate(context);
+ }
+
+ std::shared_ptr<ISSEntityUpdate> RestoreUpdateVerified(const TUpdateRestoreContext& context) const {
+ return RestoreUpdate(context).DetachResult();
}
[[nodiscard]] TConclusionStatus Initialize(const TEntityInitializationContext& context) {
@@ -34,6 +44,8 @@ public:
{
}
+
+ static std::shared_ptr<ISSEntity> GetEntityVerified(TOperationContext& context, const TPath& path);
};
} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h
index 521432745f..61953ec45a 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h
@@ -8,31 +8,47 @@
namespace NKikimr::NSchemeShard::NOlap::NAlter {
-class ISSEntityEvolution;
-
class ISSEntityUpdate {
private:
- YDB_READONLY_DEF(std::shared_ptr<ISSEntity>, OriginalEntity);
bool Initialized = false;
protected:
virtual TConclusionStatus DoInitialize(const TUpdateInitializationContext& context) = 0;
- virtual TConclusion<std::shared_ptr<ISSEntityEvolution>> DoBuildEvolution(const std::shared_ptr<ISSEntityUpdate>& selfPtr) const = 0;
+ virtual TConclusionStatus DoStart(const TUpdateStartContext& context) = 0;
+ virtual TConclusionStatus DoFinish(const TUpdateFinishContext& context) = 0;
+ virtual TString DoGetShardTxBodyString(const ui64 tabletId, const TMessageSeqNo& seqNo) const = 0;
+ virtual std::set<ui64> DoGetShardIds() const = 0;
public:
+ ISSEntityUpdate() = default;
virtual ~ISSEntityUpdate() = default;
- ISSEntityUpdate(const std::shared_ptr<ISSEntity>& originalEntity)
- : OriginalEntity(originalEntity) {
- AFL_VERIFY(OriginalEntity);
+ virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const = 0;
+
+ std::set<ui64> GetShardIds() const {
+ AFL_VERIFY(Initialized);
+ return DoGetShardIds();
}
- TConclusionStatus Initialize(const TUpdateInitializationContext& context) {
- AFL_VERIFY(!Initialized);
- Initialized = true;
- return DoInitialize(context);
+ TString GetShardTxBodyString(const ui64 tabletId, const TMessageSeqNo& seqNo) const {
+ AFL_VERIFY(Initialized);
+ return DoGetShardTxBodyString(tabletId, seqNo);
+ }
+
+ TConclusionStatus Start(const TUpdateStartContext& context) {
+ AFL_VERIFY(Initialized);
+ return DoStart(context);
+ }
+
+ TConclusionStatus Finish(const TUpdateFinishContext& context) {
+ AFL_VERIFY(Initialized);
+ return DoFinish(context);
}
- TConclusion<std::shared_ptr<ISSEntityEvolution>> BuildEvolution(const std::shared_ptr<ISSEntityUpdate>& selfPtr) const {
- AFL_VERIFY(!!selfPtr);
- return DoBuildEvolution(selfPtr);
+ TConclusionStatus Initialize(const TUpdateInitializationContext& context) {
+ AFL_VERIFY(!Initialized);
+ auto result = DoInitialize(context);
+ if (result.IsSuccess()) {
+ Initialized = true;
+ }
+ return result;
}
};
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/ya.make
index 63b5dc1a63..66a40695a6 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/ya.make
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/ya.make
@@ -3,7 +3,6 @@ LIBRARY()
SRCS(
object.cpp
update.cpp
- evolution.cpp
converter.cpp
context.cpp
)
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/common/object.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/common/object.cpp
new file mode 100644
index 0000000000..f8987b734d
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/common/object.cpp
@@ -0,0 +1,40 @@
+#include "object.h"
+#include <ydb/core/tx/schemeshard/schemeshard_impl.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+TConclusion<std::shared_ptr<ISSEntityUpdate>> TColumnTableEntity::DoRestoreUpdate(const TUpdateRestoreContext& context) const {
+ auto& ssContext = *context.GetSSOperationContext();
+ auto tableInfo = ssContext.SS->ColumnTables.GetVerifiedPtr(GetPathId());
+ if (!tableInfo) {
+ return TConclusionStatus::Fail("object not exists");
+ }
+ if (!tableInfo->AlterData) {
+ return TConclusionStatus::Fail("object not in update");
+ }
+ if (!tableInfo->AlterData->AlterBody) {
+ return TConclusionStatus::Fail("object is incorrect in update");
+ }
+ NKikimrSchemeOp::TModifyScheme mScheme;
+ *mScheme.MutableAlterColumnTable() = *tableInfo->AlterData->AlterBody;
+ mScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterColumnTable);
+ TUpdateInitializationContext uContext(&context.GetOriginalEntity(), context.GetSSOperationContext(), &mScheme, context.GetTxId());
+ return DoCreateUpdateImpl(uContext);
+}
+
+TConclusionStatus TColumnTableEntity::DoInitialize(const TEntityInitializationContext& context) {
+ TableInfo = context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(GetPathId());
+ return DoInitializeImpl(context);
+}
+
+TConclusion<std::shared_ptr<ISSEntityUpdate>> TColumnTableEntity::DoCreateUpdate(const TUpdateInitializationContext& context) const {
+ if (GetTableInfo()->AlterVersion == 0) {
+ return NKikimr::TConclusionStatus::Fail("Table is not created yet");
+ }
+ if (GetTableInfo()->AlterData) {
+ return NKikimr::TConclusionStatus::Fail("There's another Alter in flight");
+ }
+ return DoCreateUpdateImpl(context);
+}
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/common/object.h b/ydb/core/tx/schemeshard/olap/operations/alter/common/object.h
new file mode 100644
index 0000000000..215cf77b95
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/common/object.h
@@ -0,0 +1,45 @@
+#pragma once
+#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h>
+#include <ydb/core/tx/schemeshard/olap/table/table.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+class TColumnTableEntity: public ISSEntity {
+private:
+ using TBase = ISSEntity;
+ YDB_READONLY_DEF(TColumnTableInfo::TPtr, TableInfo);
+ virtual TConclusionStatus DoInitializeImpl(const TEntityInitializationContext& context) = 0;
+ virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdateImpl(const TUpdateInitializationContext& context) const = 0;
+protected:
+ TConclusion<std::shared_ptr<ISSEntityUpdate>> DoRestoreUpdate(const TUpdateRestoreContext& ssContext) const override;
+ virtual TConclusionStatus DoInitialize(const TEntityInitializationContext& context) override final;
+ virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdate(const TUpdateInitializationContext& context) const override final;;
+
+ TColumnTableEntity(const TPathId& pathId, const TColumnTableInfo::TPtr& tabletInfo)
+ : TBase(pathId)
+ , TableInfo(tabletInfo) {
+ }
+
+public:
+
+ TColumnTableEntity(const TPathId& pathId)
+ : TBase(pathId) {
+ }
+
+ const TColumnTableInfo& GetTableInfoVerified() const {
+ AFL_VERIFY(!!TableInfo);
+ return *TableInfo;
+ }
+
+ TColumnTableInfo::TPtr GetTableInfoPtrVerified() const {
+ AFL_VERIFY(!!TableInfo);
+ return TableInfo;
+ }
+
+ const NKikimrSchemeOp::TColumnDataLifeCycle& GetTableTTLProto() {
+ AFL_VERIFY(TableInfo);
+ return TableInfo->Description.GetTtlSettings();
+ }
+};
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/common/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/common/update.cpp
new file mode 100644
index 0000000000..442c678335
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/common/update.cpp
@@ -0,0 +1,31 @@
+#include "update.h"
+#include <ydb/core/tx/schemeshard/schemeshard_impl.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+TConclusionStatus TColumnTableUpdate::DoStart(const TUpdateStartContext& context) {
+ auto conclusion = DoStartImpl(context);
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ const auto pathId = context.GetObjectPath()->Base()->PathId;
+ auto tableInfo = context.GetSSOperationContext()->SS->ColumnTables.TakeVerified(pathId);
+ context.GetSSOperationContext()->SS->PersistColumnTableAlter(*context.GetDB(), pathId, *GetTargetTableInfoVerified());
+ tableInfo->AlterData = GetTargetTableInfoVerified();
+ return TConclusionStatus::Success();
+}
+
+TConclusionStatus TColumnTableUpdate::DoFinish(const TUpdateFinishContext& context) {
+ auto conclusion = DoFinishImpl(context);
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+
+ const auto pathId = context.GetObjectPath()->Base()->PathId;
+ auto tableInfo = context.GetSSOperationContext()->SS->ColumnTables.TakeAlterVerified(pathId);
+ context.GetSSOperationContext()->SS->PersistColumnTableAlterRemove(*context.GetDB(), pathId);
+ context.GetSSOperationContext()->SS->PersistColumnTable(*context.GetDB(), pathId, *tableInfo);
+ return TConclusionStatus::Success();
+}
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/common/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/common/update.h
new file mode 100644
index 0000000000..c33f18eb03
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/common/update.h
@@ -0,0 +1,58 @@
+#pragma once
+#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/context.h>
+#include <ydb/core/tx/schemeshard/olap/table/table.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+class TColumnTableUpdate: public ISSEntityUpdate {
+private:
+ using TBase = ISSEntityUpdate;
+
+ virtual std::shared_ptr<TColumnTableInfo> GetTargetTableInfo() const = 0;
+ virtual std::shared_ptr<ISSEntity> GetTargetSSEntity() const = 0;
+
+ virtual TConclusionStatus DoStart(const TUpdateStartContext& context) override final;
+ virtual TConclusionStatus DoFinish(const TUpdateFinishContext& context) override final;
+
+ virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override {
+ return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SCHEMA;
+ }
+ virtual TConclusionStatus DoInitializeImpl(const TUpdateInitializationContext& context) = 0;
+protected:
+ virtual TConclusionStatus DoStartImpl(const TUpdateStartContext& /*context*/) {
+ return TConclusionStatus::Success();
+ }
+ virtual TConclusionStatus DoFinishImpl(const TUpdateFinishContext& /*context*/) {
+ return TConclusionStatus::Success();
+ }
+ virtual TConclusionStatus DoInitialize(const TUpdateInitializationContext& context) override final {
+ if (!context.GetModification()->HasAlterColumnTable() && !context.GetModification()->HasAlterTable()) {
+ return TConclusionStatus::Fail("no update data");
+ }
+ return DoInitializeImpl(context);
+ }
+
+ std::shared_ptr<TColumnTableInfo> GetTargetTableInfoVerified() const {
+ auto result = GetTargetTableInfo();
+ AFL_VERIFY(!!result);
+ return result;
+ }
+
+ template <class T>
+ const T& GetTargetEntityAsVerified() const {
+ auto resultPtr = dynamic_pointer_cast<T>(GetTargetEntityVerified());
+ AFL_VERIFY(!!resultPtr);
+ return *resultPtr;
+ }
+
+ std::shared_ptr<ISSEntity> GetTargetEntityVerified() const {
+ auto result = GetTargetSSEntity();
+ AFL_VERIFY(!!result);
+ return result;
+ }
+
+public:
+};
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/common/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/common/ya.make
new file mode 100644
index 0000000000..24a9ac14d1
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/common/ya.make
@@ -0,0 +1,14 @@
+LIBRARY()
+
+SRCS(
+ update.cpp
+ object.cpp
+)
+
+PEERDIR(
+ ydb/core/tx/schemeshard/olap/operations/alter/abstract
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.cpp
new file mode 100644
index 0000000000..62c52b4b56
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.cpp
@@ -0,0 +1,26 @@
+#include "update.h"
+#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h>
+#include <ydb/core/tx/schemeshard/schemeshard_impl.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+NKikimr::TConclusionStatus TInStoreTableUpdate::DoStartImpl(const TUpdateStartContext& context) {
+ const auto& inStoreTable = GetTargetEntityAsVerified<TInStoreTable>();
+
+ auto tableInfo = GetTargetTableInfoVerified();
+ const auto storePathId = tableInfo->GetOlapStorePathIdVerified();
+ TPath storePath = TPath::Init(storePathId, context.GetSSOperationContext()->SS);
+
+ Y_ABORT_UNLESS(inStoreTable.GetStoreInfo()->ColumnTables.contains((*context.GetObjectPath())->PathId));
+ inStoreTable.GetStoreInfo()->ColumnTablesUnderOperation.insert((*context.GetObjectPath())->PathId);
+
+ // Sequentially chain operations in the same olap store
+ if (context.GetSSOperationContext()->SS->Operations.contains(storePath.Base()->LastTxId)) {
+ context.GetSSOperationContext()->OnComplete.Dependence(storePath.Base()->LastTxId, (*context.GetObjectPath())->LastTxId);
+ }
+ storePath.Base()->LastTxId = (*context.GetObjectPath())->LastTxId;
+ context.GetSSOperationContext()->SS->PersistLastTxId(*context.GetDB(), storePath.Base());
+ return DoStartInStoreImpl(context);
+}
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.h
new file mode 100644
index 0000000000..415945c182
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.h
@@ -0,0 +1,19 @@
+#pragma once
+#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/common/update.h>
+#include <ydb/core/tx/schemeshard/olap/ttl/update.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+class TInStoreTableUpdate: public TColumnTableUpdate {
+private:
+ using TBase = TColumnTableUpdate;
+
+ virtual TConclusionStatus DoStartImpl(const TUpdateStartContext& context) override final;
+ virtual TConclusionStatus DoStartInStoreImpl(const TUpdateStartContext& /*context*/) {
+ return TConclusionStatus::Success();
+ }
+public:
+};
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/ya.make
new file mode 100644
index 0000000000..13fb75778f
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/ya.make
@@ -0,0 +1,13 @@
+LIBRARY()
+
+SRCS(
+ update.cpp
+)
+
+PEERDIR(
+ ydb/core/tx/schemeshard/olap/operations/alter/abstract
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.cpp
new file mode 100644
index 0000000000..6dbcbfb1d7
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.cpp
@@ -0,0 +1,82 @@
+#include "update.h"
+#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h>
+#include <ydb/core/tx/schemeshard/schemeshard_impl.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+TConclusionStatus TInStoreShardsUpdate::DoInitializeImpl(const TUpdateInitializationContext& context) {
+ const auto& original = context.GetOriginalEntityAsVerified<TInStoreTable>();
+ if (!context.GetModification()->GetAlterColumnTable().HasAlterShards()) {
+ return TConclusionStatus::Fail("no data about shards altering");
+ }
+ auto& alterShards = context.GetModification()->GetAlterColumnTable().GetAlterShards();
+ Alter = alterShards;
+ AFL_VERIFY(Alter.HasModification());
+
+ for (auto&& i : Alter.GetModification().GetNewShardIds()) {
+ AFL_VERIFY(ShardIds.emplace(i).second);
+ AFL_VERIFY(NewShardIds.emplace(i).second);
+ }
+
+ for (auto&& i : Alter.GetModification().GetDeleteShardIds()) {
+ AFL_VERIFY(ShardIds.emplace(i).second);
+ AFL_VERIFY(DeleteShardIds.emplace(i).second);
+ }
+
+ auto tableInfo = original.GetTableInfoPtrVerified();
+
+ auto description = tableInfo->Description;
+
+ Sharding = tableInfo->GetShardingVerified(original.GetTableSchemaVerified());
+ Sharding->ApplyModification(Alter.GetModification()).Validate();
+ *description.MutableSharding() = Sharding->SerializeToProto();
+ for (auto&& i : Sharding->GetModifiedShardIds(Alter.GetModification())) {
+ ShardIds.emplace(i);
+ AFL_VERIFY(ModifiedShardIds.emplace(i).second);
+ }
+
+ auto targetInfo = std::make_shared<TColumnTableInfo>(tableInfo->AlterVersion + 1, std::move(description),
+ TMaybe<NKikimrSchemeOp::TColumnStoreSharding>(), context.GetModification()->GetAlterColumnTable());
+
+ TEntityInitializationContext eContext(context.GetSSOperationContext());
+ TargetInStoreTable = std::make_shared<TInStoreTable>(original.GetPathId(), targetInfo, eContext);
+
+ return TConclusionStatus::Success();
+}
+
+void TInStoreShardsUpdate::FillToShardTx(NKikimrTxColumnShard::TCreateTable& info) const {
+ info.SetPathId(TargetInStoreTable->GetPathId().LocalPathId);
+ auto& alterBody = TargetInStoreTable->GetTableInfoVerified();
+
+ AFL_VERIFY(alterBody.Description.HasSchemaPresetId());
+ const ui32 presetId = alterBody.Description.GetSchemaPresetId();
+ Y_ABORT_UNLESS(!!TargetInStoreTable->GetStoreInfo(), "Unexpected schema preset without olap store");
+ Y_ABORT_UNLESS(TargetInStoreTable->GetStoreInfo()->SchemaPresets.contains(presetId), "Failed to find schema preset %" PRIu32 " in an olap store", presetId);
+ auto& preset = TargetInStoreTable->GetStoreInfo()->SchemaPresets.at(presetId);
+ size_t presetIndex = preset.GetProtoIndex();
+ *info.MutableSchemaPreset() = TargetInStoreTable->GetStoreInfo()->GetDescription().GetSchemaPresets(presetIndex);
+
+ if (alterBody.Description.HasSchemaPresetVersionAdj()) {
+ info.SetSchemaPresetVersionAdj(alterBody.Description.GetSchemaPresetVersionAdj());
+ }
+ if (alterBody.Description.HasTtlSettings()) {
+ *info.MutableTtlSettings() = alterBody.Description.GetTtlSettings();
+ }
+}
+
+NKikimr::TConclusionStatus TInStoreShardsUpdate::DoFinishImpl(const TUpdateFinishContext& context) {
+ auto conclusion = TBase::DoFinishImpl(context);
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ auto alter = context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(TargetInStoreTable->GetPathId())->AlterData;
+ for (auto&& i : Alter.GetModification().GetOpenWriteIds()) {
+ AFL_VERIFY(!!alter);
+ Sharding->SetShardingOpenSnapshotVerified(i, context.GetSnapshotVerified());
+ }
+ AFL_VERIFY(!!alter->AlterBody);
+ *alter->Description.MutableSharding() = Sharding->SerializeToProto();
+ return TConclusionStatus::Success();
+}
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.h
new file mode 100644
index 0000000000..87d1bc7f18
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/update.h
@@ -0,0 +1,63 @@
+#pragma once
+#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.h>
+#include <ydb/core/tx/schemeshard/olap/ttl/update.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+class TInStoreShardsUpdate: public TInStoreTableUpdate {
+private:
+ using TBase = TInStoreTableUpdate;
+ NKikimrSchemeOp::TAlterShards Alter;
+ std::shared_ptr<TInStoreTable> TargetInStoreTable;
+ std::set<ui64> ShardIds;
+ std::set<ui64> NewShardIds;
+ std::set<ui64> DeleteShardIds;
+ std::set<ui64> ModifiedShardIds;
+ std::shared_ptr<NSharding::IShardingBase> Sharding;
+ virtual TConclusionStatus DoInitializeImpl(const TUpdateInitializationContext& context) override;
+ void FillToShardTx(NKikimrTxColumnShard::TCreateTable& shardAlter) const;
+
+ virtual TConclusionStatus DoFinishImpl(const TUpdateFinishContext& context) override;
+ virtual std::shared_ptr<TColumnTableInfo> GetTargetTableInfo() const override {
+ return TargetInStoreTable->GetTableInfoPtrVerified();
+ }
+ virtual std::shared_ptr<ISSEntity> GetTargetSSEntity() const override {
+ return TargetInStoreTable;
+ }
+
+ virtual TString DoGetShardTxBodyString(const ui64 tabletId, const TMessageSeqNo& seqNo) const override {
+ NKikimrTxColumnShard::TSchemaTxBody result;
+ result.MutableSeqNo()->SetGeneration(seqNo.Generation);
+ result.MutableSeqNo()->SetRound(seqNo.Round);
+ AFL_VERIFY(NewShardIds.contains(tabletId) || ModifiedShardIds.contains(tabletId) || DeleteShardIds.contains(tabletId));
+ if (NewShardIds.contains(tabletId)) {
+ auto& alter = *result.MutableEnsureTables();
+ auto& create = *alter.AddTables();
+ FillToShardTx(create);
+ create.SetPathId(TargetInStoreTable->GetPathId().LocalPathId);
+ }
+ if (DeleteShardIds.contains(tabletId)) {
+ result.MutableDropTable()->SetPathId(TargetInStoreTable->GetPathId().LocalPathId);
+ }
+ auto container = Sharding->GetTabletShardingInfoOptional(tabletId);
+ if (!!container) {
+ auto& shardingInfo = *result.MutableGranuleShardingInfo();
+ shardingInfo.SetPathId(TargetInStoreTable->GetPathId().LocalPathId);
+ shardingInfo.SetVersionId(Sharding->GetShardInfoVerified(tabletId).GetShardingVersion());
+ *shardingInfo.MutableContainer() = container.SerializeToProto();
+ AFL_VERIFY(ModifiedShardIds.contains(tabletId));
+ }
+ return result.SerializeAsString();
+ }
+
+ virtual std::set<ui64> DoGetShardIds() const override {
+ return ShardIds;
+ }
+
+public:
+
+};
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/ya.make
new file mode 100644
index 0000000000..13fb75778f
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards/ya.make
@@ -0,0 +1,13 @@
+LIBRARY()
+
+SRCS(
+ update.cpp
+)
+
+PEERDIR(
+ ydb/core/tx/schemeshard/olap/operations/alter/abstract
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.cpp
deleted file mode 100644
index 7fbdcebcc5..0000000000
--- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.cpp
+++ /dev/null
@@ -1,35 +0,0 @@
-#include "evolution.h"
-#include <ydb/core/tx/schemeshard/schemeshard_impl.h>
-
-namespace NKikimr::NSchemeShard::NOlap::NAlter {
-
-NKikimr::TConclusionStatus TInStoreSchemaEvolution::DoInitialize(const TEvolutionInitializationContext& context) {
- TColumnTableInfo::TPtr tableInfo = context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(GetPathId());
- AFL_VERIFY(!tableInfo->IsStandalone());
- TEntityInitializationContext eContext(context.GetSSOperationContext());
- Original = std::make_shared<TInStoreTable>(GetPathId(), tableInfo, eContext);
- Target = std::make_shared<TInStoreTable>(GetPathId(), tableInfo->AlterData, eContext);
- Update = std::make_shared<TInStoreSchemaUpdate>(Original, Target);
- return TConclusionStatus::Success();
-}
-
-NKikimr::TConclusionStatus TInStoreSchemaEvolution::DoStartEvolution(const TEvolutionStartContext& context) {
- const auto storePathId = Original->GetTableInfo()->GetOlapStorePathIdVerified();
- TPath storePath = TPath::Init(storePathId, context.GetSSOperationContext()->SS);
-
- Y_ABORT_UNLESS(Original->GetStoreInfo()->ColumnTables.contains((*context.GetObjectPath())->PathId));
- Original->GetStoreInfo()->ColumnTablesUnderOperation.insert((*context.GetObjectPath())->PathId);
-
- // Sequentially chain operations in the same olap store
- if (context.GetSSOperationContext()->SS->Operations.contains(storePath.Base()->LastTxId)) {
- context.GetSSOperationContext()->OnComplete.Dependence(storePath.Base()->LastTxId, (*context.GetObjectPath())->LastTxId);
- }
- storePath.Base()->LastTxId = (*context.GetObjectPath())->LastTxId;
- context.GetSSOperationContext()->SS->PersistLastTxId(*context.GetDB(), storePath.Base());
- context.GetSSOperationContext()->SS->PersistColumnTableAlter(*context.GetDB(), GetPathId(), *Target->GetTableInfo());
- auto table = context.GetSSOperationContext()->SS->ColumnTables.TakeVerified(GetPathId());
- table->AlterData = Target->GetTableInfoPtrVerified();
- return TConclusionStatus::Success();
-}
-
-} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.h
deleted file mode 100644
index a26a43d94c..0000000000
--- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.h
+++ /dev/null
@@ -1,50 +0,0 @@
-#pragma once
-#include "object.h"
-#include "update.h"
-
-#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.h>
-
-namespace NKikimr::NSchemeShard::NOlap::NAlter {
-
-class TInStoreSchemaEvolution: public ISSEntityEvolution {
-private:
- using TBase = ISSEntityEvolution;
- std::shared_ptr<TInStoreTable> Original;
- std::shared_ptr<TInStoreTable> Target;
- std::shared_ptr<TInStoreSchemaUpdate> Update;
-protected:
- virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override {
- return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SCHEMA;
- }
-
- virtual NKikimrTxColumnShard::TSchemaTxBody DoGetShardTxBody(const NKikimrTxColumnShard::TSchemaTxBody& original, const ui64 /*tabletId*/) const override {
- NKikimrTxColumnShard::TSchemaTxBody copy = original;
-
- auto& alter = *copy.MutableAlterTable();
- Update->FillToShardTx(alter, Target);
- alter.SetPathId(GetPathId().LocalPathId);
- return copy;
- }
-
- virtual TConclusionStatus DoInitialize(const TEvolutionInitializationContext& context) override;
-
- virtual TConclusionStatus DoStartEvolution(const TEvolutionStartContext& context) override;
-public:
- TInStoreSchemaEvolution(const TPathId& pathId)
- : TBase(pathId)
- {
-
- }
- TInStoreSchemaEvolution(const std::shared_ptr<TInStoreTable>& original, const std::shared_ptr<TInStoreTable>& target, const std::shared_ptr<TInStoreSchemaUpdate>& update)
- : TBase(original ? original->GetPathId() : target->GetPathId())
- , Original(original)
- , Target(target)
- , Update(update)
- {
- AFL_VERIFY(!!Original);
- AFL_VERIFY(!!Target);
- AFL_VERIFY(!!Update);
- }
-};
-
-} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.cpp
index de08a2396b..867849583e 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.cpp
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.cpp
@@ -1,14 +1,15 @@
#include "object.h"
-#include "update.h"
+#include "config_shards/update.h"
+#include "resharding/update.h"
+#include "schema/update.h"
+#include "transfer/update.h"
#include <ydb/core/tx/schemeshard/schemeshard_impl.h>
namespace NKikimr::NSchemeShard::NOlap::NAlter {
-TConclusionStatus TInStoreTable::InitializeWithTableInfo(const TColumnTableInfo::TPtr& tInfo, const TEntityInitializationContext& context) {
- TableInfo = tInfo;
- AFL_VERIFY(!!TableInfo);
- AFL_VERIFY(!TableInfo->IsStandalone());
- const auto storePathId = TableInfo->GetOlapStorePathIdVerified();
+TConclusionStatus TInStoreTable::InitializeWithTableInfo(const TEntityInitializationContext& context) {
+ AFL_VERIFY(!GetTableInfoPtrVerified()->IsStandalone());
+ const auto storePathId = GetTableInfoPtrVerified()->GetOlapStorePathIdVerified();
TPath storePath = TPath::Init(storePathId, context.GetSSOperationContext()->SS);
{
TPath::TChecker checks = storePath.Check();
@@ -28,22 +29,41 @@ TConclusionStatus TInStoreTable::InitializeWithTableInfo(const TColumnTableInfo:
schema.ParseFromLocalDB(GetTableSchemaProto().DetachResult());
TableSchema = std::move(schema);
- if (TableInfo->Description.HasTtlSettings()) {
+ if (GetTableInfoPtrVerified()->Description.HasTtlSettings()) {
TOlapTTL ttl;
- ttl.DeserializeFromProto(TableInfo->Description.GetTtlSettings()).Validate();
+ ttl.DeserializeFromProto(GetTableInfoPtrVerified()->Description.GetTtlSettings()).Validate();
TableTTL = std::move(ttl);
}
return TConclusionStatus::Success();
}
-TConclusionStatus TInStoreTable::DoInitialize(const TEntityInitializationContext& context) {
- TableInfo = context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(GetPathId());
- return InitializeWithTableInfo(TableInfo, context);
+TConclusionStatus TInStoreTable::DoInitializeImpl(const TEntityInitializationContext& context) {
+ return InitializeWithTableInfo(context);
}
-NKikimr::TConclusion<std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEntityUpdate>> TInStoreTable::DoCreateUpdate(const TUpdateInitializationContext& context, const std::shared_ptr<ISSEntity>& selfPtr) const {
- std::shared_ptr<ISSEntityUpdate> result = std::make_shared<TInStoreSchemaUpdate>(selfPtr);
+TConclusion<std::shared_ptr<ISSEntityUpdate>> TInStoreTable::DoCreateUpdateImpl(const TUpdateInitializationContext& context) const {
+ std::shared_ptr<ISSEntityUpdate> result;
+ if (context.GetModification()->HasAlterTable()) {
+ result = std::make_shared<TInStoreSchemaUpdate>();
+ } else if (context.GetModification()->HasAlterColumnTable()) {
+ auto& alter = context.GetModification()->GetAlterColumnTable();
+
+ if (alter.HasAlterShards()) {
+ if (alter.GetAlterShards().HasTransfer()) {
+ result = std::make_shared<TInStoreShardsTransfer>();
+ } else if (alter.GetAlterShards().HasModification()) {
+ result = std::make_shared<TInStoreShardsUpdate>();
+ }
+ } else if (alter.HasReshardColumnTable()) {
+ result = std::make_shared<TInStoreShardingUpdate>();
+ } else if (alter.HasAlterSchema() || alter.HasAlterTtlSettings()) {
+ result = std::make_shared<TInStoreSchemaUpdate>();
+ }
+ }
+ if (!result) {
+ return NKikimr::TConclusionStatus::Fail("Undefined modification type");
+ }
auto initConclusion = result->Initialize(context);
if (initConclusion.IsFail()) {
return initConclusion;
@@ -51,4 +71,18 @@ NKikimr::TConclusion<std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEn
return result;
}
+NKikimr::TConclusion<NKikimrSchemeOp::TColumnTableSchema> TInStoreTable::GetTableSchemaProto() const {
+ AFL_VERIFY(!!StoreInfo);
+ if (!StoreInfo->SchemaPresets.count(GetTableInfoPtrVerified()->Description.GetSchemaPresetId())) {
+ return TConclusionStatus::Fail("No preset for in-store column table");
+ }
+
+ auto& preset = StoreInfo->SchemaPresets.at(GetTableInfoPtrVerified()->Description.GetSchemaPresetId());
+ auto& presetProto = StoreInfo->GetDescription().GetSchemaPresets(preset.GetProtoIndex());
+ if (!presetProto.HasSchema()) {
+ return TConclusionStatus::Fail("No schema in preset for in-store column table");
+ }
+ return presetProto.GetSchema();
+}
+
} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h
index 5c6b9b0c02..e0fb2bd502 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h
@@ -1,5 +1,5 @@
#pragma once
-#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/common/object.h>
#include <ydb/core/tx/schemeshard/olap/table/table.h>
#include <ydb/core/tx/schemeshard/olap/store/store.h>
#include <ydb/core/tx/schemeshard/olap/schema/schema.h>
@@ -7,58 +7,32 @@
namespace NKikimr::NSchemeShard::NOlap::NAlter {
-class TInStoreTable: public ISSEntity {
+class TInStoreTable: public TColumnTableEntity {
private:
- using TBase = ISSEntity;
- YDB_READONLY_DEF(TColumnTableInfo::TPtr, TableInfo);
+ using TBase = TColumnTableEntity;
YDB_READONLY_DEF(TOlapStoreInfo::TPtr, StoreInfo);
std::optional<TOlapSchema> TableSchema;
std::optional<TOlapTTL> TableTTL;
- virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdate(const TUpdateInitializationContext& context, const std::shared_ptr<ISSEntity>& selfPtr) const override;
- virtual TConclusionStatus DoInitialize(const TEntityInitializationContext& context) override;
- [[nodiscard]] TConclusionStatus InitializeWithTableInfo(const TColumnTableInfo::TPtr& tInfo, const TEntityInitializationContext& context);
+ virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdateImpl(const TUpdateInitializationContext& context) const override;
+ virtual TConclusionStatus DoInitializeImpl(const TEntityInitializationContext& context) override;
+ [[nodiscard]] TConclusionStatus InitializeWithTableInfo(const TEntityInitializationContext& context);
public:
virtual TString GetClassName() const override {
return "IN_STORE_TABLE";
}
- TInStoreTable(const TPathId& pathId)
- : TBase(pathId) {
+ using TBase::TBase;
+ TInStoreTable(const TPathId& pathId, const std::shared_ptr<TColumnTableInfo>& tableInfo, const TEntityInitializationContext& context)
+ : TBase(pathId, tableInfo)
+ {
+ InitializeWithTableInfo(context).Validate();
}
- TInStoreTable(const TPathId& pathId, const TColumnTableInfo::TPtr& tInfo, const TEntityInitializationContext& context)
- : TBase(pathId) {
- InitializeWithTableInfo(tInfo, context).Validate();
- }
-
- const TColumnTableInfo& GetTableInfoVerified() const {
- AFL_VERIFY(!!TableInfo);
- return *TableInfo;
- }
-
- TColumnTableInfo::TPtr GetTableInfoPtrVerified() const {
- AFL_VERIFY(!!TableInfo);
- return TableInfo;
- }
-
- TConclusion<NKikimrSchemeOp::TColumnTableSchema> GetTableSchemaProto() const {
- AFL_VERIFY(!!StoreInfo);
- if (!StoreInfo->SchemaPresets.count(TableInfo->Description.GetSchemaPresetId())) {
- return TConclusionStatus::Fail("No preset for in-store column table");
- }
-
- auto& preset = StoreInfo->SchemaPresets.at(TableInfo->Description.GetSchemaPresetId());
- auto& presetProto = StoreInfo->GetDescription().GetSchemaPresets(preset.GetProtoIndex());
- if (!presetProto.HasSchema()) {
- return TConclusionStatus::Fail("No schema in preset for in-store column table");
- }
- return presetProto.GetSchema();
- }
+ TConclusion<NKikimrSchemeOp::TColumnTableSchema> GetTableSchemaProto() const;
const NKikimrSchemeOp::TColumnDataLifeCycle& GetTableTTLProto() {
- AFL_VERIFY(TableInfo);
- return TableInfo->Description.GetTtlSettings();
+ return GetTableInfoPtrVerified()->Description.GetTtlSettings();
}
const TOlapSchema& GetTableSchemaVerified() const {
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.cpp
new file mode 100644
index 0000000000..1aef72056e
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.cpp
@@ -0,0 +1,62 @@
+#include "update.h"
+#include <ydb/core/tx/columnshard/bg_tasks/abstract/task.h>
+#include <ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/task.h>
+#include <ydb/core/tx/schemeshard/schemeshard_impl.h>
+#include <ydb/core/tx/sharding/sharding.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+TConclusionStatus TInStoreShardingUpdate::DoStart(const TUpdateStartContext& context) {
+ NKikimr::NOlap::NBackground::TTask task("SPLIT_SHARDS::" + context.GetObjectPath()->PathString(), std::make_shared<NKikimr::NOlap::NBackground::TFakeStatusChannel>(),
+ std::make_shared<NOlap::NBackground::TTxChainTask>(TxChainData));
+ auto tx = context.GetSSOperationContext()->SS->BackgroundSessionsManager->TxAddTask(task);
+ if (!tx->Execute(context.GetSSOperationContext()->GetTxc(), context.GetSSOperationContext()->Ctx)) {
+ return TConclusionStatus::Fail("cannot execute transaction for write task");
+ }
+ tx->Complete(context.GetSSOperationContext()->Ctx);
+ return TConclusionStatus::Success();
+}
+
+TConclusionStatus TInStoreShardingUpdate::DoInitialize(const TUpdateInitializationContext& context) {
+ auto& inStoreTable = context.GetOriginalEntityAsVerified<TInStoreTable>();
+ AFL_VERIFY(context.GetModification()->GetAlterColumnTable().HasReshardColumnTable());
+ std::shared_ptr<NSharding::IShardingBase> sharding = inStoreTable.GetTableInfoPtrVerified()->GetShardingVerified(inStoreTable.GetTableSchemaVerified());
+ TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> alters = std::vector<NKikimrSchemeOp::TAlterShards>();
+ auto& storeInfo = *inStoreTable.GetStoreInfo();
+ auto layoutPolicy = storeInfo.GetTablesLayoutPolicy();
+ auto currentLayout = context.GetSSOperationContext()->SS->ColumnTables.GetTablesLayout(TColumnTablesLayout::ShardIdxToTabletId(
+ storeInfo.GetColumnShards(), *context.GetSSOperationContext()->SS));
+ auto tablePtr = context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(context.GetOriginalEntity().GetPathId());
+ if (context.GetModification()->GetAlterColumnTable().GetReshardColumnTable().GetIncrease()) {
+ const ui32 shardsCount = inStoreTable.GetTableInfoPtrVerified()->GetColumnShards().size();
+ auto layoutConclusion = layoutPolicy->Layout(currentLayout, shardsCount);
+ if (layoutConclusion.IsFail()) {
+ return layoutConclusion;
+ }
+ alters = sharding->BuildAddShardsModifiers(layoutConclusion->GetTabletIds());
+ } else {
+ if (inStoreTable.GetTableInfoPtrVerified()->GetColumnShards().size() % 2) {
+ return TConclusionStatus::Fail("cannot reduce shards count (possible for even shards count only)");
+ }
+ const ui32 newShardsCount = inStoreTable.GetTableInfoPtrVerified()->GetColumnShards().size() / 2;
+ auto layoutConclusion = layoutPolicy->Layout(currentLayout, newShardsCount);
+ if (layoutConclusion.IsFail()) {
+ return layoutConclusion;
+ }
+ alters = sharding->BuildReduceShardsModifiers(layoutConclusion->GetTabletIds());
+ }
+ if (alters.IsFail()) {
+ return alters;
+ }
+ for (auto&& i : *alters) {
+ NKikimrSchemeOp::TModifyScheme modification;
+ modification.SetWorkingDir(context.GetModification()->GetWorkingDir());
+ modification.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterColumnTable);
+ *modification.MutableAlterColumnTable()->MutableAlterShards() = std::move(i);
+ modification.MutableAlterColumnTable()->SetName(context.GetModification()->GetAlterColumnTable().GetName());
+ TxChainData.MutableTransactions().emplace_back(std::move(modification));
+ }
+ return TConclusionStatus::Success();
+}
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.h
new file mode 100644
index 0000000000..8b78c3ddb5
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/update.h
@@ -0,0 +1,36 @@
+#pragma once
+#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h>
+#include <ydb/core/tx/schemeshard/olap/ttl/update.h>
+#include <ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain/common.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+class TInStoreShardingUpdate: public ISSEntityUpdate {
+private:
+ using TBase = ISSEntityUpdate;
+ NOlap::NBackground::TTxChainData TxChainData;
+ virtual TString DoGetShardTxBodyString(const ui64 /*tabletId*/, const TMessageSeqNo& /*seqNo*/) const override {
+ AFL_VERIFY(false);
+ return "";
+ }
+
+ virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override {
+ return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SHARING;
+ }
+
+ virtual TConclusionStatus DoFinish(const TUpdateFinishContext& /*context*/) override {
+ return TConclusionStatus::Success();
+ }
+
+ virtual TConclusionStatus DoInitialize(const TUpdateInitializationContext& context) override;
+ virtual TConclusionStatus DoStart(const TUpdateStartContext& context) override;
+
+ virtual std::set<ui64> DoGetShardIds() const override {
+ return {};
+ }
+
+public:
+};
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/ya.make
new file mode 100644
index 0000000000..87d19eb784
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding/ya.make
@@ -0,0 +1,14 @@
+LIBRARY()
+
+SRCS(
+ update.cpp
+)
+
+PEERDIR(
+ ydb/core/tx/schemeshard/olap/operations/alter/abstract
+ ydb/core/tx/schemeshard/olap/bg_tasks/tx_chain
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.cpp
new file mode 100644
index 0000000000..bbf1845ac1
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.cpp
@@ -0,0 +1,69 @@
+#include "update.h"
+#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+NKikimr::TConclusionStatus TInStoreSchemaUpdate::DoInitializeImpl(const TUpdateInitializationContext& context) {
+ auto alter = TConverterModifyToAlter().Convert(*context.GetModification());
+ if (alter.IsFail()) {
+ return alter;
+ }
+ auto alterCS = alter.DetachResult();
+ if (alterCS.HasAlterSchema()) {
+ return TConclusionStatus::Fail("cannot modify scheme for table in store");
+ }
+ if (!alterCS.HasAlterTtlSettings()) {
+ return TConclusionStatus::Fail("no data for update");
+ }
+ AlterTTL.emplace(alterCS.GetAlterTtlSettings());
+ const auto& originalTable = context.GetOriginalEntityAsVerified<TInStoreTable>();
+ auto description = originalTable.GetTableInfoVerified().Description;
+
+ {
+ const auto& originalTableInfo = originalTable.GetTableInfoVerified();
+ const auto& storeInfo = originalTable.GetStoreInfo();
+ AFL_VERIFY(!!storeInfo)("problem", "Unexpected schema preset without olap store");
+ AFL_VERIFY(originalTableInfo.Description.HasSchemaPresetId());
+ const ui32 presetId = originalTableInfo.Description.GetSchemaPresetId();
+ AFL_VERIFY(storeInfo->SchemaPresets.contains(presetId))("problem", "Failed to find schema preset in an olap store")("id", presetId);
+ auto& preset = storeInfo->SchemaPresets.at(presetId);
+ size_t presetIndex = preset.GetProtoIndex();
+ SchemaPreset = storeInfo->GetDescription().GetSchemaPresets(presetIndex);
+ }
+
+ if (AlterTTL) {
+ TOlapSchema originalSchema;
+ originalSchema.ParseFromLocalDB(SchemaPreset->GetSchema());
+
+ auto ttl = originalTable.GetTableTTLOptional() ? *originalTable.GetTableTTLOptional() : TOlapTTL();
+ auto patch = ttl.Update(*AlterTTL);
+ if (patch.IsFail()) {
+ return patch;
+ }
+ TSimpleErrorCollector collector;
+ if (!originalSchema.ValidateTtlSettings(ttl.GetData(), collector)) {
+ return TConclusionStatus::Fail("ttl update error: " + collector->GetErrorMessage() + ". in alter constructor STANDALONE_UPDATE");
+ }
+ *description.MutableTtlSettings() = ttl.SerializeToProto();
+ }
+
+ auto targetInfo = std::make_shared<TColumnTableInfo>(context.GetOriginalEntityAsVerified<TInStoreTable>().GetTableInfoVerified().AlterVersion + 1,
+ std::move(description), TMaybe<NKikimrSchemeOp::TColumnStoreSharding>(), std::move(alterCS));
+ TEntityInitializationContext eContext(context.GetSSOperationContext());
+ TargetInStoreTable = std::make_shared<TInStoreTable>(context.GetOriginalEntity().GetPathId(), targetInfo, eContext);
+ return TConclusionStatus::Success();
+}
+
+void TInStoreSchemaUpdate::FillToShardTx(NKikimrTxColumnShard::TAlterTable& shardAlter) const {
+ auto& alterBody = TargetInStoreTable->GetTableInfoVerified();
+ *shardAlter.MutableTtlSettings() = alterBody.Description.GetTtlSettings();
+
+ AFL_VERIFY(!!SchemaPreset);
+ *shardAlter.MutableSchemaPreset() = *SchemaPreset;
+
+ if (alterBody.Description.HasSchemaPresetVersionAdj()) {
+ shardAlter.SetSchemaPresetVersionAdj(alterBody.Description.GetSchemaPresetVersionAdj());
+ }
+}
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.h
new file mode 100644
index 0000000000..4ec15b86ac
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/update.h
@@ -0,0 +1,50 @@
+#pragma once
+#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h>
+#include <ydb/core/protos/flat_scheme_op.pb.h>
+#include <ydb/core/tx/schemeshard/olap/ttl/update.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+class TInStoreSchemaUpdate: public TInStoreTableUpdate {
+private:
+ using TBase = TInStoreTableUpdate;
+ std::optional<TOlapTTLUpdate> AlterTTL;
+ std::optional<NKikimrSchemeOp::TColumnTableSchemaPreset> SchemaPreset;
+ std::shared_ptr<TInStoreTable> TargetInStoreTable;
+ virtual TConclusionStatus DoInitializeImpl(const TUpdateInitializationContext& context) override;
+
+ virtual std::shared_ptr<ISSEntity> GetTargetSSEntity() const override {
+ return TargetInStoreTable;
+ }
+
+ virtual std::shared_ptr<TColumnTableInfo> GetTargetTableInfo() const override {
+ return TargetInStoreTable->GetTableInfoPtrVerified();
+ }
+
+ virtual TString DoGetShardTxBodyString(const ui64 /*tabletId*/, const TMessageSeqNo& seqNo) const override {
+ NKikimrTxColumnShard::TSchemaTxBody result;
+ result.MutableSeqNo()->SetGeneration(seqNo.Generation);
+ result.MutableSeqNo()->SetRound(seqNo.Round);
+
+ auto& alter = *result.MutableAlterTable();
+ FillToShardTx(alter);
+ alter.SetPathId(TargetInStoreTable->GetPathId().LocalPathId);
+ return result.SerializeAsString();
+ }
+
+ void FillToShardTx(NKikimrTxColumnShard::TAlterTable& shardAlter) const;
+
+ virtual std::set<ui64> DoGetShardIds() const override {
+ return TargetInStoreTable->GetTableInfoVerified().GetShardIdsSet();
+ }
+
+public:
+
+ virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override {
+ return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SCHEMA;
+ }
+};
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/ya.make
new file mode 100644
index 0000000000..1c221f9bf5
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema/ya.make
@@ -0,0 +1,11 @@
+LIBRARY()
+
+SRCS(
+ update.cpp
+)
+
+PEERDIR(
+ ydb/core/tx/schemeshard/olap/operations/alter/abstract
+)
+
+END()
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.cpp
new file mode 100644
index 0000000000..73bbd10a3d
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.cpp
@@ -0,0 +1,44 @@
+#include "update.h"
+#include <ydb/core/tx/columnshard/data_sharing/initiator/controller/schemeshard.h>
+#include <ydb/core/tx/schemeshard/schemeshard_impl.h>
+#include <ydb/core/tx/columnshard/common/snapshot.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+NKikimr::TConclusionStatus TInStoreShardsTransfer::DoInitializeImpl(const TUpdateInitializationContext& context) {
+ if (!context.GetModification()->GetAlterColumnTable().GetAlterShards().GetTransfer().GetTransfers().size()) {
+ return TConclusionStatus::Fail("hasn't data about shards transfer");
+ }
+ auto& table = context.GetOriginalEntityAsVerified<TInStoreTable>();
+ auto sharding = table.GetTableInfo()->GetShardingVerified(table.GetTableSchemaVerified());
+ for (auto&& alter : context.GetModification()->GetAlterColumnTable().GetAlterShards().GetTransfer().GetTransfers()) {
+ NKikimrColumnShardDataSharingProto::TDestinationSession destinationSession;
+ destinationSession.SetSessionId("SHARE_TO_SHARD::" + ::ToString(alter.GetDestinationTabletId()));
+ *destinationSession.MutableInitiatorController() = NKikimr::NOlap::NDataSharing::TInitiatorControllerContainer(
+ std::make_shared<NKikimr::NOlap::NDataSharing::TSSInitiatorController>(context.GetSSOperationContext()->SS->TabletID(), 0)).SerializeToProto();
+ {
+ auto& pathIdRemap = *destinationSession.AddPathIds();
+ pathIdRemap.SetSourcePathId(context.GetOriginalEntity().GetPathId().LocalPathId);
+ pathIdRemap.SetDestPathId(context.GetOriginalEntity().GetPathId().LocalPathId);
+ }
+ ::NKikimr::NOlap::TSnapshot ssOpen = sharding->GetShardingOpenSnapshotVerified(alter.GetDestinationTabletId());
+
+ destinationSession.MutableTransferContext()->SetDestinationTabletId(alter.GetDestinationTabletId());
+ destinationSession.MutableTransferContext()->SetTxId(context.GetTxId());
+ *destinationSession.MutableTransferContext()->MutableSnapshotBarrier() = ssOpen.SerializeToProto();
+ for (auto&& i : alter.GetSourceTabletIds()) {
+ destinationSession.MutableTransferContext()->AddSourceTabletIds(i);
+ }
+ DestinationSessions.emplace_back(destinationSession);
+ AFL_VERIFY(ShardIdsUsage.emplace(alter.GetDestinationTabletId()).second);
+ }
+ const auto& inStoreOriginal = context.GetOriginalEntityAsVerified<TInStoreTable>();
+ auto targetInfo = std::make_shared<TColumnTableInfo>(inStoreOriginal.GetTableInfoVerified().AlterVersion,
+ inStoreOriginal.GetTableInfoVerified().Description, TMaybe<NKikimrSchemeOp::TColumnStoreSharding>(), context.GetModification()->GetAlterColumnTable());
+ TEntityInitializationContext eContext(context.GetSSOperationContext());
+ TargetInStoreTable = std::make_shared<TInStoreTable>(context.GetOriginalEntity().GetPathId(), targetInfo, eContext);
+
+ return TConclusionStatus::Success();
+}
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.h
new file mode 100644
index 0000000000..5b19aa69fb
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/update.h
@@ -0,0 +1,48 @@
+#pragma once
+#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/common/update.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h>
+#include <ydb/core/tx/columnshard/data_sharing/protos/sessions.pb.h>
+#include <ydb/core/tx/schemeshard/olap/ttl/update.h>
+
+namespace NKikimr::NSchemeShard::NOlap::NAlter {
+
+class TInStoreShardsTransfer: public TInStoreTableUpdate {
+private:
+ using TBase = TInStoreTableUpdate;
+ std::vector<NKikimrColumnShardDataSharingProto::TDestinationSession> DestinationSessions;
+ std::shared_ptr<TInStoreTable> TargetInStoreTable;
+ std::set<ui64> ShardIdsUsage;
+
+ virtual std::shared_ptr<TColumnTableInfo> GetTargetTableInfo() const override {
+ AFL_VERIFY(TargetInStoreTable);
+ return TargetInStoreTable->GetTableInfoPtrVerified();
+ }
+ virtual std::shared_ptr<ISSEntity> GetTargetSSEntity() const override {
+ return TargetInStoreTable;
+ }
+
+ virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override {
+ return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SHARING;
+ }
+
+ virtual TConclusionStatus DoInitializeImpl(const TUpdateInitializationContext& context) override;
+
+ virtual TString DoGetShardTxBodyString(const ui64 tabletId, const TMessageSeqNo& /*seqNo*/) const override {
+ for (auto&& i : DestinationSessions) {
+ if (i.GetTransferContext().GetDestinationTabletId() == tabletId) {
+ return i.SerializeAsString();
+ }
+ }
+ AFL_VERIFY(false);
+ return "";
+ }
+
+ virtual std::set<ui64> DoGetShardIds() const override {
+ return ShardIdsUsage;
+ }
+
+public:
+};
+
+} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/ya.make
new file mode 100644
index 0000000000..8ad440b4ea
--- /dev/null
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer/ya.make
@@ -0,0 +1,14 @@
+LIBRARY()
+
+SRCS(
+ update.cpp
+)
+
+PEERDIR(
+ ydb/core/tx/schemeshard/olap/operations/alter/abstract
+ ydb/core/tx/columnshard/data_sharing/protos
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.cpp
deleted file mode 100644
index e10fdb0b5f..0000000000
--- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.cpp
+++ /dev/null
@@ -1,94 +0,0 @@
-#include "evolution.h"
-#include "update.h"
-#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h>
-
-namespace NKikimr::NSchemeShard::NOlap::NAlter {
-
-NKikimr::TConclusionStatus TInStoreSchemaUpdate::DoInitialize(const TUpdateInitializationContext& context) {
- auto alter = TConverterModifyToAlter().Convert(*context.GetModification());
- if (alter.IsFail()) {
- return alter;
- }
- auto alterCS = alter.DetachResult();
- {
- auto conclusion = InitializeDiff(alterCS);
- if (conclusion.IsFail()) {
- return conclusion;
- }
- }
- auto description = OriginalInStoreTable->GetTableInfoVerified().Description;
- {
- auto ttlConclusion = GetTtlPatched();
- if (ttlConclusion.IsFail()) {
- return ttlConclusion;
- }
- *description.MutableTtlSettings() = ttlConclusion->SerializeToProto();
- }
-
- auto targetInfo = std::make_shared<TColumnTableInfo>(OriginalInStoreTable->GetTableInfoVerified().AlterVersion + 1, std::move(description), TMaybe<NKikimrSchemeOp::TColumnStoreSharding>(), std::move(alterCS));
- TEntityInitializationContext eContext(context.GetSSOperationContext());
- TargetInStoreTable = std::make_shared<TInStoreTable>(GetOriginalEntity()->GetPathId(), targetInfo, eContext);
-
- return TConclusionStatus::Success();
-}
-
-NKikimr::TConclusion<std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEntityEvolution>> TInStoreSchemaUpdate::DoBuildEvolution(const std::shared_ptr<ISSEntityUpdate>& updateSelfPtr) const {
- auto update = dynamic_pointer_cast<TInStoreSchemaUpdate>(updateSelfPtr);
- AFL_VERIFY(!!update);
- return std::make_shared<TInStoreSchemaEvolution>(OriginalInStoreTable, TargetInStoreTable, update);
-}
-
-NKikimr::TConclusionStatus TInStoreSchemaUpdate::InitializeDiff(const NKikimrSchemeOp::TAlterColumnTable& alterCS) {
- if (alterCS.HasAlterSchema()) {
- return TConclusionStatus::Fail("cannot modify scheme for table in store");
- }
- AlterTTL = alterCS.GetAlterTtlSettings();
- if (!AlterTTL) {
- return TConclusionStatus::Fail("no data for update");
- }
- auto ttlConclusion = GetTtlPatched();
- if (ttlConclusion.IsFail()) {
- return ttlConclusion;
- }
- return TConclusionStatus::Success();
-}
-
-NKikimr::TConclusion<NKikimr::NSchemeShard::NOlap::NAlter::TOlapTTL> TInStoreSchemaUpdate::GetTtlPatched() const {
- const TOlapSchema& originalSchema = OriginalInStoreTable->GetTableSchemaVerified();
- TOlapTTL ttl = OriginalInStoreTable->GetTableTTLOptional() ? *OriginalInStoreTable->GetTableTTLOptional() : TOlapTTL();
- auto patch = ttl.Update(*AlterTTL);
- if (patch.IsFail()) {
- return patch;
- }
- TSimpleErrorCollector collector;
- if (!originalSchema.ValidateTtlSettings(ttl.GetData(), collector)) {
- return TConclusionStatus::Fail("ttl update error: " + collector->GetErrorMessage() + ". in alter constructor IN_STORE_TABLE");
- }
- return ttl;
-}
-
-void TInStoreSchemaUpdate::FillToShardTx(NKikimrTxColumnShard::TAlterTable& shardAlter, const std::shared_ptr<TInStoreTable>& target) const {
- if (AlterTTL) {
- *shardAlter.MutableTtlSettings() = target->GetTableTTLProto();
- }
- auto& alterBody = OriginalInStoreTable->GetTableInfoVerified();
-
- AFL_VERIFY(alterBody.Description.HasSchemaPresetId());
- const ui32 presetId = alterBody.Description.GetSchemaPresetId();
- Y_ABORT_UNLESS(!!TargetInStoreTable->GetStoreInfo(),
- "Unexpected schema preset without olap store");
- Y_ABORT_UNLESS(TargetInStoreTable->GetStoreInfo()->SchemaPresets.contains(presetId),
- "Failed to find schema preset %" PRIu32
- " in an olap store",
- presetId);
- auto& preset = TargetInStoreTable->GetStoreInfo()->SchemaPresets.at(presetId);
- size_t presetIndex = preset.GetProtoIndex();
- *shardAlter.MutableSchemaPreset() =
- TargetInStoreTable->GetStoreInfo()->GetDescription().GetSchemaPresets(presetIndex);
-
- if (alterBody.Description.HasSchemaPresetVersionAdj()) {
- shardAlter.SetSchemaPresetVersionAdj(alterBody.Description.GetSchemaPresetVersionAdj());
- }
-}
-
-} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.h
deleted file mode 100644
index 37b6f2c95c..0000000000
--- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/update.h
+++ /dev/null
@@ -1,43 +0,0 @@
-#pragma once
-#include "object.h"
-#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h>
-#include <ydb/core/tx/schemeshard/olap/ttl/update.h>
-
-namespace NKikimr::NSchemeShard::NOlap::NAlter {
-
-class TInStoreSchemaUpdate: public ISSEntityUpdate {
-private:
- using TBase = ISSEntityUpdate;
- std::optional<TOlapTTLUpdate> AlterTTL;
- std::shared_ptr<TInStoreTable> OriginalInStoreTable;
- std::shared_ptr<TInStoreTable> TargetInStoreTable;
- virtual TConclusionStatus DoInitialize(const TUpdateInitializationContext& context) override;
- TConclusionStatus InitializeDiff(const NKikimrSchemeOp::TAlterColumnTable& alterCS);
- TConclusion<TOlapTTL> GetTtlPatched() const;
-public:
- TInStoreSchemaUpdate(const std::shared_ptr<ISSEntity>& originalEntity)
- : TBase(originalEntity) {
- OriginalInStoreTable = dynamic_pointer_cast<TInStoreTable>(originalEntity);
- AFL_VERIFY(OriginalInStoreTable);
- }
-
- TInStoreSchemaUpdate(const std::shared_ptr<ISSEntity>& originalEntity, const std::shared_ptr<ISSEntity>& targetEntity)
- : TBase(originalEntity) {
- OriginalInStoreTable = dynamic_pointer_cast<TInStoreTable>(originalEntity);
- AFL_VERIFY(OriginalInStoreTable);
- TargetInStoreTable = dynamic_pointer_cast<TInStoreTable>(targetEntity);
- AFL_VERIFY(TargetInStoreTable);
- AFL_VERIFY(!!TargetInStoreTable->GetTableInfoVerified().AlterBody);
- InitializeDiff(*TargetInStoreTable->GetTableInfoVerified().AlterBody).Validate();
- }
-
- void FillToShardTx(NKikimrTxColumnShard::TAlterTable& shardAlter, const std::shared_ptr<TInStoreTable>& target) const;
-
- const TOlapTTLUpdate* GetAlterTTLOptional() const {
- return AlterTTL ? &*AlterTTL : nullptr;
- }
-
- virtual TConclusion<std::shared_ptr<ISSEntityEvolution>> DoBuildEvolution(const std::shared_ptr<ISSEntityUpdate>& updateSelfPtr) const override;
-};
-
-} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/ya.make
index 4106bd5a63..6f2b79b6e0 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/in_store/ya.make
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/in_store/ya.make
@@ -2,12 +2,15 @@ LIBRARY()
SRCS(
object.cpp
- update.cpp
- evolution.cpp
)
PEERDIR(
ydb/core/tx/schemeshard/olap/operations/alter/abstract
+ ydb/core/tx/schemeshard/olap/operations/alter/in_store/config_shards
+ ydb/core/tx/schemeshard/olap/operations/alter/in_store/resharding
+ ydb/core/tx/schemeshard/olap/operations/alter/in_store/schema
+ ydb/core/tx/schemeshard/olap/operations/alter/in_store/transfer
+ ydb/core/tx/schemeshard/olap/operations/alter/in_store/common
ydb/core/tx/schemeshard/olap/ttl
ydb/core/tx/schemeshard/olap/table
ydb/core/tx/schemeshard/olap/store
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.cpp
deleted file mode 100644
index 7dfe50652d..0000000000
--- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.cpp
+++ /dev/null
@@ -1,35 +0,0 @@
-#include "evolution.h"
-#include <ydb/core/tx/schemeshard/schemeshard_impl.h>
-
-namespace NKikimr::NSchemeShard::NOlap::NAlter {
-
-NKikimr::TConclusionStatus TStandaloneSchemaEvolution::DoInitialize(const TEvolutionInitializationContext& context) {
- TColumnTableInfo::TPtr tableInfo = context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(GetPathId());
- if (!tableInfo->IsStandalone()) {
- return TConclusionStatus::Fail("incorrect path object - not standalone");
- }
- Original = std::make_shared<TStandaloneTable>(GetPathId(), tableInfo);
- Target = std::make_shared<TStandaloneTable>(GetPathId(), tableInfo->AlterData);
- {
- NKikimrSchemeOp::TModifyScheme txSchema;
- AFL_VERIFY(!!tableInfo->AlterData->AlterBody);
- *txSchema.MutableAlterColumnTable() = *tableInfo->AlterData->AlterBody;
- txSchema.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterColumnTable);
- Update = std::make_shared<TStandaloneSchemaUpdate>(Original);
- TUpdateInitializationContext uContext(context.GetSSOperationContext(), &txSchema);
- auto conclusion = Update->Initialize(uContext);
- if (conclusion.IsFail()) {
- return conclusion;
- }
- }
- return TConclusionStatus::Success();
-}
-
-NKikimr::TConclusionStatus TStandaloneSchemaEvolution::DoStartEvolution(const TEvolutionStartContext& context) {
- auto table = context.GetSSOperationContext()->SS->ColumnTables.TakeVerified(GetPathId());
- table->AlterData = Target->GetTableInfoPtrVerified();
- context.GetSSOperationContext()->SS->PersistColumnTableAlter(*context.GetDB(), GetPathId(), Target->GetTableInfoVerified());
- return TConclusionStatus::Success();
-}
-
-} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.h b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.h
deleted file mode 100644
index 86bbc1e9e0..0000000000
--- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.h
+++ /dev/null
@@ -1,49 +0,0 @@
-#pragma once
-#include "object.h"
-#include "update.h"
-#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/evolution.h>
-
-namespace NKikimr::NSchemeShard::NOlap::NAlter {
-
-class TStandaloneSchemaEvolution: public ISSEntityEvolution {
-private:
- using TBase = ISSEntityEvolution;
- std::shared_ptr<TStandaloneTable> Original;
- std::shared_ptr<TStandaloneTable> Target;
- std::shared_ptr<TStandaloneSchemaUpdate> Update;
-protected:
- virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override {
- return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SCHEMA;
- }
-
- virtual NKikimrTxColumnShard::TSchemaTxBody DoGetShardTxBody(const NKikimrTxColumnShard::TSchemaTxBody& original, const ui64 /*tabletId*/) const override {
- NKikimrTxColumnShard::TSchemaTxBody copy = original;
-
- auto& alter = *copy.MutableAlterTable();
- Update->FillToShardTx(alter, Target);
- alter.SetPathId(GetPathId().LocalPathId);
- return copy;
- }
-
- virtual TConclusionStatus DoInitialize(const TEvolutionInitializationContext& context) override;
-
- virtual TConclusionStatus DoStartEvolution(const TEvolutionStartContext& context) override;
-public:
- TStandaloneSchemaEvolution(const TPathId& pathId)
- : TBase(pathId)
- {
-
- }
- TStandaloneSchemaEvolution(const std::shared_ptr<TStandaloneTable>& original, const std::shared_ptr<TStandaloneTable>& target, const std::shared_ptr<TStandaloneSchemaUpdate>& update)
- : TBase(original ? original->GetPathId() : target->GetPathId())
- , Original(original)
- , Target(target)
- , Update(update)
- {
- AFL_VERIFY(!!Original);
- AFL_VERIFY(!!Target);
- AFL_VERIFY(!!Update);
- }
-};
-
-} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.cpp
index e6d8dc3abb..0f4e693274 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.cpp
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.cpp
@@ -4,8 +4,14 @@
namespace NKikimr::NSchemeShard::NOlap::NAlter {
-NKikimr::TConclusion<std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEntityUpdate>> TStandaloneTable::DoCreateUpdate(const TUpdateInitializationContext& context, const std::shared_ptr<ISSEntity>& selfPtr) const {
- auto result = std::make_shared<TStandaloneSchemaUpdate>(selfPtr);
+NKikimr::TConclusion<std::shared_ptr<ISSEntityUpdate>> TStandaloneTable::DoCreateUpdateImpl(const TUpdateInitializationContext& context) const {
+ std::shared_ptr<ISSEntityUpdate> result;
+ if (context.GetModification()->HasAlterTable() || context.GetModification()->HasAlterColumnTable()) {
+ result = std::make_shared<TStandaloneSchemaUpdate>();
+ }
+ if (!result) {
+ return TConclusionStatus::Fail("alter data not found");
+ }
auto conclusion = result->Initialize(context);
if (conclusion.IsFail()) {
return conclusion;
@@ -13,23 +19,21 @@ NKikimr::TConclusion<std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEn
return result;
}
-NKikimr::TConclusionStatus TStandaloneTable::DoInitialize(const TEntityInitializationContext& context) {
- return InitializeFromTableInfo(context.GetSSOperationContext()->SS->ColumnTables.GetVerifiedPtr(GetPathId()));
+NKikimr::TConclusionStatus TStandaloneTable::DoInitializeImpl(const TEntityInitializationContext& /*context*/) {
+ return InitializeFromTableInfo();
}
-NKikimr::TConclusionStatus TStandaloneTable::InitializeFromTableInfo(const TColumnTableInfo::TPtr& tableInfo) {
- AFL_VERIFY(!!tableInfo);
- TableInfo = tableInfo;
- if (!TableInfo->Description.HasSchema()) {
+NKikimr::TConclusionStatus TStandaloneTable::InitializeFromTableInfo() {
+ if (!GetTableInfoPtrVerified()->Description.HasSchema()) {
return TConclusionStatus::Fail("path id object has no schema owned for " + GetPathId().ToString());
}
TOlapSchema schema;
- schema.ParseFromLocalDB(TableInfo->Description.GetSchema());
+ schema.ParseFromLocalDB(GetTableInfoPtrVerified()->Description.GetSchema());
TableSchema = std::move(schema);
- if (TableInfo->Description.HasTtlSettings()) {
+ if (GetTableInfoPtrVerified()->Description.HasTtlSettings()) {
TOlapTTL ttl;
- ttl.DeserializeFromProto(TableInfo->Description.GetTtlSettings()).Validate();
+ ttl.DeserializeFromProto(GetTableInfoPtrVerified()->Description.GetTtlSettings()).Validate();
TableTTL = std::move(ttl);
}
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.h b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.h
index 929d61590c..bd3c445254 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.h
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.h
@@ -1,52 +1,48 @@
#pragma once
-#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/common/object.h>
#include <ydb/core/tx/schemeshard/olap/table/table.h>
#include <ydb/core/tx/schemeshard/olap/ttl/schema.h>
namespace NKikimr::NSchemeShard::NOlap::NAlter {
-class TStandaloneTable: public ISSEntity {
+class TStandaloneTable: public TColumnTableEntity {
private:
- using TBase = ISSEntity;
- TColumnTableInfo::TPtr TableInfo;
+ using TBase = TColumnTableEntity;
std::optional<TOlapSchema> TableSchema;
std::optional<TOlapTTL> TableTTL;
- virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdate(const TUpdateInitializationContext& context, const std::shared_ptr<ISSEntity>& selfPtr) const override;
- virtual TConclusionStatus DoInitialize(const TEntityInitializationContext& context) override;
+ virtual TConclusion<std::shared_ptr<ISSEntityUpdate>> DoCreateUpdateImpl(const TUpdateInitializationContext& context) const override;
+ virtual TConclusionStatus DoInitializeImpl(const TEntityInitializationContext& context) override;
+ TConclusionStatus InitializeFromTableInfo();
public:
TStandaloneTable(const TPathId& pathId)
: TBase(pathId)
{
}
- TStandaloneTable(const TPathId& pathId, const TColumnTableInfo::TPtr& tableInfo)
- : TBase(pathId)
- {
- InitializeFromTableInfo(tableInfo).Validate();
- }
- TColumnTableInfo::TPtr GetTableInfoPtrVerified() const {
- AFL_VERIFY(!!TableInfo);
- return TableInfo;
+ TStandaloneTable(const TPathId& pathId, const std::shared_ptr<TColumnTableInfo>& tableInfo)
+ : TBase(pathId, tableInfo)
+ {
+ InitializeFromTableInfo();
}
virtual TString GetClassName() const override {
return "STANDALONE_TABLE";
}
- const NKikimrSchemeOp::TColumnTableSchema& GetTableSchemaProto() {
- AFL_VERIFY(TableInfo);
- return TableInfo->Description.GetSchema();
+ const NKikimrSchemeOp::TColumnTableSchema& GetTableSchemaProto() const {
+ AFL_VERIFY(GetTableInfo());
+ return GetTableInfo()->Description.GetSchema();
}
- const TColumnTableInfo& GetTableInfoVerified() {
- AFL_VERIFY(TableInfo);
- return *TableInfo;
+ const TColumnTableInfo& GetTableInfoVerified() const {
+ AFL_VERIFY(GetTableInfo());
+ return *GetTableInfo();
}
- const NKikimrSchemeOp::TColumnDataLifeCycle& GetTableTTLProto() {
- AFL_VERIFY(TableInfo);
- return TableInfo->Description.GetTtlSettings();
+ const NKikimrSchemeOp::TColumnDataLifeCycle& GetTableTTLProto() const {
+ AFL_VERIFY(GetTableInfo());
+ return GetTableInfo()->Description.GetTtlSettings();
}
const TOlapSchema& GetTableSchemaVerified() const {
@@ -58,9 +54,6 @@ public:
return TableTTL ? &*TableTTL : nullptr;
}
- TConclusionStatus InitializeFromTableInfo(const TColumnTableInfo::TPtr& tableInfo);
-
-
};
} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp
index 172d0694b4..b94ff1888a 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.cpp
@@ -1,22 +1,11 @@
#include "update.h"
-#include "evolution.h"
#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/converter.h>
#include <ydb/core/tx/schemeshard/olap/common/common.h>
namespace NKikimr::NSchemeShard::NOlap::NAlter {
-NKikimr::TConclusion<std::shared_ptr<NKikimr::NSchemeShard::NOlap::NAlter::ISSEntityEvolution>> TStandaloneSchemaUpdate::DoBuildEvolution(const std::shared_ptr<ISSEntityUpdate>& updateSelfPtr) const {
- auto originalInfo = dynamic_pointer_cast<TStandaloneTable>(GetOriginalEntity());
- if (!originalInfo) {
- return TConclusionStatus::Fail("incorrect incoming type for patch: " + GetOriginalEntity()->GetClassName() + " in STANDALONE_UPDATE");
- }
-
- auto updateSelfCast = dynamic_pointer_cast<TStandaloneSchemaUpdate>(updateSelfPtr);
- AFL_VERIFY(!!updateSelfCast);
- return std::make_shared<TStandaloneSchemaEvolution>(OriginalStandalone, TargetStandalone, updateSelfCast);
-}
-
-NKikimr::TConclusionStatus TStandaloneSchemaUpdate::DoInitialize(const TUpdateInitializationContext& context) {
+NKikimr::TConclusionStatus TStandaloneSchemaUpdate::DoInitializeImpl(const TUpdateInitializationContext& context) {
+ const auto& originalTable = context.GetOriginalEntityAsVerified<TStandaloneTable>();
auto alter = TConverterModifyToAlter().Convert(*context.GetModification());
if (alter.IsFail()) {
return alter;
@@ -30,13 +19,15 @@ NKikimr::TConclusionStatus TStandaloneSchemaUpdate::DoInitialize(const TUpdateIn
}
AlterSchema = std::move(schemaUpdate);
}
- AlterTTL = alterCS.GetAlterTtlSettings();
+ if (alterCS.HasAlterTtlSettings()) {
+ AlterTTL = alterCS.GetAlterTtlSettings();
+ }
if (!AlterSchema && !AlterTTL) {
return TConclusionStatus::Fail("no data for update");
}
TOlapSchema originalSchema;
- originalSchema.ParseFromLocalDB(OriginalStandalone->GetTableInfoVerified().Description.GetSchema());
+ originalSchema.ParseFromLocalDB(originalTable.GetTableInfoVerified().Description.GetSchema());
TSimpleErrorCollector collector;
TOlapSchema targetSchema = originalSchema;
@@ -45,24 +36,23 @@ NKikimr::TConclusionStatus TStandaloneSchemaUpdate::DoInitialize(const TUpdateIn
return TConclusionStatus::Fail("schema update error: " + collector->GetErrorMessage() + ". in alter constructor STANDALONE_UPDATE");
}
}
- auto description = OriginalStandalone->GetTableInfoVerified().Description;
+ auto description = originalTable.GetTableInfoVerified().Description;
targetSchema.Serialize(*description.MutableSchema());
- std::optional<TOlapTTL> ttl;
+ auto ttl = originalTable.GetTableTTLOptional() ? *originalTable.GetTableTTLOptional() : TOlapTTL();
if (AlterTTL) {
- ttl = OriginalStandalone->GetTableTTLOptional() ? *OriginalStandalone->GetTableTTLOptional() : TOlapTTL();
- auto patch = ttl->Update(*AlterTTL);
+ auto patch = ttl.Update(*AlterTTL);
if (patch.IsFail()) {
return patch;
}
- if (!targetSchema.ValidateTtlSettings(ttl->GetData(), collector)) {
- return TConclusionStatus::Fail("ttl update error: " + collector->GetErrorMessage() + ". in alter constructor STANDALONE_UPDATE");
- }
- *description.MutableTtlSettings() = ttl->SerializeToProto();
+ *description.MutableTtlSettings() = ttl.SerializeToProto();
+ }
+ if (!targetSchema.ValidateTtlSettings(ttl.GetData(), collector)) {
+ return TConclusionStatus::Fail("ttl update error: " + collector->GetErrorMessage() + ". in alter constructor STANDALONE_UPDATE");
}
- auto saSharding = OriginalStandalone->GetTableInfoVerified().GetStandaloneShardingVerified();
+ auto saSharding = originalTable.GetTableInfoVerified().GetStandaloneShardingVerified();
- auto targetInfo = std::make_shared<TColumnTableInfo>(OriginalStandalone->GetTableInfoVerified().AlterVersion + 1, std::move(description), std::move(saSharding), alterCS);
- TargetStandalone = std::make_shared<TStandaloneTable>(GetOriginalEntity()->GetPathId(), targetInfo);
+ auto targetInfo = std::make_shared<TColumnTableInfo>(originalTable.GetTableInfoVerified().AlterVersion + 1, std::move(description), std::move(saSharding), alterCS);
+ TargetStandalone = std::make_shared<TStandaloneTable>(context.GetOriginalEntity().GetPathId(), targetInfo);
return TConclusionStatus::Success();
}
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.h
index 99168cc11f..3f2b62cc93 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.h
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/update.h
@@ -1,46 +1,56 @@
#pragma once
#include "object.h"
#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/common/update.h>
#include <ydb/core/tx/schemeshard/olap/schema/update.h>
#include <ydb/core/tx/schemeshard/olap/ttl/update.h>
namespace NKikimr::NSchemeShard::NOlap::NAlter {
-class TStandaloneSchemaUpdate: public ISSEntityUpdate {
+class TStandaloneSchemaUpdate: public TColumnTableUpdate {
private:
using TBase = ISSEntityUpdate;
std::optional<TOlapSchemaUpdate> AlterSchema;
std::optional<TOlapTTLUpdate> AlterTTL;
std::shared_ptr<TStandaloneTable> TargetStandalone;
- std::shared_ptr<TStandaloneTable> OriginalStandalone;
- virtual TConclusionStatus DoInitialize(const TUpdateInitializationContext& context) override;
+ virtual TConclusionStatus DoInitializeImpl(const TUpdateInitializationContext& context) override;
-public:
- TStandaloneSchemaUpdate(const std::shared_ptr<ISSEntity>& original)
- : TBase(original)
- , OriginalStandalone(dynamic_pointer_cast<TStandaloneTable>(GetOriginalEntity()))
- {
- AFL_VERIFY(!!OriginalStandalone);
+ virtual std::shared_ptr<TColumnTableInfo> GetTargetTableInfo() const override {
+ return TargetStandalone->GetTableInfoPtrVerified();
}
- void FillToShardTx(NKikimrTxColumnShard::TAlterTable& shardAlter, const std::shared_ptr<TStandaloneTable>& schema) const {
- if (AlterSchema) {
- *shardAlter.MutableSchema() = schema->GetTableSchemaProto();
- }
- if (AlterTTL) {
- *shardAlter.MutableTtlSettings() = schema->GetTableTTLProto();
- }
+ virtual std::shared_ptr<ISSEntity> GetTargetSSEntity() const override {
+ return TargetStandalone;
+ }
+
+ virtual std::set<ui64> DoGetShardIds() const override {
+ return TargetStandalone->GetTableInfoVerified().GetShardIdsSet();
}
- const TOlapSchemaUpdate* GetAlterSchemaOptional() const {
- return AlterSchema ? &*AlterSchema : nullptr;
+ virtual NKikimrTxColumnShard::ETransactionKind GetShardTransactionKind() const override {
+ return NKikimrTxColumnShard::ETransactionKind::TX_KIND_SCHEMA;
}
- const TOlapTTLUpdate* GetAlterTTLOptional() const {
- return AlterTTL ? &*AlterTTL : nullptr;
+ virtual TString DoGetShardTxBodyString(const ui64 /*tabletId*/, const TMessageSeqNo& seqNo) const override {
+ NKikimrTxColumnShard::TSchemaTxBody result;
+ result.MutableSeqNo()->SetGeneration(seqNo.Generation);
+ result.MutableSeqNo()->SetRound(seqNo.Round);
+
+ auto& alter = *result.MutableAlterTable();
+ FillToShardTx(alter);
+ alter.SetPathId(TargetStandalone->GetPathId().LocalPathId);
+ return result.SerializeAsString();
}
- virtual TConclusion<std::shared_ptr<ISSEntityEvolution>> DoBuildEvolution(const std::shared_ptr<ISSEntityUpdate>& updateSelfPtr) const override;
+ void FillToShardTx(NKikimrTxColumnShard::TAlterTable& shardAlter) const {
+ if (AlterSchema) {
+ *shardAlter.MutableSchema() = TargetStandalone->GetTableSchemaProto();
+ }
+ if (AlterTTL) {
+ *shardAlter.MutableTtlSettings() = TargetStandalone->GetTableTTLProto();
+ }
+ }
+public:
};
} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/ya.make
index be763ef9e5..808037e757 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/standalone/ya.make
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/standalone/ya.make
@@ -3,7 +3,6 @@ LIBRARY()
SRCS(
object.cpp
update.cpp
- evolution.cpp
)
PEERDIR(
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/ya.make b/ydb/core/tx/schemeshard/olap/operations/alter/ya.make
index c4cb68a39c..9408ce658e 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter/ya.make
+++ b/ydb/core/tx/schemeshard/olap/operations/alter/ya.make
@@ -2,6 +2,7 @@ LIBRARY()
PEERDIR(
ydb/core/tx/schemeshard/olap/operations/alter/abstract
+ ydb/core/tx/schemeshard/olap/operations/alter/common
ydb/core/tx/schemeshard/olap/operations/alter/in_store
ydb/core/tx/schemeshard/olap/operations/alter/standalone
)
diff --git a/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp b/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp
index cd1eb3f973..fa7eb0f7b1 100644
--- a/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp
+++ b/ydb/core/tx/schemeshard/olap/operations/alter_table.cpp
@@ -1,6 +1,5 @@
#include "alter/abstract/object.h"
#include "alter/abstract/update.h"
-#include "alter/abstract/evolution.h"
#include <ydb/core/tx/schemeshard/schemeshard__operation_part.h>
#include <ydb/core/tx/schemeshard/schemeshard__operation_common.h>
#include <ydb/core/tx/schemeshard/schemeshard_impl.h>
@@ -44,34 +43,27 @@ public:
TPath path = TPath::Init(pathId, context.SS);
TString pathString = path.PathString();
- auto tableInfo = context.SS->ColumnTables.GetVerifiedPtr(pathId);
- std::shared_ptr<ISSEntityEvolution> evolution;
- {
- TEvolutionInitializationContext eContext(&context);
- evolution = tableInfo->BuildEvolution(pathId, eContext).DetachResult();
- }
- TColumnTableInfo::TPtr alterData = tableInfo->AlterData;
- Y_ABORT_UNLESS(alterData);
+ std::shared_ptr<ISSEntity> originalEntity = ISSEntity::GetEntityVerified(context, path);
+ TUpdateRestoreContext urContext(originalEntity.get(), &context, (ui64)OperationId.GetTxId());
+ std::shared_ptr<ISSEntityUpdate> update = originalEntity->RestoreUpdateVerified(urContext);
TSimpleErrorCollector errors;
TEntityInitializationContext iContext(&context);
txState->ClearShardsInProgress();
- NKikimrTxColumnShard::TSchemaTxBody tx;
auto seqNo = context.SS->StartRound(*txState);
- context.SS->FillSeqNo(tx, seqNo);
for (auto& shard : txState->Shards) {
const TTabletId tabletId = context.SS->ShardInfos[shard.Idx].TabletID;
AFL_VERIFY(shard.TabletType == ETabletType::ColumnShard);
- auto txShard = evolution->GetShardTxBody(tx, (ui64)tabletId);
+ auto txShardString = update->GetShardTxBodyString((ui64)tabletId, seqNo);
auto event = std::make_unique<TEvColumnShard::TEvProposeTransaction>(
- evolution->GetShardTransactionKind(),
+ update->GetShardTransactionKind(),
context.SS->TabletID(),
context.Ctx.SelfID,
ui64(OperationId.GetTxId()),
- txShard.SerializeAsString(),
- context.SS->SelectProcessingParams(txState->TargetPathId));
+ txShardString,
+ context.SS->SelectProcessingParams(txState->TargetPathId), seqNo);
context.OnComplete.BindMsgToPipe(OperationId, tabletId, shard.Idx, event.release());
@@ -118,12 +110,16 @@ public:
TPathId pathId = txState->TargetPathId;
TPathElement::TPtr path = context.SS->PathsById.at(pathId);
+ TPath objPath = TPath::Init(pathId, context.SS);
NIceDb::TNiceDb db(context.GetDB());
- auto tableInfo = context.SS->ColumnTables.TakeAlterVerified(pathId);
- context.SS->PersistColumnTableAlterRemove(db, pathId);
- context.SS->PersistColumnTable(db, pathId, *tableInfo);
+ std::shared_ptr<ISSEntity> originalEntity = ISSEntity::GetEntityVerified(context, objPath);
+ TUpdateRestoreContext urContext(originalEntity.get(), &context, (ui64)OperationId.GetTxId());
+ std::shared_ptr<ISSEntityUpdate> update = originalEntity->RestoreUpdateVerified(urContext);
+
+ TUpdateFinishContext fContext(&objPath, &context, &db, NKikimr::NOlap::TSnapshot(ev->Get()->StepId, ev->Get()->TxId));
+ update->Finish(fContext).Validate();
auto parentDir = context.SS->PathsById.at(path->ParentPathId);
if (parentDir->IsLikeDirectory()) {
@@ -184,7 +180,7 @@ public:
}
bool HandleReply(TEvColumnShard::TEvNotifyTxCompletionResult::TPtr& ev, TOperationContext& context) override {
- TTxState* txState = context.SS->FindTxSafe(OperationId, TTxState::TxAlterColumnTable);
+ TTxState* txState = context.SS->FindTxSafe(OperationId, TTxState::TxAlterColumnTable);
auto shardId = TTabletId(ev->Get()->Record.GetOrigin());
auto shardIdx = context.SS->MustGetShardIdx(shardId);
Y_ABORT_UNLESS(context.SS->ShardInfos.contains(shardIdx));
@@ -295,32 +291,14 @@ public:
}
}
- auto tableInfo = context.SS->ColumnTables.GetVerifiedPtr(path.Base()->PathId);
-
- if (tableInfo->AlterVersion == 0) {
- result->SetError(NKikimrScheme::StatusMultipleModifications, "Table is not created yet");
- return result;
- }
- if (tableInfo->AlterData) {
- result->SetError(NKikimrScheme::StatusMultipleModifications, "There's another Alter in flight");
- return result;
- }
TProposeErrorCollector errors(*result);
- std::shared_ptr<ISSEntity> originalEntity;
- {
- TEntityInitializationContext iContext(&context);
- auto conclusion = tableInfo->BuildEntity(path.Base()->PathId, iContext);
- if (conclusion.IsFail()) {
- errors.AddError(conclusion.GetErrorMessage());
- return result;
- }
- originalEntity = conclusion.DetachResult();
- }
+ TEntityInitializationContext iContext(&context);
+ std::shared_ptr<ISSEntity> originalEntity = ISSEntity::GetEntityVerified(context, path);
std::shared_ptr<ISSEntityUpdate> update;
{
- TUpdateInitializationContext uContext(&context, &Transaction);
- TConclusion<std::shared_ptr<ISSEntityUpdate>> conclusion = originalEntity->CreateUpdate(uContext, originalEntity);
+ TUpdateInitializationContext uContext(&*originalEntity, &context, &Transaction, (ui64)OperationId.GetTxId());
+ TConclusion<std::shared_ptr<ISSEntityUpdate>> conclusion = originalEntity->CreateUpdate(uContext);
if (conclusion.IsFail()) {
errors.AddError(conclusion.GetErrorMessage());
return result;
@@ -328,16 +306,6 @@ public:
update = conclusion.DetachResult();
}
- std::shared_ptr<ISSEntityEvolution> evolution;
- {
- auto conclusion = update->BuildEvolution(update);
- if (conclusion.IsFail()) {
- errors.AddError(conclusion.GetErrorMessage());
- return result;
- }
- evolution = conclusion.DetachResult();
- }
-
TString errStr;
if (!context.SS->CheckApplyIf(Transaction, errStr)) {
result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr);
@@ -346,39 +314,61 @@ public:
NIceDb::TNiceDb db(context.GetDB());
- TTxState& txState = context.SS->CreateTx(OperationId, TTxState::TxAlterColumnTable, path->PathId);
- txState.State = TTxState::ConfigureParts;
+ if (update->GetShardIds().size()) {
+ TTxState& txState = context.SS->CreateTx(OperationId, TTxState::TxAlterColumnTable, path->PathId);
+ txState.State = TTxState::ConfigureParts;
- // TODO: we need to know all shards where this table is currently active
- for (ui64 columnShardId : tableInfo->GetColumnShards()) {
- auto tabletId = TTabletId(columnShardId);
- auto shardIdx = context.SS->TabletIdToShardIdx.at(tabletId);
+ for (ui64 columnShardId : update->GetShardIds()) {
+ auto tabletId = TTabletId(columnShardId);
+ auto shardIdx = context.SS->TabletIdToShardIdx.at(tabletId);
- Y_VERIFY_S(context.SS->ShardInfos.contains(shardIdx), "Unknown shardIdx " << shardIdx);
- txState.Shards.emplace_back(shardIdx, context.SS->ShardInfos[shardIdx].TabletType, TTxState::ConfigureParts);
+ Y_VERIFY_S(context.SS->ShardInfos.contains(shardIdx), "Unknown shardIdx " << shardIdx);
+ txState.Shards.emplace_back(shardIdx, context.SS->ShardInfos[shardIdx].TabletType, TTxState::ConfigureParts);
- context.SS->ShardInfos[shardIdx].CurrentTxId = OperationId.GetTxId();
- context.SS->PersistShardTx(db, shardIdx, OperationId.GetTxId());
- }
+ context.SS->ShardInfos[shardIdx].CurrentTxId = OperationId.GetTxId();
+ context.SS->PersistShardTx(db, shardIdx, OperationId.GetTxId());
+ }
- path->LastTxId = OperationId.GetTxId();
- path->PathState = TPathElement::EPathState::EPathStateAlter;
- context.SS->PersistLastTxId(db, path.Base());
+ path->LastTxId = OperationId.GetTxId();
+ path->PathState = TPathElement::EPathState::EPathStateAlter;
+ context.SS->PersistLastTxId(db, path.Base());
- {
- TEvolutionStartContext startContext(&path, &context, &db);
- auto status = evolution->StartEvolution(startContext);
- if (status.IsFail()) {
- errors.AddError(status.GetErrorMessage());
- return result;
+ {
+ TUpdateStartContext startContext(&path, &context, &db);
+ auto status = update->Start(startContext);
+ if (status.IsFail()) {
+ errors.AddError(status.GetErrorMessage());
+ return result;
+ }
+ }
+ context.SS->PersistTxState(db, OperationId);
+
+ context.OnComplete.ActivateTx(OperationId);
+
+ SetState(NextState());
+ } else {
+ {
+ {
+ TUpdateStartContext startContext(&path, &context, &db);
+ auto status = update->Start(startContext);
+ if (status.IsFail()) {
+ errors.AddError(status.GetErrorMessage());
+ return result;
+ }
+ }
+ {
+ TUpdateFinishContext fContext(&path, &context, &db, {});
+ auto status = update->Finish(fContext);
+ if (status.IsFail()) {
+ errors.AddError(status.GetErrorMessage());
+ return result;
+ }
+ }
}
+ result->SetStatus(NKikimrScheme::StatusSuccess);
+ SetState(TTxState::Done);
}
- context.SS->PersistTxState(db, OperationId);
-
- context.OnComplete.ActivateTx(OperationId);
-
- SetState(NextState());
return result;
}
diff --git a/ydb/core/tx/schemeshard/olap/operations/create_table.cpp b/ydb/core/tx/schemeshard/olap/operations/create_table.cpp
index b23286ef4d..908474451e 100644
--- a/ydb/core/tx/schemeshard/olap/operations/create_table.cpp
+++ b/ydb/core/tx/schemeshard/olap/operations/create_table.cpp
@@ -71,7 +71,7 @@ public:
}
}
tableInfo->AlterVersion = 1;
- auto shardingValidation = NSharding::TShardingBase::ValidateBehaviour(GetSchema(), tableInfo->Description.GetSharding());
+ auto shardingValidation = NSharding::IShardingBase::ValidateBehaviour(GetSchema(), tableInfo->Description.GetSharding());
if (shardingValidation.IsFail()) {
errors.AddError(shardingValidation.GetErrorMessage());
return nullptr;
@@ -170,7 +170,7 @@ private:
for (auto&& i : layoutConclusion->MutableTabletIds()) {
description.MutableSharding()->AddColumnShards(i);
}
- auto shardingObject = NSharding::TShardingBase::BuildFromProto(GetSchema(), description.GetSharding());
+ auto shardingObject = NSharding::IShardingBase::BuildFromProto(GetSchema(), description.GetSharding());
if (shardingObject.IsFail()) {
return shardingObject;
}
diff --git a/ydb/core/tx/schemeshard/olap/table/table.cpp b/ydb/core/tx/schemeshard/olap/table/table.cpp
index dfb54dcb18..9c90e758e0 100644
--- a/ydb/core/tx/schemeshard/olap/table/table.cpp
+++ b/ydb/core/tx/schemeshard/olap/table/table.cpp
@@ -1,17 +1,18 @@
#include "table.h"
#include <ydb/core/tx/schemeshard/olap/operations/alter/abstract/object.h>
-#include <ydb/core/tx/schemeshard/olap/operations/alter/standalone/evolution.h>
-#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/evolution.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/standalone/object.h>
+#include <ydb/core/tx/schemeshard/olap/operations/alter/in_store/object.h>
+#include <ydb/core/tx/columnshard/common/protos/snapshot.pb.h>
namespace NKikimr::NSchemeShard {
TColumnTableInfo::TColumnTableInfo(
ui64 alterVersion,
- NKikimrSchemeOp::TColumnTableDescription&& description,
+ const NKikimrSchemeOp::TColumnTableDescription& description,
TMaybe<NKikimrSchemeOp::TColumnStoreSharding>&& standaloneSharding,
TMaybe<NKikimrSchemeOp::TAlterColumnTable>&& alterBody)
: AlterVersion(alterVersion)
- , Description(std::move(description))
+ , Description(description)
, StandaloneSharding(std::move(standaloneSharding))
, AlterBody(std::move(alterBody))
{
@@ -38,18 +39,4 @@ TConclusion<std::shared_ptr<NOlap::NAlter::ISSEntity>> TColumnTableInfo::BuildEn
return result;
}
-TConclusion<std::shared_ptr<NOlap::NAlter::ISSEntityEvolution>> TColumnTableInfo::BuildEvolution(const TPathId& pathId, const NOlap::NAlter::TEvolutionInitializationContext& iContext) const {
- std::shared_ptr<NOlap::NAlter::ISSEntityEvolution> result;
- if (IsStandalone()) {
- result = std::make_shared<NOlap::NAlter::TStandaloneSchemaEvolution>(pathId);
- } else {
- result = std::make_shared<NOlap::NAlter::TInStoreSchemaEvolution>(pathId);
- }
- auto initConclusion = result->Initialize(iContext);
- if (initConclusion.IsFail()) {
- return initConclusion;
- }
- return result;
-}
-
} \ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/olap/table/table.h b/ydb/core/tx/schemeshard/olap/table/table.h
index 5f9b426e81..e37ede6c3e 100644
--- a/ydb/core/tx/schemeshard/olap/table/table.h
+++ b/ydb/core/tx/schemeshard/olap/table/table.h
@@ -2,6 +2,8 @@
#include <ydb/core/protos/flat_scheme_op.pb.h>
#include <ydb/core/tx/schemeshard/schemeshard_identificators.h>
#include <ydb/core/tx/schemeshard/schemeshard_info_types.h>
+#include <ydb/core/tx/sharding/sharding.h>
+#include <ydb/core/tx/columnshard/common/snapshot.h>
namespace NKikimr::NSchemeShard::NOlap::NAlter {
class ISSEntity;
@@ -24,6 +26,14 @@ public:
return PathIdFromPathId(Description.GetColumnStorePathId());
}
+ std::shared_ptr<NSharding::IShardingBase> GetShardingVerified(const TOlapSchema& olapSchema) const {
+ return NSharding::IShardingBase::BuildFromProto(olapSchema, Description.GetSharding()).DetachResult();
+ }
+
+ std::set<ui64> GetShardIdsSet() const {
+ return std::set<ui64>(Description.GetSharding().GetColumnShards().begin(), Description.GetSharding().GetColumnShards().end());
+ }
+
const auto& GetColumnShards() const {
return Description.GetSharding().GetColumnShards();
}
@@ -31,7 +41,6 @@ public:
void SetColumnShards(const std::vector<ui64>& columnShards) {
AFL_VERIFY(GetColumnShards().empty())("original", Description.DebugString());
AFL_VERIFY(columnShards.size());
- Description.MutableSharding()->SetVersion(1);
Description.MutableSharding()->MutableColumnShards()->Clear();
Description.MutableSharding()->MutableColumnShards()->Reserve(columnShards.size());
@@ -47,7 +56,7 @@ public:
TAggregatedStats Stats;
TColumnTableInfo() = default;
- TColumnTableInfo(ui64 alterVersion, NKikimrSchemeOp::TColumnTableDescription&& description,
+ TColumnTableInfo(ui64 alterVersion, const NKikimrSchemeOp::TColumnTableDescription& description,
TMaybe<NKikimrSchemeOp::TColumnStoreSharding>&& standaloneSharding,
TMaybe<NKikimrSchemeOp::TAlterColumnTable>&& alterBody = Nothing());
diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp
index b6f905f163..a9d8f20bfe 100644
--- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp
@@ -4067,9 +4067,6 @@ NKikimrSchemeOp::TPathVersion TSchemeShard::GetPathVersion(const TPath& path) co
result.SetColumnTableVersion(tableInfo->AlterVersion);
generalVersion += result.GetColumnTableVersion();
- result.SetColumnTableShardingVersion(tableInfo->Description.GetSharding().GetVersion());
- generalVersion += result.GetColumnTableShardingVersion();
-
if (tableInfo->Description.HasSchema()) {
result.SetColumnTableSchemaVersion(tableInfo->Description.GetSchema().GetVersion());
} else if (tableInfo->Description.HasSchemaPresetId() && tableInfo->GetOlapStorePathIdVerified()) {
@@ -4428,6 +4425,7 @@ void TSchemeShard::StateConfigure(STFUNC_SIG) {
HFuncTraced(TEvents::TEvUndelivered, Handle);
HFuncTraced(NKikimr::NOlap::NBackground::TEvExecuteGeneralLocalTransaction, Handle);
+ HFuncTraced(NKikimr::NOlap::NBackground::TEvRemoveSession, Handle);
HFuncTraced(TEvSchemeShard::TEvInitRootShard, Handle);
HFuncTraced(TEvSchemeShard::TEvInitTenantSchemeShard, Handle);
HFuncTraced(TEvSchemeShard::TEvMigrateSchemeShard, Handle);
@@ -4471,6 +4469,7 @@ void TSchemeShard::StateWork(STFUNC_SIG) {
HFuncTraced(TEvents::TEvUndelivered, Handle);
HFuncTraced(TEvSchemeShard::TEvInitRootShard, Handle);
HFuncTraced(NKikimr::NOlap::NBackground::TEvExecuteGeneralLocalTransaction, Handle);
+ HFuncTraced(NKikimr::NOlap::NBackground::TEvRemoveSession, Handle);
HFuncTraced(TEvSchemeShard::TEvMeasureSelfResponseTime, SelfPinger->Handle);
HFuncTraced(TEvSchemeShard::TEvWakeupToMeasureSelfResponseTime, SelfPinger->Handle);
@@ -5164,6 +5163,12 @@ void TSchemeShard::Handle(TEvSchemeShard::TEvInitRootShard::TPtr& ev, const TAct
Execute(CreateTxInitRootCompatibility(ev), ctx);
}
+void TSchemeShard::Handle(NKikimr::NOlap::NBackground::TEvRemoveSession::TPtr& ev, const TActorContext& ctx) {
+ auto txRemove = BackgroundSessionsManager->TxRemove(ev->Get()->GetClassName(), ev->Get()->GetIdentifier());
+ AFL_VERIFY(!!txRemove);
+ Execute(txRemove.release(), ctx);
+}
+
void TSchemeShard::Handle(NKikimr::NOlap::NBackground::TEvExecuteGeneralLocalTransaction::TPtr& ev, const TActorContext& ctx) {
Execute(ev->Get()->ExtractTransaction().release(), ctx);
}
diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h
index 82544ed169..1dcd1b994e 100644
--- a/ydb/core/tx/schemeshard/schemeshard_impl.h
+++ b/ydb/core/tx/schemeshard/schemeshard_impl.h
@@ -1011,6 +1011,8 @@ public:
static void FillTableBoundaries(const TTableInfo::TPtr tableInfo, google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TSplitBoundary>& boundaries);
void Handle(NKikimr::NOlap::NBackground::TEvExecuteGeneralLocalTransaction::TPtr& ev, const TActorContext& ctx);
+ void Handle(NKikimr::NOlap::NBackground::TEvRemoveSession::TPtr& ev, const TActorContext& ctx);
+
void Handle(TEvSchemeShard::TEvInitRootShard::TPtr &ev, const TActorContext &ctx);
void Handle(TEvSchemeShard::TEvInitTenantSchemeShard::TPtr &ev, const TActorContext &ctx);
diff --git a/ydb/core/tx/sharding/hash_intervals.cpp b/ydb/core/tx/sharding/hash_intervals.cpp
new file mode 100644
index 0000000000..7d972a00af
--- /dev/null
+++ b/ydb/core/tx/sharding/hash_intervals.cpp
@@ -0,0 +1,187 @@
+#include "hash_intervals.h"
+
+namespace NKikimr::NSharding::NConsistency {
+
+NKikimr::TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> TConsistencySharding64::DoBuildSplitShardsModifiers(const std::vector<ui64>& newTabletIds) const {
+ if (newTabletIds.size() != GetOrderedShardIds().size()) {
+ return TConclusionStatus::Fail("can multiple 2 only for add shards count");
+ }
+ if (!!SpecialShardingInfo) {
+ return TConclusionStatus::Fail("not unified shards distribution for consistency intervals modification");
+ }
+ const TSpecificShardingInfo shardingInfo = SpecialShardingInfo ? *SpecialShardingInfo : TSpecificShardingInfo(GetOrderedShardIds());
+ std::vector<NKikimrSchemeOp::TAlterShards> result;
+ {
+ ui32 idx = 0;
+ for (auto&& i : GetOrderedShardIds()) {
+ {
+ NKikimrSchemeOp::TAlterShards alter;
+ auto specSharding = shardingInfo.GetShardingTabletVerified(i);
+ specSharding.SetTabletId(newTabletIds[idx]);
+ specSharding.CutHalfIntervalFromStart();
+ alter.MutableModification()->AddOpenWriteIds(newTabletIds[idx]);
+ *alter.MutableModification()->MutableConsistency()->AddShards() = specSharding.SerializeToProto();
+ result.emplace_back(alter);
+ }
+ {
+ NKikimrSchemeOp::TAlterShards alter;
+ auto& transfer = *alter.MutableTransfer()->AddTransfers();
+ transfer.SetDestinationTabletId(newTabletIds[idx]);
+ transfer.AddSourceTabletIds(i);
+ result.emplace_back(alter);
+ }
+ {
+ NKikimrSchemeOp::TAlterShards alter;
+ alter.MutableModification()->AddOpenReadIds(newTabletIds[idx]);
+ auto specSharding = shardingInfo.GetShardingTabletVerified(i);
+ specSharding.CutHalfIntervalToEnd();
+ *alter.MutableModification()->MutableConsistency()->AddShards() = specSharding.SerializeToProto();
+ result.emplace_back(alter);
+ }
+ ++idx;
+ }
+ }
+ return result;
+}
+
+NKikimr::TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> TConsistencySharding64::DoBuildMergeShardsModifiers(const std::vector<ui64>& newTabletIds) const {
+ if (newTabletIds.size() * 2 != GetOrderedShardIds().size()) {
+ return TConclusionStatus::Fail("can div 2 only for reduce shards count");
+ }
+ if (!!SpecialShardingInfo) {
+ return TConclusionStatus::Fail("not unified shards distribution for consistency intervals modification");
+ }
+ const TSpecificShardingInfo shardingInfo = SpecialShardingInfo ? *SpecialShardingInfo : TSpecificShardingInfo(GetOrderedShardIds());
+ std::vector<NKikimrSchemeOp::TAlterShards> result;
+ {
+ ui32 idx = 0;
+ for (auto&& i : newTabletIds) {
+ const ui64 from1 = GetOrderedShardIds()[2 * idx];
+ const ui64 from2 = GetOrderedShardIds()[2 * idx + 1];
+ auto source1 = shardingInfo.GetShardingTabletVerified(from1);
+ auto source2 = shardingInfo.GetShardingTabletVerified(from2);
+ AFL_VERIFY(source1.GetHashIntervalRightOpened() == source2.GetHashIntervalLeftClosed());
+ {
+ NKikimrSchemeOp::TAlterShards alter;
+ TSpecificShardingInfo::TConsistencyShardingTablet newInterval(i, source1.GetHashIntervalLeftClosed(), source2.GetHashIntervalRightOpened());
+ alter.MutableModification()->AddOpenWriteIds(i);
+ *alter.MutableModification()->MutableConsistency()->AddShards() = newInterval.SerializeToProto();
+ result.emplace_back(alter);
+ }
+ {
+ NKikimrSchemeOp::TAlterShards alter;
+ auto& transfer = *alter.MutableTransfer()->AddTransfers();
+ transfer.SetDestinationTabletId(newTabletIds[idx]);
+ transfer.AddSourceTabletIds(from1);
+ transfer.AddSourceTabletIds(from2);
+ result.emplace_back(alter);
+ }
+ {
+ NKikimrSchemeOp::TAlterShards alter;
+ alter.MutableModification()->AddOpenReadIds(i);
+ alter.MutableModification()->AddCloseWriteIds(from1);
+ alter.MutableModification()->AddCloseWriteIds(from2);
+ alter.MutableModification()->AddCloseReadIds(from1);
+ alter.MutableModification()->AddCloseReadIds(from2);
+ result.emplace_back(alter);
+ }
+ ++idx;
+ }
+ }
+ return result;
+}
+
+NKikimr::TConclusionStatus TConsistencySharding64::DoApplyModification(const NKikimrSchemeOp::TShardingModification& proto) {
+ if (!proto.HasConsistency()) {
+ return TConclusionStatus::Success();
+ }
+
+ for (auto&& i : proto.GetConsistency().GetShards()) {
+ TSpecificShardingInfo::TConsistencyShardingTablet info;
+ {
+ auto conclusion = info.DeserializeFromProto(i);
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ }
+ if (!UpdateShardInfo(info)) {
+ return TConclusionStatus::Fail("no shard with same id on update consistency interval");
+ }
+ }
+ return TConclusionStatus::Success();
+}
+
+NKikimr::TConclusionStatus TConsistencySharding64::DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) {
+ auto conclusion = TBase::DoDeserializeFromProto(proto);
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ if (!proto.HasHashSharding()) {
+ return TConclusionStatus::Fail("no data for consistency sharding");
+ }
+ AFL_VERIFY(proto.GetHashSharding().GetFunction() == NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CONSISTENCY_64);
+ {
+ TSpecificShardingInfo specSharding;
+ auto parseResult = specSharding.DeserializeFromProto(proto, GetClosedWritingShardIds(), GetClosedReadingShardIds());
+ if (parseResult.IsFail()) {
+ return parseResult;
+ }
+ if (!specSharding.IsEmpty()) {
+ SpecialShardingInfo = std::move(specSharding);
+ }
+ }
+ return TConclusionStatus::Success();
+}
+
+NKikimr::TConclusionStatus TConsistencySharding64::DoOnAfterModification() {
+ AFL_VERIFY(!!SpecialShardingInfo);
+ auto result = SpecialShardingInfo->BuildActivityIndex(GetClosedWritingShardIds(), GetClosedReadingShardIds());
+ if (result.IsFail()) {
+ return result;
+ }
+
+ std::vector<ui64> shardIds;
+ if (SpecialShardingInfo->CheckUnifiedDistribution(GetOrderedShardIds().size(), shardIds)) {
+ SetOrderedShardIds(shardIds);
+ SpecialShardingInfo.reset();
+ }
+
+ return TConclusionStatus::Success();
+}
+
+THashMap<ui64, std::vector<ui32>> TConsistencySharding64::MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const {
+ std::vector<ui64> hashes = MakeHashes(batch);
+
+ if (!SpecialShardingInfo) {
+ std::vector<std::vector<ui32>> result;
+ result.resize(GetShardsCount());
+ for (auto&& i : result) {
+ i.reserve(hashes.size());
+ }
+ ui32 idx = 0;
+ for (auto&& hash : hashes) {
+ result[std::min<ui32>(hash / (Max<ui64>() / result.size()), result.size() - 1)].emplace_back(idx);
+ ++idx;
+ }
+ THashMap<ui64, std::vector<ui32>> resultHash;
+ for (ui32 i = 0; i < result.size(); ++i) {
+ if (result[i].size()) {
+ resultHash.emplace(GetShardIdByOrderIdx(i), std::move(result[i]));
+ }
+ }
+ return resultHash;
+ } else {
+ return SpecialShardingInfo->MakeShardingWrite(hashes);
+ }
+}
+
+std::shared_ptr<NKikimr::NSharding::IGranuleShardingLogic> TConsistencySharding64::DoGetTabletShardingInfoOptional(const ui64 tabletId) const {
+ if (SpecialShardingInfo) {
+ return std::make_shared<TGranuleSharding>(GetShardingColumns(), SpecialShardingInfo->GetShardingTabletVerified(tabletId));
+ } else {
+ TSpecificShardingInfo info(GetOrderedShardIds());
+ return std::make_shared<TGranuleSharding>(GetShardingColumns(), info.GetShardingTabletVerified(tabletId));
+ }
+}
+
+}
diff --git a/ydb/core/tx/sharding/hash_intervals.h b/ydb/core/tx/sharding/hash_intervals.h
new file mode 100644
index 0000000000..0f11e36628
--- /dev/null
+++ b/ydb/core/tx/sharding/hash_intervals.h
@@ -0,0 +1,385 @@
+#pragma once
+#include "hash_sharding.h"
+#include <ydb/core/protos/flat_scheme_op.pb.h>
+
+namespace NKikimr::NSharding::NConsistency {
+
+class TSpecificShardingInfo {
+public:
+ class TConsistencyShardingTablet {
+ private:
+ YDB_ACCESSOR(ui64, TabletId, 0);
+ YDB_READONLY(ui64, HashIntervalLeftClosed, 0);
+ YDB_READONLY(ui64, HashIntervalRightOpened, Max<ui64>());
+ public:
+ TConsistencyShardingTablet() = default;
+ TConsistencyShardingTablet(const ui64 tabletId, const ui64 hashIntervalLeftClosed, const ui64 hashIntervalRightOpened)
+ : TabletId(tabletId)
+ , HashIntervalLeftClosed(hashIntervalLeftClosed)
+ , HashIntervalRightOpened(hashIntervalRightOpened) {
+ AFL_VERIFY(HashIntervalLeftClosed < HashIntervalRightOpened);
+ }
+
+ void CutHalfIntervalFromStart() {
+ const ui64 toHalf = 0.5 * HashIntervalRightOpened;
+ const ui64 fromHalf = 0.5 * HashIntervalLeftClosed;
+ HashIntervalRightOpened = toHalf + fromHalf;
+ }
+
+ void CutHalfIntervalToEnd() {
+ const ui64 toHalf = 0.5 * HashIntervalRightOpened;
+ const ui64 fromHalf = 0.5 * HashIntervalLeftClosed;
+ HashIntervalLeftClosed = toHalf + fromHalf;
+ }
+
+ NKikimrSchemeOp::TConsistencyShardingTablet SerializeToProto() const {
+ NKikimrSchemeOp::TConsistencyShardingTablet result;
+ result.SetTabletId(TabletId);
+ result.SetHashIntervalLeftClosed(HashIntervalLeftClosed);
+ result.SetHashIntervalRightOpened(HashIntervalRightOpened);
+ return result;
+ }
+
+ TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TConsistencyShardingTablet& proto) {
+ TabletId = proto.GetTabletId();
+ HashIntervalLeftClosed = proto.GetHashIntervalLeftClosed();
+ HashIntervalRightOpened = proto.GetHashIntervalRightOpened();
+ AFL_VERIFY(HashIntervalLeftClosed < HashIntervalRightOpened);
+ return TConclusionStatus::Success();
+ }
+
+ bool operator<(const TConsistencyShardingTablet& item) const {
+ if (HashIntervalLeftClosed == item.HashIntervalLeftClosed) {
+ return HashIntervalRightOpened < item.HashIntervalRightOpened;
+ } else {
+ return HashIntervalLeftClosed < item.HashIntervalLeftClosed;
+ }
+ }
+ };
+
+private:
+ bool IndexConstructed = false;
+ std::vector<TConsistencyShardingTablet> SpecialSharding;
+ std::vector<TConsistencyShardingTablet*> ActiveWriteSpecialSharding;
+ std::vector<TConsistencyShardingTablet*> ActiveReadSpecialSharding;
+ ui64 GetUnifiedDistributionBorder(const ui32 idx, const ui64 shardsCount) const {
+ AFL_VERIFY(idx <= shardsCount);
+ if (idx == shardsCount) {
+ return Max<ui64>();
+ }
+ return Max<ui64>() * (1.0 * idx / shardsCount);
+ }
+
+ TConclusionStatus CheckIntervalsFilling() const {
+ {
+ ui64 currentPos = 0;
+ for (auto&& i : ActiveReadSpecialSharding) {
+ if (currentPos < i->GetHashIntervalLeftClosed()) {
+ return TConclusionStatus::Fail("sharding special intervals not covered (reading) full ui64 line");
+ } else if (currentPos > i->GetHashIntervalLeftClosed()) {
+ return TConclusionStatus::Fail("sharding intervals covered twice for reading full ui64 line");
+ }
+ currentPos = i->GetHashIntervalRightOpened();
+ }
+ if (currentPos != Max<ui64>()) {
+ return TConclusionStatus::Fail("sharding special intervals not covered (reading) full ui64 line (final segment)");
+ }
+ }
+ {
+ ui64 currentPos = 0;
+ for (auto&& i : ActiveWriteSpecialSharding) {
+ if (currentPos < i->GetHashIntervalLeftClosed()) {
+ return TConclusionStatus::Fail("sharding special intervals not covered (writing) full ui64 line");
+ }
+ currentPos = std::max<ui64>(currentPos, i->GetHashIntervalRightOpened());
+ }
+ if (currentPos != Max<ui64>()) {
+ return TConclusionStatus::Fail("sharding special intervals not covered (writing) full ui64 line (final segment)");
+ }
+ }
+ return TConclusionStatus::Success();
+ }
+
+public:
+ bool IsEmpty() const {
+ return SpecialSharding.empty();
+ }
+
+ TSpecificShardingInfo() = default;
+
+ TSpecificShardingInfo(const std::vector<ui64>& shardIds) {
+ for (ui32 i = 0; i < shardIds.size(); ++i) {
+ const ui64 start = GetUnifiedDistributionBorder(i, shardIds.size());
+ const ui64 finish = GetUnifiedDistributionBorder(i + 1, shardIds.size());
+ TConsistencyShardingTablet info(shardIds[i], start, finish);
+ SpecialSharding.emplace_back(info);
+ }
+ BuildActivityIndex(Default<std::set<ui64>>(), Default<std::set<ui64>>()).Validate();
+ }
+
+ THashMap<ui64, std::vector<ui32>> MakeShardingWrite(const std::vector<ui64> hashes) const {
+ AFL_VERIFY(IndexConstructed);
+ std::vector<std::vector<ui32>> result;
+ result.resize(ActiveWriteSpecialSharding.size());
+ for (auto&& i : result) {
+ i.reserve(hashes.size());
+ }
+ ui32 idxRecord = 0;
+ for (auto&& i : hashes) {
+ ui32 idxShard = 0;
+ bool found = false;
+ for (auto&& s : ActiveWriteSpecialSharding) {
+ if (s->GetHashIntervalLeftClosed() > i || i >= s->GetHashIntervalRightOpened()) {
+ break;
+ }
+ result[idxShard].emplace_back(idxRecord);
+ found = true;
+ ++idxShard;
+ }
+ AFL_VERIFY(found);
+ ++idxRecord;
+ }
+ THashMap<ui64, std::vector<ui32>> resultHash;
+ for (ui32 i = 0; i < result.size(); ++i) {
+ if (result[i].size()) {
+ resultHash.emplace(ActiveWriteSpecialSharding[i]->GetTabletId(), std::move(result[i]));
+ }
+ }
+ return resultHash;
+ }
+
+ void SerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const {
+ AFL_VERIFY(IndexConstructed);
+ for (auto&& i : SpecialSharding) {
+ *proto.MutableHashSharding()->AddTabletsForConsistency() = i.SerializeToProto();
+ }
+ }
+
+ TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto, const std::set<ui64>& closedForWrite, const std::set<ui64>& closedForRead) {
+ for (auto&& i : proto.GetHashSharding().GetTabletsForConsistency()) {
+ TConsistencyShardingTablet info;
+ auto conclusion = info.DeserializeFromProto(i);
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ SpecialSharding.emplace_back(std::move(info));
+ }
+ IndexConstructed = false;
+ return BuildActivityIndex(closedForWrite, closedForRead);
+ }
+
+ [[nodiscard]] TConclusionStatus BuildActivityIndex(const std::set<ui64>& closedForWrite, const std::set<ui64>& closedForRead) {
+ std::sort(SpecialSharding.begin(), SpecialSharding.end());
+ ActiveWriteSpecialSharding.clear();
+ ActiveReadSpecialSharding.clear();
+ if (SpecialSharding.empty()) {
+ return TConclusionStatus::Success();
+ }
+ for (auto&& i : SpecialSharding) {
+ if (!closedForWrite.contains(i.GetTabletId())) {
+ ActiveWriteSpecialSharding.emplace_back(&i);
+ }
+ if (!closedForRead.contains(i.GetTabletId())) {
+ ActiveReadSpecialSharding.emplace_back(&i);
+ }
+ }
+ auto result = CheckIntervalsFilling();
+ IndexConstructed = result.IsSuccess();
+ return result;
+ }
+
+ const TConsistencyShardingTablet& GetShardingTabletVerified(const ui64 tabletId) const {
+ for (auto&& i : SpecialSharding) {
+ if (i.GetTabletId() == tabletId) {
+ return i;
+ }
+ }
+ AFL_VERIFY(false);
+ return Default<TConsistencyShardingTablet>();
+ }
+
+ bool CheckUnifiedDistribution(const ui32 originalShardsCount, std::vector<ui64>& orderedShardIds) {
+ if (originalShardsCount != SpecialSharding.size()) {
+ return false;
+ }
+ AFL_VERIFY(IndexConstructed);
+ std::set<ui64> activeReadTabletIds;
+ for (auto&& i : ActiveReadSpecialSharding) {
+ activeReadTabletIds.emplace(i->GetTabletId());
+ }
+ std::set<ui64> activeWriteTabletIds;
+ for (auto&& i : ActiveWriteSpecialSharding) {
+ activeWriteTabletIds.emplace(i->GetTabletId());
+ }
+ std::set<ui64> tabletIds;
+ for (auto&& i : SpecialSharding) {
+ tabletIds.emplace(i.GetTabletId());
+ }
+ if (activeReadTabletIds != tabletIds || activeWriteTabletIds != tabletIds) {
+ return false;
+ }
+ ui32 idx = 0;
+ std::vector<ui64> shardIdsCorrectOrder;
+ for (auto&& i : SpecialSharding) {
+ const ui64 start = GetUnifiedDistributionBorder(idx, SpecialSharding.size());
+ const ui64 finish = GetUnifiedDistributionBorder(idx + 1, SpecialSharding.size());
+ if (i.GetHashIntervalLeftClosed() != start || i.GetHashIntervalRightOpened() != finish) {
+ return false;
+ }
+ shardIdsCorrectOrder.emplace_back(i.GetTabletId());
+ ++idx;
+ }
+ orderedShardIds = shardIdsCorrectOrder;
+ return true;
+ }
+
+ bool UpdateShardInfo(const TConsistencyShardingTablet& info) {
+ for (auto&& i : SpecialSharding) {
+ if (i.GetTabletId() == info.GetTabletId()) {
+ i = info;
+ IndexConstructed = false;
+ return true;
+ }
+ }
+ return false;
+ }
+
+ void AddShardInfo(const TConsistencyShardingTablet& info) {
+ for (auto&& i : SpecialSharding) {
+ AFL_VERIFY(i.GetTabletId() != info.GetTabletId());
+ }
+ SpecialSharding.emplace_back(info);
+ IndexConstructed = false;
+ }
+
+};
+
+class TConsistencySharding64: public THashShardingImpl {
+public:
+ static TString GetClassNameStatic() {
+ return "CONSISTENCY";
+ }
+private:
+ using TBase = THashShardingImpl;
+
+ std::optional<TSpecificShardingInfo> SpecialShardingInfo;
+
+ virtual TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> DoBuildSplitShardsModifiers(const std::vector<ui64>& newTabletIds) const override;
+ virtual TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> DoBuildMergeShardsModifiers(const std::vector<ui64>& newTabletIds) const override;
+
+ bool UpdateShardInfo(const TSpecificShardingInfo::TConsistencyShardingTablet& info) {
+ GetShardInfoVerified(info.GetTabletId()).IncrementVersion();
+ AFL_VERIFY(!!SpecialShardingInfo);
+ if (SpecialShardingInfo->UpdateShardInfo(info)) {
+ return true;
+ }
+ for (auto&& i : GetOrderedShardIds()) {
+ if (i == info.GetTabletId()) {
+ SpecialShardingInfo->AddShardInfo(info);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ virtual std::shared_ptr<IGranuleShardingLogic> DoGetTabletShardingInfoOptional(const ui64 tabletId) const override;
+
+ virtual TConclusionStatus DoOnBeforeModification() override {
+ if (!SpecialShardingInfo) {
+ AFL_VERIFY(!HasReadClosedShards() && !HasWriteClosedShards());
+ SpecialShardingInfo = TSpecificShardingInfo(GetOrderedShardIds());
+ }
+ return TConclusionStatus::Success();
+ }
+
+ virtual TConclusionStatus DoApplyModification(const NKikimrSchemeOp::TShardingModification& proto) override;
+ virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override {
+ TBase::DoSerializeToProto(proto);
+ if (SpecialShardingInfo) {
+ SpecialShardingInfo->SerializeToProto(proto);
+ }
+ proto.MutableHashSharding()->SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CONSISTENCY_64);
+ }
+ virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) override;
+
+ virtual TConclusionStatus DoOnAfterModification() override;
+
+ virtual std::set<ui64> DoGetModifiedShardIds(const NKikimrSchemeOp::TShardingModification& proto) const override {
+ std::set<ui64> result;
+ for (auto&& i : proto.GetConsistency().GetShards()) {
+ result.emplace(i.GetTabletId());
+ }
+ return result;
+ }
+public:
+ using TBase::TBase;
+
+ TConsistencySharding64() = default;
+
+ TConsistencySharding64(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui64 seed = 0)
+ : TBase(shardIds, columnNames, seed) {
+ }
+
+ virtual THashMap<ui64, std::vector<ui32>> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override;
+
+ virtual TString GetClassName() const override {
+ return GetClassNameStatic();
+ }
+};
+
+class TGranuleSharding: public THashGranuleSharding {
+public:
+ static TString GetClassNameStatic() {
+ return "CONSISTENCY";
+ }
+private:
+ using TBase = THashGranuleSharding;
+ TSpecificShardingInfo::TConsistencyShardingTablet Interval;
+ static const inline TFactory::TRegistrator<TGranuleSharding> Registrator = TFactory::TRegistrator<TGranuleSharding>(GetClassNameStatic());
+protected:
+ virtual std::shared_ptr<NArrow::TColumnFilter> DoGetFilter(const std::shared_ptr<arrow::Table>& table) const override {
+ const std::vector<ui64> hashes = CalcHashes(table);
+ auto result = std::make_shared<NArrow::TColumnFilter>(NArrow::TColumnFilter::BuildAllowFilter());
+ const auto getter = [&](const ui64 index) {
+ const ui64 hash = hashes[index];
+ return Interval.GetHashIntervalLeftClosed() <= hash && hash < Interval.GetHashIntervalRightOpened();
+ };
+ result->ResetWithLambda(hashes.size(), getter);
+ return result;
+
+ }
+ virtual void DoSerializeToProto(TProto& proto) const override {
+ *proto.MutableConsistency()->MutableHashing() = TBase::SerializeHashingToProto();
+ *proto.MutableConsistency()->MutableShardInfo() = Interval.SerializeToProto();
+ }
+ virtual TConclusionStatus DoDeserializeFromProto(const TProto& proto) override {
+ {
+ auto conclusion = TBase::DeserializeHashingFromProto(proto.GetConsistency().GetHashing());
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ }
+ {
+ auto conclusion = Interval.DeserializeFromProto(proto.GetConsistency().GetShardInfo());
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ }
+
+ return TConclusionStatus::Success();
+ }
+public:
+ TGranuleSharding() = default;
+
+ TGranuleSharding(const std::vector<TString>& columnNames, const TSpecificShardingInfo::TConsistencyShardingTablet& interval)
+ : TBase(columnNames)
+ , Interval(interval) {
+
+ }
+
+ virtual TString GetClassName() const override {
+ return GetClassNameStatic();
+ }
+};
+
+}
diff --git a/ydb/core/tx/sharding/hash_modulo.cpp b/ydb/core/tx/sharding/hash_modulo.cpp
new file mode 100644
index 0000000000..09dc844e65
--- /dev/null
+++ b/ydb/core/tx/sharding/hash_modulo.cpp
@@ -0,0 +1,208 @@
+#include "hash_modulo.h"
+
+namespace NKikimr::NSharding::NModulo {
+
+THashMap<ui64, std::vector<ui32>> THashShardingModuloN::MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const {
+ const std::vector<ui64> hashes = MakeHashes(batch);
+ if (!SpecialShardingInfo) {
+ THashMap<ui64, std::vector<ui32>> resultHash;
+ std::vector<std::vector<ui32>> result;
+ result.resize(GetShardsCount());
+ for (auto&& i : result) {
+ i.reserve(hashes.size());
+ }
+ ui32 idx = 0;
+ for (auto&& i : hashes) {
+ result[i % GetShardsCount()].emplace_back(idx++);
+ }
+ for (ui32 i = 0; i < result.size(); ++i) {
+ if (result[i].size()) {
+ resultHash[GetOrderedShardIds()[i]] = std::move(result[i]);
+ }
+ }
+ return resultHash;
+ } else {
+ return SpecialShardingInfo->MakeShardingWrite(hashes);
+ }
+}
+
+NKikimr::TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> THashShardingModuloN::DoBuildSplitShardsModifiers(const std::vector<ui64>& newTabletIds) const {
+ if (newTabletIds.size() != GetOrderedShardIds().size()) {
+ return TConclusionStatus::Fail("can multiple 2 only for add shards count");
+ }
+ if (!!SpecialShardingInfo) {
+ return TConclusionStatus::Fail("not unified shards distribution for module");
+ }
+ TSpecificShardingInfo info(GetOrderedShardIds());
+ std::vector<NKikimrSchemeOp::TAlterShards> result;
+ {
+ NKikimrSchemeOp::TAlterShards alter;
+ alter.MutableModification()->MutableModulo()->SetPartsCount(2 * GetOrderedShardIds().size());
+ for (auto&& i : GetOrderedShardIds()) {
+ auto specSharding = info.GetShardingTabletVerified(i);
+ AFL_VERIFY(specSharding.MutableAppropriateMods().size() == 1);
+ AFL_VERIFY(specSharding.MutableAppropriateMods().emplace(*specSharding.MutableAppropriateMods().begin() + GetOrderedShardIds().size()).second);
+ *alter.MutableModification()->MutableModulo()->AddShards() = specSharding.SerializeToProto();
+ }
+
+ result.emplace_back(std::move(alter));
+ }
+ {
+ ui32 idx = 0;
+ for (auto&& i : GetOrderedShardIds()) {
+ {
+ NKikimrSchemeOp::TAlterShards alter;
+ alter.MutableModification()->AddOpenWriteIds(newTabletIds[idx]);
+ auto specSharding = info.GetShardingTabletVerified(i);
+ specSharding.SetTabletId(newTabletIds[idx]);
+ AFL_VERIFY(specSharding.MutableAppropriateMods().size() == 1);
+ *alter.MutableModification()->MutableModulo()->AddShards() = specSharding.SerializeToProto();
+ result.emplace_back(alter);
+ }
+ {
+ NKikimrSchemeOp::TAlterShards alter;
+ auto& transfer = *alter.MutableTransfer()->AddTransfers();
+ transfer.SetDestinationTabletId(newTabletIds[idx]);
+ transfer.AddSourceTabletIds(i);
+ result.emplace_back(alter);
+ }
+ {
+ NKikimrSchemeOp::TAlterShards alter;
+ alter.MutableModification()->AddOpenReadIds(newTabletIds[idx]);
+ auto specSharding = info.GetShardingTabletVerified(i);
+ AFL_VERIFY(specSharding.MutableAppropriateMods().size() == 1);
+ const ui32 original = *specSharding.MutableAppropriateMods().begin();
+ specSharding.MutableAppropriateMods().erase(original);
+ specSharding.MutableAppropriateMods().emplace(original + GetOrderedShardIds().size());
+ *alter.MutableModification()->MutableModulo()->AddShards() = specSharding.SerializeToProto();
+ result.emplace_back(alter);
+ }
+ ++idx;
+ }
+ }
+ return result;
+}
+
+NKikimr::TConclusionStatus THashShardingModuloN::DoApplyModification(const NKikimrSchemeOp::TShardingModification& proto) {
+ AFL_VERIFY(!!SpecialShardingInfo);
+ if (!proto.HasModulo()) {
+ return TConclusionStatus::Success();
+ }
+
+ if (proto.GetModulo().HasPartsCount()) {
+ SpecialShardingInfo->SetPartsCount(proto.GetModulo().GetPartsCount());
+ }
+
+ for (auto&& i : proto.GetModulo().GetShards()) {
+ TSpecificShardingInfo::TModuloShardingTablet info;
+ {
+ auto conclusion = info.DeserializeFromProto(i);
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ }
+ if (!UpdateShardInfo(info)) {
+ return TConclusionStatus::Fail("no shard with same id on update modulo");
+ }
+ }
+ return TConclusionStatus::Success();
+}
+
+NKikimr::TConclusionStatus THashShardingModuloN::DoOnAfterModification() {
+ AFL_VERIFY(!!SpecialShardingInfo);
+
+ auto result = SpecialShardingInfo->BuildActivityIndex(GetClosedWritingShardIds(), GetClosedReadingShardIds());
+ if (result.IsFail()) {
+ return result;
+ }
+
+ std::vector<ui64> shardIdsOrdered;
+ if (SpecialShardingInfo->CheckUnifiedDistribution(GetOrderedShardIds().size(), shardIdsOrdered)) {
+ SetOrderedShardIds(shardIdsOrdered);
+ SpecialShardingInfo.reset();
+ }
+
+ return TConclusionStatus::Success();
+}
+
+NKikimr::TConclusionStatus THashShardingModuloN::DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) {
+ auto conclusion = TBase::DoDeserializeFromProto(proto);
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ if (!proto.HasHashSharding()) {
+ return TConclusionStatus::Fail("no data for modulo n sharding");
+ }
+ AFL_VERIFY(proto.GetHashSharding().GetFunction() == NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_MODULO_N);
+ {
+ TSpecificShardingInfo specialInfo;
+ auto result = specialInfo.DeserializeFromProto(proto, GetClosedWritingShardIds(), GetClosedReadingShardIds());
+ if (result.IsFail()) {
+ return result;
+ }
+ if (!specialInfo.IsEmpty()) {
+ SpecialShardingInfo = std::move(specialInfo);
+ }
+ }
+ return TConclusionStatus::Success();
+}
+
+std::shared_ptr<NKikimr::NSharding::IGranuleShardingLogic> THashShardingModuloN::DoGetTabletShardingInfoOptional(const ui64 tabletId) const {
+ if (SpecialShardingInfo) {
+ return std::make_shared<TGranuleSharding>(GetShardingColumns(), SpecialShardingInfo->GetShardingTabletVerified(tabletId), SpecialShardingInfo->GetPartsCount());
+ } else {
+ TSpecificShardingInfo info(GetOrderedShardIds());
+ return std::make_shared<TGranuleSharding>(GetShardingColumns(), info.GetShardingTabletVerified(tabletId), info.GetPartsCount());
+ }
+}
+
+NKikimr::TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> THashShardingModuloN::DoBuildMergeShardsModifiers(const std::vector<ui64>& newTabletIds) const {
+ if (newTabletIds.size() * 2 != GetOrderedShardIds().size()) {
+ return TConclusionStatus::Fail("can div 2 only for reduce shards count");
+ }
+ if (!!SpecialShardingInfo) {
+ return TConclusionStatus::Fail("not unified shards distribution for consistency intervals modification");
+ }
+ const TSpecificShardingInfo shardingInfo = SpecialShardingInfo ? *SpecialShardingInfo : TSpecificShardingInfo(GetOrderedShardIds());
+ std::vector<NKikimrSchemeOp::TAlterShards> result;
+ {
+ ui32 idx = 0;
+ for (auto&& i : newTabletIds) {
+ const ui64 from1 = GetOrderedShardIds()[2 * idx];
+ const ui64 from2 = GetOrderedShardIds()[2 * idx + 1];
+ auto source1 = shardingInfo.GetShardingTabletVerified(from1);
+ auto source2 = shardingInfo.GetShardingTabletVerified(from2);
+ AFL_VERIFY(source1.GetAppropriateMods().size() == 1);
+ AFL_VERIFY(source2.GetAppropriateMods().size() == 1);
+ AFL_VERIFY(*source1.GetAppropriateMods().begin() + 1 == *source2.GetAppropriateMods().begin());
+ {
+ NKikimrSchemeOp::TAlterShards alter;
+ TSpecificShardingInfo::TModuloShardingTablet newInterval(i, { *source1.GetAppropriateMods().begin() ,*source2.GetAppropriateMods().begin() });
+ alter.MutableModification()->AddOpenWriteIds(i);
+ *alter.MutableModification()->MutableModulo()->AddShards() = newInterval.SerializeToProto();
+ result.emplace_back(alter);
+ }
+ {
+ NKikimrSchemeOp::TAlterShards alter;
+ auto& transfer = *alter.MutableTransfer()->AddTransfers();
+ transfer.SetDestinationTabletId(newTabletIds[idx]);
+ transfer.AddSourceTabletIds(from1);
+ transfer.AddSourceTabletIds(from2);
+ result.emplace_back(alter);
+ }
+ {
+ NKikimrSchemeOp::TAlterShards alter;
+ alter.MutableModification()->AddOpenReadIds(i);
+ alter.MutableModification()->AddCloseWriteIds(from1);
+ alter.MutableModification()->AddCloseWriteIds(from2);
+ alter.MutableModification()->AddCloseReadIds(from1);
+ alter.MutableModification()->AddCloseReadIds(from2);
+ result.emplace_back(alter);
+ }
+ ++idx;
+ }
+ }
+ return result;
+}
+
+}
diff --git a/ydb/core/tx/sharding/hash_modulo.h b/ydb/core/tx/sharding/hash_modulo.h
new file mode 100644
index 0000000000..769c552bdf
--- /dev/null
+++ b/ydb/core/tx/sharding/hash_modulo.h
@@ -0,0 +1,367 @@
+#pragma once
+#include "hash_sharding.h"
+#include <ydb/core/protos/flat_scheme_op.pb.h>
+
+namespace NKikimr::NSharding::NModulo {
+
+class TSpecificShardingInfo {
+public:
+ class TModuloShardingTablet {
+ private:
+ YDB_ACCESSOR(ui64, TabletId, 0);
+ YDB_ACCESSOR_DEF(std::set<ui32>, AppropriateMods);
+ public:
+ TModuloShardingTablet() = default;
+ TModuloShardingTablet(const ui64 tabletId, const std::set<ui32>& mods)
+ : TabletId(tabletId)
+ , AppropriateMods(mods) {
+
+ }
+
+ NKikimrSchemeOp::TModuloShardingTablet SerializeToProto() const {
+ NKikimrSchemeOp::TModuloShardingTablet result;
+ result.SetTabletId(TabletId);
+ for (auto&& i : AppropriateMods) {
+ result.AddAppropriateMods(i);
+ }
+ return result;
+ }
+
+ TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TModuloShardingTablet& proto) {
+ TabletId = proto.GetTabletId();
+ for (auto&& i : proto.GetAppropriateMods()) {
+ AppropriateMods.emplace(i);
+ }
+ return TConclusionStatus::Success();
+ }
+ };
+private:
+ bool IndexConstructed = false;
+ YDB_READONLY(ui32, PartsCount, 0);
+ std::vector<TModuloShardingTablet> SpecialSharding;
+ std::vector<TModuloShardingTablet*> ActiveWriteSpecialSharding;
+ std::vector<TModuloShardingTablet*> ActiveReadSpecialSharding;
+
+public:
+ TSpecificShardingInfo() = default;
+ TSpecificShardingInfo(const std::vector<ui64>& shardIds)
+ : PartsCount(shardIds.size())
+ {
+ for (ui32 i = 0; i < shardIds.size(); ++i) {
+ TModuloShardingTablet info(shardIds[i], { i });
+ SpecialSharding.emplace_back(info);
+ }
+ BuildActivityIndex(Default<std::set<ui64>>(), Default<std::set<ui64>>()).Validate();
+ }
+
+ void SetPartsCount(const ui32 partsCount) {
+ PartsCount = partsCount;
+ }
+
+ THashMap<ui64, std::vector<ui32>> MakeShardingWrite(const std::vector<ui64>& hashes) const {
+ AFL_VERIFY(PartsCount);
+ AFL_VERIFY(IndexConstructed);
+ std::vector<std::vector<ui32>> result;
+ result.resize(ActiveWriteSpecialSharding.size());
+ THashMap<ui64, std::vector<ui32>> resultHash;
+ std::vector<std::vector<ui32>> shardsByMod;
+ ui32 idx = 0;
+ for (auto&& i : ActiveWriteSpecialSharding) {
+ for (auto&& m : i->GetAppropriateMods()) {
+ shardsByMod[m].emplace_back(idx);
+ }
+ ++idx;
+ }
+ ui32 recordIdx = 0;
+ for (auto&& i : hashes) {
+ for (auto&& s : shardsByMod[i % PartsCount]) {
+ result[s].emplace_back(recordIdx);
+ }
+ ++recordIdx;
+ }
+ for (ui32 i = 0; i < result.size(); ++i) {
+ if (result[i].size()) {
+ resultHash[ActiveWriteSpecialSharding[i]->GetTabletId()] = std::move(result[i]);
+ }
+ }
+ return resultHash;
+ }
+
+ bool CheckUnifiedDistribution(const ui32 summaryShardsCount, std::vector<ui64>& orderedShardIds) {
+ AFL_VERIFY(IndexConstructed);
+ if (summaryShardsCount != PartsCount) {
+ return false;
+ }
+ std::set<ui64> activeReadTabletIds;
+ for (auto&& i : ActiveReadSpecialSharding) {
+ activeReadTabletIds.emplace(i->GetTabletId());
+ }
+ std::set<ui64> activeWriteTabletIds;
+ for (auto&& i : ActiveWriteSpecialSharding) {
+ activeWriteTabletIds.emplace(i->GetTabletId());
+ }
+ std::set<ui64> tabletIds;
+ for (auto&& i : SpecialSharding) {
+ tabletIds.emplace(i.GetTabletId());
+ }
+ if (activeReadTabletIds != tabletIds || activeWriteTabletIds != tabletIds) {
+ return false;
+ }
+
+ std::map<ui32, ui64> orderedTabletIds;
+ for (auto&& i : SpecialSharding) {
+ if (i.GetAppropriateMods().size() != 1) {
+ return false;
+ }
+ for (auto&& m : i.GetAppropriateMods()) {
+ AFL_VERIFY(m < PartsCount)("m", m)("parts", PartsCount);
+ if (!orderedTabletIds.emplace(m, i.GetTabletId()).second) {
+ return false;
+ }
+ }
+ }
+ orderedShardIds.clear();
+ for (auto&& i : orderedTabletIds) {
+ orderedShardIds.emplace_back(i.second);
+ }
+ AFL_VERIFY(orderedTabletIds.size() == summaryShardsCount);
+ return true;
+ }
+
+ bool IsEmpty() const {
+ return SpecialSharding.empty();
+ }
+
+ TConclusionStatus BuildActivityIndex(const std::set<ui64>& closedWriteIds, const std::set<ui64>& closedReadIds) {
+ if (SpecialSharding.empty()) {
+ AFL_VERIFY(!PartsCount);
+ return TConclusionStatus::Success();
+ }
+ ActiveReadSpecialSharding.clear();
+ ActiveWriteSpecialSharding.clear();
+ std::set<ui32> modsRead;
+ std::set<ui32> modsWrite;
+ for (auto&& i : SpecialSharding) {
+ for (auto&& n : i.GetAppropriateMods()) {
+ if (PartsCount <= n) {
+ return TConclusionStatus::Fail("too large parts mod in compare with parts count");
+ }
+ if (!closedWriteIds.contains(i.GetTabletId())) {
+ modsWrite.emplace(n);
+ }
+ if (!closedReadIds.contains(i.GetTabletId())) {
+ if (!modsRead.emplace(n).second) {
+ return TConclusionStatus::Fail("read interval twice usage impossible");
+ }
+ }
+ }
+ if (!closedReadIds.contains(i.GetTabletId())) {
+ ActiveReadSpecialSharding.emplace_back(&i);
+ }
+ if (!closedWriteIds.contains(i.GetTabletId())) {
+ ActiveWriteSpecialSharding.emplace_back(&i);
+ }
+ }
+ if (modsRead.size() && modsRead.size() != PartsCount) {
+ return TConclusionStatus::Fail("incorrect sharding configuration read from proto (read): " + JoinSeq(", ", modsRead) + "; " + ::ToString(PartsCount));
+ }
+ if (modsWrite.size() && modsWrite.size() != PartsCount) {
+ return TConclusionStatus::Fail("incorrect sharding configuration read from proto (write): " + JoinSeq(", ", modsWrite) + "; " + ::ToString(PartsCount));
+ }
+ IndexConstructed = true;
+ return TConclusionStatus::Success();
+ }
+
+ TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto, const std::set<ui64>& closedWriteIds, const std::set<ui64>& closedReadIds) {
+ IndexConstructed = false;
+
+ PartsCount = proto.GetHashSharding().GetModuloPartsCount();
+ for (auto&& i : proto.GetHashSharding().GetTabletsForModulo()) {
+ TModuloShardingTablet info;
+ auto conclusion = info.DeserializeFromProto(i);
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ SpecialSharding.emplace_back(std::move(info));
+ }
+ AFL_VERIFY(PartsCount || SpecialSharding.empty());
+ return BuildActivityIndex(closedWriteIds, closedReadIds);
+ }
+
+ const TModuloShardingTablet& GetShardingTabletVerified(const ui64 tabletId) const {
+ for (auto&& i : SpecialSharding) {
+ if (i.GetTabletId() == tabletId) {
+ return i;
+ }
+ }
+ AFL_VERIFY(false);
+ return Default<TModuloShardingTablet>();
+ }
+
+ void SerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const {
+ AFL_VERIFY(PartsCount);
+ proto.MutableHashSharding()->SetModuloPartsCount(PartsCount);
+ for (auto&& i : SpecialSharding) {
+ *proto.MutableHashSharding()->AddTabletsForModulo() = i.SerializeToProto();
+ }
+ }
+
+ bool UpdateShardInfo(const TModuloShardingTablet& info) {
+ for (auto&& i : SpecialSharding) {
+ if (i.GetTabletId() == info.GetTabletId()) {
+ i = info;
+ IndexConstructed = false;
+ return true;
+ }
+ }
+ return false;
+ }
+
+ void AddShardInfo(const TModuloShardingTablet& info) {
+ IndexConstructed = false;
+ for (auto&& i : SpecialSharding) {
+ AFL_VERIFY(i.GetTabletId() != info.GetTabletId());
+ }
+ SpecialSharding.emplace_back(info);
+ }
+};
+
+class THashShardingModuloN : public THashShardingImpl {
+public:
+ static TString GetClassNameStatic() {
+ return "MODULO";
+ }
+private:
+ using TBase = THashShardingImpl;
+private:
+ std::optional<TSpecificShardingInfo> SpecialShardingInfo;
+
+ bool UpdateShardInfo(const TSpecificShardingInfo::TModuloShardingTablet& info) {
+ AFL_VERIFY(SpecialShardingInfo);
+ GetShardInfoVerified(info.GetTabletId()).IncrementVersion();
+ if (SpecialShardingInfo->UpdateShardInfo(info)) {
+ return true;
+ }
+ for (auto&& i : GetOrderedShardIds()) {
+ if (i == info.GetTabletId()) {
+ SpecialShardingInfo->AddShardInfo(info);
+ return true;
+ }
+ }
+ AFL_VERIFY(false);
+ return false;
+ }
+ virtual std::shared_ptr<IGranuleShardingLogic> DoGetTabletShardingInfoOptional(const ui64 tabletId) const override;
+
+protected:
+ virtual TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> DoBuildSplitShardsModifiers(const std::vector<ui64>& newTabletIds) const override;
+
+ virtual TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> DoBuildMergeShardsModifiers(const std::vector<ui64>& newTabletIds) const override;
+
+ virtual TConclusionStatus DoOnAfterModification() override;
+ virtual TConclusionStatus DoOnBeforeModification() override {
+ if (!SpecialShardingInfo) {
+ SpecialShardingInfo = TSpecificShardingInfo(GetOrderedShardIds());
+ }
+ return TConclusionStatus::Success();
+ }
+
+ virtual TConclusionStatus DoApplyModification(const NKikimrSchemeOp::TShardingModification& proto) override;
+ virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override {
+ TBase::DoSerializeToProto(proto);
+ if (SpecialShardingInfo) {
+ SpecialShardingInfo->SerializeToProto(proto);
+ }
+ proto.MutableHashSharding()->SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_MODULO_N);
+ }
+
+
+ virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) override;
+
+ virtual std::set<ui64> DoGetModifiedShardIds(const NKikimrSchemeOp::TShardingModification& proto) const override {
+ std::set<ui64> result;
+ for (auto&& i : proto.GetModulo().GetShards()) {
+ result.emplace(i.GetTabletId());
+ }
+ return result;
+ }
+public:
+ using TBase::TBase;
+
+ THashShardingModuloN() = default;
+
+ THashShardingModuloN(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui64 seed = 0)
+ : TBase(shardIds, columnNames, seed)
+ {}
+
+ virtual TString GetClassName() const override {
+ return GetClassNameStatic();
+ }
+
+ virtual THashMap<ui64, std::vector<ui32>> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override;
+};
+
+class TGranuleSharding: public THashGranuleSharding {
+public:
+ static TString GetClassNameStatic() {
+ return "MODULO";
+ }
+private:
+ using TBase = THashGranuleSharding;
+ ui64 PartsCount = 0;
+ TSpecificShardingInfo::TModuloShardingTablet Interval;
+ static const inline TFactory::TRegistrator<TGranuleSharding> Registrator = TFactory::TRegistrator<TGranuleSharding>(GetClassNameStatic());
+
+protected:
+ virtual std::shared_ptr<NArrow::TColumnFilter> DoGetFilter(const std::shared_ptr<arrow::Table>& table) const override {
+ const std::vector<ui64> hashes = CalcHashes(table);
+ auto result = std::make_shared<NArrow::TColumnFilter>(NArrow::TColumnFilter::BuildAllowFilter());
+ const auto getter = [&](const ui32 index) {
+ return Interval.GetAppropriateMods().contains(hashes[index] % PartsCount);
+ };
+ result->ResetWithLambda(hashes.size(), getter);
+ return result;
+
+ }
+ virtual void DoSerializeToProto(TProto& proto) const override {
+ AFL_VERIFY(PartsCount);
+ proto.MutableModulo()->SetModuloPartsCount(PartsCount);
+ *proto.MutableModulo()->MutableHashing() = TBase::SerializeHashingToProto();
+ *proto.MutableModulo()->MutableShardInfo() = Interval.SerializeToProto();
+ }
+ virtual TConclusionStatus DoDeserializeFromProto(const TProto& proto) override {
+ PartsCount = proto.GetModulo().GetModuloPartsCount();
+ if (!PartsCount) {
+ return TConclusionStatus::Fail("incorrect parts count for modulo info");
+ }
+ {
+ auto conclusion = TBase::DeserializeHashingFromProto(proto.GetModulo().GetHashing());
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ }
+ {
+ auto conclusion = Interval.DeserializeFromProto(proto.GetModulo().GetShardInfo());
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ }
+
+ return TConclusionStatus::Success();
+ }
+public:
+ TGranuleSharding() = default;
+
+ TGranuleSharding(const std::vector<TString>& columnNames, const TSpecificShardingInfo::TModuloShardingTablet& interval, const ui64 partsCount)
+ : TBase(columnNames)
+ , PartsCount(partsCount)
+ , Interval(interval) {
+ AFL_VERIFY(PartsCount);
+ }
+
+ virtual TString GetClassName() const override {
+ return GetClassNameStatic();
+ }
+};
+
+}
diff --git a/ydb/core/tx/sharding/hash_sharding.cpp b/ydb/core/tx/sharding/hash_sharding.cpp
new file mode 100644
index 0000000000..06db098990
--- /dev/null
+++ b/ydb/core/tx/sharding/hash_sharding.cpp
@@ -0,0 +1,5 @@
+#include "hash_sharding.h"
+
+namespace NKikimr::NSharding {
+
+}
diff --git a/ydb/core/tx/sharding/hash_sharding.h b/ydb/core/tx/sharding/hash_sharding.h
new file mode 100644
index 0000000000..30111f3fe8
--- /dev/null
+++ b/ydb/core/tx/sharding/hash_sharding.h
@@ -0,0 +1,104 @@
+#pragma once
+#include "sharding.h"
+#include <ydb/core/formats/arrow/hash/calcer.h>
+
+namespace NKikimr::NSharding {
+
+class THashGranuleSharding: public IGranuleShardingLogic {
+private:
+ std::optional<NArrow::NHash::TXX64> HashCalcer;
+protected:
+ std::vector<ui64> CalcHashes(const std::shared_ptr<arrow::Table>& table) const {
+ AFL_VERIFY(!!HashCalcer);
+ return HashCalcer->ExecuteToVector(table);
+ }
+
+ virtual std::set<TString> DoGetColumnNames() const final {
+ AFL_VERIFY(!!HashCalcer);
+ auto columnsVector = HashCalcer->GetColumnNames();
+ return std::set<TString>(columnsVector.begin(), columnsVector.end());
+ }
+ NKikimrSchemeOp::THashShardingInfo SerializeHashingToProto() const {
+ NKikimrSchemeOp::THashShardingInfo proto;
+ AFL_VERIFY(!!HashCalcer);
+ for (auto&& i : HashCalcer->GetColumnNames()) {
+ proto.AddColumnNames(i);
+ }
+ return proto;
+ }
+ TConclusionStatus DeserializeHashingFromProto(const NKikimrSchemeOp::THashShardingInfo& proto) {
+ AFL_VERIFY(!HashCalcer);
+ if (proto.GetColumnNames().empty()) {
+ return TConclusionStatus::Fail("no column names for THashGranuleSharding in proto");
+ }
+ std::vector<TString> columnNames;
+ for (auto&& i : proto.GetColumnNames()) {
+ columnNames.emplace_back(i);
+ }
+ HashCalcer.emplace(columnNames, NArrow::NHash::TXX64::ENoColumnPolicy::Verify, 0);
+ return TConclusionStatus::Success();
+ }
+public:
+ THashGranuleSharding() = default;
+
+ THashGranuleSharding(const std::vector<TString>& columnNames)
+ : HashCalcer(NArrow::NHash::TXX64(columnNames, NArrow::NHash::TXX64::ENoColumnPolicy::Verify, 0))
+ {
+
+ }
+};
+
+class THashShardingImpl: public IShardingBase {
+private:
+ using TBase = IShardingBase;
+ ui64 Seed = 0;
+ std::optional<NArrow::NHash::TXX64> HashCalcer;
+ YDB_READONLY_DEF(std::vector<TString>, ShardingColumns);
+protected:
+ virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override {
+ for (auto&& i : ShardingColumns) {
+ proto.MutableHashSharding()->AddColumns(i);
+ }
+ }
+ virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) override {
+ if (!proto.HasHashSharding()) {
+ return TConclusionStatus::Fail("no data about hash sharding");
+ }
+ if (!proto.GetHashSharding().GetColumns().size()) {
+ return TConclusionStatus::Fail("no columns for hash sharding");
+ }
+ for (auto&& i : proto.GetHashSharding().GetColumns()) {
+ ShardingColumns.emplace_back(i);
+ }
+ AFL_VERIFY(!HashCalcer);
+ HashCalcer.emplace(ShardingColumns, NArrow::NHash::TXX64::ENoColumnPolicy::Verify, Seed);
+ return TConclusionStatus::Success();
+ }
+public:
+ THashShardingImpl() = default;
+
+ THashShardingImpl(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui64 seed = 0)
+ : TBase(shardIds)
+ , Seed(seed)
+ , ShardingColumns(columnNames) {
+ HashCalcer.emplace(columnNames, NArrow::NHash::TXX64::ENoColumnPolicy::Verify, Seed);
+ }
+
+ virtual TString DebugString() const override {
+ return TBase::DebugString() + ";Columns: " + JoinSeq(", ", GetShardingColumns());
+ }
+
+ virtual std::vector<ui64> MakeHashes(const std::shared_ptr<arrow::RecordBatch>& batch) const {
+ AFL_VERIFY(!!HashCalcer);
+ return HashCalcer->Execute(batch).value_or(Default<std::vector<ui64>>());
+ }
+
+ template <typename T>
+ static ui64 CalcHash(const T value, const ui32 seed = 0) {
+ static_assert(std::is_arithmetic<T>::value);
+ return XXH64(&value, sizeof(value), seed);
+ }
+
+};
+
+}
diff --git a/ydb/core/tx/sharding/hash_slider.cpp b/ydb/core/tx/sharding/hash_slider.cpp
new file mode 100644
index 0000000000..d6c2f116ae
--- /dev/null
+++ b/ydb/core/tx/sharding/hash_slider.cpp
@@ -0,0 +1,55 @@
+#include "hash_slider.h"
+
+namespace NKikimr::NSharding {
+
+THashMap<ui64, std::vector<ui32>> TLogsSharding::MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const {
+ if (GetShardingColumns().size() < 2) {
+ return {};
+ }
+
+ auto tsArray = batch->GetColumnByName(GetShardingColumns()[0]);
+ if (!tsArray || tsArray->type_id() != arrow::Type::TIMESTAMP) {
+ return {};
+ }
+
+ const std::vector<ui64> hashes = MakeHashes(batch);
+ if (hashes.empty()) {
+ return {};
+ }
+
+ auto tsColumn = std::static_pointer_cast<arrow::TimestampArray>(tsArray);
+
+ std::vector<std::vector<ui32>> result;
+ result.resize(GetOrderedShardIds().size());
+ for (auto&& i : result) {
+ i.reserve(hashes.size());
+ }
+
+ for (int row = 0; row < batch->num_rows(); ++row) {
+ result[ShardNo(tsColumn->Value(row), hashes[row])].emplace_back(row);
+ }
+
+ THashMap<ui64, std::vector<ui32>> resultHash;
+ for (ui32 i = 0; i < result.size(); ++i) {
+ if (result[i].size()) {
+ resultHash.emplace(GetShardIdByOrderIdx(i), std::move(result[i]));
+ }
+ }
+
+ return resultHash;
+}
+
+NKikimr::TConclusionStatus TLogsSharding::DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) {
+ auto conclusion = TBase::DoDeserializeFromProto(proto);
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ AFL_VERIFY(proto.GetHashSharding().GetFunction() == NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CLOUD_LOGS);
+ if (proto.GetHashSharding().HasActiveShardsCount()) {
+ NumActive = proto.GetHashSharding().GetActiveShardsCount();
+ }
+
+ return TConclusionStatus::Success();
+}
+
+}
diff --git a/ydb/core/tx/sharding/hash_slider.h b/ydb/core/tx/sharding/hash_slider.h
new file mode 100644
index 0000000000..6cd61efa91
--- /dev/null
+++ b/ydb/core/tx/sharding/hash_slider.h
@@ -0,0 +1,76 @@
+#pragma once
+#include "hash_sharding.h"
+
+namespace NKikimr::NSharding {
+
+class TLogsSharding : public THashShardingImpl {
+public:
+ static constexpr ui32 DEFAULT_ACITVE_SHARDS = 10;
+ static constexpr TDuration DEFAULT_CHANGE_PERIOD = TDuration::Minutes(5);
+ static TString GetClassNameStatic() {
+ return "LOGS_SHARDING";
+ }
+private:
+ using TBase = THashShardingImpl;
+ ui32 NumActive = DEFAULT_ACITVE_SHARDS;
+ ui64 TsMin = 0;
+ ui64 ChangePeriod = DEFAULT_CHANGE_PERIOD.MicroSeconds();
+
+ virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override {
+ TBase::DoSerializeToProto(proto);
+ proto.MutableHashSharding()->SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CLOUD_LOGS);
+ proto.MutableHashSharding()->SetActiveShardsCount(NumActive);
+ }
+
+ virtual std::shared_ptr<IGranuleShardingLogic> DoGetTabletShardingInfoOptional(const ui64 /*tabletId*/) const override {
+ return nullptr;
+ }
+
+ virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) override;
+ virtual TConclusionStatus DoOnAfterModification() override {
+ return TConclusionStatus::Success();
+ }
+ virtual TConclusionStatus DoOnBeforeModification() override {
+ return TConclusionStatus::Success();
+ }
+
+ virtual std::set<ui64> DoGetModifiedShardIds(const NKikimrSchemeOp::TShardingModification& /*proto*/) const override {
+ return {};
+ }
+
+ virtual TConclusionStatus DoApplyModification(const NKikimrSchemeOp::TShardingModification& /*proto*/) override {
+ return TConclusionStatus::Fail("its impossible to modify logs sharding");
+ }
+
+ virtual TString GetClassName() const override {
+ return GetClassNameStatic();
+ }
+public:
+ TLogsSharding() = default;
+
+ TLogsSharding(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui32 shardsCountActive, TDuration changePeriod = DEFAULT_CHANGE_PERIOD)
+ : TBase(shardIds, columnNames)
+ , NumActive(Min<ui32>(shardsCountActive, GetShardsCount()))
+ , TsMin(0)
+ , ChangePeriod(changePeriod.MicroSeconds())
+ {
+ AFL_VERIFY(NumActive);
+ }
+
+ // tsMin = GetTsMin(tabletIdsMap, timestamp);
+ // tabletIds = GetTableIdsByTs(tabletIdsMap, timestamp);
+ // numIntervals = tabletIds.size() / nActive;
+ // tsInterval = (timestamp - tsMin) / changePeriod;
+ // shardNo = (hash(uid) % nActive) + (tsInterval % numIntervals) * nActive;
+ // tabletId = tabletIds[shardNo];
+ ui32 ShardNo(ui64 timestamp, const ui64 uidHash) const {
+ const ui32 tsInterval = (timestamp - TsMin) / ChangePeriod;
+ const ui32 numIntervals = std::max<ui32>(1, GetShardsCount() / NumActive);
+ return ((uidHash % NumActive) + (tsInterval % numIntervals) * NumActive) % GetShardsCount();
+ }
+
+ virtual THashMap<ui64, std::vector<ui32>> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override;
+
+};
+
+}
diff --git a/ydb/core/tx/sharding/random.cpp b/ydb/core/tx/sharding/random.cpp
new file mode 100644
index 0000000000..fc6ed93c2f
--- /dev/null
+++ b/ydb/core/tx/sharding/random.cpp
@@ -0,0 +1,21 @@
+#include "random.h"
+
+namespace NKikimr::NSharding {
+
+THashMap<ui64, std::vector<ui32>> TRandomSharding::MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const {
+ std::vector<ui64> activeShardIds;
+ for (auto&& i : GetOrderedShardIds()) {
+ if (IsActiveForWrite(i)) {
+ activeShardIds.emplace_back(i);
+ }
+ }
+ AFL_VERIFY(activeShardIds.size());
+ THashMap<ui64, std::vector<ui32>> resultHash;
+ std::vector<ui32>& sVector = resultHash.emplace(activeShardIds[RandomNumber<ui32>(activeShardIds.size())], std::vector<ui32>()).first->second;
+ for (ui32 i = 0; i < batch->num_rows(); ++i) {
+ sVector.emplace_back(i);
+ }
+ return resultHash;
+}
+
+}
diff --git a/ydb/core/tx/sharding/random.h b/ydb/core/tx/sharding/random.h
new file mode 100644
index 0000000000..cd0d40ca9f
--- /dev/null
+++ b/ydb/core/tx/sharding/random.h
@@ -0,0 +1,51 @@
+#pragma once
+#include "sharding.h"
+
+namespace NKikimr::NSharding {
+
+class TRandomSharding: public IShardingBase {
+private:
+ using TBase = IShardingBase;
+ static TString GetClassNameStatic() {
+ return "RANDOM";
+ }
+protected:
+ virtual std::shared_ptr<IGranuleShardingLogic> DoGetTabletShardingInfoOptional(const ui64 /*tabletId*/) const override {
+ return nullptr;
+ }
+
+ virtual TConclusionStatus DoOnAfterModification() override {
+ return TConclusionStatus::Success();
+ }
+ virtual TConclusionStatus DoOnBeforeModification() override {
+ return TConclusionStatus::Success();
+ }
+
+ virtual std::set<ui64> DoGetModifiedShardIds(const NKikimrSchemeOp::TShardingModification& /*proto*/) const override {
+ return {};
+ }
+
+ virtual TConclusionStatus DoApplyModification(const NKikimrSchemeOp::TShardingModification& /*proto*/) override {
+ return TConclusionStatus::Fail("its impossible to modify random sharding");
+ }
+ virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override {
+ proto.MutableRandomSharding();
+ }
+
+ virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) override {
+ if (!proto.HasRandomSharding()) {
+ return TConclusionStatus::Fail("no random sharding data");
+ }
+ return TConclusionStatus::Success();
+ }
+public:
+ using TBase::TBase;
+
+ virtual THashMap<ui64, std::vector<ui32>> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override;
+
+ virtual TString GetClassName() const override {
+ return GetClassNameStatic();
+ }
+};
+
+}
diff --git a/ydb/core/tx/sharding/sharding.cpp b/ydb/core/tx/sharding/sharding.cpp
index a2b9f390be..ba34521c21 100644
--- a/ydb/core/tx/sharding/sharding.cpp
+++ b/ydb/core/tx/sharding/sharding.cpp
@@ -1,10 +1,16 @@
#include "sharding.h"
+#include <ydb/core/protos/flat_scheme_op.pb.h>
#include <ydb/library/yql/utils/yql_panic.h>
+#include <ydb/core/tx/columnshard/common/protos/snapshot.pb.h>
#include <util/string/join.h>
+#include "hash_intervals.h"
+#include "hash_modulo.h"
+#include "hash_slider.h"
+#include "random.h"
namespace NKikimr::NSharding {
-TConclusionStatus TShardingBase::ValidateBehaviour(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo) {
+TConclusionStatus IShardingBase::ValidateBehaviour(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo) {
auto copy = shardingInfo;
if (copy.GetColumnShards().size() == 0) {
copy.AddColumnShards(1);
@@ -16,15 +22,14 @@ TConclusionStatus TShardingBase::ValidateBehaviour(const NSchemeShard::TOlapSche
return TConclusionStatus::Success();
}
-TConclusion<std::unique_ptr<TShardingBase>> TShardingBase::BuildFromProto(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingProto) {
+TConclusion<std::unique_ptr<IShardingBase>> IShardingBase::BuildFromProto(const NSchemeShard::TOlapSchema* schema, const NKikimrSchemeOp::TColumnTableSharding& shardingProto) {
if (!shardingProto.GetColumnShards().size()) {
return TConclusionStatus::Fail("config is incorrect for construct sharding behaviour");
}
- std::vector<ui64> shardIds(shardingProto.GetColumnShards().begin(), shardingProto.GetColumnShards().end());
+ std::unique_ptr<IShardingBase> result;
if (shardingProto.HasRandomSharding()) {
- return std::make_unique<TRandomSharding>(shardIds);
- }
- if (shardingProto.HasHashSharding()) {
+ result = std::make_unique<TRandomSharding>();
+ } else if (shardingProto.HasHashSharding()) {
auto& hashSharding = shardingProto.GetHashSharding();
std::vector<TString> columnNames;
if (hashSharding.GetColumns().empty()) {
@@ -32,67 +37,264 @@ TConclusion<std::unique_ptr<TShardingBase>> TShardingBase::BuildFromProto(const
} else {
for (auto&& i : hashSharding.GetColumns()) {
columnNames.emplace_back(i);
- if (!schema.GetColumns().GetByName(i)) {
- return TConclusionStatus::Fail("incorrect sharding column name: " + i);
- }
- if (!schema.GetColumns().GetByName(i)->IsKeyColumn()) {
- return TConclusionStatus::Fail("sharding column name have to been primary key column: " + i);
+ if (schema) {
+ if (!schema->GetColumns().GetByName(i)) {
+ return TConclusionStatus::Fail("incorrect sharding column name: " + i);
+ }
+ if (!schema->GetColumns().GetByName(i)->IsKeyColumn()) {
+ return TConclusionStatus::Fail("sharding column name have to been primary key column: " + i);
+ }
}
}
}
if (hashSharding.GetFunction() == NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CONSISTENCY_64) {
- return std::make_unique<TConsistencySharding64>(shardIds, columnNames, 0);
+ result = std::make_unique<NConsistency::TConsistencySharding64>();
} else if (hashSharding.GetFunction() == NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_MODULO_N) {
- return std::make_unique<THashShardingModuloN>(shardIds, columnNames, 0);
+ result = std::make_unique<NModulo::THashShardingModuloN>();
} else if (hashSharding.GetFunction() == NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CLOUD_LOGS) {
- ui32 activeShards = TLogsSharding::DEFAULT_ACITVE_SHARDS;
- if (hashSharding.HasActiveShardsCount()) {
- activeShards = hashSharding.GetActiveShardsCount();
- }
- return std::make_unique<TLogsSharding>(shardIds, columnNames, activeShards);
+ result = std::make_unique<TLogsSharding>();
+ }
+ }
+ if (!result) {
+ return TConclusionStatus::Fail("not determined type of sharding");
+ }
+ {
+ auto conclusion = result->DeserializeFromProto(shardingProto);
+ if (conclusion.IsFail()) {
+ return conclusion;
}
+ return result;
}
- return TConclusionStatus::Fail("not determined sharding type");
}
-TString TShardingBase::DebugString() const {
+TString IShardingBase::DebugString() const {
return "SHARDING";
}
-std::vector<ui32> THashShardingModuloN::MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const {
- std::vector<ui64> hashes = MakeHashes(batch);
- std::vector<ui32> result;
- result.reserve(hashes.size());
- for (auto&& i : hashes) {
- result.emplace_back(i % GetShardsCount());
+NKikimr::TConclusionStatus IShardingBase::DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) {
+ std::set<ui64> shardIdsSetOriginal;
+ if (proto.GetShardsInfo().empty()) {
+ std::vector<ui64> orderedTabletIds(proto.GetColumnShards().begin(), proto.GetColumnShards().end());
+ InitializeFromOrdered(orderedTabletIds);
+ } else {
+ std::map<ui32, ui64> sortedTabletIds;
+ for (auto&& i : proto.GetShardsInfo()) {
+ TConclusion<TShardInfo> shardInfo = TShardInfo::BuildFromProto(i);
+ if (shardInfo.IsFail()) {
+ return shardInfo;
+ }
+ AFL_VERIFY(sortedTabletIds.emplace(shardInfo->GetSequenceIdx(), shardInfo->GetTabletId()).second);
+ AFL_VERIFY(Shards.emplace(shardInfo->GetTabletId(), shardInfo.GetResult()).second);
+ }
+ for (auto&& i : sortedTabletIds) {
+ OrderedShardIds.emplace_back(i.second);
+ }
+ }
+ if (shardIdsSetOriginal.size()) {
+ return TConclusionStatus::Fail("not for all shard in sequence we have info");
+ }
+ {
+ auto conclusion = DoDeserializeFromProto(proto);
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ }
+ return TConclusionStatus::Success();
+}
+
+NKikimr::TConclusionStatus IShardingBase::ApplyModification(const NKikimrSchemeOp::TShardingModification& proto) {
+ {
+ auto conclusion = OnBeforeModification();
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ }
+ for (auto&& i : proto.GetNewShardIds()) {
+ if (!Shards.emplace(i, TShardInfo(i, OrderedShardIds.size())).second) {
+ return TConclusionStatus::Fail("shard id from NewShardIds duplication with current full shardIds list");
+ }
+ OrderedShardIds.emplace_back(i);
+ }
+ for (auto&& i : proto.GetCloseWriteIds()) {
+ auto* shardInfo = GetShardInfo(i);
+ if (!shardInfo) {
+ return TConclusionStatus::Fail("incorrect shard id from CloseWriteIds - not exists in full scope");
+ }
+ shardInfo->SetIsOpenForWrite(false);
+ }
+ for (auto&& i : proto.GetCloseReadIds()) {
+ auto* shardInfo = GetShardInfo(i);
+ if (!shardInfo) {
+ return TConclusionStatus::Fail("incorrect shard id from CloseReadIds - not exists in full scope");
+ }
+ shardInfo->SetIsOpenForRead(false);
+ }
+ for (auto&& i : proto.GetOpenWriteIds()) {
+ auto* shardInfo = GetShardInfo(i);
+ if (!shardInfo) {
+ return TConclusionStatus::Fail("incorrect shard id from OpenWriteIds - not exists in full scope");
+ }
+ shardInfo->SetIsOpenForWrite(true);
+ }
+ for (auto&& i : proto.GetOpenReadIds()) {
+ auto* shardInfo = GetShardInfo(i);
+ if (!shardInfo) {
+ return TConclusionStatus::Fail("incorrect shard id from OpenReadIds - not exists in full scope");
+ }
+ shardInfo->SetIsOpenForRead(true);
+ }
+ {
+ auto conclusion = DoApplyModification(proto);
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ }
+ {
+ auto conclusion = OnAfterModification();
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ }
+ return TConclusionStatus::Success();
+}
+
+NKikimr::TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> IShardingBase::BuildAddShardsModifiers(const std::vector<ui64>& newTabletIds) const {
+ NKikimrSchemeOp::TShardingModification startModification;
+ for (auto&& i : newTabletIds) {
+ startModification.AddNewShardIds(i);
+ startModification.AddCloseReadIds(i);
+ startModification.AddCloseWriteIds(i);
+ }
+
+ TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> infosConclusion = BuildSplitShardsModifiers(newTabletIds);
+ if (infosConclusion.IsFail()) {
+ return infosConclusion;
+ }
+ std::vector<NKikimrSchemeOp::TAlterShards> result;
+ {
+ NKikimrSchemeOp::TAlterShards startAlter;
+ *startAlter.MutableModification() = std::move(startModification);
+ result.emplace_back(std::move(startAlter));
+ }
+ for (auto&& i : *infosConclusion) {
+ result.emplace_back(std::move(i));
}
return result;
}
-std::vector<ui32> TLogsSharding::MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const {
- if (ShardingColumns.size() < 2) {
- return {};
+NKikimr::TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> IShardingBase::BuildReduceShardsModifiers(const std::vector<ui64>& newTabletIds) const {
+ NKikimrSchemeOp::TShardingModification startModification;
+ for (auto&& i : newTabletIds) {
+ startModification.AddNewShardIds(i);
+ startModification.AddCloseReadIds(i);
+ startModification.AddCloseWriteIds(i);
}
- auto tsArray = batch->GetColumnByName(ShardingColumns[0]);
- if (!tsArray || tsArray->type_id() != arrow::Type::TIMESTAMP) {
- return {};
+ TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> infosConclusion = BuildMergeShardsModifiers(newTabletIds);
+ if (infosConclusion.IsFail()) {
+ return infosConclusion;
}
+ std::vector<NKikimrSchemeOp::TAlterShards> result;
+ {
+ NKikimrSchemeOp::TAlterShards startAlter;
+ *startAlter.MutableModification() = std::move(startModification);
+ result.emplace_back(std::move(startAlter));
+ }
+ for (auto&& i : *infosConclusion) {
+ result.emplace_back(std::move(i));
+ }
+ {
+ NKikimrSchemeOp::TShardingModification finishModification;
+ for (auto&& i : GetOrderedShardIds()) {
+ finishModification.AddDeleteShardIds(i);
+ }
+ NKikimrSchemeOp::TAlterShards finishAlter;
+ *finishAlter.MutableModification() = std::move(finishModification);
+ result.emplace_back(std::move(finishAlter));
+ }
+ return result;
+}
- const std::vector<ui64> hashes = MakeHashes(batch);
- if (hashes.empty()) {
- return {};
+NKikimrSchemeOp::TColumnTableSharding IShardingBase::SerializeToProto() const {
+ NKikimrSchemeOp::TColumnTableSharding result;
+ AFL_VERIFY(Shards.size() == OrderedShardIds.size());
+ AFL_VERIFY(OrderedShardIds.size());
+ for (auto&& i : OrderedShardIds) {
+ *result.AddShardsInfo() = GetShardInfoVerified(i).SerializeToProto();
+ result.AddColumnShards(i);
}
+ DoSerializeToProto(result);
+ return result;
+}
- auto tsColumn = std::static_pointer_cast<arrow::TimestampArray>(tsArray);
- std::vector<ui32> out;
- out.reserve(batch->num_rows());
+NKikimr::TConclusion<THashMap<ui64, std::vector<NKikimr::NArrow::TSerializedBatch>>> IShardingBase::SplitByShards(const std::shared_ptr<arrow::RecordBatch>& batch, const ui64 chunkBytesLimit) {
+ THashMap<ui64, std::vector<ui32>> sharding = MakeSharding(batch);
+ THashMap<ui64, std::shared_ptr<arrow::RecordBatch>> chunks;
+ if (sharding.size() == 1) {
+ AFL_VERIFY(chunks.emplace(sharding.begin()->first, batch).second);
+ } else {
+ chunks = NArrow::ShardingSplit(batch, sharding);
+ }
+ AFL_VERIFY(chunks.size() == sharding.size());
+ NArrow::TBatchSplitttingContext context(chunkBytesLimit);
+ THashMap<ui64, std::vector<NArrow::TSerializedBatch>> result;
+ for (auto&& [tabletId, chunk]: chunks) {
+ if (!chunk) {
+ continue;
+ }
+ auto blobsSplittedConclusion = NArrow::SplitByBlobSize(chunk, context);
+ if (blobsSplittedConclusion.IsFail()) {
+ return TConclusionStatus::Fail("cannot split batch in according to limits: " + blobsSplittedConclusion.GetErrorMessage());
+ }
+ result.emplace(tabletId, blobsSplittedConclusion.DetachResult());
+ }
+ return result;
+}
- for (int row = 0; row < batch->num_rows(); ++row) {
- out.emplace_back(ShardNo(tsColumn->Value(row), hashes[row]));
+void IShardingBase::InitializeFromOrdered(const std::vector<ui64>& orderedIds) {
+ AFL_VERIFY(orderedIds.size());
+ std::set<ui64> shardIdsSetOriginal;
+ OrderedShardIds.clear();
+ Shards.clear();
+ for (auto&& i : orderedIds) {
+ AFL_VERIFY(shardIdsSetOriginal.emplace(i).second);
+ Shards.emplace(i, TShardInfo(i, OrderedShardIds.size()));
+ OrderedShardIds.emplace_back(i);
}
+}
- return out;
+NKikimrSchemeOp::TGranuleShardInfo TShardInfo::SerializeToProto() const {
+ NKikimrSchemeOp::TGranuleShardInfo result;
+ result.SetTabletId(TabletId);
+ result.SetSequenceIdx(SequenceIdx);
+ result.SetIsOpenForWrite(IsOpenForWrite);
+ result.SetIsOpenForRead(IsOpenForRead);
+ if (OpenForWriteSnapshot) {
+ *result.MutableOpenForWriteSnapshot() = OpenForWriteSnapshot->SerializeToProto();
+ }
+ result.SetShardingVersion(ShardingVersion);
+ return result;
+}
+
+NKikimr::TConclusionStatus TShardInfo::DeserializeFromProto(const NKikimrSchemeOp::TGranuleShardInfo& proto) {
+ TabletId = proto.GetTabletId();
+ SequenceIdx = proto.GetSequenceIdx();
+ IsOpenForRead = proto.GetIsOpenForRead();
+ IsOpenForWrite = proto.GetIsOpenForWrite();
+ if (proto.HasOpenForWriteSnapshot()) {
+ NKikimr::NOlap::TSnapshot ss = NKikimr::NOlap::TSnapshot::Zero();
+ auto conclusion = ss.DeserializeFromProto(proto.GetOpenForWriteSnapshot());
+ if (conclusion.IsFail()) {
+ return conclusion;
+ }
+ OpenForWriteSnapshot = ss;
+ }
+ AFL_VERIFY(proto.HasShardingVersion());
+ ShardingVersion = proto.GetShardingVersion();
+ if (!TabletId) {
+ return TConclusionStatus::Fail("incorrect TabletId for TShardInfo proto");
+ }
+ return TConclusionStatus::Success();
}
}
diff --git a/ydb/core/tx/sharding/sharding.h b/ydb/core/tx/sharding/sharding.h
index 3b16bf46ce..1b49ba9743 100644
--- a/ydb/core/tx/sharding/sharding.h
+++ b/ydb/core/tx/sharding/sharding.h
@@ -1,236 +1,320 @@
#pragma once
+#include <ydb/library/accessor/accessor.h>
+#include <ydb/core/tx/schemeshard/olap/schema/schema.h>
+#include <ydb/core/formats/arrow/size_calcer.h>
+#include <ydb/core/tx/columnshard/common/snapshot.h>
-#include "hash.h"
+namespace NKikimrSchemeOp {
+class TColumnTableSharding;
+class TGranuleShardingLogicContainer;
+}
-#include <ydb/core/formats/arrow/arrow_helpers.h>
-#include <ydb/core/formats/arrow/hash/calcer.h>
-#include <ydb/core/formats/arrow/size_calcer.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
-#include <ydb/core/tx/schemeshard/olap/schema/schema.h>
+namespace NKikimr::NSharding {
-#include <ydb/library/accessor/accessor.h>
-#include <ydb/library/conclusion/result.h>
-#include <ydb/library/conclusion/status.h>
+struct TExternalTableColumn;
-#include <contrib/libs/apache/arrow/cpp/src/arrow/compute/api.h>
-#include <contrib/libs/xxhash/xxhash.h>
+class IGranuleShardingLogic {
+public:
+ using TProto = NKikimrSchemeOp::TGranuleShardingLogicContainer;
+ using TFactory = NObjectFactory::TObjectFactory<IGranuleShardingLogic, TString>;
-#include <util/random/random.h>
+private:
+ virtual std::shared_ptr<NArrow::TColumnFilter> DoGetFilter(const std::shared_ptr<arrow::Table>& table) const = 0;
+ virtual std::set<TString> DoGetColumnNames() const = 0;
+ virtual void DoSerializeToProto(TProto& proto) const = 0;
+ virtual TConclusionStatus DoDeserializeFromProto(const TProto& proto) = 0;
-#include <type_traits>
+public:
+ IGranuleShardingLogic() = default;
+ virtual ~IGranuleShardingLogic() = default;
-namespace NKikimr::NSharding {
+ std::shared_ptr<NArrow::TColumnFilter> GetFilter(const std::shared_ptr<arrow::Table>& table) const {
+ return DoGetFilter(table);
+ }
-struct TExternalTableColumn;
+ std::shared_ptr<NArrow::TColumnFilter> GetFilter(const std::shared_ptr<arrow::RecordBatch>& rb) const {
+ return DoGetFilter(NArrow::TStatusValidator::GetValid(arrow::Table::FromRecordBatches({ rb })));
+ }
+ std::set<TString> GetColumnNames() const {
+ return DoGetColumnNames();
+ }
+
+ virtual TString GetClassName() const = 0;
-class TShardingBase {
+ void SerializeToProto(TProto& proto) const {
+ DoSerializeToProto(proto);
+ }
+ TConclusionStatus DeserializeFromProto(const TProto& proto) {
+ return DoDeserializeFromProto(proto);
+ }
+};
+
+class TGranuleShardingLogicContainer: public NBackgroundTasks::TInterfaceProtoContainer<IGranuleShardingLogic> {
private:
- YDB_READONLY_DEF(std::vector<ui64>, ShardIds);
- YDB_READONLY(ui64, Version, 1);
- TShardingBase() = default;
-protected:
- virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const = 0;
+ using TBase = NBackgroundTasks::TInterfaceProtoContainer<IGranuleShardingLogic>;
+ using TProto = IGranuleShardingLogic::TProto;
public:
- using TColumn = TExternalTableColumn;
+ using TBase::TBase;
+};
+
+class TShardInfo {
+private:
+ YDB_READONLY(ui64, TabletId, 0);
+ YDB_ACCESSOR(ui32, SequenceIdx, 0);
+ YDB_ACCESSOR(bool, IsOpenForRead, true);
+ YDB_ACCESSOR(bool, IsOpenForWrite, true);
+ YDB_ACCESSOR_DEF(std::optional<NKikimr::NOlap::TSnapshot>, OpenForWriteSnapshot);
+ YDB_READONLY(ui32, ShardingVersion, 0);
+
+ TShardInfo() = default;
+
+ TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TGranuleShardInfo& proto);
public:
- static TConclusionStatus ValidateBehaviour(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo);
- static TConclusion<std::unique_ptr<TShardingBase>> BuildFromProto(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo);
+ TShardInfo(const ui64 tabletId, const ui32 seqIdx)
+ : TabletId(tabletId)
+ , SequenceIdx(seqIdx)
+ {
- TShardingBase(const std::vector<ui64>& shardIds)
- : ShardIds(shardIds) {
+ }
+ const NKikimr::NOlap::TSnapshot& GetOpenForWriteSnapshotVerified() const {
+ AFL_VERIFY(OpenForWriteSnapshot);
+ return *OpenForWriteSnapshot;
}
- ui32 GetShardsCount() const {
- return ShardIds.size();
+ void IncrementVersion() {
+ ++ShardingVersion;
}
- NKikimrSchemeOp::TColumnTableSharding SerializeToProto() const {
- NKikimrSchemeOp::TColumnTableSharding result;
- result.SetVersion(1);
- AFL_VERIFY(ShardIds.size());
- for (auto&& i : ShardIds) {
- result.AddColumnShards(i);
+ NKikimrSchemeOp::TGranuleShardInfo SerializeToProto() const;
+
+ static TConclusion<TShardInfo> BuildFromProto(const NKikimrSchemeOp::TGranuleShardInfo& proto) {
+ TShardInfo result;
+ auto conclusion = result.DeserializeFromProto(proto);
+ if (conclusion.IsFail()) {
+ return conclusion;
}
- DoSerializeToProto(result);
return result;
}
+};
- virtual std::vector<ui32> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const = 0;
+class IShardingBase {
+private:
+ std::vector<ui64> OrderedShardIds;
+ THashMap<ui64, TShardInfo> Shards;
+ TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto);
- TConclusion<THashMap<ui64, std::vector<NArrow::TSerializedBatch>>> SplitByShards(const std::shared_ptr<arrow::RecordBatch>& batch, const ui64 chunkBytesLimit) {
- auto sharding = MakeSharding(batch);
- std::vector<std::shared_ptr<arrow::RecordBatch>> chunks;
- if (ShardIds.size() == 1) {
- chunks = {batch};
- } else {
- chunks = NArrow::ShardingSplit(batch, sharding, ShardIds.size());
- }
- AFL_VERIFY(chunks.size() == ShardIds.size());
- NArrow::TBatchSplitttingContext context(chunkBytesLimit);
- THashMap<ui64, std::vector<NArrow::TSerializedBatch>> result;
- for (ui32 i = 0; i < chunks.size(); ++i) {
- if (!chunks[i]) {
- continue;
+ void InitializeFromOrdered(const std::vector<ui64>& orderedIds);
+
+protected:
+ std::set<ui64> GetClosedWritingShardIds() const {
+ std::set<ui64> result;
+ for (auto&& i : Shards) {
+ if (!i.second.GetIsOpenForWrite()) {
+ result.emplace(i.first);
}
- auto blobsSplittedConclusion = NArrow::SplitByBlobSize(chunks[i], context);
- if (blobsSplittedConclusion.IsFail()) {
- return TConclusionStatus::Fail("cannot split batch in according to limits: " + blobsSplittedConclusion.GetErrorMessage());
+ }
+ return result;
+ }
+
+ std::set<ui64> GetClosedReadingShardIds() const {
+ std::set<ui64> result;
+ for (auto&& i : Shards) {
+ if (!i.second.GetIsOpenForRead()) {
+ result.emplace(i.first);
}
- result.emplace(ShardIds[i], blobsSplittedConclusion.DetachResult());
}
return result;
}
- virtual TString DebugString() const;
+ void SetOrderedShardIds(const std::vector<ui64>& ids) {
+ ui32 idx = 0;
+ for (auto&& i : ids) {
+ auto* shardInfo = GetShardInfo(i);
+ AFL_VERIFY(shardInfo);
+ shardInfo->SetSequenceIdx(idx++);
+ }
+ OrderedShardIds = ids;
+ AFL_VERIFY(OrderedShardIds.size() == Shards.size());
+ }
- virtual ~TShardingBase() = default;
-};
+ const std::vector<ui64>& GetOrderedShardIds() const {
+ return OrderedShardIds;
+ }
-class THashShardingImpl: public TShardingBase {
-private:
- using TBase = TShardingBase;
- const ui64 Seed;
- const NArrow::NHash::TXX64 HashCalcer;
-protected:
- const std::vector<TString> ShardingColumns;
- virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override {
- for (auto&& i : ShardingColumns) {
- proto.MutableHashSharding()->AddColumns(i);
+ bool HasReadClosedShards() const {
+ for (auto&& [_, i] : Shards) {
+ if (!i.GetIsOpenForRead()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ bool HasWriteClosedShards() const {
+ for (auto&& [_, i] : Shards) {
+ if (!i.GetIsOpenForWrite()) {
+ return true;
+ }
}
+ return false;
+ }
+ virtual TString GetClassName() const = 0;
+ virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const = 0;
+ virtual TConclusionStatus DoDeserializeFromProto(const NKikimrSchemeOp::TColumnTableSharding& proto) = 0;
+ virtual TConclusionStatus DoApplyModification(const NKikimrSchemeOp::TShardingModification& proto) = 0;
+ virtual std::set<ui64> DoGetModifiedShardIds(const NKikimrSchemeOp::TShardingModification& proto) const = 0;
+ virtual TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> DoBuildSplitShardsModifiers(const std::vector<ui64>& /*newTabletIds*/) const {
+ return TConclusionStatus::Fail("shards splitting not implemented for " + GetClassName());
}
+ virtual TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> DoBuildMergeShardsModifiers(const std::vector<ui64>& /*newTabletIds*/) const {
+ return TConclusionStatus::Fail("shards merging not implemented for " + GetClassName());
+ }
+ virtual TConclusionStatus DoOnAfterModification() = 0;
+ virtual TConclusionStatus DoOnBeforeModification() = 0;
+ virtual std::shared_ptr<IGranuleShardingLogic> DoGetTabletShardingInfoOptional(const ui64 tabletId) const = 0;
+
+public:
+ using TColumn = TExternalTableColumn;
public:
- THashShardingImpl(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui64 seed = 0)
- : TBase(shardIds)
- , Seed(seed)
- , HashCalcer(columnNames, NArrow::NHash::TXX64::ENoColumnPolicy::Verify, Seed)
- , ShardingColumns(columnNames) {
+ IShardingBase() = default;
+
+ TShardInfo& GetShardInfoVerified(const ui64 tabletId) {
+ auto it = Shards.find(tabletId);
+ AFL_VERIFY(it != Shards.end());
+ return it->second;
}
- virtual TString DebugString() const override {
- return TBase::DebugString() + ";Columns: " + JoinSeq(", ", GetShardingColumns());
+ const TShardInfo& GetShardInfoVerified(const ui64 tabletId) const {
+ auto it = Shards.find(tabletId);
+ AFL_VERIFY(it != Shards.end());
+ return it->second;
}
- virtual std::vector<ui64> MakeHashes(const std::shared_ptr<arrow::RecordBatch>& batch) const {
- return HashCalcer.Execute(batch).value_or(Default<std::vector<ui64>>());
+ TShardInfo* GetShardInfo(const ui64 tabletId) {
+ auto it = Shards.find(tabletId);
+ if (it == Shards.end()) {
+ return nullptr;
+ }
+ return &it->second;
}
- template <typename T>
- static ui64 CalcHash(const T value, const ui32 seed = 0) {
- static_assert(std::is_arithmetic<T>::value);
- return XXH64(&value, sizeof(value), seed);
+ const TShardInfo* GetShardInfo(const ui64 tabletId) const {
+ auto it = Shards.find(tabletId);
+ if (it == Shards.end()) {
+ return nullptr;
+ }
+ return &it->second;
}
- virtual const std::vector<TString>& GetShardingColumns() const {
- return ShardingColumns;
+ void SetShardingOpenSnapshotVerified(const ui64 tabletId, const NKikimr::NOlap::TSnapshot& ss) {
+ GetShardInfoVerified(tabletId).SetOpenForWriteSnapshot(ss);
}
-};
-class THashShardingModuloN : public THashShardingImpl {
-private:
- using TBase = THashShardingImpl;
-protected:
- virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override {
- TBase::DoSerializeToProto(proto);
- proto.MutableHashSharding()->SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_MODULO_N);
+ NKikimr::NOlap::TSnapshot GetShardingOpenSnapshotVerified(const ui64 tabletId) const {
+ return GetShardInfoVerified(tabletId).GetOpenForWriteSnapshotVerified();
}
-public:
- THashShardingModuloN(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui64 seed = 0)
- : TBase(shardIds, columnNames, seed)
- {}
- virtual std::vector<ui32> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override;
-};
+ TConclusionStatus OnAfterModification() {
+ return DoOnAfterModification();
+ }
-class TRandomSharding: public TShardingBase {
-private:
- using TBase = TShardingBase;
-protected:
- virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override {
- proto.MutableRandomSharding();
+ TConclusionStatus OnBeforeModification() {
+ return DoOnBeforeModification();
}
-public:
- using TBase::TBase;
- virtual std::vector<ui32> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override {
- return std::vector<ui32>(batch->num_rows(), RandomNumber<ui32>(GetShardsCount()));
+ TGranuleShardingLogicContainer GetTabletShardingInfoOptional(const ui64 tabletId) const {
+ if (IsShardClosedForWrite(tabletId)) {
+ return TGranuleShardingLogicContainer();
+ }
+ return TGranuleShardingLogicContainer(DoGetTabletShardingInfoOptional(tabletId));
}
-};
+ TConclusionStatus ApplyModification(const NKikimrSchemeOp::TShardingModification& proto);
+ std::set<ui64> GetModifiedShardIds(const NKikimrSchemeOp::TShardingModification& proto) const {
+ return DoGetModifiedShardIds(proto);
+ }
-class TConsistencySharding64: public THashShardingImpl {
-private:
- using TBase = THashShardingImpl;
+ TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> BuildSplitShardsModifiers(const std::vector<ui64>& newTabletIds) const {
+ return DoBuildSplitShardsModifiers(newTabletIds);
+ }
+
+ TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> BuildMergeShardsModifiers(const std::vector<ui64>& newTabletIds) const {
+ return DoBuildMergeShardsModifiers(newTabletIds);
+ }
+
+ TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> BuildAddShardsModifiers(const std::vector<ui64>& newTabletIds) const;
+ TConclusion<std::vector<NKikimrSchemeOp::TAlterShards>> BuildReduceShardsModifiers(const std::vector<ui64>& newTabletIds) const;
- static ui32 CalcShardIdImpl(const ui64 hash, const ui32 shardsCount) {
- AFL_VERIFY(shardsCount);
- return std::min<ui32>(hash / (Max<ui64>() / shardsCount), shardsCount - 1);
+ void CloseShardWriting(const ui64 shardId) {
+ GetShardInfoVerified(shardId).SetIsOpenForWrite(false);
}
- virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override {
- TBase::DoSerializeToProto(proto);
- proto.MutableHashSharding()->SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CONSISTENCY_64);
+ void CloseShardReading(const ui64 shardId) {
+ GetShardInfoVerified(shardId).SetIsOpenForRead(false);
}
-public:
- TConsistencySharding64(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui64 seed = 0)
- : TBase(shardIds, columnNames, seed){
+
+ bool IsActiveForWrite(const ui64 tabletId) const {
+ return GetShardInfoVerified(tabletId).GetIsOpenForWrite();
}
- virtual std::vector<ui32> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override {
- auto hashes = MakeHashes(batch);
- std::vector<ui32> result;
- result.reserve(hashes.size());
- for (auto&& i : hashes) {
- result.emplace_back(CalcShardIdImpl(i, GetShardsCount()));
+ bool IsActiveForRead(const ui64 tabletId) const {
+ return GetShardInfoVerified(tabletId).GetIsOpenForRead();
+ }
+
+ std::vector<ui64> GetActiveReadShardIds() const {
+ std::vector<ui64> result;
+ for (auto&& [_, i] : Shards) {
+ if (i.GetIsOpenForRead()) {
+ result.emplace_back(i.GetTabletId());
+ }
}
return result;
}
- template <typename T>
- static ui32 ShardNo(const T value, const ui32 shardsCount, const ui32 seed = 0) {
- Y_ASSERT(shardsCount);
- return CalcShardIdImpl(CalcHash(value, seed), shardsCount);
+ ui64 GetShardIdByOrderIdx(const ui32 shardIdx) const {
+ AFL_VERIFY(shardIdx < OrderedShardIds.size());
+ return OrderedShardIds[shardIdx];
}
-};
-// KIKIMR-11529
-class TLogsSharding : public THashShardingImpl {
-private:
- using TBase = THashShardingImpl;
- ui32 NumActive;
- ui64 TsMin;
- ui64 ChangePeriod;
+ bool IsShardExists(const ui64 shardId) const {
+ return Shards.contains(shardId);
+ }
- virtual void DoSerializeToProto(NKikimrSchemeOp::TColumnTableSharding& proto) const override {
- TBase::DoSerializeToProto(proto);
- proto.MutableHashSharding()->SetFunction(NKikimrSchemeOp::TColumnTableSharding::THashSharding::HASH_FUNCTION_CLOUD_LOGS);
- proto.MutableHashSharding()->SetActiveShardsCount(NumActive);
+ bool IsShardClosedForRead(const ui64 shardId) const {
+ return !IsActiveForRead(shardId);
}
-public:
- static constexpr ui32 DEFAULT_ACITVE_SHARDS = 10;
- static constexpr TDuration DEFAULT_CHANGE_PERIOD = TDuration::Minutes(5);
-
- TLogsSharding(const std::vector<ui64>& shardIds, const std::vector<TString>& columnNames, ui32 shardsCountActive, TDuration changePeriod = DEFAULT_CHANGE_PERIOD)
- : TBase(shardIds, columnNames)
- , NumActive(Min<ui32>(shardsCountActive, GetShardsCount()))
- , TsMin(0)
- , ChangePeriod(changePeriod.MicroSeconds())
- {}
-
- // tsMin = GetTsMin(tabletIdsMap, timestamp);
- // tabletIds = GetTableIdsByTs(tabletIdsMap, timestamp);
- // numIntervals = tabletIds.size() / nActive;
- // tsInterval = (timestamp - tsMin) / changePeriod;
- // shardNo = (hash(uid) % nActive) + (tsInterval % numIntervals) * nActive;
- // tabletId = tabletIds[shardNo];
- ui32 ShardNo(ui64 timestamp, const ui64 uidHash) const {
- ui32 tsInterval = (timestamp - TsMin) / ChangePeriod;
- ui32 numIntervals = GetShardsCount() / NumActive;
- return ((uidHash % NumActive) + (tsInterval % numIntervals) * NumActive) % GetShardsCount();
- }
-
- virtual std::vector<ui32> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const override;
+ bool IsShardClosedForWrite(const ui64 shardId) const {
+ return !IsActiveForWrite(shardId);
+ }
+
+ static TConclusionStatus ValidateBehaviour(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo);
+ static TConclusion<std::unique_ptr<IShardingBase>> BuildFromProto(const NSchemeShard::TOlapSchema& schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo) {
+ return BuildFromProto(&schema, shardingInfo);
+ }
+ static TConclusion<std::unique_ptr<IShardingBase>> BuildFromProto(const NSchemeShard::TOlapSchema* schema, const NKikimrSchemeOp::TColumnTableSharding& shardingInfo);
+ static TConclusion<std::unique_ptr<IShardingBase>> BuildFromProto(const NKikimrSchemeOp::TColumnTableSharding& shardingInfo) {
+ return BuildFromProto(nullptr, shardingInfo);
+ }
+
+ IShardingBase(const std::vector<ui64>& shardIds) {
+ InitializeFromOrdered(shardIds);
+ }
+
+ ui32 GetShardsCount() const {
+ return Shards.size();
+ }
+
+ NKikimrSchemeOp::TColumnTableSharding SerializeToProto() const;
+
+ virtual THashMap<ui64, std::vector<ui32>> MakeSharding(const std::shared_ptr<arrow::RecordBatch>& batch) const = 0;
+
+ TConclusion<THashMap<ui64, std::vector<NArrow::TSerializedBatch>>> SplitByShards(const std::shared_ptr<arrow::RecordBatch>& batch, const ui64 chunkBytesLimit);
+
+ virtual TString DebugString() const;
+
+ virtual ~IShardingBase() = default;
};
}
diff --git a/ydb/core/tx/sharding/ya.make b/ydb/core/tx/sharding/ya.make
index d0ae61d41a..9f6d68f5fb 100644
--- a/ydb/core/tx/sharding/ya.make
+++ b/ydb/core/tx/sharding/ya.make
@@ -6,6 +6,7 @@ PEERDIR(
ydb/library/yql/public/udf
ydb/core/formats/arrow/hash
ydb/core/tx/schemeshard/olap/schema
+ ydb/core/tx/columnshard/common
ydb/core/formats
ydb/core/protos
)
@@ -16,6 +17,10 @@ SRCS(
sharding.cpp
hash.cpp
unboxed_reader.cpp
+ hash_slider.cpp
+ GLOBAL hash_modulo.cpp
+ GLOBAL hash_intervals.cpp
+ random.cpp
)
END()
diff --git a/ydb/library/conclusion/status.h b/ydb/library/conclusion/status.h
index 8e798f0583..8af77479de 100644
--- a/ydb/library/conclusion/status.h
+++ b/ydb/library/conclusion/status.h
@@ -32,43 +32,43 @@ private:
public:
void Validate(const TString& processInfo = Default<TString>()) const;
- const TString& GetErrorMessage() const {
+ [[nodiscard]] const TString& GetErrorMessage() const {
return ErrorMessage ? *ErrorMessage : Default<TString>();
}
- Ydb::StatusIds::StatusCode GetStatus() const {
+ [[nodiscard]] Ydb::StatusIds::StatusCode GetStatus() const {
return Status;
}
- static TConclusionStatus Fail(const char* errorMessage) {
+ [[nodiscard]] static TConclusionStatus Fail(const char* errorMessage) {
return TConclusionStatus(errorMessage);
}
- static TConclusionStatus Fail(const TString& errorMessage) {
+ [[nodiscard]] static TConclusionStatus Fail(const TString& errorMessage) {
return TConclusionStatus(errorMessage);
}
- static TConclusionStatus Fail(const std::string& errorMessage) {
+ [[nodiscard]] static TConclusionStatus Fail(const std::string& errorMessage) {
return TConclusionStatus(errorMessage);
}
- bool IsFail() const {
+ [[nodiscard]] bool IsFail() const {
return !Ok();
}
- bool IsSuccess() const {
+ [[nodiscard]] bool IsSuccess() const {
return Ok();
}
- bool Ok() const {
+ [[nodiscard]] bool Ok() const {
return !ErrorMessage;
}
- bool operator!() const {
+ [[nodiscard]] bool operator!() const {
return !!ErrorMessage;
}
- static TConclusionStatus Success() {
+ [[nodiscard]] static TConclusionStatus Success() {
return TConclusionStatus();
}
};
diff --git a/ydb/services/bg_tasks/abstract/interface.h b/ydb/services/bg_tasks/abstract/interface.h
index ad249c2c08..68c9ab9a23 100644
--- a/ydb/services/bg_tasks/abstract/interface.h
+++ b/ydb/services/bg_tasks/abstract/interface.h
@@ -381,6 +381,21 @@ public:
Object->SerializeToProto(result);
TOperatorPolicy::SetClassName(result, Object->GetClassName());
}
+
+ TString SerializeToString() const {
+ return SerializeToProto().SerializeAsString();
+ }
+
+ TConclusionStatus DeserializeFromString(const TString& data) {
+ TProto proto;
+ if (!proto.ParseFromArray(data.data(), data.size())) {
+ return TConclusionStatus::Fail("cannot parse string as proto");
+ }
+ if (!DeserializeFromProto(proto)) {
+ return TConclusionStatus::Fail("cannot parse proto in container");
+ }
+ return TConclusionStatus::Success();
+ }
};
}