diff options
| author | ivanmorozov333 <[email protected]> | 2024-07-07 08:58:34 +0300 |
|---|---|---|
| committer | GitHub <[email protected]> | 2024-07-07 08:58:34 +0300 |
| commit | 0e7a24c74ed41be1ddd6948ddf096d9521ce808d (patch) | |
| tree | 7c5143086be3a1b3426054f45b4e5f3c3c70d79d | |
| parent | 1add9d778775150d4009a68367afd274eaf195ad (diff) | |
serialize special keys in separated thread (#6324)
| -rw-r--r-- | ydb/core/formats/arrow/size_calcer.cpp | 7 | ||||
| -rw-r--r-- | ydb/core/formats/arrow/size_calcer.h | 6 | ||||
| -rw-r--r-- | ydb/core/tx/columnshard/background_controller.cpp | 2 | ||||
| -rw-r--r-- | ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp | 2 |
4 files changed, 9 insertions, 8 deletions
diff --git a/ydb/core/formats/arrow/size_calcer.cpp b/ydb/core/formats/arrow/size_calcer.cpp index d44018faaa7..c718b780741 100644 --- a/ydb/core/formats/arrow/size_calcer.cpp +++ b/ydb/core/formats/arrow/size_calcer.cpp @@ -242,11 +242,12 @@ ui64 GetArrayDataSize(const std::shared_ptr<arrow::Array>& column) { } NKikimr::NArrow::TSerializedBatch TSerializedBatch::Build(std::shared_ptr<arrow::RecordBatch> batch, const TBatchSplitttingContext& context) { - std::optional<TFirstLastSpecialKeys> specialKeys; + std::optional<TString> specialKeys; if (context.GetFieldsForSpecialKeys().size()) { - specialKeys = TFirstLastSpecialKeys(batch, context.GetFieldsForSpecialKeys()); + specialKeys = TFirstLastSpecialKeys(batch, context.GetFieldsForSpecialKeys()).SerializeToString(); } - return TSerializedBatch(NArrow::SerializeSchema(*batch->schema()), NArrow::SerializeBatchNoCompression(batch), batch->num_rows(), NArrow::GetBatchDataSize(batch), specialKeys); + return TSerializedBatch(NArrow::SerializeSchema(*batch->schema()), NArrow::SerializeBatchNoCompression(batch), batch->num_rows(), + NArrow::GetBatchDataSize(batch), specialKeys); } TConclusionStatus TSerializedBatch::BuildWithLimit(std::shared_ptr<arrow::RecordBatch> batch, const TBatchSplitttingContext& context, std::optional<TSerializedBatch>& sbL, std::optional<TSerializedBatch>& sbR) { diff --git a/ydb/core/formats/arrow/size_calcer.h b/ydb/core/formats/arrow/size_calcer.h index 410d5517bc8..d260427ade3 100644 --- a/ydb/core/formats/arrow/size_calcer.h +++ b/ydb/core/formats/arrow/size_calcer.h @@ -74,13 +74,13 @@ private: YDB_READONLY_DEF(TString, Data); YDB_READONLY(ui32, RowsCount, 0); YDB_READONLY(ui32, RawBytes, 0); - std::optional<TFirstLastSpecialKeys> SpecialKeys; + std::optional<TString> SpecialKeys; public: size_t GetSize() const { return Data.size(); } - const TFirstLastSpecialKeys& GetSpecialKeysSafe() const { + const TString& GetSpecialKeysSafe() const { AFL_VERIFY(SpecialKeys); return *SpecialKeys; } @@ -95,7 +95,7 @@ public: static TConclusionStatus BuildWithLimit(std::shared_ptr<arrow::RecordBatch> batch, const TBatchSplitttingContext& context, std::optional<TSerializedBatch>& sbL, std::optional<TSerializedBatch>& sbR); static TSerializedBatch Build(std::shared_ptr<arrow::RecordBatch> batch, const TBatchSplitttingContext& context); - TSerializedBatch(TString&& schemaData, TString&& data, const ui32 rowsCount, const ui32 rawBytes, const std::optional<TFirstLastSpecialKeys>& specialKeys) + TSerializedBatch(TString&& schemaData, TString&& data, const ui32 rowsCount, const ui32 rawBytes, const std::optional<TString>& specialKeys) : SchemaData(schemaData) , Data(data) , RowsCount(rowsCount) diff --git a/ydb/core/tx/columnshard/background_controller.cpp b/ydb/core/tx/columnshard/background_controller.cpp index fa6a2b32eb8..7449e7d31ff 100644 --- a/ydb/core/tx/columnshard/background_controller.cpp +++ b/ydb/core/tx/columnshard/background_controller.cpp @@ -20,7 +20,7 @@ void TBackgroundController::CheckDeadlines() { void TBackgroundController::CheckDeadlinesIndexation() { for (auto&& i : ActiveIndexationTasks) { if (TMonotonic::Now() - i.second > NOlap::TCompactionLimits::CompactionTimeout) { - AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "deadline_compaction")("task_id", i.first); + AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "deadline_indexation")("task_id", i.first); Y_DEBUG_ABORT_UNLESS(false); } } diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp index 853bcb7c546..76e27462fa0 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp @@ -7,7 +7,7 @@ bool TTxWrite::InsertOneBlob(TTransactionContext& txc, const NOlap::TWideSeriali meta.SetNumRows(batch->GetRowsCount()); meta.SetRawBytes(batch->GetRawBytes()); meta.SetDirtyWriteTimeSeconds(batch.GetStartInstant().Seconds()); - meta.SetSpecialKeysRawData(batch->GetSpecialKeysSafe().SerializeToString()); + meta.SetSpecialKeysRawData(batch->GetSpecialKeysSafe()); const auto& blobRange = batch.GetRange(); Y_ABORT_UNLESS(blobRange.GetBlobId().IsValid()); |
