summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorivanmorozov333 <[email protected]>2024-07-07 08:58:34 +0300
committerGitHub <[email protected]>2024-07-07 08:58:34 +0300
commit0e7a24c74ed41be1ddd6948ddf096d9521ce808d (patch)
tree7c5143086be3a1b3426054f45b4e5f3c3c70d79d
parent1add9d778775150d4009a68367afd274eaf195ad (diff)
serialize special keys in separated thread (#6324)
-rw-r--r--ydb/core/formats/arrow/size_calcer.cpp7
-rw-r--r--ydb/core/formats/arrow/size_calcer.h6
-rw-r--r--ydb/core/tx/columnshard/background_controller.cpp2
-rw-r--r--ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp2
4 files changed, 9 insertions, 8 deletions
diff --git a/ydb/core/formats/arrow/size_calcer.cpp b/ydb/core/formats/arrow/size_calcer.cpp
index d44018faaa7..c718b780741 100644
--- a/ydb/core/formats/arrow/size_calcer.cpp
+++ b/ydb/core/formats/arrow/size_calcer.cpp
@@ -242,11 +242,12 @@ ui64 GetArrayDataSize(const std::shared_ptr<arrow::Array>& column) {
}
NKikimr::NArrow::TSerializedBatch TSerializedBatch::Build(std::shared_ptr<arrow::RecordBatch> batch, const TBatchSplitttingContext& context) {
- std::optional<TFirstLastSpecialKeys> specialKeys;
+ std::optional<TString> specialKeys;
if (context.GetFieldsForSpecialKeys().size()) {
- specialKeys = TFirstLastSpecialKeys(batch, context.GetFieldsForSpecialKeys());
+ specialKeys = TFirstLastSpecialKeys(batch, context.GetFieldsForSpecialKeys()).SerializeToString();
}
- return TSerializedBatch(NArrow::SerializeSchema(*batch->schema()), NArrow::SerializeBatchNoCompression(batch), batch->num_rows(), NArrow::GetBatchDataSize(batch), specialKeys);
+ return TSerializedBatch(NArrow::SerializeSchema(*batch->schema()), NArrow::SerializeBatchNoCompression(batch), batch->num_rows(),
+ NArrow::GetBatchDataSize(batch), specialKeys);
}
TConclusionStatus TSerializedBatch::BuildWithLimit(std::shared_ptr<arrow::RecordBatch> batch, const TBatchSplitttingContext& context, std::optional<TSerializedBatch>& sbL, std::optional<TSerializedBatch>& sbR) {
diff --git a/ydb/core/formats/arrow/size_calcer.h b/ydb/core/formats/arrow/size_calcer.h
index 410d5517bc8..d260427ade3 100644
--- a/ydb/core/formats/arrow/size_calcer.h
+++ b/ydb/core/formats/arrow/size_calcer.h
@@ -74,13 +74,13 @@ private:
YDB_READONLY_DEF(TString, Data);
YDB_READONLY(ui32, RowsCount, 0);
YDB_READONLY(ui32, RawBytes, 0);
- std::optional<TFirstLastSpecialKeys> SpecialKeys;
+ std::optional<TString> SpecialKeys;
public:
size_t GetSize() const {
return Data.size();
}
- const TFirstLastSpecialKeys& GetSpecialKeysSafe() const {
+ const TString& GetSpecialKeysSafe() const {
AFL_VERIFY(SpecialKeys);
return *SpecialKeys;
}
@@ -95,7 +95,7 @@ public:
static TConclusionStatus BuildWithLimit(std::shared_ptr<arrow::RecordBatch> batch, const TBatchSplitttingContext& context, std::optional<TSerializedBatch>& sbL, std::optional<TSerializedBatch>& sbR);
static TSerializedBatch Build(std::shared_ptr<arrow::RecordBatch> batch, const TBatchSplitttingContext& context);
- TSerializedBatch(TString&& schemaData, TString&& data, const ui32 rowsCount, const ui32 rawBytes, const std::optional<TFirstLastSpecialKeys>& specialKeys)
+ TSerializedBatch(TString&& schemaData, TString&& data, const ui32 rowsCount, const ui32 rawBytes, const std::optional<TString>& specialKeys)
: SchemaData(schemaData)
, Data(data)
, RowsCount(rowsCount)
diff --git a/ydb/core/tx/columnshard/background_controller.cpp b/ydb/core/tx/columnshard/background_controller.cpp
index fa6a2b32eb8..7449e7d31ff 100644
--- a/ydb/core/tx/columnshard/background_controller.cpp
+++ b/ydb/core/tx/columnshard/background_controller.cpp
@@ -20,7 +20,7 @@ void TBackgroundController::CheckDeadlines() {
void TBackgroundController::CheckDeadlinesIndexation() {
for (auto&& i : ActiveIndexationTasks) {
if (TMonotonic::Now() - i.second > NOlap::TCompactionLimits::CompactionTimeout) {
- AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "deadline_compaction")("task_id", i.first);
+ AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "deadline_indexation")("task_id", i.first);
Y_DEBUG_ABORT_UNLESS(false);
}
}
diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp
index 853bcb7c546..76e27462fa0 100644
--- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp
+++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp
@@ -7,7 +7,7 @@ bool TTxWrite::InsertOneBlob(TTransactionContext& txc, const NOlap::TWideSeriali
meta.SetNumRows(batch->GetRowsCount());
meta.SetRawBytes(batch->GetRawBytes());
meta.SetDirtyWriteTimeSeconds(batch.GetStartInstant().Seconds());
- meta.SetSpecialKeysRawData(batch->GetSpecialKeysSafe().SerializeToString());
+ meta.SetSpecialKeysRawData(batch->GetSpecialKeysSafe());
const auto& blobRange = batch.GetRange();
Y_ABORT_UNLESS(blobRange.GetBlobId().IsValid());