cleaning and special methods for memory control helping

6 files changed, 41 insertions, 5 deletions

diff --git a/ydb/core/tx/columnshard/engines/column_engine.h b/ydb/core/tx/columnshard/engines/column_engine.h
index e05ec552e2..6f107e84b3 100644
--- a/ydb/core/tx/columnshard/engines/column_engine.h
+++ b/ydb/core/tx/columnshard/engines/column_engine.h
@@ -259,7 +259,7 @@ struct TSelectInfo {
         return NColumnShard::TContainerAccessorWithDirection<std::vector<TPortionInfo>>(Portions, reverse);
     }
 
-    NColumnShard::TContainerAccessorWithDirection<std::vector<TGranuleRecord>> GetGranulesOrdered(bool reverse = false) const {
+    NColumnShard::TContainerAccessorWithDirection<std::vector<TGranuleRecord>> GetGranulesOrdered(const bool reverse) const {
         return NColumnShard::TContainerAccessorWithDirection<std::vector<TGranuleRecord>>(Granules, reverse);
     }
 
diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp
index 81baad9463..81c4b293c8 100644
--- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp
+++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp
@@ -1166,11 +1166,11 @@ std::shared_ptr<TSelectInfo> TColumnEngineForLogs::Select(ui64 pathId, TSnapshot
                     }
                     Y_VERIFY(outPortion.Produced());
                     if (!pkRangesFilter.IsPortionInUsage(outPortion, GetIndexInfo())) {
-                        AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "portion_skipped")
+                        AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "portion_skipped")
                             ("granule", granule)("portion", portionInfo->Portion());
                         continue;
                     } else {
-                        AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "portion_selected")
+                        AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "portion_selected")
                             ("granule", granule)("portion", portionInfo->Portion());
                     }
                     out->Portions.emplace_back(std::move(outPortion));
diff --git a/ydb/core/tx/columnshard/engines/column_features.cpp b/ydb/core/tx/columnshard/engines/column_features.cpp
index 019217fe8d..dd6341ec7a 100644
--- a/ydb/core/tx/columnshard/engines/column_features.cpp
+++ b/ydb/core/tx/columnshard/engines/column_features.cpp
@@ -2,6 +2,7 @@
 #include "index_info.h"
 #include <ydb/core/formats/arrow/serializer/full.h>
 #include <ydb/core/formats/arrow/serializer/batch_only.h>
+#include <util/string/builder.h>
 
 namespace NKikimr::NOlap {
 
@@ -61,4 +62,18 @@ std::unique_ptr<arrow::util::Codec> TColumnFeatures::GetCompressionCodec() const
     }
 }
 
+TString TColumnLoader::DebugString() const {
+    TStringBuilder result;
+    if (ExpectedSchema) {
+        result << "schema:" << ExpectedSchema->ToString() << ";";
+    }
+    if (Transformer) {
+        result << "transformer:" << Transformer->DebugString() << ";";
+    }
+    if (Deserializer) {
+        result << "deserializer:" << Deserializer->DebugString() << ";";
+    }
+    return result;
+}
+
 } // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/engines/column_features.h b/ydb/core/tx/columnshard/engines/column_features.h
index a64cc44b77..bb5e79428a 100644
--- a/ydb/core/tx/columnshard/engines/column_features.h
+++ b/ydb/core/tx/columnshard/engines/column_features.h
@@ -57,6 +57,8 @@ private:
     std::shared_ptr<arrow::Schema> ExpectedSchema;
     const ui32 ColumnId;
 public:
+    TString DebugString() const;
+
     TColumnLoader(NArrow::NTransformation::ITransformer::TPtr transformer, NArrow::NSerialization::IDeserializer::TPtr deserializer,
         const std::shared_ptr<arrow::Schema>& expectedSchema, const ui32 columnId)
         : Transformer(transformer)
diff --git a/ydb/core/tx/columnshard/engines/index_info.h b/ydb/core/tx/columnshard/engines/index_info.h
index 9d037c509a..33f98892d1 100644
--- a/ydb/core/tx/columnshard/engines/index_info.h
+++ b/ydb/core/tx/columnshard/engines/index_info.h
@@ -59,6 +59,10 @@ public:
     /// Matches name of the filed with names of the special columns.
     static bool IsSpecialColumn(const arrow::Field& field);
     static bool IsSpecialColumn(const ui32 field);
+    static ui32 GetSpecialColumnByteWidth(const ui32 field) {
+        Y_VERIFY(IsSpecialColumn(field));
+        return 8;
+    }
     static bool IsSpecialColumn(const std::string& fieldName);
     template <class TContainer>
     static bool IsSpecialColumns(const TContainer& c) {
diff --git a/ydb/core/tx/columnshard/engines/portion_info.h b/ydb/core/tx/columnshard/engines/portion_info.h
index 13e1420a16..d79e67dc51 100644
--- a/ydb/core/tx/columnshard/engines/portion_info.h
+++ b/ydb/core/tx/columnshard/engines/portion_info.h
@@ -240,6 +240,21 @@ struct TPortionInfo {
         return Meta.NumRows();
     }
 
+    ui64 GetRawBytes(const std::vector<ui32>& columnIds) const {
+        ui64 sum = 0;
+        const ui32 numRows = NumRows();
+        for (auto&& i : columnIds) {
+            if (TIndexInfo::IsSpecialColumn(i)) {
+                sum += numRows * TIndexInfo::GetSpecialColumnByteWidth(i);
+            } else {
+                auto it = Meta.ColumnMeta.find(i);
+                Y_VERIFY(it != Meta.ColumnMeta.end());
+                sum += it->second.RawBytes;
+            }
+        }
+        return sum;
+    }
+
     ui64 RawBytesSum() const {
         ui64 sum = 0;
         for (auto& [columnId, colMeta] : Meta.ColumnMeta) {
@@ -324,10 +339,10 @@ public:
             } else {
                 auto result = loader.Apply(Data);
                 if (!result.ok()) {
-                    AFL_ERROR(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "cannot unpack batch")("error", result.status().ToString());
+                    AFL_ERROR(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "cannot unpack batch")("error", result.status().ToString())("loader", loader.DebugString());
                     return nullptr;
                 }
-                return NArrow::DictionaryToArray(*result);
+                return *result;
             }
         }
     };