YT-20159: Keep segment meta in blocks

6 files changed, 32 insertions, 0 deletions

diff --git a/yt/yt/client/table_client/config.cpp b/yt/yt/client/table_client/config.cpp
index e3f8ae5da97..8c9edb04779 100644
--- a/yt/yt/client/table_client/config.cpp
+++ b/yt/yt/client/table_client/config.cpp
@@ -368,6 +368,8 @@ void TChunkWriterOptions::Register(TRegistrar registrar)
         .Default(true);
     registrar.Parameter("enable_row_count_in_columnar_statistics", &TThis::EnableRowCountInColumnarStatistics)
         .Default(true);
+    registrar.Parameter("enable_segment_meta_in_blocks", &TThis::EnableSegmentMetaInBlocks)
+        .Default(false);
 
     registrar.Parameter("schema_modification", &TThis::SchemaModification)
         .Default(ETableSchemaModification::None);
diff --git a/yt/yt/client/table_client/config.h b/yt/yt/client/table_client/config.h
index 3ee6378ff6d..df9869f549a 100644
--- a/yt/yt/client/table_client/config.h
+++ b/yt/yt/client/table_client/config.h
@@ -340,6 +340,8 @@ public:
     bool SingleColumnGroupByDefault = false;
     bool EnableColumnarValueStatistics;
     bool EnableRowCountInColumnarStatistics;
+    bool EnableSegmentMetaInBlocks;
+
     NYTree::INodePtr CastAnyToCompositeNode;
 
     ETableSchemaModification SchemaModification;
diff --git a/yt/yt/client/table_client/public.h b/yt/yt/client/table_client/public.h
index da28a221a1b..b2deadf01a7 100644
--- a/yt/yt/client/table_client/public.h
+++ b/yt/yt/client/table_client/public.h
@@ -30,6 +30,7 @@ class TSortColumnsExt;
 class TBoundaryKeysExt;
 class TBlockIndexesExt;
 class TDataBlockMetaExt;
+class TColumnGroupInfosExt;
 class TSystemBlockMetaExt;
 class TColumnarStatisticsExt;
 class TDataBlockMeta;
@@ -52,6 +53,11 @@ using TRefCountedDataBlockMetaPtr = TIntrusivePtr<TRefCountedDataBlockMeta>;
 
 ////////////////////////////////////////////////////////////////////////////////
 
+using TRefCountedColumnGroupInfosExt = TRefCountedProto<NProto::TColumnGroupInfosExt>;
+using TRefCountedColumnGroupInfosExtPtr = TIntrusivePtr<TRefCountedColumnGroupInfosExt>;
+
+////////////////////////////////////////////////////////////////////////////////
+
 using TRefCountedColumnMeta = TRefCountedProto<NProto::TColumnMetaExt>;
 using TRefCountedColumnMetaPtr = TIntrusivePtr<TRefCountedColumnMeta>;
 
diff --git a/yt/yt/core/misc/bit_packed_unsigned_vector-inl.h b/yt/yt/core/misc/bit_packed_unsigned_vector-inl.h
index fac8faad7b9..83df7d6708a 100644
--- a/yt/yt/core/misc/bit_packed_unsigned_vector-inl.h
+++ b/yt/yt/core/misc/bit_packed_unsigned_vector-inl.h
@@ -4,6 +4,8 @@
 #include "bit_packed_unsigned_vector.h"
 #endif
 
+#include "bit_packing.h"
+
 #include <util/generic/bitops.h>
 
 namespace NYT {
@@ -79,6 +81,14 @@ BitPackUnsignedVector(TRange<T> values, ui64 maxValue, ui64* dst)
     return (offset == 0 ? 0 : 1) + word - dst;
 }
 
+template <class T>
+TSharedRef BitpackVector(TRange<T> values, ui64 maxValue, ui32* size, ui8* width)
+{
+    auto data = BitPackUnsignedVector(values, maxValue);
+    *size = GetCompressedVectorSize(data.Begin());
+    *width = GetCompressedVectorWidth(data.Begin());
+    return data;
+}
 
 template <class T>
 typename std::enable_if<std::is_unsigned<T>::value, TSharedRef>::type
diff --git a/yt/yt/core/misc/bit_packing.h b/yt/yt/core/misc/bit_packing.h
index 8023c584983..35c35f9255b 100644
--- a/yt/yt/core/misc/bit_packing.h
+++ b/yt/yt/core/misc/bit_packing.h
@@ -74,6 +74,11 @@ private:
 
 ////////////////////////////////////////////////////////////////////////////////
 
+size_t GetCompressedVectorSize(const void* ptr);
+size_t GetCompressedVectorWidth(const void* ptr);
+
+////////////////////////////////////////////////////////////////////////////////
+
 } // namespace NYT
 
 #define BIT_PACKING_INL_H_
diff --git a/yt/yt_proto/yt/client/table_chunk_format/proto/chunk_meta.proto b/yt/yt_proto/yt/client/table_chunk_format/proto/chunk_meta.proto
index dfd76990c09..82c426a503f 100644
--- a/yt/yt_proto/yt/client/table_chunk_format/proto/chunk_meta.proto
+++ b/yt/yt_proto/yt/client/table_chunk_format/proto/chunk_meta.proto
@@ -238,6 +238,13 @@ message TColumnMetaExt
     repeated NTableChunkFormat.NProto.TColumnMeta columns = 1;
 }
 
+message TColumnGroupInfosExt
+{
+    repeated int32 block_group_indexes = 1;
+    repeated int32 segment_meta_offsets = 2;
+    repeated int32 column_to_group = 3;
+}
+
 message TColumnarStatisticsExt
 {
     repeated int64 column_data_weights = 1;