disable headers check (#16303)

7 files changed, 26 insertions, 15 deletions

diff --git a/ydb/core/formats/arrow/program/graph_optimization.cpp b/ydb/core/formats/arrow/program/graph_optimization.cpp
index a907512237d..e541b74442d 100644
--- a/ydb/core/formats/arrow/program/graph_optimization.cpp
+++ b/ydb/core/formats/arrow/program/graph_optimization.cpp
@@ -693,16 +693,16 @@ TConclusionStatus TGraph::Collapse() {
                 }
             }
 
-            {
-                auto conclusion = OptimizeConditionsForHeadersCheck(n.get());
-                if (conclusion.IsFail()) {
-                    return conclusion;
-                }
-                if (*conclusion) {
-                    hasChanges = true;
-                    break;
-                }
-            }
+//            {
+//                auto conclusion = OptimizeConditionsForHeadersCheck(n.get());
+//                if (conclusion.IsFail()) {
+//                    return conclusion;
+//                }
+//                if (*conclusion) {
+//                    hasChanges = true;
+//                    break;
+//                }
+//            }
 
             {
                 auto conclusion = OptimizeConditionsForStream(n.get());
diff --git a/ydb/core/formats/arrow/ut/ut_program_step.cpp b/ydb/core/formats/arrow/ut/ut_program_step.cpp
index bea0dcc463a..687777f3312 100644
--- a/ydb/core/formats/arrow/ut/ut_program_step.cpp
+++ b/ydb/core/formats/arrow/ut/ut_program_step.cpp
@@ -587,7 +587,7 @@ Y_UNIT_TEST_SUITE(ProgramStep) {
         builder.Add(std::make_shared<TProjectionProcessor>(TColumnChainInfo::BuildVector({ 1, 2 })));
         auto chain = builder.Finish().DetachResult();
         Cerr << chain->DebugDOT() << Endl;
-        AFL_VERIFY(chain->DebugStats() == "[TOTAL:Const:2;Calculation:4;Projection:1;Filter:1;FetchOriginalData:2;AssembleOriginalData:3;CheckIndexData:1;CheckHeaderData:1;StreamLogic:1;];SUB:[FetchOriginalData:1;AssembleOriginalData:1;CheckHeaderData:1;];")("debug", chain->DebugStats());
+        AFL_VERIFY(chain->DebugStats() == "[TOTAL:Const:2;Calculation:4;Projection:1;Filter:1;FetchOriginalData:2;AssembleOriginalData:3;CheckIndexData:1;StreamLogic:1;];SUB:[AssembleOriginalData:1;];")("debug", chain->DebugStats());
     }
 
     Y_UNIT_TEST(Projection) {
diff --git a/ydb/core/tx/columnshard/counters/scan.cpp b/ydb/core/tx/columnshard/counters/scan.cpp
index 36b3c6abae8..144a2b25588 100644
--- a/ydb/core/tx/columnshard/counters/scan.cpp
+++ b/ydb/core/tx/columnshard/counters/scan.cpp
@@ -24,7 +24,8 @@ TScanCounters::TScanCounters(const TString& module)
     , LinearScanIntervals(TBase::GetDeriviative("LinearScanIntervals"))
     , LogScanRecords(TBase::GetDeriviative("LogScanRecords"))
     , LogScanIntervals(TBase::GetDeriviative("LogScanIntervals"))
-    , NotIndexBlobs(TBase::GetDeriviative("Indexes/NoData/Count"))
+    , NoIndexBlobs(TBase::GetDeriviative("Indexes/NoData/Blobs/Count"))
+    , NoIndex(TBase::GetDeriviative("Indexes/NoData/Index/Count"))
     , RecordsAcceptedByIndex(TBase::GetDeriviative("Indexes/Accepted/Records"))
     , RecordsDeniedByIndex(TBase::GetDeriviative("Indexes/Denied/Records"))
     , RecordsAcceptedByHeader(TBase::GetDeriviative("Headers/Accepted/Records"))
diff --git a/ydb/core/tx/columnshard/counters/scan.h b/ydb/core/tx/columnshard/counters/scan.h
index 78047490d3e..2dd3d7451ba 100644
--- a/ydb/core/tx/columnshard/counters/scan.h
+++ b/ydb/core/tx/columnshard/counters/scan.h
@@ -149,7 +149,8 @@ private:
     NMonitoring::THistogramPtr HistogramIntervalMemoryRequiredOnFail;
     NMonitoring::THistogramPtr HistogramIntervalMemoryReduceSize;
     NMonitoring::THistogramPtr HistogramIntervalMemoryRequiredAfterReduce;
-    NMonitoring::TDynamicCounters::TCounterPtr NotIndexBlobs;
+    NMonitoring::TDynamicCounters::TCounterPtr NoIndexBlobs;
+    NMonitoring::TDynamicCounters::TCounterPtr NoIndex;
     NMonitoring::TDynamicCounters::TCounterPtr RecordsAcceptedByIndex;
     NMonitoring::TDynamicCounters::TCounterPtr RecordsDeniedByIndex;
     NMonitoring::TDynamicCounters::TCounterPtr RecordsAcceptedByHeader;
@@ -162,8 +163,11 @@ public:
         return SubColumnCounters;
     }
 
-    void OnNotIndexBlobs() const {
-        NotIndexBlobs->Add(1);
+    void OnNoIndexBlobs(const ui32 recordsCount) const {
+        NoIndexBlobs->Add(recordsCount);
+    }
+    void OnNoIndex(const ui32 recordsCount) const {
+        NoIndex->Add(recordsCount);
     }
     void OnAcceptedByIndex(const ui32 recordsCount) const {
         RecordsAcceptedByIndex->Add(recordsCount);
diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/default_fetching.h b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/default_fetching.h
index ab886429a77..f9dbeb3b9c3 100644
--- a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/default_fetching.h
+++ b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/default_fetching.h
@@ -36,6 +36,7 @@ private:
 
         void SetBlobData(const TString& data) {
             AFL_VERIFY(!Data);
+            BlobRange.reset();
             Data.emplace(data);
         }
     };
diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp
index 589b8e8632d..4269465a249 100644
--- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp
+++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp
@@ -217,6 +217,7 @@ TConclusion<NArrow::TColumnFilter> TPortionDataSource::DoCheckIndex(
     auto meta = MutableStageData().GetRemapDataToIndex(fetchContext);
     if (!meta) {
         NYDBTest::TControllers::GetColumnShardController()->OnIndexSelectProcessed({});
+        GetContext()->GetCommonContext()->GetCounters().OnNoIndex(GetRecordsCount());
         return NArrow::TColumnFilter::BuildAllowFilter();
     }
     AFL_VERIFY(meta->IsSkipIndex());
@@ -232,6 +233,7 @@ TConclusion<NArrow::TColumnFilter> TPortionDataSource::DoCheckIndex(
     const std::optional<ui64> cat = meta->CalcCategory(fetchContext.GetSubColumnName());
     const NIndexes::TIndexColumnChunked* infoPointer = GetStageData().GetIndexes()->GetIndexDataOptional(meta->GetIndexId());
     if (!infoPointer) {
+        GetContext()->GetCommonContext()->GetCounters().OnNoIndexBlobs(GetRecordsCount());
         return filter;
     }
     const auto info = *infoPointer;
diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/categories_bloom/meta.cpp b/ydb/core/tx/columnshard/engines/storage/indexes/categories_bloom/meta.cpp
index 813605c1ec0..e6d7c62b1e5 100644
--- a/ydb/core/tx/columnshard/engines/storage/indexes/categories_bloom/meta.cpp
+++ b/ydb/core/tx/columnshard/engines/storage/indexes/categories_bloom/meta.cpp
@@ -171,6 +171,9 @@ bool TIndexMeta::DoCheckValueImpl(
     AFL_VERIFY(!!category);
     AFL_VERIFY(op == EOperation::Equals)("op", op);
     const ui32 bitsCount = data.GetBitsCount();
+    if (!bitsCount) {
+        return false;
+    }
     for (ui64 hashSeed = 0; hashSeed < HashesCount; ++hashSeed) {
         const ui64 hash = NArrow::NHash::TXX64::CalcForScalar(value, hashSeed);
         if (!data.Get(hash % bitsCount)) {