summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authoratarasov5 <[email protected]>2025-06-04 15:46:50 +0300
committeratarasov5 <[email protected]>2025-06-04 16:13:45 +0300
commiteffe17ba6e34639b097ed022707a93799495101f (patch)
treed658551e40d542e32aff1a6d7e784ed8e63f13a4
parent747cc4ded4774c02c9b9b1afd288b1b0ea84a0f1 (diff)
YQL-19940: Msan sanitizing
commit_hash:a7bee9ef953705fedbc2280f5a1eac81a305944f
-rw-r--r--yql/essentials/minikql/aligned_page_pool.cpp16
-rw-r--r--yql/essentials/minikql/asan_utils.h91
-rw-r--r--yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp8
-rw-r--r--yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp7
-rw-r--r--yql/essentials/minikql/compact_hash.h14
-rw-r--r--yql/essentials/minikql/gtest_ut/allocator_ut.cpp168
-rw-r--r--yql/essentials/minikql/mkql_alloc.cpp21
-rw-r--r--yql/essentials/minikql/mkql_alloc.h26
-rw-r--r--yql/essentials/minikql/mkql_alloc_ut.cpp2
-rw-r--r--yql/essentials/minikql/mkql_string_util_ut.cpp7
-rw-r--r--yql/essentials/minikql/sanitizer_ut/sanitizer_ut.cpp330
-rw-r--r--yql/essentials/minikql/sanitizer_ut/ya.make (renamed from yql/essentials/minikql/gtest_ut/ya.make)4
-rw-r--r--yql/essentials/minikql/ya.make2
-rw-r--r--yql/essentials/public/udf/sanitizer_utils.h139
-rw-r--r--yt/yql/providers/yt/codec/yt_codec.cpp2
-rw-r--r--yt/yql/providers/yt/codec/yt_codec_io.cpp7
16 files changed, 531 insertions, 313 deletions
diff --git a/yql/essentials/minikql/aligned_page_pool.cpp b/yql/essentials/minikql/aligned_page_pool.cpp
index f98065f20e5..8b09172a1f9 100644
--- a/yql/essentials/minikql/aligned_page_pool.cpp
+++ b/yql/essentials/minikql/aligned_page_pool.cpp
@@ -10,7 +10,7 @@
#include <util/system/info.h>
#include <util/thread/lfstack.h>
-#include <yql/essentials/minikql/asan_utils.h>
+#include <yql/essentials/public/udf/sanitizer_utils.h>
#if defined(_win_)
# include <util/system/winint.h>
@@ -78,7 +78,7 @@ public:
void *page = nullptr;
if (Pages.Dequeue(&page)) {
--Count;
- SanitizerMarkInvalid(page, PageSize);
+ NYql::NUdf::SanitizerMakeRegionInaccessible(page, PageSize);
return page;
}
@@ -104,14 +104,14 @@ private:
FreePage(addr);
return GetPageSize();
}
- SanitizerMarkInvalid(addr, PageSize);
+ NYql::NUdf::SanitizerMakeRegionInaccessible(addr, PageSize);
++Count;
Pages.Enqueue(addr);
return 0;
}
void FreePage(void* addr) {
- SanitizerMarkInvalid(addr, PageSize);
+ NYql::NUdf::SanitizerMakeRegionInaccessible(addr, PageSize);
auto res = T::Munmap(addr, PageSize);
Y_DEBUG_ABORT_UNLESS(0 == res, "Madvise failed: %s", LastSystemErrorText());
}
@@ -146,7 +146,7 @@ public:
Y_DEBUG_ABORT_UNLESS(!TAlignedPagePoolImpl<T>::IsDefaultAllocatorUsed(), "No memory maps allowed while using default allocator");
void* res = T::Mmap(size);
- SanitizerMarkInvalid(res, size);
+ NYql::NUdf::SanitizerMakeRegionInaccessible(res, size);
TotalMmappedBytes += size;
return res;
}
@@ -464,7 +464,7 @@ void* TAlignedPagePoolImpl<T>::GetPageImpl() {
template <typename T>
void* TAlignedPagePoolImpl<T>::GetPage() {
auto* page = GetPageImpl();
- SanitizerMarkInvalid(page, POOL_PAGE_SIZE);
+ NYql::NUdf::SanitizerMakeRegionInaccessible(page, POOL_PAGE_SIZE);
return page;
};
@@ -476,7 +476,7 @@ void TAlignedPagePoolImpl<T>::ReturnPage(void* addr) noexcept {
return;
}
- SanitizerMarkInvalid(addr, POOL_PAGE_SIZE);
+ NYql::NUdf::SanitizerMakeRegionInaccessible(addr, POOL_PAGE_SIZE);
Y_DEBUG_ABORT_UNLESS(AllPages.find(addr) != AllPages.end());
FreePages.emplace(addr);
}
@@ -495,7 +495,7 @@ void* TAlignedPagePoolImpl<T>::GetBlock(size_t size) {
return ret;
}
auto* block = GetBlockImpl(size);
- SanitizerMarkInvalid(block, size);
+ NYql::NUdf::SanitizerMakeRegionInaccessible(block, size);
return block;
}
diff --git a/yql/essentials/minikql/asan_utils.h b/yql/essentials/minikql/asan_utils.h
deleted file mode 100644
index b404be7b635..00000000000
--- a/yql/essentials/minikql/asan_utils.h
+++ /dev/null
@@ -1,91 +0,0 @@
-#pragma once
-
-#include <cstddef>
-
-#include <util/system/compiler.h>
-#include <util/system/yassert.h>
-
-#if defined(_asan_enabled_)
- #include <sanitizer/asan_interface.h>
-#endif
-
-namespace NKikimr {
-
-inline constexpr size_t ALLOCATION_REDZONE_SIZE = 16;
-inline constexpr size_t ASAN_EXTRA_ALLOCATION_SPACE = ALLOCATION_REDZONE_SIZE * 2;
-
-constexpr void* SanitizerMarkInvalid(void* addr, size_t size) {
-#if defined(_asan_enabled_)
- if (addr == nullptr) {
- return nullptr;
- }
- __asan_poison_memory_region(addr, size);
-#else // defined(_asan_enabled_)
- Y_UNUSED(addr, size);
-#endif
- return addr;
-}
-
-constexpr void* SanitizerMarkValid(void* addr, size_t size) {
-#if defined(_asan_enabled_)
- if (addr == nullptr) {
- return nullptr;
- }
- __asan_unpoison_memory_region(addr, size);
-#else // defined(_asan_enabled_)
- Y_UNUSED(addr, size);
-#endif
- return addr;
-}
-
-constexpr size_t GetSizeToAlloc(size_t size) {
-#if defined(_asan_enabled_)
- if (size == 0) {
- return 0;
- }
- return size + 2 * ALLOCATION_REDZONE_SIZE;
-#else // defined(_asan_enabled_)
- return size;
-#endif
-}
-
-constexpr const void* GetOriginalAllocatedObject(const void* ptr, size_t size) {
-#if defined(_asan_enabled_)
- if (size == 0) {
- return ptr;
- }
- return (char*)ptr - ALLOCATION_REDZONE_SIZE;
-#else // defined(_asan_enabled_)
- Y_UNUSED(size);
- return ptr;
-#endif
-}
-
-constexpr void* WrapPointerWithRedZones(void* ptr, size_t extendedSizeWithRedzone) {
-#if defined(_asan_enabled_)
- if (extendedSizeWithRedzone == 0) {
- return ptr;
- }
- SanitizerMarkInvalid(ptr, extendedSizeWithRedzone);
- SanitizerMarkValid((char*)ptr + ALLOCATION_REDZONE_SIZE, extendedSizeWithRedzone - 2 * ALLOCATION_REDZONE_SIZE);
- return (char*)ptr + ALLOCATION_REDZONE_SIZE;
-#else // defined(_asan_enabled_)
- Y_UNUSED(extendedSizeWithRedzone);
- return ptr;
-#endif
-}
-
-constexpr const void* UnwrapPointerWithRedZones(const void* ptr, size_t size) {
-#if defined(_asan_enabled_)
- if (size == 0) {
- return ptr;
- }
- SanitizerMarkInvalid((char*)ptr - ALLOCATION_REDZONE_SIZE, 2 * ALLOCATION_REDZONE_SIZE + size);
- return (char*)ptr - ALLOCATION_REDZONE_SIZE;
-#else // defined(_asan_enabled_)
- Y_UNUSED(size);
- return ptr;
-#endif
-}
-
-} // namespace NKikimr
diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp
index 27ca8502a85..ef7c3d2e51a 100644
--- a/yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp
+++ b/yql/essentials/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp
@@ -237,7 +237,7 @@ void RunTestBlockJoin(TSetup<false>& setup, EJoinKind joinKind,
} // namespace
Y_UNIT_TEST_SUITE(TMiniKQLBlockMapJoinTestBasic) {
- constexpr size_t testSize = 1 << 14;
+ constexpr size_t testSize = 1 << 11;
constexpr size_t valueSize = 3;
static const TVector<TString> threeLetterValues = GenerateValues(valueSize);
static const TSet<ui64> fibonacci = GenerateFibonacci(testSize);
@@ -736,7 +736,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLBlockMapJoinTestBasic) {
// 2. Make input for the "right" stream.
// Huge string is used to make less rows fit into one block
- TVector<ui64> rightKeyInit(1 << 16);
+ TVector<ui64> rightKeyInit(1 << 14);
std::fill(rightKeyInit.begin(), rightKeyInit.end(), 1);
TVector<TString> rightValueInit;
std::transform(rightKeyInit.cbegin(), rightKeyInit.cend(), std::back_inserter(rightValueInit),
@@ -880,7 +880,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLBlockMapJoinTestBasic) {
} // Y_UNIT_TEST_SUITE
Y_UNIT_TEST_SUITE(TMiniKQLBlockMapJoinTestOptional) {
- constexpr size_t testSize = 1 << 14;
+ constexpr size_t testSize = 1 << 12;
constexpr size_t valueSize = 3;
static const TVector<TString> threeLetterValues = GenerateValues(valueSize);
static const TSet<ui64> fibonacci = GenerateFibonacci(testSize);
@@ -1195,7 +1195,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLBlockMapJoinTestOptional) {
} // Y_UNIT_TEST_SUITE
Y_UNIT_TEST_SUITE(TMiniKQLBlockMapJoinTestCross) {
- constexpr size_t testSize = 1 << 14;
+ constexpr size_t testSize = 1 << 12;
constexpr size_t valueSize = 3;
static const TVector<TString> threeLetterValues = GenerateValues(valueSize);
static const TSet<ui64> fibonacci = GenerateFibonacci(testSize);
diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp
index cf58a31b110..072b10ec93e 100644
--- a/yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp
+++ b/yql/essentials/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp
@@ -16,12 +16,7 @@ namespace NMiniKQL {
namespace {
-constexpr auto TotalSambles =
-#ifndef NDEBUG
-222222U;
-#else
-22222222ULL;
-#endif
+constexpr auto TotalSambles = 222222U;
}
diff --git a/yql/essentials/minikql/compact_hash.h b/yql/essentials/minikql/compact_hash.h
index 3704907d9c3..5c04ecca338 100644
--- a/yql/essentials/minikql/compact_hash.h
+++ b/yql/essentials/minikql/compact_hash.h
@@ -1,11 +1,11 @@
#pragma once
-#include <yql/essentials/utils/hash.h>
-
#include "aligned_page_pool.h"
-#include "asan_utils.h"
#include "primes.h"
+#include <yql/essentials/public/udf/sanitizer_utils.h>
+#include <yql/essentials/utils/hash.h>
+
#include <util/generic/vector.h>
#include <util/generic/ptr.h>
#include <util/generic/hash.h>
@@ -564,7 +564,7 @@ private:
}
ui16 listCount = GetSmallPageCapacity<T>(size);
Y_ASSERT(listCount >= 2);
- TListHeader* header = new (SanitizerMarkValid(GetPagePool().GetPage(), TAlignedPagePool::POOL_PAGE_SIZE)) TListHeader(SMALL_MARK, size, listCount);
+ TListHeader* header = new (NYql::NUdf::SanitizerMakeRegionAccessible(GetPagePool().GetPage(), TAlignedPagePool::POOL_PAGE_SIZE)) TListHeader(SMALL_MARK, size, listCount);
pages.PushFront(&header->ListItem);
return header;
}
@@ -581,14 +581,14 @@ private:
ui16 listCapacity = FastClp2(size);
ui16 listCount = GetMediumPageCapacity<T>(listCapacity);
Y_ASSERT(listCount >= 2);
- TListHeader* header = new (SanitizerMarkValid(GetPagePool().GetPage(), TAlignedPagePool::POOL_PAGE_SIZE)) TListHeader(MEDIUM_MARK, listCapacity, listCount);
+ TListHeader* header = new (NYql::NUdf::SanitizerMakeRegionAccessible(GetPagePool().GetPage(), TAlignedPagePool::POOL_PAGE_SIZE)) TListHeader(MEDIUM_MARK, listCapacity, listCount);
pages.PushFront(&header->ListItem);
return header;
}
template <typename T>
TLargeListHeader* GetLargeListPage() {
- TLargeListHeader* const header = new (SanitizerMarkValid(GetPagePool().GetPage(), TAlignedPagePool::POOL_PAGE_SIZE)) TLargeListHeader(GetLargePageCapacity<T>());
+ TLargeListHeader* const header = new (NYql::NUdf::SanitizerMakeRegionAccessible(GetPagePool().GetPage(), TAlignedPagePool::POOL_PAGE_SIZE)) TLargeListHeader(GetLargePageCapacity<T>());
return header;
}
@@ -1316,7 +1316,7 @@ protected:
void AllocateBuckets(size_t count) {
auto bucketsMemory = Max(sizeof(TItemNode) * count, (size_t)TAlignedPagePool::POOL_PAGE_SIZE);
- Buckets_ = (TItemNode*)SanitizerMarkValid(GetPagePool().GetBlock(bucketsMemory), bucketsMemory);
+ Buckets_ = (TItemNode*)NYql::NUdf::SanitizerMakeRegionAccessible(GetPagePool().GetBlock(bucketsMemory), bucketsMemory);
BucketsCount_ = count;
BucketsMemory_ = bucketsMemory;
for (size_t i = 0; i < count; ++i) {
diff --git a/yql/essentials/minikql/gtest_ut/allocator_ut.cpp b/yql/essentials/minikql/gtest_ut/allocator_ut.cpp
deleted file mode 100644
index f4ae5a02926..00000000000
--- a/yql/essentials/minikql/gtest_ut/allocator_ut.cpp
+++ /dev/null
@@ -1,168 +0,0 @@
-#include <yql/essentials/minikql/mkql_alloc.cpp>
-
-#include <library/cpp/testing/gtest/gtest.h>
-
-#include <cstdlib>
-
-namespace NKikimr::NMiniKQL {
-
-enum class EAllocatorType {
- DefaultAllocator,
- ArrowAllocator,
- HugeAllocator,
-};
-
-class MemoryTest: public ::testing::TestWithParam<std::tuple<int, EAllocatorType>> {
-protected:
- MemoryTest()
- : ScopedAlloc_(__LOCATION__) {
- }
-
- size_t AllocSize() const {
- return static_cast<size_t>(std::get<0>(GetParam()));
- }
-
- EAllocatorType GetAllocatorType() const {
- return std::get<1>(GetParam());
- }
-
- void* AllocateMemory(size_t size) const {
- EAllocatorType allocatorType = GetAllocatorType();
- switch (allocatorType) {
- case EAllocatorType::DefaultAllocator:
- return TWithDefaultMiniKQLAlloc::AllocWithSize(size);
- case EAllocatorType::ArrowAllocator:
- return MKQLArrowAllocate(size);
- case EAllocatorType::HugeAllocator:
- return TMKQLHugeAllocator<char>::allocate(size);
- default:
- return nullptr; // Should never reach here
- }
- }
-
- void Free(const void* mem, size_t size) const {
- EAllocatorType allocatorType = GetAllocatorType();
- switch (allocatorType) {
- case EAllocatorType::DefaultAllocator:
- TWithDefaultMiniKQLAlloc::FreeWithSize(mem, size);
- break;
- case EAllocatorType::ArrowAllocator:
- MKQLArrowFree(mem, size);
- break;
- case EAllocatorType::HugeAllocator:
- TMKQLHugeAllocator<char>::deallocate(const_cast<char*>(static_cast<const char*>(mem)), size);
- break;
- default:
- break; // Should never reach here
- }
- }
-
- void AccessMemory(volatile void* memory, ssize_t offset) const {
- volatile char* ptr = static_cast<volatile char*>(memory) + offset;
- *ptr = 'A'; // Perform a basic write operation
- }
-
-private:
- TScopedAlloc ScopedAlloc_;
-};
-
-// Test naming function
-std::string TestNameGenerator(const ::testing::TestParamInfo<MemoryTest::ParamType>& info) {
- int sizeNumber = std::get<0>(info.param);
- EAllocatorType allocatorType = std::get<1>(info.param);
-
-
- std::string allocatorName = [&] () {
- switch (allocatorType) {
- case EAllocatorType::DefaultAllocator:
- return "DefaultAllocator";
- case EAllocatorType::ArrowAllocator:
- return "ArrowAllocator";
- case EAllocatorType::HugeAllocator:
- return "HugeAllocator";
- }
- }();
-
- return "Size" + std::to_string(sizeNumber) + "With" + allocatorName + "Allocator";
-}
-
-// Out of bounds access + use after free can be tested only with
-// --sanitize=address.
-#if defined(_asan_enabled_)
-TEST_P(MemoryTest, AccessOutOfBounds) {
- size_t allocationSize = AllocSize();
-
- void* memory = AllocateMemory(allocationSize);
- ASSERT_NE(memory, nullptr) << "Memory allocation failed.";
- // Accessing valid memory.
- ASSERT_NO_THROW({
- AccessMemory(memory, 0);
- AccessMemory(memory, allocationSize - 1);
- });
-
- // Accessing invalid left memory.
- EXPECT_DEATH({ AccessMemory(memory, -1); }, "");
- EXPECT_DEATH({ AccessMemory(memory, -8); }, "");
- EXPECT_DEATH({ AccessMemory(memory, -16); }, "");
-
- // Accessing invalid right memory.
- EXPECT_DEATH({ AccessMemory(memory, allocationSize); }, "");
- EXPECT_DEATH({ AccessMemory(memory, allocationSize + 6); }, "");
- EXPECT_DEATH({ AccessMemory(memory, allocationSize + 12); }, "");
- EXPECT_DEATH({ AccessMemory(memory, allocationSize + 15); }, "");
-
- Free(memory, allocationSize);
-}
-
-TEST_P(MemoryTest, AccessAfterFree) {
- size_t allocationSize = AllocSize();
- void* memory = AllocateMemory(allocationSize);
- void* memory2 = AllocateMemory(allocationSize);
- ASSERT_NE(memory, nullptr) << "Memory allocation failed.";
- Free(memory, allocationSize);
-
- // Access after free — should crash
- EXPECT_DEATH({ AccessMemory(memory, 0); }, "");
- EXPECT_DEATH({ AccessMemory(memory, allocationSize / 2); }, "");
- EXPECT_DEATH({ AccessMemory(memory, allocationSize - 1); }, "");
-
- Free(memory2, allocationSize);
- // Access after free — should crash
- EXPECT_DEATH({ AccessMemory(memory2, 0); }, "");
- EXPECT_DEATH({ AccessMemory(memory2, allocationSize / 2); }, "");
- EXPECT_DEATH({ AccessMemory(memory2, allocationSize - 1); }, "");
-}
-
-#endif // defined(_asan_enabled_)
-
-// Double free tracked only in DEBUG mode.
-#ifndef NDEBUG
-TEST_P(MemoryTest, DoubleFree) {
- if (GetAllocatorType() == EAllocatorType::ArrowAllocator || GetAllocatorType() == EAllocatorType::HugeAllocator) {
- GTEST_SKIP() << "Arrow and Huge allocators arae not instrumented yet to track double free.";
- }
- size_t allocationSize = AllocSize();
-
- void* memory = AllocateMemory(allocationSize);
- ASSERT_NE(memory, nullptr) << "Memory allocation failed.";
-
- Free(memory, allocationSize);
-
- // Attempting double free — should crash
- EXPECT_DEATH({ Free(memory, allocationSize); }, "");
-}
-#endif // NDEBUG
-
-// Allow empty tests for MSAN and other sanitizers.
-GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(MemoryTest);
-
-INSTANTIATE_TEST_SUITE_P(MemoryTests, MemoryTest,
- ::testing::Combine(
- ::testing::Values(8, 64, 32 * 1024, 64 * 1024, 128 * 1024, 64 * 1024 * 1024),
- ::testing::Values(
- EAllocatorType::DefaultAllocator,
- EAllocatorType::ArrowAllocator,
- EAllocatorType::HugeAllocator)),
- TestNameGenerator);
-
-} // namespace NKikimr::NMiniKQL
diff --git a/yql/essentials/minikql/mkql_alloc.cpp b/yql/essentials/minikql/mkql_alloc.cpp
index a8933de2149..febddcc84d2 100644
--- a/yql/essentials/minikql/mkql_alloc.cpp
+++ b/yql/essentials/minikql/mkql_alloc.cpp
@@ -121,6 +121,7 @@ void TAllocState::InvalidateMemInfo() {
#endif
}
+Y_NO_SANITIZE("address") Y_NO_SANITIZE("memory")
size_t TAllocState::GetDeallocatedInPages() const {
size_t deallocated = 0;
for (auto x : AllPages) {
@@ -208,7 +209,7 @@ void* MKQLAllocSlow(size_t sz, TAllocState* state, const EMemorySubPool mPool) {
auto roundedSize = AlignUp(sz + sizeof(TAllocPageHeader), MKQL_ALIGNMENT);
auto capacity = Max(ui64(TAlignedPagePool::POOL_PAGE_SIZE), roundedSize);
auto currPage = (TAllocPageHeader*)state->GetBlock(capacity);
- SanitizerMarkValid(currPage, sizeof(TAllocPageHeader));
+ NYql::NUdf::SanitizerMakeRegionAccessible(currPage, sizeof(TAllocPageHeader));
currPage->Deallocated = 0;
currPage->Capacity = capacity;
currPage->Offset = roundedSize;
@@ -244,7 +245,7 @@ void* TPagedArena::AllocSlow(const size_t sz, const EMemorySubPool mPool) {
auto roundedSize = AlignUp(sz + sizeof(TAllocPageHeader), MKQL_ALIGNMENT);
auto capacity = Max(ui64(TAlignedPagePool::POOL_PAGE_SIZE), roundedSize);
currentPage = (TAllocPageHeader*)PagePool_->GetBlock(capacity);
- SanitizerMarkValid(currentPage, sizeof(TAllocPageHeader));
+ NYql::NUdf::SanitizerMakeRegionAccessible(currentPage, sizeof(TAllocPageHeader));
currentPage->Capacity = capacity;
void* ret = (char*)currentPage + sizeof(TAllocPageHeader);
currentPage->Offset = roundedSize;
@@ -285,7 +286,7 @@ void* MKQLArrowAllocateOnArena(ui64 size) {
}
page = (TMkqlArrowHeader*)GetAlignedPage();
- SanitizerMarkValid(page, sizeof(TMkqlArrowHeader));
+ NYql::NUdf::SanitizerMakeRegionAccessible(page, sizeof(TMkqlArrowHeader));
page->Offset = sizeof(TMkqlArrowHeader);
page->Size = pageSize;
page->UseCount = 1;
@@ -330,7 +331,7 @@ void* MKQLArrowAllocateImpl(ui64 size) {
}
auto* header = (TMkqlArrowHeader*)ptr;
- SanitizerMarkValid(header, sizeof(TMkqlArrowHeader));
+ NYql::NUdf::SanitizerMakeRegionAccessible(header, sizeof(TMkqlArrowHeader));
header->Offset = 0;
header->UseCount = 0;
@@ -347,9 +348,9 @@ void* MKQLArrowAllocateImpl(ui64 size) {
} // namespace
void* MKQLArrowAllocate(ui64 size) {
- auto sizeWithRedzones = GetSizeToAlloc(size);
+ auto sizeWithRedzones = NYql::NUdf::GetSizeToAlloc(size);
void* mem = MKQLArrowAllocateImpl(sizeWithRedzones);
- return WrapPointerWithRedZones(mem, sizeWithRedzones);
+ return NYql::NUdf::WrapPointerWithRedZones(mem, sizeWithRedzones);
}
void* MKQLArrowReallocate(const void* mem, ui64 prevSize, ui64 size) {
@@ -372,7 +373,7 @@ void MKQLArrowFreeOnArena(const void* ptr) {
Y_ENSURE(it != state->ArrowBuffers.end());
state->ArrowBuffers.erase(it);
}
- SanitizerMarkInvalid(page, sizeof(TMkqlArrowHeader));
+ NYql::NUdf::SanitizerMakeRegionInaccessible(page, sizeof(TMkqlArrowHeader));
ReleaseAlignedPage(page);
}
@@ -411,13 +412,13 @@ void MKQLArrowFreeImpl(const void* mem, ui64 size) {
} // namespace
void MKQLArrowFree(const void* mem, ui64 size) {
- mem = UnwrapPointerWithRedZones(mem, size);
- auto sizeWithRedzones = GetSizeToAlloc(size);
+ mem = NYql::NUdf::UnwrapPointerWithRedZones(mem, size);
+ auto sizeWithRedzones = NYql::NUdf::GetSizeToAlloc(size);
return MKQLArrowFreeImpl(mem, sizeWithRedzones);
}
void MKQLArrowUntrack(const void* mem, ui64 size) {
- mem = GetOriginalAllocatedObject(mem, size);
+ mem = NYql::NUdf::GetOriginalAllocatedObject(mem, size);
TAllocState* state = TlsAllocState;
Y_ENSURE(state);
if (!state->EnableArrowTracking) {
diff --git a/yql/essentials/minikql/mkql_alloc.h b/yql/essentials/minikql/mkql_alloc.h
index 7a7716dd659..488f4c05a8f 100644
--- a/yql/essentials/minikql/mkql_alloc.h
+++ b/yql/essentials/minikql/mkql_alloc.h
@@ -4,7 +4,7 @@
#include "mkql_mem_info.h"
#include <yql/essentials/core/pg_settings/guc_settings.h>
-#include <yql/essentials/minikql/asan_utils.h>
+#include <yql/essentials/public/udf/sanitizer_utils.h>
#include <yql/essentials/parser/pg_wrapper/interface/context.h>
#include <yql/essentials/public/udf/udf_allocator.h>
#include <yql/essentials/public/udf/udf_value.h>
@@ -119,7 +119,7 @@ struct TAllocState : public TAlignedPagePool
explicit TAllocState(const TSourceLocation& location, const TAlignedPagePoolCounters& counters, bool supportsSizedAllocators);
void KillAllBoxed();
void InvalidateMemInfo();
- Y_NO_SANITIZE("address") size_t GetDeallocatedInPages() const;
+ Y_NO_SANITIZE("address") Y_NO_SANITIZE("memory") size_t GetDeallocatedInPages() const;
static void CleanupPAllocList(TListEntry* root);
static void CleanupArrowList(TListEntry* root);
@@ -298,9 +298,9 @@ public:
}
void* Alloc(size_t sz, const EMemorySubPool pagePool = EMemorySubPool::Default) {
- sz = GetSizeToAlloc(sz);
+ sz = NYql::NUdf::GetSizeToAlloc(sz);
void* mem = AllocImpl(sz, pagePool);
- return WrapPointerWithRedZones(mem, sz);
+ return NYql::NUdf::WrapPointerWithRedZones(mem, sz);
}
void Clear() noexcept;
@@ -393,9 +393,9 @@ inline void* MKQLAllocFastWithSizeImpl(size_t sz, TAllocState* state, const EMem
}
inline void* MKQLAllocFastWithSize(size_t sz, TAllocState* state, const EMemorySubPool mPool, const TAllocLocation& location = TAllocLocation::current()) {
- sz = GetSizeToAlloc(sz);
+ sz = NYql::NUdf::GetSizeToAlloc(sz);
void* mem = MKQLAllocFastWithSizeImpl(sz, state, mPool, location);
- return WrapPointerWithRedZones(mem, sz);
+ return NYql::NUdf::WrapPointerWithRedZones(mem, sz);
}
void MKQLFreeSlow(TAllocPageHeader* header, TAllocState *state, const EMemorySubPool mPool) noexcept;
@@ -463,8 +463,8 @@ inline void MKQLFreeFastWithSizeImpl(const void* mem, size_t sz, TAllocState* st
}
inline void MKQLFreeFastWithSize(const void* mem, size_t sz, TAllocState* state, const EMemorySubPool mPool) noexcept {
- mem = UnwrapPointerWithRedZones(mem, sz);
- sz = GetSizeToAlloc(sz);
+ mem = NYql::NUdf::UnwrapPointerWithRedZones(mem, sz);
+ sz = NYql::NUdf::GetSizeToAlloc(sz);
return MKQLFreeFastWithSizeImpl(mem, sz, state, mPool);
}
@@ -598,9 +598,9 @@ struct TMKQLHugeAllocator
static pointer allocate(size_type n, const void* = nullptr)
{
- n = GetSizeToAlloc(n);
+ n = NYql::NUdf::GetSizeToAlloc(n);
void* mem = allocateImpl(n);
- return static_cast<pointer>(WrapPointerWithRedZones(mem, n));
+ return static_cast<pointer>(NYql::NUdf::WrapPointerWithRedZones(mem, n));
}
static void deallocateImpl(const_pointer p, size_type n) noexcept
@@ -611,8 +611,8 @@ struct TMKQLHugeAllocator
static void deallocate(const_pointer p, size_type n) noexcept
{
- p = static_cast<const_pointer>(UnwrapPointerWithRedZones(p, n));
- n = GetSizeToAlloc(n);
+ p = static_cast<const_pointer>(NYql::NUdf::UnwrapPointerWithRedZones(p, n));
+ n = NYql::NUdf::GetSizeToAlloc(n);
return deallocateImpl(p, n);
}
};
@@ -647,7 +647,7 @@ public:
return;
}
- auto ptr = SanitizerMarkValid(Pool.GetPage(), TAlignedPagePool::POOL_PAGE_SIZE);
+ auto ptr = NYql::NUdf::SanitizerMakeRegionAccessible(Pool.GetPage(), TAlignedPagePool::POOL_PAGE_SIZE);
IndexInLastPage = 1;
Pages.push_back(ptr);
new(ptr) T(std::move(value));
diff --git a/yql/essentials/minikql/mkql_alloc_ut.cpp b/yql/essentials/minikql/mkql_alloc_ut.cpp
index c1636375c35..27cdbf9a5bc 100644
--- a/yql/essentials/minikql/mkql_alloc_ut.cpp
+++ b/yql/essentials/minikql/mkql_alloc_ut.cpp
@@ -32,7 +32,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLAllocTest) {
Y_UNIT_TEST(TestDeallocated) {
TScopedAlloc alloc(__LOCATION__);
#if defined(_asan_enabled_)
- constexpr size_t EXTRA_ALLOCATION_SPACE = ASAN_EXTRA_ALLOCATION_SPACE;
+ constexpr size_t EXTRA_ALLOCATION_SPACE = NYql::NUdf::SANITIZER_EXTRA_ALLOCATION_SPACE;
#else // defined(_asan_enabled_)
constexpr size_t EXTRA_ALLOCATION_SPACE = 0;
#endif // defined(_asan_enabled_)
diff --git a/yql/essentials/minikql/mkql_string_util_ut.cpp b/yql/essentials/minikql/mkql_string_util_ut.cpp
index f0d5545ab73..cec1be33fff 100644
--- a/yql/essentials/minikql/mkql_string_util_ut.cpp
+++ b/yql/essentials/minikql/mkql_string_util_ut.cpp
@@ -35,6 +35,8 @@ Y_UNIT_TEST_SUITE(TMiniKQLStringUtils) {
}
}
+// Disable test since it produces too much memory for msan.
+#if !defined(_msan_enabled_)
Y_UNIT_TEST(MakeLargeString) {
TScopedAlloc alloc(__LOCATION__);
@@ -65,7 +67,10 @@ Y_UNIT_TEST_SUITE(TMiniKQLStringUtils) {
UNIT_ASSERT_VALUES_EQUAL(value.Capacity(), 0xFFFFFFF0ULL);
}
}
+#endif
+// Disable test since it produces too much memory for msan.
+#if !defined(_msan_enabled_)
Y_UNIT_TEST(ConcatLargeString) {
TScopedAlloc alloc(__LOCATION__);
@@ -85,5 +90,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLStringUtils) {
UNIT_FAIL("No exception!");
} catch (const yexception&) {}
}
+#endif
+
}
diff --git a/yql/essentials/minikql/sanitizer_ut/sanitizer_ut.cpp b/yql/essentials/minikql/sanitizer_ut/sanitizer_ut.cpp
new file mode 100644
index 00000000000..adaa6b8d69a
--- /dev/null
+++ b/yql/essentials/minikql/sanitizer_ut/sanitizer_ut.cpp
@@ -0,0 +1,330 @@
+#include <yql/essentials/minikql/mkql_alloc.h>
+
+#include <library/cpp/testing/gtest/gtest.h>
+
+#include <cstdlib>
+#include <cstdio>
+
+namespace NKikimr::NMiniKQL {
+
+enum class EAllocatorType {
+ DefaultAllocator,
+ ArrowAllocator,
+ HugeAllocator,
+};
+
+class MemoryTest: public testing::Test {
+protected:
+ MemoryTest()
+ : ScopedAlloc_(__LOCATION__) {
+ }
+
+ void AccessMemory(volatile void* memory, ssize_t offset) const {
+ volatile char* ptr = static_cast<volatile char*>(memory) + offset;
+ *ptr = 'A'; // Perform a basic write operation
+ }
+
+ void BranchMemory(const volatile void* memory, ssize_t offset) const {
+ const volatile char* ptr = static_cast<const volatile char*>(memory) + offset;
+ if (*ptr == 'A') {
+ Cerr << "Branch access" << Endl;
+ }
+ }
+
+ TAlignedPagePool& GetPagePool() {
+ return ScopedAlloc_.Ref();
+ }
+
+private:
+ TScopedAlloc ScopedAlloc_;
+};
+
+class MemoryTestWithSizeAndAllocator: public MemoryTest, public ::testing::WithParamInterface<std::tuple<int, EAllocatorType>> {
+protected:
+ size_t AllocSize() const {
+ return static_cast<size_t>(std::get<0>(GetParam()));
+ }
+
+ EAllocatorType GetAllocatorType() const {
+ return std::get<1>(GetParam());
+ }
+
+ void* AllocateMemory(size_t size) const {
+ EAllocatorType allocatorType = GetAllocatorType();
+ switch (allocatorType) {
+ case EAllocatorType::DefaultAllocator:
+ return TWithDefaultMiniKQLAlloc::AllocWithSize(size);
+ case EAllocatorType::ArrowAllocator:
+ return MKQLArrowAllocate(size);
+ case EAllocatorType::HugeAllocator:
+ return TMKQLHugeAllocator<char>::allocate(size);
+ default:
+ return nullptr; // Should never reach here
+ }
+ }
+
+ void Free(const void* mem, size_t size) const {
+ EAllocatorType allocatorType = GetAllocatorType();
+ switch (allocatorType) {
+ case EAllocatorType::DefaultAllocator:
+ TWithDefaultMiniKQLAlloc::FreeWithSize(mem, size);
+ break;
+ case EAllocatorType::ArrowAllocator:
+ MKQLArrowFree(mem, size);
+ break;
+ case EAllocatorType::HugeAllocator:
+ TMKQLHugeAllocator<char>::deallocate(const_cast<char*>(static_cast<const char*>(mem)), size);
+ break;
+ default:
+ break; // Should never reach here
+ }
+ }
+};
+
+// Test naming function
+std::string TestNameGenerator(const ::testing::TestParamInfo<MemoryTestWithSizeAndAllocator::ParamType>& info) {
+ int sizeNumber = std::get<0>(info.param);
+ EAllocatorType allocatorType = std::get<1>(info.param);
+
+ std::string allocatorName = [&]() {
+ switch (allocatorType) {
+ case EAllocatorType::DefaultAllocator:
+ return "DefaultAllocator";
+ case EAllocatorType::ArrowAllocator:
+ return "ArrowAllocator";
+ case EAllocatorType::HugeAllocator:
+ return "HugeAllocator";
+ }
+ }();
+
+ return "Size" + std::to_string(sizeNumber) + "With" + allocatorName + "Allocator";
+}
+
+// Out of bounds access + use after free can be tested only with
+// --sanitize=address.
+#if defined(_asan_enabled_)
+TEST_P(MemoryTestWithSizeAndAllocator, AccessOutOfBounds) {
+ size_t allocationSize = AllocSize();
+
+ void* memory = AllocateMemory(allocationSize);
+ ASSERT_NE(memory, nullptr) << "Memory allocation failed.";
+ // Accessing valid memory.
+ ASSERT_NO_THROW({
+ AccessMemory(memory, 0);
+ AccessMemory(memory, allocationSize - 1);
+ });
+
+ // Accessing invalid left memory.
+ EXPECT_DEATH({ AccessMemory(memory, -1); }, "");
+ EXPECT_DEATH({ AccessMemory(memory, -8); }, "");
+ EXPECT_DEATH({ AccessMemory(memory, -16); }, "");
+
+ // Accessing invalid right memory.
+ EXPECT_DEATH({ AccessMemory(memory, allocationSize); }, "");
+ EXPECT_DEATH({ AccessMemory(memory, allocationSize + 6); }, "");
+ EXPECT_DEATH({ AccessMemory(memory, allocationSize + 12); }, "");
+ EXPECT_DEATH({ AccessMemory(memory, allocationSize + 15); }, "");
+
+ Free(memory, allocationSize);
+}
+
+TEST_P(MemoryTestWithSizeAndAllocator, AccessAfterFree) {
+ size_t allocationSize = AllocSize();
+ void* memory = AllocateMemory(allocationSize);
+ void* memory2 = AllocateMemory(allocationSize);
+ ASSERT_NE(memory, nullptr) << "Memory allocation failed.";
+ Free(memory, allocationSize);
+
+ // Access after free — should crash
+ EXPECT_DEATH({ AccessMemory(memory, 0); }, "");
+ EXPECT_DEATH({ AccessMemory(memory, allocationSize / 2); }, "");
+ EXPECT_DEATH({ AccessMemory(memory, allocationSize - 1); }, "");
+
+ Free(memory2, allocationSize);
+ // Access after free — should crash
+ EXPECT_DEATH({ AccessMemory(memory2, 0); }, "");
+ EXPECT_DEATH({ AccessMemory(memory2, allocationSize / 2); }, "");
+ EXPECT_DEATH({ AccessMemory(memory2, allocationSize - 1); }, "");
+}
+
+TEST_F(MemoryTest, TestPageFromPagePool) {
+ auto* page = GetPagePool().GetPage();
+ // No access allowed.
+ EXPECT_DEATH({ AccessMemory(page, 0); }, "");
+ EXPECT_DEATH({ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE / 2); }, "");
+ EXPECT_DEATH({ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE - 1); }, "");
+
+ // Allow access.
+ NYql::NUdf::SanitizerMakeRegionAccessible(page, TAlignedPagePool::POOL_PAGE_SIZE);
+
+ // Access allowed.
+ AccessMemory(page, 0);
+ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE / 2);
+ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE - 1);
+
+ // Return page should disable access.
+ GetPagePool().ReturnPage(page);
+
+ // Page should be unaddressable after being returned.
+ EXPECT_DEATH({ AccessMemory(page, 0); }, "");
+ EXPECT_DEATH({ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE / 2); }, "");
+ EXPECT_DEATH({ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE - 1); }, "");
+}
+
+TEST_F(MemoryTest, TestBlockFromPagePool) {
+ auto* page = GetPagePool().GetBlock(TAlignedPagePool::POOL_PAGE_SIZE * 2);
+ // No access allowed.
+ EXPECT_DEATH({ AccessMemory(page, 0); }, "");
+ EXPECT_DEATH({ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE); }, "");
+ EXPECT_DEATH({ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE * 2 - 1); }, "");
+
+ // Allow access.
+ NYql::NUdf::SanitizerMakeRegionAccessible(page, TAlignedPagePool::POOL_PAGE_SIZE * 2);
+
+ // Access allowed.
+ AccessMemory(page, 0);
+ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE);
+ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE * 2 - 1);
+ // Return page.
+ GetPagePool().ReturnBlock(page, TAlignedPagePool::POOL_PAGE_SIZE * 2);
+
+ // Page should be unaddressable after being returned.
+ EXPECT_DEATH({ AccessMemory(page, 0); }, "");
+ EXPECT_DEATH({ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE); }, "");
+ EXPECT_DEATH({ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE * 2 - 1); }, "");
+}
+
+#endif // defined(_asan_enabled_)
+
+#if defined(_msan_enabled_)
+
+TEST_P(MemoryTestWithSizeAndAllocator, UninitializedAccess) {
+ size_t allocationSize = AllocSize();
+ void* memory = AllocateMemory(allocationSize);
+
+ // Check unitialized access.
+ EXPECT_DEATH({ BranchMemory(memory, 0); }, "");
+ EXPECT_DEATH({ BranchMemory(memory, AllocSize() - 1); }, "");
+
+ // Initialize memory.
+ memset(memory, 0, allocationSize);
+
+ // Check initialized access.
+ BranchMemory(memory, 0);
+ BranchMemory(memory, AllocSize() - 1);
+
+ Free(memory, allocationSize);
+}
+
+TEST_F(MemoryTest, TestPageFromPagePool) {
+ auto* page = GetPagePool().GetPage();
+
+ // No access allowed.
+ EXPECT_DEATH({ BranchMemory(page, 0); }, "");
+ EXPECT_DEATH({ BranchMemory(page, TAlignedPagePool::POOL_PAGE_SIZE / 2); }, "");
+ EXPECT_DEATH({ BranchMemory(page, TAlignedPagePool::POOL_PAGE_SIZE - 1); }, "");
+
+ // Open region.
+ NYql::NUdf::SanitizerMakeRegionAccessible(page, TAlignedPagePool::POOL_PAGE_SIZE);
+
+ // Still cannot access memory. Opening region does not make memory initialized.
+ EXPECT_DEATH({ BranchMemory(page, 0); }, "");
+ EXPECT_DEATH({ BranchMemory(page, TAlignedPagePool::POOL_PAGE_SIZE); }, "");
+ EXPECT_DEATH({ BranchMemory(page, TAlignedPagePool::POOL_PAGE_SIZE * 2 - 1); }, "");
+
+ // Allow access.
+ memset(page, 0, TAlignedPagePool::POOL_PAGE_SIZE * 2);
+
+ // Access should be valid.
+ BranchMemory(page, 0);
+ BranchMemory(page, TAlignedPagePool::POOL_PAGE_SIZE);
+ BranchMemory(page, TAlignedPagePool::POOL_PAGE_SIZE * 2 - 1);
+
+ // Return page should disable access.
+ GetPagePool().ReturnPage(page);
+}
+
+TEST_F(MemoryTest, TestScopedInterceptorDisable) {
+ // Allocate memory but don't initialize it
+ size_t allocationSize = 64;
+ char* uninitializedSrc = static_cast<char*>(TWithDefaultMiniKQLAlloc::AllocWithSize(allocationSize));
+ ASSERT_NE(uninitializedSrc, nullptr) << "Source memory allocation failed.";
+ Y_DEFER {
+ TWithDefaultMiniKQLAlloc::FreeWithSize(uninitializedSrc, allocationSize);
+ };
+
+ // Without disabling access check, using uninitialized memory should trigger MSAN.
+ EXPECT_DEATH({
+ BranchMemory(uninitializedSrc, 0);
+ }, "");
+
+ // With YQL_MSAN_FREEZE_AND_SCOPED_UNPOISON, the access check should be disabled.
+ {
+ YQL_MSAN_FREEZE_AND_SCOPED_UNPOISON(uninitializedSrc, allocationSize);
+
+ // This should not trigger MSAN.
+ BranchMemory(uninitializedSrc, 0);
+ }
+
+ // After the scope ends, access check should be re-enabled.
+ EXPECT_DEATH({
+ BranchMemory(uninitializedSrc, 0);
+ }, "");
+}
+
+TEST_F(MemoryTest, TestBlockMsan) {
+ auto* page = GetPagePool().GetBlock(TAlignedPagePool::POOL_PAGE_SIZE * 2);
+
+ // No access allowed.
+ EXPECT_DEATH({ BranchMemory(page, 0); }, "");
+ EXPECT_DEATH({ BranchMemory(page, TAlignedPagePool::POOL_PAGE_SIZE); }, "");
+ EXPECT_DEATH({ BranchMemory(page, TAlignedPagePool::POOL_PAGE_SIZE * 2 - 1); }, "");
+
+ // Open region.
+ NYql::NUdf::SanitizerMakeRegionAccessible(page, TAlignedPagePool::POOL_PAGE_SIZE);
+
+ // Still cannot access memory.
+ EXPECT_DEATH({ BranchMemory(page, 0); }, "");
+ EXPECT_DEATH({ BranchMemory(page, TAlignedPagePool::POOL_PAGE_SIZE); }, "");
+ EXPECT_DEATH({ BranchMemory(page, TAlignedPagePool::POOL_PAGE_SIZE * 2 - 1); }, "");
+
+ // Allow access via memory initialization.
+ memset(page, 0, TAlignedPagePool::POOL_PAGE_SIZE * 2);
+
+ // Access allowed.
+ AccessMemory(page, 0);
+ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE * 2);
+ AccessMemory(page, TAlignedPagePool::POOL_PAGE_SIZE * 2 - 1);
+ GetPagePool().ReturnBlock(page, TAlignedPagePool::POOL_PAGE_SIZE * 2);
+}
+
+#endif // defined(_msan_enabled_)
+
+// Double free tracked only in DEBUG mode.
+#ifndef NDEBUG
+TEST_P(MemoryTestWithSizeAndAllocator, DoubleFree) {
+ if (GetAllocatorType() == EAllocatorType::ArrowAllocator || GetAllocatorType() == EAllocatorType::HugeAllocator) {
+ GTEST_SKIP() << "Arrow and Huge allocators arae not instrumented yet to track double free.";
+ }
+ size_t allocationSize = AllocSize();
+
+ void* memory = AllocateMemory(allocationSize);
+ ASSERT_NE(memory, nullptr) << "Memory allocation failed.";
+
+ Free(memory, allocationSize);
+
+ // Attempting double free — should crash
+ EXPECT_DEATH({ Free(memory, allocationSize); }, "");
+}
+#endif // NDEBUG
+
+// Allow empty tests for MSAN and other sanitizers.
+GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(MemoryTestWithSizeAndAllocator);
+
+INSTANTIATE_TEST_SUITE_P(MemoryTestWithSizeAndAllocators, MemoryTestWithSizeAndAllocator,
+ ::testing::Combine(::testing::Values(8, 64, 32 * 1024, 64 * 1024, 128 * 1024, 64 * 1024 * 1024),
+ ::testing::Values(EAllocatorType::DefaultAllocator, EAllocatorType::ArrowAllocator,
+ EAllocatorType::HugeAllocator)),
+ TestNameGenerator);
+
+} // namespace NKikimr::NMiniKQL
diff --git a/yql/essentials/minikql/gtest_ut/ya.make b/yql/essentials/minikql/sanitizer_ut/ya.make
index ed80e31b208..7ce154ee702 100644
--- a/yql/essentials/minikql/gtest_ut/ya.make
+++ b/yql/essentials/minikql/sanitizer_ut/ya.make
@@ -1,15 +1,15 @@
GTEST()
PEERDIR(
+ contrib/libs/apache/arrow
yql/essentials/minikql
yql/essentials/minikql/invoke_builtins/llvm16
yql/essentials/public/udf/service/exception_policy
- contrib/libs/apache/arrow
yql/essentials/sql/pg_dummy
)
SRC(
- allocator_ut.cpp
+ sanitizer_ut.cpp
)
YQL_LAST_ABI_VERSION()
diff --git a/yql/essentials/minikql/ya.make b/yql/essentials/minikql/ya.make
index 1d09c07117a..f611be4b1e6 100644
--- a/yql/essentials/minikql/ya.make
+++ b/yql/essentials/minikql/ya.make
@@ -108,5 +108,5 @@ RECURSE(
RECURSE_FOR_TESTS(
benchmark
ut
- gtest_ut
+ sanitizer_ut
)
diff --git a/yql/essentials/public/udf/sanitizer_utils.h b/yql/essentials/public/udf/sanitizer_utils.h
new file mode 100644
index 00000000000..9cb745b20ba
--- /dev/null
+++ b/yql/essentials/public/udf/sanitizer_utils.h
@@ -0,0 +1,139 @@
+#pragma once
+
+#include <cstddef>
+
+#include <util/system/compiler.h>
+
+#if defined(_asan_enabled_)
+ #include <sanitizer/asan_interface.h>
+#elif defined(_msan_enabled_)
+ #include <sanitizer/msan_interface.h>
+#endif
+
+#if defined(_asan_enabled_) || defined(_msan_enabled_)
+ #include <util/generic/scope.h>
+#endif // defined(_asan_enabled_) || defined(_msan_enabled_)
+
+namespace NYql::NUdf {
+namespace NInternal {
+
+constexpr void SanitizerMarkInvalid(void* addr, size_t size) {
+ if (addr == nullptr) {
+ return;
+ }
+#if defined(_asan_enabled_)
+ __asan_poison_memory_region(addr, size);
+#elif defined(_msan_enabled_)
+ __msan_poison(addr, size);
+#else
+ Y_UNUSED(addr, size);
+#endif
+}
+constexpr void SanitizerMarkValid(void* addr, size_t size) {
+ if (addr == nullptr) {
+ return;
+ }
+#if defined(_asan_enabled_)
+ __asan_unpoison_memory_region(addr, size);
+#elif defined(_msan_enabled_)
+ __msan_unpoison(addr, size);
+#else
+ Y_UNUSED(addr, size);
+#endif
+}
+
+} // namespace NInternal
+
+#if defined(_asan_enabled_)
+inline constexpr size_t ALLOCATION_REDZONE_SIZE = 16;
+#elif defined(_msan_enabled_)
+inline constexpr size_t ALLOCATION_REDZONE_SIZE = 0;
+#else
+inline constexpr size_t ALLOCATION_REDZONE_SIZE = 0;
+#endif
+inline constexpr size_t SANITIZER_EXTRA_ALLOCATION_SPACE = ALLOCATION_REDZONE_SIZE * 2;
+
+#if defined(_msan_enabled_)
+ #define YQL_MSAN_FREEZE_AND_SCOPED_UNPOISON(addr, size) \
+ TArrayHolder<char> __yqlMaybePoisonedBuffer(new char[(size)]()); \
+ __msan_copy_shadow(__yqlMaybePoisonedBuffer.Get(), (addr), (size)); \
+ NYql::NUdf::NInternal::SanitizerMarkValid((addr), (size)); \
+ Y_DEFER { \
+ __msan_copy_shadow((addr), __yqlMaybePoisonedBuffer.Get(), (size)); \
+ }
+#else // defined (_msan_enabled_)
+ #define YQL_MSAN_FREEZE_AND_SCOPED_UNPOISON(addr, size)
+#endif // defined (_msan_enabled_)
+
+// Mark the memory as deallocated and handed over to the allocator.
+// Client -> Allocator.
+constexpr void* SanitizerMakeRegionInaccessible(void* addr, size_t size) {
+ NInternal::SanitizerMarkInvalid(addr, size);
+ return addr;
+}
+
+// Mark the memory as allocated and handed over to the client.
+// Allocator -> Client.
+constexpr void* SanitizerMakeRegionAccessible(void* addr, size_t size) {
+#if defined(_asan_enabled_)
+ NInternal::SanitizerMarkValid(addr, size);
+#elif defined(_msan_enabled_)
+ // NOTE: When we give memory from allocator to client we should mark it as invalid.
+ NInternal::SanitizerMarkInvalid(addr, size);
+#endif
+ Y_UNUSED(size);
+ return addr;
+}
+
+constexpr size_t GetSizeToAlloc(size_t size) {
+#if defined(_asan_enabled_) || defined(_msan_enabled_)
+ if (size == 0) {
+ return 0;
+ }
+ return size + 2 * ALLOCATION_REDZONE_SIZE;
+#else // defined(_asan_enabled_) || defined(_msan_enabled_)
+ return size;
+#endif // defined(_asan_enabled_) || defined(_msan_enabled_)
+}
+
+constexpr const void* GetOriginalAllocatedObject(const void* ptr, size_t size) {
+#if defined(_asan_enabled_) || defined(_msan_enabled_)
+ if (size == 0) {
+ return ptr;
+ }
+ return static_cast<const char*>(ptr) - ALLOCATION_REDZONE_SIZE;
+#else // defined(_asan_enabled_) || defined(_msan_enabled_)
+ Y_UNUSED(size);
+ return ptr;
+#endif // defined(_asan_enabled_) || defined(_msan_enabled_)
+}
+
+constexpr void* WrapPointerWithRedZones(void* ptr, size_t extendedSizeWithRedzone) {
+#if defined(_asan_enabled_) || defined(_msan_enabled_)
+ if (extendedSizeWithRedzone == 0) {
+ return ptr;
+ }
+ SanitizerMakeRegionInaccessible(ptr, extendedSizeWithRedzone);
+ SanitizerMakeRegionAccessible(static_cast<char*>(ptr) + ALLOCATION_REDZONE_SIZE, extendedSizeWithRedzone - 2 * ALLOCATION_REDZONE_SIZE);
+ return static_cast<char*>(ptr) + ALLOCATION_REDZONE_SIZE;
+#else // defined(_asan_enabled_) || defined(_msan_enabled_)
+ Y_UNUSED(extendedSizeWithRedzone);
+ return ptr;
+#endif // defined(_asan_enabled_) || defined(_msan_enabled_)
+}
+
+constexpr const void* UnwrapPointerWithRedZones(const void* ptr, size_t size) {
+#if defined(_asan_enabled_) || defined(_msan_enabled_)
+ if (size == 0) {
+ return ptr;
+ }
+ SanitizerMakeRegionInaccessible(static_cast<char*>(const_cast<void*>(ptr)) - ALLOCATION_REDZONE_SIZE,
+ 2 * ALLOCATION_REDZONE_SIZE + size);
+ return static_cast<const char*>(ptr) - ALLOCATION_REDZONE_SIZE;
+#else // defined(_asan_enabled_) || defined(_msan_enabled_)
+ Y_UNUSED(size);
+ return ptr;
+#endif // defined(_asan_enabled_) || defined(_msan_enabled_)
+}
+
+} // namespace NYql::NUdf
diff --git a/yt/yql/providers/yt/codec/yt_codec.cpp b/yt/yql/providers/yt/codec/yt_codec.cpp
index e414aa0f6d1..35cccc93191 100644
--- a/yt/yql/providers/yt/codec/yt_codec.cpp
+++ b/yt/yql/providers/yt/codec/yt_codec.cpp
@@ -1791,7 +1791,7 @@ public:
void operator=(const TTempBlockWriter&) = delete;
std::pair<char*, char*> NextEmptyBlock() override {
- auto newPage = SanitizerMarkValid(Pool_.GetPage(), TAlignedPagePool::POOL_PAGE_SIZE);
+ auto newPage = NYql::NUdf::SanitizerMakeRegionAccessible(Pool_.GetPage(), TAlignedPagePool::POOL_PAGE_SIZE);
auto header = (TPageHeader*)newPage;
header->Avail_ = 0;
header->Next_ = &Dummy_;
diff --git a/yt/yql/providers/yt/codec/yt_codec_io.cpp b/yt/yql/providers/yt/codec/yt_codec_io.cpp
index a46ecc0f680..901c4451e1b 100644
--- a/yt/yql/providers/yt/codec/yt_codec_io.cpp
+++ b/yt/yql/providers/yt/codec/yt_codec_io.cpp
@@ -413,13 +413,18 @@ private:
}
try {
+ // Note: The Arrow format actually allows reading and writing uninitialized memory.
+ // This is permitted because any "meaningful" access to the data uses a validity bitmask,
+ // which indicates whether each byte is valid.
+ YQL_MSAN_FREEZE_AND_SCOPED_UNPOISON(firstBlock->Buffer_.Get(), firstBlock->LastRecordBoundary_.value_or(firstBlock->Avail_));
Target_.Write(firstBlock->Buffer_.Get(), firstBlock->LastRecordBoundary_.value_or(firstBlock->Avail_));
if (firstBlock->LastRecordBoundary_) {
if (OnRecordBoundaryCallback_) {
OnRecordBoundaryCallback_();
}
if (firstBlock->Avail_ > *firstBlock->LastRecordBoundary_) {
- Target_.Write(firstBlock->Buffer_.Get() + *firstBlock->LastRecordBoundary_, firstBlock->Avail_ - *firstBlock->LastRecordBoundary_);
+ Target_.Write(firstBlock->Buffer_.Get() + *firstBlock->LastRecordBoundary_,
+ firstBlock->Avail_ - *firstBlock->LastRecordBoundary_);
}
}