diff options
author | Alexander Gololobov <davenger@yandex-team.com> | 2022-02-10 16:47:38 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:47:38 +0300 |
commit | fccc62e9bfdce9be2fe7e0f23479da3a5512211a (patch) | |
tree | c0748b5dcbade83af788c0abfa89c0383d6b779c /library/cpp/lfalloc/alloc_profiler | |
parent | 39608cdb86363c75ce55b2b9a69841c3b71f22cf (diff) | |
download | ydb-fccc62e9bfdce9be2fe7e0f23479da3a5512211a.tar.gz |
Restoring authorship annotation for Alexander Gololobov <davenger@yandex-team.com>. Commit 2 of 2.
Diffstat (limited to 'library/cpp/lfalloc/alloc_profiler')
-rw-r--r-- | library/cpp/lfalloc/alloc_profiler/profiler.cpp | 16 | ||||
-rw-r--r-- | library/cpp/lfalloc/alloc_profiler/profiler.h | 12 | ||||
-rw-r--r-- | library/cpp/lfalloc/alloc_profiler/profiler_ut.cpp | 150 | ||||
-rw-r--r-- | library/cpp/lfalloc/alloc_profiler/stackcollect.cpp | 156 | ||||
-rw-r--r-- | library/cpp/lfalloc/alloc_profiler/stackcollect.h | 162 | ||||
-rw-r--r-- | library/cpp/lfalloc/alloc_profiler/ut/ya.make | 38 | ||||
-rw-r--r-- | library/cpp/lfalloc/alloc_profiler/ya.make | 30 |
7 files changed, 282 insertions, 282 deletions
diff --git a/library/cpp/lfalloc/alloc_profiler/profiler.cpp b/library/cpp/lfalloc/alloc_profiler/profiler.cpp index beb0ffb289..0e30927a5a 100644 --- a/library/cpp/lfalloc/alloc_profiler/profiler.cpp +++ b/library/cpp/lfalloc/alloc_profiler/profiler.cpp @@ -8,7 +8,7 @@ #include <util/generic/vector.h> #include <util/stream/str.h> -namespace NAllocProfiler { +namespace NAllocProfiler { namespace { @@ -50,32 +50,32 @@ void DeallocationCallback(int stackId, int tag, size_t size, int sizeIdx) //////////////////////////////////////////////////////////////////////////////// -bool StartAllocationSampling(bool profileAllThreads) +bool StartAllocationSampling(bool profileAllThreads) { auto& collector = AllocationStackCollector(); collector.Clear(); - NAllocDbg::SetProfileAllThreads(profileAllThreads); + NAllocDbg::SetProfileAllThreads(profileAllThreads); NAllocDbg::SetAllocationCallback(AllocationCallback); NAllocDbg::SetDeallocationCallback(DeallocationCallback); NAllocDbg::SetAllocationSamplingEnabled(true); return true; } -bool StopAllocationSampling(IAllocationStatsDumper &out, int count) +bool StopAllocationSampling(IAllocationStatsDumper &out, int count) { NAllocDbg::SetAllocationCallback(nullptr); NAllocDbg::SetDeallocationCallback(nullptr); NAllocDbg::SetAllocationSamplingEnabled(false); auto& collector = AllocationStackCollector(); - collector.Dump(count, out); + collector.Dump(count, out); return true; } -bool StopAllocationSampling(IOutputStream& out, int count) { - TAllocationStatsDumper dumper(out); - return StopAllocationSampling(dumper, count); +bool StopAllocationSampling(IOutputStream& out, int count) { + TAllocationStatsDumper dumper(out); + return StopAllocationSampling(dumper, count); } } // namespace NProfiler diff --git a/library/cpp/lfalloc/alloc_profiler/profiler.h b/library/cpp/lfalloc/alloc_profiler/profiler.h index 592849b460..4ea49b9dcc 100644 --- a/library/cpp/lfalloc/alloc_profiler/profiler.h +++ b/library/cpp/lfalloc/alloc_profiler/profiler.h @@ -1,13 +1,13 @@ #pragma once -#include "stackcollect.h" +#include "stackcollect.h" #include <library/cpp/lfalloc/dbg_info/dbg_info.h> #include <util/generic/noncopyable.h> #include <util/stream/output.h> -namespace NAllocProfiler { +namespace NAllocProfiler { //////////////////////////////////////////////////////////////////////////////// @@ -21,9 +21,9 @@ inline bool SetProfileCurrentThread(bool value) return NAllocDbg::SetProfileCurrentThread(value); } -bool StartAllocationSampling(bool profileAllThreads = false); -bool StopAllocationSampling(IAllocationStatsDumper& out, int count = 100); -bool StopAllocationSampling(IOutputStream& out, int count = 100); +bool StartAllocationSampling(bool profileAllThreads = false); +bool StopAllocationSampling(IAllocationStatsDumper& out, int count = 100); +bool StopAllocationSampling(IOutputStream& out, int count = 100); //////////////////////////////////////////////////////////////////////////////// @@ -42,4 +42,4 @@ public: } }; -} // namespace NAllocProfiler +} // namespace NAllocProfiler diff --git a/library/cpp/lfalloc/alloc_profiler/profiler_ut.cpp b/library/cpp/lfalloc/alloc_profiler/profiler_ut.cpp index 21b667e730..4341dda6ed 100644 --- a/library/cpp/lfalloc/alloc_profiler/profiler_ut.cpp +++ b/library/cpp/lfalloc/alloc_profiler/profiler_ut.cpp @@ -1,76 +1,76 @@ -#include "profiler.h" - +#include "profiler.h" + #include <library/cpp/testing/unittest/registar.h> - -namespace NAllocProfiler { - -//////////////////////////////////////////////////////////////////////////////// - -Y_UNIT_TEST_SUITE(Profiler) { - Y_UNIT_TEST(StackCollection) - { - TStringStream str; - - NAllocProfiler::StartAllocationSampling(true); - TVector<TAutoPtr<int>> test; - // Do many allocations and no deallocations - for (int i = 0; i < 10000; ++i) { - test.push_back(new int); - } - NAllocProfiler::StopAllocationSampling(str); - //Cout << str.Str() << Endl; - -#if !defined(ARCH_AARCH64) - /* Check that output resembles this: - - STACK #2: 0 Allocs: 10 Frees: 0 CurrentSize: 40 - 0000000000492353 ?? - 000000000048781F operator new(unsigned long) +1807 - 00000000003733FA NAllocProfiler::NTestSuiteProfiler::TTestCaseStackCollection::Execute_(NUnitTest::TTestContext&) +218 - 00000000004A1938 NUnitTest::TTestBase::Run(std::__y1::function<void ()>, TString, char const*, bool) +120 - 0000000000375656 NAllocProfiler::NTestSuiteProfiler::TCurrentTest::Execute() +342 - 00000000004A20CF NUnitTest::TTestFactory::Execute() +847 - 000000000049922D NUnitTest::RunMain(int, char**) +1965 - 00007FF665778F45 __libc_start_main +245 - */ - - UNIT_ASSERT_STRING_CONTAINS(str.Str(), "StackCollection"); - UNIT_ASSERT_STRING_CONTAINS(str.Str(), "NUnitTest::TTestBase::Run"); - UNIT_ASSERT_STRING_CONTAINS(str.Str(), "NAllocProfiler::NTestSuiteProfiler::TCurrentTest::Execute"); - UNIT_ASSERT_STRING_CONTAINS(str.Str(), "NUnitTest::TTestFactory::Execute"); - UNIT_ASSERT_STRING_CONTAINS(str.Str(), "NUnitTest::RunMain"); -#endif - } - - class TAllocDumper : public NAllocProfiler::TAllocationStatsDumper { - public: - explicit TAllocDumper(IOutputStream& out) : NAllocProfiler::TAllocationStatsDumper(out) {} - - TString FormatTag(int tag) override { - UNIT_ASSERT_VALUES_EQUAL(tag, 42); - return "TAG_NAME_42"; - } - }; - - Y_UNIT_TEST(TagNames) - { - TStringStream str; - - NAllocProfiler::StartAllocationSampling(true); - TVector<TAutoPtr<int>> test; - NAllocProfiler::TProfilingScope scope(42); - // Do many allocations and no deallocations - for (int i = 0; i < 10000; ++i) { - test.push_back(new int); - } - - TAllocDumper dumper(str); - NAllocProfiler::StopAllocationSampling(dumper); - -#if !defined(ARCH_AARCH64) - UNIT_ASSERT_STRING_CONTAINS(str.Str(), "TAG_NAME_42"); -#endif - } -} - -} + +namespace NAllocProfiler { + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(Profiler) { + Y_UNIT_TEST(StackCollection) + { + TStringStream str; + + NAllocProfiler::StartAllocationSampling(true); + TVector<TAutoPtr<int>> test; + // Do many allocations and no deallocations + for (int i = 0; i < 10000; ++i) { + test.push_back(new int); + } + NAllocProfiler::StopAllocationSampling(str); + //Cout << str.Str() << Endl; + +#if !defined(ARCH_AARCH64) + /* Check that output resembles this: + + STACK #2: 0 Allocs: 10 Frees: 0 CurrentSize: 40 + 0000000000492353 ?? + 000000000048781F operator new(unsigned long) +1807 + 00000000003733FA NAllocProfiler::NTestSuiteProfiler::TTestCaseStackCollection::Execute_(NUnitTest::TTestContext&) +218 + 00000000004A1938 NUnitTest::TTestBase::Run(std::__y1::function<void ()>, TString, char const*, bool) +120 + 0000000000375656 NAllocProfiler::NTestSuiteProfiler::TCurrentTest::Execute() +342 + 00000000004A20CF NUnitTest::TTestFactory::Execute() +847 + 000000000049922D NUnitTest::RunMain(int, char**) +1965 + 00007FF665778F45 __libc_start_main +245 + */ + + UNIT_ASSERT_STRING_CONTAINS(str.Str(), "StackCollection"); + UNIT_ASSERT_STRING_CONTAINS(str.Str(), "NUnitTest::TTestBase::Run"); + UNIT_ASSERT_STRING_CONTAINS(str.Str(), "NAllocProfiler::NTestSuiteProfiler::TCurrentTest::Execute"); + UNIT_ASSERT_STRING_CONTAINS(str.Str(), "NUnitTest::TTestFactory::Execute"); + UNIT_ASSERT_STRING_CONTAINS(str.Str(), "NUnitTest::RunMain"); +#endif + } + + class TAllocDumper : public NAllocProfiler::TAllocationStatsDumper { + public: + explicit TAllocDumper(IOutputStream& out) : NAllocProfiler::TAllocationStatsDumper(out) {} + + TString FormatTag(int tag) override { + UNIT_ASSERT_VALUES_EQUAL(tag, 42); + return "TAG_NAME_42"; + } + }; + + Y_UNIT_TEST(TagNames) + { + TStringStream str; + + NAllocProfiler::StartAllocationSampling(true); + TVector<TAutoPtr<int>> test; + NAllocProfiler::TProfilingScope scope(42); + // Do many allocations and no deallocations + for (int i = 0; i < 10000; ++i) { + test.push_back(new int); + } + + TAllocDumper dumper(str); + NAllocProfiler::StopAllocationSampling(dumper); + +#if !defined(ARCH_AARCH64) + UNIT_ASSERT_STRING_CONTAINS(str.Str(), "TAG_NAME_42"); +#endif + } +} + +} diff --git a/library/cpp/lfalloc/alloc_profiler/stackcollect.cpp b/library/cpp/lfalloc/alloc_profiler/stackcollect.cpp index d608803e84..fded4e2fd1 100644 --- a/library/cpp/lfalloc/alloc_profiler/stackcollect.cpp +++ b/library/cpp/lfalloc/alloc_profiler/stackcollect.cpp @@ -5,16 +5,16 @@ #include <util/generic/algorithm.h> #include <util/generic/vector.h> #include <util/stream/format.h> -#include <util/stream/str.h> -#include <util/string/cast.h> -#include <util/string/printf.h> +#include <util/stream/str.h> +#include <util/string/cast.h> +#include <util/string/printf.h> #include <util/system/backtrace.h> #include <util/system/spinlock.h> #include <util/system/yassert.h> -namespace NAllocProfiler { - +namespace NAllocProfiler { + //////////////////////////////////////////////////////////////////////////////// template <typename T> @@ -87,11 +87,11 @@ public: return Y_ARRAY_SIZE(Frames); } - void BackTrace(const TFrameInfo* stack, TStackVec<void*, 64>& frames) const + void BackTrace(const TFrameInfo* stack, TStackVec<void*, 64>& frames) const { - frames.clear(); + frames.clear(); for (size_t i = 0; i < 100; ++i) { - frames.push_back(stack->Addr); + frames.push_back(stack->Addr); int prevInd = stack->PrevInd; if (prevInd == -1) { break; @@ -174,11 +174,11 @@ private: //////////////////////////////////////////////////////////////////////////////// -class TAllocationStackCollector::TImpl: public TStackCollector<TStats> { - using TBase = TStackCollector<TStats>; +class TAllocationStackCollector::TImpl: public TStackCollector<TStats> { + using TBase = TStackCollector<TStats>; private: - TStats Total; + TStats Total; public: int Alloc(void** stack, size_t frameCount, int tag, size_t size) @@ -203,7 +203,7 @@ public: Total.Clear(); } - void Dump(int count, IAllocationStatsDumper& out) const + void Dump(int count, IAllocationStatsDumper& out) const { const TFrameInfo* frames = TBase::GetFrames(); size_t framesCount = TBase::GetFramesCount(); @@ -225,18 +225,18 @@ public: : ls.Frees > rs.Frees; }); - out.DumpTotal(Total); + out.DumpTotal(Total); - TAllocationInfo allocInfo; + TAllocationInfo allocInfo; int printedCount = 0; for (const TFrameInfo* stack: stacks) { - allocInfo.Clear(); - allocInfo.Tag = stack->Tag; - allocInfo.Stats = stack->Stats; - TBase::BackTrace(stack, allocInfo.Stack); + allocInfo.Clear(); + allocInfo.Tag = stack->Tag; + allocInfo.Stats = stack->Stats; + TBase::BackTrace(stack, allocInfo.Stack); + + out.DumpEntry(allocInfo); - out.DumpEntry(allocInfo); - if (++printedCount >= count) { break; } @@ -268,65 +268,65 @@ void TAllocationStackCollector::Clear() Impl->Clear(); } -void TAllocationStackCollector::Dump(int count, IAllocationStatsDumper &out) const +void TAllocationStackCollector::Dump(int count, IAllocationStatsDumper &out) const { Impl->Dump(count, out); } - -TString IAllocationStatsDumper::FormatTag(int tag) { - return ToString(tag); -} - -TString IAllocationStatsDumper::FormatSize(intptr_t sz) { - return ToString(sz); -} - - -TAllocationStatsDumper::TAllocationStatsDumper(IOutputStream& out) - : PrintedCount(0) - , Out(out) - , SymbolCache(2048) -{} - -void TAllocationStatsDumper::DumpTotal(const TStats& total) { - Out << "TOTAL" - << "\tAllocs: " << total.Allocs - << "\tFrees: " << total.Frees - << "\tCurrentSize: " << FormatSize(total.CurrentSize) - << Endl; -} - -void TAllocationStatsDumper::DumpEntry(const TAllocationInfo& allocInfo) { - Out << Endl - << "STACK #" << PrintedCount+1 << ": " << FormatTag(allocInfo.Tag) - << "\tAllocs: " << allocInfo.Stats.Allocs - << "\tFrees: " << allocInfo.Stats.Frees - << "\tCurrentSize: " << FormatSize(allocInfo.Stats.CurrentSize) - << Endl; - FormatBackTrace(allocInfo.Stack.data(), allocInfo.Stack.size()); - PrintedCount++; -} - -void TAllocationStatsDumper::FormatBackTrace(void* const* stack, size_t sz) { - char name[1024]; - for (size_t i = 0; i < sz; ++i) { - TSymbol symbol; - auto it = SymbolCache.Find(stack[i]); - if (it != SymbolCache.End()) { - symbol = it.Value(); - } else { - TResolvedSymbol rs = ResolveSymbol(stack[i], name, sizeof(name)); - symbol = {rs.NearestSymbol, rs.Name}; - SymbolCache.Insert(stack[i], symbol); - } - - Out << Hex((intptr_t)stack[i], HF_FULL) << "\t" << symbol.Name; - intptr_t offset = (intptr_t)stack[i] - (intptr_t)symbol.Address; - if (offset) - Out << " +" << offset; - Out << Endl; - } -} - -} // namespace NAllocProfiler + +TString IAllocationStatsDumper::FormatTag(int tag) { + return ToString(tag); +} + +TString IAllocationStatsDumper::FormatSize(intptr_t sz) { + return ToString(sz); +} + + +TAllocationStatsDumper::TAllocationStatsDumper(IOutputStream& out) + : PrintedCount(0) + , Out(out) + , SymbolCache(2048) +{} + +void TAllocationStatsDumper::DumpTotal(const TStats& total) { + Out << "TOTAL" + << "\tAllocs: " << total.Allocs + << "\tFrees: " << total.Frees + << "\tCurrentSize: " << FormatSize(total.CurrentSize) + << Endl; +} + +void TAllocationStatsDumper::DumpEntry(const TAllocationInfo& allocInfo) { + Out << Endl + << "STACK #" << PrintedCount+1 << ": " << FormatTag(allocInfo.Tag) + << "\tAllocs: " << allocInfo.Stats.Allocs + << "\tFrees: " << allocInfo.Stats.Frees + << "\tCurrentSize: " << FormatSize(allocInfo.Stats.CurrentSize) + << Endl; + FormatBackTrace(allocInfo.Stack.data(), allocInfo.Stack.size()); + PrintedCount++; +} + +void TAllocationStatsDumper::FormatBackTrace(void* const* stack, size_t sz) { + char name[1024]; + for (size_t i = 0; i < sz; ++i) { + TSymbol symbol; + auto it = SymbolCache.Find(stack[i]); + if (it != SymbolCache.End()) { + symbol = it.Value(); + } else { + TResolvedSymbol rs = ResolveSymbol(stack[i], name, sizeof(name)); + symbol = {rs.NearestSymbol, rs.Name}; + SymbolCache.Insert(stack[i], symbol); + } + + Out << Hex((intptr_t)stack[i], HF_FULL) << "\t" << symbol.Name; + intptr_t offset = (intptr_t)stack[i] - (intptr_t)symbol.Address; + if (offset) + Out << " +" << offset; + Out << Endl; + } +} + +} // namespace NAllocProfiler diff --git a/library/cpp/lfalloc/alloc_profiler/stackcollect.h b/library/cpp/lfalloc/alloc_profiler/stackcollect.h index 7c10cd2ffd..80715ed7cb 100644 --- a/library/cpp/lfalloc/alloc_profiler/stackcollect.h +++ b/library/cpp/lfalloc/alloc_profiler/stackcollect.h @@ -2,89 +2,89 @@ #include <library/cpp/containers/stack_vector/stack_vec.h> #include <library/cpp/cache/cache.h> - + #include <util/generic/noncopyable.h> #include <util/generic/ptr.h> #include <util/stream/output.h> -namespace NAllocProfiler { - -struct TStats { - intptr_t Allocs = 0; - intptr_t Frees = 0; - intptr_t CurrentSize = 0; - - void Clear() - { - Allocs = 0; - Frees = 0; - CurrentSize = 0; - } - - void Alloc(size_t size) - { - AtomicIncrement(Allocs); - AtomicAdd(CurrentSize, size); - } - - void Free(size_t size) - { - AtomicIncrement(Frees); - AtomicSub(CurrentSize, size); - } -}; - -struct TAllocationInfo { - int Tag; - TStats Stats; - TStackVec<void*, 64> Stack; - - void Clear() { - Tag = 0; - Stats.Clear(); - Stack.clear(); - } -}; - - -class IAllocationStatsDumper { -public: - virtual ~IAllocationStatsDumper() = default; - - // Total stats - virtual void DumpTotal(const TStats& total) = 0; - - // Stats for individual stack - virtual void DumpEntry(const TAllocationInfo& allocInfo) = 0; - - // App-specific tag printer - virtual TString FormatTag(int tag); - - // Size printer (e.g. "10KB", "100MB", "over 9000") - virtual TString FormatSize(intptr_t sz); -}; - -// Default implementation -class TAllocationStatsDumper: public IAllocationStatsDumper { -public: - explicit TAllocationStatsDumper(IOutputStream& out); - void DumpTotal(const TStats& total) override; - void DumpEntry(const TAllocationInfo& allocInfo) override; - -private: - void FormatBackTrace(void* const* stack, size_t sz); - -private: - struct TSymbol { - const void* Address; - TString Name; - }; - - size_t PrintedCount; - IOutputStream& Out; - TLFUCache<void*, TSymbol> SymbolCache; -}; - +namespace NAllocProfiler { + +struct TStats { + intptr_t Allocs = 0; + intptr_t Frees = 0; + intptr_t CurrentSize = 0; + + void Clear() + { + Allocs = 0; + Frees = 0; + CurrentSize = 0; + } + + void Alloc(size_t size) + { + AtomicIncrement(Allocs); + AtomicAdd(CurrentSize, size); + } + + void Free(size_t size) + { + AtomicIncrement(Frees); + AtomicSub(CurrentSize, size); + } +}; + +struct TAllocationInfo { + int Tag; + TStats Stats; + TStackVec<void*, 64> Stack; + + void Clear() { + Tag = 0; + Stats.Clear(); + Stack.clear(); + } +}; + + +class IAllocationStatsDumper { +public: + virtual ~IAllocationStatsDumper() = default; + + // Total stats + virtual void DumpTotal(const TStats& total) = 0; + + // Stats for individual stack + virtual void DumpEntry(const TAllocationInfo& allocInfo) = 0; + + // App-specific tag printer + virtual TString FormatTag(int tag); + + // Size printer (e.g. "10KB", "100MB", "over 9000") + virtual TString FormatSize(intptr_t sz); +}; + +// Default implementation +class TAllocationStatsDumper: public IAllocationStatsDumper { +public: + explicit TAllocationStatsDumper(IOutputStream& out); + void DumpTotal(const TStats& total) override; + void DumpEntry(const TAllocationInfo& allocInfo) override; + +private: + void FormatBackTrace(void* const* stack, size_t sz); + +private: + struct TSymbol { + const void* Address; + TString Name; + }; + + size_t PrintedCount; + IOutputStream& Out; + TLFUCache<void*, TSymbol> SymbolCache; +}; + //////////////////////////////////////////////////////////////////////////////// class TAllocationStackCollector: private TNonCopyable { @@ -101,7 +101,7 @@ public: void Clear(); - void Dump(int count, IAllocationStatsDumper& out) const; + void Dump(int count, IAllocationStatsDumper& out) const; }; -} // namespace NAllocProfiler +} // namespace NAllocProfiler diff --git a/library/cpp/lfalloc/alloc_profiler/ut/ya.make b/library/cpp/lfalloc/alloc_profiler/ut/ya.make index c90a1278d5..8a7daa74af 100644 --- a/library/cpp/lfalloc/alloc_profiler/ut/ya.make +++ b/library/cpp/lfalloc/alloc_profiler/ut/ya.make @@ -1,22 +1,22 @@ UNITTEST_FOR(library/cpp/lfalloc/alloc_profiler) - -OWNER(g:rtmr g:kikimr) - -PEERDIR( + +OWNER(g:rtmr g:kikimr) + +PEERDIR( library/cpp/testing/unittest -) - -IF (ARCH_AARCH64) - PEERDIR( - contrib/libs/jemalloc - ) -ELSE() - ALLOCATOR(LF_DBG) -ENDIF() - -SRCS( - profiler_ut.cpp +) + +IF (ARCH_AARCH64) + PEERDIR( + contrib/libs/jemalloc + ) +ELSE() + ALLOCATOR(LF_DBG) +ENDIF() + +SRCS( + profiler_ut.cpp align_ut.cpp -) - -END() +) + +END() diff --git a/library/cpp/lfalloc/alloc_profiler/ya.make b/library/cpp/lfalloc/alloc_profiler/ya.make index dd1bfb0918..0f58d91767 100644 --- a/library/cpp/lfalloc/alloc_profiler/ya.make +++ b/library/cpp/lfalloc/alloc_profiler/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - -OWNER(g:rtmr g:kikimr) - -SRCS( - profiler.cpp - stackcollect.cpp -) - -PEERDIR( +LIBRARY() + +OWNER(g:rtmr g:kikimr) + +SRCS( + profiler.cpp + stackcollect.cpp +) + +PEERDIR( library/cpp/lfalloc/dbg_info library/cpp/cache -) - -END() - -RECURSE(ut) +) + +END() + +RECURSE(ut) |