diff options
author | max42 <max42@yandex-team.com> | 2023-06-30 03:37:03 +0300 |
---|---|---|
committer | max42 <max42@yandex-team.com> | 2023-06-30 03:37:03 +0300 |
commit | fac2bd72b4b31ec3238292caf8fb2a8aaa6d6c4a (patch) | |
tree | b8cbc1deb00309c7f1a7ab6df520a76cf0b5c6d7 /library/cpp/yt | |
parent | 7bf166b1a7ed0af927f230022b245af618e998c1 (diff) | |
download | ydb-fac2bd72b4b31ec3238292caf8fb2a8aaa6d6c4a.tar.gz |
YT-19324: move YT provider to ydb/library/yql
This commit is formed by the following script: https://paste.yandex-team.ru/6f92e4b8-efc5-4d34-948b-15ee2accd7e7/text.
This commit has zero effect on all projects that depend on YQL.
The summary of changes:
- `yql/providers/yt -> ydb/library/yql/providers/yt `- the whole implementation of YT provider is moved into YDB code base for further export as a part of YT YQL plugin shared library;
- `yql/providers/stat/{expr_nodes,uploader} -> ydb/library/yql/providers/stat/{expr_nodes,uploader}` - a small interface without implementation and the description of stat expr nodes;
- `yql/core/extract_predicate/ut -> ydb/library/yql/core/extract_predicate/ut`;
- `yql/core/{ut,ut_common} -> ydb/library/yql/core/{ut,ut_common}`;
- `yql/core` is gone;
- `yql/library/url_preprocessing -> ydb/library/yql/core/url_preprocessing`.
**NB**: all new targets inside `ydb/` are under `IF (NOT CMAKE_EXPORT)` clause which disables them from open-source cmake generation and ya make build. They will be enabled in the subsequent commits.
Diffstat (limited to 'library/cpp/yt')
51 files changed, 3680 insertions, 0 deletions
diff --git a/library/cpp/yt/backtrace/backtrace-inl.h b/library/cpp/yt/backtrace/backtrace-inl.h new file mode 100644 index 0000000000..b78eeffd75 --- /dev/null +++ b/library/cpp/yt/backtrace/backtrace-inl.h @@ -0,0 +1,36 @@ +#pragma once +#ifndef BACKTRACE_INL_H_ +#error "Direct inclusion of this file is not allowed, include backtrace.h" +// For the sake of sane code completion. +#include "backtrace.h" +#endif + +#include <util/system/compiler.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +template <class TCursor> +Y_NO_INLINE TBacktrace GetBacktrace( + TCursor* cursor, + TBacktraceBuffer buffer, + int framesToSkip) +{ + // Account for the current frame. + ++framesToSkip; + size_t frameCount = 0; + while (frameCount < buffer.size() && !cursor->IsFinished()) { + if (framesToSkip > 0) { + --framesToSkip; + } else { + buffer[frameCount++] = cursor->GetCurrentIP(); + } + cursor->MoveNext(); + } + return {buffer.begin(), frameCount}; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/backtrace.cpp b/library/cpp/yt/backtrace/backtrace.cpp new file mode 100644 index 0000000000..153a0a5dd0 --- /dev/null +++ b/library/cpp/yt/backtrace/backtrace.cpp @@ -0,0 +1,18 @@ +#include "backtrace.h" + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +TString SymbolizeBacktrace(TBacktrace backtrace) +{ + TString result; + SymbolizeBacktrace( + backtrace, + [&] (TStringBuf str) { result += str; }); + return result; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/backtrace.h b/library/cpp/yt/backtrace/backtrace.h new file mode 100644 index 0000000000..ea70d9558c --- /dev/null +++ b/library/cpp/yt/backtrace/backtrace.h @@ -0,0 +1,45 @@ +#pragma once + +#include <library/cpp/yt/memory/range.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +using TBacktrace = TRange<const void*>; +using TBacktraceBuffer = TMutableRange<const void*>; + +//! Obtains a backtrace via a given cursor. +/*! + * \param buffer is the buffer where the backtrace is written to + * \param framesToSkip is the number of top frames to skip + * \returns the portion of #buffer that has actually been filled + */ +template <class TCursor> +TBacktrace GetBacktrace( + TCursor* cursor, + TBacktraceBuffer buffer, + int framesToSkip); + +//! Symbolizes a backtrace invoking a given callback for each frame. +/*! + * \param backtrace Backtrace to symbolize + * \param frameCallback Callback to invoke per each frame + */ +void SymbolizeBacktrace( + TBacktrace backtrace, + const std::function<void(TStringBuf)>& frameCallback); + +//! Symbolizes a backtrace to a string. +/*! + * \param backtrace Backtrace to symbolize + */ +TString SymbolizeBacktrace(TBacktrace backtrace); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace + +#define BACKTRACE_INL_H_ +#include "backtrace-inl.h" +#undef BACKTRACE_INL_H_ diff --git a/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.cpp b/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.cpp new file mode 100644 index 0000000000..ea6e0bc08e --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.cpp @@ -0,0 +1,22 @@ +#include "dummy_cursor.h" + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +bool TDummyCursor::IsFinished() const +{ + return true; +} + +const void* TDummyCursor::GetCurrentIP() const +{ + return nullptr; +} + +void TDummyCursor::MoveNext() +{ } + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.h b/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.h new file mode 100644 index 0000000000..b47d7d2aba --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/dummy/dummy_cursor.h @@ -0,0 +1,17 @@ +#pragma once + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +class TDummyCursor +{ +public: + bool IsFinished() const; + const void* GetCurrentIP() const; + void MoveNext(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/dummy/ya.make b/library/cpp/yt/backtrace/cursors/dummy/ya.make new file mode 100644 index 0000000000..49fd7be050 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/dummy/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + dummy_cursor.cpp +) + +END() diff --git a/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.cpp b/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.cpp new file mode 100644 index 0000000000..290d30c3ce --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.cpp @@ -0,0 +1,146 @@ +#include "frame_pointer_cursor.h" + +#include <util/generic/size_literals.h> + +#include <array> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +TFramePointerCursor::TFramePointerCursor( + TSafeMemoryReader* memoryReader, + const TFramePointerCursorContext& context) + : MemoryReader_(memoryReader) + , Rip_(reinterpret_cast<const void*>(context.Rip)) + , Rbp_(reinterpret_cast<const void*>(context.Rbp)) + , StartRsp_(reinterpret_cast<const void*>(context.Rsp)) +{ } + +bool TFramePointerCursor::IsFinished() const +{ + return Finished_; +} + +const void* TFramePointerCursor::GetCurrentIP() const +{ + return Rip_; +} + +void TFramePointerCursor::MoveNext() +{ + if (Finished_) { + return; + } + + auto add = [] (auto ptr, auto delta) { + return reinterpret_cast<void*>(reinterpret_cast<intptr_t>(ptr) + delta); + }; + + auto checkPtr = [&] (auto ptr) { + ui8 data; + return MemoryReader_->Read(ptr, &data); + }; + + // We try unwinding stack manually by following frame pointers. + // + // We assume that stack does not span more than 4mb. + + if (First_) { + First_ = false; + + // For the first frame there are three special cases where naive + // unwinding would skip the caller frame. + // + // 1) Right after call instruction, rbp points to frame of a caller. + // 2) Right after "push rbp" instruction. + // 3) Right before ret instruction, rbp points to frame of a caller. + // + // We read current instruction and try to detect such cases. + // + // 55 push %rbp + // 48 89 e5 mov %rsp, %rbp + // c3 retq + + std::array<ui8, 3> data; + if (!MemoryReader_->Read(Rip_, &data)) { + Finished_ = true; + return; + } + + if (data[0] == 0xc3 || data[0] == 0x55) { + void* savedRip; + if (!MemoryReader_->Read(StartRsp_, &savedRip)) { + Finished_ = true; + return; + } + + // Avoid infinite loop. + if (Rip_ == savedRip) { + Finished_ = true; + return; + } + + // Detect garbage pointer. + if (!checkPtr(savedRip)) { + Finished_ = true; + return; + } + + Rip_ = savedRip; + return; + } + + if (data[0] == 0x48 && data[1] == 0x89 && data[2] == 0xe5) { + void* savedRip; + if (!MemoryReader_->Read(add(StartRsp_, 8), &savedRip)) { + Finished_ = true; + return; + } + + // Avoid infinite loop. + if (Rip_ == savedRip) { + Finished_ = true; + return; + } + + // Detect garbage pointer. + if (!checkPtr(savedRip)) { + Finished_ = true; + return; + } + + Rip_ = savedRip; + return; + } + } + + const void* savedRbp; + const void* savedRip; + if (!MemoryReader_->Read(Rbp_, &savedRbp) || !MemoryReader_->Read(add(Rbp_, 8), &savedRip)) { + Finished_ = true; + return; + } + + if (!checkPtr(savedRbp)) { + Finished_ = true; + return; + } + + if (!checkPtr(savedRip)) { + Finished_ = true; + return; + } + + if (savedRbp < StartRsp_ || savedRbp > add(StartRsp_, 4_MB)) { + Finished_ = true; + return; + } + + Rip_ = savedRip; + Rbp_ = savedRbp; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h b/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h new file mode 100644 index 0000000000..7a6eaf431b --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h @@ -0,0 +1,39 @@ +#pragma once + +#include <library/cpp/yt/memory/safe_memory_reader.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +struct TFramePointerCursorContext +{ + ui64 Rip; + ui64 Rsp; + ui64 Rbp; +}; + +class TFramePointerCursor +{ +public: + TFramePointerCursor( + TSafeMemoryReader* memoryReader, + const TFramePointerCursorContext& context); + + bool IsFinished() const; + const void* GetCurrentIP() const; + void MoveNext(); + +private: + TSafeMemoryReader* MemoryReader_; + bool Finished_ = false; + bool First_ = true; + + const void* Rip_ = nullptr; + const void* Rbp_ = nullptr; + const void* StartRsp_ = nullptr; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/frame_pointer/ya.make b/library/cpp/yt/backtrace/cursors/frame_pointer/ya.make new file mode 100644 index 0000000000..cb85d70315 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/frame_pointer/ya.make @@ -0,0 +1,9 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + frame_pointer_cursor.cpp +) + +END() diff --git a/library/cpp/yt/backtrace/cursors/interop/interop.cpp b/library/cpp/yt/backtrace/cursors/interop/interop.cpp new file mode 100644 index 0000000000..b4e6cfbe6e --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/interop/interop.cpp @@ -0,0 +1,102 @@ +#include "interop.h" + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +TFramePointerCursorContext FramePointerCursorContextFromUcontext(const ucontext_t& ucontext) +{ +#if defined(_linux_) + return { + .Rip = static_cast<ui64>(ucontext.uc_mcontext.gregs[REG_RIP]), + .Rsp = static_cast<ui64>(ucontext.uc_mcontext.gregs[REG_RSP]), + .Rbp = static_cast<ui64>(ucontext.uc_mcontext.gregs[REG_RBP]), + }; +#elif defined(_darwin_) + return { + .Rip = static_cast<ui64>(ucontext.uc_mcontext->__ss.__rip), + .Rsp = static_cast<ui64>(ucontext.uc_mcontext->__ss.__rsp), + .Rbp = static_cast<ui64>(ucontext.uc_mcontext->__ss.__rbp), + }; +#else + #error Unsupported platform +#endif +} + +std::optional<unw_context_t> TrySynthesizeLibunwindContextFromMachineContext( + const TContMachineContext& machineContext) +{ + unw_context_t unwindContext; + if (unw_getcontext(&unwindContext) != 0) { + return {}; + } + + // Some dirty hacks follow. + struct TUnwindContextRegisters + { + ui64 Rax; + ui64 Rbx; + ui64 Rcx; + ui64 Rdx; + ui64 Rdi; + ui64 Rsi; + ui64 Rbp; + ui64 Rsp; + ui64 R8; + ui64 R9; + ui64 R10; + ui64 R11; + ui64 R12; + ui64 R13; + ui64 R14; + ui64 R15; + ui64 Rip; + ui64 Rflags; + ui64 CS; + ui64 FS; + ui64 GS; + }; + + struct TMachineContextRegisters + { + ui64 Rbx; + ui64 Rbp; + ui64 R12; + ui64 R13; + ui64 R14; + ui64 R15; + ui64 Rsp; + ui64 Rip; + }; + + static_assert(sizeof(TContMachineContext) >= sizeof(TMachineContextRegisters)); + static_assert(sizeof(unw_context_t) >= sizeof(TUnwindContextRegisters)); + const auto* machineContextRegisters = reinterpret_cast<const TMachineContextRegisters*>(&machineContext); + auto* unwindContextRegisters = reinterpret_cast<TUnwindContextRegisters*>(&unwindContext); + #define XX(register) unwindContextRegisters->register = machineContextRegisters->register; + XX(Rbx) + XX(Rbp) + XX(R12) + XX(R13) + XX(R14) + XX(R15) + XX(Rsp) + XX(Rip) + #undef XX + return unwindContext; +} + +TFramePointerCursorContext FramePointerCursorContextFromLibunwindCursor( + const unw_cursor_t& cursor) +{ + TFramePointerCursorContext context{}; + auto& mutableCursor = const_cast<unw_cursor_t&>(cursor); + YT_VERIFY(unw_get_reg(&mutableCursor, UNW_REG_IP, &context.Rip) == 0); + YT_VERIFY(unw_get_reg(&mutableCursor, UNW_X86_64_RSP, &context.Rsp) == 0); + YT_VERIFY(unw_get_reg(&mutableCursor, UNW_X86_64_RBP, &context.Rbp) == 0); + return context; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/interop/interop.h b/library/cpp/yt/backtrace/cursors/interop/interop.h new file mode 100644 index 0000000000..62e7177107 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/interop/interop.h @@ -0,0 +1,25 @@ +#pragma once + +#include <library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h> + +#include <contrib/libs/libunwind/include/libunwind.h> + +#include <util/system/context.h> + +#include <optional> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +TFramePointerCursorContext FramePointerCursorContextFromUcontext(const ucontext_t& ucontext); + +std::optional<unw_context_t> TrySynthesizeLibunwindContextFromMachineContext( + const TContMachineContext& machineContext); + +TFramePointerCursorContext FramePointerCursorContextFromLibunwindCursor( + const unw_cursor_t& uwCursor); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/interop/ya.make b/library/cpp/yt/backtrace/cursors/interop/ya.make new file mode 100644 index 0000000000..6637f6a9b4 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/interop/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + interop.cpp +) + +PEERDIR( + library/cpp/yt/backtrace/cursors/frame_pointer + contrib/libs/libunwind +) + +END() diff --git a/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.cpp b/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.cpp new file mode 100644 index 0000000000..f814753034 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.cpp @@ -0,0 +1,70 @@ +#include "libunwind_cursor.h" + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +TLibunwindCursor::TLibunwindCursor() +{ + if (unw_getcontext(&Context_) != 0) { + Finished_ = true; + return; + } + + Initialize(); +} + +TLibunwindCursor::TLibunwindCursor(const unw_context_t& context) + : Context_(context) +{ + Initialize(); +} + +void TLibunwindCursor::Initialize() +{ + if (unw_init_local(&Cursor_, &Context_) != 0) { + Finished_ = true; + return; + } + + ReadCurrentIP(); +} + +bool TLibunwindCursor::IsFinished() const +{ + return Finished_; +} + +const void* TLibunwindCursor::GetCurrentIP() const +{ + return CurrentIP_; +} + +void TLibunwindCursor::MoveNext() +{ + if (Finished_) { + return; + } + + if (unw_step(&Cursor_) <= 0) { + Finished_ = true; + return; + } + + ReadCurrentIP(); +} + +void TLibunwindCursor::ReadCurrentIP() +{ + unw_word_t ip = 0; + if (unw_get_reg(&Cursor_, UNW_REG_IP, &ip) < 0) { + Finished_ = true; + return; + } + + CurrentIP_ = reinterpret_cast<const void*>(ip); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h b/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h new file mode 100644 index 0000000000..08b01d07ef --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/libunwind/libunwind_cursor.h @@ -0,0 +1,33 @@ +#pragma once + +#include <contrib/libs/libunwind/include/libunwind.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +class TLibunwindCursor +{ +public: + TLibunwindCursor(); + explicit TLibunwindCursor(const unw_context_t& context); + + bool IsFinished() const; + const void* GetCurrentIP() const; + void MoveNext(); + +private: + unw_context_t Context_; + unw_cursor_t Cursor_; + + bool Finished_ = false; + + const void* CurrentIP_ = nullptr; + + void Initialize(); + void ReadCurrentIP(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/cursors/libunwind/ya.make b/library/cpp/yt/backtrace/cursors/libunwind/ya.make new file mode 100644 index 0000000000..8f3a8c5284 --- /dev/null +++ b/library/cpp/yt/backtrace/cursors/libunwind/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + libunwind_cursor.cpp +) + +PEERDIR( + contrib/libs/libunwind +) + +END() diff --git a/library/cpp/yt/backtrace/symbolizers/dummy/dummy_symbolizer.cpp b/library/cpp/yt/backtrace/symbolizers/dummy/dummy_symbolizer.cpp new file mode 100644 index 0000000000..19cb41e795 --- /dev/null +++ b/library/cpp/yt/backtrace/symbolizers/dummy/dummy_symbolizer.cpp @@ -0,0 +1,25 @@ +#include <library/cpp/yt/backtrace/backtrace.h> + +#include <library/cpp/yt/string/raw_formatter.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +void SymbolizeBacktrace( + TBacktrace backtrace, + const std::function<void(TStringBuf)>& frameCallback) +{ + for (int index = 0; index < std::ssize(backtrace); ++index) { + TRawFormatter<1024> formatter; + formatter.AppendNumber(index + 1, 10, 2); + formatter.AppendString(". "); + formatter.AppendNumberAsHexWithPadding(reinterpret_cast<uintptr_t>(backtrace[index]), 12); + formatter.AppendString("\n"); + frameCallback(formatter.GetBuffer()); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/symbolizers/dwarf/dwarf_symbolizer.cpp b/library/cpp/yt/backtrace/symbolizers/dwarf/dwarf_symbolizer.cpp new file mode 100644 index 0000000000..f5d02aaa33 --- /dev/null +++ b/library/cpp/yt/backtrace/symbolizers/dwarf/dwarf_symbolizer.cpp @@ -0,0 +1,64 @@ +#include <library/cpp/yt/backtrace/backtrace.h> + +#include <library/cpp/dwarf_backtrace/backtrace.h> + +#include <library/cpp/yt/string/raw_formatter.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +void SymbolizeBacktrace( + TBacktrace backtrace, + const std::function<void(TStringBuf)>& frameCallback) +{ + auto error = NDwarf::ResolveBacktrace({backtrace.begin(), backtrace.size()}, [&] (const NDwarf::TLineInfo& info) { + TRawFormatter<1024> formatter; + formatter.AppendNumber(info.Index + 1, 10, 2); + formatter.AppendString(". "); + formatter.AppendString("0x"); + const int width = (sizeof(void*) == 8 ? 12 : 8); + // 12 for x86_64 because higher bits are always zeroed. + formatter.AppendNumber(info.Address, 16, width, '0'); + formatter.AppendString(" in "); + formatter.AppendString(info.FunctionName); + const int bytesToAppendEstimate = 4 + info.FileName.Size() + 1 + 4 /* who cares about line numbers > 9999 */ + 1; + if (formatter.GetBytesRemaining() < bytesToAppendEstimate) { + const int offset = formatter.GetBytesRemaining() - bytesToAppendEstimate; + if (formatter.GetBytesWritten() + offset >= 0) { + formatter.Advance(offset); + } + } + formatter.AppendString(" at "); + formatter.AppendString(info.FileName); + formatter.AppendChar(':'); + formatter.AppendNumber(info.Line); + if (formatter.GetBytesRemaining() == 0) { + formatter.Revert(1); + } + formatter.AppendString("\n"); + frameCallback(formatter.GetBuffer()); + // Call the callback exactly `frameCount` times, + // even if there are inline functions and one frame resolved to several lines. + // It needs for case when caller uses `frameCount` less than 100 for pretty formatting. + if (info.Index + 1 == std::ssize(backtrace)) { + return NDwarf::EResolving::Break; + } + return NDwarf::EResolving::Continue; + }); + if (error) { + TRawFormatter<1024> formatter; + formatter.AppendString("*** Error symbolizing backtrace via Dwarf\n"); + formatter.AppendString("*** Code: "); + formatter.AppendNumber(error->Code); + formatter.AppendString("\n"); + formatter.AppendString("*** Message: "); + formatter.AppendString(error->Message); + formatter.AppendString("\n"); + frameCallback(formatter.GetBuffer()); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/symbolizers/dwarf/ya.make b/library/cpp/yt/backtrace/symbolizers/dwarf/ya.make new file mode 100644 index 0000000000..bffeb676d8 --- /dev/null +++ b/library/cpp/yt/backtrace/symbolizers/dwarf/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +SRCS( + GLOBAL dwarf_symbolizer.cpp +) + +PEERDIR( + library/cpp/dwarf_backtrace + library/cpp/yt/backtrace +) + +END() + +IF (BUILD_TYPE == "DEBUG" OR BUILD_TYPE == "PROFILE") + RECURSE_FOR_TESTS( + unittests + ) +ENDIF() diff --git a/library/cpp/yt/backtrace/symbolizers/dynload/dynload_symbolizer.cpp b/library/cpp/yt/backtrace/symbolizers/dynload/dynload_symbolizer.cpp new file mode 100644 index 0000000000..37ebda8e48 --- /dev/null +++ b/library/cpp/yt/backtrace/symbolizers/dynload/dynload_symbolizer.cpp @@ -0,0 +1,113 @@ +#include <library/cpp/yt/backtrace/backtrace.h> + +#include <library/cpp/yt/string/raw_formatter.h> + +#include <util/system/compiler.h> + +#include <dlfcn.h> +#include <cxxabi.h> + +namespace NYT::NBacktrace { + +//////////////////////////////////////////////////////////////////////////////// + +namespace { + +int GetSymbolInfo(const void* pc, char* buffer, int length) +{ + TBaseFormatter formatter(buffer, length); + + // See http://www.codesourcery.com/cxx-abi/abi.html#mangling + // And, yes, dladdr() is not async signal safe. We can substitute it + // with hand-written symbolization code from google-glog in case of any trouble. + Dl_info info; + if (!dladdr(pc, &info)) { + return 0; + } + + /* + * typedef struct { + * const char *dli_fname; // Pathname of shared object that + * // contains address + * void *dli_fbase; // Address at which shared object + * // is loaded + * const char *dli_sname; // Name of nearest symbol with address + * // lower than addr + * void *dli_saddr; // Exact address of symbol named + * // in dli_sname + * } Dl_info; + * + * If no symbol matching addr could be found, then dli_sname and dli_saddr are set to NULL. + */ + + if (info.dli_sname && info.dli_saddr) { + formatter.AppendString("<"); + int demangleStatus = 0; + + if (info.dli_sname[0] == '_' && info.dli_sname[1] == 'Z') { + // This is also not async signal safe. + // But (ta-dah!) we can replace it with symbolization code from google-glob. + char* demangledName = abi::__cxa_demangle(info.dli_sname, 0, 0, &demangleStatus); + if (demangleStatus == 0) { + formatter.AppendString(demangledName); + } else { + formatter.AppendString(info.dli_sname); + } + free(demangledName); + } else { + formatter.AppendString(info.dli_sname); + } + formatter.AppendString("+"); + formatter.AppendNumber((char*)pc - (char*)info.dli_saddr); + formatter.AppendString(">"); + formatter.AppendString(" "); + } + + if (info.dli_fname && info.dli_fbase) { + formatter.AppendString("("); + formatter.AppendString(info.dli_fname); + formatter.AppendString("+"); + formatter.AppendNumber((char*)pc - (char*)info.dli_fbase); + formatter.AppendString(")"); + } + return formatter.GetBytesWritten(); +} + +void DumpStackFrameInfo(TBaseFormatter* formatter, const void* pc) +{ + formatter->AppendString("@ "); + const int width = (sizeof(void*) == 8 ? 12 : 8) + 2; + // +2 for "0x"; 12 for x86_64 because higher bits are always zeroed. + formatter->AppendNumberAsHexWithPadding(reinterpret_cast<uintptr_t>(pc), width); + formatter->AppendString(" "); + // Get the symbol from the previous address of PC, + // because PC may be in the next function. + formatter->Advance(GetSymbolInfo( + reinterpret_cast<const char*>(pc) - 1, + formatter->GetCursor(), + formatter->GetBytesRemaining())); + if (formatter->GetBytesRemaining() == 0) { + formatter->Revert(1); + } + formatter->AppendString("\n"); +} + +} // namespace + +Y_WEAK void SymbolizeBacktrace( + TBacktrace backtrace, + const std::function<void(TStringBuf)>& frameCallback) +{ + for (int i = 0; i < std::ssize(backtrace); ++i) { + TRawFormatter<1024> formatter; + formatter.Reset(); + formatter.AppendNumber(i + 1, 10, 2); + formatter.AppendString(". "); + DumpStackFrameInfo(&formatter, backtrace[i]); + frameCallback(formatter.GetBuffer()); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/unittests/backtrace_ut.cpp b/library/cpp/yt/backtrace/unittests/backtrace_ut.cpp new file mode 100644 index 0000000000..5992b69277 --- /dev/null +++ b/library/cpp/yt/backtrace/unittests/backtrace_ut.cpp @@ -0,0 +1,61 @@ +#include <gtest/gtest.h> + +#include <gmock/gmock.h> + +#include <library/cpp/yt/memory/safe_memory_reader.h> + +#include <library/cpp/yt/backtrace/cursors/frame_pointer/frame_pointer_cursor.h> + +#include <library/cpp/yt/backtrace/cursors/interop/interop.h> + +#include <util/system/compiler.h> + +#include <contrib/libs/libunwind/include/libunwind.h> + +namespace NYT::NBacktrace { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +template <int Depth, class TFn> +Y_NO_INLINE void RunInDeepStack(TFn cb) +{ + if constexpr (Depth == 0) { + cb(); + } else { + std::vector<int> touchMem; + touchMem.push_back(0); + + RunInDeepStack<Depth-1>(cb); + + DoNotOptimizeAway(touchMem); + } +} + +TEST(TFramePointerCursor, FramePointerCursor) +{ + std::vector<const void*> backtrace; + RunInDeepStack<64>([&] { + unw_context_t unwContext; + ASSERT_TRUE(unw_getcontext(&unwContext) == 0); + + unw_cursor_t unwCursor; + ASSERT_TRUE(unw_init_local(&unwCursor, &unwContext) == 0); + + TSafeMemoryReader reader; + auto fpCursorContext = NBacktrace::FramePointerCursorContextFromLibunwindCursor(unwCursor); + NBacktrace::TFramePointerCursor fpCursor(&reader, fpCursorContext); + + while (!fpCursor.IsFinished()) { + backtrace.push_back(fpCursor.GetCurrentIP()); + fpCursor.MoveNext(); + } + }); + + ASSERT_THAT(backtrace, testing::SizeIs(testing::Ge(64u))); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NBacktrace diff --git a/library/cpp/yt/backtrace/unittests/ya.make b/library/cpp/yt/backtrace/unittests/ya.make new file mode 100644 index 0000000000..89e55a95ef --- /dev/null +++ b/library/cpp/yt/backtrace/unittests/ya.make @@ -0,0 +1,20 @@ +GTEST() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +PEERDIR( + library/cpp/testing/gtest + library/cpp/yt/backtrace + library/cpp/yt/backtrace/cursors/interop + library/cpp/yt/backtrace/cursors/frame_pointer + library/cpp/yt/backtrace/cursors/libunwind + library/cpp/yt/memory +) + +IF (BUILD_TYPE == "DEBUG" OR BUILD_TYPE == "PROFILE") + SRCS( + backtrace_ut.cpp + ) +ENDIF() + +END() diff --git a/library/cpp/yt/backtrace/ya.make b/library/cpp/yt/backtrace/ya.make new file mode 100644 index 0000000000..d294082e06 --- /dev/null +++ b/library/cpp/yt/backtrace/ya.make @@ -0,0 +1,44 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + backtrace.cpp +) + +IF (OS_WINDOWS) + SRCS( + symbolizers/dummy/dummy_symbolizer.cpp + ) +ELSE() + SRCS( + symbolizers/dynload/dynload_symbolizer.cpp + ) +ENDIF() + +PEERDIR( + library/cpp/yt/string +) + +END() + +RECURSE( + cursors/dummy + cursors/frame_pointer +) + +IF (NOT OS_WINDOWS) + RECURSE( + cursors/libunwind + ) +ENDIF() + +IF (OS_LINUX) + RECURSE( + symbolizers/dwarf + ) + + RECURSE_FOR_TESTS( + unittests + ) +ENDIF() diff --git a/library/cpp/yt/containers/sharded_set-inl.h b/library/cpp/yt/containers/sharded_set-inl.h new file mode 100644 index 0000000000..67d5be58c6 --- /dev/null +++ b/library/cpp/yt/containers/sharded_set-inl.h @@ -0,0 +1,217 @@ +#ifndef SHARDED_SET_INL_H_ +#error "Direct inclusion of this file is not allowed, include sharded_set.h" +// For the sake of sane code completion. +#include "sharded_set.h" +#endif + +#include <library/cpp/yt/assert/assert.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, int N, class F, class S> +class TShardedSet<T, N, F, S>::const_iterator +{ +private: + friend class TShardedSet<T, N, F, S>; + + using TOwner = TShardedSet<T, N, F, S>; + using TShardIterator = typename S::const_iterator; + + const TOwner* const Owner_; + + int ShardIndex_; + TShardIterator ShardIterator_; + + const_iterator( + const TOwner* owner, + int shardIndex, + TShardIterator shardIterator) + : Owner_(owner) + , ShardIndex_(shardIndex) + , ShardIterator_(shardIterator) + { } + + bool IsValid() const + { + return ShardIterator_ != Owner_->Shards_[ShardIndex_].end(); + } + + void FastForward() + { + while (ShardIndex_ != N - 1 && !IsValid()) { + ++ShardIndex_; + ShardIterator_ = Owner_->Shards_[ShardIndex_].begin(); + } + } + +public: + using difference_type = typename std::iterator_traits<TShardIterator>::difference_type; + using value_type = typename std::iterator_traits<TShardIterator>::value_type; + using pointer = typename std::iterator_traits<TShardIterator>::pointer; + using reference = typename std::iterator_traits<TShardIterator>::reference; + using iterator_category = std::forward_iterator_tag; + + const_iterator& operator++() + { + ++ShardIterator_; + FastForward(); + + return *this; + } + + const_iterator operator++(int) + { + auto result = *this; + + ++ShardIterator_; + FastForward(); + + return result; + } + + bool operator==(const const_iterator& rhs) const + { + return + ShardIndex_ == rhs.ShardIndex_ && + ShardIterator_ == rhs.ShardIterator_; + } + + bool operator!=(const const_iterator& rhs) const + { + return !(*this == rhs); + } + + const T& operator*() const + { + return *ShardIterator_; + } + + const T* operator->() const + { + return &operator*(); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, int N, class F, class S> +TShardedSet<T, N, F, S>::TShardedSet(F elementToShard) + : ElementToShard_(elementToShard) +{ } + +template <class T, int N, class F, class S> +bool TShardedSet<T, N, F, S>::empty() const +{ + return size() == 0; +} + +template <class T, int N, class F, class S> +typename TShardedSet<T, N, F, S>::size_type TShardedSet<T, N, F, S>::size() const +{ + size_type result = 0; + for (const auto& shard : Shards_) { + result += shard.size(); + } + + return result; +} + +template <class T, int N, class F, class S> +const T& TShardedSet<T, N, F, S>::front() const +{ + return *begin(); +} + +template <class T, int N, class F, class S> +typename TShardedSet<T, N, F, S>::size_type TShardedSet<T, N, F, S>::count(const T& value) const +{ + return GetShard(value).count(value); +} + +template <class T, int N, class F, class S> +bool TShardedSet<T, N, F, S>::contains(const T& value) const +{ + return GetShard(value).contains(value); +} + +template <class T, int N, class F, class S> +std::pair<typename TShardedSet<T, N, F, S>::const_iterator, bool> TShardedSet<T, N, F, S>::insert(const T& value) +{ + auto shardIndex = ElementToShard_(value); + auto& shard = Shards_[shardIndex]; + auto [shardIterator, inserted] = shard.insert(value); + + const_iterator iterator(this, shardIndex, shardIterator); + return {iterator, inserted}; +} + +template <class T, int N, class F, class S> +bool TShardedSet<T, N, F, S>::erase(const T& value) +{ + return GetShard(value).erase(value); +} + +template <class T, int N, class F, class S> +void TShardedSet<T, N, F, S>::clear() +{ + for (auto& shard : Shards_) { + shard.clear(); + } +} + +template <class T, int N, class F, class S> +typename TShardedSet<T, N, F, S>::const_iterator TShardedSet<T, N, F, S>::begin() const +{ + const_iterator iterator(this, /*shardIndex*/ 0, /*shardIterator*/ Shards_[0].begin()); + iterator.FastForward(); + + return iterator; +} + +template <class T, int N, class F, class S> +typename TShardedSet<T, N, F, S>::const_iterator TShardedSet<T, N, F, S>::cbegin() const +{ + return begin(); +} + +template <class T, int N, class F, class S> +typename TShardedSet<T, N, F, S>::const_iterator TShardedSet<T, N, F, S>::end() const +{ + return const_iterator(this, /*shardIndex*/ N - 1, /*shardIterator*/ Shards_[N - 1].end()); +} + +template <class T, int N, class F, class S> +typename TShardedSet<T, N, F, S>::const_iterator TShardedSet<T, N, F, S>::cend() const +{ + return end(); +} + +template <class T, int N, class F, class S> +const S& TShardedSet<T, N, F, S>::Shard(int shardIndex) const +{ + return Shards_[shardIndex]; +} + +template <class T, int N, class F, class S> +S& TShardedSet<T, N, F, S>::MutableShard(int shardIndex) +{ + return Shards_[shardIndex]; +} + +template <class T, int N, class F, class S> +S& TShardedSet<T, N, F, S>::GetShard(const T& value) +{ + return Shards_[ElementToShard_(value)]; +} + +template <class T, int N, class F, class S> +const S& TShardedSet<T, N, F, S>::GetShard(const T& value) const +{ + return Shards_[ElementToShard_(value)]; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/containers/sharded_set.h b/library/cpp/yt/containers/sharded_set.h new file mode 100644 index 0000000000..fa24893aa4 --- /dev/null +++ b/library/cpp/yt/containers/sharded_set.h @@ -0,0 +1,69 @@ +#pragma once + +#include <util/generic/hash_set.h> + +#include <array> +#include <cstddef> +#include <utility> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +//! A set that stores elements divided into fixed amount of shards. +//! Provides access to whole set and particular shards. +//! The interface is pretty minimalistic, feel free to extend it when needed. +template <class T, int N, class F, class S = THashSet<T>> +class TShardedSet +{ +public: + using size_type = size_t; + using difference_type = ptrdiff_t; + + using value_type = T; + + class const_iterator; + + explicit TShardedSet(F elementToShard = F()); + + [[nodiscard]] bool empty() const; + + size_type size() const; + + const T& front() const; + + size_type count(const T& value) const; + + bool contains(const T& value) const; + + std::pair<const_iterator, bool> insert(const T& value); + + bool erase(const T& value); + + void clear(); + + const_iterator begin() const; + const_iterator cbegin() const; + + const_iterator end() const; + const_iterator cend() const; + + const S& Shard(int shardIndex) const; + S& MutableShard(int shardIndex); + +private: + std::array<S, N> Shards_; + + const F ElementToShard_; + + S& GetShard(const T& value); + const S& GetShard(const T& value) const; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + +#define SHARDED_SET_INL_H_ +#include "sharded_set-inl.h" +#undef SHARDED_SET_INL_H_ diff --git a/library/cpp/yt/containers/unittests/sharded_set_ut.cpp b/library/cpp/yt/containers/unittests/sharded_set_ut.cpp new file mode 100644 index 0000000000..2c4f8c5935 --- /dev/null +++ b/library/cpp/yt/containers/unittests/sharded_set_ut.cpp @@ -0,0 +1,121 @@ +#include <library/cpp/yt/containers/sharded_set.h> + +#include <library/cpp/testing/gtest/gtest.h> + +#include <random> + +namespace NYT { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +struct TIntToShard +{ + int operator()(int value) const + { + return value % 16; + } +}; + +using TSet = TShardedSet<int, 16, TIntToShard>; + +//////////////////////////////////////////////////////////////////////////////// + +TEST(CompactSetTest, Insert) +{ + TSet set; + + for (int i = 0; i < 4; i++) { + set.insert(i); + } + + for (int i = 0; i < 4; i++) { + set.insert(i); + } + + EXPECT_EQ(4u, set.size()); + + for (int i = 0; i < 4; i++) + EXPECT_EQ(1u, set.count(i)); + + EXPECT_EQ(0u, set.count(4)); +} + +TEST(CompactSetTest, Erase) +{ + TSet set; + + for (int i = 0; i < 8; i++) { + set.insert(i); + } + + EXPECT_EQ(8u, set.size()); + + // Remove elements one by one and check if all other elements are still there. + for (int i = 0; i < 8; i++) { + EXPECT_EQ(1u, set.count(i)); + EXPECT_TRUE(set.erase(i)); + EXPECT_EQ(0u, set.count(i)); + EXPECT_EQ(8u - i - 1, set.size()); + for (int j = i + 1; j < 8; j++) { + EXPECT_EQ(1u, set.count(j)); + } + } + + EXPECT_EQ(0u, set.count(8)); +} + +TEST(CompactSetTest, StressTest) +{ + TSet set; + + constexpr int Iterations = 1'000'000; + constexpr int Values = 128; + + THashSet<int> values; + + auto checkEverything = [&] { + EXPECT_EQ(values.size(), set.size()); + EXPECT_EQ(values.empty(), set.empty()); + EXPECT_EQ(values, THashSet<int>(set.begin(), set.end())); + + std::array<THashSet<int>, 16> shards; + for (int value : values) { + shards[value % 16].insert(value); + } + for (int shardIndex = 0; shardIndex < 16; ++shardIndex) { + EXPECT_EQ(shards[shardIndex], set.Shard(shardIndex)); + } + + for (int value = 0; value < Values; ++value) { + EXPECT_EQ(values.contains(value), set.contains(value)); + EXPECT_EQ(values.count(value), set.count(value)); + } + }; + + std::mt19937_64 rng(42); + + for (int iteration = 0; iteration < Iterations; ++iteration) { + if (rng() % 100 == 0) { + set.clear(); + values.clear(); + checkEverything(); + } + + int value = rng() % Values; + if (rng() % 2 == 0) { + set.insert(value); + values.insert(value); + } else { + set.erase(value); + values.erase(value); + } + + checkEverything(); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT diff --git a/library/cpp/yt/containers/unittests/ya.make b/library/cpp/yt/containers/unittests/ya.make new file mode 100644 index 0000000000..3e7cfd4311 --- /dev/null +++ b/library/cpp/yt/containers/unittests/ya.make @@ -0,0 +1,15 @@ +GTEST(unittester-containers) + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + sharded_set_ut.cpp +) + +PEERDIR( + library/cpp/yt/containers + + library/cpp/testing/gtest +) + +END() diff --git a/library/cpp/yt/cpu_clock/benchmark/benchmark.cpp b/library/cpp/yt/cpu_clock/benchmark/benchmark.cpp new file mode 100644 index 0000000000..9d300b6726 --- /dev/null +++ b/library/cpp/yt/cpu_clock/benchmark/benchmark.cpp @@ -0,0 +1,41 @@ +#include "benchmark/benchmark.h" +#include <benchmark/benchmark.h> + +#include <library/cpp/yt/cpu_clock/clock.h> + +namespace NYT { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +void BM_GetCpuInstant(benchmark::State& state) +{ + for (auto _ : state) { + benchmark::DoNotOptimize(GetCpuInstant()); + } +} + +BENCHMARK(BM_GetCpuInstant); + +void BM_GetCpuApproximateInstant(benchmark::State& state) +{ + for (auto _ : state) { + benchmark::DoNotOptimize(GetApproximateCpuInstant()); + } +} + +BENCHMARK(BM_GetCpuApproximateInstant); + +void BM_InstantNow(benchmark::State& state) +{ + for (auto _ : state) { + benchmark::DoNotOptimize(TInstant::Now()); + } +} + +BENCHMARK(BM_InstantNow); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT diff --git a/library/cpp/yt/cpu_clock/benchmark/ya.make b/library/cpp/yt/cpu_clock/benchmark/ya.make new file mode 100644 index 0000000000..4550bf5934 --- /dev/null +++ b/library/cpp/yt/cpu_clock/benchmark/ya.make @@ -0,0 +1,11 @@ +G_BENCHMARK() + +SRCS( + benchmark.cpp +) + +PEERDIR( + library/cpp/yt/cpu_clock +) + +END() diff --git a/library/cpp/yt/cpu_clock/unittests/clock_ut.cpp b/library/cpp/yt/cpu_clock/unittests/clock_ut.cpp new file mode 100644 index 0000000000..bd9cb6d4be --- /dev/null +++ b/library/cpp/yt/cpu_clock/unittests/clock_ut.cpp @@ -0,0 +1,46 @@ +#include <gtest/gtest.h> + +#include <library/cpp/yt/cpu_clock/clock.h> + +namespace NYT { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +i64 DiffMS(T a, T b) +{ + return a >= b + ? static_cast<i64>(a.MilliSeconds()) - static_cast<i64>(b.MilliSeconds()) + : DiffMS(b, a); +} + +TEST(TTimingTest, GetInstant) +{ + GetInstant(); + + EXPECT_LE(DiffMS(GetInstant(), TInstant::Now()), 10); +} + +TEST(TTimingTest, InstantVSCpuInstant) +{ + auto instant1 = TInstant::Now(); + auto cpuInstant = InstantToCpuInstant(instant1); + auto instant2 = CpuInstantToInstant(cpuInstant); + EXPECT_LE(DiffMS(instant1, instant2), 10); +} + +TEST(TTimingTest, DurationVSCpuDuration) +{ + auto cpuInstant1 = GetCpuInstant(); + constexpr auto duration1 = TDuration::MilliSeconds(100); + Sleep(duration1); + auto cpuInstant2 = GetCpuInstant(); + auto duration2 = CpuDurationToDuration(cpuInstant2 - cpuInstant1); + EXPECT_LE(DiffMS(duration1, duration2), 10); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT diff --git a/library/cpp/yt/cpu_clock/unittests/ya.make b/library/cpp/yt/cpu_clock/unittests/ya.make new file mode 100644 index 0000000000..921087c295 --- /dev/null +++ b/library/cpp/yt/cpu_clock/unittests/ya.make @@ -0,0 +1,13 @@ +GTEST() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + clock_ut.cpp +) + +PEERDIR( + library/cpp/yt/cpu_clock +) + +END() diff --git a/library/cpp/yt/farmhash/farm_hash.h b/library/cpp/yt/farmhash/farm_hash.h new file mode 100644 index 0000000000..fe4c8193a0 --- /dev/null +++ b/library/cpp/yt/farmhash/farm_hash.h @@ -0,0 +1,63 @@ +#pragma once + +#include <contrib/libs/farmhash/farmhash.h> + +#include <util/system/types.h> + +#include <util/generic/strbuf.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +using TFingerprint = ui64; + +static inline TFingerprint FarmHash(ui64 value) +{ + return ::util::Fingerprint(value); +} + +static inline TFingerprint FarmHash(const void* buf, size_t len) +{ + return ::util::Hash64(static_cast<const char*>(buf), len); +} + +static inline TFingerprint FarmHash(const void* buf, size_t len, ui64 seed) +{ + return ::util::Hash64WithSeed(static_cast<const char*>(buf), len, seed); +} + +static inline TFingerprint FarmFingerprint(ui64 value) +{ + return ::util::Fingerprint(value); +} + +static inline TFingerprint FarmFingerprint(const void* buf, size_t len) +{ + return ::util::Fingerprint64(static_cast<const char*>(buf), len); +} + +static inline TFingerprint FarmFingerprint(TStringBuf buf) +{ + return FarmFingerprint(buf.Data(), buf.Size()); +} + +static inline TFingerprint FarmFingerprint(ui64 first, ui64 second) +{ + return ::util::Fingerprint(::util::Uint128(first, second)); +} + +// Forever-fixed Google FarmHash fingerprint. +template <class T> +TFingerprint FarmFingerprint(const T* begin, const T* end) +{ + ui64 result = 0xdeadc0de; + for (const auto* value = begin; value < end; ++value) { + result = FarmFingerprint(result, FarmFingerprint(*value)); + } + return result ^ (end - begin); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/logging/logger-inl.h b/library/cpp/yt/logging/logger-inl.h new file mode 100644 index 0000000000..6f489da82d --- /dev/null +++ b/library/cpp/yt/logging/logger-inl.h @@ -0,0 +1,303 @@ +#ifndef LOGGER_INL_H_ +#error "Direct inclusion of this file is not allowed, include logger.h" +// For the sake of sane code completion. +#include "logger.h" +#endif +#undef LOGGER_INL_H_ + +#include <library/cpp/yt/yson_string/convert.h> +#include <library/cpp/yt/yson_string/string.h> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +inline bool TLogger::IsAnchorUpToDate(const TLoggingAnchor& position) const +{ + return + !Category_ || + position.CurrentVersion == Category_->ActualVersion->load(std::memory_order::relaxed); +} + +template <class... TArgs> +void TLogger::AddTag(const char* format, TArgs&&... args) +{ + AddRawTag(Format(format, std::forward<TArgs>(args)...)); +} + +template <class TType> +void TLogger::AddStructuredTag(TStringBuf key, TType value) +{ + StructuredTags_.emplace_back(key, NYson::ConvertToYsonString(value)); +} + +template <class... TArgs> +TLogger TLogger::WithTag(const char* format, TArgs&&... args) const +{ + auto result = *this; + result.AddTag(format, std::forward<TArgs>(args)...); + return result; +} + +template <class TType> +TLogger TLogger::WithStructuredTag(TStringBuf key, TType value) const +{ + auto result = *this; + result.AddStructuredTag(key, value); + return result; +} + +Y_FORCE_INLINE bool TLogger::IsLevelEnabled(ELogLevel level) const +{ + // This is the first check which is intended to be inlined next to + // logging invocation point. Check below is almost zero-cost due + // to branch prediction (which requires inlining for proper work). + if (level < MinLevel_) { + return false; + } + + // Next check is heavier and requires full log manager definition which + // is undesirable in -inl.h header file. This is why we extract it + // to a separate method which is implemented in cpp file. + return IsLevelEnabledHeavy(level); +} + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +struct TMessageStringBuilderContext +{ + TSharedMutableRef Chunk; +}; + +struct TMessageBufferTag +{ }; + +class TMessageStringBuilder + : public TStringBuilderBase +{ +public: + TSharedRef Flush(); + + // For testing only. + static void DisablePerThreadCache(); + +protected: + void DoReset() override; + void DoReserve(size_t newLength) override; + +private: + struct TPerThreadCache + { + ~TPerThreadCache(); + + TSharedMutableRef Chunk; + size_t ChunkOffset = 0; + }; + + TSharedMutableRef Buffer_; + + static thread_local TPerThreadCache* Cache_; + static thread_local bool CacheDestroyed_; + static TPerThreadCache* GetCache(); + + static constexpr size_t ChunkSize = 128_KB - 64; +}; + +inline bool HasMessageTags( + const TLoggingContext& loggingContext, + const TLogger& logger) +{ + if (logger.GetTag()) { + return true; + } + if (loggingContext.TraceLoggingTag) { + return true; + } + return false; +} + +inline void AppendMessageTags( + TStringBuilderBase* builder, + const TLoggingContext& loggingContext, + const TLogger& logger) +{ + bool printComma = false; + if (const auto& loggerTag = logger.GetTag()) { + builder->AppendString(loggerTag); + printComma = true; + } + if (auto traceLoggingTag = loggingContext.TraceLoggingTag) { + if (printComma) { + builder->AppendString(TStringBuf(", ")); + } + builder->AppendString(traceLoggingTag); + } +} + +inline void AppendLogMessage( + TStringBuilderBase* builder, + const TLoggingContext& loggingContext, + const TLogger& logger, + TRef message) +{ + if (HasMessageTags(loggingContext, logger)) { + if (message.Size() >= 1 && message[message.Size() - 1] == ')') { + builder->AppendString(TStringBuf(message.Begin(), message.Size() - 1)); + builder->AppendString(TStringBuf(", ")); + } else { + builder->AppendString(TStringBuf(message.Begin(), message.Size())); + builder->AppendString(TStringBuf(" (")); + } + AppendMessageTags(builder, loggingContext, logger); + builder->AppendChar(')'); + } else { + builder->AppendString(TStringBuf(message.Begin(), message.Size())); + } +} + +template <class... TArgs> +void AppendLogMessageWithFormat( + TStringBuilderBase* builder, + const TLoggingContext& loggingContext, + const TLogger& logger, + TStringBuf format, + TArgs&&... args) +{ + if (HasMessageTags(loggingContext, logger)) { + if (format.size() >= 2 && format[format.size() - 1] == ')') { + builder->AppendFormat(format.substr(0, format.size() - 1), std::forward<TArgs>(args)...); + builder->AppendString(TStringBuf(", ")); + } else { + builder->AppendFormat(format, std::forward<TArgs>(args)...); + builder->AppendString(TStringBuf(" (")); + } + AppendMessageTags(builder, loggingContext, logger); + builder->AppendChar(')'); + } else { + builder->AppendFormat(format, std::forward<TArgs>(args)...); + } +} + +struct TLogMessage +{ + TSharedRef MessageRef; + TStringBuf Anchor; +}; + +template <size_t Length, class... TArgs> +TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + const char (&format)[Length], + TArgs&&... args) +{ + TMessageStringBuilder builder; + AppendLogMessageWithFormat(&builder, loggingContext, logger, format, std::forward<TArgs>(args)...); + return {builder.Flush(), format}; +} + +template <class T> +TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + const T& obj) +{ + TMessageStringBuilder builder; + FormatValue(&builder, obj, TStringBuf()); + if (HasMessageTags(loggingContext, logger)) { + builder.AppendString(TStringBuf(" (")); + AppendMessageTags(&builder, loggingContext, logger); + builder.AppendChar(')'); + } + return {builder.Flush(), TStringBuf()}; +} + +inline TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + TStringBuf message) +{ + TMessageStringBuilder builder; + builder.AppendString(message); + if (HasMessageTags(loggingContext, logger)) { + builder.AppendString(TStringBuf(" (")); + AppendMessageTags(&builder, loggingContext, logger); + builder.AppendChar(')'); + } + return {builder.Flush(), message}; +} + +template <size_t Length> +TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + const char (&message)[Length]) +{ + return BuildLogMessage( + loggingContext, + logger, + TStringBuf(message)); +} + +inline TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + TSharedRef&& message) +{ + if (HasMessageTags(loggingContext, logger)) { + TMessageStringBuilder builder; + AppendLogMessage(&builder, loggingContext, logger, message); + return {builder.Flush(), TStringBuf()}; + } else { + return {std::move(message), TStringBuf()}; + } +} + +inline TLogEvent CreateLogEvent( + const TLoggingContext& loggingContext, + const TLogger& logger, + ELogLevel level) +{ + TLogEvent event; + event.Instant = loggingContext.Instant; + event.Category = logger.GetCategory(); + event.Essential = logger.IsEssential(); + event.Level = level; + event.ThreadId = loggingContext.ThreadId; + event.ThreadName = loggingContext.ThreadName; + event.FiberId = loggingContext.FiberId; + event.TraceId = loggingContext.TraceId; + event.RequestId = loggingContext.RequestId; + return event; +} + +void OnCriticalLogEvent( + const TLogger& logger, + const TLogEvent& event); + +inline void LogEventImpl( + const TLoggingContext& loggingContext, + const TLogger& logger, + ELogLevel level, + ::TSourceLocation sourceLocation, + TSharedRef message) +{ + auto event = CreateLogEvent(loggingContext, logger, level); + event.MessageKind = ELogMessageKind::Unstructured; + event.MessageRef = std::move(message); + event.Family = ELogFamily::PlainText; + event.SourceFile = sourceLocation.File; + event.SourceLine = sourceLocation.Line; + logger.Write(std::move(event)); + if (Y_UNLIKELY(event.Level >= ELogLevel::Alert)) { + OnCriticalLogEvent(logger, event); + } +} + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/logger.cpp b/library/cpp/yt/logging/logger.cpp new file mode 100644 index 0000000000..4ee5c1a01b --- /dev/null +++ b/library/cpp/yt/logging/logger.cpp @@ -0,0 +1,289 @@ +#include "logger.h" + +#include <library/cpp/yt/assert/assert.h> + +#include <library/cpp/yt/cpu_clock/clock.h> + +#include <library/cpp/yt/misc/thread_name.h> + +#include <util/system/compiler.h> +#include <util/system/thread.h> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +void OnCriticalLogEvent( + const TLogger& logger, + const TLogEvent& event) +{ + if (event.Level == ELogLevel::Fatal || + event.Level == ELogLevel::Alert && logger.GetAbortOnAlert()) + { + fprintf(stderr, "*** Aborting on critical log event\n"); + fwrite(event.MessageRef.begin(), 1, event.MessageRef.size(), stderr); + fprintf(stderr, "\n"); + YT_ABORT(); + } +} + +TSharedRef TMessageStringBuilder::Flush() +{ + return Buffer_.Slice(0, GetLength()); +} + +void TMessageStringBuilder::DisablePerThreadCache() +{ + Cache_ = nullptr; + CacheDestroyed_ = true; +} + +void TMessageStringBuilder::DoReset() +{ + Buffer_.Reset(); +} + +void TMessageStringBuilder::DoReserve(size_t newCapacity) +{ + auto oldLength = GetLength(); + newCapacity = FastClp2(newCapacity); + + auto newChunkSize = std::max(ChunkSize, newCapacity); + // Hold the old buffer until the data is copied. + auto oldBuffer = std::move(Buffer_); + auto* cache = GetCache(); + if (Y_LIKELY(cache)) { + auto oldCapacity = End_ - Begin_; + auto deltaCapacity = newCapacity - oldCapacity; + if (End_ == cache->Chunk.Begin() + cache->ChunkOffset && + cache->ChunkOffset + deltaCapacity <= cache->Chunk.Size()) + { + // Resize inplace. + Buffer_ = cache->Chunk.Slice(cache->ChunkOffset - oldCapacity, cache->ChunkOffset + deltaCapacity); + cache->ChunkOffset += deltaCapacity; + End_ = Begin_ + newCapacity; + return; + } + + if (Y_UNLIKELY(cache->ChunkOffset + newCapacity > cache->Chunk.Size())) { + cache->Chunk = TSharedMutableRef::Allocate<TMessageBufferTag>(newChunkSize, {.InitializeStorage = false}); + cache->ChunkOffset = 0; + } + + Buffer_ = cache->Chunk.Slice(cache->ChunkOffset, cache->ChunkOffset + newCapacity); + cache->ChunkOffset += newCapacity; + } else { + Buffer_ = TSharedMutableRef::Allocate<TMessageBufferTag>(newChunkSize, {.InitializeStorage = false}); + newCapacity = newChunkSize; + } + if (oldLength > 0) { + ::memcpy(Buffer_.Begin(), Begin_, oldLength); + } + Begin_ = Buffer_.Begin(); + End_ = Begin_ + newCapacity; +} + +TMessageStringBuilder::TPerThreadCache* TMessageStringBuilder::GetCache() +{ + if (Y_LIKELY(Cache_)) { + return Cache_; + } + if (CacheDestroyed_) { + return nullptr; + } + static thread_local TPerThreadCache Cache; + Cache_ = &Cache; + return Cache_; +} + +TMessageStringBuilder::TPerThreadCache::~TPerThreadCache() +{ + TMessageStringBuilder::DisablePerThreadCache(); +} + +thread_local TMessageStringBuilder::TPerThreadCache* TMessageStringBuilder::Cache_; +thread_local bool TMessageStringBuilder::CacheDestroyed_; + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +Y_WEAK TLoggingContext GetLoggingContext() +{ + return { + .Instant = GetCpuInstant(), + .ThreadId = TThread::CurrentThreadId(), + .ThreadName = GetCurrentThreadName(), + }; +} + +Y_WEAK ILogManager* GetDefaultLogManager() +{ + return nullptr; +} + +//////////////////////////////////////////////////////////////////////////////// + +thread_local ELogLevel ThreadMinLogLevel = ELogLevel::Minimum; + +void SetThreadMinLogLevel(ELogLevel minLogLevel) +{ + ThreadMinLogLevel = minLogLevel; +} + +ELogLevel GetThreadMinLogLevel() +{ + return ThreadMinLogLevel; +} + +//////////////////////////////////////////////////////////////////////////////// + +TLogger::TLogger(ILogManager* logManager, TStringBuf categoryName) + : LogManager_(logManager) + , Category_(LogManager_ ? LogManager_->GetCategory(categoryName) : nullptr) + , MinLevel_(LogManager_ ? LoggerDefaultMinLevel : NullLoggerMinLevel) +{ } + +TLogger::TLogger(TStringBuf categoryName) + : TLogger(GetDefaultLogManager(), categoryName) +{ } + +TLogger::operator bool() const +{ + return LogManager_; +} + +const TLoggingCategory* TLogger::GetCategory() const +{ + return Category_; +} + +bool TLogger::IsLevelEnabledHeavy(ELogLevel level) const +{ + // Note that we managed to reach this point, i.e. level >= MinLevel_, + // which implies that MinLevel_ != ELogLevel::Maximum, so this logger was not + // default constructed, thus it has non-trivial category. + YT_ASSERT(Category_); + + if (Category_->CurrentVersion != Category_->ActualVersion->load(std::memory_order::relaxed)) { + LogManager_->UpdateCategory(const_cast<TLoggingCategory*>(Category_)); + } + + return + level >= Category_->MinPlainTextLevel && + level >= ThreadMinLogLevel; +} + +bool TLogger::GetAbortOnAlert() const +{ + return LogManager_->GetAbortOnAlert(); +} + +bool TLogger::IsEssential() const +{ + return Essential_; +} + +void TLogger::UpdateAnchor(TLoggingAnchor* anchor) const +{ + LogManager_->UpdateAnchor(anchor); +} + +void TLogger::RegisterStaticAnchor(TLoggingAnchor* anchor, ::TSourceLocation sourceLocation, TStringBuf message) const +{ + LogManager_->RegisterStaticAnchor(anchor, sourceLocation, message); +} + +void TLogger::Write(TLogEvent&& event) const +{ + LogManager_->Enqueue(std::move(event)); +} + +void TLogger::AddRawTag(const TString& tag) +{ + if (!Tag_.empty()) { + Tag_ += ", "; + } + Tag_ += tag; +} + +TLogger TLogger::WithRawTag(const TString& tag) const +{ + auto result = *this; + result.AddRawTag(tag); + return result; +} + +TLogger TLogger::WithEssential(bool essential) const +{ + auto result = *this; + result.Essential_ = essential; + return result; +} + +TLogger TLogger::WithStructuredValidator(TStructuredValidator validator) const +{ + auto result = *this; + result.StructuredValidators_.push_back(std::move(validator)); + return result; +} + +TLogger TLogger::WithMinLevel(ELogLevel minLevel) const +{ + auto result = *this; + if (result) { + result.MinLevel_ = minLevel; + } + return result; +} + +const TString& TLogger::GetTag() const +{ + return Tag_; +} + +const TLogger::TStructuredTags& TLogger::GetStructuredTags() const +{ + return StructuredTags_; +} + +const TLogger::TStructuredValidators& TLogger::GetStructuredValidators() const +{ + return StructuredValidators_; +} + +//////////////////////////////////////////////////////////////////////////////// + +void LogStructuredEvent( + const TLogger& logger, + NYson::TYsonString message, + ELogLevel level) +{ + YT_VERIFY(message.GetType() == NYson::EYsonType::MapFragment); + + if (!logger.GetStructuredValidators().empty()) { + auto samplingRate = logger.GetCategory()->StructuredValidationSamplingRate.load(); + auto p = RandomNumber<double>(); + if (p < samplingRate) { + for (const auto& validator : logger.GetStructuredValidators()) { + validator(message); + } + } + } + + auto loggingContext = GetLoggingContext(); + auto event = NDetail::CreateLogEvent( + loggingContext, + logger, + level); + event.MessageKind = ELogMessageKind::Structured; + event.MessageRef = message.ToSharedRef(); + event.Family = ELogFamily::Structured; + logger.Write(std::move(event)); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/logger.h b/library/cpp/yt/logging/logger.h new file mode 100644 index 0000000000..cdb5584d29 --- /dev/null +++ b/library/cpp/yt/logging/logger.h @@ -0,0 +1,351 @@ +#pragma once + +#include "public.h" + +#include <library/cpp/yt/string/format.h> + +#include <library/cpp/yt/memory/ref.h> + +#include <library/cpp/yt/cpu_clock/public.h> + +#include <library/cpp/yt/yson_string/string.h> + +#include <library/cpp/yt/misc/guid.h> + +#include <library/cpp/yt/misc/thread_name.h> + +#include <library/cpp/yt/memory/leaky_singleton.h> + +#include <util/system/src_location.h> + +#include <util/generic/size_literals.h> + +#include <atomic> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +constexpr double DefaultStructuredValidationSamplingRate = 0.01; + +struct TLoggingCategory +{ + TString Name; + //! This value is used for early dropping of plaintext events in order + //! to reduce load on logging thread for events which are definitely going + //! to be dropped due to rule setup. + //! NB: this optimization is used only for plaintext events since structured + //! logging rate is negligible comparing to the plaintext logging rate. + std::atomic<ELogLevel> MinPlainTextLevel; + std::atomic<int> CurrentVersion; + std::atomic<int>* ActualVersion; + std::atomic<double> StructuredValidationSamplingRate = DefaultStructuredValidationSamplingRate; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TLoggingAnchor +{ + std::atomic<bool> Registered = false; + ::TSourceLocation SourceLocation = {TStringBuf{}, 0}; + TString AnchorMessage; + TLoggingAnchor* NextAnchor = nullptr; + + std::atomic<int> CurrentVersion = 0; + std::atomic<bool> Enabled = false; + + struct TCounter + { + std::atomic<i64> Current = 0; + i64 Previous = 0; + }; + + TCounter MessageCounter; + TCounter ByteCounter; +}; + +//////////////////////////////////////////////////////////////////////////////// +// Declare some type aliases to avoid circular dependencies. +using TThreadId = size_t; +using TFiberId = size_t; +using TTraceId = TGuid; +using TRequestId = TGuid; + +//////////////////////////////////////////////////////////////////////////////// + +DEFINE_ENUM(ELogMessageKind, + (Unstructured) + (Structured) +); + +struct TLogEvent +{ + const TLoggingCategory* Category = nullptr; + ELogLevel Level = ELogLevel::Minimum; + ELogFamily Family = ELogFamily::PlainText; + bool Essential = false; + + ELogMessageKind MessageKind = ELogMessageKind::Unstructured; + TSharedRef MessageRef; + + TCpuInstant Instant = 0; + + TThreadId ThreadId = {}; + TThreadName ThreadName = {}; + + TFiberId FiberId = {}; + + TTraceId TraceId; + TRequestId RequestId; + + TStringBuf SourceFile; + int SourceLine = -1; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct ILogManager +{ + virtual ~ILogManager() = default; + + virtual void RegisterStaticAnchor( + TLoggingAnchor* position, + ::TSourceLocation sourceLocation, + TStringBuf anchorMessage) = 0; + virtual void UpdateAnchor(TLoggingAnchor* position) = 0; + + virtual void Enqueue(TLogEvent&& event) = 0; + + virtual const TLoggingCategory* GetCategory(TStringBuf categoryName) = 0; + virtual void UpdateCategory(TLoggingCategory* category) = 0; + + virtual bool GetAbortOnAlert() const = 0; +}; + +ILogManager* GetDefaultLogManager(); + +//////////////////////////////////////////////////////////////////////////////// + +struct TLoggingContext +{ + TCpuInstant Instant; + TThreadId ThreadId; + TThreadName ThreadName; + TFiberId FiberId; + TTraceId TraceId; + TRequestId RequestId; + TStringBuf TraceLoggingTag; +}; + +TLoggingContext GetLoggingContext(); + +//////////////////////////////////////////////////////////////////////////////// + +//! Sets the minimum logging level for messages in current thread. +// NB: In fiber environment, min log level is attached to a fiber, +// so after context switch thread min log level might change. +void SetThreadMinLogLevel(ELogLevel minLogLevel); +ELogLevel GetThreadMinLogLevel(); + +//////////////////////////////////////////////////////////////////////////////// + +static constexpr auto NullLoggerMinLevel = ELogLevel::Maximum; + +// Min level for non-null logger depends on whether we are in debug or release build. +// - For release mode default behavior is to omit trace logging, +// this is done by setting logger min level to Debug by default. +// - For debug mode logger min level is set to trace by default, so that trace logging is +// allowed by logger, but still may be discarded by category min level. +#ifdef NDEBUG +static constexpr auto LoggerDefaultMinLevel = ELogLevel::Debug; +#else +static constexpr auto LoggerDefaultMinLevel = ELogLevel::Trace; +#endif + +class TLogger +{ +public: + using TStructuredValidator = std::function<void(const NYson::TYsonString&)>; + using TStructuredValidators = std::vector<TStructuredValidator>; + + using TStructuredTag = std::pair<TString, NYson::TYsonString>; + // TODO(max42): switch to TCompactVector after YT-15430. + using TStructuredTags = std::vector<TStructuredTag>; + + TLogger() = default; + TLogger(const TLogger& other) = default; + TLogger& operator=(const TLogger& other) = default; + + TLogger(ILogManager* logManager, TStringBuf categoryName); + explicit TLogger(TStringBuf categoryName); + + explicit operator bool() const; + + const TLoggingCategory* GetCategory() const; + + //! Validate that level is admitted by logger's own min level + //! and by category's min level. + bool IsLevelEnabled(ELogLevel level) const; + + bool GetAbortOnAlert() const; + + bool IsEssential() const; + + bool IsAnchorUpToDate(const TLoggingAnchor& anchor) const; + void UpdateAnchor(TLoggingAnchor* anchor) const; + void RegisterStaticAnchor(TLoggingAnchor* anchor, ::TSourceLocation sourceLocation, TStringBuf message) const; + + void Write(TLogEvent&& event) const; + + void AddRawTag(const TString& tag); + template <class... TArgs> + void AddTag(const char* format, TArgs&&... args); + + template <class TType> + void AddStructuredTag(TStringBuf key, TType value); + + TLogger WithRawTag(const TString& tag) const; + template <class... TArgs> + TLogger WithTag(const char* format, TArgs&&... args) const; + + template <class TType> + TLogger WithStructuredTag(TStringBuf key, TType value) const; + + TLogger WithStructuredValidator(TStructuredValidator validator) const; + + TLogger WithMinLevel(ELogLevel minLevel) const; + + TLogger WithEssential(bool essential = true) const; + + const TString& GetTag() const; + const TStructuredTags& GetStructuredTags() const; + + const TStructuredValidators& GetStructuredValidators() const; + +protected: + // These fields are set only during logger creation, so they are effectively const + // and accessing them is thread-safe. + ILogManager* LogManager_ = nullptr; + const TLoggingCategory* Category_ = nullptr; + bool Essential_ = false; + ELogLevel MinLevel_ = NullLoggerMinLevel; + TString Tag_; + TStructuredTags StructuredTags_; + TStructuredValidators StructuredValidators_; + +private: + //! This method checks level against category's min level. + //! Refer to comment in TLogger::IsLevelEnabled for more details. + bool IsLevelEnabledHeavy(ELogLevel level) const; +}; + +//////////////////////////////////////////////////////////////////////////////// + +void LogStructuredEvent( + const TLogger& logger, + NYson::TYsonString message, + ELogLevel level); + +//////////////////////////////////////////////////////////////////////////////// + +#ifdef YT_ENABLE_TRACE_LOGGING +#define YT_LOG_TRACE(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Trace, __VA_ARGS__) +#define YT_LOG_TRACE_IF(condition, ...) if (condition) YT_LOG_TRACE(__VA_ARGS__) +#define YT_LOG_TRACE_UNLESS(condition, ...) if (!(condition)) YT_LOG_TRACE(__VA_ARGS__) +#else +#define YT_LOG_UNUSED(...) if (true) { } else { YT_LOG_DEBUG(__VA_ARGS__); } +#define YT_LOG_TRACE(...) YT_LOG_UNUSED(__VA_ARGS__) +#define YT_LOG_TRACE_IF(condition, ...) YT_LOG_UNUSED(__VA_ARGS__) +#define YT_LOG_TRACE_UNLESS(condition, ...) YT_LOG_UNUSED(__VA_ARGS__) +#endif + +#define YT_LOG_DEBUG(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Debug, __VA_ARGS__) +#define YT_LOG_DEBUG_IF(condition, ...) if (condition) YT_LOG_DEBUG(__VA_ARGS__) +#define YT_LOG_DEBUG_UNLESS(condition, ...) if (!(condition)) YT_LOG_DEBUG(__VA_ARGS__) + +#define YT_LOG_INFO(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Info, __VA_ARGS__) +#define YT_LOG_INFO_IF(condition, ...) if (condition) YT_LOG_INFO(__VA_ARGS__) +#define YT_LOG_INFO_UNLESS(condition, ...) if (!(condition)) YT_LOG_INFO(__VA_ARGS__) + +#define YT_LOG_WARNING(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Warning, __VA_ARGS__) +#define YT_LOG_WARNING_IF(condition, ...) if (condition) YT_LOG_WARNING(__VA_ARGS__) +#define YT_LOG_WARNING_UNLESS(condition, ...) if (!(condition)) YT_LOG_WARNING(__VA_ARGS__) + +#define YT_LOG_ERROR(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Error, __VA_ARGS__) +#define YT_LOG_ERROR_IF(condition, ...) if (condition) YT_LOG_ERROR(__VA_ARGS__) +#define YT_LOG_ERROR_UNLESS(condition, ...) if (!(condition)) YT_LOG_ERROR(__VA_ARGS__) + +#define YT_LOG_ALERT(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Alert, __VA_ARGS__); +#define YT_LOG_ALERT_IF(condition, ...) if (condition) YT_LOG_ALERT(__VA_ARGS__) +#define YT_LOG_ALERT_UNLESS(condition, ...) if (!(condition)) YT_LOG_ALERT(__VA_ARGS__) + +#define YT_LOG_FATAL(...) \ + do { \ + YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Fatal, __VA_ARGS__); \ + Y_UNREACHABLE(); \ + } while(false) +#define YT_LOG_FATAL_IF(condition, ...) if (Y_UNLIKELY(condition)) YT_LOG_FATAL(__VA_ARGS__) +#define YT_LOG_FATAL_UNLESS(condition, ...) if (!Y_LIKELY(condition)) YT_LOG_FATAL(__VA_ARGS__) + +#define YT_LOG_EVENT(logger, level, ...) \ + YT_LOG_EVENT_WITH_ANCHOR(logger, level, nullptr, __VA_ARGS__) + +#define YT_LOG_EVENT_WITH_ANCHOR(logger, level, anchor, ...) \ + do { \ + const auto& logger__##__LINE__ = (logger); \ + auto level__##__LINE__ = (level); \ + \ + if (!logger__##__LINE__.IsLevelEnabled(level__##__LINE__)) { \ + break; \ + } \ + \ + auto location__##__LINE__ = __LOCATION__; \ + \ + ::NYT::NLogging::TLoggingAnchor* anchor__##__LINE__ = (anchor); \ + if (!anchor__##__LINE__) { \ + static ::NYT::TLeakyStorage<::NYT::NLogging::TLoggingAnchor> staticAnchor__##__LINE__; \ + anchor__##__LINE__ = staticAnchor__##__LINE__.Get(); \ + } \ + \ + bool anchorUpToDate__##__LINE__ = logger__##__LINE__.IsAnchorUpToDate(*anchor__##__LINE__); \ + if (anchorUpToDate__##__LINE__ && !anchor__##__LINE__->Enabled.load(std::memory_order::relaxed)) { \ + break; \ + } \ + \ + auto loggingContext__##__LINE__ = ::NYT::NLogging::GetLoggingContext(); \ + auto message__##__LINE__ = ::NYT::NLogging::NDetail::BuildLogMessage(loggingContext__##__LINE__, logger__##__LINE__, __VA_ARGS__); \ + \ + if (!anchorUpToDate__##__LINE__) { \ + logger__##__LINE__.RegisterStaticAnchor(anchor__##__LINE__, location__##__LINE__, message__##__LINE__.Anchor); \ + logger__##__LINE__.UpdateAnchor(anchor__##__LINE__); \ + } \ + \ + if (!anchor__##__LINE__->Enabled.load(std::memory_order::relaxed)) { \ + break; \ + } \ + \ + static thread_local i64 localByteCounter__##__LINE__; \ + static thread_local ui8 localMessageCounter__##__LINE__; \ + \ + localByteCounter__##__LINE__ += message__##__LINE__.MessageRef.Size(); \ + if (Y_UNLIKELY(++localMessageCounter__##__LINE__ == 0)) { \ + anchor__##__LINE__->MessageCounter.Current += 256; \ + anchor__##__LINE__->ByteCounter.Current += localByteCounter__##__LINE__; \ + localByteCounter__##__LINE__ = 0; \ + } \ + \ + ::NYT::NLogging::NDetail::LogEventImpl( \ + loggingContext__##__LINE__, \ + logger__##__LINE__, \ + level__##__LINE__, \ + location__##__LINE__, \ + std::move(message__##__LINE__.MessageRef)); \ + } while (false) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging + +#define LOGGER_INL_H_ +#include "logger-inl.h" +#undef LOGGER_INL_H_ diff --git a/library/cpp/yt/logging/public.h b/library/cpp/yt/logging/public.h new file mode 100644 index 0000000000..1e2b59ca0d --- /dev/null +++ b/library/cpp/yt/logging/public.h @@ -0,0 +1,39 @@ +#pragma once + +#include <library/cpp/yt/misc/enum.h> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +// Any change to this enum must be also propagated to FormatLevel. +DEFINE_ENUM(ELogLevel, + (Minimum) + (Trace) + (Debug) + (Info) + (Warning) + (Error) + (Alert) + (Fatal) + (Maximum) +); + +DEFINE_ENUM(ELogFamily, + (PlainText) + (Structured) +); + +//////////////////////////////////////////////////////////////////////////////// + +struct TLoggingCategory; +struct TLoggingAnchor; +struct TLogEvent; +struct TLoggingContext; + +class TLogger; +struct ILogManager; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/unittests/logger_ut.cpp b/library/cpp/yt/logging/unittests/logger_ut.cpp new file mode 100644 index 0000000000..7696ea4a83 --- /dev/null +++ b/library/cpp/yt/logging/unittests/logger_ut.cpp @@ -0,0 +1,38 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/logging/logger.h> + +namespace NYT::NLogging { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TLogger, NullByDefault) +{ + { + TLogger logger; + EXPECT_FALSE(logger); + EXPECT_FALSE(logger.IsLevelEnabled(ELogLevel::Fatal)); + } + { + TLogger logger{"Category"}; + EXPECT_FALSE(logger); + EXPECT_FALSE(logger.IsLevelEnabled(ELogLevel::Fatal)); + } +} + +TEST(TLogger, CopyOfNullLogger) +{ + TLogger nullLogger{/*logManager*/ nullptr, "Category"}; + ASSERT_FALSE(nullLogger); + + auto logger = nullLogger.WithMinLevel(ELogLevel::Debug); + + EXPECT_FALSE(logger); + EXPECT_FALSE(logger.IsLevelEnabled(ELogLevel::Fatal)); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/unittests/ya.make b/library/cpp/yt/logging/unittests/ya.make new file mode 100644 index 0000000000..42268d3db2 --- /dev/null +++ b/library/cpp/yt/logging/unittests/ya.make @@ -0,0 +1,18 @@ +GTEST(unittester-library-logging) + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +IF (NOT OS_WINDOWS) + ALLOCATOR(YT) +ENDIF() + +SRCS( + logger_ut.cpp +) + +PEERDIR( + library/cpp/testing/gtest + library/cpp/yt/logging +) + +END() diff --git a/library/cpp/yt/logging/ya.make b/library/cpp/yt/logging/ya.make new file mode 100644 index 0000000000..cf629a24b6 --- /dev/null +++ b/library/cpp/yt/logging/ya.make @@ -0,0 +1,20 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + logger.cpp +) + +PEERDIR( + library/cpp/yt/assert + library/cpp/yt/memory + library/cpp/yt/misc + library/cpp/yt/yson_string +) + +END() + +RECURSE_FOR_TESTS( + unittests +) diff --git a/library/cpp/yt/memory/leaky_ref_counted_singleton-inl.h b/library/cpp/yt/memory/leaky_ref_counted_singleton-inl.h new file mode 100644 index 0000000000..1fba63c427 --- /dev/null +++ b/library/cpp/yt/memory/leaky_ref_counted_singleton-inl.h @@ -0,0 +1,43 @@ +#ifndef LEAKY_REF_COUNTED_SINGLETON_INL_H_ +#error "Direct inclusion of this file is not allowed, include leaky_ref_counted_singleton.h" +// For the sake of sane code completion. +#include "leaky_ref_counted_singleton.h" +#endif + +#include "new.h" + +#include <atomic> +#include <mutex> + +#include <util/system/compiler.h> +#include <util/system/sanitizers.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, class... TArgs> +TIntrusivePtr<T> LeakyRefCountedSingleton(TArgs&&... args) +{ + static std::atomic<T*> Ptr; + auto* ptr = Ptr.load(std::memory_order::acquire); + if (Y_LIKELY(ptr)) { + return ptr; + } + + static std::once_flag Initialized; + std::call_once(Initialized, [&] { + auto ptr = New<T>(std::forward<TArgs>(args)...); + Ref(ptr.Get()); + Ptr.store(ptr.Get()); +#if defined(_asan_enabled_) + NSan::MarkAsIntentionallyLeaked(ptr.Get()); +#endif + }); + + return Ptr.load(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/memory/leaky_ref_counted_singleton.h b/library/cpp/yt/memory/leaky_ref_counted_singleton.h new file mode 100644 index 0000000000..d77c3c9829 --- /dev/null +++ b/library/cpp/yt/memory/leaky_ref_counted_singleton.h @@ -0,0 +1,22 @@ +#pragma once + +#include "intrusive_ptr.h" + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +#define DECLARE_LEAKY_REF_COUNTED_SINGLETON_FRIEND() \ + template <class T> \ + friend struct ::NYT::TRefCountedWrapper; + +template <class T, class... TArgs> +TIntrusivePtr<T> LeakyRefCountedSingleton(TArgs&&... args); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + +#define LEAKY_REF_COUNTED_SINGLETON_INL_H_ +#include "leaky_ref_counted_singleton-inl.h" +#undef LEAKY_REF_COUNTED_SINGLETON_INL_H_ diff --git a/library/cpp/yt/misc/arcadia_enum-inl.h b/library/cpp/yt/misc/arcadia_enum-inl.h new file mode 100644 index 0000000000..17a10bb3b2 --- /dev/null +++ b/library/cpp/yt/misc/arcadia_enum-inl.h @@ -0,0 +1,49 @@ +#pragma once +#ifndef ARCADIA_ENUM_INL_H_ +#error "Direct inclusion of this file is not allowed, include arcadia_enum.h" +// For the sake of sane code completion. +#include "arcadia_enum.h" +#endif + +#include <util/system/type_name.h> + +namespace NYT::NDetail { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +struct TArcadiaEnumTraitsImpl +{ + static constexpr bool IsBitEnum = false; + static constexpr bool IsStringSerializableEnum = false; + + static TStringBuf GetTypeName() + { + static const auto Result = TypeName<T>(); + return Result; + } + + static std::optional<TStringBuf> FindLiteralByValue(T value) + { + auto names = GetEnumNames<T>(); + auto it = names.find(value); + return it == names.end() ? std::nullopt : std::make_optional(TStringBuf(it->second)); + } + + static std::optional<T> FindValueByLiteral(TStringBuf literal) + { + static const auto LiteralToValue = [] { + THashMap<TString, T> result; + for (const auto& [value, name] : GetEnumNames<T>()) { + result.emplace(name, value); + } + return result; + }(); + auto it = LiteralToValue.find(literal); + return it == LiteralToValue.end() ? std::nullopt : std::make_optional(it->second); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NDetail diff --git a/library/cpp/yt/misc/arcadia_enum.h b/library/cpp/yt/misc/arcadia_enum.h new file mode 100644 index 0000000000..85ad182a6c --- /dev/null +++ b/library/cpp/yt/misc/arcadia_enum.h @@ -0,0 +1,18 @@ +#pragma once + +#include <util/generic/serialized_enum.h> + +//////////////////////////////////////////////////////////////////////////////// +// TEnumTraits interop for Arcadia enums + +#define YT_DEFINE_ARCADIA_ENUM_TRAITS(enumType) \ + [[maybe_unused]] inline ::NYT::NDetail::TArcadiaEnumTraitsImpl<enumType> GetEnumTraitsImpl(enumType) \ + { \ + return {}; \ + } + +//////////////////////////////////////////////////////////////////////////////// + +#define ARCADIA_ENUM_INL_H_ +#include "arcadia_enum-inl.h" +#undef ARCADIA_ENUM_INL_H_ diff --git a/library/cpp/yt/misc/property.h b/library/cpp/yt/misc/property.h new file mode 100644 index 0000000000..d5c2a26c7a --- /dev/null +++ b/library/cpp/yt/misc/property.h @@ -0,0 +1,306 @@ +#pragma once + +//////////////////////////////////////////////////////////////////////////////// + +//! Declares a trivial public read-write property that is passed by reference. +#define DECLARE_BYREF_RW_PROPERTY(type, name) \ +public: \ + type& name(); \ + const type& name() const + +//! Defines a trivial public read-write property that is passed by reference. +//! All arguments after name are used as default value (via braced-init-list). +#define DEFINE_BYREF_RW_PROPERTY(type, name, ...) \ +protected: \ + type name##_ { __VA_ARGS__ }; \ + \ +public: \ + Y_FORCE_INLINE type& name() \ + { \ + return name##_; \ + } \ + \ + Y_FORCE_INLINE const type& name() const \ + { \ + return name##_; \ + } \ + static_assert(true) + +//! Defines a trivial public read-write property that is passed by reference +//! and is not inline-initialized. +#define DEFINE_BYREF_RW_PROPERTY_NO_INIT(type, name) \ +protected: \ + type name##_; \ + \ +public: \ + Y_FORCE_INLINE type& name() \ + { \ + return name##_; \ + } \ + \ + Y_FORCE_INLINE const type& name() const \ + { \ + return name##_; \ + } \ + static_assert(true) + +//! Forwards a trivial public read-write property that is passed by reference. +#define DELEGATE_BYREF_RW_PROPERTY(declaringType, type, name, delegateTo) \ + type& declaringType::name() \ + { \ + return (delegateTo).name(); \ + } \ + \ + const type& declaringType::name() const \ + { \ + return (delegateTo).name(); \ + } \ + static_assert(true) + +//////////////////////////////////////////////////////////////////////////////// + +//! Declares a trivial public read-only property that is passed by reference. +#define DECLARE_BYREF_RO_PROPERTY(type, name) \ +public: \ + const type& name() const + +//! Defines a trivial public read-only property that is passed by reference. +//! All arguments after name are used as default value (via braced-init-list). +#define DEFINE_BYREF_RO_PROPERTY(type, name, ...) \ +protected: \ + type name##_ { __VA_ARGS__ }; \ + \ +public: \ + Y_FORCE_INLINE const type& name() const \ + { \ + return name##_; \ + } \ + static_assert(true) + +//! Defines a trivial public read-only property that is passed by reference +//! and is not inline-initialized. +#define DEFINE_BYREF_RO_PROPERTY_NO_INIT(type, name) \ +protected: \ + type name##_; \ + \ +public: \ + Y_FORCE_INLINE const type& name() const \ + { \ + return name##_; \ + } \ + static_assert(true) + +//! Forwards a trivial public read-only property that is passed by reference. +#define DELEGATE_BYREF_RO_PROPERTY(declaringType, type, name, delegateTo) \ + const type& declaringType::name() const \ + { \ + return (delegateTo).name(); \ + } \ + static_assert(true) + +//////////////////////////////////////////////////////////////////////////////// + +//! Declares a trivial public read-write property that is passed by value. +#define DECLARE_BYVAL_RW_PROPERTY(type, name) \ +public: \ + type Get##name() const; \ + void Set##name(type value) + +//! Defines a trivial public read-write property that is passed by value. +//! All arguments after name are used as default value (via braced-init-list). +#define DEFINE_BYVAL_RW_PROPERTY(type, name, ...) \ +protected: \ + type name##_ { __VA_ARGS__ }; \ + \ +public: \ + Y_FORCE_INLINE type Get##name() const \ + { \ + return name##_; \ + } \ + \ + Y_FORCE_INLINE void Set##name(type value) \ + { \ + name##_ = value; \ + } \ + static_assert(true) + + +//! Defines a trivial public read-write property that is passed by value. +//! All arguments after name are used as default value (via braced-init-list). +#define DEFINE_BYVAL_RW_PROPERTY_WITH_FLUENT_SETTER(declaringType, type, name, ...) \ +protected: \ + type name##_ { __VA_ARGS__ }; \ + \ +public: \ + Y_FORCE_INLINE type Get##name() const \ + { \ + return name##_; \ + } \ + \ + Y_FORCE_INLINE void Set##name(type value) &\ + { \ + name##_ = value; \ + } \ + \ + Y_FORCE_INLINE declaringType&& Set##name(type value) &&\ + { \ + name##_ = value; \ + return std::move(*this); \ + } \ + static_assert(true) + +//! Defines a trivial public read-write property that is passed by value +//! and is not inline-initialized. +#define DEFINE_BYVAL_RW_PROPERTY_NO_INIT(type, name, ...) \ +protected: \ + type name##_; \ + \ +public: \ + Y_FORCE_INLINE type Get##name() const \ + { \ + return name##_; \ + } \ + \ + Y_FORCE_INLINE void Set##name(type value) \ + { \ + name##_ = value; \ + } \ + static_assert(true) + +//! Forwards a trivial public read-write property that is passed by value. +#define DELEGATE_BYVAL_RW_PROPERTY(declaringType, type, name, delegateTo) \ + type declaringType::Get##name() const \ + { \ + return (delegateTo).Get##name(); \ + } \ + \ + void declaringType::Set##name(type value) \ + { \ + (delegateTo).Set##name(value); \ + } \ + static_assert(true) + +//////////////////////////////////////////////////////////////////////////////// + +//! Declares a trivial public read-only property that is passed by value. +#define DECLARE_BYVAL_RO_PROPERTY(type, name) \ +public: \ + type Get##name() const + +//! Defines a trivial public read-only property that is passed by value. +//! All arguments after name are used as default value (via braced-init-list). +#define DEFINE_BYVAL_RO_PROPERTY(type, name, ...) \ +protected: \ + type name##_ { __VA_ARGS__ }; \ + \ +public: \ + Y_FORCE_INLINE type Get##name() const \ + { \ + return name##_; \ + } \ + static_assert(true) + + +//! Defines a trivial public read-only property that is passed by value +//! and is not inline-initialized. +#define DEFINE_BYVAL_RO_PROPERTY_NO_INIT(type, name) \ +protected: \ + type name##_; \ + \ +public: \ + Y_FORCE_INLINE type Get##name() const \ + { \ + return name##_; \ + } \ + static_assert(true) + +//! Forwards a trivial public read-only property that is passed by value. +#define DELEGATE_BYVAL_RO_PROPERTY(declaringType, type, name, delegateTo) \ + type declaringType::Get##name() \ + { \ + return (delegateTo).Get##name(); \ + } \ + static_assert(true) + +//////////////////////////////////////////////////////////////////////////////// + +//! Below are macro helpers for extra properties. +//! Extra properties should be used for lazy memory allocation for properties that +//! hold default values for the majority of objects. + +//! Initializes extra property holder if it is not initialized. +#define INITIALIZE_EXTRA_PROPERTY_HOLDER(holder) \ + if (!holder##_) { \ + holder##_.reset(new decltype(holder##_)::element_type()); \ + } \ + static_assert(true) + +//! Declares an extra property holder. Holder contains extra properties values. +//! Holder is not created until some property is set with a non-default value. +//! If there is no holder property getter returns default value. +#define DECLARE_EXTRA_PROPERTY_HOLDER(type, holder) \ +public: \ + Y_FORCE_INLINE bool HasCustom##holder() const \ + { \ + return static_cast<bool>(holder##_); \ + } \ + Y_FORCE_INLINE const type* GetCustom##holder() const \ + { \ + return holder##_.get(); \ + } \ + Y_FORCE_INLINE type* GetCustom##holder() \ + { \ + return holder##_.get(); \ + } \ + Y_FORCE_INLINE void InitializeCustom##holder() \ + { \ + INITIALIZE_EXTRA_PROPERTY_HOLDER(holder); \ + } \ +private: \ + std::unique_ptr<type> holder##_; \ + static const type Default##holder##_ + +//! Defines a storage for extra properties default values. +#define DEFINE_EXTRA_PROPERTY_HOLDER(class, type, holder) \ + const type class::Default##holder##_ + +//! Defines a public read-write extra property that is passed by value. +#define DEFINE_BYVAL_RW_EXTRA_PROPERTY(holder, name) \ +public: \ + Y_FORCE_INLINE decltype(holder##_->name) Get##name() const \ + { \ + if (!holder##_) { \ + return Default##holder##_.name; \ + } \ + return holder##_->name; \ + } \ + Y_FORCE_INLINE void Set##name(decltype(holder##_->name) val) \ + { \ + if (!holder##_) { \ + if (val == Default##holder##_.name) { \ + return; \ + } \ + INITIALIZE_EXTRA_PROPERTY_HOLDER(holder); \ + } \ + holder##_->name = val; \ + } \ + static_assert(true) + +//! Defines a public read-write extra property that is passed by reference. +#define DEFINE_BYREF_RW_EXTRA_PROPERTY(holder, name) \ +public: \ + Y_FORCE_INLINE const decltype(holder##_->name)& name() const \ + { \ + if (!holder##_) { \ + return Default##holder##_.name; \ + } \ + return holder##_->name; \ + } \ + Y_FORCE_INLINE decltype(holder##_->name)& Mutable##name() \ + { \ + INITIALIZE_EXTRA_PROPERTY_HOLDER(holder); \ + return holder##_->name; \ + } \ + static_assert(true) + +//////////////////////////////////////////////////////////////////////////////// diff --git a/library/cpp/yt/string/raw_formatter.h b/library/cpp/yt/string/raw_formatter.h new file mode 100644 index 0000000000..6956330883 --- /dev/null +++ b/library/cpp/yt/string/raw_formatter.h @@ -0,0 +1,212 @@ +#pragma once + +#include "guid.h" + +#include <algorithm> +#include <array> + +#include <util/generic/strbuf.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +//! A dead-simple string formatter. +/*! + * This formatter is intended to be as simple as possible and async signal safe. + * This is the reason we do not use printf(): it does not meet signal-safety + * requirements. + */ + +class TBaseFormatter +{ +public: + TBaseFormatter(char* buffer, int length) + : Begin_(buffer) + , Cursor_(buffer) + , End_(buffer + length) + { } + + //! Returns an underlying cursor. + char* GetCursor() + { + return Cursor_; + } + + //! Returns an pointer to the underlying buffer. + const char* GetData() const + { + return Begin_; + } + + //! Returns the number of bytes written in the buffer. + int GetBytesWritten() const + { + return Cursor_ - Begin_; + } + + //! Returns the number of bytes available in the buffer. + int GetBytesRemaining() const + { + return End_ - Cursor_; + } + + //! Advances the internal cursor #count symbols forward (assuming the data is already present). + void Advance(int count) + { + Cursor_ += count; + + if (Cursor_ > End_) { + Cursor_ = End_; + } + } + + //! Drops trailing #count symbols (assuming these are present). + void Revert(int count) + { + Cursor_ -= count; + } + + //! Appends the string and updates the internal cursor. + void AppendString(const char* string) + { + while (*string != '\0' && Cursor_ < End_) { + *Cursor_++ = *string++; + } + } + + //! Appends the string and updates the internal cursor. + void AppendString(TStringBuf string) + { + size_t position = 0; + while (position < string.length() && Cursor_ < End_) { + *Cursor_++ = string[position++]; + } + } + + //! Appends a single character and updates the internal cursor. + void AppendChar(char ch) + { + if (Cursor_ < End_) { + *Cursor_++ = ch; + } + } + + //! Formats |number| in base |radix| and updates the internal cursor. + void AppendNumber(uintptr_t number, int radix = 10, int width = 0, char ch = ' ') + { + int digits = 0; + + if (radix == 16) { + // Optimize output of hex numbers. + + uintptr_t reverse = 0; + int length = 0; + do { + reverse <<= 4; + reverse |= number & 0xf; + number >>= 4; + ++length; + } while (number > 0); + + for (int index = 0; index < length && Cursor_ + digits < End_; ++index) { + unsigned int modulus = reverse & 0xf; + Cursor_[digits] = (modulus < 10 ? '0' + modulus : 'a' + modulus - 10); + ++digits; + reverse >>= 4; + } + } else { + while (Cursor_ + digits < End_) { + const int modulus = number % radix; + number /= radix; + Cursor_[digits] = (modulus < 10 ? '0' + modulus : 'a' + modulus - 10); + ++digits; + if (number == 0) { + break; + } + } + + // Reverse the bytes written. + std::reverse(Cursor_, Cursor_ + digits); + } + + if (digits < width) { + auto delta = width - digits; + std::copy(Cursor_, Cursor_ + digits, Cursor_ + delta); + std::fill(Cursor_, Cursor_ + delta, ch); + Cursor_ += width; + } else { + Cursor_ += digits; + } + } + + //! Formats |number| as hexadecimal number and updates the internal cursor. + //! Padding will be added in front if needed. + void AppendNumberAsHexWithPadding(uintptr_t number, int width) + { + char* begin = Cursor_; + AppendString("0x"); + AppendNumber(number, 16); + // Move to right and add padding in front if needed. + if (Cursor_ < begin + width) { + auto delta = begin + width - Cursor_; + std::copy(begin, Cursor_, begin + delta); + std::fill(begin, begin + delta, ' '); + Cursor_ = begin + width; + } + } + + //! Formats |guid| and updates the internal cursor. + void AppendGuid(TGuid guid) + { + if (Y_LIKELY(End_ - Cursor_ >= MaxGuidStringSize)) { + // Fast path. + Cursor_ = WriteGuidToBuffer(Cursor_, guid); + } else { + // Slow path. + std::array<char, MaxGuidStringSize> buffer; + auto* end = WriteGuidToBuffer(buffer.data(), guid); + AppendString(TStringBuf(buffer.data(), end)); + } + } + + //! Resets the underlying cursor. + void Reset() + { + Cursor_ = Begin_; + } + + TStringBuf GetBuffer() const + { + return {Begin_, Cursor_}; + } + +private: + char* const Begin_; + char* Cursor_; + char* const End_; + +}; + +template <size_t N> +class TRawFormatter + : public TBaseFormatter +{ +public: + TRawFormatter() + : TBaseFormatter(Buffer_, N) + { } + + TRawFormatter(char* buffer, int length) + : TBaseFormatter(buffer, length) + { } + +private: + char Buffer_[N]; + +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + diff --git a/library/cpp/yt/threading/unittests/count_down_latch_ut.cpp b/library/cpp/yt/threading/unittests/count_down_latch_ut.cpp new file mode 100644 index 0000000000..894bdab22a --- /dev/null +++ b/library/cpp/yt/threading/unittests/count_down_latch_ut.cpp @@ -0,0 +1,78 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/threading/count_down_latch.h> + +#include <thread> + +namespace NYT::NThreading { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +void WaitForLatch(const TCountDownLatch& latch) +{ + latch.Wait(); + EXPECT_EQ(0, latch.GetCount()); +} + +TEST(TCountDownLatch, TwoThreads) +{ + TCountDownLatch latch(2); + + std::thread t1(std::bind(&WaitForLatch, std::cref(latch))); + std::thread t2(std::bind(&WaitForLatch, std::cref(latch))); + + EXPECT_EQ(2, latch.GetCount()); + latch.CountDown(); + EXPECT_EQ(1, latch.GetCount()); + latch.CountDown(); + EXPECT_EQ(0, latch.GetCount()); + + t1.join(); + t2.join(); +} + +TEST(TCountDownLatch, TwoThreadsPredecremented) +{ + TCountDownLatch latch(2); + + EXPECT_EQ(2, latch.GetCount()); + latch.CountDown(); + EXPECT_EQ(1, latch.GetCount()); + latch.CountDown(); + EXPECT_EQ(0, latch.GetCount()); + + std::thread t1(std::bind(&WaitForLatch, std::cref(latch))); + std::thread t2(std::bind(&WaitForLatch, std::cref(latch))); + + t1.join(); + t2.join(); +} + +TEST(TCountDownLatch, TwoThreadsTwoLatches) +{ + TCountDownLatch first(1); + TCountDownLatch second(1); + + std::thread t1([&] () { + first.Wait(); + second.CountDown(); + EXPECT_EQ(0, first.GetCount()); + EXPECT_EQ(0, second.GetCount()); + }); + + std::thread t2([&] () { + first.CountDown(); + second.Wait(); + EXPECT_EQ(0, first.GetCount()); + EXPECT_EQ(0, second.GetCount()); + }); + + t1.join(); + t2.join(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NThreading diff --git a/library/cpp/yt/threading/unittests/recursive_spin_lock_ut.cpp b/library/cpp/yt/threading/unittests/recursive_spin_lock_ut.cpp new file mode 100644 index 0000000000..9c2d8f16cb --- /dev/null +++ b/library/cpp/yt/threading/unittests/recursive_spin_lock_ut.cpp @@ -0,0 +1,88 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/threading/recursive_spin_lock.h> +#include <library/cpp/yt/threading/event_count.h> + +#include <thread> + +namespace NYT::NThreading { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TRecursiveSpinLockTest, SingleThread) +{ + TRecursiveSpinLock lock; + EXPECT_FALSE(lock.IsLocked()); + EXPECT_TRUE(lock.TryAcquire()); + EXPECT_TRUE(lock.IsLocked()); + EXPECT_TRUE(lock.TryAcquire()); + EXPECT_TRUE(lock.IsLocked()); + lock.Release(); + EXPECT_TRUE(lock.IsLocked()); + lock.Release(); + EXPECT_FALSE(lock.IsLocked()); + EXPECT_TRUE(lock.TryAcquire()); + EXPECT_TRUE(lock.IsLocked()); + lock.Release(); + lock.Acquire(); + lock.Release(); +} + +TEST(TRecursiveSpinLockTest, TwoThreads) +{ + TRecursiveSpinLock lock; + TEvent e1, e2, e3, e4, e5, e6, e7; + + std::thread t1([&] { + e1.Wait(); + EXPECT_TRUE(lock.IsLocked()); + EXPECT_FALSE(lock.IsLockedByCurrentThread()); + EXPECT_FALSE(lock.TryAcquire()); + e2.NotifyOne(); + e3.Wait(); + EXPECT_TRUE(lock.IsLocked()); + EXPECT_FALSE(lock.IsLockedByCurrentThread()); + EXPECT_FALSE(lock.TryAcquire()); + e4.NotifyOne(); + e5.Wait(); + EXPECT_FALSE(lock.IsLocked()); + EXPECT_FALSE(lock.IsLockedByCurrentThread()); + EXPECT_TRUE(lock.TryAcquire()); + e6.NotifyOne(); + e7.Wait(); + lock.Release(); + }); + + std::thread t2([&] { + EXPECT_FALSE(lock.IsLocked()); + EXPECT_TRUE(lock.TryAcquire()); + EXPECT_TRUE(lock.IsLockedByCurrentThread()); + e1.NotifyOne(); + e2.Wait(); + EXPECT_TRUE(lock.TryAcquire()); + EXPECT_TRUE(lock.IsLockedByCurrentThread()); + e3.NotifyOne(); + e4.Wait(); + lock.Release(); + lock.Release(); + EXPECT_FALSE(lock.IsLocked()); + e5.NotifyOne(); + e6.Wait(); + EXPECT_TRUE(lock.IsLocked()); + EXPECT_FALSE(lock.IsLockedByCurrentThread()); + e7.NotifyOne(); + lock.Acquire(); + lock.Acquire(); + lock.Release(); + lock.Release(); + }); + + t1.join(); + t2.join(); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NThreading diff --git a/library/cpp/yt/threading/unittests/spin_wait_ut.cpp b/library/cpp/yt/threading/unittests/spin_wait_ut.cpp new file mode 100644 index 0000000000..8469634f34 --- /dev/null +++ b/library/cpp/yt/threading/unittests/spin_wait_ut.cpp @@ -0,0 +1,48 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/threading/spin_wait.h> +#include <library/cpp/yt/threading/spin_wait_hook.h> + +#include <thread> +#include <mutex> + +namespace NYT::NThreading { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +bool SpinWaitSlowPathHookInvoked; + +void SpinWaitSlowPathHook( + TCpuDuration cpuDelay, + const TSourceLocation& /*location*/, + ESpinLockActivityKind /*activityKind*/) +{ + SpinWaitSlowPathHookInvoked = true; + auto delay = CpuDurationToDuration(cpuDelay); + EXPECT_GE(delay, TDuration::Seconds(1)); + EXPECT_LE(delay, TDuration::Seconds(5)); +} + +TEST(TSpinWaitTest, SlowPathHook) +{ + static std::once_flag registerFlag; + std::call_once( + registerFlag, + [] { + RegisterSpinWaitSlowPathHook(SpinWaitSlowPathHook); + }); + SpinWaitSlowPathHookInvoked = false; + { + TSpinWait spinWait(__LOCATION__, ESpinLockActivityKind::ReadWrite); + for (int i = 0; i < 1'000'000; ++i) { + spinWait.Wait(); + } + } + EXPECT_TRUE(SpinWaitSlowPathHookInvoked); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NThreading diff --git a/library/cpp/yt/threading/unittests/ya.make b/library/cpp/yt/threading/unittests/ya.make new file mode 100644 index 0000000000..ef9b5d2995 --- /dev/null +++ b/library/cpp/yt/threading/unittests/ya.make @@ -0,0 +1,17 @@ +GTEST() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + count_down_latch_ut.cpp + recursive_spin_lock_ut.cpp + spin_wait_ut.cpp +) + +PEERDIR( + library/cpp/yt/assert + library/cpp/yt/threading + library/cpp/testing/gtest +) + +END() diff --git a/library/cpp/yt/user_job_statistics/user_job_statistics.cpp b/library/cpp/yt/user_job_statistics/user_job_statistics.cpp new file mode 100644 index 0000000000..b7fd71503d --- /dev/null +++ b/library/cpp/yt/user_job_statistics/user_job_statistics.cpp @@ -0,0 +1,133 @@ +#include "user_job_statistics.h" +#include <yt/cpp/mapreduce/common/helpers.h> +#include <util/stream/null.h> +#include <util/string/builder.h> +#include <util/system/mutex.h> +#include <util/system/env.h> + +using namespace NYtTools; + +static TMutex GlobalStatsWritingMutex; + +#if defined(_unix_) +const FHANDLE TUserJobStatsProxy::JobStatisticsHandle = 5; +#elif defined(_win_) +const FHANDLE TUserJobStatsProxy::JobStatisticsHandle = nullptr; +#endif + +static IOutputStream* CorrectHandle(const FHANDLE h) { +#if defined(_unix_) + if (fcntl(h, F_GETFD) == -1) { + return &Cerr; + } + return nullptr; +#elif defined(_win_) + return &Cerr; +#endif +} + +static TString PrintNodeSimple(const NYT::TNode& n) { + return NYT::NodeToYsonString(n, NYson::EYsonFormat::Text); +} + +void TUserJobStatsProxy::Init(IOutputStream * usingStream) { + if (usingStream == nullptr) { + usingStream = CorrectHandle(JobStatisticsHandle); + } + + if(usingStream == nullptr && GetEnv("YT_JOB_ID").empty()) { + usingStream = &Cerr; + } + + + if (usingStream == nullptr) { + TFileHandle fixedDesrc(JobStatisticsHandle); + FetchedOut = MakeHolder<TFixedBufferFileOutput>(TFile(fixedDesrc.Duplicate())); + UsingStream = FetchedOut.Get(); + fixedDesrc.Release(); + } else { + UsingStream = usingStream; + } +} + +void TUserJobStatsProxy::InitChecked(IOutputStream* def) { + IOutputStream* usingStream = CorrectHandle(JobStatisticsHandle); + + if (usingStream == nullptr && !GetEnv("YT_JOB_ID").empty()) { + TFileHandle fixedDesrc(JobStatisticsHandle); + FetchedOut = MakeHolder<TFixedBufferFileOutput>(TFile(fixedDesrc.Duplicate())); + UsingStream = FetchedOut.Get(); + fixedDesrc.Release(); + } else { + UsingStream = def; + } +} + +void TUserJobStatsProxy::InitIfNotInited(IOutputStream * usingStream) { + if (UsingStream == nullptr) { + Init(usingStream); + } +} + +void TUserJobStatsProxy::CommitStats() { + if (Stats.empty()) { + return; + } + + auto res = NYT::TNode::CreateMap(); + for (auto& p : Stats) { + res[p.first] = p.second; + } + for (auto& p : TimeStats) { + res[p.first] = p.second.MilliSeconds(); + } + with_lock(GlobalStatsWritingMutex) { + *UsingStream << PrintNodeSimple(res) << ";" << Endl; + } + Stats.clear(); +} + + +TTimeStatHolder TUserJobStatsProxy::TimerStart(TString name, bool commitOnFinish) { + return THolder(new TTimeStat(this, name, commitOnFinish)); +} + +void TUserJobStatsProxy::WriteStat(TString name, i64 val) { + auto res = NYT::TNode {} (name, val); + with_lock(GlobalStatsWritingMutex) { + *UsingStream << PrintNodeSimple(res) << ";" << Endl; + } +} + +void TUserJobStatsProxy::WriteStatNoFlush(TString name, i64 val) { + auto res = NYT::TNode {} (name, val); + with_lock(GlobalStatsWritingMutex) { + *UsingStream << (TStringBuilder{} << PrintNodeSimple(res) << ";\n"); + } +} + +TTimeStat::TTimeStat(TUserJobStatsProxy* parent, TString name, bool commit) + : Parent(parent) + , Name(name) + , Commit(commit) {} + +TTimeStat::~TTimeStat() { + Finish(); +} + +void TTimeStat::Cancel() { + Parent = nullptr; +} + +void TTimeStat::Finish() { + if (!Parent) { + return; + } + + if (Commit) { + Parent->WriteStatNoFlush(Name, Timer.Get().MilliSeconds()); + } else { + Parent->TimeStats[Name] += Timer.Get(); + } + Cancel(); +} diff --git a/library/cpp/yt/user_job_statistics/user_job_statistics.h b/library/cpp/yt/user_job_statistics/user_job_statistics.h new file mode 100644 index 0000000000..6939d20417 --- /dev/null +++ b/library/cpp/yt/user_job_statistics/user_job_statistics.h @@ -0,0 +1,58 @@ +#pragma once + +#include <util/stream/file.h> +#include <util/generic/hash.h> +#include <util/datetime/cputimer.h> + +namespace NYtTools { + class TTimeStat; + using TTimeStatHolder = THolder<TTimeStat>; + + class TUserJobStatsProxy { + public: + static const FHANDLE JobStatisticsHandle; + private: + THolder<IOutputStream> FetchedOut; + IOutputStream* UsingStream = &Cerr; + public: + // TODO: add inheritance + THashMap<TString, i64> Stats;//will be dumped in CommitStats or desctructor + THashMap<TString, TDuration> TimeStats;//will be dumped in CommitStats or desctructor + + TUserJobStatsProxy() { Init(nullptr); } + ~TUserJobStatsProxy() { + CommitStats(); + } + TUserJobStatsProxy (IOutputStream* usingStream) {Init(usingStream);} + + void Init(IOutputStream* usingStream); + void InitChecked(IOutputStream* ifNotInJob); + void InitIfNotInited(IOutputStream* usingStream); + IOutputStream* GetStream() const { return UsingStream; } + void CommitStats(); + void WriteStat(TString name, i64 val); //immidiatly wirtes stat + void WriteStatNoFlush(TString name, i64 val); //immidiatly wirtes stat but do not flush it + + //@param name name of statistic to be written in millisecs from creation to destruction + //@param commitOnFinish if false: will update state/write on job finish; if true: write stat in destructor + TTimeStatHolder TimerStart(TString name, bool commitOnFinish = false); + }; + + class TTimeStat { + TUserJobStatsProxy* Parent; + TString Name; + bool Commit; + + TTimeStat(TUserJobStatsProxy* parent, TString name, bool commit); + friend class TUserJobStatsProxy; + + TSimpleTimer Timer; + public: + ~TTimeStat(); + TDuration Get() const { + return Timer.Get(); + } + void Cancel(); + void Finish(); + }; +} diff --git a/library/cpp/yt/user_job_statistics/ya.make b/library/cpp/yt/user_job_statistics/ya.make new file mode 100644 index 0000000000..7179660b31 --- /dev/null +++ b/library/cpp/yt/user_job_statistics/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( + user_job_statistics.cpp +) + +PEERDIR( + yt/cpp/mapreduce/common +) + +END() |