diff options
author | max42 <[email protected]> | 2023-07-29 00:02:16 +0300 |
---|---|---|
committer | max42 <[email protected]> | 2023-07-29 00:02:16 +0300 |
commit | 73b89de71748a21e102d27b9f3ed1bf658766cb5 (patch) | |
tree | 188bbd2d622fa91cdcbb1b6d6d77fbc84a0646f5 /library/cpp/yt/logging | |
parent | 528e321bcc2a2b67b53aeba58c3bd88305a141ee (diff) |
YT-19210: expose YQL shared library for YT.
After this, a new target libyqlplugin.so appears. in open-source cmake build.
Diff in open-source YDB repo looks like the following: https://paste.yandex-team.ru/f302bdb4-7ef2-4362-91c7-6ca45f329264
Diffstat (limited to 'library/cpp/yt/logging')
29 files changed, 1878 insertions, 0 deletions
diff --git a/library/cpp/yt/logging/CMakeLists.darwin-x86_64.txt b/library/cpp/yt/logging/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..b9c4a4c7dba --- /dev/null +++ b/library/cpp/yt/logging/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(plain_text_formatter) + +add_library(cpp-yt-logging) +target_compile_options(cpp-yt-logging PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-logging PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-assert + cpp-yt-memory + cpp-yt-misc + cpp-yt-yson_string +) +target_sources(cpp-yt-logging PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/logger.cpp +) diff --git a/library/cpp/yt/logging/CMakeLists.linux-aarch64.txt b/library/cpp/yt/logging/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..bd299948914 --- /dev/null +++ b/library/cpp/yt/logging/CMakeLists.linux-aarch64.txt @@ -0,0 +1,26 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(plain_text_formatter) + +add_library(cpp-yt-logging) +target_compile_options(cpp-yt-logging PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-logging PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-assert + cpp-yt-memory + cpp-yt-misc + cpp-yt-yson_string +) +target_sources(cpp-yt-logging PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/logger.cpp +) diff --git a/library/cpp/yt/logging/CMakeLists.linux-x86_64.txt b/library/cpp/yt/logging/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..bd299948914 --- /dev/null +++ b/library/cpp/yt/logging/CMakeLists.linux-x86_64.txt @@ -0,0 +1,26 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(plain_text_formatter) + +add_library(cpp-yt-logging) +target_compile_options(cpp-yt-logging PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(cpp-yt-logging PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-assert + cpp-yt-memory + cpp-yt-misc + cpp-yt-yson_string +) +target_sources(cpp-yt-logging PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/logger.cpp +) diff --git a/library/cpp/yt/logging/CMakeLists.txt b/library/cpp/yt/logging/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/library/cpp/yt/logging/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/yt/logging/CMakeLists.windows-x86_64.txt b/library/cpp/yt/logging/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..5e766966b07 --- /dev/null +++ b/library/cpp/yt/logging/CMakeLists.windows-x86_64.txt @@ -0,0 +1,22 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +add_subdirectory(plain_text_formatter) + +add_library(cpp-yt-logging) +target_link_libraries(cpp-yt-logging PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-assert + cpp-yt-memory + cpp-yt-misc + cpp-yt-yson_string +) +target_sources(cpp-yt-logging PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/logger.cpp +) diff --git a/library/cpp/yt/logging/backends/arcadia/backend.cpp b/library/cpp/yt/logging/backends/arcadia/backend.cpp new file mode 100644 index 00000000000..3c6ff9f5f58 --- /dev/null +++ b/library/cpp/yt/logging/backends/arcadia/backend.cpp @@ -0,0 +1,86 @@ +#include "backend.h" + +#include <library/cpp/logger/backend.h> +#include <library/cpp/logger/record.h> + +#include <library/cpp/yt/assert/assert.h> + +#include <library/cpp/yt/logging/logger.h> + +namespace NYT::NLogging { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +ELogLevel ConvertToLogLevel(ELogPriority priority) +{ + switch (priority) { + case ELogPriority::TLOG_DEBUG: + return ELogLevel::Debug; + case ELogPriority::TLOG_INFO: + [[fallthrough]]; + case ELogPriority::TLOG_NOTICE: + return ELogLevel::Info; + case ELogPriority::TLOG_WARNING: + return ELogLevel::Warning; + case ELogPriority::TLOG_ERR: + return ELogLevel::Error; + case ELogPriority::TLOG_CRIT: + case ELogPriority::TLOG_ALERT: + return ELogLevel::Alert; + case ELogPriority::TLOG_EMERG: + return ELogLevel::Fatal; + case ELogPriority::TLOG_RESOURCES: + return ELogLevel::Maximum; + } + YT_ABORT(); +} + +class TLogBackendBridge + : public TLogBackend +{ +public: + TLogBackendBridge(const TLogger& logger) + : Logger_(logger) + { } + + void WriteData(const TLogRecord& rec) override + { + const auto logLevel = ConvertToLogLevel(rec.Priority); + if (!Logger_.IsLevelEnabled(logLevel)) { + return; + } + + // Remove trailing \n, because it will add it. + TStringBuf message(rec.Data, rec.Len); + message.ChopSuffix(TStringBuf("\n")); + // Use low-level api, because it is more convinient here. + auto loggingContext = GetLoggingContext(); + auto event = NDetail::CreateLogEvent(loggingContext, Logger_, logLevel); + event.MessageRef = NDetail::BuildLogMessage(loggingContext, Logger_, message).MessageRef; + event.Family = ELogFamily::PlainText; + Logger_.Write(std::move(event)); + } + + void ReopenLog() override + { } + + ELogPriority FiltrationLevel() const override + { + return LOG_MAX_PRIORITY; + } + +private: + const TLogger Logger_; +}; + +} // namespace + +THolder<TLogBackend> CreateArcadiaLogBackend(const TLogger& logger) +{ + return MakeHolder<TLogBackendBridge>(logger); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/backends/arcadia/backend.h b/library/cpp/yt/logging/backends/arcadia/backend.h new file mode 100644 index 00000000000..251918c9726 --- /dev/null +++ b/library/cpp/yt/logging/backends/arcadia/backend.h @@ -0,0 +1,18 @@ +#pragma once + +#include <library/cpp/yt/logging/public.h> + +#include <util/generic/ptr.h> + +class TLogBackend; + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +//! Create TLogBackend which redirects log messages to |logger|. +THolder<TLogBackend> CreateArcadiaLogBackend(const TLogger& logger); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/backends/arcadia/ya.make b/library/cpp/yt/logging/backends/arcadia/ya.make new file mode 100644 index 00000000000..ee90be8108f --- /dev/null +++ b/library/cpp/yt/logging/backends/arcadia/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + backend.cpp +) + +PEERDIR( + library/cpp/yt/assert + library/cpp/yt/logging + + library/cpp/logger +) + +END() diff --git a/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp b/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp new file mode 100644 index 00000000000..62269dc0c0d --- /dev/null +++ b/library/cpp/yt/logging/backends/stream/stream_log_manager.cpp @@ -0,0 +1,87 @@ +#include "stream_log_manager.h" + +#include <library/cpp/yt/logging/logger.h> + +#include <library/cpp/yt/logging/plain_text_formatter/formatter.h> + +#include <library/cpp/yt/string/raw_formatter.h> + +#include <library/cpp/yt/threading/fork_aware_spin_lock.h> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +class TStreamLogManager + : public ILogManager +{ +public: + explicit TStreamLogManager(IOutputStream* output) + : Output_(output) + { } + + void RegisterStaticAnchor( + TLoggingAnchor* anchor, + ::TSourceLocation /*sourceLocation*/, + TStringBuf /*anchorMessage*/) override + { + anchor->Registered = true; + } + + virtual void UpdateAnchor(TLoggingAnchor* anchor) override + { + anchor->Enabled = true; + } + + virtual void Enqueue(TLogEvent&& event) override + { + Buffer_.Reset(); + EventFormatter_.Format(&Buffer_, event); + *Output_ << Buffer_.GetBuffer() << Endl; + } + + virtual const TLoggingCategory* GetCategory(TStringBuf categoryName) override + { + if (!categoryName) { + return nullptr; + } + + auto guard = Guard(SpinLock_); + auto it = NameToCategory_.find(categoryName); + if (it == NameToCategory_.end()) { + auto category = std::make_unique<TLoggingCategory>(); + category->Name = categoryName; + category->ActualVersion = &Version_; + category->CurrentVersion = Version_.load(); + it = NameToCategory_.emplace(categoryName, std::move(category)).first; + } + return it->second.get(); + } + + virtual void UpdateCategory(TLoggingCategory* /*category*/) override + { } + + virtual bool GetAbortOnAlert() const override + { + return false; + } + +private: + IOutputStream* const Output_; + + NThreading::TForkAwareSpinLock SpinLock_; + THashMap<TString, std::unique_ptr<TLoggingCategory>> NameToCategory_; + std::atomic<int> Version_ = 1; + + TPlainTextEventFormatter EventFormatter_{/*enableSourceLocation*/ false}; + TRawFormatter<MessageBufferSize> Buffer_; +}; + +std::unique_ptr<ILogManager> CreateStreamLogManager(IOutputStream* output) +{ + return std::make_unique<TStreamLogManager>(output); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/backends/stream/stream_log_manager.h b/library/cpp/yt/logging/backends/stream/stream_log_manager.h new file mode 100644 index 00000000000..2f6794e5878 --- /dev/null +++ b/library/cpp/yt/logging/backends/stream/stream_log_manager.h @@ -0,0 +1,15 @@ +#pragma once + +#include <library/cpp/yt/logging/public.h> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +//! Creates a dead-simple implementation that synchronously logs +//! all events to #output. +std::unique_ptr<ILogManager> CreateStreamLogManager(IOutputStream* output); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp b/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp new file mode 100644 index 00000000000..cb3e244e3b9 --- /dev/null +++ b/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp @@ -0,0 +1,37 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/logging/logger.h> + +#include <library/cpp/yt/logging/backends/stream/stream_log_manager.h> + +#include <util/stream/str.h> + +#include <util/string/split.h> + +namespace NYT::NLogging { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TStreamLogManagerTest, Simple) +{ + TString str; + { + TStringOutput output(str); + auto logManager = CreateStreamLogManager(&output); + TLogger Logger(logManager.get(), "Test"); + YT_LOG_INFO("Hello world"); + } + + TVector<TStringBuf> tokens; + Split(str, "\t", tokens); + EXPECT_GE(std::ssize(tokens), 4); + EXPECT_EQ(tokens[1], "I"); + EXPECT_EQ(tokens[2], "Test"); + EXPECT_EQ(tokens[3], "Hello world"); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/backends/stream/unittests/ya.make b/library/cpp/yt/logging/backends/stream/unittests/ya.make new file mode 100644 index 00000000000..29270459faa --- /dev/null +++ b/library/cpp/yt/logging/backends/stream/unittests/ya.make @@ -0,0 +1,14 @@ +GTEST(unittester-library-logging) + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + stream_log_manager_ut.cpp +) + +PEERDIR( + library/cpp/testing/gtest + library/cpp/yt/logging/backends/stream +) + +END() diff --git a/library/cpp/yt/logging/backends/stream/ya.make b/library/cpp/yt/logging/backends/stream/ya.make new file mode 100644 index 00000000000..86e3aa046b9 --- /dev/null +++ b/library/cpp/yt/logging/backends/stream/ya.make @@ -0,0 +1,20 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + stream_log_manager.cpp +) + +PEERDIR( + library/cpp/yt/logging + library/cpp/yt/logging/plain_text_formatter + library/cpp/yt/string + library/cpp/yt/threading +) + +END() + +RECURSE_FOR_TESTS( + unittests +) diff --git a/library/cpp/yt/logging/backends/ya.make b/library/cpp/yt/logging/backends/ya.make new file mode 100644 index 00000000000..6ee4b72bfe3 --- /dev/null +++ b/library/cpp/yt/logging/backends/ya.make @@ -0,0 +1,4 @@ +RECURSE( + arcadia + stream +) diff --git a/library/cpp/yt/logging/logger-inl.h b/library/cpp/yt/logging/logger-inl.h new file mode 100644 index 00000000000..6f489da82dc --- /dev/null +++ b/library/cpp/yt/logging/logger-inl.h @@ -0,0 +1,303 @@ +#ifndef LOGGER_INL_H_ +#error "Direct inclusion of this file is not allowed, include logger.h" +// For the sake of sane code completion. +#include "logger.h" +#endif +#undef LOGGER_INL_H_ + +#include <library/cpp/yt/yson_string/convert.h> +#include <library/cpp/yt/yson_string/string.h> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +inline bool TLogger::IsAnchorUpToDate(const TLoggingAnchor& position) const +{ + return + !Category_ || + position.CurrentVersion == Category_->ActualVersion->load(std::memory_order::relaxed); +} + +template <class... TArgs> +void TLogger::AddTag(const char* format, TArgs&&... args) +{ + AddRawTag(Format(format, std::forward<TArgs>(args)...)); +} + +template <class TType> +void TLogger::AddStructuredTag(TStringBuf key, TType value) +{ + StructuredTags_.emplace_back(key, NYson::ConvertToYsonString(value)); +} + +template <class... TArgs> +TLogger TLogger::WithTag(const char* format, TArgs&&... args) const +{ + auto result = *this; + result.AddTag(format, std::forward<TArgs>(args)...); + return result; +} + +template <class TType> +TLogger TLogger::WithStructuredTag(TStringBuf key, TType value) const +{ + auto result = *this; + result.AddStructuredTag(key, value); + return result; +} + +Y_FORCE_INLINE bool TLogger::IsLevelEnabled(ELogLevel level) const +{ + // This is the first check which is intended to be inlined next to + // logging invocation point. Check below is almost zero-cost due + // to branch prediction (which requires inlining for proper work). + if (level < MinLevel_) { + return false; + } + + // Next check is heavier and requires full log manager definition which + // is undesirable in -inl.h header file. This is why we extract it + // to a separate method which is implemented in cpp file. + return IsLevelEnabledHeavy(level); +} + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +struct TMessageStringBuilderContext +{ + TSharedMutableRef Chunk; +}; + +struct TMessageBufferTag +{ }; + +class TMessageStringBuilder + : public TStringBuilderBase +{ +public: + TSharedRef Flush(); + + // For testing only. + static void DisablePerThreadCache(); + +protected: + void DoReset() override; + void DoReserve(size_t newLength) override; + +private: + struct TPerThreadCache + { + ~TPerThreadCache(); + + TSharedMutableRef Chunk; + size_t ChunkOffset = 0; + }; + + TSharedMutableRef Buffer_; + + static thread_local TPerThreadCache* Cache_; + static thread_local bool CacheDestroyed_; + static TPerThreadCache* GetCache(); + + static constexpr size_t ChunkSize = 128_KB - 64; +}; + +inline bool HasMessageTags( + const TLoggingContext& loggingContext, + const TLogger& logger) +{ + if (logger.GetTag()) { + return true; + } + if (loggingContext.TraceLoggingTag) { + return true; + } + return false; +} + +inline void AppendMessageTags( + TStringBuilderBase* builder, + const TLoggingContext& loggingContext, + const TLogger& logger) +{ + bool printComma = false; + if (const auto& loggerTag = logger.GetTag()) { + builder->AppendString(loggerTag); + printComma = true; + } + if (auto traceLoggingTag = loggingContext.TraceLoggingTag) { + if (printComma) { + builder->AppendString(TStringBuf(", ")); + } + builder->AppendString(traceLoggingTag); + } +} + +inline void AppendLogMessage( + TStringBuilderBase* builder, + const TLoggingContext& loggingContext, + const TLogger& logger, + TRef message) +{ + if (HasMessageTags(loggingContext, logger)) { + if (message.Size() >= 1 && message[message.Size() - 1] == ')') { + builder->AppendString(TStringBuf(message.Begin(), message.Size() - 1)); + builder->AppendString(TStringBuf(", ")); + } else { + builder->AppendString(TStringBuf(message.Begin(), message.Size())); + builder->AppendString(TStringBuf(" (")); + } + AppendMessageTags(builder, loggingContext, logger); + builder->AppendChar(')'); + } else { + builder->AppendString(TStringBuf(message.Begin(), message.Size())); + } +} + +template <class... TArgs> +void AppendLogMessageWithFormat( + TStringBuilderBase* builder, + const TLoggingContext& loggingContext, + const TLogger& logger, + TStringBuf format, + TArgs&&... args) +{ + if (HasMessageTags(loggingContext, logger)) { + if (format.size() >= 2 && format[format.size() - 1] == ')') { + builder->AppendFormat(format.substr(0, format.size() - 1), std::forward<TArgs>(args)...); + builder->AppendString(TStringBuf(", ")); + } else { + builder->AppendFormat(format, std::forward<TArgs>(args)...); + builder->AppendString(TStringBuf(" (")); + } + AppendMessageTags(builder, loggingContext, logger); + builder->AppendChar(')'); + } else { + builder->AppendFormat(format, std::forward<TArgs>(args)...); + } +} + +struct TLogMessage +{ + TSharedRef MessageRef; + TStringBuf Anchor; +}; + +template <size_t Length, class... TArgs> +TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + const char (&format)[Length], + TArgs&&... args) +{ + TMessageStringBuilder builder; + AppendLogMessageWithFormat(&builder, loggingContext, logger, format, std::forward<TArgs>(args)...); + return {builder.Flush(), format}; +} + +template <class T> +TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + const T& obj) +{ + TMessageStringBuilder builder; + FormatValue(&builder, obj, TStringBuf()); + if (HasMessageTags(loggingContext, logger)) { + builder.AppendString(TStringBuf(" (")); + AppendMessageTags(&builder, loggingContext, logger); + builder.AppendChar(')'); + } + return {builder.Flush(), TStringBuf()}; +} + +inline TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + TStringBuf message) +{ + TMessageStringBuilder builder; + builder.AppendString(message); + if (HasMessageTags(loggingContext, logger)) { + builder.AppendString(TStringBuf(" (")); + AppendMessageTags(&builder, loggingContext, logger); + builder.AppendChar(')'); + } + return {builder.Flush(), message}; +} + +template <size_t Length> +TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + const char (&message)[Length]) +{ + return BuildLogMessage( + loggingContext, + logger, + TStringBuf(message)); +} + +inline TLogMessage BuildLogMessage( + const TLoggingContext& loggingContext, + const TLogger& logger, + TSharedRef&& message) +{ + if (HasMessageTags(loggingContext, logger)) { + TMessageStringBuilder builder; + AppendLogMessage(&builder, loggingContext, logger, message); + return {builder.Flush(), TStringBuf()}; + } else { + return {std::move(message), TStringBuf()}; + } +} + +inline TLogEvent CreateLogEvent( + const TLoggingContext& loggingContext, + const TLogger& logger, + ELogLevel level) +{ + TLogEvent event; + event.Instant = loggingContext.Instant; + event.Category = logger.GetCategory(); + event.Essential = logger.IsEssential(); + event.Level = level; + event.ThreadId = loggingContext.ThreadId; + event.ThreadName = loggingContext.ThreadName; + event.FiberId = loggingContext.FiberId; + event.TraceId = loggingContext.TraceId; + event.RequestId = loggingContext.RequestId; + return event; +} + +void OnCriticalLogEvent( + const TLogger& logger, + const TLogEvent& event); + +inline void LogEventImpl( + const TLoggingContext& loggingContext, + const TLogger& logger, + ELogLevel level, + ::TSourceLocation sourceLocation, + TSharedRef message) +{ + auto event = CreateLogEvent(loggingContext, logger, level); + event.MessageKind = ELogMessageKind::Unstructured; + event.MessageRef = std::move(message); + event.Family = ELogFamily::PlainText; + event.SourceFile = sourceLocation.File; + event.SourceLine = sourceLocation.Line; + logger.Write(std::move(event)); + if (Y_UNLIKELY(event.Level >= ELogLevel::Alert)) { + OnCriticalLogEvent(logger, event); + } +} + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/logger.cpp b/library/cpp/yt/logging/logger.cpp new file mode 100644 index 00000000000..4ee5c1a01b0 --- /dev/null +++ b/library/cpp/yt/logging/logger.cpp @@ -0,0 +1,289 @@ +#include "logger.h" + +#include <library/cpp/yt/assert/assert.h> + +#include <library/cpp/yt/cpu_clock/clock.h> + +#include <library/cpp/yt/misc/thread_name.h> + +#include <util/system/compiler.h> +#include <util/system/thread.h> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +void OnCriticalLogEvent( + const TLogger& logger, + const TLogEvent& event) +{ + if (event.Level == ELogLevel::Fatal || + event.Level == ELogLevel::Alert && logger.GetAbortOnAlert()) + { + fprintf(stderr, "*** Aborting on critical log event\n"); + fwrite(event.MessageRef.begin(), 1, event.MessageRef.size(), stderr); + fprintf(stderr, "\n"); + YT_ABORT(); + } +} + +TSharedRef TMessageStringBuilder::Flush() +{ + return Buffer_.Slice(0, GetLength()); +} + +void TMessageStringBuilder::DisablePerThreadCache() +{ + Cache_ = nullptr; + CacheDestroyed_ = true; +} + +void TMessageStringBuilder::DoReset() +{ + Buffer_.Reset(); +} + +void TMessageStringBuilder::DoReserve(size_t newCapacity) +{ + auto oldLength = GetLength(); + newCapacity = FastClp2(newCapacity); + + auto newChunkSize = std::max(ChunkSize, newCapacity); + // Hold the old buffer until the data is copied. + auto oldBuffer = std::move(Buffer_); + auto* cache = GetCache(); + if (Y_LIKELY(cache)) { + auto oldCapacity = End_ - Begin_; + auto deltaCapacity = newCapacity - oldCapacity; + if (End_ == cache->Chunk.Begin() + cache->ChunkOffset && + cache->ChunkOffset + deltaCapacity <= cache->Chunk.Size()) + { + // Resize inplace. + Buffer_ = cache->Chunk.Slice(cache->ChunkOffset - oldCapacity, cache->ChunkOffset + deltaCapacity); + cache->ChunkOffset += deltaCapacity; + End_ = Begin_ + newCapacity; + return; + } + + if (Y_UNLIKELY(cache->ChunkOffset + newCapacity > cache->Chunk.Size())) { + cache->Chunk = TSharedMutableRef::Allocate<TMessageBufferTag>(newChunkSize, {.InitializeStorage = false}); + cache->ChunkOffset = 0; + } + + Buffer_ = cache->Chunk.Slice(cache->ChunkOffset, cache->ChunkOffset + newCapacity); + cache->ChunkOffset += newCapacity; + } else { + Buffer_ = TSharedMutableRef::Allocate<TMessageBufferTag>(newChunkSize, {.InitializeStorage = false}); + newCapacity = newChunkSize; + } + if (oldLength > 0) { + ::memcpy(Buffer_.Begin(), Begin_, oldLength); + } + Begin_ = Buffer_.Begin(); + End_ = Begin_ + newCapacity; +} + +TMessageStringBuilder::TPerThreadCache* TMessageStringBuilder::GetCache() +{ + if (Y_LIKELY(Cache_)) { + return Cache_; + } + if (CacheDestroyed_) { + return nullptr; + } + static thread_local TPerThreadCache Cache; + Cache_ = &Cache; + return Cache_; +} + +TMessageStringBuilder::TPerThreadCache::~TPerThreadCache() +{ + TMessageStringBuilder::DisablePerThreadCache(); +} + +thread_local TMessageStringBuilder::TPerThreadCache* TMessageStringBuilder::Cache_; +thread_local bool TMessageStringBuilder::CacheDestroyed_; + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +Y_WEAK TLoggingContext GetLoggingContext() +{ + return { + .Instant = GetCpuInstant(), + .ThreadId = TThread::CurrentThreadId(), + .ThreadName = GetCurrentThreadName(), + }; +} + +Y_WEAK ILogManager* GetDefaultLogManager() +{ + return nullptr; +} + +//////////////////////////////////////////////////////////////////////////////// + +thread_local ELogLevel ThreadMinLogLevel = ELogLevel::Minimum; + +void SetThreadMinLogLevel(ELogLevel minLogLevel) +{ + ThreadMinLogLevel = minLogLevel; +} + +ELogLevel GetThreadMinLogLevel() +{ + return ThreadMinLogLevel; +} + +//////////////////////////////////////////////////////////////////////////////// + +TLogger::TLogger(ILogManager* logManager, TStringBuf categoryName) + : LogManager_(logManager) + , Category_(LogManager_ ? LogManager_->GetCategory(categoryName) : nullptr) + , MinLevel_(LogManager_ ? LoggerDefaultMinLevel : NullLoggerMinLevel) +{ } + +TLogger::TLogger(TStringBuf categoryName) + : TLogger(GetDefaultLogManager(), categoryName) +{ } + +TLogger::operator bool() const +{ + return LogManager_; +} + +const TLoggingCategory* TLogger::GetCategory() const +{ + return Category_; +} + +bool TLogger::IsLevelEnabledHeavy(ELogLevel level) const +{ + // Note that we managed to reach this point, i.e. level >= MinLevel_, + // which implies that MinLevel_ != ELogLevel::Maximum, so this logger was not + // default constructed, thus it has non-trivial category. + YT_ASSERT(Category_); + + if (Category_->CurrentVersion != Category_->ActualVersion->load(std::memory_order::relaxed)) { + LogManager_->UpdateCategory(const_cast<TLoggingCategory*>(Category_)); + } + + return + level >= Category_->MinPlainTextLevel && + level >= ThreadMinLogLevel; +} + +bool TLogger::GetAbortOnAlert() const +{ + return LogManager_->GetAbortOnAlert(); +} + +bool TLogger::IsEssential() const +{ + return Essential_; +} + +void TLogger::UpdateAnchor(TLoggingAnchor* anchor) const +{ + LogManager_->UpdateAnchor(anchor); +} + +void TLogger::RegisterStaticAnchor(TLoggingAnchor* anchor, ::TSourceLocation sourceLocation, TStringBuf message) const +{ + LogManager_->RegisterStaticAnchor(anchor, sourceLocation, message); +} + +void TLogger::Write(TLogEvent&& event) const +{ + LogManager_->Enqueue(std::move(event)); +} + +void TLogger::AddRawTag(const TString& tag) +{ + if (!Tag_.empty()) { + Tag_ += ", "; + } + Tag_ += tag; +} + +TLogger TLogger::WithRawTag(const TString& tag) const +{ + auto result = *this; + result.AddRawTag(tag); + return result; +} + +TLogger TLogger::WithEssential(bool essential) const +{ + auto result = *this; + result.Essential_ = essential; + return result; +} + +TLogger TLogger::WithStructuredValidator(TStructuredValidator validator) const +{ + auto result = *this; + result.StructuredValidators_.push_back(std::move(validator)); + return result; +} + +TLogger TLogger::WithMinLevel(ELogLevel minLevel) const +{ + auto result = *this; + if (result) { + result.MinLevel_ = minLevel; + } + return result; +} + +const TString& TLogger::GetTag() const +{ + return Tag_; +} + +const TLogger::TStructuredTags& TLogger::GetStructuredTags() const +{ + return StructuredTags_; +} + +const TLogger::TStructuredValidators& TLogger::GetStructuredValidators() const +{ + return StructuredValidators_; +} + +//////////////////////////////////////////////////////////////////////////////// + +void LogStructuredEvent( + const TLogger& logger, + NYson::TYsonString message, + ELogLevel level) +{ + YT_VERIFY(message.GetType() == NYson::EYsonType::MapFragment); + + if (!logger.GetStructuredValidators().empty()) { + auto samplingRate = logger.GetCategory()->StructuredValidationSamplingRate.load(); + auto p = RandomNumber<double>(); + if (p < samplingRate) { + for (const auto& validator : logger.GetStructuredValidators()) { + validator(message); + } + } + } + + auto loggingContext = GetLoggingContext(); + auto event = NDetail::CreateLogEvent( + loggingContext, + logger, + level); + event.MessageKind = ELogMessageKind::Structured; + event.MessageRef = message.ToSharedRef(); + event.Family = ELogFamily::Structured; + logger.Write(std::move(event)); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/logger.h b/library/cpp/yt/logging/logger.h new file mode 100644 index 00000000000..686ce9251c0 --- /dev/null +++ b/library/cpp/yt/logging/logger.h @@ -0,0 +1,351 @@ +#pragma once + +#include "public.h" + +#include <library/cpp/yt/string/format.h> + +#include <library/cpp/yt/memory/ref.h> + +#include <library/cpp/yt/cpu_clock/public.h> + +#include <library/cpp/yt/yson_string/string.h> + +#include <library/cpp/yt/misc/guid.h> + +#include <library/cpp/yt/misc/thread_name.h> + +#include <library/cpp/yt/memory/leaky_singleton.h> + +#include <util/system/src_location.h> + +#include <util/generic/size_literals.h> + +#include <atomic> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +constexpr double DefaultStructuredValidationSamplingRate = 0.01; + +struct TLoggingCategory +{ + TString Name; + //! This value is used for early dropping of plaintext events in order + //! to reduce load on logging thread for events which are definitely going + //! to be dropped due to rule setup. + //! NB: this optimization is used only for plaintext events since structured + //! logging rate is negligible comparing to the plaintext logging rate. + std::atomic<ELogLevel> MinPlainTextLevel; + std::atomic<int> CurrentVersion; + std::atomic<int>* ActualVersion; + std::atomic<double> StructuredValidationSamplingRate = DefaultStructuredValidationSamplingRate; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TLoggingAnchor +{ + std::atomic<bool> Registered = false; + ::TSourceLocation SourceLocation = {TStringBuf{}, 0}; + TString AnchorMessage; + TLoggingAnchor* NextAnchor = nullptr; + + std::atomic<int> CurrentVersion = 0; + std::atomic<bool> Enabled = false; + + struct TCounter + { + std::atomic<i64> Current = 0; + i64 Previous = 0; + }; + + TCounter MessageCounter; + TCounter ByteCounter; +}; + +//////////////////////////////////////////////////////////////////////////////// +// Declare some type aliases to avoid circular dependencies. +using TThreadId = size_t; +using TFiberId = size_t; +using TTraceId = TGuid; +using TRequestId = TGuid; + +//////////////////////////////////////////////////////////////////////////////// + +DEFINE_ENUM(ELogMessageKind, + (Unstructured) + (Structured) +); + +struct TLogEvent +{ + const TLoggingCategory* Category = nullptr; + ELogLevel Level = ELogLevel::Minimum; + ELogFamily Family = ELogFamily::PlainText; + bool Essential = false; + + ELogMessageKind MessageKind = ELogMessageKind::Unstructured; + TSharedRef MessageRef; + + TCpuInstant Instant = 0; + + TThreadId ThreadId = {}; + TThreadName ThreadName = {}; + + TFiberId FiberId = {}; + + TTraceId TraceId; + TRequestId RequestId; + + TStringBuf SourceFile; + int SourceLine = -1; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct ILogManager +{ + virtual ~ILogManager() = default; + + virtual void RegisterStaticAnchor( + TLoggingAnchor* anchor, + ::TSourceLocation sourceLocation, + TStringBuf anchorMessage) = 0; + virtual void UpdateAnchor(TLoggingAnchor* anchor) = 0; + + virtual void Enqueue(TLogEvent&& event) = 0; + + virtual const TLoggingCategory* GetCategory(TStringBuf categoryName) = 0; + virtual void UpdateCategory(TLoggingCategory* category) = 0; + + virtual bool GetAbortOnAlert() const = 0; +}; + +ILogManager* GetDefaultLogManager(); + +//////////////////////////////////////////////////////////////////////////////// + +struct TLoggingContext +{ + TCpuInstant Instant; + TThreadId ThreadId; + TThreadName ThreadName; + TFiberId FiberId; + TTraceId TraceId; + TRequestId RequestId; + TStringBuf TraceLoggingTag; +}; + +TLoggingContext GetLoggingContext(); + +//////////////////////////////////////////////////////////////////////////////// + +//! Sets the minimum logging level for messages in current thread. +// NB: In fiber environment, min log level is attached to a fiber, +// so after context switch thread min log level might change. +void SetThreadMinLogLevel(ELogLevel minLogLevel); +ELogLevel GetThreadMinLogLevel(); + +//////////////////////////////////////////////////////////////////////////////// + +static constexpr auto NullLoggerMinLevel = ELogLevel::Maximum; + +// Min level for non-null logger depends on whether we are in debug or release build. +// - For release mode default behavior is to omit trace logging, +// this is done by setting logger min level to Debug by default. +// - For debug mode logger min level is set to trace by default, so that trace logging is +// allowed by logger, but still may be discarded by category min level. +#ifdef NDEBUG +static constexpr auto LoggerDefaultMinLevel = ELogLevel::Debug; +#else +static constexpr auto LoggerDefaultMinLevel = ELogLevel::Trace; +#endif + +class TLogger +{ +public: + using TStructuredValidator = std::function<void(const NYson::TYsonString&)>; + using TStructuredValidators = std::vector<TStructuredValidator>; + + using TStructuredTag = std::pair<TString, NYson::TYsonString>; + // TODO(max42): switch to TCompactVector after YT-15430. + using TStructuredTags = std::vector<TStructuredTag>; + + TLogger() = default; + TLogger(const TLogger& other) = default; + TLogger& operator=(const TLogger& other) = default; + + TLogger(ILogManager* logManager, TStringBuf categoryName); + explicit TLogger(TStringBuf categoryName); + + explicit operator bool() const; + + const TLoggingCategory* GetCategory() const; + + //! Validate that level is admitted by logger's own min level + //! and by category's min level. + bool IsLevelEnabled(ELogLevel level) const; + + bool GetAbortOnAlert() const; + + bool IsEssential() const; + + bool IsAnchorUpToDate(const TLoggingAnchor& anchor) const; + void UpdateAnchor(TLoggingAnchor* anchor) const; + void RegisterStaticAnchor(TLoggingAnchor* anchor, ::TSourceLocation sourceLocation, TStringBuf message) const; + + void Write(TLogEvent&& event) const; + + void AddRawTag(const TString& tag); + template <class... TArgs> + void AddTag(const char* format, TArgs&&... args); + + template <class TType> + void AddStructuredTag(TStringBuf key, TType value); + + TLogger WithRawTag(const TString& tag) const; + template <class... TArgs> + TLogger WithTag(const char* format, TArgs&&... args) const; + + template <class TType> + TLogger WithStructuredTag(TStringBuf key, TType value) const; + + TLogger WithStructuredValidator(TStructuredValidator validator) const; + + TLogger WithMinLevel(ELogLevel minLevel) const; + + TLogger WithEssential(bool essential = true) const; + + const TString& GetTag() const; + const TStructuredTags& GetStructuredTags() const; + + const TStructuredValidators& GetStructuredValidators() const; + +protected: + // These fields are set only during logger creation, so they are effectively const + // and accessing them is thread-safe. + ILogManager* LogManager_ = nullptr; + const TLoggingCategory* Category_ = nullptr; + bool Essential_ = false; + ELogLevel MinLevel_ = NullLoggerMinLevel; + TString Tag_; + TStructuredTags StructuredTags_; + TStructuredValidators StructuredValidators_; + +private: + //! This method checks level against category's min level. + //! Refer to comment in TLogger::IsLevelEnabled for more details. + bool IsLevelEnabledHeavy(ELogLevel level) const; +}; + +//////////////////////////////////////////////////////////////////////////////// + +void LogStructuredEvent( + const TLogger& logger, + NYson::TYsonString message, + ELogLevel level); + +//////////////////////////////////////////////////////////////////////////////// + +#ifdef YT_ENABLE_TRACE_LOGGING +#define YT_LOG_TRACE(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Trace, __VA_ARGS__) +#define YT_LOG_TRACE_IF(condition, ...) if (condition) YT_LOG_TRACE(__VA_ARGS__) +#define YT_LOG_TRACE_UNLESS(condition, ...) if (!(condition)) YT_LOG_TRACE(__VA_ARGS__) +#else +#define YT_LOG_UNUSED(...) if (true) { } else { YT_LOG_DEBUG(__VA_ARGS__); } +#define YT_LOG_TRACE(...) YT_LOG_UNUSED(__VA_ARGS__) +#define YT_LOG_TRACE_IF(condition, ...) YT_LOG_UNUSED(__VA_ARGS__) +#define YT_LOG_TRACE_UNLESS(condition, ...) YT_LOG_UNUSED(__VA_ARGS__) +#endif + +#define YT_LOG_DEBUG(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Debug, __VA_ARGS__) +#define YT_LOG_DEBUG_IF(condition, ...) if (condition) YT_LOG_DEBUG(__VA_ARGS__) +#define YT_LOG_DEBUG_UNLESS(condition, ...) if (!(condition)) YT_LOG_DEBUG(__VA_ARGS__) + +#define YT_LOG_INFO(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Info, __VA_ARGS__) +#define YT_LOG_INFO_IF(condition, ...) if (condition) YT_LOG_INFO(__VA_ARGS__) +#define YT_LOG_INFO_UNLESS(condition, ...) if (!(condition)) YT_LOG_INFO(__VA_ARGS__) + +#define YT_LOG_WARNING(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Warning, __VA_ARGS__) +#define YT_LOG_WARNING_IF(condition, ...) if (condition) YT_LOG_WARNING(__VA_ARGS__) +#define YT_LOG_WARNING_UNLESS(condition, ...) if (!(condition)) YT_LOG_WARNING(__VA_ARGS__) + +#define YT_LOG_ERROR(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Error, __VA_ARGS__) +#define YT_LOG_ERROR_IF(condition, ...) if (condition) YT_LOG_ERROR(__VA_ARGS__) +#define YT_LOG_ERROR_UNLESS(condition, ...) if (!(condition)) YT_LOG_ERROR(__VA_ARGS__) + +#define YT_LOG_ALERT(...) YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Alert, __VA_ARGS__); +#define YT_LOG_ALERT_IF(condition, ...) if (condition) YT_LOG_ALERT(__VA_ARGS__) +#define YT_LOG_ALERT_UNLESS(condition, ...) if (!(condition)) YT_LOG_ALERT(__VA_ARGS__) + +#define YT_LOG_FATAL(...) \ + do { \ + YT_LOG_EVENT(Logger, ::NYT::NLogging::ELogLevel::Fatal, __VA_ARGS__); \ + Y_UNREACHABLE(); \ + } while(false) +#define YT_LOG_FATAL_IF(condition, ...) if (Y_UNLIKELY(condition)) YT_LOG_FATAL(__VA_ARGS__) +#define YT_LOG_FATAL_UNLESS(condition, ...) if (!Y_LIKELY(condition)) YT_LOG_FATAL(__VA_ARGS__) + +#define YT_LOG_EVENT(logger, level, ...) \ + YT_LOG_EVENT_WITH_ANCHOR(logger, level, nullptr, __VA_ARGS__) + +#define YT_LOG_EVENT_WITH_ANCHOR(logger, level, anchor, ...) \ + do { \ + const auto& logger__##__LINE__ = (logger); \ + auto level__##__LINE__ = (level); \ + \ + if (!logger__##__LINE__.IsLevelEnabled(level__##__LINE__)) { \ + break; \ + } \ + \ + auto location__##__LINE__ = __LOCATION__; \ + \ + ::NYT::NLogging::TLoggingAnchor* anchor__##__LINE__ = (anchor); \ + if (!anchor__##__LINE__) { \ + static ::NYT::TLeakyStorage<::NYT::NLogging::TLoggingAnchor> staticAnchor__##__LINE__; \ + anchor__##__LINE__ = staticAnchor__##__LINE__.Get(); \ + } \ + \ + bool anchorUpToDate__##__LINE__ = logger__##__LINE__.IsAnchorUpToDate(*anchor__##__LINE__); \ + if (anchorUpToDate__##__LINE__ && !anchor__##__LINE__->Enabled.load(std::memory_order::relaxed)) { \ + break; \ + } \ + \ + auto loggingContext__##__LINE__ = ::NYT::NLogging::GetLoggingContext(); \ + auto message__##__LINE__ = ::NYT::NLogging::NDetail::BuildLogMessage(loggingContext__##__LINE__, logger__##__LINE__, __VA_ARGS__); \ + \ + if (!anchorUpToDate__##__LINE__) { \ + logger__##__LINE__.RegisterStaticAnchor(anchor__##__LINE__, location__##__LINE__, message__##__LINE__.Anchor); \ + logger__##__LINE__.UpdateAnchor(anchor__##__LINE__); \ + } \ + \ + if (!anchor__##__LINE__->Enabled.load(std::memory_order::relaxed)) { \ + break; \ + } \ + \ + static thread_local i64 localByteCounter__##__LINE__; \ + static thread_local ui8 localMessageCounter__##__LINE__; \ + \ + localByteCounter__##__LINE__ += message__##__LINE__.MessageRef.Size(); \ + if (Y_UNLIKELY(++localMessageCounter__##__LINE__ == 0)) { \ + anchor__##__LINE__->MessageCounter.Current += 256; \ + anchor__##__LINE__->ByteCounter.Current += localByteCounter__##__LINE__; \ + localByteCounter__##__LINE__ = 0; \ + } \ + \ + ::NYT::NLogging::NDetail::LogEventImpl( \ + loggingContext__##__LINE__, \ + logger__##__LINE__, \ + level__##__LINE__, \ + location__##__LINE__, \ + std::move(message__##__LINE__.MessageRef)); \ + } while (false) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging + +#define LOGGER_INL_H_ +#include "logger-inl.h" +#undef LOGGER_INL_H_ diff --git a/library/cpp/yt/logging/plain_text_formatter/CMakeLists.darwin-x86_64.txt b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.darwin-x86_64.txt new file mode 100644 index 00000000000..3be59e00aff --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.darwin-x86_64.txt @@ -0,0 +1,24 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-logging-plain_text_formatter) +target_compile_options(yt-logging-plain_text_formatter PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-logging-plain_text_formatter PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-cpu_clock + cpp-yt-logging + cpp-yt-string + cpp-yt-misc +) +target_sources(yt-logging-plain_text_formatter PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/plain_text_formatter/formatter.cpp +) diff --git a/library/cpp/yt/logging/plain_text_formatter/CMakeLists.linux-aarch64.txt b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.linux-aarch64.txt new file mode 100644 index 00000000000..2771fdd74f4 --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.linux-aarch64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-logging-plain_text_formatter) +target_compile_options(yt-logging-plain_text_formatter PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-logging-plain_text_formatter PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-cpu_clock + cpp-yt-logging + cpp-yt-string + cpp-yt-misc +) +target_sources(yt-logging-plain_text_formatter PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/plain_text_formatter/formatter.cpp +) diff --git a/library/cpp/yt/logging/plain_text_formatter/CMakeLists.linux-x86_64.txt b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.linux-x86_64.txt new file mode 100644 index 00000000000..2771fdd74f4 --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.linux-x86_64.txt @@ -0,0 +1,25 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-logging-plain_text_formatter) +target_compile_options(yt-logging-plain_text_formatter PRIVATE + -Wdeprecated-this-capture +) +target_link_libraries(yt-logging-plain_text_formatter PUBLIC + contrib-libs-linux-headers + contrib-libs-cxxsupp + yutil + cpp-yt-cpu_clock + cpp-yt-logging + cpp-yt-string + cpp-yt-misc +) +target_sources(yt-logging-plain_text_formatter PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/plain_text_formatter/formatter.cpp +) diff --git a/library/cpp/yt/logging/plain_text_formatter/CMakeLists.txt b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.txt new file mode 100644 index 00000000000..f8b31df0c11 --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.txt @@ -0,0 +1,17 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-aarch64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + include(CMakeLists.darwin-x86_64.txt) +elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA) + include(CMakeLists.windows-x86_64.txt) +elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA) + include(CMakeLists.linux-x86_64.txt) +endif() diff --git a/library/cpp/yt/logging/plain_text_formatter/CMakeLists.windows-x86_64.txt b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.windows-x86_64.txt new file mode 100644 index 00000000000..d78a3651a04 --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/CMakeLists.windows-x86_64.txt @@ -0,0 +1,21 @@ + +# This file was generated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(yt-logging-plain_text_formatter) +target_link_libraries(yt-logging-plain_text_formatter PUBLIC + contrib-libs-cxxsupp + yutil + cpp-yt-cpu_clock + cpp-yt-logging + cpp-yt-string + cpp-yt-misc +) +target_sources(yt-logging-plain_text_formatter PRIVATE + ${CMAKE_SOURCE_DIR}/library/cpp/yt/logging/plain_text_formatter/formatter.cpp +) diff --git a/library/cpp/yt/logging/plain_text_formatter/formatter.cpp b/library/cpp/yt/logging/plain_text_formatter/formatter.cpp new file mode 100644 index 00000000000..ab2181113dd --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/formatter.cpp @@ -0,0 +1,226 @@ +#include "formatter.h" + +#include <library/cpp/yt/cpu_clock/clock.h> + +#include <library/cpp/yt/misc/port.h> + +#ifdef YT_USE_SSE42 + #include <emmintrin.h> + #include <pmmintrin.h> +#endif + +namespace NYT::NLogging { + +constexpr int MessageBufferWatermarkSize = 256; + +//////////////////////////////////////////////////////////////////////////////// + +namespace { + +// Ultra-fast specialized versions of AppendNumber. +void AppendDigit(TBaseFormatter* out, ui32 value) +{ + out->AppendChar('0' + value); +} + +void AppendNumber2(TBaseFormatter* out, ui32 value) +{ + AppendDigit(out, value / 10); + AppendDigit(out, value % 10); +} + +void AppendNumber3(TBaseFormatter* out, ui32 value) +{ + AppendDigit(out, value / 100); + AppendDigit(out, (value / 10) % 10); + AppendDigit(out, value % 10); +} + +void AppendNumber4(TBaseFormatter* out, ui32 value) +{ + AppendDigit(out, value / 1000); + AppendDigit(out, (value / 100) % 10); + AppendDigit(out, (value / 10) % 10); + AppendDigit(out, value % 10); +} + +void AppendNumber6(TBaseFormatter* out, ui32 value) +{ + AppendDigit(out, value / 100000); + AppendDigit(out, (value / 10000) % 10); + AppendDigit(out, (value / 1000) % 10); + AppendDigit(out, (value / 100) % 10); + AppendDigit(out, (value / 10) % 10); + AppendDigit(out, value % 10); +} + +} // namespace + +void FormatDateTime(TBaseFormatter* out, TInstant dateTime) +{ + tm localTime; + dateTime.LocalTime(&localTime); + AppendNumber4(out, localTime.tm_year + 1900); + out->AppendChar('-'); + AppendNumber2(out, localTime.tm_mon + 1); + out->AppendChar('-'); + AppendNumber2(out, localTime.tm_mday); + out->AppendChar(' '); + AppendNumber2(out, localTime.tm_hour); + out->AppendChar(':'); + AppendNumber2(out, localTime.tm_min); + out->AppendChar(':'); + AppendNumber2(out, localTime.tm_sec); +} + +void FormatMilliseconds(TBaseFormatter* out, TInstant dateTime) +{ + AppendNumber3(out, dateTime.MilliSecondsOfSecond()); +} + +void FormatMicroseconds(TBaseFormatter* out, TInstant dateTime) +{ + AppendNumber6(out, dateTime.MicroSecondsOfSecond()); +} + +void FormatLevel(TBaseFormatter* out, ELogLevel level) +{ + static char chars[] = "?TDIWEAF?"; + out->AppendChar(chars[static_cast<int>(level)]); +} + +void FormatMessage(TBaseFormatter* out, TStringBuf message) +{ + auto current = message.begin(); + +#ifdef YT_USE_SSE42 + auto vectorLow = _mm_set1_epi8(PrintableASCIILow); + auto vectorHigh = _mm_set1_epi8(PrintableASCIIHigh); +#endif + + auto appendChar = [&] { + char ch = *current; + if (ch == '\n') { + out->AppendString("\\n"); + } else if (ch == '\t') { + out->AppendString("\\t"); + } else if (ch < PrintableASCIILow || ch > PrintableASCIIHigh) { + unsigned char unsignedCh = ch; + out->AppendString("\\x"); + out->AppendChar(IntToHexLowercase[unsignedCh >> 4]); + out->AppendChar(IntToHexLowercase[unsignedCh & 15]); + } else { + out->AppendChar(ch); + } + ++current; + }; + + while (current < message.end()) { + if (out->GetBytesRemaining() < MessageBufferWatermarkSize) { + out->AppendString(TStringBuf("...<message truncated>")); + break; + } +#ifdef YT_USE_SSE42 + // Use SSE for optimization. + if (current + 16 > message.end()) { + appendChar(); + } else { + const void* inPtr = &(*current); + void* outPtr = out->GetCursor(); + auto value = _mm_lddqu_si128(static_cast<const __m128i*>(inPtr)); + if (_mm_movemask_epi8(_mm_cmplt_epi8(value, vectorLow)) || + _mm_movemask_epi8(_mm_cmpgt_epi8(value, vectorHigh))) { + for (int index = 0; index < 16; ++index) { + appendChar(); + } + } else { + _mm_storeu_si128(static_cast<__m128i*>(outPtr), value); + out->Advance(16); + current += 16; + } + } +#else + // Unoptimized version. + appendChar(); +#endif + } +} + +//////////////////////////////////////////////////////////////////////////////// + +void TCachingDateFormatter::Format(TBaseFormatter* buffer, TInstant dateTime, bool printMicroseconds) +{ + auto currentSecond = dateTime.Seconds(); + if (CachedSecond_ != currentSecond) { + Cached_.Reset(); + FormatDateTime(&Cached_, dateTime); + CachedSecond_ = currentSecond; + } + + buffer->AppendString(Cached_.GetBuffer()); + buffer->AppendChar(','); + if (printMicroseconds) { + FormatMicroseconds(buffer, dateTime); + } else { + FormatMilliseconds(buffer, dateTime); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TPlainTextEventFormatter::TPlainTextEventFormatter(bool enableSourceLocation) + : EnableSourceLocation_(enableSourceLocation) +{ } + +void TPlainTextEventFormatter::Format(TBaseFormatter* buffer, const TLogEvent& event) +{ + CachingDateFormatter_.Format(buffer, CpuInstantToInstant(event.Instant), true); + + buffer->AppendChar('\t'); + + FormatLevel(buffer, event.Level); + + buffer->AppendChar('\t'); + + buffer->AppendString(event.Category->Name); + + buffer->AppendChar('\t'); + + FormatMessage(buffer, event.MessageRef.ToStringBuf()); + + buffer->AppendChar('\t'); + + if (event.ThreadName.Length > 0) { + buffer->AppendString(TStringBuf(event.ThreadName.Buffer.data(), event.ThreadName.Length)); + } else if (event.ThreadId != TThreadId()) { + buffer->AppendNumber(event.ThreadId, 16); + } + + buffer->AppendChar('\t'); + + if (event.FiberId != TFiberId()) { + buffer->AppendNumber(event.FiberId, 16); + } + + buffer->AppendChar('\t'); + + if (event.TraceId != TTraceId()) { + buffer->AppendGuid(event.TraceId); + } + + if (EnableSourceLocation_) { + buffer->AppendChar('\t'); + if (event.SourceFile) { + auto sourceFile = event.SourceFile; + buffer->AppendString(sourceFile.RNextTok(LOCSLASH_C)); + buffer->AppendChar(':'); + buffer->AppendNumber(event.SourceLine); + } + } + + buffer->AppendChar('\n'); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/plain_text_formatter/formatter.h b/library/cpp/yt/logging/plain_text_formatter/formatter.h new file mode 100644 index 00000000000..1c35c7c5ee0 --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/formatter.h @@ -0,0 +1,48 @@ +#pragma once + +#include <library/cpp/yt/string/raw_formatter.h> + +#include <library/cpp/yt/logging//logger.h> + +namespace NYT::NLogging { + +/////////////////////////////////////////////////////////////π/////////////////// + +constexpr int DateTimeBufferSize = 64; +constexpr int MessageBufferSize = 64_KB; + +void FormatMilliseconds(TBaseFormatter* out, TInstant dateTime); +void FormatMicroseconds(TBaseFormatter* out, TInstant dateTime); +void FormatLevel(TBaseFormatter* out, ELogLevel level); +void FormatMessage(TBaseFormatter* out, TStringBuf message); + +/////////////////////////////////////////////////////////////π/////////////////// + +class TCachingDateFormatter +{ +public: + void Format(TBaseFormatter* buffer, TInstant dateTime, bool printMicroseconds = false); + +private: + ui64 CachedSecond_ = 0; + TRawFormatter<DateTimeBufferSize> Cached_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TPlainTextEventFormatter +{ +public: + explicit TPlainTextEventFormatter(bool enableSourceLocation); + + void Format(TBaseFormatter* buffer, const TLogEvent& event); + +private: + const bool EnableSourceLocation_; + + TCachingDateFormatter CachingDateFormatter_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/plain_text_formatter/ya.make b/library/cpp/yt/logging/plain_text_formatter/ya.make new file mode 100644 index 00000000000..cb31c29d110 --- /dev/null +++ b/library/cpp/yt/logging/plain_text_formatter/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + formatter.cpp +) + +PEERDIR( + library/cpp/yt/cpu_clock + library/cpp/yt/logging + library/cpp/yt/string + library/cpp/yt/misc +) + +END() diff --git a/library/cpp/yt/logging/public.h b/library/cpp/yt/logging/public.h new file mode 100644 index 00000000000..1e2b59ca0d3 --- /dev/null +++ b/library/cpp/yt/logging/public.h @@ -0,0 +1,39 @@ +#pragma once + +#include <library/cpp/yt/misc/enum.h> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +// Any change to this enum must be also propagated to FormatLevel. +DEFINE_ENUM(ELogLevel, + (Minimum) + (Trace) + (Debug) + (Info) + (Warning) + (Error) + (Alert) + (Fatal) + (Maximum) +); + +DEFINE_ENUM(ELogFamily, + (PlainText) + (Structured) +); + +//////////////////////////////////////////////////////////////////////////////// + +struct TLoggingCategory; +struct TLoggingAnchor; +struct TLogEvent; +struct TLoggingContext; + +class TLogger; +struct ILogManager; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/unittests/logger_ut.cpp b/library/cpp/yt/logging/unittests/logger_ut.cpp new file mode 100644 index 00000000000..7696ea4a83b --- /dev/null +++ b/library/cpp/yt/logging/unittests/logger_ut.cpp @@ -0,0 +1,38 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/logging/logger.h> + +namespace NYT::NLogging { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TLogger, NullByDefault) +{ + { + TLogger logger; + EXPECT_FALSE(logger); + EXPECT_FALSE(logger.IsLevelEnabled(ELogLevel::Fatal)); + } + { + TLogger logger{"Category"}; + EXPECT_FALSE(logger); + EXPECT_FALSE(logger.IsLevelEnabled(ELogLevel::Fatal)); + } +} + +TEST(TLogger, CopyOfNullLogger) +{ + TLogger nullLogger{/*logManager*/ nullptr, "Category"}; + ASSERT_FALSE(nullLogger); + + auto logger = nullLogger.WithMinLevel(ELogLevel::Debug); + + EXPECT_FALSE(logger); + EXPECT_FALSE(logger.IsLevelEnabled(ELogLevel::Fatal)); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/unittests/ya.make b/library/cpp/yt/logging/unittests/ya.make new file mode 100644 index 00000000000..42268d3db2f --- /dev/null +++ b/library/cpp/yt/logging/unittests/ya.make @@ -0,0 +1,18 @@ +GTEST(unittester-library-logging) + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +IF (NOT OS_WINDOWS) + ALLOCATOR(YT) +ENDIF() + +SRCS( + logger_ut.cpp +) + +PEERDIR( + library/cpp/testing/gtest + library/cpp/yt/logging +) + +END() diff --git a/library/cpp/yt/logging/ya.make b/library/cpp/yt/logging/ya.make new file mode 100644 index 00000000000..e611c2e554c --- /dev/null +++ b/library/cpp/yt/logging/ya.make @@ -0,0 +1,25 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/library/cpp/yt/ya_cpp.make.inc) + +SRCS( + logger.cpp +) + +PEERDIR( + library/cpp/yt/assert + library/cpp/yt/memory + library/cpp/yt/misc + library/cpp/yt/yson_string +) + +END() + +RECURSE( + backends + plain_text_formatter +) + +RECURSE_FOR_TESTS( + unittests +) |