diff options
author | Devtools Arcadia <arcadia-devtools@yandex-team.ru> | 2022-02-07 18:08:42 +0300 |
---|---|---|
committer | Devtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net> | 2022-02-07 18:08:42 +0300 |
commit | 1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch) | |
tree | e26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/cpp/logger | |
download | ydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz |
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/cpp/logger')
70 files changed, 3099 insertions, 0 deletions
diff --git a/library/cpp/logger/all.h b/library/cpp/logger/all.h new file mode 100644 index 0000000000..ee1666844e --- /dev/null +++ b/library/cpp/logger/all.h @@ -0,0 +1,9 @@ +#pragma once + +#include "log.h" +#include "null.h" +#include "file.h" +#include "stream.h" +#include "thread.h" +#include "system.h" +#include "sync_page_cache_file.h" diff --git a/library/cpp/logger/backend.cpp b/library/cpp/logger/backend.cpp new file mode 100644 index 0000000000..b26bf5e88c --- /dev/null +++ b/library/cpp/logger/backend.cpp @@ -0,0 +1,71 @@ +#include "backend.h" +#include <util/generic/vector.h> +#include <util/system/mutex.h> +#include <util/generic/singleton.h> +#include <util/generic/yexception.h> + +namespace { + class TGlobalLogsStorage { + private: + TVector<TLogBackend*> Backends; + TMutex Mutex; + + public: + void Register(TLogBackend* backend) { + TGuard<TMutex> g(Mutex); + Backends.push_back(backend); + } + + void UnRegister(TLogBackend* backend) { + TGuard<TMutex> g(Mutex); + for (ui32 i = 0; i < Backends.size(); ++i) { + if (Backends[i] == backend) { + Backends.erase(Backends.begin() + i); + return; + } + } + Y_FAIL("Incorrect pointer for log backend"); + } + + void Reopen(bool flush) { + TGuard<TMutex> g(Mutex); + for (auto& b : Backends) { + if (flush) { + b->ReopenLog(); + } else { + b->ReopenLogNoFlush(); + } + } + } + }; +} + +template <> +class TSingletonTraits<TGlobalLogsStorage> { +public: + static const size_t Priority = 50; +}; + +ELogPriority TLogBackend::FiltrationLevel() const { + return LOG_MAX_PRIORITY; +} + +TLogBackend::TLogBackend() noexcept { + Singleton<TGlobalLogsStorage>()->Register(this); +} + +TLogBackend::~TLogBackend() { + Singleton<TGlobalLogsStorage>()->UnRegister(this); +} + +void TLogBackend::ReopenLogNoFlush() { + ReopenLog(); +} + +void TLogBackend::ReopenAllBackends(bool flush) { + Singleton<TGlobalLogsStorage>()->Reopen(flush); +} + +size_t TLogBackend::QueueSize() const { + ythrow yexception() << "Not implemented."; +} diff --git a/library/cpp/logger/backend.h b/library/cpp/logger/backend.h new file mode 100644 index 0000000000..d088726d6d --- /dev/null +++ b/library/cpp/logger/backend.h @@ -0,0 +1,30 @@ +#pragma once + +#include "priority.h" + +#include <util/generic/noncopyable.h> + +#include <cstddef> + +struct TLogRecord; + +// NOTE: be aware that all `TLogBackend`s are registred in singleton. +class TLogBackend: public TNonCopyable { +public: + TLogBackend() noexcept; + virtual ~TLogBackend(); + + virtual void WriteData(const TLogRecord& rec) = 0; + virtual void ReopenLog() = 0; + + // Does not guarantee consistency with previous WriteData() calls: + // log entries could be written to the new (reopened) log file due to + // buffering effects. + virtual void ReopenLogNoFlush(); + + virtual ELogPriority FiltrationLevel() const; + + static void ReopenAllBackends(bool flush = true); + + virtual size_t QueueSize() const; +}; diff --git a/library/cpp/logger/backend_creator.cpp b/library/cpp/logger/backend_creator.cpp new file mode 100644 index 0000000000..ea430edb83 --- /dev/null +++ b/library/cpp/logger/backend_creator.cpp @@ -0,0 +1,45 @@ +#include "backend_creator.h" +#include "stream.h" +#include "uninitialized_creator.h" +#include <util/system/yassert.h> +#include <util/stream/debug.h> +#include <util/stream/output.h> + + +THolder<TLogBackend> ILogBackendCreator::CreateLogBackend() const { + try { + return DoCreateLogBackend(); + } catch(...) { + Cdbg << "Warning: " << CurrentExceptionMessage() << ". Use stderr instead." << Endl; + } + return MakeHolder<TStreamLogBackend>(&Cerr); +} + +bool ILogBackendCreator::Init(const IInitContext& /*ctx*/) { + return true; +} + + +NJson::TJsonValue ILogBackendCreator::AsJson() const { + NJson::TJsonValue json; + ToJson(json); + return json; +} + +THolder<ILogBackendCreator> ILogBackendCreator::Create(const IInitContext& ctx) { + auto res = MakeHolder<TLogBackendCreatorUninitialized>(); + if(!res->Init(ctx)) { + Cdbg << "Cannot init log backend creator"; + return nullptr; + } + return res; +} + +TLogBackendCreatorBase::TLogBackendCreatorBase(const TString& type) + : Type(type) +{} + +void TLogBackendCreatorBase::ToJson(NJson::TJsonValue& value) const { + value["LoggerType"] = Type; + DoToJson(value); +} diff --git a/library/cpp/logger/backend_creator.h b/library/cpp/logger/backend_creator.h new file mode 100644 index 0000000000..4f0ca24a4e --- /dev/null +++ b/library/cpp/logger/backend_creator.h @@ -0,0 +1,59 @@ +#pragma once + +#include "backend.h" +#include <library/cpp/object_factory/object_factory.h> +#include <library/cpp/json/json_value.h> +#include <util/generic/vector.h> +#include <util/generic/ptr.h> +#include <util/string/cast.h> + +class ILogBackendCreator { +public: + using TFactory = NObjectFactory::TObjectFactory<ILogBackendCreator, TString>; + + class IInitContext { + public: + template<class T> + bool GetValue(TStringBuf name, T& var) const { + TString tmp; + if (!GetValue(name, tmp)) { + return false; + } + var = FromString<T>(tmp); + return true; + } + + template<class T> + T GetOrElse(TStringBuf name, const T& def) const { + T res; + return GetValue(name, res) ? res : def; + } + + virtual ~IInitContext() = default; + virtual bool GetValue(TStringBuf name, TString& var) const = 0; + virtual TVector<THolder<IInitContext>> GetChildren(TStringBuf name) const = 0; + }; + +public: + virtual ~ILogBackendCreator() = default; + THolder<TLogBackend> CreateLogBackend() const; + virtual bool Init(const IInitContext& ctx); + + NJson::TJsonValue AsJson() const; + virtual void ToJson(NJson::TJsonValue& value) const = 0; + + static THolder<ILogBackendCreator> Create(const IInitContext& ctx); + +private: + virtual THolder<TLogBackend> DoCreateLogBackend() const = 0; +}; + +class TLogBackendCreatorBase: public ILogBackendCreator { +public: + TLogBackendCreatorBase(const TString& type); + virtual void ToJson(NJson::TJsonValue& value) const override final; + +protected: + virtual void DoToJson(NJson::TJsonValue& value) const = 0; + TString Type; +}; diff --git a/library/cpp/logger/composite.cpp b/library/cpp/logger/composite.cpp new file mode 100644 index 0000000000..3ce0154ced --- /dev/null +++ b/library/cpp/logger/composite.cpp @@ -0,0 +1,18 @@ +#include "composite.h" +#include "uninitialized_creator.h" + +void TCompositeLogBackend::WriteData(const TLogRecord& rec) { + for (auto& slave: Slaves) { + slave->WriteData(rec); + } +} + +void TCompositeLogBackend::ReopenLog() { + for (auto& slave : Slaves) { + slave->ReopenLog(); + } +} + +void TCompositeLogBackend::AddLogBackend(THolder<TLogBackend>&& backend) { + Slaves.emplace_back(std::move(backend)); +} diff --git a/library/cpp/logger/composite.h b/library/cpp/logger/composite.h new file mode 100644 index 0000000000..142c61b8d5 --- /dev/null +++ b/library/cpp/logger/composite.h @@ -0,0 +1,14 @@ +#pragma once + +#include "backend.h" +#include <util/generic/vector.h> + +class TCompositeLogBackend: public TLogBackend { +public: + virtual void WriteData(const TLogRecord& rec) override; + virtual void ReopenLog() override; + virtual void AddLogBackend(THolder<TLogBackend>&& backend); + +private: + TVector<THolder<TLogBackend>> Slaves; +}; diff --git a/library/cpp/logger/composite_creator.cpp b/library/cpp/logger/composite_creator.cpp new file mode 100644 index 0000000000..2064265766 --- /dev/null +++ b/library/cpp/logger/composite_creator.cpp @@ -0,0 +1,34 @@ +#include "composite_creator.h" +#include "composite.h" +#include "uninitialized_creator.h" + +THolder<TLogBackend> TCompositeBackendCreator::DoCreateLogBackend() const { + auto res = MakeHolder<TCompositeLogBackend>(); + for (const auto& child : Children) { + res->AddLogBackend(child->CreateLogBackend()); + } + return std::move(res); +} + + +TCompositeBackendCreator::TCompositeBackendCreator() + : TLogBackendCreatorBase("composite") +{} + +bool TCompositeBackendCreator::Init(const IInitContext& ctx) { + for (const auto& child : ctx.GetChildren("SubLogger")) { + Children.emplace_back(MakeHolder<TLogBackendCreatorUninitialized>()); + if (!Children.back()->Init(*child)) { + return false; + } + } + return true; +} + +ILogBackendCreator::TFactory::TRegistrator<TCompositeBackendCreator> TCompositeBackendCreator::Registrar("composite"); + +void TCompositeBackendCreator::DoToJson(NJson::TJsonValue& value) const { + for (const auto& child: Children) { + child->ToJson(value["SubLogger"].AppendValue(NJson::JSON_MAP)); + } +} diff --git a/library/cpp/logger/composite_creator.h b/library/cpp/logger/composite_creator.h new file mode 100644 index 0000000000..877d3a2b33 --- /dev/null +++ b/library/cpp/logger/composite_creator.h @@ -0,0 +1,18 @@ +#pragma once + +#include "backend_creator.h" +#include <util/generic/vector.h> + +class TCompositeBackendCreator : public TLogBackendCreatorBase { +public: + TCompositeBackendCreator(); + virtual bool Init(const IInitContext& ctx) override; + static TFactory::TRegistrator<TCompositeBackendCreator> Registrar; + +protected: + virtual void DoToJson(NJson::TJsonValue& value) const override; + +private: + virtual THolder<TLogBackend> DoCreateLogBackend() const override; + TVector<THolder<ILogBackendCreator>> Children; +}; diff --git a/library/cpp/logger/composite_ut.cpp b/library/cpp/logger/composite_ut.cpp new file mode 100644 index 0000000000..f330419271 --- /dev/null +++ b/library/cpp/logger/composite_ut.cpp @@ -0,0 +1,98 @@ +#include "log.h" +#include <library/cpp/logger/init_context/config.h> +#include <library/cpp/logger/init_context/yconf.h> +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/yconf/patcher/unstrict_config.h> +#include <util/stream/file.h> +#include <util/system/fs.h> + +Y_UNIT_TEST_SUITE(TCompositeLogTest) +{ + TVector<TString> ReadLines(const TString & filename) { + TVector<TString> lines; + TIFStream fin(filename); + TString line; + while (fin.ReadLine(line)) { + lines.push_back(std::move(line)); + } + return lines; + } + + void Clear(const TString & filename) { + NFs::Remove(filename + "1"); + NFs::Remove(filename + "2"); + } + + void DoTestComposite(const ILogBackendCreator::IInitContext& ctx, const TString & filename) { + Clear(filename); + { + TLog log; + { + auto creator = ILogBackendCreator::Create(ctx); + log.ResetBackend(creator->CreateLogBackend()); + log.ReopenLog(); + } + log.Write(TLOG_ERR, "first\n"); + log.Write(TLOG_DEBUG, "second\n"); + } + auto data1 = ReadLines(filename + "1"); + auto data2 = ReadLines(filename + "2"); + UNIT_ASSERT_VALUES_EQUAL(data1.size(), 2); + UNIT_ASSERT(data1[0] == "first"); + UNIT_ASSERT(data1[1] == "second"); + + UNIT_ASSERT_VALUES_EQUAL(data2.size(), 1); + UNIT_ASSERT(data2[0] == "first"); + Clear(filename); + } + + Y_UNIT_TEST(TestCompositeConfig) { + TString s(R"( +{ + "LoggerType": "composite", + "SubLogger":[ + { + "LoggerType": "file", + "Path": "config_log_1" + }, { + "LoggerType": "config_log_2", + "LogLevel": "INFO" + } + ] +})"); + TStringInput si(s); + NConfig::TConfig cfg = NConfig::TConfig::FromJson(si); + //Прогоняем конфигурацию через серализацию и десериализацию + TLogBackendCreatorInitContextConfig ctx(cfg); + TString newCfg = ILogBackendCreator::Create(ctx)->AsJson().GetStringRobust(); + TStringInput si2(newCfg); + DoTestComposite(TLogBackendCreatorInitContextConfig(NConfig::TConfig::FromJson(si2)), "config_log_"); + + } + Y_UNIT_TEST(TestCompositeYConf) { + constexpr const char* CONFIG = R"( +<Logger> + LoggerType: composite + <SubLogger> + LoggerType: file + Path: yconf_log_1 + </SubLogger> + <SubLogger> + LoggerType: yconf_log_2 + LogLevel: INFO + </SubLogger> +</Logger> +)"; + TUnstrictConfig cfg; + if (!cfg.ParseMemory(CONFIG)) { + TString errors; + cfg.PrintErrors(errors); + UNIT_ASSERT_C(false, errors); + } + TLogBackendCreatorInitContextYConf ctx(*cfg.GetFirstChild("Logger")); + //Прогоняем конфигурацию через серализацию и десериализацию + TUnstrictConfig newCfg; + UNIT_ASSERT(newCfg.ParseJson(ILogBackendCreator::Create(ctx)->AsJson())); + DoTestComposite(TLogBackendCreatorInitContextYConf(*newCfg.GetRootSection()), "yconf_log_"); + } +}; diff --git a/library/cpp/logger/element.cpp b/library/cpp/logger/element.cpp new file mode 100644 index 0000000000..b510fe16e1 --- /dev/null +++ b/library/cpp/logger/element.cpp @@ -0,0 +1,38 @@ +#include "log.h" +#include "element.h" + +#include <utility> + +TLogElement::TLogElement(const TLog* parent) + : Parent_(parent) + , Priority_(Parent_->DefaultPriority()) +{ + Reset(); +} + +TLogElement::TLogElement(const TLog* parent, ELogPriority priority) + : Parent_(parent) + , Priority_(priority) +{ + Reset(); +} + +TLogElement::~TLogElement() { + try { + Finish(); + } catch (...) { + } +} + +void TLogElement::DoFlush() { + if (IsNull()) { + return; + } + + const size_t filled = Filled(); + + if (filled) { + Parent_->Write(Priority_, Data(), filled); + Reset(); + } +} diff --git a/library/cpp/logger/element.h b/library/cpp/logger/element.h new file mode 100644 index 0000000000..fc9bff851f --- /dev/null +++ b/library/cpp/logger/element.h @@ -0,0 +1,48 @@ +#pragma once + +#include "priority.h" + +#include <util/stream/tempbuf.h> + +class TLog; + +/* + * better do not use directly + */ +class TLogElement: public TTempBufOutput { +public: + TLogElement(const TLog* parent); + TLogElement(const TLog* parent, ELogPriority priority); + + TLogElement(TLogElement&&) noexcept = default; + TLogElement& operator=(TLogElement&&) noexcept = default; + + ~TLogElement() override; + + template <class T> + inline TLogElement& operator<<(const T& t) { + static_cast<IOutputStream&>(*this) << t; + + return *this; + } + + /* + * for pretty usage: logger << TLOG_ERROR << "Error description"; + */ + inline TLogElement& operator<<(ELogPriority priority) { + Flush(); + Priority_ = priority; + return *this; + } + + ELogPriority Priority() const noexcept { + return Priority_; + } + +protected: + void DoFlush() override; + +protected: + const TLog* Parent_; + ELogPriority Priority_; +}; diff --git a/library/cpp/logger/element_ut.cpp b/library/cpp/logger/element_ut.cpp new file mode 100644 index 0000000000..32edc52dfb --- /dev/null +++ b/library/cpp/logger/element_ut.cpp @@ -0,0 +1,39 @@ +#include "log.h" +#include "element.h" +#include "stream.h" + +#include <util/generic/string.h> +#include <util/stream/str.h> +#include <util/generic/ptr.h> +#include <utility> + +#include <library/cpp/testing/unittest/registar.h> + +class TLogElementTest: public TTestBase { + UNIT_TEST_SUITE(TLogElementTest); + UNIT_TEST(TestMoveCtor); + UNIT_TEST_SUITE_END(); + +public: + void TestMoveCtor(); +}; + +UNIT_TEST_SUITE_REGISTRATION(TLogElementTest); + +void TLogElementTest::TestMoveCtor() { + TStringStream output; + TLog log(MakeHolder<TStreamLogBackend>(&output)); + + THolder<TLogElement> src = MakeHolder<TLogElement>(&log); + + TString message = "Hello, World!"; + (*src) << message; + + THolder<TLogElement> dst = MakeHolder<TLogElement>(std::move(*src)); + + src.Destroy(); + UNIT_ASSERT(output.Str() == ""); + + dst.Destroy(); + UNIT_ASSERT(output.Str() == message); +} diff --git a/library/cpp/logger/file.cpp b/library/cpp/logger/file.cpp new file mode 100644 index 0000000000..15a4946eda --- /dev/null +++ b/library/cpp/logger/file.cpp @@ -0,0 +1,59 @@ +#include "file.h" +#include "record.h" + +#include <util/system/file.h> +#include <util/system/rwlock.h> +#include <util/generic/string.h> + +/* + * file log + */ +class TFileLogBackend::TImpl { +public: + inline TImpl(const TString& path) + : File_(OpenFile(path)) + { + } + + inline void WriteData(const TLogRecord& rec) { + //many writes are thread-safe + TReadGuard guard(Lock_); + + File_.Write(rec.Data, rec.Len); + } + + inline void ReopenLog() { + //but log rotate not thread-safe + TWriteGuard guard(Lock_); + + File_.LinkTo(OpenFile(File_.GetName())); + } + +private: + static inline TFile OpenFile(const TString& path) { + return TFile(path, OpenAlways | WrOnly | ForAppend | Seq | NoReuse); + } + +private: + TRWMutex Lock_; + TFile File_; +}; + +TFileLogBackend::TFileLogBackend(const TString& path) + : Impl_(new TImpl(path)) +{ +} + +TFileLogBackend::~TFileLogBackend() { +} + +void TFileLogBackend::WriteData(const TLogRecord& rec) { + Impl_->WriteData(rec); +} + +void TFileLogBackend::ReopenLog() { + TAtomicSharedPtr<TImpl> copy = Impl_; + if (copy) { + copy->ReopenLog(); + } +} diff --git a/library/cpp/logger/file.h b/library/cpp/logger/file.h new file mode 100644 index 0000000000..10b4cd0c20 --- /dev/null +++ b/library/cpp/logger/file.h @@ -0,0 +1,19 @@ +#pragma once + +#include "backend.h" + +#include <util/generic/fwd.h> +#include <util/generic/ptr.h> + +class TFileLogBackend: public TLogBackend { +public: + TFileLogBackend(const TString& path); + ~TFileLogBackend() override; + + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override; + +private: + class TImpl; + TAtomicSharedPtr<TImpl> Impl_; +}; diff --git a/library/cpp/logger/file_creator.cpp b/library/cpp/logger/file_creator.cpp new file mode 100644 index 0000000000..0a84469e43 --- /dev/null +++ b/library/cpp/logger/file_creator.cpp @@ -0,0 +1,22 @@ +#include "file_creator.h" +#include "file.h" + +TFileLogBackendCreator::TFileLogBackendCreator(const TString& path /*= TString()*/, const TString& type /*= "file"*/) + : TLogBackendCreatorBase(type) + , Path(path) +{} + +THolder<TLogBackend> TFileLogBackendCreator::DoCreateLogBackend() const { + return MakeHolder<TFileLogBackend>(Path); +} + +bool TFileLogBackendCreator::Init(const IInitContext& ctx) { + ctx.GetValue("Path", Path); + return !!Path; +} + +ILogBackendCreator::TFactory::TRegistrator<TFileLogBackendCreator> TFileLogBackendCreator::Registrar("file"); + +void TFileLogBackendCreator::DoToJson(NJson::TJsonValue& value) const { + value["Path"] = Path; +} diff --git a/library/cpp/logger/file_creator.h b/library/cpp/logger/file_creator.h new file mode 100644 index 0000000000..73e55261ad --- /dev/null +++ b/library/cpp/logger/file_creator.h @@ -0,0 +1,17 @@ +#pragma once + +#include "backend_creator.h" + +class TFileLogBackendCreator : public TLogBackendCreatorBase { +public: + TFileLogBackendCreator(const TString& path = TString(), const TString& type = "file"); + virtual bool Init(const IInitContext& ctx) override; + static TFactory::TRegistrator<TFileLogBackendCreator> Registrar; + +protected: + virtual void DoToJson(NJson::TJsonValue& value) const override; + TString Path; + +private: + virtual THolder<TLogBackend> DoCreateLogBackend() const override; +}; diff --git a/library/cpp/logger/filter.cpp b/library/cpp/logger/filter.cpp new file mode 100644 index 0000000000..300ac6b595 --- /dev/null +++ b/library/cpp/logger/filter.cpp @@ -0,0 +1 @@ +#include "filter.h" diff --git a/library/cpp/logger/filter.h b/library/cpp/logger/filter.h new file mode 100644 index 0000000000..9ef83fb58c --- /dev/null +++ b/library/cpp/logger/filter.h @@ -0,0 +1,32 @@ +#pragma once + +#include "priority.h" +#include "record.h" +#include "backend.h" +#include <util/generic/ptr.h> + +class TFilteredLogBackend: public TLogBackend { + THolder<TLogBackend> Backend; + ELogPriority Level; + +public: + TFilteredLogBackend(THolder<TLogBackend>&& t, ELogPriority level = LOG_MAX_PRIORITY) noexcept + : Backend(std::move(t)) + , Level(level) + { + } + + ELogPriority FiltrationLevel() const override { + return Level; + } + + void ReopenLog() override { + Backend->ReopenLog(); + } + + void WriteData(const TLogRecord& rec) override { + if (rec.Priority <= (ELogPriority)Level) { + Backend->WriteData(rec); + } + } +}; diff --git a/library/cpp/logger/filter_creator.cpp b/library/cpp/logger/filter_creator.cpp new file mode 100644 index 0000000000..fd5618087b --- /dev/null +++ b/library/cpp/logger/filter_creator.cpp @@ -0,0 +1,20 @@ +#include "filter_creator.h" +#include "filter.h" + +TFilteredBackendCreator::TFilteredBackendCreator(THolder<ILogBackendCreator> slave, ELogPriority priority) + : Slave(std::move(slave)) + , Priority(priority) +{} + +THolder<TLogBackend> TFilteredBackendCreator::DoCreateLogBackend() const { + return MakeHolder<TFilteredLogBackend>(Slave->CreateLogBackend(), Priority); +} + +bool TFilteredBackendCreator::Init(const IInitContext& ctx) { + return Slave->Init(ctx); +} + +void TFilteredBackendCreator::ToJson(NJson::TJsonValue& value) const { + value["LogLevel"] = ToString(Priority); + Slave->ToJson(value); +} diff --git a/library/cpp/logger/filter_creator.h b/library/cpp/logger/filter_creator.h new file mode 100644 index 0000000000..0f301457a0 --- /dev/null +++ b/library/cpp/logger/filter_creator.h @@ -0,0 +1,16 @@ +#pragma once + +#include "backend_creator.h" +#include "priority.h" + +class TFilteredBackendCreator : public ILogBackendCreator { +public: + TFilteredBackendCreator(THolder<ILogBackendCreator> slave, ELogPriority priority); + virtual bool Init(const IInitContext& ctx) override; + virtual void ToJson(NJson::TJsonValue& value) const override; + +private: + virtual THolder<TLogBackend> DoCreateLogBackend() const override; + THolder<ILogBackendCreator> Slave; + ELogPriority Priority; +}; diff --git a/library/cpp/logger/global/common.cpp b/library/cpp/logger/global/common.cpp new file mode 100644 index 0000000000..4fb05c19b4 --- /dev/null +++ b/library/cpp/logger/global/common.cpp @@ -0,0 +1,36 @@ +#include "common.h" + +#include <util/generic/yexception.h> + +namespace NLoggingImpl { + TString GetLocalTimeSSimple() { + struct tm tm; + return Strftime("%b%d_%H%M%S", Now().LocalTime(&tm)); + } + + TString PrepareToOpenLog(TString logType, const int logLevel, const bool rotation, const bool startAsDaemon) { + Y_ENSURE(logLevel >= 0 && logLevel <= (int)LOG_MAX_PRIORITY, "Incorrect log level"); + + if (rotation && TFsPath(logType).Exists()) { + TString newPath = Sprintf("%s_%s_%" PRIu64, logType.data(), NLoggingImpl::GetLocalTimeSSimple().data(), static_cast<ui64>(Now().MicroSeconds())); + TFsPath(logType).RenameTo(newPath); + } + if (startAsDaemon && (logType == "console"sv || logType == "cout"sv || logType == "cerr"sv)) { + logType = "null"; + } + + return logType; + } +} + +bool TLogFilter::CheckLoggingContext(TLog& log, const TLogRecordContext& context) { + return context.Priority <= log.FiltrationLevel(); +} + +TSimpleSharedPtr<TLogElement> TLogFilter::StartRecord(TLog& logger, const TLogRecordContext& context, TSimpleSharedPtr<TLogElement> earlier) { + if (earlier) + return earlier; + TSimpleSharedPtr<TLogElement> result(new TLogElement(&logger)); + *result << context.Priority; + return result; +} diff --git a/library/cpp/logger/global/common.h b/library/cpp/logger/global/common.h new file mode 100644 index 0000000000..7dcf650dec --- /dev/null +++ b/library/cpp/logger/global/common.h @@ -0,0 +1,149 @@ +#pragma once + +#include <util/datetime/base.h> + +#include <util/folder/path.h> +#include <util/generic/singleton.h> +#include <util/generic/string.h> +#include <util/generic/ptr.h> +#include <util/generic/yexception.h> +#include <util/string/printf.h> +#include <util/system/src_location.h> + +#include <library/cpp/logger/log.h> + +namespace NLoggingImpl { + const size_t SingletonPriority = 500; +} + +template <class T> +T* CreateDefaultLogger() { + return nullptr; +} + +namespace NLoggingImpl { + template<class T, class TTraits> + class TOperatorBase { + struct TPtr { + TPtr() + : Instance(TTraits::CreateDefault()) + { + } + + THolder<T> Instance; + }; + + public: + inline static bool Usage() { + return SingletonWithPriority<TPtr, SingletonPriority>()->Instance.Get(); + } + + inline static T* Get() { + return SingletonWithPriority<TPtr, SingletonPriority>()->Instance.Get(); + } + + inline static void Set(T* v) { + SingletonWithPriority<TPtr, SingletonPriority>()->Instance.Reset(v); + } + }; + + template<class T> + struct TLoggerTraits { + static T* CreateDefault() { + return CreateDefaultLogger<T>(); + } + }; +} + +template <class T> +class TLoggerOperator : public NLoggingImpl::TOperatorBase<T, NLoggingImpl::TLoggerTraits<T>> { +public: + inline static TLog& Log() { + Y_ASSERT(TLoggerOperator::Usage()); + return *TLoggerOperator::Get(); + } +}; + +namespace NLoggingImpl { + + TString GetLocalTimeSSimple(); + + // Returns correct log type to use + TString PrepareToOpenLog(TString logType, int logLevel, bool rotation, bool startAsDaemon); + + template <class TLoggerType> + void InitLogImpl(TString logType, const int logLevel, const bool rotation, const bool startAsDaemon) { + TLoggerOperator<TLoggerType>::Set(new TLoggerType(PrepareToOpenLog(logType, logLevel, rotation, startAsDaemon), (ELogPriority)logLevel)); + } +} + +struct TLogRecordContext { + constexpr TLogRecordContext(const TSourceLocation& sourceLocation, TStringBuf customMessage, ELogPriority priority) + : SourceLocation(sourceLocation) + , CustomMessage(customMessage) + , Priority(priority) + {} + + TSourceLocation SourceLocation; + TStringBuf CustomMessage; + ELogPriority Priority; +}; + +template <class... R> +struct TLogRecordPreprocessor; + +template <> +struct TLogRecordPreprocessor<> { + inline static bool CheckLoggingContext(TLog& /*log*/, const TLogRecordContext& /*context*/) { + return true; + } + + inline static TSimpleSharedPtr<TLogElement> StartRecord(TLog& log, const TLogRecordContext& context, TSimpleSharedPtr<TLogElement> earlier) { + if (earlier) + return earlier; + TSimpleSharedPtr<TLogElement> result(new TLogElement(&log)); + *result << context.Priority; + return result; + } +}; + +template <class H, class... R> +struct TLogRecordPreprocessor<H, R...> { + inline static bool CheckLoggingContext(TLog& log, const TLogRecordContext& context) { + return H::CheckLoggingContext(log, context) && TLogRecordPreprocessor<R...>::CheckLoggingContext(log, context); + } + + inline static TSimpleSharedPtr<TLogElement> StartRecord(TLog& log, const TLogRecordContext& context, TSimpleSharedPtr<TLogElement> earlier) { + TSimpleSharedPtr<TLogElement> first = H::StartRecord(log, context, earlier); + return TLogRecordPreprocessor<R...>::StartRecord(log, context, first); + } +}; + +struct TLogFilter { + static bool CheckLoggingContext(TLog& log, const TLogRecordContext& context); + static TSimpleSharedPtr<TLogElement> StartRecord(TLog& log, const TLogRecordContext& context, TSimpleSharedPtr<TLogElement> earlier); +}; + +class TNullLog; + +template <class TPreprocessor> +TSimpleSharedPtr<TLogElement> GetLoggerForce(TLog& log, const TLogRecordContext& context) { + if (TSimpleSharedPtr<TLogElement> result = TPreprocessor::StartRecord(log, context, nullptr)) + return result; + return new TLogElement(&TLoggerOperator<TNullLog>::Log()); +} + +namespace NPrivateGlobalLogger { + struct TEatStream { + Y_FORCE_INLINE bool operator|(const IOutputStream&) const { + return true; + } + }; +} + +#define LOGGER_GENERIC_LOG_CHECKED(logger, preprocessor, level, message) (*GetLoggerForce<preprocessor>(logger, TLogRecordContext(__LOCATION__, message, level))) +#define LOGGER_CHECKED_GENERIC_LOG(logger, preprocessor, level, message) \ + (preprocessor::CheckLoggingContext(logger, TLogRecordContext(__LOCATION__, message, level))) && NPrivateGlobalLogger::TEatStream() | (*(preprocessor::StartRecord(logger, TLogRecordContext(__LOCATION__, message, level), nullptr))) + +#define SINGLETON_GENERIC_LOG_CHECKED(type, preprocessor, level, message) LOGGER_GENERIC_LOG_CHECKED(TLoggerOperator<type>::Log(), preprocessor, level, message) +#define SINGLETON_CHECKED_GENERIC_LOG(type, preprocessor, level, message) LOGGER_CHECKED_GENERIC_LOG(TLoggerOperator<type>::Log(), preprocessor, level, message) diff --git a/library/cpp/logger/global/global.cpp b/library/cpp/logger/global/global.cpp new file mode 100644 index 0000000000..9fbd10f666 --- /dev/null +++ b/library/cpp/logger/global/global.cpp @@ -0,0 +1,43 @@ +#include "global.h" + +static void DoInitGlobalLog(THolder<TGlobalLog> logger, THolder<ILoggerFormatter> formatter) { + TLoggerOperator<TGlobalLog>::Set(logger.Release()); + if (!formatter) { + formatter.Reset(CreateRtyLoggerFormatter()); + } + TLoggerFormatterOperator::Set(formatter.Release()); +} + +void DoInitGlobalLog(const TString& logType, const int logLevel, const bool rotation, const bool startAsDaemon, THolder<ILoggerFormatter> formatter, bool threaded) { + DoInitGlobalLog( + MakeHolder<TGlobalLog>( + CreateLogBackend( + NLoggingImpl::PrepareToOpenLog(logType, logLevel, rotation, startAsDaemon), + (ELogPriority)logLevel, + threaded)), + std::move(formatter)); +} + +void DoInitGlobalLog(THolder<TLogBackend> backend, THolder<ILoggerFormatter> formatter) { + DoInitGlobalLog(THolder(new TGlobalLog(std::move(backend))), std::move(formatter)); +} + +bool GlobalLogInitialized() { + return TLoggerOperator<TGlobalLog>::Usage(); +} + +template <> +TGlobalLog* CreateDefaultLogger<TGlobalLog>() { + return new TGlobalLog("console", TLOG_INFO); +} + +template <> +TNullLog* CreateDefaultLogger<TNullLog>() { + return new TNullLog("null"); +} + +NPrivateGlobalLogger::TVerifyEvent::~TVerifyEvent() { + const TString info = Str(); + FATAL_LOG << info << Endl; + Y_FAIL("%s", info.data()); +} diff --git a/library/cpp/logger/global/global.h b/library/cpp/logger/global/global.h new file mode 100644 index 0000000000..cbe71b16ea --- /dev/null +++ b/library/cpp/logger/global/global.h @@ -0,0 +1,125 @@ +#pragma once + +#include "common.h" +#include "rty_formater.h" + +// ATTENTION! MUST CALL DoInitGlobalLog BEFORE USAGE + +bool GlobalLogInitialized(); +void DoInitGlobalLog(const TString& logType, const int logLevel, const bool rotation, const bool startAsDaemon, THolder<ILoggerFormatter> formatter = {}, bool threaded = false); +void DoInitGlobalLog(THolder<TLogBackend> backend, THolder<ILoggerFormatter> formatter = {}); + +inline void InitGlobalLog2Null() { + DoInitGlobalLog("null", TLOG_EMERG, false, false); +} + +inline void InitGlobalLog2Console(int loglevel = TLOG_INFO) { + DoInitGlobalLog("console", loglevel, false, false); +} + +class TGlobalLog: public TLog { +public: + TGlobalLog(const TString& logType, ELogPriority priority = LOG_MAX_PRIORITY) + : TLog(logType, priority) + { + } + + TGlobalLog(THolder<TLogBackend> backend) + : TLog(std::move(backend)) + { + } +}; + +template <> +TGlobalLog* CreateDefaultLogger<TGlobalLog>(); + +class TNullLog: public TLog { +public: + TNullLog(const TString& logType, ELogPriority priority = LOG_MAX_PRIORITY) + : TLog(logType, priority) + { + } + + TNullLog(THolder<TLogBackend> backend) + : TLog(std::move(backend)) + { + } +}; + +template <> +TNullLog* CreateDefaultLogger<TNullLog>(); + +template <> +class TSingletonTraits<TLoggerOperator<TGlobalLog>::TPtr> { +public: + static const size_t Priority = NLoggingImpl::SingletonPriority; +}; + +template <> +class TSingletonTraits<TLoggerOperator<TNullLog>::TPtr> { +public: + static const size_t Priority = NLoggingImpl::SingletonPriority; +}; + +#define FATAL_LOG SINGLETON_CHECKED_GENERIC_LOG(TGlobalLog, TRTYLogPreprocessor, TLOG_CRIT, "CRITICAL_INFO") +#define ALERT_LOG SINGLETON_CHECKED_GENERIC_LOG(TGlobalLog, TRTYLogPreprocessor, TLOG_ALERT, "ALERT") +#define ERROR_LOG SINGLETON_CHECKED_GENERIC_LOG(TGlobalLog, TRTYLogPreprocessor, TLOG_ERR, "ERROR") +#define WARNING_LOG SINGLETON_CHECKED_GENERIC_LOG(TGlobalLog, TRTYLogPreprocessor, TLOG_WARNING, "WARNING") +#define NOTICE_LOG SINGLETON_CHECKED_GENERIC_LOG(TGlobalLog, TRTYLogPreprocessor, TLOG_NOTICE, "NOTICE") +#define INFO_LOG SINGLETON_CHECKED_GENERIC_LOG(TGlobalLog, TRTYLogPreprocessor, TLOG_INFO, "INFO") +#define DEBUG_LOG SINGLETON_CHECKED_GENERIC_LOG(TGlobalLog, TRTYLogPreprocessor, TLOG_DEBUG, "DEBUG") +#define RESOURCES_LOG SINGLETON_CHECKED_GENERIC_LOG(TGlobalLog, TRTYLogPreprocessor, TLOG_RESOURCES, "RESOURCES") + +#define TEMPLATE_LOG(logLevel) SINGLETON_CHECKED_GENERIC_LOG(TGlobalLog, TRTYLogPreprocessor, logLevel, ToString(logLevel).data()) + +#define IS_LOG_ACTIVE(logLevel) (TLoggerOperator<TGlobalLog>::Log().FiltrationLevel() >= logLevel) + +#define RTY_MEM_LOG(Action) \ + { NOTICE_LOG << "RESOURCES On " << Action << ": " << NLoggingImpl::GetSystemResources() << Endl; }; + +#define VERIFY_WITH_LOG(expr, msg, ...) \ + do { \ + if (Y_UNLIKELY(!(expr))) { \ + FATAL_LOG << Sprintf(msg, ##__VA_ARGS__) << Endl; \ + Y_VERIFY(false, msg, ##__VA_ARGS__); \ + }; \ + } while (0); + +namespace NPrivateGlobalLogger { + class TVerifyEvent: public TStringStream { + public: + ~TVerifyEvent(); + template <class T> + inline TVerifyEvent& operator<<(const T& t) { + static_cast<IOutputStream&>(*this) << t; + + return *this; + } + }; + class TNullStream: public TStringStream { + public: + ~TNullStream() = default; + + template <class T> + inline TNullStream& operator<<(const T& /*t*/) { + return *this; + } + }; +} + +#define CHECK_WITH_LOG(expr) \ + Y_UNLIKELY(!(expr)) && NPrivateGlobalLogger::TEatStream() | NPrivateGlobalLogger::TVerifyEvent() << __LOCATION__ << ": " << #expr << "(verification failed!): " + +#if !defined(NDEBUG) && !defined(__GCCXML__) +#define ASSERT_WITH_LOG(expr) \ + Y_UNLIKELY(!(expr)) && NPrivateGlobalLogger::TEatStream() | NPrivateGlobalLogger::TVerifyEvent() << __LOCATION__ << ": " << #expr << "(verification failed!): " +#else +#define ASSERT_WITH_LOG(expr) \ + Y_UNLIKELY(false && !(expr)) && NPrivateGlobalLogger::TEatStream() | NPrivateGlobalLogger::TNullStream() +#endif + +#define CHECK_EQ_WITH_LOG(a, b) CHECK_WITH_LOG((a) == (b)) << a << " != " << b; +#define CHECK_LEQ_WITH_LOG(a, b) CHECK_WITH_LOG((a) <= (b)) << a << " > " << b; + +#define FAIL_LOG(msg, ...) VERIFY_WITH_LOG(false, msg, ##__VA_ARGS__) +#define S_FAIL_LOG CHECK_WITH_LOG(false) diff --git a/library/cpp/logger/global/rty_formater.cpp b/library/cpp/logger/global/rty_formater.cpp new file mode 100644 index 0000000000..305f8470c5 --- /dev/null +++ b/library/cpp/logger/global/rty_formater.cpp @@ -0,0 +1,94 @@ +#include "rty_formater.h" +#include <util/datetime/base.h> +#include <util/datetime/systime.h> +#include <util/stream/str.h> +#include <util/stream/printf.h> +#include <util/system/mem_info.h> +#include <util/system/yassert.h> +#include <inttypes.h> +#include <cstdio> + +namespace { + constexpr size_t LocalTimeSBufferSize = sizeof("2017-07-24 12:20:34.313 +0300"); + + size_t PrintLocalTimeS(const TInstant instant, char* const begin, const char* const end) { + Y_VERIFY(static_cast<size_t>(end - begin) >= LocalTimeSBufferSize); + + struct tm tm; + instant.LocalTime(&tm); + + // both stftime and sprintf exclude the terminating null byte from the return value + char* pos = begin; + pos += strftime(pos, end - pos, "%Y-%m-%d %H:%M:%S.", &tm); + pos += sprintf(pos, "%03" PRIu32, instant.MilliSecondsOfSecond()); + pos += strftime(pos, end - pos, " %z", &tm); + Y_VERIFY(LocalTimeSBufferSize - 1 == pos - begin); // together with Y_VERIFY above this also implies pos<=end + return (pos - begin); + } +} + +namespace NLoggingImpl { + IOutputStream& operator<<(IOutputStream& out, TLocalTimeS localTimeS) { + char buffer[LocalTimeSBufferSize]; + size_t len = PrintLocalTimeS(localTimeS.GetInstant(), buffer, buffer + sizeof(buffer)); + out.Write(buffer, len); + return out; + } + + TLocalTimeS::operator TString() const { + TString res; + res.reserve(LocalTimeSBufferSize); + res.ReserveAndResize(PrintLocalTimeS(Instant, res.begin(), res.begin() + res.capacity())); + return res; + } + + TString TLocalTimeS::operator+(const TStringBuf right) const { + TString res(*this); + res += right; + return res; + } + + TStringBuf StripFileName(TStringBuf string) { + return string.RNextTok(LOCSLASH_C); + } + + TString GetSystemResources() { + NMemInfo::TMemInfo mi = NMemInfo::GetMemInfo(); + return PrintSystemResources(mi); + } + + TString PrintSystemResources(const NMemInfo::TMemInfo& mi) { + return Sprintf(" rss=%0.3fMb, vms=%0.3fMb", mi.RSS * 1.0 / (1024 * 1024), mi.VMS * 1.0 / (1024 * 1024)); + } +} + +namespace { + class TRtyLoggerFormatter : public ILoggerFormatter { + public: + void Format(const TLogRecordContext& context, TLogElement& elem) const override { + elem << context.CustomMessage << ": " << NLoggingImpl::GetLocalTimeS() << " " + << NLoggingImpl::StripFileName(context.SourceLocation.File) << ":" << context.SourceLocation.Line; + if (context.Priority > TLOG_RESOURCES && !ExitStarted()) { + elem << NLoggingImpl::GetSystemResources(); + } + elem << " "; + } + }; +} + +ILoggerFormatter* CreateRtyLoggerFormatter() { + return new TRtyLoggerFormatter(); +} + +bool TRTYMessageFormater::CheckLoggingContext(TLog& /*logger*/, const TLogRecordContext& /*context*/) { + return true; +} + +TSimpleSharedPtr<TLogElement> TRTYMessageFormater::StartRecord(TLog& logger, const TLogRecordContext& context, TSimpleSharedPtr<TLogElement> earlier) { + if (!earlier) { + earlier.Reset(new TLogElement(&logger)); + } + + TLoggerFormatterOperator::Get()->Format(context, *earlier); + return earlier; +} diff --git a/library/cpp/logger/global/rty_formater.h b/library/cpp/logger/global/rty_formater.h new file mode 100644 index 0000000000..6532e1d769 --- /dev/null +++ b/library/cpp/logger/global/rty_formater.h @@ -0,0 +1,61 @@ +#pragma once + +#include "common.h" + +namespace NMemInfo { + struct TMemInfo; +} + +class ILoggerFormatter { +public: + virtual ~ILoggerFormatter() = default; + + virtual void Format(const TLogRecordContext&, TLogElement&) const = 0; +}; + +ILoggerFormatter* CreateRtyLoggerFormatter(); + +namespace NLoggingImpl { + class TLocalTimeS { + public: + TLocalTimeS(TInstant instant = TInstant::Now()) + : Instant(instant) + { + } + + TInstant GetInstant() const { + return Instant; + } + + operator TString() const; + TString operator+(TStringBuf right) const; + + private: + TInstant Instant; + }; + + IOutputStream& operator<<(IOutputStream& out, TLocalTimeS localTimeS); + + inline TLocalTimeS GetLocalTimeS() { + return TLocalTimeS(); + } + + TString GetSystemResources(); + TString PrintSystemResources(const NMemInfo::TMemInfo& info); + + struct TLoggerFormatterTraits { + static ILoggerFormatter* CreateDefault() { + return CreateRtyLoggerFormatter(); + } + }; +} + +class TLoggerFormatterOperator : public NLoggingImpl::TOperatorBase<ILoggerFormatter, NLoggingImpl::TLoggerFormatterTraits> { +}; + +struct TRTYMessageFormater { + static bool CheckLoggingContext(TLog& logger, const TLogRecordContext& context); + static TSimpleSharedPtr<TLogElement> StartRecord(TLog& logger, const TLogRecordContext& context, TSimpleSharedPtr<TLogElement> earlier); +}; + +using TRTYLogPreprocessor = TLogRecordPreprocessor<TLogFilter, TRTYMessageFormater>; diff --git a/library/cpp/logger/global/rty_formater_ut.cpp b/library/cpp/logger/global/rty_formater_ut.cpp new file mode 100644 index 0000000000..551a97c5bf --- /dev/null +++ b/library/cpp/logger/global/rty_formater_ut.cpp @@ -0,0 +1,31 @@ +#include "rty_formater.h" + +#include <library/cpp/testing/unittest/registar.h> + +namespace { + const TStringBuf SampleISO8601("2017-07-25T19:26:09.894000+03:00"); + const TStringBuf SampleRtyLog("2017-07-25 19:26:09.894 +0300"); +} + +Y_UNIT_TEST_SUITE(NLoggingImplTest) { + Y_UNIT_TEST(TestTLocalTimeSToStream) { + NLoggingImpl::TLocalTimeS lt(TInstant::ParseIso8601Deprecated(SampleISO8601)); + TStringStream ss; + ss << lt; + UNIT_ASSERT_EQUAL(ss.Str(), SampleRtyLog); + } + Y_UNIT_TEST(TestTLocalTimeSToString) { + NLoggingImpl::TLocalTimeS lt(TInstant::ParseIso8601Deprecated(SampleISO8601)); + UNIT_ASSERT_EQUAL(TString(lt), SampleRtyLog); + } + Y_UNIT_TEST(TestTLocalTimeSAddLeft) { + NLoggingImpl::TLocalTimeS lt(TInstant::ParseIso8601Deprecated(SampleISO8601)); + TStringBuf suffix("suffix"); + UNIT_ASSERT_EQUAL(lt + suffix, TString(SampleRtyLog) + suffix); + } + Y_UNIT_TEST(TestTLocalTimeSAddRight) { + NLoggingImpl::TLocalTimeS lt(TInstant::ParseIso8601Deprecated(SampleISO8601)); + TString prefix("prefix"); + UNIT_ASSERT_EQUAL(prefix + lt, prefix + SampleRtyLog); + } +} diff --git a/library/cpp/logger/global/ut/ya.make b/library/cpp/logger/global/ut/ya.make new file mode 100644 index 0000000000..8aea38906f --- /dev/null +++ b/library/cpp/logger/global/ut/ya.make @@ -0,0 +1,15 @@ +UNITTEST() + +OWNER(salmin) + +PEERDIR( + library/cpp/logger/global +) + +SRCDIR(library/cpp/logger/global) + +SRCS( + rty_formater_ut.cpp +) + +END() diff --git a/library/cpp/logger/global/ya.make b/library/cpp/logger/global/ya.make new file mode 100644 index 0000000000..20eb361e72 --- /dev/null +++ b/library/cpp/logger/global/ya.make @@ -0,0 +1,19 @@ +LIBRARY() + +OWNER(g:geosaas) + +PEERDIR( + library/cpp/logger +) + +IF (OS_WINDOWS) + NO_WERROR() +ENDIF() + +SRCS( + common.cpp + global.cpp + rty_formater.cpp +) + +END() diff --git a/library/cpp/logger/init_context/README.md b/library/cpp/logger/init_context/README.md new file mode 100644 index 0000000000..93564e4890 --- /dev/null +++ b/library/cpp/logger/init_context/README.md @@ -0,0 +1,5 @@ +Эта библиотека содержит две раплизации InitContext для TLogBackendCreator. + +TLogBackendCreatorInitContextYConf работает с YandexConfig (library/cpp/yconf). + +TLogBackendCreatorInitContextConfig работает с NConfig::TConfig (library/cpp/config). diff --git a/library/cpp/logger/init_context/config.cpp b/library/cpp/logger/init_context/config.cpp new file mode 100644 index 0000000000..30efa13333 --- /dev/null +++ b/library/cpp/logger/init_context/config.cpp @@ -0,0 +1,26 @@ +#include "config.h" + +TLogBackendCreatorInitContextConfig::TLogBackendCreatorInitContextConfig(const NConfig::TConfig& config) + : Config(config) +{} + +bool TLogBackendCreatorInitContextConfig::GetValue(TStringBuf name, TString& var) const { + if (Config.Has(name)) { + var = Config[name].Get<TString>(); + return true; + } + return false; +} + +TVector<THolder<ILogBackendCreator::IInitContext>> TLogBackendCreatorInitContextConfig::GetChildren(TStringBuf name) const { + TVector<THolder<IInitContext>> result; + const NConfig::TConfig& child = Config[name]; + if (child.IsA<NConfig::TArray>()) { + for (const auto& i: child.Get<NConfig::TArray>()) { + result.emplace_back(MakeHolder<TLogBackendCreatorInitContextConfig>(i)); + } + } else if (!child.IsNull()) { + result.emplace_back(MakeHolder<TLogBackendCreatorInitContextConfig>(child)); + } + return result; +} diff --git a/library/cpp/logger/init_context/config.h b/library/cpp/logger/init_context/config.h new file mode 100644 index 0000000000..8227d13176 --- /dev/null +++ b/library/cpp/logger/init_context/config.h @@ -0,0 +1,14 @@ +#pragma once + +#include <library/cpp/logger/backend_creator.h> +#include <library/cpp/config/config.h> + +class TLogBackendCreatorInitContextConfig : public ILogBackendCreator::IInitContext { +public: + TLogBackendCreatorInitContextConfig(const NConfig::TConfig& config); + virtual bool GetValue(TStringBuf name, TString& var) const override; + virtual TVector<THolder<IInitContext>> GetChildren(TStringBuf name) const override; + +private: + const NConfig::TConfig& Config; +}; diff --git a/library/cpp/logger/init_context/ya.make b/library/cpp/logger/init_context/ya.make new file mode 100644 index 0000000000..9572a34c60 --- /dev/null +++ b/library/cpp/logger/init_context/ya.make @@ -0,0 +1,20 @@ +OWNER( + pg + mvel + g:util + g:base +) + +LIBRARY() + +PEERDIR( + library/cpp/logger + library/cpp/config + library/cpp/yconf +) +SRCS( + config.cpp + yconf.cpp +) + +END() diff --git a/library/cpp/logger/init_context/yconf.cpp b/library/cpp/logger/init_context/yconf.cpp new file mode 100644 index 0000000000..c7da1d607c --- /dev/null +++ b/library/cpp/logger/init_context/yconf.cpp @@ -0,0 +1,18 @@ +#include "yconf.h" + +TLogBackendCreatorInitContextYConf::TLogBackendCreatorInitContextYConf(const TYandexConfig::Section& section) + : Section(section) +{} + +bool TLogBackendCreatorInitContextYConf::GetValue(TStringBuf name, TString& var) const { + return Section.GetDirectives().GetValue(name, var); +} + +TVector<THolder<ILogBackendCreator::IInitContext>> TLogBackendCreatorInitContextYConf::GetChildren(TStringBuf name) const { + TVector<THolder<IInitContext>> result; + auto children = Section.GetAllChildren(); + for (auto range = children.equal_range(TCiString(name)); range.first != range.second; ++range.first) { + result.emplace_back(MakeHolder<TLogBackendCreatorInitContextYConf>(*range.first->second)); + } + return result; +} diff --git a/library/cpp/logger/init_context/yconf.h b/library/cpp/logger/init_context/yconf.h new file mode 100644 index 0000000000..b1867d271d --- /dev/null +++ b/library/cpp/logger/init_context/yconf.h @@ -0,0 +1,13 @@ +#pragma once + +#include <library/cpp/logger/backend_creator.h> +#include <library/cpp/yconf/conf.h> + +class TLogBackendCreatorInitContextYConf: public ILogBackendCreator::IInitContext { +public: + TLogBackendCreatorInitContextYConf(const TYandexConfig::Section& section); + virtual bool GetValue(TStringBuf name, TString& var) const override; + virtual TVector<THolder<IInitContext>> GetChildren(TStringBuf name) const override; +private: + const TYandexConfig::Section& Section; +}; diff --git a/library/cpp/logger/log.cpp b/library/cpp/logger/log.cpp new file mode 100644 index 0000000000..e1d70cc3d2 --- /dev/null +++ b/library/cpp/logger/log.cpp @@ -0,0 +1,252 @@ +#include "log.h" +#include "uninitialized_creator.h" +#include "filter.h" +#include "null.h" +#include "stream.h" +#include "thread.h" + +#include <util/string/cast.h> +#include <util/stream/printf.h> +#include <util/system/yassert.h> +#include <util/generic/string.h> +#include <util/generic/scope.h> +#include <util/generic/yexception.h> + +THolder<TLogBackend> CreateLogBackend(const TString& fname, ELogPriority priority, bool threaded) { + TLogBackendCreatorUninitialized creator; + creator.InitCustom(fname, priority, threaded); + return creator.CreateLogBackend(); +} + +THolder<TLogBackend> CreateFilteredOwningThreadedLogBackend(const TString& fname, ELogPriority priority, size_t queueLen) { + return MakeHolder<TFilteredLogBackend>(CreateOwningThreadedLogBackend(fname, queueLen), priority); +} + +THolder<TOwningThreadedLogBackend> CreateOwningThreadedLogBackend(const TString& fname, size_t queueLen) { + return MakeHolder<TOwningThreadedLogBackend>(CreateLogBackend(fname, LOG_MAX_PRIORITY, false).Release(), queueLen); +} + +class TLog::TImpl: public TAtomicRefCount<TImpl> { + class TPriorityLogStream final: public IOutputStream { + public: + inline TPriorityLogStream(ELogPriority p, const TImpl* parent) + : Priority_(p) + , Parent_(parent) + { + } + + void DoWrite(const void* buf, size_t len) override { + Parent_->WriteData(Priority_, (const char*)buf, len); + } + + private: + ELogPriority Priority_ = LOG_DEF_PRIORITY; + const TImpl* Parent_ = nullptr; + }; + +public: + inline TImpl(THolder<TLogBackend> backend) + : Backend_(std::move(backend)) + { + } + + inline void ReopenLog() { + if (!IsOpen()) { + return; + } + + Backend_->ReopenLog(); + } + + inline void ReopenLogNoFlush() { + if (!IsOpen()) { + return; + } + + Backend_->ReopenLogNoFlush(); + } + + inline void AddLog(ELogPriority priority, const char* format, va_list args) const { + if (!IsOpen()) { + return; + } + + TPriorityLogStream ls(priority, this); + + Printf(ls, format, args); + } + + inline void ResetBackend(THolder<TLogBackend> backend) noexcept { + Backend_ = std::move(backend); + } + + inline THolder<TLogBackend> ReleaseBackend() noexcept { + return std::move(Backend_); + } + + inline bool IsNullLog() const noexcept { + return !IsOpen() || (dynamic_cast<TNullLogBackend*>(Backend_.Get()) != nullptr); + } + + inline bool IsOpen() const noexcept { + return nullptr != Backend_.Get(); + } + + inline void CloseLog() noexcept { + Backend_.Destroy(); + + Y_ASSERT(!IsOpen()); + } + + inline void WriteData(ELogPriority priority, const char* data, size_t len) const { + if (IsOpen()) { + Backend_->WriteData(TLogRecord(priority, data, len)); + } + } + + inline ELogPriority DefaultPriority() noexcept { + return DefaultPriority_; + } + + inline void SetDefaultPriority(ELogPriority priority) noexcept { + DefaultPriority_ = priority; + } + + inline ELogPriority FiltrationLevel() const noexcept { + return Backend_->FiltrationLevel(); + } + + inline size_t BackEndQueueSize() const { + return Backend_->QueueSize(); + } + +private: + THolder<TLogBackend> Backend_; + ELogPriority DefaultPriority_ = LOG_DEF_PRIORITY; +}; + +TLog::TLog() + : Impl_(MakeIntrusive<TImpl>(nullptr)) +{ +} + +TLog::TLog(const TString& fname, ELogPriority priority) + : TLog(CreateLogBackend(fname, priority, false)) +{ +} + +TLog::TLog(THolder<TLogBackend> backend) + : Impl_(MakeIntrusive<TImpl>(std::move(backend))) +{ +} + +TLog::TLog(const TLog&) = default; +TLog::TLog(TLog&&) = default; +TLog::~TLog() = default; +TLog& TLog::operator=(const TLog&) = default; +TLog& TLog::operator=(TLog&&) = default; + +bool TLog::IsOpen() const noexcept { + return Impl_->IsOpen(); +} + +void TLog::AddLog(const char* format, ...) const { + va_list args; + va_start(args, format); + + Y_DEFER { + va_end(args); + }; + + Impl_->AddLog(Impl_->DefaultPriority(), format, args); +} + +void TLog::AddLog(ELogPriority priority, const char* format, ...) const { + va_list args; + va_start(args, format); + + Y_DEFER { + va_end(args); + }; + + Impl_->AddLog(priority, format, args); +} + +void TLog::AddLogVAList(const char* format, va_list lst) { + Impl_->AddLog(Impl_->DefaultPriority(), format, lst); +} + +void TLog::ReopenLog() { + if (const auto copy = Impl_) { + copy->ReopenLog(); + } +} + +void TLog::ReopenLogNoFlush() { + if (const auto copy = Impl_) { + copy->ReopenLogNoFlush(); + } +} + +void TLog::CloseLog() { + Impl_->CloseLog(); +} + +void TLog::SetDefaultPriority(ELogPriority priority) noexcept { + Impl_->SetDefaultPriority(priority); +} + +ELogPriority TLog::FiltrationLevel() const noexcept { + return Impl_->FiltrationLevel(); +} + +ELogPriority TLog::DefaultPriority() const noexcept { + return Impl_->DefaultPriority(); +} + +bool TLog::OpenLog(const char* path, ELogPriority lp) { + if (path) { + ResetBackend(CreateLogBackend(path, lp)); + } else { + ResetBackend(MakeHolder<TStreamLogBackend>(&Cerr)); + } + + return true; +} + +void TLog::ResetBackend(THolder<TLogBackend> backend) noexcept { + Impl_->ResetBackend(std::move(backend)); +} + +bool TLog::IsNullLog() const noexcept { + return Impl_->IsNullLog(); +} + +THolder<TLogBackend> TLog::ReleaseBackend() noexcept { + return Impl_->ReleaseBackend(); +} + +void TLog::Write(ELogPriority priority, const char* data, size_t len) const { + if (Formatter_) { + const auto formated = Formatter_(priority, TStringBuf{data, len}); + Impl_->WriteData(priority, formated.data(), formated.size()); + } else { + Impl_->WriteData(priority, data, len); + } +} + +void TLog::Write(ELogPriority priority, const TStringBuf data) const { + Write(priority, data.data(), data.size()); +} + +void TLog::Write(const char* data, size_t len) const { + Write(Impl_->DefaultPriority(), data, len); +} + +void TLog::SetFormatter(TLogFormatter formatter) noexcept { + Formatter_ = std::move(formatter); +} + +size_t TLog::BackEndQueueSize() const { + return Impl_->BackEndQueueSize(); +} diff --git a/library/cpp/logger/log.h b/library/cpp/logger/log.h new file mode 100644 index 0000000000..8be984ccc8 --- /dev/null +++ b/library/cpp/logger/log.h @@ -0,0 +1,115 @@ +#pragma once + +#include "backend.h" +#include "element.h" +#include "priority.h" +#include "record.h" +#include "thread.h" + +#include <util/generic/fwd.h> +#include <util/generic/ptr.h> + +#include <functional> +#include <cstdarg> + +using TLogFormatter = std::function<TString(ELogPriority priority, TStringBuf)>; + +// Logging facilities interface. +// +// ```cpp +// TLog base; +// ... +// auto log = base; +// log.SetFormatter([reqId](ELogPriority p, TStringBuf msg) { +// return TStringBuilder() << "reqid=" << reqId << "; " << msg; +// }); +// +// log.Write(TLOG_INFO, "begin"); +// HandleRequest(...); +// log.Write(TLOG_INFO, "end"); +// ``` +// +// Users are encouraged to copy `TLog` instance. +class TLog { +public: + // Construct empty logger all writes will be spilled. + TLog(); + // Construct file logger. + TLog(const TString& fname, ELogPriority priority = LOG_MAX_PRIORITY); + // Construct any type of logger + TLog(THolder<TLogBackend> backend); + + TLog(const TLog&); + TLog(TLog&&); + ~TLog(); + TLog& operator=(const TLog&); + TLog& operator=(TLog&&); + + // Change underlying backend. + // NOTE: not thread safe. + void ResetBackend(THolder<TLogBackend> backend) noexcept; + // Reset underlying backend, `IsNullLog()` will return `true` after this call. + // NOTE: not thread safe. + THolder<TLogBackend> ReleaseBackend() noexcept; + // Check if underlying backend is defined and is not null. + // NOTE: not thread safe with respect to `ResetBackend` and `ReleaseBackend`. + bool IsNullLog() const noexcept; + + // Write message to the log. + // + // @param[in] priority Message priority to use. + // @param[in] message Message to write. + void Write(ELogPriority priority, TStringBuf message) const; + // Write message to the log using `DefaultPriority()`. + void Write(const char* data, size_t len) const; + // Write message to the log, but pass the message in a c-style. + void Write(ELogPriority priority, const char* data, size_t len) const; + + // Write message to the log in a c-like printf style. + void Y_PRINTF_FORMAT(3, 4) AddLog(ELogPriority priority, const char* format, ...) const; + // Write message to the log in a c-like printf style with `DefaultPriority()` priority. + void Y_PRINTF_FORMAT(2, 3) AddLog(const char* format, ...) const; + + // Call `ReopenLog()` of the underlying backend. + void ReopenLog(); + // Call `ReopenLogNoFlush()` of the underlying backend. + void ReopenLogNoFlush(); + // Call `QueueSize()` of the underlying backend. + size_t BackEndQueueSize() const; + + // Set log default priority. + // NOTE: not thread safe. + void SetDefaultPriority(ELogPriority priority) noexcept; + // Get default priority + ELogPriority DefaultPriority() const noexcept; + + // Call `FiltrationLevel()` of the underlying backend. + ELogPriority FiltrationLevel() const noexcept; + + // Set current log formatter. + void SetFormatter(TLogFormatter formatter) noexcept; + + template <class T> + inline TLogElement operator<<(const T& t) const { + TLogElement ret(this); + ret << t; + return ret; + } + +public: + // These methods are deprecated and present here only for compatibility reasons (for 13 years + // already ...). Do not use them. + bool OpenLog(const char* path, ELogPriority lp = LOG_MAX_PRIORITY); + bool IsOpen() const noexcept; + void AddLogVAList(const char* format, va_list lst); + void CloseLog(); + +private: + class TImpl; + TIntrusivePtr<TImpl> Impl_; + TLogFormatter Formatter_; +}; + +THolder<TLogBackend> CreateLogBackend(const TString& fname, ELogPriority priority = LOG_MAX_PRIORITY, bool threaded = false); +THolder<TLogBackend> CreateFilteredOwningThreadedLogBackend(const TString& fname, ELogPriority priority = LOG_MAX_PRIORITY, size_t queueLen = 0); +THolder<TOwningThreadedLogBackend> CreateOwningThreadedLogBackend(const TString& fname, size_t queueLen = 0); diff --git a/library/cpp/logger/log_ut.cpp b/library/cpp/logger/log_ut.cpp new file mode 100644 index 0000000000..8de46f17f5 --- /dev/null +++ b/library/cpp/logger/log_ut.cpp @@ -0,0 +1,191 @@ +#include "all.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/system/fs.h> +#include <util/system/rwlock.h> +#include <util/system/yield.h> +#include <util/memory/blob.h> +#include <util/stream/file.h> +#include <util/generic/string.h> +#include <util/generic/vector.h> + +class TLogTest: public TTestBase { + UNIT_TEST_SUITE(TLogTest); + UNIT_TEST(TestFile) + UNIT_TEST(TestFormat) + UNIT_TEST(TestWrite) + UNIT_TEST(TestThreaded) + UNIT_TEST(TestThreadedWithOverflow) + UNIT_TEST(TestNoFlush) + UNIT_TEST_SUITE_END(); + +private: + void TestFile(); + void TestFormat(); + void TestWrite(); + void TestThreaded(); + void TestThreadedWithOverflow(); + void TestNoFlush(); + void SetUp() override; + void TearDown() override; +}; + +UNIT_TEST_SUITE_REGISTRATION(TLogTest); + +#define LOGFILE "tmplogfile" + +void TLogTest::TestFile() { + { + TLog log; + + { + TLog filelog(LOGFILE); + + log = filelog; + } + + int v1 = 12; + unsigned v2 = 34; + double v3 = 3.0; + const char* v4 = "qwqwqw"; + + log.ReopenLog(); + log.AddLog("some useful data %d, %u, %lf, %s\n", v1, v2, v3, v4); + } + + TBlob data = TBlob::FromFileSingleThreaded(LOGFILE); + + UNIT_ASSERT_EQUAL(TString((const char*)data.Begin(), data.Size()), "some useful data 12, 34, 3.000000, qwqwqw\n"); +} + +void TLogTest::TestThreaded() { + { + TFileLogBackend fb(LOGFILE); + TLog log(THolder(new TThreadedLogBackend(&fb))); + + int v1 = 12; + unsigned v2 = 34; + double v3 = 3.0; + const char* v4 = "qwqwqw"; + + log.ReopenLog(); + log.AddLog("some useful data %d, %u, %lf, %s\n", v1, v2, v3, v4); + } + + TBlob data = TBlob::FromFileSingleThreaded(LOGFILE); + + UNIT_ASSERT_EQUAL(TString((const char*)data.Begin(), data.Size()), "some useful data 12, 34, 3.000000, qwqwqw\n"); +} + +void TLogTest::TestThreadedWithOverflow() { + class TFakeLogBackend: public TLogBackend { + public: + TWriteGuard Guard() { + return TWriteGuard(Lock_); + } + + void WriteData(const TLogRecord&) override { + TReadGuard guard(Lock_); + } + + void ReopenLog() override { + TWriteGuard guard(Lock_); + } + + private: + TRWMutex Lock_; + }; + + auto waitForFreeQueue = [](const TLog& log) { + ThreadYield(); + while (log.BackEndQueueSize() > 0) { + Sleep(TDuration::MilliSeconds(1)); + } + }; + + TFakeLogBackend fb; + { + TLog log(THolder(new TThreadedLogBackend(&fb, 2))); + + auto guard = fb.Guard(); + log.AddLog("first write"); + waitForFreeQueue(log); + log.AddLog("second write (first in queue)"); + log.AddLog("third write (second in queue)"); + UNIT_ASSERT_EXCEPTION(log.AddLog("fourth write (queue overflow)"), yexception); + } + + { + ui32 overflows = 0; + TLog log(THolder(new TThreadedLogBackend(&fb, 2, [&overflows] { ++overflows; }))); + + auto guard = fb.Guard(); + log.AddLog("first write"); + waitForFreeQueue(log); + log.AddLog("second write (first in queue)"); + log.AddLog("third write (second in queue)"); + UNIT_ASSERT_EQUAL(overflows, 0); + log.AddLog("fourth write (queue overflow)"); + UNIT_ASSERT_EQUAL(overflows, 1); + } +} + +void TLogTest::TestNoFlush() { + { + TFileLogBackend fb(LOGFILE); + TLog log(THolder(new TThreadedLogBackend(&fb))); + + int v1 = 12; + unsigned v2 = 34; + double v3 = 3.0; + const char* v4 = "qwqwqw"; + + log.ReopenLogNoFlush(); + log.AddLog("some useful data %d, %u, %lf, %s\n", v1, v2, v3, v4); + } + + TBlob data = TBlob::FromFileSingleThreaded(LOGFILE); + + UNIT_ASSERT_EQUAL(TString((const char*)data.Begin(), data.Size()), "some useful data 12, 34, 3.000000, qwqwqw\n"); +} + +void TLogTest::TestFormat() { + TStringStream data; + + { + TLog log(THolder(new TStreamLogBackend(&data))); + + log << "qw" + << " " + << "1234" << 1234 << " " << 12.3 << 'q' << Endl; + } + + UNIT_ASSERT_EQUAL(data.Str(), "qw 12341234 12.3q\n"); +} + +void TLogTest::TestWrite() { + TStringStream data; + TString test; + + { + TLog log(THolder(new TStreamLogBackend(&data))); + + for (size_t i = 0; i < 1000; ++i) { + TVector<char> buf(i, (char)i); + + test.append(buf.data(), buf.size()); + log.Write(buf.data(), buf.size()); + } + } + + UNIT_ASSERT_EQUAL(data.Str(), test); +} + +void TLogTest::SetUp() { + TearDown(); +} + +void TLogTest::TearDown() { + NFs::Remove(LOGFILE); +} diff --git a/library/cpp/logger/null.cpp b/library/cpp/logger/null.cpp new file mode 100644 index 0000000000..debb22f794 --- /dev/null +++ b/library/cpp/logger/null.cpp @@ -0,0 +1,13 @@ +#include "null.h" + +TNullLogBackend::TNullLogBackend() { +} + +TNullLogBackend::~TNullLogBackend() { +} + +void TNullLogBackend::WriteData(const TLogRecord&) { +} + +void TNullLogBackend::ReopenLog() { +} diff --git a/library/cpp/logger/null.h b/library/cpp/logger/null.h new file mode 100644 index 0000000000..a02f250b00 --- /dev/null +++ b/library/cpp/logger/null.h @@ -0,0 +1,12 @@ +#pragma once + +#include "backend.h" + +class TNullLogBackend: public TLogBackend { +public: + TNullLogBackend(); + ~TNullLogBackend() override; + + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override; +}; diff --git a/library/cpp/logger/null_creator.cpp b/library/cpp/logger/null_creator.cpp new file mode 100644 index 0000000000..9f63d5c739 --- /dev/null +++ b/library/cpp/logger/null_creator.cpp @@ -0,0 +1,17 @@ +#include "null_creator.h" +#include "null.h" + +THolder<TLogBackend> TNullLogBackendCreator::DoCreateLogBackend() const { + return MakeHolder<TNullLogBackend>(); +} + +ILogBackendCreator::TFactory::TRegistrator<TNullLogBackendCreator> TNullLogBackendCreator::RegistrarDevNull("/dev/null"); +ILogBackendCreator::TFactory::TRegistrator<TNullLogBackendCreator> TNullLogBackendCreator::RegistrarNull("null"); + + +void TNullLogBackendCreator::DoToJson(NJson::TJsonValue& /*value*/) const { +} + +TNullLogBackendCreator::TNullLogBackendCreator() + : TLogBackendCreatorBase("null") +{} diff --git a/library/cpp/logger/null_creator.h b/library/cpp/logger/null_creator.h new file mode 100644 index 0000000000..a5bcab63f9 --- /dev/null +++ b/library/cpp/logger/null_creator.h @@ -0,0 +1,15 @@ +#pragma once + +#include "backend_creator.h" + +class TNullLogBackendCreator : public TLogBackendCreatorBase { +public: + TNullLogBackendCreator(); + static TFactory::TRegistrator<TNullLogBackendCreator> RegistrarNull; + static TFactory::TRegistrator<TNullLogBackendCreator> RegistrarDevNull; +protected: + virtual void DoToJson(NJson::TJsonValue& value) const override; + +private: + virtual THolder<TLogBackend> DoCreateLogBackend() const override; +}; diff --git a/library/cpp/logger/priority.h b/library/cpp/logger/priority.h new file mode 100644 index 0000000000..d2a9fa0a07 --- /dev/null +++ b/library/cpp/logger/priority.h @@ -0,0 +1,15 @@ +#pragma once + +enum ELogPriority { + TLOG_EMERG = 0 /* "EMERG" */, + TLOG_ALERT = 1 /* "ALERT" */, + TLOG_CRIT = 2 /* "CRITICAL_INFO" */, + TLOG_ERR = 3 /* "ERROR" */, + TLOG_WARNING = 4 /* "WARNING" */, + TLOG_NOTICE = 5 /* "NOTICE" */, + TLOG_INFO = 6 /* "INFO" */, + TLOG_DEBUG = 7 /* "DEBUG" */, + TLOG_RESOURCES = 8 /* "RESOURCES" */ +}; +#define LOG_MAX_PRIORITY TLOG_RESOURCES +#define LOG_DEF_PRIORITY TLOG_INFO diff --git a/library/cpp/logger/record.h b/library/cpp/logger/record.h new file mode 100644 index 0000000000..c28a7785fd --- /dev/null +++ b/library/cpp/logger/record.h @@ -0,0 +1,18 @@ +#pragma once + +#include "priority.h" + +#include <util/system/defaults.h> + +struct TLogRecord { + const char* Data; + size_t Len; + ELogPriority Priority; + + inline TLogRecord(ELogPriority priority, const char* data, size_t len) noexcept + : Data(data) + , Len(len) + , Priority(priority) + { + } +}; diff --git a/library/cpp/logger/rotating_file.cpp b/library/cpp/logger/rotating_file.cpp new file mode 100644 index 0000000000..a62f48f25d --- /dev/null +++ b/library/cpp/logger/rotating_file.cpp @@ -0,0 +1,86 @@ +#include "rotating_file.h" +#include "file.h" +#include "record.h" + +#include <util/string/builder.h> +#include <util/system/fstat.h> +#include <util/system/rwlock.h> +#include <util/system/fs.h> +#include <util/system/atomic.h> +#include <util/generic/string.h> + +/* + * rotating file log + * if Size_ > MaxSizeBytes + * Path.(N-1) -> Path.N + * Path.(N-2) -> Path.(N-1) + * ... + * Path.1 -> Path.2 + * Path -> Path.1 + */ +class TRotatingFileLogBackend::TImpl { +public: + inline TImpl(const TString& path, const ui64 maxSizeBytes, const ui32 rotatedFilesCount) + : Log_(path) + , Path_(path) + , MaxSizeBytes_(maxSizeBytes) + , Size_(TFileStat(Path_).Size) + , RotatedFilesCount_(rotatedFilesCount) + { + Y_ENSURE(RotatedFilesCount_ != 0); + } + + inline void WriteData(const TLogRecord& rec) { + if (static_cast<ui64>(AtomicGet(Size_)) > MaxSizeBytes_) { + TWriteGuard guard(Lock_); + if (static_cast<ui64>(AtomicGet(Size_)) > MaxSizeBytes_) { + TString newLogPath(TStringBuilder{} << Path_ << "." << RotatedFilesCount_); + for (size_t fileId = RotatedFilesCount_ - 1; fileId; --fileId) { + TString oldLogPath(TStringBuilder{} << Path_ << "." << fileId); + NFs::Rename(oldLogPath, newLogPath); + newLogPath = oldLogPath; + } + NFs::Rename(Path_, newLogPath); + Log_.ReopenLog(); + AtomicSet(Size_, 0); + } + } + TReadGuard guard(Lock_); + Log_.WriteData(rec); + AtomicAdd(Size_, rec.Len); + } + + inline void ReopenLog() { + TWriteGuard guard(Lock_); + + Log_.ReopenLog(); + AtomicSet(Size_, TFileStat(Path_).Size); + } + +private: + TRWMutex Lock_; + TFileLogBackend Log_; + const TString Path_; + const ui64 MaxSizeBytes_; + TAtomic Size_; + const ui32 RotatedFilesCount_; +}; + +TRotatingFileLogBackend::TRotatingFileLogBackend(const TString& path, const ui64 maxSizeByte, const ui32 rotatedFilesCount) + : Impl_(new TImpl(path, maxSizeByte, rotatedFilesCount)) +{ +} + +TRotatingFileLogBackend::~TRotatingFileLogBackend() { +} + +void TRotatingFileLogBackend::WriteData(const TLogRecord& rec) { + Impl_->WriteData(rec); +} + +void TRotatingFileLogBackend::ReopenLog() { + TAtomicSharedPtr<TImpl> copy = Impl_; + if (copy) { + copy->ReopenLog(); + } +} diff --git a/library/cpp/logger/rotating_file.h b/library/cpp/logger/rotating_file.h new file mode 100644 index 0000000000..cb047f25fb --- /dev/null +++ b/library/cpp/logger/rotating_file.h @@ -0,0 +1,20 @@ +#pragma once + +#include "backend.h" + +#include <util/generic/fwd.h> +#include <util/generic/ptr.h> + +class TRotatingFileLogBackend: public TLogBackend { +public: + TRotatingFileLogBackend(const TString& preRotatePath, const TString& postRotatePath, const ui64 maxSizeBytes); + TRotatingFileLogBackend(const TString& path, const ui64 maxSizeBytes, const ui32 rotatedFilesCount); + ~TRotatingFileLogBackend() override; + + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override; + +private: + class TImpl; + TAtomicSharedPtr<TImpl> Impl_; +}; diff --git a/library/cpp/logger/rotating_file_creator.cpp b/library/cpp/logger/rotating_file_creator.cpp new file mode 100644 index 0000000000..6f71b68573 --- /dev/null +++ b/library/cpp/logger/rotating_file_creator.cpp @@ -0,0 +1,28 @@ +#include "rotating_file_creator.h" +#include "rotating_file.h" + +THolder<TLogBackend> TRotatingFileLogBackendCreator::DoCreateLogBackend() const { + return MakeHolder<TRotatingFileLogBackend>(Path, MaxSizeBytes, RotatedFilesCount); +} + + +TRotatingFileLogBackendCreator::TRotatingFileLogBackendCreator() + : TFileLogBackendCreator("", "rotating") +{} + +bool TRotatingFileLogBackendCreator::Init(const IInitContext& ctx) { + if (!TFileLogBackendCreator::Init(ctx)) { + return false; + } + ctx.GetValue("MaxSizeBytes", MaxSizeBytes); + ctx.GetValue("RotatedFilesCount", RotatedFilesCount); + return true; +} + +ILogBackendCreator::TFactory::TRegistrator<TRotatingFileLogBackendCreator> TRotatingFileLogBackendCreator::Registrar("rotating"); + +void TRotatingFileLogBackendCreator::DoToJson(NJson::TJsonValue& value) const { + TFileLogBackendCreator::DoToJson(value); + value["MaxSizeBytes"] = MaxSizeBytes; + value["RotatedFilesCount"] = RotatedFilesCount; +} diff --git a/library/cpp/logger/rotating_file_creator.h b/library/cpp/logger/rotating_file_creator.h new file mode 100644 index 0000000000..b2e94584da --- /dev/null +++ b/library/cpp/logger/rotating_file_creator.h @@ -0,0 +1,18 @@ +#pragma once + +#include "file_creator.h" + +class TRotatingFileLogBackendCreator : public TFileLogBackendCreator { +public: + TRotatingFileLogBackendCreator(); + virtual bool Init(const IInitContext& ctx) override; + static TFactory::TRegistrator<TRotatingFileLogBackendCreator> Registrar; + +protected: + virtual void DoToJson(NJson::TJsonValue& value) const override; + +private: + virtual THolder<TLogBackend> DoCreateLogBackend() const override; + ui64 MaxSizeBytes = Max<ui64>(); + ui64 RotatedFilesCount = Max<ui64>(); +}; diff --git a/library/cpp/logger/rotating_file_ut.cpp b/library/cpp/logger/rotating_file_ut.cpp new file mode 100644 index 0000000000..84966933d9 --- /dev/null +++ b/library/cpp/logger/rotating_file_ut.cpp @@ -0,0 +1,57 @@ +#include "rotating_file.h" +#include "record.h" + +#include <util/generic/string.h> +#include <util/system/fstat.h> +#include <util/system/fs.h> + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/testing/unittest/tests_data.h> + +Y_UNIT_TEST_SUITE(NewRotatingFileSuite) { + const TString PATH = GetWorkPath() + "/my.log"; + + Y_UNIT_TEST(TestFileWrite) { + TRotatingFileLogBackend log(PATH, 4000, 2); + TString data = "my data"; + log.WriteData(TLogRecord(ELogPriority::TLOG_INFO, data.data(), data.size())); + UNIT_ASSERT_C(TFileStat(PATH).Size > 0, "file " << PATH << " has zero size"); + } + + Y_UNIT_TEST(TestFileRotate) { + const ui64 maxSize = 40; + TRotatingFileLogBackend log(PATH, maxSize, 2); + TStringBuilder data; + for (size_t i = 0; i < 10; ++i) + data << "data\n"; + log.WriteData(TLogRecord(ELogPriority::TLOG_INFO, data.data(), data.size())); + UNIT_ASSERT_C(TFileStat(PATH).Size > 0, "file " << PATH << " has zero size"); + data.clear(); + data << "more data"; + log.WriteData(TLogRecord(ELogPriority::TLOG_INFO, data.data(), data.size())); + UNIT_ASSERT_C(TFileStat(PATH).Size > 0, "file " << PATH << " has zero size"); + UNIT_ASSERT_C(TFileStat(TStringBuilder{} << PATH << ".1").Size > 0, "file " << PATH << ".1 has zero size"); + UNIT_ASSERT_C(TFileStat(PATH).Size < maxSize, "size of file " << PATH << " is greater than the size limit of " << maxSize << " bytes"); + } + + Y_UNIT_TEST(TestDoubleFileRotate) { + const ui64 maxSize = 40; + TRotatingFileLogBackend log(PATH, maxSize, 2); + TStringBuilder data; + for (size_t i = 0; i < 10; ++i) + data << "data\n"; + log.WriteData(TLogRecord(ELogPriority::TLOG_INFO, data.data(), data.size())); + UNIT_ASSERT_C(TFileStat(PATH).Size > 0, "file " << PATH << " has zero size"); + log.WriteData(TLogRecord(ELogPriority::TLOG_INFO, data.data(), data.size())); + UNIT_ASSERT_C(TFileStat(PATH).Size > 0, "file " << PATH << " has zero size"); + UNIT_ASSERT_C(TFileStat(TStringBuilder{} << PATH << ".1").Size > 0, "file " << PATH << ".1 has zero size"); + UNIT_ASSERT_C(TFileStat(PATH).Size > maxSize, "size of file " << PATH << " is lesser than was written"); + data.clear(); + data << "more data"; + log.WriteData(TLogRecord(ELogPriority::TLOG_INFO, data.data(), data.size())); + UNIT_ASSERT_C(TFileStat(PATH).Size > 0, "file " << PATH << " has zero size"); + UNIT_ASSERT_C(TFileStat(TStringBuilder{} << PATH << ".1").Size > 0, "file " << PATH << ".1 has zero size"); + UNIT_ASSERT_C(TFileStat(TStringBuilder{} << PATH << ".2").Size > 0, "file " << PATH << ".2 has zero size"); + UNIT_ASSERT_C(TFileStat(PATH).Size < maxSize, "size of file " << PATH << " is greater than the size limit of " << maxSize << " bytes"); + } +} diff --git a/library/cpp/logger/stream.cpp b/library/cpp/logger/stream.cpp new file mode 100644 index 0000000000..96787ad94b --- /dev/null +++ b/library/cpp/logger/stream.cpp @@ -0,0 +1,19 @@ +#include "stream.h" +#include "record.h" + +#include <util/stream/output.h> + +TStreamLogBackend::TStreamLogBackend(IOutputStream* slave) + : Slave_(slave) +{ +} + +TStreamLogBackend::~TStreamLogBackend() { +} + +void TStreamLogBackend::WriteData(const TLogRecord& rec) { + Slave_->Write(rec.Data, rec.Len); +} + +void TStreamLogBackend::ReopenLog() { +} diff --git a/library/cpp/logger/stream.h b/library/cpp/logger/stream.h new file mode 100644 index 0000000000..feb240afcb --- /dev/null +++ b/library/cpp/logger/stream.h @@ -0,0 +1,17 @@ +#pragma once + +#include "backend.h" + +class IOutputStream; + +class TStreamLogBackend: public TLogBackend { +public: + TStreamLogBackend(IOutputStream* slave); + ~TStreamLogBackend() override; + + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override; + +private: + IOutputStream* Slave_; +}; diff --git a/library/cpp/logger/stream_creator.cpp b/library/cpp/logger/stream_creator.cpp new file mode 100644 index 0000000000..6246f9bf9c --- /dev/null +++ b/library/cpp/logger/stream_creator.cpp @@ -0,0 +1,32 @@ +#include "stream_creator.h" +#include "stream.h" + +THolder<TLogBackend> TCerrLogBackendCreator::DoCreateLogBackend() const { + return MakeHolder<TStreamLogBackend>(&Cerr); +} + + +TCerrLogBackendCreator::TCerrLogBackendCreator() + : TLogBackendCreatorBase("cerr") +{} + +void TCerrLogBackendCreator::DoToJson(NJson::TJsonValue& /*value*/) const { +} + +ILogBackendCreator::TFactory::TRegistrator<TCerrLogBackendCreator> TCerrLogBackendCreator::RegistrarCerr("cerr"); +ILogBackendCreator::TFactory::TRegistrator<TCerrLogBackendCreator> TCerrLogBackendCreator::RegistrarConsole("console"); + + +THolder<TLogBackend> TCoutLogBackendCreator::DoCreateLogBackend() const { + return MakeHolder<TStreamLogBackend>(&Cout); +} + + +TCoutLogBackendCreator::TCoutLogBackendCreator() + : TLogBackendCreatorBase("cout") +{} + +ILogBackendCreator::TFactory::TRegistrator<TCoutLogBackendCreator> TCoutLogBackendCreator::Registrar("cout"); + +void TCoutLogBackendCreator::DoToJson(NJson::TJsonValue& /*value*/) const { +} diff --git a/library/cpp/logger/stream_creator.h b/library/cpp/logger/stream_creator.h new file mode 100644 index 0000000000..cd66e986c0 --- /dev/null +++ b/library/cpp/logger/stream_creator.h @@ -0,0 +1,28 @@ +#pragma once + +#include "backend_creator.h" + +class TCerrLogBackendCreator : public TLogBackendCreatorBase { +public: + TCerrLogBackendCreator(); + static TFactory::TRegistrator<TCerrLogBackendCreator> RegistrarCerr; + static TFactory::TRegistrator<TCerrLogBackendCreator> RegistrarConsole; + +protected: + virtual void DoToJson(NJson::TJsonValue& value) const override; + +private: + virtual THolder<TLogBackend> DoCreateLogBackend() const override; +}; + +class TCoutLogBackendCreator : public TLogBackendCreatorBase { +public: + TCoutLogBackendCreator(); + static TFactory::TRegistrator<TCoutLogBackendCreator> Registrar; + +protected: + virtual void DoToJson(NJson::TJsonValue& value) const override; + +private: + virtual THolder<TLogBackend> DoCreateLogBackend() const override; +}; diff --git a/library/cpp/logger/sync_page_cache_file.cpp b/library/cpp/logger/sync_page_cache_file.cpp new file mode 100644 index 0000000000..a0e93a78d7 --- /dev/null +++ b/library/cpp/logger/sync_page_cache_file.cpp @@ -0,0 +1,125 @@ +#include "sync_page_cache_file.h" +#include "record.h" + +#include <util/generic/buffer.h> +#include <util/system/file.h> +#include <util/system/info.h> +#include <util/system/mutex.h> +#include <util/system/rwlock.h> +#include <util/system/align.h> + +class TSyncPageCacheFileLogBackend::TImpl: public TNonCopyable { +public: + TImpl(const TString& path, size_t maxBufferSize, size_t maxPendingCacheSize) + : File_{OpenFile(path)} + , MaxBufferSize_{maxBufferSize} + , MaxPendingCacheSize_{maxPendingCacheSize} + , Buffer_{maxBufferSize} + { + ResetPtrs(); + } + + ~TImpl() noexcept { + try { + Write(); + FlushSync(GuaranteedWrittenPtr_, WrittenPtr_); + } catch (...) { + } + } + + void WriteData(const TLogRecord& rec) { + TGuard guard{Lock_}; + + Buffer_.Append(rec.Data, rec.Len); + if (Buffer_.size() >= MaxBufferSize_) { + const i64 prevAlignedEndPtr = PageAlignedWrittenPtr_; + Write(); + + if (prevAlignedEndPtr < PageAlignedWrittenPtr_) { + FlushAsync(prevAlignedEndPtr, PageAlignedWrittenPtr_); + } + + const i64 minPendingCacheOffset = PageAlignedWrittenPtr_ - MaxPendingCacheSize_; + if (minPendingCacheOffset > GuaranteedWrittenPtr_) { + FlushSync(GuaranteedWrittenPtr_, minPendingCacheOffset); + } + } + } + + void ReopenLog() { + TGuard guard{Lock_}; + + Write(); + FlushSync(GuaranteedWrittenPtr_, WrittenPtr_); + + File_.LinkTo(OpenFile(File_.GetName())); + + ResetPtrs(); + } + +private: + void ResetPtrs() { + WrittenPtr_ = File_.GetLength(); + PageAlignedWrittenPtr_ = AlignDown(WrittenPtr_, GetPageSize()); + GuaranteedWrittenPtr_ = WrittenPtr_; + } + + static TFile OpenFile(const TString& path) { + return TFile{path, OpenAlways | WrOnly | ForAppend | Seq | NoReuse}; + } + + static i64 GetPageSize() { + static const i64 pageSize = NSystemInfo::GetPageSize(); + Y_ASSUME(IsPowerOf2(pageSize)); + return pageSize; + } + + void Write() { + File_.Write(Buffer_.Data(), Buffer_.Size()); + WrittenPtr_ += Buffer_.Size(); + PageAlignedWrittenPtr_ = AlignDown(WrittenPtr_, GetPageSize()); + Buffer_.Clear(); + } + + void FlushAsync(const i64 from, const i64 to) { + File_.FlushCache(from, to - from, /* wait = */ false); + } + + void FlushSync(const i64 from, const i64 to) { + const i64 begin = AlignDown(from, GetPageSize()); + const i64 end = AlignUp(to, GetPageSize()); + const i64 length = end - begin; + + File_.FlushCache(begin, length, /* wait = */ true); + File_.EvictCache(begin, length); + + GuaranteedWrittenPtr_ = to; + } + +private: + TMutex Lock_; + TFile File_; + + const size_t MaxBufferSize_ = 0; + const size_t MaxPendingCacheSize_ = 0; + + TBuffer Buffer_; + i64 WrittenPtr_ = 0; + i64 PageAlignedWrittenPtr_ = 0; + i64 GuaranteedWrittenPtr_ = 0; +}; + +TSyncPageCacheFileLogBackend::TSyncPageCacheFileLogBackend(const TString& path, size_t maxBufferSize, size_t maxPengingCacheSize) + : Impl_(MakeHolder<TImpl>(path, maxBufferSize, maxPengingCacheSize)) +{} + +TSyncPageCacheFileLogBackend::~TSyncPageCacheFileLogBackend() { +} + +void TSyncPageCacheFileLogBackend::WriteData(const TLogRecord& rec) { + Impl_->WriteData(rec); +} + +void TSyncPageCacheFileLogBackend::ReopenLog() { + Impl_->ReopenLog(); +} diff --git a/library/cpp/logger/sync_page_cache_file.h b/library/cpp/logger/sync_page_cache_file.h new file mode 100644 index 0000000000..a36340651c --- /dev/null +++ b/library/cpp/logger/sync_page_cache_file.h @@ -0,0 +1,19 @@ +#pragma once + +#include "backend.h" + +#include <util/generic/fwd.h> +#include <util/generic/ptr.h> + +class TSyncPageCacheFileLogBackend final: public TLogBackend { +public: + TSyncPageCacheFileLogBackend(const TString& path, size_t maxBufferSize, size_t maxPendingCacheSize); + ~TSyncPageCacheFileLogBackend(); + + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; diff --git a/library/cpp/logger/sync_page_cache_file_creator.cpp b/library/cpp/logger/sync_page_cache_file_creator.cpp new file mode 100644 index 0000000000..bf91d20c23 --- /dev/null +++ b/library/cpp/logger/sync_page_cache_file_creator.cpp @@ -0,0 +1,28 @@ +#include "sync_page_cache_file_creator.h" +#include "sync_page_cache_file.h" + +THolder<TLogBackend> TSyncPageCacheFileLogBackendCreator::DoCreateLogBackend() const { + return MakeHolder<TSyncPageCacheFileLogBackend>(Path, MaxBufferSize, MaxPendingCacheSize); +} + + +TSyncPageCacheFileLogBackendCreator::TSyncPageCacheFileLogBackendCreator() + : TFileLogBackendCreator("", "sync_page") +{} + +bool TSyncPageCacheFileLogBackendCreator::Init(const IInitContext& ctx) { + if (!TFileLogBackendCreator::Init(ctx)) { + return false; + } + ctx.GetValue("MaxBufferSize", MaxBufferSize); + ctx.GetValue("MaxPendingCacheSize", MaxPendingCacheSize); + return true; +} + +ILogBackendCreator::TFactory::TRegistrator<TSyncPageCacheFileLogBackendCreator> TSyncPageCacheFileLogBackendCreator::Registrar("sync_page"); + +void TSyncPageCacheFileLogBackendCreator::DoToJson(NJson::TJsonValue& value) const { + TFileLogBackendCreator::DoToJson(value); + value["MaxBufferSize"] = MaxBufferSize; + value["MaxPendingCacheSize"] = MaxPendingCacheSize; +} diff --git a/library/cpp/logger/sync_page_cache_file_creator.h b/library/cpp/logger/sync_page_cache_file_creator.h new file mode 100644 index 0000000000..7148150222 --- /dev/null +++ b/library/cpp/logger/sync_page_cache_file_creator.h @@ -0,0 +1,18 @@ +#pragma once + +#include "file_creator.h" + +class TSyncPageCacheFileLogBackendCreator : public TFileLogBackendCreator { +public: + TSyncPageCacheFileLogBackendCreator(); + virtual bool Init(const IInitContext& ctx) override; + static TFactory::TRegistrator<TSyncPageCacheFileLogBackendCreator> Registrar; + +protected: + virtual void DoToJson(NJson::TJsonValue& value) const override; + +private: + virtual THolder<TLogBackend> DoCreateLogBackend() const override; + size_t MaxBufferSize = Max<size_t>(); + size_t MaxPendingCacheSize = Max<size_t>(); +}; diff --git a/library/cpp/logger/system.cpp b/library/cpp/logger/system.cpp new file mode 100644 index 0000000000..42233f63d2 --- /dev/null +++ b/library/cpp/logger/system.cpp @@ -0,0 +1,87 @@ +#include <util/stream/output.h> +#include <util/stream/null.h> +#include <util/system/compat.h> +#include <util/system/yassert.h> +#include <util/system/defaults.h> +#include <util/generic/singleton.h> +#include <util/generic/utility.h> + +#if defined(_unix_) +#include <syslog.h> +#endif + +#include "system.h" +#include "record.h" +#include "stream.h" + +TSysLogBackend::TSysLogBackend(const char* ident, EFacility facility, int flags) + : Ident(ident) + , Facility(facility) + , Flags(flags) +{ +#if defined(_unix_) + Y_ASSERT(TSYSLOG_LOCAL0 <= facility && facility <= TSYSLOG_LOCAL7); + + static const int f2sf[] = { + LOG_LOCAL0, + LOG_LOCAL1, + LOG_LOCAL2, + LOG_LOCAL3, + LOG_LOCAL4, + LOG_LOCAL5, + LOG_LOCAL6, + LOG_LOCAL7}; + + int sysflags = LOG_NDELAY | LOG_PID; + + if (flags & LogPerror) { + sysflags |= LOG_PERROR; + } + + if (flags & LogCons) { + sysflags |= LOG_CONS; + } + + openlog(Ident.data(), sysflags, f2sf[(size_t)facility]); +#endif +} + +TSysLogBackend::~TSysLogBackend() { +#if defined(_unix_) + closelog(); +#endif +} + +void TSysLogBackend::WriteData(const TLogRecord& rec) { +#if defined(_unix_) + syslog(ELogPriority2SyslogPriority(rec.Priority), "%.*s", (int)rec.Len, rec.Data); +#else + Y_UNUSED(rec); +#endif +} + +void TSysLogBackend::ReopenLog() { +} + +int TSysLogBackend::ELogPriority2SyslogPriority(ELogPriority priority) { +#if defined(_unix_) + return Min(int(priority), (int)LOG_PRIMASK); +#else + // trivial conversion + return int(priority); +#endif +} + +namespace { + class TSysLogInstance: public TLog { + public: + inline TSysLogInstance() + : TLog(MakeHolder<TStreamLogBackend>(&Cnull)) + { + } + }; +} + +TLog& SysLogInstance() { + return *Singleton<TSysLogInstance>(); +} diff --git a/library/cpp/logger/system.h b/library/cpp/logger/system.h new file mode 100644 index 0000000000..b8c60b3023 --- /dev/null +++ b/library/cpp/logger/system.h @@ -0,0 +1,66 @@ +#pragma once + +#include "log.h" +#include "backend.h" +#include "priority.h" + +#define YSYSLOG(priority, ...) SysLogInstance().AddLog((priority), __VA_ARGS__) +#define YSYSLOGINIT_FLAGS(ident, facility, flags) \ + struct TLogIniter { \ + TLogIniter() { \ + SysLogInstance().ResetBackend(THolder<TLogBackend>( \ + (ident) ? (TLogBackend*)(new TSysLogBackend((ident), (facility), (flags))) : (TLogBackend*)(new TNullLogBackend())));\ + } \ + } Y_CAT(loginit, __LINE__); + +#define YSYSLOGINIT(ident, facility) YSYSLOGINIT_FLAGS((ident), (facility), 0) + +class TSysLogBackend: public TLogBackend { +public: + enum EFacility { + TSYSLOG_LOCAL0 = 0, + TSYSLOG_LOCAL1 = 1, + TSYSLOG_LOCAL2 = 2, + TSYSLOG_LOCAL3 = 3, + TSYSLOG_LOCAL4 = 4, + TSYSLOG_LOCAL5 = 5, + TSYSLOG_LOCAL6 = 6, + TSYSLOG_LOCAL7 = 7 + }; + + enum EFlags { + LogPerror = 1, + LogCons = 2 + }; + + TSysLogBackend(const char* ident, EFacility facility, int flags = 0); + ~TSysLogBackend() override; + + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override; + + virtual TString GetIdent() const { + return Ident; + } + + virtual EFacility GetFacility() const { + return Facility; + } + + virtual int GetFlags() const { + return Flags; + } + +protected: + int ELogPriority2SyslogPriority(ELogPriority priority); + + TString Ident; + EFacility Facility; + int Flags; +}; + +/* + * return system-wide logger instance + * better do not use in real programs(instead of robot, of course) + */ +TLog& SysLogInstance(); diff --git a/library/cpp/logger/system_creator.cpp b/library/cpp/logger/system_creator.cpp new file mode 100644 index 0000000000..e1cd02d422 --- /dev/null +++ b/library/cpp/logger/system_creator.cpp @@ -0,0 +1,25 @@ +#include "system_creator.h" + +THolder<TLogBackend> TSysLogBackendCreator::DoCreateLogBackend() const { + return MakeHolder<TSysLogBackend>(Ident.c_str(), Facility, Flags); +} + + +TSysLogBackendCreator::TSysLogBackendCreator() + : TLogBackendCreatorBase("system") +{} + +bool TSysLogBackendCreator::Init(const IInitContext& ctx) { + ctx.GetValue("Ident", Ident); + ctx.GetValue("Facility", (int&)Facility); + ctx.GetValue("Flags", Flags); + return true; +} + +ILogBackendCreator::TFactory::TRegistrator<TSysLogBackendCreator> TSysLogBackendCreator::Registrar("system"); + +void TSysLogBackendCreator::DoToJson(NJson::TJsonValue& value) const { + value["Ident"] = Ident; + value["Facility"] = (int&)Facility; + value["Flags"] = Flags; +} diff --git a/library/cpp/logger/system_creator.h b/library/cpp/logger/system_creator.h new file mode 100644 index 0000000000..e5321267e9 --- /dev/null +++ b/library/cpp/logger/system_creator.h @@ -0,0 +1,20 @@ +#pragma once + +#include "backend_creator.h" +#include "system.h" + +class TSysLogBackendCreator : public TLogBackendCreatorBase { +public: + TSysLogBackendCreator(); + virtual bool Init(const IInitContext& ctx) override; + static TFactory::TRegistrator<TSysLogBackendCreator> Registrar; + +protected: + virtual void DoToJson(NJson::TJsonValue& value) const override; + +private: + virtual THolder<TLogBackend> DoCreateLogBackend() const override; + TString Ident; + TSysLogBackend::EFacility Facility = TSysLogBackend::TSYSLOG_LOCAL0; + int Flags = 0; +}; diff --git a/library/cpp/logger/thread.cpp b/library/cpp/logger/thread.cpp new file mode 100644 index 0000000000..0ccf9e374b --- /dev/null +++ b/library/cpp/logger/thread.cpp @@ -0,0 +1,165 @@ +#include "thread.h" +#include "record.h" + +#include <util/thread/pool.h> +#include <util/system/event.h> +#include <util/memory/addstorage.h> +#include <util/generic/ptr.h> +#include <util/generic/yexception.h> + +class TThreadedLogBackend::TImpl { + class TRec: public IObjectInQueue, public TAdditionalStorage<TRec>, public TLogRecord { + public: + inline TRec(TImpl* parent, const TLogRecord& rec) + : TLogRecord(rec.Priority, (const char*)AdditionalData(), rec.Len) + , Parent_(parent) + { + memcpy(AdditionalData(), rec.Data, rec.Len); + } + + inline ~TRec() override { + } + + private: + void Process(void* /*tsr*/) override { + THolder<TRec> This(this); + + Parent_->Slave_->WriteData(*this); + } + + private: + TImpl* Parent_; + }; + + class TReopener: public IObjectInQueue, public TSystemEvent, public TAtomicRefCount<TReopener> { + public: + inline TReopener(TImpl* parent) + : Parent_(parent) + { + Ref(); + } + + inline ~TReopener() override { + } + + private: + void Process(void* /*tsr*/) override { + try { + Parent_->Slave_->ReopenLog(); + } catch (...) { + } + + Signal(); + UnRef(); + } + + private: + TImpl* Parent_; + }; + +public: + inline TImpl(TLogBackend* slave, size_t queuelen, std::function<void()> queueOverflowCallback = {}) + : Slave_(slave) + , QueueOverflowCallback_(std::move(queueOverflowCallback)) + { + Queue_.Start(1, queuelen); + } + + inline ~TImpl() { + Queue_.Stop(); + } + + inline void WriteData(const TLogRecord& rec) { + THolder<TRec> obj(new (rec.Len) TRec(this, rec)); + + if (Queue_.Add(obj.Get())) { + Y_UNUSED(obj.Release()); + return; + } + + if (QueueOverflowCallback_) { + QueueOverflowCallback_(); + } else { + ythrow yexception() << "log queue exhausted"; + } + } + + // Write an emergency message when the memory allocator is corrupted. + // The TThreadedLogBackend object can't be used after this method is called. + inline void WriteEmergencyData(const TLogRecord& rec) noexcept { + Queue_.Stop(); + Slave_->WriteData(rec); + } + + inline void ReopenLog() { + TIntrusivePtr<TReopener> reopener(new TReopener(this)); + + if (!Queue_.Add(reopener.Get())) { + reopener->UnRef(); // Ref() was called in constructor + ythrow yexception() << "log queue exhausted"; + } + + reopener->Wait(); + } + + inline void ReopenLogNoFlush() { + Slave_->ReopenLogNoFlush(); + } + + inline size_t QueueSize() const { + return Queue_.Size(); + } + +private: + TLogBackend* Slave_; + TThreadPool Queue_{"ThreadedLogBack"}; + const std::function<void()> QueueOverflowCallback_; +}; + +TThreadedLogBackend::TThreadedLogBackend(TLogBackend* slave) + : Impl_(new TImpl(slave, 0)) +{ +} + +TThreadedLogBackend::TThreadedLogBackend(TLogBackend* slave, size_t queuelen, std::function<void()> queueOverflowCallback) + : Impl_(new TImpl(slave, queuelen, std::move(queueOverflowCallback))) +{ +} + +TThreadedLogBackend::~TThreadedLogBackend() { +} + +void TThreadedLogBackend::WriteData(const TLogRecord& rec) { + Impl_->WriteData(rec); +} + +void TThreadedLogBackend::ReopenLog() { + Impl_->ReopenLog(); +} + +void TThreadedLogBackend::ReopenLogNoFlush() { + Impl_->ReopenLogNoFlush(); +} + +void TThreadedLogBackend::WriteEmergencyData(const TLogRecord& rec) { + Impl_->WriteEmergencyData(rec); +} + +size_t TThreadedLogBackend::QueueSize() const { + return Impl_->QueueSize(); +} + +TOwningThreadedLogBackend::TOwningThreadedLogBackend(TLogBackend* slave) + : THolder<TLogBackend>(slave) + , TThreadedLogBackend(Get()) +{ +} + +TOwningThreadedLogBackend::TOwningThreadedLogBackend(TLogBackend* slave, size_t queuelen, std::function<void()> queueOverflowCallback) + : THolder<TLogBackend>(slave) + , TThreadedLogBackend(Get(), queuelen, std::move(queueOverflowCallback)) +{ +} + +TOwningThreadedLogBackend::~TOwningThreadedLogBackend() { +} diff --git a/library/cpp/logger/thread.h b/library/cpp/logger/thread.h new file mode 100644 index 0000000000..65f7a88e87 --- /dev/null +++ b/library/cpp/logger/thread.h @@ -0,0 +1,34 @@ +#pragma once + +#include "backend.h" + +#include <util/generic/ptr.h> + +#include <functional> + +class TThreadedLogBackend: public TLogBackend { +public: + TThreadedLogBackend(TLogBackend* slave); + TThreadedLogBackend(TLogBackend* slave, size_t queuelen, std::function<void()> queueOverflowCallback = {}); + ~TThreadedLogBackend() override; + + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override; + void ReopenLogNoFlush() override; + size_t QueueSize() const override; + + // Write an emergency message when the memory allocator is corrupted. + // The TThreadedLogBackend object can't be used after this method is called. + void WriteEmergencyData(const TLogRecord& rec); + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +class TOwningThreadedLogBackend: private THolder<TLogBackend>, public TThreadedLogBackend { +public: + TOwningThreadedLogBackend(TLogBackend* slave); + TOwningThreadedLogBackend(TLogBackend* slave, size_t queuelen, std::function<void()> queueOverflowCallback = {}); + ~TOwningThreadedLogBackend() override; +}; diff --git a/library/cpp/logger/thread_creator.cpp b/library/cpp/logger/thread_creator.cpp new file mode 100644 index 0000000000..8f5cfe2782 --- /dev/null +++ b/library/cpp/logger/thread_creator.cpp @@ -0,0 +1,30 @@ +#include "thread_creator.h" +#include "thread.h" + +TOwningThreadedLogBackendCreator::TOwningThreadedLogBackendCreator(THolder<ILogBackendCreator>&& slave) + : Slave(std::move(slave)) +{} + +THolder<TLogBackend> TOwningThreadedLogBackendCreator::DoCreateLogBackend() const { + return MakeHolder<TOwningThreadedLogBackend>(Slave->CreateLogBackend().Release(), QueueLen, QueueOverflowCallback); +} + +bool TOwningThreadedLogBackendCreator::Init(const IInitContext& ctx) { + ctx.GetValue("QueueLen", QueueLen); + return Slave->Init(ctx); +} + + +void TOwningThreadedLogBackendCreator::ToJson(NJson::TJsonValue& value) const { + value["QueueLen"] = QueueLen; + value["Threaded"] = true; + Slave->ToJson(value); +} + +void TOwningThreadedLogBackendCreator::SetQueueOverflowCallback(std::function<void()> callback) { + QueueOverflowCallback = std::move(callback); +} + +void TOwningThreadedLogBackendCreator::SetQueueLen(size_t len) { + QueueLen = len; +} diff --git a/library/cpp/logger/thread_creator.h b/library/cpp/logger/thread_creator.h new file mode 100644 index 0000000000..9e5ee9571d --- /dev/null +++ b/library/cpp/logger/thread_creator.h @@ -0,0 +1,20 @@ +#pragma once + +#include "backend_creator.h" + +#include <functional> + +class TOwningThreadedLogBackendCreator: public ILogBackendCreator { +public: + TOwningThreadedLogBackendCreator(THolder<ILogBackendCreator>&& slave); + virtual bool Init(const IInitContext& ctx) override; + virtual void ToJson(NJson::TJsonValue& value) const override; + void SetQueueOverflowCallback(std::function<void()> callback); + void SetQueueLen(size_t len); + +private: + virtual THolder<TLogBackend> DoCreateLogBackend() const override; + THolder<ILogBackendCreator> Slave; + std::function<void()> QueueOverflowCallback = {}; + size_t QueueLen = 0; +}; diff --git a/library/cpp/logger/uninitialized_creator.cpp b/library/cpp/logger/uninitialized_creator.cpp new file mode 100644 index 0000000000..26dd168529 --- /dev/null +++ b/library/cpp/logger/uninitialized_creator.cpp @@ -0,0 +1,50 @@ +#include "uninitialized_creator.h" +#include "filter_creator.h" +#include "thread_creator.h" +#include "file_creator.h" +#include "null_creator.h" +#include <util/string/cast.h> + +THolder<TLogBackend> TLogBackendCreatorUninitialized::DoCreateLogBackend() const { + return Slave->CreateLogBackend(); +} + +void TLogBackendCreatorUninitialized::InitCustom(const TString& type, ELogPriority priority, bool threaded) { + if (!type) { + Slave = MakeHolder<TNullLogBackendCreator>(); + } else if (TFactory::Has(type)) { + Slave = TFactory::MakeHolder(type); + } else { + Slave = MakeHolder<TFileLogBackendCreator>(type); + } + + if (threaded) { + Slave = MakeHolder<TOwningThreadedLogBackendCreator>(std::move(Slave)); + } + + if (priority != LOG_MAX_PRIORITY) { + Slave = MakeHolder<TFilteredBackendCreator>(std::move(Slave), priority); + } +} + +bool TLogBackendCreatorUninitialized::Init(const IInitContext& ctx) { + auto type = ctx.GetOrElse("LoggerType", TString()); + bool threaded = ctx.GetOrElse("Threaded", false); + ELogPriority priority = LOG_MAX_PRIORITY; + TString prStr; + if (ctx.GetValue("LogLevel", prStr)) { + if (!TryFromString(prStr, priority)) { + priority = (ELogPriority)FromString<int>(prStr); + } + } + InitCustom(type, priority, threaded); + return Slave->Init(ctx); +} + + +void TLogBackendCreatorUninitialized::ToJson(NJson::TJsonValue& value) const { + Y_VERIFY(Slave, "Serialization off uninitialized LogBackendCreator"); + Slave->ToJson(value); +} + +ILogBackendCreator::TFactory::TRegistrator<TLogBackendCreatorUninitialized> TLogBackendCreatorUninitialized::Registrar(""); diff --git a/library/cpp/logger/uninitialized_creator.h b/library/cpp/logger/uninitialized_creator.h new file mode 100644 index 0000000000..16ee0b92f6 --- /dev/null +++ b/library/cpp/logger/uninitialized_creator.h @@ -0,0 +1,16 @@ +#pragma once + +#include "backend_creator.h" + +class TLogBackendCreatorUninitialized : public ILogBackendCreator { +public: + void InitCustom(const TString& type, ELogPriority priority, bool threaded); + virtual bool Init(const IInitContext& ctx) override; + virtual void ToJson(NJson::TJsonValue& value) const override; + + static TFactory::TRegistrator<TLogBackendCreatorUninitialized> Registrar; + +private: + virtual THolder<TLogBackend> DoCreateLogBackend() const override; + THolder<ILogBackendCreator> Slave; +}; diff --git a/library/cpp/logger/ut/ya.make b/library/cpp/logger/ut/ya.make new file mode 100644 index 0000000000..2a461c1353 --- /dev/null +++ b/library/cpp/logger/ut/ya.make @@ -0,0 +1,20 @@ +UNITTEST() + +OWNER(pg) + +PEERDIR( + ADDINCL library/cpp/logger + library/cpp/logger/init_context + library/cpp/yconf/patcher +) + +SRCDIR(library/cpp/logger) + +SRCS( + log_ut.cpp + element_ut.cpp + rotating_file_ut.cpp + composite_ut.cpp +) + +END() diff --git a/library/cpp/logger/ya.make b/library/cpp/logger/ya.make new file mode 100644 index 0000000000..00a5263cba --- /dev/null +++ b/library/cpp/logger/ya.make @@ -0,0 +1,47 @@ +OWNER( + pg + mvel + g:util + g:base +) + +LIBRARY() + +GENERATE_ENUM_SERIALIZATION(priority.h) + +PEERDIR ( + library/cpp/json +) + +SRCS( + all.h + backend.cpp + backend_creator.cpp + composite.cpp + GLOBAL composite_creator.cpp + element.cpp + file.cpp + GLOBAL file_creator.cpp + filter.cpp + filter_creator.cpp + log.cpp + null.cpp + GLOBAL null_creator.cpp + priority.h + record.h + rotating_file.cpp + GLOBAL rotating_file_creator.cpp + stream.cpp + GLOBAL stream_creator.cpp + sync_page_cache_file.cpp + GLOBAL sync_page_cache_file_creator.cpp + system.cpp + GLOBAL system_creator.cpp + thread.cpp + thread_creator.cpp + GLOBAL uninitialized_creator.cpp +) + +END() + +RECURSE_FOR_TESTS(ut) |