1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
|
#pragma once
#include <library/cpp/unified_agent_client/f_maybe.h>
#include <library/cpp/unified_agent_client/throttling.h>
#include <library/cpp/unified_agent_client/formatters.h>
#include <library/cpp/logger/log.h>
#include <library/cpp/monlib/dynamic_counters/counters.h>
#include <util/generic/string.h>
#include <util/string/join.h>
#include <util/string/printf.h>
#include <util/system/file.h>
#include <atomic>
#include <format>
#define YLOG(logPriority, message, logger) \
do { \
const auto __logPriority = logPriority; \
if (auto* log = logger.Accept(__logPriority, false); log != nullptr) { \
logger.Log(*log, __logPriority, message); \
} \
} while (false)
// Logging with throttling
#define YLOG_T(logger, logPriority, ...) \
do { \
const auto _logPriority = logPriority; \
static ::NUnifiedAgent::TLogger::TThrottleState _throttleState; \
auto [_log, _dropped] = logger.Accept(_logPriority, false, _throttleState); \
if (_log != nullptr) { \
if (_dropped > 0) { \
logger.Log(*_log, _logPriority, std::format("{} [+{} suppressed]", \
::NUnifiedAgent::YLogFormatMessage(__VA_ARGS__), _dropped)); \
} else { \
logger.Log(*_log, _logPriority, ::NUnifiedAgent::YLogFormatMessage(__VA_ARGS__)); \
} \
} \
} while (false)
#define YLOG_EMERG(msg) YLOG(TLOG_EMERG, msg, Logger)
#define YLOG_ALERT(msg) YLOG(TLOG_ALERT, msg, Logger)
#define YLOG_CRIT(msg) YLOG(TLOG_CRIT, msg, Logger)
#define YLOG_ERR(msg) YLOG(TLOG_ERR, msg, Logger)
#define YLOG_WARNING(msg) YLOG(TLOG_WARNING, msg, Logger)
#define YLOG_NOTICE(msg) YLOG(TLOG_NOTICE, msg, Logger)
#define YLOG_INFO(msg) YLOG(TLOG_INFO, msg, Logger)
#define YLOG_DEBUG(msg) YLOG(TLOG_DEBUG, msg, Logger)
#define YLOG_RESOURCES(msg) YLOG(TLOG_RESOURCES , msg, Logger)
#define YLOG_FATAL(msg) \
YLOG(TLOG_CRIT, msg, Logger); \
_Exit(1);
#define YLOG_EMERG_F(fmt, ...) YLOG_EMERG(std::format(fmt, __VA_ARGS__))
#define YLOG_ALERT_F(fmt, ...) YLOG_ALERT(std::format(fmt, __VA_ARGS__))
#define YLOG_CRIT_F(fmt, ...) YLOG_CRIT(std::format(fmt, __VA_ARGS__))
#define YLOG_ERR_F(fmt, ...) YLOG_ERR(std::format(fmt, __VA_ARGS__))
#define YLOG_WARNING_F(fmt, ...) YLOG_WARNING(std::format(fmt, __VA_ARGS__))
#define YLOG_NOTICE_F(fmt, ...) YLOG_NOTICE(std::format(fmt, __VA_ARGS__))
#define YLOG_INFO_F(fmt, ...) YLOG_INFO(std::format(fmt, __VA_ARGS__))
#define YLOG_DEBUG_F(fmt, ...) YLOG_DEBUG(std::format(fmt, __VA_ARGS__))
#define YLOG_RESOURCES_F(fmt, ...) YLOG_RESOURCES(std::format(fmt, __VA_ARGS__))
#define YLOG_FATAL_F(fmt, ...) YLOG_FATAL(std::format(fmt, __VA_ARGS__))
// Logging with throttling and metrics increment:
// Errors for ERR+ priorities, RecordsReceived/RecordsDropped
#define YLOG_EMERG_T(...) YLOG_T(Logger, TLOG_EMERG, __VA_ARGS__)
#define YLOG_ALERT_T(...) YLOG_T(Logger, TLOG_ALERT, __VA_ARGS__)
#define YLOG_CRITICAL_T(...) YLOG_T(Logger, TLOG_CRIT, __VA_ARGS__)
#define YLOG_ERROR_T(...) YLOG_T(Logger, TLOG_ERR, __VA_ARGS__)
#define YLOG_WARNING_T(...) YLOG_T(Logger, TLOG_WARNING, __VA_ARGS__)
#define YLOG_NOTICE_T(...) YLOG_T(Logger, TLOG_NOTICE, __VA_ARGS__)
#define YLOG_INFO_T(...) YLOG_T(Logger, TLOG_INFO, __VA_ARGS__)
#define YLOG_DEBUG_T(...) YLOG_T(Logger, TLOG_DEBUG, __VA_ARGS__)
#define YLOG_RESOURCES_T(...) YLOG_T(Logger, TLOG_RESOURCES , __VA_ARGS__)
namespace NUnifiedAgent {
// Helper functions to format messages
template <typename T>
decltype(auto) YLogFormatMessage(T&& msg) {
return std::forward<T>(msg);
}
// Overload for multiple arguments (with formatting)
template <typename... Args>
std::string YLogFormatMessage(std::format_string<Args...> fmt, Args&&... args) {
return std::format(fmt, std::forward<Args>(args)...);
}
class TScopeLogger;
class TLogger {
public:
// These counters must outlive the TLogger instance
struct TCounters {
NMonitoring::TDeprecatedCounter* Errors;
NMonitoring::TDeprecatedCounter* DroppedBytes;
NMonitoring::TDeprecatedCounter* RecordsReceived;
NMonitoring::TDeprecatedCounter* RecordsDropped;
};
// Throttling state for a single log location
struct TThrottleState {
std::atomic<ui32> TimeSlot{0};
std::atomic<ui32> Logged{0};
std::atomic<ui32> Dropped{0};
};
TLogger(TLog& log, TFMaybe<size_t> rateLimitBytes, TCounters counters = {});
TLogger(TLog& log, TFMaybe<size_t> rateLimitBytes, ui32 maxLogsPerSlot, ui32 slotSeconds, TCounters counters = {});
virtual ~TLogger() = default;
void AddLog(TLog& log);
void StartTracing(ELogPriority logPriority) noexcept;
void FinishTracing() noexcept;
TScopeLogger Child(const TString& v, NMonitoring::TDeprecatedCounter* errors = nullptr);
bool HasRateLimit() const noexcept {
return Throttler_ != nullptr;
}
friend class TScopeLogger;
protected:
// Accept with throttling support
// Returns: pair of (TLog* if should log or nullptr if dropped, number of previously suppressed logs)
std::pair<TLog*, ui32> Accept(ELogPriority logPriority, NMonitoring::TDeprecatedCounter* errors, TThrottleState& state) const;
virtual void Log(TLog& log, ELogPriority logPriority, const TStringBuf message, const TString& scope) const;
TLog* Accept(ELogPriority logPriority, NMonitoring::TDeprecatedCounter* errors) const noexcept {
if ((logPriority <= TLOG_ERR) && (errors != nullptr)) {
++(*errors);
}
auto* result = CurrentLogContext_.load(std::memory_order_acquire);
return result != nullptr && static_cast<int>(logPriority) <= static_cast<int>(result->Priority)
? &result->Log
: nullptr;
}
private:
struct TLogContext {
TLog Log;
ELogPriority Priority;
};
class TThrottlerWithLock {
public:
explicit TThrottlerWithLock(size_t rateLimitBytes);
bool TryConsume(double tokens);
private:
TThrottler Throttler_;
TAdaptiveLock Lock_;
};
private:
void SetCurrentLogContext(TLogContext& logContext);
TLogContext& GetOrCreateTracingLogContext(ELogPriority logPriority);
void SetTracing(TLogContext& logContext, const char* action);
private:
TLogContext DefaultLogContext_;
TVector<THolder<TLogContext>> TracingLogContexts_;
std::atomic<TLogContext*> CurrentLogContext_;
TCounters Counters_;
const ui32 MaxLogsPerSlot_;
const ui32 SlotSeconds_;
const THolder<TThrottlerWithLock> Throttler_;
TAdaptiveLock Lock_;
TVector<TLog> AdditionalLogs_;
};
class TScopeLogger {
public:
TScopeLogger();
void Log(TLog& log, ELogPriority logPriority, const TStringBuf message) const {
if (Logger_) {
Logger_->Log(log, logPriority, message, Scope_);
}
}
TLog* Accept(ELogPriority logPriority, bool silent) const noexcept {
return Logger_ ? Logger_->Accept(logPriority, silent ? nullptr : Errors_) : nullptr;
}
// Accept with throttling support
std::pair<TLog*, ui32> Accept(ELogPriority logPriority, bool silent, TLogger::TThrottleState& state) const {
return Logger_
? Logger_->Accept(logPriority, silent ? nullptr : Errors_, state)
: std::make_pair(nullptr, 0);
}
TScopeLogger Child(const TString& v, NMonitoring::TDeprecatedCounter* errors = nullptr) const;
TLogger* Unwrap() const noexcept {
return Logger_;
}
ELogPriority FiltrationLevel() const {
if (Logger_) {
return Logger_->CurrentLogContext_.load()->Priority;
}
return LOG_DEF_PRIORITY;
}
friend class TLogger;
private:
TScopeLogger(TLogger* logger, TString scope, NMonitoring::TDeprecatedCounter* errors);
private:
TLogger* Logger_;
TString Scope_;
NMonitoring::TDeprecatedCounter* Errors_;
};
} // namespace NUnifiedAgent
|
