aboutsummaryrefslogtreecommitdiffstats
path: root/library/cpp
diff options
context:
space:
mode:
authorAlexander Smirnov <alex@ydb.tech>2024-06-03 08:43:13 +0000
committerAlexander Smirnov <alex@ydb.tech>2024-06-03 08:43:13 +0000
commite8b793b8275c229a46b4211f83c03cb074c9ba7c (patch)
tree831d4507c2b234635f45b573fcd03a4b96133bb5 /library/cpp
parent89c3654dc08129116422f621b50086305f68bc9c (diff)
parent28ff234f758e7fd2f42f229a4b55ff70506f0015 (diff)
downloadydb-e8b793b8275c229a46b4211f83c03cb074c9ba7c.tar.gz
Merge branch 'rightlib' into mergelibs-240603-0842
Diffstat (limited to 'library/cpp')
-rw-r--r--library/cpp/histogram/adaptive/adaptive_histogram.cpp22
-rw-r--r--library/cpp/histogram/adaptive/block_histogram.cpp22
-rw-r--r--library/cpp/http/io/headers.h1
-rw-r--r--library/cpp/http/simple/http_client.cpp19
-rw-r--r--library/cpp/http/simple/http_client.h7
-rw-r--r--library/cpp/http/simple/http_client_options.h10
-rw-r--r--library/cpp/http/simple/ut/http_ut.cpp135
-rw-r--r--library/cpp/ipv6_address/ipv6_address.cpp16
-rw-r--r--library/cpp/ipv6_address/ipv6_address.h10
-rw-r--r--library/cpp/ipv6_address/ut/ipv6_address_ut.cpp20
-rw-r--r--library/cpp/protobuf/yql/ya.make2
-rw-r--r--library/cpp/tld/tlds-alpha-by-domain.txt2
-rw-r--r--library/cpp/yt/logging/logger-inl.h2
-rw-r--r--library/cpp/yt/logging/logger.h26
-rw-r--r--library/cpp/yt/logging/static_analysis-inl.h139
-rw-r--r--library/cpp/yt/logging/static_analysis.h22
-rw-r--r--library/cpp/yt/logging/unittests/static_analysis_ut.cpp41
-rw-r--r--library/cpp/yt/logging/unittests/ya.make1
-rw-r--r--library/cpp/yt/misc/source_location.cpp26
-rw-r--r--library/cpp/yt/misc/source_location.h18
-rw-r--r--library/cpp/yt/string/format_analyser-inl.h143
-rw-r--r--library/cpp/yt/string/format_analyser.h102
22 files changed, 756 insertions, 30 deletions
diff --git a/library/cpp/histogram/adaptive/adaptive_histogram.cpp b/library/cpp/histogram/adaptive/adaptive_histogram.cpp
index 4bc4f63cda0..bfe76330e08 100644
--- a/library/cpp/histogram/adaptive/adaptive_histogram.cpp
+++ b/library/cpp/histogram/adaptive/adaptive_histogram.cpp
@@ -7,6 +7,8 @@
#include <util/system/backtrace.h>
+#include <format>
+
namespace NKiwiAggr {
TAdaptiveHistogram::TAdaptiveHistogram(size_t intervals, ui64 id, TQualityFunction qualityFunc)
: Id(id)
@@ -71,7 +73,10 @@ namespace NKiwiAggr {
void TAdaptiveHistogram::Merge(const THistogram& histo, double multiplier) {
if (!IsValidFloat(histo.GetMinValue()) || !IsValidFloat(histo.GetMaxValue())) {
- fprintf(stderr, "Merging in histogram id %lu: skip bad histo with minvalue %f maxvalue %f\n", Id, histo.GetMinValue(), histo.GetMaxValue());
+ Cerr << std::format(
+ "Merging in histogram id {}: skip bad histo with minvalue {} maxvalue {}\n",
+ Id, histo.GetMinValue(), histo.GetMaxValue()
+ );
return;
}
if (histo.FreqSize() == 0) {
@@ -87,7 +92,10 @@ namespace NKiwiAggr {
double value = histo.GetPosition(j);
double weight = histo.GetFreq(j);
if (!IsValidFloat(value) || !IsValidFloat(weight)) {
- fprintf(stderr, "Merging in histogram id %lu: skip bad value %f weight %f\n", Id, value, weight);
+ Cerr << std::format(
+ "Merging in histogram id {}: skip bad value {} weight {}\n",
+ Id, value, weight
+ );
continue;
}
Add(value, weight * multiplier);
@@ -100,7 +108,10 @@ namespace NKiwiAggr {
for (size_t j = 0; j < histo.FreqSize(); ++j) {
double weight = histo.GetFreq(j);
if (!IsValidFloat(pos) || !IsValidFloat(weight)) {
- fprintf(stderr, "Merging in histogram id %lu: skip bad value %f weight %f\n", Id, pos, weight);
+ Cerr << std::format(
+ "Merging in histogram id {}: skip bad value {} weight {}\n",
+ Id, pos, weight
+ );
pos += histo.GetBinRange();
continue;
}
@@ -218,7 +229,10 @@ namespace NKiwiAggr {
double value = histo.GetPosition(i);
double weight = histo.GetFreq(i);
if (!IsValidFloat(value) || !IsValidFloat(weight)) {
- fprintf(stderr, "FromProto in histogram id %lu: skip bad value %f weight %f\n", Id, value, weight);
+ Cerr << std::format(
+ "FromProto in histogram id {}: skip bad value {} weight {}\n",
+ Id, value, weight
+ );
continue;
}
Add(value, weight);
diff --git a/library/cpp/histogram/adaptive/block_histogram.cpp b/library/cpp/histogram/adaptive/block_histogram.cpp
index 55337488ac1..ff0383761fa 100644
--- a/library/cpp/histogram/adaptive/block_histogram.cpp
+++ b/library/cpp/histogram/adaptive/block_histogram.cpp
@@ -10,6 +10,8 @@
#include <util/generic/ymath.h>
#include <util/string/printf.h>
+#include <format>
+
namespace {
struct TEmpty {
};
@@ -138,7 +140,10 @@ namespace NKiwiAggr {
void TBlockHistogram::Merge(const THistogram& histo, double multiplier) {
if (!IsValidFloat(histo.GetMinValue()) || !IsValidFloat(histo.GetMaxValue())) {
- fprintf(stderr, "Merging in histogram id %lu: skip bad histo with minvalue %f maxvalue %f\n", Id, histo.GetMinValue(), histo.GetMaxValue());
+ Cerr << std::format(
+ "Merging in histogram id {}: skip bad histo with minvalue {} maxvalue {}\n",
+ Id, histo.GetMinValue(), histo.GetMaxValue()
+ );
return;
}
if (histo.FreqSize() == 0) {
@@ -154,7 +159,10 @@ namespace NKiwiAggr {
double value = histo.GetPosition(j);
double weight = histo.GetFreq(j);
if (!IsValidFloat(value) || !IsValidFloat(weight)) {
- fprintf(stderr, "Merging in histogram id %lu: skip bad value %f weight %f\n", Id, value, weight);
+ Cerr << std::format(
+ "Merging in histogram id {}: skip bad value {} weight {}\n",
+ Id, value, weight
+ );
continue;
}
Add(value, weight * multiplier);
@@ -167,7 +175,10 @@ namespace NKiwiAggr {
for (size_t j = 0; j < histo.FreqSize(); ++j) {
double weight = histo.GetFreq(j);
if (!IsValidFloat(pos) || !IsValidFloat(weight)) {
- fprintf(stderr, "Merging in histogram id %lu: skip bad value %f weight %f\n", Id, pos, weight);
+ Cerr << std::format(
+ "Merging in histogram id {}: skip bad value {} weight {}\n",
+ Id, pos, weight
+ );
pos += histo.GetBinRange();
continue;
}
@@ -229,7 +240,10 @@ namespace NKiwiAggr {
double value = histo.GetPosition(i);
double weight = histo.GetFreq(i);
if (!IsValidFloat(value) || !IsValidFloat(weight)) {
- fprintf(stderr, "FromProto in histogram id %lu: skip bad value %f weight %f\n", Id, value, weight);
+ Cerr << std::format(
+ "FromProto in histogram id {}: skip bad value {} weight {}\n",
+ Id, value, weight
+ );
continue;
}
Bins[i].first = value;
diff --git a/library/cpp/http/io/headers.h b/library/cpp/http/io/headers.h
index 3ae40683e2d..72d3dec4bb0 100644
--- a/library/cpp/http/io/headers.h
+++ b/library/cpp/http/io/headers.h
@@ -4,7 +4,6 @@
#include <util/generic/deque.h>
#include <util/generic/string.h>
#include <util/generic/strbuf.h>
-#include <util/generic/vector.h> // XXX unused - remove after fixing transitive includes.
#include <util/string/cast.h>
class IInputStream;
diff --git a/library/cpp/http/simple/http_client.cpp b/library/cpp/http/simple/http_client.cpp
index 818dc048ad1..342236f9a33 100644
--- a/library/cpp/http/simple/http_client.cpp
+++ b/library/cpp/http/simple/http_client.cpp
@@ -301,8 +301,14 @@ TKeepAliveHttpClient TSimpleHttpClient::CreateClient() const {
void TSimpleHttpClient::PrepareClient(TKeepAliveHttpClient&) const {
}
+TRedirectableHttpClient::TRedirectableHttpClient(const TOptions& options)
+ : TSimpleHttpClient(options)
+ , Opts(options)
+{
+}
+
TRedirectableHttpClient::TRedirectableHttpClient(const TString& host, ui32 port, TDuration socketTimeout, TDuration connectTimeout)
- : TSimpleHttpClient(host, port, socketTimeout, connectTimeout)
+ : TRedirectableHttpClient(TOptions().Host(host).Port(port).SocketTimeout(socketTimeout).ConnectTimeout(connectTimeout))
{
}
@@ -315,6 +321,10 @@ void TRedirectableHttpClient::PrepareClient(TKeepAliveHttpClient& cl) const {
void TRedirectableHttpClient::ProcessResponse(const TStringBuf relativeUrl, THttpInput& input, IOutputStream* output, const unsigned statusCode) const {
for (auto i = input.Headers().Begin(), e = input.Headers().End(); i != e; ++i) {
if (0 == TString::compare(i->Name(), TStringBuf("Location"))) {
+ if (Opts.MaxRedirectCount() == 0) {
+ ythrow THttpRequestException(statusCode) << "Exceeds MaxRedirectCount limit, code " << statusCode << " at " << Host << relativeUrl;
+ }
+
TVector<TString> request_url_parts, request_body_parts;
size_t splitted_index = 0;
@@ -339,7 +349,12 @@ void TRedirectableHttpClient::ProcessResponse(const TStringBuf relativeUrl, THtt
}
}
- TRedirectableHttpClient cl(url, port, TDuration::Seconds(60), TDuration::Seconds(60));
+ auto opts = Opts;
+ opts.Host(url);
+ opts.Port(port);
+ opts.MaxRedirectCount(opts.MaxRedirectCount() - 1);
+
+ TRedirectableHttpClient cl(opts);
if (HttpsVerification) {
cl.EnableVerificationForHttps();
}
diff --git a/library/cpp/http/simple/http_client.h b/library/cpp/http/simple/http_client.h
index c01b11ba43f..eab92d42da5 100644
--- a/library/cpp/http/simple/http_client.h
+++ b/library/cpp/http/simple/http_client.h
@@ -185,12 +185,19 @@ private:
class TRedirectableHttpClient: public TSimpleHttpClient {
public:
+ using TOptions = TSimpleHttpClientOptions;
+
+ explicit TRedirectableHttpClient(const TOptions& options);
+
TRedirectableHttpClient(const TString& host, ui32 port, TDuration socketTimeout = TDuration::Seconds(5),
TDuration connectTimeout = TDuration::Seconds(30));
private:
void PrepareClient(TKeepAliveHttpClient& cl) const override;
void ProcessResponse(const TStringBuf relativeUrl, THttpInput& input, IOutputStream* output, const unsigned statusCode) const override;
+
+private:
+ TOptions Opts;
};
namespace NPrivate {
diff --git a/library/cpp/http/simple/http_client_options.h b/library/cpp/http/simple/http_client_options.h
index f2e964a462a..603ca5103af 100644
--- a/library/cpp/http/simple/http_client_options.h
+++ b/library/cpp/http/simple/http_client_options.h
@@ -51,9 +51,19 @@ public:
return ConnectTimeout_;
}
+ TSelf& MaxRedirectCount(int count) {
+ MaxRedirectCount_ = count;
+ return *this;
+ }
+
+ ui16 MaxRedirectCount() const noexcept {
+ return MaxRedirectCount_;
+ }
+
private:
TString Host_;
ui16 Port_;
TDuration SocketTimeout_ = TDuration::Seconds(5);
TDuration ConnectTimeout_ = TDuration::Seconds(30);
+ int MaxRedirectCount_ = INT_MAX;
};
diff --git a/library/cpp/http/simple/ut/http_ut.cpp b/library/cpp/http/simple/ut/http_ut.cpp
index bf7e767428e..7768fdc4faa 100644
--- a/library/cpp/http/simple/ut/http_ut.cpp
+++ b/library/cpp/http/simple/ut/http_ut.cpp
@@ -71,6 +71,76 @@ Y_UNIT_TEST_SUITE(SimpleHttp) {
}
};
+ class TScenario {
+ public:
+ struct TElem {
+ TString Url;
+ int Status = HTTP_OK;
+ TString Content{};
+ };
+
+ TScenario(const TVector<TElem>& seq, ui16 port = 80, TDuration sleep = TDuration())
+ : Seq_(seq)
+ , Sleep_(sleep)
+ , Port_(port)
+ {
+ }
+
+ bool DoReply(const TRequestReplier::TReplyParams& params, TRequestReplier* replier) {
+ const auto parsed = TParsedHttpFull(params.Input.FirstLine());
+ const auto url = parsed.Request;
+ params.Input.ReadAll();
+
+ UNIT_ASSERT(SeqIdx_ < Seq_.size());
+ auto& elem = Seq_[SeqIdx_++];
+
+ UNIT_ASSERT_VALUES_EQUAL(elem.Url, url);
+
+ Sleep(Sleep_);
+
+ if (elem.Status == -1) {
+ replier->ResetConnection(); // RST / ECONNRESET
+ return true;
+ }
+
+ THttpResponse resp((HttpCodes)elem.Status);
+
+ if (elem.Status >= 300 && elem.Status < 400) {
+ UNIT_ASSERT(SeqIdx_ < Seq_.size());
+ resp.AddHeader("Location", TStringBuilder() << "http://localhost:" << Port_ << Seq_[SeqIdx_].Url);
+ }
+
+ resp.SetContent(elem.Content);
+ resp.OutTo(params.Output);
+
+ return true;
+ }
+
+ void VerifyInvariants() {
+ UNIT_ASSERT_VALUES_EQUAL(SeqIdx_, Seq_.size());
+ }
+
+ private:
+ TVector<TElem> Seq_;
+ size_t SeqIdx_ = 0;
+ TDuration Sleep_;
+ ui16 Port_;
+ };
+
+ class TScenarioReplier: public TRequestReplier {
+ TScenario* Scenario_ = nullptr;
+
+ public:
+ TScenarioReplier(TScenario* scenario)
+ : Scenario_(scenario)
+ {
+ }
+
+ bool DoReply(const TReplyParams& params) override {
+ return Scenario_->DoReply(params, this);
+ }
+ };
+
class TCodedPong: public TRequestReplier {
HttpCodes Code_;
@@ -129,6 +199,32 @@ Y_UNIT_TEST_SUITE(SimpleHttp) {
}
};
+ static void TestRedirectCountParam(int maxRedirectCount, int redirectCount) {
+ TPortManager pm;
+ ui16 port = pm.GetPort(80);
+
+ TVector<TScenario::TElem> steps;
+ for (int i = 0; i < redirectCount; ++i) {
+ steps.push_back({"/any", 302});
+ }
+ steps.push_back({"/any", 200, "Hello"});
+ TScenario scenario(steps, port);
+
+ NMock::TMockServer server(createOptions(port, true), [&scenario]() { return new TScenarioReplier(&scenario); });
+
+ TRedirectableHttpClient cl(TSimpleHttpClientOptions().Host("localhost").Port(port).MaxRedirectCount(maxRedirectCount));
+ UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
+
+ TStringStream s;
+ if (maxRedirectCount >= redirectCount) {
+ UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/any", &s));
+ UNIT_ASSERT_VALUES_EQUAL("Hello", s.Str());
+ scenario.VerifyInvariants();
+ } else {
+ UNIT_ASSERT_EXCEPTION_CONTAINS(cl.DoGet("/any", &s), THttpRequestException, "");
+ }
+ }
+
Y_UNIT_TEST(simpleSuccessful) {
TPortManager pm;
ui16 port = pm.GetPort(80);
@@ -274,6 +370,45 @@ Y_UNIT_TEST_SUITE(SimpleHttp) {
}
}
+ Y_UNIT_TEST(redirectCountDefault) {
+ TPortManager pm;
+ ui16 port = pm.GetPort(80);
+
+ TScenario scenario({
+ {"/any", 307},
+ {"/any?param=1", 302},
+ {"/any?param=1", 302},
+ {"/any?param=1", 302},
+ {"/any?param=1", 302},
+ {"/any?param=1", 302},
+ {"/any?param=1", 302},
+ {"/any?param=1", 302},
+ {"/any?param=1", 302},
+ {"/any?param=1", 302},
+ {"/any?param=2", 200, "Hello"}
+ }, port);
+
+ NMock::TMockServer server(createOptions(port, true), [&scenario]() { return new TScenarioReplier(&scenario); });
+
+ TRedirectableHttpClient cl("localhost", port);
+ UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount());
+
+ TStringStream s;
+ UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/any", &s));
+ UNIT_ASSERT_VALUES_EQUAL("Hello", s.Str());
+
+ scenario.VerifyInvariants();
+ }
+
+ Y_UNIT_TEST(redirectCountN) {
+ TestRedirectCountParam(0, 0);
+ TestRedirectCountParam(0, 1);
+ TestRedirectCountParam(1, 1);
+ TestRedirectCountParam(3, 3);
+ TestRedirectCountParam(20, 20);
+ TestRedirectCountParam(20, 21);
+ }
+
Y_UNIT_TEST(redirectable) {
TPortManager pm;
ui16 port = pm.GetPort(80);
diff --git a/library/cpp/ipv6_address/ipv6_address.cpp b/library/cpp/ipv6_address/ipv6_address.cpp
index c2b33bcbe86..aed4fd46722 100644
--- a/library/cpp/ipv6_address/ipv6_address.cpp
+++ b/library/cpp/ipv6_address/ipv6_address.cpp
@@ -118,6 +118,7 @@ TIpv6Address TIpv6Address::FromString(TStringBuf str) noexcept {
TString TIpv6Address::ToString(bool* ok) const noexcept {
return ToString(true, ok);
}
+
TString TIpv6Address::ToString(bool PrintScopeId, bool* ok) const noexcept {
TString result;
bool isOk = true;
@@ -271,14 +272,17 @@ IOutputStream& operator<<(IOutputStream& out, const TIpv6Address& ipv6Address) n
}
TString THostAddressAndPort::ToString() const noexcept {
- TStringStream Str;
- Str << *this;
- return Str.Str();
+ return ToString({});
}
-IOutputStream& operator<<(IOutputStream& Out, const THostAddressAndPort& HostAddressAndPort) noexcept {
- Out << HostAddressAndPort.Ip << ":" << HostAddressAndPort.Port;
- return Out;
+TString THostAddressAndPort::ToString(THostAddressAndPortPrintOptions options) const noexcept {
+ bool isBrackets = Ip.IsIpv6();
+ return TString::Join(
+ isBrackets ? "[" : "",
+ Ip.ToString(options.PrintScopeId),
+ isBrackets ? "]:" : ":",
+ ::ToString(Port)
+ );
}
namespace {
diff --git a/library/cpp/ipv6_address/ipv6_address.h b/library/cpp/ipv6_address/ipv6_address.h
index 44e5e79012a..93f8349d4cd 100644
--- a/library/cpp/ipv6_address/ipv6_address.h
+++ b/library/cpp/ipv6_address/ipv6_address.h
@@ -72,6 +72,10 @@ public:
return Ip != 0 && (Type_ == Ipv6 || Type_ == Ipv4);
}
+ constexpr bool IsIpv6() const noexcept {
+ return Type_ == Ipv6;
+ }
+
explicit constexpr operator bool() const noexcept {
return IsValid();
}
@@ -164,6 +168,10 @@ constexpr TIpv6Address Get1() noexcept {
return {1, TIpv6Address::Ipv6};
}
+struct THostAddressAndPortPrintOptions {
+ bool PrintScopeId = false;
+};
+
struct THostAddressAndPort {
constexpr THostAddressAndPort() noexcept = default;
constexpr THostAddressAndPort(const TIpv6Address& i, TIpPort p) noexcept {
@@ -182,11 +190,11 @@ struct THostAddressAndPort {
}
TString ToString() const noexcept;
+ TString ToString(THostAddressAndPortPrintOptions options) const noexcept;
TIpv6Address Ip {};
TIpPort Port {0};
};
-IOutputStream& operator<<(IOutputStream& Out, const THostAddressAndPort& HostAddressAndPort) noexcept;
///
/// Returns
diff --git a/library/cpp/ipv6_address/ut/ipv6_address_ut.cpp b/library/cpp/ipv6_address/ut/ipv6_address_ut.cpp
index f23aabe0eea..08f4ad22ae9 100644
--- a/library/cpp/ipv6_address/ut/ipv6_address_ut.cpp
+++ b/library/cpp/ipv6_address/ut/ipv6_address_ut.cpp
@@ -6,12 +6,14 @@ class TIpv6AddressTest: public TTestBase {
UNIT_TEST_SUITE(TIpv6AddressTest);
UNIT_TEST(ParseHostAndMayBePortFromString_data);
UNIT_TEST(CheckAddressValidity)
+ UNIT_TEST(CheckToStringConversion)
UNIT_TEST_SUITE_END();
private:
void ParseHostAndMayBePortFromString_data();
void CheckAddressValidity();
void HashCompileTest();
+ void CheckToStringConversion();
};
UNIT_TEST_SUITE_REGISTRATION(TIpv6AddressTest);
@@ -106,6 +108,24 @@ void TIpv6AddressTest::CheckAddressValidity() {
static_assert(Get1() == TIpv6Address(0, 0, 0, 0, 0, 0, 0, 1));
}
+void TIpv6AddressTest::CheckToStringConversion() {
+ {
+ TString ipPort = "[2aa2::786b]:789";
+ bool ok;
+ auto result = ParseHostAndMayBePortFromString(ipPort, 80, ok);
+ auto hostAddressAndPort = std::get<THostAddressAndPort>(result);
+ UNIT_ASSERT_EQUAL(hostAddressAndPort.ToString({}), ipPort);
+ UNIT_ASSERT_EQUAL(hostAddressAndPort.ToString(), ipPort);
+ }
+ {
+ TString ipPort = "[2aa2::786b%25]:789";
+ bool ok;
+ auto result = ParseHostAndMayBePortFromString(ipPort, 80, ok);
+ auto hostAddressAndPort = std::get<THostAddressAndPort>(result);
+ UNIT_ASSERT_EQUAL(hostAddressAndPort.ToString({.PrintScopeId = true}), ipPort);
+ }
+}
+
void TIpv6AddressTest::HashCompileTest() {
std::unordered_set<TIpv6Address> test;
Y_UNUSED(test);
diff --git a/library/cpp/protobuf/yql/ya.make b/library/cpp/protobuf/yql/ya.make
index 29c21da4024..088890befc7 100644
--- a/library/cpp/protobuf/yql/ya.make
+++ b/library/cpp/protobuf/yql/ya.make
@@ -12,4 +12,6 @@ PEERDIR(
library/cpp/string_utils/base64
)
+GENERATE_ENUM_SERIALIZATION(descriptor.h)
+
END()
diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt
index 07a1789b175..bf5f9f36549 100644
--- a/library/cpp/tld/tlds-alpha-by-domain.txt
+++ b/library/cpp/tld/tlds-alpha-by-domain.txt
@@ -1,4 +1,4 @@
-# Version 2024051300, Last Updated Mon May 13 07:07:01 2024 UTC
+# Version 2024060100, Last Updated Sat Jun 1 07:07:02 2024 UTC
AAA
AARP
ABB
diff --git a/library/cpp/yt/logging/logger-inl.h b/library/cpp/yt/logging/logger-inl.h
index dcf40d9c119..b20526532d5 100644
--- a/library/cpp/yt/logging/logger-inl.h
+++ b/library/cpp/yt/logging/logger-inl.h
@@ -277,6 +277,7 @@ inline void LogEventImpl(
const TLogger& logger,
ELogLevel level,
::TSourceLocation sourceLocation,
+ TLoggingAnchor* anchor,
TSharedRef message)
{
auto event = CreateLogEvent(loggingContext, logger, level);
@@ -285,6 +286,7 @@ inline void LogEventImpl(
event.Family = ELogFamily::PlainText;
event.SourceFile = sourceLocation.File;
event.SourceLine = sourceLocation.Line;
+ event.Anchor = anchor;
logger.Write(std::move(event));
if (Y_UNLIKELY(event.Level >= ELogLevel::Alert)) {
OnCriticalLogEvent(logger, event);
diff --git a/library/cpp/yt/logging/logger.h b/library/cpp/yt/logging/logger.h
index 693ea2b9cd2..abeb9a1c9f0 100644
--- a/library/cpp/yt/logging/logger.h
+++ b/library/cpp/yt/logging/logger.h
@@ -22,6 +22,10 @@
#include <atomic>
+#if !defined(NDEBUG) && !defined(YT_DISABLE_FORMAT_STATIC_ANALYSIS)
+ #include "static_analysis.h"
+#endif
+
namespace NYT::NLogging {
////////////////////////////////////////////////////////////////////////////////
@@ -56,7 +60,7 @@ struct TLoggingAnchor
struct TCounter
{
- std::atomic<i64> Current = 0;
+ i64 Current = 0;
i64 Previous = 0;
};
@@ -100,6 +104,8 @@ struct TLogEvent
TStringBuf SourceFile;
int SourceLine = -1;
+
+ TLoggingAnchor* Anchor = nullptr;
};
////////////////////////////////////////////////////////////////////////////////
@@ -294,6 +300,12 @@ void LogStructuredEvent(
#define YT_LOG_EVENT(logger, level, ...) \
YT_LOG_EVENT_WITH_ANCHOR(logger, level, nullptr, __VA_ARGS__)
+#if !defined(NDEBUG) && !defined(YT_DISABLE_FORMAT_STATIC_ANALYSIS)
+ #define YT_LOG_CHECK_FORMAT(...) STATIC_ANALYSIS_CHECK_LOG_FORMAT(__VA_ARGS__)
+#else
+ #define YT_LOG_CHECK_FORMAT(...)
+#endif
+
#define YT_LOG_EVENT_WITH_ANCHOR(logger, level, anchor, ...) \
do { \
const auto& logger__ = (logger)(); \
@@ -317,6 +329,7 @@ void LogStructuredEvent(
} \
\
auto loggingContext__ = ::NYT::NLogging::GetLoggingContext(); \
+ YT_LOG_CHECK_FORMAT(__VA_ARGS__); \
auto message__ = ::NYT::NLogging::NDetail::BuildLogMessage(loggingContext__, logger__, __VA_ARGS__); \
\
if (!anchorUpToDate__) { \
@@ -328,21 +341,12 @@ void LogStructuredEvent(
break; \
} \
\
- static thread_local i64 localByteCounter__; \
- static thread_local ui8 localMessageCounter__; \
- \
- localByteCounter__ += message__.MessageRef.Size(); \
- if (Y_UNLIKELY(++localMessageCounter__ == 0)) { \
- anchor__->MessageCounter.Current += 256; \
- anchor__->ByteCounter.Current += localByteCounter__; \
- localByteCounter__ = 0; \
- } \
- \
::NYT::NLogging::NDetail::LogEventImpl( \
loggingContext__, \
logger__, \
level__, \
location__, \
+ anchor__, \
std::move(message__.MessageRef)); \
} while (false)
diff --git a/library/cpp/yt/logging/static_analysis-inl.h b/library/cpp/yt/logging/static_analysis-inl.h
new file mode 100644
index 00000000000..d4ec5343bc1
--- /dev/null
+++ b/library/cpp/yt/logging/static_analysis-inl.h
@@ -0,0 +1,139 @@
+#ifndef STATIC_ANALYSIS_INL_H_
+#error "Direct inclusion of this file is not allowed, include static_analysis.h"
+// For the sake of sane code completion.
+#include "static_analysis.h"
+#endif
+
+#include <library/cpp/yt/misc/preprocessor.h>
+
+#include <library/cpp/yt/string/format_analyser.h>
+
+#include <string_view>
+#include <variant> // monostate
+
+namespace NYT::NLogging::NDetail {
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Tag for dispatching proper TFormatArg specialization.
+template <class T>
+struct TLoggerFormatArg
+{ };
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Stateless constexpr way of capturing arg types
+// without invoking any ctors. With the help of macros
+// can turn non-constexpr argument pack of arguments
+// into constexpr pack of types.
+template <class... TArgs>
+struct TLoggerFormatArgs
+{ };
+
+// Used for macro conversion. Purposefully undefined.
+template <class... TArgs>
+TLoggerFormatArgs<TArgs...> AsFormatArgs(TArgs&&...);
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <bool First, bool Second>
+struct TAnalyserDispatcher
+{
+ template <class... TArgs>
+ static consteval void Do(std::string_view, std::string_view, TLoggerFormatArgs<TArgs...>)
+ {
+ // Give up :(
+ // We can't crash here, because, for example, YT_LOG_ERROR(error) exists
+ // and we can't really check if error is actually TError or something else here.
+ // and probably shouldn't bother trying.
+ }
+};
+
+template <bool Second>
+struct TAnalyserDispatcher<true, Second>
+{
+ template <class TFirst, class... TArgs>
+ static consteval void Do(std::string_view str, std::string_view, TLoggerFormatArgs<TFirst, TArgs...>)
+ {
+ // Remove outer \"'s generated by PP_STRINGIZE.
+ auto stripped = std::string_view(std::begin(str) + 1, std::size(str) - 2);
+ ::NYT::NDetail::TFormatAnalyser::ValidateFormat<TLoggerFormatArg<TArgs>...>(stripped);
+ }
+};
+
+template <>
+struct TAnalyserDispatcher<false, true>
+{
+ template <class TFirst, class TSecond, class... TArgs>
+ static consteval void Do(std::string_view, std::string_view str, TLoggerFormatArgs<TFirst, TSecond, TArgs...>)
+ {
+ // Remove outer \"'s generated by PP_STRINGIZE.
+ auto stripped = std::string_view(std::begin(str) + 1, std::size(str) - 2);
+ ::NYT::NDetail::TFormatAnalyser::ValidateFormat<TLoggerFormatArg<TArgs>...>(stripped);
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+// This value is never read since homogenization works for unevaluated expressions.
+inline constexpr auto InvalidToken = std::monostate{};
+
+////////////////////////////////////////////////////////////////////////////////
+
+#define PP_VA_PICK_1_IMPL(N, ...) N
+#define PP_VA_PICK_2_IMPL(_1, N, ...) N
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! Parameter pack parsing.
+
+#define STATIC_ANALYSIS_CAPTURE_TYPES(...) \
+ decltype(::NYT::NLogging::NDetail::AsFormatArgs(__VA_ARGS__)){}
+
+#define STATIC_ANALYSIS_FIRST_TOKEN(...) \
+ PP_STRINGIZE( \
+ PP_VA_PICK_1_IMPL(__VA_ARGS__ __VA_OPT__(,) ::NYT::NLogging::NDetail::InvalidToken))
+
+#define STATIC_ANALYSIS_SECOND_TOKEN(...) \
+ PP_STRINGIZE(\
+ PP_VA_PICK_2_IMPL( \
+ __VA_ARGS__ __VA_OPT__(,) \
+ ::NYT::NLogging::NDetail::InvalidToken, \
+ ::NYT::NLogging::NDetail::InvalidToken))
+
+#define STATIC_ANALYSIS_FIRST_TOKEN_COND(...) \
+ STATIC_ANALYSIS_FIRST_TOKEN(__VA_ARGS__)[0] == '\"'
+
+#define STATIC_ANALYSIS_SECOND_TOKEN_COND(...) \
+ STATIC_ANALYSIS_SECOND_TOKEN(__VA_ARGS__)[0] == '\"'
+
+#undef STATIC_ANALYSIS_CHECK_LOG_FORMAT
+#define STATIC_ANALYSIS_CHECK_LOG_FORMAT(...) \
+ ::NYT \
+ ::NLogging \
+ ::NDetail \
+ ::TAnalyserDispatcher< \
+ STATIC_ANALYSIS_FIRST_TOKEN_COND(__VA_ARGS__), \
+ STATIC_ANALYSIS_SECOND_TOKEN_COND(__VA_ARGS__) \
+ >::Do( \
+ STATIC_ANALYSIS_FIRST_TOKEN(__VA_ARGS__), \
+ STATIC_ANALYSIS_SECOND_TOKEN(__VA_ARGS__), \
+ STATIC_ANALYSIS_CAPTURE_TYPES(__VA_ARGS__))
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NLogging::NDetail
+
+template <class T>
+struct NYT::TFormatArg<NYT::NLogging::NDetail::TLoggerFormatArg<T>>
+ : public NYT::TFormatArgBase
+{
+ // We mix in '\"' and ' ' which is an artifact of logging stringize.
+ // We want to support YT_LOG_XXX("Value: %" PRIu64, 42)
+ // for plantform independent prints of numbers.
+ // String below may be converted to a token:
+ // "\"Value: %\" \"u\""
+ // Thus adding a \" \" sequence.
+ static constexpr auto FlagSpecifiers
+ = TFormatArgBase::ExtendFlags</*Hot*/ false, 2, std::array{'\"', ' '}, /*TFrom*/ NYT::TFormatArg<T>>();
+};
diff --git a/library/cpp/yt/logging/static_analysis.h b/library/cpp/yt/logging/static_analysis.h
new file mode 100644
index 00000000000..a335d8c6cc1
--- /dev/null
+++ b/library/cpp/yt/logging/static_analysis.h
@@ -0,0 +1,22 @@
+#pragma once
+
+namespace NYT::NLogging {
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Performs a compile-time check of log arguments validity.
+// Valid argument lists are:
+// 1. (format, args...)
+// 2. (error, format, args...)
+// If format is not a string literal or argument list
+// is not valid, no check is made -- macro turns to
+// a no-op.
+#define STATIC_ANALYSIS_CHECK_LOG_FORMAT(...)
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace NYT::NLogging
+
+#define STATIC_ANALYSIS_INL_H_
+#include "static_analysis-inl.h"
+#undef STATIC_ANALYSIS_INL_H_
diff --git a/library/cpp/yt/logging/unittests/static_analysis_ut.cpp b/library/cpp/yt/logging/unittests/static_analysis_ut.cpp
new file mode 100644
index 00000000000..1c705dc9674
--- /dev/null
+++ b/library/cpp/yt/logging/unittests/static_analysis_ut.cpp
@@ -0,0 +1,41 @@
+#include <library/cpp/testing/gtest/gtest.h>
+
+#include <library/cpp/yt/logging/logger.h>
+
+namespace NYT::NLogging {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+TEST(TStaticAnalysis, ValidFormats)
+{
+ // Mock for actual error -- we only care that
+ // it is some runtime object.
+ [[maybe_unused]] struct TError
+ { } error;
+
+ YT_LOG_CHECK_FORMAT("Hello");
+ YT_LOG_CHECK_FORMAT("Hello %v", "World!");
+ YT_LOG_CHECK_FORMAT("Hello %qv", "World!");
+ YT_LOG_CHECK_FORMAT(error);
+ YT_LOG_CHECK_FORMAT(error, "Hello");
+ YT_LOG_CHECK_FORMAT(error, "Hello %Qhs", "World!");
+ YT_LOG_CHECK_FORMAT("Hello %%");
+ YT_LOG_CHECK_FORMAT("Hello %" PRIu64, 42);
+}
+
+// Uncomment this test to see that we don't have false negatives!
+// TEST(TStaticAnalysis, InvalidFormats)
+// {
+// YT_LOG_CHECK_FORMAT("Hello", 1);
+// YT_LOG_CHECK_FORMAT("Hello %");
+// YT_LOG_CHECK_FORMAT("Hello %false");
+// YT_LOG_CHECK_FORMAT("Hello ", "World");
+// YT_LOG_CHECK_FORMAT("Hello ", "(World: %v)", 42);
+// YT_LOG_CHECK_FORMAT("Hello %lbov", 42); // There is no 'b' flag.
+// }
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+} // namespace NYT::NLogging
diff --git a/library/cpp/yt/logging/unittests/ya.make b/library/cpp/yt/logging/unittests/ya.make
index 42268d3db2f..021b0d09d6a 100644
--- a/library/cpp/yt/logging/unittests/ya.make
+++ b/library/cpp/yt/logging/unittests/ya.make
@@ -8,6 +8,7 @@ ENDIF()
SRCS(
logger_ut.cpp
+ static_analysis_ut.cpp
)
PEERDIR(
diff --git a/library/cpp/yt/misc/source_location.cpp b/library/cpp/yt/misc/source_location.cpp
index 8d22d43636c..3fe45e23a76 100644
--- a/library/cpp/yt/misc/source_location.cpp
+++ b/library/cpp/yt/misc/source_location.cpp
@@ -1,11 +1,37 @@
#include "source_location.h"
+#include <library/cpp/yt/string/format.h>
+
#include <string.h>
namespace NYT {
////////////////////////////////////////////////////////////////////////////////
+#ifdef __cpp_lib_source_location
+
+void FormatValue(TStringBuilderBase* builder, const std::source_location& location, TStringBuf /*format*/)
+{
+ if (location.file_name() != nullptr) {
+ builder->AppendFormat(
+ "%v:%v:%v",
+ location.file_name(),
+ location.line(),
+ location.column());
+ } else {
+ builder->AppendString("<unknown>");
+ }
+}
+
+TString ToString(const std::source_location& location)
+{
+ return ToStringViaBuilder(location);
+}
+
+#endif // __cpp_lib_source_location
+
+////////////////////////////////////////////////////////////////////////////////
+
const char* TSourceLocation::GetFileName() const
{
return FileName_;
diff --git a/library/cpp/yt/misc/source_location.h b/library/cpp/yt/misc/source_location.h
index 84213eea708..38a6f83c804 100644
--- a/library/cpp/yt/misc/source_location.h
+++ b/library/cpp/yt/misc/source_location.h
@@ -1,9 +1,27 @@
#pragma once
+#include <util/generic/string.h>
+
+#ifdef __cpp_lib_source_location
+#include <source_location>
+#endif // __cpp_lib_source_location
+
namespace NYT {
////////////////////////////////////////////////////////////////////////////////
+// TODO(dgolear): Drop when LLVM-14 is eradicated.
+#ifdef __cpp_lib_source_location
+
+class TStringBuilderBase;
+
+void FormatValue(TStringBuilderBase* builder, const std::source_location& location, TStringBuf /*format*/);
+TString ToString(const std::source_location& location);
+
+#endif // __cpp_lib_source_location
+
+////////////////////////////////////////////////////////////////////////////////
+
class TSourceLocation
{
public:
diff --git a/library/cpp/yt/string/format_analyser-inl.h b/library/cpp/yt/string/format_analyser-inl.h
new file mode 100644
index 00000000000..a548a06072a
--- /dev/null
+++ b/library/cpp/yt/string/format_analyser-inl.h
@@ -0,0 +1,143 @@
+#ifndef FORMAT_ANALYSER_INL_H_
+#error "Direct inclusion of this file is not allowed, include format_analyser.h"
+// For the sake of sane code completion.
+#include "format_analyser.h"
+#endif
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace NDetail {
+
+consteval bool Contains(std::string_view sv, char symbol)
+{
+ return sv.find(symbol) != std::string_view::npos;
+}
+
+template <class... TArgs>
+consteval void TFormatAnalyser::ValidateFormat(std::string_view fmt)
+{
+ std::array<std::string_view, sizeof...(TArgs)> markers = {};
+ std::array<TSpecifiers, sizeof...(TArgs)> specifiers{GetSpecifiers<TArgs>()...};
+
+ int markerCount = 0;
+ int currentMarkerStart = -1;
+
+ for (int idx = 0; idx < std::ssize(fmt); ++idx) {
+ auto symbol = fmt[idx];
+
+ // Parse verbatim text.
+ if (currentMarkerStart == -1) {
+ if (symbol == IntroductorySymbol) {
+ // Marker maybe begins.
+ currentMarkerStart = idx;
+ }
+ continue;
+ }
+
+ // NB: We check for %% first since
+ // in order to verify if symbol is a specifier
+ // we need markerCount to be within range of our
+ // specifier array.
+ if (symbol == IntroductorySymbol) {
+ if (currentMarkerStart + 1 != idx) {
+ // '%a% detected'
+ CrashCompilerWrongTermination("You may not terminate flag sequence other than %% with \'%\' symbol");
+ return;
+ }
+ // '%%' detected --- skip
+ currentMarkerStart = -1;
+ continue;
+ }
+
+ // We are inside of marker.
+ if (markerCount == std::ssize(markers)) {
+ // To many markers
+ CrashCompilerNotEnoughArguments("Number of arguments supplied to format is smaller than the number of flag sequences");
+ return;
+ }
+
+ if (Contains(specifiers[markerCount].Conversion, symbol)) {
+ // Marker has finished.
+
+ markers[markerCount]
+ = std::string_view(fmt.begin() + currentMarkerStart, idx - currentMarkerStart + 1);
+ currentMarkerStart = -1;
+ ++markerCount;
+
+ continue;
+ }
+
+ if (!Contains(specifiers[markerCount].Flags, symbol)) {
+ CrashCompilerWrongFlagSpecifier("Symbol is not a valid flag specifier; See FlagSpecifiers");
+ }
+ }
+
+ if (currentMarkerStart != -1) {
+ // Runaway marker.
+ CrashCompilerMissingTermination("Unterminated flag sequence detected; Use \'%%\' to type plain %");
+ return;
+ }
+
+ if (markerCount < std::ssize(markers)) {
+ // Missing markers.
+ CrashCompilerTooManyArguments("Number of arguments supplied to format is greater than the number of flag sequences");
+ return;
+ }
+
+ // TODO(arkady-e1ppa): Consider per-type verification
+ // of markers.
+}
+
+template <class TArg>
+consteval auto TFormatAnalyser::GetSpecifiers()
+{
+ static_assert(CFormattable<TArg>, "Your specialization of TFormatArg is broken");
+
+ return TSpecifiers{
+ .Conversion = std::string_view{
+ std::data(TFormatArg<TArg>::ConversionSpecifiers),
+ std::size(TFormatArg<TArg>::ConversionSpecifiers)},
+ .Flags = std::string_view{
+ std::data(TFormatArg<TArg>::FlagSpecifiers),
+ std::size(TFormatArg<TArg>::FlagSpecifiers)},
+ };
+}
+
+} // namespace NDetail
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <bool Hot, size_t N, std::array<char, N> List, class TFrom>
+consteval auto TFormatArgBase::ExtendConversion()
+{
+ static_assert(CFormattable<TFrom>);
+ return AppendArrays<Hot, N, List, &TFrom::ConversionSpecifiers>();
+}
+
+template <bool Hot, size_t N, std::array<char, N> List, class TFrom>
+consteval auto TFormatArgBase::ExtendFlags()
+{
+ static_assert(CFormattable<TFrom>);
+ return AppendArrays<Hot, N, List, &TFrom::FlagSpecifiers>();
+}
+
+template <bool Hot, size_t N, std::array<char, N> List, auto* From>
+consteval auto TFormatArgBase::AppendArrays()
+{
+ auto& from = *From;
+ return [] <size_t... I, size_t... J> (
+ std::index_sequence<I...>,
+ std::index_sequence<J...>) {
+ if constexpr (Hot) {
+ return std::array{List[J]..., from[I]...};
+ } else {
+ return std::array{from[I]..., List[J]...};
+ }
+ } (
+ std::make_index_sequence<std::size(from)>(),
+ std::make_index_sequence<N>());
+ }
+
+} // namespace NYT::NDetail
diff --git a/library/cpp/yt/string/format_analyser.h b/library/cpp/yt/string/format_analyser.h
new file mode 100644
index 00000000000..4afe3f1ca8b
--- /dev/null
+++ b/library/cpp/yt/string/format_analyser.h
@@ -0,0 +1,102 @@
+#pragma once
+
+#include <array>
+#include <string_view>
+
+namespace NYT {
+
+////////////////////////////////////////////////////////////////////////////////
+
+namespace NDetail {
+
+struct TFormatAnalyser
+{
+public:
+ template <class... TArgs>
+ static consteval void ValidateFormat(std::string_view fmt);
+
+private:
+ // Non-constexpr function call will terminate compilation.
+ // Purposefully undefined and non-constexpr/consteval
+ static void CrashCompilerNotEnoughArguments(std::string_view msg);
+ static void CrashCompilerTooManyArguments(std::string_view msg);
+ static void CrashCompilerWrongTermination(std::string_view msg);
+ static void CrashCompilerMissingTermination(std::string_view msg);
+ static void CrashCompilerWrongFlagSpecifier(std::string_view msg);
+
+ struct TSpecifiers
+ {
+ std::string_view Conversion;
+ std::string_view Flags;
+ };
+ template <class TArg>
+ static consteval auto GetSpecifiers();
+
+ static constexpr char IntroductorySymbol = '%';
+};
+
+} // namespace NDetail
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Base used for flag checks for each type independently.
+// Use it for overrides.
+struct TFormatArgBase
+{
+public:
+ // TODO(arkady-e1ppa): Consider more strict formatting rules.
+ static constexpr std::array ConversionSpecifiers = {
+ 'v', '1', 'c', 's', 'd', 'i', 'o',
+ 'x', 'X', 'u', 'f', 'F', 'e', 'E',
+ 'a', 'A', 'g', 'G', 'n', 'p'
+ };
+
+ static constexpr std::array FlagSpecifiers = {
+ '-', '+', ' ', '#', '0',
+ '1', '2', '3', '4', '5',
+ '6', '7', '8', '9',
+ '*', '.', 'h', 'l', 'j',
+ 'z', 't', 'L', 'q', 'Q'
+ };
+
+ template <class T>
+ static constexpr bool IsSpecifierList = requires (T t) {
+ [] <size_t N> (std::array<char, N>) { } (t);
+ };
+
+ // Hot = |true| adds specifiers to the beggining
+ // of a new array.
+ template <bool Hot, size_t N, std::array<char, N> List, class TFrom = TFormatArgBase>
+ static consteval auto ExtendConversion();
+
+ template <bool Hot, size_t N, std::array<char, N> List, class TFrom = TFormatArgBase>
+ static consteval auto ExtendFlags();
+
+private:
+ template <bool Hot, size_t N, std::array<char, N> List, auto* From>
+ static consteval auto AppendArrays();
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+template <class T>
+struct TFormatArg
+ : public TFormatArgBase
+{ };
+
+// Semantic requirement:
+// Said field must be constexpr.
+template <class T>
+concept CFormattable = requires {
+ TFormatArg<T>::ConversionSpecifiers;
+ requires TFormatArgBase::IsSpecifierList<decltype(TFormatArg<T>::ConversionSpecifiers)>;
+
+ TFormatArg<T>::FlagSpecifiers;
+ requires TFormatArgBase::IsSpecifierList<decltype(TFormatArg<T>::FlagSpecifiers)>;
+};
+
+} // namespace NYT
+
+#define FORMAT_ANALYSER_INL_H_
+#include "format_analyser-inl.h"
+#undef FORMAT_ANALYSER_INL_H_