diff options
author | Alexander Smirnov <alex@ydb.tech> | 2024-06-03 08:43:13 +0000 |
---|---|---|
committer | Alexander Smirnov <alex@ydb.tech> | 2024-06-03 08:43:13 +0000 |
commit | e8b793b8275c229a46b4211f83c03cb074c9ba7c (patch) | |
tree | 831d4507c2b234635f45b573fcd03a4b96133bb5 /library/cpp | |
parent | 89c3654dc08129116422f621b50086305f68bc9c (diff) | |
parent | 28ff234f758e7fd2f42f229a4b55ff70506f0015 (diff) | |
download | ydb-e8b793b8275c229a46b4211f83c03cb074c9ba7c.tar.gz |
Merge branch 'rightlib' into mergelibs-240603-0842
Diffstat (limited to 'library/cpp')
22 files changed, 756 insertions, 30 deletions
diff --git a/library/cpp/histogram/adaptive/adaptive_histogram.cpp b/library/cpp/histogram/adaptive/adaptive_histogram.cpp index 4bc4f63cda0..bfe76330e08 100644 --- a/library/cpp/histogram/adaptive/adaptive_histogram.cpp +++ b/library/cpp/histogram/adaptive/adaptive_histogram.cpp @@ -7,6 +7,8 @@ #include <util/system/backtrace.h> +#include <format> + namespace NKiwiAggr { TAdaptiveHistogram::TAdaptiveHistogram(size_t intervals, ui64 id, TQualityFunction qualityFunc) : Id(id) @@ -71,7 +73,10 @@ namespace NKiwiAggr { void TAdaptiveHistogram::Merge(const THistogram& histo, double multiplier) { if (!IsValidFloat(histo.GetMinValue()) || !IsValidFloat(histo.GetMaxValue())) { - fprintf(stderr, "Merging in histogram id %lu: skip bad histo with minvalue %f maxvalue %f\n", Id, histo.GetMinValue(), histo.GetMaxValue()); + Cerr << std::format( + "Merging in histogram id {}: skip bad histo with minvalue {} maxvalue {}\n", + Id, histo.GetMinValue(), histo.GetMaxValue() + ); return; } if (histo.FreqSize() == 0) { @@ -87,7 +92,10 @@ namespace NKiwiAggr { double value = histo.GetPosition(j); double weight = histo.GetFreq(j); if (!IsValidFloat(value) || !IsValidFloat(weight)) { - fprintf(stderr, "Merging in histogram id %lu: skip bad value %f weight %f\n", Id, value, weight); + Cerr << std::format( + "Merging in histogram id {}: skip bad value {} weight {}\n", + Id, value, weight + ); continue; } Add(value, weight * multiplier); @@ -100,7 +108,10 @@ namespace NKiwiAggr { for (size_t j = 0; j < histo.FreqSize(); ++j) { double weight = histo.GetFreq(j); if (!IsValidFloat(pos) || !IsValidFloat(weight)) { - fprintf(stderr, "Merging in histogram id %lu: skip bad value %f weight %f\n", Id, pos, weight); + Cerr << std::format( + "Merging in histogram id {}: skip bad value {} weight {}\n", + Id, pos, weight + ); pos += histo.GetBinRange(); continue; } @@ -218,7 +229,10 @@ namespace NKiwiAggr { double value = histo.GetPosition(i); double weight = histo.GetFreq(i); if (!IsValidFloat(value) || !IsValidFloat(weight)) { - fprintf(stderr, "FromProto in histogram id %lu: skip bad value %f weight %f\n", Id, value, weight); + Cerr << std::format( + "FromProto in histogram id {}: skip bad value {} weight {}\n", + Id, value, weight + ); continue; } Add(value, weight); diff --git a/library/cpp/histogram/adaptive/block_histogram.cpp b/library/cpp/histogram/adaptive/block_histogram.cpp index 55337488ac1..ff0383761fa 100644 --- a/library/cpp/histogram/adaptive/block_histogram.cpp +++ b/library/cpp/histogram/adaptive/block_histogram.cpp @@ -10,6 +10,8 @@ #include <util/generic/ymath.h> #include <util/string/printf.h> +#include <format> + namespace { struct TEmpty { }; @@ -138,7 +140,10 @@ namespace NKiwiAggr { void TBlockHistogram::Merge(const THistogram& histo, double multiplier) { if (!IsValidFloat(histo.GetMinValue()) || !IsValidFloat(histo.GetMaxValue())) { - fprintf(stderr, "Merging in histogram id %lu: skip bad histo with minvalue %f maxvalue %f\n", Id, histo.GetMinValue(), histo.GetMaxValue()); + Cerr << std::format( + "Merging in histogram id {}: skip bad histo with minvalue {} maxvalue {}\n", + Id, histo.GetMinValue(), histo.GetMaxValue() + ); return; } if (histo.FreqSize() == 0) { @@ -154,7 +159,10 @@ namespace NKiwiAggr { double value = histo.GetPosition(j); double weight = histo.GetFreq(j); if (!IsValidFloat(value) || !IsValidFloat(weight)) { - fprintf(stderr, "Merging in histogram id %lu: skip bad value %f weight %f\n", Id, value, weight); + Cerr << std::format( + "Merging in histogram id {}: skip bad value {} weight {}\n", + Id, value, weight + ); continue; } Add(value, weight * multiplier); @@ -167,7 +175,10 @@ namespace NKiwiAggr { for (size_t j = 0; j < histo.FreqSize(); ++j) { double weight = histo.GetFreq(j); if (!IsValidFloat(pos) || !IsValidFloat(weight)) { - fprintf(stderr, "Merging in histogram id %lu: skip bad value %f weight %f\n", Id, pos, weight); + Cerr << std::format( + "Merging in histogram id {}: skip bad value {} weight {}\n", + Id, pos, weight + ); pos += histo.GetBinRange(); continue; } @@ -229,7 +240,10 @@ namespace NKiwiAggr { double value = histo.GetPosition(i); double weight = histo.GetFreq(i); if (!IsValidFloat(value) || !IsValidFloat(weight)) { - fprintf(stderr, "FromProto in histogram id %lu: skip bad value %f weight %f\n", Id, value, weight); + Cerr << std::format( + "FromProto in histogram id {}: skip bad value {} weight {}\n", + Id, value, weight + ); continue; } Bins[i].first = value; diff --git a/library/cpp/http/io/headers.h b/library/cpp/http/io/headers.h index 3ae40683e2d..72d3dec4bb0 100644 --- a/library/cpp/http/io/headers.h +++ b/library/cpp/http/io/headers.h @@ -4,7 +4,6 @@ #include <util/generic/deque.h> #include <util/generic/string.h> #include <util/generic/strbuf.h> -#include <util/generic/vector.h> // XXX unused - remove after fixing transitive includes. #include <util/string/cast.h> class IInputStream; diff --git a/library/cpp/http/simple/http_client.cpp b/library/cpp/http/simple/http_client.cpp index 818dc048ad1..342236f9a33 100644 --- a/library/cpp/http/simple/http_client.cpp +++ b/library/cpp/http/simple/http_client.cpp @@ -301,8 +301,14 @@ TKeepAliveHttpClient TSimpleHttpClient::CreateClient() const { void TSimpleHttpClient::PrepareClient(TKeepAliveHttpClient&) const { } +TRedirectableHttpClient::TRedirectableHttpClient(const TOptions& options) + : TSimpleHttpClient(options) + , Opts(options) +{ +} + TRedirectableHttpClient::TRedirectableHttpClient(const TString& host, ui32 port, TDuration socketTimeout, TDuration connectTimeout) - : TSimpleHttpClient(host, port, socketTimeout, connectTimeout) + : TRedirectableHttpClient(TOptions().Host(host).Port(port).SocketTimeout(socketTimeout).ConnectTimeout(connectTimeout)) { } @@ -315,6 +321,10 @@ void TRedirectableHttpClient::PrepareClient(TKeepAliveHttpClient& cl) const { void TRedirectableHttpClient::ProcessResponse(const TStringBuf relativeUrl, THttpInput& input, IOutputStream* output, const unsigned statusCode) const { for (auto i = input.Headers().Begin(), e = input.Headers().End(); i != e; ++i) { if (0 == TString::compare(i->Name(), TStringBuf("Location"))) { + if (Opts.MaxRedirectCount() == 0) { + ythrow THttpRequestException(statusCode) << "Exceeds MaxRedirectCount limit, code " << statusCode << " at " << Host << relativeUrl; + } + TVector<TString> request_url_parts, request_body_parts; size_t splitted_index = 0; @@ -339,7 +349,12 @@ void TRedirectableHttpClient::ProcessResponse(const TStringBuf relativeUrl, THtt } } - TRedirectableHttpClient cl(url, port, TDuration::Seconds(60), TDuration::Seconds(60)); + auto opts = Opts; + opts.Host(url); + opts.Port(port); + opts.MaxRedirectCount(opts.MaxRedirectCount() - 1); + + TRedirectableHttpClient cl(opts); if (HttpsVerification) { cl.EnableVerificationForHttps(); } diff --git a/library/cpp/http/simple/http_client.h b/library/cpp/http/simple/http_client.h index c01b11ba43f..eab92d42da5 100644 --- a/library/cpp/http/simple/http_client.h +++ b/library/cpp/http/simple/http_client.h @@ -185,12 +185,19 @@ private: class TRedirectableHttpClient: public TSimpleHttpClient { public: + using TOptions = TSimpleHttpClientOptions; + + explicit TRedirectableHttpClient(const TOptions& options); + TRedirectableHttpClient(const TString& host, ui32 port, TDuration socketTimeout = TDuration::Seconds(5), TDuration connectTimeout = TDuration::Seconds(30)); private: void PrepareClient(TKeepAliveHttpClient& cl) const override; void ProcessResponse(const TStringBuf relativeUrl, THttpInput& input, IOutputStream* output, const unsigned statusCode) const override; + +private: + TOptions Opts; }; namespace NPrivate { diff --git a/library/cpp/http/simple/http_client_options.h b/library/cpp/http/simple/http_client_options.h index f2e964a462a..603ca5103af 100644 --- a/library/cpp/http/simple/http_client_options.h +++ b/library/cpp/http/simple/http_client_options.h @@ -51,9 +51,19 @@ public: return ConnectTimeout_; } + TSelf& MaxRedirectCount(int count) { + MaxRedirectCount_ = count; + return *this; + } + + ui16 MaxRedirectCount() const noexcept { + return MaxRedirectCount_; + } + private: TString Host_; ui16 Port_; TDuration SocketTimeout_ = TDuration::Seconds(5); TDuration ConnectTimeout_ = TDuration::Seconds(30); + int MaxRedirectCount_ = INT_MAX; }; diff --git a/library/cpp/http/simple/ut/http_ut.cpp b/library/cpp/http/simple/ut/http_ut.cpp index bf7e767428e..7768fdc4faa 100644 --- a/library/cpp/http/simple/ut/http_ut.cpp +++ b/library/cpp/http/simple/ut/http_ut.cpp @@ -71,6 +71,76 @@ Y_UNIT_TEST_SUITE(SimpleHttp) { } }; + class TScenario { + public: + struct TElem { + TString Url; + int Status = HTTP_OK; + TString Content{}; + }; + + TScenario(const TVector<TElem>& seq, ui16 port = 80, TDuration sleep = TDuration()) + : Seq_(seq) + , Sleep_(sleep) + , Port_(port) + { + } + + bool DoReply(const TRequestReplier::TReplyParams& params, TRequestReplier* replier) { + const auto parsed = TParsedHttpFull(params.Input.FirstLine()); + const auto url = parsed.Request; + params.Input.ReadAll(); + + UNIT_ASSERT(SeqIdx_ < Seq_.size()); + auto& elem = Seq_[SeqIdx_++]; + + UNIT_ASSERT_VALUES_EQUAL(elem.Url, url); + + Sleep(Sleep_); + + if (elem.Status == -1) { + replier->ResetConnection(); // RST / ECONNRESET + return true; + } + + THttpResponse resp((HttpCodes)elem.Status); + + if (elem.Status >= 300 && elem.Status < 400) { + UNIT_ASSERT(SeqIdx_ < Seq_.size()); + resp.AddHeader("Location", TStringBuilder() << "http://localhost:" << Port_ << Seq_[SeqIdx_].Url); + } + + resp.SetContent(elem.Content); + resp.OutTo(params.Output); + + return true; + } + + void VerifyInvariants() { + UNIT_ASSERT_VALUES_EQUAL(SeqIdx_, Seq_.size()); + } + + private: + TVector<TElem> Seq_; + size_t SeqIdx_ = 0; + TDuration Sleep_; + ui16 Port_; + }; + + class TScenarioReplier: public TRequestReplier { + TScenario* Scenario_ = nullptr; + + public: + TScenarioReplier(TScenario* scenario) + : Scenario_(scenario) + { + } + + bool DoReply(const TReplyParams& params) override { + return Scenario_->DoReply(params, this); + } + }; + class TCodedPong: public TRequestReplier { HttpCodes Code_; @@ -129,6 +199,32 @@ Y_UNIT_TEST_SUITE(SimpleHttp) { } }; + static void TestRedirectCountParam(int maxRedirectCount, int redirectCount) { + TPortManager pm; + ui16 port = pm.GetPort(80); + + TVector<TScenario::TElem> steps; + for (int i = 0; i < redirectCount; ++i) { + steps.push_back({"/any", 302}); + } + steps.push_back({"/any", 200, "Hello"}); + TScenario scenario(steps, port); + + NMock::TMockServer server(createOptions(port, true), [&scenario]() { return new TScenarioReplier(&scenario); }); + + TRedirectableHttpClient cl(TSimpleHttpClientOptions().Host("localhost").Port(port).MaxRedirectCount(maxRedirectCount)); + UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount()); + + TStringStream s; + if (maxRedirectCount >= redirectCount) { + UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/any", &s)); + UNIT_ASSERT_VALUES_EQUAL("Hello", s.Str()); + scenario.VerifyInvariants(); + } else { + UNIT_ASSERT_EXCEPTION_CONTAINS(cl.DoGet("/any", &s), THttpRequestException, ""); + } + } + Y_UNIT_TEST(simpleSuccessful) { TPortManager pm; ui16 port = pm.GetPort(80); @@ -274,6 +370,45 @@ Y_UNIT_TEST_SUITE(SimpleHttp) { } } + Y_UNIT_TEST(redirectCountDefault) { + TPortManager pm; + ui16 port = pm.GetPort(80); + + TScenario scenario({ + {"/any", 307}, + {"/any?param=1", 302}, + {"/any?param=1", 302}, + {"/any?param=1", 302}, + {"/any?param=1", 302}, + {"/any?param=1", 302}, + {"/any?param=1", 302}, + {"/any?param=1", 302}, + {"/any?param=1", 302}, + {"/any?param=1", 302}, + {"/any?param=2", 200, "Hello"} + }, port); + + NMock::TMockServer server(createOptions(port, true), [&scenario]() { return new TScenarioReplier(&scenario); }); + + TRedirectableHttpClient cl("localhost", port); + UNIT_ASSERT_VALUES_EQUAL(0, server.GetClientCount()); + + TStringStream s; + UNIT_ASSERT_NO_EXCEPTION(cl.DoGet("/any", &s)); + UNIT_ASSERT_VALUES_EQUAL("Hello", s.Str()); + + scenario.VerifyInvariants(); + } + + Y_UNIT_TEST(redirectCountN) { + TestRedirectCountParam(0, 0); + TestRedirectCountParam(0, 1); + TestRedirectCountParam(1, 1); + TestRedirectCountParam(3, 3); + TestRedirectCountParam(20, 20); + TestRedirectCountParam(20, 21); + } + Y_UNIT_TEST(redirectable) { TPortManager pm; ui16 port = pm.GetPort(80); diff --git a/library/cpp/ipv6_address/ipv6_address.cpp b/library/cpp/ipv6_address/ipv6_address.cpp index c2b33bcbe86..aed4fd46722 100644 --- a/library/cpp/ipv6_address/ipv6_address.cpp +++ b/library/cpp/ipv6_address/ipv6_address.cpp @@ -118,6 +118,7 @@ TIpv6Address TIpv6Address::FromString(TStringBuf str) noexcept { TString TIpv6Address::ToString(bool* ok) const noexcept { return ToString(true, ok); } + TString TIpv6Address::ToString(bool PrintScopeId, bool* ok) const noexcept { TString result; bool isOk = true; @@ -271,14 +272,17 @@ IOutputStream& operator<<(IOutputStream& out, const TIpv6Address& ipv6Address) n } TString THostAddressAndPort::ToString() const noexcept { - TStringStream Str; - Str << *this; - return Str.Str(); + return ToString({}); } -IOutputStream& operator<<(IOutputStream& Out, const THostAddressAndPort& HostAddressAndPort) noexcept { - Out << HostAddressAndPort.Ip << ":" << HostAddressAndPort.Port; - return Out; +TString THostAddressAndPort::ToString(THostAddressAndPortPrintOptions options) const noexcept { + bool isBrackets = Ip.IsIpv6(); + return TString::Join( + isBrackets ? "[" : "", + Ip.ToString(options.PrintScopeId), + isBrackets ? "]:" : ":", + ::ToString(Port) + ); } namespace { diff --git a/library/cpp/ipv6_address/ipv6_address.h b/library/cpp/ipv6_address/ipv6_address.h index 44e5e79012a..93f8349d4cd 100644 --- a/library/cpp/ipv6_address/ipv6_address.h +++ b/library/cpp/ipv6_address/ipv6_address.h @@ -72,6 +72,10 @@ public: return Ip != 0 && (Type_ == Ipv6 || Type_ == Ipv4); } + constexpr bool IsIpv6() const noexcept { + return Type_ == Ipv6; + } + explicit constexpr operator bool() const noexcept { return IsValid(); } @@ -164,6 +168,10 @@ constexpr TIpv6Address Get1() noexcept { return {1, TIpv6Address::Ipv6}; } +struct THostAddressAndPortPrintOptions { + bool PrintScopeId = false; +}; + struct THostAddressAndPort { constexpr THostAddressAndPort() noexcept = default; constexpr THostAddressAndPort(const TIpv6Address& i, TIpPort p) noexcept { @@ -182,11 +190,11 @@ struct THostAddressAndPort { } TString ToString() const noexcept; + TString ToString(THostAddressAndPortPrintOptions options) const noexcept; TIpv6Address Ip {}; TIpPort Port {0}; }; -IOutputStream& operator<<(IOutputStream& Out, const THostAddressAndPort& HostAddressAndPort) noexcept; /// /// Returns diff --git a/library/cpp/ipv6_address/ut/ipv6_address_ut.cpp b/library/cpp/ipv6_address/ut/ipv6_address_ut.cpp index f23aabe0eea..08f4ad22ae9 100644 --- a/library/cpp/ipv6_address/ut/ipv6_address_ut.cpp +++ b/library/cpp/ipv6_address/ut/ipv6_address_ut.cpp @@ -6,12 +6,14 @@ class TIpv6AddressTest: public TTestBase { UNIT_TEST_SUITE(TIpv6AddressTest); UNIT_TEST(ParseHostAndMayBePortFromString_data); UNIT_TEST(CheckAddressValidity) + UNIT_TEST(CheckToStringConversion) UNIT_TEST_SUITE_END(); private: void ParseHostAndMayBePortFromString_data(); void CheckAddressValidity(); void HashCompileTest(); + void CheckToStringConversion(); }; UNIT_TEST_SUITE_REGISTRATION(TIpv6AddressTest); @@ -106,6 +108,24 @@ void TIpv6AddressTest::CheckAddressValidity() { static_assert(Get1() == TIpv6Address(0, 0, 0, 0, 0, 0, 0, 1)); } +void TIpv6AddressTest::CheckToStringConversion() { + { + TString ipPort = "[2aa2::786b]:789"; + bool ok; + auto result = ParseHostAndMayBePortFromString(ipPort, 80, ok); + auto hostAddressAndPort = std::get<THostAddressAndPort>(result); + UNIT_ASSERT_EQUAL(hostAddressAndPort.ToString({}), ipPort); + UNIT_ASSERT_EQUAL(hostAddressAndPort.ToString(), ipPort); + } + { + TString ipPort = "[2aa2::786b%25]:789"; + bool ok; + auto result = ParseHostAndMayBePortFromString(ipPort, 80, ok); + auto hostAddressAndPort = std::get<THostAddressAndPort>(result); + UNIT_ASSERT_EQUAL(hostAddressAndPort.ToString({.PrintScopeId = true}), ipPort); + } +} + void TIpv6AddressTest::HashCompileTest() { std::unordered_set<TIpv6Address> test; Y_UNUSED(test); diff --git a/library/cpp/protobuf/yql/ya.make b/library/cpp/protobuf/yql/ya.make index 29c21da4024..088890befc7 100644 --- a/library/cpp/protobuf/yql/ya.make +++ b/library/cpp/protobuf/yql/ya.make @@ -12,4 +12,6 @@ PEERDIR( library/cpp/string_utils/base64 ) +GENERATE_ENUM_SERIALIZATION(descriptor.h) + END() diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt index 07a1789b175..bf5f9f36549 100644 --- a/library/cpp/tld/tlds-alpha-by-domain.txt +++ b/library/cpp/tld/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2024051300, Last Updated Mon May 13 07:07:01 2024 UTC +# Version 2024060100, Last Updated Sat Jun 1 07:07:02 2024 UTC AAA AARP ABB diff --git a/library/cpp/yt/logging/logger-inl.h b/library/cpp/yt/logging/logger-inl.h index dcf40d9c119..b20526532d5 100644 --- a/library/cpp/yt/logging/logger-inl.h +++ b/library/cpp/yt/logging/logger-inl.h @@ -277,6 +277,7 @@ inline void LogEventImpl( const TLogger& logger, ELogLevel level, ::TSourceLocation sourceLocation, + TLoggingAnchor* anchor, TSharedRef message) { auto event = CreateLogEvent(loggingContext, logger, level); @@ -285,6 +286,7 @@ inline void LogEventImpl( event.Family = ELogFamily::PlainText; event.SourceFile = sourceLocation.File; event.SourceLine = sourceLocation.Line; + event.Anchor = anchor; logger.Write(std::move(event)); if (Y_UNLIKELY(event.Level >= ELogLevel::Alert)) { OnCriticalLogEvent(logger, event); diff --git a/library/cpp/yt/logging/logger.h b/library/cpp/yt/logging/logger.h index 693ea2b9cd2..abeb9a1c9f0 100644 --- a/library/cpp/yt/logging/logger.h +++ b/library/cpp/yt/logging/logger.h @@ -22,6 +22,10 @@ #include <atomic> +#if !defined(NDEBUG) && !defined(YT_DISABLE_FORMAT_STATIC_ANALYSIS) + #include "static_analysis.h" +#endif + namespace NYT::NLogging { //////////////////////////////////////////////////////////////////////////////// @@ -56,7 +60,7 @@ struct TLoggingAnchor struct TCounter { - std::atomic<i64> Current = 0; + i64 Current = 0; i64 Previous = 0; }; @@ -100,6 +104,8 @@ struct TLogEvent TStringBuf SourceFile; int SourceLine = -1; + + TLoggingAnchor* Anchor = nullptr; }; //////////////////////////////////////////////////////////////////////////////// @@ -294,6 +300,12 @@ void LogStructuredEvent( #define YT_LOG_EVENT(logger, level, ...) \ YT_LOG_EVENT_WITH_ANCHOR(logger, level, nullptr, __VA_ARGS__) +#if !defined(NDEBUG) && !defined(YT_DISABLE_FORMAT_STATIC_ANALYSIS) + #define YT_LOG_CHECK_FORMAT(...) STATIC_ANALYSIS_CHECK_LOG_FORMAT(__VA_ARGS__) +#else + #define YT_LOG_CHECK_FORMAT(...) +#endif + #define YT_LOG_EVENT_WITH_ANCHOR(logger, level, anchor, ...) \ do { \ const auto& logger__ = (logger)(); \ @@ -317,6 +329,7 @@ void LogStructuredEvent( } \ \ auto loggingContext__ = ::NYT::NLogging::GetLoggingContext(); \ + YT_LOG_CHECK_FORMAT(__VA_ARGS__); \ auto message__ = ::NYT::NLogging::NDetail::BuildLogMessage(loggingContext__, logger__, __VA_ARGS__); \ \ if (!anchorUpToDate__) { \ @@ -328,21 +341,12 @@ void LogStructuredEvent( break; \ } \ \ - static thread_local i64 localByteCounter__; \ - static thread_local ui8 localMessageCounter__; \ - \ - localByteCounter__ += message__.MessageRef.Size(); \ - if (Y_UNLIKELY(++localMessageCounter__ == 0)) { \ - anchor__->MessageCounter.Current += 256; \ - anchor__->ByteCounter.Current += localByteCounter__; \ - localByteCounter__ = 0; \ - } \ - \ ::NYT::NLogging::NDetail::LogEventImpl( \ loggingContext__, \ logger__, \ level__, \ location__, \ + anchor__, \ std::move(message__.MessageRef)); \ } while (false) diff --git a/library/cpp/yt/logging/static_analysis-inl.h b/library/cpp/yt/logging/static_analysis-inl.h new file mode 100644 index 00000000000..d4ec5343bc1 --- /dev/null +++ b/library/cpp/yt/logging/static_analysis-inl.h @@ -0,0 +1,139 @@ +#ifndef STATIC_ANALYSIS_INL_H_ +#error "Direct inclusion of this file is not allowed, include static_analysis.h" +// For the sake of sane code completion. +#include "static_analysis.h" +#endif + +#include <library/cpp/yt/misc/preprocessor.h> + +#include <library/cpp/yt/string/format_analyser.h> + +#include <string_view> +#include <variant> // monostate + +namespace NYT::NLogging::NDetail { + +//////////////////////////////////////////////////////////////////////////////// + +// Tag for dispatching proper TFormatArg specialization. +template <class T> +struct TLoggerFormatArg +{ }; + +//////////////////////////////////////////////////////////////////////////////// + +// Stateless constexpr way of capturing arg types +// without invoking any ctors. With the help of macros +// can turn non-constexpr argument pack of arguments +// into constexpr pack of types. +template <class... TArgs> +struct TLoggerFormatArgs +{ }; + +// Used for macro conversion. Purposefully undefined. +template <class... TArgs> +TLoggerFormatArgs<TArgs...> AsFormatArgs(TArgs&&...); + +//////////////////////////////////////////////////////////////////////////////// + +template <bool First, bool Second> +struct TAnalyserDispatcher +{ + template <class... TArgs> + static consteval void Do(std::string_view, std::string_view, TLoggerFormatArgs<TArgs...>) + { + // Give up :( + // We can't crash here, because, for example, YT_LOG_ERROR(error) exists + // and we can't really check if error is actually TError or something else here. + // and probably shouldn't bother trying. + } +}; + +template <bool Second> +struct TAnalyserDispatcher<true, Second> +{ + template <class TFirst, class... TArgs> + static consteval void Do(std::string_view str, std::string_view, TLoggerFormatArgs<TFirst, TArgs...>) + { + // Remove outer \"'s generated by PP_STRINGIZE. + auto stripped = std::string_view(std::begin(str) + 1, std::size(str) - 2); + ::NYT::NDetail::TFormatAnalyser::ValidateFormat<TLoggerFormatArg<TArgs>...>(stripped); + } +}; + +template <> +struct TAnalyserDispatcher<false, true> +{ + template <class TFirst, class TSecond, class... TArgs> + static consteval void Do(std::string_view, std::string_view str, TLoggerFormatArgs<TFirst, TSecond, TArgs...>) + { + // Remove outer \"'s generated by PP_STRINGIZE. + auto stripped = std::string_view(std::begin(str) + 1, std::size(str) - 2); + ::NYT::NDetail::TFormatAnalyser::ValidateFormat<TLoggerFormatArg<TArgs>...>(stripped); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +// This value is never read since homogenization works for unevaluated expressions. +inline constexpr auto InvalidToken = std::monostate{}; + +//////////////////////////////////////////////////////////////////////////////// + +#define PP_VA_PICK_1_IMPL(N, ...) N +#define PP_VA_PICK_2_IMPL(_1, N, ...) N + +//////////////////////////////////////////////////////////////////////////////// + +//! Parameter pack parsing. + +#define STATIC_ANALYSIS_CAPTURE_TYPES(...) \ + decltype(::NYT::NLogging::NDetail::AsFormatArgs(__VA_ARGS__)){} + +#define STATIC_ANALYSIS_FIRST_TOKEN(...) \ + PP_STRINGIZE( \ + PP_VA_PICK_1_IMPL(__VA_ARGS__ __VA_OPT__(,) ::NYT::NLogging::NDetail::InvalidToken)) + +#define STATIC_ANALYSIS_SECOND_TOKEN(...) \ + PP_STRINGIZE(\ + PP_VA_PICK_2_IMPL( \ + __VA_ARGS__ __VA_OPT__(,) \ + ::NYT::NLogging::NDetail::InvalidToken, \ + ::NYT::NLogging::NDetail::InvalidToken)) + +#define STATIC_ANALYSIS_FIRST_TOKEN_COND(...) \ + STATIC_ANALYSIS_FIRST_TOKEN(__VA_ARGS__)[0] == '\"' + +#define STATIC_ANALYSIS_SECOND_TOKEN_COND(...) \ + STATIC_ANALYSIS_SECOND_TOKEN(__VA_ARGS__)[0] == '\"' + +#undef STATIC_ANALYSIS_CHECK_LOG_FORMAT +#define STATIC_ANALYSIS_CHECK_LOG_FORMAT(...) \ + ::NYT \ + ::NLogging \ + ::NDetail \ + ::TAnalyserDispatcher< \ + STATIC_ANALYSIS_FIRST_TOKEN_COND(__VA_ARGS__), \ + STATIC_ANALYSIS_SECOND_TOKEN_COND(__VA_ARGS__) \ + >::Do( \ + STATIC_ANALYSIS_FIRST_TOKEN(__VA_ARGS__), \ + STATIC_ANALYSIS_SECOND_TOKEN(__VA_ARGS__), \ + STATIC_ANALYSIS_CAPTURE_TYPES(__VA_ARGS__)) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging::NDetail + +template <class T> +struct NYT::TFormatArg<NYT::NLogging::NDetail::TLoggerFormatArg<T>> + : public NYT::TFormatArgBase +{ + // We mix in '\"' and ' ' which is an artifact of logging stringize. + // We want to support YT_LOG_XXX("Value: %" PRIu64, 42) + // for plantform independent prints of numbers. + // String below may be converted to a token: + // "\"Value: %\" \"u\"" + // Thus adding a \" \" sequence. + static constexpr auto FlagSpecifiers + = TFormatArgBase::ExtendFlags</*Hot*/ false, 2, std::array{'\"', ' '}, /*TFrom*/ NYT::TFormatArg<T>>(); +}; diff --git a/library/cpp/yt/logging/static_analysis.h b/library/cpp/yt/logging/static_analysis.h new file mode 100644 index 00000000000..a335d8c6cc1 --- /dev/null +++ b/library/cpp/yt/logging/static_analysis.h @@ -0,0 +1,22 @@ +#pragma once + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +// Performs a compile-time check of log arguments validity. +// Valid argument lists are: +// 1. (format, args...) +// 2. (error, format, args...) +// If format is not a string literal or argument list +// is not valid, no check is made -- macro turns to +// a no-op. +#define STATIC_ANALYSIS_CHECK_LOG_FORMAT(...) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging + +#define STATIC_ANALYSIS_INL_H_ +#include "static_analysis-inl.h" +#undef STATIC_ANALYSIS_INL_H_ diff --git a/library/cpp/yt/logging/unittests/static_analysis_ut.cpp b/library/cpp/yt/logging/unittests/static_analysis_ut.cpp new file mode 100644 index 00000000000..1c705dc9674 --- /dev/null +++ b/library/cpp/yt/logging/unittests/static_analysis_ut.cpp @@ -0,0 +1,41 @@ +#include <library/cpp/testing/gtest/gtest.h> + +#include <library/cpp/yt/logging/logger.h> + +namespace NYT::NLogging { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TStaticAnalysis, ValidFormats) +{ + // Mock for actual error -- we only care that + // it is some runtime object. + [[maybe_unused]] struct TError + { } error; + + YT_LOG_CHECK_FORMAT("Hello"); + YT_LOG_CHECK_FORMAT("Hello %v", "World!"); + YT_LOG_CHECK_FORMAT("Hello %qv", "World!"); + YT_LOG_CHECK_FORMAT(error); + YT_LOG_CHECK_FORMAT(error, "Hello"); + YT_LOG_CHECK_FORMAT(error, "Hello %Qhs", "World!"); + YT_LOG_CHECK_FORMAT("Hello %%"); + YT_LOG_CHECK_FORMAT("Hello %" PRIu64, 42); +} + +// Uncomment this test to see that we don't have false negatives! +// TEST(TStaticAnalysis, InvalidFormats) +// { +// YT_LOG_CHECK_FORMAT("Hello", 1); +// YT_LOG_CHECK_FORMAT("Hello %"); +// YT_LOG_CHECK_FORMAT("Hello %false"); +// YT_LOG_CHECK_FORMAT("Hello ", "World"); +// YT_LOG_CHECK_FORMAT("Hello ", "(World: %v)", 42); +// YT_LOG_CHECK_FORMAT("Hello %lbov", 42); // There is no 'b' flag. +// } + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/unittests/ya.make b/library/cpp/yt/logging/unittests/ya.make index 42268d3db2f..021b0d09d6a 100644 --- a/library/cpp/yt/logging/unittests/ya.make +++ b/library/cpp/yt/logging/unittests/ya.make @@ -8,6 +8,7 @@ ENDIF() SRCS( logger_ut.cpp + static_analysis_ut.cpp ) PEERDIR( diff --git a/library/cpp/yt/misc/source_location.cpp b/library/cpp/yt/misc/source_location.cpp index 8d22d43636c..3fe45e23a76 100644 --- a/library/cpp/yt/misc/source_location.cpp +++ b/library/cpp/yt/misc/source_location.cpp @@ -1,11 +1,37 @@ #include "source_location.h" +#include <library/cpp/yt/string/format.h> + #include <string.h> namespace NYT { //////////////////////////////////////////////////////////////////////////////// +#ifdef __cpp_lib_source_location + +void FormatValue(TStringBuilderBase* builder, const std::source_location& location, TStringBuf /*format*/) +{ + if (location.file_name() != nullptr) { + builder->AppendFormat( + "%v:%v:%v", + location.file_name(), + location.line(), + location.column()); + } else { + builder->AppendString("<unknown>"); + } +} + +TString ToString(const std::source_location& location) +{ + return ToStringViaBuilder(location); +} + +#endif // __cpp_lib_source_location + +//////////////////////////////////////////////////////////////////////////////// + const char* TSourceLocation::GetFileName() const { return FileName_; diff --git a/library/cpp/yt/misc/source_location.h b/library/cpp/yt/misc/source_location.h index 84213eea708..38a6f83c804 100644 --- a/library/cpp/yt/misc/source_location.h +++ b/library/cpp/yt/misc/source_location.h @@ -1,9 +1,27 @@ #pragma once +#include <util/generic/string.h> + +#ifdef __cpp_lib_source_location +#include <source_location> +#endif // __cpp_lib_source_location + namespace NYT { //////////////////////////////////////////////////////////////////////////////// +// TODO(dgolear): Drop when LLVM-14 is eradicated. +#ifdef __cpp_lib_source_location + +class TStringBuilderBase; + +void FormatValue(TStringBuilderBase* builder, const std::source_location& location, TStringBuf /*format*/); +TString ToString(const std::source_location& location); + +#endif // __cpp_lib_source_location + +//////////////////////////////////////////////////////////////////////////////// + class TSourceLocation { public: diff --git a/library/cpp/yt/string/format_analyser-inl.h b/library/cpp/yt/string/format_analyser-inl.h new file mode 100644 index 00000000000..a548a06072a --- /dev/null +++ b/library/cpp/yt/string/format_analyser-inl.h @@ -0,0 +1,143 @@ +#ifndef FORMAT_ANALYSER_INL_H_ +#error "Direct inclusion of this file is not allowed, include format_analyser.h" +// For the sake of sane code completion. +#include "format_analyser.h" +#endif + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +consteval bool Contains(std::string_view sv, char symbol) +{ + return sv.find(symbol) != std::string_view::npos; +} + +template <class... TArgs> +consteval void TFormatAnalyser::ValidateFormat(std::string_view fmt) +{ + std::array<std::string_view, sizeof...(TArgs)> markers = {}; + std::array<TSpecifiers, sizeof...(TArgs)> specifiers{GetSpecifiers<TArgs>()...}; + + int markerCount = 0; + int currentMarkerStart = -1; + + for (int idx = 0; idx < std::ssize(fmt); ++idx) { + auto symbol = fmt[idx]; + + // Parse verbatim text. + if (currentMarkerStart == -1) { + if (symbol == IntroductorySymbol) { + // Marker maybe begins. + currentMarkerStart = idx; + } + continue; + } + + // NB: We check for %% first since + // in order to verify if symbol is a specifier + // we need markerCount to be within range of our + // specifier array. + if (symbol == IntroductorySymbol) { + if (currentMarkerStart + 1 != idx) { + // '%a% detected' + CrashCompilerWrongTermination("You may not terminate flag sequence other than %% with \'%\' symbol"); + return; + } + // '%%' detected --- skip + currentMarkerStart = -1; + continue; + } + + // We are inside of marker. + if (markerCount == std::ssize(markers)) { + // To many markers + CrashCompilerNotEnoughArguments("Number of arguments supplied to format is smaller than the number of flag sequences"); + return; + } + + if (Contains(specifiers[markerCount].Conversion, symbol)) { + // Marker has finished. + + markers[markerCount] + = std::string_view(fmt.begin() + currentMarkerStart, idx - currentMarkerStart + 1); + currentMarkerStart = -1; + ++markerCount; + + continue; + } + + if (!Contains(specifiers[markerCount].Flags, symbol)) { + CrashCompilerWrongFlagSpecifier("Symbol is not a valid flag specifier; See FlagSpecifiers"); + } + } + + if (currentMarkerStart != -1) { + // Runaway marker. + CrashCompilerMissingTermination("Unterminated flag sequence detected; Use \'%%\' to type plain %"); + return; + } + + if (markerCount < std::ssize(markers)) { + // Missing markers. + CrashCompilerTooManyArguments("Number of arguments supplied to format is greater than the number of flag sequences"); + return; + } + + // TODO(arkady-e1ppa): Consider per-type verification + // of markers. +} + +template <class TArg> +consteval auto TFormatAnalyser::GetSpecifiers() +{ + static_assert(CFormattable<TArg>, "Your specialization of TFormatArg is broken"); + + return TSpecifiers{ + .Conversion = std::string_view{ + std::data(TFormatArg<TArg>::ConversionSpecifiers), + std::size(TFormatArg<TArg>::ConversionSpecifiers)}, + .Flags = std::string_view{ + std::data(TFormatArg<TArg>::FlagSpecifiers), + std::size(TFormatArg<TArg>::FlagSpecifiers)}, + }; +} + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +template <bool Hot, size_t N, std::array<char, N> List, class TFrom> +consteval auto TFormatArgBase::ExtendConversion() +{ + static_assert(CFormattable<TFrom>); + return AppendArrays<Hot, N, List, &TFrom::ConversionSpecifiers>(); +} + +template <bool Hot, size_t N, std::array<char, N> List, class TFrom> +consteval auto TFormatArgBase::ExtendFlags() +{ + static_assert(CFormattable<TFrom>); + return AppendArrays<Hot, N, List, &TFrom::FlagSpecifiers>(); +} + +template <bool Hot, size_t N, std::array<char, N> List, auto* From> +consteval auto TFormatArgBase::AppendArrays() +{ + auto& from = *From; + return [] <size_t... I, size_t... J> ( + std::index_sequence<I...>, + std::index_sequence<J...>) { + if constexpr (Hot) { + return std::array{List[J]..., from[I]...}; + } else { + return std::array{from[I]..., List[J]...}; + } + } ( + std::make_index_sequence<std::size(from)>(), + std::make_index_sequence<N>()); + } + +} // namespace NYT::NDetail diff --git a/library/cpp/yt/string/format_analyser.h b/library/cpp/yt/string/format_analyser.h new file mode 100644 index 00000000000..4afe3f1ca8b --- /dev/null +++ b/library/cpp/yt/string/format_analyser.h @@ -0,0 +1,102 @@ +#pragma once + +#include <array> +#include <string_view> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +struct TFormatAnalyser +{ +public: + template <class... TArgs> + static consteval void ValidateFormat(std::string_view fmt); + +private: + // Non-constexpr function call will terminate compilation. + // Purposefully undefined and non-constexpr/consteval + static void CrashCompilerNotEnoughArguments(std::string_view msg); + static void CrashCompilerTooManyArguments(std::string_view msg); + static void CrashCompilerWrongTermination(std::string_view msg); + static void CrashCompilerMissingTermination(std::string_view msg); + static void CrashCompilerWrongFlagSpecifier(std::string_view msg); + + struct TSpecifiers + { + std::string_view Conversion; + std::string_view Flags; + }; + template <class TArg> + static consteval auto GetSpecifiers(); + + static constexpr char IntroductorySymbol = '%'; +}; + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +// Base used for flag checks for each type independently. +// Use it for overrides. +struct TFormatArgBase +{ +public: + // TODO(arkady-e1ppa): Consider more strict formatting rules. + static constexpr std::array ConversionSpecifiers = { + 'v', '1', 'c', 's', 'd', 'i', 'o', + 'x', 'X', 'u', 'f', 'F', 'e', 'E', + 'a', 'A', 'g', 'G', 'n', 'p' + }; + + static constexpr std::array FlagSpecifiers = { + '-', '+', ' ', '#', '0', + '1', '2', '3', '4', '5', + '6', '7', '8', '9', + '*', '.', 'h', 'l', 'j', + 'z', 't', 'L', 'q', 'Q' + }; + + template <class T> + static constexpr bool IsSpecifierList = requires (T t) { + [] <size_t N> (std::array<char, N>) { } (t); + }; + + // Hot = |true| adds specifiers to the beggining + // of a new array. + template <bool Hot, size_t N, std::array<char, N> List, class TFrom = TFormatArgBase> + static consteval auto ExtendConversion(); + + template <bool Hot, size_t N, std::array<char, N> List, class TFrom = TFormatArgBase> + static consteval auto ExtendFlags(); + +private: + template <bool Hot, size_t N, std::array<char, N> List, auto* From> + static consteval auto AppendArrays(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +struct TFormatArg + : public TFormatArgBase +{ }; + +// Semantic requirement: +// Said field must be constexpr. +template <class T> +concept CFormattable = requires { + TFormatArg<T>::ConversionSpecifiers; + requires TFormatArgBase::IsSpecifierList<decltype(TFormatArg<T>::ConversionSpecifiers)>; + + TFormatArg<T>::FlagSpecifiers; + requires TFormatArgBase::IsSpecifierList<decltype(TFormatArg<T>::FlagSpecifiers)>; +}; + +} // namespace NYT + +#define FORMAT_ANALYSER_INL_H_ +#include "format_analyser-inl.h" +#undef FORMAT_ANALYSER_INL_H_ |