diff options
author | Alexander Smirnov <alex@ydb.tech> | 2024-10-28 20:34:11 +0000 |
---|---|---|
committer | Alexander Smirnov <alex@ydb.tech> | 2024-10-28 20:34:11 +0000 |
commit | ef9875b11a33dbd25e92bc6b4cf692c18c9ba0ce (patch) | |
tree | 1f2fd4e4d9e585da35937b42fbda5f854af04728 /library | |
parent | 37ae9cc90160b53eb0e22021c47b3996a01cd656 (diff) | |
parent | e3c8507a3d1cb090278f211232ddfde3bedc54d4 (diff) | |
download | ydb-ef9875b11a33dbd25e92bc6b4cf692c18c9ba0ce.tar.gz |
Merge branch 'rightlib' into mergelibs-241028-2033
Diffstat (limited to 'library')
21 files changed, 596 insertions, 112 deletions
diff --git a/library/cpp/containers/bitseq/bititerator.h b/library/cpp/containers/bitseq/bititerator.h index efe7ba873b..cc18af572f 100644 --- a/library/cpp/containers/bitseq/bititerator.h +++ b/library/cpp/containers/bitseq/bititerator.h @@ -1,8 +1,8 @@ #pragma once #include "traits.h" +#include <bit> -#include <library/cpp/pop_count/popcount.h> template <typename T> class TBitIterator { @@ -49,7 +49,7 @@ public: if (!Mask) return *Data & TTraits::ElemMask(count); - auto usedBits = (size_t)PopCount(Mask - 1); + auto usedBits = (size_t)std::popcount(Mask - 1u); TWord result = Current >> usedBits; auto leftInCurrent = TTraits::NumBits - usedBits; if (count <= leftInCurrent) @@ -72,7 +72,7 @@ public: return Current & TTraits::ElemMask(count); } - auto usedBits = (size_t)PopCount(Mask - 1); + auto usedBits = (size_t)std::popcount(Mask - 1u); TWord result = Current >> usedBits; auto leftInCurrent = TTraits::NumBits - usedBits; if (count < leftInCurrent) { @@ -97,7 +97,7 @@ public: if (!count) return; - int leftInCurrent = (size_t)PopCount(~(Mask - 1)); + int leftInCurrent = std::popcount(static_cast<TWord>(~(Mask - 1u))); if (count < leftInCurrent) { Mask <<= count; return; diff --git a/library/cpp/containers/bitseq/bitvector.h b/library/cpp/containers/bitseq/bitvector.h index 3f8fd81ee5..c1d1e13cd7 100644 --- a/library/cpp/containers/bitseq/bitvector.h +++ b/library/cpp/containers/bitseq/bitvector.h @@ -2,11 +2,12 @@ #include "traits.h" -#include <library/cpp/pop_count/popcount.h> #include <util/generic/vector.h> #include <util/ysaveload.h> +#include <bit> + template <typename T> class TReadonlyBitVector; @@ -113,7 +114,7 @@ public: size_t Count() const { size_t count = 0; for (size_t i = 0; i < Data_.size(); ++i) { - count += (size_t)PopCount(Data_[i]); + count += (size_t)std::popcount(Data_[i]); } return count; } diff --git a/library/cpp/containers/bitseq/ya.make b/library/cpp/containers/bitseq/ya.make index a59c3e765e..49249f366f 100644 --- a/library/cpp/containers/bitseq/ya.make +++ b/library/cpp/containers/bitseq/ya.make @@ -2,7 +2,6 @@ LIBRARY() PEERDIR( util/draft - library/cpp/pop_count ) SRCS( diff --git a/library/cpp/http/server/http_ex.cpp b/library/cpp/http/server/http_ex.cpp index fc43d2defc..0681da10ff 100644 --- a/library/cpp/http/server/http_ex.cpp +++ b/library/cpp/http/server/http_ex.cpp @@ -87,7 +87,8 @@ bool THttpClientRequestExtension::ProcessHeaders(TBaseServerRequestData& rd, TBl } TBuffer buf(SafeIntegerCast<size_t>(contentLength)); - buf.Resize(Input().Load(buf.Data(), (size_t)contentLength)); + Input().LoadOrFail(buf.Data(), (size_t)contentLength); + buf.Resize((size_t)contentLength); postData = TBlob::FromBuffer(buf); } else { postData = TBlob::FromStream(Input()); diff --git a/library/cpp/pop_count/benchmark/main.cpp b/library/cpp/pop_count/benchmark/main.cpp index 41ea3c91cc..1af2d157b6 100644 --- a/library/cpp/pop_count/benchmark/main.cpp +++ b/library/cpp/pop_count/benchmark/main.cpp @@ -5,6 +5,8 @@ #include <library/cpp/pop_count/popcount.h> #include <library/cpp/testing/benchmark/bench.h> +#include <bit> + template <class F, class I> inline void DoRun(F&& f, I&& i) { const ui64 n = i.Iterations(); @@ -21,6 +23,13 @@ Y_CPU_BENCHMARK(PopCount_8, iface) { iface); } +Y_CPU_BENCHMARK(std_popcount_8, iface) { + DoRun([](ui8 x) { + return std::popcount<ui8>(x); + }, + iface); +} + Y_CPU_BENCHMARK(PopCount_16, iface) { DoRun([](ui16 x) { return PopCount<ui16>(x); @@ -28,6 +37,13 @@ Y_CPU_BENCHMARK(PopCount_16, iface) { iface); } +Y_CPU_BENCHMARK(std_popcount_16, iface) { + DoRun([](ui16 x) { + return std::popcount<ui16>(x); + }, + iface); +} + Y_CPU_BENCHMARK(PopCount_32, iface) { DoRun([](ui32 x) { return PopCount<ui32>(x); @@ -35,6 +51,13 @@ Y_CPU_BENCHMARK(PopCount_32, iface) { iface); } +Y_CPU_BENCHMARK(std_popcount_32, iface) { + DoRun([](ui32 x) { + return std::popcount<ui32>(x); + }, + iface); +} + Y_CPU_BENCHMARK(PopCount_64, iface) { DoRun([](ui64 x) { return PopCount<ui64>(x); @@ -42,6 +65,13 @@ Y_CPU_BENCHMARK(PopCount_64, iface) { iface); } +Y_CPU_BENCHMARK(std_popcount_64, iface) { + DoRun([](ui64 x) { + return std::popcount<ui64>(x); + }, + iface); +} + #if !defined(_MSC_VER) Y_CPU_BENCHMARK(BUILTIN_64, iface) { DoRun([](ui64 x) { diff --git a/library/cpp/regex/hyperscan/hyperscan.cpp b/library/cpp/regex/hyperscan/hyperscan.cpp index ba85e9bbab..5b6069d4a3 100644 --- a/library/cpp/regex/hyperscan/hyperscan.cpp +++ b/library/cpp/regex/hyperscan/hyperscan.cpp @@ -10,6 +10,7 @@ #include <contrib/libs/hyperscan/runtime_avx512/hs_runtime.h> #include <util/generic/singleton.h> +#include <util/system/sanitizers.h> namespace NHyperscan { using TSerializedDatabase = THolder<char, TDeleter<decltype(&free), &free>>; @@ -18,7 +19,8 @@ namespace NHyperscan { namespace NPrivate { ERuntime DetectCurrentRuntime() { - if (NX86::HaveAVX512F() && NX86::HaveAVX512BW()) { + // TODO: Remove MSanIsOn check upon DEVTOOLSSUPPORT-49258 resolution + if (NX86::HaveAVX512F() && NX86::HaveAVX512BW() && !NSan::MSanIsOn()) { return ERuntime::AVX512; } else if (NX86::HaveAVX() && NX86::HaveAVX2()) { return ERuntime::AVX2; diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt index 0e38681a50..fbf58d0ddb 100644 --- a/library/cpp/tld/tlds-alpha-by-domain.txt +++ b/library/cpp/tld/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2024101900, Last Updated Sat Oct 19 07:07:02 2024 UTC +# Version 2024102500, Last Updated Fri Oct 25 07:07:02 2024 UTC AAA AARP ABB diff --git a/library/cpp/yt/error/error_attributes-inl.h b/library/cpp/yt/error/error_attributes-inl.h new file mode 100644 index 0000000000..a05e62d0e9 --- /dev/null +++ b/library/cpp/yt/error/error_attributes-inl.h @@ -0,0 +1,72 @@ +#ifndef ERROR_ATTRIBUTES_INL_H_ +#error "Direct inclusion of this file is not allowed, include error_attributes.h" +// For the sake of sane code completion. +#include "error_attributes.h" +#endif + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +T TErrorAttributes::GetAndRemove(const TString& key) +{ + auto result = Get<T>(key); + Remove(key); + return result; +} + +template <class T> +T TErrorAttributes::Get(TStringBuf key, const T& defaultValue) const +{ + return Find<T>(key).value_or(defaultValue); +} + +template <class T> +T TErrorAttributes::GetAndRemove(const TString& key, const T& defaultValue) +{ + auto result = Find<T>(key); + if (result) { + Remove(key); + return *result; + } else { + return defaultValue; + } +} + +template <class T> +typename TOptionalTraits<T>::TOptional TErrorAttributes::FindAndRemove(const TString& key) +{ + auto result = Find<T>(key); + if (result) { + Remove(key); + } + return result; +} + +template <CMergeableDictionary TDictionary> +void TErrorAttributes::MergeFrom(const TDictionary& dict) +{ + using TTraits = TMergeDictionariesTraits<TDictionary>; + + for (const auto& [key, value] : TTraits::MakeIterableView(dict)) { + SetYson(key, value); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +template <> +struct TMergeDictionariesTraits<TErrorAttributes> +{ + static auto MakeIterableView(const TErrorAttributes& attributes) + { + return attributes.ListPairs(); + } +}; + +static_assert(CMergeableDictionary<TErrorAttributes>); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/error/error_attributes.cpp b/library/cpp/yt/error/error_attributes.cpp new file mode 100644 index 0000000000..2c1b542314 --- /dev/null +++ b/library/cpp/yt/error/error_attributes.cpp @@ -0,0 +1,62 @@ +#include "error_attributes.h" + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +TErrorAttributes::TErrorAttributes(void* attributes) + : Attributes_(attributes) +{ } + +void TErrorAttributes::Clear() +{ + for (const auto& key : ListKeys()) { + Remove(key); + } +} + +NYson::TYsonString TErrorAttributes::GetYsonAndRemove(const TString& key) +{ + auto result = GetYson(key); + Remove(key); + return result; +} + +bool TErrorAttributes::Contains(TStringBuf key) const +{ + return FindYson(key).operator bool(); +} + +bool operator == (const TErrorAttributes& lhs, const TErrorAttributes& rhs) +{ + auto lhsPairs = lhs.ListPairs(); + auto rhsPairs = rhs.ListPairs(); + if (lhsPairs.size() != rhsPairs.size()) { + return false; + } + + std::sort(lhsPairs.begin(), lhsPairs.end(), [] (const auto& lhs, const auto& rhs) { + return lhs.first < rhs.first; + }); + std::sort(rhsPairs.begin(), rhsPairs.end(), [] (const auto& lhs, const auto& rhs) { + return lhs.first < rhs.first; + }); + + for (auto index = 0; index < std::ssize(lhsPairs); ++index) { + if (lhsPairs[index].first != rhsPairs[index].first) { + return false; + } + } + + for (auto index = 0; index < std::ssize(lhsPairs); ++index) { + if (lhsPairs[index].second != rhsPairs[index].second) { + return false; + } + } + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/error/error_attributes.h b/library/cpp/yt/error/error_attributes.h new file mode 100644 index 0000000000..88771f8360 --- /dev/null +++ b/library/cpp/yt/error/error_attributes.h @@ -0,0 +1,105 @@ +#pragma once + +#include "mergeable_dictionary.h" + +#include <library/cpp/yt/misc/optional.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +// For now this is just an opaque handle to error attributes +// used to remove dependency on IAttributeDictionary in public API. +// Eventually it would be a simple hash map. +// NB(arkady-e1ppa): For now most methods are defined in yt/yt/core/misc/stripped_error.cpp +// eventually they will be moved here. +class TErrorAttributes +{ +public: + using TKey = TString; + using TValue = NYson::TYsonString; + using TKeyValuePair = std::pair<TKey, TValue>; + + //! Returns the list of all keys in the dictionary. + std::vector<TString> ListKeys() const; + + //! Returns the list of all key-value pairs in the dictionary. + std::vector<TKeyValuePair> ListPairs() const; + + //! Returns the value of the attribute (null indicates that the attribute is not found). + NYson::TYsonString FindYson(TStringBuf key) const; + + //! Sets the value of the attribute. + void SetYson(const TString& key, const NYson::TYsonString& value); + + //! Removes the attribute. + //! Returns |true| if the attribute was removed or |false| if there is no attribute with this key. + bool Remove(const TString& key); + + //! Removes all attributes. + void Clear(); + + //! Returns the value of the attribute (throws an exception if the attribute is not found). + NYson::TYsonString GetYson(TStringBuf key) const; + + //! Same as #GetYson but removes the value. + NYson::TYsonString GetYsonAndRemove(const TString& key); + + //! Returns |true| iff the given key is present. + bool Contains(TStringBuf key) const; + + // TODO(arkady-e1ppa): By default deserialization is located at yt/core + // consider using deserialization of some default types (guid, string, int, double) + // to be supported and everything else not supported without inclusion of yt/core. + //! Finds the attribute and deserializes its value. + //! Throws if no such value is found. + template <class T> + T Get(TStringBuf key) const; + + //! Same as #Get but removes the value. + template <class T> + T GetAndRemove(const TString& key); + + //! Finds the attribute and deserializes its value. + //! Uses default value if no such attribute is found. + template <class T> + T Get(TStringBuf key, const T& defaultValue) const; + + //! Same as #Get but removes the value if it exists. + template <class T> + T GetAndRemove(const TString& key, const T& defaultValue); + + //! Finds the attribute and deserializes its value. + //! Returns null if no such attribute is found. + template <class T> + typename TOptionalTraits<T>::TOptional Find(TStringBuf key) const; + + //! Same as #Find but removes the value if it exists. + template <class T> + typename TOptionalTraits<T>::TOptional FindAndRemove(const TString& key); + + template <CMergeableDictionary TDictionary> + void MergeFrom(const TDictionary& dict); + +private: + void* Attributes_; // IAttributesDictionary* + + friend class TErrorOr<void>; + explicit TErrorAttributes(void* attributes); + + TErrorAttributes(const TErrorAttributes& other) = default; + TErrorAttributes& operator= (const TErrorAttributes& other) = default; + + TErrorAttributes(TErrorAttributes&& other) = default; + TErrorAttributes& operator= (TErrorAttributes&& other) = default; +}; + +bool operator == (const TErrorAttributes& lhs, const TErrorAttributes& rhs); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + +#define ERROR_ATTRIBUTES_INL_H_ +#include "error_attributes-inl.h" +#undef ERROR_ATTRIBUTES_INL_H_ diff --git a/library/cpp/yt/error/mergeable_dictionary.h b/library/cpp/yt/error/mergeable_dictionary.h new file mode 100644 index 0000000000..90597059e4 --- /dev/null +++ b/library/cpp/yt/error/mergeable_dictionary.h @@ -0,0 +1,57 @@ +#pragma once + +#include "public.h" + +#include <ranges> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +// Can be specialized to make your dictionary satisfy CMergeableDictionary. +template <class TDictionary> +struct TMergeDictionariesTraits +{ + static auto MakeIterableView(const TDictionary& dict) + requires false; +}; + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +template <class T> +struct TMergeableDictionaryImpl +{ + using TView = std::invoke_result_t<decltype(&TMergeDictionariesTraits<T>::MakeIterableView), const T&>; + using TIterator = std::ranges::iterator_t<TView>; + using TValue = typename std::iterator_traits<TIterator>::value_type; + + static constexpr bool ValidSize = requires { + { std::tuple_size<TValue>::value } -> std::same_as<const size_t&>; + } && (std::tuple_size<TValue>::value == 2); + + static constexpr bool CorrectTupleElements = requires { + typename std::tuple_element<0, TValue>::type; + std::same_as<typename std::tuple_element<0, TValue>::type, TString>; + + typename std::tuple_element<1, TValue>::type; + std::same_as<typename std::tuple_element<1, TValue>::type, NYson::TYsonString>; + }; +}; + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +concept CMergeableDictionary = + requires (const T& dict) { + TMergeDictionariesTraits<T>::MakeIterableView(dict); + } && + NDetail::TMergeableDictionaryImpl<T>::ValidSize && + NDetail::TMergeableDictionaryImpl<T>::CorrectTupleElements; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/error/public.h b/library/cpp/yt/error/public.h new file mode 100644 index 0000000000..63b95497c7 --- /dev/null +++ b/library/cpp/yt/error/public.h @@ -0,0 +1,17 @@ +#pragma once + +#include <library/cpp/yt/yson_string/string.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +class TErrorOr; + +class TErrorAttributes; +struct TOriginAttributes; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/error/ya.make b/library/cpp/yt/error/ya.make index f5bd946199..60d3e9a92f 100644 --- a/library/cpp/yt/error/ya.make +++ b/library/cpp/yt/error/ya.make @@ -9,9 +9,11 @@ PEERDIR( library/cpp/yt/misc library/cpp/yt/threading library/cpp/yt/string + library/cpp/yt/yson_string # TODO(arkady-e1ppa): eliminate ) SRCS( + error_attributes.cpp origin_attributes.cpp ) diff --git a/library/cpp/yt/misc/enum-inl.h b/library/cpp/yt/misc/enum-inl.h index 5e1d04da1f..f1fb9f63e1 100644 --- a/library/cpp/yt/misc/enum-inl.h +++ b/library/cpp/yt/misc/enum-inl.h @@ -11,6 +11,7 @@ #include <util/generic/cast.h> #include <algorithm> +#include <numeric> #include <stdexcept> namespace NYT { @@ -38,6 +39,14 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// +template <class T> +constexpr std::optional<T> TryGetEnumUnknownValueImpl(T) +{ + return std::nullopt; +} + +//////////////////////////////////////////////////////////////////////////////// + namespace NDetail { template <typename TValues> @@ -132,6 +141,13 @@ constexpr bool CheckDomainNames(const TNames& names) return std::nullopt; \ } \ \ + static constexpr bool IsKnownValue(T value) \ + { \ + return false \ + PP_FOR_EACH(ENUM__IS_KNOWN_VALUE_ITEM, seq) \ + ; \ + } \ + \ static constexpr const std::array<TStringBuf, DomainSize>& GetDomainNames() \ { \ return Names; \ @@ -184,6 +200,19 @@ constexpr bool CheckDomainNames(const TNames& names) #define ENUM__GET_DOMAIN_NAMES_ITEM_ATOMIC(item) \ TStringBuf(PP_STRINGIZE(item)), +#define ENUM__IS_KNOWN_VALUE_ITEM(item) \ + PP_IF( \ + PP_IS_SEQUENCE(item), \ + ENUM__IS_KNOWN_VALUE_ITEM_SEQ, \ + ENUM__IS_KNOWN_VALUE_ITEM_ATOMIC \ + )(item) + +#define ENUM__IS_KNOWN_VALUE_ITEM_SEQ(seq) \ + ENUM__IS_KNOWN_VALUE_ITEM_ATOMIC(PP_ELEMENT(seq, 0)) + +#define ENUM__IS_KNOWN_VALUE_ITEM_ATOMIC(item) \ + || value == T::item + #define ENUM__VALIDATE_UNIQUE(enumType) \ static_assert(IsMonotonic || ::NYT::NDetail::CheckValuesUnique(Values), \ "Enumeration " #enumType " contains duplicate values"); @@ -241,6 +270,13 @@ constexpr T TEnumTraitsWithKnownDomain<T, true>::GetMaxValue() } template <class T> +constexpr T TEnumTraitsWithKnownDomain<T, true>::GetAllSetValue() + requires (TEnumTraitsImpl<T>::IsBitEnum) +{ + return TEnumTraitsImpl<T>::GetAllSetValue(); +} + +template <class T> std::vector<T> TEnumTraitsWithKnownDomain<T, true>::Decompose(T value) requires (TEnumTraitsImpl<T>::IsBitEnum) { @@ -262,6 +298,13 @@ TStringBuf TEnumTraits<T, true>::GetTypeName() } template <class T> +constexpr std::optional<T> TEnumTraits<T, true>::TryGetUnknownValue() +{ + using NYT::TryGetEnumUnknownValueImpl; + return TryGetEnumUnknownValueImpl(T()); +} + +template <class T> std::optional<T> TEnumTraits<T, true>::FindValueByLiteral(TStringBuf literal) { return TEnumTraitsImpl<T>::FindValueByLiteral(literal); @@ -274,6 +317,13 @@ std::optional<TStringBuf> TEnumTraits<T, true>::FindLiteralByValue(T value) } template <class T> +constexpr bool TEnumTraits<T, true>::IsKnownValue(T value) + requires (!TEnumTraitsImpl<T>::IsBitEnum) +{ + return TEnumTraitsImpl<T>::IsKnownValue(value); +} + +template <class T> TString TEnumTraits<T, true>::ToString(T value) { using ::ToString; @@ -308,7 +358,7 @@ T TEnumTraits<T, true>::FromString(TStringBuf literal) return T(ToUnderlying(lhs) op ToUnderlying(rhs)); \ } \ \ - [[maybe_unused]] inline T& operator assignOp (T& lhs, T rhs) \ + [[maybe_unused]] inline constexpr T& operator assignOp (T& lhs, T rhs) \ { \ lhs = T(ToUnderlying(lhs) op ToUnderlying(rhs)); \ return lhs; \ @@ -326,13 +376,13 @@ T TEnumTraits<T, true>::FromString(TStringBuf literal) return T(ToUnderlying(lhs) op rhs); \ } \ \ - [[maybe_unused]] inline T& operator assignOp (T& lhs, size_t rhs) \ + [[maybe_unused]] inline constexpr T& operator assignOp (T& lhs, size_t rhs) \ { \ lhs = T(ToUnderlying(lhs) op rhs); \ return lhs; \ } -#define ENUM__BITWISE_OPS(enumType) \ +#define ENUM__BITWISE_OPS(enumType) \ ENUM__BINARY_BITWISE_OPERATOR(enumType, &=, &) \ ENUM__BINARY_BITWISE_OPERATOR(enumType, |=, | ) \ ENUM__BINARY_BITWISE_OPERATOR(enumType, ^=, ^) \ @@ -340,6 +390,16 @@ T TEnumTraits<T, true>::FromString(TStringBuf literal) ENUM__BIT_SHIFT_OPERATOR(enumType, <<=, << ) \ ENUM__BIT_SHIFT_OPERATOR(enumType, >>=, >> ) +#define ENUM__ALL_SET_VALUE(enumType, seq) \ + static constexpr enumType GetAllSetValue() \ + { \ + return std::accumulate( \ + Values.begin(), \ + Values.end(), \ + enumType(), \ + [] (auto a, auto b) { return a | b; }); \ + } + //////////////////////////////////////////////////////////////////////////////// template <typename E> diff --git a/library/cpp/yt/misc/enum.h b/library/cpp/yt/misc/enum.h index 4d40ab8ec4..d9611b2823 100644 --- a/library/cpp/yt/misc/enum.h +++ b/library/cpp/yt/misc/enum.h @@ -33,18 +33,11 @@ template <class T> using TEnumTraitsImpl = decltype(GetEnumTraitsImpl(T())); template <class T> -constexpr bool IsEnumDomainSizeKnown() -{ - if constexpr(requires{ TEnumTraitsImpl<T>::DomainSize; }) { - return true; - } else { - return false; - } -} +constexpr std::optional<T> TryGetEnumUnknownValueImpl(T); template < class T, - bool = IsEnumDomainSizeKnown<T>() + bool DomainSizeKnown = requires{ TEnumTraitsImpl<T>::DomainSize; } > struct TEnumTraitsWithKnownDomain { }; @@ -62,7 +55,7 @@ struct TEnumTraits }; template <class T> -struct TEnumTraitsWithKnownDomain<T, true> +struct TEnumTraitsWithKnownDomain<T, /*DomainSizeKnown*/ true> { static constexpr int GetDomainSize(); @@ -76,6 +69,8 @@ struct TEnumTraitsWithKnownDomain<T, true> requires (!TEnumTraitsImpl<T>::IsBitEnum); // For bit enums only. + static constexpr T GetAllSetValue() + requires (TEnumTraitsImpl<T>::IsBitEnum); static std::vector<T> Decompose(T value) requires (TEnumTraitsImpl<T>::IsBitEnum); }; @@ -91,8 +86,11 @@ struct TEnumTraits<T, true> static TStringBuf GetTypeName(); + static constexpr std::optional<T> TryGetUnknownValue(); static std::optional<TStringBuf> FindLiteralByValue(T value); static std::optional<T> FindValueByLiteral(TStringBuf literal); + static constexpr bool IsKnownValue(T value) + requires (!TEnumTraitsImpl<T>::IsBitEnum); static TString ToString(T value); static T FromString(TStringBuf literal); @@ -102,7 +100,7 @@ struct TEnumTraits<T, true> //! Defines a smart enumeration with a specific underlying type. /*! - * \param enumType Enumeration enumType. + * \param enumType Enumeration type. * \param seq Enumeration domain encoded as a <em>sequence</em>. * \param underlyingType Underlying type. */ @@ -127,35 +125,37 @@ struct TEnumTraits<T, true> //! Defines a smart enumeration with a specific underlying type. /*! - * \param enumType Enumeration enumType. + * \param enumType Enumeration type. * \param seq Enumeration domain encoded as a <em>sequence</em>. * \param underlyingType Underlying type. */ #define DEFINE_BIT_ENUM_WITH_UNDERLYING_TYPE(enumType, underlyingType, seq) \ ENUM__CLASS(enumType, underlyingType, seq) \ + ENUM__BITWISE_OPS(enumType) \ ENUM__BEGIN_TRAITS(enumType, underlyingType, true, false, seq) \ ENUM__VALIDATE_UNIQUE(enumType) \ + ENUM__ALL_SET_VALUE(enumType, seq) \ ENUM__END_TRAITS(enumType) \ - ENUM__BITWISE_OPS(enumType) \ static_assert(true) //! Defines a smart enumeration with a specific underlying type. //! Duplicate enumeration values are allowed. /*! - * \param enumType Enumeration enumType. + * \param enumType Enumeration type. * \param seq Enumeration domain encoded as a <em>sequence</em>. * \param underlyingType Underlying type. */ #define DEFINE_AMBIGUOUS_BIT_ENUM_WITH_UNDERLYING_TYPE(enumType, underlyingType, seq) \ ENUM__CLASS(enumType, underlyingType, seq) \ + ENUM__BITWISE_OPS(enumType) \ ENUM__BEGIN_TRAITS(enumType, underlyingType, true, false, seq) \ + ENUM__ALL_SET_VALUE(enumType, seq) \ ENUM__END_TRAITS(enumType) \ - ENUM__BITWISE_OPS(enumType) \ static_assert(true) //! Defines a smart enumeration with the default |unsigned int| underlying type. /*! - * \param enumType Enumeration enumType. + * \param enumType Enumeration type. * \param seq Enumeration domain encoded as a <em>sequence</em>. */ #define DEFINE_BIT_ENUM(enumType, seq) \ @@ -163,7 +163,7 @@ struct TEnumTraits<T, true> //! Defines a smart enumeration with a specific underlying type and IsStringSerializable attribute. /*! - * \param enumType Enumeration enumType. + * \param enumType Enumeration type. * \param seq Enumeration domain encoded as a <em>sequence</em>. * \param underlyingType Underlying type. */ @@ -186,6 +186,18 @@ struct TEnumTraits<T, true> #define DEFINE_STRING_SERIALIZABLE_ENUM(enumType, seq) \ DEFINE_STRING_SERIALIZABLE_ENUM_WITH_UNDERLYING_TYPE(enumType, int, seq) +//! When enum from another representation (e.g. string or protobuf integer), +//! instructs the parser to treat undeclared values as |unknownValue|. +/*! + * \param enumType Enumeration type. + * \param unknownValue A sentinel value of #enumType. + */ +#define DEFINE_ENUM_UNKNOWN_VALUE(enumType, unknownValue) \ + [[maybe_unused]] constexpr std::optional<enumType> TryGetEnumUnknownValueImpl(enumType) \ + { \ + return enumType::unknownValue; \ + } + //////////////////////////////////////////////////////////////////////////////// //! Returns |true| iff the enumeration value is not bitwise zero. diff --git a/library/cpp/yt/misc/unittests/enum_ut.cpp b/library/cpp/yt/misc/unittests/enum_ut.cpp index 63b8666ae1..fc7feed227 100644 --- a/library/cpp/yt/misc/unittests/enum_ut.cpp +++ b/library/cpp/yt/misc/unittests/enum_ut.cpp @@ -50,6 +50,13 @@ DEFINE_ENUM_WITH_UNDERLYING_TYPE(ECardinal, char, ((South) (3)) ); +DEFINE_ENUM(EWithUnknown, + (First) + (Second) + (Unknown) +); +DEFINE_ENUM_UNKNOWN_VALUE(EWithUnknown, Unknown); + //////////////////////////////////////////////////////////////////////////////// template <class T, size_t N> @@ -203,6 +210,22 @@ TEST(TEnumTest, DomainValues) EXPECT_EQ(colorValues, ToVector(TEnumTraits<EColor>::GetDomainValues())); } +TEST(TEnumTest, IsKnownValue) +{ + EXPECT_TRUE(TEnumTraits<ESimple>::IsKnownValue(ESimple::X)); + EXPECT_TRUE(TEnumTraits<ESimple>::IsKnownValue(ESimple::Y)); + EXPECT_TRUE(TEnumTraits<ESimple>::IsKnownValue(ESimple::Z)); + + EXPECT_FALSE(TEnumTraits<ESimple>::IsKnownValue(static_cast<ESimple>(100))); + + EXPECT_TRUE(TEnumTraits<EColor>::IsKnownValue(EColor::Red)); +} + +TEST(TEnumTest, AllSetValue) +{ + EXPECT_EQ(TEnumTraits<EFlag>::GetAllSetValue(), EFlag::_1 | EFlag::_2 | EFlag::_3 | EFlag::_4); +} + TEST(TEnumTest, Decompose1) { auto f = EFlag(0); @@ -280,6 +303,12 @@ TEST(TEnumTest, Cast) } } +TEST(TEnumTest, UnknownValue) +{ + EXPECT_EQ(TEnumTraits<EColor>::TryGetUnknownValue(), std::nullopt); + EXPECT_EQ(TEnumTraits<EWithUnknown>::TryGetUnknownValue(), EWithUnknown::Unknown); +} + //////////////////////////////////////////////////////////////////////////////// } // namespace diff --git a/library/cpp/yt/string/enum-inl.h b/library/cpp/yt/string/enum-inl.h index 41f7197d15..84b0941d15 100644 --- a/library/cpp/yt/string/enum-inl.h +++ b/library/cpp/yt/string/enum-inl.h @@ -5,6 +5,8 @@ #endif #include "format.h" +#include "string.h" +#include "string_builder.h" #include <library/cpp/yt/exception/exception.h> @@ -23,110 +25,106 @@ void ThrowMalformedEnumValueException( TStringBuf value); void FormatUnknownEnumValue( - TStringBuilderBase* builder, + auto* builder, TStringBuf name, - i64 value); + auto value) +{ + builder->AppendFormat("%v::unknown-%v", name, ToUnderlying(value)); +} } // namespace NDetail template <class T> -std::optional<T> TryParseEnum(TStringBuf value) +std::optional<T> TryParseEnum(TStringBuf str, bool enableUnknown) { - auto tryFromString = [] (TStringBuf value) -> std::optional<T> { - if (auto decodedValue = TryDecodeEnumValue(value)) { - auto enumValue = TEnumTraits<T>::FindValueByLiteral(*decodedValue); - return enumValue ? enumValue : TEnumTraits<T>::FindValueByLiteral(value); - } + auto tryParseToken = [&] (TStringBuf token) -> std::optional<T> { + if (auto optionalValue = TEnumTraits<T>::FindValueByLiteral(token)) { + return *optionalValue; - auto reportError = [value] () { - throw TSimpleException(Format("Enum value %Qv is neither in a proper underscore case nor in a format \"%v(123)\"", - value, - TEnumTraits<T>::GetTypeName())); - }; - - TStringBuf typeName; - auto isTypeNameCorrect = value.NextTok('(', typeName) && typeName == TEnumTraits<T>::GetTypeName(); - if (!isTypeNameCorrect) { - reportError(); } - TStringBuf enumValue; - std::underlying_type_t<T> underlyingValue = 0; - auto isEnumValueCorrect = value.NextTok(')', enumValue) && TryFromString(enumValue, underlyingValue); - if (!isEnumValueCorrect) { - reportError(); + if (auto optionalDecodedValue = TryDecodeEnumValue(token)) { + if (auto optionalValue = TEnumTraits<T>::FindValueByLiteral(*optionalDecodedValue)) { + return *optionalValue; + } } - auto isParsingComplete = value.empty(); - if (!isParsingComplete) { - reportError(); + if (enableUnknown) { + if constexpr (constexpr auto optionalUnknownValue = TEnumTraits<T>::TryGetUnknownValue()) { + return *optionalUnknownValue; + } } - return static_cast<T>(underlyingValue); + return std::nullopt; }; if constexpr (TEnumTraits<T>::IsBitEnum) { T result{}; TStringBuf token; - while (value.NextTok('|', token)) { - if (auto scalar = tryFromString(StripString(token))) { - result |= *scalar; + while (str.NextTok('|', token)) { + if (auto optionalValue = tryParseToken(StripString(token))) { + result |= *optionalValue; } else { return {}; } } return result; } else { - return tryFromString(value); + return tryParseToken(str); } } template <class T> -T ParseEnum(TStringBuf value) +T ParseEnum(TStringBuf str) { - if (auto optionalResult = TryParseEnum<T>(value)) { + if (auto optionalResult = TryParseEnum<T>(str, /*enableUnkown*/ true)) { return *optionalResult; } - NYT::NDetail::ThrowMalformedEnumValueException(TEnumTraits<T>::GetTypeName(), value); + NYT::NDetail::ThrowMalformedEnumValueException(TEnumTraits<T>::GetTypeName(), str); } template <class T> void FormatEnum(TStringBuilderBase* builder, T value, bool lowerCase) { - auto formatScalarValue = [builder, lowerCase] (T value) { - auto optionalLiteral = TEnumTraits<T>::FindLiteralByValue(value); - if (!optionalLiteral) { - NYT::NDetail::FormatUnknownEnumValue( - builder, - TEnumTraits<T>::GetTypeName(), - ToUnderlying(value)); - return; - } - + auto formatLiteral = [&] (auto* builder, TStringBuf literal) { if (lowerCase) { - CamelCaseToUnderscoreCase(builder, *optionalLiteral); + CamelCaseToUnderscoreCase(builder, literal); } else { - builder->AppendString(*optionalLiteral); + builder->AppendString(literal); } }; if constexpr (TEnumTraits<T>::IsBitEnum) { - if (TEnumTraits<T>::FindLiteralByValue(value)) { - formatScalarValue(value); + if (None(value)) { + // Avoid empty string if possible. + if (auto optionalLiteral = TEnumTraits<T>::FindLiteralByValue(value)) { + formatLiteral(builder, *optionalLiteral); + } return; } - auto first = true; - for (auto scalarValue : TEnumTraits<T>::GetDomainValues()) { - if (Any(value & scalarValue)) { - if (!first) { - builder->AppendString(TStringBuf(" | ")); - } - first = false; - formatScalarValue(scalarValue); + + TDelimitedStringBuilderWrapper delimitedBuilder(builder, " | "); + + T printedValue{}; + for (auto currentValue : TEnumTraits<T>::GetDomainValues()) { + // Check if currentValue is viable and non-redunant. + if ((value & currentValue) == currentValue && (printedValue | currentValue) != printedValue) { + formatLiteral(&delimitedBuilder, *TEnumTraits<T>::FindLiteralByValue(currentValue)); + printedValue |= currentValue; } } + + // Handle the remainder. + if (printedValue != value) { + NYT::NDetail::FormatUnknownEnumValue(&delimitedBuilder, TEnumTraits<T>::GetTypeName(), value & ~printedValue); + } } else { - formatScalarValue(value); + if (auto optionalLiteral = TEnumTraits<T>::FindLiteralByValue(value)) { + formatLiteral(builder, *optionalLiteral); + return; + } + + NYT::NDetail::FormatUnknownEnumValue(builder, TEnumTraits<T>::GetTypeName(), value); } } diff --git a/library/cpp/yt/string/enum.cpp b/library/cpp/yt/string/enum.cpp index e61b813fd2..935c3e6e3e 100644 --- a/library/cpp/yt/string/enum.cpp +++ b/library/cpp/yt/string/enum.cpp @@ -6,13 +6,22 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// -template <bool failOnError> +namespace NDetail { + +void ThrowMalformedEnumValueException(TStringBuf typeName, TStringBuf value) +{ + throw TSimpleException(Format("Error parsing %v value %Qv", + typeName, + value)); +} + +template <bool ThrowOnError> std::optional<TString> DecodeEnumValueImpl(TStringBuf value) { auto camelValue = UnderscoreCaseToCamelCase(value); auto underscoreValue = CamelCaseToUnderscoreCase(camelValue); if (value != underscoreValue) { - if constexpr (failOnError) { + if constexpr (ThrowOnError) { throw TSimpleException(Format("Enum value %Qv is not in a proper underscore case; did you mean %Qv?", value, underscoreValue)); @@ -23,14 +32,16 @@ std::optional<TString> DecodeEnumValueImpl(TStringBuf value) return camelValue; } +} // namespace NDetail + std::optional<TString> TryDecodeEnumValue(TStringBuf value) { - return DecodeEnumValueImpl<false>(value); + return NDetail::DecodeEnumValueImpl<false>(value); } TString DecodeEnumValue(TStringBuf value) { - auto decodedValue = DecodeEnumValueImpl<true>(value); + auto decodedValue = NDetail::DecodeEnumValueImpl<true>(value); YT_VERIFY(decodedValue); return *decodedValue; } @@ -40,22 +51,6 @@ TString EncodeEnumValue(TStringBuf value) return CamelCaseToUnderscoreCase(value); } -namespace NDetail { - -void ThrowMalformedEnumValueException(TStringBuf typeName, TStringBuf value) -{ - throw TSimpleException(Format("Error parsing %v value %Qv", - typeName, - value)); -} - -void FormatUnknownEnumValue(TStringBuilderBase* builder, TStringBuf name, i64 value) -{ - builder->AppendFormat("%v(%v)", name, value); -} - -} // namespace NDetail - //////////////////////////////////////////////////////////////////////////////// } // namespace NYT diff --git a/library/cpp/yt/string/enum.h b/library/cpp/yt/string/enum.h index 792e09402d..a0be527583 100644 --- a/library/cpp/yt/string/enum.h +++ b/library/cpp/yt/string/enum.h @@ -15,10 +15,10 @@ TString DecodeEnumValue(TStringBuf value); TString EncodeEnumValue(TStringBuf value); template <class T> -std::optional<T> TryParseEnum(TStringBuf value); +std::optional<T> TryParseEnum(TStringBuf str, bool enableUnknown = false); template <class T> -T ParseEnum(TStringBuf value); +T ParseEnum(TStringBuf str); template <class T> void FormatEnum(TStringBuilderBase* builder, T value, bool lowerCase); diff --git a/library/cpp/yt/string/unittests/enum_ut.cpp b/library/cpp/yt/string/unittests/enum_ut.cpp index be5387151a..39dbdfd207 100644 --- a/library/cpp/yt/string/unittests/enum_ut.cpp +++ b/library/cpp/yt/string/unittests/enum_ut.cpp @@ -27,13 +27,39 @@ DEFINE_BIT_ENUM(ELangs, ((Rust) (0x04)) ((Python) (0x08)) ((JavaScript) (0x10)) + ((CppGo) (0x03)) + ((All) (0x1f)) ); +DEFINE_BIT_ENUM(ELangsWithUnknown, + ((None) (0x00)) + ((Cpp) (0x01)) + ((Go) (0x02)) + ((Rust) (0x04)) + ((Python) (0x08)) + ((JavaScript) (0x10)) + ((CppGo) (0x03)) + ((All) (0x1f)) + ((Unknown) (0x20)) +); +DEFINE_ENUM_UNKNOWN_VALUE(ELangsWithUnknown, Unknown); + DEFINE_ENUM(ECustomDomainName, ((A) (1) ("value_a")) ((B) (2) ("value_b")) ); +DEFINE_ENUM(EColorWithUnknown, + (Red) + (Green) + (Unknown) +); +DEFINE_ENUM_UNKNOWN_VALUE(EColorWithUnknown, Unknown); + +DEFINE_BIT_ENUM(EBitEnumWithoutNone, + ((Some)(1)) +); + TEST(TFormatTest, Enum) { EXPECT_EQ("Red", Format("%v", EColor::Red)); @@ -42,7 +68,8 @@ TEST(TFormatTest, Enum) EXPECT_EQ("BlackAndWhite", Format("%v", EColor::BlackAndWhite)); EXPECT_EQ("black_and_white", Format("%lv", EColor::BlackAndWhite)); - EXPECT_EQ("EColor(100)", Format("%v", EColor(100))); + EXPECT_EQ("EColor::unknown-100", Format("%v", EColor(100))); + EXPECT_EQ("Cpp | ELangs::unknown-32", Format("%v", ELangs(0x21))); EXPECT_EQ("JavaScript", Format("%v", ELangs::JavaScript)); EXPECT_EQ("java_script", Format("%lv", ELangs::JavaScript)); @@ -50,6 +77,8 @@ TEST(TFormatTest, Enum) EXPECT_EQ("None", Format("%v", ELangs::None)); EXPECT_EQ("none", Format("%lv", ELangs::None)); + EXPECT_EQ("", Format("%v", EBitEnumWithoutNone())); + EXPECT_EQ("Cpp | Go", Format("%v", ELangs::Cpp | ELangs::Go)); EXPECT_EQ("cpp | go", Format("%lv", ELangs::Cpp | ELangs::Go)); @@ -64,6 +93,19 @@ TEST(TFormatEnumTest, FormatEnumWithCustomDomainName) EXPECT_EQ("value_b", FormatEnum(ECustomDomainName::B)); } +TEST(TParseEnumTest, ParseUnknown) +{ + EXPECT_EQ(std::nullopt, TryParseEnum<EColor>("yellow")); + EXPECT_THROW(ParseEnum<EColor>("yellow"), TSimpleException); + EXPECT_EQ(std::nullopt, TryParseEnum<EColorWithUnknown>("yellow")); + EXPECT_EQ(EColorWithUnknown::Unknown, TryParseEnum<EColorWithUnknown>("yellow", /*enableUnknown*/ true)); + EXPECT_EQ(EColorWithUnknown::Unknown, ParseEnum<EColorWithUnknown>("yellow")); + EXPECT_EQ(std::nullopt, TryParseEnum<ELangs>("cpp | haskell")); + EXPECT_THROW(ParseEnum<ELangs>("cpp | haskell"), TSimpleException); + EXPECT_EQ(ELangsWithUnknown::Cpp | ELangsWithUnknown::Unknown, TryParseEnum<ELangsWithUnknown>("cpp | haskell", /*enableUnknown*/ true)); + EXPECT_EQ(ELangsWithUnknown::Cpp | ELangsWithUnknown::Unknown, ParseEnum<ELangsWithUnknown>("cpp | haskell")); +} + TEST(TParseEnumTest, ParseEnumWithCustomDomainName) { EXPECT_EQ(ECustomDomainName::A, TryParseEnum<ECustomDomainName>("value_a")); @@ -77,7 +119,7 @@ TEST(TParseEnumTest, ParseBitEnum) EXPECT_EQ(ELangs::Cpp, TryParseEnum<ELangs>("cpp")); EXPECT_EQ(ELangs::Cpp | ELangs::Rust, TryParseEnum<ELangs>("cpp|rust")); EXPECT_EQ(ELangs::Cpp | ELangs::Rust, TryParseEnum<ELangs>("cpp | rust")); - EXPECT_EQ(std::nullopt, TryParseEnum<ELangs>("unk | rust")); + EXPECT_EQ(std::nullopt, TryParseEnum<ELangs>("haskell | rust")); } //////////////////////////////////////////////////////////////////////////////// diff --git a/library/python/testing/style/rules.py b/library/python/testing/style/rules.py index 1f4d283769..0de9dcdbca 100644 --- a/library/python/testing/style/rules.py +++ b/library/python/testing/style/rules.py @@ -16,7 +16,7 @@ def _path_skip_reason(path, skip_links=True): if '/generated/' in path: return "path '{}' contains '/generated/'".format(path) - if path and '/contrib/' in path: + if path and '/contrib/' in path and '/.yandex_meta/' not in path: return "path '{}' contains '/contrib/'".format(path) if path and '/vendor/' in path: |