diff options
author | Andrey Chulkov <achulkov2@nebius.com> | 2024-10-15 14:10:17 +0300 |
---|---|---|
committer | robot-piglet <robot-piglet@yandex-team.com> | 2024-10-15 14:27:22 +0300 |
commit | 05cf1efb77e08298322326fe748b6bcdb7b6526e (patch) | |
tree | b55a13dc9277ebf98872ee1ddee93e8f24781265 | |
parent | 2ff5baf34b9dea61580028eb4c4afde7601232a7 (diff) | |
download | ydb-05cf1efb77e08298322326fe748b6bcdb7b6526e.tar.gz |
Increase max decimal precision to 76 (decimal256)
* Changelog entry
Type: feature
Component: type system
Support for decimals with precision from 35 to 76.
This PR adds decimal256 support to YT.
Additionally:
- Decimal256 is supported in skiff as int256.
- ClickHouse decimal precisions from 35 to 76 are supported in CHYT.
---
Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/798
Co-authored-by: ermolovd <ermolovd@yandex-team.com>
Co-authored-by: dakovalkov <dakovalkov@yandex-team.com>
commit_hash:73c4809966cf4c625e6007d31b5dde14bd80e829
-rw-r--r-- | library/cpp/skiff/public.h | 2 | ||||
-rw-r--r-- | library/cpp/skiff/skiff.cpp | 70 | ||||
-rw-r--r-- | library/cpp/skiff/skiff.h | 29 | ||||
-rw-r--r-- | library/cpp/skiff/skiff_schema-inl.h | 2 | ||||
-rw-r--r-- | library/cpp/skiff/skiff_validator.cpp | 2 | ||||
-rw-r--r-- | library/cpp/skiff/unittests/skiff_ut.cpp | 46 | ||||
-rw-r--r-- | yt/yt/client/table_client/logical_type.h | 2 | ||||
-rw-r--r-- | yt/yt/library/decimal/decimal.cpp | 515 | ||||
-rw-r--r-- | yt/yt/library/decimal/decimal.h | 25 | ||||
-rw-r--r-- | yt/yt/library/decimal/unittests/decimal_ut.cpp | 164 | ||||
-rw-r--r-- | yt/yt/library/formats/arrow_parser.cpp | 51 | ||||
-rw-r--r-- | yt/yt/library/formats/skiff_parser.cpp | 1 | ||||
-rw-r--r-- | yt/yt/library/formats/skiff_writer.cpp | 4 | ||||
-rw-r--r-- | yt/yt/library/formats/skiff_yson_converter-inl.h | 20 | ||||
-rw-r--r-- | yt/yt/library/formats/skiff_yson_converter.cpp | 8 |
15 files changed, 812 insertions, 129 deletions
diff --git a/library/cpp/skiff/public.h b/library/cpp/skiff/public.h index d67c6f26ee..aa2c0a5bb4 100644 --- a/library/cpp/skiff/public.h +++ b/library/cpp/skiff/public.h @@ -15,11 +15,13 @@ enum class EWireType Int32 /* "int32" */, Int64 /* "int64" */, Int128 /* "int128" */, + Int256 /* "int256" */, Uint8 /* "uint8" */, Uint16 /* "uint16" */, Uint32 /* "uint32" */, Uint64 /* "uint64" */, Uint128 /* "uint128" */, + Uint256 /* "uint256" */, Double /* "double" */, Boolean /* "boolean" */, String32 /* "string32" */, diff --git a/library/cpp/skiff/skiff.cpp b/library/cpp/skiff/skiff.cpp index cbdbdfe364..9b42628cc6 100644 --- a/library/cpp/skiff/skiff.cpp +++ b/library/cpp/skiff/skiff.cpp @@ -32,6 +32,18 @@ bool operator!=(TUint128 lhs, TUint128 rhs) //////////////////////////////////////////////////////////////////////////////// +bool operator==(const TInt256& lhs, const TInt256& rhs) +{ + return lhs.Parts == rhs.Parts; +} + +bool operator==(const TUint256& lhs, const TUint256& rhs) +{ + return lhs.Parts == rhs.Parts; +} + +//////////////////////////////////////////////////////////////////////////////// + TUncheckedSkiffParser::TUncheckedSkiffParser(IZeroCopyInput* underlying) : Underlying_(underlying) , Buffer_(512 * 1024) @@ -95,6 +107,26 @@ TUint128 TUncheckedSkiffParser::ParseUint128() return {low, high}; } +TInt256 TUncheckedSkiffParser::ParseInt256() +{ + TInt256 result; + for (auto& part : result.Parts) { + part = ParseSimple<ui64>(); + } + + return result; +} + +TUint256 TUncheckedSkiffParser::ParseUint256() +{ + TUint256 result; + for (auto& part : result.Parts) { + part = ParseSimple<ui64>(); + } + + return result; +} + double TUncheckedSkiffParser::ParseDouble() { return ParseSimple<double>(); @@ -274,6 +306,18 @@ TUint128 TCheckedSkiffParser::ParseUint128() return Parser_.ParseUint128(); } +TInt256 TCheckedSkiffParser::ParseInt256() +{ + Validator_->OnSimpleType(EWireType::Int256); + return Parser_.ParseInt256(); +} + +TUint256 TCheckedSkiffParser::ParseUint256() +{ + Validator_->OnSimpleType(EWireType::Uint256); + return Parser_.ParseUint256(); +} + double TCheckedSkiffParser::ParseDouble() { Validator_->OnSimpleType(EWireType::Double); @@ -389,6 +433,20 @@ void TUncheckedSkiffWriter::WriteUint128(TUint128 value) WriteSimple<ui64>(value.High); } +void TUncheckedSkiffWriter::WriteInt256(const TInt256& value) +{ + for (auto part : value.Parts) { + WriteSimple<ui64>(part); + } +} + +void TUncheckedSkiffWriter::WriteUint256(const TUint256& value) +{ + for (auto part : value.Parts) { + WriteSimple<ui64>(part); + } +} + void TUncheckedSkiffWriter::WriteUint8(ui8 value) { WriteSimple<ui8>(value); @@ -551,6 +609,18 @@ void TCheckedSkiffWriter::WriteUint128(TUint128 value) Writer_.WriteUint128(value); } +void TCheckedSkiffWriter::WriteInt256(TInt256 value) +{ + Validator_->OnSimpleType(EWireType::Int256); + Writer_.WriteInt256(std::move(value)); +} + +void TCheckedSkiffWriter::WriteUint256(TUint256 value) +{ + Validator_->OnSimpleType(EWireType::Uint256); + Writer_.WriteUint256(std::move(value)); +} + void TCheckedSkiffWriter::WriteString32(TStringBuf value) { Validator_->OnSimpleType(EWireType::String32); diff --git a/library/cpp/skiff/skiff.h b/library/cpp/skiff/skiff.h index 183c112700..c1315081d3 100644 --- a/library/cpp/skiff/skiff.h +++ b/library/cpp/skiff/skiff.h @@ -49,6 +49,23 @@ bool operator!=(TUint128 lhs, TUint128 rhs); //////////////////////////////////////////////////////////////////////////////// +struct TInt256 +{ + std::array<ui64, 4> Parts; +}; + +struct TUint256 +{ + std::array<ui64, 4> Parts; +}; + +// Operator != is synthesized since C++ 20. +bool operator==(const TInt256& lhs, const TInt256& rhs); + +bool operator==(const TUint256& lhs, const TUint256& rhs); + +//////////////////////////////////////////////////////////////////////////////// + class TUncheckedSkiffParser { public: @@ -68,6 +85,9 @@ public: TInt128 ParseInt128(); TUint128 ParseUint128(); + TInt256 ParseInt256(); + TUint256 ParseUint256(); + double ParseDouble(); bool ParseBoolean(); @@ -127,6 +147,9 @@ public: TInt128 ParseInt128(); TUint128 ParseUint128(); + TInt256 ParseInt256(); + TUint256 ParseUint256(); + double ParseDouble(); bool ParseBoolean(); @@ -177,6 +200,9 @@ public: void WriteInt128(TInt128 value); void WriteUint128(TUint128 value); + void WriteInt256(const TInt256& value); + void WriteUint256(const TUint256& value); + void WriteString32(TStringBuf value); void WriteYson32(TStringBuf value); @@ -223,6 +249,9 @@ public: void WriteInt128(TInt128 value); void WriteUint128(TUint128 value); + void WriteInt256(TInt256 value); + void WriteUint256(TUint256 value); + void WriteString32(TStringBuf value); void WriteYson32(TStringBuf value); diff --git a/library/cpp/skiff/skiff_schema-inl.h b/library/cpp/skiff/skiff_schema-inl.h index 853ff36738..a977194180 100644 --- a/library/cpp/skiff/skiff_schema-inl.h +++ b/library/cpp/skiff/skiff_schema-inl.h @@ -19,12 +19,14 @@ inline bool IsSimpleType(EWireType type) case EWireType::Int32: case EWireType::Int64: case EWireType::Int128: + case EWireType::Int256: case EWireType::Uint8: case EWireType::Uint16: case EWireType::Uint32: case EWireType::Uint64: case EWireType::Uint128: + case EWireType::Uint256: case EWireType::Double: case EWireType::Boolean: diff --git a/library/cpp/skiff/skiff_validator.cpp b/library/cpp/skiff/skiff_validator.cpp index 76dd3b7600..a2e9e1db90 100644 --- a/library/cpp/skiff/skiff_validator.cpp +++ b/library/cpp/skiff/skiff_validator.cpp @@ -363,12 +363,14 @@ std::shared_ptr<IValidatorNode> CreateUsageValidatorNode(const std::shared_ptr<T case EWireType::Int32: case EWireType::Int64: case EWireType::Int128: + case EWireType::Int256: case EWireType::Uint8: case EWireType::Uint16: case EWireType::Uint32: case EWireType::Uint64: case EWireType::Uint128: + case EWireType::Uint256: case EWireType::Double: case EWireType::Boolean: diff --git a/library/cpp/skiff/unittests/skiff_ut.cpp b/library/cpp/skiff/unittests/skiff_ut.cpp index 5e4c709611..cdb09e7d52 100644 --- a/library/cpp/skiff/unittests/skiff_ut.cpp +++ b/library/cpp/skiff/unittests/skiff_ut.cpp @@ -209,6 +209,29 @@ Y_UNIT_TEST_SUITE(Skiff) UNIT_ASSERT_EQUAL(parser.ParseInt128(), val2); } + Y_UNIT_TEST(TestInt256) + { + TBufferStream bufferStream; + + auto schema = CreateSimpleTypeSchema(EWireType::Int256); + + const TInt256 val1 = {0x1924cd4aeb9ced82, 0x0885e83f456d6a7e, 0xe9ba36585eccae1a, 0x7854b6f9ce448be9}; + const TInt256 val2 = {0xe9ba36585eccae1a, 0x1924cd4aeb9ced82, 0x0885e83f456d6a7e, static_cast<ui64>(-0x7854b6f9ce448be9)}; + + TCheckedSkiffWriter writer(schema, &bufferStream); + writer.WriteInt256(val1); + writer.WriteInt256(val2); + writer.Finish(); + + UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()), + "82ed9ceb4acd2419" "7e6a6d453fe88508" "1aaecc5e5836bae9" "e98b44cef9b65478" + "1aaecc5e5836bae9" "82ed9ceb4acd2419" "7e6a6d453fe88508" "1774bb310649ab87"); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_EQUAL(parser.ParseInt256(), val1); + UNIT_ASSERT_EQUAL(parser.ParseInt256(), val2); + } + Y_UNIT_TEST(TestUint128) { TBufferStream bufferStream; @@ -232,6 +255,29 @@ Y_UNIT_TEST_SUITE(Skiff) UNIT_ASSERT_EQUAL(parser.ParseUint128(), val2); } + Y_UNIT_TEST(TestUint256) + { + TBufferStream bufferStream; + + auto schema = CreateSimpleTypeSchema(EWireType::Uint256); + + const auto val1 = TUint256{0x1924cd4aeb9ced82, 0x7854b6f9ce448be9, 0x8854b6f9ce448be9, 0x0885e83f456d6a7e}; + const auto val2 = TUint256{0xe9ba36585eccae1a, 0x8854b6f9ce448be9, 0x1924cd4aeb9ced82, 0xabacabadabacaba0}; + + TCheckedSkiffWriter writer(schema, &bufferStream); + writer.WriteUint256(val1); + writer.WriteUint256(val2); + writer.Finish(); + + UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()), + "82ed9ceb4acd2419" "e98b44cef9b65478" "e98b44cef9b65488" "7e6a6d453fe88508" + "1aaecc5e5836bae9" "e98b44cef9b65488" "82ed9ceb4acd2419" "a0abacabadabacab"); + + TCheckedSkiffParser parser(schema, &bufferStream); + UNIT_ASSERT_EQUAL(parser.ParseUint256(), val1); + UNIT_ASSERT_EQUAL(parser.ParseUint256(), val2); + } + Y_UNIT_TEST(TestBoolean) { auto schema = CreateSimpleTypeSchema(EWireType::Boolean); diff --git a/yt/yt/client/table_client/logical_type.h b/yt/yt/client/table_client/logical_type.h index 904ff9708b..bd482df984 100644 --- a/yt/yt/client/table_client/logical_type.h +++ b/yt/yt/client/table_client/logical_type.h @@ -170,7 +170,7 @@ class TDecimalLogicalType { public: static constexpr int MinPrecision = 1; - static constexpr int MaxPrecision = 35; + static constexpr int MaxPrecision = 76; public: TDecimalLogicalType(int precision, int scale); diff --git a/yt/yt/library/decimal/decimal.cpp b/yt/yt/library/decimal/decimal.cpp index e1674f84e2..3df4b44028 100644 --- a/yt/yt/library/decimal/decimal.cpp +++ b/yt/yt/library/decimal/decimal.cpp @@ -12,24 +12,263 @@ namespace NYT::NDecimal { //////////////////////////////////////////////////////////////////////////////// +// We use the same type used for binary representation of 256 bit decimals for +// implementing the necessary arithmetic operations. +using i256 = TDecimal::TValue256; + +// We chose to only implement a small subset of operations and functions necessary +// for converting 256 bit decimals to and from text. +// Rationale: there do not seem to be any good implementations of 256-bit arithmetic +// that we are comfortable depending on (for now). There are some big number +// implementations around openssl, but these types are intended for arbitrarily +// large integers and use dynamic allocations. It also does not make sense to commit +// to implementing full-fledged 256-bit arithmetics when we actually use a minority +// of operations. +// When we have a good enough int256 either in library/util or in contrib, we can +// switch to it easily. + +//////////////////////////////////////////////////////////////////////////////// + +constexpr i256 operator-(i256 value) noexcept +{ + // Invert. + for (int partIndex = 0; partIndex < std::ssize(value.Parts); ++partIndex) { + value.Parts[partIndex] = ~value.Parts[partIndex]; + } + + // Add 1. + for (int partIndex = 0; partIndex < std::ssize(value.Parts) && ++value.Parts[partIndex] == 0; ++partIndex) { } + + return value; +} + +constexpr std::strong_ordering operator<=>(const i256& lhs, const i256& rhs) +{ + bool lhsIsNegative = lhs.Parts.back() & (1u << 31); + bool rhsIsNegative = rhs.Parts.back() & (1u << 31); + + if (lhsIsNegative && !rhsIsNegative) { + return std::strong_ordering::less; + } + + if (!lhsIsNegative && rhsIsNegative) { + return std::strong_ordering::greater; + } + + for (int partIndex = std::ssize(lhs.Parts) - 1; partIndex >= 0; --partIndex) { + if (lhs.Parts[partIndex] != rhs.Parts[partIndex]) { + return lhs.Parts[partIndex] <=> rhs.Parts[partIndex]; + } + } + + return std::strong_ordering::equal; +} + +// Not synthesized by default :( +constexpr bool operator==(const i256& lhs, const i256& rhs) +{ + return lhs.Parts == rhs.Parts; +} + +//////////////////////////////////////////////////////////////////////////////// + +// Some operations require working with an explicitly unsigned type, since they +// might cause integer overflow, which is not very acceptable for signed types. +// It is also easier to implement some arithmetic operations (like *= 10) by +// shifting bits. +struct TUnsignedValue256 +{ + std::array<ui32, 8> Parts; +}; + +using ui256 = TUnsignedValue256; + +static_assert(sizeof(ui256) == sizeof(i256)); + +//////////////////////////////////////////////////////////////////////////////// + +constexpr bool operator==(const ui256& lhs, const ui256& rhs) +{ + return lhs.Parts == rhs.Parts; +} + +constexpr ui256 operator+(ui256 lhs, const ui256& rhs) +{ + ui64 carry = 0; + for (int partIndex = 0; partIndex < std::ssize(lhs.Parts); ++partIndex) { + carry += lhs.Parts[partIndex]; + carry += rhs.Parts[partIndex]; + lhs.Parts[partIndex] = carry; + carry >>= 32; + } + + return lhs; +} + +template <int Shift> +Y_FORCE_INLINE constexpr ui256 ShiftUp(ui256 value) +{ + static_assert(Shift >= 0 && Shift <= 32); + + value.Parts.back() <<= Shift; + for (int partIndex = std::ssize(value.Parts) - 2; partIndex >= 0; --partIndex) { + value.Parts[partIndex + 1] |= value.Parts[partIndex] >> (32 - Shift); + value.Parts[partIndex] <<= Shift; + } + + return value; +} + +//////////////////////////////////////////////////////////////////////////////// + template <typename T> constexpr bool ValidDecimalUnderlyingInteger = std::is_same_v<T, i32> || std::is_same_v<T, i64> || - std::is_same_v<T, i128>; + std::is_same_v<T, i128> || + std::is_same_v<T, i256>; + +template <typename T> +constexpr bool ValidDecimalUnderlyingUnsignedInteger = + std::is_same_v<T, ui32> || + std::is_same_v<T, ui64> || + std::is_same_v<T, ui128> || + std::is_same_v<T, ui256>; + +template <typename T> +Y_FORCE_INLINE constexpr T GetNan() +{ + if constexpr (std::is_same_v<T, i256>) { + constexpr i32 i32Max = std::numeric_limits<i32>::max(); + constexpr ui32 ui32Max = std::numeric_limits<ui32>::max(); + + return {ui32Max, ui32Max, ui32Max, ui32Max, ui32Max, ui32Max, ui32Max, i32Max}; + } else { + return std::numeric_limits<T>::max(); + } +} + +template <typename T> +Y_FORCE_INLINE constexpr T GetPlusInf() +{ + if constexpr (std::is_same_v<T, i256>) { + constexpr i32 i32Max = std::numeric_limits<i32>::max(); + constexpr ui32 ui32Max = std::numeric_limits<ui32>::max(); + + return {ui32Max - 1, ui32Max, ui32Max, ui32Max, ui32Max, ui32Max, ui32Max, i32Max}; + } else { + return std::numeric_limits<T>::max() - 1; + } +} template <typename T> struct TDecimalTraits { static_assert(ValidDecimalUnderlyingInteger<T>); - static constexpr T Nan = std::numeric_limits<T>::max(); - static constexpr T PlusInf = std::numeric_limits<T>::max() - 1; + static constexpr T Nan = GetNan<T>(); + static constexpr T PlusInf = GetPlusInf<T>(); static constexpr T MinusInf = -PlusInf; - - static constexpr T MinSpecialValue = PlusInf; }; +template <typename T> +Y_FORCE_INLINE constexpr bool IsNegativeInteger(T value) +{ + static_assert(ValidDecimalUnderlyingInteger<T>); + + if constexpr (std::is_same_v<T, i256>) { + return value.Parts.back() & (1u << 31); + } else { + return value < 0; + } +} + +template <typename T> +Y_FORCE_INLINE constexpr auto DecimalIntegerToUnsigned(T value) +{ + static_assert(ValidDecimalUnderlyingInteger<T>); + + if constexpr (std::is_same_v<T, i256>) { + return ui256{value.Parts}; + } else if constexpr (std::is_same_v<T, i128>) { + return ui128(value); + } else { + using TU = std::make_unsigned_t<T>; + return static_cast<TU>(value); + } +} + +template <typename T> +Y_FORCE_INLINE constexpr auto DecimalIntegerToSigned(T value) +{ + static_assert(ValidDecimalUnderlyingUnsignedInteger<T>); + + if constexpr (std::is_same_v<T, ui256>) { + return i256{value.Parts}; + } else if constexpr (std::is_same_v<T, ui128>) { + return i128(value); + } else { + using TS = std::make_signed_t<T>; + return static_cast<TS>(value); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +//! The functions below are used for implementing conversion to/from text to binary decimal. +//! We actually do not need any arithmetic operations beside v *= 10, v /= 10, addition and negation. +//! so they are the only ones actually implemented for our custom i256/ui256. + +template <typename T> +Y_FORCE_INLINE constexpr auto FlipMSB(T value) +{ + static_assert(ValidDecimalUnderlyingInteger<T>); + + if constexpr (std::is_same_v<T, i256>) { + value.Parts.back() ^= (1u << 31); + return value; + } else { + constexpr auto One = DecimalIntegerToUnsigned(T{1}); + // Bit operations are only valid with unsigned types. + return T(DecimalIntegerToUnsigned(value) ^ (One << (sizeof(T) * 8 - 1))); + } +} + +template <typename T> +Y_FORCE_INLINE constexpr ui32 GetNextDigit(T value, T* nextValue) +{ + static_assert(ValidDecimalUnderlyingUnsignedInteger<T>); + + if constexpr (std::is_same_v<T, ui256>) { + ui64 remainder = 0; + for (int partIndex = std::ssize(value.Parts) - 1; partIndex >= 0; --partIndex) { + // Everything should fit into long long since we are dividing by 10. + auto step = std::lldiv(value.Parts[partIndex] + (remainder << 32), 10); + value.Parts[partIndex] = step.quot; + remainder = step.rem; + } + *nextValue = value; + return remainder; + } else { + constexpr auto Ten = T{10}; + *nextValue = value / Ten; + return static_cast<ui32>(value % Ten); + } +} + +template <typename T> +Y_FORCE_INLINE constexpr T MultiplyByTen(T value) +{ + static_assert(ValidDecimalUnderlyingUnsignedInteger<T>); + + if constexpr (std::is_same_v<T, ui256>) { + // 2 * (4 * v + v) = 10v. + return ShiftUp<1>(ShiftUp<2>(value) + value); + } else { + return value * DecimalIntegerToUnsigned(10); + } +} + //////////////////////////////////////////////////////////////////////////////// constexpr int GetDecimalBinaryValueSize(int precision) @@ -39,34 +278,41 @@ constexpr int GetDecimalBinaryValueSize(int precision) return 4; } else if (precision <= 18) { return 8; - } else if (precision <= 35) { + } else if (precision <= 38) { return 16; + } else if (precision <= 76) { + return 32; } } return 0; } -static constexpr i128 DecimalIntegerMaxValueTable[] = { - i128{0}, // 0 - i128{9}, // 1 - i128{99}, // 2 - i128{999}, // 3 - i128{9999}, // 4 - i128{99999}, // 5 - i128{999999}, // 6 - i128{9999999}, // 7 - i128{99999999}, // 8 - i128{999999999}, // 9 - i128{9999999999ul}, // 10 - i128{99999999999ul}, // 11 - i128{999999999999ul}, // 12 - i128{9999999999999ul}, // 13 - i128{99999999999999ul}, // 14 - i128{999999999999999ul}, // 15 - i128{9999999999999999ul}, // 16 - i128{99999999999999999ul}, // 17 - i128{999999999999999999ul}, // 18 +static constexpr i32 Decimal32IntegerMaxValueTable[] = { + 0, // 0 + 9, // 1 + 99, // 2 + 999, // 3 + 9999, // 4 + 99999, // 5 + 999999, // 6 + 9999999, // 7 + 99999999, // 8 + 999999999, // 9 +}; +static constexpr i64 Decimal64IntegerMaxValueTable[] = { + 9999999999ul, // 10 + 99999999999ul, // 11 + 999999999999ul, // 12 + 9999999999999ul, // 13 + 99999999999999ul, // 14 + 999999999999999ul, // 15 + 9999999999999999ul, // 16 + 99999999999999999ul, // 17 + 999999999999999999ul, // 18 +}; + +static constexpr i128 Decimal128IntegerMaxValueTable[] = { // 128 bits // // Generated by fair Python script: @@ -79,7 +325,7 @@ static constexpr i128 DecimalIntegerMaxValueTable[] = { // hex_value[-16:], // hex_value[:-16] or "0", // precision)) - // for i in range(19, 36): + // for i in range(19, 39): // print_max_decimal(i) // i128{static_cast<ui64>(0x8ac7230489e7fffful)} | (i128{static_cast<ui64>(0x0ul)} << 64), // 19 @@ -99,37 +345,106 @@ static constexpr i128 DecimalIntegerMaxValueTable[] = { i128{static_cast<ui64>(0x38c15b09fffffffful)} | (i128{static_cast<ui64>(0x314dc6448d93ul)} << 64), // 33 i128{static_cast<ui64>(0x378d8e63fffffffful)} | (i128{static_cast<ui64>(0x1ed09bead87c0ul)} << 64), // 34 i128{static_cast<ui64>(0x2b878fe7fffffffful)} | (i128{static_cast<ui64>(0x13426172c74d82ul)} << 64), // 35 + i128{static_cast<ui64>(0xb34b9f0ffffffffful)} | (i128{static_cast<ui64>(0xc097ce7bc90715ul)} << 64), // 36 + i128{static_cast<ui64>(0x00f4369ffffffffful)} | (i128{static_cast<ui64>(0x785ee10d5da46d9ul)} << 64), // 37 + i128{static_cast<ui64>(0x098a223ffffffffful)} | (i128{static_cast<ui64>(0x4b3b4ca85a86c47aul)} << 64), // 38 +}; + +static constexpr i256 Decimal256IntegerMaxValueTable[] = { + // 256 bits + // + // Generated by fair Python script: + // + // def print_max_decimal(precision): + // max_value = int("9" * precision) + // hex_value = hex(max_value)[2:] # strip 0x + // hex_value = hex_value.strip("L") + // parts = [hex_value[-8 * i:-8 * (i - 1) if i > 1 else len(hex_value)] or "0" for i in range(1, 9)] + // assert sum(int(v, 16) * 2 ** (32 * i) for i, v in enumerate(parts)) == max_value + // assert int(parts[-1], 16) < 2 ** 31 - 1 + // joined_parts = ", ".join(f"static_cast<ui32>(0x{part}u)" for part in parts) + // print(f"{{{joined_parts}}}, // {precision}") + // + // for i in range(39, 77): + // print_max_decimal(i) + // + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x5f65567fu), static_cast<ui32>(0x8943acc4u), static_cast<ui32>(0xf050fe93u), static_cast<ui32>(0x2u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 39 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xb9f560ffu), static_cast<ui32>(0x5ca4bfabu), static_cast<ui32>(0x6329f1c3u), static_cast<ui32>(0x1du), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 40 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x4395c9ffu), static_cast<ui32>(0x9e6f7cb5u), static_cast<ui32>(0xdfa371a1u), static_cast<ui32>(0x125u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 41 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xa3d9e3ffu), static_cast<ui32>(0x305adf14u), static_cast<ui32>(0xbc627050u), static_cast<ui32>(0xb7au), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 42 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x6682e7ffu), static_cast<ui32>(0xe38cb6ceu), static_cast<ui32>(0x5bd86321u), static_cast<ui32>(0x72cbu), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 43 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x011d0fffu), static_cast<ui32>(0xe37f2410u), static_cast<ui32>(0x9673df52u), static_cast<ui32>(0x47bf1u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 44 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x0b229fffu), static_cast<ui32>(0xe2f768a0u), static_cast<ui32>(0xe086b93cu), static_cast<ui32>(0x2cd76fu), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 45 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x6f5a3fffu), static_cast<ui32>(0xddaa1640u), static_cast<ui32>(0xc5433c60u), static_cast<ui32>(0x1c06a5eu), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 46 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x59867fffu), static_cast<ui32>(0xa8a4de84u), static_cast<ui32>(0xb4a05bc8u), static_cast<ui32>(0x118427b3u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 47 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x7f40ffffu), static_cast<ui32>(0x9670b12bu), static_cast<ui32>(0x0e4395d6u), static_cast<ui32>(0xaf298d05u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 48 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xf889ffffu), static_cast<ui32>(0xe066ebb2u), static_cast<ui32>(0x8ea3da61u), static_cast<ui32>(0xd79f8232u), static_cast<ui32>(0x6u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 49 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xb563ffffu), static_cast<ui32>(0xc40534fdu), static_cast<ui32>(0x926687d2u), static_cast<ui32>(0x6c3b15f9u), static_cast<ui32>(0x44u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 50 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x15e7ffffu), static_cast<ui32>(0xa83411e9u), static_cast<ui32>(0xb8014e3bu), static_cast<ui32>(0x3a4edbbfu), static_cast<ui32>(0x2acu), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 51 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xdb0fffffu), static_cast<ui32>(0x9208b31au), static_cast<ui32>(0x300d0e54u), static_cast<ui32>(0x4714957du), static_cast<ui32>(0x1abau), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 52 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x8e9fffffu), static_cast<ui32>(0xb456ff0cu), static_cast<ui32>(0xe0828f4du), static_cast<ui32>(0xc6cdd6e3u), static_cast<ui32>(0x10b46u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 53 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x923fffffu), static_cast<ui32>(0x0b65f67du), static_cast<ui32>(0xc5199909u), static_cast<ui32>(0xc40a64e6u), static_cast<ui32>(0xa70c3u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 54 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xb67fffffu), static_cast<ui32>(0x71fba0e7u), static_cast<ui32>(0xb2fffa5au), static_cast<ui32>(0xa867f103u), static_cast<ui32>(0x6867a5u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 55 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x20ffffffu), static_cast<ui32>(0x73d4490du), static_cast<ui32>(0xfdffc788u), static_cast<ui32>(0x940f6a24u), static_cast<ui32>(0x4140c78u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 56 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x49ffffffu), static_cast<ui32>(0x864ada83u), static_cast<ui32>(0xebfdcb54u), static_cast<ui32>(0xc89a2571u), static_cast<ui32>(0x28c87cb5u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 57 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xe3ffffffu), static_cast<ui32>(0x3eec8920u), static_cast<ui32>(0x37e9f14du), static_cast<ui32>(0xd6057673u), static_cast<ui32>(0x97d4df19u), static_cast<ui32>(0x1u), static_cast<ui32>(0x0u)}, // 58 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xe7ffffffu), static_cast<ui32>(0x753d5b48u), static_cast<ui32>(0x2f236d04u), static_cast<ui32>(0x5c36a080u), static_cast<ui32>(0xee50b702u), static_cast<ui32>(0xfu), static_cast<ui32>(0x0u)}, // 59 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x0fffffffu), static_cast<ui32>(0x946590d9u), static_cast<ui32>(0xd762422cu), static_cast<ui32>(0x9a224501u), static_cast<ui32>(0x4f272617u), static_cast<ui32>(0x9fu), static_cast<ui32>(0x0u)}, // 60 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x9fffffffu), static_cast<ui32>(0xcbf7a87au), static_cast<ui32>(0x69d695bdu), static_cast<ui32>(0x0556b212u), static_cast<ui32>(0x17877cecu), static_cast<ui32>(0x639u), static_cast<ui32>(0x0u)}, // 61 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x3fffffffu), static_cast<ui32>(0xf7ac94cau), static_cast<ui32>(0x2261d969u), static_cast<ui32>(0x3562f4b8u), static_cast<ui32>(0xeb4ae138u), static_cast<ui32>(0x3e3au), static_cast<ui32>(0x0u)}, // 62 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x7fffffffu), static_cast<ui32>(0xacbdcfe6u), static_cast<ui32>(0x57d27e23u), static_cast<ui32>(0x15dd8f31u), static_cast<ui32>(0x30eccc32u), static_cast<ui32>(0x26e4du), static_cast<ui32>(0x0u)}, // 63 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xbf6a1f00u), static_cast<ui32>(0x6e38ed64u), static_cast<ui32>(0xdaa797edu), static_cast<ui32>(0xe93ff9f4u), static_cast<ui32>(0x184f03u), static_cast<ui32>(0x0u)}, // 64 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x7a253609u), static_cast<ui32>(0x4e3945efu), static_cast<ui32>(0x8a8bef46u), static_cast<ui32>(0x1c7fc390u), static_cast<ui32>(0xf31627u), static_cast<ui32>(0x0u)}, // 65 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xc5741c63u), static_cast<ui32>(0x0e3cbb5au), static_cast<ui32>(0x697758bfu), static_cast<ui32>(0x1cfda3a5u), static_cast<ui32>(0x97edd87u), static_cast<ui32>(0x0u)}, // 66 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xb6891be7u), static_cast<ui32>(0x8e5f518bu), static_cast<ui32>(0x1ea97776u), static_cast<ui32>(0x21e86476u), static_cast<ui32>(0x5ef4a747u), static_cast<ui32>(0x0u)}, // 67 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x215b170fu), static_cast<ui32>(0x8fb92f75u), static_cast<ui32>(0x329eaaa1u), static_cast<ui32>(0x5313ec9du), static_cast<ui32>(0xb58e88c7u), static_cast<ui32>(0x3u)}, // 68 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x4d8ee69fu), static_cast<ui32>(0x9d3bda93u), static_cast<ui32>(0xfa32aa4fu), static_cast<ui32>(0x3ec73e23u), static_cast<ui32>(0x179157c9u), static_cast<ui32>(0x25u)}, // 69 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x0795023fu), static_cast<ui32>(0x245689c1u), static_cast<ui32>(0xc5faa71cu), static_cast<ui32>(0x73c86d67u), static_cast<ui32>(0xebad6ddcu), static_cast<ui32>(0x172u)}, // 70 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x4bd2167fu), static_cast<ui32>(0x6b61618au), static_cast<ui32>(0xbbca8719u), static_cast<ui32>(0x85d4460du), static_cast<ui32>(0x34c64a9cu), static_cast<ui32>(0xe7du)}, // 71 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xf634e0ffu), static_cast<ui32>(0x31cdcf66u), static_cast<ui32>(0x55e946feu), static_cast<ui32>(0x3a4abc89u), static_cast<ui32>(0x0fbeea1du), static_cast<ui32>(0x90e4u)}, // 72 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x9e10c9ffu), static_cast<ui32>(0xf20a1a05u), static_cast<ui32>(0x5b1cc5edu), static_cast<ui32>(0x46eb5d5du), static_cast<ui32>(0x9d752524u), static_cast<ui32>(0x5a8e8u)}, // 73 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x2ca7e3ffu), static_cast<ui32>(0x74650438u), static_cast<ui32>(0x8f1fbb4bu), static_cast<ui32>(0xc531a5a5u), static_cast<ui32>(0x2693736au), static_cast<ui32>(0x389916u)}, // 74 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xbe8ee7ffu), static_cast<ui32>(0x8bf22a31u), static_cast<ui32>(0x973d50f2u), static_cast<ui32>(0xb3f07877u), static_cast<ui32>(0x81c2822bu), static_cast<ui32>(0x235faddu)}, // 75 + {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x71950fffu), static_cast<ui32>(0x7775a5f1u), static_cast<ui32>(0xe8652979u), static_cast<ui32>(0x0764b4abu), static_cast<ui32>(0x119915b5u), static_cast<ui32>(0x161bcca7u)}, // 76 }; template<typename T> -Y_FORCE_INLINE constexpr T GetDecimalMaxIntegerValue(int precision) +Y_FORCE_INLINE constexpr auto GetDecimalMaxIntegerValue(int precision) { static_assert(ValidDecimalUnderlyingInteger<T>); - if (TDecimal::GetValueBinarySize(precision) <= sizeof(T)) { - return DecimalIntegerMaxValueTable[precision]; - } else { + if (TDecimal::GetValueBinarySize(precision) > static_cast<int>(sizeof(T))) { YT_ABORT(); } -} -template <typename T> -Y_FORCE_INLINE constexpr auto DecimalIntegerToUnsigned(T value) -{ - static_assert(ValidDecimalUnderlyingInteger<T>); - - if constexpr (std::is_same_v<T, i128>) { - return ui128(value); + if constexpr (std::is_same_v<T, i32>) { + YT_VERIFY(precision <= 9); + return Decimal32IntegerMaxValueTable[precision]; + } else if constexpr (std::is_same_v<T, i64>) { + YT_VERIFY(precision >= 10 && precision <= 18); + return Decimal64IntegerMaxValueTable[precision - 10]; + } else if constexpr (std::is_same_v<T, i128>) { + YT_VERIFY(precision >= 19 && precision <= 38); + return Decimal128IntegerMaxValueTable[precision - 19]; + } else if constexpr (std::is_same_v<T, i256>) { + YT_VERIFY(precision >= 39 && precision <= 76); + return Decimal256IntegerMaxValueTable[precision - 39]; } else { - using TU = std::make_unsigned_t<T>; - return static_cast<TU>(value); + YT_ABORT(); } } template <typename T> static Y_FORCE_INLINE T DecimalHostToInet(T value) { - if constexpr (std::is_same_v<T, i128> || std::is_same_v<T, ui128>) { + if constexpr (std::is_same_v<T, i256> || std::is_same_v<T, ui256>) { + for (int partIndex = 0; partIndex < std::ssize(value.Parts) / 2; ++partIndex) { + value.Parts[partIndex] = ::HostToInet(value.Parts[partIndex]); + value.Parts[std::size(value.Parts) - 1 - partIndex] = ::HostToInet(value.Parts[std::size(value.Parts) - 1 - partIndex]); + std::swap(value.Parts[partIndex], value.Parts[std::size(value.Parts) - 1 - partIndex]); + } + return value; + } else if constexpr (std::is_same_v<T, i128> || std::is_same_v<T, ui128>) { return T(::HostToInet(GetLow(value)), ::HostToInet(GetHigh(value))); } else { return ::HostToInet(value); @@ -139,7 +454,14 @@ static Y_FORCE_INLINE T DecimalHostToInet(T value) template <typename T> static Y_FORCE_INLINE T DecimalInetToHost(T value) { - if constexpr (std::is_same_v<T, i128> || std::is_same_v<T, ui128>) { + if constexpr (std::is_same_v<T, i256> || std::is_same_v<T, ui256>) { + for (int partIndex = 0; partIndex < std::ssize(value.Parts) / 2; ++partIndex) { + value.Parts[partIndex] = ::InetToHost(value.Parts[partIndex]); + value.Parts[std::size(value.Parts) - 1 - partIndex] = ::InetToHost(value.Parts[std::size(value.Parts) - 1 - partIndex]); + std::swap(value.Parts[partIndex], value.Parts[std::size(value.Parts) - 1 - partIndex]); + } + return value; + } else if constexpr (std::is_same_v<T, i128> || std::is_same_v<T, ui128>) { return T(::InetToHost(GetLow(value)), ::InetToHost(GetHigh(value))); } else { return ::InetToHost(value); @@ -151,22 +473,14 @@ static T DecimalBinaryToIntegerUnchecked(TStringBuf binaryValue) { T result; memcpy(&result, binaryValue.data(), sizeof(result)); - result = DecimalInetToHost(result); - - constexpr auto one = DecimalIntegerToUnsigned(T{1}); - result = static_cast<T>(DecimalIntegerToUnsigned(result) ^ (one << (sizeof(T) * 8 - 1))); - - return result; + return FlipMSB(DecimalInetToHost(result)); } template<typename T> static void DecimalIntegerToBinaryUnchecked(T decodedValue, void* buf) { - auto unsignedValue = DecimalIntegerToUnsigned(decodedValue); - constexpr auto one = DecimalIntegerToUnsigned(T{1}); - unsignedValue ^= (one << (sizeof(T) * 8 - 1)); - unsignedValue = DecimalHostToInet(unsignedValue); - memcpy(buf, &unsignedValue, sizeof(unsignedValue)); + auto preparedValue = DecimalHostToInet(FlipMSB(decodedValue)); + memcpy(buf, &preparedValue, sizeof(preparedValue)); } static void CheckDecimalValueSize(TStringBuf value, int precision, int scale) @@ -192,12 +506,6 @@ static Y_FORCE_INLINE TStringBuf PlaceOnBuffer(TStringBuf value, char* buffer) template<typename T> static TStringBuf WriteTextDecimalUnchecked(T decodedValue, int scale, char* buffer) { - i8 digits[std::numeric_limits<T>::digits + 1] = {0,}; - static constexpr auto ten = DecimalIntegerToUnsigned(T{10}); - - const bool negative = decodedValue < 0; - auto absValue = DecimalIntegerToUnsigned(negative ? -decodedValue : decodedValue); - if (decodedValue == TDecimalTraits<T>::MinusInf) { static constexpr TStringBuf minusInf = "-inf"; return PlaceOnBuffer(minusInf, buffer); @@ -209,10 +517,14 @@ static TStringBuf WriteTextDecimalUnchecked(T decodedValue, int scale, char* buf return PlaceOnBuffer(nan, buffer); } + i8 digits[TDecimal::MaxTextSize] = {0,}; + + bool negative = IsNegativeInteger(decodedValue); + auto absValue = DecimalIntegerToUnsigned(negative ? -decodedValue : decodedValue); + auto* curDigit = digits; - while (absValue > 0) { - *curDigit = static_cast<int>(absValue % ten); - absValue = absValue / ten; + while (absValue != DecimalIntegerToUnsigned(T{0})) { + *curDigit = GetNextDigit(absValue, &absValue); curDigit++; } YT_VERIFY(curDigit <= digits + std::size(digits)); @@ -313,35 +625,40 @@ T DecimalTextToInteger(TStringBuf textValue, int precision, int scale) break; } - T result = 0; + // This value can overflow in the process of parsing the text value. + // We do throw an exception later, but UB is not good for your health. + auto result = DecimalIntegerToUnsigned(T{0}); int beforePoint = 0; int afterPoint = 0; + + auto addDigit = [&] (auto digit) { + // We use this type to avoid warnings about casting signed types to unsigned/narrowing ints. + // Ugly, but this way we don't need to define cumbersome constructors for TValue256. + ui16 currentDigit = *digit - '0'; + if (currentDigit < 0 || currentDigit > 9) { + ThrowInvalidDecimal(textValue, precision, scale); + } + + result = MultiplyByTen(result); + result = result + DecimalIntegerToUnsigned(T{currentDigit}); + }; + for (; cur != end; ++cur) { if (*cur == '.') { ++cur; for (; cur != end; ++cur) { - int currentDigit = *cur - '0'; - result *= 10; - result += currentDigit; + addDigit(cur); ++afterPoint; - if (currentDigit < 0 || currentDigit > 9) { - ThrowInvalidDecimal(textValue, precision, scale); - } } break; } - int currentDigit = *cur - '0'; - result *= 10; - result += currentDigit; + addDigit(cur); ++beforePoint; - if (currentDigit < 0 || currentDigit > 9) { - ThrowInvalidDecimal(textValue, precision, scale); - } } for (; afterPoint < scale; ++afterPoint) { - result *= 10; + result = MultiplyByTen(result); } if (afterPoint > scale) { @@ -352,7 +669,10 @@ T DecimalTextToInteger(TStringBuf textValue, int precision, int scale) ThrowInvalidDecimal(textValue, precision, scale, "too many digits before decimal point"); } - return negative ? -result : result; + // This cast is guaranteed by the checks above to be correct. + auto signedResult = DecimalIntegerToSigned(result); + // This is safe: the range of representable values fits into [-signed_max, signed_max] for each underlying type. + return negative ? -signedResult : signedResult; } template<typename T> @@ -374,6 +694,8 @@ TStringBuf TDecimal::BinaryToText(TStringBuf binaryDecimal, int precision, int s return DecimalBinaryToTextUncheckedImpl<i64>(binaryDecimal, scale, buffer); case 16: return DecimalBinaryToTextUncheckedImpl<i128>(binaryDecimal, scale, buffer); + case 32: + return DecimalBinaryToTextUncheckedImpl<i256>(binaryDecimal, scale, buffer); } CheckDecimalValueSize(binaryDecimal, precision, scale); YT_ABORT(); @@ -410,8 +732,10 @@ TStringBuf TDecimal::TextToBinary(TStringBuf textValue, int precision, int scale return TextToBinaryImpl<i64>(textValue, precision, scale, buffer); case 16: return TextToBinaryImpl<i128>(textValue, precision, scale, buffer); + case 32: + return TextToBinaryImpl<i256>(textValue, precision, scale, buffer); default: - static_assert(GetDecimalBinaryValueSize(TDecimal::MaxPrecision) == 16); + static_assert(GetDecimalBinaryValueSize(TDecimal::MaxPrecision) == 32); YT_ABORT(); } } @@ -444,7 +768,7 @@ static void ValidateDecimalBinaryValueImpl(TStringBuf binaryDecimal, int precisi { T decoded = DecimalBinaryToIntegerUnchecked<T>(binaryDecimal); - const T maxValue = static_cast<T>(DecimalIntegerMaxValueTable[precision]); + auto maxValue = GetDecimalMaxIntegerValue<T>(precision); if (-maxValue <= decoded && decoded <= maxValue) { return; @@ -478,11 +802,14 @@ void TDecimal::ValidateBinaryValue(TStringBuf binaryDecimal, int precision, int return ValidateDecimalBinaryValueImpl<i64>(binaryDecimal, precision, scale); case 16: return ValidateDecimalBinaryValueImpl<i128>(binaryDecimal, precision, scale); + case 32: + return ValidateDecimalBinaryValueImpl<i256>(binaryDecimal, precision, scale); default: - static_assert(GetDecimalBinaryValueSize(TDecimal::MaxPrecision) == 16); + static_assert(GetDecimalBinaryValueSize(TDecimal::MaxPrecision) == 32); YT_ABORT(); } } + template <typename T> Y_FORCE_INLINE void CheckDecimalIntBits(int precision) { @@ -495,6 +822,14 @@ Y_FORCE_INLINE void CheckDecimalIntBits(int precision) } } +Y_FORCE_INLINE void CheckDecimalFitsInto128Bits(int precision) +{ + if (precision > 38) { + THROW_ERROR_EXCEPTION("Decimal<%v, ?> does not fit into int128", + precision); + } +} + int TDecimal::GetValueBinarySize(int precision) { const auto result = GetDecimalBinaryValueSize(precision); @@ -535,7 +870,7 @@ TStringBuf TDecimal::WriteBinary128(int precision, TValue128 value, char* buffer return TStringBuf{buffer, sizeof(TValue128)}; } -TStringBuf TDecimal::WriteBinaryVariadic(int precision, TValue128 value, char* buffer, size_t bufferLength) +TStringBuf TDecimal::WriteBinary128Variadic(int precision, TValue128 value, char* buffer, size_t bufferLength) { const size_t resultLength = GetValueBinarySize(precision); switch (resultLength) { @@ -550,6 +885,16 @@ TStringBuf TDecimal::WriteBinaryVariadic(int precision, TValue128 value, char* b } } +TStringBuf TDecimal::WriteBinary256(int precision, TValue256 value, char* buffer, size_t bufferLength) +{ + const size_t resultLength = GetValueBinarySize(precision); + CheckDecimalIntBits<TValue256>(precision); + YT_VERIFY(bufferLength >= resultLength); + + DecimalIntegerToBinaryUnchecked(std::move(value), buffer); + return TStringBuf{buffer, sizeof(TValue256)}; +} + template <typename T> Y_FORCE_INLINE void CheckBufferLength(int precision, size_t bufferLength) { @@ -581,6 +926,12 @@ TDecimal::TValue128 TDecimal::ParseBinary128(int precision, TStringBuf buffer) return {GetLow(result), static_cast<i64>(GetHigh(result))}; } +TDecimal::TValue256 TDecimal::ParseBinary256(int precision, TStringBuf buffer) +{ + CheckBufferLength<i256>(precision, buffer.Size()); + return DecimalBinaryToIntegerUnchecked<i256>(buffer); +} + //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NDecimal diff --git a/yt/yt/library/decimal/decimal.h b/yt/yt/library/decimal/decimal.h index 8a6cf34a8f..27375d3904 100644 --- a/yt/yt/library/decimal/decimal.h +++ b/yt/yt/library/decimal/decimal.h @@ -5,6 +5,8 @@ #include <util/system/defaults.h> #include <util/generic/string.h> +#include <array> + namespace NYT::NDecimal { //////////////////////////////////////////////////////////////////////////////// @@ -12,6 +14,9 @@ namespace NYT::NDecimal { class TDecimal { public: + //! Both types defined below represent signed values. They are only intended + //! to be used for manipulating binary data, since they don't actually expose + //! any arithmetic operations. You should avoid dealing with individual parts. struct TValue128 { ui64 Low; @@ -19,15 +24,21 @@ public: }; static_assert(sizeof(TValue128) == 2 * sizeof(ui64)); + struct TValue256 + { + std::array<ui32, 8> Parts; + }; + static_assert(sizeof(TValue256) == 4 * sizeof(ui64)); + public: - // Maximum precision supported by YT - static constexpr int MaxPrecision = 35; - static constexpr int MaxBinarySize = 16; + //! Maximum precision supported by YT. + static constexpr int MaxPrecision = 76; + static constexpr int MaxBinarySize = 32; // NB. Sometimes we print values that exceed MaxPrecision (e.g. in error messages) - // MaxTextSize is chosen so we can print ANY i128 number as decimal. + // MaxTextSize is chosen so we can print ANY i256 number as decimal. static constexpr int MaxTextSize = - std::numeric_limits<ui128>::digits + 1 // max number of digits in ui128 number + 77 // length of 2^63 in decimal form + 1 // possible decimal point + 1; // possible minus sign @@ -49,13 +60,15 @@ public: static TStringBuf WriteBinary32(int precision, i32 value, char* buffer, size_t bufferLength); static TStringBuf WriteBinary64(int precision, i64 value, char* buffer, size_t bufferLength); static TStringBuf WriteBinary128(int precision, TValue128 value, char* buffer, size_t bufferLength); + static TStringBuf WriteBinary256(int precision, TValue256 value, char* buffer, size_t bufferLength); // Writes either 32-bit, 64-bit or 128-bit binary value depending on precision, provided a TValue128. - static TStringBuf WriteBinaryVariadic(int precision, TValue128 value, char* buffer, size_t bufferLength); + static TStringBuf WriteBinary128Variadic(int precision, TValue128 value, char* buffer, size_t bufferLength); static i32 ParseBinary32(int precision, TStringBuf buffer); static i64 ParseBinary64(int precision, TStringBuf buffer); static TValue128 ParseBinary128(int precision, TStringBuf buffer); + static TValue256 ParseBinary256(int precision, TStringBuf buffer); }; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/library/decimal/unittests/decimal_ut.cpp b/yt/yt/library/decimal/unittests/decimal_ut.cpp index ba8be4ab15..50c550707c 100644 --- a/yt/yt/library/decimal/unittests/decimal_ut.cpp +++ b/yt/yt/library/decimal/unittests/decimal_ut.cpp @@ -98,6 +98,150 @@ TEST(TDecimal, TestTextBinaryConversion) "61449825198266175750309883089040771", "800BD5B5D5F0C73E0C9CD4943298B583"); + // A few more test cases with big numbers for various precisions generated by python snippet: + // import random + // def print_test_case(plus, binary_size, precision): + // textval = "".join(random.choice("0123456789") for _ in range(precision)) + // if not plus: + // textval = "-" + textval + // binval = hex(2 ** (binary_size * 8 - 1) + int(textval)) + // binval = binval[2:].strip('L') # strip 0x and final 'L' + // binval = binval.upper() + // print( + // "TEST_TEXT_BINARY_CONVERSION(\n" + // " {precision}, 0,\n" + // " \"{text}\",\n" + // " \"{binary}\");\n" + // .format(precision=precision, text=textval.lstrip("0"), binary=binval) + // ) + // random.seed(42) + // + // for binary_size, precision in (4, 9), (8, 18), (16, 35), (16, 38), (32, 39), (32, 76): + // print_test_case(False, binary_size, precision) + // print_test_case(False, binary_size, precision) + // print_test_case(True, binary_size, precision) + // print_test_case(True, binary_size, precision) + + TEST_TEXT_BINARY_CONVERSION( + 9, 0, + "-104332181", + "79C8046B"); + + TEST_TEXT_BINARY_CONVERSION( + 9, 0, + "-960013389", + "46C75BB3"); + + TEST_TEXT_BINARY_CONVERSION( + 9, 0, + "83863794", + "84FFA8F2"); + + TEST_TEXT_BINARY_CONVERSION( + 9, 0, + "26542351", + "8195010F"); + + TEST_TEXT_BINARY_CONVERSION( + 18, 0, + "-161559407816184959", + "7DC2069316FE6B81"); + + TEST_TEXT_BINARY_CONVERSION( + 18, 0, + "-310341316475255341", + "7BB17237885D85D3"); + + TEST_TEXT_BINARY_CONVERSION( + 18, 0, + "928327648350305641", + "8CE21513E317FD69"); + + TEST_TEXT_BINARY_CONVERSION( + 18, 0, + "395376724238849696", + "857CA90930B6D6A0"); + + TEST_TEXT_BINARY_CONVERSION( + 35, 0, + "-53287101226916697848018451462704828", + "7FF5BCBE39B5F1A05ED5F0135FD2B144"); + + TEST_TEXT_BINARY_CONVERSION( + 35, 0, + "-14893252880957015430391171822782489", + "7FFD21B4B88705D84E5A0ECF560DF7E7"); + + TEST_TEXT_BINARY_CONVERSION( + 35, 0, + "63834657871331509839301031051834738", + "800C4B4AA7E81EF6726D934AA12AB572"); + + TEST_TEXT_BINARY_CONVERSION( + 35, 0, + "29973763116566701065133387262473178", + "8005C5D2141747EF1198888FB96033DA"); + + TEST_TEXT_BINARY_CONVERSION( + 38, 0, + "-10801326773602606474687234309805009788", + "77DFBD795731FA57AD40A913BEB49484"); + + TEST_TEXT_BINARY_CONVERSION( + 38, 0, + "-20812191361939909169985435346247510799", + "7057B7BE1F13B6F1D6DB2F569BA994F1"); + + TEST_TEXT_BINARY_CONVERSION( + 38, 0, + "11838425135427849808412411824493534874", + "88E7FF6C4CB25E5053056568535B9E9A"); + + TEST_TEXT_BINARY_CONVERSION( + 38, 0, + "1640052427868011280598262045053315869", + "813BDCD3E2BEF11A4624071A88406B1D"); + + TEST_TEXT_BINARY_CONVERSION( + 39, 0, + "-232260256342160733754330365414586850142", + "7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF51444D3EF60040D9A177B4EBDCD1E4A2"); + + TEST_TEXT_BINARY_CONVERSION( + 39, 0, + "-940196556981693406088356159514846564823", + "7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD3CACBACF9473427D637FA1697111EE29"); + + TEST_TEXT_BINARY_CONVERSION( + 39, 0, + "662994680443699577738721489513433200379", + "80000000000000000000000000000001F2C8217C5832413CEDC2D8713542F6FB"); + + TEST_TEXT_BINARY_CONVERSION( + 39, 0, + "176936763201632870831727889579868727743", + "80000000000000000000000000000000851CC7F2FA54AF2AC8A1868AB524A9BF"); + + TEST_TEXT_BINARY_CONVERSION( + 76, 0, + "-4873471434558122362316658760366909670546688937346706562729806990162720465375", + "7539B681CC20CEEDAB96C358A49B105EB5C51C958B66D69F946E52B2C4CBDE21"); + + TEST_TEXT_BINARY_CONVERSION( + 76, 0, + "-5646417080531003309232719374529912419049663193149190586518506716572628498776", + "73843DBE5B00AA3065C94D18C3DE5B54ABBCB7AD4AB9775BD7308BA12A14C6A8"); + + TEST_TEXT_BINARY_CONVERSION( + 76, 0, + "9453147379965075273545494808313678377701436349578856855744431351823374989413", + "94E64AB40D1C30A9A2BFEA9C96B7584F559D5572641F0C90E86764F3187BB865"); + + TEST_TEXT_BINARY_CONVERSION( + 76, 0, + "4352408240084271094777520471167190229413186999386774964990913341232812067974", + "899F603224EC624E2351628FF4637023C84FDBB0E2BBCBBB9395D8DD19B30C86"); + #undef TEST_TEXT_BINARY_CONVERSION } @@ -127,20 +271,27 @@ TEST(TDecimal, TestPrecisionScaleLimits) EXPECT_THROW_WITH_SUBSTRING(TDecimal::TextToBinary("314.15", 5, 3), "too many digits before decimal point"); EXPECT_THROW_WITH_SUBSTRING(TDecimal::TextToBinary("-314.15", 5, 3), "too many digits before decimal point"); + // This group of tests checks that text values which cause signed overflow throw valid exceptions. + // The values for each precision are equal to 2^(binary_size - 1) + 0.11. + EXPECT_THROW_WITH_SUBSTRING(TDecimal::TextToBinary("2147483647.11", 9, 2), "too many digits before decimal point"); + EXPECT_THROW_WITH_SUBSTRING(TDecimal::TextToBinary("9223372036854775807.11", 18, 2), "too many digits before decimal point"); + EXPECT_THROW_WITH_SUBSTRING(TDecimal::TextToBinary("170141183460469231731687303715884105727.11", 35, 2), "too many digits before decimal point"); + EXPECT_THROW_WITH_SUBSTRING(TDecimal::TextToBinary("57896044618658097711785492504343953926634992332820282019728792003956564819967.11", TDecimal::MaxPrecision, 2), "too many digits before decimal point"); + // Sometimes we want to print values that are not representable with given precision // (e.g. in error messages we sometimes want to print text value of invalid decimal to explain that it has // more digits than allowed by precision). // // Here we test that extreme values are printed ok. - auto maxBinaryDecimal = HexDecode("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD"); - auto minBinaryDecimal1 = HexDecode("00000000000000000000000000000000"); - auto minBinaryDecimal2 = HexDecode("00000000000000000000000000000003"); + auto maxBinaryDecimal = HexDecode("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD"); + auto minBinaryDecimal1 = HexDecode("0000000000000000000000000000000000000000000000000000000000000000"); + auto minBinaryDecimal2 = HexDecode("0000000000000000000000000000000000000000000000000000000000000003"); EXPECT_EQ(TDecimal::MaxBinarySize, std::ssize(maxBinaryDecimal)); // If max TDecimal::MaxBinarySize ever increases EXPECT_EQ(TDecimal::MaxBinarySize, std::ssize(minBinaryDecimal1)); // please update this test EXPECT_EQ(TDecimal::MaxBinarySize, std::ssize(minBinaryDecimal2)); // with better values. - EXPECT_EQ("1701411834604692317316873037158841057.25", TDecimal::BinaryToText(maxBinaryDecimal, TDecimal::MaxPrecision, 2)); - EXPECT_EQ("-1701411834604692317316873037158841057.28", TDecimal::BinaryToText(minBinaryDecimal1, TDecimal::MaxPrecision, 2)); - EXPECT_EQ("-1701411834604692317316873037158841057.25", TDecimal::BinaryToText(minBinaryDecimal2, TDecimal::MaxPrecision, 2)); + EXPECT_EQ("578960446186580977117854925043439539266349923328202820197287920039565648199.65", TDecimal::BinaryToText(maxBinaryDecimal, TDecimal::MaxPrecision, 2)); + EXPECT_EQ("-578960446186580977117854925043439539266349923328202820197287920039565648199.68", TDecimal::BinaryToText(minBinaryDecimal1, TDecimal::MaxPrecision, 2)); + EXPECT_EQ("-578960446186580977117854925043439539266349923328202820197287920039565648199.65", TDecimal::BinaryToText(minBinaryDecimal2, TDecimal::MaxPrecision, 2)); } TEST(TDecimal, TestValidation) @@ -148,6 +299,7 @@ TEST(TDecimal, TestValidation) EXPECT_NO_THROW(TDecimal::ValidateBinaryValue(HexDecode("8000013A"), 3, 2)); EXPECT_NO_THROW(TDecimal::ValidateBinaryValue(HexDecode("80000000" "0000013A"), 10, 2)); EXPECT_NO_THROW(TDecimal::ValidateBinaryValue(HexDecode("80000000" "00000000" "00000000" "0000013A"), 35, 2)); + EXPECT_NO_THROW(TDecimal::ValidateBinaryValue(HexDecode("80000000" "00000000" "00000000" "00000000" "00000000" "00000000" "00000000" "0000013A"), 76, 2)); } class TDecimalWithPrecisionTest diff --git a/yt/yt/library/formats/arrow_parser.cpp b/yt/yt/library/formats/arrow_parser.cpp index cc31e25335..f34127990a 100644 --- a/yt/yt/library/formats/arrow_parser.cpp +++ b/yt/yt/library/formats/arrow_parser.cpp @@ -163,28 +163,17 @@ public: return ParseNull(); } - // Decimal types. For now, YT natively supports only Decimal128 with scale up to 35. - // Thus, we represent short enough decimals as native YT decimals, and wider decimals as - // their decimal string representation; but the latter is subject to change whenever we - // get the native support for Decimal128 with scale up to 38 or Decimal256 with scale up to 76. arrow::Status Visit(const arrow::Decimal128Type& type) override { - constexpr int MaximumYTDecimalPrecision = 35; - if (type.precision() <= MaximumYTDecimalPrecision) { - return ParseStringLikeArray<arrow::Decimal128Array>([&] (const TStringBuf& value, i64 columnId) { - return MakeDecimalBinaryValue(value, columnId, type.precision()); - }); - } else { - return ParseStringLikeArray<arrow::Decimal128Array>([&] (const TStringBuf& value, i64 columnId) { - return MakeDecimalTextValue<arrow::Decimal128>(value, columnId, type.scale()); - }); - } + return ParseStringLikeArray<arrow::Decimal128Array>([&] (const TStringBuf& value, i64 columnId) { + return MakeDecimalBinaryValue<TDecimal::TValue128>(value, columnId, type.precision()); + }); } arrow::Status Visit(const arrow::Decimal256Type& type) override { return ParseStringLikeArray<arrow::Decimal256Array>([&] (const TStringBuf& value, i64 columnId) { - return MakeDecimalTextValue<arrow::Decimal256>(value, columnId, type.scale()); + return MakeDecimalBinaryValue<TDecimal::TValue256>(value, columnId, type.precision()); }); } @@ -294,33 +283,31 @@ private: return arrow::Status::OK(); } + template <class TUnderlyingValueType> TUnversionedValue MakeDecimalBinaryValue(const TStringBuf& value, i64 columnId, int precision) { - // NB: arrow wire representation of Decimal128 is little-endian and (obviously) 128 bit, + // NB: Arrow wire representation of Decimal128 is little-endian and (obviously) 128 bit, // while YT in-memory representation of Decimal is big-endian, variadic-length of either 32 bit, 64 bit or 128 bit, // and MSB-flipped to ensure lexical sorting order. - TDecimal::TValue128 value128; - YT_VERIFY(value.size() == sizeof(value128)); - std::memcpy(&value128, value.data(), value.size()); + // Representation of Decimal256 is similar, but only 256 bits. + TUnderlyingValueType decimalValue; + YT_VERIFY(value.size() == sizeof(decimalValue)); + std::memcpy(&decimalValue, value.data(), value.size()); - const auto maxByteCount = sizeof(value128); + const auto maxByteCount = sizeof(decimalValue); char* buffer = BufferForStringLikeValues_->Preallocate(maxByteCount); - auto decimalBinary = TDecimal::WriteBinaryVariadic(precision, value128, buffer, maxByteCount); + TStringBuf decimalBinary; + if constexpr (std::is_same_v<TUnderlyingValueType, TDecimal::TValue128>) { + decimalBinary = TDecimal::WriteBinary128Variadic(precision, decimalValue, buffer, maxByteCount); + } else if constexpr (std::is_same_v<TUnderlyingValueType, TDecimal::TValue256>) { + decimalBinary = TDecimal::WriteBinary256(precision, decimalValue, buffer, maxByteCount); + } else { + static_assert(std::is_same_v<TUnderlyingValueType, TDecimal::TValue256>, "Unexpected decimal type"); + } BufferForStringLikeValues_->Advance(decimalBinary.size()); return MakeUnversionedStringValue(decimalBinary, columnId); } - - template <class TArrowDecimalType> - TUnversionedValue MakeDecimalTextValue(const TStringBuf& value, i64 columnId, int scale) - { - TArrowDecimalType decimal(reinterpret_cast<const uint8_t*>(value.data())); - auto string = decimal.ToString(scale); - char* buffer = BufferForStringLikeValues_->Preallocate(string.size()); - std::memcpy(buffer, string.data(), string.size()); - BufferForStringLikeValues_->Advance(string.size()); - return MakeUnversionedStringValue(TStringBuf(buffer, string.size()), columnId); - } }; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/library/formats/skiff_parser.cpp b/yt/yt/library/formats/skiff_parser.cpp index edff9ffa0b..febde44f42 100644 --- a/yt/yt/library/formats/skiff_parser.cpp +++ b/yt/yt/library/formats/skiff_parser.cpp @@ -300,6 +300,7 @@ const auto precision = denullifiedType.GetPrecision(); CASE(EWireType::Int32); CASE(EWireType::Int64); CASE(EWireType::Int128); + CASE(EWireType::Int256); #undef CASE case EWireType::Yson32: return CreatePrimitiveTypeConverter(wireType, fieldDescription.IsRequired(), columnId, ysonConverter); diff --git a/yt/yt/library/formats/skiff_writer.cpp b/yt/yt/library/formats/skiff_writer.cpp index 3bc6965219..0984e72c8f 100644 --- a/yt/yt/library/formats/skiff_writer.cpp +++ b/yt/yt/library/formats/skiff_writer.cpp @@ -506,6 +506,10 @@ TUnversionedValueToSkiffConverter CreateDecimalValueConverter( return CreatePrimitiveValueConverter<EValueType::String>( isRequired, TDecimalSkiffWriter<EWireType::Int128>(precision)); + case EWireType::Int256: + return CreatePrimitiveValueConverter<EValueType::String>( + isRequired, + TDecimalSkiffWriter<EWireType::Int256>(precision)); case EWireType::Yson32: return CreatePrimitiveValueConverter(wireType, isRequired); default: diff --git a/yt/yt/library/formats/skiff_yson_converter-inl.h b/yt/yt/library/formats/skiff_yson_converter-inl.h index 43dabc63a5..8d5a2f8efd 100644 --- a/yt/yt/library/formats/skiff_yson_converter-inl.h +++ b/yt/yt/library/formats/skiff_yson_converter-inl.h @@ -70,8 +70,18 @@ Y_FORCE_INLINE TStringBuf TDecimalSkiffParser<SkiffWireType>::operator() (NSkiff TDecimal::TValue128{skiffValue.Low, skiffValue.High}, Buffer_, sizeof(Buffer_)); + } else if constexpr (SkiffWireType == EWireType::Int256) { + const auto skiffValue = parser->ParseInt256(); + TDecimal::TValue256 decimalValue; + static_assert(sizeof(decimalValue) == sizeof(skiffValue)); + std::memcpy(&decimalValue, &skiffValue, sizeof(decimalValue)); + return TDecimal::WriteBinary256( + Precision_, + std::move(decimalValue), + Buffer_, + sizeof(Buffer_)); } else { - static_assert(SkiffWireType == EWireType::Int128); + static_assert(SkiffWireType == EWireType::Int256); } } @@ -99,9 +109,15 @@ void TDecimalSkiffWriter<SkiffWireType>::operator()(TStringBuf value, NSkiff::TC } else if constexpr (SkiffWireType == EWireType::Int128) { auto intValue = TDecimal::ParseBinary128(Precision_, value); writer->WriteInt128(TInt128{intValue.Low, intValue.High}); + } else if constexpr (SkiffWireType == EWireType::Int256) { + auto intValue = TDecimal::ParseBinary256(Precision_, value); + TInt256 skiffValue; + static_assert(sizeof(skiffValue) == sizeof(intValue)); + std::memcpy(&skiffValue, &intValue, sizeof(skiffValue)); + writer->WriteInt256(std::move(skiffValue)); } else { // poor man's static_assert(false) - static_assert(SkiffWireType == EWireType::Int128); + static_assert(SkiffWireType == EWireType::Int256); } } diff --git a/yt/yt/library/formats/skiff_yson_converter.cpp b/yt/yt/library/formats/skiff_yson_converter.cpp index 82fac3ef1b..ebde4dcb3d 100644 --- a/yt/yt/library/formats/skiff_yson_converter.cpp +++ b/yt/yt/library/formats/skiff_yson_converter.cpp @@ -748,6 +748,10 @@ TYsonToSkiffConverter CreateDecimalYsonToSkiffConverter( return CreatePrimitiveTypeYsonToSkiffConverter<EYsonItemType::StringValue>( std::move(descriptor), TDecimalSkiffWriter<EWireType::Int128>(precision)); + case EWireType::Int256: + return CreatePrimitiveTypeYsonToSkiffConverter<EYsonItemType::StringValue>( + std::move(descriptor), + TDecimalSkiffWriter<EWireType::Int256>(precision)); case EWireType::Yson32: return CreatePrimitiveTypeYsonToSkiffConverter(std::move(descriptor), wireType); default: @@ -1815,6 +1819,8 @@ TSkiffToYsonConverter CreateDecimalSkiffToYsonConverter( return TPrimitiveTypeSkiffToYsonConverter(TDecimalSkiffParser<EWireType::Int64>(precision)); case EWireType::Int128: return TPrimitiveTypeSkiffToYsonConverter(TDecimalSkiffParser<EWireType::Int128>(precision)); + case EWireType::Int256: + return TPrimitiveTypeSkiffToYsonConverter(TDecimalSkiffParser<EWireType::Int256>(precision)); case EWireType::Yson32: return CreatePrimitiveTypeSkiffToYsonConverter(wireType); default: @@ -1899,6 +1905,8 @@ void CheckSkiffWireTypeForDecimal(int precision, NSkiff::EWireType wireType) skiffBinarySize = sizeof(i64); } else if (wireType == NSkiff::EWireType::Int128) { skiffBinarySize = 2 * sizeof(i64); + } else if (wireType == NSkiff::EWireType::Int256) { + skiffBinarySize = 4 * sizeof(i64); } if (decimalBinarySize != skiffBinarySize) { |