aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAndrey Chulkov <achulkov2@nebius.com>2024-10-15 14:10:17 +0300
committerrobot-piglet <robot-piglet@yandex-team.com>2024-10-15 14:27:22 +0300
commit05cf1efb77e08298322326fe748b6bcdb7b6526e (patch)
treeb55a13dc9277ebf98872ee1ddee93e8f24781265
parent2ff5baf34b9dea61580028eb4c4afde7601232a7 (diff)
downloadydb-05cf1efb77e08298322326fe748b6bcdb7b6526e.tar.gz
Increase max decimal precision to 76 (decimal256)
* Changelog entry Type: feature Component: type system Support for decimals with precision from 35 to 76. This PR adds decimal256 support to YT. Additionally: - Decimal256 is supported in skiff as int256. - ClickHouse decimal precisions from 35 to 76 are supported in CHYT. --- Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/798 Co-authored-by: ermolovd <ermolovd@yandex-team.com> Co-authored-by: dakovalkov <dakovalkov@yandex-team.com> commit_hash:73c4809966cf4c625e6007d31b5dde14bd80e829
-rw-r--r--library/cpp/skiff/public.h2
-rw-r--r--library/cpp/skiff/skiff.cpp70
-rw-r--r--library/cpp/skiff/skiff.h29
-rw-r--r--library/cpp/skiff/skiff_schema-inl.h2
-rw-r--r--library/cpp/skiff/skiff_validator.cpp2
-rw-r--r--library/cpp/skiff/unittests/skiff_ut.cpp46
-rw-r--r--yt/yt/client/table_client/logical_type.h2
-rw-r--r--yt/yt/library/decimal/decimal.cpp515
-rw-r--r--yt/yt/library/decimal/decimal.h25
-rw-r--r--yt/yt/library/decimal/unittests/decimal_ut.cpp164
-rw-r--r--yt/yt/library/formats/arrow_parser.cpp51
-rw-r--r--yt/yt/library/formats/skiff_parser.cpp1
-rw-r--r--yt/yt/library/formats/skiff_writer.cpp4
-rw-r--r--yt/yt/library/formats/skiff_yson_converter-inl.h20
-rw-r--r--yt/yt/library/formats/skiff_yson_converter.cpp8
15 files changed, 812 insertions, 129 deletions
diff --git a/library/cpp/skiff/public.h b/library/cpp/skiff/public.h
index d67c6f26ee..aa2c0a5bb4 100644
--- a/library/cpp/skiff/public.h
+++ b/library/cpp/skiff/public.h
@@ -15,11 +15,13 @@ enum class EWireType
Int32 /* "int32" */,
Int64 /* "int64" */,
Int128 /* "int128" */,
+ Int256 /* "int256" */,
Uint8 /* "uint8" */,
Uint16 /* "uint16" */,
Uint32 /* "uint32" */,
Uint64 /* "uint64" */,
Uint128 /* "uint128" */,
+ Uint256 /* "uint256" */,
Double /* "double" */,
Boolean /* "boolean" */,
String32 /* "string32" */,
diff --git a/library/cpp/skiff/skiff.cpp b/library/cpp/skiff/skiff.cpp
index cbdbdfe364..9b42628cc6 100644
--- a/library/cpp/skiff/skiff.cpp
+++ b/library/cpp/skiff/skiff.cpp
@@ -32,6 +32,18 @@ bool operator!=(TUint128 lhs, TUint128 rhs)
////////////////////////////////////////////////////////////////////////////////
+bool operator==(const TInt256& lhs, const TInt256& rhs)
+{
+ return lhs.Parts == rhs.Parts;
+}
+
+bool operator==(const TUint256& lhs, const TUint256& rhs)
+{
+ return lhs.Parts == rhs.Parts;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
TUncheckedSkiffParser::TUncheckedSkiffParser(IZeroCopyInput* underlying)
: Underlying_(underlying)
, Buffer_(512 * 1024)
@@ -95,6 +107,26 @@ TUint128 TUncheckedSkiffParser::ParseUint128()
return {low, high};
}
+TInt256 TUncheckedSkiffParser::ParseInt256()
+{
+ TInt256 result;
+ for (auto& part : result.Parts) {
+ part = ParseSimple<ui64>();
+ }
+
+ return result;
+}
+
+TUint256 TUncheckedSkiffParser::ParseUint256()
+{
+ TUint256 result;
+ for (auto& part : result.Parts) {
+ part = ParseSimple<ui64>();
+ }
+
+ return result;
+}
+
double TUncheckedSkiffParser::ParseDouble()
{
return ParseSimple<double>();
@@ -274,6 +306,18 @@ TUint128 TCheckedSkiffParser::ParseUint128()
return Parser_.ParseUint128();
}
+TInt256 TCheckedSkiffParser::ParseInt256()
+{
+ Validator_->OnSimpleType(EWireType::Int256);
+ return Parser_.ParseInt256();
+}
+
+TUint256 TCheckedSkiffParser::ParseUint256()
+{
+ Validator_->OnSimpleType(EWireType::Uint256);
+ return Parser_.ParseUint256();
+}
+
double TCheckedSkiffParser::ParseDouble()
{
Validator_->OnSimpleType(EWireType::Double);
@@ -389,6 +433,20 @@ void TUncheckedSkiffWriter::WriteUint128(TUint128 value)
WriteSimple<ui64>(value.High);
}
+void TUncheckedSkiffWriter::WriteInt256(const TInt256& value)
+{
+ for (auto part : value.Parts) {
+ WriteSimple<ui64>(part);
+ }
+}
+
+void TUncheckedSkiffWriter::WriteUint256(const TUint256& value)
+{
+ for (auto part : value.Parts) {
+ WriteSimple<ui64>(part);
+ }
+}
+
void TUncheckedSkiffWriter::WriteUint8(ui8 value)
{
WriteSimple<ui8>(value);
@@ -551,6 +609,18 @@ void TCheckedSkiffWriter::WriteUint128(TUint128 value)
Writer_.WriteUint128(value);
}
+void TCheckedSkiffWriter::WriteInt256(TInt256 value)
+{
+ Validator_->OnSimpleType(EWireType::Int256);
+ Writer_.WriteInt256(std::move(value));
+}
+
+void TCheckedSkiffWriter::WriteUint256(TUint256 value)
+{
+ Validator_->OnSimpleType(EWireType::Uint256);
+ Writer_.WriteUint256(std::move(value));
+}
+
void TCheckedSkiffWriter::WriteString32(TStringBuf value)
{
Validator_->OnSimpleType(EWireType::String32);
diff --git a/library/cpp/skiff/skiff.h b/library/cpp/skiff/skiff.h
index 183c112700..c1315081d3 100644
--- a/library/cpp/skiff/skiff.h
+++ b/library/cpp/skiff/skiff.h
@@ -49,6 +49,23 @@ bool operator!=(TUint128 lhs, TUint128 rhs);
////////////////////////////////////////////////////////////////////////////////
+struct TInt256
+{
+ std::array<ui64, 4> Parts;
+};
+
+struct TUint256
+{
+ std::array<ui64, 4> Parts;
+};
+
+// Operator != is synthesized since C++ 20.
+bool operator==(const TInt256& lhs, const TInt256& rhs);
+
+bool operator==(const TUint256& lhs, const TUint256& rhs);
+
+////////////////////////////////////////////////////////////////////////////////
+
class TUncheckedSkiffParser
{
public:
@@ -68,6 +85,9 @@ public:
TInt128 ParseInt128();
TUint128 ParseUint128();
+ TInt256 ParseInt256();
+ TUint256 ParseUint256();
+
double ParseDouble();
bool ParseBoolean();
@@ -127,6 +147,9 @@ public:
TInt128 ParseInt128();
TUint128 ParseUint128();
+ TInt256 ParseInt256();
+ TUint256 ParseUint256();
+
double ParseDouble();
bool ParseBoolean();
@@ -177,6 +200,9 @@ public:
void WriteInt128(TInt128 value);
void WriteUint128(TUint128 value);
+ void WriteInt256(const TInt256& value);
+ void WriteUint256(const TUint256& value);
+
void WriteString32(TStringBuf value);
void WriteYson32(TStringBuf value);
@@ -223,6 +249,9 @@ public:
void WriteInt128(TInt128 value);
void WriteUint128(TUint128 value);
+ void WriteInt256(TInt256 value);
+ void WriteUint256(TUint256 value);
+
void WriteString32(TStringBuf value);
void WriteYson32(TStringBuf value);
diff --git a/library/cpp/skiff/skiff_schema-inl.h b/library/cpp/skiff/skiff_schema-inl.h
index 853ff36738..a977194180 100644
--- a/library/cpp/skiff/skiff_schema-inl.h
+++ b/library/cpp/skiff/skiff_schema-inl.h
@@ -19,12 +19,14 @@ inline bool IsSimpleType(EWireType type)
case EWireType::Int32:
case EWireType::Int64:
case EWireType::Int128:
+ case EWireType::Int256:
case EWireType::Uint8:
case EWireType::Uint16:
case EWireType::Uint32:
case EWireType::Uint64:
case EWireType::Uint128:
+ case EWireType::Uint256:
case EWireType::Double:
case EWireType::Boolean:
diff --git a/library/cpp/skiff/skiff_validator.cpp b/library/cpp/skiff/skiff_validator.cpp
index 76dd3b7600..a2e9e1db90 100644
--- a/library/cpp/skiff/skiff_validator.cpp
+++ b/library/cpp/skiff/skiff_validator.cpp
@@ -363,12 +363,14 @@ std::shared_ptr<IValidatorNode> CreateUsageValidatorNode(const std::shared_ptr<T
case EWireType::Int32:
case EWireType::Int64:
case EWireType::Int128:
+ case EWireType::Int256:
case EWireType::Uint8:
case EWireType::Uint16:
case EWireType::Uint32:
case EWireType::Uint64:
case EWireType::Uint128:
+ case EWireType::Uint256:
case EWireType::Double:
case EWireType::Boolean:
diff --git a/library/cpp/skiff/unittests/skiff_ut.cpp b/library/cpp/skiff/unittests/skiff_ut.cpp
index 5e4c709611..cdb09e7d52 100644
--- a/library/cpp/skiff/unittests/skiff_ut.cpp
+++ b/library/cpp/skiff/unittests/skiff_ut.cpp
@@ -209,6 +209,29 @@ Y_UNIT_TEST_SUITE(Skiff)
UNIT_ASSERT_EQUAL(parser.ParseInt128(), val2);
}
+ Y_UNIT_TEST(TestInt256)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Int256);
+
+ const TInt256 val1 = {0x1924cd4aeb9ced82, 0x0885e83f456d6a7e, 0xe9ba36585eccae1a, 0x7854b6f9ce448be9};
+ const TInt256 val2 = {0xe9ba36585eccae1a, 0x1924cd4aeb9ced82, 0x0885e83f456d6a7e, static_cast<ui64>(-0x7854b6f9ce448be9)};
+
+ TCheckedSkiffWriter writer(schema, &bufferStream);
+ writer.WriteInt256(val1);
+ writer.WriteInt256(val2);
+ writer.Finish();
+
+ UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+ "82ed9ceb4acd2419" "7e6a6d453fe88508" "1aaecc5e5836bae9" "e98b44cef9b65478"
+ "1aaecc5e5836bae9" "82ed9ceb4acd2419" "7e6a6d453fe88508" "1774bb310649ab87");
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_EQUAL(parser.ParseInt256(), val1);
+ UNIT_ASSERT_EQUAL(parser.ParseInt256(), val2);
+ }
+
Y_UNIT_TEST(TestUint128)
{
TBufferStream bufferStream;
@@ -232,6 +255,29 @@ Y_UNIT_TEST_SUITE(Skiff)
UNIT_ASSERT_EQUAL(parser.ParseUint128(), val2);
}
+ Y_UNIT_TEST(TestUint256)
+ {
+ TBufferStream bufferStream;
+
+ auto schema = CreateSimpleTypeSchema(EWireType::Uint256);
+
+ const auto val1 = TUint256{0x1924cd4aeb9ced82, 0x7854b6f9ce448be9, 0x8854b6f9ce448be9, 0x0885e83f456d6a7e};
+ const auto val2 = TUint256{0xe9ba36585eccae1a, 0x8854b6f9ce448be9, 0x1924cd4aeb9ced82, 0xabacabadabacaba0};
+
+ TCheckedSkiffWriter writer(schema, &bufferStream);
+ writer.WriteUint256(val1);
+ writer.WriteUint256(val2);
+ writer.Finish();
+
+ UNIT_ASSERT_VALUES_EQUAL(HexEncode(bufferStream.Buffer()),
+ "82ed9ceb4acd2419" "e98b44cef9b65478" "e98b44cef9b65488" "7e6a6d453fe88508"
+ "1aaecc5e5836bae9" "e98b44cef9b65488" "82ed9ceb4acd2419" "a0abacabadabacab");
+
+ TCheckedSkiffParser parser(schema, &bufferStream);
+ UNIT_ASSERT_EQUAL(parser.ParseUint256(), val1);
+ UNIT_ASSERT_EQUAL(parser.ParseUint256(), val2);
+ }
+
Y_UNIT_TEST(TestBoolean)
{
auto schema = CreateSimpleTypeSchema(EWireType::Boolean);
diff --git a/yt/yt/client/table_client/logical_type.h b/yt/yt/client/table_client/logical_type.h
index 904ff9708b..bd482df984 100644
--- a/yt/yt/client/table_client/logical_type.h
+++ b/yt/yt/client/table_client/logical_type.h
@@ -170,7 +170,7 @@ class TDecimalLogicalType
{
public:
static constexpr int MinPrecision = 1;
- static constexpr int MaxPrecision = 35;
+ static constexpr int MaxPrecision = 76;
public:
TDecimalLogicalType(int precision, int scale);
diff --git a/yt/yt/library/decimal/decimal.cpp b/yt/yt/library/decimal/decimal.cpp
index e1674f84e2..3df4b44028 100644
--- a/yt/yt/library/decimal/decimal.cpp
+++ b/yt/yt/library/decimal/decimal.cpp
@@ -12,24 +12,263 @@ namespace NYT::NDecimal {
////////////////////////////////////////////////////////////////////////////////
+// We use the same type used for binary representation of 256 bit decimals for
+// implementing the necessary arithmetic operations.
+using i256 = TDecimal::TValue256;
+
+// We chose to only implement a small subset of operations and functions necessary
+// for converting 256 bit decimals to and from text.
+// Rationale: there do not seem to be any good implementations of 256-bit arithmetic
+// that we are comfortable depending on (for now). There are some big number
+// implementations around openssl, but these types are intended for arbitrarily
+// large integers and use dynamic allocations. It also does not make sense to commit
+// to implementing full-fledged 256-bit arithmetics when we actually use a minority
+// of operations.
+// When we have a good enough int256 either in library/util or in contrib, we can
+// switch to it easily.
+
+////////////////////////////////////////////////////////////////////////////////
+
+constexpr i256 operator-(i256 value) noexcept
+{
+ // Invert.
+ for (int partIndex = 0; partIndex < std::ssize(value.Parts); ++partIndex) {
+ value.Parts[partIndex] = ~value.Parts[partIndex];
+ }
+
+ // Add 1.
+ for (int partIndex = 0; partIndex < std::ssize(value.Parts) && ++value.Parts[partIndex] == 0; ++partIndex) { }
+
+ return value;
+}
+
+constexpr std::strong_ordering operator<=>(const i256& lhs, const i256& rhs)
+{
+ bool lhsIsNegative = lhs.Parts.back() & (1u << 31);
+ bool rhsIsNegative = rhs.Parts.back() & (1u << 31);
+
+ if (lhsIsNegative && !rhsIsNegative) {
+ return std::strong_ordering::less;
+ }
+
+ if (!lhsIsNegative && rhsIsNegative) {
+ return std::strong_ordering::greater;
+ }
+
+ for (int partIndex = std::ssize(lhs.Parts) - 1; partIndex >= 0; --partIndex) {
+ if (lhs.Parts[partIndex] != rhs.Parts[partIndex]) {
+ return lhs.Parts[partIndex] <=> rhs.Parts[partIndex];
+ }
+ }
+
+ return std::strong_ordering::equal;
+}
+
+// Not synthesized by default :(
+constexpr bool operator==(const i256& lhs, const i256& rhs)
+{
+ return lhs.Parts == rhs.Parts;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Some operations require working with an explicitly unsigned type, since they
+// might cause integer overflow, which is not very acceptable for signed types.
+// It is also easier to implement some arithmetic operations (like *= 10) by
+// shifting bits.
+struct TUnsignedValue256
+{
+ std::array<ui32, 8> Parts;
+};
+
+using ui256 = TUnsignedValue256;
+
+static_assert(sizeof(ui256) == sizeof(i256));
+
+////////////////////////////////////////////////////////////////////////////////
+
+constexpr bool operator==(const ui256& lhs, const ui256& rhs)
+{
+ return lhs.Parts == rhs.Parts;
+}
+
+constexpr ui256 operator+(ui256 lhs, const ui256& rhs)
+{
+ ui64 carry = 0;
+ for (int partIndex = 0; partIndex < std::ssize(lhs.Parts); ++partIndex) {
+ carry += lhs.Parts[partIndex];
+ carry += rhs.Parts[partIndex];
+ lhs.Parts[partIndex] = carry;
+ carry >>= 32;
+ }
+
+ return lhs;
+}
+
+template <int Shift>
+Y_FORCE_INLINE constexpr ui256 ShiftUp(ui256 value)
+{
+ static_assert(Shift >= 0 && Shift <= 32);
+
+ value.Parts.back() <<= Shift;
+ for (int partIndex = std::ssize(value.Parts) - 2; partIndex >= 0; --partIndex) {
+ value.Parts[partIndex + 1] |= value.Parts[partIndex] >> (32 - Shift);
+ value.Parts[partIndex] <<= Shift;
+ }
+
+ return value;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
template <typename T>
constexpr bool ValidDecimalUnderlyingInteger =
std::is_same_v<T, i32> ||
std::is_same_v<T, i64> ||
- std::is_same_v<T, i128>;
+ std::is_same_v<T, i128> ||
+ std::is_same_v<T, i256>;
+
+template <typename T>
+constexpr bool ValidDecimalUnderlyingUnsignedInteger =
+ std::is_same_v<T, ui32> ||
+ std::is_same_v<T, ui64> ||
+ std::is_same_v<T, ui128> ||
+ std::is_same_v<T, ui256>;
+
+template <typename T>
+Y_FORCE_INLINE constexpr T GetNan()
+{
+ if constexpr (std::is_same_v<T, i256>) {
+ constexpr i32 i32Max = std::numeric_limits<i32>::max();
+ constexpr ui32 ui32Max = std::numeric_limits<ui32>::max();
+
+ return {ui32Max, ui32Max, ui32Max, ui32Max, ui32Max, ui32Max, ui32Max, i32Max};
+ } else {
+ return std::numeric_limits<T>::max();
+ }
+}
+
+template <typename T>
+Y_FORCE_INLINE constexpr T GetPlusInf()
+{
+ if constexpr (std::is_same_v<T, i256>) {
+ constexpr i32 i32Max = std::numeric_limits<i32>::max();
+ constexpr ui32 ui32Max = std::numeric_limits<ui32>::max();
+
+ return {ui32Max - 1, ui32Max, ui32Max, ui32Max, ui32Max, ui32Max, ui32Max, i32Max};
+ } else {
+ return std::numeric_limits<T>::max() - 1;
+ }
+}
template <typename T>
struct TDecimalTraits
{
static_assert(ValidDecimalUnderlyingInteger<T>);
- static constexpr T Nan = std::numeric_limits<T>::max();
- static constexpr T PlusInf = std::numeric_limits<T>::max() - 1;
+ static constexpr T Nan = GetNan<T>();
+ static constexpr T PlusInf = GetPlusInf<T>();
static constexpr T MinusInf = -PlusInf;
-
- static constexpr T MinSpecialValue = PlusInf;
};
+template <typename T>
+Y_FORCE_INLINE constexpr bool IsNegativeInteger(T value)
+{
+ static_assert(ValidDecimalUnderlyingInteger<T>);
+
+ if constexpr (std::is_same_v<T, i256>) {
+ return value.Parts.back() & (1u << 31);
+ } else {
+ return value < 0;
+ }
+}
+
+template <typename T>
+Y_FORCE_INLINE constexpr auto DecimalIntegerToUnsigned(T value)
+{
+ static_assert(ValidDecimalUnderlyingInteger<T>);
+
+ if constexpr (std::is_same_v<T, i256>) {
+ return ui256{value.Parts};
+ } else if constexpr (std::is_same_v<T, i128>) {
+ return ui128(value);
+ } else {
+ using TU = std::make_unsigned_t<T>;
+ return static_cast<TU>(value);
+ }
+}
+
+template <typename T>
+Y_FORCE_INLINE constexpr auto DecimalIntegerToSigned(T value)
+{
+ static_assert(ValidDecimalUnderlyingUnsignedInteger<T>);
+
+ if constexpr (std::is_same_v<T, ui256>) {
+ return i256{value.Parts};
+ } else if constexpr (std::is_same_v<T, ui128>) {
+ return i128(value);
+ } else {
+ using TS = std::make_signed_t<T>;
+ return static_cast<TS>(value);
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+//! The functions below are used for implementing conversion to/from text to binary decimal.
+//! We actually do not need any arithmetic operations beside v *= 10, v /= 10, addition and negation.
+//! so they are the only ones actually implemented for our custom i256/ui256.
+
+template <typename T>
+Y_FORCE_INLINE constexpr auto FlipMSB(T value)
+{
+ static_assert(ValidDecimalUnderlyingInteger<T>);
+
+ if constexpr (std::is_same_v<T, i256>) {
+ value.Parts.back() ^= (1u << 31);
+ return value;
+ } else {
+ constexpr auto One = DecimalIntegerToUnsigned(T{1});
+ // Bit operations are only valid with unsigned types.
+ return T(DecimalIntegerToUnsigned(value) ^ (One << (sizeof(T) * 8 - 1)));
+ }
+}
+
+template <typename T>
+Y_FORCE_INLINE constexpr ui32 GetNextDigit(T value, T* nextValue)
+{
+ static_assert(ValidDecimalUnderlyingUnsignedInteger<T>);
+
+ if constexpr (std::is_same_v<T, ui256>) {
+ ui64 remainder = 0;
+ for (int partIndex = std::ssize(value.Parts) - 1; partIndex >= 0; --partIndex) {
+ // Everything should fit into long long since we are dividing by 10.
+ auto step = std::lldiv(value.Parts[partIndex] + (remainder << 32), 10);
+ value.Parts[partIndex] = step.quot;
+ remainder = step.rem;
+ }
+ *nextValue = value;
+ return remainder;
+ } else {
+ constexpr auto Ten = T{10};
+ *nextValue = value / Ten;
+ return static_cast<ui32>(value % Ten);
+ }
+}
+
+template <typename T>
+Y_FORCE_INLINE constexpr T MultiplyByTen(T value)
+{
+ static_assert(ValidDecimalUnderlyingUnsignedInteger<T>);
+
+ if constexpr (std::is_same_v<T, ui256>) {
+ // 2 * (4 * v + v) = 10v.
+ return ShiftUp<1>(ShiftUp<2>(value) + value);
+ } else {
+ return value * DecimalIntegerToUnsigned(10);
+ }
+}
+
////////////////////////////////////////////////////////////////////////////////
constexpr int GetDecimalBinaryValueSize(int precision)
@@ -39,34 +278,41 @@ constexpr int GetDecimalBinaryValueSize(int precision)
return 4;
} else if (precision <= 18) {
return 8;
- } else if (precision <= 35) {
+ } else if (precision <= 38) {
return 16;
+ } else if (precision <= 76) {
+ return 32;
}
}
return 0;
}
-static constexpr i128 DecimalIntegerMaxValueTable[] = {
- i128{0}, // 0
- i128{9}, // 1
- i128{99}, // 2
- i128{999}, // 3
- i128{9999}, // 4
- i128{99999}, // 5
- i128{999999}, // 6
- i128{9999999}, // 7
- i128{99999999}, // 8
- i128{999999999}, // 9
- i128{9999999999ul}, // 10
- i128{99999999999ul}, // 11
- i128{999999999999ul}, // 12
- i128{9999999999999ul}, // 13
- i128{99999999999999ul}, // 14
- i128{999999999999999ul}, // 15
- i128{9999999999999999ul}, // 16
- i128{99999999999999999ul}, // 17
- i128{999999999999999999ul}, // 18
+static constexpr i32 Decimal32IntegerMaxValueTable[] = {
+ 0, // 0
+ 9, // 1
+ 99, // 2
+ 999, // 3
+ 9999, // 4
+ 99999, // 5
+ 999999, // 6
+ 9999999, // 7
+ 99999999, // 8
+ 999999999, // 9
+};
+static constexpr i64 Decimal64IntegerMaxValueTable[] = {
+ 9999999999ul, // 10
+ 99999999999ul, // 11
+ 999999999999ul, // 12
+ 9999999999999ul, // 13
+ 99999999999999ul, // 14
+ 999999999999999ul, // 15
+ 9999999999999999ul, // 16
+ 99999999999999999ul, // 17
+ 999999999999999999ul, // 18
+};
+
+static constexpr i128 Decimal128IntegerMaxValueTable[] = {
// 128 bits
//
// Generated by fair Python script:
@@ -79,7 +325,7 @@ static constexpr i128 DecimalIntegerMaxValueTable[] = {
// hex_value[-16:],
// hex_value[:-16] or "0",
// precision))
- // for i in range(19, 36):
+ // for i in range(19, 39):
// print_max_decimal(i)
//
i128{static_cast<ui64>(0x8ac7230489e7fffful)} | (i128{static_cast<ui64>(0x0ul)} << 64), // 19
@@ -99,37 +345,106 @@ static constexpr i128 DecimalIntegerMaxValueTable[] = {
i128{static_cast<ui64>(0x38c15b09fffffffful)} | (i128{static_cast<ui64>(0x314dc6448d93ul)} << 64), // 33
i128{static_cast<ui64>(0x378d8e63fffffffful)} | (i128{static_cast<ui64>(0x1ed09bead87c0ul)} << 64), // 34
i128{static_cast<ui64>(0x2b878fe7fffffffful)} | (i128{static_cast<ui64>(0x13426172c74d82ul)} << 64), // 35
+ i128{static_cast<ui64>(0xb34b9f0ffffffffful)} | (i128{static_cast<ui64>(0xc097ce7bc90715ul)} << 64), // 36
+ i128{static_cast<ui64>(0x00f4369ffffffffful)} | (i128{static_cast<ui64>(0x785ee10d5da46d9ul)} << 64), // 37
+ i128{static_cast<ui64>(0x098a223ffffffffful)} | (i128{static_cast<ui64>(0x4b3b4ca85a86c47aul)} << 64), // 38
+};
+
+static constexpr i256 Decimal256IntegerMaxValueTable[] = {
+ // 256 bits
+ //
+ // Generated by fair Python script:
+ //
+ // def print_max_decimal(precision):
+ // max_value = int("9" * precision)
+ // hex_value = hex(max_value)[2:] # strip 0x
+ // hex_value = hex_value.strip("L")
+ // parts = [hex_value[-8 * i:-8 * (i - 1) if i > 1 else len(hex_value)] or "0" for i in range(1, 9)]
+ // assert sum(int(v, 16) * 2 ** (32 * i) for i, v in enumerate(parts)) == max_value
+ // assert int(parts[-1], 16) < 2 ** 31 - 1
+ // joined_parts = ", ".join(f"static_cast<ui32>(0x{part}u)" for part in parts)
+ // print(f"{{{joined_parts}}}, // {precision}")
+ //
+ // for i in range(39, 77):
+ // print_max_decimal(i)
+ //
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x5f65567fu), static_cast<ui32>(0x8943acc4u), static_cast<ui32>(0xf050fe93u), static_cast<ui32>(0x2u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 39
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xb9f560ffu), static_cast<ui32>(0x5ca4bfabu), static_cast<ui32>(0x6329f1c3u), static_cast<ui32>(0x1du), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 40
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x4395c9ffu), static_cast<ui32>(0x9e6f7cb5u), static_cast<ui32>(0xdfa371a1u), static_cast<ui32>(0x125u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 41
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xa3d9e3ffu), static_cast<ui32>(0x305adf14u), static_cast<ui32>(0xbc627050u), static_cast<ui32>(0xb7au), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 42
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x6682e7ffu), static_cast<ui32>(0xe38cb6ceu), static_cast<ui32>(0x5bd86321u), static_cast<ui32>(0x72cbu), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 43
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x011d0fffu), static_cast<ui32>(0xe37f2410u), static_cast<ui32>(0x9673df52u), static_cast<ui32>(0x47bf1u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 44
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x0b229fffu), static_cast<ui32>(0xe2f768a0u), static_cast<ui32>(0xe086b93cu), static_cast<ui32>(0x2cd76fu), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 45
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x6f5a3fffu), static_cast<ui32>(0xddaa1640u), static_cast<ui32>(0xc5433c60u), static_cast<ui32>(0x1c06a5eu), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 46
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x59867fffu), static_cast<ui32>(0xa8a4de84u), static_cast<ui32>(0xb4a05bc8u), static_cast<ui32>(0x118427b3u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 47
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x7f40ffffu), static_cast<ui32>(0x9670b12bu), static_cast<ui32>(0x0e4395d6u), static_cast<ui32>(0xaf298d05u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 48
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xf889ffffu), static_cast<ui32>(0xe066ebb2u), static_cast<ui32>(0x8ea3da61u), static_cast<ui32>(0xd79f8232u), static_cast<ui32>(0x6u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 49
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xb563ffffu), static_cast<ui32>(0xc40534fdu), static_cast<ui32>(0x926687d2u), static_cast<ui32>(0x6c3b15f9u), static_cast<ui32>(0x44u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 50
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x15e7ffffu), static_cast<ui32>(0xa83411e9u), static_cast<ui32>(0xb8014e3bu), static_cast<ui32>(0x3a4edbbfu), static_cast<ui32>(0x2acu), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 51
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xdb0fffffu), static_cast<ui32>(0x9208b31au), static_cast<ui32>(0x300d0e54u), static_cast<ui32>(0x4714957du), static_cast<ui32>(0x1abau), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 52
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x8e9fffffu), static_cast<ui32>(0xb456ff0cu), static_cast<ui32>(0xe0828f4du), static_cast<ui32>(0xc6cdd6e3u), static_cast<ui32>(0x10b46u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 53
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x923fffffu), static_cast<ui32>(0x0b65f67du), static_cast<ui32>(0xc5199909u), static_cast<ui32>(0xc40a64e6u), static_cast<ui32>(0xa70c3u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 54
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xb67fffffu), static_cast<ui32>(0x71fba0e7u), static_cast<ui32>(0xb2fffa5au), static_cast<ui32>(0xa867f103u), static_cast<ui32>(0x6867a5u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 55
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x20ffffffu), static_cast<ui32>(0x73d4490du), static_cast<ui32>(0xfdffc788u), static_cast<ui32>(0x940f6a24u), static_cast<ui32>(0x4140c78u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 56
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x49ffffffu), static_cast<ui32>(0x864ada83u), static_cast<ui32>(0xebfdcb54u), static_cast<ui32>(0xc89a2571u), static_cast<ui32>(0x28c87cb5u), static_cast<ui32>(0x0u), static_cast<ui32>(0x0u)}, // 57
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xe3ffffffu), static_cast<ui32>(0x3eec8920u), static_cast<ui32>(0x37e9f14du), static_cast<ui32>(0xd6057673u), static_cast<ui32>(0x97d4df19u), static_cast<ui32>(0x1u), static_cast<ui32>(0x0u)}, // 58
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xe7ffffffu), static_cast<ui32>(0x753d5b48u), static_cast<ui32>(0x2f236d04u), static_cast<ui32>(0x5c36a080u), static_cast<ui32>(0xee50b702u), static_cast<ui32>(0xfu), static_cast<ui32>(0x0u)}, // 59
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x0fffffffu), static_cast<ui32>(0x946590d9u), static_cast<ui32>(0xd762422cu), static_cast<ui32>(0x9a224501u), static_cast<ui32>(0x4f272617u), static_cast<ui32>(0x9fu), static_cast<ui32>(0x0u)}, // 60
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x9fffffffu), static_cast<ui32>(0xcbf7a87au), static_cast<ui32>(0x69d695bdu), static_cast<ui32>(0x0556b212u), static_cast<ui32>(0x17877cecu), static_cast<ui32>(0x639u), static_cast<ui32>(0x0u)}, // 61
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x3fffffffu), static_cast<ui32>(0xf7ac94cau), static_cast<ui32>(0x2261d969u), static_cast<ui32>(0x3562f4b8u), static_cast<ui32>(0xeb4ae138u), static_cast<ui32>(0x3e3au), static_cast<ui32>(0x0u)}, // 62
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x7fffffffu), static_cast<ui32>(0xacbdcfe6u), static_cast<ui32>(0x57d27e23u), static_cast<ui32>(0x15dd8f31u), static_cast<ui32>(0x30eccc32u), static_cast<ui32>(0x26e4du), static_cast<ui32>(0x0u)}, // 63
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xbf6a1f00u), static_cast<ui32>(0x6e38ed64u), static_cast<ui32>(0xdaa797edu), static_cast<ui32>(0xe93ff9f4u), static_cast<ui32>(0x184f03u), static_cast<ui32>(0x0u)}, // 64
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x7a253609u), static_cast<ui32>(0x4e3945efu), static_cast<ui32>(0x8a8bef46u), static_cast<ui32>(0x1c7fc390u), static_cast<ui32>(0xf31627u), static_cast<ui32>(0x0u)}, // 65
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xc5741c63u), static_cast<ui32>(0x0e3cbb5au), static_cast<ui32>(0x697758bfu), static_cast<ui32>(0x1cfda3a5u), static_cast<ui32>(0x97edd87u), static_cast<ui32>(0x0u)}, // 66
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xb6891be7u), static_cast<ui32>(0x8e5f518bu), static_cast<ui32>(0x1ea97776u), static_cast<ui32>(0x21e86476u), static_cast<ui32>(0x5ef4a747u), static_cast<ui32>(0x0u)}, // 67
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x215b170fu), static_cast<ui32>(0x8fb92f75u), static_cast<ui32>(0x329eaaa1u), static_cast<ui32>(0x5313ec9du), static_cast<ui32>(0xb58e88c7u), static_cast<ui32>(0x3u)}, // 68
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x4d8ee69fu), static_cast<ui32>(0x9d3bda93u), static_cast<ui32>(0xfa32aa4fu), static_cast<ui32>(0x3ec73e23u), static_cast<ui32>(0x179157c9u), static_cast<ui32>(0x25u)}, // 69
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x0795023fu), static_cast<ui32>(0x245689c1u), static_cast<ui32>(0xc5faa71cu), static_cast<ui32>(0x73c86d67u), static_cast<ui32>(0xebad6ddcu), static_cast<ui32>(0x172u)}, // 70
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x4bd2167fu), static_cast<ui32>(0x6b61618au), static_cast<ui32>(0xbbca8719u), static_cast<ui32>(0x85d4460du), static_cast<ui32>(0x34c64a9cu), static_cast<ui32>(0xe7du)}, // 71
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xf634e0ffu), static_cast<ui32>(0x31cdcf66u), static_cast<ui32>(0x55e946feu), static_cast<ui32>(0x3a4abc89u), static_cast<ui32>(0x0fbeea1du), static_cast<ui32>(0x90e4u)}, // 72
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x9e10c9ffu), static_cast<ui32>(0xf20a1a05u), static_cast<ui32>(0x5b1cc5edu), static_cast<ui32>(0x46eb5d5du), static_cast<ui32>(0x9d752524u), static_cast<ui32>(0x5a8e8u)}, // 73
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x2ca7e3ffu), static_cast<ui32>(0x74650438u), static_cast<ui32>(0x8f1fbb4bu), static_cast<ui32>(0xc531a5a5u), static_cast<ui32>(0x2693736au), static_cast<ui32>(0x389916u)}, // 74
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xbe8ee7ffu), static_cast<ui32>(0x8bf22a31u), static_cast<ui32>(0x973d50f2u), static_cast<ui32>(0xb3f07877u), static_cast<ui32>(0x81c2822bu), static_cast<ui32>(0x235faddu)}, // 75
+ {static_cast<ui32>(0xffffffffu), static_cast<ui32>(0xffffffffu), static_cast<ui32>(0x71950fffu), static_cast<ui32>(0x7775a5f1u), static_cast<ui32>(0xe8652979u), static_cast<ui32>(0x0764b4abu), static_cast<ui32>(0x119915b5u), static_cast<ui32>(0x161bcca7u)}, // 76
};
template<typename T>
-Y_FORCE_INLINE constexpr T GetDecimalMaxIntegerValue(int precision)
+Y_FORCE_INLINE constexpr auto GetDecimalMaxIntegerValue(int precision)
{
static_assert(ValidDecimalUnderlyingInteger<T>);
- if (TDecimal::GetValueBinarySize(precision) <= sizeof(T)) {
- return DecimalIntegerMaxValueTable[precision];
- } else {
+ if (TDecimal::GetValueBinarySize(precision) > static_cast<int>(sizeof(T))) {
YT_ABORT();
}
-}
-template <typename T>
-Y_FORCE_INLINE constexpr auto DecimalIntegerToUnsigned(T value)
-{
- static_assert(ValidDecimalUnderlyingInteger<T>);
-
- if constexpr (std::is_same_v<T, i128>) {
- return ui128(value);
+ if constexpr (std::is_same_v<T, i32>) {
+ YT_VERIFY(precision <= 9);
+ return Decimal32IntegerMaxValueTable[precision];
+ } else if constexpr (std::is_same_v<T, i64>) {
+ YT_VERIFY(precision >= 10 && precision <= 18);
+ return Decimal64IntegerMaxValueTable[precision - 10];
+ } else if constexpr (std::is_same_v<T, i128>) {
+ YT_VERIFY(precision >= 19 && precision <= 38);
+ return Decimal128IntegerMaxValueTable[precision - 19];
+ } else if constexpr (std::is_same_v<T, i256>) {
+ YT_VERIFY(precision >= 39 && precision <= 76);
+ return Decimal256IntegerMaxValueTable[precision - 39];
} else {
- using TU = std::make_unsigned_t<T>;
- return static_cast<TU>(value);
+ YT_ABORT();
}
}
template <typename T>
static Y_FORCE_INLINE T DecimalHostToInet(T value)
{
- if constexpr (std::is_same_v<T, i128> || std::is_same_v<T, ui128>) {
+ if constexpr (std::is_same_v<T, i256> || std::is_same_v<T, ui256>) {
+ for (int partIndex = 0; partIndex < std::ssize(value.Parts) / 2; ++partIndex) {
+ value.Parts[partIndex] = ::HostToInet(value.Parts[partIndex]);
+ value.Parts[std::size(value.Parts) - 1 - partIndex] = ::HostToInet(value.Parts[std::size(value.Parts) - 1 - partIndex]);
+ std::swap(value.Parts[partIndex], value.Parts[std::size(value.Parts) - 1 - partIndex]);
+ }
+ return value;
+ } else if constexpr (std::is_same_v<T, i128> || std::is_same_v<T, ui128>) {
return T(::HostToInet(GetLow(value)), ::HostToInet(GetHigh(value)));
} else {
return ::HostToInet(value);
@@ -139,7 +454,14 @@ static Y_FORCE_INLINE T DecimalHostToInet(T value)
template <typename T>
static Y_FORCE_INLINE T DecimalInetToHost(T value)
{
- if constexpr (std::is_same_v<T, i128> || std::is_same_v<T, ui128>) {
+ if constexpr (std::is_same_v<T, i256> || std::is_same_v<T, ui256>) {
+ for (int partIndex = 0; partIndex < std::ssize(value.Parts) / 2; ++partIndex) {
+ value.Parts[partIndex] = ::InetToHost(value.Parts[partIndex]);
+ value.Parts[std::size(value.Parts) - 1 - partIndex] = ::InetToHost(value.Parts[std::size(value.Parts) - 1 - partIndex]);
+ std::swap(value.Parts[partIndex], value.Parts[std::size(value.Parts) - 1 - partIndex]);
+ }
+ return value;
+ } else if constexpr (std::is_same_v<T, i128> || std::is_same_v<T, ui128>) {
return T(::InetToHost(GetLow(value)), ::InetToHost(GetHigh(value)));
} else {
return ::InetToHost(value);
@@ -151,22 +473,14 @@ static T DecimalBinaryToIntegerUnchecked(TStringBuf binaryValue)
{
T result;
memcpy(&result, binaryValue.data(), sizeof(result));
- result = DecimalInetToHost(result);
-
- constexpr auto one = DecimalIntegerToUnsigned(T{1});
- result = static_cast<T>(DecimalIntegerToUnsigned(result) ^ (one << (sizeof(T) * 8 - 1)));
-
- return result;
+ return FlipMSB(DecimalInetToHost(result));
}
template<typename T>
static void DecimalIntegerToBinaryUnchecked(T decodedValue, void* buf)
{
- auto unsignedValue = DecimalIntegerToUnsigned(decodedValue);
- constexpr auto one = DecimalIntegerToUnsigned(T{1});
- unsignedValue ^= (one << (sizeof(T) * 8 - 1));
- unsignedValue = DecimalHostToInet(unsignedValue);
- memcpy(buf, &unsignedValue, sizeof(unsignedValue));
+ auto preparedValue = DecimalHostToInet(FlipMSB(decodedValue));
+ memcpy(buf, &preparedValue, sizeof(preparedValue));
}
static void CheckDecimalValueSize(TStringBuf value, int precision, int scale)
@@ -192,12 +506,6 @@ static Y_FORCE_INLINE TStringBuf PlaceOnBuffer(TStringBuf value, char* buffer)
template<typename T>
static TStringBuf WriteTextDecimalUnchecked(T decodedValue, int scale, char* buffer)
{
- i8 digits[std::numeric_limits<T>::digits + 1] = {0,};
- static constexpr auto ten = DecimalIntegerToUnsigned(T{10});
-
- const bool negative = decodedValue < 0;
- auto absValue = DecimalIntegerToUnsigned(negative ? -decodedValue : decodedValue);
-
if (decodedValue == TDecimalTraits<T>::MinusInf) {
static constexpr TStringBuf minusInf = "-inf";
return PlaceOnBuffer(minusInf, buffer);
@@ -209,10 +517,14 @@ static TStringBuf WriteTextDecimalUnchecked(T decodedValue, int scale, char* buf
return PlaceOnBuffer(nan, buffer);
}
+ i8 digits[TDecimal::MaxTextSize] = {0,};
+
+ bool negative = IsNegativeInteger(decodedValue);
+ auto absValue = DecimalIntegerToUnsigned(negative ? -decodedValue : decodedValue);
+
auto* curDigit = digits;
- while (absValue > 0) {
- *curDigit = static_cast<int>(absValue % ten);
- absValue = absValue / ten;
+ while (absValue != DecimalIntegerToUnsigned(T{0})) {
+ *curDigit = GetNextDigit(absValue, &absValue);
curDigit++;
}
YT_VERIFY(curDigit <= digits + std::size(digits));
@@ -313,35 +625,40 @@ T DecimalTextToInteger(TStringBuf textValue, int precision, int scale)
break;
}
- T result = 0;
+ // This value can overflow in the process of parsing the text value.
+ // We do throw an exception later, but UB is not good for your health.
+ auto result = DecimalIntegerToUnsigned(T{0});
int beforePoint = 0;
int afterPoint = 0;
+
+ auto addDigit = [&] (auto digit) {
+ // We use this type to avoid warnings about casting signed types to unsigned/narrowing ints.
+ // Ugly, but this way we don't need to define cumbersome constructors for TValue256.
+ ui16 currentDigit = *digit - '0';
+ if (currentDigit < 0 || currentDigit > 9) {
+ ThrowInvalidDecimal(textValue, precision, scale);
+ }
+
+ result = MultiplyByTen(result);
+ result = result + DecimalIntegerToUnsigned(T{currentDigit});
+ };
+
for (; cur != end; ++cur) {
if (*cur == '.') {
++cur;
for (; cur != end; ++cur) {
- int currentDigit = *cur - '0';
- result *= 10;
- result += currentDigit;
+ addDigit(cur);
++afterPoint;
- if (currentDigit < 0 || currentDigit > 9) {
- ThrowInvalidDecimal(textValue, precision, scale);
- }
}
break;
}
- int currentDigit = *cur - '0';
- result *= 10;
- result += currentDigit;
+ addDigit(cur);
++beforePoint;
- if (currentDigit < 0 || currentDigit > 9) {
- ThrowInvalidDecimal(textValue, precision, scale);
- }
}
for (; afterPoint < scale; ++afterPoint) {
- result *= 10;
+ result = MultiplyByTen(result);
}
if (afterPoint > scale) {
@@ -352,7 +669,10 @@ T DecimalTextToInteger(TStringBuf textValue, int precision, int scale)
ThrowInvalidDecimal(textValue, precision, scale, "too many digits before decimal point");
}
- return negative ? -result : result;
+ // This cast is guaranteed by the checks above to be correct.
+ auto signedResult = DecimalIntegerToSigned(result);
+ // This is safe: the range of representable values fits into [-signed_max, signed_max] for each underlying type.
+ return negative ? -signedResult : signedResult;
}
template<typename T>
@@ -374,6 +694,8 @@ TStringBuf TDecimal::BinaryToText(TStringBuf binaryDecimal, int precision, int s
return DecimalBinaryToTextUncheckedImpl<i64>(binaryDecimal, scale, buffer);
case 16:
return DecimalBinaryToTextUncheckedImpl<i128>(binaryDecimal, scale, buffer);
+ case 32:
+ return DecimalBinaryToTextUncheckedImpl<i256>(binaryDecimal, scale, buffer);
}
CheckDecimalValueSize(binaryDecimal, precision, scale);
YT_ABORT();
@@ -410,8 +732,10 @@ TStringBuf TDecimal::TextToBinary(TStringBuf textValue, int precision, int scale
return TextToBinaryImpl<i64>(textValue, precision, scale, buffer);
case 16:
return TextToBinaryImpl<i128>(textValue, precision, scale, buffer);
+ case 32:
+ return TextToBinaryImpl<i256>(textValue, precision, scale, buffer);
default:
- static_assert(GetDecimalBinaryValueSize(TDecimal::MaxPrecision) == 16);
+ static_assert(GetDecimalBinaryValueSize(TDecimal::MaxPrecision) == 32);
YT_ABORT();
}
}
@@ -444,7 +768,7 @@ static void ValidateDecimalBinaryValueImpl(TStringBuf binaryDecimal, int precisi
{
T decoded = DecimalBinaryToIntegerUnchecked<T>(binaryDecimal);
- const T maxValue = static_cast<T>(DecimalIntegerMaxValueTable[precision]);
+ auto maxValue = GetDecimalMaxIntegerValue<T>(precision);
if (-maxValue <= decoded && decoded <= maxValue) {
return;
@@ -478,11 +802,14 @@ void TDecimal::ValidateBinaryValue(TStringBuf binaryDecimal, int precision, int
return ValidateDecimalBinaryValueImpl<i64>(binaryDecimal, precision, scale);
case 16:
return ValidateDecimalBinaryValueImpl<i128>(binaryDecimal, precision, scale);
+ case 32:
+ return ValidateDecimalBinaryValueImpl<i256>(binaryDecimal, precision, scale);
default:
- static_assert(GetDecimalBinaryValueSize(TDecimal::MaxPrecision) == 16);
+ static_assert(GetDecimalBinaryValueSize(TDecimal::MaxPrecision) == 32);
YT_ABORT();
}
}
+
template <typename T>
Y_FORCE_INLINE void CheckDecimalIntBits(int precision)
{
@@ -495,6 +822,14 @@ Y_FORCE_INLINE void CheckDecimalIntBits(int precision)
}
}
+Y_FORCE_INLINE void CheckDecimalFitsInto128Bits(int precision)
+{
+ if (precision > 38) {
+ THROW_ERROR_EXCEPTION("Decimal<%v, ?> does not fit into int128",
+ precision);
+ }
+}
+
int TDecimal::GetValueBinarySize(int precision)
{
const auto result = GetDecimalBinaryValueSize(precision);
@@ -535,7 +870,7 @@ TStringBuf TDecimal::WriteBinary128(int precision, TValue128 value, char* buffer
return TStringBuf{buffer, sizeof(TValue128)};
}
-TStringBuf TDecimal::WriteBinaryVariadic(int precision, TValue128 value, char* buffer, size_t bufferLength)
+TStringBuf TDecimal::WriteBinary128Variadic(int precision, TValue128 value, char* buffer, size_t bufferLength)
{
const size_t resultLength = GetValueBinarySize(precision);
switch (resultLength) {
@@ -550,6 +885,16 @@ TStringBuf TDecimal::WriteBinaryVariadic(int precision, TValue128 value, char* b
}
}
+TStringBuf TDecimal::WriteBinary256(int precision, TValue256 value, char* buffer, size_t bufferLength)
+{
+ const size_t resultLength = GetValueBinarySize(precision);
+ CheckDecimalIntBits<TValue256>(precision);
+ YT_VERIFY(bufferLength >= resultLength);
+
+ DecimalIntegerToBinaryUnchecked(std::move(value), buffer);
+ return TStringBuf{buffer, sizeof(TValue256)};
+}
+
template <typename T>
Y_FORCE_INLINE void CheckBufferLength(int precision, size_t bufferLength)
{
@@ -581,6 +926,12 @@ TDecimal::TValue128 TDecimal::ParseBinary128(int precision, TStringBuf buffer)
return {GetLow(result), static_cast<i64>(GetHigh(result))};
}
+TDecimal::TValue256 TDecimal::ParseBinary256(int precision, TStringBuf buffer)
+{
+ CheckBufferLength<i256>(precision, buffer.Size());
+ return DecimalBinaryToIntegerUnchecked<i256>(buffer);
+}
+
////////////////////////////////////////////////////////////////////////////////
} // namespace NYT::NDecimal
diff --git a/yt/yt/library/decimal/decimal.h b/yt/yt/library/decimal/decimal.h
index 8a6cf34a8f..27375d3904 100644
--- a/yt/yt/library/decimal/decimal.h
+++ b/yt/yt/library/decimal/decimal.h
@@ -5,6 +5,8 @@
#include <util/system/defaults.h>
#include <util/generic/string.h>
+#include <array>
+
namespace NYT::NDecimal {
////////////////////////////////////////////////////////////////////////////////
@@ -12,6 +14,9 @@ namespace NYT::NDecimal {
class TDecimal
{
public:
+ //! Both types defined below represent signed values. They are only intended
+ //! to be used for manipulating binary data, since they don't actually expose
+ //! any arithmetic operations. You should avoid dealing with individual parts.
struct TValue128
{
ui64 Low;
@@ -19,15 +24,21 @@ public:
};
static_assert(sizeof(TValue128) == 2 * sizeof(ui64));
+ struct TValue256
+ {
+ std::array<ui32, 8> Parts;
+ };
+ static_assert(sizeof(TValue256) == 4 * sizeof(ui64));
+
public:
- // Maximum precision supported by YT
- static constexpr int MaxPrecision = 35;
- static constexpr int MaxBinarySize = 16;
+ //! Maximum precision supported by YT.
+ static constexpr int MaxPrecision = 76;
+ static constexpr int MaxBinarySize = 32;
// NB. Sometimes we print values that exceed MaxPrecision (e.g. in error messages)
- // MaxTextSize is chosen so we can print ANY i128 number as decimal.
+ // MaxTextSize is chosen so we can print ANY i256 number as decimal.
static constexpr int MaxTextSize =
- std::numeric_limits<ui128>::digits + 1 // max number of digits in ui128 number
+ 77 // length of 2^63 in decimal form
+ 1 // possible decimal point
+ 1; // possible minus sign
@@ -49,13 +60,15 @@ public:
static TStringBuf WriteBinary32(int precision, i32 value, char* buffer, size_t bufferLength);
static TStringBuf WriteBinary64(int precision, i64 value, char* buffer, size_t bufferLength);
static TStringBuf WriteBinary128(int precision, TValue128 value, char* buffer, size_t bufferLength);
+ static TStringBuf WriteBinary256(int precision, TValue256 value, char* buffer, size_t bufferLength);
// Writes either 32-bit, 64-bit or 128-bit binary value depending on precision, provided a TValue128.
- static TStringBuf WriteBinaryVariadic(int precision, TValue128 value, char* buffer, size_t bufferLength);
+ static TStringBuf WriteBinary128Variadic(int precision, TValue128 value, char* buffer, size_t bufferLength);
static i32 ParseBinary32(int precision, TStringBuf buffer);
static i64 ParseBinary64(int precision, TStringBuf buffer);
static TValue128 ParseBinary128(int precision, TStringBuf buffer);
+ static TValue256 ParseBinary256(int precision, TStringBuf buffer);
};
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/library/decimal/unittests/decimal_ut.cpp b/yt/yt/library/decimal/unittests/decimal_ut.cpp
index ba8be4ab15..50c550707c 100644
--- a/yt/yt/library/decimal/unittests/decimal_ut.cpp
+++ b/yt/yt/library/decimal/unittests/decimal_ut.cpp
@@ -98,6 +98,150 @@ TEST(TDecimal, TestTextBinaryConversion)
"61449825198266175750309883089040771",
"800BD5B5D5F0C73E0C9CD4943298B583");
+ // A few more test cases with big numbers for various precisions generated by python snippet:
+ // import random
+ // def print_test_case(plus, binary_size, precision):
+ // textval = "".join(random.choice("0123456789") for _ in range(precision))
+ // if not plus:
+ // textval = "-" + textval
+ // binval = hex(2 ** (binary_size * 8 - 1) + int(textval))
+ // binval = binval[2:].strip('L') # strip 0x and final 'L'
+ // binval = binval.upper()
+ // print(
+ // "TEST_TEXT_BINARY_CONVERSION(\n"
+ // " {precision}, 0,\n"
+ // " \"{text}\",\n"
+ // " \"{binary}\");\n"
+ // .format(precision=precision, text=textval.lstrip("0"), binary=binval)
+ // )
+ // random.seed(42)
+ //
+ // for binary_size, precision in (4, 9), (8, 18), (16, 35), (16, 38), (32, 39), (32, 76):
+ // print_test_case(False, binary_size, precision)
+ // print_test_case(False, binary_size, precision)
+ // print_test_case(True, binary_size, precision)
+ // print_test_case(True, binary_size, precision)
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 9, 0,
+ "-104332181",
+ "79C8046B");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 9, 0,
+ "-960013389",
+ "46C75BB3");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 9, 0,
+ "83863794",
+ "84FFA8F2");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 9, 0,
+ "26542351",
+ "8195010F");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 18, 0,
+ "-161559407816184959",
+ "7DC2069316FE6B81");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 18, 0,
+ "-310341316475255341",
+ "7BB17237885D85D3");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 18, 0,
+ "928327648350305641",
+ "8CE21513E317FD69");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 18, 0,
+ "395376724238849696",
+ "857CA90930B6D6A0");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 35, 0,
+ "-53287101226916697848018451462704828",
+ "7FF5BCBE39B5F1A05ED5F0135FD2B144");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 35, 0,
+ "-14893252880957015430391171822782489",
+ "7FFD21B4B88705D84E5A0ECF560DF7E7");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 35, 0,
+ "63834657871331509839301031051834738",
+ "800C4B4AA7E81EF6726D934AA12AB572");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 35, 0,
+ "29973763116566701065133387262473178",
+ "8005C5D2141747EF1198888FB96033DA");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 38, 0,
+ "-10801326773602606474687234309805009788",
+ "77DFBD795731FA57AD40A913BEB49484");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 38, 0,
+ "-20812191361939909169985435346247510799",
+ "7057B7BE1F13B6F1D6DB2F569BA994F1");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 38, 0,
+ "11838425135427849808412411824493534874",
+ "88E7FF6C4CB25E5053056568535B9E9A");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 38, 0,
+ "1640052427868011280598262045053315869",
+ "813BDCD3E2BEF11A4624071A88406B1D");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 39, 0,
+ "-232260256342160733754330365414586850142",
+ "7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF51444D3EF60040D9A177B4EBDCD1E4A2");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 39, 0,
+ "-940196556981693406088356159514846564823",
+ "7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD3CACBACF9473427D637FA1697111EE29");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 39, 0,
+ "662994680443699577738721489513433200379",
+ "80000000000000000000000000000001F2C8217C5832413CEDC2D8713542F6FB");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 39, 0,
+ "176936763201632870831727889579868727743",
+ "80000000000000000000000000000000851CC7F2FA54AF2AC8A1868AB524A9BF");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 76, 0,
+ "-4873471434558122362316658760366909670546688937346706562729806990162720465375",
+ "7539B681CC20CEEDAB96C358A49B105EB5C51C958B66D69F946E52B2C4CBDE21");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 76, 0,
+ "-5646417080531003309232719374529912419049663193149190586518506716572628498776",
+ "73843DBE5B00AA3065C94D18C3DE5B54ABBCB7AD4AB9775BD7308BA12A14C6A8");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 76, 0,
+ "9453147379965075273545494808313678377701436349578856855744431351823374989413",
+ "94E64AB40D1C30A9A2BFEA9C96B7584F559D5572641F0C90E86764F3187BB865");
+
+ TEST_TEXT_BINARY_CONVERSION(
+ 76, 0,
+ "4352408240084271094777520471167190229413186999386774964990913341232812067974",
+ "899F603224EC624E2351628FF4637023C84FDBB0E2BBCBBB9395D8DD19B30C86");
+
#undef TEST_TEXT_BINARY_CONVERSION
}
@@ -127,20 +271,27 @@ TEST(TDecimal, TestPrecisionScaleLimits)
EXPECT_THROW_WITH_SUBSTRING(TDecimal::TextToBinary("314.15", 5, 3), "too many digits before decimal point");
EXPECT_THROW_WITH_SUBSTRING(TDecimal::TextToBinary("-314.15", 5, 3), "too many digits before decimal point");
+ // This group of tests checks that text values which cause signed overflow throw valid exceptions.
+ // The values for each precision are equal to 2^(binary_size - 1) + 0.11.
+ EXPECT_THROW_WITH_SUBSTRING(TDecimal::TextToBinary("2147483647.11", 9, 2), "too many digits before decimal point");
+ EXPECT_THROW_WITH_SUBSTRING(TDecimal::TextToBinary("9223372036854775807.11", 18, 2), "too many digits before decimal point");
+ EXPECT_THROW_WITH_SUBSTRING(TDecimal::TextToBinary("170141183460469231731687303715884105727.11", 35, 2), "too many digits before decimal point");
+ EXPECT_THROW_WITH_SUBSTRING(TDecimal::TextToBinary("57896044618658097711785492504343953926634992332820282019728792003956564819967.11", TDecimal::MaxPrecision, 2), "too many digits before decimal point");
+
// Sometimes we want to print values that are not representable with given precision
// (e.g. in error messages we sometimes want to print text value of invalid decimal to explain that it has
// more digits than allowed by precision).
//
// Here we test that extreme values are printed ok.
- auto maxBinaryDecimal = HexDecode("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD");
- auto minBinaryDecimal1 = HexDecode("00000000000000000000000000000000");
- auto minBinaryDecimal2 = HexDecode("00000000000000000000000000000003");
+ auto maxBinaryDecimal = HexDecode("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD");
+ auto minBinaryDecimal1 = HexDecode("0000000000000000000000000000000000000000000000000000000000000000");
+ auto minBinaryDecimal2 = HexDecode("0000000000000000000000000000000000000000000000000000000000000003");
EXPECT_EQ(TDecimal::MaxBinarySize, std::ssize(maxBinaryDecimal)); // If max TDecimal::MaxBinarySize ever increases
EXPECT_EQ(TDecimal::MaxBinarySize, std::ssize(minBinaryDecimal1)); // please update this test
EXPECT_EQ(TDecimal::MaxBinarySize, std::ssize(minBinaryDecimal2)); // with better values.
- EXPECT_EQ("1701411834604692317316873037158841057.25", TDecimal::BinaryToText(maxBinaryDecimal, TDecimal::MaxPrecision, 2));
- EXPECT_EQ("-1701411834604692317316873037158841057.28", TDecimal::BinaryToText(minBinaryDecimal1, TDecimal::MaxPrecision, 2));
- EXPECT_EQ("-1701411834604692317316873037158841057.25", TDecimal::BinaryToText(minBinaryDecimal2, TDecimal::MaxPrecision, 2));
+ EXPECT_EQ("578960446186580977117854925043439539266349923328202820197287920039565648199.65", TDecimal::BinaryToText(maxBinaryDecimal, TDecimal::MaxPrecision, 2));
+ EXPECT_EQ("-578960446186580977117854925043439539266349923328202820197287920039565648199.68", TDecimal::BinaryToText(minBinaryDecimal1, TDecimal::MaxPrecision, 2));
+ EXPECT_EQ("-578960446186580977117854925043439539266349923328202820197287920039565648199.65", TDecimal::BinaryToText(minBinaryDecimal2, TDecimal::MaxPrecision, 2));
}
TEST(TDecimal, TestValidation)
@@ -148,6 +299,7 @@ TEST(TDecimal, TestValidation)
EXPECT_NO_THROW(TDecimal::ValidateBinaryValue(HexDecode("8000013A"), 3, 2));
EXPECT_NO_THROW(TDecimal::ValidateBinaryValue(HexDecode("80000000" "0000013A"), 10, 2));
EXPECT_NO_THROW(TDecimal::ValidateBinaryValue(HexDecode("80000000" "00000000" "00000000" "0000013A"), 35, 2));
+ EXPECT_NO_THROW(TDecimal::ValidateBinaryValue(HexDecode("80000000" "00000000" "00000000" "00000000" "00000000" "00000000" "00000000" "0000013A"), 76, 2));
}
class TDecimalWithPrecisionTest
diff --git a/yt/yt/library/formats/arrow_parser.cpp b/yt/yt/library/formats/arrow_parser.cpp
index cc31e25335..f34127990a 100644
--- a/yt/yt/library/formats/arrow_parser.cpp
+++ b/yt/yt/library/formats/arrow_parser.cpp
@@ -163,28 +163,17 @@ public:
return ParseNull();
}
- // Decimal types. For now, YT natively supports only Decimal128 with scale up to 35.
- // Thus, we represent short enough decimals as native YT decimals, and wider decimals as
- // their decimal string representation; but the latter is subject to change whenever we
- // get the native support for Decimal128 with scale up to 38 or Decimal256 with scale up to 76.
arrow::Status Visit(const arrow::Decimal128Type& type) override
{
- constexpr int MaximumYTDecimalPrecision = 35;
- if (type.precision() <= MaximumYTDecimalPrecision) {
- return ParseStringLikeArray<arrow::Decimal128Array>([&] (const TStringBuf& value, i64 columnId) {
- return MakeDecimalBinaryValue(value, columnId, type.precision());
- });
- } else {
- return ParseStringLikeArray<arrow::Decimal128Array>([&] (const TStringBuf& value, i64 columnId) {
- return MakeDecimalTextValue<arrow::Decimal128>(value, columnId, type.scale());
- });
- }
+ return ParseStringLikeArray<arrow::Decimal128Array>([&] (const TStringBuf& value, i64 columnId) {
+ return MakeDecimalBinaryValue<TDecimal::TValue128>(value, columnId, type.precision());
+ });
}
arrow::Status Visit(const arrow::Decimal256Type& type) override
{
return ParseStringLikeArray<arrow::Decimal256Array>([&] (const TStringBuf& value, i64 columnId) {
- return MakeDecimalTextValue<arrow::Decimal256>(value, columnId, type.scale());
+ return MakeDecimalBinaryValue<TDecimal::TValue256>(value, columnId, type.precision());
});
}
@@ -294,33 +283,31 @@ private:
return arrow::Status::OK();
}
+ template <class TUnderlyingValueType>
TUnversionedValue MakeDecimalBinaryValue(const TStringBuf& value, i64 columnId, int precision)
{
- // NB: arrow wire representation of Decimal128 is little-endian and (obviously) 128 bit,
+ // NB: Arrow wire representation of Decimal128 is little-endian and (obviously) 128 bit,
// while YT in-memory representation of Decimal is big-endian, variadic-length of either 32 bit, 64 bit or 128 bit,
// and MSB-flipped to ensure lexical sorting order.
- TDecimal::TValue128 value128;
- YT_VERIFY(value.size() == sizeof(value128));
- std::memcpy(&value128, value.data(), value.size());
+ // Representation of Decimal256 is similar, but only 256 bits.
+ TUnderlyingValueType decimalValue;
+ YT_VERIFY(value.size() == sizeof(decimalValue));
+ std::memcpy(&decimalValue, value.data(), value.size());
- const auto maxByteCount = sizeof(value128);
+ const auto maxByteCount = sizeof(decimalValue);
char* buffer = BufferForStringLikeValues_->Preallocate(maxByteCount);
- auto decimalBinary = TDecimal::WriteBinaryVariadic(precision, value128, buffer, maxByteCount);
+ TStringBuf decimalBinary;
+ if constexpr (std::is_same_v<TUnderlyingValueType, TDecimal::TValue128>) {
+ decimalBinary = TDecimal::WriteBinary128Variadic(precision, decimalValue, buffer, maxByteCount);
+ } else if constexpr (std::is_same_v<TUnderlyingValueType, TDecimal::TValue256>) {
+ decimalBinary = TDecimal::WriteBinary256(precision, decimalValue, buffer, maxByteCount);
+ } else {
+ static_assert(std::is_same_v<TUnderlyingValueType, TDecimal::TValue256>, "Unexpected decimal type");
+ }
BufferForStringLikeValues_->Advance(decimalBinary.size());
return MakeUnversionedStringValue(decimalBinary, columnId);
}
-
- template <class TArrowDecimalType>
- TUnversionedValue MakeDecimalTextValue(const TStringBuf& value, i64 columnId, int scale)
- {
- TArrowDecimalType decimal(reinterpret_cast<const uint8_t*>(value.data()));
- auto string = decimal.ToString(scale);
- char* buffer = BufferForStringLikeValues_->Preallocate(string.size());
- std::memcpy(buffer, string.data(), string.size());
- BufferForStringLikeValues_->Advance(string.size());
- return MakeUnversionedStringValue(TStringBuf(buffer, string.size()), columnId);
- }
};
////////////////////////////////////////////////////////////////////////////////
diff --git a/yt/yt/library/formats/skiff_parser.cpp b/yt/yt/library/formats/skiff_parser.cpp
index edff9ffa0b..febde44f42 100644
--- a/yt/yt/library/formats/skiff_parser.cpp
+++ b/yt/yt/library/formats/skiff_parser.cpp
@@ -300,6 +300,7 @@ const auto precision = denullifiedType.GetPrecision();
CASE(EWireType::Int32);
CASE(EWireType::Int64);
CASE(EWireType::Int128);
+ CASE(EWireType::Int256);
#undef CASE
case EWireType::Yson32:
return CreatePrimitiveTypeConverter(wireType, fieldDescription.IsRequired(), columnId, ysonConverter);
diff --git a/yt/yt/library/formats/skiff_writer.cpp b/yt/yt/library/formats/skiff_writer.cpp
index 3bc6965219..0984e72c8f 100644
--- a/yt/yt/library/formats/skiff_writer.cpp
+++ b/yt/yt/library/formats/skiff_writer.cpp
@@ -506,6 +506,10 @@ TUnversionedValueToSkiffConverter CreateDecimalValueConverter(
return CreatePrimitiveValueConverter<EValueType::String>(
isRequired,
TDecimalSkiffWriter<EWireType::Int128>(precision));
+ case EWireType::Int256:
+ return CreatePrimitiveValueConverter<EValueType::String>(
+ isRequired,
+ TDecimalSkiffWriter<EWireType::Int256>(precision));
case EWireType::Yson32:
return CreatePrimitiveValueConverter(wireType, isRequired);
default:
diff --git a/yt/yt/library/formats/skiff_yson_converter-inl.h b/yt/yt/library/formats/skiff_yson_converter-inl.h
index 43dabc63a5..8d5a2f8efd 100644
--- a/yt/yt/library/formats/skiff_yson_converter-inl.h
+++ b/yt/yt/library/formats/skiff_yson_converter-inl.h
@@ -70,8 +70,18 @@ Y_FORCE_INLINE TStringBuf TDecimalSkiffParser<SkiffWireType>::operator() (NSkiff
TDecimal::TValue128{skiffValue.Low, skiffValue.High},
Buffer_,
sizeof(Buffer_));
+ } else if constexpr (SkiffWireType == EWireType::Int256) {
+ const auto skiffValue = parser->ParseInt256();
+ TDecimal::TValue256 decimalValue;
+ static_assert(sizeof(decimalValue) == sizeof(skiffValue));
+ std::memcpy(&decimalValue, &skiffValue, sizeof(decimalValue));
+ return TDecimal::WriteBinary256(
+ Precision_,
+ std::move(decimalValue),
+ Buffer_,
+ sizeof(Buffer_));
} else {
- static_assert(SkiffWireType == EWireType::Int128);
+ static_assert(SkiffWireType == EWireType::Int256);
}
}
@@ -99,9 +109,15 @@ void TDecimalSkiffWriter<SkiffWireType>::operator()(TStringBuf value, NSkiff::TC
} else if constexpr (SkiffWireType == EWireType::Int128) {
auto intValue = TDecimal::ParseBinary128(Precision_, value);
writer->WriteInt128(TInt128{intValue.Low, intValue.High});
+ } else if constexpr (SkiffWireType == EWireType::Int256) {
+ auto intValue = TDecimal::ParseBinary256(Precision_, value);
+ TInt256 skiffValue;
+ static_assert(sizeof(skiffValue) == sizeof(intValue));
+ std::memcpy(&skiffValue, &intValue, sizeof(skiffValue));
+ writer->WriteInt256(std::move(skiffValue));
} else {
// poor man's static_assert(false)
- static_assert(SkiffWireType == EWireType::Int128);
+ static_assert(SkiffWireType == EWireType::Int256);
}
}
diff --git a/yt/yt/library/formats/skiff_yson_converter.cpp b/yt/yt/library/formats/skiff_yson_converter.cpp
index 82fac3ef1b..ebde4dcb3d 100644
--- a/yt/yt/library/formats/skiff_yson_converter.cpp
+++ b/yt/yt/library/formats/skiff_yson_converter.cpp
@@ -748,6 +748,10 @@ TYsonToSkiffConverter CreateDecimalYsonToSkiffConverter(
return CreatePrimitiveTypeYsonToSkiffConverter<EYsonItemType::StringValue>(
std::move(descriptor),
TDecimalSkiffWriter<EWireType::Int128>(precision));
+ case EWireType::Int256:
+ return CreatePrimitiveTypeYsonToSkiffConverter<EYsonItemType::StringValue>(
+ std::move(descriptor),
+ TDecimalSkiffWriter<EWireType::Int256>(precision));
case EWireType::Yson32:
return CreatePrimitiveTypeYsonToSkiffConverter(std::move(descriptor), wireType);
default:
@@ -1815,6 +1819,8 @@ TSkiffToYsonConverter CreateDecimalSkiffToYsonConverter(
return TPrimitiveTypeSkiffToYsonConverter(TDecimalSkiffParser<EWireType::Int64>(precision));
case EWireType::Int128:
return TPrimitiveTypeSkiffToYsonConverter(TDecimalSkiffParser<EWireType::Int128>(precision));
+ case EWireType::Int256:
+ return TPrimitiveTypeSkiffToYsonConverter(TDecimalSkiffParser<EWireType::Int256>(precision));
case EWireType::Yson32:
return CreatePrimitiveTypeSkiffToYsonConverter(wireType);
default:
@@ -1899,6 +1905,8 @@ void CheckSkiffWireTypeForDecimal(int precision, NSkiff::EWireType wireType)
skiffBinarySize = sizeof(i64);
} else if (wireType == NSkiff::EWireType::Int128) {
skiffBinarySize = 2 * sizeof(i64);
+ } else if (wireType == NSkiff::EWireType::Int256) {
+ skiffBinarySize = 4 * sizeof(i64);
}
if (decimalBinarySize != skiffBinarySize) {