diff options
author | whatsername <whatsername@yandex-team.com> | 2024-03-21 12:38:52 +0300 |
---|---|---|
committer | whatsername <whatsername@yandex-team.com> | 2024-03-21 12:49:42 +0300 |
commit | f2d8ab4111c1f5dba066eb9df8a2ecd86cd8c192 (patch) | |
tree | 73ac9be80fb07e711b489a5ee6bcf2fcdda0f667 | |
parent | 345654a416ff6d02ba04db39de7e1965b47aa844 (diff) | |
download | ydb-f2d8ab4111c1f5dba066eb9df8a2ecd86cd8c192.tar.gz |
YT-18458: Introduce wide types into mapreduce interface
7ae047ef618cc44d7dd3e817dc27f2336d9e38c3
-rw-r--r-- | library/cpp/type_info/fwd.h | 12 | ||||
-rw-r--r-- | library/cpp/type_info/type.cpp | 132 | ||||
-rw-r--r-- | library/cpp/type_info/type.h | 198 | ||||
-rw-r--r-- | library/cpp/type_info/type_complexity.cpp | 4 | ||||
-rw-r--r-- | library/cpp/type_info/type_constructors.h | 12 | ||||
-rw-r--r-- | library/cpp/type_info/type_equivalence.cpp | 20 | ||||
-rw-r--r-- | library/cpp/type_info/type_factory.cpp | 32 | ||||
-rw-r--r-- | library/cpp/type_info/type_factory.h | 28 | ||||
-rw-r--r-- | library/cpp/type_info/type_io.cpp | 40 | ||||
-rw-r--r-- | library/cpp/type_info/type_list.h | 12 | ||||
-rw-r--r-- | yt/cpp/mapreduce/client/skiff.cpp | 6 | ||||
-rw-r--r-- | yt/cpp/mapreduce/interface/common.cpp | 27 | ||||
-rw-r--r-- | yt/cpp/mapreduce/interface/common.h | 9 | ||||
-rw-r--r-- | yt/cpp/mapreduce/interface/common_ut.cpp | 10 | ||||
-rw-r--r-- | yt/cpp/mapreduce/interface/serialize.cpp | 5 |
15 files changed, 545 insertions, 2 deletions
diff --git a/library/cpp/type_info/fwd.h b/library/cpp/type_info/fwd.h index 644932f677..127bbbeebe 100644 --- a/library/cpp/type_info/fwd.h +++ b/library/cpp/type_info/fwd.h @@ -100,6 +100,18 @@ namespace NTi { class TUuidType; using TUuidTypePtr = TIntrusiveConstPtr<TUuidType>; + class TDate32Type; + using TDate32TypePtr = TIntrusiveConstPtr<TDate32Type>; + + class TDatetime64Type; + using TDatetime64TypePtr = TIntrusiveConstPtr<TDatetime64Type>; + + class TTimestamp64Type; + using TTimestamp64TypePtr = TIntrusiveConstPtr<TTimestamp64Type>; + + class TInterval64Type; + using TInterval64TypePtr = TIntrusiveConstPtr<TInterval64Type>; + class TOptionalType; using TOptionalTypePtr = TIntrusiveConstPtr<TOptionalType>; diff --git a/library/cpp/type_info/type.cpp b/library/cpp/type_info/type.cpp index fb084fa18b..9197f31f2f 100644 --- a/library/cpp/type_info/type.cpp +++ b/library/cpp/type_info/type.cpp @@ -864,6 +864,98 @@ namespace NTi { Y_UNUSED(factory); } + TDate32Type::TDate32Type() + : TPrimitiveType({}, EPrimitiveTypeName::Date32) + { + } + + TDate32TypePtr TDate32Type::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TDate32Type* TDate32Type::InstanceRaw() { + static auto singleton = TDate32Type(); + return &singleton; + } + + const TDate32Type* TDate32Type::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TDate32Type::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TDatetime64Type::TDatetime64Type() + : TPrimitiveType({}, EPrimitiveTypeName::Datetime64) + { + } + + TDatetime64TypePtr TDatetime64Type::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TDatetime64Type* TDatetime64Type::InstanceRaw() { + static auto singleton = TDatetime64Type(); + return &singleton; + } + + const TDatetime64Type* TDatetime64Type::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TDatetime64Type::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TTimestamp64Type::TTimestamp64Type() + : TPrimitiveType({}, EPrimitiveTypeName::Timestamp64) + { + } + + TTimestamp64TypePtr TTimestamp64Type::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TTimestamp64Type* TTimestamp64Type::InstanceRaw() { + static auto singleton = TTimestamp64Type(); + return &singleton; + } + + const TTimestamp64Type* TTimestamp64Type::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TTimestamp64Type::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + + TInterval64Type::TInterval64Type() + : TPrimitiveType({}, EPrimitiveTypeName::Interval64) + { + } + + TInterval64TypePtr TInterval64Type::Instance() { + return InstanceRaw()->AsPtr(); + } + + const TInterval64Type* TInterval64Type::InstanceRaw() { + static auto singleton = TInterval64Type(); + return &singleton; + } + + const TInterval64Type* TInterval64Type::Clone(ITypeFactoryInternal& factory) const noexcept { + Y_UNUSED(factory); + return InstanceRaw(); + } + + void TInterval64Type::Drop(ITypeFactoryInternal& factory) noexcept { + Y_UNUSED(factory); + } + TOptionalType::TOptionalType(TMaybe<ui64> hash, const TType* item) noexcept : TType(hash, ETypeName::Optional) , Item_(item) @@ -1371,6 +1463,22 @@ namespace NTi { return NPrivate::GetDefaultHeapFactory()->Uuid(); } + TDate32TypePtr Date32() { + return NPrivate::GetDefaultHeapFactory()->Date32(); + } + + TDatetime64TypePtr Datetime64() { + return NPrivate::GetDefaultHeapFactory()->Datetime64(); + } + + TTimestamp64TypePtr Timestamp64() { + return NPrivate::GetDefaultHeapFactory()->Timestamp64(); + } + + TInterval64TypePtr Interval64() { + return NPrivate::GetDefaultHeapFactory()->Interval64(); + } + TOptionalTypePtr Optional(TTypePtr item) { return NPrivate::GetDefaultHeapFactory()->Optional(std::move(item)); } @@ -1549,6 +1657,26 @@ Y_DECLARE_OUT_SPEC(, NTi::TUuidType, o, v) { o << "Uuid"; } +Y_DECLARE_OUT_SPEC(, NTi::TDate32Type, o, v) { + Y_UNUSED(v); + o << "Date32"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TDatetime64Type, o, v) { + Y_UNUSED(v); + o << "Datetime64"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TTimestamp64Type, o, v) { + Y_UNUSED(v); + o << "Timestamp64"; +} + +Y_DECLARE_OUT_SPEC(, NTi::TInterval64Type, o, v) { + Y_UNUSED(v); + o << "Interval64"; +} + Y_DECLARE_OUT_SPEC(, NTi::TOptionalType, o, v) { o << "Optional<" << *v.GetItemTypeRaw() << ">"; } @@ -1651,6 +1779,10 @@ static_assert(std::is_trivially_destructible_v<NTi::TDecimalType>); static_assert(std::is_trivially_destructible_v<NTi::TJsonType>); static_assert(std::is_trivially_destructible_v<NTi::TYsonType>); static_assert(std::is_trivially_destructible_v<NTi::TUuidType>); +static_assert(std::is_trivially_destructible_v<NTi::TDate32Type>); +static_assert(std::is_trivially_destructible_v<NTi::TDatetime64Type>); +static_assert(std::is_trivially_destructible_v<NTi::TTimestamp64Type>); +static_assert(std::is_trivially_destructible_v<NTi::TInterval64Type>); static_assert(std::is_trivially_destructible_v<NTi::TOptionalType>); static_assert(std::is_trivially_destructible_v<NTi::TListType>); static_assert(std::is_trivially_destructible_v<NTi::TDictType>); diff --git a/library/cpp/type_info/type.h b/library/cpp/type_info/type.h index ea148c364d..0176e4ce23 100644 --- a/library/cpp/type_info/type.h +++ b/library/cpp/type_info/type.h @@ -171,6 +171,22 @@ namespace NTi { inline TUuidTypePtr AsUuid() const noexcept; inline const TUuidType* AsUuidRaw() const noexcept; + inline bool IsDate32() const noexcept; + inline TDate32TypePtr AsDate32() const noexcept; + inline const TDate32Type* AsDate32Raw() const noexcept; + + inline bool IsDatetime64() const noexcept; + inline TDatetime64TypePtr AsDatetime64() const noexcept; + inline const TDatetime64Type* AsDatetime64Raw() const noexcept; + + inline bool IsTimestamp64() const noexcept; + inline TTimestamp64TypePtr AsTimestamp64() const noexcept; + inline const TTimestamp64Type* AsTimestamp64Raw() const noexcept; + + inline bool IsInterval64() const noexcept; + inline TInterval64TypePtr AsInterval64() const noexcept; + inline const TInterval64Type* AsInterval64Raw() const noexcept; + inline bool IsOptional() const noexcept; inline TOptionalTypePtr AsOptional() const noexcept; inline const TOptionalType* AsOptionalRaw() const noexcept; @@ -952,7 +968,7 @@ namespace NTi { void Drop(ITypeFactoryInternal& factory) noexcept; }; - /// An absolute point in time in range `[1970-01-01, 2106-01-01)`, precision up to microseconds. + /// An absolute point in time in range `[1970-01-01, 2106-01-01)`, precision up to milliseconds. class TTimestampType final: public TPrimitiveType { friend class TType; friend class ITypeFactoryInternal; @@ -1189,6 +1205,102 @@ namespace NTi { void Drop(ITypeFactoryInternal& factory) noexcept; }; + /// An absolute point in time in range `[-144169-01-01, 148108-01-01)`, precision up to days (Unix epoch 1970-01-01 - 0 days). + class TDate32Type final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TDate32TypePtr AsPtr() const noexcept { + return const_cast<TDate32Type*>(this); + } + + private: + explicit TDate32Type(); + + public: + static TDate32TypePtr Instance(); + static const TDate32Type* InstanceRaw(); + + protected: + const TDate32Type* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// An absolute point in time in range `[-144169-01-01, 148108-01-01)`, precision up to seconds (Unix epoch 1970-01-01 - 0 seconds). + class TDatetime64Type final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TDatetime64TypePtr AsPtr() const noexcept { + return const_cast<TDatetime64Type*>(this); + } + + private: + explicit TDatetime64Type(); + + public: + static TDatetime64TypePtr Instance(); + static const TDatetime64Type* InstanceRaw(); + + protected: + const TDatetime64Type* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// An absolute point in time in range `[-144169-01-01, 148108-01-01)`, precision up to milliseconds (Unix epoch 1970-01-01 - 0 milliseconds). + class TTimestamp64Type final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TTimestamp64TypePtr AsPtr() const noexcept { + return const_cast<TTimestamp64Type*>(this); + } + + private: + explicit TTimestamp64Type(); + + public: + static TTimestamp64TypePtr Instance(); + static const TTimestamp64Type* InstanceRaw(); + + protected: + const TTimestamp64Type* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + + /// Signed delta between two timestamps64. + class TInterval64Type final: public TPrimitiveType { + friend class TType; + friend class ITypeFactoryInternal; + friend class ITypeFactory; + friend class IPoolTypeFactory; + + public: + TInterval64TypePtr AsPtr() const noexcept { + return const_cast<TInterval64Type*>(this); + } + + private: + explicit TInterval64Type(); + + public: + static TInterval64TypePtr Instance(); + static const TInterval64Type* InstanceRaw(); + + protected: + const TInterval64Type* Clone(ITypeFactoryInternal& factory) const noexcept; + void Drop(ITypeFactoryInternal& factory) noexcept; + }; + /// Object which can store a value or a singular `NULL` value. /// /// This type is used to encode a value or its absence. @@ -2040,6 +2152,58 @@ namespace NTi { return static_cast<const TUuidType*>(this); } + bool TType::IsDate32() const noexcept { + return TypeName_ == ETypeName::Date32; + } + + TDate32TypePtr TType::AsDate32() const noexcept { + return AsDate32Raw()->AsPtr(); + } + + const TDate32Type* TType::AsDate32Raw() const noexcept { + Y_ABORT_UNLESS(IsDate32()); + return static_cast<const TDate32Type*>(this); + } + + bool TType::IsDatetime64() const noexcept { + return TypeName_ == ETypeName::Datetime64; + } + + TDatetime64TypePtr TType::AsDatetime64() const noexcept { + return AsDatetime64Raw()->AsPtr(); + } + + const TDatetime64Type* TType::AsDatetime64Raw() const noexcept { + Y_ABORT_UNLESS(IsDatetime64()); + return static_cast<const TDatetime64Type*>(this); + } + + bool TType::IsTimestamp64() const noexcept { + return TypeName_ == ETypeName::Timestamp64; + } + + TTimestamp64TypePtr TType::AsTimestamp64() const noexcept { + return AsTimestamp64Raw()->AsPtr(); + } + + const TTimestamp64Type* TType::AsTimestamp64Raw() const noexcept { + Y_ABORT_UNLESS(IsTimestamp64()); + return static_cast<const TTimestamp64Type*>(this); + } + + bool TType::IsInterval64() const noexcept { + return TypeName_ == ETypeName::Interval64; + } + + TInterval64TypePtr TType::AsInterval64() const noexcept { + return AsInterval64Raw()->AsPtr(); + } + + const TInterval64Type* TType::AsInterval64Raw() const noexcept { + Y_ABORT_UNLESS(IsInterval64()); + return static_cast<const TInterval64Type*>(this); + } + bool TType::IsOptional() const noexcept { return TypeName_ == ETypeName::Optional; } @@ -2186,6 +2350,14 @@ namespace NTi { return std::forward<V>(visitor)(this->AsYson()); case ETypeName::Uuid: return std::forward<V>(visitor)(this->AsUuid()); + case ETypeName::Date32: + return std::forward<V>(visitor)(this->AsDate32()); + case ETypeName::Datetime64: + return std::forward<V>(visitor)(this->AsDatetime64()); + case ETypeName::Timestamp64: + return std::forward<V>(visitor)(this->AsTimestamp64()); + case ETypeName::Interval64: + return std::forward<V>(visitor)(this->AsInterval64()); case ETypeName::Void: return std::forward<V>(visitor)(this->AsVoid()); case ETypeName::Null: @@ -2260,6 +2432,14 @@ namespace NTi { return std::forward<V>(visitor)(this->AsYsonRaw()); case ETypeName::Uuid: return std::forward<V>(visitor)(this->AsUuidRaw()); + case ETypeName::Date32: + return std::forward<V>(visitor)(this->AsDate32Raw()); + case ETypeName::Datetime64: + return std::forward<V>(visitor)(this->AsDatetime64Raw()); + case ETypeName::Timestamp64: + return std::forward<V>(visitor)(this->AsTimestamp64Raw()); + case ETypeName::Interval64: + return std::forward<V>(visitor)(this->AsInterval64Raw()); case ETypeName::Void: return std::forward<V>(visitor)(this->AsVoidRaw()); case ETypeName::Null: @@ -2334,6 +2514,14 @@ namespace NTi { return std::forward<V>(visitor)(this->AsYson()); case EPrimitiveTypeName::Uuid: return std::forward<V>(visitor)(this->AsUuid()); + case EPrimitiveTypeName::Date32: + return std::forward<V>(visitor)(this->AsDate32()); + case EPrimitiveTypeName::Datetime64: + return std::forward<V>(visitor)(this->AsDatetime64()); + case EPrimitiveTypeName::Timestamp64: + return std::forward<V>(visitor)(this->AsTimestamp64()); + case EPrimitiveTypeName::Interval64: + return std::forward<V>(visitor)(this->AsInterval64()); } Y_UNREACHABLE(); @@ -2390,6 +2578,14 @@ namespace NTi { return std::forward<V>(visitor)(this->AsYsonRaw()); case EPrimitiveTypeName::Uuid: return std::forward<V>(visitor)(this->AsUuidRaw()); + case NTi::EPrimitiveTypeName::Date32: + return std::forward<V>(visitor)(this->AsDate32Raw()); + case NTi::EPrimitiveTypeName::Datetime64: + return std::forward<V>(visitor)(this->AsDatetime64Raw()); + case NTi::EPrimitiveTypeName::Timestamp64: + return std::forward<V>(visitor)(this->AsTimestamp64Raw()); + case NTi::EPrimitiveTypeName::Interval64: + return std::forward<V>(visitor)(this->AsInterval64Raw()); } Y_UNREACHABLE(); diff --git a/library/cpp/type_info/type_complexity.cpp b/library/cpp/type_info/type_complexity.cpp index b994cc2187..572940bff8 100644 --- a/library/cpp/type_info/type_complexity.cpp +++ b/library/cpp/type_info/type_complexity.cpp @@ -37,6 +37,10 @@ int ComputeTypeComplexity(const TType* type) case ETypeName::Json: case ETypeName::Yson: case ETypeName::Uuid: + case ETypeName::Date32: + case ETypeName::Datetime64: + case ETypeName::Timestamp64: + case ETypeName::Interval64: case ETypeName::Void: case ETypeName::Null: return 1; diff --git a/library/cpp/type_info/type_constructors.h b/library/cpp/type_info/type_constructors.h index 439d5a1887..50c48169a5 100644 --- a/library/cpp/type_info/type_constructors.h +++ b/library/cpp/type_info/type_constructors.h @@ -80,6 +80,18 @@ namespace NTi { /// Create new `Uuid` type using the default heap factory. TUuidTypePtr Uuid(); + /// Create new `Date32` type using the default heap factory. + TDate32TypePtr Date32(); + + /// Create new `Datetime64` type using the default heap factory. + TDatetime64TypePtr Datetime64(); + + /// Create new `Timestamp64` type using the default heap factory. + TTimestamp64TypePtr Timestamp64(); + + /// Create new `Interval64` type using the default heap factory. + TInterval64TypePtr Interval64(); + /// Create new `Optional` type using the default heap factory. TOptionalTypePtr Optional(TTypePtr item); diff --git a/library/cpp/type_info/type_equivalence.cpp b/library/cpp/type_info/type_equivalence.cpp index 9f22f0308c..8288782cc7 100644 --- a/library/cpp/type_info/type_equivalence.cpp +++ b/library/cpp/type_info/type_equivalence.cpp @@ -150,6 +150,26 @@ namespace NTi::NEq { } template <bool IgnoreHash> + bool StrictlyEqual(const TDate32Type&, const TDate32Type&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TDatetime64Type&, const TDatetime64Type&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TTimestamp64Type&, const TTimestamp64Type&) { + return true; + } + + template <bool IgnoreHash> + bool StrictlyEqual(const TInterval64Type&, const TInterval64Type&) { + return true; + } + + template <bool IgnoreHash> bool StrictlyEqual(const TOptionalType& lhs, const TOptionalType& rhs) { return StrictlyEqual<IgnoreHash>(lhs.GetItemTypeRaw(), rhs.GetItemTypeRaw()); } diff --git a/library/cpp/type_info/type_factory.cpp b/library/cpp/type_info/type_factory.cpp index cc4206e012..661e31c7a9 100644 --- a/library/cpp/type_info/type_factory.cpp +++ b/library/cpp/type_info/type_factory.cpp @@ -217,6 +217,38 @@ namespace NTi { return TUuidType::InstanceRaw(); } + TDate32TypePtr ITypeFactory::Date32() { + return TDate32Type::Instance(); + } + + const TDate32Type* IPoolTypeFactory::Date32Raw() { + return TDate32Type::InstanceRaw(); + } + + TDatetime64TypePtr ITypeFactory::Datetime64() { + return TDatetime64Type::Instance(); + } + + const TDatetime64Type* IPoolTypeFactory::Datetime64Raw() { + return TDatetime64Type::InstanceRaw(); + } + + TTimestamp64TypePtr ITypeFactory::Timestamp64() { + return TTimestamp64Type::Instance(); + } + + const TTimestamp64Type* IPoolTypeFactory::Timestamp64Raw() { + return TTimestamp64Type::InstanceRaw(); + } + + TInterval64TypePtr ITypeFactory::Interval64() { + return TInterval64Type::Instance(); + } + + const TInterval64Type* IPoolTypeFactory::Interval64Raw() { + return TInterval64Type::InstanceRaw(); + } + TOptionalTypePtr ITypeFactory::Optional(TTypePtr item) { return TOptionalType::Create(*this, std::move(item)); } diff --git a/library/cpp/type_info/type_factory.h b/library/cpp/type_info/type_factory.h index df47c7082b..1d21094355 100644 --- a/library/cpp/type_info/type_factory.h +++ b/library/cpp/type_info/type_factory.h @@ -626,6 +626,18 @@ namespace NTi { /// Create a new `Uuid` type. See `NTi::TUuidType` for more info. TUuidTypePtr Uuid(); + /// Create a new `Date32` type. See `NTi::TDate32Type` for more info. + TDate32TypePtr Date32(); + + /// Create a new `Datetime64` type. See `NTi::TDatetime64Type` for more info. + TDatetime64TypePtr Datetime64(); + + /// Create a new `Timestamp64` type. See `NTi::TTimestamp64Type` for more info. + TTimestamp64TypePtr Timestamp64(); + + /// Create a new `Interval64` type. See `NTi::TInterval64Type` for more info. + TInterval64TypePtr Interval64(); + /// Create a new `Optional` type. See `NTi::TOptionalType` for more info. /// If `item` is managed by some other factory, it will be deep-copied into this factory. TOptionalTypePtr Optional(TTypePtr item); @@ -812,6 +824,22 @@ namespace NTi { /// The returned object will live for as long as this factory lives. const TUuidType* UuidRaw(); + /// Create a new `Date32` type. See `NTi::TDate32Type` for more info. + /// The returned object will live for as long as this factory lives. + const TDate32Type* Date32Raw(); + + /// Create a new `Datetime64` type. See `NTi::TDatetime64Type` for more info. + /// The returned object will live for as long as this factory lives. + const TDatetime64Type* Datetime64Raw(); + + /// Create a new `Timestamp64` type. See `NTi::TTimestamp64Type` for more info. + /// The returned object will live for as long as this factory lives. + const TTimestamp64Type* Timestamp64Raw(); + + /// Create a new `Interval64` type. See `NTi::TInterval64Type` for more info. + /// The returned object will live for as long as this factory lives. + const TInterval64Type* Interval64Raw(); + /// Create a new `Optional` type. See `NTi::TOptionalType` for more info. /// The returned object will live for as long as this factory lives. const TOptionalType* OptionalRaw(const TType* item); diff --git a/library/cpp/type_info/type_io.cpp b/library/cpp/type_info/type_io.cpp index 90b98c8baa..986a9d6264 100644 --- a/library/cpp/type_info/type_io.cpp +++ b/library/cpp/type_info/type_io.cpp @@ -291,6 +291,18 @@ namespace NTi::NIo { case ETypeName::Uuid: type = TUuidType::InstanceRaw(); break; + case ETypeName::Date32: + type = TDate32Type::InstanceRaw(); + break; + case ETypeName::Datetime64: + type = TDatetime64Type::InstanceRaw(); + break; + case ETypeName::Timestamp64: + type = TTimestamp64Type::InstanceRaw(); + break; + case ETypeName::Interval64: + type = TInterval64Type::InstanceRaw(); + break; case ETypeName::Void: type = TVoidType::InstanceRaw(); break; @@ -710,6 +722,18 @@ namespace NTi::NIo { [&consumer](const TUuidType*) { consumer.OnScalarString("uuid"); }, + [&consumer](const TDate32Type*) { + consumer.OnScalarString("date32"); + }, + [&consumer](const TDatetime64Type*) { + consumer.OnScalarString("datetime64"); + }, + [&consumer](const TTimestamp64Type*) { + consumer.OnScalarString("timestamp64"); + }, + [&consumer](const TInterval64Type*) { + consumer.OnScalarString("interval64"); + }, [&consumer](const TDecimalType* t) { consumer.OnBeginMap(); @@ -968,6 +992,18 @@ namespace NTi::NIo { [&consumer](const TUuidType*) { WriteDataType(consumer, EPrimitiveTypeName::Uuid); }, + [&consumer](const TDate32Type*) { + WriteDataType(consumer, EPrimitiveTypeName::Date32); + }, + [&consumer](const TDatetime64Type*) { + WriteDataType(consumer, EPrimitiveTypeName::Datetime64); + }, + [&consumer](const TTimestamp64Type*) { + WriteDataType(consumer, EPrimitiveTypeName::Timestamp64); + }, + [&consumer](const TInterval64Type*) { + WriteDataType(consumer, EPrimitiveTypeName::Interval64); + }, [&consumer](const TDecimalType* t) { consumer.OnBeginList(); consumer.OnScalarString("DataType"); @@ -1142,6 +1178,10 @@ namespace NTi::NIo { [](const TJsonType*) -> TStringBuf { return "string"; }, [](const TYsonType*) -> TStringBuf { return "any"; }, [](const TUuidType*) -> TStringBuf { return "string"; }, + [](const TDate32Type*) -> TStringBuf { return "int64"; }, + [](const TDatetime64Type*) -> TStringBuf { return "int64"; }, + [](const TTimestamp64Type*) -> TStringBuf { return "int64"; }, + [](const TInterval64Type*) -> TStringBuf { return "int64"; }, [](const TDecimalType*) -> TStringBuf { return "string"; }, [](const TOptionalType*) -> TStringBuf { return "any"; }, [](const TListType*) -> TStringBuf { return "any"; }, diff --git a/library/cpp/type_info/type_list.h b/library/cpp/type_info/type_list.h index 8d0c1db88b..2a4d13c27f 100644 --- a/library/cpp/type_info/type_list.h +++ b/library/cpp/type_info/type_list.h @@ -59,6 +59,11 @@ namespace NTi { Json, Yson, Uuid, + + Date32, + Datetime64, + Timestamp64, + Interval64, }; /// Enum with names of all types, including primitives. @@ -98,8 +103,13 @@ namespace NTi { Yson, Uuid, + Date32, + Datetime64, + Timestamp64, + Interval64, + FIRST_PRIMITIVE = Bool, - LAST_PRIMITIVE = Uuid, + LAST_PRIMITIVE = Interval64, // // # Singular types diff --git a/yt/cpp/mapreduce/client/skiff.cpp b/yt/cpp/mapreduce/client/skiff.cpp index 1eb2e80d79..f6e52f44e8 100644 --- a/yt/cpp/mapreduce/client/skiff.cpp +++ b/yt/cpp/mapreduce/client/skiff.cpp @@ -94,6 +94,12 @@ NSkiff::EWireType ValueTypeToSkiffType(EValueType valueType) case VT_INTERVAL: return EWireType::Int64; + + case VT_DATE32: + case VT_DATETIME64: + case VT_TIMESTAMP64: + case VT_INTERVAL64: + return EWireType::Int64; }; ythrow yexception() << "Cannot convert EValueType '" << valueType << "' to NSkiff::EWireType"; } diff --git a/yt/cpp/mapreduce/interface/common.cpp b/yt/cpp/mapreduce/interface/common.cpp index 28f98713f6..0a46c0b583 100644 --- a/yt/cpp/mapreduce/interface/common.cpp +++ b/yt/cpp/mapreduce/interface/common.cpp @@ -192,6 +192,15 @@ static NTi::TTypePtr OldTypeToTypeV3(EValueType type) return NTi::Float(); case VT_JSON: return NTi::Json(); + + case VT_DATE32: + return NTi::Date32(); + case VT_DATETIME64: + return NTi::Datetime64(); + case VT_TIMESTAMP64: + return NTi::Timestamp64(); + case VT_INTERVAL64: + return NTi::Interval64(); } } @@ -254,6 +263,15 @@ static std::pair<EValueType, bool> Simplify(const NTi::TTypePtr& type) case ETypeName::Yson: return {VT_ANY, true}; + case ETypeName::Date32: + return {VT_DATE32, true}; + case ETypeName::Datetime64: + return {VT_DATETIME64, true}; + case ETypeName::Timestamp64: + return {VT_TIMESTAMP64, true}; + case ETypeName::Interval64: + return {VT_INTERVAL64, true}; + case ETypeName::Void: return {VT_VOID, false}; case ETypeName::Null: @@ -633,6 +651,15 @@ TString ToString(EValueType type) case VT_JSON: return "json"; + + case VT_DATE32: + return "date32"; + case VT_DATETIME64: + return "datetime64"; + case VT_TIMESTAMP64: + return "timestamp64"; + case VT_INTERVAL64: + return "interval64"; } ythrow yexception() << "Invalid value type " << static_cast<int>(type); } diff --git a/yt/cpp/mapreduce/interface/common.h b/yt/cpp/mapreduce/interface/common.h index ead8284b57..8b3edf48c4 100644 --- a/yt/cpp/mapreduce/interface/common.h +++ b/yt/cpp/mapreduce/interface/common.h @@ -403,6 +403,15 @@ enum EValueType : int VT_FLOAT, /// Json, sequence of bytes that is valid json. VT_JSON, + + // Date32, number of days shifted from Unix epoch, which is 0 (signed) + VT_DATE32, + // Datetime64, number of seconds shifted from Unix epoch, which is 0 (signed) + VT_DATETIME64, + // Timestamp64, number of milliseconds shifted from Unix epoch, which is 0 (signed) + VT_TIMESTAMP64, + // Interval64, difference between two timestamps64 (signed) + VT_INTERVAL64, }; /// diff --git a/yt/cpp/mapreduce/interface/common_ut.cpp b/yt/cpp/mapreduce/interface/common_ut.cpp index f9f0664039..05d7495ca2 100644 --- a/yt/cpp/mapreduce/interface/common_ut.cpp +++ b/yt/cpp/mapreduce/interface/common_ut.cpp @@ -228,6 +228,16 @@ Y_UNIT_TEST_SUITE(Common) UNIT_ASSERT_VALUES_EQUAL(column.Type(), VT_DATE); } { + auto column = TColumnSchema().Type(NTi::Interval64()); + UNIT_ASSERT_VALUES_EQUAL(column.Required(), true); + UNIT_ASSERT_VALUES_EQUAL(column.Type(), VT_INTERVAL64); + } + { + auto column = TColumnSchema().Type(NTi::Optional(NTi::Date32())); + UNIT_ASSERT_VALUES_EQUAL(column.Required(), false); + UNIT_ASSERT_VALUES_EQUAL(column.Type(), VT_DATE32); + } + { auto column = TColumnSchema().Type(NTi::Null()); UNIT_ASSERT_VALUES_EQUAL(column.Required(), false); UNIT_ASSERT_VALUES_EQUAL(column.Type(), VT_NULL); diff --git a/yt/cpp/mapreduce/interface/serialize.cpp b/yt/cpp/mapreduce/interface/serialize.cpp index 18cfaaa9b7..9be114b45e 100644 --- a/yt/cpp/mapreduce/interface/serialize.cpp +++ b/yt/cpp/mapreduce/interface/serialize.cpp @@ -129,6 +129,11 @@ void Deserialize(EValueType& valueType, const TNode& node) {"interval", VT_INTERVAL}, {"float", VT_FLOAT}, {"json", VT_JSON}, + + {"date32", VT_DATE32}, + {"datetime64", VT_DATETIME64}, + {"timestamp64", VT_TIMESTAMP64}, + {"interval64", VT_INTERVAL64}, }; auto it = str2ValueType.find(nodeStr); |