diff options
author | uzhas <uzhas@ydb.tech> | 2022-08-05 13:31:51 +0300 |
---|---|---|
committer | uzhas <uzhas@ydb.tech> | 2022-08-05 13:31:51 +0300 |
commit | 535d0b70fe9e03c1897c7f284d070a13bb4b89bf (patch) | |
tree | b84e9b268012e83d898276dae25af06f7dc9996c | |
parent | 68d8479d07e0b689ba126cf2c159ae7f11afb9cf (diff) | |
download | ydb-535d0b70fe9e03c1897c7f284d070a13bb4b89bf.tar.gz |
support pg types in UDF ABI
28 files changed, 664 insertions, 181 deletions
diff --git a/CMakeLists.darwin.txt b/CMakeLists.darwin.txt index c160c45c51b..2e34cfb148d 100644 --- a/CMakeLists.darwin.txt +++ b/CMakeLists.darwin.txt @@ -409,8 +409,8 @@ add_subdirectory(ydb/library/yql/public/udf) add_subdirectory(ydb/library/yql/public/decimal) add_subdirectory(ydb/library/yql/utils) add_subdirectory(ydb/library/dynumber) -add_subdirectory(ydb/library/yql/public/udf/tz) add_subdirectory(ydb/library/yql/parser/pg_catalog) +add_subdirectory(ydb/library/yql/public/udf/tz) add_subdirectory(ydb/core/mon) add_subdirectory(library/cpp/string_utils/url) add_subdirectory(ydb/core/node_whiteboard) diff --git a/CMakeLists.linux.txt b/CMakeLists.linux.txt index cf6df6b997e..d1424fe5f5c 100644 --- a/CMakeLists.linux.txt +++ b/CMakeLists.linux.txt @@ -412,8 +412,8 @@ add_subdirectory(ydb/library/yql/public/udf) add_subdirectory(ydb/library/yql/public/decimal) add_subdirectory(ydb/library/yql/utils) add_subdirectory(ydb/library/dynumber) -add_subdirectory(ydb/library/yql/public/udf/tz) add_subdirectory(ydb/library/yql/parser/pg_catalog) +add_subdirectory(ydb/library/yql/public/udf/tz) add_subdirectory(ydb/core/mon) add_subdirectory(library/cpp/string_utils/url) add_subdirectory(ydb/core/node_whiteboard) diff --git a/ydb/library/yql/minikql/CMakeLists.txt b/ydb/library/yql/minikql/CMakeLists.txt index d4c1c7539b3..82d34c624c7 100644 --- a/ydb/library/yql/minikql/CMakeLists.txt +++ b/ydb/library/yql/minikql/CMakeLists.txt @@ -26,10 +26,10 @@ target_link_libraries(library-yql-minikql PUBLIC ydb-library-binary_json ydb-library-dynumber yql-minikql-dom + yql-parser-pg_catalog yql-public-udf public-udf-tz library-yql-utils - yql-parser-pg_catalog ) target_sources(library-yql-minikql PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/aligned_page_pool.cpp diff --git a/ydb/library/yql/minikql/computation/mkql_value_builder.cpp b/ydb/library/yql/minikql/computation/mkql_value_builder.cpp index 6bfd8319df6..7561ef034ef 100644 --- a/ydb/library/yql/minikql/computation/mkql_value_builder.cpp +++ b/ydb/library/yql/minikql/computation/mkql_value_builder.cpp @@ -7,6 +7,10 @@ #include <util/system/env.h> +namespace NYql { + std::unique_ptr<NUdf::IPgBuilder> CreatePgBuilder(); +} + namespace NKikimr { namespace NMiniKQL { @@ -16,6 +20,7 @@ namespace NMiniKQL { TDefaultValueBuilder::TDefaultValueBuilder(const THolderFactory& holderFactory, NUdf::EValidatePolicy policy) : HolderFactory_(holderFactory) , Policy_(policy) + , PgBuilder_(NYql::CreatePgBuilder()) {} void TDefaultValueBuilder::SetSecureParamsProvider(const NUdf::ISecureParamsProvider* provider) { diff --git a/ydb/library/yql/minikql/computation/mkql_value_builder.h b/ydb/library/yql/minikql/computation/mkql_value_builder.h index b293b263a3d..0b8eeeb9de6 100644 --- a/ydb/library/yql/minikql/computation/mkql_value_builder.h +++ b/ydb/library/yql/minikql/computation/mkql_value_builder.h @@ -21,7 +21,7 @@ class TDefaultValueBuilder final: public NUdf::IValueBuilder, private TNonCopyab public NUdf::IDateBuilder { public: - TDefaultValueBuilder(const THolderFactory& holderFactory, NUdf::EValidatePolicy policy = NUdf::EValidatePolicy::Fail); + explicit TDefaultValueBuilder(const THolderFactory& holderFactory, NUdf::EValidatePolicy policy = NUdf::EValidatePolicy::Fail); void SetSecureParamsProvider(const NUdf::ISecureParamsProvider* provider); void RethrowAtTerminate(); @@ -56,11 +56,10 @@ public: bool GetSecureParam(NUdf::TStringRef key, NUdf::TStringRef &value) const final; const NUdf::TSourcePosition* CalleePosition() const final; - NUdf::TUnboxedValue Run(const NUdf::TSourcePosition& callee, const NUdf::IBoxedValue& value, const NUdf::TUnboxedValuePod* args) const; - NUdf::TFlatDataBlockPtr NewFlatDataBlock(ui32 initialSize, ui32 initialCapacity) const; - NUdf::TFlatArrayBlockPtr NewFlatArrayBlock(ui32 count) const; - NUdf::TSingleBlockPtr NewSingleBlock(const NUdf::TUnboxedValue& value) const; - + NUdf::TUnboxedValue Run(const NUdf::TSourcePosition& callee, const NUdf::IBoxedValue& value, const NUdf::TUnboxedValuePod* args) const final; + NUdf::TFlatDataBlockPtr NewFlatDataBlock(ui32 initialSize, ui32 initialCapacity) const final; + NUdf::TFlatArrayBlockPtr NewFlatArrayBlock(ui32 count) const final; + NUdf::TSingleBlockPtr NewSingleBlock(const NUdf::TUnboxedValue& value) const final; bool MakeDate(ui32 year, ui32 month, ui32 day, ui16& value) const final; bool SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const final; @@ -86,9 +85,14 @@ public: bool FullSplitDatetime2(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 timezoneId = 0) const final; + const NUdf::IPgBuilder& GetPgBuilder() const final { + return *PgBuilder_; + } + private: const THolderFactory& HolderFactory_; NUdf::EValidatePolicy Policy_; + std::unique_ptr<NUdf::IPgBuilder> PgBuilder_; const NUdf::ISecureParamsProvider* SecureParamsProvider_ = nullptr; const NUdf::TSourcePosition** CalleePositionPtr_ = nullptr; mutable bool Rethrow_ = false; diff --git a/ydb/library/yql/minikql/mkql_type_builder.cpp b/ydb/library/yql/minikql/mkql_type_builder.cpp index 013e45e40dd..9daa64a8375 100644 --- a/ydb/library/yql/minikql/mkql_type_builder.cpp +++ b/ydb/library/yql/minikql/mkql_type_builder.cpp @@ -6,6 +6,8 @@ #include <library/cpp/containers/stack_vector/stack_vec.h> #include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h> +#include <ydb/library/yql/parser/pg_catalog/catalog.h> +#include <array> // TODO: remove const_casts @@ -15,6 +17,37 @@ namespace { static const TString UdfName("UDF"); +class TPgTypeIndex { + static constexpr ui32 MaxOid = 15000; + using TUdfTypes = std::array<NYql::NUdf::TPgTypeDescription, MaxOid>; + TUdfTypes Types; + +public: + TPgTypeIndex() { + NYql::NPg::EnumTypes([&](ui32 typeId, const NYql::NPg::TTypeDesc& t) { + Y_VERIFY(typeId < Types.size()); + auto& e = Types[typeId]; + e.Name = t.Name; + e.TypeId = t.TypeId; + e.Typelen = t.TypeLen; + e.ArrayTypeId = t.ArrayTypeId; + e.ElementTypeId = t.ElementTypeId; + e.PassByValue = t.PassByValue; + }); + } + + const NYql::NUdf::TPgTypeDescription* Resolve(ui32 typeId) const { + if (typeId >= Types.size()) { + return nullptr; + } + auto& e = Types[typeId]; + if (!e.TypeId) { + return nullptr; + } + return &e; + } +}; + class TCallablePayload : public NUdf::ICallablePayload { public: TCallablePayload(NMiniKQL::TNode* node) @@ -1308,6 +1341,10 @@ NUdf::TType* TFunctionTypeInfoBuilder::Tagged(const NUdf::TType* baseType, const return TTaggedType::Create(const_cast<TType*>(static_cast<const TType*>(baseType)), tag, Env_); } +NUdf::TType* TFunctionTypeInfoBuilder::TFunctionTypeInfoBuilder::Pg(ui32 typeId) const { + return TPgType::Create(typeId, Env_); +} + bool TFunctionTypeInfoBuilder::GetSecureParam(NUdf::TStringRef key, NUdf::TStringRef& value) const { if (SecureParamsProvider_) return SecureParamsProvider_->GetSecureParam(key, value); @@ -1603,6 +1640,7 @@ case NMiniKQL::TType::EKind::TypeKind: { \ MKQL_HANDLE_UDF_TYPE(Stream) MKQL_HANDLE_UDF_TYPE(Resource) MKQL_HANDLE_UDF_TYPE(Tagged) + MKQL_HANDLE_UDF_TYPE(Pg) default: Y_VERIFY_DEBUG(false, "Wrong MQKL type kind %s", mkqlType->GetKindAsStr().data()); } @@ -1620,6 +1658,10 @@ bool TTypeInfoHelper::IsSameType(const NUdf::TType* type1, const NUdf::TType* ty return mkqlType1->IsSameType(*mkqlType2); } +const NYql::NUdf::TPgTypeDescription* TTypeInfoHelper::FindPgTypeDescription(ui32 typeId) const { + return HugeSingleton<TPgTypeIndex>()->Resolve(typeId); +} + void TTypeInfoHelper::DoData(const NMiniKQL::TDataType* dt, NUdf::ITypeVisitor* v) { const auto typeId = dt->GetSchemeType(); v->OnDataType(typeId); @@ -1713,6 +1755,12 @@ void TTypeInfoHelper::DoTagged(const NMiniKQL::TTaggedType* tt, NUdf::ITypeVisit } } +void TTypeInfoHelper::DoPg(const NMiniKQL::TPgType* tt, NUdf::ITypeVisitor* v) { + if (v->IsCompatibleTo(NUdf::MakeAbiCompatibilityVersion(2, 25))) { + v->OnPg(tt->GetTypeId()); + } +} + NUdf::IHash::TPtr MakeHashImpl(const NMiniKQL::TType* type) { switch (type->GetKind()) { case NMiniKQL::TType::EKind::Data: { diff --git a/ydb/library/yql/minikql/mkql_type_builder.h b/ydb/library/yql/minikql/mkql_type_builder.h index 58b75138619..baf7b4106e7 100644 --- a/ydb/library/yql/minikql/mkql_type_builder.h +++ b/ydb/library/yql/minikql/mkql_type_builder.h @@ -117,6 +117,7 @@ public: NUdf::ISetTypeBuilder::TPtr Set() const override; NUdf::IEnumTypeBuilder::TPtr Enum(ui32 expectedItems = 10) const override; NUdf::TType* Tagged(const NUdf::TType* baseType, const NUdf::TStringRef& tag) const override; + NUdf::TType* Pg(ui32 typeId) const override; bool GetSecureParam(NUdf::TStringRef key, NUdf::TStringRef& value) const override; private: @@ -146,6 +147,7 @@ public: NUdf::ETypeKind GetTypeKind(const NUdf::TType* type) const override; void VisitType(const NUdf::TType* type, NUdf::ITypeVisitor* visitor) const override; bool IsSameType(const NUdf::TType* type1, const NUdf::TType* type2) const override; + const NYql::NUdf::TPgTypeDescription* FindPgTypeDescription(ui32 typeId) const override; private: static void DoData(const NMiniKQL::TDataType* dt, NUdf::ITypeVisitor* v); @@ -159,6 +161,7 @@ private: static void DoStream(const NMiniKQL::TStreamType* st, NUdf::ITypeVisitor* v); static void DoResource(const NMiniKQL::TResourceType* rt, NUdf::ITypeVisitor* v); static void DoTagged(const NMiniKQL::TTaggedType* tt, NUdf::ITypeVisitor* v); + static void DoPg(const NMiniKQL::TPgType* tt, NUdf::ITypeVisitor* v); }; NUdf::IHash::TPtr MakeHashImpl(const NMiniKQL::TType* type); diff --git a/ydb/library/yql/parser/pg_catalog/catalog.cpp b/ydb/library/yql/parser/pg_catalog/catalog.cpp index b5e880a0561..26cb20c6202 100644 --- a/ydb/library/yql/parser/pg_catalog/catalog.cpp +++ b/ydb/library/yql/parser/pg_catalog/catalog.cpp @@ -1224,6 +1224,13 @@ const TTypeDesc& LookupType(ui32 typeId) { return *typePtr; } +void EnumTypes(std::function<void(ui32, const TTypeDesc&)> f) { + const auto& catalog = TCatalog::Instance(); + for (const auto& [typeId, desc] : catalog.Types) { + f(typeId, desc); + } +} + bool HasCast(ui32 sourceId, ui32 targetId) { const auto& catalog = TCatalog::Instance(); return catalog.CastsByDir.contains(std::make_pair(sourceId, targetId)); diff --git a/ydb/library/yql/parser/pg_catalog/catalog.h b/ydb/library/yql/parser/pg_catalog/catalog.h index e3a7bbb4ec4..ad62ce037fd 100644 --- a/ydb/library/yql/parser/pg_catalog/catalog.h +++ b/ydb/library/yql/parser/pg_catalog/catalog.h @@ -145,6 +145,7 @@ bool HasReturnSetProc(const TStringBuf& name); bool HasType(const TStringBuf& name); const TTypeDesc& LookupType(const TString& name); const TTypeDesc& LookupType(ui32 typeId); +void EnumTypes(std::function<void(ui32, const TTypeDesc&)> f); bool HasCast(ui32 sourceId, ui32 targetId); const TCastDesc& LookupCast(ui32 sourceId, ui32 targetId); diff --git a/ydb/library/yql/parser/pg_wrapper/CMakeLists.txt b/ydb/library/yql/parser/pg_wrapper/CMakeLists.txt index 275cbfcb094..745feb4ff88 100644 --- a/ydb/library/yql/parser/pg_wrapper/CMakeLists.txt +++ b/ydb/library/yql/parser/pg_wrapper/CMakeLists.txt @@ -107,14 +107,15 @@ target_include_directories(yql-parser-pg_wrapper PRIVATE target_link_libraries(yql-parser-pg_wrapper PUBLIC contrib-libs-cxxsupp yutil - yql-public-issue library-cpp-resource + library-cpp-yson + library-yql-core yql-minikql-computation yql-parser-pg_catalog - library-yql-core providers-common-codec + yql-public-issue + yql-public-udf library-yql-utils - library-cpp-yson contrib-libs-icu contrib-libs-libc_compat Iconv::Iconv diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp index 85f5a0e274a..3267c60de79 100644 --- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp +++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp @@ -11,6 +11,7 @@ #include <ydb/library/yql/minikql/mkql_type_builder.h> #include <ydb/library/yql/providers/common/codec/yql_pg_codec.h> #include <ydb/library/yql/parser/pg_catalog/catalog.h> +#include <ydb/library/yql/public/udf/udf_value_builder.h> #include <ydb/library/yql/core/yql_pg_utils.h> #include <ydb/library/yql/utils/fp_bits.h> #include <library/cpp/yson/detail.h> @@ -991,6 +992,129 @@ private: bool ArrayCast = false; }; + +template <NUdf::EDataSlot Slot> +NUdf::TUnboxedValuePod ConvertToPgValue(NUdf::TUnboxedValuePod value, TMaybe<NUdf::EDataSlot> actualSlot = {}) { +#ifndef NDEBUG + // todo: improve checks + if (actualSlot && Slot != *actualSlot) { + throw yexception() << "Invalid data slot in ConvertToPgValue, expected " << Slot << ", but actual: " << *actualSlot; + } +#else + Y_UNUSED(actualSlot); +#endif + + switch (Slot) { + case NUdf::EDataSlot::Bool: + return ScalarDatumToPod(BoolGetDatum(value.Get<bool>())); + case NUdf::EDataSlot::Int16: + return ScalarDatumToPod(Int16GetDatum(value.Get<i16>())); + case NUdf::EDataSlot::Int32: + return ScalarDatumToPod(Int32GetDatum(value.Get<i32>())); + case NUdf::EDataSlot::Int64: + return ScalarDatumToPod(Int64GetDatum(value.Get<i64>())); + case NUdf::EDataSlot::Float: + return ScalarDatumToPod(Float4GetDatum(value.Get<float>())); + case NUdf::EDataSlot::Double: + return ScalarDatumToPod(Float8GetDatum(value.Get<double>())); + case NUdf::EDataSlot::String: + case NUdf::EDataSlot::Utf8: { + const auto& ref = value.AsStringRef(); + return PointerDatumToPod((Datum)MakeVar(ref)); + } + default: + ythrow yexception() << "Unexpected data slot in ConvertToPgValue: " << Slot; + } +} + +template <NUdf::EDataSlot Slot, bool IsCString> +NUdf::TUnboxedValuePod ConvertFromPgValue(NUdf::TUnboxedValuePod value) { + switch (Slot) { + case NUdf::EDataSlot::Bool: + return NUdf::TUnboxedValuePod((bool)DatumGetBool(ScalarDatumFromPod(value))); + case NUdf::EDataSlot::Int16: + return NUdf::TUnboxedValuePod((i16)DatumGetInt16(ScalarDatumFromPod(value))); + case NUdf::EDataSlot::Int32: + return NUdf::TUnboxedValuePod((i32)DatumGetInt32(ScalarDatumFromPod(value))); + case NUdf::EDataSlot::Int64: + return NUdf::TUnboxedValuePod((i64)DatumGetInt64(ScalarDatumFromPod(value))); + case NUdf::EDataSlot::Float: + return NUdf::TUnboxedValuePod((float)DatumGetFloat4(ScalarDatumFromPod(value))); + case NUdf::EDataSlot::Double: + return NUdf::TUnboxedValuePod((double)DatumGetFloat8(ScalarDatumFromPod(value))); + case NUdf::EDataSlot::String: + case NUdf::EDataSlot::Utf8: + if (IsCString) { + auto x = (const char*)value.AsBoxed().Get() + PallocHdrSize; + return MakeString(TStringBuf(x)); + } else { + auto x = (const text*)((const char*)value.AsBoxed().Get() + PallocHdrSize); + return MakeString(GetVarBuf(x)); + } + default: + ythrow yexception() << "Unexpected data slot in ConvertFromPgValue: " << Slot; + } +} + +NUdf::TUnboxedValuePod ConvertFromPgValue(NUdf::TUnboxedValuePod source, ui32 sourceTypeId, NKikimr::NMiniKQL::TType* targetType) { +#ifndef NDEBUG + // todo: vallidate targetType + Y_UNUSED(targetType); +#else + Y_UNUSED(targetType); +#endif + + switch (sourceTypeId) { + case BOOLOID: + return ConvertFromPgValue<NUdf::EDataSlot::Bool, false>(source); + case INT2OID: + return ConvertFromPgValue<NUdf::EDataSlot::Int16, false>(source); + case INT4OID: + return ConvertFromPgValue<NUdf::EDataSlot::Int32, false>(source); + case INT8OID: + return ConvertFromPgValue<NUdf::EDataSlot::Int64, false>(source); + case FLOAT4OID: + return ConvertFromPgValue<NUdf::EDataSlot::Float, false>(source); + case FLOAT8OID: + return ConvertFromPgValue<NUdf::EDataSlot::Double, false>(source); + case TEXTOID: + case VARCHAROID: + return ConvertFromPgValue<NUdf::EDataSlot::Utf8, false>(source); + case BYTEAOID: + return ConvertFromPgValue<NUdf::EDataSlot::String, false>(source); + case CSTRINGOID: + return ConvertFromPgValue<NUdf::EDataSlot::Utf8, true>(source); + default: + ythrow yexception() << "Unsupported type: " << NPg::LookupType(sourceTypeId).Name; + } +} + +NUdf::TUnboxedValuePod ConvertToPgValue(NUdf::TUnboxedValuePod source, NKikimr::NMiniKQL::TType* sourceType, ui32 targetTypeId) { + auto sourceDataType = AS_TYPE(TDataType, sourceType); + YQL_ENSURE(sourceDataType); + + switch (targetTypeId) { + case BOOLOID: + return ConvertToPgValue<NUdf::EDataSlot::Bool>(source, sourceDataType->GetDataSlot()); + case INT2OID: + return ConvertToPgValue<NUdf::EDataSlot::Int16>(source, sourceDataType->GetDataSlot()); + case INT4OID: + return ConvertToPgValue<NUdf::EDataSlot::Int32>(source, sourceDataType->GetDataSlot()); + case INT8OID: + return ConvertToPgValue<NUdf::EDataSlot::Int64>(source, sourceDataType->GetDataSlot()); + case FLOAT4OID: + return ConvertToPgValue<NUdf::EDataSlot::Float>(source, sourceDataType->GetDataSlot()); + case FLOAT8OID: + return ConvertToPgValue<NUdf::EDataSlot::Double>(source, sourceDataType->GetDataSlot()); + case TEXTOID: + return ConvertToPgValue<NUdf::EDataSlot::Utf8>(source, sourceDataType->GetDataSlot()); + case BYTEAOID: + return ConvertToPgValue<NUdf::EDataSlot::String>(source, sourceDataType->GetDataSlot()); + default: + ythrow yexception() << "Unsupported type: " << NPg::LookupType(targetTypeId).Name; + } +} + template <NUdf::EDataSlot Slot, bool IsCString> class TFromPg : public TMutableComputationNode<TFromPg<Slot, IsCString>> { typedef TMutableComputationNode<TFromPg<Slot, IsCString>> TBaseComputation; @@ -1006,33 +1130,7 @@ public: if (!value) { return value.Release(); } - - switch (Slot) - { - case NUdf::EDataSlot::Bool: - return NUdf::TUnboxedValuePod((bool)DatumGetBool(ScalarDatumFromPod(value))); - case NUdf::EDataSlot::Int16: - return NUdf::TUnboxedValuePod((i16)DatumGetInt16(ScalarDatumFromPod(value))); - case NUdf::EDataSlot::Int32: - return NUdf::TUnboxedValuePod((i32)DatumGetInt32(ScalarDatumFromPod(value))); - case NUdf::EDataSlot::Int64: - return NUdf::TUnboxedValuePod((i64)DatumGetInt64(ScalarDatumFromPod(value))); - case NUdf::EDataSlot::Float: - return NUdf::TUnboxedValuePod((float)DatumGetFloat4(ScalarDatumFromPod(value))); - case NUdf::EDataSlot::Double: - return NUdf::TUnboxedValuePod((double)DatumGetFloat8(ScalarDatumFromPod(value))); - case NUdf::EDataSlot::String: - case NUdf::EDataSlot::Utf8: - if (IsCString) { - auto x = (const char*)value.AsBoxed().Get() + PallocHdrSize; - return MakeString(TStringBuf(x)); - } else { - auto x = (const text*)((const char*)value.AsBoxed().Get() + PallocHdrSize); - return MakeString(GetVarBuf(x)); - } - default: - Y_UNREACHABLE(); - } + return ConvertFromPgValue<Slot, IsCString>(value); } private: @@ -1059,28 +1157,7 @@ public: return value.Release(); } - switch (Slot) - { - case NUdf::EDataSlot::Bool: - return ScalarDatumToPod(BoolGetDatum(value.Get<bool>())); - case NUdf::EDataSlot::Int16: - return ScalarDatumToPod(Int16GetDatum(value.Get<i16>())); - case NUdf::EDataSlot::Int32: - return ScalarDatumToPod(Int32GetDatum(value.Get<i32>())); - case NUdf::EDataSlot::Int64: - return ScalarDatumToPod(Int64GetDatum(value.Get<i64>())); - case NUdf::EDataSlot::Float: - return ScalarDatumToPod(Float4GetDatum(value.Get<float>())); - case NUdf::EDataSlot::Double: - return ScalarDatumToPod(Float8GetDatum(value.Get<double>())); - case NUdf::EDataSlot::String: - case NUdf::EDataSlot::Utf8: { - const auto& ref = value.AsStringRef(); - return PointerDatumToPod((Datum)MakeVar(ref)); - } - default: - Y_UNREACHABLE(); - } + return ConvertToPgValue<Slot>(value); } private: @@ -1513,29 +1590,42 @@ TString PgValueToNativeText(const NUdf::TUnboxedValuePod& value, ui32 pgTypeId) outFuncId = NPg::LookupProc("array_out", { 0 }).ProcId; } - FmgrInfo finfo; - Zero(finfo); - Y_ENSURE(outFuncId); - fmgr_info(outFuncId, &finfo); - Y_ENSURE(!finfo.fn_retset); - Y_ENSURE(finfo.fn_addr); - Y_ENSURE(finfo.fn_nargs == 1); - LOCAL_FCINFO(callInfo, 1); - Zero(*callInfo); - callInfo->flinfo = &finfo; - callInfo->nargs = 1; - callInfo->fncollation = DEFAULT_COLLATION_OID; - callInfo->isnull = false; - callInfo->args[0] = { typeInfo.PassByValue ? - ScalarDatumFromPod(value) : - PointerDatumFromPod(value), false }; - auto str = (char*)finfo.fn_addr(callInfo); - Y_ENSURE(!callInfo->isnull); - Y_DEFER{ - pfree(str); - }; + PG_TRY(); + { + FmgrInfo finfo; + Zero(finfo); + Y_ENSURE(outFuncId); + fmgr_info(outFuncId, &finfo); + Y_ENSURE(!finfo.fn_retset); + Y_ENSURE(finfo.fn_addr); + Y_ENSURE(finfo.fn_nargs == 1); + LOCAL_FCINFO(callInfo, 1); + Zero(*callInfo); + callInfo->flinfo = &finfo; + callInfo->nargs = 1; + callInfo->fncollation = DEFAULT_COLLATION_OID; + callInfo->isnull = false; + callInfo->args[0] = { typeInfo.PassByValue ? + ScalarDatumFromPod(value) : + PointerDatumFromPod(value), false }; + auto str = (char*)finfo.fn_addr(callInfo); + Y_ENSURE(!callInfo->isnull); + Y_DEFER{ + pfree(str); + }; - return TString(str); + return TString(str); + } + PG_CATCH(); + { + auto error_data = CopyErrorData(); + TStringBuilder errMsg; + errMsg << "Error in 'out' function: " << NPg::LookupProc(outFuncId).Name << ", reason: " << error_data->message; + FreeErrorData(error_data); + FlushErrorState(); + UdfTerminate(errMsg.c_str()); + } + PG_END_TRY(); } template <typename F> @@ -1555,32 +1645,45 @@ void PgValueToNativeBinaryImpl(const NUdf::TUnboxedValuePod& value, ui32 pgTypeI sendFuncId = NPg::LookupProc("array_send", { 0 }).ProcId; } - FmgrInfo finfo; - Zero(finfo); - Y_ENSURE(sendFuncId); - fmgr_info(sendFuncId, &finfo); - Y_ENSURE(!finfo.fn_retset); - Y_ENSURE(finfo.fn_addr); - Y_ENSURE(finfo.fn_nargs == 1); - LOCAL_FCINFO(callInfo, 1); - Zero(*callInfo); - callInfo->flinfo = &finfo; - callInfo->nargs = 1; - callInfo->fncollation = DEFAULT_COLLATION_OID; - callInfo->isnull = false; - callInfo->args[0] = { typeInfo.PassByValue ? - ScalarDatumFromPod(value) : - PointerDatumFromPod(value), false }; - - auto x = (text*)finfo.fn_addr(callInfo); - Y_ENSURE(!callInfo->isnull); - Y_DEFER{ - pfree(x); - }; + PG_TRY(); + { + FmgrInfo finfo; + Zero(finfo); + Y_ENSURE(sendFuncId); + fmgr_info(sendFuncId, &finfo); + Y_ENSURE(!finfo.fn_retset); + Y_ENSURE(finfo.fn_addr); + Y_ENSURE(finfo.fn_nargs == 1); + LOCAL_FCINFO(callInfo, 1); + Zero(*callInfo); + callInfo->flinfo = &finfo; + callInfo->nargs = 1; + callInfo->fncollation = DEFAULT_COLLATION_OID; + callInfo->isnull = false; + callInfo->args[0] = { typeInfo.PassByValue ? + ScalarDatumFromPod(value) : + PointerDatumFromPod(value), false }; + + auto x = (text*)finfo.fn_addr(callInfo); + Y_ENSURE(!callInfo->isnull); + Y_DEFER{ + pfree(x); + }; - auto s = GetVarBuf(x); - ui32 len = s.Size(); - f(TStringBuf(s.Data(), s.Size())); + auto s = GetVarBuf(x); + ui32 len = s.Size(); + f(TStringBuf(s.Data(), s.Size())); + } + PG_CATCH(); + { + auto error_data = CopyErrorData(); + TStringBuilder errMsg; + errMsg << "Error in 'send' function: " << NPg::LookupProc(sendFuncId).Name << ", reason: " << error_data->message; + FreeErrorData(error_data); + FlushErrorState(); + UdfTerminate(errMsg.c_str()); + } + PG_END_TRY(); } TString PgValueToNativeBinary(const NUdf::TUnboxedValuePod& value, ui32 pgTypeId) { @@ -1768,27 +1871,40 @@ NUdf::TUnboxedValue PgValueFromNativeBinary(const TStringBuf binary, ui32 pgType receiveFuncId = NPg::LookupProc("array_recv", { 0,0,0 }).ProcId; } - FmgrInfo finfo; - Zero(finfo); - Y_ENSURE(receiveFuncId); - fmgr_info(receiveFuncId, &finfo); - Y_ENSURE(!finfo.fn_retset); - Y_ENSURE(finfo.fn_addr); - Y_ENSURE(finfo.fn_nargs >= 1 && finfo.fn_nargs <= 3); - LOCAL_FCINFO(callInfo, 3); - Zero(*callInfo); - callInfo->flinfo = &finfo; - callInfo->nargs = 3; - callInfo->fncollation = DEFAULT_COLLATION_OID; - callInfo->isnull = false; - callInfo->args[0] = { (Datum)&stringInfo, false }; - callInfo->args[1] = { ObjectIdGetDatum(typeIOParam), false }; - callInfo->args[2] = { Int32GetDatum(-1), false }; - - auto x = finfo.fn_addr(callInfo); - Y_ENSURE(!callInfo->isnull); - Y_ENSURE(stringInfo.cursor == stringInfo.len); - return typeInfo.PassByValue ? ScalarDatumToPod(x) : PointerDatumToPod(x); + PG_TRY(); + { + FmgrInfo finfo; + Zero(finfo); + Y_ENSURE(receiveFuncId); + fmgr_info(receiveFuncId, &finfo); + Y_ENSURE(!finfo.fn_retset); + Y_ENSURE(finfo.fn_addr); + Y_ENSURE(finfo.fn_nargs >= 1 && finfo.fn_nargs <= 3); + LOCAL_FCINFO(callInfo, 3); + Zero(*callInfo); + callInfo->flinfo = &finfo; + callInfo->nargs = 3; + callInfo->fncollation = DEFAULT_COLLATION_OID; + callInfo->isnull = false; + callInfo->args[0] = { (Datum)&stringInfo, false }; + callInfo->args[1] = { ObjectIdGetDatum(typeIOParam), false }; + callInfo->args[2] = { Int32GetDatum(-1), false }; + + auto x = finfo.fn_addr(callInfo); + Y_ENSURE(!callInfo->isnull); + Y_ENSURE(stringInfo.cursor == stringInfo.len); + return typeInfo.PassByValue ? ScalarDatumToPod(x) : PointerDatumToPod(x); + } + PG_CATCH(); + { + auto error_data = CopyErrorData(); + TStringBuilder errMsg; + errMsg << "Error in 'recv' function: " << NPg::LookupProc(receiveFuncId).Name << ", reason: " << error_data->message; + FreeErrorData(error_data); + FlushErrorState(); + UdfTerminate(errMsg.c_str()); + } + PG_END_TRY(); } NUdf::TUnboxedValue PgValueFromNativeText(const TStringBuf text, ui32 pgTypeId) { @@ -1802,26 +1918,39 @@ NUdf::TUnboxedValue PgValueFromNativeText(const TStringBuf text, ui32 pgTypeId) inFuncId = NPg::LookupProc("array_in", { 0,0,0 }).ProcId; } - FmgrInfo finfo; - Zero(finfo); - Y_ENSURE(inFuncId); - fmgr_info(inFuncId, &finfo); - Y_ENSURE(!finfo.fn_retset); - Y_ENSURE(finfo.fn_addr); - Y_ENSURE(finfo.fn_nargs >= 1 && finfo.fn_nargs <= 3); - LOCAL_FCINFO(callInfo, 3); - Zero(*callInfo); - callInfo->flinfo = &finfo; - callInfo->nargs = 3; - callInfo->fncollation = DEFAULT_COLLATION_OID; - callInfo->isnull = false; - callInfo->args[0] = { (Datum)str.c_str(), false }; - callInfo->args[1] = { ObjectIdGetDatum(typeIOParam), false }; - callInfo->args[2] = { Int32GetDatum(-1), false }; - - auto x = finfo.fn_addr(callInfo); - Y_ENSURE(!callInfo->isnull); - return typeInfo.PassByValue ? ScalarDatumToPod(x) : PointerDatumToPod(x); + PG_TRY(); + { + FmgrInfo finfo; + Zero(finfo); + Y_ENSURE(inFuncId); + fmgr_info(inFuncId, &finfo); + Y_ENSURE(!finfo.fn_retset); + Y_ENSURE(finfo.fn_addr); + Y_ENSURE(finfo.fn_nargs >= 1 && finfo.fn_nargs <= 3); + LOCAL_FCINFO(callInfo, 3); + Zero(*callInfo); + callInfo->flinfo = &finfo; + callInfo->nargs = 3; + callInfo->fncollation = DEFAULT_COLLATION_OID; + callInfo->isnull = false; + callInfo->args[0] = { (Datum)str.c_str(), false }; + callInfo->args[1] = { ObjectIdGetDatum(typeIOParam), false }; + callInfo->args[2] = { Int32GetDatum(-1), false }; + + auto x = finfo.fn_addr(callInfo); + Y_ENSURE(!callInfo->isnull); + return typeInfo.PassByValue ? ScalarDatumToPod(x) : PointerDatumToPod(x); + } + PG_CATCH(); + { + auto error_data = CopyErrorData(); + TStringBuilder errMsg; + errMsg << "Error in 'in' function: " << NPg::LookupProc(inFuncId).Name << ", reason: " << error_data->message; + FreeErrorData(error_data); + FlushErrorState(); + UdfTerminate(errMsg.c_str()); + } + PG_END_TRY(); } NUdf::TUnboxedValue PgValueFromString(const TStringBuf s, ui32 pgTypeId) { @@ -2603,6 +2732,51 @@ void PgReleaseThreadContext(void* ctx) { } // namespace NMiniKQL } // namespace NKikimr +namespace NYql { + +class TPgBuilderImpl : public NUdf::IPgBuilder { +public: + NUdf::TUnboxedValue ValueFromText(ui32 typeId, const NUdf::TStringRef& value, NUdf::TStringValue& error) const override { + try { + return NCommon::PgValueFromNativeText(static_cast<TStringBuf>(value), typeId); + } catch (const std::exception& e) { + error = NUdf::TStringValue(TStringBuf(e.what())); + } + return NUdf::TUnboxedValue(); + } + + NUdf::TUnboxedValue ValueFromBinary(ui32 typeId, const NUdf::TStringRef& value, NUdf::TStringValue& error) const override { + try { + return NCommon::PgValueFromNativeBinary(static_cast<TStringBuf>(value), typeId); + } catch (const std::exception& e) { + error = NUdf::TStringValue(TStringBuf(e.what())); + } + return NUdf::TUnboxedValue(); + } + + NUdf::TUnboxedValue ConvertFromPg(NUdf::TUnboxedValue source, ui32 sourceTypeId, NUdf::TType* targetType) const override { + return ConvertFromPgValue(source, sourceTypeId, static_cast<NKikimr::NMiniKQL::TType*>(targetType)); + } + + NUdf::TUnboxedValue ConvertToPg(NUdf::TUnboxedValue source, NUdf::TType* sourceType, ui32 targetTypeId) const override { + return ConvertToPgValue(source, static_cast<NKikimr::NMiniKQL::TType*>(sourceType), targetTypeId); + } + + NUdf::TUnboxedValue NewString(i32 typeLen, ui32 targetTypeId, NUdf::TStringRef data) const override { + // todo: implement + Y_UNUSED(typeLen); + Y_UNUSED(targetTypeId); + Y_UNUSED(data); + return NUdf::TUnboxedValue(); + } +}; + +std::unique_ptr<NUdf::IPgBuilder> CreatePgBuilder() { + return std::make_unique<TPgBuilderImpl>(); +} + +} // namespace NYql + extern "C" { void yql_canonize_float4(float4* x) { @@ -2647,6 +2821,4 @@ void get_type_io_data(Oid typid, } } -} - - +} // extern "C" diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.h b/ydb/library/yql/parser/pg_wrapper/comp_factory.h index 9f27db24a92..8e17a3134b0 100644 --- a/ydb/library/yql/parser/pg_wrapper/comp_factory.h +++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.h @@ -7,4 +7,3 @@ namespace NYql { NKikimr::NMiniKQL::TComputationNodeFactory GetPgFactory(); } - diff --git a/ydb/library/yql/public/udf/udf_counter.h b/ydb/library/yql/public/udf/udf_counter.h index 5c09dfe2347..2350b597785 100644 --- a/ydb/library/yql/public/udf/udf_counter.h +++ b/ydb/library/yql/public/udf/udf_counter.h @@ -1,5 +1,6 @@ #pragma once #include "udf_types.h" +#include "udf_type_size_check.h" #include <library/cpp/deprecated/atomic/atomic.h> namespace NYql { diff --git a/ydb/library/yql/public/udf/udf_pg_type_description.h b/ydb/library/yql/public/udf/udf_pg_type_description.h new file mode 100644 index 00000000000..859dba6c54d --- /dev/null +++ b/ydb/library/yql/public/udf/udf_pg_type_description.h @@ -0,0 +1,28 @@ +#pragma once + +#include "udf_string_ref.h" +#include "udf_type_size_check.h" +#include "udf_version.h" + +namespace NYql { +namespace NUdf { + +////////////////////////////////////////////////////////////////////////////// +// TPgTypeDescription +////////////////////////////////////////////////////////////////////////////// + +struct TPgTypeDescription1 { + TStringRef Name; + ui32 TypeId = 0; + i32 Typelen = 0; + ui32 ArrayTypeId = 0; + ui32 ElementTypeId = 0; + bool PassByValue = false; +}; + +using TPgTypeDescription = TPgTypeDescription1; + +UDF_ASSERT_TYPE_SIZE(TPgTypeDescription1, 40); + +} // namspace NUdf +} // namspace NYql diff --git a/ydb/library/yql/public/udf/udf_registrator.h b/ydb/library/yql/public/udf/udf_registrator.h index 4d195d7eda6..0bc0647257a 100644 --- a/ydb/library/yql/public/udf/udf_registrator.h +++ b/ydb/library/yql/public/udf/udf_registrator.h @@ -4,6 +4,7 @@ #include "udf_types.h" #include "udf_ptr.h" #include "udf_string.h" +#include "udf_type_size_check.h" #include "udf_value.h" #include <functional> diff --git a/ydb/library/yql/public/udf/udf_string.h b/ydb/library/yql/public/udf/udf_string.h index 7c687ffbf48..34f74a69a5b 100644 --- a/ydb/library/yql/public/udf/udf_string.h +++ b/ydb/library/yql/public/udf/udf_string.h @@ -3,6 +3,7 @@ #include "udf_types.h" #include "udf_allocator.h" #include "udf_string_ref.h" +#include "udf_type_size_check.h" #include <util/generic/strbuf.h> #include <util/system/align.h> @@ -106,6 +107,9 @@ class TStringValue public: inline explicit TStringValue(ui32 size): Data_(AllocateData(size, size)) {} inline explicit TStringValue(TData* data): Data_(data) { Ref(); } + inline explicit TStringValue(TStringRef data): Data_(AllocateData(data.Size(), data.Size())) { + memcpy(Data_->Data(), data.Data(), data.Size()); + } inline TStringValue(const TStringValue& rhs): Data_(rhs.Data_) { Ref(); } inline TStringValue(TStringValue&& rhs): Data_(rhs.Data_) { rhs.Data_ = nullptr; } diff --git a/ydb/library/yql/public/udf/udf_string_ref.h b/ydb/library/yql/public/udf/udf_string_ref.h index 75a82b81e6a..3c48fd0c64c 100644 --- a/ydb/library/yql/public/udf/udf_string_ref.h +++ b/ydb/library/yql/public/udf/udf_string_ref.h @@ -1,6 +1,6 @@ #pragma once -#include "udf_types.h" +#include "udf_type_size_check.h" #include <util/generic/strbuf.h> diff --git a/ydb/library/yql/public/udf/udf_type_builder.h b/ydb/library/yql/public/udf/udf_type_builder.h index 8bfc73a449d..5954430577d 100644 --- a/ydb/library/yql/public/udf/udf_type_builder.h +++ b/ydb/library/yql/public/udf/udf_type_builder.h @@ -4,6 +4,7 @@ #include "udf_types.h" #include "udf_ptr.h" #include "udf_string_ref.h" +#include "udf_type_size_check.h" #include "udf_value.h" #include <type_traits> @@ -57,6 +58,9 @@ struct TStream { using ItemType = T; }; template <typename T, const char* Tag> struct TTagged { using BaseType = T; }; +template <ui32 TypeId> +struct TPg; + ////////////////////////////////////////////////////////////////////////////// // ITypeBuilder ////////////////////////////////////////////////////////////////////////////// @@ -583,7 +587,16 @@ public: }; #endif -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 22) +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +class IFunctionTypeInfoBuilder13: public IFunctionTypeInfoBuilder12 { +public: + virtual TType* Pg(ui32 typeId) const = 0; +}; +#endif + +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +using IFunctionTypeInfoBuilderImpl = IFunctionTypeInfoBuilder13; +#elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 22) using IFunctionTypeInfoBuilderImpl = IFunctionTypeInfoBuilder12; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) using IFunctionTypeInfoBuilderImpl = IFunctionTypeInfoBuilder11; @@ -809,6 +822,15 @@ struct TTypeBuilderHelper<TTagged<T, Tag>> { }; #endif +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +template <ui32 TypeId> +struct TTypeBuilderHelper<TPg<TypeId>> { + static TType* Build(const IFunctionTypeInfoBuilder& builder) { + return builder.Pg(TypeId); + } +}; +#endif + template <> struct TCallableArgsHelper<> { static void Arg( diff --git a/ydb/library/yql/public/udf/udf_type_inspection.h b/ydb/library/yql/public/udf/udf_type_inspection.h index f11ea0ff072..a940cb2d429 100644 --- a/ydb/library/yql/public/udf/udf_type_inspection.h +++ b/ydb/library/yql/public/udf/udf_type_inspection.h @@ -59,7 +59,17 @@ public: }; #endif -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +class TStubTypeVisitor5: public TStubTypeVisitor4 +{ +public: + void OnPg(ui32 typeId) override; +}; +#endif + +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +using TStubTypeVisitor = TStubTypeVisitor5; +#elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) using TStubTypeVisitor = TStubTypeVisitor4; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15) using TStubTypeVisitor = TStubTypeVisitor3; @@ -445,6 +455,31 @@ private: }; #endif +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +////////////////////////////////////////////////////////////////////////////// +// TPgTypeInspector +////////////////////////////////////////////////////////////////////////////// +class TPgTypeInspector: public TStubTypeVisitor +{ +public: + TPgTypeInspector(const ITypeInfoHelper& typeHelper, const TType* type) { + if (typeHelper.GetTypeKind(type) == ETypeKind::Pg) { + typeHelper.VisitType(type, this); + } + } + + explicit operator bool() const { return TypeId_ != 0; } + ui32 GetTypeId() const { return TypeId_; } + +private: + void OnPg(ui32 typeId) override { + TypeId_ = typeId; + } + + ui32 TypeId_ = 0; +}; +#endif + inline void TStubTypeVisitor1::OnDataType(TDataTypeId typeId) { Y_UNUSED(typeId); @@ -520,6 +555,11 @@ inline void TStubTypeVisitor4::OnTagged(const TType*, TStringRef) { Y_FAIL("Not implemented"); } #endif +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +inline void TStubTypeVisitor5::OnPg(ui32) { + Y_FAIL("Not implemented"); +} +#endif } // namspace NUdf } // namspace NYql diff --git a/ydb/library/yql/public/udf/udf_type_printer.cpp b/ydb/library/yql/public/udf/udf_type_printer.cpp index ceb13b783ca..64c120e4a20 100644 --- a/ydb/library/yql/public/udf/udf_type_printer.cpp +++ b/ydb/library/yql/public/udf/udf_type_printer.cpp @@ -7,8 +7,8 @@ namespace { static thread_local IOutputStream * Output_ = nullptr; } -TTypePrinter1::TTypePrinter1(const ITypeInfoHelper& typeHelper, const TType* type) - : TypeHelper_(typeHelper), Type_(type) +TTypePrinter1::TTypePrinter1(const ITypeInfoHelper1& typeHelper, const TType* type) + : TypeHelper1_(typeHelper), Type_(type) {} void TTypePrinter1::Out(IOutputStream &o) const { @@ -18,13 +18,13 @@ void TTypePrinter1::Out(IOutputStream &o) const { } void TTypePrinter1::OutImpl(const TType* type) const { - switch (TypeHelper_.GetTypeKind(type)) { + switch (TypeHelper1_.GetTypeKind(type)) { case ETypeKind::Null: *Output_ << "Null"; break; case ETypeKind::Void: *Output_ << "Void"; break; case ETypeKind::EmptyList: *Output_ << "EmptyList"; break; case ETypeKind::EmptyDict: *Output_ << "EmptyDict"; break; case ETypeKind::Unknown: *Output_ << "Unknown"; break; - default: TypeHelper_.VisitType(type, const_cast<TTypePrinter1*>(this)); + default: TypeHelper1_.VisitType(type, const_cast<TTypePrinter1*>(this)); } } @@ -117,5 +117,16 @@ void TTypePrinter1::OnTaggedImpl(const TType* baseType, TStringRef tag) { *Output_ << ",'" << std::string_view(tag) << "'>"; } +TTypePrinter5::TTypePrinter5(const ITypeInfoHelper2& typeHelper2, const TType* type) + : TTypePrinter4(typeHelper2, type) + , TypeHelper2_(typeHelper2) +{} + +void TTypePrinter5::OnPgImpl(ui32 typeId) { + auto* description = TypeHelper2_.FindPgTypeDescription(typeId); + Y_VERIFY(description); + *Output_ << std::string_view(description->Name); +} + } } diff --git a/ydb/library/yql/public/udf/udf_type_printer.h b/ydb/library/yql/public/udf/udf_type_printer.h index 749118c9bc8..a50d1175d8a 100644 --- a/ydb/library/yql/public/udf/udf_type_printer.h +++ b/ydb/library/yql/public/udf/udf_type_printer.h @@ -9,7 +9,7 @@ namespace NUdf { class TTypePrinter1 : private ITypeVisitor { public: - TTypePrinter1(const ITypeInfoHelper& typeHelper, const TType* type); + TTypePrinter1(const ITypeInfoHelper1& typeHelper, const TType* type); void Out(IOutputStream &o) const; @@ -28,7 +28,7 @@ protected: void OnResourceImpl(TStringRef tag); void OnTaggedImpl(const TType* baseType, TStringRef tag); - const ITypeInfoHelper& TypeHelper_; + const ITypeInfoHelper1& TypeHelper1_; const TType* Type_; }; @@ -68,7 +68,28 @@ protected: }; #endif -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +class TTypePrinter5 : public TTypePrinter4 { +public: + using TTypePrinter4::TTypePrinter4; + TTypePrinter5(const ITypeInfoHelper2& typeHelper, const TType* type); + +protected: + void OnPg(ui32 typeId) final { + OnPgImpl(typeId); + } + +private: + void OnPgImpl(ui32 typeId); + +private: + const ITypeInfoHelper2& TypeHelper2_; +}; +#endif + +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +using TTypePrinter = TTypePrinter5; +#elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) using TTypePrinter = TTypePrinter4; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15) using TTypePrinter = TTypePrinter3; diff --git a/ydb/library/yql/public/udf/udf_type_size_check.h b/ydb/library/yql/public/udf/udf_type_size_check.h new file mode 100644 index 00000000000..1433abf3620 --- /dev/null +++ b/ydb/library/yql/public/udf/udf_type_size_check.h @@ -0,0 +1,7 @@ +#pragma once + +#define UDF_ASSERT_TYPE_SIZE(type, expectedSize) \ + static_assert(sizeof(type) <= (expectedSize), \ + "Size of " #type " mismatches expected size. " \ + "Expected size is " #expectedSize) + diff --git a/ydb/library/yql/public/udf/udf_types.h b/ydb/library/yql/public/udf/udf_types.h index c2b67c7cbbc..11298a78822 100644 --- a/ydb/library/yql/public/udf/udf_types.h +++ b/ydb/library/yql/public/udf/udf_types.h @@ -2,17 +2,38 @@ #include "udf_ptr.h" #include "udf_data_type.h" +#include "udf_pg_type_description.h" +#include "udf_type_size_check.h" #include "udf_version.h" namespace NYql { namespace NUdf { -class TStringRef; - // opaque type info using TType = void; -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) + +#define UDF_TYPE_KIND_MAP(XX) \ + XX(Unknown) \ + XX(Data) \ + XX(Struct) \ + XX(List) \ + XX(Optional) \ + XX(Tuple) \ + XX(Dict) \ + XX(Callable) \ + XX(Resource) \ + XX(Void) \ + XX(Variant) \ + XX(Stream) \ + XX(Null) \ + XX(EmptyList) \ + XX(EmptyDict) \ + XX(Tagged) \ + XX(Pg) + +#elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) #define UDF_TYPE_KIND_MAP(XX) \ XX(Unknown) \ @@ -75,11 +96,6 @@ enum ETypeKind ENUM_TO_STRING(ETypeKind, UDF_TYPE_KIND_MAP) -#define UDF_ASSERT_TYPE_SIZE(type, expectedSize) \ - static_assert(sizeof(type) == (expectedSize), \ - "Size of " #type " exceeds expected size. " \ - "Expected size is " #expectedSize) - ////////////////////////////////////////////////////////////////////////////// // ICallablePayload ////////////////////////////////////////////////////////////////////////////// @@ -160,7 +176,16 @@ public: }; #endif -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +class ITypeVisitor5: public ITypeVisitor4 { +public: + virtual void OnPg(ui32 typeId) = 0; +}; +#endif + +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +using ITypeVisitor = ITypeVisitor5; +#elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) using ITypeVisitor = ITypeVisitor4; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15) using ITypeVisitor = ITypeVisitor3; @@ -175,13 +200,13 @@ UDF_ASSERT_TYPE_SIZE(ITypeVisitor, 16); ////////////////////////////////////////////////////////////////////////////// // ITypeInfoHelper ////////////////////////////////////////////////////////////////////////////// -class ITypeInfoHelper +class ITypeInfoHelper1 { public: - using TPtr = TRefCountedPtr<ITypeInfoHelper>; + using TPtr = TRefCountedPtr<ITypeInfoHelper1>; public: - virtual ~ITypeInfoHelper() = default; + virtual ~ITypeInfoHelper1() = default; virtual ETypeKind GetTypeKind(const TType* type) const = 0; virtual void VisitType(const TType* type, ITypeVisitor* visitor) const = 0; @@ -212,6 +237,20 @@ private: } }; +class ITypeInfoHelper2 : public ITypeInfoHelper1 { +public: + using TPtr = TRefCountedPtr<ITypeInfoHelper2>; + +public: + virtual const TPgTypeDescription* FindPgTypeDescription(ui32 typeId) const = 0; +}; + +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +using ITypeInfoHelper = ITypeInfoHelper2; +#else +using ITypeInfoHelper = ITypeInfoHelper1; +#endif + UDF_ASSERT_TYPE_SIZE(ITypeInfoHelper, 16); UDF_ASSERT_TYPE_SIZE(ITypeInfoHelper::TPtr, 8); diff --git a/ydb/library/yql/public/udf/udf_value.h b/ydb/library/yql/public/udf/udf_value.h index d57114bec14..a8ae44ce940 100644 --- a/ydb/library/yql/public/udf/udf_value.h +++ b/ydb/library/yql/public/udf/udf_value.h @@ -3,6 +3,7 @@ #include "udf_allocator.h" #include "udf_string.h" #include "udf_terminator.h" +#include "udf_type_size_check.h" #include "udf_version.h" #include <ydb/library/yql/public/decimal/yql_decimal.h> diff --git a/ydb/library/yql/public/udf/udf_value_builder.h b/ydb/library/yql/public/udf/udf_value_builder.h index 096f735a2b1..344fbd1d121 100644 --- a/ydb/library/yql/public/udf/udf_value_builder.h +++ b/ydb/library/yql/public/udf/udf_value_builder.h @@ -4,6 +4,7 @@ #include "udf_types.h" #include "udf_type_builder.h" #include "udf_string.h" +#include "udf_type_size_check.h" #include "udf_value.h" #include <array> @@ -103,6 +104,20 @@ class IDateBuilder: public IDateBuilder1 {}; UDF_ASSERT_TYPE_SIZE(IDateBuilder, 8); /////////////////////////////////////////////////////////////////////////////// +// IPgBuilder +/////////////////////////////////////////////////////////////////////////////// +class IPgBuilder { +public: + virtual ~IPgBuilder() {} + virtual TUnboxedValue ValueFromText(ui32 typeId, const TStringRef& value, TStringValue& error) const = 0; + virtual TUnboxedValue ValueFromBinary(ui32 typeId, const TStringRef& value, TStringValue& error) const = 0; + virtual TUnboxedValue ConvertFromPg(TUnboxedValue source, ui32 sourceTypeId, TType* targetType) const = 0; + virtual TUnboxedValue ConvertToPg(TUnboxedValue source, TType* sourceType, ui32 targetTypeId) const = 0; + virtual TUnboxedValue NewString(i32 typeLen, ui32 targetTypeId, TStringRef data) const = 0; +}; +UDF_ASSERT_TYPE_SIZE(IPgBuilder, 8); + +/////////////////////////////////////////////////////////////////////////////// // IValueBuilder /////////////////////////////////////////////////////////////////////////////// class IValueBuilder1 @@ -178,7 +193,16 @@ public: }; #endif -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 19) +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +class IValueBuilder6: public IValueBuilder5 { +public: + virtual const IPgBuilder& GetPgBuilder() const = 0; +}; +#endif + +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) +class IValueBuilder: public IValueBuilder6 {}; +#elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 19) class IValueBuilder: public IValueBuilder5 {}; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 17) class IValueBuilder: public IValueBuilder4 {}; diff --git a/ydb/library/yql/public/udf/udf_version.h b/ydb/library/yql/public/udf/udf_version.h index e31e518d44f..35735928439 100644 --- a/ydb/library/yql/public/udf/udf_version.h +++ b/ydb/library/yql/public/udf/udf_version.h @@ -7,7 +7,7 @@ namespace NYql { namespace NUdf { #define CURRENT_UDF_ABI_VERSION_MAJOR 2 -#define CURRENT_UDF_ABI_VERSION_MINOR 24 +#define CURRENT_UDF_ABI_VERSION_MINOR 25 #define CURRENT_UDF_ABI_VERSION_PATCH 0 #ifdef USE_CURRENT_UDF_ABI_VERSION diff --git a/ydb/library/yql/sql/pg_dummy/CMakeLists.txt b/ydb/library/yql/sql/pg_dummy/CMakeLists.txt index 4a8548ba6f2..e8f7d109a06 100644 --- a/ydb/library/yql/sql/pg_dummy/CMakeLists.txt +++ b/ydb/library/yql/sql/pg_dummy/CMakeLists.txt @@ -15,11 +15,12 @@ target_link_libraries(yql-sql-pg_dummy PUBLIC contrib-libs-cxxsupp yutil library-yql-ast - yql-sql-settings - providers-common-codec - yql-minikql-computation - library-yql-minikql library-yql-core + library-yql-minikql + yql-minikql-computation + providers-common-codec + yql-public-udf + yql-sql-settings ) target_sources(yql-sql-pg_dummy PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp diff --git a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp index 6be057fc6da..0b47dc6e420 100644 --- a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp +++ b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp @@ -4,6 +4,7 @@ #include <ydb/library/yql/minikql/computation/presort_impl.h> #include <ydb/library/yql/core/yql_pg_utils.h> #include <ydb/library/yql/minikql/mkql_type_builder.h> +#include <ydb/library/yql/public/udf/udf_value_builder.h> namespace NSQLTranslationPG { @@ -209,4 +210,46 @@ bool ParsePgIntervalModifier(const TString& str, i32& ret) { return false; } +class TPgDummyBuilder : public NUdf::IPgBuilder { +public: + NUdf::TUnboxedValue ValueFromText(ui32 typeId, const NUdf::TStringRef& value, NUdf::TStringValue& error) const override { + Y_UNUSED(typeId); + Y_UNUSED(value); + error = NUdf::TStringValue(TStringBuf("TPgDummyBuilder::ValueFromText does nothing")); + return NUdf::TUnboxedValue(); + } + + NUdf::TUnboxedValue ValueFromBinary(ui32 typeId, const NUdf::TStringRef& value, NUdf::TStringValue& error) const override { + Y_UNUSED(typeId); + Y_UNUSED(value); + error = NUdf::TStringValue(TStringBuf("TPgDummyBuilder::ValueFromBinary does nothing")); + return NUdf::TUnboxedValue(); + } + + NUdf::TUnboxedValue ConvertFromPg(NUdf::TUnboxedValue source, ui32 sourceTypeId, NUdf::TType* targetType) const override { + Y_UNUSED(source); + Y_UNUSED(sourceTypeId); + Y_UNUSED(targetType); + ythrow yexception() << "TPgDummyBuilder::ConvertFromPg does nothing"; + } + + NUdf::TUnboxedValue ConvertToPg(NUdf::TUnboxedValue source, NUdf::TType* sourceType, ui32 targetTypeId) const override { + Y_UNUSED(source); + Y_UNUSED(sourceType); + Y_UNUSED(targetTypeId); + ythrow yexception() << "TPgDummyBuilder::ConvertToPg does nothing"; + } + + NUdf::TUnboxedValue NewString(i32 typeLen, ui32 targetTypeId, NUdf::TStringRef data) const override { + Y_UNUSED(typeLen); + Y_UNUSED(targetTypeId); + Y_UNUSED(data); + ythrow yexception() << "TPgDummyBuilder::NewString does nothing"; + } +}; + +std::unique_ptr<NUdf::IPgBuilder> CreatePgBuilder() { + return std::make_unique<TPgDummyBuilder>(); +} + } // NYql |