diff options
author | vvvv <vvvv@ydb.tech> | 2023-04-24 16:43:04 +0300 |
---|---|---|
committer | vvvv <vvvv@ydb.tech> | 2023-04-24 16:43:04 +0300 |
commit | 03289407980859c479993e521fbd90b32ba7abad (patch) | |
tree | df620c8c660c2f75456b84a6d8eab58a71a4353e | |
parent | f593cbec1a40e96fa6d4133d1324edcafa1e5213 (diff) | |
download | ydb-03289407980859c479993e521fbd90b32ba7abad.tar.gz |
Support of wide sort for all types, fixes for PG
faster arrays
arrays, tests
initial
23 files changed, 249 insertions, 102 deletions
diff --git a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp index 62b6b3c1dba..9077a8531f1 100644 --- a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp +++ b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp @@ -4029,25 +4029,22 @@ TExprNode::TPtr OptimizeTopOrSort(const TExprNode::TPtr& node, TExprContext& ctx } }); - const auto itemType = node->GetTypeAnn()->Cast<TFlowExprType>()->GetItemType()->Cast<TStructExprType>(); std::unordered_set<size_t> unique; std::vector<size_t> sorted; if (node->Tail().Tail().IsCallable("Member") && &node->Tail().Tail().Head() == &node->Tail().Head().Head()) { - if (const auto it = indexes.find(&node->Tail().Tail().Tail()); indexes.cend() == it) + if (const auto it = indexes.find(&node->Tail().Tail().Tail()); indexes.cend() == it) { return node; - else if (IsDataOrOptionalOfData(itemType->FindItemType(node->Tail().Tail().Tail().Content()))) + } else { sorted.emplace_back(it->second); - else - return node; + } } else if (node->Tail().Tail().IsList()) { for (const auto& field : node->Tail().Tail().Children()) { if (field->IsCallable("Member") && &field->Head() == &node->Tail().Head().Head()) - if (const auto it = indexes.find(&field->Tail()); indexes.cend() == it) + if (const auto it = indexes.find(&field->Tail()); indexes.cend() == it) { return node; - else if (IsDataOrOptionalOfData(itemType->FindItemType(field->Tail().Content()))) + } else { sorted.emplace_back(it->second); - else - return node; + } else return node; } diff --git a/ydb/library/yql/core/type_ann/type_ann_wide.cpp b/ydb/library/yql/core/type_ann/type_ann_wide.cpp index dcb0f8dce1b..f6f1c922e3b 100644 --- a/ydb/library/yql/core/type_ann/type_ann_wide.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_wide.cpp @@ -693,12 +693,7 @@ bool ValidateWideTopKeys(TExprNode& keys, const TTypeAnnotationNode::TListType& return false; } - const TDataExprType* type = nullptr; - if (bool opt; !EnsureDataOrOptionalOfData( item->Head().Pos(), types[index], opt, type, ctx)) { - return false; - } - - if (!EnsureComparableType(item->Pos(), *type, ctx)) { + if (!EnsureComparableType(item->Pos(), *types[index], ctx)) { return false; } } else { diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_block_reader.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_block_reader.cpp index 5fdcf2a7b9f..90b1c8047f1 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_block_reader.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_block_reader.cpp @@ -42,11 +42,9 @@ public: template<typename TStringType, bool Nullable, NUdf::EPgStringType PgString> class TStringBlockItemConverter : public IBlockItemConverter { public: - void SetPgBuilder(const NUdf::IPgBuilder* pgBuilder, i32 typeLen, ui32 pgTypeId) { + void SetPgBuilder(const NUdf::IPgBuilder* pgBuilder) { Y_ENSURE(PgString != NUdf::EPgStringType::None); PgBuilder = pgBuilder; - TypeLen = typeLen; - PgTypeId = pgTypeId; } NUdf::TUnboxedValuePod MakeValue(TBlockItem item, const THolderFactory& holderFactory) const final { @@ -57,8 +55,10 @@ public: } } - if constexpr (PgString != NUdf::EPgStringType::None) { - return PgBuilder->NewString(TypeLen, PgTypeId, item.AsStringRef()).Release(); + if constexpr (PgString == NUdf::EPgStringType::CString) { + return PgBuilder->MakeCString(item.AsStringRef().Data()).Release(); + } else if constexpr (PgString == NUdf::EPgStringType::Text) { + return PgBuilder->MakeText(item.AsStringRef().Data()).Release(); } else { return MakeString(item.AsStringRef()); } @@ -184,11 +184,11 @@ struct TConverterTraits { } else { if (desc.Typelen == -1) { auto ret = std::make_unique<TStrings<arrow::BinaryType, true, NUdf::EPgStringType::Text>>(); - ret->SetPgBuilder(pgBuilder, desc.Typelen, desc.TypeId); + ret->SetPgBuilder(pgBuilder); return ret; } else { auto ret = std::make_unique<TStrings<arrow::BinaryType, true, NUdf::EPgStringType::CString>>(); - ret->SetPgBuilder(pgBuilder, desc.Typelen, desc.TypeId); + ret->SetPgBuilder(pgBuilder); return ret; } } diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_wide_top_sort.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_wide_top_sort.cpp index 39aa47b86dd..8e361e4bcd0 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_wide_top_sort.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_wide_top_sort.cpp @@ -2,6 +2,7 @@ #include "mkql_llvm_base.h" #include <ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h> +#include <ydb/library/yql/minikql/computation/presort.h> #include <ydb/library/yql/minikql/mkql_node_builder.h> #include <ydb/library/yql/minikql/mkql_node_cast.h> #include <ydb/library/yql/minikql/defs.h> @@ -14,16 +15,56 @@ namespace NMiniKQL { namespace { +struct TKeyInfo { + NUdf::EDataSlot Slot; + bool IsOptional; + NUdf::ICompare::TPtr Compare; + TType* PresortType = nullptr; + std::optional<TGenericPresortEncoder> LeftPacker; + std::optional<TGenericPresortEncoder> RightPacker; +}; + struct TMyValueCompare { - TMyValueCompare(const TKeyTypes& types) - : Types(types) - {} + TMyValueCompare(const std::vector<TKeyInfo>& keys) + : Keys(keys) + { + for (auto& key : Keys) { + if (key.PresortType) { + key.LeftPacker.emplace(key.PresortType); + key.RightPacker.emplace(key.PresortType); + } + } + } int operator()(const bool* directions, const NUdf::TUnboxedValuePod* left, const NUdf::TUnboxedValuePod* right) const { - return CompareValues(left, right, Types, directions); + for (auto i = 0u; i < Keys.size(); ++i) { + auto& key = Keys[i]; + int cmp; + if (key.Compare) { + cmp = key.Compare->Compare(left[i], right[i]); + if (!directions[i]) { + cmp = -cmp; + } + } else if (key.LeftPacker) { + auto strLeft = key.LeftPacker->Encode(left[i], false); + auto strRight = key.RightPacker->Encode(right[i], false); + cmp = strLeft.compare(strRight); + if (!directions[i]) { + cmp = -cmp; + } + } else { + cmp = CompareValues(key.Slot, directions[i], key.IsOptional, left[i], right[i]); + } + + if (cmp) { + return cmp; + } + } + + return 0; } - const TKeyTypes& Types; + mutable std::vector<TKeyInfo> Keys; }; using TComparePtr = int(*)(const bool*, const NUdf::TUnboxedValuePod*, const NUdf::TUnboxedValuePod*); @@ -213,9 +254,21 @@ class TWideTopWrapper: public TStatefulWideFlowCodegeneratorNode<TWideTopWrapper { using TBaseComputation = TStatefulWideFlowCodegeneratorNode<TWideTopWrapper<Sort, HasCount>>; public: - TWideTopWrapper(TComputationMutables& mutables, IComputationWideFlowNode* flow, IComputationNode* count, TComputationNodePtrVector&& directions, TKeyTypes&& keyTypes, std::vector<ui32>&& indexes, std::vector<EValueRepresentation>&& representations) - : TBaseComputation(mutables, flow, EValueRepresentation::Boxed), Flow(flow), Count(count), Directions(std::move(directions)), KeyTypes(std::move(keyTypes)), Indexes(std::move(indexes)), Representations(std::move(representations)) - {} + TWideTopWrapper(TComputationMutables& mutables, IComputationWideFlowNode* flow, IComputationNode* count, TComputationNodePtrVector&& directions, std::vector<TKeyInfo>&& keys, + std::vector<ui32>&& indexes, std::vector<EValueRepresentation>&& representations) + : TBaseComputation(mutables, flow, EValueRepresentation::Boxed), Flow(flow), Count(count), Directions(std::move(directions)), Keys(std::move(keys)) + , Indexes(std::move(indexes)), Representations(std::move(representations)) + { + for (const auto& x : Keys) { + if (x.Compare || x.PresortType) { + KeyTypes.clear(); + HasComplexType = true; + break; + } + + KeyTypes.emplace_back(x.Slot, x.IsOptional); + } + } EFetchResult DoCalculate(NUdf::TUnboxedValue& state, TComputationContext& ctx, NUdf::TUnboxedValue*const* output) const { if (!state.HasValue()) { @@ -381,7 +434,7 @@ public: } } else { - for (auto i = 0U; i < KeyTypes.size(); ++i) { + for (auto i = 0U; i < Keys.size(); ++i) { const auto item = getres.second[Indexes[i]](ctx, block); new StoreInst(item, placeholders[i], block); } @@ -400,11 +453,11 @@ public: block = push; - for (auto i = 0U; i < KeyTypes.size(); ++i) { + for (auto i = 0U; i < Keys.size(); ++i) { ValueAddRef(Representations[i], placeholders[i], ctx, block); } - for (auto i = KeyTypes.size(); i < Representations.size(); ++i) { + for (auto i = Keys.size(); i < Representations.size(); ++i) { const auto item = getres.second[Indexes[i]](ctx, block); ValueAddRef(Representations[i], item, ctx, block); new StoreInst(item, placeholders[i], block); @@ -415,7 +468,7 @@ public: block = skip; - for (auto i = 0U; i < KeyTypes.size(); ++i) { + for (auto i = 0U; i < Keys.size(); ++i) { ValueCleanup(Representations[i], placeholders[i], ctx, block); new StoreInst(ConstantInt::get(valueType, 0), placeholders[i], block); } @@ -454,9 +507,9 @@ public: private: void MakeState(TComputationContext& ctx, NUdf::TUnboxedValue& state, ui64 count, const bool* directions) const { #ifdef MKQL_DISABLE_CODEGEN - state = ctx.HolderFactory.Create<TState<HasCount>>(count, directions, Directions.size(), TMyValueCompare(KeyTypes), Indexes); + state = ctx.HolderFactory.Create<TState<HasCount>>(count, directions, Directions.size(), TMyValueCompare(Keys), Indexes); #else - state = ctx.HolderFactory.Create<TState<HasCount>>(count, directions, Directions.size(), ctx.ExecuteLLVM && Compare ? TCompareFunc(Compare) : TCompareFunc(TMyValueCompare(KeyTypes)), Indexes); + state = ctx.HolderFactory.Create<TState<HasCount>>(count, directions, Directions.size(), ctx.ExecuteLLVM && Compare ? TCompareFunc(Compare) : TCompareFunc(TMyValueCompare(Keys)), Indexes); #endif } @@ -473,9 +526,12 @@ private: IComputationWideFlowNode *const Flow; IComputationNode *const Count; const TComputationNodePtrVector Directions; - const TKeyTypes KeyTypes; + const std::vector<TKeyInfo> Keys; const std::vector<ui32> Indexes; const std::vector<EValueRepresentation> Representations; + TKeyTypes KeyTypes; + bool HasComplexType = false; + #ifndef MKQL_DISABLE_CODEGEN TComparePtr Compare = nullptr; @@ -494,7 +550,9 @@ private: } void GenerateFunctions(const NYql::NCodegen::ICodegen::TPtr& codegen) final { - codegen->ExportSymbol(CompareFunc = GenerateCompareFunction(codegen, MakeName(), KeyTypes)); + if (!HasComplexType) { + codegen->ExportSymbol(CompareFunc = GenerateCompareFunction(codegen, MakeName(), KeyTypes)); + } } #endif }; @@ -519,16 +577,30 @@ IComputationNode* WrapWideTopT(TCallable& callable, const TComputationNodeFactor const auto inputWideComponents = GetWideComponents(AS_TYPE(TFlowType, callable.GetType()->GetReturnType())); std::vector<ui32> indexes(inputWideComponents.size()); - TKeyTypes keyTypes(keyWidth); - std::unordered_set<ui32> keyIndexes; - for (auto i = 0U; i < keyTypes.size(); ++i) { - const auto keyIndex = AS_VALUE(TDataLiteral, callable.GetInput(((i + 1U) << 1U) - offset))->AsValue().Get<ui32>(); - indexes[i] = keyIndex; + std::unordered_set<ui32> keyIndexes; + std::vector<TKeyInfo> keys(keyWidth); + for (auto i = 0U; i < keyWidth; ++i) { + const auto keyIndex = AS_VALUE(TDataLiteral, callable.GetInput(((i + 1U) << 1U) - offset))->AsValue().Get<ui32>(); + indexes[i] = keyIndex; keyIndexes.emplace(keyIndex); - keyTypes[i].first = *UnpackOptionalData(inputWideComponents[keyIndex], keyTypes[i].second)->GetDataSlot(); + + bool isTuple; + bool encoded; + bool useIHash; + TKeyTypes oneKeyTypes; + GetDictionaryKeyTypes(inputWideComponents[keyIndex], oneKeyTypes, isTuple,encoded, useIHash, false); + if (useIHash) { + keys[i].Compare = MakeCompareImpl(inputWideComponents[keyIndex]); + } else if (encoded) { + keys[i].PresortType = inputWideComponents[keyIndex]; + } else { + Y_ENSURE(oneKeyTypes.size() == 1); + keys[i].Slot = oneKeyTypes.front().first; + keys[i].IsOptional = oneKeyTypes.front().second; + } } - size_t payloadPos = keyTypes.size(); + size_t payloadPos = keyWidth; for (auto i = 0U; i < indexes.size(); ++i) { if (keyIndexes.contains(i)) { continue; @@ -546,7 +618,8 @@ IComputationNode* WrapWideTopT(TCallable& callable, const TComputationNodeFactor std::generate(directions.begin(), directions.end(), [&](){ return LocateNode(ctx.NodeLocator, callable, ++++index); }); if (const auto wide = dynamic_cast<IComputationWideFlowNode*>(flow)) { - return new TWideTopWrapper<Sort, HasCount>(ctx.Mutables, wide, count, std::move(directions), std::move(keyTypes), std::move(indexes), std::move(representations)); + return new TWideTopWrapper<Sort, HasCount>(ctx.Mutables, wide, count, std::move(directions), std::move(keys), + std::move(indexes), std::move(representations)); } THROW yexception() << "Expected wide flow."; diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp index 5e4088b6f35..0c399e66b17 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp @@ -3939,7 +3939,7 @@ IComputationNode* TNodeFactory::CreateImmutableNode(NUdf::TUnboxedValue&& value) return new TUnboxedImmutableCodegeneratorNode(&MemInfo, std::move(value)); } -void GetDictionaryKeyTypes(TType* keyType, TKeyTypes& types, bool& isTuple, bool& encoded, bool& useIHash) { +void GetDictionaryKeyTypes(TType* keyType, TKeyTypes& types, bool& isTuple, bool& encoded, bool& useIHash, bool expandTuple) { isTuple = false; encoded = false; useIHash = false; @@ -3954,7 +3954,7 @@ void GetDictionaryKeyTypes(TType* keyType, TKeyTypes& types, bool& isTuple, bool keyType = AS_TYPE(TOptionalType, keyType)->GetItemType(); } - if (keyType->IsTuple()) { + if (expandTuple && keyType->IsTuple()) { auto tuple = AS_TYPE(TTupleType, keyType); for (ui32 i = 0; i < tuple->GetElementsCount(); ++i) { bool isOptional; diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h index eaf0bd90b47..96c8012df53 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h @@ -626,7 +626,7 @@ private: TComputationMutables& Mutables; }; -void GetDictionaryKeyTypes(TType* keyType, TKeyTypes& types, bool& isTuple, bool& encoded, bool& useIHash); +void GetDictionaryKeyTypes(TType* keyType, TKeyTypes& types, bool& isTuple, bool& encoded, bool& useIHash, bool expandTuple = true); struct TContainerCacheOnContext : private TNonCopyable { TContainerCacheOnContext(TComputationMutables& mutables); diff --git a/ydb/library/yql/parser/pg_catalog/catalog.cpp b/ydb/library/yql/parser/pg_catalog/catalog.cpp index 4ffe25c4c51..b4cbcb17659 100644 --- a/ydb/library/yql/parser/pg_catalog/catalog.cpp +++ b/ydb/library/yql/parser/pg_catalog/catalog.cpp @@ -442,6 +442,7 @@ public: arrayType.ElementTypeId = arrayType.TypeId; arrayType.TypeId = LastType.ArrayTypeId; arrayType.PassByValue = false; + arrayType.TypeLen = -1; Types[LastType.ArrayTypeId] = arrayType; } diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp index efdf042b959..5f75abf925b 100644 --- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp +++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp @@ -2627,17 +2627,23 @@ void PgDestroyContext(const std::string_view& contextType, void* ctx) { } } -template <bool PassByValue> +template <bool PassByValue, bool IsArray> class TPgHash : public NUdf::IHash { public: TPgHash(const NYql::NPg::TTypeDesc& typeDesc) : TypeDesc(typeDesc) { - Y_ENSURE(PassByValue == TypeDesc.PassByValue); - Y_ENSURE(TypeDesc.HashProcId); + auto hashProcId = TypeDesc.HashProcId; + if constexpr (IsArray) { + const auto& elemDesc = NYql::NPg::LookupType(TypeDesc.ElementTypeId); + Y_ENSURE(elemDesc.HashProcId); + hashProcId = NYql::NPg::LookupProc("hash_array", { 0, 0 }).ProcId; + } + + Y_ENSURE(hashProcId);; Zero(FInfoHash); - fmgr_info(TypeDesc.HashProcId, &FInfoHash); + fmgr_info(hashProcId, &FInfoHash); Y_ENSURE(!FInfoHash.fn_retset); Y_ENSURE(FInfoHash.fn_addr); Y_ENSURE(FInfoHash.fn_nargs == 1); @@ -2672,43 +2678,59 @@ private: NUdf::IHash::TPtr MakePgHash(const NMiniKQL::TPgType* type) { const auto& typeDesc = NYql::NPg::LookupType(type->GetTypeId()); if (typeDesc.PassByValue) { - return new TPgHash<true>(typeDesc); + return new TPgHash<true, false>(typeDesc); + } else if (typeDesc.TypeId == typeDesc.ArrayTypeId) { + return new TPgHash<false, true>(typeDesc); } else { - return new TPgHash<false>(typeDesc); + return new TPgHash<false, false>(typeDesc); } } -template <bool PassByValue> -class TPgCompare : public NUdf::ICompare, public NUdf::TBlockItemComparatorBase<TPgCompare<PassByValue>, true> { +template <bool PassByValue, bool IsArray> +class TPgCompare : public NUdf::ICompare, public NUdf::TBlockItemComparatorBase<TPgCompare<PassByValue, IsArray>, true> { public: TPgCompare(const NYql::NPg::TTypeDesc& typeDesc) : TypeDesc(typeDesc) { - Y_ENSURE(PassByValue == TypeDesc.PassByValue); - Y_ENSURE(TypeDesc.LessProcId); - Y_ENSURE(TypeDesc.CompareProcId); - Y_ENSURE(TypeDesc.EqualProcId); - - Zero(FInfoLess); - fmgr_info(TypeDesc.LessProcId, &FInfoLess); - Y_ENSURE(!FInfoLess.fn_retset); - Y_ENSURE(FInfoLess.fn_addr); - Y_ENSURE(FInfoLess.fn_nargs == 2); - - Zero(FInfoCompare); - fmgr_info(TypeDesc.CompareProcId, &FInfoCompare); + Zero(FInfoLess); + Zero(FInfoCompare); + Zero(FInfoEquals); + + auto lessProcId = TypeDesc.LessProcId; + auto compareProcId = TypeDesc.CompareProcId; + auto equalProcId = TypeDesc.EqualProcId; + if constexpr (IsArray) { + const auto& elemDesc = NYql::NPg::LookupType(TypeDesc.ElementTypeId); + Y_ENSURE(elemDesc.CompareProcId); + + compareProcId = NYql::NPg::LookupProc("btarraycmp", { 0, 0 }).ProcId; + } else { + Y_ENSURE(lessProcId); + Y_ENSURE(equalProcId); + + fmgr_info(lessProcId, &FInfoLess); + Y_ENSURE(!FInfoLess.fn_retset); + Y_ENSURE(FInfoLess.fn_addr); + Y_ENSURE(FInfoLess.fn_nargs == 2); + + fmgr_info(equalProcId, &FInfoEquals); + Y_ENSURE(!FInfoEquals.fn_retset); + Y_ENSURE(FInfoEquals.fn_addr); + Y_ENSURE(FInfoEquals.fn_nargs == 2); + } + + Y_ENSURE(compareProcId); + fmgr_info(compareProcId, &FInfoCompare); Y_ENSURE(!FInfoCompare.fn_retset); Y_ENSURE(FInfoCompare.fn_addr); Y_ENSURE(FInfoCompare.fn_nargs == 2); - - Zero(FInfoEquals); - fmgr_info(TypeDesc.EqualProcId, &FInfoEquals); - Y_ENSURE(!FInfoEquals.fn_retset); - Y_ENSURE(FInfoEquals.fn_addr); - Y_ENSURE(FInfoEquals.fn_nargs == 2); } bool Less(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) const override { + if constexpr (IsArray) { + return Compare(lhs, rhs) < 0; + } + LOCAL_FCINFO(callInfo, 2); Zero(*callInfo); callInfo->flinfo = const_cast<FmgrInfo*>(&FInfoLess); // don't copy becase of ICompare isn't threadsafe @@ -2790,6 +2812,10 @@ public: } bool DoEquals(NUdf::TBlockItem lhs, NUdf::TBlockItem rhs) const { + if constexpr (IsArray) { + return DoCompare(lhs, rhs) == 0; + } + LOCAL_FCINFO(callInfo, 2); Zero(*callInfo); callInfo->flinfo = const_cast<FmgrInfo*>(&FInfoEquals); // don't copy becase of ICompare isn't threadsafe @@ -2809,6 +2835,10 @@ public: } bool DoLess(NUdf::TBlockItem lhs, NUdf::TBlockItem rhs) const { + if constexpr (IsArray) { + return DoCompare(lhs, rhs) < 0; + } + LOCAL_FCINFO(callInfo, 2); Zero(*callInfo); callInfo->flinfo = const_cast<FmgrInfo*>(&FInfoLess); // don't copy becase of ICompare isn't threadsafe @@ -2836,32 +2866,43 @@ private: NUdf::ICompare::TPtr MakePgCompare(const NMiniKQL::TPgType* type) { const auto& typeDesc = NYql::NPg::LookupType(type->GetTypeId()); if (typeDesc.PassByValue) { - return new TPgCompare<true>(typeDesc); + return new TPgCompare<true, false>(typeDesc); + } else if (typeDesc.TypeId == typeDesc.ArrayTypeId) { + return new TPgCompare<false, true>(typeDesc); } else { - return new TPgCompare<false>(typeDesc); + return new TPgCompare<false, false>(typeDesc); } } NUdf::IBlockItemComparator::TPtr MakePgItemComparator(ui32 typeId) { const auto& typeDesc = NYql::NPg::LookupType(typeId); if (typeDesc.PassByValue) { - return new TPgCompare<true>(typeDesc); + return new TPgCompare<true, false>(typeDesc); + } else if (typeDesc.TypeId == typeDesc.ArrayTypeId) { + return new TPgCompare<false, true>(typeDesc); } else { - return new TPgCompare<false>(typeDesc); + return new TPgCompare<false, false>(typeDesc); } } -template <bool PassByValue> +template <bool PassByValue, bool IsArray> class TPgEquate: public NUdf::IEquate { public: TPgEquate(const NYql::NPg::TTypeDesc& typeDesc) : TypeDesc(typeDesc) { - Y_ENSURE(PassByValue == TypeDesc.PassByValue); - Y_ENSURE(TypeDesc.EqualProcId); + auto equalProcId = TypeDesc.EqualProcId; + if constexpr (IsArray) { + const auto& elemDesc = NYql::NPg::LookupType(TypeDesc.ElementTypeId); + Y_ENSURE(elemDesc.CompareProcId); + + equalProcId = NYql::NPg::LookupProc("btarraycmp", { 0, 0 }).ProcId; + } + + Y_ENSURE(equalProcId); Zero(FInfoEquate); - fmgr_info(TypeDesc.EqualProcId, &FInfoEquate); + fmgr_info(equalProcId, &FInfoEquate); Y_ENSURE(!FInfoEquate.fn_retset); Y_ENSURE(FInfoEquate.fn_addr); Y_ENSURE(FInfoEquate.fn_nargs == 2); @@ -2895,6 +2936,10 @@ public: auto x = FInfoEquate.fn_addr(callInfo); Y_ENSURE(!callInfo->isnull); + if constexpr (IsArray) { + return DatumGetInt32(x) == 0; + } + return DatumGetBool(x); } @@ -2907,9 +2952,11 @@ private: NUdf::IEquate::TPtr MakePgEquate(const TPgType* type) { const auto& typeDesc = NYql::NPg::LookupType(type->GetTypeId()); if (typeDesc.PassByValue) { - return new TPgEquate<true>(typeDesc); + return new TPgEquate<true, false>(typeDesc); + } else if (typeDesc.TypeId == typeDesc.ArrayTypeId) { + return new TPgEquate<false, true>(typeDesc); } else { - return new TPgEquate<false>(typeDesc); + return new TPgEquate<false, false>(typeDesc); } } @@ -2988,12 +3035,26 @@ public: NUdf::TStringRef AsCStringBuffer(const NUdf::TUnboxedValue& value) const override { auto x = (const char*)value.AsBoxed().Get() + PallocHdrSize; - return { x, strlen(x) }; + return { x, strlen(x) + 1}; } NUdf::TStringRef AsTextBuffer(const NUdf::TUnboxedValue& value) const override { auto x = (const text*)((const char*)value.AsBoxed().Get() + PallocHdrSize); - return GetVarBuf(x); + return { (const char*)x, GetFullVarSize(x) }; + } + + NUdf::TUnboxedValue MakeCString(const char* value) const override { + auto len = 1 + strlen(value); + char* ret = (char*)palloc(len); + memcpy(ret, value, len); + return PointerDatumToPod((Datum)ret); + } + + NUdf::TUnboxedValue MakeText(const char* value) const override { + auto len = GetFullVarSize((const text*)value); + char* ret = (char*)palloc(len); + memcpy(ret, value, len); + return PointerDatumToPod((Datum)ret); } }; diff --git a/ydb/library/yql/public/udf/udf_value_builder.h b/ydb/library/yql/public/udf/udf_value_builder.h index bc384adb9b3..6b41af3ea11 100644 --- a/ydb/library/yql/public/udf/udf_value_builder.h +++ b/ydb/library/yql/public/udf/udf_value_builder.h @@ -138,8 +138,18 @@ public: }; #endif +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 33) +class IPgBuilder3: public IPgBuilder2 +{ +public: + virtual TUnboxedValue MakeCString(const char* value) const = 0; + virtual TUnboxedValue MakeText(const char* value) const = 0; +}; +#endif -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 31) +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 33) +class IPgBuilder: public IPgBuilder3 {}; +#elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 31) class IPgBuilder: public IPgBuilder2 {}; #else class IPgBuilder: public IPgBuilder1 {}; diff --git a/ydb/library/yql/public/udf/udf_version.h b/ydb/library/yql/public/udf/udf_version.h index e1d13de0643..e1a33d560f6 100644 --- a/ydb/library/yql/public/udf/udf_version.h +++ b/ydb/library/yql/public/udf/udf_version.h @@ -7,7 +7,7 @@ namespace NYql { namespace NUdf { #define CURRENT_UDF_ABI_VERSION_MAJOR 2 -#define CURRENT_UDF_ABI_VERSION_MINOR 32 +#define CURRENT_UDF_ABI_VERSION_MINOR 33 #define CURRENT_UDF_ABI_VERSION_PATCH 0 #ifdef USE_CURRENT_UDF_ABI_VERSION diff --git a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp index 9ec3e2427d5..58b35faba2b 100644 --- a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp +++ b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp @@ -291,6 +291,16 @@ public: Y_UNUSED(value); ythrow yexception() << "TPgDummyBuilder::AsTextBuffer does nothing"; } + + NUdf::TUnboxedValue MakeCString(const char* value) const override { + Y_UNUSED(value); + ythrow yexception() << "TPgDummyBuilder::MakeCString does nothing"; + } + + NUdf::TUnboxedValue MakeText(const char* value) const override { + Y_UNUSED(value); + ythrow yexception() << "TPgDummyBuilder::MakeText does nothing"; + } }; std::unique_ptr<NUdf::IPgBuilder> CreatePgBuilder() { diff --git a/ydb/library/yql/udfs/common/datetime2/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/udfs/common/datetime2/CMakeLists.darwin-x86_64.txt index 91de393c57b..7ffcfc538e8 100644 --- a/ydb/library/yql/udfs/common/datetime2/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/udfs/common/datetime2/CMakeLists.darwin-x86_64.txt @@ -23,7 +23,7 @@ target_link_libraries(datetime2_udf INTERFACE add_global_library_for(datetime2_udf.global datetime2_udf) target_compile_options(datetime2_udf.global PRIVATE -DUDF_ABI_VERSION_MAJOR=2 - -DUDF_ABI_VERSION_MINOR=31 + -DUDF_ABI_VERSION_MINOR=33 -DUDF_ABI_VERSION_PATCH=0 ) target_link_libraries(datetime2_udf.global PUBLIC diff --git a/ydb/library/yql/udfs/common/datetime2/CMakeLists.linux-aarch64.txt b/ydb/library/yql/udfs/common/datetime2/CMakeLists.linux-aarch64.txt index 6b5c6fcef0e..c92e34d53ed 100644 --- a/ydb/library/yql/udfs/common/datetime2/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/udfs/common/datetime2/CMakeLists.linux-aarch64.txt @@ -24,7 +24,7 @@ target_link_libraries(datetime2_udf INTERFACE add_global_library_for(datetime2_udf.global datetime2_udf) target_compile_options(datetime2_udf.global PRIVATE -DUDF_ABI_VERSION_MAJOR=2 - -DUDF_ABI_VERSION_MINOR=31 + -DUDF_ABI_VERSION_MINOR=33 -DUDF_ABI_VERSION_PATCH=0 ) target_link_libraries(datetime2_udf.global PUBLIC diff --git a/ydb/library/yql/udfs/common/datetime2/CMakeLists.linux-x86_64.txt b/ydb/library/yql/udfs/common/datetime2/CMakeLists.linux-x86_64.txt index 6b5c6fcef0e..c92e34d53ed 100644 --- a/ydb/library/yql/udfs/common/datetime2/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/udfs/common/datetime2/CMakeLists.linux-x86_64.txt @@ -24,7 +24,7 @@ target_link_libraries(datetime2_udf INTERFACE add_global_library_for(datetime2_udf.global datetime2_udf) target_compile_options(datetime2_udf.global PRIVATE -DUDF_ABI_VERSION_MAJOR=2 - -DUDF_ABI_VERSION_MINOR=31 + -DUDF_ABI_VERSION_MINOR=33 -DUDF_ABI_VERSION_PATCH=0 ) target_link_libraries(datetime2_udf.global PUBLIC diff --git a/ydb/library/yql/udfs/common/datetime2/CMakeLists.windows-x86_64.txt b/ydb/library/yql/udfs/common/datetime2/CMakeLists.windows-x86_64.txt index 91de393c57b..7ffcfc538e8 100644 --- a/ydb/library/yql/udfs/common/datetime2/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/udfs/common/datetime2/CMakeLists.windows-x86_64.txt @@ -23,7 +23,7 @@ target_link_libraries(datetime2_udf INTERFACE add_global_library_for(datetime2_udf.global datetime2_udf) target_compile_options(datetime2_udf.global PRIVATE -DUDF_ABI_VERSION_MAJOR=2 - -DUDF_ABI_VERSION_MINOR=31 + -DUDF_ABI_VERSION_MINOR=33 -DUDF_ABI_VERSION_PATCH=0 ) target_link_libraries(datetime2_udf.global PUBLIC diff --git a/ydb/library/yql/udfs/common/url_base/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/udfs/common/url_base/CMakeLists.darwin-x86_64.txt index 64b76ee37f1..53c776455f0 100644 --- a/ydb/library/yql/udfs/common/url_base/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/udfs/common/url_base/CMakeLists.darwin-x86_64.txt @@ -20,7 +20,7 @@ target_link_libraries(url_udf INTERFACE add_global_library_for(url_udf.global url_udf) target_compile_options(url_udf.global PRIVATE -DUDF_ABI_VERSION_MAJOR=2 - -DUDF_ABI_VERSION_MINOR=31 + -DUDF_ABI_VERSION_MINOR=33 -DUDF_ABI_VERSION_PATCH=0 ) target_link_libraries(url_udf.global PUBLIC diff --git a/ydb/library/yql/udfs/common/url_base/CMakeLists.linux-aarch64.txt b/ydb/library/yql/udfs/common/url_base/CMakeLists.linux-aarch64.txt index 91a0105d1fd..a27a9fce966 100644 --- a/ydb/library/yql/udfs/common/url_base/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/udfs/common/url_base/CMakeLists.linux-aarch64.txt @@ -21,7 +21,7 @@ target_link_libraries(url_udf INTERFACE add_global_library_for(url_udf.global url_udf) target_compile_options(url_udf.global PRIVATE -DUDF_ABI_VERSION_MAJOR=2 - -DUDF_ABI_VERSION_MINOR=31 + -DUDF_ABI_VERSION_MINOR=33 -DUDF_ABI_VERSION_PATCH=0 ) target_link_libraries(url_udf.global PUBLIC diff --git a/ydb/library/yql/udfs/common/url_base/CMakeLists.linux-x86_64.txt b/ydb/library/yql/udfs/common/url_base/CMakeLists.linux-x86_64.txt index 91a0105d1fd..a27a9fce966 100644 --- a/ydb/library/yql/udfs/common/url_base/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/udfs/common/url_base/CMakeLists.linux-x86_64.txt @@ -21,7 +21,7 @@ target_link_libraries(url_udf INTERFACE add_global_library_for(url_udf.global url_udf) target_compile_options(url_udf.global PRIVATE -DUDF_ABI_VERSION_MAJOR=2 - -DUDF_ABI_VERSION_MINOR=31 + -DUDF_ABI_VERSION_MINOR=33 -DUDF_ABI_VERSION_PATCH=0 ) target_link_libraries(url_udf.global PUBLIC diff --git a/ydb/library/yql/udfs/common/url_base/CMakeLists.windows-x86_64.txt b/ydb/library/yql/udfs/common/url_base/CMakeLists.windows-x86_64.txt index 64b76ee37f1..53c776455f0 100644 --- a/ydb/library/yql/udfs/common/url_base/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/udfs/common/url_base/CMakeLists.windows-x86_64.txt @@ -20,7 +20,7 @@ target_link_libraries(url_udf INTERFACE add_global_library_for(url_udf.global url_udf) target_compile_options(url_udf.global PRIVATE -DUDF_ABI_VERSION_MAJOR=2 - -DUDF_ABI_VERSION_MINOR=31 + -DUDF_ABI_VERSION_MINOR=33 -DUDF_ABI_VERSION_PATCH=0 ) target_link_libraries(url_udf.global PUBLIC diff --git a/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.darwin-x86_64.txt index 022e8f5f575..b73a2ccb7dd 100644 --- a/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.darwin-x86_64.txt @@ -10,7 +10,7 @@ add_library(common-url_base-lib) target_compile_options(common-url_base-lib PRIVATE -DUDF_ABI_VERSION_MAJOR=2 - -DUDF_ABI_VERSION_MINOR=31 + -DUDF_ABI_VERSION_MINOR=33 -DUDF_ABI_VERSION_PATCH=0 ) target_link_libraries(common-url_base-lib PUBLIC diff --git a/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.linux-aarch64.txt b/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.linux-aarch64.txt index f76c72fac0a..f1127d3de7f 100644 --- a/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.linux-aarch64.txt @@ -10,7 +10,7 @@ add_library(common-url_base-lib) target_compile_options(common-url_base-lib PRIVATE -DUDF_ABI_VERSION_MAJOR=2 - -DUDF_ABI_VERSION_MINOR=31 + -DUDF_ABI_VERSION_MINOR=33 -DUDF_ABI_VERSION_PATCH=0 ) target_link_libraries(common-url_base-lib PUBLIC diff --git a/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.linux-x86_64.txt b/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.linux-x86_64.txt index f76c72fac0a..f1127d3de7f 100644 --- a/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.linux-x86_64.txt @@ -10,7 +10,7 @@ add_library(common-url_base-lib) target_compile_options(common-url_base-lib PRIVATE -DUDF_ABI_VERSION_MAJOR=2 - -DUDF_ABI_VERSION_MINOR=31 + -DUDF_ABI_VERSION_MINOR=33 -DUDF_ABI_VERSION_PATCH=0 ) target_link_libraries(common-url_base-lib PUBLIC diff --git a/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.windows-x86_64.txt b/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.windows-x86_64.txt index 022e8f5f575..b73a2ccb7dd 100644 --- a/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/udfs/common/url_base/lib/CMakeLists.windows-x86_64.txt @@ -10,7 +10,7 @@ add_library(common-url_base-lib) target_compile_options(common-url_base-lib PRIVATE -DUDF_ABI_VERSION_MAJOR=2 - -DUDF_ABI_VERSION_MINOR=31 + -DUDF_ABI_VERSION_MINOR=33 -DUDF_ABI_VERSION_PATCH=0 ) target_link_libraries(common-url_base-lib PUBLIC |