diff options
author | vvvv <vvvv@ydb.tech> | 2023-10-05 19:51:48 +0300 |
---|---|---|
committer | vvvv <vvvv@ydb.tech> | 2023-10-05 20:10:09 +0300 |
commit | 12a033df34f005acaa1aa94b2d472b3e69705235 (patch) | |
tree | 862c3c09ff38d9e9e7260227105052a6196358ef | |
parent | 3d570e3d169572e58569f1ab96b039b9c949862e (diff) | |
download | ydb-12a033df34f005acaa1aa94b2d472b3e69705235.tar.gz |
YQL-16728 syscache override (initial impl), support of output records
33 files changed, 776 insertions, 125 deletions
diff --git a/ydb/docs/ru/core/postgresql/_includes/functions.md b/ydb/docs/ru/core/postgresql/_includes/functions.md index af6a20fd4d..cb1bad7cce 100644 --- a/ydb/docs/ru/core/postgresql/_includes/functions.md +++ b/ydb/docs/ru/core/postgresql/_includes/functions.md @@ -3103,19 +3103,19 @@ jsonb_array_length('[]') → 0 ```||
||json_each ( json ) → setof record ( key text, value json )
jsonb_each ( jsonb ) → setof record ( key text, value jsonb )|
-Expands the top-level JSON object into a set of key/value pairs. (NOT SUPPORTED)|
+Expands the top-level JSON object into a set of key/value pairs.|
```sql
-#SELECT * FROM json_each('{"a":"foo", "b":"bar"}') as a → [
+SELECT * FROM json_each('{"a":"foo", "b":"bar"}') as a → [
a,"foo"
b,"bar"
]
```||
||json_each_text ( json ) → setof record ( key text, value text )
jsonb_each_text ( jsonb ) → setof record ( key text, value text )|
-Expands the top-level JSON object into a set of key/value pairs. The returned values will be of type text. (NOT SUPPORTED)|
+Expands the top-level JSON object into a set of key/value pairs. The returned values will be of type text.|
```sql
-#SELECT * FROM json_each_text('{"a":"foo", "b":"bar"}') as a → [
+SELECT * FROM json_each_text('{"a":"foo", "b":"bar"}') as a → [
a,foo
b,bar
]
diff --git a/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp b/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp index b37d5f2e44..9331835ab1 100644 --- a/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp +++ b/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp @@ -922,7 +922,7 @@ void FillInputIndices(const TExprNode::TPtr& from, const TExprNode::TPtr& finalE bool foundColumn = false; ui32 inputIndex = 0; for (; inputIndex < from->Tail().ChildrenSize(); ++inputIndex) { - const auto& inputAlias = from->Tail().Child(inputIndex)->Child(1)->Content(); + auto inputAlias = from->Tail().Child(inputIndex)->Child(1)->Content(); const auto& read = from->Tail().Child(inputIndex)->Head(); const auto& columns = from->Tail().Child(inputIndex)->Tail(); if (x.second.first != Max<ui32>() && x.second.first != inputIndex) { @@ -930,11 +930,38 @@ void FillInputIndices(const TExprNode::TPtr& from, const TExprNode::TPtr& finalE } if (read.IsCallable("PgResolvedCall")) { - Y_ENSURE(!inputAlias.empty()); - Y_ENSURE(columns.ChildrenSize() == 0 || columns.ChildrenSize() == 1); - auto memberName = NTypeAnnImpl::MakeAliasedColumn(inputAlias, - (columns.ChildrenSize() == 1) ? columns.Head().Content() : inputAlias); - foundColumn = (memberName == x.first); + if (inputAlias.empty()) { + inputAlias = read.Head().Content(); + } + + auto itemType = GetSeqItemType(read.GetTypeAnn()); + if (!itemType) { + itemType = read.GetTypeAnn(); + } + + if (itemType->GetKind() == ETypeAnnotationKind::Struct) { + if (columns.ChildrenSize() == 0) { + auto structType = itemType->Cast<TStructExprType>(); + for (const auto& item : structType->GetItems()) { + if (NTypeAnnImpl::MakeAliasedColumn(inputAlias, item->GetName()) == x.first) { + foundColumn = true; + break; + } + } + } else { + for (const auto& child : columns.Children()) { + if (NTypeAnnImpl::MakeAliasedColumn(inputAlias, child->Content()) == x.first) { + foundColumn = true; + break; + } + } + } + } else { + Y_ENSURE(columns.ChildrenSize() == 0 || columns.ChildrenSize() == 1); + auto memberName = NTypeAnnImpl::MakeAliasedColumn(inputAlias, + (columns.ChildrenSize() == 1) ? columns.Head().Content() : inputAlias); + foundColumn = (memberName == x.first); + } } else { if (alias && alias != inputAlias) { continue; @@ -994,43 +1021,92 @@ void FillInputIndices(const TExprNode::TPtr& from, const TExprNode::TPtr& finalE } TExprNode::TListType BuildCleanedColumns(TPositionHandle pos, const TExprNode::TPtr& from, const TUsedColumns& usedColumns, - TVector<TString>& inputAliases, THashMap<TString, ui32>& memberToInput, TExprContext& ctx) { + TVector<TString>& inputAliases, THashMap<TString, ui32>& memberToInput, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) { TExprNode::TListType cleanedInputs; for (ui32 i = 0; i < from->Tail().ChildrenSize(); ++i) { auto list = from->Tail().Child(i)->HeadPtr(); - const auto& inputAlias = from->Tail().Child(i)->Child(1)->Content(); + auto originalList = list; + auto inputAlias = from->Tail().Child(i)->Child(1)->Content(); inputAliases.push_back(TString(inputAlias)); if (list->IsCallable("PgResolvedCall")) { const auto& columns = from->Tail().Child(i)->Tail(); - Y_ENSURE(!inputAlias.empty()); - Y_ENSURE(columns.ChildrenSize() == 0 || columns.ChildrenSize() == 1); - auto memberName = (columns.ChildrenSize() == 1) ? columns.Head().Content() : inputAlias; - if (list->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List) { - list = ctx.Builder(pos) - .Callable("OrderedMap") - .Add(0, list) - .Lambda(1) - .Param("item") - .Callable("AsStruct") - .List(0) - .Atom(0, memberName) - .Arg(1, "item") + if (inputAlias.empty()) { + inputAliases.back() = inputAlias = list->Head().Content(); + } + + auto itemType = GetSeqItemType(list->GetTypeAnn()); + if (!itemType) { + itemType = list->GetTypeAnn(); + } + + if (itemType->GetKind() == ETypeAnnotationKind::Struct) { + if (list->GetTypeAnn()->GetKind() != ETypeAnnotationKind::List) { + list = ctx.Builder(pos) + .Callable("AsList") + .Add(0, list) + .Seal() + .Build(); + } + + if (columns.ChildrenSize() != 0) { + // rename columns + list = ctx.Builder(pos) + .Callable("OrderedMap") + .Add(0, list) + .Lambda(1) + .Param("item") + .Callable("AsStruct") + .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder & { + auto originalColumnOrder = typeCtx.LookupColumnOrder(*originalList); + YQL_ENSURE(originalColumnOrder); + YQL_ENSURE(originalColumnOrder->size() == columns.ChildrenSize()); + for (ui32 i = 0; i < columns.ChildrenSize(); ++i) { + parent.List(i) + .Atom(0, columns.Child(i)->Content()) + .Callable(1, "Member") + .Arg(0, "item") + .Atom(1, (*originalColumnOrder)[i]) + .Seal() + .Seal(); + } + + return parent; + }) .Seal() .Seal() .Seal() - .Seal() - .Build(); + .Build(); + } } else { - list = ctx.Builder(pos) - .Callable("AsList") - .Callable(0, "AsStruct") - .List(0) - .Atom(0, memberName) - .Add(1, list) + Y_ENSURE(columns.ChildrenSize() == 0 || columns.ChildrenSize() == 1); + auto memberName = (columns.ChildrenSize() == 1) ? columns.Head().Content() : inputAlias; + if (list->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List) { + list = ctx.Builder(pos) + .Callable("OrderedMap") + .Add(0, list) + .Lambda(1) + .Param("item") + .Callable("AsStruct") + .List(0) + .Atom(0, memberName) + .Arg(1, "item") + .Seal() + .Seal() .Seal() .Seal() - .Seal() - .Build(); + .Build(); + } else { + list = ctx.Builder(pos) + .Callable("AsList") + .Callable(0, "AsStruct") + .List(0) + .Atom(0, memberName) + .Add(1, list) + .Seal() + .Seal() + .Seal() + .Build(); + } } } @@ -3104,7 +3180,7 @@ TExprNode::TPtr ExpandPgSelectImpl(const TExprNode::TPtr& node, TExprContext& ct FillInputIndices(from, finalExtTypes, usedColumns, optCtx); THashMap<TString, ui32> memberToInput; - cleanedInputs = BuildCleanedColumns(node->Pos(), from, usedColumns, inputAliases, memberToInput, ctx); + cleanedInputs = BuildCleanedColumns(node->Pos(), from, usedColumns, inputAliases, memberToInput, ctx, *optCtx.Types); if (cleanedInputs.size() == 1) { list = cleanedInputs.front(); } else { diff --git a/ydb/library/yql/core/type_ann/type_ann_pg.cpp b/ydb/library/yql/core/type_ann/type_ann_pg.cpp index 8b94576937..46395ff771 100644 --- a/ydb/library/yql/core/type_ann/type_ann_pg.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_pg.cpp @@ -107,6 +107,7 @@ IGraphTransformer::TStatus PgCallWrapper(const TExprNode::TPtr& input, TExprNode return IGraphTransformer::TStatus::Error; } + bool rangeFunction = false; for (const auto& setting : input->Child(isResolved ? 2 : 1)->Children()) { if (!EnsureTupleMinSize(*setting, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; @@ -117,10 +118,13 @@ IGraphTransformer::TStatus PgCallWrapper(const TExprNode::TPtr& input, TExprNode } auto content = setting->Head().Content(); - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), - TStringBuilder() << "Unexpected setting " << content << " in function " << name)); - - return IGraphTransformer::TStatus::Error; + if (content == "range") { + rangeFunction = true; + } else { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), + TStringBuilder() << "Unexpected setting " << content << " in function " << name)); + return IGraphTransformer::TStatus::Error; + } } TVector<ui32> argTypes; @@ -167,12 +171,36 @@ IGraphTransformer::TStatus PgCallWrapper(const TExprNode::TPtr& input, TExprNode } const TTypeAnnotationNode* result = ctx.Expr.MakeType<TPgExprType>(proc.ResultType); + TMaybe<TColumnOrder> resultColumnOrder; + if (proc.ResultType == NPg::RecordOid && rangeFunction) { + if (proc.OutputArgNames.empty()) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), + TStringBuilder() << "Aggregate function " << name << " cannot be used in FROM")); + return IGraphTransformer::TStatus::Error; + } + + resultColumnOrder.ConstructInPlace(); + TVector<const TItemExprType*> items; + for (size_t i = 0; i < proc.OutputArgTypes.size(); ++i) { + resultColumnOrder->push_back(proc.OutputArgNames[i]); + items.push_back(ctx.Expr.MakeType<TItemExprType>( + proc.OutputArgNames[i], + ctx.Expr.MakeType<TPgExprType>(proc.OutputArgTypes[i]))); + } + + result = ctx.Expr.MakeType<TStructExprType>(items); + } + if (proc.ReturnSet) { result = ctx.Expr.MakeType<TListExprType>(result); } input->SetTypeAnn(result); - return IGraphTransformer::TStatus::Ok; + if (resultColumnOrder) { + return ctx.Types.SetColumnOrder(*input, *resultColumnOrder, ctx.Expr); + } else { + return IGraphTransformer::TStatus::Ok; + } } else { try { const auto procOrType = NPg::LookupProcWithCasts(TString(name), argTypes); @@ -3139,27 +3167,51 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN const TStructExprType* inputStructType = nullptr; if (isRangeFunction) { if (alias.empty()) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(option->Head().Pos()), - "Empty alias for range function is not allowed")); - return IGraphTransformer::TStatus::Error; + alias = TString(p->Head().Head().Content()); } - if (p->Child(2)->ChildrenSize() > 0 && p->Child(2)->ChildrenSize() != 1) { - ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(option->Head().Pos()), - TStringBuilder() << "Expected exactly one column name for range function, but got: " << p->Child(2)->ChildrenSize())); - return IGraphTransformer::TStatus::Error; - } - - auto memberName = (p->Child(2)->ChildrenSize() == 1) ? p->Child(2)->Head().Content() : alias; - TVector<const TItemExprType*> items; auto itemType = p->Head().GetTypeAnn(); if (itemType->GetKind() == ETypeAnnotationKind::List) { itemType = itemType->Cast<TListExprType>()->GetItemType(); } - items.push_back(ctx.Expr.MakeType<TItemExprType>(memberName, itemType)); - inputStructType = ctx.Expr.MakeType<TStructExprType>(items); - columnOrder = TColumnOrder({ TString(memberName) }); + if (itemType->GetKind() == ETypeAnnotationKind::Struct) { + inputStructType = itemType->Cast<TStructExprType>(); + if (p->Child(2)->ChildrenSize() > 0) { + if (p->Child(2)->ChildrenSize() != inputStructType->GetSize()) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(option->Head().Pos()), + TStringBuilder() << "Expected exactly " << inputStructType->GetSize() << " column names for range function, but got: " << p->Child(2)->ChildrenSize())); + return IGraphTransformer::TStatus::Error; + } + + TVector<const TItemExprType*> newItems; + TColumnOrder newOrder; + for (ui32 i = 0; i < inputStructType->GetSize(); ++i) { + auto newName = TString(p->Child(2)->Child(i)->Content()); + newOrder.push_back(newName); + newItems.push_back(ctx.Expr.MakeType<TItemExprType>(newName, inputStructType->GetItems()[i]->GetItemType())); + } + + inputStructType = ctx.Expr.MakeType<TStructExprType>(newItems); + if (!inputStructType->Validate(option->Head().Pos(), ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + columnOrder = newOrder; + } + } else { + if (p->Child(2)->ChildrenSize() > 0 && p->Child(2)->ChildrenSize() != 1) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(option->Head().Pos()), + TStringBuilder() << "Expected exactly one column name for range function, but got: " << p->Child(2)->ChildrenSize())); + return IGraphTransformer::TStatus::Error; + } + + auto memberName = (p->Child(2)->ChildrenSize() == 1) ? p->Child(2)->Head().Content() : alias; + TVector<const TItemExprType*> items; + items.push_back(ctx.Expr.MakeType<TItemExprType>(memberName, itemType)); + inputStructType = ctx.Expr.MakeType<TStructExprType>(items); + columnOrder = TColumnOrder({ TString(memberName) }); + } } else { if (!EnsureListType(p->Head(), ctx.Expr)) { diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp index 0231d19210..fab06f6df6 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.cpp +++ b/ydb/library/yql/minikql/mkql_program_builder.cpp @@ -5490,13 +5490,14 @@ TRuntimeNode TProgramBuilder::PgConst(TPgType* pgType, const std::string_view& v TRuntimeNode TProgramBuilder::PgResolvedCall(bool useContext, const std::string_view& name, ui32 id, const TArrayRef<const TRuntimeNode>& args, - TType* returnType) { - if constexpr (RuntimeVersion < 30U) { + TType* returnType, bool rangeFunction) { + if constexpr (RuntimeVersion < 45U) { THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__; } TCallableBuilder callableBuilder(Env, __func__, returnType); callableBuilder.Add(NewDataLiteral(useContext)); + callableBuilder.Add(NewDataLiteral(rangeFunction)); callableBuilder.Add(NewDataLiteral<NUdf::EDataSlot::String>(name)); callableBuilder.Add(NewDataLiteral(id)); for (const auto& arg : args) { diff --git a/ydb/library/yql/minikql/mkql_program_builder.h b/ydb/library/yql/minikql/mkql_program_builder.h index 4be297d5ef..4d072b63ce 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.h +++ b/ydb/library/yql/minikql/mkql_program_builder.h @@ -686,7 +686,7 @@ public: TRuntimeNode PgConst(TPgType* pgType, const std::string_view& value, TRuntimeNode typeMod = {}); TRuntimeNode PgResolvedCall(bool useContext, const std::string_view& name, ui32 id, - const TArrayRef<const TRuntimeNode>& args, TType* returnType); + const TArrayRef<const TRuntimeNode>& args, TType* returnType, bool rangeFunction); TRuntimeNode PgCast(TRuntimeNode input, TType* returnType, TRuntimeNode typeMod = {}); TRuntimeNode FromPg(TRuntimeNode input, TType* returnType); TRuntimeNode ToPg(TRuntimeNode input, TType* returnType); diff --git a/ydb/library/yql/minikql/mkql_runtime_version.h b/ydb/library/yql/minikql/mkql_runtime_version.h index 29eb20a452..a5650b04fc 100644 --- a/ydb/library/yql/minikql/mkql_runtime_version.h +++ b/ydb/library/yql/minikql/mkql_runtime_version.h @@ -24,7 +24,7 @@ namespace NMiniKQL { // 1. Bump this version every time incompatible runtime nodes are introduced. // 2. Make sure you provide runtime node generation for previous runtime versions. #ifndef MKQL_RUNTIME_VERSION -#define MKQL_RUNTIME_VERSION 44U +#define MKQL_RUNTIME_VERSION 45U #endif // History: diff --git a/ydb/library/yql/parser/pg_catalog/catalog.cpp b/ydb/library/yql/parser/pg_catalog/catalog.cpp index d6f7d6690f..183114333f 100644 --- a/ydb/library/yql/parser/pg_catalog/catalog.cpp +++ b/ydb/library/yql/parser/pg_catalog/catalog.cpp @@ -11,7 +11,6 @@ namespace NYql::NPg { constexpr ui32 InvalidOid = 0; constexpr ui32 Int2VectorOid = 22; constexpr ui32 OidVectorOid = 30; -constexpr ui32 RecordOid = 2249; //constexpr ui32 AnyElementOid = 2283; //constexpr ui32 AnyNonArrayOid = 2776; //constexpr ui32 AnyCompatibleOid = 5077; @@ -390,17 +389,74 @@ public: LastProc.IsStrict = (value == "t"); } else if (key == "proretset") { LastProc.ReturnSet = (value == "t"); + } else if (key == "proallargtypes") { + AllArgTypesStr = value; + } else if (key == "proargmodes") { + ArgModesStr = value; + } else if (key == "proargnames") { + ArgNamesStr = value; } } void OnFinish() override { if (IsSupported) { + if (!ArgModesStr.empty()) { + Y_ENSURE(!AllArgTypesStr.empty()); + Y_ENSURE(ArgModesStr.front() == '{'); + Y_ENSURE(ArgModesStr.back() == '}'); + TVector<TString> modes; + Split(ArgModesStr.substr(1, ArgModesStr.size() - 2), ",", modes); + Y_ENSURE(modes.size() >= LastProc.ArgTypes.size()); + for (size_t i = 0; i < modes.size(); ++i) { + if (i < LastProc.ArgTypes.size()) { + if (modes[i] != "i") { + IsSupported = false; + break; + } + } else if (modes[i] != "o") { + IsSupported = false; + break; + } + } + } + } + + if (IsSupported) { + if (!ArgNamesStr.empty()) { + Y_ENSURE(ArgNamesStr.front() == '{'); + Y_ENSURE(ArgNamesStr.back() == '}'); + TVector<TString> names; + Split(ArgNamesStr.substr(1, ArgNamesStr.size() - 2), ",", names); + Y_ENSURE(names.size() >= LastProc.ArgTypes.size()); + LastProc.OutputArgNames.insert(LastProc.OutputArgNames.begin(), names.begin() + LastProc.ArgTypes.size(), names.end()); + } + + if (!AllArgTypesStr.empty()) { + Y_ENSURE(!ArgModesStr.empty()); + Y_ENSURE(AllArgTypesStr.front() == '{'); + Y_ENSURE(AllArgTypesStr.back() == '}'); + TVector<TString> types; + Split(AllArgTypesStr.substr(1, AllArgTypesStr.size() - 2), ",", types); + Y_ENSURE(types.size() >= LastProc.ArgTypes.size()); + + for (size_t i = LastProc.ArgTypes.size(); i < types.size(); ++i) { + auto idPtr = TypeByName.FindPtr(types[i]); + Y_ENSURE(idPtr); + LastProc.OutputArgTypes.push_back(*idPtr); + } + } + } + + if (IsSupported) { Y_ENSURE(!LastProc.Name.empty()); Procs[LastProc.ProcId] = LastProc; } IsSupported = true; LastProc = TProcDesc(); + AllArgTypesStr = ""; + ArgModesStr = ""; + ArgNamesStr = ""; } private: @@ -408,6 +464,9 @@ private: const THashMap<TString, ui32>& TypeByName; TProcDesc LastProc; bool IsSupported = true; + TString AllArgTypesStr; + TString ArgModesStr; + TString ArgNamesStr; }; struct TLazyTypeInfo { diff --git a/ydb/library/yql/parser/pg_catalog/catalog.h b/ydb/library/yql/parser/pg_catalog/catalog.h index a26d2740c4..d7aec784cb 100644 --- a/ydb/library/yql/parser/pg_catalog/catalog.h +++ b/ydb/library/yql/parser/pg_catalog/catalog.h @@ -10,6 +10,7 @@ namespace NYql::NPg { constexpr ui32 UnknownOid = 705; constexpr ui32 AnyOid = 2276; constexpr ui32 AnyArrayOid = 2277; +constexpr ui32 RecordOid = 2249; // copied from pg_class.h enum class ERelPersistence : char @@ -50,6 +51,8 @@ struct TProcDesc { bool IsStrict = true; EProcKind Kind = EProcKind::Function; bool ReturnSet = false; + TVector<TString> OutputArgNames; + TVector<ui32> OutputArgTypes; }; // Copied from pg_collation_d.h diff --git a/ydb/library/yql/parser/pg_wrapper/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/parser/pg_wrapper/CMakeLists.darwin-x86_64.txt index d740c6709f..f971cfe75f 100644 --- a/ydb/library/yql/parser/pg_wrapper/CMakeLists.darwin-x86_64.txt +++ b/ydb/library/yql/parser/pg_wrapper/CMakeLists.darwin-x86_64.txt @@ -161,6 +161,7 @@ target_sources(yql-parser-pg_wrapper PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/superuser.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/config.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/cost_mocks.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/syscache.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/port/pg_crc32c_sse42.c ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/port/pg_crc32c_sse42_choose.c ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/access/brin/brin.c diff --git a/ydb/library/yql/parser/pg_wrapper/CMakeLists.linux-aarch64.txt b/ydb/library/yql/parser/pg_wrapper/CMakeLists.linux-aarch64.txt index 12f45af138..e3f88c861d 100644 --- a/ydb/library/yql/parser/pg_wrapper/CMakeLists.linux-aarch64.txt +++ b/ydb/library/yql/parser/pg_wrapper/CMakeLists.linux-aarch64.txt @@ -160,6 +160,7 @@ target_sources(yql-parser-pg_wrapper PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/superuser.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/config.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/cost_mocks.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/syscache.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/access/brin/brin.c ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/access/brin/brin_bloom.c ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/access/brin/brin_inclusion.c diff --git a/ydb/library/yql/parser/pg_wrapper/CMakeLists.linux-x86_64.txt b/ydb/library/yql/parser/pg_wrapper/CMakeLists.linux-x86_64.txt index 38925c93a5..bf641d6dd5 100644 --- a/ydb/library/yql/parser/pg_wrapper/CMakeLists.linux-x86_64.txt +++ b/ydb/library/yql/parser/pg_wrapper/CMakeLists.linux-x86_64.txt @@ -162,6 +162,7 @@ target_sources(yql-parser-pg_wrapper PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/superuser.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/config.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/cost_mocks.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/syscache.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/port/pg_crc32c_sse42.c ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/port/pg_crc32c_sse42_choose.c ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/access/brin/brin.c diff --git a/ydb/library/yql/parser/pg_wrapper/CMakeLists.windows-x86_64.txt b/ydb/library/yql/parser/pg_wrapper/CMakeLists.windows-x86_64.txt index 433e46b638..0a114e0460 100644 --- a/ydb/library/yql/parser/pg_wrapper/CMakeLists.windows-x86_64.txt +++ b/ydb/library/yql/parser/pg_wrapper/CMakeLists.windows-x86_64.txt @@ -176,6 +176,7 @@ target_sources(yql-parser-pg_wrapper PRIVATE ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/superuser.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/config.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/cost_mocks.cpp + ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/syscache.cpp ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/port/pg_crc32c_sse42.c ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/port/pg_crc32c_sse42_choose.c ${CMAKE_SOURCE_DIR}/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/access/brin/brin.c diff --git a/ydb/library/yql/parser/pg_wrapper/arena_ctx.cpp b/ydb/library/yql/parser/pg_wrapper/arena_ctx.cpp index 4ca471c3e6..411fc8666b 100644 --- a/ydb/library/yql/parser/pg_wrapper/arena_ctx.cpp +++ b/ydb/library/yql/parser/pg_wrapper/arena_ctx.cpp @@ -75,22 +75,32 @@ const MemoryContextMethods MyMethods = { __thread TArenaMemoryContext* TArenaMemoryContext::Current = nullptr; TArenaMemoryContext::TArenaMemoryContext() { - Prev = Current; - Current = this; - PrevContext = CurrentMemoryContext; - - CurrentMemoryContext = (MemoryContext)malloc(sizeof(MemoryContextData)); - MemoryContextCreate(CurrentMemoryContext, + MyContext = (MemoryContext)malloc(sizeof(MemoryContextData)); + MemoryContextCreate(MyContext, T_AllocSetContext, &MyMethods, nullptr, "arena"); + Acquire(); } TArenaMemoryContext::~TArenaMemoryContext() { - free(CurrentMemoryContext); + Release(); + free(MyContext); +} + +void TArenaMemoryContext::Acquire() { + PrevContext = CurrentMemoryContext; + CurrentMemoryContext = MyContext; + Prev = Current; + Current = this; +} + +void TArenaMemoryContext::Release() { CurrentMemoryContext = PrevContext; + PrevContext = nullptr; Current = Prev; + Prev = nullptr; } } diff --git a/ydb/library/yql/parser/pg_wrapper/arena_ctx.h b/ydb/library/yql/parser/pg_wrapper/arena_ctx.h index f7b98999c5..4cbcfacb30 100644 --- a/ydb/library/yql/parser/pg_wrapper/arena_ctx.h +++ b/ydb/library/yql/parser/pg_wrapper/arena_ctx.h @@ -14,11 +14,15 @@ public: return Current->Pool; } + void Acquire(); + void Release(); + private: segmented_string_pool Pool; + MemoryContext PrevContext = nullptr; + MemoryContext MyContext = nullptr; static __thread TArenaMemoryContext* Current; TArenaMemoryContext* Prev = nullptr; - MemoryContext PrevContext = nullptr; }; } diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp index 6337bb8cf7..16c081af5f 100644 --- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp +++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp @@ -37,10 +37,12 @@ extern "C" { #include "utils/arrayaccess.h" #include "utils/lsyscache.h" #include "utils/datetime.h" +#include "utils/typcache.h" #include "nodes/execnodes.h" #include "executor/executor.h" #include "lib/stringinfo.h" #include "miscadmin.h" +#include "funcapi.h" #include "thread_inits.h" #undef Abs @@ -78,6 +80,9 @@ struct TMainContext { MemoryContextData Data; MemoryContext PrevCurrentMemoryContext = nullptr; MemoryContext PrevErrorContext = nullptr; + MemoryContext PrevCacheMemoryContext = nullptr; + RecordCacheState CurrentRecordCacheState = { NULL, NULL, NULL, 0, 0, INVALID_TUPLEDESC_IDENTIFIER }; + RecordCacheState PrevRecordCacheState; TimestampTz StartTimestamp; pg_stack_base_t PrevStackBase; TString LastError; @@ -339,7 +344,7 @@ class TPgResolvedCallBase : public TMutableComputationNode<TDerived> { typedef TMutableComputationNode<TDerived> TBaseComputation; public: TPgResolvedCallBase(TComputationMutables& mutables, const std::string_view& name, ui32 id, - TComputationNodePtrVector&& argNodes, TVector<TType*>&& argTypes, bool isList) + TComputationNodePtrVector&& argNodes, TVector<TType*>&& argTypes, bool isList, const TStructType* structType) : TBaseComputation(mutables) , Name(name) , Id(id) @@ -347,6 +352,7 @@ public: , RetTypeDesc(NPg::LookupType(ProcDesc.ResultType)) , ArgNodes(std::move(argNodes)) , ArgTypes(std::move(argTypes)) + , StructType(structType) { Zero(FInfo); Y_ENSURE(Id); @@ -387,6 +393,7 @@ protected: const NPg::TTypeDesc RetTypeDesc; const TComputationNodePtrVector ArgNodes; const TVector<TType*> ArgTypes; + const TStructType* StructType; TVector<NPg::TTypeDesc> ArgDesc; }; @@ -408,7 +415,7 @@ class TPgResolvedCall : public TPgResolvedCallBase<TPgResolvedCall<UseContext>> public: TPgResolvedCall(TComputationMutables& mutables, const std::string_view& name, ui32 id, TComputationNodePtrVector&& argNodes, TVector<TType*>&& argTypes) - : TBaseComputation(mutables, name, id, std::move(argNodes), std::move(argTypes), false) + : TBaseComputation(mutables, name, id, std::move(argNodes), std::move(argTypes), false, nullptr) , StateIndex(mutables.CurValueIndex++) { } @@ -481,13 +488,17 @@ private: class TIterator : public TComputationValue<TIterator> { public: TIterator(TMemoryUsageInfo* memInfo, const std::string_view& name, const TUnboxedValueVector& args, - const TVector<NPg::TTypeDesc>& argDesc, const NPg::TTypeDesc& retTypeDesc, const FmgrInfo* fInfo) + const TVector<NPg::TTypeDesc>& argDesc, const NPg::TTypeDesc& retTypeDesc, const NPg::TProcDesc& procDesc, + const FmgrInfo* fInfo, const TStructType* structType, const THolderFactory& holderFactory) : TComputationValue<TIterator>(memInfo) , Name(name) , Args(args) , ArgDesc(argDesc) , RetTypeDesc(retTypeDesc) + , ProcDesc(procDesc) , CallInfo(argDesc.size(), fInfo) + , StructType(structType) + , HolderFactory(holderFactory) { auto& callInfo = CallInfo.Ref(); callInfo.resultinfo = (fmNodePtr)&RSInfo.Ref(); @@ -497,8 +508,30 @@ private: rsInfo.returnMode = SFRM_ValuePerCall; rsInfo.setResult = nullptr; rsInfo.setDesc = nullptr; - rsInfo.expectedDesc = CreateTemplateTupleDesc(1); - TupleDescInitEntry(rsInfo.expectedDesc, (AttrNumber) 1, nullptr, RetTypeDesc.TypeId, -1, 0); + if (RetTypeDesc.TypeId != RECORDOID) { + rsInfo.expectedDesc = CreateTemplateTupleDesc(1); + TupleDescInitEntry(rsInfo.expectedDesc, (AttrNumber) 1, nullptr, RetTypeDesc.TypeId, -1, 0); + } else { + if (StructType) { + YQL_ENSURE(ProcDesc.OutputArgNames.size() == ProcDesc.OutputArgTypes.size()); + YQL_ENSURE(ProcDesc.OutputArgNames.size() == StructType->GetMembersCount()); + StructIndicies.resize(StructType->GetMembersCount()); + } + + rsInfo.expectedDesc = CreateTemplateTupleDesc(ProcDesc.OutputArgTypes.size()); + for (size_t i = 0; i < ProcDesc.OutputArgTypes.size(); ++i) { + auto attrName = ProcDesc.OutputArgNames.empty() ? nullptr : ProcDesc.OutputArgNames[i].c_str(); + TupleDescInitEntry(rsInfo.expectedDesc, (AttrNumber) 1 + i, attrName, ProcDesc.OutputArgTypes[i], -1, 0); + if (StructType) { + auto index = StructType->FindMemberIndex(ProcDesc.OutputArgNames[i]); + YQL_ENSURE(index); + StructIndicies[i] = *index; + } + } + + rsInfo.expectedDesc = BlessTupleDesc(rsInfo.expectedDesc); + } + TupleSlot = MakeSingleTupleTableSlot(rsInfo.expectedDesc, &TTSOpsMinimalTuple); for (ui32 i = 0; i < args.size(); ++i) { const auto& value = args[i]; @@ -541,6 +574,7 @@ private: tuplestore_select_read_pointer(RSInfo.Ref().setResult, readPtr); return CopyTuple(value); } else { + YQL_ENSURE(!StructType); if (RSInfo.Ref().isDone == ExprEndResult) { IsFinished = true; return false; @@ -563,53 +597,87 @@ private: } slot_getallattrs(TupleSlot); - Y_ENSURE(TupleSlot->tts_nvalid == 1); - if (TupleSlot->tts_isnull[0]) { - value = NUdf::TUnboxedValuePod(); + if (RetTypeDesc.TypeId == RECORDOID) { + if (StructType) { + Y_ENSURE(TupleSlot->tts_nvalid == StructType->GetMembersCount()); + NUdf::TUnboxedValue* itemsPtr; + value = HolderFactory.CreateDirectArrayHolder(StructType->GetMembersCount(), itemsPtr); + for (ui32 i = 0; i < StructType->GetMembersCount(); ++i) { + itemsPtr[StructIndicies[i]] = CloneTupleItem(i); + } + } else { + // whole record value + auto tupleDesc = RSInfo.Ref().expectedDesc; + auto tuple = ExecCopySlotHeapTuple(TupleSlot); + auto result = (HeapTupleHeader) palloc(tuple->t_len); + memcpy(result, tuple->t_data, tuple->t_len); + HeapTupleHeaderSetDatumLength(result, tuple->t_len); + HeapTupleHeaderSetTypeId(result, tupleDesc->tdtypeid); + HeapTupleHeaderSetTypMod(result, tupleDesc->tdtypmod); + heap_freetuple(tuple); + value = PointerDatumToPod(HeapTupleHeaderGetDatum(result)); + } } else { - auto datum = TupleSlot->tts_values[0]; + Y_ENSURE(TupleSlot->tts_nvalid == 1); + value = CloneTupleItem(0); + } + + return true; + } + + NUdf::TUnboxedValuePod CloneTupleItem(ui32 index) { + if (TupleSlot->tts_isnull[index]) { + return NUdf::TUnboxedValuePod(); + } else { + auto datum = TupleSlot->tts_values[index]; if (RetTypeDesc.PassByValue) { - value = ScalarDatumToPod(datum); + return ScalarDatumToPod(datum); } else if (RetTypeDesc.TypeLen == -1) { const text* orig = (const text*)datum; - value = PointerDatumToPod((Datum)MakeVar(GetVarBuf(orig))); + return PointerDatumToPod((Datum)MakeVar(GetVarBuf(orig))); } else { Y_ENSURE(RetTypeDesc.TypeLen == -2); const char* orig = (const char*)datum; - value = PointerDatumToPod((Datum)MakeCString(orig)); + return PointerDatumToPod((Datum)MakeCString(orig)); } - } - - return true; + } } const std::string_view Name; TUnboxedValueVector Args; const TVector<NPg::TTypeDesc>& ArgDesc; const NPg::TTypeDesc& RetTypeDesc; + const NPg::TProcDesc& ProcDesc; TExprContextHolder ExprContextHolder; TFunctionCallInfo CallInfo; + const TStructType* StructType; + const THolderFactory& HolderFactory; TReturnSetInfo RSInfo; bool IsFinished = false; TupleTableSlot* TupleSlot = nullptr; + TVector<ui32> StructIndicies; }; TListValue(TMemoryUsageInfo* memInfo, TComputationContext& compCtx, const std::string_view& name, TUnboxedValueVector&& args, const TVector<NPg::TTypeDesc>& argDesc, - const NPg::TTypeDesc& retTypeDesc, const FmgrInfo* fInfo) + const NPg::TTypeDesc& retTypeDesc, const NPg::TProcDesc& procDesc, const FmgrInfo* fInfo, + const TStructType* structType, const THolderFactory& holderFactory) : TCustomListValue(memInfo) , CompCtx(compCtx) , Name(name) , Args(args) , ArgDesc(argDesc) , RetTypeDesc(retTypeDesc) + , ProcDesc(procDesc) , FInfo(fInfo) + , StructType(structType) + , HolderFactory(holderFactory) { } private: NUdf::TUnboxedValue GetListIterator() const final { - return CompCtx.HolderFactory.Create<TIterator>(Name, Args, ArgDesc, RetTypeDesc, FInfo); + return CompCtx.HolderFactory.Create<TIterator>(Name, Args, ArgDesc, RetTypeDesc, ProcDesc, FInfo, StructType, CompCtx.HolderFactory); } TComputationContext& CompCtx; @@ -617,13 +685,16 @@ private: TUnboxedValueVector Args; const TVector<NPg::TTypeDesc>& ArgDesc; const NPg::TTypeDesc& RetTypeDesc; + const NPg::TProcDesc& ProcDesc; const FmgrInfo* FInfo; + const TStructType* StructType; + const THolderFactory& HolderFactory; }; public: TPgResolvedMultiCall(TComputationMutables& mutables, const std::string_view& name, ui32 id, - TComputationNodePtrVector&& argNodes, TVector<TType*>&& argTypes) - : TBaseComputation(mutables, name, id, std::move(argNodes), std::move(argTypes), true) + TComputationNodePtrVector&& argNodes, TVector<TType*>&& argTypes, const TStructType* structType) + : TBaseComputation(mutables, name, id, std::move(argNodes), std::move(argTypes), true, structType) { } @@ -635,7 +706,7 @@ public: args.push_back(value); } - return compCtx.HolderFactory.Create<TListValue>(compCtx, Name, std::move(args), ArgDesc, RetTypeDesc, &FInfo); + return compCtx.HolderFactory.Create<TListValue>(compCtx, Name, std::move(args), ArgDesc, RetTypeDesc, ProcDesc, &FInfo, StructType, compCtx.HolderFactory); } }; @@ -1792,23 +1863,35 @@ TComputationNodeFactory GetPgFactory() { if (name == "PgResolvedCall") { const auto useContextData = AS_VALUE(TDataLiteral, callable.GetInput(0)); - const auto nameData = AS_VALUE(TDataLiteral, callable.GetInput(1)); - const auto idData = AS_VALUE(TDataLiteral, callable.GetInput(2)); + const auto rangeFunctionData = AS_VALUE(TDataLiteral, callable.GetInput(1)); + const auto nameData = AS_VALUE(TDataLiteral, callable.GetInput(2)); + const auto idData = AS_VALUE(TDataLiteral, callable.GetInput(3)); auto useContext = useContextData->AsValue().Get<bool>(); + auto rangeFunction = rangeFunctionData->AsValue().Get<bool>(); auto name = nameData->AsValue().AsStringRef(); auto id = idData->AsValue().Get<ui32>(); TComputationNodePtrVector argNodes; TVector<TType*> argTypes; - for (ui32 i = 3; i < callable.GetInputsCount(); ++i) { + for (ui32 i = 4; i < callable.GetInputsCount(); ++i) { argNodes.emplace_back(LocateNode(ctx.NodeLocator, callable, i)); argTypes.emplace_back(callable.GetInput(i).GetStaticType()); } - const bool isList = callable.GetType()->GetReturnType()->IsList(); + const auto returnType = callable.GetType()->GetReturnType(); + const bool isList = returnType->IsList(); + const auto itemType = isList ? AS_TYPE(TListType, returnType)->GetItemType() : returnType; + const TStructType* structType = nullptr; + if (rangeFunction) { + if (itemType->IsStruct()) { + structType = AS_TYPE(TStructType, itemType); + } + } + if (isList) { YQL_ENSURE(!useContext); - return new TPgResolvedMultiCall(ctx.Mutables, name, id, std::move(argNodes), std::move(argTypes)); + return new TPgResolvedMultiCall(ctx.Mutables, name, id, std::move(argNodes), std::move(argTypes), structType); } else { + YQL_ENSURE(!structType); if (useContext) { return new TPgResolvedCall<true>(ctx.Mutables, name, id, std::move(argNodes), std::move(argTypes)); } else { @@ -3341,7 +3424,10 @@ void PgAcquireThreadContext(void* ctx) { auto main = (TMainContext*)ctx; main->PrevCurrentMemoryContext = CurrentMemoryContext; main->PrevErrorContext = ErrorContext; - CurrentMemoryContext = ErrorContext = (MemoryContext)&main->Data; + main->PrevCacheMemoryContext = CacheMemoryContext; + SaveRecordCacheState(&main->PrevRecordCacheState); + LoadRecordCacheState(&main->CurrentRecordCacheState); + CurrentMemoryContext = ErrorContext = CacheMemoryContext = (MemoryContext)&main->Data; SetParallelStartTimestamps(main->StartTimestamp, main->StartTimestamp); main->PrevStackBase = set_stack_base(); yql_error_report_active = true; @@ -3353,6 +3439,9 @@ void PgReleaseThreadContext(void* ctx) { auto main = (TMainContext*)ctx; CurrentMemoryContext = main->PrevCurrentMemoryContext; ErrorContext = main->PrevErrorContext; + CacheMemoryContext = main->PrevCacheMemoryContext; + SaveRecordCacheState(&main->CurrentRecordCacheState); + LoadRecordCacheState(&main->PrevRecordCacheState); restore_stack_base(main->PrevStackBase); yql_error_report_active = false; } diff --git a/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/utils/cache/catcache.c b/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/utils/cache/catcache.c index 7c9d9bf8f2..b216464086 100644 --- a/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/utils/cache/catcache.c +++ b/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/utils/cache/catcache.c @@ -1204,7 +1204,6 @@ SearchCatCacheInternal(CatCache *cache, Datum v3, Datum v4) { - return NULL; Datum arguments[CATCACHE_MAXKEYS]; uint32 hashValue; Index hashIndex; diff --git a/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/utils/cache/syscache.c b/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/utils/cache/syscache.c index 0f50330909..0c0cd3d999 100644 --- a/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/utils/cache/syscache.c +++ b/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/utils/cache/syscache.c @@ -1111,7 +1111,7 @@ InitCatalogCachePhase2(void) * CAUTION: The tuple that is returned must NOT be freed by the caller! */ HeapTuple -SearchSysCache(int cacheId, +SearchSysCache_original(int cacheId, Datum key1, Datum key2, Datum key3, @@ -1124,7 +1124,7 @@ SearchSysCache(int cacheId, } HeapTuple -SearchSysCache1(int cacheId, +SearchSysCache1_original(int cacheId, Datum key1) { Assert(cacheId >= 0 && cacheId < SysCacheSize && @@ -1135,7 +1135,7 @@ SearchSysCache1(int cacheId, } HeapTuple -SearchSysCache2(int cacheId, +SearchSysCache2_original(int cacheId, Datum key1, Datum key2) { Assert(cacheId >= 0 && cacheId < SysCacheSize && @@ -1146,7 +1146,7 @@ SearchSysCache2(int cacheId, } HeapTuple -SearchSysCache3(int cacheId, +SearchSysCache3_original(int cacheId, Datum key1, Datum key2, Datum key3) { Assert(cacheId >= 0 && cacheId < SysCacheSize && @@ -1157,7 +1157,7 @@ SearchSysCache3(int cacheId, } HeapTuple -SearchSysCache4(int cacheId, +SearchSysCache4_original(int cacheId, Datum key1, Datum key2, Datum key3, Datum key4) { Assert(cacheId >= 0 && cacheId < SysCacheSize && @@ -1172,7 +1172,7 @@ SearchSysCache4(int cacheId, * Release previously grabbed reference count on a tuple */ void -ReleaseSysCache(HeapTuple tuple) +ReleaseSysCache_original(HeapTuple tuple) { ReleaseCatCache(tuple); } diff --git a/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/utils/cache/typcache.c b/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/utils/cache/typcache.c index 6860026d04..f4dee21d19 100644 --- a/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/utils/cache/typcache.c +++ b/ydb/library/yql/parser/pg_wrapper/postgresql/src/backend/utils/cache/typcache.c @@ -285,6 +285,24 @@ static __thread int32 NextRecordTypmod = 0; /* number of entries used */ */ static __thread uint64 tupledesc_id_counter = INVALID_TUPLEDESC_IDENTIFIER; +void LoadRecordCacheState(RecordCacheState* state) { + RecordCacheHash = state->RecordCacheHash; + RecordCacheArray = state->RecordCacheArray; + RecordIdentifierArray = state->RecordIdentifierArray; + RecordCacheArrayLen = state->RecordCacheArrayLen; + NextRecordTypmod = state->NextRecordTypmod; + tupledesc_id_counter = state->tupledesc_id_counter; +} + +extern void SaveRecordCacheState(RecordCacheState* state) { + state->RecordCacheHash = RecordCacheHash; + state->RecordCacheArray = RecordCacheArray; + state->RecordIdentifierArray = RecordIdentifierArray; + state->RecordCacheArrayLen = RecordCacheArrayLen; + state->NextRecordTypmod = NextRecordTypmod; + state->tupledesc_id_counter = tupledesc_id_counter; +} + static void load_typcache_tupdesc(TypeCacheEntry *typentry); static void load_rangetype_info(TypeCacheEntry *typentry); static void load_multirangetype_info(TypeCacheEntry *typentry); diff --git a/ydb/library/yql/parser/pg_wrapper/postgresql/src/include/utils/typcache.h b/ydb/library/yql/parser/pg_wrapper/postgresql/src/include/utils/typcache.h index f3bdc6fba6..57da08e272 100644 --- a/ydb/library/yql/parser/pg_wrapper/postgresql/src/include/utils/typcache.h +++ b/ydb/library/yql/parser/pg_wrapper/postgresql/src/include/utils/typcache.h @@ -206,4 +206,18 @@ extern void SharedRecordTypmodRegistryInit(SharedRecordTypmodRegistry *, extern void SharedRecordTypmodRegistryAttach(SharedRecordTypmodRegistry *); +struct HTAB; + +typedef struct RecordCacheState { + struct HTAB *RecordCacheHash; + TupleDesc *RecordCacheArray; + uint64 *RecordIdentifierArray; + int32 RecordCacheArrayLen; + int32 NextRecordTypmod; + uint64 tupledesc_id_counter; +} RecordCacheState; + +extern void LoadRecordCacheState(RecordCacheState* state); +extern void SaveRecordCacheState(RecordCacheState* state); + #endif /* TYPCACHE_H */ diff --git a/ydb/library/yql/parser/pg_wrapper/syscache.cpp b/ydb/library/yql/parser/pg_wrapper/syscache.cpp new file mode 100644 index 0000000000..8cb0240d0e --- /dev/null +++ b/ydb/library/yql/parser/pg_wrapper/syscache.cpp @@ -0,0 +1,267 @@ +#define SortBy PG_SortBy +#define TypeName PG_TypeName + +#include "pg_compat.h" + +extern "C" { +#include "utils/syscache.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_type.h" +#include "catalog/pg_type_d.h" +#include "access/htup_details.h" +} + +#undef TypeName +#undef SortBy +#undef LOG +#undef INFO +#undef NOTICE +#undef WARNING +//#undef ERROR +#undef FATAL +#undef PANIC +#undef open +#undef fopen +#undef bind +#undef locale_t + +#include "arena_ctx.h" +#include "utils.h" + +#include <ydb/library/yql/parser/pg_catalog/catalog.h> +#include <unordered_map> +#include <functional> +#include <tuple> + +namespace NYql { +namespace { + +using THeapTupleKey = std::tuple<Datum, Datum, Datum, Datum>; +using THeapTupleHasher = std::function<size_t(const THeapTupleKey&)>; +using THeapTupleEquals = std::function<bool(const THeapTupleKey&, const THeapTupleKey&)>; +using TSysCacheHashMap = std::unordered_map<THeapTupleKey, HeapTuple, THeapTupleHasher, THeapTupleEquals>; + +size_t OidHasher1(const THeapTupleKey& key) { + return std::hash<Oid>()((Oid)std::get<0>(key)); +} + +bool OidEquals1(const THeapTupleKey& key1, const THeapTupleKey& key2) { + return (Oid)std::get<0>(key1) == (Oid)std::get<0>(key2); +} + +struct TSysCache { + TArenaMemoryContext Arena; + std::unique_ptr<TSysCacheHashMap> Maps[SysCacheSize]; + + static const TSysCache& Instance() { + return *Singleton<TSysCache>(); + } + + TSysCache() + { + InitializeProcs(); + InitializeTypes(); + Arena.Release(); + } + + ~TSysCache() { + Arena.Acquire(); + } + + static void FillDatum(ui32 count, Datum* values, bool* nulls, ui32 attrNum, Datum value) { + Y_ENSURE(attrNum > 0 && attrNum <= count); + values[attrNum - 1] = value; + nulls[attrNum - 1] = false; + } + + static void FillAttr(TupleDesc tupleDesc, ui32 attrNum, Oid type) { + Y_ENSURE(attrNum > 0 && attrNum <= tupleDesc->natts); + TupleDescInitEntry(tupleDesc, attrNum, nullptr, type, -1, 0); + } + + void InitializeProcs() { + auto& map = Maps[PROCOID] = std::make_unique<TSysCacheHashMap>(0, OidHasher1, OidEquals1); + TupleDesc tupleDesc = CreateTemplateTupleDesc(Natts_pg_proc); + FillAttr(tupleDesc, Anum_pg_proc_oid, OIDOID); + FillAttr(tupleDesc, Anum_pg_proc_proname, NAMEOID); + FillAttr(tupleDesc, Anum_pg_proc_pronamespace, OIDOID); + FillAttr(tupleDesc, Anum_pg_proc_proowner, OIDOID); + FillAttr(tupleDesc, Anum_pg_proc_prolang, OIDOID); + FillAttr(tupleDesc, Anum_pg_proc_procost, FLOAT4OID); + FillAttr(tupleDesc, Anum_pg_proc_prorows, FLOAT4OID); + FillAttr(tupleDesc, Anum_pg_proc_provariadic, OIDOID); + FillAttr(tupleDesc, Anum_pg_proc_prosupport, REGPROCOID); + FillAttr(tupleDesc, Anum_pg_proc_prokind, CHAROID); + FillAttr(tupleDesc, Anum_pg_proc_prosecdef, BOOLOID); + FillAttr(tupleDesc, Anum_pg_proc_proleakproof, BOOLOID); + FillAttr(tupleDesc, Anum_pg_proc_proisstrict, BOOLOID); + FillAttr(tupleDesc, Anum_pg_proc_proretset, BOOLOID); + FillAttr(tupleDesc, Anum_pg_proc_provolatile, CHAROID); + FillAttr(tupleDesc, Anum_pg_proc_proparallel, CHAROID); + FillAttr(tupleDesc, Anum_pg_proc_pronargs, INT2OID); + FillAttr(tupleDesc, Anum_pg_proc_pronargdefaults, INT2OID); + FillAttr(tupleDesc, Anum_pg_proc_prorettype, OIDOID); + FillAttr(tupleDesc, Anum_pg_proc_proargtypes, OIDVECTOROID); + FillAttr(tupleDesc, Anum_pg_proc_proallargtypes, OIDARRAYOID); + FillAttr(tupleDesc, Anum_pg_proc_proargmodes, TEXTARRAYOID); + FillAttr(tupleDesc, Anum_pg_proc_proargnames, TEXTARRAYOID); + FillAttr(tupleDesc, Anum_pg_proc_proargdefaults, PG_NODE_TREEOID); + FillAttr(tupleDesc, Anum_pg_proc_protrftypes, OIDARRAYOID); + FillAttr(tupleDesc, Anum_pg_proc_prosrc, TEXTOID); + FillAttr(tupleDesc, Anum_pg_proc_probin, TEXTOID); + FillAttr(tupleDesc, Anum_pg_proc_prosqlbody, PG_NODE_TREEOID); + FillAttr(tupleDesc, Anum_pg_proc_proconfig, TEXTARRAYOID); + FillAttr(tupleDesc, Anum_pg_proc_proacl, ACLITEMARRAYOID); + + NPg::EnumProc([&](ui32 oid, const NPg::TProcDesc& desc){ + auto key = THeapTupleKey(oid, 0, 0, 0); + + Datum values[Natts_pg_proc]; + bool nulls[Natts_pg_proc]; + Zero(values); + std::fill_n(nulls, Natts_pg_proc, true); + std::fill_n(nulls, Anum_pg_proc_prorettype, false); // fixed part of Form_pg_proc + FillDatum(Natts_pg_proc, values, nulls, Anum_pg_proc_oid, oid); + FillDatum(Natts_pg_proc, values, nulls, Anum_pg_proc_prorettype, desc.ResultType); + auto name = MakeFixedString(desc.Name, NPg::LookupType(NAMEOID).TypeLen); + FillDatum(Natts_pg_proc, values, nulls, Anum_pg_proc_proname, (Datum)name); + HeapTuple h = heap_form_tuple(tupleDesc, values, nulls); + auto row = (Form_pg_proc)GETSTRUCT(h); + Y_ENSURE(row->oid == oid); + Y_ENSURE(row->prorettype == desc.ResultType); + Y_ENSURE(NameStr(row->proname) == desc.Name); + map->emplace(key, h); + }); + } + + void InitializeTypes() { + auto& map = Maps[TYPEOID] = std::make_unique<TSysCacheHashMap>(0, OidHasher1, OidEquals1); + TupleDesc tupleDesc = CreateTemplateTupleDesc(Natts_pg_type); + FillAttr(tupleDesc, Anum_pg_type_oid, OIDOID); + FillAttr(tupleDesc, Anum_pg_type_typname, NAMEOID); + FillAttr(tupleDesc, Anum_pg_type_typnamespace, OIDOID); + FillAttr(tupleDesc, Anum_pg_type_typowner, OIDOID); + FillAttr(tupleDesc, Anum_pg_type_typlen, INT2OID); + FillAttr(tupleDesc, Anum_pg_type_typbyval, BOOLOID); + FillAttr(tupleDesc, Anum_pg_type_typtype, CHAROID); + FillAttr(tupleDesc, Anum_pg_type_typcategory, CHAROID); + FillAttr(tupleDesc, Anum_pg_type_typispreferred, BOOLOID); + FillAttr(tupleDesc, Anum_pg_type_typisdefined, BOOLOID); + FillAttr(tupleDesc, Anum_pg_type_typdelim, CHAROID); + FillAttr(tupleDesc, Anum_pg_type_typrelid, OIDOID); + FillAttr(tupleDesc, Anum_pg_type_typsubscript, REGPROCOID); + FillAttr(tupleDesc, Anum_pg_type_typelem, OIDOID); + FillAttr(tupleDesc, Anum_pg_type_typarray, OIDOID); + FillAttr(tupleDesc, Anum_pg_type_typinput, REGPROCOID); + FillAttr(tupleDesc, Anum_pg_type_typoutput, REGPROCOID); + FillAttr(tupleDesc, Anum_pg_type_typreceive, REGPROCOID); + FillAttr(tupleDesc, Anum_pg_type_typsend, REGPROCOID); + FillAttr(tupleDesc, Anum_pg_type_typmodin, REGPROCOID); + FillAttr(tupleDesc, Anum_pg_type_typmodout, REGPROCOID); + FillAttr(tupleDesc, Anum_pg_type_typanalyze, REGPROCOID); + FillAttr(tupleDesc, Anum_pg_type_typalign, CHAROID); + FillAttr(tupleDesc, Anum_pg_type_typstorage, CHAROID); + FillAttr(tupleDesc, Anum_pg_type_typnotnull, BOOLOID); + FillAttr(tupleDesc, Anum_pg_type_typbasetype, OIDOID); + FillAttr(tupleDesc, Anum_pg_type_typtypmod, INT4OID); + FillAttr(tupleDesc, Anum_pg_type_typndims, INT4OID); + FillAttr(tupleDesc, Anum_pg_type_typcollation, OIDOID); + FillAttr(tupleDesc, Anum_pg_type_typdefaultbin, PG_NODE_TREEOID); + FillAttr(tupleDesc, Anum_pg_type_typdefault, TEXTOID); + FillAttr(tupleDesc, Anum_pg_type_typacl, ACLITEMARRAYOID); + + NPg::EnumTypes([&](ui32 oid, const NPg::TTypeDesc& desc){ + auto key = THeapTupleKey(oid, 0, 0, 0); + + Datum values[Natts_pg_type]; + bool nulls[Natts_pg_type]; + Zero(values); + std::fill_n(nulls, Natts_pg_type, true); + std::fill_n(nulls, Anum_pg_type_typcollation, false); // fixed part of Form_pg_type + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_oid, oid); + auto name = MakeFixedString(desc.Name, NPg::LookupType(NAMEOID).TypeLen); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typname, (Datum)name); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typbyval, desc.PassByValue); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typlen, desc.TypeLen); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typtype, (char)desc.TypType); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typcategory, desc.Category); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typispreferred, desc.IsPreferred); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typisdefined, true); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typdelim, desc.TypeDelim); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typarray, desc.ArrayTypeId); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typelem, desc.ElementTypeId); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typinput, desc.InFuncId); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typoutput, desc.OutFuncId); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typreceive, desc.ReceiveFuncId); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typsend, desc.SendFuncId); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typmodin, desc.TypeModInFuncId); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typmodout, desc.TypeModOutFuncId); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typalign, desc.TypeAlign); + FillDatum(Natts_pg_type, values, nulls, Anum_pg_type_typstorage, TYPSTORAGE_PLAIN); + HeapTuple h = heap_form_tuple(tupleDesc, values, nulls); + auto row = (Form_pg_type)GETSTRUCT(h); + Y_ENSURE(row->oid == oid); + Y_ENSURE(NameStr(row->typname) == desc.Name); + Y_ENSURE(row->typlen == desc.TypeLen); + Y_ENSURE(row->typbyval == desc.PassByValue); + Y_ENSURE(row->typtype == (char)desc.TypType); + Y_ENSURE(row->typcategory == desc.Category); + Y_ENSURE(row->typispreferred == desc.IsPreferred); + Y_ENSURE(row->typisdefined == true); + Y_ENSURE(row->typdelim == desc.TypeDelim); + Y_ENSURE(row->typelem == desc.ElementTypeId); + Y_ENSURE(row->typarray == desc.ArrayTypeId); + Y_ENSURE(row->typinput == desc.InFuncId); + Y_ENSURE(row->typoutput == desc.OutFuncId); + Y_ENSURE(row->typreceive == desc.ReceiveFuncId); + Y_ENSURE(row->typsend == desc.SendFuncId); + Y_ENSURE(row->typmodin == desc.TypeModInFuncId); + Y_ENSURE(row->typmodout == desc.TypeModOutFuncId); + Y_ENSURE(row->typalign == desc.TypeAlign); + Y_ENSURE(row->typstorage == TYPSTORAGE_PLAIN); + map->emplace(key, h); + }); + + } +}; + +} +} + + +HeapTuple SearchSysCache(int cacheId, Datum key1, Datum key2, Datum key3, Datum key4) { + Y_ENSURE(cacheId >= 0 && cacheId < SysCacheSize); + const auto& map = NYql::TSysCache::Instance().Maps[cacheId]; + if (!map) { + return nullptr; + } + + auto it = map->find(std::make_tuple(key1, key2, key3, key4)); + if (it == map->end()) { + return nullptr; + } + + return it->second; +} + +HeapTuple SearchSysCache1(int cacheId, Datum key1) { + return SearchSysCache(cacheId, key1, 0, 0, 0); +} + +HeapTuple SearchSysCache2(int cacheId, Datum key1, Datum key2) { + return SearchSysCache(cacheId, key1, key2, 0, 0); +} + +HeapTuple SearchSysCache3(int cacheId, Datum key1, Datum key2, Datum key3) { + return SearchSysCache(cacheId, key1, key2, key3, 0); +} + +HeapTuple SearchSysCache4(int cacheId, Datum key1, Datum key2, Datum key3, Datum key4) { + return SearchSysCache(cacheId, key1, key2, key3, key4); +} + +void ReleaseSysCache(HeapTuple tuple) { + Y_UNUSED(tuple); +} + + diff --git a/ydb/library/yql/parser/pg_wrapper/test/test_doc.py b/ydb/library/yql/parser/pg_wrapper/test/test_doc.py index a3acfbcdbb..32e486d85c 100644 --- a/ydb/library/yql/parser/pg_wrapper/test/test_doc.py +++ b/ydb/library/yql/parser/pg_wrapper/test/test_doc.py @@ -32,8 +32,7 @@ def run_one(item): return (line, input, output, None, e, elapsed_time) -def convert_value(data, output): - cell = data[0] +def convert_cell(cell, output): if cell is None: value = 'NULL' elif isinstance(cell, bytes): @@ -49,11 +48,21 @@ def convert_value(data, output): return (value, output) +def convert_value(data, output): + if len(data) == 1: + return convert_cell(data[0], output) + lst = [convert_cell(x[0], x[1]) for x in zip(data, output.split(","))] + return (",".join(x[0] for x in lst), ",".join(x[1] for x in lst)) + + def test_doc(): skip_before = get_param("skip_before") + stop_at = get_param("stop_at") if skip_before is not None: print("WILL SKIP TESTS BEFORE: ", skip_before) - doc_src = yatest.common.source_path("ydb/docs/ru/core/postgresql/functions.md") + if stop_at is not None: + print("WILL STOP AT: ", stop_at) + doc_src = yatest.common.source_path("ydb/docs/ru/core/postgresql/_includes/functions.md") with open(doc_src) as f: doc_data = f.readlines() in_code = False @@ -67,8 +76,10 @@ def test_doc(): skip_in_progress = skip_before is not None for raw_line in doc_data: line = raw_line.strip() + if stop_at is not None and line.startswith("## " + stop_at): + break if skip_in_progress: - if line.startswith("# " + skip_before): + if line.startswith("## " + skip_before): skip_in_progress = False continue if set_of is not None: diff --git a/ydb/library/yql/parser/pg_wrapper/test/ya.make b/ydb/library/yql/parser/pg_wrapper/test/ya.make index 39bdd36910..5aedce2294 100644 --- a/ydb/library/yql/parser/pg_wrapper/test/ya.make +++ b/ydb/library/yql/parser/pg_wrapper/test/ya.make @@ -13,7 +13,7 @@ REQUIREMENTS( ) DATA( - arcadia/ydb/docs/ru/core/postgresql/functions.md + arcadia/ydb/docs/ru/core/postgresql/_includes/functions.md arcadia/ydb/library/yql/cfg/udf_test arcadia/ydb/library/yql/mount ) diff --git a/ydb/library/yql/parser/pg_wrapper/utils.h b/ydb/library/yql/parser/pg_wrapper/utils.h index d3ec83d9e6..f7f61860ef 100644 --- a/ydb/library/yql/parser/pg_wrapper/utils.h +++ b/ydb/library/yql/parser/pg_wrapper/utils.h @@ -1,4 +1,7 @@ #pragma once +#include <ydb/library/yql/minikql/mkql_alloc.h> +#include <ydb/library/yql/minikql/computation/mkql_block_item.h> +#include <ydb/library/yql/parser/pg_catalog/catalog.h> namespace NYql { @@ -117,6 +120,19 @@ inline char* MakeCString(TStringBuf s) { return ret; } +inline char* MakeFixedStringNotFilled(size_t size) { + char* ret = (char*)palloc(size); + memset(ret, 0, size); + return ret; +} + +inline char* MakeFixedString(TStringBuf s, size_t size) { + auto ret = MakeFixedStringNotFilled(size); + Y_ENSURE(s.Size() + 1 <= size); + memcpy(ret, s.Data(), s.Size()); + return ret; +} + inline text* MakeVarNotFilled(size_t size) { text* ret = (text*)palloc(size + VARHDRSZ); UpdateCleanVarSize(ret, size); diff --git a/ydb/library/yql/parser/pg_wrapper/ya.make b/ydb/library/yql/parser/pg_wrapper/ya.make index 3d899e1806..b1efb2e912 100644 --- a/ydb/library/yql/parser/pg_wrapper/ya.make +++ b/ydb/library/yql/parser/pg_wrapper/ya.make @@ -38,6 +38,7 @@ SRCS( superuser.cpp config.cpp cost_mocks.cpp + syscache.cpp ) IF (ARCH_X86_64) diff --git a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp index 85ea5fbd31..1942487859 100644 --- a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp +++ b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp @@ -2529,8 +2529,15 @@ TMkqlCommonCallableCompiler::TShared::TShared() { args.push_back(MkqlBuildExpr(*node.Child(i), ctx)); } + bool rangeFunction = false; + for (const auto& child : node.Child(2)->Children()) { + if (child->Head().Content() == "range") { + rangeFunction = true; + } + } + auto returnType = BuildType(node, *node.GetTypeAnn(), ctx.ProgramBuilder); - return ctx.ProgramBuilder.PgResolvedCall(node.IsCallable("PgResolvedCallCtx"), name, id, args, returnType); + return ctx.ProgramBuilder.PgResolvedCall(node.IsCallable("PgResolvedCallCtx"), name, id, args, returnType, rangeFunction); }); AddCallable("PgResolvedOp", [](const TExprNode& node, TMkqlBuildContext& ctx) { @@ -2544,7 +2551,7 @@ TMkqlCommonCallableCompiler::TShared::TShared() { } auto returnType = BuildType(node, *node.GetTypeAnn(), ctx.ProgramBuilder); - return ctx.ProgramBuilder.PgResolvedCall(false, procName, procId, args, returnType); + return ctx.ProgramBuilder.PgResolvedCall(false, procName, procId, args, returnType, false); }); AddCallable("BlockPgResolvedCall", [](const TExprNode& node, TMkqlBuildContext& ctx) { diff --git a/ydb/library/yql/sql/pg/pg_sql.cpp b/ydb/library/yql/sql/pg/pg_sql.cpp index 1d0c603dcc..f6114c26bc 100644 --- a/ydb/library/yql/sql/pg/pg_sql.cpp +++ b/ydb/library/yql/sql/pg/pg_sql.cpp @@ -2514,13 +2514,10 @@ public: TString alias; TVector<TString> colnames; - if (!value->alias) { - AddError("RangeFunction: expected alias"); - return {}; - } - - if (!ParseAlias(value->alias, alias, colnames)) { - return {}; + if (value->alias) { + if (!ParseAlias(value->alias, alias, colnames)) { + return {}; + } } auto funcNode = ListNodeNth(value->functions, 0); @@ -2539,7 +2536,13 @@ public: settings.AllowColumns = false; settings.AllowReturnSet = true; settings.Scope = "RANGE FUNCTION"; - auto func = ParseExpr(ListNodeNth(lst, 0), settings); + auto node = ListNodeNth(lst, 0); + if (NodeTag(node) != T_FuncCall) { + AddError("RangeFunction: extected FuncCall"); + return {}; + } + + auto func = ParseFuncCall(CAST_NODE(FuncCall, node), settings, true); if (!func) { return {}; } @@ -2734,7 +2737,7 @@ public: return ParseNullTestExpr(CAST_NODE(NullTest, node), settings); } case T_FuncCall: { - return ParseFuncCall(CAST_NODE(FuncCall, node), settings); + return ParseFuncCall(CAST_NODE(FuncCall, node), settings, false); } case T_A_ArrayExpr: { return ParseAArrayExpr(CAST_NODE(A_ArrayExpr, node), settings); @@ -3052,7 +3055,7 @@ public: return L(A("PgSubLink"), QA(linkType), L(A("Void")), L(A("Void")), rowTest, L(A("lambda"), QL(), select)); } - TAstNode* ParseFuncCall(const FuncCall* value, const TExprSettings& settings) { + TAstNode* ParseFuncCall(const FuncCall* value, const TExprSettings& settings, bool rangeFunction) { AT_LOCATION(value); if (ListLength(value->agg_order) > 0) { AddError("FuncCall: unsupported agg_order"); @@ -3167,6 +3170,10 @@ public: callSettings.push_back(QL(QA("distinct"))); } + if (rangeFunction) { + callSettings.push_back(QL(QA("range"))); + } + args.push_back(QVL(callSettings.data(), callSettings.size())); if (value->agg_star) { if (name != "count") { diff --git a/ydb/library/yql/sql/v1/builtin.cpp b/ydb/library/yql/sql/v1/builtin.cpp index fc4945e436..ab025274e9 100644 --- a/ydb/library/yql/sql/v1/builtin.cpp +++ b/ydb/library/yql/sql/v1/builtin.cpp @@ -732,6 +732,7 @@ public: } }; +template <bool RangeFunction> class TYqlPgCall : public TCallNode { public: TYqlPgCall(TPosition pos, const TVector<TNodePtr>& args) @@ -754,12 +755,12 @@ public: } Args[0] = BuildQuotedAtom(Args[0]->GetPos(), Args[0]->GetLiteralValue()); - Args.insert(Args.begin() + 1, Q(Y())); + Args.insert(Args.begin() + 1, RangeFunction ? Q(Y(Q(Y(Q("range"))))) : Q(Y())); return TCallNode::DoInit(ctx, src); } TNodePtr DoClone() const final { - return new TYqlPgCall(Pos, CloneContainer(Args)); + return new TYqlPgCall<RangeFunction>(Pos, CloneContainer(Args)); } }; @@ -2933,7 +2934,8 @@ struct TBuiltinFuncData { {"pgtype", BuildSimpleBuiltinFactoryCallback<TYqlPgType>() }, {"pgconst", BuildSimpleBuiltinFactoryCallback<TYqlPgConst>() }, {"pgop", BuildSimpleBuiltinFactoryCallback<TYqlPgOp>() }, - {"pgcall", BuildSimpleBuiltinFactoryCallback<TYqlPgCall>() }, + {"pgcall", BuildSimpleBuiltinFactoryCallback<TYqlPgCall<false>>() }, + {"pgrangecall", BuildSimpleBuiltinFactoryCallback<TYqlPgCall<true>>() }, {"pgcast", BuildSimpleBuiltinFactoryCallback<TYqlPgCast>() }, {"frompg", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("FromPg", 1, 1) }, {"topg", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("ToPg", 1, 1) }, @@ -3379,7 +3381,7 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec TVector<TNodePtr> pgCallArgs; pgCallArgs.push_back(BuildLiteralRawString(pos, name)); pgCallArgs.insert(pgCallArgs.end(), args.begin(), args.end()); - return new TYqlPgCall(pos, pgCallArgs); + return new TYqlPgCall<false>(pos, pgCallArgs); } } else if (name == "MakeLibraPreprocessor") { if (args.size() != 1) { diff --git a/ydb/library/yql/tests/sql/suites/pg/range_function_multi.sql b/ydb/library/yql/tests/sql/suites/pg/range_function_multi.sql index a7524fb83f..4f188c9000 100644 --- a/ydb/library/yql/tests/sql/suites/pg/range_function_multi.sql +++ b/ydb/library/yql/tests/sql/suites/pg/range_function_multi.sql @@ -1,2 +1,4 @@ --!syntax_pg -select * from generate_series(1,10) x +select * from generate_series(1,10); +select * from generate_series(1,10) x; +select * from generate_series(1,10) x(y); diff --git a/ydb/library/yql/tests/sql/suites/pg/range_function_multi_record.sql b/ydb/library/yql/tests/sql/suites/pg/range_function_multi_record.sql new file mode 100644 index 0000000000..da9330fc6a --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/pg/range_function_multi_record.sql @@ -0,0 +1,4 @@ +--!syntax_pg +select * from json_each('{"a":"foo", "b":"bar"}'); +select * from json_each('{"a":"foo", "b":"bar"}') as f; +select * from json_each('{"a":"foo", "b":"bar"}') as f(x,y); diff --git a/ydb/library/yql/tests/sql/suites/pg/range_function_scalar.sql b/ydb/library/yql/tests/sql/suites/pg/range_function_scalar.sql index 30296c5cc9..e5b0525ddd 100644 --- a/ydb/library/yql/tests/sql/suites/pg/range_function_scalar.sql +++ b/ydb/library/yql/tests/sql/suites/pg/range_function_scalar.sql @@ -1,3 +1,4 @@ --!syntax_pg -select x from upper('abc') x; -select a from upper('abc') x(a); +select * from upper('abc'); +select * from upper('abc') x; +select * from upper('abc') x(a); diff --git a/ydb/library/yql/tests/sql/suites/pg/set_of_as_records.cfg b/ydb/library/yql/tests/sql/suites/pg/set_of_as_records.cfg new file mode 100644 index 0000000000..bb349dd8ab --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/pg/set_of_as_records.cfg @@ -0,0 +1 @@ +providers yt diff --git a/ydb/library/yql/tests/sql/suites/pg/set_of_as_records.sql b/ydb/library/yql/tests/sql/suites/pg/set_of_as_records.sql new file mode 100644 index 0000000000..f205c5f947 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/pg/set_of_as_records.sql @@ -0,0 +1 @@ +select Pg::json_each(pgjson('{"a":"foo", "b":"bar"}'))
\ No newline at end of file diff --git a/ydb/library/yql/tests/sql/suites/pg/set_of_as_structs.sql b/ydb/library/yql/tests/sql/suites/pg/set_of_as_structs.sql new file mode 100644 index 0000000000..06242eeaa9 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/pg/set_of_as_structs.sql @@ -0,0 +1,2 @@ +SELECT * FROM + AS_TABLE(PgRangeCall("json_each", pgjson('{"a":"foo", "b":"bar"}')));
\ No newline at end of file |