YQL-13710 get rid of pgtypes selector

ref:8b130b14b87797497c797a23dff6413be842fdc7

8 files changed, 73 insertions, 77 deletions

diff --git a/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp b/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp
index dbd69626d7..4da3152bad 100644
--- a/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp
@@ -5,6 +5,27 @@
 
 namespace NYql {
 
+TExprNode::TPtr BuildFilter(TPositionHandle pos, const TExprNode::TPtr& list, const TExprNode::TPtr& filter, TExprContext& ctx) {
+    return ctx.Builder(pos)
+        .Callable("Filter")
+            .Add(0, list)
+            .Lambda(1)
+                .Param("row")
+                .Callable("Coalesce")
+                    .Callable(0, "FromPg")
+                        .Apply(0, filter->Tail().TailPtr())
+                            .With(0, "row")
+                        .Seal()
+                    .Seal()
+                    .Callable(1, "Bool")
+                        .Atom(0, "0")
+                    .Seal()
+                .Seal()
+            .Seal()
+        .Seal()
+        .Build();
+}
+
 TExprNode::TPtr ExpandPositionalUnionAll(const TExprNode& node, const TVector<TColumnOrder>& columnOrders,
     TExprNode::TListType children, TExprContext& ctx, TOptimizeContext& optCtx) {
     auto targetColumnOrder = optCtx.Types->LookupColumnOrder(node);
@@ -163,7 +184,7 @@ TExprNode::TListType BuildCleanedColumns(TPositionHandle pos, const TExprNode::T
     TExprNode::TListType cleanedInputs;
     for (ui32 i = 0; i < from->Tail().ChildrenSize(); ++i) {
         auto cleaned = ctx.Builder(pos)
-            .Callable("OrderedMap")
+            .Callable("Map")
                 .Add(0, from->Tail().Child(i)->HeadPtr())
                 .Lambda(1)
                     .Param("row")
@@ -219,7 +240,7 @@ std::tuple<TVector<ui32>, TExprNode::TListType> BuildJoinGroups(TPositionHandle
             auto join = groupTuple->Child(i);
             auto joinType = join->Child(0)->Content();
             auto cartesian = ctx.Builder(pos)
-                    .Callable("OrderedFlatMap")
+                    .Callable("FlatMap")
                         .Add(0, current)
                         .Lambda(1)
                             .Param("x")
@@ -245,36 +266,40 @@ std::tuple<TVector<ui32>, TExprNode::TListType> BuildJoinGroups(TPositionHandle
 
             auto buildMinus = [&](auto left, auto right) {
                 return ctx.Builder(pos)
-                    .Callable("OrderedFlatMap")
+                    .Callable("Filter")
                         .Add(0, left)
                         .Lambda(1)
                             .Param("x")
-                            .Callable("OptionalIf")
-                                .Callable(0, "Not")
-                                    .Callable(0, "HasItems")
-                                        .Callable(0, "Filter")
-                                            .Add(0, right)
-                                            .Lambda(1)
-                                                .Param("y")
-                                                .Apply(join->Tail().TailPtr())
-                                                    .With(0)
-                                                        .Callable("FlattenMembers")
-                                                            .List(0)
-                                                                .Atom(0, "")
-                                                                .Arg(1,"x")
-                                                            .Seal()
-                                                            .List(1)
-                                                                .Atom(0, "")
-                                                                .Arg(1, "y")
+                            .Callable(0, "Not")
+                                .Callable(0, "HasItems")
+                                    .Callable(0, "Filter")
+                                        .Add(0, right)
+                                        .Lambda(1)
+                                            .Param("y")
+                                            .Callable("Coalesce")
+                                                .Callable(0, "FromPg")
+                                                    .Apply(0, join->Tail().TailPtr())
+                                                        .With(0)
+                                                            .Callable("FlattenMembers")
+                                                                .List(0)
+                                                                    .Atom(0, "")
+                                                                    .Arg(1,"x")
+                                                                .Seal()
+                                                                .List(1)
+                                                                    .Atom(0, "")
+                                                                    .Arg(1, "y")
+                                                                .Seal()
                                                             .Seal()
-                                                        .Seal()
-                                                    .Done()
+                                                        .Done()
+                                                    .Seal()
+                                                .Seal()
+                                                .Callable(1, "Bool")
+                                                    .Atom(0, "0")
                                                 .Seal()
                                             .Seal()
                                         .Seal()
                                     .Seal()
                                 .Seal()
-                                .Arg(1, "x")
                             .Seal()
                         .Seal()
                     .Seal()
@@ -283,17 +308,7 @@ std::tuple<TVector<ui32>, TExprNode::TListType> BuildJoinGroups(TPositionHandle
 
             TExprNode::TPtr filteredCartesian;
             if (joinType != "cross") {
-                filteredCartesian = ctx.Builder(pos)
-                    .Callable("OrderedFilter")
-                        .Add(0, cartesian)
-                        .Lambda(1)
-                            .Param("row")
-                            .Apply(join->Tail().TailPtr())
-                                .With(0, "row")
-                            .Seal()
-                        .Seal()
-                    .Seal()
-                    .Build();
+                filteredCartesian = BuildFilter(pos, cartesian, join, ctx);
             }
 
             if (joinType == "cross") {
@@ -425,27 +440,6 @@ TExprNode::TPtr BuildProjectionLambda(TPositionHandle pos, const TExprNode::TPtr
         .Build();
 }
 
-TExprNode::TPtr BuildFilter(TPositionHandle pos, const TExprNode::TPtr& list, const TExprNode::TPtr& filter, TExprContext& ctx) {
-    return ctx.Builder(pos)
-        .Callable("Filter")
-            .Add(0, list)
-            .Lambda(1)
-                .Param("row")
-                .Callable("Coalesce")
-                    .Callable(0, "FromPg")
-                        .Apply(0, filter->Tail().TailPtr())
-                            .With(0, "row")
-                        .Seal()
-                    .Seal()
-                    .Callable(1, "Bool")
-                        .Atom(0, "0")
-                    .Seal()
-                .Seal()
-            .Seal()
-        .Seal()
-        .Build();
-}
-
 using TAggs = TVector<std::pair<TExprNode::TPtr, TExprNode::TPtr>>;
 
 std::tuple<TAggs, TNodeMap<ui32>> GatherAggregations(const TExprNode::TPtr& projectionLambda, const TExprNode::TPtr& having) {
diff --git a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp
index 8ba5ddd645..3afd887442 100644
--- a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp
@@ -1271,6 +1271,10 @@ template <typename TMapType, typename TFlatMapType>
 TExprNode::TPtr ConvertMapToFlatmap(TMapType map, TExprContext& ctx) {
     auto list = map.Input();
     auto lambda = map.Lambda();
+    if (list.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Pg ||
+        lambda.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Pg) {
+        return map.Ptr();
+    }
 
     auto ret = Build<TFlatMapType>(ctx, map.Pos())
             .Input(list)
@@ -1292,6 +1296,10 @@ template <typename TFilterType, typename TFlatMapType>
 TExprNode::TPtr ConvertFilterToFlatmap(TFilterType filter, TExprContext& ctx, TOptimizeContext& optCtx) {
     const auto& list = filter.Input();
     const auto& lambda = filter.Lambda();
+    if (list.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Pg ||
+        lambda.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Pg) {
+        return filter.Ptr();
+    }
 
     if (const auto& limit = filter.Limit()) {
         const auto ret = Build<TCoTake>(ctx, filter.Pos())
diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp
index 42dc50af7a..e5542d4b23 100644
--- a/ydb/library/yql/core/type_ann/type_ann_core.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp
@@ -5699,6 +5699,11 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
                 return IGraphTransformer::TStatus::Repeat;
             }
 
+            if (arg->GetTypeAnn() && arg->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Pg) {
+                types.emplace_back(arg->GetTypeAnn());
+                continue;
+            }
+
             if (!EnsureOptionalType(*arg, ctx.Expr)) {
                 return IGraphTransformer::TStatus::Error;
             }
diff --git a/ydb/library/yql/core/type_ann/type_ann_list.cpp b/ydb/library/yql/core/type_ann/type_ann_list.cpp
index d0a8e2b4ae..f15219fbca 100644
--- a/ydb/library/yql/core/type_ann/type_ann_list.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_list.cpp
@@ -555,7 +555,7 @@ namespace {
                 if (calcSpec->IsCallable("WindowTraits")) {
                     auto finishType = calcSpec->Child(4)->GetTypeAnn();
                     if (frameCanBeEmpty) {
-                        bool isOptional = finishType->GetKind() == ETypeAnnotationKind::Optional;
+                        bool isOptional = finishType->IsOptionalOrNull();
 
                         auto defVal = calcSpec->Child(5);
                         if (defVal->IsCallable("Null") && !isOptional) {
@@ -577,7 +577,6 @@ namespace {
         return IGraphTransformer::TStatus::Ok;
     }
 
-
     IGraphTransformer::TStatus ValidateTraitsDefaultValue(const TExprNode::TPtr& input, ui32 defaultValueIndex,
         const TTypeAnnotationNode& finishType, TStringBuf finishName, TExprNode::TPtr& output, TContext& ctx)
     {
@@ -2359,7 +2358,7 @@ namespace {
                 YQL_ENSURE(it != members.end());
                 memberType = it->second.first;
                 if (it->second.first->GetKind() != ETypeAnnotationKind::Error && it->second.second < inputsCount) {
-                    if (memberType->GetKind() != ETypeAnnotationKind::Optional) {
+                    if (!memberType->IsOptionalOrNull()) {
                         memberType = ctx.Expr.MakeType<TOptionalExprType>(memberType);
                     }
                 }
@@ -3400,7 +3399,7 @@ namespace {
             return IGraphTransformer::TStatus::Error;
         }
 
-        const auto optAnn = ctx.Expr.MakeType<TOptionalExprType>(stateType);
+        const auto optAnn = (stateType->GetKind() == ETypeAnnotationKind::Pg) ? stateType : ctx.Expr.MakeType<TOptionalExprType>(stateType);
         input->SetTypeAnn(optAnn);
         return IGraphTransformer::TStatus::Ok;
     }
diff --git a/ydb/library/yql/core/type_ann/type_ann_pg.cpp b/ydb/library/yql/core/type_ann/type_ann_pg.cpp
index 1ea520bc7f..53d1d0d566 100644
--- a/ydb/library/yql/core/type_ann/type_ann_pg.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_pg.cpp
@@ -1635,8 +1635,7 @@ IGraphTransformer::TStatus RebuildLambdaColumns(const TExprNode::TPtr& root, con
 void MakeOptionalColumns(const TStructExprType*& structType, TExprContext& ctx) {
     bool needRebuild = false;
     for (const auto& item : structType->GetItems()) {
-        if (item->GetItemType()->GetKind() != ETypeAnnotationKind::Optional
-            && item->GetItemType()->GetKind() != ETypeAnnotationKind::Null) {
+        if (!item->GetItemType()->IsOptionalOrNull()) {
             needRebuild = true;
             break;
         }
@@ -1648,8 +1647,7 @@ void MakeOptionalColumns(const TStructExprType*& structType, TExprContext& ctx)
 
     auto newItems = structType->GetItems();
     for (auto& item : newItems) {
-        if (item->GetItemType()->GetKind() != ETypeAnnotationKind::Optional
-            && item->GetItemType()->GetKind() != ETypeAnnotationKind::Null) {
+        if (!item->GetItemType()->IsOptionalOrNull()) {
             item = ctx.MakeType<TItemExprType>(item->GetName(), ctx.MakeType<TOptionalExprType>(item->GetItemType()));
         }
     }
@@ -2230,16 +2228,7 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
                                     return IGraphTransformer::TStatus::Error;
                                 }
 
-                                auto predicate = ctx.Expr.Builder(quals.Pos())
-                                    .Callable("Coalesce")
-                                        .Add(0, newRoot)
-                                        .Callable(1, "Bool")
-                                            .Atom(0, "0")
-                                        .Seal()
-                                    .Seal()
-                                    .Build();
-
-                                auto newLambda = ctx.Expr.NewLambda(quals.Pos(), std::move(arguments), std::move(predicate));
+                                auto newLambda = ctx.Expr.NewLambda(quals.Pos(), std::move(arguments), std::move(newRoot));
 
                                 auto newChildren = quals.ChildrenList();
                                 newChildren[0] = typeNode;
diff --git a/ydb/library/yql/core/yql_expr_type_annotation.cpp b/ydb/library/yql/core/yql_expr_type_annotation.cpp
index 81c8360ea3..420abc946b 100644
--- a/ydb/library/yql/core/yql_expr_type_annotation.cpp
+++ b/ydb/library/yql/core/yql_expr_type_annotation.cpp
@@ -3363,7 +3363,7 @@ bool EnsureCodeResourceType(const TExprNode& node, TExprContext& ctx) {
 
 const TTypeAnnotationNode* MakeSequenceType(ETypeAnnotationKind sequenceKind, const TTypeAnnotationNode& itemType, TExprContext& ctx) {
     switch (sequenceKind) {
-        case ETypeAnnotationKind::Optional: return ctx.MakeType<TOptionalExprType>(&itemType);
+        case ETypeAnnotationKind::Optional: return itemType.GetKind() == ETypeAnnotationKind::Pg ? &itemType : ctx.MakeType<TOptionalExprType>(&itemType);
         case ETypeAnnotationKind::Pg:       return &itemType;
         case ETypeAnnotationKind::Flow:     return ctx.MakeType<TFlowExprType>(&itemType);
         case ETypeAnnotationKind::List:     return ctx.MakeType<TListExprType>(&itemType);
diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp
index 461a280a63..4eda1c327f 100644
--- a/ydb/library/yql/minikql/mkql_program_builder.cpp
+++ b/ydb/library/yql/minikql/mkql_program_builder.cpp
@@ -1938,6 +1938,10 @@ TRuntimeNode TProgramBuilder::NewDecimalLiteral(NYql::NDecimal::TInt128 data, ui
 }
 
 TRuntimeNode TProgramBuilder::NewOptional(TRuntimeNode data) {
+    if (data.GetStaticType()->IsPg()) {
+        return data;
+    }
+
     auto type = TOptionalType::Create(data.GetStaticType(), Env);
     return TRuntimeNode(TOptionalLiteral::Create(data, type, Env), true);
 }
diff --git a/ydb/library/yql/sql/settings/translation_settings.cpp b/ydb/library/yql/sql/settings/translation_settings.cpp
index c03821f3c0..f66258a1fc 100644
--- a/ydb/library/yql/sql/settings/translation_settings.cpp
+++ b/ydb/library/yql/sql/settings/translation_settings.cpp
@@ -106,9 +106,6 @@ namespace NSQLTranslation {
                 settings.AnsiLexer = true;
             } else if (value == "syntax_pg") {
                 settings.PgParser = true;
-                settings.PgTypes = false;
-            } else if (value == "syntax_pg_types") {
-                settings.PgParser = true;
             } else {
                 issues.AddIssue(NYql::YqlIssue(NYql::TPosition(0, lineNumber), NYql::TIssuesIds::DEFAULT_ERROR,
                     TStringBuilder() << "Unknown SQL translation setting: " << value));