Use unknown type for PG null

9 files changed, 145 insertions, 8 deletions

diff --git a/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp b/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp
index a55e3bd766..43a3c81ebd 100644
--- a/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_pgselect.cpp
@@ -3009,7 +3009,7 @@ TExprNode::TPtr ExpandPgSelectImpl(const TExprNode::TPtr& node, TExprContext& ct
         TExprNode::TPtr list;
         if (values) {
             YQL_ENSURE(!result);
-            list = BuildValues(node->Pos(), values, targetColumns, ctx);
+            list = ctx.NewCallable(node->Pos(), "PgReplaceUnknown", { BuildValues(node->Pos(), values, targetColumns, ctx) });
         } else {
             YQL_ENSURE(result);
             YQL_ENSURE(!targetColumns, "target columns for projection are not supported yet");
@@ -3130,6 +3130,8 @@ TExprNode::TPtr ExpandPgSelectImpl(const TExprNode::TPtr& node, TExprContext& ct
                 .Seal()
                 .Build();
 
+            list = ctx.NewCallable(node->Pos(), "PgReplaceUnknown", { list });
+
             if (distinctAll) {
                 YQL_ENSURE(!extraSortColumns);
                 list = ctx.NewCallable(node->Pos(), "SqlAggregateAll", { list });
diff --git a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp
index 2b0a876da4..0b20383ce5 100644
--- a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp
+++ b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp
@@ -689,6 +689,18 @@ TExprNode::TPtr ExpandCastOverData(const TExprNode::TPtr& input, TExprContext& c
 }
 
 template <bool Strong>
+TExprNode::TPtr ExpandCastOverPg(const TExprNode::TPtr& input, TExprContext& ctx) {
+    auto to = input->GetTypeAnn()->Cast<TPgExprType>();
+    YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Pg";
+    return ctx.Builder(input->Pos())
+        .Callable("PgCast")
+            .Add(0, input->HeadPtr())
+            .Add(1, ExpandType(input->Pos(), *to, ctx))
+        .Seal()
+        .Build();
+}
+
+template <bool Strong>
 TExprNode::TPtr ExpandCastOverOptionalData(const TExprNode::TPtr& input, TExprContext& ctx) {
     if constexpr (Strong) {
         YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content()
@@ -1556,6 +1568,7 @@ TExprNode::TPtr ExpandCast(const TExprNode::TPtr& input, TExprContext& ctx) {
     if (sKind == tKind) {
         switch (sKind) {
             case ETypeAnnotationKind::Data:     return ExpandCastOverData<Strong>(input, ctx);
+            case ETypeAnnotationKind::Pg:       return ExpandCastOverPg<Strong>(input, ctx);
             case ETypeAnnotationKind::Optional: return ExpandCastOverOptional<Strong>(input, ctx);
             case ETypeAnnotationKind::List:     return ExpandCastOverList<Strong>(input, ctx);
             case ETypeAnnotationKind::Dict:     return ExpandCastOverDict<Strong>(input, ctx);
diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp
index e28a0e4583..ed7de115d5 100644
--- a/ydb/library/yql/core/type_ann/type_ann_core.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp
@@ -11790,6 +11790,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
         Functions["PgQualifiedStar"] = &PgQualifiedStarWrapper;
         Functions["PgColumnRef"] = &PgColumnRefWrapper;
         Functions["PgResultItem"] = &PgResultItemWrapper;
+        Functions["PgReplaceUnknown"] = &PgReplaceUnknownWrapper;
         Functions["PgWhere"] = &PgWhereWrapper;
         Functions["PgSort"] = &PgSortWrapper;
         Functions["PgGroup"] = &PgWhereWrapper;
diff --git a/ydb/library/yql/core/type_ann/type_ann_pg.cpp b/ydb/library/yql/core/type_ann/type_ann_pg.cpp
index 1aa7c0591d..f2086dfb64 100644
--- a/ydb/library/yql/core/type_ann/type_ann_pg.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_pg.cpp
@@ -17,6 +17,21 @@ namespace NYql {
 
 namespace NTypeAnnImpl {
 
+bool AdjustUnknownType(TVector<const TItemExprType*>& outputItems, TExprContext& ctx) {
+    bool replaced = false;
+    for (auto& item : outputItems) {
+        if (item->GetItemType()->GetKind() == ETypeAnnotationKind::Pg &&
+            item->GetItemType()->Cast<TPgExprType>()->GetName() == "unknown") {
+                replaced = true;
+                item = ctx.MakeType<TItemExprType>(
+                    item->GetName(),
+                    ctx.MakeType<TPgExprType>(NPg::LookupType("text").TypeId));
+            }
+    }
+
+    return replaced;
+}
+
 bool ValidateInputTypes(TExprNode& node, TExprContext& ctx) {
     if (!EnsureTuple(node, ctx)) {
         return false;
@@ -171,6 +186,8 @@ const TTypeAnnotationNode* FromPgImpl(TPositionHandle pos, const TTypeAnnotation
         dataType = ctx.MakeType<TDataExprType>(EDataSlot::Utf8);
     } else if (name == "bytea") {
         dataType = ctx.MakeType<TDataExprType>(EDataSlot::String);
+    } else if (name == "unknown") {
+        return ctx.MakeType<TNullExprType>();
     } else {
         ctx.AddError(TIssue(ctx.GetPosition(pos),
             TStringBuilder() << "Unsupported type: " << name));
@@ -623,6 +640,48 @@ IGraphTransformer::TStatus PgResultItemWrapper(const TExprNode::TPtr& input, TEx
     return IGraphTransformer::TStatus::Ok;
 }
 
+IGraphTransformer::TStatus PgReplaceUnknownWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+    if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
+        return IGraphTransformer::TStatus::Error;
+    }
+
+    if (!EnsureListType(input->Head(), ctx.Expr)) {
+        return IGraphTransformer::TStatus::Error;
+    }
+
+    auto listType = input->Head().GetTypeAnn()->Cast<TListExprType>();
+    if (!EnsureStructType(input->Head().Pos(), *listType->GetItemType(), ctx.Expr)) {
+        return IGraphTransformer::TStatus::Error;
+    }
+
+    auto structType = listType->GetItemType()->Cast<TStructExprType>();
+    auto newItemTypes = structType->GetItems();
+    if (!AdjustUnknownType(newItemTypes, ctx.Expr)) {
+        output = input->HeadPtr();
+        return IGraphTransformer::TStatus::Repeat;
+    }
+
+    auto newStructType = ctx.Expr.MakeType<TStructExprType>(newItemTypes);
+    if (!newStructType->Validate(input->Pos(), ctx.Expr)) {
+        return IGraphTransformer::TStatus::Error;
+    }
+
+    output = ctx.Expr.Builder(input->Pos())
+        .Callable("OrderedMap")
+            .Add(0, input->HeadPtr())
+            .Lambda(1)
+                .Param("row")
+                .Callable("SafeCast")
+                    .Arg(0, "row")
+                    .Add(1, ExpandType(input->Pos(), *newStructType, ctx.Expr))
+                .Seal()
+            .Seal()
+        .Seal()
+        .Build();
+
+    return IGraphTransformer::TStatus::Repeat;
+}
+
 IGraphTransformer::TStatus PgWhereWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
     Y_UNUSED(output);
     if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
@@ -922,7 +981,8 @@ IGraphTransformer::TStatus PgConstWrapper(const TExprNode::TPtr& input, TExprNod
         }
     }
 
-    input->SetTypeAnn(input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType());
+    auto retType = input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+    input->SetTypeAnn(retType);
     return IGraphTransformer::TStatus::Ok;
 }
 
@@ -966,7 +1026,13 @@ IGraphTransformer::TStatus PgCastWrapper(const TExprNode::TPtr& input, TExprNode
 
     if (inputTypeId != 0 && inputTypeId != targetTypeId) {
         bool fail = false;
-        const auto& inputDesc = NPg::LookupType(inputTypeId);
+        const auto& rawInputDesc = NPg::LookupType(inputTypeId);
+        const NPg::TTypeDesc* inputDescPtr = &rawInputDesc;
+        if (rawInputDesc.Name == "unknown") {
+            inputDescPtr = &NPg::LookupType("text");
+        }
+
+        const NPg::TTypeDesc& inputDesc = *inputDescPtr;
         const auto& targetDesc = NPg::LookupType(targetTypeId);
         const bool isInputArray = (inputDesc.TypeId == inputDesc.ArrayTypeId);
         const bool isTargetArray = (targetDesc.TypeId == targetDesc.ArrayTypeId);
@@ -2620,6 +2686,7 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
                         outputItems.push_back(ctx.Expr.MakeType<TItemExprType>(columnName, tupleType->GetItems()[i]));
                     }
 
+                    AdjustUnknownType(outputItems, ctx.Expr);
                     outputRowType = ctx.Expr.MakeType<TStructExprType>(outputItems);
                     if (!outputRowType->Validate(names->Pos(), ctx.Expr)) {
                         return IGraphTransformer::TStatus::Error;
@@ -2869,6 +2936,7 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN
                             return IGraphTransformer::TStatus::Repeat;
                         }
 
+                        AdjustUnknownType(outputItems, ctx.Expr);
                         outputRowType = ctx.Expr.MakeType<TStructExprType>(outputItems);
                         if (!outputRowType->Validate(data.Pos(), ctx.Expr)) {
                             return IGraphTransformer::TStatus::Error;
@@ -4310,23 +4378,32 @@ IGraphTransformer::TStatus PgInWrapper(const TExprNode::TPtr& input, TExprNode::
         return IGraphTransformer::TStatus::Error;
     }
 
-    if (!listItemType->IsEquatable()) {
+    if (itemTypePg && !listItemType->IsEquatable()) {
         ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()),
             TStringBuilder() << "Cannot compare items of type: " << NPg::LookupType(itemTypePg).Name));
     }
 
-    if (!inputType->IsEquatable()) {
+    if (inputTypePg && !inputType->IsEquatable()) {
         ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()),
             TStringBuilder() << "Cannot compare items of type: " << NPg::LookupType(inputTypePg).Name));
     }
 
+    if (!itemTypePg || !inputTypePg) {
+        output = ctx.Expr.Builder(input->Pos())
+            .Callable("Nothing")
+                .Callable(0, "PgType")
+                    .Atom(0, "bool")
+                .Seal()
+            .Seal()
+            .Build();
+    }
+
     auto result = ctx.Expr.MakeType<TPgExprType>(NPg::LookupType("bool").TypeId);
     input->SetTypeAnn(result);
     return IGraphTransformer::TStatus::Ok;
 }
 
 IGraphTransformer::TStatus PgBetweenWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
-    Y_UNUSED(output);
     if (!EnsureArgsCount(*input, 3, ctx.Expr)) {
         return IGraphTransformer::TStatus::Error;
     }
@@ -4373,6 +4450,16 @@ IGraphTransformer::TStatus PgBetweenWrapper(const TExprNode::TPtr& input, TExprN
         return IGraphTransformer::TStatus::Error;
     }
 
+    if (!elemType) {
+        output = ctx.Expr.Builder(input->Pos())
+            .Callable("Nothing")
+                .Callable(0, "PgType")
+                    .Atom(0, "bool")
+                .Seal()
+            .Seal()
+            .Build();
+    }
+
     auto result = ctx.Expr.MakeType<TPgExprType>(NPg::LookupType("bool").TypeId);
     input->SetTypeAnn(result);
     return IGraphTransformer::TStatus::Ok;
diff --git a/ydb/library/yql/core/type_ann/type_ann_pg.h b/ydb/library/yql/core/type_ann/type_ann_pg.h
index 86c9b53347..fb7310f53b 100644
--- a/ydb/library/yql/core/type_ann/type_ann_pg.h
+++ b/ydb/library/yql/core/type_ann/type_ann_pg.h
@@ -30,6 +30,7 @@ IGraphTransformer::TStatus PgAggWrapper(const TExprNode::TPtr& input, TExprNode:
 IGraphTransformer::TStatus PgQualifiedStarWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
 IGraphTransformer::TStatus PgColumnRefWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
 IGraphTransformer::TStatus PgResultItemWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
+IGraphTransformer::TStatus PgReplaceUnknownWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
 IGraphTransformer::TStatus PgWhereWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
 IGraphTransformer::TStatus PgSortWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
 IGraphTransformer::TStatus PgWindowWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
diff --git a/ydb/library/yql/core/yql_expr_type_annotation.cpp b/ydb/library/yql/core/yql_expr_type_annotation.cpp
index 703efc6a5b..41f4a7db53 100644
--- a/ydb/library/yql/core/yql_expr_type_annotation.cpp
+++ b/ydb/library/yql/core/yql_expr_type_annotation.cpp
@@ -110,6 +110,13 @@ IGraphTransformer::TStatus TryConvertToImpl(TExprContext& ctx, TExprNode::TPtr&
             node = ctx.NewCallable(node->Pos(), "Nothing", { ExpandType(node->Pos(), expectedType, ctx) });
             return IGraphTransformer::TStatus::Repeat;
         }
+
+        if (sourceType.GetKind() == ETypeAnnotationKind::Pg) {
+            if (sourceType.Cast<TPgExprType>()->GetName() == "unknown") {
+                node = ctx.NewCallable(node->Pos(), "PgCast", { node, ExpandType(node->Pos(), expectedType, ctx) });
+                return IGraphTransformer::TStatus::Repeat;
+            }
+        }
     }
 
     if (expectedType.GetKind() == ETypeAnnotationKind::Optional) {
@@ -940,6 +947,10 @@ NUdf::TCastResultOptions CastResult(const TDataExprType* source, const TDataExpr
 
 template <bool Strong>
 NUdf::TCastResultOptions CastResult(const TPgExprType* source, const TPgExprType* target) {
+    if (source->GetName() == "unknown") {
+        return NUdf::ECastOptions::Complete;
+    }
+
     if (source->GetId() != target->GetId()) {
         return NUdf::ECastOptions::Impossible;
     }
@@ -1299,6 +1310,14 @@ const TPgExprType* CommonType(TPositionHandle pos, const TPgExprType* one, const
         return one;
     }
 
+    if (one->GetName() == "unknown") {
+        return two;
+    }
+
+    if (two->GetName() == "unknown") {
+        return one;
+    }
+
     ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Cannot infer common type for " << one->GetName() << " and " << two->GetName()));
     return nullptr;
 }
@@ -5437,6 +5456,10 @@ bool ExtractPgType(const TTypeAnnotationNode* type, ui32& pgType, bool& convertT
         return false;
     } else {
         pgType = type->Cast<TPgExprType>()->GetId();
+        if (NPg::LookupType(pgType).Name == "unknown") {
+            pgType = 0;
+        }
+
         return true;
     }
 }
@@ -5875,7 +5898,9 @@ TExprNode::TPtr ExpandPgAggregationTraits(TPositionHandle pos, const NPg::TAggre
             .Seal()
             .Build();
 
-        if (lambda->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Null) {
+        if ((lambda->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Null) ||
+            (lambda->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Pg &&
+            lambda->GetTypeAnn()->Cast<TPgExprType>()->GetName() == "unknown")) {
             initLambda = ctx.Builder(pos)
                 .Lambda()
                     .Param("row")
diff --git a/ydb/library/yql/parser/pg_catalog/catalog.cpp b/ydb/library/yql/parser/pg_catalog/catalog.cpp
index 38a23938e3..5e7c704ecc 100644
--- a/ydb/library/yql/parser/pg_catalog/catalog.cpp
+++ b/ydb/library/yql/parser/pg_catalog/catalog.cpp
@@ -9,6 +9,7 @@
 namespace NYql::NPg {
 
 constexpr ui32 InvalidOid = 0;
+constexpr ui32 UnknownOid = 705;
 constexpr ui32 AnyOid = 2276;
 constexpr ui32 AnyArrayOid = 2277;
 //constexpr ui32 AnyElementOid = 2283;
@@ -46,6 +47,10 @@ bool IsCompatibleTo(ui32 actualType, ui32 expectedType, const TTypes& types) {
         return true;
     }
 
+    if (actualType == UnknownOid) {
+        return true;
+    }
+
     if (expectedType == AnyOid) {
         return true;
     }
diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp
index 53298ba799..bff63c78ab 100644
--- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp
+++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp
@@ -1771,6 +1771,9 @@ TComputationNodeFactory GetPgFactory() {
                 ui32 sourceId = 0;
                 if (!inputType->IsNull()) {
                     sourceId = AS_TYPE(TPgType, inputType)->GetTypeId();
+                    if (sourceId == UNKNOWNOID) {
+                        sourceId = TEXTOID;
+                    }
                 }
 
                 auto returnType = callable.GetType()->GetReturnType();
diff --git a/ydb/library/yql/sql/pg/pg_sql.cpp b/ydb/library/yql/sql/pg/pg_sql.cpp
index 4dcc46b2e6..1131f584c2 100644
--- a/ydb/library/yql/sql/pg/pg_sql.cpp
+++ b/ydb/library/yql/sql/pg/pg_sql.cpp
@@ -2144,7 +2144,7 @@ public:
             return L(A("PgConst"), QAX(ToString(StrVal(val))), L(A("PgType"), QA("text")));
         }
         case T_Null: {
-            return L(A("PgCast"), L(A("Null")), L(A("PgType"), QA("text")));
+            return L(A("PgCast"), L(A("Null")), L(A("PgType"), QA("unknown")));
         }
         default:
             ValueNotImplemented(value, val);