Apply GH commits

Apply GH: Extract prefix and entries in backup-related sql (#10807)

Apply GH: Fix syntax for Column Family (#10781)

Apply GH: Case-insensitive mode for searching modules and functions (#10842)

Apply GH: Fixed i/o for pg_proc (#10914)

Apply GH: An option to render SQL transalation with Seq! (#11015)
commit_hash:d2d2fcdef2bbd0434236aef325aa071c7e39c526

1 files changed, 123 insertions, 46 deletions

diff --git a/yql/essentials/sql/v1/query.cpp b/yql/essentials/sql/v1/query.cpp
index 5548587090e..56dd8bd63d7 100644
--- a/yql/essentials/sql/v1/query.cpp
+++ b/yql/essentials/sql/v1/query.cpp
@@ -2693,15 +2693,44 @@ TNodePtr BuildWriteResult(TPosition pos, const TString& label, TNodePtr settings
 
 class TYqlProgramNode: public TAstListNode {
 public:
-    TYqlProgramNode(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped)
+    TYqlProgramNode(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped, bool useSeq)
         : TAstListNode(pos)
         , Blocks(blocks)
         , TopLevel(topLevel)
         , Scoped(scoped)
+        , UseSeq(useSeq)
     {}
 
     bool DoInit(TContext& ctx, ISource* src) override {
         bool hasError = false;
+        INode::TPtr currentWorldsHolder;
+        INode::TPtr seqNode;
+        if (UseSeq) {
+            currentWorldsHolder = new TAstListNodeImpl(GetPos());
+            seqNode = new TAstListNodeImpl(GetPos());
+            seqNode->Add("Seq!","world");
+        }
+
+        INode* currentWorlds = UseSeq ? currentWorldsHolder.Get() : this;
+        auto flushCurrentWorlds = [&](bool changeSeq, bool finish) {
+            currentWorldsHolder->Add(Y("return","world"));
+            auto lambda = BuildLambda(GetPos(), Y("world"), Y("block", Q(currentWorldsHolder)));
+            seqNode->Add(lambda);
+
+            if (finish) {
+                Add(Y("let", "world", seqNode));
+            } else {
+                currentWorldsHolder = new TAstListNodeImpl(GetPos());
+                currentWorlds = currentWorldsHolder.Get();
+            }
+
+            if (changeSeq) {
+                Add(Y("let", "world", seqNode));
+                seqNode = new TAstListNodeImpl(GetPos());
+                seqNode->Add("Seq!","world");
+            }
+        };
+
         if (TopLevel) {
             for (auto& var: ctx.Variables) {
                 if (!var.second.second->Init(ctx, src)) {
@@ -2798,33 +2827,33 @@ public:
                 auto resultSink = Y("DataSink", BuildQuotedAtom(Pos, TString(ResultProviderName)));
 
                 for (const auto& warningPragma : ctx.WarningPolicy.GetRules()) {
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
                         BuildQuotedAtom(Pos, "Warning"), BuildQuotedAtom(Pos, warningPragma.GetPattern()),
                             BuildQuotedAtom(Pos, to_lower(ToString(warningPragma.GetAction()))))));
                 }
 
                 if (ctx.ResultSizeLimit > 0) {
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", resultSink,
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", resultSink,
                         BuildQuotedAtom(Pos, "SizeLimit"), BuildQuotedAtom(Pos, ToString(ctx.ResultSizeLimit)))));
                 }
 
                 if (!ctx.PragmaPullUpFlatMapOverJoin) {
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
                         BuildQuotedAtom(Pos, "DisablePullUpFlatMapOverJoin"))));
                 }
 
                 if (ctx.FilterPushdownOverJoinOptionalSide) {
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
                         BuildQuotedAtom(Pos, "FilterPushdownOverJoinOptionalSide"))));
                 }
 
                 if (!ctx.RotateJoinTree) {
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
                         BuildQuotedAtom(Pos, "RotateJoinTree"), BuildQuotedAtom(Pos, "false"))));
                 }
 
                 if (ctx.DiscoveryMode) {
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
                         BuildQuotedAtom(Pos, "DiscoveryMode"))));
                 }
 
@@ -2835,12 +2864,12 @@ public:
                     } else if (ctx.DqEngineForce) {
                         mode = "force";
                     }
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
                         BuildQuotedAtom(Pos, "DqEngine"), BuildQuotedAtom(Pos, mode))));
                 }
 
                 if (ctx.CostBasedOptimizer) {
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
                         BuildQuotedAtom(Pos, "CostBasedOptimizer"), BuildQuotedAtom(Pos, ctx.CostBasedOptimizer))));
                 }
 
@@ -2850,43 +2879,43 @@ public:
                         pragmaName = "JsonQueryReturnsJsonDocument";
                     }
 
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, pragmaName))));
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, pragmaName))));
                 }
 
                 if (ctx.OrderedColumns) {
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
                         BuildQuotedAtom(Pos, "OrderedColumns"))));
                 }
 
                 if (ctx.PqReadByRtmrCluster) {
                     auto pqSourceAll = Y("DataSource", BuildQuotedAtom(Pos, TString(PqProviderName)), BuildQuotedAtom(Pos, "$all"));
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", pqSourceAll,
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", pqSourceAll,
                         BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "PqReadByRtmrCluster_"), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster))));
 
                     auto rtmrSourceAll = Y("DataSource", BuildQuotedAtom(Pos, TString(RtmrProviderName)), BuildQuotedAtom(Pos, "$all"));
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSourceAll,
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSourceAll,
                         BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "PqReadByRtmrCluster_"), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster))));
 
                     if (ctx.PqReadByRtmrCluster != "dq") {
                         // set any dynamic settings for particular RTMR cluster for CommitAll!
                         auto rtmrSource = Y("DataSource", BuildQuotedAtom(Pos, TString(RtmrProviderName)), BuildQuotedAtom(Pos, ctx.PqReadByRtmrCluster));
-                        Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSource,
+                        currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", rtmrSource,
                             BuildQuotedAtom(Pos, "Attr"), BuildQuotedAtom(Pos, "Dummy_"), BuildQuotedAtom(Pos, "1"))));
                     }
                 }
 
                 if (ctx.YsonCastToString.Defined()) {
                     const TString pragmaName = *ctx.YsonCastToString ? "YsonCastToString" : "DisableYsonCastToString";
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, pragmaName))));
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, pragmaName))));
                 }
 
                 if (ctx.UseBlocks) {
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "UseBlocks"))));
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource, BuildQuotedAtom(Pos, "UseBlocks"))));
                 }
 
                 if (ctx.BlockEngineEnable) {
                     TString mode = ctx.BlockEngineForce ? "force" : "auto";
-                    Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
+                    currentWorlds->Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
                         BuildQuotedAtom(Pos, "BlockEngine"), BuildQuotedAtom(Pos, mode))));
                 }
             }
@@ -2914,22 +2943,41 @@ public:
             Add(Y("let", data.first, node));
         }
 
+        if (UseSeq) {
+            flushCurrentWorlds(false, false);
+        }
+
         for (auto& block: Blocks) {
             const auto subqueryAliasPtr = block->SubqueryAlias();
             if (subqueryAliasPtr) {
                 if (block->UsedSubquery()) {
+                    if (UseSeq) {
+                        flushCurrentWorlds(true, false);
+                    }
+
                     const auto& ref = block->GetLabel();
                     YQL_ENSURE(!ref.empty());
                     Add(block);
-                    Add(Y("let", "world", Y("Nth", *subqueryAliasPtr, Q("0"))));
+                    currentWorlds->Add(Y("let", "world", Y("Nth", *subqueryAliasPtr, Q("0"))));
                     Add(Y("let", ref, Y("Nth", *subqueryAliasPtr, Q("1"))));
                 }
             } else {
                 const auto& ref = block->GetLabel();
-                Add(Y("let", ref ? ref : "world", block));
+                if (ref) {
+                    Add(Y("let", ref, block));
+                } else {
+                    currentWorlds->Add(Y("let", "world", block));
+                    if (UseSeq) {
+                        flushCurrentWorlds(false, false);
+                    }
+                }
             }
         }
 
+        if (UseSeq) {
+            flushCurrentWorlds(false, true);
+        }
+
         if (TopLevel) {
             if (ctx.UniversalAliases) {
                 decltype(Nodes) preparedNodes;
@@ -2974,10 +3022,11 @@ private:
     TVector<TNodePtr> Blocks;
     const bool TopLevel;
     TScopedStatePtr Scoped;
+    const bool UseSeq;
 };
 
-TNodePtr BuildQuery(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped) {
-    return new TYqlProgramNode(pos, blocks, topLevel, scoped);
+TNodePtr BuildQuery(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped, bool useSeq) {
+    return new TYqlProgramNode(pos, blocks, topLevel, scoped, useSeq);
 }
 
 class TPragmaNode final: public INode {
@@ -3285,16 +3334,18 @@ class TBaseBackupCollectionNode
 public:
     TBaseBackupCollectionNode(
         TPosition pos,
+        const TString& prefix,
         const TString& objectId,
         const TObjectOperatorContext& context)
             : TBase(pos)
             , TObjectOperatorContext(context)
+            , Prefix(prefix)
             , Id(objectId)
     {}
 
     bool DoInit(TContext& ctx, ISource* src) final {
         auto keys = Y("Key");
-        keys = L(keys, Q(Y(Q("backupCollection"), Y("String", BuildQuotedAtom(Pos, Id)))));
+        keys = L(keys, Q(Y(Q("backupCollection"), Y("String", BuildQuotedAtom(Pos, Id)), Y("String", BuildQuotedAtom(Pos, Prefix)))));
         auto options = this->FillOptions(ctx, Y());
 
         Add("block", Q(Y(
@@ -3309,6 +3360,7 @@ public:
     virtual INode::TPtr FillOptions(TContext& ctx, INode::TPtr options) const = 0;
 
 protected:
+    TString Prefix;
     TString Id;
 };
 
@@ -3319,10 +3371,11 @@ class TCreateBackupCollectionNode
 public:
     TCreateBackupCollectionNode(
         TPosition pos,
+        const TString& prefix,
         const TString& objectId,
         const TCreateBackupCollectionParameters& params,
         const TObjectOperatorContext& context)
-            : TBase(pos, objectId, context)
+            : TBase(pos, prefix, objectId, context)
             , Params(params)
     {}
 
@@ -3331,7 +3384,7 @@ public:
 
         auto settings = Y();
         for (auto& [key, value] : Params.Settings) {
-            settings->Add(Q(Y(BuildQuotedAtom(Pos, key), value.Build())));
+            settings->Add(Q(Y(BuildQuotedAtom(Pos, key), Y("String", value.Build()))));
         }
         options->Add(Q(Y(Q("settings"), Q(settings))));
 
@@ -3349,7 +3402,7 @@ public:
     }
 
     TPtr DoClone() const final {
-        return new TCreateBackupCollectionNode(GetPos(), Id, Params, *this);
+        return new TCreateBackupCollectionNode(GetPos(), Prefix, Id, Params, *this);
     }
 
 private:
@@ -3363,10 +3416,11 @@ class TAlterBackupCollectionNode
 public:
     TAlterBackupCollectionNode(
         TPosition pos,
+        const TString& prefix,
         const TString& objectId,
         const TAlterBackupCollectionParameters& params,
         const TObjectOperatorContext& context)
-            : TBase(pos, objectId, context)
+            : TBase(pos, prefix, objectId, context)
             , Params(params)
     {}
 
@@ -3375,7 +3429,7 @@ public:
 
         auto settings = Y();
         for (auto& [key, value] : Params.Settings) {
-            settings->Add(Q(Y(BuildQuotedAtom(Pos, key), value.Build())));
+            settings->Add(Q(Y(BuildQuotedAtom(Pos, key), Y("String", value.Build()))));
         }
         options->Add(Q(Y(Q("settings"), Q(settings))));
 
@@ -3403,7 +3457,7 @@ public:
     }
 
     TPtr DoClone() const final {
-        return new TAlterBackupCollectionNode(GetPos(), Id, Params, *this);
+        return new TAlterBackupCollectionNode(GetPos(), Prefix, Id, Params, *this);
     }
 
 private:
@@ -3417,10 +3471,11 @@ class TDropBackupCollectionNode
 public:
     TDropBackupCollectionNode(
         TPosition pos,
+        const TString& prefix,
         const TString& objectId,
         const TDropBackupCollectionParameters&,
         const TObjectOperatorContext& context)
-            : TBase(pos, objectId, context)
+            : TBase(pos, prefix, objectId, context)
     {}
 
     virtual INode::TPtr FillOptions(TContext&, INode::TPtr options) const final {
@@ -3431,29 +3486,38 @@ public:
 
     TPtr DoClone() const final {
         TDropBackupCollectionParameters params;
-        return new TDropBackupCollectionNode(GetPos(), Id, params, *this);
+        return new TDropBackupCollectionNode(GetPos(), Prefix, Id, params, *this);
     }
 };
 
-TNodePtr BuildCreateBackupCollection(TPosition pos, const TString& id,
+TNodePtr BuildCreateBackupCollection(
+    TPosition pos,
+    const TString& prefix,
+    const TString& id,
     const TCreateBackupCollectionParameters& params,
     const TObjectOperatorContext& context)
 {
-    return new TCreateBackupCollectionNode(pos, id, params, context);
+    return new TCreateBackupCollectionNode(pos, prefix, id, params, context);
 }
 
-TNodePtr BuildAlterBackupCollection(TPosition pos, const TString& id,
+TNodePtr BuildAlterBackupCollection(
+    TPosition pos,
+    const TString& prefix,
+    const TString& id,
     const TAlterBackupCollectionParameters& params,
     const TObjectOperatorContext& context)
 {
-    return new TAlterBackupCollectionNode(pos, id, params, context);
+    return new TAlterBackupCollectionNode(pos, prefix, id, params, context);
 }
 
-TNodePtr BuildDropBackupCollection(TPosition pos, const TString& id,
+TNodePtr BuildDropBackupCollection(
+    TPosition pos,
+    const TString& prefix,
+    const TString& id,
     const TDropBackupCollectionParameters& params,
     const TObjectOperatorContext& context)
 {
-    return new TDropBackupCollectionNode(pos, id, params, context);
+    return new TDropBackupCollectionNode(pos, prefix, id, params, context);
 }
 
 class TBackupNode final
@@ -3464,11 +3528,13 @@ class TBackupNode final
 public:
     TBackupNode(
         TPosition pos,
+        const TString& prefix,
         const TString& id,
         const TBackupParameters& params,
         const TObjectOperatorContext& context)
             : TBase(pos)
             , TObjectOperatorContext(context)
+            , Prefix(prefix)
             , Id(id)
             , Params(params)
     {
@@ -3477,13 +3543,14 @@ public:
 
     bool DoInit(TContext& ctx, ISource* src) override {
         auto keys = Y("Key");
-        keys = L(keys, Q(Y(Q("backup"), Y("String", BuildQuotedAtom(Pos, Id)))));
+        keys = L(keys, Q(Y(Q("backup"), Y("String", BuildQuotedAtom(Pos, Id)), Y("String", BuildQuotedAtom(Pos, Prefix)))));
 
         auto opts = Y();
-        opts->Add(Q(Y(Q("mode"), Q("backup"))));
 
         if (Params.Incremental) {
-            opts->Add(Q(Y(Q("incremental"))));
+            opts->Add(Q(Y(Q("mode"), Q("backupIncremental"))));
+        } else {
+            opts->Add(Q(Y(Q("mode"), Q("backup"))));
         }
 
         Add("block", Q(Y(
@@ -3496,18 +3563,22 @@ public:
     }
 
     TPtr DoClone() const final {
-        return new TBackupNode(GetPos(), Id, Params, *this);
+        return new TBackupNode(GetPos(), Prefix, Id, Params, *this);
     }
 private:
+    TString Prefix;
     TString Id;
     TBackupParameters Params;
 };
 
-TNodePtr BuildBackup(TPosition pos, const TString& id,
+TNodePtr BuildBackup(
+    TPosition pos,
+    const TString& prefix,
+    const TString& id,
     const TBackupParameters& params,
     const TObjectOperatorContext& context)
 {
-    return new TBackupNode(pos, id, params, context);
+    return new TBackupNode(pos, prefix, id, params, context);
 }
 
 class TRestoreNode final
@@ -3518,11 +3589,13 @@ class TRestoreNode final
 public:
     TRestoreNode(
         TPosition pos,
+        const TString& prefix,
         const TString& id,
         const TRestoreParameters& params,
         const TObjectOperatorContext& context)
             : TBase(pos)
             , TObjectOperatorContext(context)
+            , Prefix(prefix)
             , Id(id)
             , Params(params)
     {
@@ -3531,7 +3604,7 @@ public:
 
     bool DoInit(TContext& ctx, ISource* src) override {
         auto keys = Y("Key");
-        keys = L(keys, Q(Y(Q("restore"), Y("String", BuildQuotedAtom(Pos, Id)))));
+        keys = L(keys, Q(Y(Q("restore"), Y("String", BuildQuotedAtom(Pos, Id)), Y("String", BuildQuotedAtom(Pos, Prefix)))));
 
         auto opts = Y();
         opts->Add(Q(Y(Q("mode"), Q("restore"))));
@@ -3550,18 +3623,22 @@ public:
     }
 
     TPtr DoClone() const final {
-        return new TRestoreNode(GetPos(), Id, Params, *this);
+        return new TRestoreNode(GetPos(), Prefix, Id, Params, *this);
     }
 private:
+    TString Prefix;
     TString Id;
     TRestoreParameters Params;
 };
 
-TNodePtr BuildRestore(TPosition pos, const TString& id,
+TNodePtr BuildRestore(
+    TPosition pos,
+    const TString& prefix,
+    const TString& id,
     const TRestoreParameters& params,
     const TObjectOperatorContext& context)
 {
-    return new TRestoreNode(pos, id, params, context);
+    return new TRestoreNode(pos, prefix, id, params, context);
 }
 
 } // namespace NSQLTranslationV1