Reduce query compilation overhead. (KIKIMR-14779)

ref:e968f151b8ab3324123c57bad1cc510ddd76b35b

7 files changed, 49 insertions, 16 deletions

diff --git a/ydb/core/kqp/host/kqp_runner.cpp b/ydb/core/kqp/host/kqp_runner.cpp
index 581f75e0486..52fef46c598 100644
--- a/ydb/core/kqp/host/kqp_runner.cpp
+++ b/ydb/core/kqp/host/kqp_runner.cpp
@@ -516,7 +516,7 @@ private:
         }
 
         auto query = kqlQuery->Ptr();
-        YQL_CLOG(INFO, ProviderKqp) << "Initial KQL query: " << KqpExprToPrettyString(*query, ctx);
+        YQL_CLOG(DEBUG, ProviderKqp) << "Initial KQL query: " << KqpExprToPrettyString(*query, ctx);
 
         TransformCtx->Reset();
         TransformCtx->Settings = NKikimrKqp::TKqlSettings();
@@ -529,7 +529,9 @@ private:
             return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
         }
 
-        YQL_CLOG(INFO, ProviderKqp) << "Optimized KQL query: " << KqpExprToPrettyString(*optimizedQuery, ctx);
+        YQL_CLOG(TRACE, ProviderKqp) << "PhysicalOptimizeTransformer: "
+            << TransformerStatsToYson(PhysicalOptimizeTransformer->GetStatistics());
+        YQL_CLOG(DEBUG, ProviderKqp) << "Optimized KQL query: " << KqpExprToPrettyString(*optimizedQuery, ctx);
 
         BuildQueryCtx->Reset();
         PhysicalBuildQueryTransformer->Rewind();
@@ -540,6 +542,9 @@ private:
             return MakeKikimrResultHolder(ResultFromErrors<IKqpHost::TQueryResult>(ctx.IssueManager.GetIssues()));
         }
 
+        YQL_CLOG(TRACE, ProviderKqp) << "PhysicalBuildQueryTransformer: "
+            << TransformerStatsToYson(PhysicalBuildQueryTransformer->GetStatistics());
+
         PhysicalPeepholeTransformer->Rewind();
         auto transformedQuery = builtQuery;
         status = InstantTransform(*PhysicalPeepholeTransformer, transformedQuery, ctx);
@@ -549,7 +554,9 @@ private:
                 ctx.IssueManager.GetIssues()));
         }
 
-        YQL_CLOG(INFO, ProviderKqp) << "Physical KQL query: " << KqpExprToPrettyString(*builtQuery, ctx);
+        YQL_CLOG(TRACE, ProviderKqp) << "PhysicalPeepholeTransformer: "
+            << TransformerStatsToYson(PhysicalPeepholeTransformer->GetStatistics());
+        YQL_CLOG(DEBUG, ProviderKqp) << "Physical KQL query: " << KqpExprToPrettyString(*builtQuery, ctx);
 
         auto& preparedQuery = *TransformCtx->QueryCtx->PreparingQuery;
         TKqpPhysicalQuery physicalQuery(transformedQuery);
diff --git a/ydb/core/kqp/opt/kqp_opt_build_txs.cpp b/ydb/core/kqp/opt/kqp_opt_build_txs.cpp
index f35ba23f666..ac4c6b02e41 100644
--- a/ydb/core/kqp/opt/kqp_opt_build_txs.cpp
+++ b/ydb/core/kqp/opt/kqp_opt_build_txs.cpp
@@ -155,6 +155,7 @@ private:
     }
 
     static bool AreAllStagesKqpPure(const TVector<TDqPhyStage>& stages) {
+        // TODO: Avoid lambda analysis here, use sources/sinks for table interaction.
         return std::all_of(stages.begin(), stages.end(), [](const auto& x) { return IsKqpPureLambda(x.Program()); });
     }
 
diff --git a/ydb/core/kqp/opt/kqp_opt_impl.h b/ydb/core/kqp/opt/kqp_opt_impl.h
index 6db7f968e9a..da0bae03c80 100644
--- a/ydb/core/kqp/opt/kqp_opt_impl.h
+++ b/ydb/core/kqp/opt/kqp_opt_impl.h
@@ -20,12 +20,12 @@ static inline void DumpAppliedRule(const TString& name, const NYql::TExprNode::T
         builder << KqpExprToPrettyString(*input, ctx) << Endl;
         builder << "Expression after rule application: " << Endl;
         builder << KqpExprToPrettyString(*output, ctx);
-        YQL_CLOG(INFO, ProviderKqp) << builder;
+        YQL_CLOG(TRACE, ProviderKqp) << builder;
     }
 #else
     Y_UNUSED(ctx);
     if (input != output) {
-        YQL_CLOG(INFO, ProviderKqp) << name;
+        YQL_CLOG(TRACE, ProviderKqp) << name;
     }
 #endif
 }
diff --git a/ydb/core/kqp/opt/kqp_opt_phy_finalize.cpp b/ydb/core/kqp/opt/kqp_opt_phy_finalize.cpp
index cb04ddbee82..5c4e648c8e6 100644
--- a/ydb/core/kqp/opt/kqp_opt_phy_finalize.cpp
+++ b/ydb/core/kqp/opt/kqp_opt_phy_finalize.cpp
@@ -17,14 +17,25 @@ namespace {
 TStatus KqpBuildPureExprStagesResult(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx,
     const TKqpOptimizeContext& kqpCtx)
 {
+    output = input;
+
     TExprBase inputExpr(input);
     auto query = inputExpr.Cast<TKqlQuery>();
 
-    auto hasPrecomputes = [](TExprBase node) {
-        auto precompute = FindNode(node.Ptr(), [](const NYql::TExprNode::TPtr& node) {
-            return TMaybeNode<TDqPhyPrecompute>(node).IsValid();
-        });
+    if (query.Results().Empty()) {
+        return TStatus::Ok;
+    }
 
+    auto predicate = [](const NYql::TExprNode::TPtr& node) {
+        return TMaybeNode<TDqPhyPrecompute>(node).IsValid();
+    };
+
+    auto filter = [](const NYql::TExprNode::TPtr& node) {
+        return !TMaybeNode<TCoLambda>(node).IsValid();
+    };
+
+    auto hasPrecomputes = [&](TExprBase node) {
+        auto precompute = FindNode(node.Ptr(), filter, predicate);
         return precompute != nullptr;
     };
 
@@ -215,6 +226,8 @@ NYql::IGraphTransformer::TStatus PerformGlobalRule(const TString& ruleName, cons
 TAutoPtr<IGraphTransformer> CreateKqpFinalizingOptTransformer(const TIntrusivePtr<TKqpOptimizeContext>& kqpCtx) {
     return CreateFunctorTransformer(
         [kqpCtx](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) -> TStatus {
+            output = input;
+
             PERFORM_GLOBAL_RULE("ReplicateMultiUsedConnection", input, output, ctx,
                 [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
                     YQL_ENSURE(TKqlQuery::Match(input.Get()));
@@ -241,7 +254,7 @@ TAutoPtr<IGraphTransformer> CreateKqpFinalizingOptTransformer(const TIntrusivePt
                     return KqpDuplicateResults(input, output, ctx);
                 });
 
-            YQL_CLOG(INFO, ProviderKqp) << "FinalizingOptimized KQL query: " << KqpExprToPrettyString(*input, ctx);
+            YQL_CLOG(TRACE, ProviderKqp) << "FinalizingOptimized KQL query: " << KqpExprToPrettyString(*input, ctx);
 
             return TStatus::Ok;
         });
diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp
index 2dded70c9d3..432483b606b 100644
--- a/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp
+++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp
@@ -254,7 +254,7 @@ TVector<TExprBase> ConvertComparisonNode(const TExprBase& nodeIn, const TExprNod
             if (!IsSupportedCast(maybeSafeCast.Cast())) {
                 return NullNode;
             }
-            
+
             return node;
         }
 
@@ -612,13 +612,13 @@ TMaybeNode<TExprBase> SafeCastPredicatePushdown(const TCoFlatMap& flatmap,
 
     return ComparisonPushdown(parameters, predicate, ctx, pos, input);
 }
-	
+
 TMaybeNode<TExprBase> SimplePredicatePushdown(const TCoCompare& predicate, TExprContext& ctx, TPositionHandle pos,
     const TExprNode* lambdaArg, const TExprBase& input)
 {
     if (!IsSupportedPredicate(predicate)) {
         return NullNode;
-    }	
+    }
 
     auto parameters = ExtractComparisonParameters(predicate, lambdaArg, input);
 
@@ -629,7 +629,7 @@ TMaybeNode<TExprBase> SimplePredicatePushdown(const TCoCompare& predicate, TExpr
     return ComparisonPushdown(parameters, predicate, ctx, pos, input);
 }
 
-	
+
 TMaybeNode<TExprBase> CoalescePushdown(const TCoCoalesce& coalesce, TExprContext& ctx, TPositionHandle pos,
     const TExprNode* lambdaArg, const TExprBase& input)
 {
@@ -745,7 +745,7 @@ TExprBase KqpPushOlapFilter(TExprBase node, TExprContext& ctx, const TKqpOptimiz
     const auto& lambda = flatmap.Lambda();
     auto lambdaArg = lambda.Args().Arg(0).Raw();
 
-    YQL_CLOG(INFO, ProviderKqp) << "Initial OLAP lambda: " << KqpExprToPrettyString(lambda, ctx);
+    YQL_CLOG(TRACE, ProviderKqp) << "Initial OLAP lambda: " << KqpExprToPrettyString(lambda, ctx);
 
     auto maybeOptionalIf = lambda.Body().Maybe<TCoOptionalIf>();
 
@@ -780,7 +780,7 @@ TExprBase KqpPushOlapFilter(TExprBase node, TExprContext& ctx, const TKqpOptimiz
             .Build()
         .Done();
 
-    YQL_CLOG(INFO, ProviderKqp) << "Pushed OLAP lambda: " << KqpExprToPrettyString(newProcessLambda, ctx);
+    YQL_CLOG(TRACE, ProviderKqp) << "Pushed OLAP lambda: " << KqpExprToPrettyString(newProcessLambda, ctx);
 
 #ifdef ENABLE_COLUMNS_PRUNING
     TMaybeNode<TCoAtomList> readColumns = BuildColumnsFromLambda(lambda, ctx, node.Pos());
diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp
index 426ff16596f..138b615701f 100644
--- a/ydb/library/yql/providers/common/provider/yql_provider.cpp
+++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp
@@ -441,6 +441,15 @@ void TransformerStatsToYson(const TString& name, const IGraphTransformer::TStati
     writer.OnEndMap();
 }
 
+TString TransformerStatsToYson(const IGraphTransformer::TStatistics& stats, NYson::EYsonFormat format) {
+    TStringStream out;
+    NYson::TYsonWriter writer(&out, format);
+
+    TransformerStatsToYson("", stats, writer);
+
+    return out.Str();
+}
+
 bool FillUsedFilesImpl(
     const TExprNode& node,
     TUserDataTable& files,
diff --git a/ydb/library/yql/providers/common/provider/yql_provider.h b/ydb/library/yql/providers/common/provider/yql_provider.h
index a09f759a766..2c2a38a9b5d 100644
--- a/ydb/library/yql/providers/common/provider/yql_provider.h
+++ b/ydb/library/yql/providers/common/provider/yql_provider.h
@@ -100,6 +100,9 @@ TVector<TString> GetStructFields(const TTypeAnnotationNode* type);
 
 void TransformerStatsToYson(const TString& name, const IGraphTransformer::TStatistics& stats, NYson::TYsonWriter& writer);
 
+TString TransformerStatsToYson(const IGraphTransformer::TStatistics& stats, NYson::EYsonFormat format
+    = NYson::EYsonFormat::Pretty);
+
 void FillSecureParams(const TExprNode::TPtr& node, const TTypeAnnotationContext& types, THashMap<TString, TString>& secureParams);
 
 bool FillUsedFiles(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const TUserDataTable& crutches = {});