YQL-14803: hidden query, no publish

ref:66fd153755a9ba6d61be3290da5c70809ab60490

5 files changed, 39 insertions, 10 deletions

diff --git a/ydb/library/yql/core/facade/yql_facade.cpp b/ydb/library/yql/core/facade/yql_facade.cpp
index 035975464f5..6df51169cad 100644
--- a/ydb/library/yql/core/facade/yql_facade.cpp
+++ b/ydb/library/yql/core/facade/yql_facade.cpp
@@ -183,9 +183,10 @@ TProgramPtr TProgramFactory::Create(
 TProgramPtr TProgramFactory::Create(
         const TString& filename,
         const TString& sourceCode,
-        const TString& sessionId)
+        const TString& sessionId,
+        bool hidden)
 {
-    auto randomProvider = UseRepeatableRandomAndTimeProviders_ && !UseUnrepeatableRandom ?
+    auto randomProvider = UseRepeatableRandomAndTimeProviders_ && !UseUnrepeatableRandom && !hidden ?
         CreateDeterministicRandomProvider(1) : CreateDefaultRandomProvider();
     auto timeProvider = UseRepeatableRandomAndTimeProviders_ ?
         CreateDeterministicTimeProvider(10000000) : CreateDefaultTimeProvider();
@@ -198,7 +199,7 @@ TProgramPtr TProgramFactory::Create(
     // make UserDataTable_ copy here
     return new TProgram(FunctionRegistry_, randomProvider, timeProvider, NextUniqueId_, DataProvidersInit_,
         UserDataTable_, CredentialTables_, UserCredentials_, moduleResolver, udfResolver, udfIndex, udfIndexPackageSet, FileStorage_,
-        GatewaysConfig_, filename, sourceCode, sessionId, Runner_, EnableRangeComputeFor_);
+        GatewaysConfig_, filename, sourceCode, sessionId, Runner_, EnableRangeComputeFor_, hidden);
 }
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -223,7 +224,8 @@ TProgram::TProgram(
         const TString& sourceCode,
         const TString& sessionId,
         const TString& runner,
-        bool enableRangeComputeFor
+        bool enableRangeComputeFor,
+        bool hidden
     )
     : FunctionRegistry_(functionRegistry)
     , RandomProvider_(randomProvider)
@@ -249,6 +251,7 @@ TProgram::TProgram(
     , ResultFormat_(NYson::EYsonFormat::Binary)
     , OutputFormat_(NYson::EYsonFormat::Pretty)
     , EnableRangeComputeFor_(enableRangeComputeFor)
+    , Hidden_(hidden)
 {
     if (SessionId_.empty()) {
         SessionId_ = CreateGuidAsString();
@@ -764,7 +767,7 @@ TProgram::TFutureStatus TProgram::RunAsync(
     if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) {
         return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
     }
-    TypeCtx_->IsReadOnly = false;
+    TypeCtx_->IsReadOnly = Hidden_;
 
     for (const auto& dp : DataProviders_) {
         if (!dp.RemoteClusterProvider || !dp.RemoteRun) {
@@ -838,7 +841,7 @@ TProgram::TFutureStatus TProgram::RunAsyncWithConfig(
     if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) {
         return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
     }
-    TypeCtx_->IsReadOnly = false;
+    TypeCtx_->IsReadOnly = Hidden_;
 
     for (const auto& dp : DataProviders_) {
         if (!dp.RemoteClusterProvider || !dp.RemoteRun) {
@@ -996,6 +999,7 @@ TFuture<IGraphTransformer::TStatus> TProgram::AsyncTransformWithFallback(bool ap
             FallbackCounter ++;
             // don't execute recapture again
             ExprCtx_->Step.Done(TExprStep::Recapture);
+            BeforeFallback();
             return AsyncTransformWithFallback(false);
         }
         return res;
@@ -1124,7 +1128,7 @@ TMaybe<TString> TProgram::GetTasksInfo() {
     }
 }
 
-TMaybe<TString> TProgram::GetStatistics(bool totalOnly) {
+TMaybe<TString> TProgram::GetStatistics(bool totalOnly, THashMap<TString, TStringBuf> extraYsons) {
     if (!TypeCtx_) {
         return Nothing();
     }
@@ -1174,6 +1178,13 @@ TMaybe<TString> TProgram::GetStatistics(bool totalOnly) {
 
     writer.OnEndMap(); // system
 
+    // extra
+    for (const auto &[k, extraYson] : extraYsons) {
+        writer.OnKeyedItem(k);
+        writer.OnRaw(extraYson);
+        hasStatistics = true;
+    }
+
     // Footer
     writer.OnEndMap();
     writer.OnEndMap();
@@ -1273,6 +1284,7 @@ TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& us
     if (DiagnosticFormat_) {
         typeAnnotationContext->Diagnostics = true;
     }
+    typeAnnotationContext->Hidden = Hidden_;
 
     if (UdfIndex_ && UdfIndexPackageSet_) {
         // setup default versions at the beginning
@@ -1295,6 +1307,9 @@ TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& us
             ProgressWriter_,
             OperationOptions_
         );
+        if (Hidden_ && !dp.SupportsHidden) {
+            continue;
+        }
 
         providerNames.insert(dp.Names.begin(), dp.Names.end());
         DataProviders_.emplace_back(dp);
diff --git a/ydb/library/yql/core/facade/yql_facade.h b/ydb/library/yql/core/facade/yql_facade.h
index 4ba75c5ba78..1fd21181b7e 100644
--- a/ydb/library/yql/core/facade/yql_facade.h
+++ b/ydb/library/yql/core/facade/yql_facade.h
@@ -61,7 +61,8 @@ public:
     TProgramPtr Create(
             const TString& filename,
             const TString& sourceCode,
-            const TString& sessionId = TString());
+            const TString& sessionId = TString(),
+            bool hidden = false);
 
     void UnrepeatableRandom();
 private:
@@ -218,7 +219,7 @@ public:
 
     TMaybe<TString> GetTasksInfo();
 
-    TMaybe<TString> GetStatistics(bool totalOnly = false);
+    TMaybe<TString> GetStatistics(bool totalOnly = false, THashMap<TString, TStringBuf> extraYsons = {});
 
     TMaybe<TString> GetDiscoveredData();
 
@@ -293,6 +294,10 @@ public:
 
     IPlanBuilder& GetPlanBuilder();
 
+    void SetBeforeFallback(std::function<void()>&& func) {
+        BeforeFallback = std::move(func);
+    }
+
 private:
     TProgram(
         const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
@@ -313,7 +318,8 @@ private:
         const TString& sourceCode,
         const TString& sessionId,
         const TString& runner,
-        bool enableRangeComputeFor);
+        bool enableRangeComputeFor,
+        bool hidden);
 
     TTypeAnnotationContextPtr BuildTypeAnnotationContext(const TString& username);
     TTypeAnnotationContextPtr GetAnnotationContext() const;
@@ -387,6 +393,8 @@ private:
     TString ExtractedQueryParametersMetadataYson_;
     const bool EnableRangeComputeFor_;
     i64 FallbackCounter = 0;
+    std::function<void()> BeforeFallback = [](){};
+    bool Hidden_ = false;
 };
 
 } // namspace NYql
diff --git a/ydb/library/yql/core/yql_data_provider.h b/ydb/library/yql/core/yql_data_provider.h
index f8ecac31f8a..ad2667e3e19 100644
--- a/ydb/library/yql/core/yql_data_provider.h
+++ b/ydb/library/yql/core/yql_data_provider.h
@@ -194,6 +194,7 @@ struct TDataProviderInfo {
     TIntrusivePtr<IDataProvider> Sink;
     bool SupportFullResultDataSink = false;
     bool WaitForActiveProcesses = true;
+    bool SupportsHidden = false;
 
     std::function<TMaybe<TString>(const TMaybe<TSet<TString>>& usedClusters, const TMaybe<TSet<TString>>& usedProviders,
         ESourceSyntax syntax)> RemoteClusterProvider;
diff --git a/ydb/library/yql/core/yql_type_annotation.h b/ydb/library/yql/core/yql_type_annotation.h
index 6333bd75992..003b6bd1258 100644
--- a/ydb/library/yql/core/yql_type_annotation.h
+++ b/ydb/library/yql/core/yql_type_annotation.h
@@ -235,6 +235,7 @@ struct TTypeAnnotationContext: public TThrRefBase {
     std::tuple<std::optional<ui64>, std::optional<double>, std::optional<TGUID>> CachedRandom;
 
     std::optional<bool> InitializeResult;
+    bool Hidden = false;
 
     template <typename T>
     T GetRandom() const noexcept;
diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto
index 89f50b15f56..7f8959bd9af 100644
--- a/ydb/library/yql/providers/common/proto/gateways_config.proto
+++ b/ydb/library/yql/providers/common/proto/gateways_config.proto
@@ -410,6 +410,10 @@ message TDqGatewayConfig {
     repeated string DefaultAnalyzeQueryForUsers = 4;
 
     repeated TAttr DefaultSettings = 102;
+
+    optional uint32 WithHiddenPercentage = 5 [default = 0];
+    repeated TDefaultAutoByHourPercentage WithHiddenByHour = 6;
+    repeated string NoWithHiddenForUsers = 7;
 }
 
 /////////////////////////////// Yql Core ///////////////////////////////