Save Secret Names to Query Plan and get Secret Values in runtime

1. В прото описании read стейджа (ExternalSource) помещается structured token (json), в котором вместо секрета находится его название
2. Также на верхнем уровне графа (PhyTx) передается весь список используемых секретов
3. Эти секреты будут перед запуском ресольвиться в значения (TBD)
4. Перед созданием тасок ссылки на секреты заменяются на полученные значения

18 files changed, 179 insertions, 12 deletions

diff --git a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp
index 0e023ec038..08c4b6c501 100644
--- a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp
+++ b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp
@@ -1678,7 +1678,7 @@ private:
                             }
                             break;
                         case NKqpProto::TKqpSource::kExternalSource:
-                            BuildReadTasksFromSource(stageInfo);
+                            BuildReadTasksFromSource(stageInfo, {});
                             break;
                         default:
                             YQL_ENSURE(false, "unknown source type");
diff --git a/ydb/core/kqp/executer_actor/kqp_executer_impl.h b/ydb/core/kqp/executer_actor/kqp_executer_impl.h
index 7fa3567aeb..31092da6b8 100644
--- a/ydb/core/kqp/executer_actor/kqp_executer_impl.h
+++ b/ydb/core/kqp/executer_actor/kqp_executer_impl.h
@@ -31,6 +31,7 @@
 #include <ydb/library/yql/dq/proto/dq_tasks.pb.h>
 #include <ydb/library/yql/dq/runtime/dq_transport.h>
 #include <ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h>
+#include <ydb/library/yql/providers/common/structured_token/yql_token_builder.h>
 #include <ydb/library/yql/public/issue/yql_issue.h>
 #include <ydb/library/yql/public/issue/yql_issue_message.h>
 
@@ -704,7 +705,7 @@ protected:
         }
     }
 
-    void BuildReadTasksFromSource(TStageInfo& stageInfo) {
+    void BuildReadTasksFromSource(TStageInfo& stageInfo, TMap<TString, TString> secureParams) {
         const auto& stage = stageInfo.Meta.GetStage(stageInfo.Id);
 
         YQL_ENSURE(stage.GetSources(0).HasExternalSource());
@@ -720,7 +721,14 @@ protected:
             input.SourceSettings = externalSource.GetSettings();
             input.SourceType = externalSource.GetType();
 
-            task.Meta.DqTaskParams.emplace(externalSource.GetTaskParamKey(), partitionParam);
+            task.Meta.TaskParams.emplace(externalSource.GetTaskParamKey(), partitionParam);
+
+            auto sourceName = externalSource.GetSourceName();
+            if (sourceName) {
+                auto structuredToken = NYql::CreateStructuredTokenParser(externalSource.GetAuthInfo()).ToBuilder().ReplaceReferences(secureParams).ToJson();
+                task.Meta.SecureParams.emplace(sourceName, structuredToken);
+            }
+
             task.Meta.Type = TTaskMeta::TTaskType::Compute;
 
         }
diff --git a/ydb/core/kqp/executer_actor/kqp_tasks_graph.cpp b/ydb/core/kqp/executer_actor/kqp_tasks_graph.cpp
index 8ce4530e9d..e0ba573bbf 100644
--- a/ydb/core/kqp/executer_actor/kqp_tasks_graph.cpp
+++ b/ydb/core/kqp/executer_actor/kqp_tasks_graph.cpp
@@ -1035,11 +1035,11 @@ void SerializeTaskToProto(const TKqpTasksGraph& tasksGraph, const TTask& task, N
         result->SetMetaId(task.GetMetaIdUnsafe());
     }
 
-    for (const auto& [paramName, paramValue] : task.Meta.DqTaskParams) {
+    for (const auto& [paramName, paramValue] : task.Meta.TaskParams) {
         (*result->MutableTaskParams())[paramName] = paramValue;
     }
 
-    for (const auto& [paramName, paramValue] : task.Meta.DqSecureParams) {
+    for (const auto& [paramName, paramValue] : task.Meta.SecureParams) {
         (*result->MutableSecureParams())[paramName] = paramValue;
     }
 
diff --git a/ydb/core/kqp/executer_actor/kqp_tasks_graph.h b/ydb/core/kqp/executer_actor/kqp_tasks_graph.h
index fbde955300..768a1a7858 100644
--- a/ydb/core/kqp/executer_actor/kqp_tasks_graph.h
+++ b/ydb/core/kqp/executer_actor/kqp_tasks_graph.h
@@ -161,8 +161,8 @@ public:
     TActorId ExecuterId;
     ui32 Type = Unknown;
 
-    THashMap<TString, TString> DqTaskParams; // Params for sources/sinks
-    THashMap<TString, TString> DqSecureParams;
+    THashMap<TString, TString> TaskParams; // Params for sources/sinks
+    THashMap<TString, TString> SecureParams;
 
     enum TTaskType : ui32 {
         Unknown = 0,
diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp
index 617c24a2c7..b92c96f7a9 100644
--- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp
@@ -197,10 +197,19 @@ public:
             LoadResults.clear();
             return false;
         }
-        it->second->AddCluster(metadata.ExternalSource.DataSourcePath, {{
+
+        THashMap<TString, TString> properties = {{
             {"location", metadata.ExternalSource.DataSourceLocation },
             {"installation", metadata.ExternalSource.DataSourceInstallation }
-            }});
+        }};
+
+        if (metadata.ExternalSource.DataSourceAuth.identity_case() == NKikimrSchemeOp::TAuth::kServiceAccount) {
+            properties["serviceAccountId"] = metadata.ExternalSource.DataSourceAuth.GetServiceAccount().GetId();
+            properties["serviceAccountIdSignatureReference"] = metadata.ExternalSource.DataSourceAuth.GetServiceAccount().GetSecretName();
+        }
+
+        it->second->AddCluster(metadata.ExternalSource.DataSourcePath, properties);
+
         return true;
     }
 
diff --git a/ydb/core/kqp/query_compiler/kqp_query_compiler.cpp b/ydb/core/kqp/query_compiler/kqp_query_compiler.cpp
index d4104c7f02..a1d02627d4 100644
--- a/ydb/core/kqp/query_compiler/kqp_query_compiler.cpp
+++ b/ydb/core/kqp/query_compiler/kqp_query_compiler.cpp
@@ -17,6 +17,7 @@
 #include <ydb/library/yql/minikql/mkql_node_serialization.h>
 #include <ydb/library/yql/providers/common/mkql/yql_type_mkql.h>
 #include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
+#include <ydb/library/yql/providers/common/structured_token/yql_token_builder.h>
 #include <ydb/library/yql/providers/dq/common/yql_dq_settings.h>
 
 namespace NKikimr {
@@ -834,6 +835,10 @@ private:
 
             FillTable(*tableMeta, std::move(tableColumns), *txProto.AddTables());
         }
+
+        for (const auto& secretName : SecretNames) {
+            txProto.AddSecretNames(secretName);
+        }
     }
 
     void FillKqpSource(const TDqSource& source, NKqpProto::TKqpSource* protoSource, bool allowSystemColumns,
@@ -930,6 +935,17 @@ private:
             for (const TString& partitionParam : partitionParams) {
                 externalSource.AddPartitionedTaskParams(partitionParam);
             }
+
+            THashMap<TString, TString> secureParams;
+            NYql::NCommon::FillSecureParams(source.Ptr(), TypesCtx, secureParams);
+            if (!secureParams.empty()) {
+                YQL_ENSURE(secureParams.size() == 1, "Only one SecureParams per source allowed");
+                auto it = secureParams.begin();
+                externalSource.SetSourceName(it->first);
+                auto token = it->second;
+                externalSource.SetAuthInfo(token);
+                CreateStructuredTokenParser(token).ListReferences(SecretNames);
+            }
         }
     }
 
@@ -1101,6 +1117,7 @@ private:
     TKqlCompileContext KqlCtx;
     TIntrusivePtr<NCommon::IMkqlCallableCompiler> KqlCompiler;
     TTypeAnnotationContext& TypesCtx;
+    TSet<TString> SecretNames;
 };
 
 } // namespace
diff --git a/ydb/core/protos/kqp_physical.proto b/ydb/core/protos/kqp_physical.proto
index 46730dda52..5d8f86f453 100644
--- a/ydb/core/protos/kqp_physical.proto
+++ b/ydb/core/protos/kqp_physical.proto
@@ -307,6 +307,9 @@ message TKqpExternalSource {
     // Partitioning
     string TaskParamKey = 3;
     repeated string PartitionedTaskParams = 4;
+
+    string SourceName = 5;
+    string AuthInfo = 6;
 }
 
 message TKqpSource {
@@ -363,6 +366,7 @@ message TKqpPhyTx {
     bool HasEffects = 6; // at least one stage has flag TKqpPhyStage::IsEffectStage set
     repeated TKqpPhyTable Tables = 7;
     TKqpSchemeOperation SchemeOperation = 8;
+    repeated string SecretNames = 9;
 }
 
 message TKqpTableInfo {
diff --git a/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.darwin-x86_64.txt
index fc4d1508d0..010ecdd3b2 100644
--- a/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.darwin-x86_64.txt
@@ -25,6 +25,7 @@ target_link_options(yql-providers-common-structured_token-ut PRIVATE
 )
 target_sources(yql-providers-common-structured_token-ut PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/structured_token/ut/yql_structured_token_ut.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/structured_token/ut/yql_token_builder_ut.cpp
 )
 set_property(
   TARGET
diff --git a/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.linux-aarch64.txt
index d0de17a01c..55b58634b3 100644
--- a/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.linux-aarch64.txt
@@ -30,6 +30,7 @@ target_link_options(yql-providers-common-structured_token-ut PRIVATE
 )
 target_sources(yql-providers-common-structured_token-ut PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/structured_token/ut/yql_structured_token_ut.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/structured_token/ut/yql_token_builder_ut.cpp
 )
 set_property(
   TARGET
diff --git a/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.linux-x86_64.txt
index 972ef1ff0c..7072a49530 100644
--- a/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.linux-x86_64.txt
@@ -31,6 +31,7 @@ target_link_options(yql-providers-common-structured_token-ut PRIVATE
 )
 target_sources(yql-providers-common-structured_token-ut PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/structured_token/ut/yql_structured_token_ut.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/structured_token/ut/yql_token_builder_ut.cpp
 )
 set_property(
   TARGET
diff --git a/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.windows-x86_64.txt
index 35955e06c3..37018fc8d9 100644
--- a/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/providers/common/structured_token/ut/CMakeLists.windows-x86_64.txt
@@ -20,6 +20,7 @@ target_link_libraries(yql-providers-common-structured_token-ut PUBLIC
 )
 target_sources(yql-providers-common-structured_token-ut PRIVATE
   ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/structured_token/ut/yql_structured_token_ut.cpp
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/common/structured_token/ut/yql_token_builder_ut.cpp
 )
 set_property(
   TARGET
diff --git a/ydb/library/yql/providers/common/structured_token/ut/ya.make b/ydb/library/yql/providers/common/structured_token/ut/ya.make
index 90f62995d5..787199c2d6 100644
--- a/ydb/library/yql/providers/common/structured_token/ut/ya.make
+++ b/ydb/library/yql/providers/common/structured_token/ut/ya.make
@@ -2,6 +2,7 @@ UNITTEST_FOR(ydb/library/yql/providers/common/structured_token)
 
 SRCS(
     yql_structured_token_ut.cpp
+    yql_token_builder_ut.cpp
 )
 
 END()
diff --git a/ydb/library/yql/providers/common/structured_token/ut/yql_token_builder_ut.cpp b/ydb/library/yql/providers/common/structured_token/ut/yql_token_builder_ut.cpp
index 3675014d48..b7d4e8d501 100644
--- a/ydb/library/yql/providers/common/structured_token/ut/yql_token_builder_ut.cpp
+++ b/ydb/library/yql/providers/common/structured_token/ut/yql_token_builder_ut.cpp
@@ -30,6 +30,28 @@ Y_UNIT_TEST_SUITE(TokenBuilderTest) {
         UNIT_ASSERT_VALUES_EQUAL(sign, "my_sa_sign");
     }
 
+    Y_UNIT_TEST(ServiceAccountIdWithSecret) {
+        TStructuredTokenBuilder b;
+        b.SetServiceAccountIdAuthWithSecret("my_sa_id", "my_sa_sign_reference");
+        UNIT_ASSERT_VALUES_EQUAL(R"({"sa_id":"my_sa_id","sa_id_signature_ref":"my_sa_sign_reference"})", b.ToJson());
+        const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson());
+        UNIT_ASSERT(p.HasServiceAccountIdAuth());
+        UNIT_ASSERT(!p.HasBasicAuth());
+        UNIT_ASSERT(!p.HasIAMToken());
+        UNIT_ASSERT(!p.IsNoAuth());
+        TString id, sign, reference;
+        UNIT_ASSERT(p.GetServiceAccountIdAuth(id, sign, reference));
+        UNIT_ASSERT_VALUES_EQUAL(id, "my_sa_id");
+        UNIT_ASSERT_VALUES_EQUAL(sign, "");
+        UNIT_ASSERT_VALUES_EQUAL(reference, "my_sa_sign_reference");
+        TSet<TString> references;
+        p.ListReferences(references);
+        UNIT_ASSERT_VALUES_EQUAL(references.size(), 1);
+        UNIT_ASSERT(references.contains("my_sa_sign_reference"));
+        b.ReplaceReferences({{"my_sa_sign_reference", "my_sa_sign_value"}});
+        UNIT_ASSERT_VALUES_EQUAL(R"({"sa_id":"my_sa_id","sa_id_signature":"my_sa_sign_value"})", b.ToJson());
+    }
+
     Y_UNIT_TEST(BasicAuth) {
         TStructuredTokenBuilder b;
         b.SetBasicAuth("my_login", "my_passw");
@@ -45,6 +67,28 @@ Y_UNIT_TEST_SUITE(TokenBuilderTest) {
         UNIT_ASSERT_VALUES_EQUAL(password, "my_passw");
     }
 
+    Y_UNIT_TEST(BasicAuthWithSecret) {
+        TStructuredTokenBuilder b;
+        b.SetBasicAuthWithSecret("my_login", "my_passw_reference");
+        UNIT_ASSERT_VALUES_EQUAL(R"({"basic_login":"my_login","basic_password_ref":"my_passw_reference"})", b.ToJson());
+        const TStructuredTokenParser p = CreateStructuredTokenParser(b.ToJson());
+        UNIT_ASSERT(!p.HasServiceAccountIdAuth());
+        UNIT_ASSERT(p.HasBasicAuth());
+        UNIT_ASSERT(!p.HasIAMToken());
+        UNIT_ASSERT(!p.IsNoAuth());
+        TString login, password, reference;
+        UNIT_ASSERT(p.GetBasicAuth(login, password, reference));
+        UNIT_ASSERT_VALUES_EQUAL(login, "my_login");
+        UNIT_ASSERT_VALUES_EQUAL(password, "");
+        UNIT_ASSERT_VALUES_EQUAL(reference, "my_passw_reference");
+        TSet<TString> references;
+        p.ListReferences(references);
+        UNIT_ASSERT_VALUES_EQUAL(references.size(), 1);
+        UNIT_ASSERT(references.contains("my_passw_reference"));
+        b.ReplaceReferences({{"my_passw_reference", "my_passw_value"}});
+        UNIT_ASSERT_VALUES_EQUAL(R"({"basic_login":"my_login","basic_password":"my_passw_value"})", b.ToJson());
+    }
+
     Y_UNIT_TEST(IAMToken) {
         TStructuredTokenBuilder b;
         b.SetIAMToken("my_token");
diff --git a/ydb/library/yql/providers/common/structured_token/yql_structured_token.cpp b/ydb/library/yql/providers/common/structured_token/yql_structured_token.cpp
index 84a2f06b3c..31a527763b 100644
--- a/ydb/library/yql/providers/common/structured_token/yql_structured_token.cpp
+++ b/ydb/library/yql/providers/common/structured_token/yql_structured_token.cpp
@@ -36,6 +36,11 @@ TStructuredToken& TStructuredToken::SetField(const TString& name, const TString&
     return *this;
 }
 
+TStructuredToken& TStructuredToken::ClearField(const TString& name) {
+    Data.erase(name);
+    return *this;
+}
+
 TString TStructuredToken::ToJson() const {
     TStringStream output;
     // set "format output" to false, no need for extra indents
diff --git a/ydb/library/yql/providers/common/structured_token/yql_structured_token.h b/ydb/library/yql/providers/common/structured_token/yql_structured_token.h
index 6ee1f478b7..2789723663 100644
--- a/ydb/library/yql/providers/common/structured_token/yql_structured_token.h
+++ b/ydb/library/yql/providers/common/structured_token/yql_structured_token.h
@@ -17,6 +17,7 @@ public:
     TMaybe<TString> FindField(const TString& name) const;
     bool HasField(const TString& name) const;
     TStructuredToken& SetField(const TString& name, const TString& value);
+    TStructuredToken& ClearField(const TString& name);
     TString ToJson() const;
 
 private:
diff --git a/ydb/library/yql/providers/common/structured_token/yql_token_builder.cpp b/ydb/library/yql/providers/common/structured_token/yql_token_builder.cpp
index 3bace92c54..65437f34fc 100644
--- a/ydb/library/yql/providers/common/structured_token/yql_token_builder.cpp
+++ b/ydb/library/yql/providers/common/structured_token/yql_token_builder.cpp
@@ -16,12 +16,24 @@ TStructuredTokenBuilder& TStructuredTokenBuilder::SetServiceAccountIdAuth(const
     return *this;
 }
 
+TStructuredTokenBuilder& TStructuredTokenBuilder::SetServiceAccountIdAuthWithSecret(const TString& accountId, const TString& accountIdSignatureReference) {
+    Data.SetField("sa_id", accountId);
+    Data.SetField("sa_id_signature_ref", accountIdSignatureReference);
+    return *this;
+}
+
 TStructuredTokenBuilder& TStructuredTokenBuilder::SetBasicAuth(const TString& login, const TString& password) {
     Data.SetField("basic_login", login);
     Data.SetField("basic_password", password);
     return *this;
 }
 
+TStructuredTokenBuilder& TStructuredTokenBuilder::SetBasicAuthWithSecret(const TString& login, const TString& passwordReference) {
+    Data.SetField("basic_login", login);
+    Data.SetField("basic_password_ref", passwordReference);
+    return *this;
+}
+
 TStructuredTokenBuilder& TStructuredTokenBuilder::SetIAMToken(const TString& token) {
     Data.SetField("token", token);
     return *this;
@@ -32,6 +44,20 @@ TStructuredTokenBuilder& TStructuredTokenBuilder::SetNoAuth() {
     return *this;
 }
 
+TStructuredTokenBuilder& TStructuredTokenBuilder::ReplaceReferences(const TMap<TString, TString> secrets) {
+    if (Data.HasField("basic_password_ref")) {
+        auto reference = Data.GetField("basic_password_ref");
+        Data.ClearField("basic_password_ref");
+        Data.SetField("basic_password", secrets.at(reference));
+    }
+    if (Data.HasField("sa_id_signature_ref")) {
+        auto reference = Data.GetField("sa_id_signature_ref");
+        Data.ClearField("sa_id_signature_ref");
+        Data.SetField("sa_id_signature", secrets.at(reference));
+    }
+    return *this;
+}
+
 TString TStructuredTokenBuilder::ToJson() const {
     return Data.ToJson();
 }
@@ -48,8 +74,14 @@ bool TStructuredTokenParser::HasServiceAccountIdAuth() const {
 }
 
 bool TStructuredTokenParser::GetServiceAccountIdAuth(TString& accountId, TString& accountIdSignature) const  {
+    TString accountIdSignatureReference;
+    return GetServiceAccountIdAuth(accountId, accountIdSignature, accountIdSignatureReference);
+}
+
+bool TStructuredTokenParser::GetServiceAccountIdAuth(TString& accountId, TString& accountIdSignature, TString& accountIdSignatureReference) const {
     accountId = Data.GetField("sa_id");
-    accountIdSignature = Data.GetField("sa_id_signature");
+    accountIdSignature = Data.GetFieldOrDefault("sa_id_signature", "");
+    accountIdSignatureReference = Data.GetFieldOrDefault("sa_id_signature_ref", "");
     return true;
 }
 
@@ -58,8 +90,14 @@ bool TStructuredTokenParser::HasBasicAuth() const {
 }
 
 bool TStructuredTokenParser::GetBasicAuth(TString& login, TString& password) const {
+    TString passwordReference;
+    return GetBasicAuth(login, password, passwordReference);
+}
+
+bool TStructuredTokenParser::GetBasicAuth(TString& login, TString& password, TString& passwordReference) const {
     login = Data.GetField("basic_login");
-    password = Data.GetField("basic_password");
+    password = Data.GetFieldOrDefault("basic_password", "");
+    passwordReference = Data.GetFieldOrDefault("basic_password_ref", "");
     return true;
 }
 
@@ -75,6 +113,15 @@ bool TStructuredTokenParser::IsNoAuth() const {
     return Data.HasField("no_auth");
 }
 
+void TStructuredTokenParser::ListReferences(TSet<TString>& references) const {
+    if (Data.HasField("basic_password_ref")) {
+        references.insert(Data.GetField("basic_password_ref"));
+    }
+    if (Data.HasField("sa_id_signature_ref")) {
+        references.insert(Data.GetField("sa_id_signature_ref"));
+    }
+}
+
 TStructuredTokenBuilder TStructuredTokenParser::ToBuilder() const {
     return TStructuredTokenBuilder(Data);
 }
@@ -85,6 +132,7 @@ TStructuredTokenParser CreateStructuredTokenParser(const TString& content = {})
 
 TString ComposeStructuredTokenJsonForServiceAccount(const TString& serviceAccountId, const TString& serviceAccountIdSignature, const TString& token) {
     TStructuredTokenBuilder result;
+    
     if (serviceAccountId && serviceAccountIdSignature) {
         result.SetServiceAccountIdAuth(serviceAccountId, serviceAccountIdSignature);
         return result.ToJson();
@@ -99,4 +147,16 @@ TString ComposeStructuredTokenJsonForServiceAccount(const TString& serviceAccoun
     return result.ToJson();
 }
 
+TString ComposeStructuredTokenJsonForServiceAccountWithSecret(const TString& serviceAccountId, const TString& serviceAccountIdSignatureSecretName) {
+    TStructuredTokenBuilder result;
+    
+    if (serviceAccountId && serviceAccountIdSignatureSecretName) {
+        result.SetServiceAccountIdAuthWithSecret(serviceAccountId, serviceAccountIdSignatureSecretName);
+        return result.ToJson();
+    }
+
+    result.SetNoAuth();
+    return result.ToJson();
+}
+
 }
diff --git a/ydb/library/yql/providers/common/structured_token/yql_token_builder.h b/ydb/library/yql/providers/common/structured_token/yql_token_builder.h
index aec2659446..5859bb8ece 100644
--- a/ydb/library/yql/providers/common/structured_token/yql_token_builder.h
+++ b/ydb/library/yql/providers/common/structured_token/yql_token_builder.h
@@ -2,6 +2,8 @@
 
 #include <ydb/library/yql/providers/common/structured_token/yql_structured_token.h>
 
+#include <util/generic/set.h>
+
 namespace NYql {
 
 class TStructuredTokenBuilder {
@@ -11,9 +13,12 @@ public:
     TStructuredTokenBuilder(TStructuredTokenBuilder&&) = default;
 
     TStructuredTokenBuilder& SetServiceAccountIdAuth(const TString& accountId, const TString& accountIdSignature);
+    TStructuredTokenBuilder& SetServiceAccountIdAuthWithSecret(const TString& accountId, const TString& accountIdSignatureReference);
     TStructuredTokenBuilder& SetBasicAuth(const TString& login, const TString& password);
+    TStructuredTokenBuilder& SetBasicAuthWithSecret(const TString& login, const TString& passwordReference);
     TStructuredTokenBuilder& SetIAMToken(const TString& token);
     TStructuredTokenBuilder& SetNoAuth();
+    TStructuredTokenBuilder& ReplaceReferences(const TMap<TString, TString> secrets);
 
     TString ToJson() const;
 
@@ -26,11 +31,14 @@ public:
     explicit TStructuredTokenParser(TStructuredToken&& data);
     bool HasServiceAccountIdAuth() const;
     bool GetServiceAccountIdAuth(TString& accountId, TString& accountIdSignature) const;
+    bool GetServiceAccountIdAuth(TString& accountId, TString& accountIdSignature, TString& accountIdSignatureReference) const;
     bool HasBasicAuth() const;
     bool GetBasicAuth(TString& login, TString& password) const;
+    bool GetBasicAuth(TString& login, TString& password, TString& passwordReference) const;
     bool HasIAMToken() const;
     TString GetIAMToken() const;
     bool IsNoAuth() const;
+    void ListReferences(TSet<TString>& references) const;
 
     TStructuredTokenBuilder ToBuilder() const;
 
@@ -40,4 +48,5 @@ private:
 
 TStructuredTokenParser CreateStructuredTokenParser(const TString& content);
 TString ComposeStructuredTokenJsonForServiceAccount(const TString& serviceAccountId, const TString& serviceAccountIdSignature, const TString& token);
+TString ComposeStructuredTokenJsonForServiceAccountWithSecret(const TString& serviceAccountId, const TString& serviceAccountIdSignatureSecretName);
 }
diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp
index 7d5672179b..561af786b5 100644
--- a/ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp
+++ b/ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp
@@ -32,7 +32,12 @@ public:
     void AddCluster(const TString& name, const THashMap<TString, TString>& properties) override {
         auto& settings = State_->Configuration->Clusters[name];
         settings.Url = properties.Value("location", "");
-        State_->Configuration->Tokens[name] = ComposeStructuredTokenJsonForServiceAccount(properties.Value("serviceAccountId", ""), properties.Value("serviceAccountIdSignature", ""), properties.Value("authToken", ""));
+        auto signReference = properties.Value("serviceAccountIdSignatureReference", "");
+        if (signReference) {
+            State_->Configuration->Tokens[name] = ComposeStructuredTokenJsonForServiceAccountWithSecret(properties.Value("serviceAccountId", ""), signReference);
+        } else {
+            State_->Configuration->Tokens[name] = ComposeStructuredTokenJsonForServiceAccount(properties.Value("serviceAccountId", ""), properties.Value("serviceAccountIdSignature", ""), properties.Value("authToken", ""));
+        }
     }
 
     TStringBuf GetName() const override {