new auth types have been added

8 files changed, 155 insertions, 8 deletions

diff --git a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp
index 7a762c7036f..5079f336e6c 100644
--- a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp
+++ b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp
@@ -2231,6 +2231,10 @@ private:
 
         if (settings.AuthMethod == "NONE") {
             externaDataSourceDesc.MutableAuth()->MutableNone();
+        } else if (settings.AuthMethod == "SERVICE_ACCOUNT") {
+            auto& sa = *externaDataSourceDesc.MutableAuth()->MutableServiceAccount();
+            sa.SetId(settings.ServiceAccount.Id);
+            sa.SetSecretName(settings.ServiceAccount.SecretName);
         }
     }
 
diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp
index b27072bc9d3..3ac91473636 100644
--- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp
@@ -181,6 +181,8 @@ namespace {
         out.ExternalDataSource = settings.GetObjectId();
         out.SourceType = GetOrDefault(settings, "source_type");
         out.AuthMethod = GetOrDefault(settings, "auth_method");
+        out.ServiceAccount.Id = GetOrDefault(settings, "service_account_id");
+        out.ServiceAccount.SecretName = GetOrDefault(settings, "service_account_secret_name");
         out.Installation = GetOrDefault(settings, "installation");
         out.Location = GetOrDefault(settings, "location");
         return out;
diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h
index e700162e66c..8c977865f59 100644
--- a/ydb/core/kqp/provider/yql_kikimr_gateway.h
+++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h
@@ -587,11 +587,17 @@ struct TDropExternalTableSettings {
 };
 
 struct TCreateExternalDataSourceSettings {
+    struct TServiceAccount {
+        TString Id;
+        TString SecretName;
+    };
+
     TString ExternalDataSource;
     TString SourceType;
     TString Location;
     TString Installation;
     TString AuthMethod;
+    TServiceAccount ServiceAccount;
 };
 
 struct TAlterExternalDataSourceSettings {
diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp
index bfd71fecfa5..2ad5a6df1fb 100644
--- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp
+++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp
@@ -4303,6 +4303,37 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
         UNIT_ASSERT(externalDataSource.ExternalDataSourceInfo->Description.GetAuth().HasNone());
     }
 
+    Y_UNIT_TEST(CreateExternalDataSourceWithSa) {
+        TKikimrRunner kikimr;
+        kikimr.GetTestServer().GetRuntime()->GetAppData(0).FeatureFlags.SetEnableExternalDataSources(true);
+        auto db = kikimr.GetTableClient();
+        auto session = db.CreateSession().GetValueSync().GetSession();
+        TString externalDataSourceName = "/Root/ExternalDataSource";
+        auto query = TStringBuilder() << R"(
+            CREATE EXTERNAL DATA SOURCE `)" << externalDataSourceName << R"(` WITH (
+                SOURCE_TYPE="ObjectStorage",
+                LOCATION="my-bucket",
+                AUTH_METHOD="SERVICE_ACCOUNT",
+                SERVICE_ACCOUNT_ID="mysa",
+                SERVICE_ACCOUNT_SECRET_NAME="mysasignature"
+            );)";
+        auto result = session.ExecuteSchemeQuery(query).GetValueSync();
+        UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
+
+        auto& runtime = *kikimr.GetTestServer().GetRuntime();
+        auto externalDataSourceDesc = Navigate(runtime, runtime.AllocateEdgeActor(), externalDataSourceName, NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown);
+        const auto& externalDataSource = externalDataSourceDesc->ResultSet.at(0);
+        UNIT_ASSERT_EQUAL(externalDataSource.Kind, NSchemeCache::TSchemeCacheNavigate::EKind::KindExternalDataSource);
+        UNIT_ASSERT(externalDataSource.ExternalDataSourceInfo);
+        UNIT_ASSERT_VALUES_EQUAL(externalDataSource.ExternalDataSourceInfo->Description.GetSourceType(), "ObjectStorage");
+        UNIT_ASSERT_VALUES_EQUAL(externalDataSource.ExternalDataSourceInfo->Description.GetInstallation(), "");
+        UNIT_ASSERT_VALUES_EQUAL(externalDataSource.ExternalDataSourceInfo->Description.GetLocation(), "my-bucket");
+        UNIT_ASSERT_VALUES_EQUAL(externalDataSource.ExternalDataSourceInfo->Description.GetName(), SplitPath(externalDataSourceName).back());
+        UNIT_ASSERT(externalDataSource.ExternalDataSourceInfo->Description.GetAuth().HasServiceAccount());
+        UNIT_ASSERT_VALUES_EQUAL(externalDataSource.ExternalDataSourceInfo->Description.GetAuth().GetServiceAccount().GetId(), "mysa");
+        UNIT_ASSERT_VALUES_EQUAL(externalDataSource.ExternalDataSourceInfo->Description.GetAuth().GetServiceAccount().GetSecretName(), "mysasignature");
+    }
+
     Y_UNIT_TEST(DisableCreateExternalDataSource) {
         TKikimrRunner kikimr;
         auto db = kikimr.GetTableClient();
@@ -4333,7 +4364,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
             );)";
         auto result = session.ExecuteSchemeQuery(query).GetValueSync();
         UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR);
-        UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Authorization method not specified");
+        UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Unknown AUTH_METHOD = UNKNOWN");
     }
 
     Y_UNIT_TEST(DropExternalDataSource) {
diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
index 7e50110e647..eec4c42fbb9 100644
--- a/ydb/core/protos/flat_scheme_op.proto
+++ b/ydb/core/protos/flat_scheme_op.proto
@@ -1732,9 +1732,15 @@ message TExternalTableDescription {
 message NoneAuth {
 }
 
+message ServiceAccountAuth {
+    optional string Id = 1;
+    optional string SecretName = 2;
+}
+
 message TAuth {
     oneof identity {
         NoneAuth None = 3;
+        ServiceAccountAuth ServiceAccount = 4;
     }
 }
 
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_external_data_source.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_data_source.cpp
index c15a012a63a..4afcfdacddb 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_create_external_data_source.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_data_source.cpp
@@ -50,6 +50,7 @@ bool ValidateAuth(const NKikimrSchemeOp::TAuth& auth, TString& errStr) {
             errStr = "Authorization method not specified";
             return false;
         }
+        case NKikimrSchemeOp::TAuth::kServiceAccount:
         case NKikimrSchemeOp::TAuth::kNone: {
             return true;
         }
diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp
index 65399f2d03d..3cd320f850c 100644
--- a/ydb/library/yql/sql/v1/sql.cpp
+++ b/ydb/library/yql/sql/v1/sql.cpp
@@ -878,6 +878,7 @@ protected:
     bool PermissionNameClause(const TRule_permission_name& node, TDeferredAtom& result);
     bool PermissionNameClause(const TRule_permission_id& node, TDeferredAtom& result);
 
+    bool ValidateAuthMethod(const std::map<TString, TDeferredAtom>& result);
     bool ValidateExternalTable(const TCreateTableParameters& params);
 private:
     bool SimpleTableRefCoreImpl(const TRule_simple_table_ref_core& node, TTableRef& result);
@@ -10200,7 +10201,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
             // ALTER TABLESTORE object_ref alter_table_store_action (COMMA alter_table_store_action)*;
             auto& node = core.GetAlt_sql_stmt_core38().GetRule_alter_table_store_stmt1();
             TObjectOperatorContext context(Ctx.Scoped);
-            
+
             if (node.GetRule_object_ref3().HasBlock1()) {
                 if (!ClusterExpr(node.GetRule_object_ref3().GetBlock1().GetRule_cluster_expr1(),
                     false, context.ServiceId, context.Cluster)) {
@@ -11859,7 +11860,7 @@ namespace {
         YQL_ENSURE(columnType, "Unknown column type");
         result["TYPE"] = TDeferredAtom(pos, columnType);
         if (!nullable) {
-            result["NOT_NULL"] = TDeferredAtom(pos, "true");    
+            result["NOT_NULL"] = TDeferredAtom(pos, "true");
         }
         return true;
     }
@@ -11902,7 +11903,8 @@ bool TSqlTranslation::StoreDataSourceSettingsEntry(const TIdentifier& id, const
         return false;
     }
 
-    if (IsIn({"source_type", "installation", "location", "auth_method"}, key)) {
+    if (IsIn({"source_type", "installation", "location",
+              "auth_method", "service_account_id", "service_account_secret_name"}, key)) {
         if (!StoreString(*value, result[key], Ctx, to_upper(key))) {
             return false;
         }
@@ -11925,13 +11927,11 @@ bool TSqlTranslation::ParseExternalDataSourceSettings(std::map<TString, TDeferre
             return false;
         }
     }
-
     if (result.find("source_type") == result.end()) {
         Ctx.Error() << "SOURCE_TYPE requires key";
         return false;
     }
-    if (result.find("auth_method") == result.end()) {
-        Ctx.Error() << "AUTH_METHOD requires key";
+    if (!ValidateAuthMethod(result)) {
         return false;
     }
     if (result.find("installation") == result.end() && result.find("location") == result.end()) {
@@ -11941,6 +11941,40 @@ bool TSqlTranslation::ParseExternalDataSourceSettings(std::map<TString, TDeferre
     return true;
 }
 
+bool TSqlTranslation::ValidateAuthMethod(const std::map<TString, TDeferredAtom>& result) {
+    const static TSet<TStringBuf> allAuthFields{
+        "service_account_id",
+        "service_account_secret_name"
+    };
+    const static TMap<TStringBuf, TSet<TStringBuf>> authMethodFields{
+        {"NONE", {}},
+        {"SERVICE_ACCOUNT", {"service_account_id", "service_account_secret_name"}}
+    };
+    auto authMethodIt = result.find("auth_method");
+    if (authMethodIt == result.end() || authMethodIt->second.GetLiteral() == nullptr) {
+        Ctx.Error() << "AUTH_METHOD requires key";
+        return false;
+    }
+    const auto& authMethod = *authMethodIt->second.GetLiteral();
+    auto it = authMethodFields.find(authMethod);
+    if (it == authMethodFields.end()) {
+        Ctx.Error() << "Unknown AUTH_METHOD = " << authMethod;
+        return false;
+    }
+    const auto& currentAuthFields = it->second;
+    for (const auto& authField: allAuthFields) {
+        if (currentAuthFields.contains(authField) && !result.contains(TString{authField})) {
+            Ctx.Error() << to_upper(TString{authField}) << " requires key";
+            return false;
+        }
+        if (!currentAuthFields.contains(authField) && result.contains(TString{authField})) {
+            Ctx.Error() << to_upper(TString{authField}) << " key is not supported for AUTH_METHOD = " << authMethod;
+            return false;
+        }
+    }
+    return true;
+}
+
 bool TSqlTranslation::ValidateExternalTable(const TCreateTableParameters& params) {
     if (params.TableType != ETableType::ExternalTable) {
         return true;
diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp
index 0667828b586..0718c3c64c7 100644
--- a/ydb/library/yql/sql/v1/sql_ut.cpp
+++ b/ydb/library/yql/sql/v1/sql_ut.cpp
@@ -5066,6 +5066,32 @@ Y_UNIT_TEST_SUITE(ExternalDataSource) {
         UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]);
     }
 
+    Y_UNIT_TEST(CreateExternalDataSourceWithServiceAccount) {
+        NYql::TAstParseResult res = SqlToYql(R"(
+                USE plato;
+                CREATE EXTERNAL DATA SOURCE MyDataSource WITH (
+                    SOURCE_TYPE="ObjectStorage",
+                    LOCATION="my-bucket",
+                    AUTH_METHOD="SERVICE_ACCOUNT",
+                    SERVICE_ACCOUNT_ID="sa",
+                    SERVICE_ACCOUNT_SECRET_NAME="secret_name"
+                );
+            )");
+        UNIT_ASSERT(res.Root);
+
+        TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) {
+            if (word == "Write") {
+                UNIT_ASSERT_STRING_CONTAINS(line, R"#('('('"auth_method" '"SERVICE_ACCOUNT") '('"location" '"my-bucket") '('"service_account_id" '"sa") '('"service_account_secret_name" '"secret_name") '('"source_type" '"ObjectStorage"))#");
+                UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find("createObject"));
+            }
+        };
+
+        TWordCountHive elementStat = { {TString("Write"), 0} };
+        VerifyProgram(res, elementStat, verifyLine);
+
+        UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Write"]);
+    }
+
     Y_UNIT_TEST(CreateExternalDataSourceWithTablePrefix) {
         NYql::TAstParseResult res = SqlToYql(R"(
                 USE plato;
@@ -5113,7 +5139,6 @@ Y_UNIT_TEST_SUITE(ExternalDataSource) {
                 );
             )" , "<main>:5:33: Error: INSTALLATION or LOCATION must be specified\n");
 
-
         ExpectFailWithError(R"(
                 USE plato;
                 CREATE EXTERNAL DATA SOURCE MyDataSource WITH (
@@ -5131,6 +5156,44 @@ Y_UNIT_TEST_SUITE(ExternalDataSource) {
                     OTHER="VALUE"
                 );
             )" , "<main>:7:21: Error: Unknown external data source setting: OTHER\n");
+
+        ExpectFailWithError(R"(
+                USE plato;
+                CREATE EXTERNAL DATA SOURCE MyDataSource WITH (
+                    SOURCE_TYPE="ObjectStorage",
+                    LOCATION="my-bucket",
+                    AUTH_METHOD="NONE1"
+                );
+            )" , "<main>:6:33: Error: Unknown AUTH_METHOD = NONE1\n");
+
+        ExpectFailWithError(R"(
+                USE plato;
+                CREATE EXTERNAL DATA SOURCE MyDataSource WITH (
+                    SOURCE_TYPE="ObjectStorage",
+                    LOCATION="my-bucket",
+                    AUTH_METHOD="SERVICE_ACCOUNT"
+                );
+            )" , "<main>:6:33: Error: SERVICE_ACCOUNT_ID requires key\n");
+
+        ExpectFailWithError(R"(
+                USE plato;
+                CREATE EXTERNAL DATA SOURCE MyDataSource WITH (
+                    SOURCE_TYPE="ObjectStorage",
+                    LOCATION="my-bucket",
+                    AUTH_METHOD="SERVICE_ACCOUNT",
+                    SERVICE_ACCOUNT_ID="s1"
+                );
+            )" , "<main>:7:40: Error: SERVICE_ACCOUNT_SECRET_NAME requires key\n");
+
+        ExpectFailWithError(R"(
+                USE plato;
+                CREATE EXTERNAL DATA SOURCE MyDataSource WITH (
+                    SOURCE_TYPE="ObjectStorage",
+                    LOCATION="my-bucket",
+                    AUTH_METHOD="SERVICE_ACCOUNT",
+                    SERVICE_ACCOUNT_SECRET_NAME="s1"
+                );
+            )" , "<main>:7:49: Error: SERVICE_ACCOUNT_ID requires key\n");
     }
 
     Y_UNIT_TEST(DropExternalDataSourceWithTablePrefix) {