diff options
author | Ilia Shakhov <pixcc@ydb.tech> | 2024-09-13 20:56:01 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-09-13 20:56:01 +0300 |
commit | 3dce9fe9acdecf75f27f115e05e8985535fd76fd (patch) | |
tree | c35e336b566288686b6b83319a2091491fb78de1 | |
parent | f405ee4e9988846158a7efde1ee0b49aefe2d5c8 (diff) | |
download | ydb-3dce9fe9acdecf75f27f115e05e8985535fd76fd.tar.gz |
Add import ACL from S3 for tables (#9181)
18 files changed, 482 insertions, 24 deletions
diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 1b6e9af9bc..ca50d3a6a8 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4380,6 +4380,12 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { item.Scheme = scheme; } + if (rowset.HaveValue<Schema::ImportItems::Permissions>()) { + Ydb::Scheme::ModifyPermissionsRequest permissions; + Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(permissions, rowset.GetValue<Schema::ImportItems::Permissions>())); + item.Permissions = permissions; + } + item.State = static_cast<TImportInfo::EState>(rowset.GetValue<Schema::ImportItems::State>()); item.WaitTxId = rowset.GetValueOrDefault<Schema::ImportItems::WaitTxId>(InvalidTxId); item.NextIndexIdx = rowset.GetValueOrDefault<Schema::ImportItems::NextIndexIdx>(0); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp index d3217e001c..6d68ce4975 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp @@ -212,6 +212,9 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr if (tx.HasAlterUserAttributes()) { scheme.MutableAlterUserAttributes()->CopyFrom(tx.GetAlterUserAttributes()); } + if (tx.HasModifyACL()) { + scheme.MutableModifyACL()->CopyFrom(tx.GetModifyACL()); + } result.push_back(CreateNewTable(NextPartId(nextId, result), scheme, sequences)); } diff --git a/ydb/core/tx/schemeshard/schemeshard_import.cpp b/ydb/core/tx/schemeshard/schemeshard_import.cpp index 222f785228..968897b29b 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import.cpp @@ -174,6 +174,11 @@ void TSchemeShard::PersistImportItemScheme(NIceDb::TNiceDb& db, const TImportInf db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Update( NIceDb::TUpdate<Schema::ImportItems::Scheme>(item.Scheme.SerializeAsString()) ); + if (item.Permissions.Defined()) { + db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Update( + NIceDb::TUpdate<Schema::ImportItems::Permissions>(item.Permissions->SerializeAsString()) + ); + } } void TSchemeShard::PersistImportItemDstPathId(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx) { diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp index bc9943538e..ea5f379f22 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp @@ -3,6 +3,7 @@ #include <ydb/core/base/path.h> #include <ydb/core/ydb_convert/table_description.h> +#include <ydb/core/ydb_convert/ydb_convert.h> namespace NKikimr { namespace NSchemeShard { @@ -20,10 +21,6 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateTablePropose( auto propose = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(ui64(txId), ss->TabletID()); auto& record = propose->Record; - if (importInfo->UserSID) { - record.SetOwner(*importInfo->UserSID); - } - auto& modifyScheme = *record.AddTransaction(); modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateIndexedTable); modifyScheme.SetInternal(true); @@ -66,6 +63,15 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateTablePropose( } } + if (importInfo->UserSID) { + record.SetOwner(*importInfo->UserSID); + } + FillOwner(record, item.Permissions); + + if (!FillACL(modifyScheme, item.Permissions, error)) { + return nullptr; + } + return propose; } diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp index b826e94e42..c802f2a419 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp @@ -23,12 +23,18 @@ using namespace Aws::Client; using namespace Aws::S3; using namespace Aws; +// Downloads scheme-related objects from S3 class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> { static TString SchemeKeyFromSettings(const Ydb::Import::ImportFromS3Settings& settings, ui32 itemIdx) { Y_ABORT_UNLESS(itemIdx < (ui32)settings.items_size()); return TStringBuilder() << settings.items(itemIdx).source_prefix() << "/scheme.pb"; } + static TString PermissionsKeyFromSettings(const Ydb::Import::ImportFromS3Settings& settings, ui32 itemIdx) { + Y_ABORT_UNLESS(itemIdx < (ui32)settings.items_size()); + return TStringBuilder() << settings.items(itemIdx).source_prefix() << "/permissions.pb"; + } + void HeadObject(const TString& key) { auto request = Model::HeadObjectRequest() .WithKey(key); @@ -36,10 +42,10 @@ class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> { Send(Client, new TEvExternalStorage::TEvHeadObjectRequest(request)); } - void Handle(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { + void HandleScheme(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - LOG_D("Handle TEvExternalStorage::TEvHeadObjectResponse" + LOG_D("HandleScheme TEvExternalStorage::TEvHeadObjectResponse" << ": self# " << SelfId() << ", result# " << result); @@ -51,6 +57,25 @@ class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> { GetObject(SchemeKey, std::make_pair(0, contentLength - 1)); } + void HandlePermissions(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + LOG_D("HandlePermissions TEvExternalStorage::TEvHeadObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + if (result.GetError().GetErrorType() == S3Errors::RESOURCE_NOT_FOUND + || result.GetError().GetErrorType() == S3Errors::NO_SUCH_KEY) { + Reply(); // permissions are optional + return; + } else if (!CheckResult(result, "HeadObject")) { + return; + } + + const auto contentLength = result.GetResult().GetContentLength(); + GetObject(PermissionsKey, std::make_pair(0, contentLength - 1)); + } + void GetObject(const TString& key, const std::pair<ui64, ui64>& range) { auto request = Model::GetObjectRequest() .WithKey(key) @@ -59,11 +84,11 @@ class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> { Send(Client, new TEvExternalStorage::TEvGetObjectRequest(request)); } - void Handle(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) { + void HandleScheme(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) { const auto& msg = *ev->Get(); const auto& result = msg.Result; - LOG_D("Handle TEvExternalStorage::TEvGetObjectResponse" + LOG_D("HandleScheme TEvExternalStorage::TEvGetObjectResponse" << ": self# " << SelfId() << ", result# " << result); @@ -74,7 +99,7 @@ class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> { Y_ABORT_UNLESS(ItemIdx < ImportInfo->Items.size()); auto& item = ImportInfo->Items.at(ItemIdx); - LOG_T("Trying to parse" + LOG_T("Trying to parse scheme" << ": self# " << SelfId() << ", body# " << SubstGlobalCopy(msg.Body, "\n", "\\n")); @@ -82,6 +107,38 @@ class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> { return Reply(false, "Cannot parse scheme"); } + if (NeedDownloadPermissions) { + StartDownloadingPermissions(); + } else { + Reply(); + } + } + + void HandlePermissions(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) { + const auto& msg = *ev->Get(); + const auto& result = msg.Result; + + LOG_D("HandlePermissions TEvExternalStorage::TEvGetObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + if (!CheckResult(result, "GetObject")) { + return; + } + + Y_ABORT_UNLESS(ItemIdx < ImportInfo->Items.size()); + auto& item = ImportInfo->Items.at(ItemIdx); + + LOG_T("Trying to parse permissions" + << ": self# " << SelfId() + << ", body# " << SubstGlobalCopy(msg.Body, "\n", "\\n")); + + Ydb::Scheme::ModifyPermissionsRequest permissions; + if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &permissions)) { + return Reply(false, "Cannot parse permissions"); + } + item.Permissions = std::move(permissions); + Reply(); } @@ -123,6 +180,33 @@ class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> { TActor::PassAway(); } + void Download(const TString& key) { + if (Client) { + Send(Client, new TEvents::TEvPoisonPill()); + } + Client = RegisterWithSameMailbox(CreateS3Wrapper(ExternalStorageConfig->ConstructStorageOperator())); + + HeadObject(key); + } + + void DownloadScheme() { + Download(SchemeKey); + } + + void DownloadPermissions() { + Download(PermissionsKey); + } + + void ResetRetries() { + Attempt = 0; + } + + void StartDownloadingPermissions() { + ResetRetries(); + DownloadPermissions(); + Become(&TThis::StateDownloadPermissions); + } + public: explicit TSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx) : ExternalStorageConfig(new NWrappers::NExternalStorage::TS3ExternalStorageConfig(importInfo->Settings)) @@ -130,26 +214,33 @@ public: , ImportInfo(importInfo) , ItemIdx(itemIdx) , SchemeKey(SchemeKeyFromSettings(importInfo->Settings, itemIdx)) + , PermissionsKey(PermissionsKeyFromSettings(importInfo->Settings, itemIdx)) , Retries(importInfo->Settings.number_of_retries()) + , NeedDownloadPermissions(!importInfo->Settings.no_acl()) { } void Bootstrap() { - if (Client) { - Send(Client, new TEvents::TEvPoisonPill()); - } - Client = RegisterWithSameMailbox(CreateS3Wrapper(ExternalStorageConfig->ConstructStorageOperator())); + DownloadScheme(); + Become(&TThis::StateDownloadScheme); + } + + STATEFN(StateDownloadScheme) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandleScheme); + hFunc(TEvExternalStorage::TEvGetObjectResponse, HandleScheme); - HeadObject(SchemeKey); - Become(&TThis::StateWork); + sFunc(TEvents::TEvWakeup, DownloadScheme); + sFunc(TEvents::TEvPoisonPill, PassAway); + } } - STATEFN(StateWork) { + STATEFN(StateDownloadPermissions) { switch (ev->GetTypeRewrite()) { - hFunc(TEvExternalStorage::TEvHeadObjectResponse, Handle); - hFunc(TEvExternalStorage::TEvGetObjectResponse, Handle); + hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandlePermissions); + hFunc(TEvExternalStorage::TEvGetObjectResponse, HandlePermissions); - sFunc(TEvents::TEvWakeup, Bootstrap); + sFunc(TEvents::TEvWakeup, DownloadPermissions); sFunc(TEvents::TEvPoisonPill, PassAway); } } @@ -161,6 +252,7 @@ private: const ui32 ItemIdx; const TString SchemeKey; + const TString PermissionsKey; const ui32 Retries; ui32 Attempt = 0; @@ -168,6 +260,8 @@ private: TDuration Delay = TDuration::Minutes(1); static constexpr TDuration MaxDelay = TDuration::Minutes(10); + const bool NeedDownloadPermissions = true; + TActorId Client; }; // TSchemeGetter diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index d4281b7003..0eb7b65196 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2769,6 +2769,7 @@ struct TImportInfo: public TSimpleRefCount<TImportInfo> { TString DstPathName; TPathId DstPathId; Ydb::Table::CreateTableRequest Scheme; + TMaybeFail<Ydb::Scheme::ModifyPermissionsRequest> Permissions; EState State = EState::GetScheme; ESubState SubState = ESubState::AllocateTxId; diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h index 7144a194b9..17750e6c3d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_schema.h +++ b/ydb/core/tx/schemeshard/schemeshard_schema.h @@ -1520,6 +1520,7 @@ struct Schema : NIceDb::Schema { struct DstPathOwnerId : Column<4, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; }; struct DstPathLocalId : Column<5, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; }; struct Scheme : Column<6, NScheme::NTypeIds::String> {}; + struct Permissions : Column<11, NScheme::NTypeIds::String> {}; struct State : Column<7, NScheme::NTypeIds::Byte> {}; struct WaitTxId : Column<8, NScheme::NTypeIds::Uint64> { using Type = TTxId; }; @@ -1534,6 +1535,7 @@ struct Schema : NIceDb::Schema { DstPathOwnerId, DstPathLocalId, Scheme, + Permissions, State, WaitTxId, NextIndexIdx, diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index 04bee847da..934a6e7e00 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -1207,9 +1207,9 @@ TCheckFunc HasOwner(const TString& owner) { }; } -void CheckEffectiveRight(const NKikimrScheme::TEvDescribeSchemeResult& record, const TString& right, bool mustHave) { +void CheckRight(const NKikimrScheme::TEvDescribeSchemeResult& record, const TString& right, bool mustHave, bool isEffective) { const auto& self = record.GetPathDescription().GetSelf(); - TSecurityObject src(self.GetOwner(), self.GetEffectiveACL(), false); + TSecurityObject src(self.GetOwner(), isEffective ? self.GetEffectiveACL() : self.GetACL(), false); NACLib::TSecurityObject required; required.FromString(right); @@ -1233,6 +1233,22 @@ void CheckEffectiveRight(const NKikimrScheme::TEvDescribeSchemeResult& record, c } } +TCheckFunc HasRight(const TString& right) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + CheckRight(record, right, true, true); + }; +} + +TCheckFunc HasNotRight(const TString& right) { + return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { + CheckRight(record, right, false, true); + }; +} + +void CheckEffectiveRight(const NKikimrScheme::TEvDescribeSchemeResult& record, const TString& right, bool mustHave) { + CheckRight(record, right, mustHave, true); +} + TCheckFunc HasEffectiveRight(const TString& right) { return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) { CheckEffectiveRight(record, right, true); diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h index 2720b34718..a28b0908ce 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h @@ -168,6 +168,8 @@ namespace NLs { TCheckFunc BackupHistoryCount(ui64 count); TCheckFunc HasOwner(const TString& owner); + TCheckFunc HasRight(const TString& right); + TCheckFunc HasNotRight(const TString& right); TCheckFunc HasEffectiveRight(const TString& right); TCheckFunc HasNotEffectiveRight(const TString& right); diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp index 2b558341b0..ee1de957a1 100644 --- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp +++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp @@ -138,6 +138,7 @@ namespace { struct TTestDataWithScheme { TString Scheme; + TString Permissions; TVector<TTestData> Data; TTestDataWithScheme() = default; @@ -240,9 +241,14 @@ namespace { } } - TTestDataWithScheme GenerateTestData(const TString& scheme, const TVector<std::pair<TString, ui64>>& shardsConfig) { + TTestDataWithScheme GenerateTestData( + const TString& scheme, + const TVector<std::pair<TString, ui64>>& shardsConfig, + const TString& permissions = "") + { TTestDataWithScheme result; result.Scheme = scheme; + result.Permissions = permissions; for (const auto& [keyPrefix, count] : shardsConfig) { result.Data.push_back(GenerateTestData(keyPrefix, count)); @@ -256,6 +262,9 @@ namespace { for (const auto& [prefix, item] : data) { result.emplace(prefix + "/scheme.pb", item.Scheme); + if (item.Permissions) { + result.emplace(prefix + "/permissions.pb", item.Permissions); + } for (ui32 i = 0; i < item.Data.size(); ++i) { const auto& data = item.Data.at(i); result.emplace(Sprintf("%s/data_%02d%s", prefix.data(), i, data.Ext().c_str()), data.Data); @@ -3753,6 +3762,240 @@ Y_UNIT_TEST_SUITE(TImportTests) { UNIT_ASSERT_VALUES_EQUAL(entry.GetProgress(), Ydb::Import::ImportProgress::PROGRESS_PREPARING); UNIT_ASSERT_VALUES_EQUAL(entry.GetUserSID(), userSID); } + + Y_UNIT_TEST(TablePermissions) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + const auto permissions = R"( + actions { + change_owner: "eve" + } + actions { + grant { + subject: "alice" + permission_names: "ydb.generic.read" + } + } + actions { + grant { + subject: "alice" + permission_names: "ydb.generic.write" + } + } + actions { + grant { + subject: "bob" + permission_names: "ydb.generic.read" + } + } + )"; + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}, permissions); + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", port)); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { + NLs::PathExist, + NLs::HasOwner("eve"), + NLs::HasRight("+R:alice"), + NLs::HasRight("+W:alice"), + NLs::HasRight("+R:bob") + }); + } + + Y_UNIT_TEST(UnexpectedPermission) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + const auto permissions = R"( + actions { + change_owner: "eve" + } + actions { + grant { + subject: "alice" + permission_names: "ydb.unexpected.permission" + } + } + )"; + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}, permissions); + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", port)); + env.TestWaitNotification(runtime, txId); + + auto desc = TestGetImport(runtime, txId, "/MyRoot", Ydb::StatusIds::CANCELLED); + auto entry = desc.GetResponse().GetEntry(); + UNIT_ASSERT_VALUES_EQUAL(entry.GetProgress(), Ydb::Import::ImportProgress::PROGRESS_CANCELLED); + } + + Y_UNIT_TEST(CorruptedPermissions) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + const auto permissions = R"( + corrupted + )"; + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}, permissions); + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + } + )", port)); + env.TestWaitNotification(runtime, txId); + + auto desc = TestGetImport(runtime, txId, "/MyRoot", Ydb::StatusIds::CANCELLED); + auto entry = desc.GetResponse().GetEntry(); + UNIT_ASSERT_VALUES_EQUAL(entry.GetProgress(), Ydb::Import::ImportProgress::PROGRESS_CANCELLED); + } + + Y_UNIT_TEST(NoACLOption) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + const auto permissions = R"( + actions { + change_owner: "eve" + } + actions { + grant { + subject: "alice" + permission_names: "ydb.generic.read" + } + } + actions { + grant { + subject: "alice" + permission_names: "ydb.generic.write" + } + } + actions { + grant { + subject: "bob" + permission_names: "ydb.generic.read" + } + } + )"; + + const auto data = GenerateTestData(R"( + columns { + name: "key" + type { optional_type { item { type_id: UTF8 } } } + } + columns { + name: "value" + type { optional_type { item { type_id: UTF8 } } } + } + primary_key: "key" + )", {{"a", 1}}, permissions); + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + + TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port)); + UNIT_ASSERT(s3Mock.Start()); + + const TString userSID = "user@builtin"; + TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"( + ImportFromS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_prefix: "" + destination_path: "/MyRoot/Table" + } + no_acl: true + } + )", port), userSID); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), { + NLs::PathExist, + NLs::HasOwner(userSID), + NLs::HasNotRight("+R:alice"), + NLs::HasNotRight("+W:alice"), + NLs::HasNotRight("+R:bob") + }); + } } Y_UNIT_TEST_SUITE(TImportWithRebootsTests) { diff --git a/ydb/core/ydb_convert/ydb_convert.cpp b/ydb/core/ydb_convert/ydb_convert.cpp index 20f04c1e5b..d2d84d2d26 100644 --- a/ydb/core/ydb_convert/ydb_convert.cpp +++ b/ydb/core/ydb_convert/ydb_convert.cpp @@ -20,6 +20,28 @@ namespace NKikimr { +namespace { + + bool FillAllowPermissions(NACLib::TDiffACL& out, const Ydb::Scheme::Permissions& in, TString& error) { + for (const auto& permission : in.permission_names()) { + try { + auto aclAttrs = ConvertYdbPermissionNameToACLAttrs(permission); + out.AddAccess( + NACLib::EAccessType::Allow, + aclAttrs.AccessMask, + in.subject(), + aclAttrs.InheritanceType + ); + } catch (const std::exception& e) { + error = e.what(); + return false; + } + } + return true; + } + +} // anonymous namespace + template<typename TOut> Y_FORCE_INLINE void ConvertMiniKQLTupleTypeToYdbType(const NKikimrMiniKQL::TTupleType& protoTupleType, TOut& output) { const ui32 elementsCount = static_cast<ui32>(protoTupleType.ElementSize()); @@ -1410,4 +1432,35 @@ void ProtoValueFromCell(NYdb::TValueBuilder& vb, const NScheme::TTypeInfo& typeI } } +bool FillACL(NKikimrSchemeOp::TModifyScheme& out, + const TMaybeFail<Ydb::Scheme::ModifyPermissionsRequest>& in, + TString& error) { + if (in.Empty()) { + return true; + } + + NACLib::TDiffACL diffACL; + for (const auto& action : in->actions()) { + if (action.has_grant() && !FillAllowPermissions(diffACL, action.grant(), error)) { + return false; + } + } + out.MutableModifyACL()->SetDiffACL(diffACL.SerializeAsString()); + + return true; +} + +void FillOwner(NKikimrScheme::TEvModifySchemeTransaction& out, + const TMaybeFail<Ydb::Scheme::ModifyPermissionsRequest>& in) { + if (in.Empty()) { + return; + } + + for (const auto& action : in->actions()) { + if (action.has_change_owner()) { + out.SetOwner(action.change_owner()); + } + } +} + } // namespace NKikimr diff --git a/ydb/core/ydb_convert/ydb_convert.h b/ydb/core/ydb_convert/ydb_convert.h index ec112f1d78..7ff8ce6338 100644 --- a/ydb/core/ydb_convert/ydb_convert.h +++ b/ydb/core/ydb_convert/ydb_convert.h @@ -57,5 +57,11 @@ bool CellFromProtoVal(NScheme::TTypeInfo type, i32 typmod, const Ydb::Value* vp, void ProtoValueFromCell(NYdb::TValueBuilder& vb, const NScheme::TTypeInfo& typeInfo, const TCell& cell); +bool FillACL(NKikimrSchemeOp::TModifyScheme& out, + const TMaybeFail<Ydb::Scheme::ModifyPermissionsRequest>& in, + TString& error); + +void FillOwner(NKikimrScheme::TEvModifySchemeTransaction& out, + const TMaybeFail<Ydb::Scheme::ModifyPermissionsRequest>& in); } // namespace NKikimr diff --git a/ydb/public/api/protos/ydb_import.proto b/ydb/public/api/protos/ydb_import.proto index f03f56ef26..f4cc476733 100644 --- a/ydb/public/api/protos/ydb_import.proto +++ b/ydb/public/api/protos/ydb_import.proto @@ -42,7 +42,8 @@ message ImportFromS3Settings { The object name begins with 'source_prefix'. This prefix is followed by: * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero; - * '/scheme.pb' - object with information about scheme, indexes, etc. + * '/scheme.pb' - object with information about scheme, indexes, etc; + * '/permissions.pb' - object with information about ACL and owner. */ string source_prefix = 1 [(required) = true]; @@ -67,6 +68,10 @@ message ImportFromS3Settings { // details: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html // it is especially useful for custom s3 implementations bool disable_virtual_addressing = 10; + + // Prevent importing of ACL and owner. If true, objects are created with empty ACL + // and their owner will be the user who started the import. + bool no_acl = 11; } message ImportFromS3Result { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp index e93089f01b..5c4d636e61 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp @@ -95,6 +95,9 @@ void TCommandImportFromS3::Config(TConfig& config) { config.Opts->AddLongOption("use-virtual-addressing", "S3 bucket virtual addressing") .RequiredArgument("BOOL").StoreResult<bool>(&UseVirtualAddressing).DefaultValue("true"); + config.Opts->AddLongOption("no-acl", "Prevent importing of ACL and owner") + .RequiredArgument("BOOL").StoreTrue(&NoACL).DefaultValue("false"); + AddDeprecatedJsonOption(config); AddOutputFormats(config, { EDataFormat::Pretty, EDataFormat::ProtoJsonBase64 }); config.Opts->MutuallyExclusive("json", "format"); @@ -135,6 +138,7 @@ int TCommandImportFromS3::Run(TConfig& config) { } settings.NumberOfRetries(NumberOfRetries); + settings.NoACL(NoACL); #if defined(_win32_) for (const auto& item : Items) { settings.AppendItem({item.Source, item.Destination}); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_import.h b/ydb/public/lib/ydb_cli/commands/ydb_service_import.h index 1cfab9c531..321bdacf27 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_import.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_import.h @@ -38,6 +38,7 @@ private: TString Description; ui32 NumberOfRetries = 10; bool UseVirtualAddressing = true; + bool NoACL = true; }; class TCommandImportFromFile : public TClientCommandTree { diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.cpp b/ydb/public/sdk/cpp/client/ydb_import/import.cpp index 98031b5596..47fc97e906 100644 --- a/ydb/public/sdk/cpp/client/ydb_import/import.cpp +++ b/ydb/public/sdk/cpp/client/ydb_import/import.cpp @@ -153,6 +153,10 @@ TFuture<TImportFromS3Response> TImportClient::ImportFromS3(const TImportFromS3Se request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.GetRef()); } + if (settings.NoACL_) { + request.mutable_settings()->set_no_acl(settings.NoACL_.GetRef()); + } + request.mutable_settings()->set_disable_virtual_addressing(!settings.UseVirtualAddressing_); return Impl_->ImportFromS3(std::move(request), settings); diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.h b/ydb/public/sdk/cpp/client/ydb_import/import.h index 865fec384c..8f479cfd90 100644 --- a/ydb/public/sdk/cpp/client/ydb_import/import.h +++ b/ydb/public/sdk/cpp/client/ydb_import/import.h @@ -41,6 +41,7 @@ struct TImportFromS3Settings : public TOperationRequestSettings<TImportFromS3Set FLUENT_SETTING_VECTOR(TItem, Item); FLUENT_SETTING_OPTIONAL(TString, Description); FLUENT_SETTING_OPTIONAL(ui32, NumberOfRetries); + FLUENT_SETTING_OPTIONAL(bool, NoACL); }; class TImportFromS3Response : public TOperation { diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema index 50e2aedd62..19aeda8889 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema @@ -5839,6 +5839,11 @@ "ColumnId": 10, "ColumnName": "Issue", "ColumnType": "Utf8" + }, + { + "ColumnId": 11, + "ColumnName": "Permissions", + "ColumnType": "String" } ], "ColumnsDropped": [], @@ -5854,7 +5859,8 @@ 7, 8, 9, - 10 + 10, + 11 ], "RoomID": 0, "Codec": 0, |