aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorIlia Shakhov <pixcc@ydb.tech>2024-09-13 20:56:01 +0300
committerGitHub <noreply@github.com>2024-09-13 20:56:01 +0300
commit3dce9fe9acdecf75f27f115e05e8985535fd76fd (patch)
treec35e336b566288686b6b83319a2091491fb78de1
parentf405ee4e9988846158a7efde1ee0b49aefe2d5c8 (diff)
downloadydb-3dce9fe9acdecf75f27f115e05e8985535fd76fd.tar.gz
Add import ACL from S3 for tables (#9181)
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__init.cpp6
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp3
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_import.cpp5
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp14
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp124
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_info_types.h1
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_schema.h2
-rw-r--r--ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp20
-rw-r--r--ydb/core/tx/schemeshard/ut_helpers/ls_checks.h2
-rw-r--r--ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp245
-rw-r--r--ydb/core/ydb_convert/ydb_convert.cpp53
-rw-r--r--ydb/core/ydb_convert/ydb_convert.h6
-rw-r--r--ydb/public/api/protos/ydb_import.proto7
-rw-r--r--ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp4
-rw-r--r--ydb/public/lib/ydb_cli/commands/ydb_service_import.h1
-rw-r--r--ydb/public/sdk/cpp/client/ydb_import/import.cpp4
-rw-r--r--ydb/public/sdk/cpp/client/ydb_import/import.h1
-rw-r--r--ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema8
18 files changed, 482 insertions, 24 deletions
diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp
index 1b6e9af9bc..ca50d3a6a8 100644
--- a/ydb/core/tx/schemeshard/schemeshard__init.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp
@@ -4380,6 +4380,12 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> {
item.Scheme = scheme;
}
+ if (rowset.HaveValue<Schema::ImportItems::Permissions>()) {
+ Ydb::Scheme::ModifyPermissionsRequest permissions;
+ Y_ABORT_UNLESS(ParseFromStringNoSizeLimit(permissions, rowset.GetValue<Schema::ImportItems::Permissions>()));
+ item.Permissions = permissions;
+ }
+
item.State = static_cast<TImportInfo::EState>(rowset.GetValue<Schema::ImportItems::State>());
item.WaitTxId = rowset.GetValueOrDefault<Schema::ImportItems::WaitTxId>(InvalidTxId);
item.NextIndexIdx = rowset.GetValueOrDefault<Schema::ImportItems::NextIndexIdx>(0);
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp
index d3217e001c..6d68ce4975 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp
@@ -212,6 +212,9 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
if (tx.HasAlterUserAttributes()) {
scheme.MutableAlterUserAttributes()->CopyFrom(tx.GetAlterUserAttributes());
}
+ if (tx.HasModifyACL()) {
+ scheme.MutableModifyACL()->CopyFrom(tx.GetModifyACL());
+ }
result.push_back(CreateNewTable(NextPartId(nextId, result), scheme, sequences));
}
diff --git a/ydb/core/tx/schemeshard/schemeshard_import.cpp b/ydb/core/tx/schemeshard/schemeshard_import.cpp
index 222f785228..968897b29b 100644
--- a/ydb/core/tx/schemeshard/schemeshard_import.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_import.cpp
@@ -174,6 +174,11 @@ void TSchemeShard::PersistImportItemScheme(NIceDb::TNiceDb& db, const TImportInf
db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Update(
NIceDb::TUpdate<Schema::ImportItems::Scheme>(item.Scheme.SerializeAsString())
);
+ if (item.Permissions.Defined()) {
+ db.Table<Schema::ImportItems>().Key(importInfo->Id, itemIdx).Update(
+ NIceDb::TUpdate<Schema::ImportItems::Permissions>(item.Permissions->SerializeAsString())
+ );
+ }
}
void TSchemeShard::PersistImportItemDstPathId(NIceDb::TNiceDb& db, const TImportInfo::TPtr importInfo, ui32 itemIdx) {
diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp
index bc9943538e..ea5f379f22 100644
--- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp
@@ -3,6 +3,7 @@
#include <ydb/core/base/path.h>
#include <ydb/core/ydb_convert/table_description.h>
+#include <ydb/core/ydb_convert/ydb_convert.h>
namespace NKikimr {
namespace NSchemeShard {
@@ -20,10 +21,6 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateTablePropose(
auto propose = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(ui64(txId), ss->TabletID());
auto& record = propose->Record;
- if (importInfo->UserSID) {
- record.SetOwner(*importInfo->UserSID);
- }
-
auto& modifyScheme = *record.AddTransaction();
modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateIndexedTable);
modifyScheme.SetInternal(true);
@@ -66,6 +63,15 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateTablePropose(
}
}
+ if (importInfo->UserSID) {
+ record.SetOwner(*importInfo->UserSID);
+ }
+ FillOwner(record, item.Permissions);
+
+ if (!FillACL(modifyScheme, item.Permissions, error)) {
+ return nullptr;
+ }
+
return propose;
}
diff --git a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp
index b826e94e42..c802f2a419 100644
--- a/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_import_scheme_getter.cpp
@@ -23,12 +23,18 @@ using namespace Aws::Client;
using namespace Aws::S3;
using namespace Aws;
+// Downloads scheme-related objects from S3
class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> {
static TString SchemeKeyFromSettings(const Ydb::Import::ImportFromS3Settings& settings, ui32 itemIdx) {
Y_ABORT_UNLESS(itemIdx < (ui32)settings.items_size());
return TStringBuilder() << settings.items(itemIdx).source_prefix() << "/scheme.pb";
}
+ static TString PermissionsKeyFromSettings(const Ydb::Import::ImportFromS3Settings& settings, ui32 itemIdx) {
+ Y_ABORT_UNLESS(itemIdx < (ui32)settings.items_size());
+ return TStringBuilder() << settings.items(itemIdx).source_prefix() << "/permissions.pb";
+ }
+
void HeadObject(const TString& key) {
auto request = Model::HeadObjectRequest()
.WithKey(key);
@@ -36,10 +42,10 @@ class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> {
Send(Client, new TEvExternalStorage::TEvHeadObjectRequest(request));
}
- void Handle(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) {
+ void HandleScheme(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) {
const auto& result = ev->Get()->Result;
- LOG_D("Handle TEvExternalStorage::TEvHeadObjectResponse"
+ LOG_D("HandleScheme TEvExternalStorage::TEvHeadObjectResponse"
<< ": self# " << SelfId()
<< ", result# " << result);
@@ -51,6 +57,25 @@ class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> {
GetObject(SchemeKey, std::make_pair(0, contentLength - 1));
}
+ void HandlePermissions(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) {
+ const auto& result = ev->Get()->Result;
+
+ LOG_D("HandlePermissions TEvExternalStorage::TEvHeadObjectResponse"
+ << ": self# " << SelfId()
+ << ", result# " << result);
+
+ if (result.GetError().GetErrorType() == S3Errors::RESOURCE_NOT_FOUND
+ || result.GetError().GetErrorType() == S3Errors::NO_SUCH_KEY) {
+ Reply(); // permissions are optional
+ return;
+ } else if (!CheckResult(result, "HeadObject")) {
+ return;
+ }
+
+ const auto contentLength = result.GetResult().GetContentLength();
+ GetObject(PermissionsKey, std::make_pair(0, contentLength - 1));
+ }
+
void GetObject(const TString& key, const std::pair<ui64, ui64>& range) {
auto request = Model::GetObjectRequest()
.WithKey(key)
@@ -59,11 +84,11 @@ class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> {
Send(Client, new TEvExternalStorage::TEvGetObjectRequest(request));
}
- void Handle(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) {
+ void HandleScheme(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) {
const auto& msg = *ev->Get();
const auto& result = msg.Result;
- LOG_D("Handle TEvExternalStorage::TEvGetObjectResponse"
+ LOG_D("HandleScheme TEvExternalStorage::TEvGetObjectResponse"
<< ": self# " << SelfId()
<< ", result# " << result);
@@ -74,7 +99,7 @@ class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> {
Y_ABORT_UNLESS(ItemIdx < ImportInfo->Items.size());
auto& item = ImportInfo->Items.at(ItemIdx);
- LOG_T("Trying to parse"
+ LOG_T("Trying to parse scheme"
<< ": self# " << SelfId()
<< ", body# " << SubstGlobalCopy(msg.Body, "\n", "\\n"));
@@ -82,6 +107,38 @@ class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> {
return Reply(false, "Cannot parse scheme");
}
+ if (NeedDownloadPermissions) {
+ StartDownloadingPermissions();
+ } else {
+ Reply();
+ }
+ }
+
+ void HandlePermissions(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) {
+ const auto& msg = *ev->Get();
+ const auto& result = msg.Result;
+
+ LOG_D("HandlePermissions TEvExternalStorage::TEvGetObjectResponse"
+ << ": self# " << SelfId()
+ << ", result# " << result);
+
+ if (!CheckResult(result, "GetObject")) {
+ return;
+ }
+
+ Y_ABORT_UNLESS(ItemIdx < ImportInfo->Items.size());
+ auto& item = ImportInfo->Items.at(ItemIdx);
+
+ LOG_T("Trying to parse permissions"
+ << ": self# " << SelfId()
+ << ", body# " << SubstGlobalCopy(msg.Body, "\n", "\\n"));
+
+ Ydb::Scheme::ModifyPermissionsRequest permissions;
+ if (!google::protobuf::TextFormat::ParseFromString(msg.Body, &permissions)) {
+ return Reply(false, "Cannot parse permissions");
+ }
+ item.Permissions = std::move(permissions);
+
Reply();
}
@@ -123,6 +180,33 @@ class TSchemeGetter: public TActorBootstrapped<TSchemeGetter> {
TActor::PassAway();
}
+ void Download(const TString& key) {
+ if (Client) {
+ Send(Client, new TEvents::TEvPoisonPill());
+ }
+ Client = RegisterWithSameMailbox(CreateS3Wrapper(ExternalStorageConfig->ConstructStorageOperator()));
+
+ HeadObject(key);
+ }
+
+ void DownloadScheme() {
+ Download(SchemeKey);
+ }
+
+ void DownloadPermissions() {
+ Download(PermissionsKey);
+ }
+
+ void ResetRetries() {
+ Attempt = 0;
+ }
+
+ void StartDownloadingPermissions() {
+ ResetRetries();
+ DownloadPermissions();
+ Become(&TThis::StateDownloadPermissions);
+ }
+
public:
explicit TSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx)
: ExternalStorageConfig(new NWrappers::NExternalStorage::TS3ExternalStorageConfig(importInfo->Settings))
@@ -130,26 +214,33 @@ public:
, ImportInfo(importInfo)
, ItemIdx(itemIdx)
, SchemeKey(SchemeKeyFromSettings(importInfo->Settings, itemIdx))
+ , PermissionsKey(PermissionsKeyFromSettings(importInfo->Settings, itemIdx))
, Retries(importInfo->Settings.number_of_retries())
+ , NeedDownloadPermissions(!importInfo->Settings.no_acl())
{
}
void Bootstrap() {
- if (Client) {
- Send(Client, new TEvents::TEvPoisonPill());
- }
- Client = RegisterWithSameMailbox(CreateS3Wrapper(ExternalStorageConfig->ConstructStorageOperator()));
+ DownloadScheme();
+ Become(&TThis::StateDownloadScheme);
+ }
+
+ STATEFN(StateDownloadScheme) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandleScheme);
+ hFunc(TEvExternalStorage::TEvGetObjectResponse, HandleScheme);
- HeadObject(SchemeKey);
- Become(&TThis::StateWork);
+ sFunc(TEvents::TEvWakeup, DownloadScheme);
+ sFunc(TEvents::TEvPoisonPill, PassAway);
+ }
}
- STATEFN(StateWork) {
+ STATEFN(StateDownloadPermissions) {
switch (ev->GetTypeRewrite()) {
- hFunc(TEvExternalStorage::TEvHeadObjectResponse, Handle);
- hFunc(TEvExternalStorage::TEvGetObjectResponse, Handle);
+ hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandlePermissions);
+ hFunc(TEvExternalStorage::TEvGetObjectResponse, HandlePermissions);
- sFunc(TEvents::TEvWakeup, Bootstrap);
+ sFunc(TEvents::TEvWakeup, DownloadPermissions);
sFunc(TEvents::TEvPoisonPill, PassAway);
}
}
@@ -161,6 +252,7 @@ private:
const ui32 ItemIdx;
const TString SchemeKey;
+ const TString PermissionsKey;
const ui32 Retries;
ui32 Attempt = 0;
@@ -168,6 +260,8 @@ private:
TDuration Delay = TDuration::Minutes(1);
static constexpr TDuration MaxDelay = TDuration::Minutes(10);
+ const bool NeedDownloadPermissions = true;
+
TActorId Client;
}; // TSchemeGetter
diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h
index d4281b7003..0eb7b65196 100644
--- a/ydb/core/tx/schemeshard/schemeshard_info_types.h
+++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h
@@ -2769,6 +2769,7 @@ struct TImportInfo: public TSimpleRefCount<TImportInfo> {
TString DstPathName;
TPathId DstPathId;
Ydb::Table::CreateTableRequest Scheme;
+ TMaybeFail<Ydb::Scheme::ModifyPermissionsRequest> Permissions;
EState State = EState::GetScheme;
ESubState SubState = ESubState::AllocateTxId;
diff --git a/ydb/core/tx/schemeshard/schemeshard_schema.h b/ydb/core/tx/schemeshard/schemeshard_schema.h
index 7144a194b9..17750e6c3d 100644
--- a/ydb/core/tx/schemeshard/schemeshard_schema.h
+++ b/ydb/core/tx/schemeshard/schemeshard_schema.h
@@ -1520,6 +1520,7 @@ struct Schema : NIceDb::Schema {
struct DstPathOwnerId : Column<4, NScheme::NTypeIds::Uint64> { using Type = TOwnerId; };
struct DstPathLocalId : Column<5, NScheme::NTypeIds::Uint64> { using Type = TLocalPathId; };
struct Scheme : Column<6, NScheme::NTypeIds::String> {};
+ struct Permissions : Column<11, NScheme::NTypeIds::String> {};
struct State : Column<7, NScheme::NTypeIds::Byte> {};
struct WaitTxId : Column<8, NScheme::NTypeIds::Uint64> { using Type = TTxId; };
@@ -1534,6 +1535,7 @@ struct Schema : NIceDb::Schema {
DstPathOwnerId,
DstPathLocalId,
Scheme,
+ Permissions,
State,
WaitTxId,
NextIndexIdx,
diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp
index 04bee847da..934a6e7e00 100644
--- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp
+++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp
@@ -1207,9 +1207,9 @@ TCheckFunc HasOwner(const TString& owner) {
};
}
-void CheckEffectiveRight(const NKikimrScheme::TEvDescribeSchemeResult& record, const TString& right, bool mustHave) {
+void CheckRight(const NKikimrScheme::TEvDescribeSchemeResult& record, const TString& right, bool mustHave, bool isEffective) {
const auto& self = record.GetPathDescription().GetSelf();
- TSecurityObject src(self.GetOwner(), self.GetEffectiveACL(), false);
+ TSecurityObject src(self.GetOwner(), isEffective ? self.GetEffectiveACL() : self.GetACL(), false);
NACLib::TSecurityObject required;
required.FromString(right);
@@ -1233,6 +1233,22 @@ void CheckEffectiveRight(const NKikimrScheme::TEvDescribeSchemeResult& record, c
}
}
+TCheckFunc HasRight(const TString& right) {
+ return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) {
+ CheckRight(record, right, true, true);
+ };
+}
+
+TCheckFunc HasNotRight(const TString& right) {
+ return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) {
+ CheckRight(record, right, false, true);
+ };
+}
+
+void CheckEffectiveRight(const NKikimrScheme::TEvDescribeSchemeResult& record, const TString& right, bool mustHave) {
+ CheckRight(record, right, mustHave, true);
+}
+
TCheckFunc HasEffectiveRight(const TString& right) {
return [=] (const NKikimrScheme::TEvDescribeSchemeResult& record) {
CheckEffectiveRight(record, right, true);
diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h
index 2720b34718..a28b0908ce 100644
--- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h
+++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.h
@@ -168,6 +168,8 @@ namespace NLs {
TCheckFunc BackupHistoryCount(ui64 count);
TCheckFunc HasOwner(const TString& owner);
+ TCheckFunc HasRight(const TString& right);
+ TCheckFunc HasNotRight(const TString& right);
TCheckFunc HasEffectiveRight(const TString& right);
TCheckFunc HasNotEffectiveRight(const TString& right);
diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp
index 2b558341b0..ee1de957a1 100644
--- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp
+++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp
@@ -138,6 +138,7 @@ namespace {
struct TTestDataWithScheme {
TString Scheme;
+ TString Permissions;
TVector<TTestData> Data;
TTestDataWithScheme() = default;
@@ -240,9 +241,14 @@ namespace {
}
}
- TTestDataWithScheme GenerateTestData(const TString& scheme, const TVector<std::pair<TString, ui64>>& shardsConfig) {
+ TTestDataWithScheme GenerateTestData(
+ const TString& scheme,
+ const TVector<std::pair<TString, ui64>>& shardsConfig,
+ const TString& permissions = "")
+ {
TTestDataWithScheme result;
result.Scheme = scheme;
+ result.Permissions = permissions;
for (const auto& [keyPrefix, count] : shardsConfig) {
result.Data.push_back(GenerateTestData(keyPrefix, count));
@@ -256,6 +262,9 @@ namespace {
for (const auto& [prefix, item] : data) {
result.emplace(prefix + "/scheme.pb", item.Scheme);
+ if (item.Permissions) {
+ result.emplace(prefix + "/permissions.pb", item.Permissions);
+ }
for (ui32 i = 0; i < item.Data.size(); ++i) {
const auto& data = item.Data.at(i);
result.emplace(Sprintf("%s/data_%02d%s", prefix.data(), i, data.Ext().c_str()), data.Data);
@@ -3753,6 +3762,240 @@ Y_UNIT_TEST_SUITE(TImportTests) {
UNIT_ASSERT_VALUES_EQUAL(entry.GetProgress(), Ydb::Import::ImportProgress::PROGRESS_PREPARING);
UNIT_ASSERT_VALUES_EQUAL(entry.GetUserSID(), userSID);
}
+
+ Y_UNIT_TEST(TablePermissions) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 100;
+
+ const auto permissions = R"(
+ actions {
+ change_owner: "eve"
+ }
+ actions {
+ grant {
+ subject: "alice"
+ permission_names: "ydb.generic.read"
+ }
+ }
+ actions {
+ grant {
+ subject: "alice"
+ permission_names: "ydb.generic.write"
+ }
+ }
+ actions {
+ grant {
+ subject: "bob"
+ permission_names: "ydb.generic.read"
+ }
+ }
+ )";
+
+ const auto data = GenerateTestData(R"(
+ columns {
+ name: "key"
+ type { optional_type { item { type_id: UTF8 } } }
+ }
+ columns {
+ name: "value"
+ type { optional_type { item { type_id: UTF8 } } }
+ }
+ primary_key: "key"
+ )", {{"a", 1}}, permissions);
+
+ TPortManager portManager;
+ const ui16 port = portManager.GetPort();
+
+ TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port));
+ UNIT_ASSERT(s3Mock.Start());
+
+ TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"(
+ ImportFromS3Settings {
+ endpoint: "localhost:%d"
+ scheme: HTTP
+ items {
+ source_prefix: ""
+ destination_path: "/MyRoot/Table"
+ }
+ }
+ )", port));
+ env.TestWaitNotification(runtime, txId);
+
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {
+ NLs::PathExist,
+ NLs::HasOwner("eve"),
+ NLs::HasRight("+R:alice"),
+ NLs::HasRight("+W:alice"),
+ NLs::HasRight("+R:bob")
+ });
+ }
+
+ Y_UNIT_TEST(UnexpectedPermission) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 100;
+
+ const auto permissions = R"(
+ actions {
+ change_owner: "eve"
+ }
+ actions {
+ grant {
+ subject: "alice"
+ permission_names: "ydb.unexpected.permission"
+ }
+ }
+ )";
+
+ const auto data = GenerateTestData(R"(
+ columns {
+ name: "key"
+ type { optional_type { item { type_id: UTF8 } } }
+ }
+ columns {
+ name: "value"
+ type { optional_type { item { type_id: UTF8 } } }
+ }
+ primary_key: "key"
+ )", {{"a", 1}}, permissions);
+
+ TPortManager portManager;
+ const ui16 port = portManager.GetPort();
+
+ TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port));
+ UNIT_ASSERT(s3Mock.Start());
+
+ TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"(
+ ImportFromS3Settings {
+ endpoint: "localhost:%d"
+ scheme: HTTP
+ items {
+ source_prefix: ""
+ destination_path: "/MyRoot/Table"
+ }
+ }
+ )", port));
+ env.TestWaitNotification(runtime, txId);
+
+ auto desc = TestGetImport(runtime, txId, "/MyRoot", Ydb::StatusIds::CANCELLED);
+ auto entry = desc.GetResponse().GetEntry();
+ UNIT_ASSERT_VALUES_EQUAL(entry.GetProgress(), Ydb::Import::ImportProgress::PROGRESS_CANCELLED);
+ }
+
+ Y_UNIT_TEST(CorruptedPermissions) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 100;
+
+ const auto permissions = R"(
+ corrupted
+ )";
+
+ const auto data = GenerateTestData(R"(
+ columns {
+ name: "key"
+ type { optional_type { item { type_id: UTF8 } } }
+ }
+ columns {
+ name: "value"
+ type { optional_type { item { type_id: UTF8 } } }
+ }
+ primary_key: "key"
+ )", {{"a", 1}}, permissions);
+
+ TPortManager portManager;
+ const ui16 port = portManager.GetPort();
+
+ TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port));
+ UNIT_ASSERT(s3Mock.Start());
+
+ TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"(
+ ImportFromS3Settings {
+ endpoint: "localhost:%d"
+ scheme: HTTP
+ items {
+ source_prefix: ""
+ destination_path: "/MyRoot/Table"
+ }
+ }
+ )", port));
+ env.TestWaitNotification(runtime, txId);
+
+ auto desc = TestGetImport(runtime, txId, "/MyRoot", Ydb::StatusIds::CANCELLED);
+ auto entry = desc.GetResponse().GetEntry();
+ UNIT_ASSERT_VALUES_EQUAL(entry.GetProgress(), Ydb::Import::ImportProgress::PROGRESS_CANCELLED);
+ }
+
+ Y_UNIT_TEST(NoACLOption) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 100;
+
+ const auto permissions = R"(
+ actions {
+ change_owner: "eve"
+ }
+ actions {
+ grant {
+ subject: "alice"
+ permission_names: "ydb.generic.read"
+ }
+ }
+ actions {
+ grant {
+ subject: "alice"
+ permission_names: "ydb.generic.write"
+ }
+ }
+ actions {
+ grant {
+ subject: "bob"
+ permission_names: "ydb.generic.read"
+ }
+ }
+ )";
+
+ const auto data = GenerateTestData(R"(
+ columns {
+ name: "key"
+ type { optional_type { item { type_id: UTF8 } } }
+ }
+ columns {
+ name: "value"
+ type { optional_type { item { type_id: UTF8 } } }
+ }
+ primary_key: "key"
+ )", {{"a", 1}}, permissions);
+
+ TPortManager portManager;
+ const ui16 port = portManager.GetPort();
+
+ TS3Mock s3Mock(ConvertTestData(data), TS3Mock::TSettings(port));
+ UNIT_ASSERT(s3Mock.Start());
+
+ const TString userSID = "user@builtin";
+ TestImport(runtime, ++txId, "/MyRoot", Sprintf(R"(
+ ImportFromS3Settings {
+ endpoint: "localhost:%d"
+ scheme: HTTP
+ items {
+ source_prefix: ""
+ destination_path: "/MyRoot/Table"
+ }
+ no_acl: true
+ }
+ )", port), userSID);
+ env.TestWaitNotification(runtime, txId);
+
+ TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {
+ NLs::PathExist,
+ NLs::HasOwner(userSID),
+ NLs::HasNotRight("+R:alice"),
+ NLs::HasNotRight("+W:alice"),
+ NLs::HasNotRight("+R:bob")
+ });
+ }
}
Y_UNIT_TEST_SUITE(TImportWithRebootsTests) {
diff --git a/ydb/core/ydb_convert/ydb_convert.cpp b/ydb/core/ydb_convert/ydb_convert.cpp
index 20f04c1e5b..d2d84d2d26 100644
--- a/ydb/core/ydb_convert/ydb_convert.cpp
+++ b/ydb/core/ydb_convert/ydb_convert.cpp
@@ -20,6 +20,28 @@
namespace NKikimr {
+namespace {
+
+ bool FillAllowPermissions(NACLib::TDiffACL& out, const Ydb::Scheme::Permissions& in, TString& error) {
+ for (const auto& permission : in.permission_names()) {
+ try {
+ auto aclAttrs = ConvertYdbPermissionNameToACLAttrs(permission);
+ out.AddAccess(
+ NACLib::EAccessType::Allow,
+ aclAttrs.AccessMask,
+ in.subject(),
+ aclAttrs.InheritanceType
+ );
+ } catch (const std::exception& e) {
+ error = e.what();
+ return false;
+ }
+ }
+ return true;
+ }
+
+} // anonymous namespace
+
template<typename TOut>
Y_FORCE_INLINE void ConvertMiniKQLTupleTypeToYdbType(const NKikimrMiniKQL::TTupleType& protoTupleType, TOut& output) {
const ui32 elementsCount = static_cast<ui32>(protoTupleType.ElementSize());
@@ -1410,4 +1432,35 @@ void ProtoValueFromCell(NYdb::TValueBuilder& vb, const NScheme::TTypeInfo& typeI
}
}
+bool FillACL(NKikimrSchemeOp::TModifyScheme& out,
+ const TMaybeFail<Ydb::Scheme::ModifyPermissionsRequest>& in,
+ TString& error) {
+ if (in.Empty()) {
+ return true;
+ }
+
+ NACLib::TDiffACL diffACL;
+ for (const auto& action : in->actions()) {
+ if (action.has_grant() && !FillAllowPermissions(diffACL, action.grant(), error)) {
+ return false;
+ }
+ }
+ out.MutableModifyACL()->SetDiffACL(diffACL.SerializeAsString());
+
+ return true;
+}
+
+void FillOwner(NKikimrScheme::TEvModifySchemeTransaction& out,
+ const TMaybeFail<Ydb::Scheme::ModifyPermissionsRequest>& in) {
+ if (in.Empty()) {
+ return;
+ }
+
+ for (const auto& action : in->actions()) {
+ if (action.has_change_owner()) {
+ out.SetOwner(action.change_owner());
+ }
+ }
+}
+
} // namespace NKikimr
diff --git a/ydb/core/ydb_convert/ydb_convert.h b/ydb/core/ydb_convert/ydb_convert.h
index ec112f1d78..7ff8ce6338 100644
--- a/ydb/core/ydb_convert/ydb_convert.h
+++ b/ydb/core/ydb_convert/ydb_convert.h
@@ -57,5 +57,11 @@ bool CellFromProtoVal(NScheme::TTypeInfo type, i32 typmod, const Ydb::Value* vp,
void ProtoValueFromCell(NYdb::TValueBuilder& vb, const NScheme::TTypeInfo& typeInfo, const TCell& cell);
+bool FillACL(NKikimrSchemeOp::TModifyScheme& out,
+ const TMaybeFail<Ydb::Scheme::ModifyPermissionsRequest>& in,
+ TString& error);
+
+void FillOwner(NKikimrScheme::TEvModifySchemeTransaction& out,
+ const TMaybeFail<Ydb::Scheme::ModifyPermissionsRequest>& in);
} // namespace NKikimr
diff --git a/ydb/public/api/protos/ydb_import.proto b/ydb/public/api/protos/ydb_import.proto
index f03f56ef26..f4cc476733 100644
--- a/ydb/public/api/protos/ydb_import.proto
+++ b/ydb/public/api/protos/ydb_import.proto
@@ -42,7 +42,8 @@ message ImportFromS3Settings {
The object name begins with 'source_prefix'.
This prefix is followed by:
* '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero;
- * '/scheme.pb' - object with information about scheme, indexes, etc.
+ * '/scheme.pb' - object with information about scheme, indexes, etc;
+ * '/permissions.pb' - object with information about ACL and owner.
*/
string source_prefix = 1 [(required) = true];
@@ -67,6 +68,10 @@ message ImportFromS3Settings {
// details: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html
// it is especially useful for custom s3 implementations
bool disable_virtual_addressing = 10;
+
+ // Prevent importing of ACL and owner. If true, objects are created with empty ACL
+ // and their owner will be the user who started the import.
+ bool no_acl = 11;
}
message ImportFromS3Result {
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp
index e93089f01b..5c4d636e61 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp
+++ b/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp
@@ -95,6 +95,9 @@ void TCommandImportFromS3::Config(TConfig& config) {
config.Opts->AddLongOption("use-virtual-addressing", "S3 bucket virtual addressing")
.RequiredArgument("BOOL").StoreResult<bool>(&UseVirtualAddressing).DefaultValue("true");
+ config.Opts->AddLongOption("no-acl", "Prevent importing of ACL and owner")
+ .RequiredArgument("BOOL").StoreTrue(&NoACL).DefaultValue("false");
+
AddDeprecatedJsonOption(config);
AddOutputFormats(config, { EDataFormat::Pretty, EDataFormat::ProtoJsonBase64 });
config.Opts->MutuallyExclusive("json", "format");
@@ -135,6 +138,7 @@ int TCommandImportFromS3::Run(TConfig& config) {
}
settings.NumberOfRetries(NumberOfRetries);
+ settings.NoACL(NoACL);
#if defined(_win32_)
for (const auto& item : Items) {
settings.AppendItem({item.Source, item.Destination});
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_import.h b/ydb/public/lib/ydb_cli/commands/ydb_service_import.h
index 1cfab9c531..321bdacf27 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_service_import.h
+++ b/ydb/public/lib/ydb_cli/commands/ydb_service_import.h
@@ -38,6 +38,7 @@ private:
TString Description;
ui32 NumberOfRetries = 10;
bool UseVirtualAddressing = true;
+ bool NoACL = true;
};
class TCommandImportFromFile : public TClientCommandTree {
diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.cpp b/ydb/public/sdk/cpp/client/ydb_import/import.cpp
index 98031b5596..47fc97e906 100644
--- a/ydb/public/sdk/cpp/client/ydb_import/import.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_import/import.cpp
@@ -153,6 +153,10 @@ TFuture<TImportFromS3Response> TImportClient::ImportFromS3(const TImportFromS3Se
request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.GetRef());
}
+ if (settings.NoACL_) {
+ request.mutable_settings()->set_no_acl(settings.NoACL_.GetRef());
+ }
+
request.mutable_settings()->set_disable_virtual_addressing(!settings.UseVirtualAddressing_);
return Impl_->ImportFromS3(std::move(request), settings);
diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.h b/ydb/public/sdk/cpp/client/ydb_import/import.h
index 865fec384c..8f479cfd90 100644
--- a/ydb/public/sdk/cpp/client/ydb_import/import.h
+++ b/ydb/public/sdk/cpp/client/ydb_import/import.h
@@ -41,6 +41,7 @@ struct TImportFromS3Settings : public TOperationRequestSettings<TImportFromS3Set
FLUENT_SETTING_VECTOR(TItem, Item);
FLUENT_SETTING_OPTIONAL(TString, Description);
FLUENT_SETTING_OPTIONAL(ui32, NumberOfRetries);
+ FLUENT_SETTING_OPTIONAL(bool, NoACL);
};
class TImportFromS3Response : public TOperation {
diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema
index 50e2aedd62..19aeda8889 100644
--- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema
+++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_schemeshard_/flat_schemeshard.schema
@@ -5839,6 +5839,11 @@
"ColumnId": 10,
"ColumnName": "Issue",
"ColumnType": "Utf8"
+ },
+ {
+ "ColumnId": 11,
+ "ColumnName": "Permissions",
+ "ColumnType": "String"
}
],
"ColumnsDropped": [],
@@ -5854,7 +5859,8 @@
7,
8,
9,
- 10
+ 10,
+ 11
],
"RoomID": 0,
"Codec": 0,