diff options
| author | Vasily Gerasimov <[email protected]> | 2025-05-21 12:46:41 +0300 |
|---|---|---|
| committer | GitHub <[email protected]> | 2025-05-21 12:46:41 +0300 |
| commit | ddb9664139d8ea8a4b7b674279a047d01869f4f0 (patch) | |
| tree | 2467c5f5e9fdaaa499e5b7807acd9a67fc8117a8 | |
| parent | 2ab1e1916633d2f5be41e19ee8d2fb3a8dd8523b (diff) | |
Implement ImportService.ListObjectsInS3Export on server side (#18454)
23 files changed, 1021 insertions, 107 deletions
diff --git a/ydb/core/backup/common/metadata.cpp b/ydb/core/backup/common/metadata.cpp index d54b8798618..99456735330 100644 --- a/ydb/core/backup/common/metadata.cpp +++ b/ydb/core/backup/common/metadata.cpp @@ -1,5 +1,7 @@ #include "metadata.h" +#include <ydb/core/base/path.h> + #include <library/cpp/json/json_writer.h> #include <library/cpp/json/json_reader.h> @@ -53,6 +55,31 @@ TMetadata TMetadata::Deserialize(const TString& metadata) { return result; } +TString TSchemaMapping::Serialize() const { + TString content; + TStringOutput ss(content); + NJson::TJsonWriter writer(&ss, false); + + writer.OpenMap(); + writer.WriteKey("exportedObjects"); + writer.OpenMap(); + for (const auto& item : Items) { + writer.WriteKey(item.ObjectPath); + writer.OpenMap(); + writer.Write("exportPrefix", item.ExportPrefix); + if (item.IV) { + writer.Write("iv", item.IV->GetHexString()); + } + writer.CloseMap(); + } + writer.CloseMap(); + writer.CloseMap(); + + writer.Flush(); + ss.Flush(); + return content; +} + bool TSchemaMapping::Deserialize(const TString& jsonContent, TString& error) { NJson::TJsonValue json; if (!NJson::ReadJsonTree(jsonContent, &json)) { @@ -97,4 +124,35 @@ bool TSchemaMapping::Deserialize(const TString& jsonContent, TString& error) { return true; } +TString NormalizeItemPath(const TString& path) { + TString result = CanonizePath(path); + if (result.size() > 1 && result.front() == '/') { + result.erase(0, 1); + } + return result; +} + +TString NormalizeItemPrefix(TString prefix) { + // Cut slshes from the beginning and from the end + size_t toCut = 0; + while (toCut < prefix.size() && prefix[toCut] == '/') { + ++toCut; + } + if (toCut) { + prefix.erase(0, toCut); + } + + while (prefix && prefix.back() == '/') { + prefix.pop_back(); + } + return prefix; +} + +TString NormalizeExportPrefix(TString prefix) { + while (prefix && prefix.back() == '/') { + prefix.pop_back(); + } + return prefix; +} + } diff --git a/ydb/core/backup/common/metadata.h b/ydb/core/backup/common/metadata.h index 7479f7ee41c..1b7f8cb9001 100644 --- a/ydb/core/backup/common/metadata.h +++ b/ydb/core/backup/common/metadata.h @@ -57,6 +57,10 @@ private: TMaybeFail<ui64> Version; }; +TString NormalizeItemPath(const TString& path); +TString NormalizeItemPrefix(TString prefix); +TString NormalizeExportPrefix(TString prefix); + class TSchemaMapping { public: struct TItem { @@ -67,6 +71,7 @@ public: TSchemaMapping() = default; + TString Serialize() const; bool Deserialize(const TString& jsonContent, TString& error); public: diff --git a/ydb/core/backup/common/metadata_ut.cpp b/ydb/core/backup/common/metadata_ut.cpp new file mode 100644 index 00000000000..4dfceb3b6e9 --- /dev/null +++ b/ydb/core/backup/common/metadata_ut.cpp @@ -0,0 +1,35 @@ +#include "metadata.h" + +#include <library/cpp/testing/unittest/registar.h> + +namespace NKikimr::NBackup { + +Y_UNIT_TEST_SUITE(PathsNormalizationTest) { + Y_UNIT_TEST(NormalizeItemPath) { + UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPath("/a/b/c/"), "a/b/c"); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPath("/"), ""); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPath(""), ""); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPath("//"), ""); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPath("///a///b///"), "a/b"); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPath("a/b/c"), "a/b/c"); + } + + Y_UNIT_TEST(NormalizeItemPrefix) { + UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPrefix("///a///b///c///"), "a///b///c"); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPrefix("a///b///c"), "a///b///c"); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPrefix("//"), ""); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPrefix("/"), ""); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPrefix(""), ""); + } + + Y_UNIT_TEST(NormalizeExportPrefix) { + UNIT_ASSERT_STRINGS_EQUAL(NormalizeExportPrefix("///a///"), "///a"); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeExportPrefix("a/"), "a"); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeExportPrefix("a"), "a"); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeExportPrefix("/prefix//"), "/prefix"); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeExportPrefix(""), ""); + UNIT_ASSERT_STRINGS_EQUAL(NormalizeExportPrefix("/prefix"), "/prefix"); + } +} + +} // namespace NKikimr::NBackup diff --git a/ydb/core/backup/common/ut/ya.make b/ydb/core/backup/common/ut/ya.make index 5978f64e27c..188e2d3c195 100644 --- a/ydb/core/backup/common/ut/ya.make +++ b/ydb/core/backup/common/ut/ya.make @@ -2,6 +2,7 @@ UNITTEST_FOR(ydb/core/backup/common) SRCS( encryption_ut.cpp + metadata_ut.cpp ) END() diff --git a/ydb/core/grpc_services/rpc_list_objects_in_s3_export.cpp b/ydb/core/grpc_services/rpc_list_objects_in_s3_export.cpp new file mode 100644 index 00000000000..7e68226272f --- /dev/null +++ b/ydb/core/grpc_services/rpc_list_objects_in_s3_export.cpp @@ -0,0 +1,203 @@ +#include "service_import.h" +#include "rpc_deferrable.h" + +#include <ydb/core/base/tablet_pipe.h> +#include <ydb/core/grpc_services/base/base.h> +#include <ydb/core/tx/scheme_cache/scheme_cache.h> +#include <ydb/core/tx/schemeshard/schemeshard_import.h> +#include <ydb/public/api/protos/ydb_import.pb.h> + +#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, "[ListObjectsInS3Export] " << SelfId() << " " << stream) +#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_PROXY, "[ListObjectsInS3Export] " << SelfId() << " " << stream) +#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_PROXY, "[ListObjectsInS3Export] " << SelfId() << " " << stream) +#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, "[ListObjectsInS3Export] " << SelfId() << " " << stream) +#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::TX_PROXY, "[ListObjectsInS3Export] " << SelfId() << " " << stream) +#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_PROXY, "[ListObjectsInS3Export] " << SelfId() << " " << stream) + +namespace NKikimr::NGRpcService { + +using TEvListObjectsInS3ExportRequest = TGrpcRequestOperationCall<Ydb::Import::ListObjectsInS3ExportRequest, + Ydb::Import::ListObjectsInS3ExportResponse>; + +class TListObjectsInS3ExportRPC: public TRpcOperationRequestActor<TListObjectsInS3ExportRPC, TEvListObjectsInS3ExportRequest> { +public: + using TBase = TRpcOperationRequestActor<TListObjectsInS3ExportRPC, TEvListObjectsInS3ExportRequest>; + using TRpcOperationRequestActor<TListObjectsInS3ExportRPC, TEvListObjectsInS3ExportRequest>::TRpcOperationRequestActor; + + explicit TListObjectsInS3ExportRPC(IRequestOpCtx* request) + : TBase(request) + , UserToken(CreateUserToken(request)) + { + } + + STATEFN(StateFunc) { + switch (ev->GetTypeRewrite()) { + hFunc(NKikimr::NSchemeShard::TEvImport::TEvListObjectsInS3ExportResponse, Handle); + hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); + + hFunc(TEvTabletPipe::TEvClientConnected, Handle); + hFunc(TEvTabletPipe::TEvClientDestroyed, Handle); + default: + return StateFuncBase(ev); + } + } + + void Bootstrap() { + if (!Request_ || !Request_->GetDatabaseName()) { + return Reply(Ydb::StatusIds::BAD_REQUEST, "Database name is not specified", NKikimrIssues::TIssuesIds::YDB_API_VALIDATION_ERROR, NActors::TActivationContext::AsActorContext()); + } + + ResolveDatabase(); + + Become(&TListObjectsInS3ExportRPC::StateFunc); + } + + void ResolveDatabase() { + LOG_D("Resolve database" + << ": name# " << Request_->GetDatabaseName()); + + auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>(); + request->DatabaseName = *Request_->GetDatabaseName(); + + auto& entry = request->ResultSet.emplace_back(); + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath; + entry.Path = NKikimr::SplitPath(*Request_->GetDatabaseName()); + + Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release())); + } + + void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { + const auto& request = ev->Get()->Request; + + LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult" + << ": request# " << (request ? request->ToString(*AppData()->TypeRegistry) : "nullptr")); + + if (request->ResultSet.empty()) { + return Reply(Ydb::StatusIds::SCHEME_ERROR, "Scheme error", NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, NActors::TActivationContext::AsActorContext()); + } + + const auto& entry = request->ResultSet.front(); + + if (request->ErrorCount > 0) { + switch (entry.Status) { + case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok: + break; + case NSchemeCache::TSchemeCacheNavigate::EStatus::AccessDenied: + return Reply(Ydb::StatusIds::UNAUTHORIZED, "Access denied", NKikimrIssues::TIssuesIds::ACCESS_DENIED, NActors::TActivationContext::AsActorContext()); + case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown: + case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown: + return Reply(Ydb::StatusIds::SCHEME_ERROR, "Unknown database", NKikimrIssues::TIssuesIds::PATH_NOT_EXIST, NActors::TActivationContext::AsActorContext()); + case NSchemeCache::TSchemeCacheNavigate::EStatus::LookupError: + case NSchemeCache::TSchemeCacheNavigate::EStatus::RedirectLookupError: + return Reply(Ydb::StatusIds::UNAVAILABLE, "Database lookup error", NKikimrIssues::TIssuesIds::RESOLVE_LOOKUP_ERROR, NActors::TActivationContext::AsActorContext()); + default: + return Reply(Ydb::StatusIds::SCHEME_ERROR, "Scheme error", NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, NActors::TActivationContext::AsActorContext()); + } + } + + if (!this->CheckDatabaseAccess(CanonizePath(entry.Path), entry.SecurityObject)) { + return; + } + + auto domainInfo = entry.DomainInfo; + if (!domainInfo) { + LOG_E("Got empty domain info"); + return Reply(Ydb::StatusIds::INTERNAL_ERROR, "Internal error", NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, NActors::TActivationContext::AsActorContext()); + } + + SchemeShardId = domainInfo->ExtractSchemeShard(); + SendRequestToSchemeShard(); + } + + bool CheckDatabaseAccess(const TString& path, TIntrusivePtr<TSecurityObject> securityObject) { + const ui32 access = NACLib::DescribeSchema; + + if (!UserToken || !securityObject) { + return true; + } + + if (securityObject->CheckAccess(access, *UserToken)) { + return true; + } + + Reply(Ydb::StatusIds::UNAUTHORIZED, + TStringBuilder() << "Access denied" + << ": for# " << UserToken->GetUserSID() + << ", path# " << path + << ", access# " << NACLib::AccessRightsToString(access), + NKikimrIssues::TIssuesIds::ACCESS_DENIED, + NActors::TActivationContext::AsActorContext()); + return false; + } + + void SendRequestToSchemeShard() { + LOG_D("Send request: schemeShardId# " << SchemeShardId); + + if (!PipeClient) { + NTabletPipe::TClientConfig config; + config.RetryPolicy = {.RetryLimitCount = 3}; + PipeClient = this->RegisterWithSameMailbox(NTabletPipe::CreateClient(this->SelfId(), SchemeShardId, config)); + } + + auto request = MakeHolder<NSchemeShard::TEvImport::TEvListObjectsInS3ExportRequest>(); + + *request->Record.MutableOperationParams() = GetProtoRequest()->operation_params(); + *request->Record.MutableSettings() = GetProtoRequest()->settings(); + request->Record.SetPageSize(GetProtoRequest()->page_size()); + request->Record.SetPageToken(GetProtoRequest()->page_token()); + + NTabletPipe::SendData(this->SelfId(), PipeClient, std::move(request), 0, Span_.GetTraceId()); + } + + void Handle(NKikimr::NSchemeShard::TEvImport::TEvListObjectsInS3ExportResponse::TPtr& ev) { + const auto& record = ev->Get()->Record; + + LOG_D("Handle TListObjectsInS3ExportRPC::TEvListObjectsInS3ExportResponse" + << ": record# " << record.ShortDebugString()); + + if (record.GetStatus() != Ydb::StatusIds::SUCCESS) { + return Reply(record.GetStatus(), record.GetIssues(), NActors::TActivationContext::AsActorContext()); + } else { + return ReplyWithResult(record.GetStatus(), record.GetIssues(), record.GetResult(), NActors::TActivationContext::AsActorContext()); + } + } + + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { + if (ev->Get()->Status != NKikimrProto::OK) { + DeliveryProblem(); + } + } + + void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr&) { + DeliveryProblem(); + } + + void DeliveryProblem() { + LOG_W("Delivery problem"); + Reply(Ydb::StatusIds::UNAVAILABLE, "Delivery problem", NKikimrIssues::TIssuesIds::DEFAULT_ERROR, NActors::TActivationContext::AsActorContext()); + } + + void PassAway() override { + NTabletPipe::CloseClient(this->SelfId(), PipeClient); + TBase::PassAway(); + } + + static THolder<const NACLib::TUserToken> CreateUserToken(IRequestOpCtx* request) { + if (const auto& userToken = request->GetSerializedToken()) { + return MakeHolder<NACLib::TUserToken>(userToken); + } else { + return {}; + } + } + +private: + ui64 SchemeShardId = 0; + TActorId PipeClient; + const THolder<const NACLib::TUserToken> UserToken; +}; + +void DoListObjectsInS3ExportRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& f) { + f.RegisterActor(new TListObjectsInS3ExportRPC(p.release())); +} + +} // namespace NKikimr::NGRpcService diff --git a/ydb/core/grpc_services/service_import.h b/ydb/core/grpc_services/service_import.h index 971e02f199a..56f89329acd 100644 --- a/ydb/core/grpc_services/service_import.h +++ b/ydb/core/grpc_services/service_import.h @@ -9,6 +9,7 @@ class IRequestOpCtx; class IFacilityProvider; void DoImportFromS3Request(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& f); +void DoListObjectsInS3ExportRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& f); void DoImportDataRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& f); } diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make index 3e703900201..80ae3a0bfde 100644 --- a/ydb/core/grpc_services/ya.make +++ b/ydb/core/grpc_services/ya.make @@ -60,6 +60,7 @@ SRCS( rpc_kh_describe.cpp rpc_kh_snapshots.cpp rpc_kqp_base.cpp + rpc_list_objects_in_s3_export.cpp rpc_list_operations.cpp rpc_load_rows.cpp rpc_log_store.cpp diff --git a/ydb/core/protos/import.proto b/ydb/core/protos/import.proto index 78ce80452d3..5f4ed3ba48e 100644 --- a/ydb/core/protos/import.proto +++ b/ydb/core/protos/import.proto @@ -129,3 +129,16 @@ message TListImportsResponse { message TEvListImportsResponse { optional TListImportsResponse Response = 1; } + +message TEvListObjectsInS3ExportRequest { + optional Ydb.Operations.OperationParams OperationParams = 1; + optional Ydb.Import.ListObjectsInS3ExportSettings Settings = 2; + optional int64 PageSize = 3; + optional string PageToken = 4; +} + +message TEvListObjectsInS3ExportResponse { + optional Ydb.StatusIds.StatusCode Status = 1; + repeated Ydb.Issue.IssueMessage Issues = 2; + optional Ydb.Import.ListObjectsInS3ExportResult Result = 3; +} diff --git a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp index fed6310d198..8665e12f17f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp @@ -408,10 +408,7 @@ private: return true; } - TString commonDestinationPrefix = exportSettings.destination_prefix(); - if (commonDestinationPrefix.back() == '/') { - commonDestinationPrefix.pop_back(); - } + TString commonDestinationPrefix = NBackup::NormalizeExportPrefix(exportSettings.destination_prefix()); TMaybe<NBackup::TEncryptionIV> iv; if (exportSettings.has_encryption_settings()) { @@ -435,28 +432,20 @@ private: if (exportPath.StartsWith(sourcePathRoot)) { exportPath = exportPath.substr(sourcePathRoot.size() + 1); // cut all prefix + '/' } + exportPath = NBackup::NormalizeItemPath(exportPath); // Path without leading slash schemaMappingItem.SetSourcePath(exportPath); TString destinationPrefix; if (!exportItem.destination_prefix().empty()) { TString& itemPrefix = *exportItem.mutable_destination_prefix(); - if (itemPrefix[0] == '/') { - destinationPrefix = itemPrefix = itemPrefix.substr(1); - } else { - destinationPrefix = itemPrefix; - } - if (itemPrefix.back() == '/') { - itemPrefix.pop_back(); - } + destinationPrefix = itemPrefix = NBackup::NormalizeItemPrefix(itemPrefix); } else { std::stringstream itemPrefix; if (exportSettings.has_encryption_settings()) { // Anonymize object name in export itemPrefix << std::setfill('0') << std::setw(3) << std::right << itemIndex; } else { - TStringBuf exportPathBuf = exportPath; - exportPathBuf.SkipPrefix("/"); - itemPrefix << exportPathBuf; + itemPrefix << exportPath; } destinationPrefix = itemPrefix.str(); } diff --git a/ydb/core/tx/schemeshard/schemeshard_export_uploaders.cpp b/ydb/core/tx/schemeshard/schemeshard_export_uploaders.cpp index b3510f1a8e2..0a853d20dc9 100644 --- a/ydb/core/tx/schemeshard/schemeshard_export_uploaders.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_export_uploaders.cpp @@ -2,6 +2,7 @@ #include "schemeshard_export_uploaders.h" #include <ydb/core/backup/common/encryption.h> +#include <ydb/core/backup/common/metadata.h> #include <ydb/core/protos/flat_scheme_op.pb.h> #include <ydb/core/tx/datashard/export_common.h> #include <ydb/core/tx/schemeshard/schemeshard_export_helpers.h> @@ -495,34 +496,21 @@ private: } bool AddSchemaMappingJson() { - TString content; - TStringOutput ss(content); - NJson::TJsonWriter writer(&ss, false); - - writer.OpenMap(); - writer.WriteKey("exportedObjects"); - writer.OpenMap(); + NBackup::TSchemaMapping schemaMapping; for (const auto& item : ExportMetadata.GetSchemaMapping()) { - writer.WriteKey(item.GetSourcePath()); - writer.OpenMap(); - writer.Write("exportPrefix", item.GetDestinationPrefix()); - if (item.HasIV()) { - writer.Write("iv", NBackup::TEncryptionIV::FromBinaryString(item.GetIV()).GetHexString()); - } - writer.CloseMap(); + schemaMapping.Items.emplace_back(NBackup::TSchemaMapping::TItem{ + .ExportPrefix = item.GetDestinationPrefix(), + .ObjectPath = item.GetSourcePath(), + .IV = item.HasIV() ? TMaybe<NBackup::TEncryptionIV>(NBackup::TEncryptionIV::FromBinaryString(item.GetIV())) : Nothing() + }); } - writer.CloseMap(); - writer.CloseMap(); - - writer.Flush(); - ss.Flush(); TMaybe<NBackup::TEncryptionIV> iv; if (IV) { iv = NBackup::TEncryptionIV::Combine(*IV, NBackup::EBackupFileType::SchemaMapping, 0, 0); } - return AddFile("SchemaMapping/mapping.json", content, iv, Key); + return AddFile("SchemaMapping/mapping.json", schemaMapping.Serialize(), iv, Key); } void ProcessQueue() { diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 7702fb3375b..570f5353ac4 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -4991,6 +4991,7 @@ void TSchemeShard::StateWork(STFUNC_SIG) { HFuncTraced(TEvImport::TEvCancelImportRequest, Handle); HFuncTraced(TEvImport::TEvForgetImportRequest, Handle); HFuncTraced(TEvImport::TEvListImportsRequest, Handle); + HFuncTraced(TEvImport::TEvListObjectsInS3ExportRequest, Handle); HFuncTraced(TEvPrivate::TEvImportSchemeReady, Handle); HFuncTraced(TEvPrivate::TEvImportSchemaMappingReady, Handle); HFuncTraced(TEvPrivate::TEvImportSchemeQueryResult, Handle); diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 14873f05ac9..10ba657bb6b 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -1364,6 +1364,7 @@ public: void Handle(TEvImport::TEvCancelImportRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvImport::TEvForgetImportRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvImport::TEvListImportsRequest::TPtr& ev, const TActorContext& ctx); + void Handle(TEvImport::TEvListObjectsInS3ExportRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvPrivate::TEvImportSchemeReady::TPtr& ev, const TActorContext& ctx); void Handle(TEvPrivate::TEvImportSchemaMappingReady::TPtr& ev, const TActorContext& ctx); void Handle(TEvPrivate::TEvImportSchemeQueryResult::TPtr& ev, const TActorContext& ctx); diff --git a/ydb/core/tx/schemeshard/schemeshard_import.cpp b/ydb/core/tx/schemeshard/schemeshard_import.cpp index 6bad76e26e8..4d6d2838552 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import.cpp @@ -1,6 +1,7 @@ #include "schemeshard_import.h" #include "schemeshard_import_helpers.h" #include "schemeshard_impl.h" +#include "schemeshard_import_getters.h" #include <util/generic/xrange.h> @@ -262,6 +263,10 @@ void TSchemeShard::Handle(TEvImport::TEvListImportsRequest::TPtr& ev, const TAct Execute(CreateTxListImports(ev), ctx); } +void TSchemeShard::Handle(TEvImport::TEvListObjectsInS3ExportRequest::TPtr& ev, const TActorContext&) { + Register(CreateListObjectsInS3ExportGetter(std::move(ev))); +} + void TSchemeShard::Handle(TEvPrivate::TEvImportSchemeReady::TPtr& ev, const TActorContext& ctx) { Execute(CreateTxProgressImport(ev), ctx); } diff --git a/ydb/core/tx/schemeshard/schemeshard_import.h b/ydb/core/tx/schemeshard/schemeshard_import.h index 41e45e57689..0cc005a8755 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import.h +++ b/ydb/core/tx/schemeshard/schemeshard_import.h @@ -19,6 +19,8 @@ struct TEvImport { EvForgetImportResponse, EvListImportsRequest, EvListImportsResponse, + EvListObjectsInS3ExportRequest, + EvListObjectsInS3ExportResponse, EvEnd }; @@ -163,6 +165,12 @@ struct TEvImport { DECLARE_EVENT_CLASS(EvListImportsResponse) { }; + DECLARE_EVENT_CLASS(EvListObjectsInS3ExportRequest) { + }; + + DECLARE_EVENT_CLASS(EvListObjectsInS3ExportResponse) { + }; + #undef DECLARE_EVENT_CLASS }; // TEvImport diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp index 88fcac80971..df2c1a83680 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp @@ -267,8 +267,8 @@ private: importInfo.Items.reserve(settings.items().size()); for (ui32 itemIdx : xrange(settings.items().size())) { - const auto& dstPath = settings.items(itemIdx).destination_path(); - if (!dstPaths.insert(dstPath).second) { + const TString& dstPath = settings.items(itemIdx).destination_path(); + if (!dstPaths.insert(NBackup::NormalizeItemPath(dstPath)).second) { explain = TStringBuilder() << "Duplicate destination_path: " << dstPath; return false; } @@ -278,8 +278,8 @@ private: } auto& item = importInfo.Items.emplace_back(dstPath); - item.SrcPrefix = settings.items(itemIdx).source_prefix(); - item.SrcPath = settings.items(itemIdx).source_path(); + item.SrcPrefix = NBackup::NormalizeExportPrefix(settings.items(itemIdx).source_prefix()); + item.SrcPath = NBackup::NormalizeItemPath(settings.items(itemIdx).source_path()); } return true; @@ -1079,8 +1079,8 @@ private: } else { dstRoot = CanonizePath(importInfo->Settings.destination_path()); } - TString sourcePrefix = importInfo->Settings.source_prefix(); - if (sourcePrefix && sourcePrefix.back() != '/') { + TString sourcePrefix = NBackup::NormalizeExportPrefix(importInfo->Settings.source_prefix()); + if (sourcePrefix) { sourcePrefix.push_back('/'); } auto combineDstPath = [&](const TString& path) -> TString { @@ -1092,9 +1092,7 @@ private: } }; auto init = [&](const NBackup::TSchemaMapping::TItem& schemaMappingItem, NSchemeShard::TImportInfo::TItem& item) { - TStringBuf exportPrefix(schemaMappingItem.ExportPrefix); - exportPrefix.SkipPrefix("/"); - item.SrcPrefix = TStringBuilder() << sourcePrefix << exportPrefix; + item.SrcPrefix = TStringBuilder() << sourcePrefix << NBackup::NormalizeItemPrefix(schemaMappingItem.ExportPrefix); item.SrcPath = schemaMappingItem.ObjectPath; item.ExportItemIV = schemaMappingItem.IV; }; @@ -1115,19 +1113,18 @@ private: TMapping schemaMappingObjectPathIndex; for (size_t i = 0; i < importInfo->SchemaMapping->Items.size(); ++i) { const auto& schemaMappingItem = importInfo->SchemaMapping->Items[i]; - schemaMappingPrefixIndex[schemaMappingItem.ExportPrefix] = i; - schemaMappingObjectPathIndex[CanonizePath(schemaMappingItem.ObjectPath)] = i; + schemaMappingPrefixIndex[NBackup::NormalizeItemPrefix(schemaMappingItem.ExportPrefix)] = i; + schemaMappingObjectPathIndex[NBackup::NormalizeItemPath(schemaMappingItem.ObjectPath)] = i; } for (auto& item : importInfo->Items) { - TString dstPath = CanonizePath(item.DstPathName); TMapping::iterator mappingIt; if (item.SrcPrefix) { - mappingIt = schemaMappingPrefixIndex.find(item.SrcPrefix); + mappingIt = schemaMappingPrefixIndex.find(NBackup::NormalizeItemPrefix(item.SrcPrefix)); if (mappingIt == schemaMappingPrefixIndex.end()) { return CancelAndPersist(db, importInfo, -1, {}, TStringBuilder() << "cannot find prefix \"" << item.SrcPrefix << "\" in schema mapping"); } } else if (item.SrcPath) { - mappingIt = schemaMappingObjectPathIndex.find(CanonizePath(item.SrcPath)); + mappingIt = schemaMappingObjectPathIndex.find(NBackup::NormalizeItemPath(item.SrcPath)); if (mappingIt == schemaMappingObjectPathIndex.end()) { return CancelAndPersist(db, importInfo, -1, {}, TStringBuilder() << "cannot find source path \"" << item.SrcPath << "\" in schema mapping"); } diff --git a/ydb/core/tx/schemeshard/schemeshard_import_getters.cpp b/ydb/core/tx/schemeshard/schemeshard_import_getters.cpp index 595aab2ee07..f6a89453f80 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_getters.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_import_getters.cpp @@ -19,6 +19,8 @@ #include <util/string/subst.h> +#include <algorithm> + namespace NKikimr { namespace NSchemeShard { @@ -31,18 +33,43 @@ using namespace Aws; static constexpr TDuration MaxDelay = TDuration::Minutes(10); +struct TGetterSettings { + NWrappers::IExternalStorageConfig::TPtr ExternalStorageConfig; + ui32 Retries; + TMaybe<NBackup::TEncryptionKey> Key; + TMaybe<NBackup::TEncryptionIV> IV; + + static TGetterSettings FromImportInfo(const TImportInfo::TPtr& importInfo, TMaybe<NBackup::TEncryptionIV> iv) { + TGetterSettings settings; + settings.ExternalStorageConfig.reset(new NWrappers::NExternalStorage::TS3ExternalStorageConfig(importInfo->Settings)); + settings.Retries = importInfo->Settings.number_of_retries(); + if (importInfo->Settings.has_encryption_settings()) { + settings.Key = NBackup::TEncryptionKey(importInfo->Settings.encryption_settings().symmetric_key().key()); + } + settings.IV = std::move(iv); + return settings; + } + + static TGetterSettings FromRequest(const TEvImport::TEvListObjectsInS3ExportRequest::TPtr& ev) { + TGetterSettings settings; + settings.ExternalStorageConfig.reset(new NWrappers::NExternalStorage::TS3ExternalStorageConfig(ev->Get()->Record.settings())); + settings.Retries = ev->Get()->Record.settings().number_of_retries(); + if (ev->Get()->Record.settings().has_encryption_settings()) { + settings.Key = NBackup::TEncryptionKey(ev->Get()->Record.settings().encryption_settings().symmetric_key().key()); + } + return settings; + } +}; + template <class TDerived> class TGetterFromS3 : public TActorBootstrapped<TDerived> { protected: - explicit TGetterFromS3(TImportInfo::TPtr importInfo, TMaybe<NBackup::TEncryptionIV> iv) - : ImportInfo(std::move(importInfo)) - , ExternalStorageConfig(new NWrappers::NExternalStorage::TS3ExternalStorageConfig(ImportInfo->Settings)) - , IV(std::move(iv)) - , Retries(ImportInfo->Settings.number_of_retries()) + explicit TGetterFromS3(TGetterSettings&& settings) + : ExternalStorageConfig(std::move(settings.ExternalStorageConfig)) + , Key(std::move(settings.Key)) + , IV(std::move(settings.IV)) + , Retries(settings.Retries) { - if (ImportInfo->Settings.has_encryption_settings()) { - Key = NBackup::TEncryptionKey(ImportInfo->Settings.encryption_settings().symmetric_key().key()); - } } void HeadObject(const TString& key, bool autoAddEncSuffix = true) { @@ -89,7 +116,7 @@ protected: } TString GetKey(TString key, bool autoAddEncSuffix = true) { - if (autoAddEncSuffix && ImportInfo->Settings.has_encryption_settings()) { + if (autoAddEncSuffix && Key) { key += ".enc"; } return key; @@ -114,10 +141,28 @@ protected: Delay = Min(Delay * ++Attempt, MaxDelay); this->Schedule(Delay, new TEvents::TEvWakeup()); } else { - Reply(false, TStringBuilder() << "S3 error: " << error.GetMessage().c_str()); + Reply(error.ShouldRetry() ? Ydb::StatusIds::EXTERNAL_ERROR : Ydb::StatusIds::BAD_REQUEST, TStringBuilder() << "S3 error: " << error.GetMessage().c_str()); } } + template <typename TResult> + bool IsNoSuchKeyError(const TResult& result) { + if (result.IsSuccess()) { + return false; + } + const auto& err = result.GetError(); + if (err.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) { + return true; + } + if (err.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND) { + return true; + } + if (err.GetExceptionName() == "NoSuchKey") { + return true; + } + return false; + } + void ResetRetries() { Attempt = 0; } @@ -132,7 +177,7 @@ protected: result.assign(buffer.Data(), buffer.Size()); return true; } catch (const std::exception& ex) { - Reply(false, ex.what()); + Reply(Ydb::StatusIds::BAD_REQUEST, ex.what()); return false; } } @@ -148,7 +193,7 @@ protected: result.assign(buffer.Data(), buffer.Size()); return true; } catch (const std::exception& ex) { - Reply(false, ex.what()); + Reply(Ydb::StatusIds::BAD_REQUEST, ex.what()); return false; } } @@ -156,10 +201,9 @@ protected: return true; } - virtual void Reply(bool success = true, const TString& error = TString()) = 0; + virtual void Reply(Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS, const TString& error = TString()) = 0; protected: - TImportInfo::TPtr ImportInfo; NWrappers::IExternalStorageConfig::TPtr ExternalStorageConfig; TActorId Client; TMaybe<NBackup::TEncryptionKey> Key; @@ -388,11 +432,11 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> { } else if (IsTopic(SchemeKey)) { Ydb::Topic::CreateTopicRequest request; if (!google::protobuf::TextFormat::ParseFromString(content, &request)) { - return Reply(false, "Cannot parse topic scheme"); + return Reply(Ydb::StatusIds::BAD_REQUEST, "Cannot parse topic scheme"); } item.Topic = request; } else if (!google::protobuf::TextFormat::ParseFromString(content, &item.Scheme)) { - return Reply(false, "Cannot parse scheme"); + return Reply(Ydb::StatusIds::BAD_REQUEST, "Cannot parse scheme"); } auto nextStep = [this]() { @@ -436,7 +480,7 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> { Ydb::Scheme::ModifyPermissionsRequest permissions; if (!google::protobuf::TextFormat::ParseFromString(content, &permissions)) { - return Reply(false, "Cannot parse permissions"); + return Reply(Ydb::StatusIds::BAD_REQUEST, "Cannot parse permissions"); } item.Permissions = std::move(permissions); @@ -465,7 +509,7 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> { TString expectedChecksum = msg.Body.substr(0, msg.Body.find(' ')); if (expectedChecksum != CurrentObjectChecksum) { - return Reply(false, TStringBuilder() << "Checksum mismatch for " << CurrentObjectKey + return Reply(Ydb::StatusIds::BAD_REQUEST, TStringBuilder() << "Checksum mismatch for " << CurrentObjectKey << " expected# " << expectedChecksum << ", got# " << CurrentObjectChecksum); } @@ -499,7 +543,7 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> { Ydb::Table::ChangefeedDescription changefeed; if (!google::protobuf::TextFormat::ParseFromString(content, &changefeed)) { - return Reply(false, "Cannot parse сhangefeed"); + return Reply(Ydb::StatusIds::BAD_REQUEST, "Cannot parse changefeed"); } *item.Changefeeds.MutableChangefeeds(IndexDownloadedChangefeed)->MutableChangefeed() = std::move(changefeed); @@ -542,7 +586,7 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> { Ydb::Topic::DescribeTopicResult topic; if (!google::protobuf::TextFormat::ParseFromString(content, &topic)) { - return Reply(false, "Cannot parse topic"); + return Reply(Ydb::StatusIds::BAD_REQUEST, "Cannot parse topic"); } *item.Changefeeds.MutableChangefeeds(IndexDownloadedChangefeed)->MutableTopic() = std::move(topic); @@ -597,10 +641,10 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> { } else { Reply(); } - } - void Reply(bool success = true, const TString& error = TString()) override { + void Reply(Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS, const TString& error = TString()) override { + const bool success = (statusCode == Ydb::StatusIds::SUCCESS); LOG_I("Reply" << ": self# " << SelfId() << ", success# " << success @@ -665,7 +709,8 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> { public: explicit TSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx, TMaybe<NBackup::TEncryptionIV> iv) - : TGetterFromS3<TSchemeGetter>(std::move(importInfo), std::move(iv)) + : TGetterFromS3<TSchemeGetter>(TGetterSettings::FromImportInfo(importInfo, std::move(iv))) + , ImportInfo(std::move(importInfo)) , ReplyTo(replyTo) , ItemIdx(itemIdx) , MetadataKey(MetadataKeyFromSettings(*ImportInfo, itemIdx)) @@ -743,6 +788,7 @@ public: } private: + TImportInfo::TPtr ImportInfo; const TActorId ReplyTo; const ui32 ItemIdx; @@ -856,14 +902,15 @@ class TSchemaMappingGetter : public TGetterFromS3<TSchemaMappingGetter> { ImportInfo->SchemaMapping.ConstructInPlace(); TString error; if (!ImportInfo->SchemaMapping->Deserialize(content, error)) { - Reply(false, error); + Reply(Ydb::StatusIds::BAD_REQUEST, error); return; } Reply(); } - void Reply(bool success = true, const TString& error = TString()) override { + void Reply(Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS, const TString& error = TString()) override { + const bool success = (statusCode == Ydb::StatusIds::SUCCESS); LOG_I("Reply" << ": self# " << SelfId() << ", success# " << success @@ -890,12 +937,12 @@ class TSchemaMappingGetter : public TGetterFromS3<TSchemaMappingGetter> { bool ProcessMetadata(const TString& content) { NJson::TJsonValue json; if (!NJson::ReadJsonTree(content, &json)) { - Reply(false, "Failed to parse metadata json"); + Reply(Ydb::StatusIds::BAD_REQUEST, "Failed to parse metadata json"); return false; } const NJson::TJsonValue& kind = json["kind"]; if (kind.GetString() != "SchemaMappingV0") { - Reply(false, TStringBuilder() << "Unknown kind of metadata json: " << kind.GetString()); + Reply(Ydb::StatusIds::BAD_REQUEST, TStringBuilder() << "Unknown kind of metadata json: " << kind.GetString()); return false; } return true; @@ -903,7 +950,8 @@ class TSchemaMappingGetter : public TGetterFromS3<TSchemaMappingGetter> { public: TSchemaMappingGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo) - : TGetterFromS3<TSchemaMappingGetter>(std::move(importInfo), Nothing()) + : TGetterFromS3<TSchemaMappingGetter>(TGetterSettings::FromImportInfo(importInfo, Nothing())) + , ImportInfo(std::move(importInfo)) , ReplyTo(replyTo) , MetadataKey(SchemaMappingMetadataKeyFromSettings(*ImportInfo)) , SchemaMappingKey(SchemaMappingKeyFromSettings(*ImportInfo)) @@ -936,11 +984,276 @@ public: } private: + TImportInfo::TPtr ImportInfo; const TActorId ReplyTo; const TString MetadataKey; const TString SchemaMappingKey; }; // TSchemaMappingGetter +class TListObjectsInS3ExportGetter : public TGetterFromS3<TListObjectsInS3ExportGetter> { + class TPathFilter { + public: + void Build(const Ydb::Import::ListObjectsInS3ExportSettings& settings) { + for (const auto& item : settings.items()) { + TString path = NBackup::NormalizeItemPath(item.path()); + if (path) { + Paths.emplace(path); + PathPrefixes.emplace_back(path + "/"); + } + } + } + + bool Match(const TString& path) const { + if (Paths.empty()) { + return true; + } + + if (Paths.contains(path)) { + return true; + } + + for (const TString& prefix : PathPrefixes) { + if (path.StartsWith(prefix)) { // So this path is contained in a directory that is specified by user in request + return true; + } + } + + return false; + } + + private: + std::vector<TString> PathPrefixes; + THashSet<TString> Paths; + }; +public: + TListObjectsInS3ExportGetter(TEvImport::TEvListObjectsInS3ExportRequest::TPtr&& ev) + : TGetterFromS3<TListObjectsInS3ExportGetter>(TGetterSettings::FromRequest(ev)) + , Request(std::move(ev)) + { + } + + void Bootstrap() { + if (ParseParameters()) { + CreateClient(); + DownloadSchemaMapping(); + } + } + + bool ParseParameters() { + const auto& req = Request->Get()->Record; + if (req.GetPageSize() < 0) { + Reply(Ydb::StatusIds::BAD_REQUEST, "Page size should be greater than or equal to 0"); + return false; + } + PageSize = static_cast<size_t>(req.GetPageSize()); + if (req.GetPageToken()) { + if (!TryFromString(req.GetPageToken(), StartPos)) { + Reply(Ydb::StatusIds::BAD_REQUEST, "Failed to parse page token"); + return false; + } + if (req.GetPageSize() == 0) { + Reply(Ydb::StatusIds::BAD_REQUEST, "Page size should be greater than 0"); + return false; + } + } + if (NBackup::NormalizeExportPrefix(Request->Get()->Record.GetSettings().prefix()).empty()) { + Reply(Ydb::StatusIds::BAD_REQUEST, "Empty S3 prefix specified"); + return false; + } + return true; + } + + void DownloadSchemaMapping() { + ResetRetries(); + Download(GetSchemaMappingKey()); + Become(&TThis::StateDownloadSchemaMapping); + } + + void HandleSchemaMapping(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { + const auto& result = ev->Get()->Result; + + LOG_D("HandleSchemaMapping TEvExternalStorage::TEvHeadObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + if (IsNoSuchKeyError(result)) { + return ListObjectsInS3Prefix(); + } + + if (!CheckResult(result, "HeadObject")) { + return; + } + + GetObject(GetSchemaMappingKey(), result.GetResult().GetContentLength()); + } + + void HandleSchemaMapping(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) { + const auto& msg = *ev->Get(); + const auto& result = msg.Result; + + LOG_D("HandleSchemaMapping TEvExternalStorage::TEvGetObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + if (IsNoSuchKeyError(result)) { + return ListObjectsInS3Prefix(); + } + + if (!CheckResult(result, "GetObject")) { + return; + } + + TString content; + if (!MaybeDecryptAndSaveIV(msg.Body, content)) { + return; + } + + LOG_T("Trying to parse schema mapping" + << ": self# " << SelfId() + << ", schemaMappingKey# " << GetSchemaMappingKey() + << ", body# " << SubstGlobalCopy(content, "\n", "\\n")); + + TString error; + NBackup::TSchemaMapping schemaMapping; + if (!schemaMapping.Deserialize(content, error)) { + Reply(Ydb::StatusIds::BAD_REQUEST, error); + return; + } + + ProcessItemsAndReply(std::move(schemaMapping.Items)); + } + + void ListObjectsInS3Prefix() { + ResetRetries(); + ListObjects(GetExportPrefix()); + Become(&TThis::StateListObjects); + } + + void HandleListObjects(TEvExternalStorage::TEvListObjectsResponse::TPtr& ev) { + const auto& result = ev.Get()->Get()->Result; + LOG_D("HandleListObjects TEvExternalStorage::TEvListObjectResponse" + << ": self# " << SelfId() + << ", result# " << result); + + if (!CheckResult(result, "ListObjects")) { + return; + } + + const auto& objects = result.GetResult().GetContents(); + const TString prefix = GetExportPrefix(); + const TString suffix = TString("/metadata.json") + (Key ? ".enc" : ""); + std::vector<NBackup::TSchemaMapping::TItem> items; + items.reserve(objects.size()); + for (const auto& obj : objects) { + TStringBuf key(obj.GetKey()); + // Skip prefix + // Prefix also may be added with the bucket name here, so cut bucket name also + size_t prefixPos = key.find(prefix); + if (prefixPos == TStringBuf::npos) { + LOG_D("Unexpected key found: " << key); + continue; + } + key = key.SubString(prefixPos + prefix.size(), TStringBuf::npos); + + // Every backup object has metadata.json. + // Process only keys with metadata.json suffix. + if (key.ChopSuffix(suffix)) { + TString keyStr(key); + items.emplace_back(NBackup::TSchemaMapping::TItem{ + .ExportPrefix = keyStr, + .ObjectPath = keyStr, + }); + } + } + + ProcessItemsAndReply(std::move(items)); + } + + STATEFN(StateDownloadSchemaMapping) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandleSchemaMapping); + hFunc(TEvExternalStorage::TEvGetObjectResponse, HandleSchemaMapping); + + sFunc(TEvents::TEvWakeup, DownloadSchemaMapping); + } + } + + STATEFN(StateListObjects) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvExternalStorage::TEvListObjectsResponse, HandleListObjects); + + sFunc(TEvents::TEvWakeup, ListObjectsInS3Prefix); + sFunc(TEvents::TEvPoisonPill, PassAway); + } + } + + void Reply(Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS, const TString& error = TString()) override { + LOG_I("Reply" + << ": self# " << SelfId() + << ", status# " << static_cast<int>(statusCode) + << ", error# " << error); + + auto result = MakeHolder<TEvImport::TEvListObjectsInS3ExportResponse>(); + result->Record.set_status(statusCode); + if (error) { + result->Record.add_issues()->set_message(error); + } + if (statusCode == Ydb::StatusIds::SUCCESS) { + result->Record.mutable_result()->Swap(&Result); + } + Send(Request->Sender, std::move(result)); + PassAway(); + } + + TString GetSchemaMappingKey() const { + return TStringBuilder() << NBackup::NormalizeExportPrefix(Request->Get()->Record.GetSettings().prefix()) << "/SchemaMapping/mapping.json"; + } + + TString GetExportPrefix() const { + return NBackup::NormalizeExportPrefix(Request->Get()->Record.GetSettings().prefix()) + "/"; + } + + void ProcessItemsAndReply(std::vector<NBackup::TSchemaMapping::TItem>&& items) { + std::sort(items.begin(), items.end(), [](const NBackup::TSchemaMapping::TItem& i1, const NBackup::TSchemaMapping::TItem& i2) { + return i1.ObjectPath < i2.ObjectPath; + }); + + PathFilter.Build(Request->Get()->Record.GetSettings()); + + size_t pos = 0; + for (const auto& item : items) { + if (!PathFilter.Match(item.ObjectPath)) { + continue; + } + + if (PageSize && pos >= StartPos + PageSize) { // Calc only items that suit filter + NextPos = pos; + break; + } + + if (pos >= StartPos) { + auto* result = Result.add_items(); + result->set_path(item.ObjectPath); + result->set_prefix(item.ExportPrefix); + } + + ++pos; + } + if (NextPos) { + Result.set_next_page_token(ToString(NextPos)); + } + Reply(); + } + +private: + TEvImport::TEvListObjectsInS3ExportRequest::TPtr Request; + Ydb::Import::ListObjectsInS3ExportResult Result; + size_t StartPos = 0; + size_t PageSize = 0; + size_t NextPos = 0; + TPathFilter PathFilter; +}; + IActor* CreateSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx, TMaybe<NBackup::TEncryptionIV> iv) { return new TSchemeGetter(replyTo, std::move(importInfo), itemIdx, std::move(iv)); } @@ -949,5 +1262,9 @@ IActor* CreateSchemaMappingGetter(const TActorId& replyTo, TImportInfo::TPtr imp return new TSchemaMappingGetter(replyTo, std::move(importInfo)); } +IActor* CreateListObjectsInS3ExportGetter(TEvImport::TEvListObjectsInS3ExportRequest::TPtr&& ev) { + return new TListObjectsInS3ExportGetter(std::move(ev)); +} + } // NSchemeShard } // NKikimr diff --git a/ydb/core/tx/schemeshard/schemeshard_import_getters.h b/ydb/core/tx/schemeshard/schemeshard_import_getters.h index 32b108c20e9..dddd9427573 100644 --- a/ydb/core/tx/schemeshard/schemeshard_import_getters.h +++ b/ydb/core/tx/schemeshard/schemeshard_import_getters.h @@ -2,6 +2,7 @@ #include "defs.h" #include "schemeshard_info_types.h" +#include "schemeshard_import.h" namespace NKikimr { namespace NSchemeShard { @@ -10,5 +11,7 @@ IActor* CreateSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo IActor* CreateSchemaMappingGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo); +IActor* CreateListObjectsInS3ExportGetter(TEvImport::TEvListObjectsInS3ExportRequest::TPtr&& ev); + } // NSchemeShard } // NKikimr diff --git a/ydb/core/wrappers/s3_storage_config.cpp b/ydb/core/wrappers/s3_storage_config.cpp index 4d0c61ce441..bab3941d926 100644 --- a/ydb/core/wrappers/s3_storage_config.cpp +++ b/ydb/core/wrappers/s3_storage_config.cpp @@ -46,6 +46,33 @@ namespace { namespace NPrivate { +template <class TMessage, class TEnum> +Aws::Http::Scheme ParseSchemeImpl(TEnum scheme, bool abortOnFailure = true) { + switch (scheme) { + case TMessage::HTTP: + return Aws::Http::Scheme::HTTP; + case TMessage::HTTPS: + return Aws::Http::Scheme::HTTPS; + default: + if (abortOnFailure) { + Y_ABORT("Unknown scheme"); + } + return Aws::Http::Scheme::HTTP; + } +} + +Aws::Http::Scheme ParseScheme(NKikimrSchemeOp::TS3Settings::EScheme scheme, bool abortOnFailure = true) { + return ParseSchemeImpl<NKikimrSchemeOp::TS3Settings>(scheme, abortOnFailure); +} + +Aws::Http::Scheme ParseScheme(Ydb::Import::ImportFromS3Settings::Scheme scheme, bool abortOnFailure = true) { + return ParseSchemeImpl<Ydb::Import::ImportFromS3Settings>(scheme, abortOnFailure); +} + +Aws::Http::Scheme ParseScheme(Ydb::Export::ExportToS3Settings::Scheme scheme, bool abortOnFailure = true) { + return ParseSchemeImpl<Ydb::Export::ExportToS3Settings>(scheme, abortOnFailure); +} + template <typename TSettings> Aws::Client::ClientConfiguration ConfigFromSettings(const TSettings& settings) { Aws::Client::ClientConfiguration config; @@ -60,16 +87,7 @@ Aws::Client::ClientConfiguration ConfigFromSettings(const TSettings& settings) { config.connectTimeoutMs = 10000; config.maxConnections = threadsCount; - switch (settings.scheme()) { - case TSettings::HTTP: - config.scheme = Aws::Http::Scheme::HTTP; - break; - case TSettings::HTTPS: - config.scheme = Aws::Http::Scheme::HTTPS; - break; - default: - Y_ABORT("Unknown scheme"); - } + config.scheme = NPrivate::ParseScheme(settings.scheme()); return config; } @@ -104,16 +122,7 @@ Aws::Client::ClientConfiguration TS3ExternalStorageConfig::ConfigFromSettings(co config.maxConnections = settings.HasMaxConnectionsCount() ? settings.GetMaxConnectionsCount() : settings.GetExecutorThreadsCount(); config.caPath = "/etc/ssl/certs"; - switch (settings.GetScheme()) { - case NKikimrSchemeOp::TS3Settings::HTTP: - config.scheme = Aws::Http::Scheme::HTTP; - break; - case NKikimrSchemeOp::TS3Settings::HTTPS: - config.scheme = Aws::Http::Scheme::HTTPS; - break; - default: - Y_ABORT("Unknown scheme"); - } + config.scheme = NPrivate::ParseScheme(settings.GetScheme()); if (settings.HasRegion()) { config.region = settings.GetRegion(); @@ -127,16 +136,7 @@ Aws::Client::ClientConfiguration TS3ExternalStorageConfig::ConfigFromSettings(co config.proxyHost = settings.GetProxyHost(); config.proxyPort = settings.GetProxyPort(); - switch (settings.GetProxyScheme()) { - case NKikimrSchemeOp::TS3Settings::HTTP: - config.proxyScheme = Aws::Http::Scheme::HTTP; - break; - case NKikimrSchemeOp::TS3Settings::HTTPS: - config.proxyScheme = Aws::Http::Scheme::HTTPS; - break; - default: - break; - } + config.proxyScheme = NPrivate::ParseScheme(settings.GetProxyScheme(), false); } return config; @@ -146,6 +146,10 @@ Aws::Client::ClientConfiguration TS3ExternalStorageConfig::ConfigFromSettings(co return NPrivate::ConfigFromSettings(settings); } +Aws::Client::ClientConfiguration TS3ExternalStorageConfig::ConfigFromSettings(const Ydb::Import::ListObjectsInS3ExportSettings& settings) { + return NPrivate::ConfigFromSettings(settings); +} + Aws::Client::ClientConfiguration TS3ExternalStorageConfig::ConfigFromSettings(const Ydb::Export::ExportToS3Settings& settings) { return NPrivate::ConfigFromSettings(settings); } @@ -158,6 +162,10 @@ Aws::Auth::AWSCredentials TS3ExternalStorageConfig::CredentialsFromSettings(cons return NPrivate::CredentialsFromSettings(settings); } +Aws::Auth::AWSCredentials TS3ExternalStorageConfig::CredentialsFromSettings(const Ydb::Import::ListObjectsInS3ExportSettings& settings) { + return NPrivate::CredentialsFromSettings(settings); +} + Aws::Auth::AWSCredentials TS3ExternalStorageConfig::CredentialsFromSettings(const Ydb::Export::ExportToS3Settings& settings) { return NPrivate::CredentialsFromSettings(settings); } @@ -178,6 +186,14 @@ TS3ExternalStorageConfig::TS3ExternalStorageConfig(const Ydb::Import::ImportFrom Bucket = settings.bucket(); } +TS3ExternalStorageConfig::TS3ExternalStorageConfig(const Ydb::Import::ListObjectsInS3ExportSettings& settings) + : Config(ConfigFromSettings(settings)) + , Credentials(CredentialsFromSettings(settings)) + , UseVirtualAddressing(!settings.disable_virtual_addressing()) +{ + Bucket = settings.bucket(); +} + TS3ExternalStorageConfig::TS3ExternalStorageConfig(const Ydb::Export::ExportToS3Settings& settings) : Config(ConfigFromSettings(settings)) , Credentials(CredentialsFromSettings(settings)) diff --git a/ydb/core/wrappers/s3_storage_config.h b/ydb/core/wrappers/s3_storage_config.h index cc5bbfeaf09..f6f3b10d39b 100644 --- a/ydb/core/wrappers/s3_storage_config.h +++ b/ydb/core/wrappers/s3_storage_config.h @@ -26,6 +26,8 @@ private: static Aws::Auth::AWSCredentials CredentialsFromSettings(const NKikimrSchemeOp::TS3Settings& settings); static Aws::Client::ClientConfiguration ConfigFromSettings(const Ydb::Import::ImportFromS3Settings& settings); static Aws::Auth::AWSCredentials CredentialsFromSettings(const Ydb::Import::ImportFromS3Settings& settings); + static Aws::Client::ClientConfiguration ConfigFromSettings(const Ydb::Import::ListObjectsInS3ExportSettings& settings); + static Aws::Auth::AWSCredentials CredentialsFromSettings(const Ydb::Import::ListObjectsInS3ExportSettings& settings); static Aws::Client::ClientConfiguration ConfigFromSettings(const Ydb::Export::ExportToS3Settings& settings); static Aws::Auth::AWSCredentials CredentialsFromSettings(const Ydb::Export::ExportToS3Settings& settings); @@ -46,6 +48,7 @@ public: TS3ExternalStorageConfig(const NKikimrSchemeOp::TS3Settings& settings); TS3ExternalStorageConfig(const Ydb::Import::ImportFromS3Settings& settings); + TS3ExternalStorageConfig(const Ydb::Import::ListObjectsInS3ExportSettings& settings); TS3ExternalStorageConfig(const Ydb::Export::ExportToS3Settings& settings); TS3ExternalStorageConfig(const Aws::Auth::AWSCredentials& credentials, const Aws::Client::ClientConfiguration& config, const TString& bucket); }; diff --git a/ydb/services/ydb/backup_ut/list_objects_in_s3_export_ut.cpp b/ydb/services/ydb/backup_ut/list_objects_in_s3_export_ut.cpp new file mode 100644 index 00000000000..7a9239fc009 --- /dev/null +++ b/ydb/services/ydb/backup_ut/list_objects_in_s3_export_ut.cpp @@ -0,0 +1,219 @@ +#include "s3_backup_test_base.h" +#include <ydb/core/backup/common/metadata.h> + +#include <fmt/format.h> + +using namespace NYdb; +using namespace fmt::literals; + +class TListObjectsInS3ExportTestFixture : public TS3BackupTestFixture { + void SetUp(NUnitTest::TTestContext& /* context */) override { + auto res = YdbQueryClient().ExecuteQuery(R"sql( + CREATE TABLE `/Root/Table0` ( + key Uint32 NOT NULL, + value String, + PRIMARY KEY (key) + ); + + CREATE TABLE `/Root/dir1/Table1` ( + key Uint32 NOT NULL, + value String, + PRIMARY KEY (key) + ); + + CREATE TABLE `/Root/dir1/dir2/Table2` ( + key Uint32 NOT NULL, + value String, + PRIMARY KEY (key) + ); + )sql", NQuery::TTxControl::NoTx()).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString()); + + // Empty dir + auto mkdir = YdbSchemeClient().MakeDirectory("/Root/dir1/dir2/dir3").GetValueSync(); + UNIT_ASSERT_C(mkdir.IsSuccess(), mkdir.GetIssues().ToString()); + } + + void TearDown(NUnitTest::TTestContext& /* context */) override { + } +}; + +Y_UNIT_TEST_SUITE_F(ListObjectsInS3Export, TListObjectsInS3ExportTestFixture) { + Y_UNIT_TEST(ExportWithSchemaMapping) { + { + NExport::TExportToS3Settings exportSettings = MakeExportSettings("", "Prefix//"); + auto res = YdbExportClient().ExportToS3(exportSettings).GetValueSync(); + WaitOpSuccess(res); + } + + ValidateListObjectInS3Export({ + {"Table0", "Table0"}, + {"dir1/Table1", "dir1/Table1"}, + {"dir1/dir2/Table2", "dir1/dir2/Table2"}, + }, "Prefix"); + } + + Y_UNIT_TEST(ExportWithoutSchemaMapping) { + { + NExport::TExportToS3Settings exportSettings = MakeExportSettings("", ""); + exportSettings + .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/Root/Table0", .Dst = "/Prefix/t0"}) + .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/Root/dir1/Table1", .Dst = "/Prefix/d1/t1"}) + .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/Root/dir1/dir2/Table2", .Dst = "/Prefix/d1/d2/t2"}); + auto res = YdbExportClient().ExportToS3(exportSettings).GetValueSync(); + WaitOpSuccess(res); + } + + ValidateListObjectInS3Export({ + {"t0", "t0"}, + {"d1/t1", "d1/t1"}, + {"d1/d2/t2", "d1/d2/t2"}, + }, "Prefix"); + + ValidateListObjectInS3Export({ + {"t1", "t1"}, + {"d2/t2", "d2/t2"}, + }, "Prefix/d1"); + } + + Y_UNIT_TEST(ExportWithEncryption) { + { + NExport::TExportToS3Settings exportSettings = MakeExportSettings("", "Prefix"); + exportSettings + .SymmetricEncryption(NExport::TExportToS3Settings::TEncryptionAlgorithm::AES_128_GCM, "Cool random key!") + .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/Root/Table0"}) + .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/dir1/Table1"}) + .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "dir1/dir2//Table2"}); + auto res = YdbExportClient().ExportToS3(exportSettings).GetValueSync(); + WaitOpSuccess(res); + } + + NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix//"); + listSettings + .SymmetricKey("Cool random key!"); + + ValidateListObjectPathsInS3Export({ + "Table0", + "dir1/Table1", + "dir1/dir2/Table2", + }, listSettings); + } + + Y_UNIT_TEST(ExportWithWrongEncryptionKey) { + { + NExport::TExportToS3Settings exportSettings = MakeExportSettings("", "Prefix"); + exportSettings + .SymmetricEncryption(NExport::TExportToS3Settings::TEncryptionAlgorithm::AES_128_GCM, "Cool random key!") + .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/Root/Table0"}) + .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/dir1/Table1"}) + .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "dir1/dir2//Table2"}); + auto res = YdbExportClient().ExportToS3(exportSettings).GetValueSync(); + WaitOpSuccess(res); + } + + NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix//"); + listSettings + .SymmetricKey("Cool and random)"); + + auto res = YdbImportClient().ListObjectsInS3Export(listSettings).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::BAD_REQUEST, "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString()); + } + + Y_UNIT_TEST(PagingParameters) { + NBackup::TSchemaMapping schemaMapping; + constexpr size_t ItemsCount = 12000; + constexpr size_t ItemsPerFolder = ItemsCount / 5; + constexpr size_t ItemsPerSubfolder = ItemsPerFolder / 3; + for (size_t i = 0; i < ItemsCount; ++i) { + TStringBuilder objectPath; + objectPath << "Folder" << (i % 5) << "/Subfolder" << (i % 3) << "/Object" << i; + schemaMapping.Items.emplace_back(NBackup::TSchemaMapping::TItem{ + .ExportPrefix = TStringBuilder() << "Prefix" << i, + .ObjectPath = objectPath, + }); + } + S3Mock().GetData()["/test_bucket/Prefix/SchemaMapping/mapping.json"] = schemaMapping.Serialize(); + + // Without paging + { + NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix//"); + auto res = YdbImportClient().ListObjectsInS3Export(listSettings).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.GetItems().size(), ItemsCount); + } + + // From beginning to the end + { + auto addItemsByFilter = [&](size_t expectedItemsCount, const TString& prefix1 = {}, const TString& prefix2 = {}) { + THashSet<TString> items; + i64 pageSize = 42; + TString nextPageToken; + do { + NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix//"); + if (prefix1) { + listSettings.AppendItem(NYdb::NImport::TListObjectsInS3ExportSettings::TItem{.Path = prefix1}); + } + if (prefix2) { + listSettings.AppendItem(NYdb::NImport::TListObjectsInS3ExportSettings::TItem{.Path = prefix2}); + } + auto res = YdbImportClient().ListObjectsInS3Export(listSettings, pageSize, nextPageToken).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString()); + UNIT_ASSERT_GT(res.GetItems().size(), 0); + UNIT_ASSERT_LE(res.GetItems().size(), static_cast<size_t>(pageSize)); + for (const auto& item : res.GetItems()) { + UNIT_ASSERT_C(items.emplace(item.Path).second, "Duplicate item: {" << item.Prefix << ", " << item.Path << "}. Listing result: " << res); + } + + ++pageSize; // just for fun + nextPageToken = res.NextPageToken(); + } while (!nextPageToken.empty()); + UNIT_ASSERT_VALUES_EQUAL(items.size(), expectedItemsCount); + }; + + addItemsByFilter(ItemsCount); + addItemsByFilter(ItemsPerFolder, "/Folder2"); + addItemsByFilter(ItemsPerSubfolder, "Folder1/Subfolder1//"); + addItemsByFilter(ItemsPerSubfolder, "Folder1/Subfolder1//"); + addItemsByFilter(1, "Folder0/Subfolder2//Object620"); + addItemsByFilter(ItemsPerSubfolder + 1, "Folder0/Subfolder2//Object620", "Folder1/Subfolder1"); + addItemsByFilter(1, "Folder0/Subfolder2//Object620", "Folder1/Subfolder"); // treat not as a prefix, but as a directory, so Subfolder don't match anything + addItemsByFilter(2, "Folder0/Subfolder2//Object620", "Folder1/Subfolder1/Object931"); + } + } + + Y_UNIT_TEST(ParametersValidation) { + { + NExport::TExportToS3Settings exportSettings = MakeExportSettings("", "Prefix//"); + auto res = YdbExportClient().ExportToS3(exportSettings).GetValueSync(); + WaitOpSuccess(res); + } + + { + // No prefix + NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings(""); + auto res = YdbImportClient().ListObjectsInS3Export(listSettings).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::BAD_REQUEST, "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString()); + } + + { + // Negative page size + NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix"); + auto res = YdbImportClient().ListObjectsInS3Export(listSettings, -42).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::BAD_REQUEST, "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString()); + } + + { + // Big page size + NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix"); + auto res = YdbImportClient().ListObjectsInS3Export(listSettings, 100500).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::BAD_REQUEST, "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString()); + } + + { + // Wrong page token + NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix"); + auto res = YdbImportClient().ListObjectsInS3Export(listSettings, 42, "incorrect page token").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::BAD_REQUEST, "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString()); + } + } +} diff --git a/ydb/services/ydb/backup_ut/s3_backup_test_base.h b/ydb/services/ydb/backup_ut/s3_backup_test_base.h index 9b130aea8dd..473e4e9bb57 100644 --- a/ydb/services/ydb/backup_ut/s3_backup_test_base.h +++ b/ydb/services/ydb/backup_ut/s3_backup_test_base.h @@ -152,6 +152,20 @@ protected: return importSettings; } + NYdb::NImport::TListObjectsInS3ExportSettings MakeListObjectsInS3ExportSettings(const TString& prefix) { + NYdb::NImport::TListObjectsInS3ExportSettings listSettings; + listSettings + .Endpoint(TStringBuilder() << "localhost:" << S3Port()) + .Bucket("test_bucket") + .Scheme(NYdb::ES3Scheme::HTTP) + .AccessKey("test_key") + .SecretKey("test_secret"); + if (prefix) { + listSettings.Prefix(prefix); + } + return listSettings; + } + void ValidateS3FileList(const TSet<TString>& paths, const TString& prefix = {}) { TSet<TString> keys; for (const auto& [key, _] : S3Mock().GetData()) { @@ -178,6 +192,40 @@ protected: } } + void ValidateListObjectInS3Export(const TSet<std::pair<TString /*prefix*/, TString /*path*/>>& paths, const NYdb::NImport::TListObjectsInS3ExportSettings& listSettings) { + auto res = YdbImportClient().ListObjectsInS3Export(listSettings).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString()); + + TSet<std::pair<TString, TString>> pathsInResponse; + for (const auto& item : res.GetItems()) { + bool inserted = pathsInResponse.emplace(item.Prefix, item.Path).second; + UNIT_ASSERT_C(inserted, "Duplicate item: {" << item.Prefix << ", " << item.Path << "}. Listing result: " << res); + } + + UNIT_ASSERT_VALUES_EQUAL_C(pathsInResponse, paths, "Listing result: " << res); + } + + void ValidateListObjectInS3Export(const TSet<std::pair<TString /*prefix*/, TString /*path*/>>& paths, const TString& exportPrefix) { + ValidateListObjectInS3Export(paths, MakeListObjectsInS3ExportSettings(exportPrefix)); + } + + void ValidateListObjectPathsInS3Export(const TSet<TString>& paths, const NYdb::NImport::TListObjectsInS3ExportSettings& listSettings) { + auto res = YdbImportClient().ListObjectsInS3Export(listSettings).GetValueSync(); + UNIT_ASSERT_C(res.IsSuccess(), "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString()); + + TSet<TString> pathsInResponse; + for (const auto& item : res.GetItems()) { + bool inserted = pathsInResponse.emplace(item.Path).second; + UNIT_ASSERT_C(inserted, "Duplicate item: {" << item.Prefix << ", " << item.Path << "}. Listing result: " << res); + } + + UNIT_ASSERT_VALUES_EQUAL_C(pathsInResponse, paths, "Listing result: " << res); + } + + void ValidateListObjectPathsInS3Export(const TSet<TString>& paths, const TString& exportPrefix) { + ValidateListObjectPathsInS3Export(paths, MakeListObjectsInS3ExportSettings(exportPrefix)); + } + TString DebugListDir(const TString& path) { // Debug listing for specified dir auto res = YdbSchemeClient().ListDirectory(path).GetValueSync(); TStringBuilder l; diff --git a/ydb/services/ydb/backup_ut/ya.make b/ydb/services/ydb/backup_ut/ya.make index e5f3e895506..f0966d7b2c4 100644 --- a/ydb/services/ydb/backup_ut/ya.make +++ b/ydb/services/ydb/backup_ut/ya.make @@ -12,6 +12,7 @@ ENDIF() SRCS( backup_path_ut.cpp encrypted_backup_ut.cpp + list_objects_in_s3_export_ut.cpp ydb_backup_ut.cpp ) diff --git a/ydb/services/ydb/ydb_import.cpp b/ydb/services/ydb/ydb_import.cpp index 7a91d096a53..12ebfd5af58 100644 --- a/ydb/services/ydb/ydb_import.cpp +++ b/ydb/services/ydb/ydb_import.cpp @@ -24,6 +24,7 @@ void TGRpcYdbImportService::SetupIncomingRequests(NYdbGrpc::TLoggerPtr logger) { #NAME, logger, getCounterBlock("import", #NAME))->Run(); ADD_REQUEST(ImportFromS3, ImportFromS3Request, ImportFromS3Response, DoImportFromS3Request); + ADD_REQUEST(ListObjectsInS3Export, ListObjectsInS3ExportRequest, ListObjectsInS3ExportResponse, DoListObjectsInS3ExportRequest); ADD_REQUEST(ImportData, ImportDataRequest, ImportDataResponse, DoImportDataRequest); #undef ADD_REQUEST |
