summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorVasily Gerasimov <[email protected]>2025-05-21 12:46:41 +0300
committerGitHub <[email protected]>2025-05-21 12:46:41 +0300
commitddb9664139d8ea8a4b7b674279a047d01869f4f0 (patch)
tree2467c5f5e9fdaaa499e5b7807acd9a67fc8117a8
parent2ab1e1916633d2f5be41e19ee8d2fb3a8dd8523b (diff)
Implement ImportService.ListObjectsInS3Export on server side (#18454)
-rw-r--r--ydb/core/backup/common/metadata.cpp58
-rw-r--r--ydb/core/backup/common/metadata.h5
-rw-r--r--ydb/core/backup/common/metadata_ut.cpp35
-rw-r--r--ydb/core/backup/common/ut/ya.make1
-rw-r--r--ydb/core/grpc_services/rpc_list_objects_in_s3_export.cpp203
-rw-r--r--ydb/core/grpc_services/service_import.h1
-rw-r--r--ydb/core/grpc_services/ya.make1
-rw-r--r--ydb/core/protos/import.proto13
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_export__create.cpp19
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_export_uploaders.cpp28
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_impl.cpp1
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_impl.h1
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_import.cpp5
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_import.h8
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_import__create.cpp25
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_import_getters.cpp373
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_import_getters.h3
-rw-r--r--ydb/core/wrappers/s3_storage_config.cpp76
-rw-r--r--ydb/core/wrappers/s3_storage_config.h3
-rw-r--r--ydb/services/ydb/backup_ut/list_objects_in_s3_export_ut.cpp219
-rw-r--r--ydb/services/ydb/backup_ut/s3_backup_test_base.h48
-rw-r--r--ydb/services/ydb/backup_ut/ya.make1
-rw-r--r--ydb/services/ydb/ydb_import.cpp1
23 files changed, 1021 insertions, 107 deletions
diff --git a/ydb/core/backup/common/metadata.cpp b/ydb/core/backup/common/metadata.cpp
index d54b8798618..99456735330 100644
--- a/ydb/core/backup/common/metadata.cpp
+++ b/ydb/core/backup/common/metadata.cpp
@@ -1,5 +1,7 @@
#include "metadata.h"
+#include <ydb/core/base/path.h>
+
#include <library/cpp/json/json_writer.h>
#include <library/cpp/json/json_reader.h>
@@ -53,6 +55,31 @@ TMetadata TMetadata::Deserialize(const TString& metadata) {
return result;
}
+TString TSchemaMapping::Serialize() const {
+ TString content;
+ TStringOutput ss(content);
+ NJson::TJsonWriter writer(&ss, false);
+
+ writer.OpenMap();
+ writer.WriteKey("exportedObjects");
+ writer.OpenMap();
+ for (const auto& item : Items) {
+ writer.WriteKey(item.ObjectPath);
+ writer.OpenMap();
+ writer.Write("exportPrefix", item.ExportPrefix);
+ if (item.IV) {
+ writer.Write("iv", item.IV->GetHexString());
+ }
+ writer.CloseMap();
+ }
+ writer.CloseMap();
+ writer.CloseMap();
+
+ writer.Flush();
+ ss.Flush();
+ return content;
+}
+
bool TSchemaMapping::Deserialize(const TString& jsonContent, TString& error) {
NJson::TJsonValue json;
if (!NJson::ReadJsonTree(jsonContent, &json)) {
@@ -97,4 +124,35 @@ bool TSchemaMapping::Deserialize(const TString& jsonContent, TString& error) {
return true;
}
+TString NormalizeItemPath(const TString& path) {
+ TString result = CanonizePath(path);
+ if (result.size() > 1 && result.front() == '/') {
+ result.erase(0, 1);
+ }
+ return result;
+}
+
+TString NormalizeItemPrefix(TString prefix) {
+ // Cut slshes from the beginning and from the end
+ size_t toCut = 0;
+ while (toCut < prefix.size() && prefix[toCut] == '/') {
+ ++toCut;
+ }
+ if (toCut) {
+ prefix.erase(0, toCut);
+ }
+
+ while (prefix && prefix.back() == '/') {
+ prefix.pop_back();
+ }
+ return prefix;
+}
+
+TString NormalizeExportPrefix(TString prefix) {
+ while (prefix && prefix.back() == '/') {
+ prefix.pop_back();
+ }
+ return prefix;
+}
+
}
diff --git a/ydb/core/backup/common/metadata.h b/ydb/core/backup/common/metadata.h
index 7479f7ee41c..1b7f8cb9001 100644
--- a/ydb/core/backup/common/metadata.h
+++ b/ydb/core/backup/common/metadata.h
@@ -57,6 +57,10 @@ private:
TMaybeFail<ui64> Version;
};
+TString NormalizeItemPath(const TString& path);
+TString NormalizeItemPrefix(TString prefix);
+TString NormalizeExportPrefix(TString prefix);
+
class TSchemaMapping {
public:
struct TItem {
@@ -67,6 +71,7 @@ public:
TSchemaMapping() = default;
+ TString Serialize() const;
bool Deserialize(const TString& jsonContent, TString& error);
public:
diff --git a/ydb/core/backup/common/metadata_ut.cpp b/ydb/core/backup/common/metadata_ut.cpp
new file mode 100644
index 00000000000..4dfceb3b6e9
--- /dev/null
+++ b/ydb/core/backup/common/metadata_ut.cpp
@@ -0,0 +1,35 @@
+#include "metadata.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr::NBackup {
+
+Y_UNIT_TEST_SUITE(PathsNormalizationTest) {
+ Y_UNIT_TEST(NormalizeItemPath) {
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPath("/a/b/c/"), "a/b/c");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPath("/"), "");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPath(""), "");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPath("//"), "");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPath("///a///b///"), "a/b");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPath("a/b/c"), "a/b/c");
+ }
+
+ Y_UNIT_TEST(NormalizeItemPrefix) {
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPrefix("///a///b///c///"), "a///b///c");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPrefix("a///b///c"), "a///b///c");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPrefix("//"), "");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPrefix("/"), "");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeItemPrefix(""), "");
+ }
+
+ Y_UNIT_TEST(NormalizeExportPrefix) {
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeExportPrefix("///a///"), "///a");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeExportPrefix("a/"), "a");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeExportPrefix("a"), "a");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeExportPrefix("/prefix//"), "/prefix");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeExportPrefix(""), "");
+ UNIT_ASSERT_STRINGS_EQUAL(NormalizeExportPrefix("/prefix"), "/prefix");
+ }
+}
+
+} // namespace NKikimr::NBackup
diff --git a/ydb/core/backup/common/ut/ya.make b/ydb/core/backup/common/ut/ya.make
index 5978f64e27c..188e2d3c195 100644
--- a/ydb/core/backup/common/ut/ya.make
+++ b/ydb/core/backup/common/ut/ya.make
@@ -2,6 +2,7 @@ UNITTEST_FOR(ydb/core/backup/common)
SRCS(
encryption_ut.cpp
+ metadata_ut.cpp
)
END()
diff --git a/ydb/core/grpc_services/rpc_list_objects_in_s3_export.cpp b/ydb/core/grpc_services/rpc_list_objects_in_s3_export.cpp
new file mode 100644
index 00000000000..7e68226272f
--- /dev/null
+++ b/ydb/core/grpc_services/rpc_list_objects_in_s3_export.cpp
@@ -0,0 +1,203 @@
+#include "service_import.h"
+#include "rpc_deferrable.h"
+
+#include <ydb/core/base/tablet_pipe.h>
+#include <ydb/core/grpc_services/base/base.h>
+#include <ydb/core/tx/scheme_cache/scheme_cache.h>
+#include <ydb/core/tx/schemeshard/schemeshard_import.h>
+#include <ydb/public/api/protos/ydb_import.pb.h>
+
+#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, "[ListObjectsInS3Export] " << SelfId() << " " << stream)
+#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_PROXY, "[ListObjectsInS3Export] " << SelfId() << " " << stream)
+#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::TX_PROXY, "[ListObjectsInS3Export] " << SelfId() << " " << stream)
+#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::TX_PROXY, "[ListObjectsInS3Export] " << SelfId() << " " << stream)
+#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::TX_PROXY, "[ListObjectsInS3Export] " << SelfId() << " " << stream)
+#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_PROXY, "[ListObjectsInS3Export] " << SelfId() << " " << stream)
+
+namespace NKikimr::NGRpcService {
+
+using TEvListObjectsInS3ExportRequest = TGrpcRequestOperationCall<Ydb::Import::ListObjectsInS3ExportRequest,
+ Ydb::Import::ListObjectsInS3ExportResponse>;
+
+class TListObjectsInS3ExportRPC: public TRpcOperationRequestActor<TListObjectsInS3ExportRPC, TEvListObjectsInS3ExportRequest> {
+public:
+ using TBase = TRpcOperationRequestActor<TListObjectsInS3ExportRPC, TEvListObjectsInS3ExportRequest>;
+ using TRpcOperationRequestActor<TListObjectsInS3ExportRPC, TEvListObjectsInS3ExportRequest>::TRpcOperationRequestActor;
+
+ explicit TListObjectsInS3ExportRPC(IRequestOpCtx* request)
+ : TBase(request)
+ , UserToken(CreateUserToken(request))
+ {
+ }
+
+ STATEFN(StateFunc) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(NKikimr::NSchemeShard::TEvImport::TEvListObjectsInS3ExportResponse, Handle);
+ hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle);
+
+ hFunc(TEvTabletPipe::TEvClientConnected, Handle);
+ hFunc(TEvTabletPipe::TEvClientDestroyed, Handle);
+ default:
+ return StateFuncBase(ev);
+ }
+ }
+
+ void Bootstrap() {
+ if (!Request_ || !Request_->GetDatabaseName()) {
+ return Reply(Ydb::StatusIds::BAD_REQUEST, "Database name is not specified", NKikimrIssues::TIssuesIds::YDB_API_VALIDATION_ERROR, NActors::TActivationContext::AsActorContext());
+ }
+
+ ResolveDatabase();
+
+ Become(&TListObjectsInS3ExportRPC::StateFunc);
+ }
+
+ void ResolveDatabase() {
+ LOG_D("Resolve database"
+ << ": name# " << Request_->GetDatabaseName());
+
+ auto request = MakeHolder<NSchemeCache::TSchemeCacheNavigate>();
+ request->DatabaseName = *Request_->GetDatabaseName();
+
+ auto& entry = request->ResultSet.emplace_back();
+ entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpPath;
+ entry.Path = NKikimr::SplitPath(*Request_->GetDatabaseName());
+
+ Send(MakeSchemeCacheID(), new TEvTxProxySchemeCache::TEvNavigateKeySet(request.Release()));
+ }
+
+ void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
+ const auto& request = ev->Get()->Request;
+
+ LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult"
+ << ": request# " << (request ? request->ToString(*AppData()->TypeRegistry) : "nullptr"));
+
+ if (request->ResultSet.empty()) {
+ return Reply(Ydb::StatusIds::SCHEME_ERROR, "Scheme error", NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, NActors::TActivationContext::AsActorContext());
+ }
+
+ const auto& entry = request->ResultSet.front();
+
+ if (request->ErrorCount > 0) {
+ switch (entry.Status) {
+ case NSchemeCache::TSchemeCacheNavigate::EStatus::Ok:
+ break;
+ case NSchemeCache::TSchemeCacheNavigate::EStatus::AccessDenied:
+ return Reply(Ydb::StatusIds::UNAUTHORIZED, "Access denied", NKikimrIssues::TIssuesIds::ACCESS_DENIED, NActors::TActivationContext::AsActorContext());
+ case NSchemeCache::TSchemeCacheNavigate::EStatus::RootUnknown:
+ case NSchemeCache::TSchemeCacheNavigate::EStatus::PathErrorUnknown:
+ return Reply(Ydb::StatusIds::SCHEME_ERROR, "Unknown database", NKikimrIssues::TIssuesIds::PATH_NOT_EXIST, NActors::TActivationContext::AsActorContext());
+ case NSchemeCache::TSchemeCacheNavigate::EStatus::LookupError:
+ case NSchemeCache::TSchemeCacheNavigate::EStatus::RedirectLookupError:
+ return Reply(Ydb::StatusIds::UNAVAILABLE, "Database lookup error", NKikimrIssues::TIssuesIds::RESOLVE_LOOKUP_ERROR, NActors::TActivationContext::AsActorContext());
+ default:
+ return Reply(Ydb::StatusIds::SCHEME_ERROR, "Scheme error", NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, NActors::TActivationContext::AsActorContext());
+ }
+ }
+
+ if (!this->CheckDatabaseAccess(CanonizePath(entry.Path), entry.SecurityObject)) {
+ return;
+ }
+
+ auto domainInfo = entry.DomainInfo;
+ if (!domainInfo) {
+ LOG_E("Got empty domain info");
+ return Reply(Ydb::StatusIds::INTERNAL_ERROR, "Internal error", NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, NActors::TActivationContext::AsActorContext());
+ }
+
+ SchemeShardId = domainInfo->ExtractSchemeShard();
+ SendRequestToSchemeShard();
+ }
+
+ bool CheckDatabaseAccess(const TString& path, TIntrusivePtr<TSecurityObject> securityObject) {
+ const ui32 access = NACLib::DescribeSchema;
+
+ if (!UserToken || !securityObject) {
+ return true;
+ }
+
+ if (securityObject->CheckAccess(access, *UserToken)) {
+ return true;
+ }
+
+ Reply(Ydb::StatusIds::UNAUTHORIZED,
+ TStringBuilder() << "Access denied"
+ << ": for# " << UserToken->GetUserSID()
+ << ", path# " << path
+ << ", access# " << NACLib::AccessRightsToString(access),
+ NKikimrIssues::TIssuesIds::ACCESS_DENIED,
+ NActors::TActivationContext::AsActorContext());
+ return false;
+ }
+
+ void SendRequestToSchemeShard() {
+ LOG_D("Send request: schemeShardId# " << SchemeShardId);
+
+ if (!PipeClient) {
+ NTabletPipe::TClientConfig config;
+ config.RetryPolicy = {.RetryLimitCount = 3};
+ PipeClient = this->RegisterWithSameMailbox(NTabletPipe::CreateClient(this->SelfId(), SchemeShardId, config));
+ }
+
+ auto request = MakeHolder<NSchemeShard::TEvImport::TEvListObjectsInS3ExportRequest>();
+
+ *request->Record.MutableOperationParams() = GetProtoRequest()->operation_params();
+ *request->Record.MutableSettings() = GetProtoRequest()->settings();
+ request->Record.SetPageSize(GetProtoRequest()->page_size());
+ request->Record.SetPageToken(GetProtoRequest()->page_token());
+
+ NTabletPipe::SendData(this->SelfId(), PipeClient, std::move(request), 0, Span_.GetTraceId());
+ }
+
+ void Handle(NKikimr::NSchemeShard::TEvImport::TEvListObjectsInS3ExportResponse::TPtr& ev) {
+ const auto& record = ev->Get()->Record;
+
+ LOG_D("Handle TListObjectsInS3ExportRPC::TEvListObjectsInS3ExportResponse"
+ << ": record# " << record.ShortDebugString());
+
+ if (record.GetStatus() != Ydb::StatusIds::SUCCESS) {
+ return Reply(record.GetStatus(), record.GetIssues(), NActors::TActivationContext::AsActorContext());
+ } else {
+ return ReplyWithResult(record.GetStatus(), record.GetIssues(), record.GetResult(), NActors::TActivationContext::AsActorContext());
+ }
+ }
+
+ void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) {
+ if (ev->Get()->Status != NKikimrProto::OK) {
+ DeliveryProblem();
+ }
+ }
+
+ void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr&) {
+ DeliveryProblem();
+ }
+
+ void DeliveryProblem() {
+ LOG_W("Delivery problem");
+ Reply(Ydb::StatusIds::UNAVAILABLE, "Delivery problem", NKikimrIssues::TIssuesIds::DEFAULT_ERROR, NActors::TActivationContext::AsActorContext());
+ }
+
+ void PassAway() override {
+ NTabletPipe::CloseClient(this->SelfId(), PipeClient);
+ TBase::PassAway();
+ }
+
+ static THolder<const NACLib::TUserToken> CreateUserToken(IRequestOpCtx* request) {
+ if (const auto& userToken = request->GetSerializedToken()) {
+ return MakeHolder<NACLib::TUserToken>(userToken);
+ } else {
+ return {};
+ }
+ }
+
+private:
+ ui64 SchemeShardId = 0;
+ TActorId PipeClient;
+ const THolder<const NACLib::TUserToken> UserToken;
+};
+
+void DoListObjectsInS3ExportRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& f) {
+ f.RegisterActor(new TListObjectsInS3ExportRPC(p.release()));
+}
+
+} // namespace NKikimr::NGRpcService
diff --git a/ydb/core/grpc_services/service_import.h b/ydb/core/grpc_services/service_import.h
index 971e02f199a..56f89329acd 100644
--- a/ydb/core/grpc_services/service_import.h
+++ b/ydb/core/grpc_services/service_import.h
@@ -9,6 +9,7 @@ class IRequestOpCtx;
class IFacilityProvider;
void DoImportFromS3Request(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& f);
+void DoListObjectsInS3ExportRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& f);
void DoImportDataRequest(std::unique_ptr<IRequestOpCtx> p, const IFacilityProvider& f);
}
diff --git a/ydb/core/grpc_services/ya.make b/ydb/core/grpc_services/ya.make
index 3e703900201..80ae3a0bfde 100644
--- a/ydb/core/grpc_services/ya.make
+++ b/ydb/core/grpc_services/ya.make
@@ -60,6 +60,7 @@ SRCS(
rpc_kh_describe.cpp
rpc_kh_snapshots.cpp
rpc_kqp_base.cpp
+ rpc_list_objects_in_s3_export.cpp
rpc_list_operations.cpp
rpc_load_rows.cpp
rpc_log_store.cpp
diff --git a/ydb/core/protos/import.proto b/ydb/core/protos/import.proto
index 78ce80452d3..5f4ed3ba48e 100644
--- a/ydb/core/protos/import.proto
+++ b/ydb/core/protos/import.proto
@@ -129,3 +129,16 @@ message TListImportsResponse {
message TEvListImportsResponse {
optional TListImportsResponse Response = 1;
}
+
+message TEvListObjectsInS3ExportRequest {
+ optional Ydb.Operations.OperationParams OperationParams = 1;
+ optional Ydb.Import.ListObjectsInS3ExportSettings Settings = 2;
+ optional int64 PageSize = 3;
+ optional string PageToken = 4;
+}
+
+message TEvListObjectsInS3ExportResponse {
+ optional Ydb.StatusIds.StatusCode Status = 1;
+ repeated Ydb.Issue.IssueMessage Issues = 2;
+ optional Ydb.Import.ListObjectsInS3ExportResult Result = 3;
+}
diff --git a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp
index fed6310d198..8665e12f17f 100644
--- a/ydb/core/tx/schemeshard/schemeshard_export__create.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_export__create.cpp
@@ -408,10 +408,7 @@ private:
return true;
}
- TString commonDestinationPrefix = exportSettings.destination_prefix();
- if (commonDestinationPrefix.back() == '/') {
- commonDestinationPrefix.pop_back();
- }
+ TString commonDestinationPrefix = NBackup::NormalizeExportPrefix(exportSettings.destination_prefix());
TMaybe<NBackup::TEncryptionIV> iv;
if (exportSettings.has_encryption_settings()) {
@@ -435,28 +432,20 @@ private:
if (exportPath.StartsWith(sourcePathRoot)) {
exportPath = exportPath.substr(sourcePathRoot.size() + 1); // cut all prefix + '/'
}
+ exportPath = NBackup::NormalizeItemPath(exportPath); // Path without leading slash
schemaMappingItem.SetSourcePath(exportPath);
TString destinationPrefix;
if (!exportItem.destination_prefix().empty()) {
TString& itemPrefix = *exportItem.mutable_destination_prefix();
- if (itemPrefix[0] == '/') {
- destinationPrefix = itemPrefix = itemPrefix.substr(1);
- } else {
- destinationPrefix = itemPrefix;
- }
- if (itemPrefix.back() == '/') {
- itemPrefix.pop_back();
- }
+ destinationPrefix = itemPrefix = NBackup::NormalizeItemPrefix(itemPrefix);
} else {
std::stringstream itemPrefix;
if (exportSettings.has_encryption_settings()) {
// Anonymize object name in export
itemPrefix << std::setfill('0') << std::setw(3) << std::right << itemIndex;
} else {
- TStringBuf exportPathBuf = exportPath;
- exportPathBuf.SkipPrefix("/");
- itemPrefix << exportPathBuf;
+ itemPrefix << exportPath;
}
destinationPrefix = itemPrefix.str();
}
diff --git a/ydb/core/tx/schemeshard/schemeshard_export_uploaders.cpp b/ydb/core/tx/schemeshard/schemeshard_export_uploaders.cpp
index b3510f1a8e2..0a853d20dc9 100644
--- a/ydb/core/tx/schemeshard/schemeshard_export_uploaders.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_export_uploaders.cpp
@@ -2,6 +2,7 @@
#include "schemeshard_export_uploaders.h"
#include <ydb/core/backup/common/encryption.h>
+#include <ydb/core/backup/common/metadata.h>
#include <ydb/core/protos/flat_scheme_op.pb.h>
#include <ydb/core/tx/datashard/export_common.h>
#include <ydb/core/tx/schemeshard/schemeshard_export_helpers.h>
@@ -495,34 +496,21 @@ private:
}
bool AddSchemaMappingJson() {
- TString content;
- TStringOutput ss(content);
- NJson::TJsonWriter writer(&ss, false);
-
- writer.OpenMap();
- writer.WriteKey("exportedObjects");
- writer.OpenMap();
+ NBackup::TSchemaMapping schemaMapping;
for (const auto& item : ExportMetadata.GetSchemaMapping()) {
- writer.WriteKey(item.GetSourcePath());
- writer.OpenMap();
- writer.Write("exportPrefix", item.GetDestinationPrefix());
- if (item.HasIV()) {
- writer.Write("iv", NBackup::TEncryptionIV::FromBinaryString(item.GetIV()).GetHexString());
- }
- writer.CloseMap();
+ schemaMapping.Items.emplace_back(NBackup::TSchemaMapping::TItem{
+ .ExportPrefix = item.GetDestinationPrefix(),
+ .ObjectPath = item.GetSourcePath(),
+ .IV = item.HasIV() ? TMaybe<NBackup::TEncryptionIV>(NBackup::TEncryptionIV::FromBinaryString(item.GetIV())) : Nothing()
+ });
}
- writer.CloseMap();
- writer.CloseMap();
-
- writer.Flush();
- ss.Flush();
TMaybe<NBackup::TEncryptionIV> iv;
if (IV) {
iv = NBackup::TEncryptionIV::Combine(*IV, NBackup::EBackupFileType::SchemaMapping, 0, 0);
}
- return AddFile("SchemaMapping/mapping.json", content, iv, Key);
+ return AddFile("SchemaMapping/mapping.json", schemaMapping.Serialize(), iv, Key);
}
void ProcessQueue() {
diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp
index 7702fb3375b..570f5353ac4 100644
--- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp
@@ -4991,6 +4991,7 @@ void TSchemeShard::StateWork(STFUNC_SIG) {
HFuncTraced(TEvImport::TEvCancelImportRequest, Handle);
HFuncTraced(TEvImport::TEvForgetImportRequest, Handle);
HFuncTraced(TEvImport::TEvListImportsRequest, Handle);
+ HFuncTraced(TEvImport::TEvListObjectsInS3ExportRequest, Handle);
HFuncTraced(TEvPrivate::TEvImportSchemeReady, Handle);
HFuncTraced(TEvPrivate::TEvImportSchemaMappingReady, Handle);
HFuncTraced(TEvPrivate::TEvImportSchemeQueryResult, Handle);
diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h
index 14873f05ac9..10ba657bb6b 100644
--- a/ydb/core/tx/schemeshard/schemeshard_impl.h
+++ b/ydb/core/tx/schemeshard/schemeshard_impl.h
@@ -1364,6 +1364,7 @@ public:
void Handle(TEvImport::TEvCancelImportRequest::TPtr& ev, const TActorContext& ctx);
void Handle(TEvImport::TEvForgetImportRequest::TPtr& ev, const TActorContext& ctx);
void Handle(TEvImport::TEvListImportsRequest::TPtr& ev, const TActorContext& ctx);
+ void Handle(TEvImport::TEvListObjectsInS3ExportRequest::TPtr& ev, const TActorContext& ctx);
void Handle(TEvPrivate::TEvImportSchemeReady::TPtr& ev, const TActorContext& ctx);
void Handle(TEvPrivate::TEvImportSchemaMappingReady::TPtr& ev, const TActorContext& ctx);
void Handle(TEvPrivate::TEvImportSchemeQueryResult::TPtr& ev, const TActorContext& ctx);
diff --git a/ydb/core/tx/schemeshard/schemeshard_import.cpp b/ydb/core/tx/schemeshard/schemeshard_import.cpp
index 6bad76e26e8..4d6d2838552 100644
--- a/ydb/core/tx/schemeshard/schemeshard_import.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_import.cpp
@@ -1,6 +1,7 @@
#include "schemeshard_import.h"
#include "schemeshard_import_helpers.h"
#include "schemeshard_impl.h"
+#include "schemeshard_import_getters.h"
#include <util/generic/xrange.h>
@@ -262,6 +263,10 @@ void TSchemeShard::Handle(TEvImport::TEvListImportsRequest::TPtr& ev, const TAct
Execute(CreateTxListImports(ev), ctx);
}
+void TSchemeShard::Handle(TEvImport::TEvListObjectsInS3ExportRequest::TPtr& ev, const TActorContext&) {
+ Register(CreateListObjectsInS3ExportGetter(std::move(ev)));
+}
+
void TSchemeShard::Handle(TEvPrivate::TEvImportSchemeReady::TPtr& ev, const TActorContext& ctx) {
Execute(CreateTxProgressImport(ev), ctx);
}
diff --git a/ydb/core/tx/schemeshard/schemeshard_import.h b/ydb/core/tx/schemeshard/schemeshard_import.h
index 41e45e57689..0cc005a8755 100644
--- a/ydb/core/tx/schemeshard/schemeshard_import.h
+++ b/ydb/core/tx/schemeshard/schemeshard_import.h
@@ -19,6 +19,8 @@ struct TEvImport {
EvForgetImportResponse,
EvListImportsRequest,
EvListImportsResponse,
+ EvListObjectsInS3ExportRequest,
+ EvListObjectsInS3ExportResponse,
EvEnd
};
@@ -163,6 +165,12 @@ struct TEvImport {
DECLARE_EVENT_CLASS(EvListImportsResponse) {
};
+ DECLARE_EVENT_CLASS(EvListObjectsInS3ExportRequest) {
+ };
+
+ DECLARE_EVENT_CLASS(EvListObjectsInS3ExportResponse) {
+ };
+
#undef DECLARE_EVENT_CLASS
}; // TEvImport
diff --git a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp
index 88fcac80971..df2c1a83680 100644
--- a/ydb/core/tx/schemeshard/schemeshard_import__create.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_import__create.cpp
@@ -267,8 +267,8 @@ private:
importInfo.Items.reserve(settings.items().size());
for (ui32 itemIdx : xrange(settings.items().size())) {
- const auto& dstPath = settings.items(itemIdx).destination_path();
- if (!dstPaths.insert(dstPath).second) {
+ const TString& dstPath = settings.items(itemIdx).destination_path();
+ if (!dstPaths.insert(NBackup::NormalizeItemPath(dstPath)).second) {
explain = TStringBuilder() << "Duplicate destination_path: " << dstPath;
return false;
}
@@ -278,8 +278,8 @@ private:
}
auto& item = importInfo.Items.emplace_back(dstPath);
- item.SrcPrefix = settings.items(itemIdx).source_prefix();
- item.SrcPath = settings.items(itemIdx).source_path();
+ item.SrcPrefix = NBackup::NormalizeExportPrefix(settings.items(itemIdx).source_prefix());
+ item.SrcPath = NBackup::NormalizeItemPath(settings.items(itemIdx).source_path());
}
return true;
@@ -1079,8 +1079,8 @@ private:
} else {
dstRoot = CanonizePath(importInfo->Settings.destination_path());
}
- TString sourcePrefix = importInfo->Settings.source_prefix();
- if (sourcePrefix && sourcePrefix.back() != '/') {
+ TString sourcePrefix = NBackup::NormalizeExportPrefix(importInfo->Settings.source_prefix());
+ if (sourcePrefix) {
sourcePrefix.push_back('/');
}
auto combineDstPath = [&](const TString& path) -> TString {
@@ -1092,9 +1092,7 @@ private:
}
};
auto init = [&](const NBackup::TSchemaMapping::TItem& schemaMappingItem, NSchemeShard::TImportInfo::TItem& item) {
- TStringBuf exportPrefix(schemaMappingItem.ExportPrefix);
- exportPrefix.SkipPrefix("/");
- item.SrcPrefix = TStringBuilder() << sourcePrefix << exportPrefix;
+ item.SrcPrefix = TStringBuilder() << sourcePrefix << NBackup::NormalizeItemPrefix(schemaMappingItem.ExportPrefix);
item.SrcPath = schemaMappingItem.ObjectPath;
item.ExportItemIV = schemaMappingItem.IV;
};
@@ -1115,19 +1113,18 @@ private:
TMapping schemaMappingObjectPathIndex;
for (size_t i = 0; i < importInfo->SchemaMapping->Items.size(); ++i) {
const auto& schemaMappingItem = importInfo->SchemaMapping->Items[i];
- schemaMappingPrefixIndex[schemaMappingItem.ExportPrefix] = i;
- schemaMappingObjectPathIndex[CanonizePath(schemaMappingItem.ObjectPath)] = i;
+ schemaMappingPrefixIndex[NBackup::NormalizeItemPrefix(schemaMappingItem.ExportPrefix)] = i;
+ schemaMappingObjectPathIndex[NBackup::NormalizeItemPath(schemaMappingItem.ObjectPath)] = i;
}
for (auto& item : importInfo->Items) {
- TString dstPath = CanonizePath(item.DstPathName);
TMapping::iterator mappingIt;
if (item.SrcPrefix) {
- mappingIt = schemaMappingPrefixIndex.find(item.SrcPrefix);
+ mappingIt = schemaMappingPrefixIndex.find(NBackup::NormalizeItemPrefix(item.SrcPrefix));
if (mappingIt == schemaMappingPrefixIndex.end()) {
return CancelAndPersist(db, importInfo, -1, {}, TStringBuilder() << "cannot find prefix \"" << item.SrcPrefix << "\" in schema mapping");
}
} else if (item.SrcPath) {
- mappingIt = schemaMappingObjectPathIndex.find(CanonizePath(item.SrcPath));
+ mappingIt = schemaMappingObjectPathIndex.find(NBackup::NormalizeItemPath(item.SrcPath));
if (mappingIt == schemaMappingObjectPathIndex.end()) {
return CancelAndPersist(db, importInfo, -1, {}, TStringBuilder() << "cannot find source path \"" << item.SrcPath << "\" in schema mapping");
}
diff --git a/ydb/core/tx/schemeshard/schemeshard_import_getters.cpp b/ydb/core/tx/schemeshard/schemeshard_import_getters.cpp
index 595aab2ee07..f6a89453f80 100644
--- a/ydb/core/tx/schemeshard/schemeshard_import_getters.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_import_getters.cpp
@@ -19,6 +19,8 @@
#include <util/string/subst.h>
+#include <algorithm>
+
namespace NKikimr {
namespace NSchemeShard {
@@ -31,18 +33,43 @@ using namespace Aws;
static constexpr TDuration MaxDelay = TDuration::Minutes(10);
+struct TGetterSettings {
+ NWrappers::IExternalStorageConfig::TPtr ExternalStorageConfig;
+ ui32 Retries;
+ TMaybe<NBackup::TEncryptionKey> Key;
+ TMaybe<NBackup::TEncryptionIV> IV;
+
+ static TGetterSettings FromImportInfo(const TImportInfo::TPtr& importInfo, TMaybe<NBackup::TEncryptionIV> iv) {
+ TGetterSettings settings;
+ settings.ExternalStorageConfig.reset(new NWrappers::NExternalStorage::TS3ExternalStorageConfig(importInfo->Settings));
+ settings.Retries = importInfo->Settings.number_of_retries();
+ if (importInfo->Settings.has_encryption_settings()) {
+ settings.Key = NBackup::TEncryptionKey(importInfo->Settings.encryption_settings().symmetric_key().key());
+ }
+ settings.IV = std::move(iv);
+ return settings;
+ }
+
+ static TGetterSettings FromRequest(const TEvImport::TEvListObjectsInS3ExportRequest::TPtr& ev) {
+ TGetterSettings settings;
+ settings.ExternalStorageConfig.reset(new NWrappers::NExternalStorage::TS3ExternalStorageConfig(ev->Get()->Record.settings()));
+ settings.Retries = ev->Get()->Record.settings().number_of_retries();
+ if (ev->Get()->Record.settings().has_encryption_settings()) {
+ settings.Key = NBackup::TEncryptionKey(ev->Get()->Record.settings().encryption_settings().symmetric_key().key());
+ }
+ return settings;
+ }
+};
+
template <class TDerived>
class TGetterFromS3 : public TActorBootstrapped<TDerived> {
protected:
- explicit TGetterFromS3(TImportInfo::TPtr importInfo, TMaybe<NBackup::TEncryptionIV> iv)
- : ImportInfo(std::move(importInfo))
- , ExternalStorageConfig(new NWrappers::NExternalStorage::TS3ExternalStorageConfig(ImportInfo->Settings))
- , IV(std::move(iv))
- , Retries(ImportInfo->Settings.number_of_retries())
+ explicit TGetterFromS3(TGetterSettings&& settings)
+ : ExternalStorageConfig(std::move(settings.ExternalStorageConfig))
+ , Key(std::move(settings.Key))
+ , IV(std::move(settings.IV))
+ , Retries(settings.Retries)
{
- if (ImportInfo->Settings.has_encryption_settings()) {
- Key = NBackup::TEncryptionKey(ImportInfo->Settings.encryption_settings().symmetric_key().key());
- }
}
void HeadObject(const TString& key, bool autoAddEncSuffix = true) {
@@ -89,7 +116,7 @@ protected:
}
TString GetKey(TString key, bool autoAddEncSuffix = true) {
- if (autoAddEncSuffix && ImportInfo->Settings.has_encryption_settings()) {
+ if (autoAddEncSuffix && Key) {
key += ".enc";
}
return key;
@@ -114,10 +141,28 @@ protected:
Delay = Min(Delay * ++Attempt, MaxDelay);
this->Schedule(Delay, new TEvents::TEvWakeup());
} else {
- Reply(false, TStringBuilder() << "S3 error: " << error.GetMessage().c_str());
+ Reply(error.ShouldRetry() ? Ydb::StatusIds::EXTERNAL_ERROR : Ydb::StatusIds::BAD_REQUEST, TStringBuilder() << "S3 error: " << error.GetMessage().c_str());
}
}
+ template <typename TResult>
+ bool IsNoSuchKeyError(const TResult& result) {
+ if (result.IsSuccess()) {
+ return false;
+ }
+ const auto& err = result.GetError();
+ if (err.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) {
+ return true;
+ }
+ if (err.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND) {
+ return true;
+ }
+ if (err.GetExceptionName() == "NoSuchKey") {
+ return true;
+ }
+ return false;
+ }
+
void ResetRetries() {
Attempt = 0;
}
@@ -132,7 +177,7 @@ protected:
result.assign(buffer.Data(), buffer.Size());
return true;
} catch (const std::exception& ex) {
- Reply(false, ex.what());
+ Reply(Ydb::StatusIds::BAD_REQUEST, ex.what());
return false;
}
}
@@ -148,7 +193,7 @@ protected:
result.assign(buffer.Data(), buffer.Size());
return true;
} catch (const std::exception& ex) {
- Reply(false, ex.what());
+ Reply(Ydb::StatusIds::BAD_REQUEST, ex.what());
return false;
}
}
@@ -156,10 +201,9 @@ protected:
return true;
}
- virtual void Reply(bool success = true, const TString& error = TString()) = 0;
+ virtual void Reply(Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS, const TString& error = TString()) = 0;
protected:
- TImportInfo::TPtr ImportInfo;
NWrappers::IExternalStorageConfig::TPtr ExternalStorageConfig;
TActorId Client;
TMaybe<NBackup::TEncryptionKey> Key;
@@ -388,11 +432,11 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
} else if (IsTopic(SchemeKey)) {
Ydb::Topic::CreateTopicRequest request;
if (!google::protobuf::TextFormat::ParseFromString(content, &request)) {
- return Reply(false, "Cannot parse topic scheme");
+ return Reply(Ydb::StatusIds::BAD_REQUEST, "Cannot parse topic scheme");
}
item.Topic = request;
} else if (!google::protobuf::TextFormat::ParseFromString(content, &item.Scheme)) {
- return Reply(false, "Cannot parse scheme");
+ return Reply(Ydb::StatusIds::BAD_REQUEST, "Cannot parse scheme");
}
auto nextStep = [this]() {
@@ -436,7 +480,7 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
Ydb::Scheme::ModifyPermissionsRequest permissions;
if (!google::protobuf::TextFormat::ParseFromString(content, &permissions)) {
- return Reply(false, "Cannot parse permissions");
+ return Reply(Ydb::StatusIds::BAD_REQUEST, "Cannot parse permissions");
}
item.Permissions = std::move(permissions);
@@ -465,7 +509,7 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
TString expectedChecksum = msg.Body.substr(0, msg.Body.find(' '));
if (expectedChecksum != CurrentObjectChecksum) {
- return Reply(false, TStringBuilder() << "Checksum mismatch for " << CurrentObjectKey
+ return Reply(Ydb::StatusIds::BAD_REQUEST, TStringBuilder() << "Checksum mismatch for " << CurrentObjectKey
<< " expected# " << expectedChecksum
<< ", got# " << CurrentObjectChecksum);
}
@@ -499,7 +543,7 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
Ydb::Table::ChangefeedDescription changefeed;
if (!google::protobuf::TextFormat::ParseFromString(content, &changefeed)) {
- return Reply(false, "Cannot parse сhangefeed");
+ return Reply(Ydb::StatusIds::BAD_REQUEST, "Cannot parse changefeed");
}
*item.Changefeeds.MutableChangefeeds(IndexDownloadedChangefeed)->MutableChangefeed() = std::move(changefeed);
@@ -542,7 +586,7 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
Ydb::Topic::DescribeTopicResult topic;
if (!google::protobuf::TextFormat::ParseFromString(content, &topic)) {
- return Reply(false, "Cannot parse topic");
+ return Reply(Ydb::StatusIds::BAD_REQUEST, "Cannot parse topic");
}
*item.Changefeeds.MutableChangefeeds(IndexDownloadedChangefeed)->MutableTopic() = std::move(topic);
@@ -597,10 +641,10 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
} else {
Reply();
}
-
}
- void Reply(bool success = true, const TString& error = TString()) override {
+ void Reply(Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS, const TString& error = TString()) override {
+ const bool success = (statusCode == Ydb::StatusIds::SUCCESS);
LOG_I("Reply"
<< ": self# " << SelfId()
<< ", success# " << success
@@ -665,7 +709,8 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
public:
explicit TSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx, TMaybe<NBackup::TEncryptionIV> iv)
- : TGetterFromS3<TSchemeGetter>(std::move(importInfo), std::move(iv))
+ : TGetterFromS3<TSchemeGetter>(TGetterSettings::FromImportInfo(importInfo, std::move(iv)))
+ , ImportInfo(std::move(importInfo))
, ReplyTo(replyTo)
, ItemIdx(itemIdx)
, MetadataKey(MetadataKeyFromSettings(*ImportInfo, itemIdx))
@@ -743,6 +788,7 @@ public:
}
private:
+ TImportInfo::TPtr ImportInfo;
const TActorId ReplyTo;
const ui32 ItemIdx;
@@ -856,14 +902,15 @@ class TSchemaMappingGetter : public TGetterFromS3<TSchemaMappingGetter> {
ImportInfo->SchemaMapping.ConstructInPlace();
TString error;
if (!ImportInfo->SchemaMapping->Deserialize(content, error)) {
- Reply(false, error);
+ Reply(Ydb::StatusIds::BAD_REQUEST, error);
return;
}
Reply();
}
- void Reply(bool success = true, const TString& error = TString()) override {
+ void Reply(Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS, const TString& error = TString()) override {
+ const bool success = (statusCode == Ydb::StatusIds::SUCCESS);
LOG_I("Reply"
<< ": self# " << SelfId()
<< ", success# " << success
@@ -890,12 +937,12 @@ class TSchemaMappingGetter : public TGetterFromS3<TSchemaMappingGetter> {
bool ProcessMetadata(const TString& content) {
NJson::TJsonValue json;
if (!NJson::ReadJsonTree(content, &json)) {
- Reply(false, "Failed to parse metadata json");
+ Reply(Ydb::StatusIds::BAD_REQUEST, "Failed to parse metadata json");
return false;
}
const NJson::TJsonValue& kind = json["kind"];
if (kind.GetString() != "SchemaMappingV0") {
- Reply(false, TStringBuilder() << "Unknown kind of metadata json: " << kind.GetString());
+ Reply(Ydb::StatusIds::BAD_REQUEST, TStringBuilder() << "Unknown kind of metadata json: " << kind.GetString());
return false;
}
return true;
@@ -903,7 +950,8 @@ class TSchemaMappingGetter : public TGetterFromS3<TSchemaMappingGetter> {
public:
TSchemaMappingGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo)
- : TGetterFromS3<TSchemaMappingGetter>(std::move(importInfo), Nothing())
+ : TGetterFromS3<TSchemaMappingGetter>(TGetterSettings::FromImportInfo(importInfo, Nothing()))
+ , ImportInfo(std::move(importInfo))
, ReplyTo(replyTo)
, MetadataKey(SchemaMappingMetadataKeyFromSettings(*ImportInfo))
, SchemaMappingKey(SchemaMappingKeyFromSettings(*ImportInfo))
@@ -936,11 +984,276 @@ public:
}
private:
+ TImportInfo::TPtr ImportInfo;
const TActorId ReplyTo;
const TString MetadataKey;
const TString SchemaMappingKey;
}; // TSchemaMappingGetter
+class TListObjectsInS3ExportGetter : public TGetterFromS3<TListObjectsInS3ExportGetter> {
+ class TPathFilter {
+ public:
+ void Build(const Ydb::Import::ListObjectsInS3ExportSettings& settings) {
+ for (const auto& item : settings.items()) {
+ TString path = NBackup::NormalizeItemPath(item.path());
+ if (path) {
+ Paths.emplace(path);
+ PathPrefixes.emplace_back(path + "/");
+ }
+ }
+ }
+
+ bool Match(const TString& path) const {
+ if (Paths.empty()) {
+ return true;
+ }
+
+ if (Paths.contains(path)) {
+ return true;
+ }
+
+ for (const TString& prefix : PathPrefixes) {
+ if (path.StartsWith(prefix)) { // So this path is contained in a directory that is specified by user in request
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ private:
+ std::vector<TString> PathPrefixes;
+ THashSet<TString> Paths;
+ };
+public:
+ TListObjectsInS3ExportGetter(TEvImport::TEvListObjectsInS3ExportRequest::TPtr&& ev)
+ : TGetterFromS3<TListObjectsInS3ExportGetter>(TGetterSettings::FromRequest(ev))
+ , Request(std::move(ev))
+ {
+ }
+
+ void Bootstrap() {
+ if (ParseParameters()) {
+ CreateClient();
+ DownloadSchemaMapping();
+ }
+ }
+
+ bool ParseParameters() {
+ const auto& req = Request->Get()->Record;
+ if (req.GetPageSize() < 0) {
+ Reply(Ydb::StatusIds::BAD_REQUEST, "Page size should be greater than or equal to 0");
+ return false;
+ }
+ PageSize = static_cast<size_t>(req.GetPageSize());
+ if (req.GetPageToken()) {
+ if (!TryFromString(req.GetPageToken(), StartPos)) {
+ Reply(Ydb::StatusIds::BAD_REQUEST, "Failed to parse page token");
+ return false;
+ }
+ if (req.GetPageSize() == 0) {
+ Reply(Ydb::StatusIds::BAD_REQUEST, "Page size should be greater than 0");
+ return false;
+ }
+ }
+ if (NBackup::NormalizeExportPrefix(Request->Get()->Record.GetSettings().prefix()).empty()) {
+ Reply(Ydb::StatusIds::BAD_REQUEST, "Empty S3 prefix specified");
+ return false;
+ }
+ return true;
+ }
+
+ void DownloadSchemaMapping() {
+ ResetRetries();
+ Download(GetSchemaMappingKey());
+ Become(&TThis::StateDownloadSchemaMapping);
+ }
+
+ void HandleSchemaMapping(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) {
+ const auto& result = ev->Get()->Result;
+
+ LOG_D("HandleSchemaMapping TEvExternalStorage::TEvHeadObjectResponse"
+ << ": self# " << SelfId()
+ << ", result# " << result);
+
+ if (IsNoSuchKeyError(result)) {
+ return ListObjectsInS3Prefix();
+ }
+
+ if (!CheckResult(result, "HeadObject")) {
+ return;
+ }
+
+ GetObject(GetSchemaMappingKey(), result.GetResult().GetContentLength());
+ }
+
+ void HandleSchemaMapping(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) {
+ const auto& msg = *ev->Get();
+ const auto& result = msg.Result;
+
+ LOG_D("HandleSchemaMapping TEvExternalStorage::TEvGetObjectResponse"
+ << ": self# " << SelfId()
+ << ", result# " << result);
+
+ if (IsNoSuchKeyError(result)) {
+ return ListObjectsInS3Prefix();
+ }
+
+ if (!CheckResult(result, "GetObject")) {
+ return;
+ }
+
+ TString content;
+ if (!MaybeDecryptAndSaveIV(msg.Body, content)) {
+ return;
+ }
+
+ LOG_T("Trying to parse schema mapping"
+ << ": self# " << SelfId()
+ << ", schemaMappingKey# " << GetSchemaMappingKey()
+ << ", body# " << SubstGlobalCopy(content, "\n", "\\n"));
+
+ TString error;
+ NBackup::TSchemaMapping schemaMapping;
+ if (!schemaMapping.Deserialize(content, error)) {
+ Reply(Ydb::StatusIds::BAD_REQUEST, error);
+ return;
+ }
+
+ ProcessItemsAndReply(std::move(schemaMapping.Items));
+ }
+
+ void ListObjectsInS3Prefix() {
+ ResetRetries();
+ ListObjects(GetExportPrefix());
+ Become(&TThis::StateListObjects);
+ }
+
+ void HandleListObjects(TEvExternalStorage::TEvListObjectsResponse::TPtr& ev) {
+ const auto& result = ev.Get()->Get()->Result;
+ LOG_D("HandleListObjects TEvExternalStorage::TEvListObjectResponse"
+ << ": self# " << SelfId()
+ << ", result# " << result);
+
+ if (!CheckResult(result, "ListObjects")) {
+ return;
+ }
+
+ const auto& objects = result.GetResult().GetContents();
+ const TString prefix = GetExportPrefix();
+ const TString suffix = TString("/metadata.json") + (Key ? ".enc" : "");
+ std::vector<NBackup::TSchemaMapping::TItem> items;
+ items.reserve(objects.size());
+ for (const auto& obj : objects) {
+ TStringBuf key(obj.GetKey());
+ // Skip prefix
+ // Prefix also may be added with the bucket name here, so cut bucket name also
+ size_t prefixPos = key.find(prefix);
+ if (prefixPos == TStringBuf::npos) {
+ LOG_D("Unexpected key found: " << key);
+ continue;
+ }
+ key = key.SubString(prefixPos + prefix.size(), TStringBuf::npos);
+
+ // Every backup object has metadata.json.
+ // Process only keys with metadata.json suffix.
+ if (key.ChopSuffix(suffix)) {
+ TString keyStr(key);
+ items.emplace_back(NBackup::TSchemaMapping::TItem{
+ .ExportPrefix = keyStr,
+ .ObjectPath = keyStr,
+ });
+ }
+ }
+
+ ProcessItemsAndReply(std::move(items));
+ }
+
+ STATEFN(StateDownloadSchemaMapping) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandleSchemaMapping);
+ hFunc(TEvExternalStorage::TEvGetObjectResponse, HandleSchemaMapping);
+
+ sFunc(TEvents::TEvWakeup, DownloadSchemaMapping);
+ }
+ }
+
+ STATEFN(StateListObjects) {
+ switch (ev->GetTypeRewrite()) {
+ hFunc(TEvExternalStorage::TEvListObjectsResponse, HandleListObjects);
+
+ sFunc(TEvents::TEvWakeup, ListObjectsInS3Prefix);
+ sFunc(TEvents::TEvPoisonPill, PassAway);
+ }
+ }
+
+ void Reply(Ydb::StatusIds::StatusCode statusCode = Ydb::StatusIds::SUCCESS, const TString& error = TString()) override {
+ LOG_I("Reply"
+ << ": self# " << SelfId()
+ << ", status# " << static_cast<int>(statusCode)
+ << ", error# " << error);
+
+ auto result = MakeHolder<TEvImport::TEvListObjectsInS3ExportResponse>();
+ result->Record.set_status(statusCode);
+ if (error) {
+ result->Record.add_issues()->set_message(error);
+ }
+ if (statusCode == Ydb::StatusIds::SUCCESS) {
+ result->Record.mutable_result()->Swap(&Result);
+ }
+ Send(Request->Sender, std::move(result));
+ PassAway();
+ }
+
+ TString GetSchemaMappingKey() const {
+ return TStringBuilder() << NBackup::NormalizeExportPrefix(Request->Get()->Record.GetSettings().prefix()) << "/SchemaMapping/mapping.json";
+ }
+
+ TString GetExportPrefix() const {
+ return NBackup::NormalizeExportPrefix(Request->Get()->Record.GetSettings().prefix()) + "/";
+ }
+
+ void ProcessItemsAndReply(std::vector<NBackup::TSchemaMapping::TItem>&& items) {
+ std::sort(items.begin(), items.end(), [](const NBackup::TSchemaMapping::TItem& i1, const NBackup::TSchemaMapping::TItem& i2) {
+ return i1.ObjectPath < i2.ObjectPath;
+ });
+
+ PathFilter.Build(Request->Get()->Record.GetSettings());
+
+ size_t pos = 0;
+ for (const auto& item : items) {
+ if (!PathFilter.Match(item.ObjectPath)) {
+ continue;
+ }
+
+ if (PageSize && pos >= StartPos + PageSize) { // Calc only items that suit filter
+ NextPos = pos;
+ break;
+ }
+
+ if (pos >= StartPos) {
+ auto* result = Result.add_items();
+ result->set_path(item.ObjectPath);
+ result->set_prefix(item.ExportPrefix);
+ }
+
+ ++pos;
+ }
+ if (NextPos) {
+ Result.set_next_page_token(ToString(NextPos));
+ }
+ Reply();
+ }
+
+private:
+ TEvImport::TEvListObjectsInS3ExportRequest::TPtr Request;
+ Ydb::Import::ListObjectsInS3ExportResult Result;
+ size_t StartPos = 0;
+ size_t PageSize = 0;
+ size_t NextPos = 0;
+ TPathFilter PathFilter;
+};
+
IActor* CreateSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo, ui32 itemIdx, TMaybe<NBackup::TEncryptionIV> iv) {
return new TSchemeGetter(replyTo, std::move(importInfo), itemIdx, std::move(iv));
}
@@ -949,5 +1262,9 @@ IActor* CreateSchemaMappingGetter(const TActorId& replyTo, TImportInfo::TPtr imp
return new TSchemaMappingGetter(replyTo, std::move(importInfo));
}
+IActor* CreateListObjectsInS3ExportGetter(TEvImport::TEvListObjectsInS3ExportRequest::TPtr&& ev) {
+ return new TListObjectsInS3ExportGetter(std::move(ev));
+}
+
} // NSchemeShard
} // NKikimr
diff --git a/ydb/core/tx/schemeshard/schemeshard_import_getters.h b/ydb/core/tx/schemeshard/schemeshard_import_getters.h
index 32b108c20e9..dddd9427573 100644
--- a/ydb/core/tx/schemeshard/schemeshard_import_getters.h
+++ b/ydb/core/tx/schemeshard/schemeshard_import_getters.h
@@ -2,6 +2,7 @@
#include "defs.h"
#include "schemeshard_info_types.h"
+#include "schemeshard_import.h"
namespace NKikimr {
namespace NSchemeShard {
@@ -10,5 +11,7 @@ IActor* CreateSchemeGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo
IActor* CreateSchemaMappingGetter(const TActorId& replyTo, TImportInfo::TPtr importInfo);
+IActor* CreateListObjectsInS3ExportGetter(TEvImport::TEvListObjectsInS3ExportRequest::TPtr&& ev);
+
} // NSchemeShard
} // NKikimr
diff --git a/ydb/core/wrappers/s3_storage_config.cpp b/ydb/core/wrappers/s3_storage_config.cpp
index 4d0c61ce441..bab3941d926 100644
--- a/ydb/core/wrappers/s3_storage_config.cpp
+++ b/ydb/core/wrappers/s3_storage_config.cpp
@@ -46,6 +46,33 @@ namespace {
namespace NPrivate {
+template <class TMessage, class TEnum>
+Aws::Http::Scheme ParseSchemeImpl(TEnum scheme, bool abortOnFailure = true) {
+ switch (scheme) {
+ case TMessage::HTTP:
+ return Aws::Http::Scheme::HTTP;
+ case TMessage::HTTPS:
+ return Aws::Http::Scheme::HTTPS;
+ default:
+ if (abortOnFailure) {
+ Y_ABORT("Unknown scheme");
+ }
+ return Aws::Http::Scheme::HTTP;
+ }
+}
+
+Aws::Http::Scheme ParseScheme(NKikimrSchemeOp::TS3Settings::EScheme scheme, bool abortOnFailure = true) {
+ return ParseSchemeImpl<NKikimrSchemeOp::TS3Settings>(scheme, abortOnFailure);
+}
+
+Aws::Http::Scheme ParseScheme(Ydb::Import::ImportFromS3Settings::Scheme scheme, bool abortOnFailure = true) {
+ return ParseSchemeImpl<Ydb::Import::ImportFromS3Settings>(scheme, abortOnFailure);
+}
+
+Aws::Http::Scheme ParseScheme(Ydb::Export::ExportToS3Settings::Scheme scheme, bool abortOnFailure = true) {
+ return ParseSchemeImpl<Ydb::Export::ExportToS3Settings>(scheme, abortOnFailure);
+}
+
template <typename TSettings>
Aws::Client::ClientConfiguration ConfigFromSettings(const TSettings& settings) {
Aws::Client::ClientConfiguration config;
@@ -60,16 +87,7 @@ Aws::Client::ClientConfiguration ConfigFromSettings(const TSettings& settings) {
config.connectTimeoutMs = 10000;
config.maxConnections = threadsCount;
- switch (settings.scheme()) {
- case TSettings::HTTP:
- config.scheme = Aws::Http::Scheme::HTTP;
- break;
- case TSettings::HTTPS:
- config.scheme = Aws::Http::Scheme::HTTPS;
- break;
- default:
- Y_ABORT("Unknown scheme");
- }
+ config.scheme = NPrivate::ParseScheme(settings.scheme());
return config;
}
@@ -104,16 +122,7 @@ Aws::Client::ClientConfiguration TS3ExternalStorageConfig::ConfigFromSettings(co
config.maxConnections = settings.HasMaxConnectionsCount() ? settings.GetMaxConnectionsCount() : settings.GetExecutorThreadsCount();
config.caPath = "/etc/ssl/certs";
- switch (settings.GetScheme()) {
- case NKikimrSchemeOp::TS3Settings::HTTP:
- config.scheme = Aws::Http::Scheme::HTTP;
- break;
- case NKikimrSchemeOp::TS3Settings::HTTPS:
- config.scheme = Aws::Http::Scheme::HTTPS;
- break;
- default:
- Y_ABORT("Unknown scheme");
- }
+ config.scheme = NPrivate::ParseScheme(settings.GetScheme());
if (settings.HasRegion()) {
config.region = settings.GetRegion();
@@ -127,16 +136,7 @@ Aws::Client::ClientConfiguration TS3ExternalStorageConfig::ConfigFromSettings(co
config.proxyHost = settings.GetProxyHost();
config.proxyPort = settings.GetProxyPort();
- switch (settings.GetProxyScheme()) {
- case NKikimrSchemeOp::TS3Settings::HTTP:
- config.proxyScheme = Aws::Http::Scheme::HTTP;
- break;
- case NKikimrSchemeOp::TS3Settings::HTTPS:
- config.proxyScheme = Aws::Http::Scheme::HTTPS;
- break;
- default:
- break;
- }
+ config.proxyScheme = NPrivate::ParseScheme(settings.GetProxyScheme(), false);
}
return config;
@@ -146,6 +146,10 @@ Aws::Client::ClientConfiguration TS3ExternalStorageConfig::ConfigFromSettings(co
return NPrivate::ConfigFromSettings(settings);
}
+Aws::Client::ClientConfiguration TS3ExternalStorageConfig::ConfigFromSettings(const Ydb::Import::ListObjectsInS3ExportSettings& settings) {
+ return NPrivate::ConfigFromSettings(settings);
+}
+
Aws::Client::ClientConfiguration TS3ExternalStorageConfig::ConfigFromSettings(const Ydb::Export::ExportToS3Settings& settings) {
return NPrivate::ConfigFromSettings(settings);
}
@@ -158,6 +162,10 @@ Aws::Auth::AWSCredentials TS3ExternalStorageConfig::CredentialsFromSettings(cons
return NPrivate::CredentialsFromSettings(settings);
}
+Aws::Auth::AWSCredentials TS3ExternalStorageConfig::CredentialsFromSettings(const Ydb::Import::ListObjectsInS3ExportSettings& settings) {
+ return NPrivate::CredentialsFromSettings(settings);
+}
+
Aws::Auth::AWSCredentials TS3ExternalStorageConfig::CredentialsFromSettings(const Ydb::Export::ExportToS3Settings& settings) {
return NPrivate::CredentialsFromSettings(settings);
}
@@ -178,6 +186,14 @@ TS3ExternalStorageConfig::TS3ExternalStorageConfig(const Ydb::Import::ImportFrom
Bucket = settings.bucket();
}
+TS3ExternalStorageConfig::TS3ExternalStorageConfig(const Ydb::Import::ListObjectsInS3ExportSettings& settings)
+ : Config(ConfigFromSettings(settings))
+ , Credentials(CredentialsFromSettings(settings))
+ , UseVirtualAddressing(!settings.disable_virtual_addressing())
+{
+ Bucket = settings.bucket();
+}
+
TS3ExternalStorageConfig::TS3ExternalStorageConfig(const Ydb::Export::ExportToS3Settings& settings)
: Config(ConfigFromSettings(settings))
, Credentials(CredentialsFromSettings(settings))
diff --git a/ydb/core/wrappers/s3_storage_config.h b/ydb/core/wrappers/s3_storage_config.h
index cc5bbfeaf09..f6f3b10d39b 100644
--- a/ydb/core/wrappers/s3_storage_config.h
+++ b/ydb/core/wrappers/s3_storage_config.h
@@ -26,6 +26,8 @@ private:
static Aws::Auth::AWSCredentials CredentialsFromSettings(const NKikimrSchemeOp::TS3Settings& settings);
static Aws::Client::ClientConfiguration ConfigFromSettings(const Ydb::Import::ImportFromS3Settings& settings);
static Aws::Auth::AWSCredentials CredentialsFromSettings(const Ydb::Import::ImportFromS3Settings& settings);
+ static Aws::Client::ClientConfiguration ConfigFromSettings(const Ydb::Import::ListObjectsInS3ExportSettings& settings);
+ static Aws::Auth::AWSCredentials CredentialsFromSettings(const Ydb::Import::ListObjectsInS3ExportSettings& settings);
static Aws::Client::ClientConfiguration ConfigFromSettings(const Ydb::Export::ExportToS3Settings& settings);
static Aws::Auth::AWSCredentials CredentialsFromSettings(const Ydb::Export::ExportToS3Settings& settings);
@@ -46,6 +48,7 @@ public:
TS3ExternalStorageConfig(const NKikimrSchemeOp::TS3Settings& settings);
TS3ExternalStorageConfig(const Ydb::Import::ImportFromS3Settings& settings);
+ TS3ExternalStorageConfig(const Ydb::Import::ListObjectsInS3ExportSettings& settings);
TS3ExternalStorageConfig(const Ydb::Export::ExportToS3Settings& settings);
TS3ExternalStorageConfig(const Aws::Auth::AWSCredentials& credentials, const Aws::Client::ClientConfiguration& config, const TString& bucket);
};
diff --git a/ydb/services/ydb/backup_ut/list_objects_in_s3_export_ut.cpp b/ydb/services/ydb/backup_ut/list_objects_in_s3_export_ut.cpp
new file mode 100644
index 00000000000..7a9239fc009
--- /dev/null
+++ b/ydb/services/ydb/backup_ut/list_objects_in_s3_export_ut.cpp
@@ -0,0 +1,219 @@
+#include "s3_backup_test_base.h"
+#include <ydb/core/backup/common/metadata.h>
+
+#include <fmt/format.h>
+
+using namespace NYdb;
+using namespace fmt::literals;
+
+class TListObjectsInS3ExportTestFixture : public TS3BackupTestFixture {
+ void SetUp(NUnitTest::TTestContext& /* context */) override {
+ auto res = YdbQueryClient().ExecuteQuery(R"sql(
+ CREATE TABLE `/Root/Table0` (
+ key Uint32 NOT NULL,
+ value String,
+ PRIMARY KEY (key)
+ );
+
+ CREATE TABLE `/Root/dir1/Table1` (
+ key Uint32 NOT NULL,
+ value String,
+ PRIMARY KEY (key)
+ );
+
+ CREATE TABLE `/Root/dir1/dir2/Table2` (
+ key Uint32 NOT NULL,
+ value String,
+ PRIMARY KEY (key)
+ );
+ )sql", NQuery::TTxControl::NoTx()).GetValueSync();
+ UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
+
+ // Empty dir
+ auto mkdir = YdbSchemeClient().MakeDirectory("/Root/dir1/dir2/dir3").GetValueSync();
+ UNIT_ASSERT_C(mkdir.IsSuccess(), mkdir.GetIssues().ToString());
+ }
+
+ void TearDown(NUnitTest::TTestContext& /* context */) override {
+ }
+};
+
+Y_UNIT_TEST_SUITE_F(ListObjectsInS3Export, TListObjectsInS3ExportTestFixture) {
+ Y_UNIT_TEST(ExportWithSchemaMapping) {
+ {
+ NExport::TExportToS3Settings exportSettings = MakeExportSettings("", "Prefix//");
+ auto res = YdbExportClient().ExportToS3(exportSettings).GetValueSync();
+ WaitOpSuccess(res);
+ }
+
+ ValidateListObjectInS3Export({
+ {"Table0", "Table0"},
+ {"dir1/Table1", "dir1/Table1"},
+ {"dir1/dir2/Table2", "dir1/dir2/Table2"},
+ }, "Prefix");
+ }
+
+ Y_UNIT_TEST(ExportWithoutSchemaMapping) {
+ {
+ NExport::TExportToS3Settings exportSettings = MakeExportSettings("", "");
+ exportSettings
+ .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/Root/Table0", .Dst = "/Prefix/t0"})
+ .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/Root/dir1/Table1", .Dst = "/Prefix/d1/t1"})
+ .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/Root/dir1/dir2/Table2", .Dst = "/Prefix/d1/d2/t2"});
+ auto res = YdbExportClient().ExportToS3(exportSettings).GetValueSync();
+ WaitOpSuccess(res);
+ }
+
+ ValidateListObjectInS3Export({
+ {"t0", "t0"},
+ {"d1/t1", "d1/t1"},
+ {"d1/d2/t2", "d1/d2/t2"},
+ }, "Prefix");
+
+ ValidateListObjectInS3Export({
+ {"t1", "t1"},
+ {"d2/t2", "d2/t2"},
+ }, "Prefix/d1");
+ }
+
+ Y_UNIT_TEST(ExportWithEncryption) {
+ {
+ NExport::TExportToS3Settings exportSettings = MakeExportSettings("", "Prefix");
+ exportSettings
+ .SymmetricEncryption(NExport::TExportToS3Settings::TEncryptionAlgorithm::AES_128_GCM, "Cool random key!")
+ .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/Root/Table0"})
+ .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/dir1/Table1"})
+ .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "dir1/dir2//Table2"});
+ auto res = YdbExportClient().ExportToS3(exportSettings).GetValueSync();
+ WaitOpSuccess(res);
+ }
+
+ NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix//");
+ listSettings
+ .SymmetricKey("Cool random key!");
+
+ ValidateListObjectPathsInS3Export({
+ "Table0",
+ "dir1/Table1",
+ "dir1/dir2/Table2",
+ }, listSettings);
+ }
+
+ Y_UNIT_TEST(ExportWithWrongEncryptionKey) {
+ {
+ NExport::TExportToS3Settings exportSettings = MakeExportSettings("", "Prefix");
+ exportSettings
+ .SymmetricEncryption(NExport::TExportToS3Settings::TEncryptionAlgorithm::AES_128_GCM, "Cool random key!")
+ .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/Root/Table0"})
+ .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "/dir1/Table1"})
+ .AppendItem(NExport::TExportToS3Settings::TItem{.Src = "dir1/dir2//Table2"});
+ auto res = YdbExportClient().ExportToS3(exportSettings).GetValueSync();
+ WaitOpSuccess(res);
+ }
+
+ NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix//");
+ listSettings
+ .SymmetricKey("Cool and random)");
+
+ auto res = YdbImportClient().ListObjectsInS3Export(listSettings).GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::BAD_REQUEST, "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString());
+ }
+
+ Y_UNIT_TEST(PagingParameters) {
+ NBackup::TSchemaMapping schemaMapping;
+ constexpr size_t ItemsCount = 12000;
+ constexpr size_t ItemsPerFolder = ItemsCount / 5;
+ constexpr size_t ItemsPerSubfolder = ItemsPerFolder / 3;
+ for (size_t i = 0; i < ItemsCount; ++i) {
+ TStringBuilder objectPath;
+ objectPath << "Folder" << (i % 5) << "/Subfolder" << (i % 3) << "/Object" << i;
+ schemaMapping.Items.emplace_back(NBackup::TSchemaMapping::TItem{
+ .ExportPrefix = TStringBuilder() << "Prefix" << i,
+ .ObjectPath = objectPath,
+ });
+ }
+ S3Mock().GetData()["/test_bucket/Prefix/SchemaMapping/mapping.json"] = schemaMapping.Serialize();
+
+ // Without paging
+ {
+ NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix//");
+ auto res = YdbImportClient().ListObjectsInS3Export(listSettings).GetValueSync();
+ UNIT_ASSERT_C(res.IsSuccess(), "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString());
+ UNIT_ASSERT_VALUES_EQUAL(res.GetItems().size(), ItemsCount);
+ }
+
+ // From beginning to the end
+ {
+ auto addItemsByFilter = [&](size_t expectedItemsCount, const TString& prefix1 = {}, const TString& prefix2 = {}) {
+ THashSet<TString> items;
+ i64 pageSize = 42;
+ TString nextPageToken;
+ do {
+ NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix//");
+ if (prefix1) {
+ listSettings.AppendItem(NYdb::NImport::TListObjectsInS3ExportSettings::TItem{.Path = prefix1});
+ }
+ if (prefix2) {
+ listSettings.AppendItem(NYdb::NImport::TListObjectsInS3ExportSettings::TItem{.Path = prefix2});
+ }
+ auto res = YdbImportClient().ListObjectsInS3Export(listSettings, pageSize, nextPageToken).GetValueSync();
+ UNIT_ASSERT_C(res.IsSuccess(), "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString());
+ UNIT_ASSERT_GT(res.GetItems().size(), 0);
+ UNIT_ASSERT_LE(res.GetItems().size(), static_cast<size_t>(pageSize));
+ for (const auto& item : res.GetItems()) {
+ UNIT_ASSERT_C(items.emplace(item.Path).second, "Duplicate item: {" << item.Prefix << ", " << item.Path << "}. Listing result: " << res);
+ }
+
+ ++pageSize; // just for fun
+ nextPageToken = res.NextPageToken();
+ } while (!nextPageToken.empty());
+ UNIT_ASSERT_VALUES_EQUAL(items.size(), expectedItemsCount);
+ };
+
+ addItemsByFilter(ItemsCount);
+ addItemsByFilter(ItemsPerFolder, "/Folder2");
+ addItemsByFilter(ItemsPerSubfolder, "Folder1/Subfolder1//");
+ addItemsByFilter(ItemsPerSubfolder, "Folder1/Subfolder1//");
+ addItemsByFilter(1, "Folder0/Subfolder2//Object620");
+ addItemsByFilter(ItemsPerSubfolder + 1, "Folder0/Subfolder2//Object620", "Folder1/Subfolder1");
+ addItemsByFilter(1, "Folder0/Subfolder2//Object620", "Folder1/Subfolder"); // treat not as a prefix, but as a directory, so Subfolder don't match anything
+ addItemsByFilter(2, "Folder0/Subfolder2//Object620", "Folder1/Subfolder1/Object931");
+ }
+ }
+
+ Y_UNIT_TEST(ParametersValidation) {
+ {
+ NExport::TExportToS3Settings exportSettings = MakeExportSettings("", "Prefix//");
+ auto res = YdbExportClient().ExportToS3(exportSettings).GetValueSync();
+ WaitOpSuccess(res);
+ }
+
+ {
+ // No prefix
+ NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("");
+ auto res = YdbImportClient().ListObjectsInS3Export(listSettings).GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::BAD_REQUEST, "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString());
+ }
+
+ {
+ // Negative page size
+ NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix");
+ auto res = YdbImportClient().ListObjectsInS3Export(listSettings, -42).GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::BAD_REQUEST, "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString());
+ }
+
+ {
+ // Big page size
+ NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix");
+ auto res = YdbImportClient().ListObjectsInS3Export(listSettings, 100500).GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::BAD_REQUEST, "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString());
+ }
+
+ {
+ // Wrong page token
+ NYdb::NImport::TListObjectsInS3ExportSettings listSettings = MakeListObjectsInS3ExportSettings("Prefix");
+ auto res = YdbImportClient().ListObjectsInS3Export(listSettings, 42, "incorrect page token").GetValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(res.GetStatus(), EStatus::BAD_REQUEST, "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString());
+ }
+ }
+}
diff --git a/ydb/services/ydb/backup_ut/s3_backup_test_base.h b/ydb/services/ydb/backup_ut/s3_backup_test_base.h
index 9b130aea8dd..473e4e9bb57 100644
--- a/ydb/services/ydb/backup_ut/s3_backup_test_base.h
+++ b/ydb/services/ydb/backup_ut/s3_backup_test_base.h
@@ -152,6 +152,20 @@ protected:
return importSettings;
}
+ NYdb::NImport::TListObjectsInS3ExportSettings MakeListObjectsInS3ExportSettings(const TString& prefix) {
+ NYdb::NImport::TListObjectsInS3ExportSettings listSettings;
+ listSettings
+ .Endpoint(TStringBuilder() << "localhost:" << S3Port())
+ .Bucket("test_bucket")
+ .Scheme(NYdb::ES3Scheme::HTTP)
+ .AccessKey("test_key")
+ .SecretKey("test_secret");
+ if (prefix) {
+ listSettings.Prefix(prefix);
+ }
+ return listSettings;
+ }
+
void ValidateS3FileList(const TSet<TString>& paths, const TString& prefix = {}) {
TSet<TString> keys;
for (const auto& [key, _] : S3Mock().GetData()) {
@@ -178,6 +192,40 @@ protected:
}
}
+ void ValidateListObjectInS3Export(const TSet<std::pair<TString /*prefix*/, TString /*path*/>>& paths, const NYdb::NImport::TListObjectsInS3ExportSettings& listSettings) {
+ auto res = YdbImportClient().ListObjectsInS3Export(listSettings).GetValueSync();
+ UNIT_ASSERT_C(res.IsSuccess(), "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString());
+
+ TSet<std::pair<TString, TString>> pathsInResponse;
+ for (const auto& item : res.GetItems()) {
+ bool inserted = pathsInResponse.emplace(item.Prefix, item.Path).second;
+ UNIT_ASSERT_C(inserted, "Duplicate item: {" << item.Prefix << ", " << item.Path << "}. Listing result: " << res);
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL_C(pathsInResponse, paths, "Listing result: " << res);
+ }
+
+ void ValidateListObjectInS3Export(const TSet<std::pair<TString /*prefix*/, TString /*path*/>>& paths, const TString& exportPrefix) {
+ ValidateListObjectInS3Export(paths, MakeListObjectsInS3ExportSettings(exportPrefix));
+ }
+
+ void ValidateListObjectPathsInS3Export(const TSet<TString>& paths, const NYdb::NImport::TListObjectsInS3ExportSettings& listSettings) {
+ auto res = YdbImportClient().ListObjectsInS3Export(listSettings).GetValueSync();
+ UNIT_ASSERT_C(res.IsSuccess(), "Status: " << res.GetStatus() << ". Issues: " << res.GetIssues().ToString());
+
+ TSet<TString> pathsInResponse;
+ for (const auto& item : res.GetItems()) {
+ bool inserted = pathsInResponse.emplace(item.Path).second;
+ UNIT_ASSERT_C(inserted, "Duplicate item: {" << item.Prefix << ", " << item.Path << "}. Listing result: " << res);
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL_C(pathsInResponse, paths, "Listing result: " << res);
+ }
+
+ void ValidateListObjectPathsInS3Export(const TSet<TString>& paths, const TString& exportPrefix) {
+ ValidateListObjectPathsInS3Export(paths, MakeListObjectsInS3ExportSettings(exportPrefix));
+ }
+
TString DebugListDir(const TString& path) { // Debug listing for specified dir
auto res = YdbSchemeClient().ListDirectory(path).GetValueSync();
TStringBuilder l;
diff --git a/ydb/services/ydb/backup_ut/ya.make b/ydb/services/ydb/backup_ut/ya.make
index e5f3e895506..f0966d7b2c4 100644
--- a/ydb/services/ydb/backup_ut/ya.make
+++ b/ydb/services/ydb/backup_ut/ya.make
@@ -12,6 +12,7 @@ ENDIF()
SRCS(
backup_path_ut.cpp
encrypted_backup_ut.cpp
+ list_objects_in_s3_export_ut.cpp
ydb_backup_ut.cpp
)
diff --git a/ydb/services/ydb/ydb_import.cpp b/ydb/services/ydb/ydb_import.cpp
index 7a91d096a53..12ebfd5af58 100644
--- a/ydb/services/ydb/ydb_import.cpp
+++ b/ydb/services/ydb/ydb_import.cpp
@@ -24,6 +24,7 @@ void TGRpcYdbImportService::SetupIncomingRequests(NYdbGrpc::TLoggerPtr logger) {
#NAME, logger, getCounterBlock("import", #NAME))->Run();
ADD_REQUEST(ImportFromS3, ImportFromS3Request, ImportFromS3Response, DoImportFromS3Request);
+ ADD_REQUEST(ListObjectsInS3Export, ListObjectsInS3ExportRequest, ListObjectsInS3ExportResponse, DoListObjectsInS3ExportRequest);
ADD_REQUEST(ImportData, ImportDataRequest, ImportDataResponse, DoImportDataRequest);
#undef ADD_REQUEST