S3 add switch for virtual style addressing (#1150)

* add s3 virtual addressing support

15 files changed, 37 insertions, 4 deletions

diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
index 45cb5d406a..83e8533ba5 100644
--- a/ydb/core/protos/flat_scheme_op.proto
+++ b/ydb/core/protos/flat_scheme_op.proto
@@ -972,6 +972,7 @@ message TS3Settings {
     optional string Region = 12;
     optional TSecretableVariable SecretableAccessKey = 13;
     optional TSecretableVariable SecretableSecretKey = 14;
+    optional bool UseVirtualAddressing = 15 [default = true];
 
     message TLimits {
         optional uint32 ReadBatchSize = 1 [default = 8388608]; // 8 MB
@@ -1872,4 +1873,4 @@ message TViewDescription {
     optional NKikimrProto.TPathID PathId = 2;
     optional uint64 Version = 3;
     optional string QueryText = 4;
-}
\ No newline at end of file
+}
diff --git a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp
index 20cb8cd42a..c4514bec90 100644
--- a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp
@@ -141,6 +141,7 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> BackupPropose(
             backupSettings.SetSecretKey(exportSettings.secret_key());
             backupSettings.SetObjectKeyPattern(exportSettings.items(itemIdx).destination_prefix());
             backupSettings.SetStorageClass(exportSettings.storage_class());
+            backupSettings.SetUseVirtualAddressing(!exportSettings.disable_virtual_addressing());
 
             switch (exportSettings.scheme()) {
             case Ydb::Export::ExportToS3Settings::HTTP:
diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp
index 2f9941f9e3..7d81246bd7 100644
--- a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp
@@ -126,6 +126,7 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> RestorePropose(
             restoreSettings.SetAccessKey(importInfo->Settings.access_key());
             restoreSettings.SetSecretKey(importInfo->Settings.secret_key());
             restoreSettings.SetObjectKeyPattern(importInfo->Settings.items(itemIdx).source_prefix());
+            restoreSettings.SetUseVirtualAddressing(!importInfo->Settings.disable_virtual_addressing());
 
             switch (importInfo->Settings.scheme()) {
             case Ydb::Import::ImportFromS3Settings::HTTP:
diff --git a/ydb/core/wrappers/s3_storage.h b/ydb/core/wrappers/s3_storage.h
index bbf608b260..4cbb58687f 100644
--- a/ydb/core/wrappers/s3_storage.h
+++ b/ydb/core/wrappers/s3_storage.h
@@ -105,12 +105,13 @@ public:
     TS3ExternalStorage(const Aws::Client::ClientConfiguration& config,
         const Aws::Auth::AWSCredentials& credentials,
         const TString& bucket, const Aws::S3::Model::StorageClass storageClass,
-        bool verbose = true)
+        bool verbose = true,
+        bool useVirtualAdressing = true)
         : Client(new Aws::S3::S3Client(
             credentials,
             config,
             Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
-            /*useVirtualAddressing=*/ true))
+            useVirtualAdressing))
         , Config(config)
         , Credentials(credentials)
         , Bucket(bucket)
diff --git a/ydb/core/wrappers/s3_storage_config.cpp b/ydb/core/wrappers/s3_storage_config.cpp
index 65d071826c..079cd7fb45 100644
--- a/ydb/core/wrappers/s3_storage_config.cpp
+++ b/ydb/core/wrappers/s3_storage_config.cpp
@@ -183,7 +183,7 @@ TString TS3ExternalStorageConfig::DoGetStorageId() const {
 }
 
 IExternalStorageOperator::TPtr TS3ExternalStorageConfig::DoConstructStorageOperator(bool verbose) const {
-    return std::make_shared<TS3ExternalStorage>(Config, Credentials, Bucket, StorageClass, verbose);
+    return std::make_shared<TS3ExternalStorage>(Config, Credentials, Bucket, StorageClass, verbose, UseVirtualAddressing);
 }
 
 TS3ExternalStorageConfig::TS3ExternalStorageConfig(const Ydb::Import::ImportFromS3Settings& settings): Config(ConfigFromSettings(settings))
@@ -204,6 +204,7 @@ TS3ExternalStorageConfig::TS3ExternalStorageConfig(const NKikimrSchemeOp::TS3Set
     : Config(ConfigFromSettings(settings))
     , Credentials(CredentialsFromSettings(settings))
     , StorageClass(ConvertStorageClass(settings.GetStorageClass()))
+    , UseVirtualAddressing(settings.GetUseVirtualAddressing())
 {
     Bucket = settings.GetBucket();
 }
diff --git a/ydb/core/wrappers/s3_storage_config.h b/ydb/core/wrappers/s3_storage_config.h
index 85a15a4eb8..6555e109b4 100644
--- a/ydb/core/wrappers/s3_storage_config.h
+++ b/ydb/core/wrappers/s3_storage_config.h
@@ -29,6 +29,7 @@ private:
     Aws::Client::ClientConfiguration Config;
     const Aws::Auth::AWSCredentials Credentials;
     YDB_READONLY(Aws::S3::Model::StorageClass, StorageClass, Aws::S3::Model::StorageClass::STANDARD);
+    YDB_READONLY(bool, UseVirtualAddressing, true);
 
     static Aws::Client::ClientConfiguration ConfigFromSettings(const NKikimrSchemeOp::TS3Settings& settings);
     static Aws::Auth::AWSCredentials CredentialsFromSettings(const NKikimrSchemeOp::TS3Settings& settings);
diff --git a/ydb/public/api/protos/ydb_export.proto b/ydb/public/api/protos/ydb_export.proto
index 4e7d33c2bd..b69df42678 100644
--- a/ydb/public/api/protos/ydb_export.proto
+++ b/ydb/public/api/protos/ydb_export.proto
@@ -114,6 +114,12 @@ message ExportToS3Settings {
 
     // Region to use in requests
     string region = 11;
+
+    // disables virtual hosting style buckets aws s3 feature
+    // it changes the way bucket appended to url. e.g. https//bucket_name.example.com/ vs https://example.com/bucket_name
+    // details: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html
+    // it is especially useful for custom s3 implementations
+    bool disable_virtual_addressing = 12;
 }
 
 message ExportToS3Result {
diff --git a/ydb/public/api/protos/ydb_import.proto b/ydb/public/api/protos/ydb_import.proto
index 53eb5a8814..f03f56ef26 100644
--- a/ydb/public/api/protos/ydb_import.proto
+++ b/ydb/public/api/protos/ydb_import.proto
@@ -61,6 +61,12 @@ message ImportFromS3Settings {
 
     // Region to use in requests
     string region = 9;
+
+    // disables virtual hosting style buckets aws s3 feature
+    // it changes the way bucket appended to url. e.g. https//bucket_name.example.com/ vs https://example.com/bucket_name
+    // details: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html
+    // it is especially useful for custom s3 implementations
+    bool disable_virtual_addressing = 10;
 }
 
 message ImportFromS3Result {
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp
index 47230af16b..8e9edb7dab 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp
+++ b/ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp
@@ -277,6 +277,9 @@ void TCommandExportToS3::Config(TConfig& config) {
             << "    - zstd-N (N is compression level, e.g. zstd-3)" << Endl)
         .RequiredArgument("STRING").StoreResult(&Compression);
 
+    config.Opts->AddLongOption("use-virtual-addressing", "S3 bucket virtual addressing")
+        .RequiredArgument("BOOL").StoreResult<bool>(&UseVirtualAddressing).DefaultValue("true");
+
     AddDeprecatedJsonOption(config);
     AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 });
     config.Opts->MutuallyExclusive("json", "format");
@@ -312,6 +315,7 @@ int TCommandExportToS3::Run(TConfig& config) {
     settings.Bucket(AwsBucket);
     settings.AccessKey(AwsAccessKey);
     settings.SecretKey(AwsSecretKey);
+    settings.UseVirtualAddressing(UseVirtualAddressing);
 
     for (const auto& item : Items) {
         settings.AppendItem({item.Source, item.Destination});
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_export.h b/ydb/public/lib/ydb_cli/commands/ydb_service_export.h
index 56d54c216c..671e56fe8d 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_service_export.h
+++ b/ydb/public/lib/ydb_cli/commands/ydb_service_export.h
@@ -71,6 +71,7 @@ private:
     TString Description;
     ui32 NumberOfRetries = 10;
     TString Compression;
+    bool UseVirtualAddressing = true;
 };
 
 }
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp
index 6903d02b3a..64f98bdfbe 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp
+++ b/ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp
@@ -92,6 +92,9 @@ void TCommandImportFromS3::Config(TConfig& config) {
     config.Opts->AddLongOption("retries", "Number of retries")
         .RequiredArgument("NUM").StoreResult(&NumberOfRetries).DefaultValue(NumberOfRetries);
 
+    config.Opts->AddLongOption("use-virtual-addressing", "S3 bucket virtual addressing")
+        .RequiredArgument("BOOL").StoreResult<bool>(&UseVirtualAddressing).DefaultValue("true");
+
     AddDeprecatedJsonOption(config);
     AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 });
     config.Opts->MutuallyExclusive("json", "format");
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_import.h b/ydb/public/lib/ydb_cli/commands/ydb_service_import.h
index 941465f15a..90bddf76a7 100644
--- a/ydb/public/lib/ydb_cli/commands/ydb_service_import.h
+++ b/ydb/public/lib/ydb_cli/commands/ydb_service_import.h
@@ -37,6 +37,7 @@ private:
     TVector<TItem> Items;
     TString Description;
     ui32 NumberOfRetries = 10;
+    bool UseVirtualAddressing = true;
 };
 
 class TCommandImportFromFile : public TClientCommandTree {
diff --git a/ydb/public/sdk/cpp/client/ydb_export/export.cpp b/ydb/public/sdk/cpp/client/ydb_export/export.cpp
index 25c7e10d39..ce769e3cf5 100644
--- a/ydb/public/sdk/cpp/client/ydb_export/export.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_export/export.cpp
@@ -200,6 +200,8 @@ TFuture<TExportToS3Response> TExportClient::ExportToS3(const TExportToS3Settings
         request.mutable_settings()->set_compression(*settings.Compression_);
     }
 
+    request.mutable_settings()->set_disable_virtual_addressing(!settings.UseVirtualAddressing_);
+
     return Impl_->ExportToS3(std::move(request), settings);
 }
 
diff --git a/ydb/public/sdk/cpp/client/ydb_import/import.cpp b/ydb/public/sdk/cpp/client/ydb_import/import.cpp
index b7d2496743..927fb65425 100644
--- a/ydb/public/sdk/cpp/client/ydb_import/import.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_import/import.cpp
@@ -159,6 +159,8 @@ TFuture<TImportFromS3Response> TImportClient::ImportFromS3(const TImportFromS3Se
         request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.GetRef());
     }
 
+    request.mutable_settings()->set_disable_virtual_addressing(!settings.UseVirtualAddressing_);
+
     return Impl_->ImportFromS3(std::move(request), settings);
 }
 
diff --git a/ydb/public/sdk/cpp/client/ydb_types/s3_settings.h b/ydb/public/sdk/cpp/client/ydb_types/s3_settings.h
index dca95fdb7a..7c5b80a69d 100644
--- a/ydb/public/sdk/cpp/client/ydb_types/s3_settings.h
+++ b/ydb/public/sdk/cpp/client/ydb_types/s3_settings.h
@@ -18,6 +18,8 @@ struct TS3Settings {
     FLUENT_SETTING(TString, Bucket);
     FLUENT_SETTING(TString, AccessKey);
     FLUENT_SETTING(TString, SecretKey);
+    // true by default for backward compatibility
+    FLUENT_SETTING_DEFAULT(bool, UseVirtualAddressing, true);
 };
 
 } // namespace NYdb