S3 add switch for virtual style addressing

ydb/core/protos/flat_scheme_op.proto

@@ -935,6 +935,7 @@ message TS3Settings {
     optional string Region = 12;
     optional TSecretableVariable SecretableAccessKey = 13;
     optional TSecretableVariable SecretableSecretKey = 14;
+    optional bool UseVirtualAddressing = 15 [default = true];
 
     message TLimits {
         optional uint32 ReadBatchSize = 1 [default = 8388608]; // 8 MB
@@ -1835,4 +1836,4 @@ message TViewDescription {
     optional NKikimrProto.TPathID PathId = 2;
     optional uint64 Version = 3;
     optional string QueryText = 4;
-}
+}

ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp

@@ -141,6 +141,7 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> BackupPropose(
             backupSettings.SetSecretKey(exportSettings.secret_key());
             backupSettings.SetObjectKeyPattern(exportSettings.items(itemIdx).destination_prefix());
             backupSettings.SetStorageClass(exportSettings.storage_class());
+            backupSettings.SetUseVirtualAddressing(!exportSettings.disable_virtual_addressing());
 
             switch (exportSettings.scheme()) {
             case Ydb::Export::ExportToS3Settings::HTTP:

ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp

@@ -126,6 +126,7 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> RestorePropose(
             restoreSettings.SetAccessKey(importInfo->Settings.access_key());
             restoreSettings.SetSecretKey(importInfo->Settings.secret_key());
             restoreSettings.SetObjectKeyPattern(importInfo->Settings.items(itemIdx).source_prefix());
+            restoreSettings.SetUseVirtualAddressing(!importInfo->Settings.disable_virtual_addressing());
 
             switch (importInfo->Settings.scheme()) {
             case Ydb::Import::ImportFromS3Settings::HTTP:

ydb/core/wrappers/s3_storage.h

@@ -105,12 +105,13 @@ class TS3ExternalStorage: public IExternalStorageOperator, TS3User {
     TS3ExternalStorage(const Aws::Client::ClientConfiguration& config,
         const Aws::Auth::AWSCredentials& credentials,
         const TString& bucket, const Aws::S3::Model::StorageClass storageClass,
-        bool verbose = true)
+        bool verbose = true,
+        bool useVirtualAdressing = true)
         : Client(new Aws::S3::S3Client(
             credentials,
             config,
             Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
-            /*useVirtualAddressing=*/ true))
+            useVirtualAdressing))
         , Config(config)
         , Credentials(credentials)
         , Bucket(bucket)

ydb/core/wrappers/s3_storage_config.cpp

@@ -183,7 +183,7 @@ TString TS3ExternalStorageConfig::DoGetStorageId() const {
 }
 
 IExternalStorageOperator::TPtr TS3ExternalStorageConfig::DoConstructStorageOperator(bool verbose) const {
-    return std::make_shared<TS3ExternalStorage>(Config, Credentials, Bucket, StorageClass, verbose);
+    return std::make_shared<TS3ExternalStorage>(Config, Credentials, Bucket, StorageClass, verbose, UseVirtualAddressing);
 }
 
 TS3ExternalStorageConfig::TS3ExternalStorageConfig(const Ydb::Import::ImportFromS3Settings& settings): Config(ConfigFromSettings(settings))
@@ -204,6 +204,7 @@ TS3ExternalStorageConfig::TS3ExternalStorageConfig(const NKikimrSchemeOp::TS3Set
     : Config(ConfigFromSettings(settings))
     , Credentials(CredentialsFromSettings(settings))
     , StorageClass(ConvertStorageClass(settings.GetStorageClass()))
+    , UseVirtualAddressing(settings.GetUseVirtualAddressing())
 {
     Bucket = settings.GetBucket();
 }

ydb/core/wrappers/s3_storage_config.h

@@ -29,6 +29,7 @@ class TS3ExternalStorageConfig: public IExternalStorageConfig, TS3User {
     Aws::Client::ClientConfiguration Config;
     const Aws::Auth::AWSCredentials Credentials;
     YDB_READONLY(Aws::S3::Model::StorageClass, StorageClass, Aws::S3::Model::StorageClass::STANDARD);
+    YDB_READONLY(bool, UseVirtualAddressing, true);
 
     static Aws::Client::ClientConfiguration ConfigFromSettings(const NKikimrSchemeOp::TS3Settings& settings);
     static Aws::Auth::AWSCredentials CredentialsFromSettings(const NKikimrSchemeOp::TS3Settings& settings);

ydb/public/api/protos/ydb_export.proto

@@ -114,6 +114,12 @@ message ExportToS3Settings {
 
     // Region to use in requests
     string region = 11;
+
+    // disables virtual hosting style buckets aws s3 feature
+    // it changes the way bucket appended to url. e.g. https//bucket_name.example.com/ vs https://example.com/bucket_name
+    // details: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html
+    // it is especially useful for custom s3 implementations
+    bool disable_virtual_addressing = 12;
 }
 
 message ExportToS3Result {

ydb/public/api/protos/ydb_import.proto

@@ -61,6 +61,12 @@ message ImportFromS3Settings {
 
     // Region to use in requests
     string region = 9;
+
+    // disables virtual hosting style buckets aws s3 feature
+    // it changes the way bucket appended to url. e.g. https//bucket_name.example.com/ vs https://example.com/bucket_name
+    // details: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html
+    // it is especially useful for custom s3 implementations
+    bool disable_virtual_addressing = 10;
 }
 
 message ImportFromS3Result {

ydb/public/lib/ydb_cli/commands/ydb_service_export.cpp

@@ -277,6 +277,9 @@ void TCommandExportToS3::Config(TConfig& config) {
             << "    - zstd-N (N is compression level, e.g. zstd-3)" << Endl)
         .RequiredArgument("STRING").StoreResult(&Compression);
 
+    config.Opts->AddLongOption("use-virtual-addressing", "S3 bucket virtual addressing")
+        .RequiredArgument("BOOL").StoreResult<bool>(&UseVirtualAddressing).DefaultValue("true");
+
     AddDeprecatedJsonOption(config);
     AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 });
     config.Opts->MutuallyExclusive("json", "format");
@@ -312,6 +315,7 @@ int TCommandExportToS3::Run(TConfig& config) {
     settings.Bucket(AwsBucket);
     settings.AccessKey(AwsAccessKey);
     settings.SecretKey(AwsSecretKey);
+    settings.UseVirtualAddressing(UseVirtualAddressing);
 
     for (const auto& item : Items) {
         settings.AppendItem({item.Source, item.Destination});

ydb/public/lib/ydb_cli/commands/ydb_service_export.h

@@ -71,6 +71,7 @@ class TCommandExportToS3 : public TYdbOperationCommand,
     TString Description;
     ui32 NumberOfRetries = 10;
     TString Compression;
+    bool UseVirtualAddressing = true;
 };
 
 }

ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp

@@ -92,6 +92,9 @@ void TCommandImportFromS3::Config(TConfig& config) {
     config.Opts->AddLongOption("retries", "Number of retries")
         .RequiredArgument("NUM").StoreResult(&NumberOfRetries).DefaultValue(NumberOfRetries);
 
+    config.Opts->AddLongOption("use-virtual-addressing", "S3 bucket virtual addressing")
+        .RequiredArgument("BOOL").StoreResult<bool>(&UseVirtualAddressing).DefaultValue("true");
+
     AddDeprecatedJsonOption(config);
     AddFormats(config, { EOutputFormat::Pretty, EOutputFormat::ProtoJsonBase64 });
     config.Opts->MutuallyExclusive("json", "format");

ydb/public/lib/ydb_cli/commands/ydb_service_import.h

@@ -37,6 +37,7 @@ class TCommandImportFromS3 : public TYdbOperationCommand,
     TVector<TItem> Items;
     TString Description;
     ui32 NumberOfRetries = 10;
+    bool UseVirtualAddressing = true;
 };
 
 class TCommandImportFromFile : public TClientCommandTree {

ydb/public/sdk/cpp/client/ydb_export/export.cpp

@@ -200,6 +200,8 @@ TFuture<TExportToS3Response> TExportClient::ExportToS3(const TExportToS3Settings
         request.mutable_settings()->set_compression(*settings.Compression_);
     }
 
+    request.mutable_settings()->set_disable_virtual_addressing(!settings.UseVirtualAddressing_);
+
     return Impl_->ExportToS3(std::move(request), settings);
 }
 

ydb/public/sdk/cpp/client/ydb_import/import.cpp

@@ -159,6 +159,8 @@ TFuture<TImportFromS3Response> TImportClient::ImportFromS3(const TImportFromS3Se
         request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.GetRef());
     }
 
+    request.mutable_settings()->set_disable_virtual_addressing(!settings.UseVirtualAddressing_);
+
     return Impl_->ImportFromS3(std::move(request), settings);
 }
 

ydb/public/sdk/cpp/client/ydb_types/s3_settings.h

@@ -18,6 +18,8 @@ struct TS3Settings {
     FLUENT_SETTING(TString, Bucket);
     FLUENT_SETTING(TString, AccessKey);
     FLUENT_SETTING(TString, SecretKey);
+    // true by default for backward compatibility
+    FLUENT_SETTING_DEFAULT(bool, UseVirtualAddressing, true);
 };
 
 } // namespace NYdb