Add S3 settings to BlobDepot configuration

ydb-platform · Feb 24, 2025 · 5f0aa80 · 5f0aa80
1 parent 50d5375
commit 5f0aa80
Show file tree

Hide file tree

Showing 16 changed files with 99 additions and 64 deletions.
diff --git a/ydb/core/protos/blob_depot_config.proto b/ydb/core/protos/blob_depot_config.proto
@@ -1,4 +1,5 @@
 import "ydb/core/protos/hive.proto";
+import "ydb/core/protos/s3_settings.proto";
 
 package NKikimrBlobDepot;
 
@@ -17,6 +18,28 @@ message TChannelProfile {
     optional TChannelKind.E ChannelKind = 4;
 }
 
+message TS3BackendSettings {
+    optional NKikimrSchemeOp.TS3Settings Settings = 1; // how to connect to S3 (mandatory field)
+
+    // data (when written) is first stored in ordinary data channel and then asynchronously transferred to S3 storage by
+    // BlobDepot tablet; amount of stored data can be controlled by following settings
+    //
+    // future operation may include modes when data is uploaded by agents directly to the storage
+
+    // when do we fully stop processing new queries until some data is uploaded
+    optional uint64 MaxPendingBytes = 2;
+
+    // when do we start throttling (if set, this must be strictly less than MaxPendingBytes, which also must be set)
+    optional uint64 ThrottleStartBytes = 3;
+
+    // upload rate (bytes per second) at ThrottleStartBytes amount of data (which linearly drops to 0 as amount increases
+    // up to MaxPendingBytes)
+    optional uint64 ThrottleMaxBytesPerSecond = 4;
+
+    // number of concurrent puts
+    optional uint32 AsyncUploadPutsInFlight = 5 [default = 1];
+}
+
 message TBlobDepotConfig {
     repeated TChannelProfile ChannelProfiles = 1;
     optional uint32 VirtualGroupId = 2;
@@ -25,4 +48,5 @@ message TBlobDepotConfig {
     optional bool HiveContacted = 5;
     optional NKikimrHive.TEvCreateTablet HiveParams = 6; // extra hive parameters
     optional uint64 TenantHiveId = 7;
+    optional TS3BackendSettings S3BackendSettings = 8; // if set, then this tablet stores all its data in S3 (except for some cache/inflight)
 }
diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
@@ -8,6 +8,7 @@ import "ydb/core/protos/follower_group.proto";
 import "ydb/core/protos/index_builder.proto";
 import "ydb/core/protos/pqconfig.proto";
 import "ydb/core/protos/replication.proto";
+import "ydb/core/protos/s3_settings.proto";
 import "ydb/core/protos/schemeshard/operations.proto";
 import "ydb/core/protos/subdomains.proto";
 import "ydb/core/protos/table_stats.proto";
@@ -21,7 +22,6 @@ import "ydb/library/actors/protos/actors.proto";
 import "ydb/library/formats/arrow/protos/accessor.proto";
 import "ydb/library/mkql_proto/protos/minikql.proto";
 import "ydb/public/api/protos/ydb_coordination.proto";
-import "ydb/public/api/protos/ydb_export.proto";
 import "ydb/public/api/protos/ydb_table.proto";
 import "ydb/public/api/protos/ydb_topic.proto";
 import "ydb/public/api/protos/ydb_value.proto";
@@ -476,7 +476,7 @@ enum EOlapProgramType {
 
 message TStorageTierConfig {
     optional string Name = 1;
-    optional TS3Settings ObjectStorage = 2;
+    optional NKikimrSchemeOp.TS3Settings ObjectStorage = 2;
     optional TCompressionOptions Compression = 3;
 }
 
@@ -1162,62 +1162,9 @@ message TYTSettings {
     optional bool UseTypeV3 = 8;
 };
 
-message TSecretId {
-    optional string Id = 1;
-    optional string OwnerId = 2;
-}
-
-message TSecretValue {
-    optional string Data = 1;
-}
-
-message TSecretableVariable {
-    oneof Data {
-        TSecretId SecretId = 1;
-        TSecretValue Value = 2;
-    }
-}
-
-message TS3Settings {
-    enum EScheme {
-        HTTP = 0;
-        HTTPS = 1;
-    }
-
-    optional string Endpoint = 1;
-    optional EScheme Scheme = 2 [default = HTTPS];
-    optional string Bucket = 3;
-    optional string ObjectKeyPattern = 4; // dst for backup, src for restore
-    optional string AccessKey = 5;
-    optional string SecretKey = 6;
-    optional Ydb.Export.ExportToS3Settings.StorageClass StorageClass = 7 [default = STANDARD];
-    optional bool VerifySSL = 8;
-    optional string ProxyHost = 9;
-    optional uint32 ProxyPort = 10;
-    optional EScheme ProxyScheme = 11;
-    optional string Region = 12;
-    optional TSecretableVariable SecretableAccessKey = 13;
-    optional TSecretableVariable SecretableSecretKey = 14;
-    optional bool UseVirtualAddressing = 15 [default = true];
-
-    message TLimits {
-        optional uint32 ReadBatchSize = 1 [default = 8388608]; // 8 MB
-        optional uint32 MinWriteBatchSize = 2 [default = 5242880]; // 5 MB
-        reserved 3; // ReadBufferSizeLimit
-    };
-
-    optional TLimits Limits = 100;
-    optional uint32 RequestTimeoutMs = 101;
-    optional uint32 HttpRequestTimeoutMs = 102;
-    optional uint32 ConnectionTimeoutMs = 103;
-
-    optional uint32 ExecutorThreadsCount = 104 [default = 32];
-    optional uint32 MaxConnectionsCount = 105 [default = 32];
-};
-
 message TTaskCleaner {
     optional uint64 PathId = 1;
-    optional TS3Settings StorageSettings = 2;
+    optional NKikimrSchemeOp.TS3Settings StorageSettings = 2;
     optional string TieringId = 3;
 }
 
@@ -1233,7 +1180,7 @@ message TBackupTask {
 
     oneof Settings {
         TYTSettings YTSettings = 4;
-        TS3Settings S3Settings = 9;
+        NKikimrSchemeOp.TS3Settings S3Settings = 9;
     }
 
     optional TPathDescription Table = 10; // for further restore
@@ -1269,7 +1216,7 @@ message TRestoreTask {
     optional uint32 NumberOfRetries = 5;
 
     oneof Settings {
-        TS3Settings S3Settings = 6;
+        NKikimrSchemeOp.TS3Settings S3Settings = 6;
     }
 
     optional bool ValidateChecksums = 7; // currently available for s3

diff --git a/ydb/core/protos/s3_settings.proto b/ydb/core/protos/s3_settings.proto
@@ -0,0 +1,57 @@
+import "ydb/public/api/protos/ydb_export.proto";
+
+package NKikimrSchemeOp;
+option java_package = "ru.yandex.kikimr.proto";
+
+message TSecretId {
+    optional string Id = 1;
+    optional string OwnerId = 2;
+}
+
+message TSecretValue {
+    optional string Data = 1;
+}
+
+message TSecretableVariable {
+    oneof Data {
+        TSecretId SecretId = 1;
+        TSecretValue Value = 2;
+    }
+}
+
+message TS3Settings {
+    enum EScheme {
+        HTTP = 0;
+        HTTPS = 1;
+    }
+
+    optional string Endpoint = 1;
+    optional EScheme Scheme = 2 [default = HTTPS];
+    optional string Bucket = 3;
+    optional string ObjectKeyPattern = 4; // dst for backup, src for restore
+    optional string AccessKey = 5;
+    optional string SecretKey = 6;
+    optional Ydb.Export.ExportToS3Settings.StorageClass StorageClass = 7 [default = STANDARD];
+    optional bool VerifySSL = 8;
+    optional string ProxyHost = 9;
+    optional uint32 ProxyPort = 10;
+    optional EScheme ProxyScheme = 11;
+    optional string Region = 12;
+    optional TSecretableVariable SecretableAccessKey = 13;
+    optional TSecretableVariable SecretableSecretKey = 14;
+    optional bool UseVirtualAddressing = 15 [default = true];
+
+    message TLimits {
+        optional uint32 ReadBatchSize = 1 [default = 8388608]; // 8 MB
+        optional uint32 MinWriteBatchSize = 2 [default = 5242880]; // 5 MB
+        reserved 3; // ReadBufferSizeLimit
+    };
+
+    optional TLimits Limits = 100;
+    optional uint32 RequestTimeoutMs = 101;
+    optional uint32 HttpRequestTimeoutMs = 102;
+    optional uint32 ConnectionTimeoutMs = 103;
+
+    optional uint32 ExecutorThreadsCount = 104 [default = 32];
+    optional uint32 MaxConnectionsCount = 105 [default = 32];
+}
diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make
@@ -113,6 +113,7 @@ SRCS(
     query_stats.proto
     replication.proto
     resource_broker.proto
+    s3_settings.proto
     scheme_board.proto
     scheme_board_mon.proto
     scheme_log.proto

diff --git a/ydb/core/tx/columnshard/export/protos/storage.proto b/ydb/core/tx/columnshard/export/protos/storage.proto
@@ -1,4 +1,4 @@
-import "ydb/core/protos/flat_scheme_op.proto";
+import "ydb/core/protos/s3_settings.proto";
 
 package NKikimrColumnShardExportProto;
 

diff --git a/ydb/core/tx/datashard/export_s3.h b/ydb/core/tx/datashard/export_s3.h
@@ -7,6 +7,8 @@
 #include "export_iface.h"
 #include "export_s3_buffer.h"
 
+#include <ydb/core/protos/s3_settings.pb.h>
+
 namespace NKikimr {
 namespace NDataShard {
 

diff --git a/ydb/core/tx/datashard/extstorage_usage_config.h b/ydb/core/tx/datashard/extstorage_usage_config.h
@@ -6,6 +6,7 @@
 
 #include <ydb/core/base/events.h>
 #include <ydb/core/protos/flat_scheme_op.pb.h>
+#include <ydb/core/protos/s3_settings.pb.h>
 #include <ydb/public/api/protos/ydb_export.pb.h>
 
 #include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/StorageClass.h>

diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp
@@ -3,6 +3,7 @@
 #include "schemeshard__data_erasure_manager.h"
 
 #include <ydb/core/protos/table_stats.pb.h>  // for TStoragePoolsStats
+#include <ydb/core/protos/s3_settings.pb.h>
 #include <ydb/core/scheme/scheme_types_proto.h>
 #include <ydb/core/tablet/tablet_exception.h>
 #include <ydb/core/tablet_flat/flat_cxx_database.h>

diff --git a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp
@@ -2,6 +2,7 @@
 #include "schemeshard_path_describer.h"
 
 #include <ydb/core/ydb_convert/compression.h>
+#include <ydb/core/protos/s3_settings.pb.h>
 #include <ydb/public/api/protos/ydb_export.pb.h>
 
 #include <util/string/builder.h>

diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp
@@ -14,6 +14,7 @@
 #include <ydb/core/protos/feature_flags.pb.h>
 #include <ydb/core/protos/table_stats.pb.h>  // for TStoragePoolsStats
 #include <ydb/core/protos/auth.pb.h>
+#include <ydb/core/protos/s3_settings.pb.h>
 #include <ydb/core/engine/mkql_proto.h>
 #include <ydb/core/sys_view/partition_stats/partition_stats.h>
 #include <ydb/core/statistics/events.h>

diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp
@@ -4,6 +4,7 @@
 #include <ydb/core/base/path.h>
 #include <ydb/core/ydb_convert/table_description.h>
 #include <ydb/core/ydb_convert/ydb_convert.h>
+#include <ydb/core/protos/s3_settings.pb.h>
 
 namespace NKikimr {
 namespace NSchemeShard {

diff --git a/ydb/core/tx/tiering/tier/object.h b/ydb/core/tx/tiering/tier/object.h
@@ -1,5 +1,4 @@
 #pragma once
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 
 #include <ydb/library/accessor/accessor.h>
 #include <ydb/library/conclusion/status.h>

diff --git a/ydb/core/tx/tiering/tier/s3_uri.h b/ydb/core/tx/tiering/tier/s3_uri.h
@@ -1,4 +1,4 @@
-#include <ydb/core/protos/flat_scheme_op.pb.h>
+#include <ydb/core/protos/s3_settings.pb.h>
 
 #include <ydb/library/accessor/accessor.h>
 #include <ydb/library/conclusion/result.h>

diff --git a/ydb/core/wrappers/abstract.h b/ydb/core/wrappers/abstract.h
@@ -2,7 +2,7 @@
 
 #include <ydb/library/actors/core/actorsystem.h>
 
-#include <ydb/core/protos/flat_scheme_op.pb.h>
+#include <ydb/core/protos/s3_settings.pb.h>
 #include <ydb/core/wrappers/events/abstract.h>
 #include <ydb/core/wrappers/events/common.h>
 #include <ydb/core/wrappers/events/get_object.h>

diff --git a/ydb/core/wrappers/s3_storage_config.h b/ydb/core/wrappers/s3_storage_config.h
@@ -5,7 +5,7 @@
 #include "abstract.h"
 
 #include <ydb/core/base/events.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
+#include <ydb/core/protos/s3_settings.pb.h>
 #include <ydb/library/accessor/accessor.h>
 #include <ydb/public/api/protos/ydb_import.pb.h>
 #include <ydb/public/api/protos/ydb_export.pb.h>

diff --git a/ydb/services/metadata/secret/accessor/secret_id.h b/ydb/services/metadata/secret/accessor/secret_id.h
@@ -1,6 +1,6 @@
 #pragma once
 #include <ydb/core/base/appdata.h>
-
+#include <ydb/core/protos/s3_settings.pb.h>
 #include <ydb/library/accessor/accessor.h>
 #include <util/generic/overloaded.h>