Add S3 settings to BlobDepot configuration (ydb-platform#14942)

serbel324 · Feb 24, 2025 · 9d32e6d · 9d32e6d
1 parent 8d7d1c8
commit 9d32e6d
Show file tree

Hide file tree

Showing 32 changed files with 103 additions and 84 deletions.
diff --git a/ydb/core/backup/common/backup_restore_traits.cpp b/ydb/core/backup/common/backup_restore_traits.cpp
@@ -1,6 +1,5 @@
 #include "backup_restore_traits.h"
 
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 #include <ydb/library/yverify_stream/yverify_stream.h>
 
 #include <util/generic/hash.h>

diff --git a/ydb/core/backup/s3/export_s3_uploader.cpp b/ydb/core/backup/s3/export_s3_uploader.cpp
@@ -7,7 +7,6 @@
 #include "extstorage_usage_config.h"
 
 #include <ydb/core/base/appdata.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 #include <ydb/library/services/services.pb.h>
 #include <ydb/core/wrappers/s3_storage_config.h>
 #include <ydb/core/wrappers/s3_wrapper.h>

diff --git a/ydb/core/backup/s3/extstorage_usage_config.h b/ydb/core/backup/s3/extstorage_usage_config.h
@@ -5,7 +5,6 @@
 #include "backup_restore_traits.h"
 
 #include <ydb/core/base/events.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 #include <ydb/public/api/protos/ydb_export.pb.h>
 
 #include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/StorageClass.h>

diff --git a/ydb/core/external_sources/object_storage.cpp b/ydb/core/external_sources/object_storage.cpp
@@ -9,7 +9,6 @@
 #include <ydb/core/external_sources/object_storage/inference/infer_config.h>
 #include <ydb/core/kqp/gateway/actors/kqp_ic_gateway_actors.h>
 #include <ydb/core/protos/external_sources.pb.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 #include <ydb/library/yql/providers/common/http_gateway/yql_http_gateway.h>
 #include <yql/essentials/providers/common/provider/yql_provider_names.h>
 #include <yql/essentials/providers/common/structured_token/yql_token_builder.h>

diff --git a/ydb/core/formats/arrow/serializer/native.h b/ydb/core/formats/arrow/serializer/native.h
@@ -5,7 +5,6 @@
 
 #include <ydb/core/base/appdata_fwd.h>
 #include <ydb/core/protos/config.pb.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 
 #include <ydb/library/accessor/accessor.h>
 #include <ydb/library/conclusion/result.h>

diff --git a/ydb/core/protos/blob_depot_config.proto b/ydb/core/protos/blob_depot_config.proto
@@ -1,4 +1,5 @@
 import "ydb/core/protos/hive.proto";
+import "ydb/core/protos/s3_settings.proto";
 
 package NKikimrBlobDepot;
 
@@ -17,6 +18,28 @@ message TChannelProfile {
     optional TChannelKind.E ChannelKind = 4;
 }
 
+message TS3BackendSettings {
+    optional NKikimrSchemeOp.TS3Settings Settings = 1; // how to connect to S3 (mandatory field)
+
+    // data (when written) is first stored in ordinary data channel and then asynchronously transferred to S3 storage by
+    // BlobDepot tablet; amount of stored data can be controlled by following settings
+    //
+    // future operation may include modes when data is uploaded by agents directly to the storage
+
+    // when do we fully stop processing new queries until some data is uploaded
+    optional uint64 MaxPendingBytes = 2;
+
+    // when do we start throttling (if set, this must be strictly less than MaxPendingBytes, which also must be set)
+    optional uint64 ThrottleStartBytes = 3;
+
+    // upload rate (bytes per second) at ThrottleStartBytes amount of data (which linearly drops to 0 as amount increases
+    // up to MaxPendingBytes)
+    optional uint64 ThrottleMaxBytesPerSecond = 4;
+
+    // number of concurrent puts
+    optional uint32 AsyncUploadPutsInFlight = 5 [default = 1];
+}
+
 message TBlobDepotConfig {
     repeated TChannelProfile ChannelProfiles = 1;
     optional uint32 VirtualGroupId = 2;
@@ -25,4 +48,5 @@ message TBlobDepotConfig {
     optional bool HiveContacted = 5;
     optional NKikimrHive.TEvCreateTablet HiveParams = 6; // extra hive parameters
     optional uint64 TenantHiveId = 7;
+    optional TS3BackendSettings S3BackendSettings = 8; // if set, then this tablet stores all its data in S3 (except for some cache/inflight)
 }
diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
@@ -8,6 +8,7 @@ import "ydb/core/protos/follower_group.proto";
 import "ydb/core/protos/index_builder.proto";
 import "ydb/core/protos/pqconfig.proto";
 import "ydb/core/protos/replication.proto";
+import "ydb/core/protos/s3_settings.proto";
 import "ydb/core/protos/schemeshard/operations.proto";
 import "ydb/core/protos/subdomains.proto";
 import "ydb/core/protos/table_stats.proto";
@@ -21,7 +22,6 @@ import "ydb/library/actors/protos/actors.proto";
 import "ydb/library/formats/arrow/protos/accessor.proto";
 import "ydb/library/mkql_proto/protos/minikql.proto";
 import "ydb/public/api/protos/ydb_coordination.proto";
-import "ydb/public/api/protos/ydb_export.proto";
 import "ydb/public/api/protos/ydb_table.proto";
 import "ydb/public/api/protos/ydb_topic.proto";
 import "ydb/public/api/protos/ydb_value.proto";
@@ -476,7 +476,7 @@ enum EOlapProgramType {
 
 message TStorageTierConfig {
     optional string Name = 1;
-    optional TS3Settings ObjectStorage = 2;
+    optional NKikimrSchemeOp.TS3Settings ObjectStorage = 2;
     optional TCompressionOptions Compression = 3;
 }
 
@@ -1162,62 +1162,9 @@ message TYTSettings {
     optional bool UseTypeV3 = 8;
 };
 
-message TSecretId {
-    optional string Id = 1;
-    optional string OwnerId = 2;
-}
-
-message TSecretValue {
-    optional string Data = 1;
-}
-
-message TSecretableVariable {
-    oneof Data {
-        TSecretId SecretId = 1;
-        TSecretValue Value = 2;
-    }
-}
-
-message TS3Settings {
-    enum EScheme {
-        HTTP = 0;
-        HTTPS = 1;
-    }
-
-    optional string Endpoint = 1;
-    optional EScheme Scheme = 2 [default = HTTPS];
-    optional string Bucket = 3;
-    optional string ObjectKeyPattern = 4; // dst for backup, src for restore
-    optional string AccessKey = 5;
-    optional string SecretKey = 6;
-    optional Ydb.Export.ExportToS3Settings.StorageClass StorageClass = 7 [default = STANDARD];
-    optional bool VerifySSL = 8;
-    optional string ProxyHost = 9;
-    optional uint32 ProxyPort = 10;
-    optional EScheme ProxyScheme = 11;
-    optional string Region = 12;
-    optional TSecretableVariable SecretableAccessKey = 13;
-    optional TSecretableVariable SecretableSecretKey = 14;
-    optional bool UseVirtualAddressing = 15 [default = true];
-
-    message TLimits {
-        optional uint32 ReadBatchSize = 1 [default = 8388608]; // 8 MB
-        optional uint32 MinWriteBatchSize = 2 [default = 5242880]; // 5 MB
-        reserved 3; // ReadBufferSizeLimit
-    };
-
-    optional TLimits Limits = 100;
-    optional uint32 RequestTimeoutMs = 101;
-    optional uint32 HttpRequestTimeoutMs = 102;
-    optional uint32 ConnectionTimeoutMs = 103;
-
-    optional uint32 ExecutorThreadsCount = 104 [default = 32];
-    optional uint32 MaxConnectionsCount = 105 [default = 32];
-};
-
 message TTaskCleaner {
     optional uint64 PathId = 1;
-    optional TS3Settings StorageSettings = 2;
+    optional NKikimrSchemeOp.TS3Settings StorageSettings = 2;
     optional string TieringId = 3;
 }
 
@@ -1233,7 +1180,7 @@ message TBackupTask {
 
     oneof Settings {
         TYTSettings YTSettings = 4;
-        TS3Settings S3Settings = 9;
+        NKikimrSchemeOp.TS3Settings S3Settings = 9;
     }
 
     optional TPathDescription Table = 10; // for further restore
@@ -1269,7 +1216,7 @@ message TRestoreTask {
     optional uint32 NumberOfRetries = 5;
 
     oneof Settings {
-        TS3Settings S3Settings = 6;
+        NKikimrSchemeOp.TS3Settings S3Settings = 6;
     }
 
     optional bool ValidateChecksums = 7; // currently available for s3

diff --git a/ydb/core/protos/s3_settings.proto b/ydb/core/protos/s3_settings.proto
@@ -0,0 +1,57 @@
+import "ydb/public/api/protos/ydb_export.proto";
+
+package NKikimrSchemeOp;
+option java_package = "ru.yandex.kikimr.proto";
+
+message TSecretId {
+    optional string Id = 1;
+    optional string OwnerId = 2;
+}
+
+message TSecretValue {
+    optional string Data = 1;
+}
+
+message TSecretableVariable {
+    oneof Data {
+        TSecretId SecretId = 1;
+        TSecretValue Value = 2;
+    }
+}
+
+message TS3Settings {
+    enum EScheme {
+        HTTP = 0;
+        HTTPS = 1;
+    }
+
+    optional string Endpoint = 1;
+    optional EScheme Scheme = 2 [default = HTTPS];
+    optional string Bucket = 3;
+    optional string ObjectKeyPattern = 4; // dst for backup, src for restore
+    optional string AccessKey = 5;
+    optional string SecretKey = 6;
+    optional Ydb.Export.ExportToS3Settings.StorageClass StorageClass = 7 [default = STANDARD];
+    optional bool VerifySSL = 8;
+    optional string ProxyHost = 9;
+    optional uint32 ProxyPort = 10;
+    optional EScheme ProxyScheme = 11;
+    optional string Region = 12;
+    optional TSecretableVariable SecretableAccessKey = 13;
+    optional TSecretableVariable SecretableSecretKey = 14;
+    optional bool UseVirtualAddressing = 15 [default = true];
+
+    message TLimits {
+        optional uint32 ReadBatchSize = 1 [default = 8388608]; // 8 MB
+        optional uint32 MinWriteBatchSize = 2 [default = 5242880]; // 5 MB
+        reserved 3; // ReadBufferSizeLimit
+    };
+
+    optional TLimits Limits = 100;
+    optional uint32 RequestTimeoutMs = 101;
+    optional uint32 HttpRequestTimeoutMs = 102;
+    optional uint32 ConnectionTimeoutMs = 103;
+
+    optional uint32 ExecutorThreadsCount = 104 [default = 32];
+    optional uint32 MaxConnectionsCount = 105 [default = 32];
+}
diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make
@@ -113,6 +113,7 @@ SRCS(
     query_stats.proto
     replication.proto
     resource_broker.proto
+    s3_settings.proto
     scheme_board.proto
     scheme_board_mon.proto
     scheme_log.proto

diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h
@@ -3,7 +3,6 @@
 
 #include <ydb/core/tablet_flat/flat_cxx_database.h>
 #include <ydb/core/tx/long_tx_service/public/types.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 #include <ydb/core/protos/tx_columnshard.pb.h>
 #include <ydb/core/tx/columnshard/engines/insert_table/insert_table.h>
 #include <ydb/core/tx/columnshard/engines/column_engine.h>

diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp
@@ -4,7 +4,6 @@
 #include "portions/constructor_portion.h"
 
 #include <ydb/core/protos/config.pb.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 #include <ydb/core/tx/columnshard/columnshard_schema.h>
 #include <ydb/core/tx/sharding/sharding.h>
 

diff --git a/ydb/core/tx/columnshard/export/common/identifier.cpp b/ydb/core/tx/columnshard/export/common/identifier.cpp
@@ -1,7 +1,6 @@
 #include "identifier.h"
 #include <ydb/core/tx/columnshard/export/protos/task.pb.h>
 #include <ydb/core/protos/tx_columnshard.pb.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 #include <util/string/builder.h>
 
 namespace NKikimr::NOlap::NExport {
@@ -46,4 +45,4 @@ TString TIdentifier::ToString() const {
     return TStringBuilder() << "path_id=" << PathId << ";";
 }
 
-}
+}
diff --git a/ydb/core/tx/columnshard/export/protos/storage.proto b/ydb/core/tx/columnshard/export/protos/storage.proto
@@ -1,4 +1,4 @@
-import "ydb/core/protos/flat_scheme_op.proto";
+import "ydb/core/protos/s3_settings.proto";
 
 package NKikimrColumnShardExportProto;
 

diff --git a/ydb/core/tx/columnshard/export/session/selector/abstract/selector.cpp b/ydb/core/tx/columnshard/export/session/selector/abstract/selector.cpp
@@ -1,7 +1,6 @@
 #include "selector.h"
 #include <ydb/core/tx/columnshard/export/session/selector/backup/selector.h>
 #include <ydb/core/protos/tx_columnshard.pb.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 
 namespace NKikimr::NOlap::NExport {
 
@@ -13,4 +12,4 @@ NKikimr::TConclusion<TSelectorContainer> TSelectorContainer::BuildFromProto(cons
     return TSelectorContainer(std::make_shared<TBackupSelector>(parsed.DetachResult()));
 }
 
-}
+}
diff --git a/ydb/core/tx/datashard/export_s3.h b/ydb/core/tx/datashard/export_s3.h
@@ -7,6 +7,8 @@
 #include "export_iface.h"
 #include "export_s3_buffer.h"
 
+#include <ydb/core/protos/s3_settings.pb.h>
+
 namespace NKikimr {
 namespace NDataShard {
 

diff --git a/ydb/core/tx/datashard/extstorage_usage_config.h b/ydb/core/tx/datashard/extstorage_usage_config.h
@@ -6,6 +6,7 @@
 
 #include <ydb/core/base/events.h>
 #include <ydb/core/protos/flat_scheme_op.pb.h>
+#include <ydb/core/protos/s3_settings.pb.h>
 #include <ydb/public/api/protos/ydb_export.pb.h>
 
 #include <contrib/libs/aws-sdk-cpp/aws-cpp-sdk-s3/include/aws/s3/model/StorageClass.h>

diff --git a/ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h b/ydb/core/tx/schemeshard/olap/operations/alter/abstract/update.h
@@ -1,6 +1,5 @@
 #pragma once
 #include "object.h"
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 #include <ydb/core/scheme/scheme_pathid.h>
 #include <ydb/core/tx/schemeshard/schemeshard__operation_part.h>
 
@@ -52,4 +51,4 @@ class ISSEntityUpdate {
     }
 };
 
-}
+}
diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp
@@ -3,6 +3,7 @@
 #include "schemeshard__data_erasure_manager.h"
 
 #include <ydb/core/protos/table_stats.pb.h>  // for TStoragePoolsStats
+#include <ydb/core/protos/s3_settings.pb.h>
 #include <ydb/core/scheme/scheme_types_proto.h>
 #include <ydb/core/tablet/tablet_exception.h>
 #include <ydb/core/tablet_flat/flat_cxx_database.h>

diff --git a/ydb/core/tx/schemeshard/schemeshard__init_populator.cpp b/ydb/core/tx/schemeshard/schemeshard__init_populator.cpp
@@ -1,7 +1,6 @@
 #include "schemeshard_impl.h"
 #include "schemeshard_path_describer.h"
 
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 #include <ydb/core/protos/flat_tx_scheme.pb.h>
 #include <ydb/core/tx/scheme_board/populator.h>
 

diff --git a/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp
@@ -2,6 +2,7 @@
 #include "schemeshard_path_describer.h"
 
 #include <ydb/core/ydb_convert/compression.h>
+#include <ydb/core/protos/s3_settings.pb.h>
 #include <ydb/public/api/protos/ydb_export.pb.h>
 
 #include <util/string/builder.h>

diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp
@@ -14,6 +14,7 @@
 #include <ydb/core/protos/feature_flags.pb.h>
 #include <ydb/core/protos/table_stats.pb.h>  // for TStoragePoolsStats
 #include <ydb/core/protos/auth.pb.h>
+#include <ydb/core/protos/s3_settings.pb.h>
 #include <ydb/core/engine/mkql_proto.h>
 #include <ydb/core/sys_view/partition_stats/partition_stats.h>
 #include <ydb/core/statistics/events.h>

diff --git a/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp b/ydb/core/tx/schemeshard/schemeshard_import_flow_proposals.cpp
@@ -4,6 +4,7 @@
 #include <ydb/core/base/path.h>
 #include <ydb/core/ydb_convert/table_description.h>
 #include <ydb/core/ydb_convert/ydb_convert.h>
+#include <ydb/core/protos/s3_settings.pb.h>
 
 namespace NKikimr {
 namespace NSchemeShard {

diff --git a/ydb/core/tx/schemeshard/ut_external_table_reboots/ut_external_table_reboots.cpp b/ydb/core/tx/schemeshard/ut_external_table_reboots/ut_external_table_reboots.cpp
@@ -1,7 +1,6 @@
 #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h>
 
 #include <ydb/core/tx/datashard/datashard.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 
 #include <google/protobuf/text_format.h>
 

diff --git a/ydb/core/tx/schemeshard/ut_subdomain_reboots/ut_subdomain_reboots.cpp b/ydb/core/tx/schemeshard/ut_subdomain_reboots/ut_subdomain_reboots.cpp
@@ -1,7 +1,6 @@
 #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h>
 
 #include <ydb/core/tx/datashard/datashard.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 
 #include <google/protobuf/text_format.h>
 

diff --git a/ydb/core/tx/schemeshard/ut_user_attributes_reboots/ut_user_attributes_reboots.cpp b/ydb/core/tx/schemeshard/ut_user_attributes_reboots/ut_user_attributes_reboots.cpp
@@ -1,7 +1,6 @@
 #include <ydb/core/tx/schemeshard/ut_helpers/helpers.h>
 
 #include <ydb/core/tx/datashard/datashard.h>
-#include <ydb/core/protos/flat_scheme_op.pb.h>
 
 #include <google/protobuf/text_format.h>