[release-17.0] Address vttablet memory usage with backups to Azure Bl…

…ob Service (#13770) (#13775)

Signed-off-by: Matt Lord <mattalord@gmail.com>
Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com>

go/flags/endtoend/vtbackup.txt

@@ -3,8 +3,9 @@ Usage of vtbackup:
       --alsologtostderr                                             log to standard error as well as files
       --azblob_backup_account_key_file string                       Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).
       --azblob_backup_account_name string                           Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used.
+      --azblob_backup_buffer_size int                               The memory buffer size to use in bytes, per file or stripe, when streaming to Azure Blob Service. (default 104857600)
       --azblob_backup_container_name string                         Azure Blob Container Name.
-      --azblob_backup_parallelism int                               Azure Blob operation parallelism (requires extra memory when increased). (default 1)
+      --azblob_backup_parallelism int                               Azure Blob operation parallelism (requires extra memory when increased -- a multiple of azblob_backup_buffer_size). (default 1)
       --azblob_backup_storage_root string                           Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/').
       --backup_engine_implementation string                         Specifies which implementation to use for creating new backups (builtin or xtrabackup). Restores will always be done with whichever engine created a given backup. (default "builtin")
       --backup_storage_block_size int                               if backup_storage_compress is true, backup_storage_block_size sets the byte size for each block while compressing (default is 250000). (default 250000)

go/flags/endtoend/vtctld.txt

@@ -3,8 +3,9 @@ Usage of vtctld:
       --alsologtostderr                                                  log to standard error as well as files
       --azblob_backup_account_key_file string                            Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).
       --azblob_backup_account_name string                                Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used.
+      --azblob_backup_buffer_size int                                    The memory buffer size to use in bytes, per file or stripe, when streaming to Azure Blob Service. (default 104857600)
       --azblob_backup_container_name string                              Azure Blob Container Name.
-      --azblob_backup_parallelism int                                    Azure Blob operation parallelism (requires extra memory when increased). (default 1)
+      --azblob_backup_parallelism int                                    Azure Blob operation parallelism (requires extra memory when increased -- a multiple of azblob_backup_buffer_size). (default 1)
       --azblob_backup_storage_root string                                Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/').
       --backup_engine_implementation string                              Specifies which implementation to use for creating new backups (builtin or xtrabackup). Restores will always be done with whichever engine created a given backup. (default "builtin")
       --backup_storage_block_size int                                    if backup_storage_compress is true, backup_storage_block_size sets the byte size for each block while compressing (default is 250000). (default 250000)

go/flags/endtoend/vttablet.txt

@@ -4,8 +4,9 @@ Usage of vttablet:
       --app_pool_size int                                                Size of the connection pool for app connections (default 40)
       --azblob_backup_account_key_file string                            Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).
       --azblob_backup_account_name string                                Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used.
+      --azblob_backup_buffer_size int                                    The memory buffer size to use in bytes, per file or stripe, when streaming to Azure Blob Service. (default 104857600)
       --azblob_backup_container_name string                              Azure Blob Container Name.
-      --azblob_backup_parallelism int                                    Azure Blob operation parallelism (requires extra memory when increased). (default 1)
+      --azblob_backup_parallelism int                                    Azure Blob operation parallelism (requires extra memory when increased -- a multiple of azblob_backup_buffer_size). (default 1)
       --azblob_backup_storage_root string                                Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/').
       --backup_engine_implementation string                              Specifies which implementation to use for creating new backups (builtin or xtrabackup). Restores will always be done with whichever engine created a given backup. (default "builtin")
       --backup_storage_block_size int                                    if backup_storage_compress is true, backup_storage_block_size sets the byte size for each block while compressing (default is 250000). (default 250000)

go/vt/mysqlctl/azblobbackupstorage/azblob.go

@@ -73,6 +73,14 @@ var (
 		},
 	)
 
+	azBlobBufferSize = viperutil.Configure(
+		configKey("buffer_size"),
+		viperutil.Options[int]{
+			Default:  100 << (10 * 2), // 100 MiB
+			FlagName: "azblob_buffer_size",
+		},
+	)
+
 	azBlobParallelism = viperutil.Configure(
 		configKey("parallelism"),
 		viperutil.Options[int]{
@@ -91,7 +99,8 @@ func registerFlags(fs *pflag.FlagSet) {
 	fs.String("azblob_backup_account_key_file", accountKeyFile.Default(), "Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).")
 	fs.String("azblob_backup_container_name", containerName.Default(), "Azure Blob Container Name.")
 	fs.String("azblob_backup_storage_root", storageRoot.Default(), "Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/').")
-	fs.Int("azblob_backup_parallelism", azBlobParallelism.Default(), "Azure Blob operation parallelism (requires extra memory when increased).")
+	fs.Int("azblob_backup_buffer_size", azBlobBufferSize.Default(), "The memory buffer size to use in bytes, per file or stripe, when streaming to Azure Blob Service.")
+	fs.Int("azblob_backup_parallelism", azBlobParallelism.Default(), "Azure Blob operation parallelism (requires extra memory when increased -- a multiple of azblob_backup_buffer_size).")
 
 	viperutil.BindFlags(fs, accountName, accountKeyFile, containerName, storageRoot, azBlobParallelism)
 }
@@ -248,7 +257,7 @@ func (bh *AZBlobBackupHandle) AddFile(ctx context.Context, filename string, file
 	go func() {
 		defer bh.waitGroup.Done()
 		_, err := azblob.UploadStreamToBlockBlob(bh.ctx, reader, blockBlobURL, azblob.UploadStreamToBlockBlobOptions{
-			BufferSize: azblob.BlockBlobMaxStageBlockBytes,
+			BufferSize: azBlobBufferSize.Get(),
 			MaxBuffers: azBlobParallelism.Get(),
 		})
 		if err != nil {