diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index 98dc99c0e17..f9602cbc20d 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -3,6 +3,7 @@ Usage of vtbackup: --alsologtostderr log to standard error as well as files --azblob_backup_account_key_file string Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path). --azblob_backup_account_name string Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used. + --azblob_backup_buffer_size int The memory buffer size to use in bytes, per file or stripe, when streaming to Azure Blob Service. (default 104857600) --azblob_backup_container_name string Azure Blob Container Name. --azblob_backup_parallelism int Azure Blob operation parallelism (requires extra memory when increased). (default 1) --azblob_backup_storage_root string Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/'). diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index 0c165a66a5a..a252a7f1ff8 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -3,6 +3,7 @@ Usage of vtctld: --alsologtostderr log to standard error as well as files --azblob_backup_account_key_file string Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path). --azblob_backup_account_name string Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used. + --azblob_backup_buffer_size int The memory buffer size to use in bytes, per file or stripe, when streaming to Azure Blob Service. (default 104857600) --azblob_backup_container_name string Azure Blob Container Name. --azblob_backup_parallelism int Azure Blob operation parallelism (requires extra memory when increased). (default 1) --azblob_backup_storage_root string Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/'). diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 32160387009..9902b4b2076 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -4,6 +4,7 @@ Usage of vttablet: --app_pool_size int Size of the connection pool for app connections (default 40) --azblob_backup_account_key_file string Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path). --azblob_backup_account_name string Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used. + --azblob_backup_buffer_size int The memory buffer size to use in bytes, per file or stripe, when streaming to Azure Blob Service. (default 104857600) --azblob_backup_container_name string Azure Blob Container Name. --azblob_backup_parallelism int Azure Blob operation parallelism (requires extra memory when increased). (default 1) --azblob_backup_storage_root string Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/'). diff --git a/go/vt/mysqlctl/azblobbackupstorage/azblob.go b/go/vt/mysqlctl/azblobbackupstorage/azblob.go index 660abcc5008..144b9435437 100644 --- a/go/vt/mysqlctl/azblobbackupstorage/azblob.go +++ b/go/vt/mysqlctl/azblobbackupstorage/azblob.go @@ -52,6 +52,7 @@ var ( storageRoot string azBlobParallelism int + azBlobBufferSize = 100 << (10 * 2) // 100 MiB ) func registerFlags(fs *pflag.FlagSet) { @@ -59,6 +60,7 @@ func registerFlags(fs *pflag.FlagSet) { fs.StringVar(&accountKeyFile, "azblob_backup_account_key_file", "", "Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).") fs.StringVar(&containerName, "azblob_backup_container_name", "", "Azure Blob Container Name.") fs.StringVar(&storageRoot, "azblob_backup_storage_root", "", "Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/').") + fs.IntVar(&azBlobBufferSize, "azblob_backup_buffer_size", azBlobBufferSize, "The memory buffer size to use in bytes, per file or stripe, when streaming to Azure Blob Service.") fs.IntVar(&azBlobParallelism, "azblob_backup_parallelism", 1, "Azure Blob operation parallelism (requires extra memory when increased).") } @@ -218,7 +220,7 @@ func (bh *AZBlobBackupHandle) AddFile(ctx context.Context, filename string, file go func() { defer bh.waitGroup.Done() _, err := azblob.UploadStreamToBlockBlob(bh.ctx, reader, blockBlobURL, azblob.UploadStreamToBlockBlobOptions{ - BufferSize: azblob.BlockBlobMaxStageBlockBytes, + BufferSize: azBlobBufferSize, MaxBuffers: azBlobParallelism, }) if err != nil {