diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs index 1a1b82f90c86..0af5fb342107 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs @@ -25,21 +25,12 @@ internal class AppendBlobStorageResource : StorageResourceItemInternal public override string ProviderId => "blob"; - /// - /// Defines the recommended Transfer Type for the storage resource. - /// protected override DataTransferOrder TransferType => DataTransferOrder.Sequential; - /// - /// Defines the maximum chunk size for the storage resource. - /// + protected override long MaxSupportedSingleTransferSize => Constants.Blob.Append.MaxAppendBlockBytes; + protected override long MaxSupportedChunkSize => Constants.Blob.Append.MaxAppendBlockBytes; - /// - /// Length of the storage resource. This information is obtained during a GetStorageResources API call. - /// - /// Will return default if the length was not set by a GetStorageResources API call. - /// protected override long? Length => ResourceProperties?.ResourceLength; internal AppendBlobStorageResource() diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs index 6630d1d5cdbb..1e518152bc61 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs @@ -34,26 +34,12 @@ internal class BlockBlobStorageResource : StorageResourceItemInternal public override string ProviderId => "blob"; - /// - /// Defines the recommended Transfer Type of the storage resource. - /// protected override DataTransferOrder TransferType => DataTransferOrder.Unordered; - /// - /// Store Max Initial Size that a Put Blob can get to. - /// - internal static long _maxInitialSize => Constants.Blob.Block.Pre_2019_12_12_MaxUploadBytes; + protected override long MaxSupportedSingleTransferSize => Constants.Blob.Block.MaxUploadBytes; - /// - /// Defines the maximum chunk size for the storage resource. - /// protected override long MaxSupportedChunkSize => Constants.Blob.Block.MaxStageBytes; - /// - /// Length of the storage resource. This information is can obtained during a GetStorageResources API call. - /// - /// Will return default if the length was not set by a GetStorageResources API call. - /// protected override long? Length => ResourceProperties?.ResourceLength; /// @@ -168,7 +154,7 @@ await BlobClient.UploadAsync( DataMovementBlobsExtensions.GetBlobUploadOptions( _options, overwrite, - _maxInitialSize, + MaxSupportedSingleTransferSize, // We don't want any internal partioning options?.SourceProperties), cancellationToken: cancellationToken).ConfigureAwait(false); return; diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs index a24a8c9a7989..329bc8e30746 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs @@ -26,21 +26,12 @@ internal class PageBlobStorageResource : StorageResourceItemInternal public override string ProviderId => "blob"; - /// - /// Defines the recommended Transfer Type for the storage resource. - /// protected override DataTransferOrder TransferType => DataTransferOrder.Unordered; - /// - /// Defines the maximum chunk size for the storage resource. - /// + protected override long MaxSupportedSingleTransferSize => Constants.Blob.Page.MaxPageBlockBytes; + protected override long MaxSupportedChunkSize => Constants.Blob.Page.MaxPageBlockBytes; - /// - /// Length of the storage resource. This information is obtained during a GetStorageResources API call. - /// - /// Will return default if the length was not set by a GetStorageResources API call. - /// protected override long? Length => ResourceProperties?.ResourceLength; public PageBlobStorageResource() diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs index 762c4a3d34c5..bf57e614a4ff 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs @@ -28,6 +28,8 @@ internal class ShareFileStorageResource : StorageResourceItemInternal protected override DataTransferOrder TransferType => DataTransferOrder.Unordered; + protected override long MaxSupportedSingleTransferSize => DataMovementShareConstants.MaxRange; + protected override long MaxSupportedChunkSize => DataMovementShareConstants.MaxRange; protected override long? Length => ResourceProperties?.ResourceLength; diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs index dc0c6be48809..128331d09d4b 100644 --- a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs @@ -177,6 +177,7 @@ protected StorageResourceItem() { } protected internal override bool IsContainer { get { throw null; } } protected internal abstract long? Length { get; } protected internal abstract long MaxSupportedChunkSize { get; } + protected internal abstract long MaxSupportedSingleTransferSize { get; } protected internal abstract string ResourceId { get; } protected internal Azure.Storage.DataMovement.StorageResourceItemProperties ResourceProperties { get { throw null; } set { } } protected internal abstract Azure.Storage.DataMovement.DataTransferOrder TransferType { get; } diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs index dc0c6be48809..128331d09d4b 100644 --- a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs @@ -177,6 +177,7 @@ protected StorageResourceItem() { } protected internal override bool IsContainer { get { throw null; } } protected internal abstract long? Length { get; } protected internal abstract long MaxSupportedChunkSize { get; } + protected internal abstract long MaxSupportedSingleTransferSize { get; } protected internal abstract string ResourceId { get; } protected internal Azure.Storage.DataMovement.StorageResourceItemProperties ResourceProperties { get { throw null; } set { } } protected internal abstract Azure.Storage.DataMovement.DataTransferOrder TransferType { get; } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/DataTransferOptions.cs b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferOptions.cs index 0b8fb84930b7..54afe76ba5c3 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/DataTransferOptions.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferOptions.cs @@ -17,7 +17,7 @@ public class DataTransferOptions : IEquatable /// The default value is 4 MiB. /// /// When resuming a transfer, the default value will be the value specified - /// when the transfer was first started. + /// when the transfer was first started but can still be overriden. /// /// This value may be clamped to the maximum allowed for the particular transfer/resource type. /// @@ -31,7 +31,7 @@ public class DataTransferOptions : IEquatable /// The default value is 32 MiB. /// /// When resuming a transfer, the default value will be the value specified - /// when the transfer was first started. + /// when the transfer was first started but can still be overriden. /// /// This value may be clamped to the maximum allowed for the particular transfer/resource type. /// diff --git a/sdk/storage/Azure.Storage.DataMovement/src/JobPartInternal.cs b/sdk/storage/Azure.Storage.DataMovement/src/JobPartInternal.cs index 536fb3501e80..72fde2348d2e 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/JobPartInternal.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/JobPartInternal.cs @@ -185,11 +185,11 @@ internal JobPartInternal( SingleTransferCompletedEventHandler = singleTransferEventHandler; ClientDiagnostics = clientDiagnostics; - // Set transfer sizes to user specified values or default + // Set transfer sizes to user specified values or default, // clamped to max supported chunk size for the destination. _initialTransferSize = Math.Min( initialTransferSize ?? DataMovementConstants.DefaultInitialTransferSize, - _destinationResource.MaxSupportedChunkSize); + _destinationResource.MaxSupportedSingleTransferSize); _transferChunkSize = Math.Min( transferChunkSize ?? DataMovementConstants.DefaultChunkSize, _destinationResource.MaxSupportedChunkSize); diff --git a/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs index fa9f17ca859e..f9a231c0047f 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs @@ -23,22 +23,12 @@ internal class LocalFileStorageResource : StorageResourceItem public override string ProviderId => "local"; - /// - /// Defines the recommended Transfer Type of the resource - /// protected internal override DataTransferOrder TransferType => DataTransferOrder.Sequential; - /// - /// Defines the maximum chunk size for the storage resource. - /// - /// TODO: consider changing this. + protected internal override long MaxSupportedSingleTransferSize => Constants.Blob.Block.MaxStageBytes; + protected internal override long MaxSupportedChunkSize => Constants.Blob.Block.MaxStageBytes; - /// - /// Length of the storage resource. This information is can obtained during a GetStorageResources API call. - /// - /// Will return default if the length was not set by a GetStorageResources API call. - /// protected internal override long? Length => default; /// diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceItem.cs b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceItem.cs index 624b30ce8244..360f0cc4eaa1 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceItem.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceItem.cs @@ -27,6 +27,12 @@ protected StorageResourceItem() { } /// protected internal abstract DataTransferOrder TransferType { get; } + /// + /// Defines the maximum supported size for the storage resource to be created + /// in a single API call. + /// + protected internal abstract long MaxSupportedSingleTransferSize { get; } + /// /// Defines the maximum supported chunk size for the storage resource. /// diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/CleanUpTransferTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/CleanUpTransferTests.cs index 985cc958ba89..83138f49c05b 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/CleanUpTransferTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/CleanUpTransferTests.cs @@ -47,6 +47,8 @@ private Mock GetRemoteDestinationResource(bool throwOnDelet .Returns("BlockBlob"); mock.Setup(b => b.ProviderId) .Returns("blob"); + mock.Setup(b => b.MaxSupportedSingleTransferSize) + .Returns(Constants.GB); mock.Setup(b => b.MaxSupportedChunkSize) .Returns(Constants.GB); mock.Setup(b => b.GetSourceCheckpointData()) @@ -114,7 +116,8 @@ public async Task CleanupAfterFailureAsync() destMock.Verify(b => b.Uri, Times.Exactly(6)); destMock.Verify(b => b.ProviderId, Times.Once()); destMock.Verify(b => b.ResourceId, Times.Once()); - destMock.Verify(b => b.MaxSupportedChunkSize, Times.Exactly(2)); + destMock.Verify(b => b.MaxSupportedSingleTransferSize, Times.Once()); + destMock.Verify(b => b.MaxSupportedChunkSize, Times.Once()); destMock.Verify(b => b.GetDestinationCheckpointData(), Times.Once()); destMock.Verify(b => b.SetPermissionsAsync( sourceMock.Object, @@ -157,7 +160,8 @@ public async Task ErrorThrownDuringCleanup() destMock.Verify(b => b.Uri, Times.Exactly(6)); destMock.Verify(b => b.ProviderId, Times.Once()); destMock.Verify(b => b.ResourceId, Times.Once()); - destMock.Verify(b => b.MaxSupportedChunkSize, Times.Exactly(2)); + destMock.Verify(b => b.MaxSupportedSingleTransferSize, Times.Once()); + destMock.Verify(b => b.MaxSupportedChunkSize, Times.Once()); destMock.Verify(b => b.GetDestinationCheckpointData(), Times.Once()); destMock.Verify(b => b.SetPermissionsAsync( sourceMock.Object, diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResourceItem.cs b/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResourceItem.cs index c335ed07894b..1ac228538d3f 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResourceItem.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResourceItem.cs @@ -22,6 +22,8 @@ internal class MockStorageResourceItem : StorageResourceItem protected internal override DataTransferOrder TransferType { get; } + protected internal override long MaxSupportedSingleTransferSize => Constants.GB; + protected internal override long MaxSupportedChunkSize => Constants.GB; protected internal override long? Length { get; } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/ServiceToServiceJobPartTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/ServiceToServiceJobPartTests.cs index ce5ce1d569f8..d8997a22e3d4 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/ServiceToServiceJobPartTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/ServiceToServiceJobPartTests.cs @@ -134,6 +134,7 @@ public async Task ProcessPartToChunkAsync_OneShot() .Returns(Task.CompletedTask); mockDestination.Setup(resource => resource.CopyFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(Task.CompletedTask); + mockDestination.Setup(r => r.MaxSupportedSingleTransferSize).Returns(Constants.MB); mockDestination.Setup(r => r.MaxSupportedChunkSize).Returns(Constants.MB); // Set up default checkpointer with transfer job @@ -203,6 +204,7 @@ public async Task ProcessPartToChunkAsync_Chunks() .Returns(Task.CompletedTask); mockDestination.Setup(resource => resource.CompleteTransferAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(Task.CompletedTask); + mockDestination.Setup(r => r.MaxSupportedSingleTransferSize).Returns(length - 1); mockDestination.Setup(r => r.MaxSupportedChunkSize).Returns(chunkSize); // Set up default checkpointer with transfer job diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs index 9ab272964c93..a2937af8d8f4 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs @@ -21,6 +21,8 @@ internal class MemoryStorageResourceItem : StorageResourceItem protected internal override DataTransferOrder TransferType => DataTransferOrder.Unordered; + protected internal override long MaxSupportedSingleTransferSize => long.MaxValue; + protected internal override long MaxSupportedChunkSize => long.MaxValue; protected internal override long? Length => Buffer.Length; diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StorageResourceTestWrappers.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StorageResourceTestWrappers.cs index 732697d8a409..af4b429a800f 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StorageResourceTestWrappers.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StorageResourceTestWrappers.cs @@ -45,6 +45,8 @@ private T ThrowOrDo(Func func) => _throwScopeManager.InScope protected internal override DataTransferOrder TransferType => ThrowOr(_inner.TransferType); + protected internal override long MaxSupportedSingleTransferSize => ThrowOr(_inner.MaxSupportedSingleTransferSize); + protected internal override long MaxSupportedChunkSize => ThrowOr(_inner.MaxSupportedChunkSize); protected internal override long? Length => ThrowOr(_inner.Length); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StreamToUriJobPartTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StreamToUriJobPartTests.cs index a308aec49c0f..a8349cc9c29f 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/StreamToUriJobPartTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/StreamToUriJobPartTests.cs @@ -135,6 +135,7 @@ public async Task ProcessPartToChunkAsync_OneShot() Mock mockDestination = GetServiceStorageResourceItem(); mockDestination.Setup(resource => resource.CopyFromStreamAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(Task.CompletedTask); + mockDestination.Setup(r => r.MaxSupportedSingleTransferSize).Returns(Constants.MB); mockDestination.Setup(r => r.MaxSupportedChunkSize).Returns(Constants.MB); // Set up source with properties and read stream @@ -211,7 +212,9 @@ public async Task ProcessPartToChunkAsync_Chunks() .Returns(Task.CompletedTask); mockDestination.Setup(resource => resource.CompleteTransferAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(Task.CompletedTask); + mockDestination.Setup(r => r.MaxSupportedSingleTransferSize).Returns(length - 1); mockDestination.Setup(r => r.MaxSupportedChunkSize).Returns(chunkSize); + var data = GetRandomBuffer(length); using var stream = new MemoryStream(data); using var stream2 = new MemoryStream(data); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/TransferManagerTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/TransferManagerTests.cs index 18d972f649cc..f0b16da5a8bc 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/TransferManagerTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/TransferManagerTests.cs @@ -460,6 +460,7 @@ public static void BasicSetup( items.Source.SetupGet(r => r.Length).Returns(itemSize); items.Destination.SetupGet(r => r.TransferType).Returns(default(DataTransferOrder)); + items.Destination.SetupGet(r => r.MaxSupportedSingleTransferSize).Returns(Constants.GB); items.Destination.SetupGet(r => r.MaxSupportedChunkSize).Returns(Constants.GB); items.Source.Setup(r => r.GetPropertiesAsync(It.IsAny())) @@ -560,6 +561,7 @@ public static void VerifyDestinationResourceOnJobProcess(this Mock r.Uri); dstResource.VerifyGet(r => r.ResourceId); + dstResource.VerifyGet(r => r.MaxSupportedSingleTransferSize); dstResource.VerifyGet(r => r.MaxSupportedChunkSize); }