[Storage] [DataMovement] [Tests] Moving Single Transfer Blob Tests to DataMovement.Blobs.Test package

sdk/storage/Azure.Storage.Common/src/Shared/StorageVersionExtensions.cs

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
-
+#if BlobDataMovementSDK
+extern alias BaseBlobs;
+#endif
 using System;
 
 // Alias the ServiceVersion enum used by the service importing this shared
@@ -19,7 +21,7 @@
 #elif DataMovementSDK
     Azure.Storage.Blobs.BlobClientOptions.ServiceVersion;
 #elif BlobDataMovementSDK
-    Azure.Storage.Blobs.BlobClientOptions.ServiceVersion;
+    BaseBlobs::Azure.Storage.Blobs.BlobClientOptions.ServiceVersion;
 #elif ShareDataMovementSDK
     Azure.Storage.Files.Shares.ShareClientOptions.ServiceVersion;
 #else

sdk/storage/Azure.Storage.DataMovement.Blobs/assets.json

@@ -0,0 +1,6 @@
+{
+  "AssetsRepo": "Azure/azure-sdk-assets",
+  "AssetsRepoPrefixPath": "net",
+  "TagPrefix": "net/storage/Azure.Storage.DataMovement.Blobs",
+  "Tag": "net/storage/Azure.Storage.DataMovement.Blobs_cccab594d0"
+}

sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobStartTransferCopyTests.cs

@@ -0,0 +1,269 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+extern alias DMBlobs;
+extern alias BaseBlobs;
+
+using System;
+using System.Threading.Tasks;
+using Azure.Storage.Test;
+using Azure.Storage.Test.Shared;
+using Azure.Storage.DataMovement.Tests;
+using DMBlobs::Azure.Storage.DataMovement.Blobs;
+using BaseBlobs::Azure.Storage.Blobs;
+using BaseBlobs::Azure.Storage.Blobs.Models;
+using BaseBlobs::Azure.Storage.Blobs.Specialized;
+using System.IO;
+using Azure.Core;
+using Azure.Core.TestFramework;
+using Azure.Storage.Shared;
+using NUnit.Framework;
+using Metadata = System.Collections.Generic.IDictionary<string, string>;
+
+namespace Azure.Storage.DataMovement.Blobs.Tests
+{
+    [DataMovementBlobsClientTestFixture]
+    public class AppendBlobStartTransferCopyTests : StartTransferCopyTestBase
+        <BlobServiceClient,
+        BlobContainerClient,
+        AppendBlobClient,
+        BlobClientOptions,
+        BlobServiceClient,
+        BlobContainerClient,
+        AppendBlobClient,
+        BlobClientOptions,
+        StorageTestEnvironment>
+    {
+        private readonly AccessTier _defaultAccessTier = AccessTier.Cold;
+        private const string _defaultContentType = "text/plain";
+        private const string _defaultContentLanguage = "en-US";
+        private const string _defaultContentDisposition = "inline";
+        private const string _defaultCacheControl = "no-cache";
+        private readonly Metadata _defaultMetadata = DataProvider.BuildMetadata();
+        private const int DefaultBufferSize = 4 * 1024 * 1024;
+        private const int MaxReliabilityRetries = 5;
+        private const string _blobResourcePrefix = "test-blob-";
+        private const string _expectedOverwriteExceptionMessage = "BlobAlreadyExists";
+        protected readonly BlobClientOptions.ServiceVersion _serviceVersion;
+
+        public AppendBlobStartTransferCopyTests(bool async, BlobClientOptions.ServiceVersion serviceVersion)
+            : base(async, _expectedOverwriteExceptionMessage, _blobResourcePrefix, null /* RecordedTestMode.Record /* to re-record */)
+        {
+            _serviceVersion = serviceVersion;
+            SourceClientBuilder = ClientBuilderExtensions.GetNewBlobsClientBuilder(Tenants, serviceVersion);
+            DestinationClientBuilder = ClientBuilderExtensions.GetNewBlobsClientBuilder(Tenants, serviceVersion);
+        }
+
+        protected override async Task<bool> SourceExistsAsync(AppendBlobClient objectClient)
+            => await objectClient.ExistsAsync();
+
+        protected override async Task<bool> DestinationExistsAsync(AppendBlobClient objectClient)
+            => await objectClient.ExistsAsync();
+
+        protected override async Task<IDisposingContainer<BlobContainerClient>> GetSourceDisposingContainerAsync(BlobServiceClient service = null, string containerName = null)
+            => await SourceClientBuilder.GetTestContainerAsync(service, containerName);
+
+        protected override async Task<IDisposingContainer<BlobContainerClient>> GetDestinationDisposingContainerAsync(BlobServiceClient service = null, string containerName = null)
+            => await DestinationClientBuilder.GetTestContainerAsync(service, containerName);
+
+        private async Task<AppendBlobClient> GetAppendBlobClientAsync(
+            BlobContainerClient container,
+            long? objectLength = null,
+            bool createResource = false,
+            string objectName = null,
+            BlobClientOptions options = null,
+            Stream contents = null)
+        {
+            objectName ??= GetNewObjectName();
+            AppendBlobClient blobClient = container.GetAppendBlobClient(objectName);
+
+            if (createResource)
+            {
+                if (!objectLength.HasValue)
+                {
+                    throw new InvalidOperationException($"Cannot create a blob without size specified. Either set {nameof(createResource)} to false or specify a {nameof(objectLength)}.");
+                }
+
+                if (contents != default)
+                {
+                    await UploadAppendBlocksAsync(blobClient, contents);
+                }
+                else
+                {
+                    var data = GetRandomBuffer(objectLength.Value);
+                    using Stream originalStream = await CreateLimitedMemoryStream(objectLength.Value);
+                    await UploadAppendBlocksAsync(blobClient, originalStream);
+                }
+            }
+            Uri sourceUri = blobClient.GenerateSasUri(BaseBlobs::Azure.Storage.Sas.BlobSasPermissions.All, Recording.UtcNow.AddDays(1));
+            return InstrumentClient(new AppendBlobClient(sourceUri, GetOptions()));
+        }
+
+        private async Task UploadAppendBlocksAsync(AppendBlobClient blobClient, Stream contents)
+        {
+            await blobClient.CreateIfNotExistsAsync();
+            long offset = 0;
+            long size = contents.Length;
+            long blockSize = Math.Min(DefaultBufferSize, size);
+            while (offset < size)
+            {
+                Stream partStream = WindowStream.GetWindow(contents, blockSize);
+                await blobClient.AppendBlockAsync(partStream);
+                offset += blockSize;
+            }
+        }
+
+        protected override Task<AppendBlobClient> GetSourceObjectClientAsync(
+            BlobContainerClient container,
+            long? objectLength = null,
+            bool createResource = false,
+            string objectName = null,
+            BlobClientOptions options = null,
+            Stream contents = null)
+            => GetAppendBlobClientAsync(
+                container,
+                objectLength,
+                createResource,
+                objectName,
+                options,
+                contents);
+
+        protected override StorageResourceItem GetSourceStorageResourceItem(AppendBlobClient objectClient)
+            => new AppendBlobStorageResource(objectClient);
+
+        protected override Task<Stream> SourceOpenReadAsync(AppendBlobClient objectClient)
+            => objectClient.OpenReadAsync();
+
+        protected override Task<AppendBlobClient> GetDestinationObjectClientAsync(
+            BlobContainerClient container,
+            long? objectLength = null,
+            bool createResource = false,
+            string objectName = null,
+            BlobClientOptions options = null,
+            Stream contents = null)
+            => GetAppendBlobClientAsync(
+                container,
+                objectLength,
+                createResource,
+                objectName,
+                options,
+                contents);
+
+        protected override StorageResourceItem GetDestinationStorageResourceItem(
+            AppendBlobClient objectClient,
+            TransferPropertiesTestType type = TransferPropertiesTestType.Default)
+        {
+            AppendBlobStorageResourceOptions options = default;
+            if (type == TransferPropertiesTestType.NewProperties)
+            {
+                options = new AppendBlobStorageResourceOptions
+                {
+                    ContentDisposition = new(_defaultContentDisposition),
+                    ContentLanguage = new(_defaultContentLanguage),
+                    CacheControl = new(_defaultCacheControl),
+                    ContentType = new(_defaultContentType)
+                };
+            }
+            else if (type == TransferPropertiesTestType.NoPreserve)
+            {
+                options = new AppendBlobStorageResourceOptions
+                {
+                    ContentDisposition = new(false),
+                    ContentLanguage = new(false),
+                    CacheControl = new(false),
+                    ContentType = new(false)
+                };
+            }
+            else if (type == TransferPropertiesTestType.Preserve)
+            {
+                options = new AppendBlobStorageResourceOptions
+                {
+                    ContentDisposition = new(true),
+                    ContentLanguage = new(true),
+                    CacheControl = new(true),
+                    ContentType = new(true)
+                };
+            }
+            return new AppendBlobStorageResource(objectClient, options);
+        }
+
+        protected override Task<Stream> DestinationOpenReadAsync(AppendBlobClient objectClient)
+            => objectClient.OpenReadAsync();
+
+        protected override async Task VerifyPropertiesCopyAsync(
+            DataTransfer transfer,
+            TransferPropertiesTestType transferPropertiesTestType,
+            TestEventsRaised testEventsRaised,
+            AppendBlobClient sourceClient,
+            AppendBlobClient destinationClient)
+        {
+            // Verify completion
+            Assert.NotNull(transfer);
+            Assert.IsTrue(transfer.HasCompleted);
+            Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State);
+            // Verify Copy - using original source File and Copying the destination
+            await testEventsRaised.AssertSingleCompletedCheck();
+            using Stream sourceStream = await sourceClient.OpenReadAsync();
+            using Stream destinationStream = await destinationClient.OpenReadAsync();
+            Assert.AreEqual(sourceStream, destinationStream);
+
+            if (transferPropertiesTestType == TransferPropertiesTestType.NoPreserve)
+            {
+                BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync();
+
+                Assert.IsEmpty(destinationProperties.Metadata);
+                Assert.IsNull(destinationProperties.ContentDisposition);
+                Assert.IsNull(destinationProperties.ContentLanguage);
+                Assert.IsNull(destinationProperties.CacheControl);
+
+                GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync();
+                Assert.IsEmpty(destinationTags.Tags);
+            }
+            else if (transferPropertiesTestType == TransferPropertiesTestType.NewProperties)
+            {
+                BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync();
+
+                Assert.That(_defaultMetadata, Is.EqualTo(destinationProperties.Metadata));
+                Assert.AreEqual(_defaultAccessTier.ToString(), destinationProperties.AccessTier);
+                Assert.AreEqual(_defaultContentDisposition, destinationProperties.ContentDisposition);
+                Assert.AreEqual(_defaultContentLanguage, destinationProperties.ContentLanguage);
+                Assert.AreEqual(_defaultCacheControl, destinationProperties.CacheControl);
+                Assert.AreEqual(_defaultContentType, destinationProperties.ContentType);
+            }
+            else //(transferPropertiesTestType == TransferPropertiesTestType.Default ||
+                 //transferPropertiesTestType == TransferPropertiesTestType.Preserve)
+            {
+                BlobProperties sourceProperties = await sourceClient.GetPropertiesAsync();
+                BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync();
+
+                Assert.That(sourceProperties.Metadata, Is.EqualTo(destinationProperties.Metadata));
+                Assert.AreEqual(sourceProperties.AccessTier, destinationProperties.AccessTier);
+                Assert.AreEqual(sourceProperties.ContentDisposition, destinationProperties.ContentDisposition);
+                Assert.AreEqual(sourceProperties.ContentLanguage, destinationProperties.ContentLanguage);
+                Assert.AreEqual(sourceProperties.CacheControl, destinationProperties.CacheControl);
+                Assert.AreEqual(sourceProperties.ContentType, destinationProperties.ContentType);
+            }
+        }
+
+        public BlobClientOptions GetOptions()
+        {
+            var options = new BlobClientOptions((BlobClientOptions.ServiceVersion)_serviceVersion)
+            {
+                Diagnostics = { IsLoggingEnabled = true },
+                Retry =
+                {
+                    Mode = RetryMode.Exponential,
+                    MaxRetries = MaxReliabilityRetries,
+                    Delay = TimeSpan.FromSeconds(Mode == RecordedTestMode.Playback ? 0.01 : 1),
+                    MaxDelay = TimeSpan.FromSeconds(Mode == RecordedTestMode.Playback ? 0.1 : 60)
+                },
+            };
+            if (Mode != RecordedTestMode.Live)
+            {
+                options.AddPolicy(new RecordedClientRequestIdPolicy(Recording), HttpPipelinePosition.PerCall);
+            }
+
+            return InstrumentClientOptions(options);
+        }
+    }
+}