Parallel download of a blob to a stream or local file (Azure#7429)

JoshLove-msft · Sep 10, 2019 · 5011caf · 5011caf
1 parent 7d0d7b9
commit 5011caf
Show file tree

Hide file tree

Showing 61 changed files with 57,491 additions and 196 deletions.
diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs
diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTests.cs
@@ -52,6 +52,8 @@ public void Ctor_ConnectionString()
             Assert.AreEqual("accountName", builder.AccountName);
         }
 
+        #region Sequential Download
+
         [Test]
         public async Task DownloadAsync()
         {
@@ -307,6 +309,94 @@ async Task Verify(Response<BlobDownloadInfo> response)
                 }
             }
         }
+        #endregion Sequential Download
+
+        #region Parallel Download
+
+        private async Task ParallelDownloadFileAndVerify(
+            long size,
+            long singleBlockThreshold,
+            ParallelTransferOptions parallelTransferOptions)
+        {
+            var data = this.GetRandomBuffer(size);
+            var path = Path.GetTempFileName();
+
+            try
+            {
+                using (this.GetNewContainer(out var container))
+                {
+                    var name = this.GetNewBlobName();
+                    var blob = this.InstrumentClient(container.GetBlobClient(name));
+
+                    using (var stream = new MemoryStream(data))
+                    {
+                        await blob.UploadAsync(stream);
+                    }
+
+                    var destination = new FileInfo(path);
+
+                    await blob.StagedDownloadAsync(
+                        destination,
+                        singleBlockThreshold: singleBlockThreshold,
+                        parallelTransferOptions: parallelTransferOptions
+                        );
+
+                    using (var resultStream = destination.OpenRead())
+                    {
+                        TestHelper.AssertSequenceEqual(data, resultStream.AsBytes());
+                    }
+                }
+            }
+            finally
+            {
+                if (File.Exists(path))
+                {
+                    File.Delete(path);
+                }
+            }
+        }
+
+        [Test]
+        [TestCase(512)]
+        [TestCase(1 * Constants.KB)]
+        [TestCase(2 * Constants.KB)]
+        [TestCase(4 * Constants.KB)]
+        [TestCase(10 * Constants.KB)]
+        [TestCase(20 * Constants.KB)]
+        [TestCase(30 * Constants.KB)]
+        [TestCase(50 * Constants.KB)]
+        [TestCase(501 * Constants.KB)]
+        public async Task DownloadFileAsync_Parallel_SmallBlobs(long size) =>
+            // Use a 1KB threshold so we get a lot of individual blocks
+            await this.ParallelDownloadFileAndVerify(size, Constants.KB, new ParallelTransferOptions { MaximumTransferLength = Constants.KB });
+
+        [Test]
+        [Category("Live")]
+        [TestCase(33 * Constants.MB, 1)]
+        [TestCase(33 * Constants.MB, 4)]
+        [TestCase(33 * Constants.MB, 8)]
+        [TestCase(33 * Constants.MB, 16)]
+        [TestCase(33 * Constants.MB, null)]
+        [TestCase(257 * Constants.MB, 1)]
+        [TestCase(257 * Constants.MB, 4)]
+        [TestCase(257 * Constants.MB, 8)]
+        [TestCase(257 * Constants.MB, 16)]
+        [TestCase(257 * Constants.MB, null)]
+        [TestCase(1 * Constants.GB, 1)]
+        [TestCase(1 * Constants.GB, 4)]
+        [TestCase(1 * Constants.GB, 8)]
+        [TestCase(1 * Constants.GB, 16)]
+        [TestCase(1 * Constants.GB, null)]
+        public async Task DownloadFileAsync_Parallel_LargeBlobs(long size, int? maximumThreadCount)
+        {
+            // TODO: #6781 We don't want to add 1GB of random data in the recordings
+            if (this.Mode == RecordedTestMode.Live)
+            {
+                await this.ParallelDownloadFileAndVerify(size, 16 * Constants.MB, new ParallelTransferOptions { MaximumThreadCount = maximumThreadCount });
+            }
+        }
+
+        #endregion Parallel Download
 
         [Test]
         public async Task StartCopyFromUriAsync()

diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobClientTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobClientTests.cs
@@ -48,6 +48,8 @@ public void Ctor_ConnectionString()
             Assert.AreEqual("accountName", builder.AccountName);
         }
 
+        #region Upload
+
         [Test]
         public async Task UploadAsync_Stream()
         {
@@ -446,7 +448,7 @@ await blob.StagedUploadAsync(
         [TestCase(501 * Constants.KB)]
         public async Task UploadStreamAsync_SmallBlobs(long size) =>
             // Use a 1KB threshold so we get a lot of individual blocks
-            await this.UploadStreamAndVerify(size, Constants.KB, new ParallelTransferOptions { MaximumBlockLength = Constants.KB });
+            await this.UploadStreamAndVerify(size, Constants.KB, new ParallelTransferOptions { MaximumTransferLength = Constants.KB });
 
         [Test]
         [TestCase(512)]
@@ -460,7 +462,7 @@ public async Task UploadStreamAsync_SmallBlobs(long size) =>
         [TestCase(501 * Constants.KB)]
         public async Task UploadFileAsync_SmallBlobs(long size) =>
             // Use a 1KB threshold so we get a lot of individual blocks
-            await this.UploadFileAndVerify(size, Constants.KB, new ParallelTransferOptions { MaximumBlockLength = Constants.KB });
+            await this.UploadFileAndVerify(size, Constants.KB, new ParallelTransferOptions { MaximumTransferLength = Constants.KB });
 
         [Test]
         [TestCase(33 * Constants.MB, 1)]
@@ -512,5 +514,7 @@ public async Task UploadFileAsync_LargeBlobs(long size, int? maximumThreadCount)
                 await this.UploadFileAndVerify(size, 16 * Constants.MB, new ParallelTransferOptions { MaximumThreadCount = maximumThreadCount });
             }
         }
+
+        #endregion Upload
     }
 }
diff --git a/...ssionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,1).json b/...ssionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,1).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...Records/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,1)Async.json b/...Records/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,1)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...sionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,16).json b/...sionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,16).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...ecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,16)Async.json b/...ecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,16)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...ssionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,4).json b/...ssionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,4).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...Records/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,4)Async.json b/...Records/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,4)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...ssionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,8).json b/...ssionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,8).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...Records/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,8)Async.json b/...Records/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,8)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...onRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,null).json b/...onRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,null).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...ords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,null)Async.json b/...ords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(1073741824,null)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...essionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,1).json b/...essionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,1).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...nRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,1)Async.json b/...nRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,1)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...ssionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,16).json b/...ssionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,16).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...Records/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,16)Async.json b/...Records/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,16)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...essionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,4).json b/...essionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,4).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...nRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,4)Async.json b/...nRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,4)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...essionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,8).json b/...essionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,8).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...nRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,8)Async.json b/...nRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,8)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...ionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,null).json b/...ionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,null).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...cords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,null)Async.json b/...cords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(269484032,null)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...SessionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,1).json b/...SessionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,1).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...onRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,1)Async.json b/...onRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,1)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...essionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,16).json b/...essionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,16).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...nRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,16)Async.json b/...nRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,16)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...SessionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,4).json b/...SessionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,4).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...onRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,4)Async.json b/...onRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,4)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...SessionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,8).json b/...SessionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,8).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...onRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,8)Async.json b/...onRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,8)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...sionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,null).json b/...sionRecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,null).json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}
diff --git a/...ecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,null)Async.json b/...ecords/BlobBaseClientTests/DownloadFileAsync_Parallel_LargeBlobs(34603008,null)Async.json
@@ -0,0 +1,4 @@
+{
+  "Entries": [],
+  "Variables": {}
+}