Checkpointer use testing (#47224)

* fix and test reporting * testing * test checpointer use - item transfer * container transfer test * use memory checkpointer in tests * fix test * fix scope * simplify container test * missing assert * remove tostring used for debugging
Azure · Dec 3, 2024 · 68e212d · 68e212d
1 parent 1d41111
commit 68e212d
Show file tree

Hide file tree

Showing 4 changed files with 163 additions and 91 deletions.
diff --git a/sdk/storage/Azure.Storage.DataMovement/src/TransferJobInternal.cs b/sdk/storage/Azure.Storage.DataMovement/src/TransferJobInternal.cs
@@ -508,6 +508,7 @@ await TransferFailedEventHandler.RaiseAsync(
                         ClientDiagnostics)
                         .ConfigureAwait(false);
                 }
+                _dataTransfer.TransferStatus.SetFailedItem();
             }
 
             try

diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/AssemblyInfo.cs b/sdk/storage/Azure.Storage.DataMovement/tests/AssemblyInfo.cs
@@ -0,0 +1,6 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")]
diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryTransferCheckpointer.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryTransferCheckpointer.cs
@@ -57,7 +57,7 @@ public JobPartPlanHeader Plan
 
     public Dictionary<string, Job> Jobs { get; set; } = new();
 
-    public Task AddNewJobAsync(string transferId, StorageResource source, StorageResource destination, CancellationToken cancellationToken = default)
+    public virtual Task AddNewJobAsync(string transferId, StorageResource source, StorageResource destination, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (Jobs.ContainsKey(transferId))
@@ -71,12 +71,12 @@ public Task AddNewJobAsync(string transferId, StorageResource source, StorageRes
             Operation = JobPlanOperation.Upload, // TODO
             Source = source,
             Destination = destination,
-            Status = new()
+            Status = new(DataTransferState.Queued, false, false),
         };
         return Task.CompletedTask;
     }
 
-    public Task AddNewJobPartAsync(string transferId, int partNumber, JobPartPlanHeader header, CancellationToken cancellationToken = default)
+    public virtual Task AddNewJobPartAsync(string transferId, int partNumber, JobPartPlanHeader header, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (!Jobs.TryGetValue(transferId, out Job job))
@@ -90,18 +90,18 @@ public Task AddNewJobPartAsync(string transferId, int partNumber, JobPartPlanHea
         job.Parts.Add(partNumber, new()
         {
             Plan = header,
-            Status = new()
+            Status = new(DataTransferState.Queued, false, false),
         });
         return Task.CompletedTask;
     }
 
-    public Task<int> GetCurrentJobPartCountAsync(string transferId, CancellationToken cancellationToken = default)
+    public virtual Task<int> GetCurrentJobPartCountAsync(string transferId, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         return Task.FromResult(Jobs.TryGetValue(transferId, out Job job) ? job.Parts.Count : 0);
     }
 
-    public Task<DataTransferProperties> GetDataTransferPropertiesAsync(string transferId, CancellationToken cancellationToken = default)
+    public virtual Task<DataTransferProperties> GetDataTransferPropertiesAsync(string transferId, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (!Jobs.TryGetValue(transferId, out Job job))
@@ -119,7 +119,7 @@ public Task<DataTransferProperties> GetDataTransferPropertiesAsync(string transf
         });
     }
 
-    public Task<JobPartPlanHeader> GetJobPartAsync(string transferId, int partNumber, CancellationToken cancellationToken = default)
+    public virtual Task<JobPartPlanHeader> GetJobPartAsync(string transferId, int partNumber, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (!Jobs.TryGetValue(transferId, out Job job))
@@ -133,7 +133,7 @@ public Task<JobPartPlanHeader> GetJobPartAsync(string transferId, int partNumber
         return Task.FromResult(part.Plan);
     }
 
-    public Task<DataTransferStatus> GetJobStatusAsync(string transferId, CancellationToken cancellationToken = default)
+    public virtual Task<DataTransferStatus> GetJobStatusAsync(string transferId, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (!Jobs.TryGetValue(transferId, out Job job))
@@ -143,19 +143,19 @@ public Task<DataTransferStatus> GetJobStatusAsync(string transferId, Cancellatio
         return Task.FromResult(job.Status.DeepCopy());
     }
 
-    public Task<List<string>> GetStoredTransfersAsync(CancellationToken cancellationToken = default)
+    public virtual Task<List<string>> GetStoredTransfersAsync(CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         return Task.FromResult(Jobs.Keys.ToList());
     }
 
-    public Task<bool> IsEnumerationCompleteAsync(string transferId, CancellationToken cancellationToken = default)
+    public virtual Task<bool> IsEnumerationCompleteAsync(string transferId, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         return Task.FromResult(Jobs.TryGetValue(transferId, out Job job) && job.EnumerationComplete);
     }
 
-    public Task SetEnumerationCompleteAsync(string transferId, CancellationToken cancellationToken = default)
+    public virtual Task SetEnumerationCompleteAsync(string transferId, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (Jobs.TryGetValue(transferId, out Job job))
@@ -165,7 +165,7 @@ public Task SetEnumerationCompleteAsync(string transferId, CancellationToken can
         return Task.CompletedTask;
     }
 
-    public Task SetJobPartStatusAsync(string transferId, int partNumber, DataTransferStatus status, CancellationToken cancellationToken = default)
+    public virtual Task SetJobPartStatusAsync(string transferId, int partNumber, DataTransferStatus status, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (Jobs.TryGetValue(transferId, out Job job) && job.Parts.TryGetValue(partNumber, out JobPart part))
@@ -175,7 +175,7 @@ public Task SetJobPartStatusAsync(string transferId, int partNumber, DataTransfe
         return Task.CompletedTask;
     }
 
-    public Task SetJobStatusAsync(string transferId, DataTransferStatus status, CancellationToken cancellationToken = default)
+    public virtual Task SetJobStatusAsync(string transferId, DataTransferStatus status, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         if (Jobs.TryGetValue(transferId, out Job job))
@@ -185,7 +185,7 @@ public Task SetJobStatusAsync(string transferId, DataTransferStatus status, Canc
         return Task.CompletedTask;
     }
 
-    public Task<bool> TryRemoveStoredTransferAsync(string transferId, CancellationToken cancellationToken = default)
+    public virtual Task<bool> TryRemoveStoredTransferAsync(string transferId, CancellationToken cancellationToken = default)
     {
         CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
         return Task.FromResult(Jobs.Remove(transferId));
-Original file line number
+Diff line change
@@ Expand Up / @@ -508,6 +508,7 @@ await TransferFailedEventHandler.RaiseAsync( @@
                             ClientDiagnostics)
                             .ConfigureAwait(false);
                     }
+                    _dataTransfer.TransferStatus.SetFailedItem();
                 }
                 try
@@ Expand Down @@