From 3cedf86c8838eb2b7cfae89a52f94d2a8f937edf Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Wed, 14 Dec 2022 14:11:32 -0800 Subject: [PATCH] Adds check for run state and postProcessState values in CheckAzurePipelinesTestResults (https://github.com/dotnet/arcade/issues/11942) (#11957) --- .../Sdk/CheckAzurePipelinesTestResults.cs | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk/CheckAzurePipelinesTestResults.cs b/src/Microsoft.DotNet.Helix/Sdk/CheckAzurePipelinesTestResults.cs index 015ff553cb0..fccd8a868ee 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/CheckAzurePipelinesTestResults.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/CheckAzurePipelinesTestResults.cs @@ -43,7 +43,13 @@ private async Task CheckTestResultsAsync(HttpClient client) { foreach (int testRunId in TestRunIds) { - JObject data = await RetryAsync( + bool runComplete = false; + int triesToWait = 3; + JObject data = null; + + do + { + data = await RetryAsync( async () => { using var req = new HttpRequestMessage( @@ -52,6 +58,16 @@ private async Task CheckTestResultsAsync(HttpClient client) using HttpResponseMessage res = await client.SendAsync(req); return await ParseResponseAsync(req, res); }); + // This retry does not use the RetryAsync() function as that one only retries for network/timeout issues + triesToWait--; + runComplete = CheckAzurePipelinesTestRunIsComplete(data); + if (!runComplete && triesToWait > 0) + { + Log.LogWarning($"Test run {testRunId} is not in completed state. Will check back in 10 seconds."); + await Task.Delay(10000); + } + } + while (!runComplete && triesToWait > 0); if (data != null && data["runStatistics"] is JArray runStatistics) { @@ -69,6 +85,22 @@ private async Task CheckTestResultsAsync(HttpClient client) } } + private bool CheckAzurePipelinesTestRunIsComplete(JObject data) + { + // Context: https://github.com/dotnet/arcade/issues/11942 + // it seems it's possible if checking immediately after a run is closed to not see all results + // Since we pass/fail build tasks based off failed test items, it's very important that we not miss this. + // This check will add logging if /_apis/test/runs/ manages to get called while incomplete. + if (data == null) + { + return false; + } + var stateCompleted = data["state"]?.Value()?.Equals("Completed"); + var postProcessStateCompleted = data["postProcessState"]?.Value()?.Equals("Complete"); + + return (stateCompleted == true && postProcessStateCompleted == true); + } + private async Task LogErrorsForFailedRun(HttpClient client, int testRunId) { JObject data = await RetryAsync(