Support setting a test specific timeout through test.json. (redhat-de…

…veloper#81) This enables setting a higher timeout than the default for tests that are expected to take longer.
nicrowe00 · Sep 7, 2023 · 24e42c3 · 24e42c3
1 parent c8e77e4
commit 24e42c3
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -61,6 +61,7 @@ a `test.json` file. An example of this file:
       "versionSpecific": false,
       "type": "xunit",
       "cleanup": true,
+      "timeoutMultiplier": 1.0,
       "ignoredRIDs": [
         "fedora",
         "fedora.29",
@@ -169,6 +170,21 @@ trait `blue` is set, or both `os=fedora` and `arch=x64` are set.
        ]
 ```
 
+- `timeoutMultiplier`
+
+This is a number, that scales the default timeout for a specific test.
+The default timeout can be set through the command-line `--timeout`
+argument.
+
+Example:
+
+A test with the following `timeoutMultiplier` will be allowed to take twice
+as long.
+
+```
+      "timeoutMultiplier": 2.0
+```
+
 ## Notes on Writing Tests
 
 Some notes for writing tests:

diff --git a/Turkey/Program.cs b/Turkey/Program.cs
@@ -44,14 +44,14 @@ public static async Task<int> Run(string testRoot,
                                           IEnumerable<string> trait,
                                           int timeout)
         {
-            TimeSpan timeoutForEachTest;
+            TimeSpan defaultTimeout;
             if (timeout == 0)
             {
-                timeoutForEachTest = new TimeSpan(hours: 0, minutes: 5, seconds: 0);
+                defaultTimeout = new TimeSpan(hours: 0, minutes: 5, seconds: 0);
             }
             else
             {
-                timeoutForEachTest = new TimeSpan(hours: 0, minutes: 0, seconds: timeout);
+                defaultTimeout = new TimeSpan(hours: 0, minutes: 0, seconds: timeout);
             }
 
             var currentDirectory = new DirectoryInfo(Directory.GetCurrentDirectory());
@@ -129,7 +129,7 @@ public static async Task<int> Run(string testRoot,
                 verboseOutput: verbose,
                 nuGetConfig: nuGetConfig);
 
-            var results = await runner.ScanAndRunAsync(testOutputs, logDir.FullName, timeoutForEachTest);
+            var results = await runner.ScanAndRunAsync(testOutputs, logDir.FullName, defaultTimeout);
 
             int exitCode = (results.Failed == 0) ? 0 : 1;
             return exitCode;

diff --git a/Turkey/Test.cs b/Turkey/Test.cs
@@ -17,6 +17,7 @@ public class TestDescriptor
         public bool VersionSpecific { get; set; }
         public string Type { get; set; }
         public bool Cleanup { get; set; }
+        public double TimeoutMultiplier { get; set; } = 1.0;
         public List<string> IgnoredRIDs { get; set; } = new();
         public List<string> SkipWhen { get; set; } = new();
     }

diff --git a/Turkey/TestOutputFormat.cs b/Turkey/TestOutputFormat.cs
@@ -21,7 +21,10 @@ public async override Task AfterParsingTestAsync(string name, bool enabled)
 
             public async override Task AfterRunningTestAsync(string name, TestResult result, StringBuilder testLog, TimeSpan testTime)
             {
-                string elapsedTime = testTime.TotalMilliseconds.ToString();
+                int minutes = (int)testTime.TotalMinutes;
+                int seconds = (int)Math.Ceiling(testTime.TotalSeconds - 60 * minutes);
+                string elapsedTime = minutes == 0 ? $"{seconds}s"
+                                                  : $"{minutes}m {seconds}s";
                 string resultOutput = null;
                 if (Console.IsOutputRedirected || Console.IsErrorRedirected)
                 {
@@ -31,7 +34,7 @@ public async override Task AfterRunningTestAsync(string name, TestResult result,
                         case TestResult.Failed: resultOutput = "FAIL"; break;
                         case TestResult.Skipped: resultOutput = "SKIP"; break;
                     }
-                    Console.WriteLine($"[{resultOutput}]\t({elapsedTime}ms)");
+                    Console.WriteLine($"[{resultOutput}]\t({elapsedTime})");
                 }
                 else
                 {
@@ -41,7 +44,7 @@ public async override Task AfterRunningTestAsync(string name, TestResult result,
                         case TestResult.Failed: resultOutput = "\u001b[31mFAIL\u001b[0m"; break;
                         case TestResult.Skipped: resultOutput = "SKIP"; break;
                     }
-                    Console.WriteLine($"[{resultOutput}]\t({elapsedTime}ms)");
+                    Console.WriteLine($"[{resultOutput}]\t({elapsedTime})");
                 }
             }
 

diff --git a/Turkey/TestRunner.cs b/Turkey/TestRunner.cs
@@ -68,7 +68,7 @@ public TestRunner(SystemUnderTest system, DirectoryInfo root, bool verboseOutput
             this.nuGetConfig = nuGetConfig;
         }
 
-        public async Task<TestResults> ScanAndRunAsync(List<TestOutput> outputs, string logDir, TimeSpan timeoutForEachTest)
+        public async Task<TestResults> ScanAndRunAsync(List<TestOutput> outputs, string logDir, TimeSpan defaultTimeout)
         {
             await outputs.ForEachAsync(output => output.AtStartupAsync());
 
@@ -105,7 +105,8 @@ public async Task<TestResults> ScanAndRunAsync(List<TestOutput> outputs, string
 
                 await outputs.ForEachAsync(output => output.AfterParsingTestAsync(testName, !test.Skip));
 
-                using var cts = timeoutForEachTest == TimeSpan.Zero ? null : new CancellationTokenSource(timeoutForEachTest);
+                TimeSpan testTimeout = test.Descriptor.TimeoutMultiplier * defaultTimeout;
+                using var cts = testTimeout == TimeSpan.Zero ? null : new CancellationTokenSource(testTimeout);
                 var cancellationToken = cts is null ? default : cts.Token;
                 Action<string> testLogger = null;