Skip to content

Commit

Permalink
Add support for skipping tests using traits.
Browse files Browse the repository at this point in the history
  • Loading branch information
@tmds @omajid
tmds authored and omajid committed May 2, 2023
1 parent 9328139 commit 15f6ae9
Show file tree
Hide file tree
Showing 9 changed files with 234 additions and 17 deletions.
26 changes: 26 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ a `test.json` file. An example of this file:
"fedora.34-s390x",
"rhel.7",
"rhel.7-arm64"
],
"skipWhen": [
"blue",
"os=fedora,arch=x64"
]
}

Expand Down Expand Up @@ -143,6 +147,28 @@ the following keys:
See https://docs.microsoft.com/en-us/dotnet/core/rid-catalog for
more details. Not all the RIDs are fully supported yet.

- `skipWhen`

This is a list of conditions. If one (or more) conditions in the list
match the test environment, then the test is skipped.

A condition is a combination of traits separated by commas.

The test runner injects a few traits based on the system.
Additional traits can be added using the `--trait` flag.

Example:

A test with the following `skipWhen` will be skipped if the
trait `blue` is set, or both `os=fedora` and `arch=x64` are set.

```
"skipWhen": [
"blue",
"os=fedora,arch=x64"
]
```

## Notes on Writing Tests

Some notes for writing tests:
Expand Down
55 changes: 53 additions & 2 deletions Turkey.Tests/ProgramTest.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
using System.Threading.Tasks;

using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using System.Linq;
using Xunit;

namespace Turkey.Tests
Expand Down Expand Up @@ -42,5 +44,54 @@ public void GetNuGetWorking()
// TODO
// var nugetConfig = Program.GetNuGetConfigIfNeeded(nuget, sourceBuild, netCoreAppVersion);
}

public static IEnumerable<object[]> SystemTraits_MemberData()
{
// note: the version numbers are intentionally chosen to be different.
Version runtimeVersion = Version.Parse("6.5");
Version sdkVersion = Version.Parse("3.1");

string[] expectedVersionTraits = new[] { "version=6.5", "version=6" };
string expectedArch = $"arch={OSArchitectureName}";

// default traits.
yield return new object[] { runtimeVersion, sdkVersion, Array.Empty<string>(), Array.Empty<string>(), CombineTraits() };

// 'os=..' and 'rid=...' are added for the platform rids.
yield return new object[] { runtimeVersion, sdkVersion, new[] { "linux-x64", "fedora.37-x64", "linux-musl-x64" }, Array.Empty<string>(),
CombineTraits(new[] { "os=linux", "os=fedora.37", "os=linux-musl",
"rid=linux-x64", "rid=fedora.37-x64", "rid=linux-musl-x64" } ) };

// additional traits are added.
yield return new object[] { runtimeVersion, sdkVersion, Array.Empty<string>(), new[] { "blue", "green" },
CombineTraits(new[] { "blue", "green" } ) };

string[] CombineTraits(string[] expectedAdditionalTraits = null)
=> expectedVersionTraits
.Concat(new[] { expectedArch })
.Concat(expectedAdditionalTraits ?? Array.Empty<string>())
.ToArray();
}

[Theory]
[MemberData(nameof(SystemTraits_MemberData))]
public void SystemTraits(Version runtimeVersion, Version sdkVersion, string[] rids, string[] additionalTraits, string[] expectedTraits)
{
IReadOnlySet<string> systemTraits = Program.CreateTraits(runtimeVersion, sdkVersion, new List<string>(rids), additionalTraits);

Assert.Equal(expectedTraits.OrderBy(s => s), systemTraits.OrderBy(s => s));
}

private static string OSArchitectureName
=> RuntimeInformation.OSArchitecture switch
{
Architecture.X86 => "x86",
Architecture.X64 => "x64",
Architecture.Arm => "arm",
Architecture.Arm64 => "arm64",
Architecture.S390x => "s390x",
(Architecture)8 => "ppc64le", // not defined for 'net6.0' target.
_ => throw new NotSupportedException(),
};
}
}
79 changes: 72 additions & 7 deletions Turkey.Tests/TestParserTest.cs
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;

Expand All @@ -12,7 +13,7 @@ public class TestParserTests
public void DisabledTestShouldBeSkipped()
{
TestParser parser = new TestParser();
SystemUnderTest system = new SystemUnderTest(null, null, null, null);
SystemUnderTest system = new SystemUnderTest(null, null, null, null, null);
TestDescriptor test = new TestDescriptor()
{
Enabled = false,
Expand Down Expand Up @@ -42,7 +43,8 @@ public void TestShouldBeRunForSameOrHigherVersions(string version, bool expected
runtimeVersion: Version.Parse(version),
sdkVersion: null,
platformIds: new List<string>(),
environmentVariables: null);
environmentVariables: null,
traits: null);

TestDescriptor test = new TestDescriptor()
{
Expand Down Expand Up @@ -77,7 +79,8 @@ public void VersionSpecificTestShouldBeRunForSameMajorMinorVersion(string versio
runtimeVersion: Version.Parse(version),
sdkVersion: null,
platformIds: new List<string>(),
environmentVariables: null);
environmentVariables: null,
traits: null);
TestDescriptor test = new TestDescriptor()
{
Enabled = true,
Expand Down Expand Up @@ -114,7 +117,8 @@ public void VersionSpecificTestWithWildcardShouldBeRunForSameMajorVersion(string
runtimeVersion: Version.Parse(version),
sdkVersion: null,
platformIds: new List<string>(),
environmentVariables: null);
environmentVariables: null,
traits: null);
TestDescriptor test = new TestDescriptor()
{
Enabled = true,
Expand All @@ -136,7 +140,8 @@ public void MissingIgnoredRIDsIsOkay()
runtimeVersion: Version.Parse("2.1"),
sdkVersion: null,
platformIds: new string[] { "linux" }.ToList(),
environmentVariables: null);
environmentVariables: null,
traits: null);
TestDescriptor test = new TestDescriptor()
{
Enabled = true,
Expand Down Expand Up @@ -164,7 +169,8 @@ public void TestShouldNotRunOnIgnoredPlatforms(string[] currentPlatforms, string
runtimeVersion: Version.Parse("2.1"),
sdkVersion: null,
platformIds: currentPlatforms.ToList(),
environmentVariables: null);
environmentVariables: null,
traits: null);
TestDescriptor test = new TestDescriptor()
{
Enabled = true,
Expand All @@ -189,7 +195,8 @@ public void SdkTestsShouldRunOnlyWithSdk(string sdkVersion, bool requiresSdk, bo
runtimeVersion: Version.Parse("3.1"),
sdkVersion: Version.Parse(sdkVersion),
platformIds: new List<string>(),
environmentVariables: null);
environmentVariables: null,
traits: null);
TestDescriptor test = new TestDescriptor()
{
Enabled = true,
Expand All @@ -202,5 +209,63 @@ public void SdkTestsShouldRunOnlyWithSdk(string sdkVersion, bool requiresSdk, bo

Assert.Equal(expectedToRun, shouldRun);
}

public static IEnumerable<object[]> SkipTestForTraits_MemberData()
{
const string Trait1 = nameof(Trait1);
const string Trait2 = nameof(Trait2);
string[] Empty = Array.Empty<string>();

// Empty 'skipWhen' has no skips.
yield return new object[] { Empty, Empty, false };
yield return new object[] { new [] { Trait1 }, Empty, false };
yield return new object[] { new string [] { Trait1, Trait2 }, Empty, false };

// Single match gets skipped.
yield return new object[] { new [] { Trait1 }, new string [] { Trait1 }, true };
yield return new object[] { new [] { Trait1, Trait2 }, new string [] { Trait1 }, true };

// No match is enabled.
yield return new object[] { Empty, new string [] { Trait1 }, false };
yield return new object[] { new [] { Trait2 }, new string [] { Trait1 }, false };

// Comma separated traits work as AND.
yield return new object[] { Empty, new string [] { $"{Trait1},{Trait2}" }, false };
yield return new object[] { new [] { Trait1 }, new string [] { $"{Trait1},{Trait2}" }, false };
yield return new object[] { new [] { Trait2 }, new string [] { $"{Trait1},{Trait2}" }, false };
yield return new object[] { new [] { Trait1, Trait2 }, new string [] { $"{Trait1},{Trait2}" }, true };

// Items work as OR.
yield return new object[] { Empty, new string [] { Trait1, Trait2 }, false };
yield return new object[] { new [] { Trait1 }, new string [] { Trait1, Trait2 }, true };
yield return new object[] { new [] { Trait2 }, new string [] { Trait1, Trait2 }, true };
yield return new object[] { new [] { Trait1, Trait2 }, new string [] { Trait1, Trait2 }, true };
}


[Theory]
[MemberData(nameof(SkipTestForTraits_MemberData))]
public void SkipTestForTraits(string[] systemTraits, string[] skipWhen, bool expectedToSkip)
{
TestParser parser = new TestParser();

SystemUnderTest system = new SystemUnderTest(
runtimeVersion: Version.Parse("3.1"),
sdkVersion: null,
platformIds: null,
environmentVariables: null,
traits: new HashSet<string>(systemTraits));
TestDescriptor test = new TestDescriptor()
{
Enabled = true,
VersionSpecific = false,
SkipWhen = new List<string>(skipWhen),
Version = "2.1"
};

var shouldRun = parser.ShouldRunTest(system, test);

Assert.Equal(expectedToSkip, !shouldRun);
}
}
}
2 changes: 1 addition & 1 deletion Turkey.Tests/Turkey.Tests.csproj
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<TargetFramework>net6.0</TargetFramework>
<IsPackable>false</IsPackable>
<RollForward>Major</RollForward>
</PropertyGroup>
Expand Down
50 changes: 47 additions & 3 deletions Turkey/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using System.CommandLine;
using System.CommandLine.Invocation;
using System.Threading;
using System.Runtime.InteropServices;

namespace Turkey
{
Expand All @@ -33,11 +34,19 @@ public class Program
new string[] { "--timeout", "-t" },
"Set the timeout duration for test in seconds");

public static readonly Option<IEnumerable<string>> traitOption = new Option<IEnumerable<string>>(
new string[] { "--trait" },
"Add a trait which is used to disable tests.")
{
Arity = ArgumentArity.ZeroOrMore
};

public static async Task<int> Run(string testRoot,
bool verbose,
bool compatible,
string logDirectory,
string additionalFeed,
IEnumerable<string> trait,
int timeout)
{
TimeSpan timeoutForEachTest;
Expand Down Expand Up @@ -103,11 +112,15 @@ public static async Task<int> Run(string testRoot,
var sanitizer = new EnvironmentVariableSanitizer();
var envVars = sanitizer.SanitizeCurrentEnvironmentVariables();

var traits = CreateTraits(dotnet.LatestRuntimeVersion, dotnet.LatestSdkVersion, platformIds, trait);
Console.WriteLine($"Tests matching these traits will be skipped: {string.Join(", ", traits.OrderBy(s => s))}.");

SystemUnderTest system = new SystemUnderTest(
runtimeVersion: dotnet.LatestRuntimeVersion,
sdkVersion: dotnet.LatestSdkVersion,
platformIds: platformIds,
environmentVariables: envVars
environmentVariables: envVars,
traits: traits
);

Version packageVersion = dotnet.LatestRuntimeVersion;
Expand Down Expand Up @@ -191,6 +204,37 @@ public static async Task<string> GenerateNuGetConfigIfNeededAsync(string additio
return null;
}

public static IReadOnlySet<string> CreateTraits(Version runtimeVersion, Version sdkVersion, List<string> rids, IEnumerable<string> additionalTraits)
{
var traits = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

// Add 'version=' traits.
traits.Add($"version={runtimeVersion.Major}.{runtimeVersion.Minor}");
traits.Add($"version={runtimeVersion.Major}");

// Add 'os=', 'rid=' traits.
foreach (var rid in rids)
{
traits.Add($"rid={rid}");
if (rid.LastIndexOf('-') is int offset && offset != -1)
{
traits.Add($"os={rid.Substring(0, offset)}");
}
}

// Add 'arch=' trait.
string arch = RuntimeInformation.OSArchitecture.ToString().ToLowerInvariant();
traits.Add($"arch={arch}");

// Add additional traits.
foreach (var skipTrait in additionalTraits)
{
traits.Add(skipTrait);
}

return traits;
}

public static async Task<string> GetProdConFeedUrlIfNeededAsync(NuGet nuget, SourceBuild sourceBuild, Version netCoreAppVersion)
{
bool live = await nuget.IsPackageLiveAsync("runtime.linux-x64.Microsoft.NetCore.DotNetAppHost", netCoreAppVersion);
Expand All @@ -204,9 +248,8 @@ public static async Task<string> GetProdConFeedUrlIfNeededAsync(NuGet nuget, Sou

static async Task<int> Main(string[] args)
{
Func<string, bool, bool, string, string, int, Task<int>> action = Run;
var rootCommand = new RootCommand(description: "A test runner for running standalone bash-based or xunit tests");
rootCommand.Handler = CommandHandler.Create(action);
rootCommand.Handler = CommandHandler.Create(Run);

var testRootArgument = new Argument<string>();
testRootArgument.Name = "testRoot";
Expand All @@ -218,6 +261,7 @@ static async Task<int> Main(string[] args)
rootCommand.AddOption(verboseOption);
rootCommand.AddOption(logDirectoryOption);
rootCommand.AddOption(additionalFeedOption);
rootCommand.AddOption(traitOption);
rootCommand.AddOption(timeoutOption);

return await rootCommand.InvokeAsync(args);
Expand Down
3 changes: 2 additions & 1 deletion Turkey/Test.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ public class TestDescriptor
public bool VersionSpecific { get; set; }
public string Type { get; set; }
public bool Cleanup { get; set; }
public List<string> IgnoredRIDs { get; set; }
public List<string> IgnoredRIDs { get; set; } = new();
public List<string> SkipWhen { get; set; } = new();
}

// TODO is this a strongly-typed enum in C#?
Expand Down
24 changes: 24 additions & 0 deletions Turkey/TestParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,30 @@ public bool ShouldRunTest(SystemUnderTest system, TestDescriptor test)
}
}

foreach (var skipCondition in test.SkipWhen)
{
// a skipCondition is formatted as comma-separated traits: 'green,age=21'
// the condition is true when all traits are present in the test environment.

bool skipConditionMatches = true;

foreach (var skipConditionTrait in skipCondition.Split(',', StringSplitOptions.RemoveEmptyEntries)
.Select(s => s.Trim())
.Where(s => s.Length > 0))
{
if (!system.Traits.Contains(skipConditionTrait))
{
skipConditionMatches = false;
break;
}
}

if (skipConditionMatches)
{
return false;
}
}

return true;
}

Expand Down
Loading

0 comments on commit 15f6ae9

Please sign in to comment.