-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add remaining scripts for package tests
- Loading branch information
1 parent
8f71425
commit b88ac87
Showing
6 changed files
with
491 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| using System.Xml; | ||
|
|
||
| ////////////////////////////////////////////////////////////////////// | ||
| // PACKAGE TEST | ||
| ////////////////////////////////////////////////////////////////////// | ||
|
|
||
| public class PackageTest | ||
| { | ||
| public string Description { get; set; } | ||
| public string[] TestConsoleVersions { get; set; } | ||
| public string Arguments { get; set; } | ||
| public ExpectedResult ExpectedResult { get; set; } | ||
| } | ||
|
|
||
| ////////////////////////////////////////////////////////////////////// | ||
| // PACKAGE TESTER | ||
| ////////////////////////////////////////////////////////////////////// | ||
|
|
||
| const string DEFAULT_TEST_RESULT_FILE = "TestResult.xml"; | ||
|
|
||
| public abstract class PackageTester | ||
| { | ||
| protected BuildParameters _parameters; | ||
| protected ICakeContext _context; | ||
|
|
||
| public PackageTester(BuildParameters parameters) | ||
| { | ||
| _parameters = parameters; | ||
| _context = parameters.Context; | ||
|
|
||
| PackageTests.Add(new PackageTest() | ||
| { | ||
| Description = "Integration Tests using NUnit V2", | ||
| Arguments = "bin/Release/v2-tests/v2-test-assembly.dll", | ||
| TestConsoleVersions = new string[] { "3.12.0", "3.11.1", "3.10.0" }, | ||
| ExpectedResult = new ExpectedResult("Passed") | ||
| { | ||
| Total = 75, | ||
| Passed = 75, | ||
| Failed = 0, | ||
| Warnings = 0, | ||
| Inconclusive = 0, | ||
| Skipped = 0, | ||
| Assemblies = new[] { new ExpectedAssemblyResult( | ||
| System.IO.Path.GetFullPath("bin/Release/v2-tests/v2-test-assembly.dll"), "net-2.0") } | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| protected abstract string PackageName { get; } | ||
| protected abstract string PackageUnderTest { get; } | ||
| public abstract string InstallDirectory { get; } | ||
|
|
||
| public PackageCheck[] PackageChecks { get; set; } | ||
| public List<PackageTest> PackageTests = new List<PackageTest>(); | ||
|
|
||
| public void RunPackageTests() | ||
| { | ||
| var reporter = new ResultReporter(PackageName); | ||
|
|
||
| foreach (var packageTest in PackageTests) | ||
| { | ||
| foreach (var consoleVersion in packageTest.TestConsoleVersions) | ||
| { | ||
| // Delete result file ahead of time so we don't mistakenly | ||
| // read a left-over file from another test run. Leave the | ||
| // file after the run in case we need it to debug a failure. | ||
| if (_context.FileExists(DEFAULT_TEST_RESULT_FILE)) | ||
| _context.DeleteFile(DEFAULT_TEST_RESULT_FILE); | ||
|
|
||
| DisplayBanner(packageTest.Description + " - Console Version " + consoleVersion); | ||
|
|
||
| RunConsoleTest(consoleVersion, packageTest.Arguments); | ||
|
|
||
| try | ||
| { | ||
| var result = new ActualResult(DEFAULT_TEST_RESULT_FILE); | ||
| var report = new PackageTestReport(packageTest, consoleVersion, result); | ||
| reporter.AddReport(report); | ||
|
|
||
| Console.WriteLine(report.Errors.Count == 0 | ||
| ? "\nSUCCESS: Test Result matches expected result!" | ||
| : "\nERROR: Test Result not as expected!"); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| reporter.AddReport(new PackageTestReport(packageTest, consoleVersion, ex)); | ||
|
|
||
| Console.WriteLine($"\nERROR: No result found."); | ||
| Console.WriteLine(ex.ToString()); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| bool anyErrors = reporter.ReportResults(); | ||
| Console.WriteLine(); | ||
|
|
||
| // All package tests are run even if one of them fails. If there are | ||
| // any errors, we stop the run at this point. | ||
| if (anyErrors) | ||
| throw new Exception("One or more package tests had errors!"); | ||
| } | ||
|
|
||
| private void RunConsoleTest(string consoleVersion, string arguments) | ||
| { | ||
| string runner = _parameters.GetPathToConsoleRunner(consoleVersion); | ||
|
|
||
| if (InstallDirectory.EndsWith(CHOCO_ID + "/")) | ||
| { | ||
| // We are using nuget packages for the runner, so add an extra | ||
| // addins file to allow detecting chocolatey packages | ||
| string runnerDir = System.IO.Path.GetDirectoryName(runner); | ||
| using (var writer = new StreamWriter(runnerDir + "/choco.engine.addins")) | ||
| writer.WriteLine("../../nunit-extension-*/tools/"); | ||
| } | ||
|
|
||
| Console.WriteLine("Args: " + arguments); | ||
| _context.StartProcess(runner, arguments); | ||
|
|
||
| // Should have created the result file | ||
| if (!_context.FileExists(DEFAULT_TEST_RESULT_FILE)) | ||
| throw new System.Exception($"The result file was not created."); | ||
|
|
||
| } | ||
|
|
||
| private void DisplayBanner(string message) | ||
| { | ||
| Console.WriteLine(); | ||
| Console.WriteLine("======================================================="); | ||
| Console.WriteLine(message); | ||
| Console.WriteLine("======================================================="); | ||
| } | ||
| } | ||
|
|
||
| public class NuGetPackageTester : PackageTester | ||
| { | ||
| public NuGetPackageTester(BuildParameters parameters) : base(parameters) { } | ||
|
|
||
| protected override string PackageName => _parameters.NuGetPackageName; | ||
| protected override string PackageUnderTest => _parameters.NuGetPackage; | ||
| public override string InstallDirectory => _parameters.NuGetInstallDirectory; | ||
| } | ||
|
|
||
| public class ChocolateyPackageTester : PackageTester | ||
| { | ||
| public ChocolateyPackageTester(BuildParameters parameters) : base(parameters) { } | ||
|
|
||
| protected override string PackageName => _parameters.ChocolateyPackageName; | ||
| protected override string PackageUnderTest => _parameters.ChocolateyPackage; | ||
| public override string InstallDirectory => _parameters.ChocolateyInstallDirectory; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,172 @@ | ||
| // *********************************************************************** | ||
| // Copyright (c) Charlie Poole and TestCentric GUI contributors. | ||
| // Licensed under the MIT License. See LICENSE.txt in root directory. | ||
| // *********************************************************************** | ||
|
|
||
| public class PackageTestReport | ||
| { | ||
| public PackageTest Test; | ||
| public string ConsoleVersion; | ||
| public ActualResult Result; | ||
| public List<string> Errors; | ||
| public List<string> Warnings; | ||
|
|
||
| public PackageTestReport(PackageTest test, string consoleVersion, ActualResult actualResult) | ||
| { | ||
| Test = test; | ||
| ConsoleVersion = consoleVersion; | ||
| Result = actualResult; | ||
| Errors = new List<string>(); | ||
| Warnings = new List<string>(); | ||
|
|
||
| var expectedResult = test.ExpectedResult; | ||
|
|
||
| ReportMissingFiles(); | ||
|
|
||
| if (actualResult.OverallResult == null) | ||
| Errors.Add(" The test-run element has no result attribute."); | ||
| else if (expectedResult.OverallResult != actualResult.OverallResult) | ||
| Errors.Add($" Expected: Overall Result = {expectedResult.OverallResult} But was: {actualResult.OverallResult}"); | ||
| CheckCounter("Test Count", expectedResult.Total, actualResult.Total); | ||
| CheckCounter("Passed", expectedResult.Passed, actualResult.Passed); | ||
| CheckCounter("Failed", expectedResult.Failed, actualResult.Failed); | ||
| CheckCounter("Warnings", expectedResult.Warnings, actualResult.Warnings); | ||
| CheckCounter("Inconclusive", expectedResult.Inconclusive, actualResult.Inconclusive); | ||
| CheckCounter("Skipped", expectedResult.Skipped, actualResult.Skipped); | ||
|
|
||
| var expectedAssemblies = expectedResult.Assemblies; | ||
| var actualAssemblies = actualResult.Assemblies; | ||
|
|
||
| for (int i = 0; i < expectedAssemblies.Length && i < actualAssemblies.Length; i++) | ||
| { | ||
| var expected = expectedAssemblies[i]; | ||
| var actual = actualAssemblies[i]; | ||
|
|
||
| if (expected.Name != actual.Name) | ||
| Errors.Add($" Expected: {expected.Name} But was: { actual.Name}"); | ||
| else if (actual.Runtime == null) | ||
| Warnings.Add($"Unable to determine actual runtime used for {expected.Name}"); | ||
| else if (!expected.Runtime.StartsWith(actual.Runtime)) | ||
| Errors.Add($" Assembly {actual.Name} Expected: {expected.Runtime} But was: {actual.Runtime}"); | ||
| } | ||
|
|
||
| for (int i = actualAssemblies.Length; i < expectedAssemblies.Length; i++) | ||
| Errors.Add($" Assembly {expectedAssemblies[i].Name} was not found"); | ||
|
|
||
| for (int i = expectedAssemblies.Length; i < actualAssemblies.Length; i++) | ||
| Errors.Add($" Found unexpected assembly {actualAssemblies[i].Name}"); | ||
| } | ||
|
|
||
| public PackageTestReport(PackageTest test, string consoleVersion, Exception ex) | ||
| { | ||
| Test = test; | ||
| ConsoleVersion = consoleVersion; | ||
| Result = null; | ||
| Errors = new List<string>(); | ||
| Errors.Add($" {ex.Message}"); | ||
| } | ||
|
|
||
| public void Display(int index) | ||
| { | ||
| Console.WriteLine(); | ||
| Console.WriteLine($"{index}. {Test.Description}"); | ||
| Console.WriteLine($" ConsoleVersion: {ConsoleVersion}"); | ||
| Console.WriteLine($" Args: {Test.Arguments}"); | ||
| Console.WriteLine(); | ||
|
|
||
| foreach (var error in Errors) | ||
| Console.WriteLine(error); | ||
|
|
||
| if (Errors.Count == 0) | ||
| { | ||
| Console.WriteLine(" SUCCESS: Test Result matches expected result!"); | ||
| } | ||
| else | ||
| { | ||
| Console.WriteLine(); | ||
| Console.WriteLine(" ERROR: Test Result not as expected!"); | ||
| } | ||
|
|
||
| foreach (var warning in Warnings) | ||
| Console.WriteLine(" WARNING: " + warning); | ||
| } | ||
|
|
||
| // File level errors, like missing or mal-formatted files, need to be highlighted | ||
| // because otherwise it's hard to detect the cause of the problem without debugging. | ||
| // This method finds and reports that type of error. | ||
| private void ReportMissingFiles() | ||
| { | ||
| // Start with all the top-level test suites. Note that files that | ||
| // cannot be found show up as Unknown as do unsupported file types. | ||
| var suites = Result.Xml.SelectNodes( | ||
| "//test-suite[@type='Unknown'] | //test-suite[@type='Project'] | //test-suite[@type='Assembly']"); | ||
|
|
||
| // If there is no top-level suite, it generally means the file format could not be interpreted | ||
| if (suites.Count == 0) | ||
| Errors.Add(" No top-level suites! Possible empty command-line or misformed project."); | ||
|
|
||
| foreach (XmlNode suite in suites) | ||
| { | ||
| // Narrow down to the specific failures we want | ||
| string runState = GetAttribute(suite, "runstate"); | ||
| string suiteResult = GetAttribute(suite, "result"); | ||
| string label = GetAttribute(suite, "label"); | ||
| string site = suite.Attributes["site"]?.Value ?? "Test"; | ||
| if (runState == "NotRunnable" || suiteResult == "Failed" && site == "Test" && (label == "Invalid" || label == "Error")) | ||
| { | ||
| string message = suite.SelectSingleNode("reason/message")?.InnerText; | ||
| Errors.Add($" {message}"); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private void CheckCounter(string label, int expected, int actual) | ||
| { | ||
| // If expected value of counter is negative, it means no check is needed | ||
| if (expected >= 0 && expected != actual) | ||
| Errors.Add($" Expected: {label} = {expected} But was: {actual}"); | ||
| } | ||
|
|
||
| private string GetAttribute(XmlNode node, string name) | ||
| { | ||
| return node.Attributes[name]?.Value; | ||
| } | ||
| } | ||
|
|
||
| public class ResultReporter | ||
| { | ||
| private string _packageName; | ||
| private List<PackageTestReport> _reports = new List<PackageTestReport>(); | ||
|
|
||
| public ResultReporter(string packageName) | ||
| { | ||
| _packageName = packageName; | ||
| } | ||
|
|
||
| public void AddReport(PackageTestReport report) | ||
| { | ||
| _reports.Add(report); | ||
| } | ||
|
|
||
| public bool ReportResults() | ||
| { | ||
| Console.WriteLine("\n=================================================="); ; | ||
| Console.WriteLine($"Test Results for {_packageName}"); | ||
| Console.WriteLine("=================================================="); ; | ||
|
|
||
| Console.WriteLine("\nTest Environment"); | ||
| Console.WriteLine($" OS Version: {Environment.OSVersion.VersionString}"); | ||
| Console.WriteLine($" CLR Version: {Environment.Version}\n"); | ||
|
|
||
| int index = 0; | ||
| bool hasErrors = false; | ||
|
|
||
| foreach (var report in _reports) | ||
| { | ||
| hasErrors |= report.Errors.Count > 0; | ||
| report.Display(++index); | ||
| } | ||
|
|
||
| return hasErrors; | ||
| } | ||
| } |
Oops, something went wrong.