diff --git a/NUnit3TestAdapter.sln b/NUnit3TestAdapter.sln index 68b5f37c..d985e756 100644 --- a/NUnit3TestAdapter.sln +++ b/NUnit3TestAdapter.sln @@ -11,6 +11,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution build.ps1 = build.ps1 src\Common.props = src\Common.props DisableAppDomain.runsettings = DisableAppDomain.runsettings + lib.cake = lib.cake LICENSE.txt = LICENSE.txt src\native-assembly\NativeTests.dll = src\native-assembly\NativeTests.dll NuGet.Config = NuGet.Config diff --git a/build.cake b/build.cake index 3d297795..fc5be9a7 100644 --- a/build.cake +++ b/build.cake @@ -16,9 +16,9 @@ var configuration = Argument("configuration", "Release"); var version = "3.13.0"; var modifier = ""; -var dbgSuffix = configuration == "Debug" ? "-dbg" : ""; +var dbgSuffix = configuration.ToLower() == "debug" ? "-dbg" : ""; var packageVersion = version + modifier + dbgSuffix; - +Information("PackageVersion is "+packageVersion); if (BuildSystem.IsRunningOnAppVeyor) { var tag = AppVeyor.Environment.Repository.Tag; diff --git a/src/NUnitTestAdapter/AdapterSettings.cs b/src/NUnitTestAdapter/AdapterSettings.cs index e81c4264..ad1964f0 100644 --- a/src/NUnitTestAdapter/AdapterSettings.cs +++ b/src/NUnitTestAdapter/AdapterSettings.cs @@ -250,9 +250,7 @@ public void Load(string settingsXml) UseVsKeepEngineRunning = GetInnerTextAsBool(nunitNode, nameof(UseVsKeepEngineRunning), false); BasePath = GetInnerTextWithLog(nunitNode, nameof(BasePath)); PrivateBinPath = GetInnerTextWithLog(nunitNode, nameof(PrivateBinPath)); - var testOutput = GetInnerTextWithLog(nunitNode, nameof(TestOutputXml)); - if (!string.IsNullOrEmpty(testOutput)) - TestOutputXml = ValidatedPath(testOutput, nameof(TestOutputXml)); + TestOutputXml = GetInnerTextWithLog(nunitNode, nameof(TestOutputXml)); RandomSeed = GetInnerTextAsNullableInt(nunitNode, nameof(RandomSeed)); RandomSeedSpecified = RandomSeed.HasValue; if (!RandomSeedSpecified) diff --git a/src/NUnitTestAdapter/NUnit3TestExecutor.cs b/src/NUnitTestAdapter/NUnit3TestExecutor.cs index 9abe7b68..7c9dfa7e 100644 --- a/src/NUnitTestAdapter/NUnit3TestExecutor.cs +++ b/src/NUnitTestAdapter/NUnit3TestExecutor.cs @@ -56,6 +56,8 @@ public NUnit3TestExecutor() public IFrameworkHandle FrameworkHandle { get; private set; } private TfsTestFilter TfsFilter { get; set; } + private string TestOutputXmlFolder { get; set; } = ""; + // NOTE: an earlier version of this code had a FilterBuilder // property. This seemed to make sense, because we instantiate // it in two different places. However, the existence of an @@ -107,7 +109,7 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame } } - TestLog.Info(string.Format("NUnit Adapter {0}: Test execution complete", AdapterVersion)); + TestLog.Info($"NUnit Adapter {AdapterVersion}: Test execution complete"); Unload(); } @@ -158,8 +160,7 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame void ITestExecutor.Cancel() { - if (_activeRunner != null) - _activeRunner.StopRun(true); + _activeRunner?.StopRun(true); } #endregion @@ -230,7 +231,7 @@ private void RunAssembly(string assemblyPath, TestFilter filter) try { _activeRunner = GetRunnerFor(assemblyPath); - + CreateTestOutputFolder(); var loadResult = _activeRunner.Explore(TestFilter.Empty); #if !NETCOREAPP1_0 dumpXml?.AddString(loadResult.AsString()); @@ -335,9 +336,8 @@ private void GenerateTestOutput(XmlNode testResults, string assemblyPath) { if (!Settings.UseTestOutputXml) return; -#if NETCOREAPP1_0 -#else - var path = Path.Combine(Settings.TestOutputXml, $"{Path.GetFileNameWithoutExtension(assemblyPath)}.xml"); +#if !NETCOREAPP1_0 + var path = Path.Combine(TestOutputXmlFolder, $"{Path.GetFileNameWithoutExtension(assemblyPath)}.xml"); var resultService = TestEngine.Services.GetService(); // Following null argument should work for nunit3 format. Empty array is OK as well. // If you decide to handle other formats in the runsettings, it needs more work. @@ -356,6 +356,30 @@ private NUnitTestFilterBuilder CreateTestFilterBuilder() #endif } + + private void CreateTestOutputFolder() + { + if (!Settings.UseTestOutputXml) + { + return; + } + + var path = Path.IsPathRooted(Settings.TestOutputXml) + ? Settings.TestOutputXml + : Path.Combine(WorkDir, Settings.TestOutputXml); + try + { + Directory.CreateDirectory(path); + TestOutputXmlFolder = path; + TestLog.Info($" Test Output folder checked/created : {path} "); + } + catch (UnauthorizedAccessException) + { + TestLog.Error($" Failed creating test output folder at {path}"); + throw; + } + } + #endregion } } diff --git a/src/NUnitTestAdapter/NUnitTestAdapter.cs b/src/NUnitTestAdapter/NUnitTestAdapter.cs index 07a2fb4e..3bd2c00b 100644 --- a/src/NUnitTestAdapter/NUnitTestAdapter.cs +++ b/src/NUnitTestAdapter/NUnitTestAdapter.cs @@ -88,6 +88,8 @@ protected NUnitTestAdapter() // Our logger used to display messages protected TestLogger TestLog { get; private set; } + protected string WorkDir { get; private set; } + private static string exeName; public static bool IsRunningUnderIDE @@ -129,7 +131,7 @@ protected void Initialize(IDiscoveryContext context, IMessageLogger messageLogge { Settings.Load(context); TestLog.Verbosity = Settings.Verbosity; - CheckDirectories(); + } catch (Exception e) { @@ -138,14 +140,7 @@ protected void Initialize(IDiscoveryContext context, IMessageLogger messageLogge } } - private void CheckDirectories() - { - if (Settings.UseTestOutputXml) - { - Directory.CreateDirectory(Settings.TestOutputXml); - TestLog.Info($" Test Output folder checked/created : {Settings.TestOutputXml} "); - } - } + protected ITestRunner GetRunnerFor(string assemblyName) { @@ -226,7 +221,8 @@ private TestPackage CreateTestPackage(string assemblyName) if (!Directory.Exists(workDir)) Directory.CreateDirectory(workDir); package.Settings[PackageSettings.WorkDirectory] = workDir; - + WorkDir = workDir; + // CreateTestOutputFolder(workDir); return package; } diff --git a/src/NUnitTestAdapterTests/AdapterSettingsTests.cs b/src/NUnitTestAdapterTests/AdapterSettingsTests.cs index 585982f7..a98d1f1f 100644 --- a/src/NUnitTestAdapterTests/AdapterSettingsTests.cs +++ b/src/NUnitTestAdapterTests/AdapterSettingsTests.cs @@ -194,8 +194,7 @@ public void TestOutputSetting() Assert.That(_settings.UseTestOutputXml); Assert.Multiple(() => { - Assert.That(_settings.TestOutputXml, Does.Contain(@"\my\work\dir")); - Assert.That(Path.IsPathRooted(_settings.TestOutputXml), Is.True); + Assert.That(_settings.TestOutputXml, Does.Contain(@"/my/work/dir")); }); } @@ -203,6 +202,7 @@ public void TestOutputSetting() /// /// Workdir set, and is absolute, TestOutputXml is relative /// + [Ignore("Is not handled in the test executor, not in the test settings")] [Test] public void TestOutputSettingWithWorkDir() { @@ -210,7 +210,7 @@ public void TestOutputSettingWithWorkDir() Assert.That(_settings.UseTestOutputXml,"Settings not loaded properly"); Assert.Multiple(() => { - Assert.That(_settings.TestOutputXml, Does.Contain(@"\my\testoutput\dir"),"Content not correct"); + Assert.That(_settings.TestOutputXml, Does.Contain(@"\my/testoutput/dir"),"Content not correct"); Assert.That(_settings.TestOutputXml, Does.StartWith(@"C:\"),"Not correct start drive"); Assert.That(Path.IsPathRooted(_settings.TestOutputXml), Is.True,"Path not properly rooted"); }); diff --git a/src/NUnitTestAdapterTests/Fakes/FakeRunContext.cs b/src/NUnitTestAdapterTests/Fakes/FakeRunContext.cs index 8c578b4e..bd2ea916 100644 --- a/src/NUnitTestAdapterTests/Fakes/FakeRunContext.cs +++ b/src/NUnitTestAdapterTests/Fakes/FakeRunContext.cs @@ -34,6 +34,11 @@ public FakeRunContext() : base(new FakeRunSettings()) { } + + public FakeRunContext(FakeRunSettings fakeRunSettings) : base(fakeRunSettings) + { + } + #region IRunContext Members bool IRunContext.InIsolation diff --git a/src/NUnitTestAdapterTests/Fakes/FakeRunSettings.cs b/src/NUnitTestAdapterTests/Fakes/FakeRunSettings.cs index 413bee8c..117cc3be 100644 --- a/src/NUnitTestAdapterTests/Fakes/FakeRunSettings.cs +++ b/src/NUnitTestAdapterTests/Fakes/FakeRunSettings.cs @@ -1,5 +1,5 @@ // *********************************************************************** -// Copyright (c) 2012 Charlie Poole, Terje Sandstrom +// Copyright (c) 2012-2019 Charlie Poole, Terje Sandstrom // // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the @@ -33,9 +33,25 @@ ISettingsProvider IRunSettings.GetSettings(string settingsName) throw new NotImplementedException(); } - public string SettingsXml + public virtual string SettingsXml => ""; + } + + class FakeRunSettingsForTestOutput : FakeRunSettings + { + public override string SettingsXml => "TestResults"; + } + + class FakeRunSettingsForTestOutputAndWorkDir : FakeRunSettings + { + private readonly string _testOutput; + private readonly string _workDir; + + public FakeRunSettingsForTestOutputAndWorkDir(string testOutput,string workDir) { - get { return ""; } + _workDir = workDir; + _testOutput = testOutput; } + public override string SettingsXml => $"{_workDir}{_testOutput}"; } + } diff --git a/src/NUnitTestAdapterTests/TestExecutionTests.cs b/src/NUnitTestAdapterTests/TestExecutionTests.cs index 4d43fe69..5b736b9a 100644 --- a/src/NUnitTestAdapterTests/TestExecutionTests.cs +++ b/src/NUnitTestAdapterTests/TestExecutionTests.cs @@ -1,5 +1,5 @@ // *********************************************************************** -// Copyright (c) 2012-2017 Charlie Poole, Terje Sandstrom +// Copyright (c) 2012-2019 Charlie Poole, Terje Sandstrom // // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the @@ -35,15 +35,42 @@ namespace NUnit.VisualStudio.TestAdapter.Tests { + /// + /// ResultSummary Helper Class + /// + public class ResultSummary + { + private readonly Dictionary summary; + + public ResultSummary(IEnumerable results) + { + summary = new Dictionary(); + + foreach (TestResult result in results) + { + var outcome = result.Outcome; + summary[outcome] = GetCount(outcome) + 1; + } + } + + private int GetCount(TestOutcome outcome) + { + return summary.ContainsKey(outcome) + ? summary[outcome] + : 0; + } + } + + [Category("TestExecution")] public class TestExecutionTests { - private string MockAssemblyPath; + private string MockAssemblyPath; static readonly IRunContext Context = new FakeRunContext(); private FakeFrameworkHandle testLog; - ResultSummary Summary { get; set; } + ResultSummary Summary { get; set; } [OneTimeSetUp] @@ -52,7 +79,7 @@ public void LoadMockassembly() MockAssemblyPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "mock-assembly.dll"); // Sanity check to be sure we have the correct version of mock-assembly.dll - Assert.That(MockAssembly.TestsAtRuntime , Is.EqualTo(MockAssembly.Tests), + Assert.That(MockAssembly.TestsAtRuntime, Is.EqualTo(MockAssembly.Tests), "The reference to mock-assembly.dll appears to be the wrong version"); testLog = new FakeFrameworkHandle(); @@ -188,31 +215,73 @@ private TestResult GetTestResult(string displayName) .FirstOrDefault(); } - #region Nested ResultSummary Helper Class - private class ResultSummary + } + + + + + + [Category("TestExecution")] + public class TestExecutionTestsForTestOutput + { + private string _mockAssemblyPath; + private string _mockAssemblyFolder; + + private FakeFrameworkHandle testLog; + ResultSummary Summary { get; set; } + + + [OneTimeSetUp] + public void LoadMockAssembly() { - private readonly Dictionary summary; + _mockAssemblyPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "mock-assembly.dll"); + _mockAssemblyFolder = Path.GetDirectoryName(_mockAssemblyPath); + // Sanity check to be sure we have the correct version of mock-assembly.dll + Assert.That(MockAssembly.TestsAtRuntime, Is.EqualTo(MockAssembly.Tests), + "The reference to mock-assembly.dll appears to be the wrong version"); + testLog = new FakeFrameworkHandle(); - public ResultSummary(IEnumerable results) - { - summary = new Dictionary(); - - foreach(TestResult result in results) - { - var outcome = result.Outcome; - summary[outcome] = GetCount(outcome) + 1; - } - } + // Load the NUnit mock-assembly.dll once for this test, saving + // the list of test cases sent to the discovery sink - private int GetCount(TestOutcome outcome) - { - return summary.ContainsKey(outcome) - ? summary[outcome] - : 0; - } + var testResults = testLog.Events + .Where(e => e.EventType == FakeFrameworkHandle.EventType.RecordResult) + .Select(e => e.TestResult) + .ToList(); + Summary = new ResultSummary(testResults); + } +#if !NETCOREAPP1_0 + [Test] + public void ThatTestOutputXmlHasBeenCreatedBelowAssemblyFolder() + { + var context = new FakeRunContext(new FakeRunSettingsForTestOutput()); + + TestAdapterUtils.CreateExecutor().RunTests(new[] { _mockAssemblyPath }, context, testLog); + + var expectedFolder = Path.Combine(_mockAssemblyFolder, "TestResults"); + Assert.That(Directory.Exists(expectedFolder), $"Folder {expectedFolder} not created"); + var expectedFile = Path.Combine(expectedFolder, "mock-assembly.xml"); + Assert.That(File.Exists(expectedFile), $"File {expectedFile} not found"); + } + + + [Test] + public void ThatTestOutputXmlHasBeenAtWorkDirLocation() + { + var temp = Path.GetTempPath(); + var context = new FakeRunContext(new FakeRunSettingsForTestOutputAndWorkDir("TestResult", Path.Combine(temp, "NUnit"))); + + var executor = TestAdapterUtils.CreateExecutor(); + executor.RunTests(new[] { _mockAssemblyPath }, context, testLog); + var expectedFolder = Path.Combine(Path.GetTempPath(), "NUnit", "TestResult"); + Assert.That(Directory.Exists(expectedFolder), $"Folder {expectedFolder} not created"); + var expectedFile = Path.Combine(expectedFolder, "mock-assembly.xml"); + Assert.That(File.Exists(expectedFile), $"File {expectedFile} not found"); } +#endif - #endregion } + + }