Handling of relative paths for TestOutputXml fixed #596

NUnit3TestAdapter.sln

@@ -11,6 +11,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		build.ps1 = build.ps1
 		src\Common.props = src\Common.props
 		DisableAppDomain.runsettings = DisableAppDomain.runsettings
+		lib.cake = lib.cake
 		LICENSE.txt = LICENSE.txt
 		src\native-assembly\NativeTests.dll = src\native-assembly\NativeTests.dll
 		NuGet.Config = NuGet.Config

build.cake

@@ -16,9 +16,9 @@ var configuration = Argument("configuration", "Release");
 var version = "3.13.0";
 var modifier = "";
 
-var dbgSuffix = configuration == "Debug" ? "-dbg" : "";
+var dbgSuffix = configuration.ToLower() == "debug" ? "-dbg" : "";
 var packageVersion = version + modifier + dbgSuffix;
-
+Information("PackageVersion is "+packageVersion);
 if (BuildSystem.IsRunningOnAppVeyor)
 {
     var tag = AppVeyor.Environment.Repository.Tag;

src/NUnitTestAdapter/AdapterSettings.cs

@@ -250,9 +250,7 @@ public void Load(string settingsXml)
             UseVsKeepEngineRunning = GetInnerTextAsBool(nunitNode, nameof(UseVsKeepEngineRunning), false);
             BasePath = GetInnerTextWithLog(nunitNode, nameof(BasePath));
             PrivateBinPath = GetInnerTextWithLog(nunitNode, nameof(PrivateBinPath));
-            var testOutput = GetInnerTextWithLog(nunitNode, nameof(TestOutputXml));
-            if (!string.IsNullOrEmpty(testOutput))
-                TestOutputXml = ValidatedPath(testOutput, nameof(TestOutputXml));
+            TestOutputXml = GetInnerTextWithLog(nunitNode, nameof(TestOutputXml));
             RandomSeed = GetInnerTextAsNullableInt(nunitNode, nameof(RandomSeed));
             RandomSeedSpecified = RandomSeed.HasValue;
             if (!RandomSeedSpecified)

src/NUnitTestAdapter/NUnit3TestExecutor.cs

@@ -56,6 +56,8 @@ public NUnit3TestExecutor()
         public IFrameworkHandle FrameworkHandle { get; private set; }
         private TfsTestFilter TfsFilter { get; set; }
 
+        private string TestOutputXmlFolder { get; set; } = "";
+
         // NOTE: an earlier version of this code had a FilterBuilder
         // property. This seemed to make sense, because we instantiate
         // it in two different places. However, the existence of an
@@ -107,7 +109,7 @@ public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrame
                 }
             }
 
-            TestLog.Info(string.Format("NUnit Adapter {0}: Test execution complete", AdapterVersion));
+            TestLog.Info($"NUnit Adapter {AdapterVersion}: Test execution complete");
             Unload();
         }
 
@@ -158,8 +160,7 @@ public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrame
 
         void ITestExecutor.Cancel()
         {
-            if (_activeRunner != null)
-                _activeRunner.StopRun(true);
+            _activeRunner?.StopRun(true);
         }
 
         #endregion
@@ -230,7 +231,7 @@ private void RunAssembly(string assemblyPath, TestFilter filter)
             try
             {
                 _activeRunner = GetRunnerFor(assemblyPath);
-
+                CreateTestOutputFolder();
                 var loadResult = _activeRunner.Explore(TestFilter.Empty);
 #if !NETCOREAPP1_0
                 dumpXml?.AddString(loadResult.AsString());
@@ -335,9 +336,8 @@ private void GenerateTestOutput(XmlNode testResults, string assemblyPath)
         {
             if (!Settings.UseTestOutputXml)
                 return;
-#if NETCOREAPP1_0
-#else
-            var path = Path.Combine(Settings.TestOutputXml, $"{Path.GetFileNameWithoutExtension(assemblyPath)}.xml");
+#if !NETCOREAPP1_0
+            var path = Path.Combine(TestOutputXmlFolder, $"{Path.GetFileNameWithoutExtension(assemblyPath)}.xml");
             var resultService = TestEngine.Services.GetService<IResultService>();
             // Following null argument should work for nunit3 format. Empty array is OK as well.
             // If you decide to handle other formats in the runsettings, it needs more work.
@@ -356,6 +356,30 @@ private NUnitTestFilterBuilder CreateTestFilterBuilder()
 #endif
         }
 
+
+        private void CreateTestOutputFolder()
+        {
+            if (!Settings.UseTestOutputXml)
+            {
+                return;
+            }
+
+            var path = Path.IsPathRooted(Settings.TestOutputXml)
+                ? Settings.TestOutputXml
+                : Path.Combine(WorkDir, Settings.TestOutputXml);
+            try
+            {
+                Directory.CreateDirectory(path);
+                TestOutputXmlFolder = path;
+                TestLog.Info($"  Test Output folder checked/created : {path} ");
+            }
+            catch (UnauthorizedAccessException)
+            {
+                TestLog.Error($"   Failed creating test output folder at {path}");
+                throw;
+            }
+        }
+
         #endregion
     }
 }

src/NUnitTestAdapter/NUnitTestAdapter.cs

@@ -88,6 +88,8 @@ protected NUnitTestAdapter()
         // Our logger used to display messages
         protected TestLogger TestLog { get; private set; }
 
+        protected string WorkDir { get; private set; }
+
         private static string exeName;
 
         public static bool IsRunningUnderIDE
@@ -129,7 +131,7 @@ protected void Initialize(IDiscoveryContext context, IMessageLogger messageLogge
             {
                 Settings.Load(context);
                 TestLog.Verbosity = Settings.Verbosity;
-                CheckDirectories();
+
             }
             catch (Exception e)
             {
@@ -138,14 +140,7 @@ protected void Initialize(IDiscoveryContext context, IMessageLogger messageLogge
             }
         }
 
-        private void CheckDirectories()
-        {
-            if (Settings.UseTestOutputXml)
-            {
-                Directory.CreateDirectory(Settings.TestOutputXml);
-                TestLog.Info($"  Test Output folder checked/created : {Settings.TestOutputXml} ");
-            }
-        }
+
 
         protected ITestRunner GetRunnerFor(string assemblyName)
         {
@@ -226,7 +221,8 @@ private TestPackage CreateTestPackage(string assemblyName)
             if (!Directory.Exists(workDir))
                 Directory.CreateDirectory(workDir);
             package.Settings[PackageSettings.WorkDirectory] = workDir;
-
+            WorkDir = workDir;
+         //   CreateTestOutputFolder(workDir);
             return package;
         }
 

src/NUnitTestAdapterTests/AdapterSettingsTests.cs

@@ -194,23 +194,23 @@ public void TestOutputSetting()
             Assert.That(_settings.UseTestOutputXml);
             Assert.Multiple(() =>
             {
-                Assert.That(_settings.TestOutputXml, Does.Contain(@"\my\work\dir"));
-                Assert.That(Path.IsPathRooted(_settings.TestOutputXml), Is.True);
+                Assert.That(_settings.TestOutputXml, Does.Contain(@"/my/work/dir"));
             });
 
         }
 
         /// <summary>
         /// Workdir set, and is absolute,  TestOutputXml is relative
         /// </summary>
+        [Ignore("Is not handled in the test executor, not in the test settings")]
         [Test]
         public void TestOutputSettingWithWorkDir()
         {
             _settings.Load(@"<RunSettings><NUnit><WorkDirectory>C:\Whatever</WorkDirectory><TestOutputXml>my/testoutput/dir</TestOutputXml></NUnit></RunSettings>");
             Assert.That(_settings.UseTestOutputXml,"Settings not loaded properly");
             Assert.Multiple(() =>
             {
-                Assert.That(_settings.TestOutputXml, Does.Contain(@"\my\testoutput\dir"),"Content not correct");
+                Assert.That(_settings.TestOutputXml, Does.Contain(@"\my/testoutput/dir"),"Content not correct");
                 Assert.That(_settings.TestOutputXml, Does.StartWith(@"C:\"),"Not correct start drive");
                 Assert.That(Path.IsPathRooted(_settings.TestOutputXml), Is.True,"Path not properly rooted");
             });

src/NUnitTestAdapterTests/Fakes/FakeRunContext.cs

@@ -34,6 +34,11 @@ public FakeRunContext() : base(new FakeRunSettings())
         {
         }
 
+
+        public FakeRunContext(FakeRunSettings fakeRunSettings) : base(fakeRunSettings)
+        {
+        }
+
         #region IRunContext Members
 
         bool IRunContext.InIsolation

src/NUnitTestAdapterTests/Fakes/FakeRunSettings.cs

@@ -1,5 +1,5 @@
 // ***********************************************************************
-// Copyright (c) 2012 Charlie Poole, Terje Sandstrom
+// Copyright (c) 2012-2019 Charlie Poole, Terje Sandstrom
 //
 // Permission is hereby granted, free of charge, to any person obtaining
 // a copy of this software and associated documentation files (the
@@ -33,9 +33,25 @@ ISettingsProvider IRunSettings.GetSettings(string settingsName)
             throw new NotImplementedException();
         }
 
-        public string SettingsXml
+        public virtual string SettingsXml => "<RunSettings/>";
+    }
+
+    class FakeRunSettingsForTestOutput : FakeRunSettings
+    {
+        public override string SettingsXml => "<RunSettings><NUnit><TestOutputXml>TestResults</TestOutputXml></NUnit></RunSettings>";
+    }
+
+    class FakeRunSettingsForTestOutputAndWorkDir : FakeRunSettings
+    {
+        private readonly string _testOutput;
+        private readonly string _workDir;
+
+        public FakeRunSettingsForTestOutputAndWorkDir(string testOutput,string workDir)
         {
-            get { return "<RunSettings/>"; }
+            _workDir = workDir;
+            _testOutput = testOutput;
         }
+        public override string SettingsXml => $"<RunSettings><NUnit><WorkDirectory>{_workDir}</WorkDirectory><TestOutputXml>{_testOutput}</TestOutputXml></NUnit></RunSettings>";
     }
+
 }

src/NUnitTestAdapterTests/TestExecutionTests.cs

@@ -1,5 +1,5 @@
 // ***********************************************************************
-// Copyright (c) 2012-2017 Charlie Poole, Terje Sandstrom
+// Copyright (c) 2012-2019 Charlie Poole, Terje Sandstrom
 //
 // Permission is hereby granted, free of charge, to any person obtaining
 // a copy of this software and associated documentation files (the
@@ -35,15 +35,42 @@
 
 namespace NUnit.VisualStudio.TestAdapter.Tests
 {
+    /// <summary>
+    /// ResultSummary Helper Class
+    /// </summary>
+    public class ResultSummary
+    {
+        private readonly Dictionary<TestOutcome, int> summary;
+
+        public ResultSummary(IEnumerable<TestResult> results)
+        {
+            summary = new Dictionary<TestOutcome, int>();
+
+            foreach (TestResult result in results)
+            {
+                var outcome = result.Outcome;
+                summary[outcome] = GetCount(outcome) + 1;
+            }
+        }
+
+        private int GetCount(TestOutcome outcome)
+        {
+            return summary.ContainsKey(outcome)
+                ? summary[outcome]
+                : 0;
+        }
+    }
+
+
 
     [Category("TestExecution")]
     public class TestExecutionTests
     {
-        private string MockAssemblyPath; 
+        private string MockAssemblyPath;
         static readonly IRunContext Context = new FakeRunContext();
 
         private FakeFrameworkHandle testLog;
-        ResultSummary Summary { get;  set; }    
+        ResultSummary Summary { get; set; }
 
 
         [OneTimeSetUp]
@@ -52,7 +79,7 @@ public void LoadMockassembly()
             MockAssemblyPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "mock-assembly.dll");
 
             // Sanity check to be sure we have the correct version of mock-assembly.dll
-            Assert.That(MockAssembly.TestsAtRuntime , Is.EqualTo(MockAssembly.Tests),
+            Assert.That(MockAssembly.TestsAtRuntime, Is.EqualTo(MockAssembly.Tests),
                 "The reference to mock-assembly.dll appears to be the wrong version");
             testLog = new FakeFrameworkHandle();
 
@@ -188,31 +215,73 @@ private TestResult GetTestResult(string displayName)
                             .FirstOrDefault();
         }
 
-        #region Nested ResultSummary Helper Class
 
-        private class ResultSummary
+    }
+
+
+
+
+
+    [Category("TestExecution")]
+    public class TestExecutionTestsForTestOutput
+    {
+        private string _mockAssemblyPath;
+        private string _mockAssemblyFolder;
+
+        private FakeFrameworkHandle testLog;
+        ResultSummary Summary { get; set; }
+
+
+        [OneTimeSetUp]
+        public void LoadMockAssembly()
         {
-            private readonly Dictionary<TestOutcome, int> summary;
+            _mockAssemblyPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "mock-assembly.dll");
+            _mockAssemblyFolder = Path.GetDirectoryName(_mockAssemblyPath);
+            // Sanity check to be sure we have the correct version of mock-assembly.dll
+            Assert.That(MockAssembly.TestsAtRuntime, Is.EqualTo(MockAssembly.Tests),
+                "The reference to mock-assembly.dll appears to be the wrong version");
+            testLog = new FakeFrameworkHandle();
 
-            public ResultSummary(IEnumerable<TestResult> results)
-            {
-                summary = new Dictionary<TestOutcome, int>();
-
-                foreach(TestResult result in results)
-                {
-                    var outcome = result.Outcome;
-                    summary[outcome] = GetCount(outcome) + 1;
-                }
-            }
+            // Load the NUnit mock-assembly.dll once for this test, saving
+            // the list of test cases sent to the discovery sink
 
-            private int GetCount(TestOutcome outcome)
-            {
-                return summary.ContainsKey(outcome)
-                    ? summary[outcome]
-                    : 0;
-            }
+            var testResults = testLog.Events
+                .Where(e => e.EventType == FakeFrameworkHandle.EventType.RecordResult)
+                .Select(e => e.TestResult)
+                .ToList();
+            Summary = new ResultSummary(testResults);
+        }
+#if !NETCOREAPP1_0
+        [Test]
+        public void ThatTestOutputXmlHasBeenCreatedBelowAssemblyFolder()
+        {
+            var context = new FakeRunContext(new FakeRunSettingsForTestOutput());
+
+            TestAdapterUtils.CreateExecutor().RunTests(new[] { _mockAssemblyPath }, context, testLog);
+
+            var expectedFolder = Path.Combine(_mockAssemblyFolder, "TestResults");
+            Assert.That(Directory.Exists(expectedFolder), $"Folder {expectedFolder} not created");
+            var expectedFile = Path.Combine(expectedFolder, "mock-assembly.xml");
+            Assert.That(File.Exists(expectedFile), $"File {expectedFile} not found");
+        }
+
+
+        [Test]
+        public void ThatTestOutputXmlHasBeenAtWorkDirLocation()
+        {
+            var temp = Path.GetTempPath();
+            var context = new FakeRunContext(new FakeRunSettingsForTestOutputAndWorkDir("TestResult", Path.Combine(temp, "NUnit")));
+
+            var executor = TestAdapterUtils.CreateExecutor();
+            executor.RunTests(new[] { _mockAssemblyPath }, context, testLog);
+            var expectedFolder = Path.Combine(Path.GetTempPath(), "NUnit", "TestResult");
+            Assert.That(Directory.Exists(expectedFolder), $"Folder {expectedFolder} not created");
+            var expectedFile = Path.Combine(expectedFolder, "mock-assembly.xml");
+            Assert.That(File.Exists(expectedFile), $"File {expectedFile} not found");
         }
+#endif
 
-        #endregion
     }
+
+
 }