diff --git a/scripts/build.ps1 b/scripts/build.ps1 index b05081f0db..74df19cec3 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -59,7 +59,7 @@ $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1 # Dotnet build doesn't support --packages yet. See https://github.com/dotnet/cli/issues/2712 $env:NUGET_PACKAGES = $env:TP_PACKAGES_DIR $env:NUGET_EXE_Version = "3.4.3" -$env:DOTNET_CLI_VERSION = "latest" +$env:DOTNET_CLI_VERSION = "1.0.0-preview5-004384" # # Build configuration diff --git a/src/Microsoft.TestPlatform.CoreUtilities/ValidateArg.cs b/src/Microsoft.TestPlatform.CoreUtilities/ValidateArg.cs index aad87d6d51..e911a9117a 100644 --- a/src/Microsoft.TestPlatform.CoreUtilities/ValidateArg.cs +++ b/src/Microsoft.TestPlatform.CoreUtilities/ValidateArg.cs @@ -74,6 +74,32 @@ public static string NotNullOrEmpty([ValidatedNotNull]string arg, string paramet return arg; } + /// + /// Validate a string is not null, empty or consists only of white-space characters. + /// + /// + /// Input string. + /// + /// + /// Name of the parameter to validate. + /// + /// + /// Validated string. + /// + /// + /// Thrown if the input string is null null, empty or consists only of white-space characters. + /// + [DebuggerStepThrough] + public static string NotNullOrWhiteSpace([ValidatedNotNull]string arg, string parameterName) + { + if (string.IsNullOrWhiteSpace(arg)) + { + throw new ArgumentNullException(parameterName); + } + + return arg; + } + /// /// Throws ArgumentOutOfRangeException if the argument is less than zero. /// diff --git a/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs b/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs index 1f6fc5da42..906b401c8f 100644 --- a/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs +++ b/src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs @@ -209,6 +209,12 @@ private static bool TryGetFrameworkXml(XPathNavigator runSettingsNavigator, out Func validator = (string xml) => { + if (Framework.FromString(xml) != null) + { + // Allow TargetFrameworkMoniker values like .NETFramework,Version=v4.5, ".NETCoreApp,Version=v1.0 + return true; + } + var value = (FrameworkVersion)Enum.Parse(typeof(FrameworkVersion), xml, true); if (!Enum.IsDefined(typeof(FrameworkVersion), value) || value == FrameworkVersion.None) diff --git a/src/vstest.console/CommandLine/CommandLineOptions.cs b/src/vstest.console/CommandLine/CommandLineOptions.cs index 3e57adafc1..d14aeb4214 100644 --- a/src/vstest.console/CommandLine/CommandLineOptions.cs +++ b/src/vstest.console/CommandLine/CommandLineOptions.cs @@ -221,12 +221,22 @@ internal Framework TargetFrameworkVersion this.FrameworkVersionSpecified = true; } } - + /// /// Gets a value indicating whether /Framework has been specified on command line or not. /// internal bool FrameworkVersionSpecified { get; private set; } + /// + /// Gets or sets the results directory for test run. + /// + internal string ResultsDirectory { get; set; } + + /// + /// Gets or sets the /setting switch value. i.e path to settings file. + /// + internal string SettingsFile { get; set; } + #endregion #region Public Methods diff --git a/src/vstest.console/CommandLine/Executor.cs b/src/vstest.console/CommandLine/Executor.cs index aa9eaef58c..63ffce4a69 100644 --- a/src/vstest.console/CommandLine/Executor.cs +++ b/src/vstest.console/CommandLine/Executor.cs @@ -35,6 +35,8 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.Utilities; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; + using Microsoft.VisualStudio.TestPlatform.Common; using CommandLineResources = Microsoft.VisualStudio.TestPlatform.CommandLine.Resources.Resources; @@ -190,6 +192,9 @@ private int GetArgumentProcessors(string[] args, out List pr var processorsToAlwaysExecute = processorFactory.GetArgumentProcessorsToAlwaysExecute(); processors.AddRange(processorsToAlwaysExecute); + // Initialize Runsettings with defaults + RunSettingsManager.Instance.AddDefaultRunSettings(); + // Ensure we have an action argument. this.EnsureActionArgumentIsPresent(processors, processorFactory); diff --git a/src/vstest.console/Processors/CLIRunSettingsArgumentProcessor.cs b/src/vstest.console/Processors/CLIRunSettingsArgumentProcessor.cs index ac41bcb65f..610f32ad3c 100644 --- a/src/vstest.console/Processors/CLIRunSettingsArgumentProcessor.cs +++ b/src/vstest.console/Processors/CLIRunSettingsArgumentProcessor.cs @@ -13,6 +13,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Processors using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; using CommandLineResources = Microsoft.VisualStudio.TestPlatform.CommandLine.Resources.Resources; @@ -114,41 +115,8 @@ public void Initialize(string[] arguments) // Load up the run settings and set it as the active run settings. try { - var doc = new XmlDocument(); - - if (this.runSettingsManager.ActiveRunSettings != null && !string.IsNullOrEmpty(this.runSettingsManager.ActiveRunSettings.SettingsXml)) - { - var settingsXml = this.runSettingsManager.ActiveRunSettings.SettingsXml; - -#if net46 - using (var reader = XmlReader.Create(new StringReader(settingsXml), new XmlReaderSettings() { XmlResolver = null, CloseInput = true, DtdProcessing = DtdProcessing.Prohibit })) - { -#else - using (var reader = XmlReader.Create(new StringReader(settingsXml), new XmlReaderSettings() { CloseInput = true, DtdProcessing = DtdProcessing.Prohibit })) - { -#endif - doc.Load(reader); - } - } - else - { -#if net46 - doc = (XmlDocument)XmlRunSettingsUtilities.CreateDefaultRunSettings(); -#else - using (var reader = XmlReader.Create(new StringReader(XmlRunSettingsUtilities.CreateDefaultRunSettings().CreateNavigator().OuterXml), new XmlReaderSettings() { CloseInput = true, DtdProcessing = DtdProcessing.Prohibit })) - { - doc.Load(reader); - } -#endif - } - // Append / Override run settings supplied in CLI - CreateOrOverwriteRunSettings(doc, arguments); - - // Set Active Run Settings. - var runSettings = new RunSettings(); - runSettings.LoadSettingsXml(doc.OuterXml); - this.runSettingsManager.SetActiveRunSettings(runSettings); + CreateOrOverwriteRunSettings(this.runSettingsManager, arguments); } catch (XPathException exception) { @@ -166,7 +134,7 @@ public ArgumentProcessorResult Execute() return ArgumentProcessorResult.Success; } - private void CreateOrOverwriteRunSettings(XmlDocument xmlDoc, string[] args) + private void CreateOrOverwriteRunSettings(IRunSettingsProvider runSettingsProvider, string[] args) { var length = args.Length; @@ -186,37 +154,8 @@ private void CreateOrOverwriteRunSettings(XmlDocument xmlDoc, string[] args) continue; } - // Check if the key exists. - var xPath = key.Replace('.', '/'); - var node = xmlDoc.SelectSingleNode(string.Format("//RunSettings/{0}", xPath)); - - if (node == null) - { - node = CreateNode(xmlDoc, key.Split('.')); - } - - node.InnerText = value; + runSettingsProvider.UpdateRunSettingsNode(key, value); } } - - private XmlNode CreateNode(XmlDocument doc, string[] xPath) - { - XmlNode node = null; - XmlNode parent = doc.DocumentElement; - - for (int i = 0; i < xPath.Length; i++) - { - node = parent.SelectSingleNode(xPath[i]); - - if (node == null) - { - node = parent.AppendChild(doc.CreateElement(xPath[i])); - } - - parent = node; - } - - return node; - } } } diff --git a/src/vstest.console/Processors/FrameworkArgumentProcessor.cs b/src/vstest.console/Processors/FrameworkArgumentProcessor.cs index 1311afd957..cf29800818 100644 --- a/src/vstest.console/Processors/FrameworkArgumentProcessor.cs +++ b/src/vstest.console/Processors/FrameworkArgumentProcessor.cs @@ -6,7 +6,12 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Processors using System; using System.Diagnostics.Contracts; using System.Globalization; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; + using Microsoft.VisualStudio.TestPlatform.Common; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; + using Microsoft.VisualStudio.TestPlatform.Utilities; using CommandLineResources = Microsoft.VisualStudio.TestPlatform.CommandLine.Resources.Resources; @@ -54,7 +59,7 @@ public Lazy Executor { if (this.executor == null) { - this.executor = new Lazy(() => new FrameworkArgumentExecutor(CommandLineOptions.Instance)); + this.executor = new Lazy(() => new FrameworkArgumentExecutor(CommandLineOptions.Instance, RunSettingsManager.Instance)); } return this.executor; @@ -94,6 +99,10 @@ internal class FrameworkArgumentExecutor : IArgumentExecutor /// private CommandLineOptions commandLineOptions; + private IRunSettingsProvider runSettingsManager; + + public const string RunSettingsPath = "RunConfiguration.TargetFrameworkVersion"; + #endregion #region Constructor @@ -101,14 +110,16 @@ internal class FrameworkArgumentExecutor : IArgumentExecutor /// /// Default constructor. /// - /// - /// The options. - /// - public FrameworkArgumentExecutor(CommandLineOptions options) + /// The options. + /// The runsettings manager. + public FrameworkArgumentExecutor(CommandLineOptions options, IRunSettingsProvider runSettingsManager) { Contract.Requires(options != null); + Contract.Requires(runSettingsManager != null); this.commandLineOptions = options; + this.runSettingsManager = runSettingsManager; } + #endregion #region IArgumentExecutor @@ -126,11 +137,27 @@ public void Initialize(string argument) var validFramework = Framework.FromString(argument); if (validFramework == null) - { + { throw new CommandLineException( string.Format(CultureInfo.CurrentCulture, CommandLineResources.InvalidFrameworkVersion, argument)); } - commandLineOptions.TargetFrameworkVersion = validFramework; + + this.commandLineOptions.TargetFrameworkVersion = validFramework; + + if (this.commandLineOptions.TargetFrameworkVersion != Framework.DefaultFramework + && !string.IsNullOrWhiteSpace(this.commandLineOptions.SettingsFile) + && MSTestSettingsUtilities.IsLegacyTestSettingsFile(this.commandLineOptions.SettingsFile)) + { + // Legacy testsettings file support only default target framework. + IOutput output = ConsoleOutput.Instance; + output.Warning(CommandLineResources.TestSettingsFrameworkMismatch, + this.commandLineOptions.TargetFrameworkVersion.ToString(), Framework.DefaultFramework.ToString()); + } + else + { + this.runSettingsManager.UpdateRunSettingsNode(FrameworkArgumentExecutor.RunSettingsPath, + validFramework.ToString()); + } if (EqtTrace.IsInfoEnabled) { diff --git a/src/vstest.console/Processors/ListTestsArgumentProcessor.cs b/src/vstest.console/Processors/ListTestsArgumentProcessor.cs index 08a26c9fe5..f3180c5222 100644 --- a/src/vstest.console/Processors/ListTestsArgumentProcessor.cs +++ b/src/vstest.console/Processors/ListTestsArgumentProcessor.cs @@ -199,6 +199,7 @@ public ArgumentProcessorResult Execute() { Contract.Assert(this.output != null); Contract.Assert(this.commandLineOptions != null); + Contract.Assert(!string.IsNullOrWhiteSpace(this.runSettingsManager?.ActiveRunSettings?.SettingsXml)); if (this.commandLineOptions.Sources.Count() <= 0) { @@ -207,7 +208,7 @@ public ArgumentProcessorResult Execute() this.output.WriteLine(CommandLineResources.ListTestsHeaderMessage, OutputLevel.Information); - var runSettings = RunSettingsUtilities.GetRunSettings(this.runSettingsManager, this.commandLineOptions); + var runSettings = this.runSettingsManager.ActiveRunSettings.SettingsXml; var success = this.testRequestManager.DiscoverTests( new DiscoveryRequestPayload() { Sources = this.commandLineOptions.Sources, RunSettings = runSettings }, diff --git a/src/vstest.console/Processors/ParallelArgumentProcessor.cs b/src/vstest.console/Processors/ParallelArgumentProcessor.cs index 3ff835b7e0..55cb54b1e4 100644 --- a/src/vstest.console/Processors/ParallelArgumentProcessor.cs +++ b/src/vstest.console/Processors/ParallelArgumentProcessor.cs @@ -7,6 +7,10 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Processors using System.Diagnostics.Contracts; using System.Globalization; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; + using Microsoft.VisualStudio.TestPlatform.Common; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; + using CommandLineResources = Microsoft.VisualStudio.TestPlatform.CommandLine.Resources.Resources; /// @@ -49,7 +53,7 @@ public Lazy Executor { if (this.executor == null) { - this.executor = new Lazy(() => new ParallelArgumentExecutor(CommandLineOptions.Instance)); + this.executor = new Lazy(() => new ParallelArgumentExecutor(CommandLineOptions.Instance, RunSettingsManager.Instance)); } return this.executor; @@ -89,6 +93,10 @@ internal class ParallelArgumentExecutor : IArgumentExecutor /// private CommandLineOptions commandLineOptions; + private IRunSettingsProvider runSettingsManager; + + public const string RunSettingsPath = "RunConfiguration.MaxCpuCount"; + #endregion #region Constructor @@ -96,14 +104,16 @@ internal class ParallelArgumentExecutor : IArgumentExecutor /// /// Default constructor. /// - /// - /// The options. - /// - public ParallelArgumentExecutor(CommandLineOptions options) + /// The options. + /// The runsettings manager. + public ParallelArgumentExecutor(CommandLineOptions options, IRunSettingsProvider runSettingsManager) { Contract.Requires(options != null); + Contract.Requires(runSettingsManager != null); this.commandLineOptions = options; + this.runSettingsManager = runSettingsManager; } + #endregion #region IArgumentExecutor @@ -122,6 +132,7 @@ public void Initialize(string argument) } commandLineOptions.Parallel = true; + this.runSettingsManager.UpdateRunSettingsNode(ParallelArgumentExecutor.RunSettingsPath, "0"); } /// diff --git a/src/vstest.console/Processors/PlatformArgumentProcessor.cs b/src/vstest.console/Processors/PlatformArgumentProcessor.cs index e5dd73170b..1337f5518d 100644 --- a/src/vstest.console/Processors/PlatformArgumentProcessor.cs +++ b/src/vstest.console/Processors/PlatformArgumentProcessor.cs @@ -7,6 +7,9 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Processors using System.Diagnostics.Contracts; using System.Globalization; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; + using Microsoft.VisualStudio.TestPlatform.Common; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using CommandLineResources = Microsoft.VisualStudio.TestPlatform.CommandLine.Resources.Resources; @@ -54,7 +57,7 @@ public Lazy Executor { if (this.executor == null) { - this.executor = new Lazy(() => new PlatformArgumentExecutor(CommandLineOptions.Instance)); + this.executor = new Lazy(() => new PlatformArgumentExecutor(CommandLineOptions.Instance, RunSettingsManager.Instance)); } return this.executor; @@ -94,6 +97,10 @@ internal class PlatformArgumentExecutor : IArgumentExecutor /// private CommandLineOptions commandLineOptions; + private IRunSettingsProvider runSettingsManager; + + public const string RunSettingsPath = "RunConfiguration.TargetPlatform"; + #endregion #region Constructor @@ -101,14 +108,16 @@ internal class PlatformArgumentExecutor : IArgumentExecutor /// /// Default constructor. /// - /// - /// The options. - /// - public PlatformArgumentExecutor(CommandLineOptions options) + /// The options. + /// The runsettings manager. + public PlatformArgumentExecutor(CommandLineOptions options, IRunSettingsProvider runSettingsManager) { Contract.Requires(options != null); + Contract.Requires(runSettingsManager != null); this.commandLineOptions = options; + this.runSettingsManager = runSettingsManager; } + #endregion #region IArgumentExecutor @@ -134,6 +143,7 @@ public void Initialize(string argument) if (validPlatform) { this.commandLineOptions.TargetArchitecture = platform; + this.runSettingsManager.UpdateRunSettingsNode(PlatformArgumentExecutor.RunSettingsPath, platform.ToString()); } else { diff --git a/src/vstest.console/Processors/ResultsDirectoryArgumentProcessor.cs b/src/vstest.console/Processors/ResultsDirectoryArgumentProcessor.cs new file mode 100644 index 0000000000..7213972650 --- /dev/null +++ b/src/vstest.console/Processors/ResultsDirectoryArgumentProcessor.cs @@ -0,0 +1,171 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Processors +{ + using System; + using System.Diagnostics.Contracts; + using System.IO; + using System.Security; + + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; + using Microsoft.VisualStudio.TestPlatform.Common; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; + using CommandLineResources = Microsoft.VisualStudio.TestPlatform.CommandLine.Resources.Resources; + + /// + /// Allows the user to specify a path to save test results. + /// + internal class ResultsDirectoryArgumentProcessor : IArgumentProcessor + { + #region Constants + + /// + /// The name of the command line argument that the ListTestsArgumentExecutor handles. + /// + public const string CommandName = "/ResultsDirectory"; + + private const string RunSettingsPath = "RunConfiguration.ResultsDirectory"; + #endregion + + private Lazy metadata; + + private Lazy executor; + + /// + /// Gets the metadata. + /// + public Lazy Metadata + { + get + { + if (this.metadata == null) + { + this.metadata = new Lazy(() => new ResultsDirectoryArgumentProcessorCapabilities()); + } + + return this.metadata; + } + } + + /// + /// Gets or sets the executor. + /// + public Lazy Executor + { + get + { + if (this.executor == null) + { + this.executor = new Lazy(() => new ResultsDirectoryArgumentExecutor(CommandLineOptions.Instance, RunSettingsManager.Instance)); + } + + return this.executor; + } + + set + { + this.executor = value; + } + } + } + + /// + /// The argument capabilities. + /// + internal class ResultsDirectoryArgumentProcessorCapabilities : BaseArgumentProcessorCapabilities + { + public override string CommandName => ResultsDirectoryArgumentProcessor.CommandName; + + public override bool AllowMultiple => false; + + public override bool IsAction => false; + + public override ArgumentProcessorPriority Priority => ArgumentProcessorPriority.AutoUpdateRunSettings; + + public override string HelpContentResourceName => CommandLineResources.ResultsDirectoryArgumentHelp; + + public override HelpContentPriority HelpPriority => HelpContentPriority.ResultsDirectoryArgumentProcessorHelpPriority; + } + + /// + /// The argument executor. + /// + internal class ResultsDirectoryArgumentExecutor : IArgumentExecutor + { + #region Fields + + /// + /// Used for getting sources. + /// + private CommandLineOptions commandLineOptions; + + private IRunSettingsProvider runSettingsManager; + + public const string RunSettingsPath = "RunConfiguration.ResultsDirectory"; + + #endregion + + #region Constructor + + /// + /// Default constructor. + /// + /// The options. + /// The test platform + public ResultsDirectoryArgumentExecutor(CommandLineOptions options, IRunSettingsProvider runSettingsManager) + { + Contract.Requires(options != null); + Contract.Requires(runSettingsManager != null); + + this.commandLineOptions = options; + this.runSettingsManager = runSettingsManager; + } + + #endregion + + #region IArgumentExecutor + + /// + /// Initializes with the argument that was provided with the command. + /// + /// Argument that was provided with the command. + public void Initialize(string argument) + { + if (string.IsNullOrWhiteSpace(argument)) + { + throw new CommandLineException(CommandLineResources.ResultsDirectoryValueRequired); + } + + try + { + // to check valid directory path + var di = new DirectoryInfo(argument); + + if (!Path.IsPathRooted(argument)) + { + argument = Path.GetFullPath(argument); + } + } + catch (Exception ex) when( ex is NotSupportedException || ex is SecurityException || ex is ArgumentException || ex is PathTooLongException) + { + throw new CommandLineException(string.Format(CommandLineResources.InvalidResultsDirectoryPathCommand, argument, ex.Message)); + } + + this.commandLineOptions.ResultsDirectory = argument; + this.runSettingsManager.UpdateRunSettingsNode(ResultsDirectoryArgumentExecutor.RunSettingsPath, argument); + } + + /// + /// Executes the argument processor. + /// + /// The . + public ArgumentProcessorResult Execute() + { + // Nothing to do since we updated the parameter during initialize parameter + return ArgumentProcessorResult.Success; + } + + #endregion + } +} diff --git a/src/vstest.console/Processors/RunSettingsArgumentProcessor.cs b/src/vstest.console/Processors/RunSettingsArgumentProcessor.cs index 0c217e860a..4998fa4eff 100644 --- a/src/vstest.console/Processors/RunSettingsArgumentProcessor.cs +++ b/src/vstest.console/Processors/RunSettingsArgumentProcessor.cs @@ -17,6 +17,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Processors using Microsoft.VisualStudio.TestPlatform.Utilities; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; using CommandLineResources = Microsoft.VisualStudio.TestPlatform.CommandLine.Resources.Resources; @@ -119,18 +120,17 @@ public void Initialize(string argument) } Contract.EndContractBlock(); - + // Load up the run settings and set it as the active run settings. try { IXPathNavigable document = this.GetRunSettingsDocument(argument); - - var runSettings = new RunSettings(); + this.runSettingsManager.UpdateRunSettings(document.CreateNavigator().OuterXml); - // Currently do not see the need to load the settings providers in the console process. - runSettings.LoadSettingsXml(document.CreateNavigator().OuterXml); + //Add default runsettings values if not exists in given runsettings file. + this.runSettingsManager.AddDefaultRunSettings(); - this.runSettingsManager.SetActiveRunSettings(runSettings); + this.commandLineOptions.SettingsFile = argument; } catch (XmlException exception) { @@ -172,20 +172,7 @@ private IXPathNavigable GetRunSettingsDocument(string runSettingsFile) else { runSettingsDocument = XmlRunSettingsUtilities.CreateDefaultRunSettings(); - - FrameworkVersion frameworkVersion = FrameworkVersion.Framework45; - - if (this.commandLineOptions.FrameworkVersionSpecified && this.commandLineOptions.TargetFrameworkVersion != Framework.DefaultFramework) - { - IOutput output = ConsoleOutput.Instance; - output.Warning(CommandLineResources.TestSettingsFrameworkMismatch, this.commandLineOptions.TargetFrameworkVersion.ToString(), Framework.DefaultFramework.ToString()); - } - - var architecture = this.commandLineOptions.ArchitectureSpecified - ? this.commandLineOptions.TargetArchitecture - : Architecture.X86; - - runSettingsDocument = MSTestSettingsUtilities.Import(runSettingsFile, runSettingsDocument, architecture, frameworkVersion); + runSettingsDocument = MSTestSettingsUtilities.Import(runSettingsFile, runSettingsDocument, Architecture.X86, FrameworkVersion.Framework45); } if (this.commandLineOptions.EnableCodeCoverage == true) diff --git a/src/vstest.console/Processors/RunSpecificTestsArgumentProcessor.cs b/src/vstest.console/Processors/RunSpecificTestsArgumentProcessor.cs index 9c4b6df55c..da70550ac0 100644 --- a/src/vstest.console/Processors/RunSpecificTestsArgumentProcessor.cs +++ b/src/vstest.console/Processors/RunSpecificTestsArgumentProcessor.cs @@ -152,7 +152,7 @@ public RunSpecificTestsArgumentExecutor( this.commandLineOptions = options; this.testRequestManager = testRequestManager; - this.runSettingsManager = RunSettingsManager.Instance; + this.runSettingsManager = runSettingsProvider; this.output = ConsoleOutput.Instance; this.discoveryEventsRegistrar = new DiscoveryEventsRegistrar(this.discoveryRequest_OnDiscoveredTests); } @@ -190,6 +190,7 @@ public ArgumentProcessorResult Execute() Contract.Assert(this.output != null); Contract.Assert(this.commandLineOptions != null); Contract.Assert(this.testRequestManager != null); + Contract.Assert(!string.IsNullOrWhiteSpace(this.runSettingsManager.ActiveRunSettings.SettingsXml)); if (this.commandLineOptions.Sources.Count() <= 0) { @@ -203,7 +204,7 @@ public ArgumentProcessorResult Execute() bool result = false; - this.effectiveRunSettings = RunSettingsUtilities.GetRunSettings(this.runSettingsManager, this.commandLineOptions); + this.effectiveRunSettings = this.runSettingsManager.ActiveRunSettings.SettingsXml; // Discover tests from sources and filter on every discovery reported. result = this.DiscoverTestsAndSelectSpecified(this.commandLineOptions.Sources); diff --git a/src/vstest.console/Processors/RunTestsArgumentProcessor.cs b/src/vstest.console/Processors/RunTestsArgumentProcessor.cs index cffa9c0837..d71603f54b 100644 --- a/src/vstest.console/Processors/RunTestsArgumentProcessor.cs +++ b/src/vstest.console/Processors/RunTestsArgumentProcessor.cs @@ -128,7 +128,7 @@ public RunTestsArgumentExecutor( this.output = ConsoleOutput.Instance; - this.runSettingsManager = RunSettingsManager.Instance; + this.runSettingsManager = runSettingsProvider; this.testRequestManager = testRequestManager; this.testRunEventsRegistrar = new TestRunRequestEventsRegistrar(); } @@ -146,6 +146,7 @@ public void Initialize(string argument) public ArgumentProcessorResult Execute() { Contract.Assert(this.commandLineOptions != null); + Contract.Assert(!string.IsNullOrWhiteSpace(this.runSettingsManager?.ActiveRunSettings?.SettingsXml)); // Ensure a test source file was provided var anySource = this.commandLineOptions.Sources.FirstOrDefault(); @@ -173,7 +174,7 @@ private bool RunTests(IEnumerable sources) EqtTrace.Info("RunTestsArgumentProcessor:Execute: Test run is starting."); } - var runSettings = RunSettingsUtilities.GetRunSettings(this.runSettingsManager, this.commandLineOptions); + var runSettings = this.runSettingsManager.ActiveRunSettings.SettingsXml; if (EqtTrace.IsVerboseEnabled) { diff --git a/src/vstest.console/Processors/Utilities/ArgumentProcessorFactory.cs b/src/vstest.console/Processors/Utilities/ArgumentProcessorFactory.cs index e0683c555d..241ab9efbe 100644 --- a/src/vstest.console/Processors/Utilities/ArgumentProcessorFactory.cs +++ b/src/vstest.console/Processors/Utilities/ArgumentProcessorFactory.cs @@ -229,7 +229,8 @@ public IEnumerable GetArgumentProcessorsToAlwaysExecute() new EnableLoggerArgumentProcessor(), new ParallelArgumentProcessor(), new EnableDiagArgumentProcessor(), - new CLIRunSettingsArgumentProcessor() + new CLIRunSettingsArgumentProcessor(), + new ResultsDirectoryArgumentProcessor() }; /// diff --git a/src/vstest.console/Processors/Utilities/ArgumentProcessorPriority.cs b/src/vstest.console/Processors/Utilities/ArgumentProcessorPriority.cs index a551970e8d..6f9c4db1a4 100644 --- a/src/vstest.console/Processors/Utilities/ArgumentProcessorPriority.cs +++ b/src/vstest.console/Processors/Utilities/ArgumentProcessorPriority.cs @@ -33,14 +33,14 @@ internal enum ArgumentProcessorPriority TestAdapterPath = 1, /// - /// Priority of processors that needs to update runsettings. + /// Priority of processors related to Run Settings. /// - AutoUpdateRunSettings = 3, + RunSettings = 3, /// - /// Priority of processors related to Run Settings. + /// Priority of processors that needs to update runsettings. /// - RunSettings = 5, + AutoUpdateRunSettings = 5, /// /// Priority of processors related to CLI Run Settings. diff --git a/src/vstest.console/Processors/Utilities/HelpContentPriority.cs b/src/vstest.console/Processors/Utilities/HelpContentPriority.cs index 16b515bc95..61af351a0a 100644 --- a/src/vstest.console/Processors/Utilities/HelpContentPriority.cs +++ b/src/vstest.console/Processors/Utilities/HelpContentPriority.cs @@ -5,6 +5,35 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Processors { /// /// Attribute to be used for displaying the help content in required order. + /// + /// Order of settings display (based on lifecycle at https://blogs.msdn.microsoft.com/visualstudioalm/2016/07/25/evolving-the-visual-studio-test-platform-part-1/). + /// + /// Selection + /// --Tests + /// --TestCaseFilter + /// + /// Configure + /// --Framework + /// --Platform + /// --Settings + /// --CLI runsettings + /// + /// Run/Discover + /// --ListTests + /// --Parallel + /// --TestAdapterPath + /// + /// Diagnose/Report + /// --Diag + /// --Logger + /// --ResultsDirectory + /// + /// IDE Automation + /// --ParentProcessId + /// --Port + /// + /// Help + /// -–Help /// internal enum HelpContentPriority { @@ -18,50 +47,40 @@ internal enum HelpContentPriority /// RunTestsArgumentProcessorHelpPriority, - /// - /// RunSettingsArgumentProcessor Help - /// - RunSettingsArgumentProcessorHelpPriority, - /// /// RunSpecificTestsArgumentProcessor Help /// RunSpecificTestsArgumentProcessorHelpPriority, /// - /// EnableCodeCoverageArgumentProcessor Help - /// - EnableCodeCoverageArgumentProcessorHelpPriority, - - /// - /// InIsolationArgumentProcessor Help + /// TestCaseFilterArgumentProcessor Help /// - InIsolationArgumentProcessorHelpPriority, + TestCaseFilterArgumentProcessorHelpPriority, /// - /// DisableAutoFakesArgumentProcessor Help + /// FrameworkArgumentProcessor Help /// - DisableAutoFakesArgumentProcessorHelpPriority, + FrameworkArgumentProcessorHelpPriority, /// - /// UseVsixArgumentProcessor Help + /// PlatformArgumentProcessor Help /// - UseVsixArgumentProcessorHelpPriority, + PlatformArgumentProcessorHelpPriority, /// - /// TestAdapterPathArgumentProcessor Help + /// RunSettingsArgumentProcessor Help /// - TestAdapterPathArgumentProcessorHelpPriority, + RunSettingsArgumentProcessorHelpPriority, /// - /// PlatformArgumentProcessor Help + /// CLIRunSettingsArgumentProcessor Help /// - PlatformArgumentProcessorHelpPriority, + CLIRunSettingsArgumentProcessorHelpPriority, /// - /// FrameworkArgumentProcessor Help + /// ListTestsArgumentExecutor Help /// - FrameworkArgumentProcessorHelpPriority, + ListTestsArgumentProcessorHelpPriority, /// /// ParallelArgumentProcessor Help @@ -69,14 +88,14 @@ internal enum HelpContentPriority ParallelArgumentProcessorHelpPriority, /// - /// TestCaseFilterArgumentProcessor Help + /// TestAdapterPathArgumentProcessor Help /// - TestCaseFilterArgumentProcessorHelpPriority, + TestAdapterPathArgumentProcessorHelpPriority, /// - /// HelpArgumentExecutor + /// EnableDiagArgumentProcessor Help /// - HelpArgumentProcessorHelpPriority, + EnableDiagArgumentProcessorHelpPriority, /// /// EnableLoggerArgumentProcessor Help @@ -84,48 +103,63 @@ internal enum HelpContentPriority EnableLoggerArgumentProcessorHelpPriority, /// - /// ListTestsArgumentExecutor Help + /// ResultsDirectoryArgumentProcessor Help /// - ListTestsArgumentProcessorHelpPriority, + ResultsDirectoryArgumentProcessorHelpPriority, /// - /// ListDiscoverersArgumentProcessor Help + /// PortArgumentProcessor Help /// - ListDiscoverersArgumentProcessorHelpPriority, + ParentProcessIdArgumentProcessorHelpPriority, /// - /// ListExecutorsArgumentProcessor Help + /// PortArgumentProcessor Help /// - ListExecutorsArgumentProcessorHelpPriority, + PortArgumentProcessorHelpPriority, /// - /// ListLoggersArgumentProcessor Help + /// HelpArgumentExecutor /// - ListLoggersArgumentProcessorHelpPriority, + HelpArgumentProcessorHelpPriority, /// - /// ListSettingProviderArgumentProcessor Help + /// EnableCodeCoverageArgumentProcessor Help /// - ListSettingsProvidersArgumentProcessorHelpPriority, + EnableCodeCoverageArgumentProcessorHelpPriority, /// - /// PortArgumentProcessor Help + /// InIsolationArgumentProcessor Help /// - ParentProcessIdArgumentProcessorHelpPriority, + InIsolationArgumentProcessorHelpPriority, /// - /// PortArgumentProcessor Help + /// DisableAutoFakesArgumentProcessor Help /// - PortArgumentProcessorHelpPriority, + DisableAutoFakesArgumentProcessorHelpPriority, /// - /// EnableDiagArgumentProcessor Help + /// UseVsixArgumentProcessor Help /// - EnableDiagArgumentProcessorHelpPriority, + UseVsixArgumentProcessorHelpPriority, /// - /// CLIRunSettingsArgumentProcessor Help + /// ListDiscoverersArgumentProcessor Help + /// + ListDiscoverersArgumentProcessorHelpPriority, + + /// + /// ListExecutorsArgumentProcessor Help + /// + ListExecutorsArgumentProcessorHelpPriority, + + /// + /// ListLoggersArgumentProcessor Help + /// + ListLoggersArgumentProcessorHelpPriority, + + /// + /// ListSettingProviderArgumentProcessor Help /// - CLIRunSettingsArgumentProcessorHelpPriority + ListSettingsProvidersArgumentProcessorHelpPriority } } diff --git a/src/vstest.console/Processors/Utilities/RunSettingsProviderExtensions.cs b/src/vstest.console/Processors/Utilities/RunSettingsProviderExtensions.cs new file mode 100644 index 0000000000..3d89867df6 --- /dev/null +++ b/src/vstest.console/Processors/Utilities/RunSettingsProviderExtensions.cs @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities +{ + using System; + using System.Diagnostics.CodeAnalysis; + using System.IO; + using System.Xml; + using System.Diagnostics.Contracts; + + using Microsoft.VisualStudio.TestPlatform.Utilities; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; + using Microsoft.VisualStudio.TestPlatform.Common; + + /// + /// Utilities to get the run settings from the provider and the commandline options specified. + /// + internal static class RunSettingsProviderExtensions + { + public const string EmptyRunSettings = @""; + + public static void UpdateRunSettings(this IRunSettingsProvider runSettingsProvider, string runsettingsXml) + { + ValidateArg.NotNull(runSettingsProvider, nameof(runSettingsProvider)); + ValidateArg.NotNullOrWhiteSpace(runsettingsXml, nameof(runsettingsXml)); + + var runSettings = new RunSettings(); + runSettings.LoadSettingsXml(runsettingsXml); + runSettingsProvider.SetActiveRunSettings(runSettings); + } + + public static void AddDefaultRunSettings(this IRunSettingsProvider runSettingsProvider) + { + ValidateArg.NotNull(runSettingsProvider, nameof(runSettingsProvider)); + + var runSettingsXml = runSettingsProvider.ActiveRunSettings?.SettingsXml; + + if (string.IsNullOrWhiteSpace(runSettingsXml)) + { + runSettingsXml = EmptyRunSettings; + } + + runSettingsXml = AddDefaultRunSettings(runSettingsXml); + runSettingsProvider.UpdateRunSettings(runSettingsXml); + } + + public static void UpdateRunSettingsNode(this IRunSettingsProvider runSettingsProvider, string key, string data) + { + ValidateArg.NotNull(runSettingsProvider, nameof(runSettingsProvider)); + ValidateArg.NotNullOrWhiteSpace(key, nameof(key)); + ValidateArg.NotNull(data, nameof(data)); + + var xmlDocument = runSettingsProvider.GetRunSettingXmlDocument(); + RunSettingsProviderExtensions.UpdateRunSettingsXmlDocument(xmlDocument, key, data); + runSettingsProvider.UpdateRunSettings(xmlDocument.OuterXml); + } + + public static string QueryRunSettingsNode(this IRunSettingsProvider runSettingsProvider, string key) + { + ValidateArg.NotNull(runSettingsProvider, nameof(runSettingsProvider)); + ValidateArg.NotNullOrWhiteSpace(key, nameof(key)); + + var xmlDocument = runSettingsProvider.GetRunSettingXmlDocument(); + var node = GetXmlNode(xmlDocument, key); + return node?.InnerText; + } + + /// + /// Gets the effective run settings adding the default run settings if not already present. + /// + /// The run settings XML. + /// Effective run settings. + [SuppressMessage("Microsoft.Security.Xml", "CA3053:UseXmlSecureResolver", + Justification = "XmlDocument.XmlResolver is not available in core. Suppress until fxcop issue is fixed.")] + private static string AddDefaultRunSettings(string runSettings) + { + var architecture = Constants.DefaultPlatform; + var framework = Framework.DefaultFramework; + var defaultResultsDirectory = Path.Combine(Directory.GetCurrentDirectory(), Constants.ResultsDirectoryName); + + using (var stream = new StringReader(runSettings)) + using (var reader = XmlReader.Create(stream, XmlRunSettingsUtilities.ReaderSettings)) + { + var document = new XmlDocument(); + document.Load(reader); + + var navigator = document.CreateNavigator(); + + InferRunSettingsHelper.UpdateRunSettingsWithUserProvidedSwitches(navigator, architecture, framework, defaultResultsDirectory); + return navigator.OuterXml; + } + } + + private static XmlNode CreateNode(XmlDocument doc, string xPath) + { + var path = xPath.Split('.'); + XmlNode node = null; + XmlNode parent = doc.DocumentElement; + + for (var i = 0; i < path.Length; i++) + { + node = parent.SelectSingleNode(path[i]) ?? parent.AppendChild(doc.CreateElement(path[i])); + parent = node; + } + + return node; + } + + private static XmlNode GetXmlNode(XmlDocument xmlDocument, string key) + { + var xPath = key.Replace('.', '/'); + var node = xmlDocument.SelectSingleNode(string.Format("//RunSettings/{0}", xPath)); + return node; + } + + private static XmlDocument GetRunSettingXmlDocument(this IRunSettingsProvider runSettingsProvider) + { + ValidateArg.NotNull(runSettingsProvider, nameof(runSettingsProvider)); + + var doc = new XmlDocument(); + + if (runSettingsProvider.ActiveRunSettings != null && + !string.IsNullOrEmpty(runSettingsProvider.ActiveRunSettings.SettingsXml)) + { + var settingsXml = runSettingsProvider.ActiveRunSettings.SettingsXml; + +#if NET46 + using (var reader = XmlReader.Create(new StringReader(settingsXml), new XmlReaderSettings() { XmlResolver = null, CloseInput = true, DtdProcessing = DtdProcessing.Prohibit })) + { +#else + using ( + var reader = XmlReader.Create(new StringReader(settingsXml), + new XmlReaderSettings() {CloseInput = true, DtdProcessing = DtdProcessing.Prohibit})) + { +#endif + doc.Load(reader); + } + } + else + { +#if NET46 + doc = (XmlDocument) XmlRunSettingsUtilities.CreateDefaultRunSettings(); +#else + using ( + var reader = + XmlReader.Create( + new StringReader( + XmlRunSettingsUtilities.CreateDefaultRunSettings().CreateNavigator().OuterXml), + new XmlReaderSettings() {CloseInput = true, DtdProcessing = DtdProcessing.Prohibit})) + { + doc.Load(reader); + } +#endif + } + return doc; + } + + private static void UpdateRunSettingsXmlDocument(XmlDocument xmlDocument, string key, string data) + { + var node = GetXmlNode(xmlDocument, key) ?? RunSettingsProviderExtensions.CreateNode(xmlDocument, key); + node.InnerText = data; + } + } +} diff --git a/src/vstest.console/Processors/Utilities/RunSettingsUtilities.cs b/src/vstest.console/Processors/Utilities/RunSettingsUtilities.cs deleted file mode 100644 index 499c74eaa6..0000000000 --- a/src/vstest.console/Processors/Utilities/RunSettingsUtilities.cs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities -{ - using System.Diagnostics.CodeAnalysis; - using System.IO; - using System.Xml; - - using Microsoft.VisualStudio.TestPlatform.Utilities; - using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; - using Microsoft.VisualStudio.TestPlatform.ObjectModel; - using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; - - /// - /// Utilities to get the run settings from the provider and the commandline options specified. - /// - internal class RunSettingsUtilities - { - private const string EmptyRunSettings = @""; - - /// - /// Gets the run settings to be used for the session. - /// - /// The current provider of run settings. - /// The command line options specified. - /// - internal static string GetRunSettings(IRunSettingsProvider runSettingsProvider, CommandLineOptions commandlineOptions) - { - var runSettings = runSettingsProvider?.ActiveRunSettings?.SettingsXml; - - if (string.IsNullOrWhiteSpace(runSettings)) - { - runSettings = EmptyRunSettings; - } - - runSettings = GetEffectiveRunSettings(runSettings, commandlineOptions); - - return runSettings; - } - - /// - /// Gets the effective run settings adding the commandline options to the run settings if not already present. - /// - /// The run settings XML. - /// The command line options. - /// Effective run settings. - [SuppressMessage("Microsoft.Security.Xml", "CA3053:UseXmlSecureResolver", - Justification = "XmlDocument.XmlResolver is not available in core. Suppress until fxcop issue is fixed.")] - private static string GetEffectiveRunSettings(string runSettings, CommandLineOptions commandLineOptions) - { - var architecture = Constants.DefaultPlatform; - - if (commandLineOptions != null && commandLineOptions.ArchitectureSpecified) - { - architecture = commandLineOptions.TargetArchitecture; - } - - var framework = Framework.DefaultFramework; - - if (commandLineOptions != null && commandLineOptions.FrameworkVersionSpecified) - { - framework = commandLineOptions.TargetFrameworkVersion; - } - - var defaultResultsDirectory = Path.Combine(Directory.GetCurrentDirectory(), Constants.ResultsDirectoryName); - - using (var stream = new StringReader(runSettings)) - using (var reader = XmlReader.Create(stream, XmlRunSettingsUtilities.ReaderSettings)) - { - var document = new XmlDocument(); - document.Load(reader); - - var navigator = document.CreateNavigator(); - - InferRunSettingsHelper.UpdateRunSettingsWithUserProvidedSwitches(navigator, architecture, framework, defaultResultsDirectory); - - if (commandLineOptions != null && commandLineOptions.Parallel) - { - ParallelRunSettingsUtilities.UpdateRunSettingsWithParallelSettingIfNotConfigured(navigator); - } - - return navigator.OuterXml; - } - } - } -} diff --git a/src/vstest.console/Resources/Resources.Designer.cs b/src/vstest.console/Resources/Resources.Designer.cs index 3cbe0f1e62..0f55d62a8b 100644 --- a/src/vstest.console/Resources/Resources.Designer.cs +++ b/src/vstest.console/Resources/Resources.Designer.cs @@ -646,7 +646,18 @@ public static string InvalidPortArgument { return ResourceManager.GetString("InvalidPortArgument", resourceCulture); } } - + + /// + /// Looks up a localized string similar to The path '{0}' specified in the 'ResultsDirectory' is invalid. Error: {1}. + /// + public static string InvalidResultsDirectoryPathCommand + { + get + { + return ResourceManager.GetString("InvalidResultsDirectoryPathCommand", resourceCulture); + } + } + /// /// Looks up a localized string similar to The path '{0}' specified in the 'TestAdapterPath' is invalid. Error: {1}. /// @@ -1001,7 +1012,31 @@ public static string ProjectPathNotFound { return ResourceManager.GetString("ProjectPathNotFound", resourceCulture); } } - + + /// + /// Looks up a localized string similar to --ResultsDirectory|/ResultsDirectory + /// Test results directory will be created in specified path if not exists. + /// Example /ResultsDirectory:<pathToResultsDirectory>. + /// + internal static string ResultsDirectoryArgumentHelp + { + get + { + return ResourceManager.GetString("ResultsDirectoryArgumentHelp", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The /ResultsDirectory parameter requires a value, where the test results should be saved. Example: /ResultsDirectory:c:\MyTestResultsDirectory. + /// + public static string ResultsDirectoryValueRequired + { + get + { + return ResourceManager.GetString("ResultsDirectoryValueRequired", resourceCulture); + } + } + /// /// Looks up a localized string similar to --Settings|/Settings:<Settings File> /// Settings to use when running tests.. diff --git a/src/vstest.console/Resources/Resources.resx b/src/vstest.console/Resources/Resources.resx index f657d11e6d..7165b68a1c 100644 --- a/src/vstest.console/Resources/Resources.resx +++ b/src/vstest.console/Resources/Resources.resx @@ -623,4 +623,15 @@ One or more runsettings provided contain invalid token + + --ResultsDirectory|/ResultsDirectory + Test results directory will be created in specified path if not exists. + Example /ResultsDirectory:<pathToResultsDirectory> + + + The /ResultsDirectory parameter requires a value, where the test results should be saved. Example: /ResultsDirectory:c:\MyTestResultsDirectory + + + The path '{0}' specified in the 'ResultsDirectory' is invalid. Error: {1} + \ No newline at end of file diff --git a/src/vstest.console/vstest.console.csproj b/src/vstest.console/vstest.console.csproj index d1c31038a8..94fbee354d 100644 --- a/src/vstest.console/vstest.console.csproj +++ b/src/vstest.console/vstest.console.csproj @@ -1,4 +1,4 @@ - + ..\..\ diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index ef3837f583..477f15a088 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -74,5 +74,27 @@ protected string GetTargetFramworkForRunsettings() return targetFramework; } + + protected string GetTestHostProcessName(string targetPlatform) + { + var testHostProcessName = string.Empty; + if (this.IsDesktopTargetFramework()) + { + if (string.Equals(targetPlatform, "x86", StringComparison.OrdinalIgnoreCase)) + { + testHostProcessName = "testhost.x86"; + } + else + { + testHostProcessName = "testhost"; + } + } + else + { + testHostProcessName = "dotnet"; + } + + return testHostProcessName; + } } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs index 34ac55692e..e409a8825a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs @@ -28,7 +28,7 @@ public void TrxLoggerShouldProperlyOverwriteFile(string runnerFramework, string arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); this.InvokeVsTest(arguments); - var trxLogFilePath = Path.Combine(AppContext.BaseDirectory, "TestResults", trxFileName); + var trxLogFilePath = Path.Combine(Directory.GetCurrentDirectory(), "TestResults", trxFileName); Assert.IsTrue(IsValidXml(trxLogFilePath), "Invalid content in Trx log file"); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs new file mode 100644 index 0000000000..5a4a3c9de9 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.AcceptanceTests +{ + using System.IO; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class ResultsDirectoryTests : AcceptanceTestBase + { + [CustomDataTestMethod] + [NET46TargetFramework] + [NETCORETargetFramework] + public void TrxFileShouldBeCreatedInResultsDirectory(string runnerFramework, string targetFramework, string targetRuntime) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime); + var arguments = PrepareArguments(this.GetSampleTestAssembly(), this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue); + var trxFileName = "TestResults.trx"; + var resultsDir = Path.GetTempPath(); + var trxFilePath = Path.Combine(resultsDir, trxFileName); + arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); + arguments = string.Concat(arguments, $" /ResultsDirectory:{resultsDir}"); + + // Delete if already exists + File.Delete(trxFilePath); + + this.InvokeVsTest(arguments); + + Assert.IsTrue(File.Exists(trxFilePath), $"Expected Trx file: {trxFilePath} not created in results directory"); + } + + //Getting Current directory C:\Windows\system32 https://github.com/Microsoft/vstest/issues/311 + //Failed to create relative directory + [Ignore] + [CustomDataTestMethod] + [NET46TargetFramework] + [NETCORETargetFramework] + public void ResultsDirectoryRelativePathShouldWork(string runnerFramework, string targetFramework, string targetRuntime) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime); + var arguments = PrepareArguments(this.GetSampleTestAssembly(), this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue); + var trxFileName = "TestResults.trx"; + var relativeDirectory = @"relative\directory"; + var resultsDirectory = Path.Combine(Directory.GetCurrentDirectory(), relativeDirectory); + + var trxFilePath = Path.Combine(resultsDirectory , trxFileName); + arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); + arguments = string.Concat(arguments, $" /ResultsDirectory:{relativeDirectory}"); + + if (Directory.Exists(resultsDirectory)) + { + Directory.Delete(resultsDirectory); + } + + this.InvokeVsTest(arguments); + + Assert.IsTrue(File.Exists(trxFilePath), $"Expected Trx file: {trxFilePath} not created in results directory"); + } + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index 484cf7db8d..7497a3acc6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -15,115 +15,166 @@ namespace Microsoft.TestPlatform.AcceptanceTests [TestClass] public class RunsettingsTests : AcceptanceTestBase { + #region Runsettings precedence tests + /// + /// Command line run settings should have high precedence among settings file, cli runsettings and cli switches + /// [CustomDataTestMethod] [NET46TargetFramework] [NETCORETargetFramework] - public void RunTestExecutionWithRunSettingsWithoutParallelAndPlatformX86(string runnerFramework, string targetFramework, string targetRuntime) + public void CommandLineRunSettingsShouldWinAmongAllOptions(string runnerFramework, string targetFramework, string targetRuntime) { AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime); - string testhostProcessName; - int expectedProcessCreated; - if (this.IsDesktopTargetFramework()) - { - testhostProcessName = "testhost.x86"; - expectedProcessCreated = 1; - } - else - { - testhostProcessName = "dotnet"; - if (this.IsDesktopRunner()) - { - expectedProcessCreated = 2; - } - else - { - // includes launcher dotnet process - expectedProcessCreated = 3; - } - } + var targetPlatform = "x86"; + var testhostProcessName = this.GetTestHostProcessName(targetPlatform); + var expectedNumOfProcessCreated = GetExpectedNumOfProcessCreatedForWithoutParallel(); + + // passing parallel + var runConfigurationDictionary = new Dictionary + { + { "MaxCpuCount", "0" }, + { "TargetFrameworkVersion", this.GetTargetFramworkForRunsettings() }, + { "TestAdaptersPaths", this.GetTestAdapterPath() } + }; + // passing different platform + var additionalArgs = "/Platform:x64"; + + var runSettingsArgs = String.Join( + " ", + new string[] + { + "RunConfiguration.MaxCpuCount=1", + string.Concat("RunConfiguration.TargetPlatform=",targetPlatform), + string.Concat("RunConfiguration.TargetFrameworkVersion=" , this.GetTargetFramworkForRunsettings()), + string.Concat("RunConfiguration.TestAdaptersPaths=" , this.GetTestAdapterPath()) + }); + + this.RunTestWithRunSettings(runConfigurationDictionary, runSettingsArgs, additionalArgs, testhostProcessName, expectedNumOfProcessCreated); + } + + /// + /// Command line run settings should have high precedence btween cli runsettings and cli switches. + /// + [CustomDataTestMethod] + [NET46TargetFramework] + [NETCORETargetFramework] + public void CLIRunsettingsShouldWinBetweenCLISwitchesAndCLIRunsettings(string runnerFramework, string targetFramework, string targetRuntime) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime); + + var targetPlatform = "x86"; + var testhostProcessName = this.GetTestHostProcessName(targetPlatform); + var expectedNumOfProcessCreated = GetExpectedNumOfProcessCreatedForWithoutParallel(); + // Pass parallel + var additionalArgs = "/Parallel"; + + // Pass non parallel + var runSettingsArgs = String.Join( + " ", + new string[] + { + "RunConfiguration.MaxCpuCount=1", + string.Concat("RunConfiguration.TargetPlatform=",targetPlatform), + string.Concat("RunConfiguration.TargetFrameworkVersion=" , this.GetTargetFramworkForRunsettings()), + string.Concat("RunConfiguration.TestAdaptersPaths=" , this.GetTestAdapterPath()) + }); + + this.RunTestWithRunSettings(null, runSettingsArgs, additionalArgs, testhostProcessName, expectedNumOfProcessCreated); + } + + /// + /// Command line switches should have high precedence if runsetting file and commandline switch specified + /// + /// + /// + /// + [CustomDataTestMethod] + [NET46TargetFramework] + [NETCORETargetFramework] + public void CommandLineSwitchesShouldWinBetweenSettingsFileAndCommandLineSwitches(string runnerFramework, string targetFramework, string targetRuntime) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime); + + var targetPlatform = "x86"; + var testhostProcessName = this.GetTestHostProcessName(targetPlatform); + var expectedNumOfProcessCreated = GetExpectedNumOfProcessCreatedForWithoutParallel(); + + // passing different platform var runConfigurationDictionary = new Dictionary { { "MaxCpuCount", "1" }, - { "TargetPlatform", "x86" }, + { "TargetPlatform", "x64" }, { "TargetFrameworkVersion", this.GetTargetFramworkForRunsettings() }, { "TestAdaptersPaths", this.GetTestAdapterPath() } }; - this.RunTestWithRunSettings(runConfigurationDictionary, testhostProcessName, expectedProcessCreated); + var additionalArgs = "/Platform:x86"; + + this.RunTestWithRunSettings(runConfigurationDictionary, null, additionalArgs, testhostProcessName, expectedNumOfProcessCreated); } + #endregion + [CustomDataTestMethod] [NET46TargetFramework] [NETCORETargetFramework] - public void RunTestExecutionWithRunSettingsParamsAsArguments(string runnerFramework, string targetFramework, string targetRuntime) + public void RunSettingsWithoutParallelAndPlatformX86(string runnerFramework, string targetFramework, string targetRuntime) { AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime); - string testhostProcessName; - int expectedProcessCreated; - if (this.IsDesktopTargetFramework()) - { - testhostProcessName = "testhost.x86"; - expectedProcessCreated = 1; - } - else - { - testhostProcessName = "dotnet"; - if (this.IsDesktopRunner()) - { - expectedProcessCreated = 2; - } - else - { - // includes launcher dotnet process - expectedProcessCreated = 3; - } - } + var targetPlatform = "x86"; + var testhostProcessName = this.GetTestHostProcessName(targetPlatform); + var expectedNumOfProcessCreated = GetExpectedNumOfProcessCreatedForWithoutParallel(); + + var runConfigurationDictionary = new Dictionary + { + { "MaxCpuCount", "1" }, + { "TargetPlatform", targetPlatform }, + { "TargetFrameworkVersion", this.GetTargetFramworkForRunsettings() }, + { "TestAdaptersPaths", this.GetTestAdapterPath() } + }; + this.RunTestWithRunSettings(runConfigurationDictionary, null, null, testhostProcessName, expectedNumOfProcessCreated); + } + + [CustomDataTestMethod] + [NET46TargetFramework] + [NETCORETargetFramework] + public void RunSettingsParamsAsArguments(string runnerFramework, string targetFramework, string targetRuntime) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime); + + var targetPlatform = "x86"; + var testhostProcessName = this.GetTestHostProcessName(targetPlatform); + var expectedNumOfProcessCreated = GetExpectedNumOfProcessCreatedForWithoutParallel(); var runSettingsArgs = String.Join( " ", new string[] { - "RunConfiguration.MaxCpuCount=1", "RunConfiguration.TargetPlatform=x86", + "RunConfiguration.MaxCpuCount=1", + string.Concat("RunConfiguration.TargetPlatform=",targetPlatform), string.Concat("RunConfiguration.TargetFrameworkVersion=" , this.GetTargetFramworkForRunsettings()), string.Concat("RunConfiguration.TestAdaptersPaths=" , this.GetTestAdapterPath()) }); - this.RunTestWithRunSettingsAndRunSettingsParamsAsArguments(null, runSettingsArgs, testhostProcessName, expectedProcessCreated); + this.RunTestWithRunSettings(null, runSettingsArgs, null, testhostProcessName, expectedNumOfProcessCreated); } [CustomDataTestMethod] [NET46TargetFramework] [NETCORETargetFramework] - public void RunTestExecutionWithRunSettingsAndRunSettingsParamsAsArguments(string runnerFramework, string targetFramework, string targetRuntime) + public void RunSettingsAndRunSettingsParamsAsArguments(string runnerFramework, string targetFramework, string targetRuntime) { AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime); - string testhostProcessName; - int expectedProcessCreated; - if (this.IsDesktopTargetFramework()) - { - testhostProcessName = "testhost.x86"; - expectedProcessCreated = 1; - } - else - { - testhostProcessName = "dotnet"; - if (this.IsDesktopRunner()) - { - expectedProcessCreated = 2; - } - else - { - // includes launcher dotnet process - expectedProcessCreated = 3; - } - } + var targetPlatform = "x86"; + var testhostProcessName = this.GetTestHostProcessName(targetPlatform); + var expectedNumOfProcessCreated = GetExpectedNumOfProcessCreatedForWithoutParallel(); var runConfigurationDictionary = new Dictionary { { "MaxCpuCount", "2" }, - { "TargetPlatform", "x86" }, + { "TargetPlatform", targetPlatform }, { "TargetFrameworkVersion", this.GetTargetFramworkForRunsettings() }, { "TestAdaptersPaths", this.GetTestAdapterPath() } }; @@ -132,44 +183,38 @@ public void RunTestExecutionWithRunSettingsAndRunSettingsParamsAsArguments(strin " ", new string[] { - "RunConfiguration.MaxCpuCount=1", "RunConfiguration.TargetPlatform=x86", + "RunConfiguration.MaxCpuCount=1", + string.Concat("RunConfiguration.TargetPlatform=",targetPlatform), string.Concat("RunConfiguration.TargetFrameworkVersion=" , this.GetTargetFramworkForRunsettings()), string.Concat("RunConfiguration.TestAdaptersPaths=" , this.GetTestAdapterPath()) }); - this.RunTestWithRunSettingsAndRunSettingsParamsAsArguments(runConfigurationDictionary, runSettingsArgs, testhostProcessName, expectedProcessCreated); + this.RunTestWithRunSettings(runConfigurationDictionary, runSettingsArgs, null, testhostProcessName, expectedNumOfProcessCreated); } [CustomDataTestMethod] [NET46TargetFramework] [NETCORETargetFramework] - public void RunTestExecutionWithRunSettingsWithParallelAndPlatformX64(string runnerFramework, string targetFramework, string targetRuntime) + public void RunSettingsWithParallelAndPlatformX64(string runnerFramework, string targetFramework, string targetRuntime) { AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime); - string testhostProcessName; - int expectedProcessCreated = 2; - if (this.IsDesktopTargetFramework()) - { - testhostProcessName = "testhost"; - } - else + var targetPlatform = "x64"; + var testhostProcessName = this.GetTestHostProcessName(targetPlatform); + var expectedProcessCreated = 2; + if (!this.IsDesktopTargetFramework() && !this.IsDesktopRunner()) { - testhostProcessName = "dotnet"; - if (!this.IsDesktopRunner()) - { - expectedProcessCreated = 3; - } + expectedProcessCreated = 3; } var runConfigurationDictionary = new Dictionary { { "MaxCpuCount", "2" }, - { "TargetPlatform", "x64" }, + { "TargetPlatform", targetPlatform }, { "TargetFrameworkVersion", this.GetTargetFramworkForRunsettings()}, { "TestAdaptersPaths", this.GetTestAdapterPath() } }; - this.RunTestWithRunSettings(runConfigurationDictionary, testhostProcessName, expectedProcessCreated); + this.RunTestWithRunSettings(runConfigurationDictionary, null, null, testhostProcessName, expectedProcessCreated); } // Known issue https://github.com/Microsoft/vstest/issues/135 @@ -177,7 +222,7 @@ public void RunTestExecutionWithRunSettingsWithParallelAndPlatformX64(string run [CustomDataTestMethod] [NET46TargetFramework] [NETCORETargetFramework] - public void RunTestExecutionWithTestAdapterPathFromRunSettings(string runnerFramework, string targetFramework, string targetRuntime) + public void TestAdapterPathFromRunSettings(string runnerFramework, string targetFramework, string targetRuntime) { AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime); @@ -191,7 +236,6 @@ public void RunTestExecutionWithTestAdapterPathFromRunSettings(string runnerFram string.Empty, runsettingsFilePath, this.FrameworkArgValue); - this.InvokeVsTest(arguments); this.ValidateSummaryStatus(1, 1, 1); File.Delete(runsettingsFilePath); @@ -206,40 +250,13 @@ private string GetRunsettingsFilePath(Dictionary runConfiguratio return runsettingsPath; } - private void RunTestWithRunSettings( - Dictionary runConfigurationDictionary, - string testhostProcessName, - int expectedProcessCreated) + private void RunTestWithRunSettings(Dictionary runConfigurationDictionary, + string runSettingsArgs, string additionalArgs, string testhostProcessName, int expectedNumOfProcessCreated) { var assemblyPaths = this.BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); - var runsettingsPath = this.GetRunsettingsFilePath(runConfigurationDictionary); - var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), runsettingsPath, this.FrameworkArgValue); - var cts = new CancellationTokenSource(); - var numOfProcessCreatedTask = NumberOfProcessLaunchedUtility.NumberOfProcessCreated( - cts, - testhostProcessName); - - this.InvokeVsTest(arguments); - cts.Cancel(); - Assert.AreEqual( - expectedProcessCreated, - numOfProcessCreatedTask.Result, - $"Number of {testhostProcessName} process created, expected: {expectedProcessCreated} actual: {numOfProcessCreatedTask.Result} args: {arguments} runner path: {this.testEnvironment.GetConsoleRunnerPath()}"); - this.ValidateSummaryStatus(2, 2, 2); - File.Delete(runsettingsPath); - } - - private void RunTestWithRunSettingsAndRunSettingsParamsAsArguments( - Dictionary runConfigurationDictionary, - string runSettingsArgs, - string testhostProcessName, - int expectedProcessCreated) - { - var assemblyPaths = - this.BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); - string runsettingsPath = string.Empty; + var runsettingsPath = string.Empty; if (runConfigurationDictionary != null) { @@ -247,7 +264,17 @@ private void RunTestWithRunSettingsAndRunSettingsParamsAsArguments( } var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), runsettingsPath, this.FrameworkArgValue); - arguments = string.Concat(arguments, " -- ", runSettingsArgs); + + if (!string.IsNullOrWhiteSpace(runSettingsArgs)) + { + arguments = string.Concat(arguments, " -- ", runSettingsArgs); + } + + if (!string.IsNullOrWhiteSpace(additionalArgs)) + { + arguments = string.Concat(arguments, " ", additionalArgs); + } + var cts = new CancellationTokenSource(); var numOfProcessCreatedTask = NumberOfProcessLaunchedUtility.NumberOfProcessCreated( cts, @@ -256,11 +283,32 @@ private void RunTestWithRunSettingsAndRunSettingsParamsAsArguments( this.InvokeVsTest(arguments); cts.Cancel(); + // assert Assert.AreEqual( - expectedProcessCreated, + expectedNumOfProcessCreated, numOfProcessCreatedTask.Result, - $"Number of {testhostProcessName} process created, expected: {expectedProcessCreated} actual: {numOfProcessCreatedTask.Result} args: {arguments} runner path: {this.testEnvironment.GetConsoleRunnerPath()}"); + $"Number of {testhostProcessName} process created, expected: {expectedNumOfProcessCreated} actual: {numOfProcessCreatedTask.Result} args: {arguments} runner path: {this.testEnvironment.GetConsoleRunnerPath()}"); this.ValidateSummaryStatus(2, 2, 2); + + //cleanup + if (!string.IsNullOrWhiteSpace(runsettingsPath)) + { + File.Delete(runsettingsPath); + } + } + + private int GetExpectedNumOfProcessCreatedForWithoutParallel() + { + int expectedNumOfProcessCreated; + if (this.IsDesktopTargetFramework()) + { + expectedNumOfProcessCreated = 1; + } + else + { + expectedNumOfProcessCreated = this.IsDesktopRunner() ? 2 : 3; + } + return expectedNumOfProcessCreated; } } } diff --git a/test/Microsoft.TestPlatform.Utilities.UnitTests/InferRunSettingsHelperTests.cs b/test/Microsoft.TestPlatform.Utilities.UnitTests/InferRunSettingsHelperTests.cs index 81725f5d75..0ef6078d69 100644 --- a/test/Microsoft.TestPlatform.Utilities.UnitTests/InferRunSettingsHelperTests.cs +++ b/test/Microsoft.TestPlatform.Utilities.UnitTests/InferRunSettingsHelperTests.cs @@ -122,6 +122,21 @@ public void UpdateRunSettingsShouldNotUpdateFrameworkIfRunSettingsAlreadyHasIt() StringAssert.Contains(xml, ".NETFramework,Version=v4.0"); } + //TargetFrameworkMoniker + + [TestMethod] + public void UpdateRunSettingsShouldAllowTargetFrameworkMonikerValue() + { + + var settings = @".NETFramework,Version=v4.0"; + var navigator = this.GetNavigator(settings); + + InferRunSettingsHelper.UpdateRunSettingsWithUserProvidedSwitches(navigator, Architecture.X64, Framework.DefaultFramework, "temp"); + + var xml = navigator.OuterXml; + + StringAssert.Contains(xml, ".NETFramework,Version=v4.0"); + } [TestMethod] public void UpdateRunSettingsShouldNotUpdateResultsDirectoryIfRunSettingsAlreadyHasIt() diff --git a/test/vstest.console.UnitTests/ExecutorUnitTests.cs b/test/vstest.console.UnitTests/ExecutorUnitTests.cs index 45250cac62..3ea428153f 100644 --- a/test/vstest.console.UnitTests/ExecutorUnitTests.cs +++ b/test/vstest.console.UnitTests/ExecutorUnitTests.cs @@ -10,6 +10,9 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests using CoreUtilities.Tracing.Interfaces; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Microsoft.VisualStudio.TestPlatform.Common; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; using Moq; @@ -73,6 +76,20 @@ public void ExecutorEmptyArgsCallRunTestsProcessor() // "First Printed message must be Microsoft Copyright"); } + /// + /// Executor should set default runsettings value even there is no processor + /// + [TestMethod] + public void ExecuteShouldInitializeDefaultRunsettings() + { + var mockOutput = new MockOutput(); + var exitCode = new Executor(mockOutput, this.mockTestPlatformEventSource.Object).Execute(null); + RunConfiguration runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(RunSettingsManager.Instance.ActiveRunSettings.SettingsXml); + Assert.AreEqual(runConfiguration.ResultsDirectory, Constants.DefaultResultsDirectory); + Assert.AreEqual(runConfiguration.TargetFrameworkVersion.ToString(), Framework.DefaultFramework.ToString()); + Assert.AreEqual(runConfiguration.TargetPlatform, Constants.DefaultPlatform); + } + [TestMethod] public void ExecuteShouldInstrumentVsTestConsoleStart() { diff --git a/test/vstest.console.UnitTests/Processors/CLIRunSettingsArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/CLIRunSettingsArgumentProcessorTests.cs index 1896d368ac..329ca5b5c6 100644 --- a/test/vstest.console.UnitTests/Processors/CLIRunSettingsArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/CLIRunSettingsArgumentProcessorTests.cs @@ -12,17 +12,24 @@ namespace vstest.console.UnitTests.Processors using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; using Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests; using Microsoft.VisualStudio.TestPlatform.Common; - using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; - using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] public class CLIRunSettingsArgumentProcessorTests { + private TestableRunSettingsProvider settingsProvider; + private CLIRunSettingsArgumentExecutor executor; private const string DefaultRunSettings = "\r\n\r\n \r\n \r\n \r\n"; private const string RunSettingsWithDeploymentDisabled = "\r\n\r\n \r\n \r\n \r\n \r\n False\r\n \r\n"; private const string RunSettingsWithDeploymentEnabled = "\r\n\r\n \r\n \r\n \r\n \r\n True\r\n \r\n"; + [TestInitialize] + public void Init() + { + this.settingsProvider = new TestableRunSettingsProvider(); + this.executor = new CLIRunSettingsArgumentExecutor(this.settingsProvider); + } + [TestMethod] public void GetMetadataShouldReturnRunSettingsArgumentProcessorCapabilities() { @@ -63,45 +70,35 @@ public void CapabilitiesShouldReturnAppropriateProperties() [TestMethod] public void InitializeShouldNotThrowExceptionIfArgumentIsNull() { - var settingsProvider = new TestableRunSettingsProvider(); - var executor = new CLIRunSettingsArgumentExecutor(null); + this.executor.Initialize((string[])null); - executor.Initialize((string[])null); - - Assert.IsNull(settingsProvider.ActiveRunSettings); + Assert.IsNull(this.settingsProvider.ActiveRunSettings); } [TestMethod] public void InitializeShouldNotThrowExceptionIfArgumentIsEmpty() { - var settingsProvider = new TestableRunSettingsProvider(); - var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); - - executor.Initialize(new string[0]); + this.executor.Initialize(new string[0]); - Assert.IsNull(settingsProvider.ActiveRunSettings); + Assert.IsNull(this.settingsProvider.ActiveRunSettings); } [TestMethod] - public void InitializeShouldCreateDefaultRunSettingsIfArgumentsHasOnlyWhiteSpace() + public void InitializeShouldCreateEmptyRunSettingsIfArgumentsHasOnlyWhiteSpace() { - var settingsProvider = new TestableRunSettingsProvider(); - var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); + this.executor.Initialize(new string[] { " " }); - executor.Initialize(new string[] { " " }); - - Assert.AreEqual(DefaultRunSettings, settingsProvider.ActiveRunSettings.SettingsXml); + Assert.IsNull(this.settingsProvider.ActiveRunSettings); } [TestMethod] public void InitializeShouldSetValueInRunSettings() { var args = new string[] { "MSTest.DeploymentEnabled=False" }; - var settingsProvider = new TestableRunSettingsProvider(); - var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); - executor.Initialize(args); - Assert.IsNotNull(settingsProvider.ActiveRunSettings); + this.executor.Initialize(args); + + Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); Assert.AreEqual(RunSettingsWithDeploymentDisabled, settingsProvider.ActiveRunSettings.SettingsXml); } @@ -109,12 +106,10 @@ public void InitializeShouldSetValueInRunSettings() public void InitializeShouldIgnoreKeyIfValueIsNotPassed() { var args = new string[] { "MSTest.DeploymentEnabled=False", "MSTest1" }; - var settingsProvider = new TestableRunSettingsProvider(); - var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); - executor.Initialize(args); + this.executor.Initialize(args); - Assert.IsNotNull(settingsProvider.ActiveRunSettings); + Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); Assert.AreEqual(RunSettingsWithDeploymentDisabled, settingsProvider.ActiveRunSettings.SettingsXml); } @@ -122,12 +117,10 @@ public void InitializeShouldIgnoreKeyIfValueIsNotPassed() public void InitializeShouldIgnoreWhiteSpaceInBeginningOrEndOfKey() { var args = new string[] { " MSTest.DeploymentEnabled =False" }; - var settingsProvider = new TestableRunSettingsProvider(); - var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); - executor.Initialize(args); + this.executor.Initialize(args); - Assert.IsNotNull(settingsProvider.ActiveRunSettings); + Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); Assert.AreEqual(RunSettingsWithDeploymentDisabled, settingsProvider.ActiveRunSettings.SettingsXml); } @@ -135,10 +128,8 @@ public void InitializeShouldIgnoreWhiteSpaceInBeginningOrEndOfKey() public void InitializeShouldIgnoreThrowExceptionIfKeyHasWhiteSpace() { var args = new string[] { "MST est.DeploymentEnabled=False" }; - var settingsProvider = new TestableRunSettingsProvider(); - var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); - Action action = () => executor.Initialize(args); + Action action = () => this.executor.Initialize(args); ExceptionUtilities.ThrowsException( action, @@ -149,12 +140,10 @@ public void InitializeShouldIgnoreThrowExceptionIfKeyHasWhiteSpace() public void InitializeShouldEncodeXMLIfInvalidXMLCharsArePresent() { var args = new string[] { "MSTest.DeploymentEnabled=F>a>\r\n\r\n \r\n \r\n \r\n \r\n F>a><l<se\r\n \r\n", settingsProvider.ActiveRunSettings.SettingsXml); } @@ -162,44 +151,40 @@ public void InitializeShouldEncodeXMLIfInvalidXMLCharsArePresent() public void InitializeShouldIgnoreIfKeyIsNotPassed() { var args = new string[] { "MSTest.DeploymentEnabled=False", "=value" }; - var settingsProvider = new TestableRunSettingsProvider(); - var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); - executor.Initialize(args); + this.executor.Initialize(args); - Assert.IsNotNull(settingsProvider.ActiveRunSettings); + Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); Assert.AreEqual(RunSettingsWithDeploymentDisabled, settingsProvider.ActiveRunSettings.SettingsXml); } [TestMethod] public void InitializeShouldIgnoreIfEmptyValueIsPassed() { - var settingsProvider = new TestableRunSettingsProvider(); + var runSettings = new RunSettings(); runSettings.LoadSettingsXml(DefaultRunSettings); - settingsProvider.SetActiveRunSettings(runSettings); + this.settingsProvider.SetActiveRunSettings(runSettings); var args = new string[] { "MSTest.DeploymentEnabled=" }; - var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); - executor.Initialize(args); + this.executor.Initialize(args); - Assert.IsNotNull(settingsProvider.ActiveRunSettings); - Assert.AreEqual(DefaultRunSettings, settingsProvider.ActiveRunSettings.SettingsXml); + Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); + Assert.AreEqual(DefaultRunSettings, this.settingsProvider.ActiveRunSettings.SettingsXml); } [TestMethod] public void InitializeShouldOverwriteValueIfNodeAlreadyExists() { - var settingsProvider = new TestableRunSettingsProvider(); + var runSettings = new RunSettings(); runSettings.LoadSettingsXml(DefaultRunSettings); settingsProvider.SetActiveRunSettings(runSettings); var args = new string[] { "MSTest.DeploymentEnabled=True" }; - var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); - executor.Initialize(args); + this.executor.Initialize(args); - Assert.IsNotNull(settingsProvider.ActiveRunSettings); + Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); Assert.AreEqual(RunSettingsWithDeploymentEnabled, settingsProvider.ActiveRunSettings.SettingsXml); } @@ -207,37 +192,18 @@ public void InitializeShouldOverwriteValueIfNodeAlreadyExists() [TestMethod] public void InitializeShouldOverwriteValueIfWhitSpaceIsPassedAndNodeAlreadyExists() { - var settingsProvider = new TestableRunSettingsProvider(); + var runSettings = new RunSettings(); runSettings.LoadSettingsXml(DefaultRunSettings); settingsProvider.SetActiveRunSettings(runSettings); var args = new string[] { "MSTest.DeploymentEnabled= " }; - var executor = new CLIRunSettingsArgumentExecutor(settingsProvider); - executor.Initialize(args); + this.executor.Initialize(args); - Assert.IsNotNull(settingsProvider.ActiveRunSettings); + Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); Assert.AreEqual("\r\n\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n", settingsProvider.ActiveRunSettings.SettingsXml); } #endregion - - #region private - - private class TestableRunSettingsProvider : IRunSettingsProvider - { - public RunSettings ActiveRunSettings - { - get; - set; - } - - public void SetActiveRunSettings(RunSettings runSettings) - { - this.ActiveRunSettings = runSettings; - } - } - - #endregion } } diff --git a/test/vstest.console.UnitTests/Processors/FrameworkArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/FrameworkArgumentProcessorTests.cs index c495f7e215..82560c891f 100644 --- a/test/vstest.console.UnitTests/Processors/FrameworkArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/FrameworkArgumentProcessorTests.cs @@ -5,10 +5,22 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors { using Microsoft.VisualStudio.TestTools.UnitTesting; using TestPlatform.CommandLine.Processors; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; + using vstest.console.UnitTests.Processors; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; [TestClass] public class FrameworkArgumentProcessorTests { + private FrameworkArgumentExecutor executor; + private TestableRunSettingsProvider runSettingsProvider; + + [TestInitialize] + public void Init() + { + this.runSettingsProvider = new TestableRunSettingsProvider(); + this.executor = new FrameworkArgumentExecutor(CommandLineOptions.Instance, runSettingsProvider); + } [TestCleanup] public void TestCleanup() { @@ -23,7 +35,7 @@ public void GetMetadataShouldReturnFrameworkArgumentProcessorCapabilities() } [TestMethod] - public void GetExecuterShouldReturnFrameworkArgumentProcessorCapabilities() + public void GetExecuterShouldReturnFrameworkArgumentExecutor() { var processor = new FrameworkArgumentProcessor(); Assert.IsTrue(processor.Executor.Value is FrameworkArgumentExecutor); @@ -54,18 +66,15 @@ public void CapabilitiesShouldReturnAppropriateProperties() [TestMethod] public void InitializeShouldThrowIfArgumentIsNull() { - var executor = new FrameworkArgumentExecutor(CommandLineOptions.Instance); ExceptionUtilities.ThrowsException( - () => executor.Initialize(null), + () => this.executor.Initialize(null), "The /Framework argument requires the target .Net Framework version for the test run. Example: /Framework:\".NETFramework,Version=v4.6\""); } [TestMethod] public void InitializeShouldThrowIfArgumentIsEmpty() { - var executor = new FrameworkArgumentExecutor(CommandLineOptions.Instance); - ExceptionUtilities.ThrowsException( () => executor.Initialize(" "), "The /Framework argument requires the target .Net Framework version for the test run. Example: /Framework:\".NETFramework,Version=v4.6\""); @@ -74,39 +83,44 @@ public void InitializeShouldThrowIfArgumentIsEmpty() [TestMethod] public void InitializeShouldThrowIfArgumentIsInvalid() { - var executor = new FrameworkArgumentExecutor(CommandLineOptions.Instance); - ExceptionUtilities.ThrowsException( - () => executor.Initialize("foo"), + () => this.executor.Initialize("foo"), "Invalid .Net Framework version:{0}. Please give the fullname of the TargetFramework. Other supported .Net Framework versions are Framework35, Framework40 and Framework45.", "foo"); } - + [TestMethod] - public void InitializeShouldSetCommandLineOptionsFramework() + public void InitializeShouldSetCommandLineOptionsAndRunSettingsFramework() { - var executor = new FrameworkArgumentExecutor(CommandLineOptions.Instance); - - executor.Initialize(".NETCoreApp,Version=v1.0"); + this.executor.Initialize(".NETCoreApp,Version=v1.0"); Assert.AreEqual(".NETCoreApp,Version=v1.0", CommandLineOptions.Instance.TargetFrameworkVersion.Name); + Assert.AreEqual(".NETCoreApp,Version=v1.0", this.runSettingsProvider.QueryRunSettingsNode(FrameworkArgumentExecutor.RunSettingsPath)); } [TestMethod] public void InitializeShouldSetCommandLineOptionsFrameworkForOlderFrameworks() { - var executor = new FrameworkArgumentExecutor(CommandLineOptions.Instance); - - executor.Initialize("Framework35"); + this.executor.Initialize("Framework35"); Assert.AreEqual(".NETFramework,Version=v3.5", CommandLineOptions.Instance.TargetFrameworkVersion.Name); + Assert.AreEqual(".NETFramework,Version=v3.5", this.runSettingsProvider.QueryRunSettingsNode(FrameworkArgumentExecutor.RunSettingsPath)); } [TestMethod] public void InitializeShouldSetCommandLineOptionsFrameworkForCaseInsensitiveFramework() { - var executor = new FrameworkArgumentExecutor(CommandLineOptions.Instance); - - executor.Initialize(".netcoreApp,Version=v1.0"); + this.executor.Initialize(".netcoreApp,Version=v1.0"); Assert.AreEqual(".netcoreApp,Version=v1.0", CommandLineOptions.Instance.TargetFrameworkVersion.Name); + Assert.AreEqual(".netcoreApp,Version=v1.0", this.runSettingsProvider.QueryRunSettingsNode(FrameworkArgumentExecutor.RunSettingsPath)); + } + + [TestMethod] + public void InitializeShouldNotSetFrameworkIfSettingsFileIsLegacy() + { + this.runSettingsProvider.UpdateRunSettingsNode(FrameworkArgumentExecutor.RunSettingsPath, FrameworkVersion.Framework45.ToString()); + CommandLineOptions.Instance.SettingsFile = @"c:\tmp\settings.testsettings"; + this.executor.Initialize(".NETFramework,Version=v3.5"); + Assert.AreEqual(".NETFramework,Version=v3.5", CommandLineOptions.Instance.TargetFrameworkVersion.Name); + Assert.AreEqual(FrameworkVersion.Framework45.ToString(), this.runSettingsProvider.QueryRunSettingsNode(FrameworkArgumentExecutor.RunSettingsPath)); } #endregion @@ -116,11 +130,10 @@ public void InitializeShouldSetCommandLineOptionsFrameworkForCaseInsensitiveFram [TestMethod] public void ExecuteShouldReturnSuccess() { - var executor = new FrameworkArgumentExecutor(CommandLineOptions.Instance); - - Assert.AreEqual(ArgumentProcessorResult.Success, executor.Execute()); + Assert.AreEqual(ArgumentProcessorResult.Success, this.executor.Execute()); } #endregion + } } diff --git a/test/vstest.console.UnitTests/Processors/ListTestsArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/ListTestsArgumentProcessorTests.cs index 03ee3aaf06..59c1c26851 100644 --- a/test/vstest.console.UnitTests/Processors/ListTestsArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/ListTestsArgumentProcessorTests.cs @@ -10,6 +10,11 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors using Microsoft.VisualStudio.TestPlatform.Client; using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; + using Microsoft.VisualStudio.TestPlatform.Common; + using Microsoft.VisualStudio.TestPlatform.Client.RequestHelper; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; + using vstest.console.UnitTests.Processors; using Moq; using ObjectModel; @@ -33,6 +38,25 @@ public class ListTestsArgumentProcessorTests private readonly Mock mockTestPlatformEventSource; + private static ListTestsArgumentExecutor GetExecutor(ITestRequestManager testRequestManager, IOutput output) + { + var runSettingsProvider = new TestableRunSettingsProvider(); + runSettingsProvider.AddDefaultRunSettings(); + var listTestsArgumentExecutor = + new ListTestsArgumentExecutor( + CommandLineOptions.Instance, + runSettingsProvider, + testRequestManager, + output ?? ConsoleOutput.Instance); + return listTestsArgumentExecutor; + } + + [TestCleanup] + public void Cleanup() + { + CommandLineOptions.Instance.Reset(); + } + public ListTestsArgumentProcessorTests() { this.mockFileHelper = new Mock(); @@ -46,7 +70,7 @@ public ListTestsArgumentProcessorTests() [TestMethod] public void GetMetadataShouldReturnListTestsArgumentProcessorCapabilities() { - ListTestsArgumentProcessor processor = new ListTestsArgumentProcessor(); + var processor = new ListTestsArgumentProcessor(); Assert.IsTrue(processor.Metadata.Value is ListTestsArgumentProcessorCapabilities); } @@ -56,7 +80,7 @@ public void GetMetadataShouldReturnListTestsArgumentProcessorCapabilities() [TestMethod] public void GetExecuterShouldReturnListTestsArgumentProcessorCapabilities() { - ListTestsArgumentProcessor processor = new ListTestsArgumentProcessor(); + var processor = new ListTestsArgumentProcessor(); Assert.IsTrue(processor.Executor.Value is ListTestsArgumentExecutor); } @@ -65,7 +89,7 @@ public void GetExecuterShouldReturnListTestsArgumentProcessorCapabilities() [TestMethod] public void CapabilitiesShouldReturnAppropriateProperties() { - ListTestsArgumentProcessorCapabilities capabilities = new ListTestsArgumentProcessorCapabilities(); + var capabilities = new ListTestsArgumentProcessorCapabilities(); Assert.AreEqual("/ListTests", capabilities.CommandName); Assert.AreEqual("/lt", capabilities.ShortCommandName); Assert.AreEqual("-lt|--ListTests|/lt|/ListTests:\n Lists discovered tests from the given test container.", capabilities.HelpContentResourceName); @@ -85,15 +109,10 @@ public void CapabilitiesShouldReturnAppropriateProperties() [TestMethod] public void ExecutorInitializeWithValidSourceShouldAddItToTestSources() { - CommandLineOptions.Instance.Reset(); - CommandLineOptions.Instance.FileHelper = this.mockFileHelper.Object; var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, new TestPlatform(), TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - ListTestsArgumentExecutor executor = new ListTestsArgumentExecutor( - CommandLineOptions.Instance, - null, - testRequestManager); + var executor = GetExecutor(testRequestManager, null); executor.Initialize(this.dummyTestFilePath); @@ -106,10 +125,7 @@ public void ExecutorExecuteForNoSourcesShouldReturnFail() CommandLineOptions.Instance.Reset(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, new TestPlatform(), TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - ListTestsArgumentExecutor executor = new ListTestsArgumentExecutor( - CommandLineOptions.Instance, - null, - testRequestManager); + var executor = GetExecutor(testRequestManager, null); Assert.ThrowsException(() => executor.Execute()); } @@ -124,15 +140,11 @@ public void ExecutorExecuteShouldCatchTestPlatformExceptionAndReturnFail() mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny())).Returns(mockDiscoveryRequest.Object); this.ResetAndAddSourceToCommandLineOptions(); - + var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - ListTestsArgumentExecutor listTestsArgumentExecutor = - new ListTestsArgumentExecutor( - CommandLineOptions.Instance, - null, - testRequestManager); + var executor = GetExecutor(testRequestManager, null); - ArgumentProcessorResult argumentProcessorResult = listTestsArgumentExecutor.Execute(); + var argumentProcessorResult = executor.Execute(); Assert.AreEqual(ArgumentProcessorResult.Fail, argumentProcessorResult); } @@ -148,13 +160,9 @@ public void ExecutorExecuteShouldCatchSettingsExceptionAndReturnFail() this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - ListTestsArgumentExecutor listTestsArgumentExecutor = - new ListTestsArgumentExecutor( - CommandLineOptions.Instance, - null, - testRequestManager); + var listTestsArgumentExecutor = GetExecutor(testRequestManager, null); - ArgumentProcessorResult argumentProcessorResult = listTestsArgumentExecutor.Execute(); + var argumentProcessorResult = listTestsArgumentExecutor.Execute(); Assert.AreEqual(ArgumentProcessorResult.Fail, argumentProcessorResult); } @@ -170,13 +178,9 @@ public void ExecutorExecuteShouldCatchInvalidOperationExceptionAndReturnFail() this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - ListTestsArgumentExecutor listTestsArgumentExecutor = - new ListTestsArgumentExecutor( - CommandLineOptions.Instance, - null, - testRequestManager); + var listTestsArgumentExecutor = GetExecutor(testRequestManager, null); - ArgumentProcessorResult argumentProcessorResult = listTestsArgumentExecutor.Execute(); + var argumentProcessorResult = listTestsArgumentExecutor.Execute(); Assert.AreEqual(ArgumentProcessorResult.Fail, argumentProcessorResult); } @@ -188,14 +192,11 @@ public void ExecutorExecuteShouldThrowOtherExceptions() mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Throws(new Exception("DummyException")); mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny())).Returns(mockDiscoveryRequest.Object); - + this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new ListTestsArgumentExecutor( - CommandLineOptions.Instance, - null, - testRequestManager); + var executor = GetExecutor(testRequestManager, null); Assert.ThrowsException(() => executor.Execute()); } @@ -203,7 +204,6 @@ public void ExecutorExecuteShouldThrowOtherExceptions() [TestMethod] public void ExecutorExecuteShouldOutputDiscoveredTestsAndReturnSuccess() { - var mockDiscoveryRequest = new Mock(); var mockConsoleOutput = new Mock(); @@ -211,7 +211,7 @@ public void ExecutorExecuteShouldOutputDiscoveredTestsAndReturnSuccess() // Assert mockDiscoveryRequest.Verify(dr => dr.DiscoverAsync(), Times.Once); - + mockConsoleOutput.Verify((IOutput co) => co.WriteLine(" Test1", OutputLevel.Information)); mockConsoleOutput.Verify((IOutput co) => co.WriteLine(" Test2", OutputLevel.Information)); } @@ -242,7 +242,7 @@ public void ListTestArgumentProcessorExecuteShouldInstrumentDiscoveryRequestStop private void RunListTestArgumentProcessorExecuteWithMockSetup(Mock mockDiscoveryRequest, Mock mockConsoleOutput) { var mockTestPlatform = new Mock(); - List list = new List(); + var list = new List(); list.Add(new TestCase("Test1", new Uri("http://FooTestUri1"), "Source1")); list.Add(new TestCase("Test2", new Uri("http://FooTestUri2"), "Source2")); mockDiscoveryRequest.Setup(dr => dr.DiscoverAsync()).Raises(dr => dr.OnDiscoveredTests += null, new DiscoveredTestsEventArgs(list)); @@ -253,7 +253,7 @@ private void RunListTestArgumentProcessorExecuteWithMockSetup(Mock( - () => executor.Initialize("123"), + () => this.executor.Initialize("123"), "Argument " + 123 + " is not expected in the 'Parallel' command. Specify the command without the argument (Example: vstest.console.exe myTests.dll /Parallel) and try again."); } [TestMethod] public void InitializeShouldSetParallelValue() { - var executor = new ParallelArgumentExecutor(CommandLineOptions.Instance); - executor.Initialize(null); + this.executor.Initialize(null); Assert.IsTrue(CommandLineOptions.Instance.Parallel, "Parallel option must be set to true."); + Assert.AreEqual("0", this.runSettingsProvider.QueryRunSettingsNode(ParallelArgumentExecutor.RunSettingsPath)); } #endregion @@ -77,9 +87,7 @@ public void InitializeShouldSetParallelValue() [TestMethod] public void ExecuteShouldReturnSuccess() { - var executor = new ParallelArgumentExecutor(CommandLineOptions.Instance); - - Assert.AreEqual(ArgumentProcessorResult.Success, executor.Execute()); + Assert.AreEqual(ArgumentProcessorResult.Success, this.executor.Execute()); } #endregion diff --git a/test/vstest.console.UnitTests/Processors/PlatformArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/PlatformArgumentProcessorTests.cs index a38d159680..1309ca5aba 100644 --- a/test/vstest.console.UnitTests/Processors/PlatformArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/PlatformArgumentProcessorTests.cs @@ -3,12 +3,25 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors { + using System; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestPlatform.CommandLine.Processors; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; + using vstest.console.UnitTests.Processors; [TestClass] public class PlatformArgumentProcessorTests { + private PlatformArgumentExecutor executor; + private TestableRunSettingsProvider runSettingsProvider; + + [TestInitialize] + public void Init() + { + this.runSettingsProvider = new TestableRunSettingsProvider(); + this.executor = new PlatformArgumentExecutor(CommandLineOptions.Instance, runSettingsProvider); + } + [TestCleanup] public void TestCleanup() { @@ -23,7 +36,7 @@ public void GetMetadataShouldReturnPlatformArgumentProcessorCapabilities() } [TestMethod] - public void GetExecuterShouldReturnPlatformArgumentProcessorCapabilities() + public void GetExecuterShouldReturnPlatformArgumentExecutor() { var processor = new PlatformArgumentProcessor(); Assert.IsTrue(processor.Executor.Value is PlatformArgumentExecutor); @@ -54,30 +67,24 @@ public void CapabilitiesShouldReturnAppropriateProperties() [TestMethod] public void InitializeShouldThrowIfArgumentIsNull() { - var executor = new PlatformArgumentExecutor(CommandLineOptions.Instance); - ExceptionUtilities.ThrowsException( - () => executor.Initialize(null), + () => this.executor.Initialize(null), "The /Platform argument requires the target platform type for the test run to be provided. Example: /Platform:x86"); } [TestMethod] public void InitializeShouldThrowIfArgumentIsEmpty() { - var executor = new PlatformArgumentExecutor(CommandLineOptions.Instance); - ExceptionUtilities.ThrowsException( - () => executor.Initialize(" "), + () => this.executor.Initialize(" "), "The /Platform argument requires the target platform type for the test run to be provided. Example: /Platform:x86"); } [TestMethod] public void InitializeShouldThrowIfArgumentIsNotAnArchitecture() { - var executor = new PlatformArgumentExecutor(CommandLineOptions.Instance); - ExceptionUtilities.ThrowsException( - () => executor.Initialize("foo"), + () => this.executor.Initialize("foo"), "Invalid platform type:{0}. Valid platform types are x86, x64 and Arm.", "foo"); } @@ -85,10 +92,8 @@ public void InitializeShouldThrowIfArgumentIsNotAnArchitecture() [TestMethod] public void InitializeShouldThrowIfArgumentIsNotASupportedArchitecture() { - var executor = new PlatformArgumentExecutor(CommandLineOptions.Instance); - ExceptionUtilities.ThrowsException( - () => executor.Initialize("AnyCPU"), + () => this.executor.Initialize("AnyCPU"), "Invalid platform type:{0}. Valid platform types are x86, x64 and Arm.", "AnyCPU"); } @@ -96,19 +101,17 @@ public void InitializeShouldThrowIfArgumentIsNotASupportedArchitecture() [TestMethod] public void InitializeShouldSetCommandLineOptionsArchitecture() { - var executor = new PlatformArgumentExecutor(CommandLineOptions.Instance); - - executor.Initialize("x64"); + this.executor.Initialize("x64"); Assert.AreEqual(ObjectModel.Architecture.X64, CommandLineOptions.Instance.TargetArchitecture); + Assert.AreEqual(ObjectModel.Architecture.X64.ToString(), this.runSettingsProvider.QueryRunSettingsNode(PlatformArgumentExecutor.RunSettingsPath)); } [TestMethod] public void InitializeShouldNotConsiderCaseSensitivityOfTheArgumentPassed() { - var executor = new PlatformArgumentExecutor(CommandLineOptions.Instance); - executor.Initialize("ArM"); Assert.AreEqual(ObjectModel.Architecture.ARM, CommandLineOptions.Instance.TargetArchitecture); + Assert.AreEqual(ObjectModel.Architecture.ARM.ToString(), this.runSettingsProvider.QueryRunSettingsNode(PlatformArgumentExecutor.RunSettingsPath)); } #endregion @@ -118,9 +121,7 @@ public void InitializeShouldNotConsiderCaseSensitivityOfTheArgumentPassed() [TestMethod] public void ExecuteShouldReturnSuccess() { - var executor = new PlatformArgumentExecutor(CommandLineOptions.Instance); - - Assert.AreEqual(ArgumentProcessorResult.Success, executor.Execute()); + Assert.AreEqual(ArgumentProcessorResult.Success, this.executor.Execute()); } #endregion diff --git a/test/vstest.console.UnitTests/Processors/ResultsDirectoryArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/ResultsDirectoryArgumentProcessorTests.cs new file mode 100644 index 0000000000..73ffc53fb2 --- /dev/null +++ b/test/vstest.console.UnitTests/Processors/ResultsDirectoryArgumentProcessorTests.cs @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors +{ + using Microsoft.VisualStudio.TestTools.UnitTesting; + using TestPlatform.CommandLine.Processors; + using System; + using System.IO; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; + using vstest.console.UnitTests.Processors; + + [TestClass] + public class ResultsDirectoryArgumentProcessorTests + { + private ResultsDirectoryArgumentExecutor executor; + private TestableRunSettingsProvider runSettingsProvider; + + [TestInitialize] + public void Init() + { + this.runSettingsProvider = new TestableRunSettingsProvider(); + this.executor = new ResultsDirectoryArgumentExecutor(CommandLineOptions.Instance, this.runSettingsProvider); + } + + [TestCleanup] + public void TestCleanup() + { + CommandLineOptions.Instance.Reset(); + } + + [TestMethod] + public void GetMetadataShouldReturnResultsDirectoryArgumentProcessorCapabilities() + { + var processor = new ResultsDirectoryArgumentProcessor(); + Assert.IsTrue(processor.Metadata.Value is ResultsDirectoryArgumentProcessorCapabilities); + } + + [TestMethod] + public void GetExecuterShouldReturnResultsDirectoryArgumentExecutor() + { + var processor = new ResultsDirectoryArgumentProcessor(); + Assert.IsTrue(processor.Executor.Value is ResultsDirectoryArgumentExecutor); + } + + #region ResultsDirectoryArgumentProcessorCapabilities tests + + [TestMethod] + public void CapabilitiesShouldReturnAppropriateProperties() + { + var capabilities = new ResultsDirectoryArgumentProcessorCapabilities(); + Assert.AreEqual("/ResultsDirectory", capabilities.CommandName); + Assert.AreEqual("--ResultsDirectory|/ResultsDirectory\n Test results directory will be created in specified path if not exists. \n Example: /ResultsDirectory:", capabilities.HelpContentResourceName); + + Assert.AreEqual(HelpContentPriority.ResultsDirectoryArgumentProcessorHelpPriority, capabilities.HelpPriority); + Assert.AreEqual(false, capabilities.IsAction); + Assert.AreEqual(ArgumentProcessorPriority.AutoUpdateRunSettings, capabilities.Priority); + + Assert.AreEqual(false, capabilities.AllowMultiple); + Assert.AreEqual(false, capabilities.AlwaysExecute); + Assert.AreEqual(false, capabilities.IsSpecialCommand); + } + + #endregion + + #region ResultsDirectoryArgumentExecutor Initialize tests + + [TestMethod] + public void InitializeShouldThrowIfArgumentIsNull() + { + string folder = null; + var message = + @"The /ResultsDirectory parameter requires a value, where the test results should be saved. Example: /ResultsDirectory:c:\MyTestResultsDirectory"; + this.InitializeExceptionTestTemplate(folder, message); + } + + [TestMethod] + public void InitializeShouldThrowIfArgumentIsAWhiteSpace() + { + var folder = " "; + var message = + @"The /ResultsDirectory parameter requires a value, where the test results should be saved. Example: /ResultsDirectory:c:\MyTestResultsDirectory"; + this.InitializeExceptionTestTemplate(folder, message); + } + + [TestMethod] + public void InitializeShouldThrowIfGivenPathisIllegal() + { + var folder = @"c:\som<\illegal\path\"; + var message = string.Format( + @"The path '{0}' specified in the 'ResultsDirectory' is invalid. Error: {1}", + folder, + "Illegal characters in path."); + this.InitializeExceptionTestTemplate(folder, message); + } + + [TestMethod] + public void InitializeShouldThrowIfPathIsNotSupported() + { + + var folder = @"c:\path\to\in:valid"; + + var message = string.Format( + @"The path '{0}' specified in the 'ResultsDirectory' is invalid. Error: {1}", + folder, + "The given path's format is not supported."); + this.InitializeExceptionTestTemplate(folder, message); + } + + private void InitializeExceptionTestTemplate(string folder, string message) + { + var isExceptionThrown = false; + + try + { + this.executor.Initialize(folder); + } + catch (Exception ex) + { + isExceptionThrown = true; + Assert.IsTrue(ex is CommandLineException); + Assert.AreEqual(message, ex.Message); + } + + Assert.IsTrue(isExceptionThrown); + } + + [TestMethod] + public void InitializeShouldSetCommandLineOptionsAndRunSettingsForRelativePathValue() + { + var relativePath = @".\relative\path"; + var absolutePath = Path.GetFullPath(relativePath); + this.executor.Initialize(relativePath); + Assert.AreEqual(absolutePath, CommandLineOptions.Instance.ResultsDirectory); + Assert.AreEqual(absolutePath, this.runSettingsProvider.QueryRunSettingsNode(ResultsDirectoryArgumentExecutor.RunSettingsPath)); + } + + [TestMethod] + public void InitializeShouldSetCommandLineOptionsAndRunSettingsForAbsolutePathValue() + { + var absolutePath = @"c:\Users\someone\testresults"; + this.executor.Initialize(absolutePath); + Assert.AreEqual(absolutePath, CommandLineOptions.Instance.ResultsDirectory); + Assert.AreEqual(absolutePath, this.runSettingsProvider.QueryRunSettingsNode(ResultsDirectoryArgumentExecutor.RunSettingsPath)); + } + + #endregion + + #region ResultsDirectoryArgumentExecutor Execute tests + + [TestMethod] + public void ExecuteShouldReturnSuccess() + { + Assert.AreEqual(ArgumentProcessorResult.Success, executor.Execute()); + } + + #endregion + } +} diff --git a/test/vstest.console.UnitTests/Processors/RunSettingsArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/RunSettingsArgumentProcessorTests.cs new file mode 100644 index 0000000000..6a9d704dd9 --- /dev/null +++ b/test/vstest.console.UnitTests/Processors/RunSettingsArgumentProcessorTests.cs @@ -0,0 +1,301 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors +{ + using System; + using System.IO; + using System.Xml; + + using Microsoft.VisualStudio.TestPlatform.Common; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; + using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using vstest.console.UnitTests.Processors; + + using Moq; + + [TestClass] + public class RunSettingsArgumentProcessorTests + { + private IRunSettingsProvider settingsProvider; + + [TestInitialize] + public void Init() + { + this.settingsProvider = new TestableRunSettingsProvider(); + } + + [TestCleanup] + public void TestCleanup() + { + CommandLineOptions.Instance.Reset(); + } + + [TestMethod] + public void GetMetadataShouldReturnRunSettingsArgumentProcessorCapabilities() + { + var processor = new RunSettingsArgumentProcessor(); + Assert.IsTrue(processor.Metadata.Value is RunSettingsArgumentProcessorCapabilities); + } + + [TestMethod] + public void GetExecuterShouldReturnRunSettingsArgumentExecutor() + { + var processor = new RunSettingsArgumentProcessor(); + Assert.IsTrue(processor.Executor.Value is RunSettingsArgumentExecutor); + } + + #region RunSettingsArgumentProcessorCapabilities tests + + [TestMethod] + public void CapabilitiesShouldReturnAppropriateProperties() + { + var capabilities = new RunSettingsArgumentProcessorCapabilities(); + Assert.AreEqual("/Settings", capabilities.CommandName); + Assert.AreEqual("--Settings|/Settings:\n Settings to use when running tests.", capabilities.HelpContentResourceName); + + Assert.AreEqual(HelpContentPriority.RunSettingsArgumentProcessorHelpPriority, capabilities.HelpPriority); + Assert.AreEqual(false, capabilities.IsAction); + Assert.AreEqual(ArgumentProcessorPriority.RunSettings, capabilities.Priority); + + Assert.AreEqual(false, capabilities.AllowMultiple); + Assert.AreEqual(false, capabilities.AlwaysExecute); + Assert.AreEqual(false, capabilities.IsSpecialCommand); + } + + #endregion + + #region RunSettingsArgumentExecutor tests + + [TestMethod] + public void InitializeShouldThrowExceptionIfArgumentIsNull() + { + Action action = () => new RunSettingsArgumentExecutor(CommandLineOptions.Instance, null).Initialize(null); + + ExceptionUtilities.ThrowsException( + action, + "The /Settings parameter requires a settings file to be provided."); + } + + [TestMethod] + public void InitializeShouldThrowExceptionIfArgumentIsWhiteSpace() + { + Action action = () => new RunSettingsArgumentExecutor(CommandLineOptions.Instance, null).Initialize(" "); + + ExceptionUtilities.ThrowsException( + action, + "The /Settings parameter requires a settings file to be provided."); + } + + [TestMethod] + public void InitializeShouldThrowExceptionIfFileDoesNotExist() + { + var fileName = "C:\\Imaginary\\nonExistentFile.txt"; + + var executor = new RunSettingsArgumentExecutor(CommandLineOptions.Instance, null); + var mockFileHelper = new Mock(); + mockFileHelper.Setup(fh => fh.Exists(It.IsAny())).Returns(false); + + executor.FileHelper = mockFileHelper.Object; + + ExceptionUtilities.ThrowsException( + () => executor.Initialize(fileName), + "The Settings file '{0}' could not be found.", + fileName); + } + + [TestMethod] + public void InitializeShouldThrowIfRunSettingsSchemaDoesNotMatch() + { + // Arrange. + var fileName = "C:\\temp\\r.runsettings"; + var settingsXml = ""; + + CommandLineOptions.Instance.EnableCodeCoverage = true; + var executor = new TestableRunSettingsArgumentExecutor( + CommandLineOptions.Instance, + this.settingsProvider, + settingsXml); + + // Setup mocks. + var mockFileHelper = new Mock(); + mockFileHelper.Setup(fh => fh.Exists(It.IsAny())).Returns(true); + + executor.FileHelper = mockFileHelper.Object; + + // Act and Assert. + ExceptionUtilities.ThrowsException( + () => executor.Initialize(fileName), + "Settings file provided do not confirm to required format."); + } + + [TestMethod] + public void InitializeShouldSetActiveRunSettings() + { + // Arrange. + var fileName = "C:\\temp\\r.runsettings"; + var settingsXml = ""; + + var executor = new TestableRunSettingsArgumentExecutor( + CommandLineOptions.Instance, + this.settingsProvider, + settingsXml); + + // Setup mocks. + var mockFileHelper = new Mock(); + mockFileHelper.Setup(fh => fh.Exists(It.IsAny())).Returns(true); + executor.FileHelper = mockFileHelper.Object; + + // Act. + executor.Initialize(fileName); + + // Assert. + Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); + Assert.AreEqual(fileName, CommandLineOptions.Instance.SettingsFile); + } + + [TestMethod] + public void InitializeShouldSetSettingsFileForCommandLineOptions() + { + // Arrange. + var fileName = "C:\\temp\\r.runsettings"; + var settingsXml = ""; + + var executor = new TestableRunSettingsArgumentExecutor( + CommandLineOptions.Instance, + this.settingsProvider, + settingsXml); + + // Setup mocks. + var mockFileHelper = new Mock(); + mockFileHelper.Setup(fh => fh.Exists(It.IsAny())).Returns(true); + executor.FileHelper = mockFileHelper.Object; + + // Act. + executor.Initialize(fileName); + + // Assert. + Assert.AreEqual(fileName, CommandLineOptions.Instance.SettingsFile); + } + + [TestMethod] + public void InitializeShouldAddDefaultSettingsIfNotPresent() + { + // Arrange. + var fileName = "C:\\temp\\r.runsettings"; + var settingsXml = ""; + + var executor = new TestableRunSettingsArgumentExecutor( + CommandLineOptions.Instance, + this.settingsProvider, + settingsXml); + + // Setup mocks. + var mockFileHelper = new Mock(); + mockFileHelper.Setup(fh => fh.Exists(It.IsAny())).Returns(true); + executor.FileHelper = mockFileHelper.Object; + + // Act. + executor.Initialize(fileName); + + // Assert. + Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); + RunConfiguration runConfiguration = + XmlRunSettingsUtilities.GetRunConfigurationNode(this.settingsProvider.ActiveRunSettings.SettingsXml); + Assert.AreEqual(runConfiguration.ResultsDirectory, Constants.DefaultResultsDirectory); + Assert.AreEqual(runConfiguration.TargetFrameworkVersion.ToString(), Framework.DefaultFramework.ToString()); + Assert.AreEqual(runConfiguration.TargetPlatform, Constants.DefaultPlatform); + + } + + [TestMethod] + public void InitializeShouldSetActiveRunSettingsForTestSettingsFiles() + { + // Arrange. + var fileName = "C:\\temp\\r.testsettings"; + var settingsXml = ""; + + var executor = new TestableRunSettingsArgumentExecutor( + CommandLineOptions.Instance, + this.settingsProvider, + settingsXml); + + // Setup mocks. + var mockFileHelper = new Mock(); + mockFileHelper.Setup(fh => fh.Exists(It.IsAny())).Returns(true); + executor.FileHelper = mockFileHelper.Object; + + // Act. + executor.Initialize(fileName); + + // Assert. + Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); + StringAssert.Contains(this.settingsProvider.ActiveRunSettings.SettingsXml, $"\r\n \r\n {Constants.DefaultPlatform}\r\n {Framework.FromString(FrameworkVersion.Framework45.ToString()).Name}\r\n {Constants.DefaultResultsDirectory}\r\n \r\n \r\n C:\\temp\\r.testsettings\r\n true\r\n \r\n \r\n \r\n \r\n"); + } + + [TestMethod] + public void InitializeShouldSetActiveRunSettingsWithCodeCoverageData() + { + // Arrange. + var fileName = "C:\\temp\\r.runsettings"; + var settingsXml = ""; + + CommandLineOptions.Instance.EnableCodeCoverage = true; + var executor = new TestableRunSettingsArgumentExecutor( + CommandLineOptions.Instance, + this.settingsProvider, + settingsXml); + + // Setup mocks. + var mockFileHelper = new Mock(); + mockFileHelper.Setup(fh => fh.Exists(It.IsAny())).Returns(true); + executor.FileHelper = mockFileHelper.Object; + + // Act. + executor.Initialize(fileName); + + // Assert. + Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); + StringAssert.Contains(this.settingsProvider.ActiveRunSettings.SettingsXml, $"\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n .*CPPUnitTestFramework.*\r\n .*vstest.console.*\r\n .*microsoft.intellitrace.*\r\n .*vstest.executionengine.*\r\n .*vstest.discoveryengine.*\r\n .*microsoft.teamfoundation.testplatform.*\r\n .*microsoft.visualstudio.testplatform.*\r\n .*microsoft.visualstudio.testwindow.*\r\n .*microsoft.visualstudio.mstest.*\r\n .*microsoft.visualstudio.qualitytools.*\r\n .*microsoft.vssdk.testhostadapter.*\r\n .*microsoft.vssdk.testhostframework.*\r\n .*qtagent32.*\r\n .*msvcr.*dll$\r\n .*msvcp.*dll$\r\n .*clr.dll$\r\n .*clr.ni.dll$\r\n .*clrjit.dll$\r\n .*clrjit.ni.dll$\r\n .*mscoree.dll$\r\n .*mscoreei.dll$\r\n .*mscoreei.ni.dll$\r\n .*mscorlib.dll$\r\n .*mscorlib.ni.dll$\r\n \r\n \r\n True\r\n True\r\n True\r\n false\r\n \r\n \r\n \r\n ^std::.*\r\n ^ATL::.*\r\n .*::__GetTestMethodInfo.*\r\n .*__CxxPureMSILEntry.*\r\n ^Microsoft::VisualStudio::CppCodeCoverageFramework::.*\r\n ^Microsoft::VisualStudio::CppUnitTestFramework::.*\r\n .*::YOU_CAN_ONLY_DESIGNATE_ONE_.*\r\n ^__.*\r\n .*::__.*\r\n \r\n \r\n \r\n \r\n ^System.Diagnostics.DebuggerHiddenAttribute$\r\n ^System.Diagnostics.DebuggerNonUserCodeAttribute$\r\n ^System.Runtime.CompilerServices.CompilerGeneratedAttribute$\r\n ^System.CodeDom.Compiler.GeneratedCodeAttribute$\r\n ^System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverageAttribute$\r\n \r\n \r\n \r\n \r\n .*\\\\atlmfc\\\\.*\r\n .*\\\\vctools\\\\.*\r\n .*\\\\public\\\\sdk\\\\.*\r\n .*\\\\externalapis\\\\.*\r\n .*\\\\microsoft sdks\\\\.*\r\n .*\\\\vc\\\\include\\\\.*\r\n .*\\\\msclr\\\\.*\r\n .*\\\\ucrt\\\\.*\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n {Constants.DefaultResultsDirectory}\r\n {Constants.DefaultPlatform}\r\n {Framework.DefaultFramework.Name}\r\n \r\n"); + } + + #endregion + + #region Testable Implementations + + private class TestableRunSettingsArgumentExecutor : RunSettingsArgumentExecutor + { + private string runSettingsString; + + internal TestableRunSettingsArgumentExecutor( + CommandLineOptions commandLineOptions, + IRunSettingsProvider runSettingsManager, + string runSettings) + : base(commandLineOptions, runSettingsManager) + + { + this.runSettingsString = runSettings; + } + + protected override XmlReader GetReaderForFile(string runSettingsFile) + { + if (this.runSettingsString == null) + { + return null; + } + + var reader = new StringReader(this.runSettingsString); + var xmlReader = XmlReader.Create(reader, XmlRunSettingsUtilities.ReaderSettings); + + return xmlReader; + } + } + + #endregion + } +} diff --git a/test/vstest.console.UnitTests/Processors/RunSettingsArgumentProcessortTests.cs b/test/vstest.console.UnitTests/Processors/RunSettingsArgumentProcessortTests.cs deleted file mode 100644 index 0976a48423..0000000000 --- a/test/vstest.console.UnitTests/Processors/RunSettingsArgumentProcessortTests.cs +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors -{ - using System; - using System.IO; - using System.Xml; - - using Microsoft.VisualStudio.TestPlatform.Common; - using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; - using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; - using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; - - using Moq; - - [TestClass] - public class RunSettingsArgumentProcessortTests - { - [TestCleanup] - public void TestCleanup() - { - CommandLineOptions.Instance.Reset(); - } - - [TestMethod] - public void GetMetadataShouldReturnRunSettingsArgumentProcessorCapabilities() - { - var processor = new RunSettingsArgumentProcessor(); - Assert.IsTrue(processor.Metadata.Value is RunSettingsArgumentProcessorCapabilities); - } - - [TestMethod] - public void GetExecuterShouldReturnRunSettingsArgumentProcessorCapabilities() - { - var processor = new RunSettingsArgumentProcessor(); - Assert.IsTrue(processor.Executor.Value is RunSettingsArgumentExecutor); - } - - #region RunSettingsArgumentProcessorCapabilities tests - - [TestMethod] - public void CapabilitiesShouldReturnAppropriateProperties() - { - var capabilities = new RunSettingsArgumentProcessorCapabilities(); - Assert.AreEqual("/Settings", capabilities.CommandName); - Assert.AreEqual("--Settings|/Settings:\n Settings to use when running tests.", capabilities.HelpContentResourceName); - - Assert.AreEqual(HelpContentPriority.RunSettingsArgumentProcessorHelpPriority, capabilities.HelpPriority); - Assert.AreEqual(false, capabilities.IsAction); - Assert.AreEqual(ArgumentProcessorPriority.RunSettings, capabilities.Priority); - - Assert.AreEqual(false, capabilities.AllowMultiple); - Assert.AreEqual(false, capabilities.AlwaysExecute); - Assert.AreEqual(false, capabilities.IsSpecialCommand); - } - - #endregion - - #region RunSettingsArgumentExecutor tests - - [TestMethod] - public void InitializeShouldThrowExceptionIfArgumentIsNull() - { - Action action = () => new RunSettingsArgumentExecutor(CommandLineOptions.Instance, null).Initialize(null); - - ExceptionUtilities.ThrowsException( - action, - "The /Settings parameter requires a settings file to be provided."); - } - - [TestMethod] - public void InitializeShouldThrowExceptionIfArgumentIsWhiteSpace() - { - Action action = () => new RunSettingsArgumentExecutor(CommandLineOptions.Instance, null).Initialize(" "); - - ExceptionUtilities.ThrowsException( - action, - "The /Settings parameter requires a settings file to be provided."); - } - - [TestMethod] - public void InitializeShouldThrowExceptionIfFileDoesNotExist() - { - var fileName = "C:\\Imaginary\\nonExistentFile.txt"; - - var executor = new RunSettingsArgumentExecutor(CommandLineOptions.Instance, null); - var mockFileHelper = new Mock(); - mockFileHelper.Setup(fh => fh.Exists(It.IsAny())).Returns(false); - - executor.FileHelper = mockFileHelper.Object; - - ExceptionUtilities.ThrowsException( - () => executor.Initialize(fileName), - "The Settings file '{0}' could not be found.", - fileName); - } - - [TestMethod] - public void InitializeShouldThrowIfRunSettingsSchemaDoesNotMatch() - { - // Arrange. - var fileName = "C:\\temp\\r.runsettings"; - var settingsXml = ""; - - var settingsProvider = new TestableRunSettingsProvider(); - - CommandLineOptions.Instance.EnableCodeCoverage = true; - var executor = new TestableRunSettingsArgumentExecutor( - CommandLineOptions.Instance, - settingsProvider, - settingsXml); - - // Setup mocks. - var mockFileHelper = new Mock(); - mockFileHelper.Setup(fh => fh.Exists(It.IsAny())).Returns(true); - - executor.FileHelper = mockFileHelper.Object; - - // Act and Assert. - ExceptionUtilities.ThrowsException( - () => executor.Initialize(fileName), - "Settings file provided do not confirm to required format."); - } - - [TestMethod] - public void InitializeShouldSetActiveRunSettings() - { - // Arrange. - var fileName = "C:\\temp\\r.runsettings"; - var settingsXml = ""; - - var settingsProvider = new TestableRunSettingsProvider(); - - var executor = new TestableRunSettingsArgumentExecutor( - CommandLineOptions.Instance, - settingsProvider, - settingsXml); - - // Setup mocks. - var mockFileHelper = new Mock(); - mockFileHelper.Setup(fh => fh.Exists(It.IsAny())).Returns(true); - executor.FileHelper = mockFileHelper.Object; - - // Act. - executor.Initialize(fileName); - - // Assert. - Assert.IsNotNull(settingsProvider.ActiveRunSettings); - StringAssert.Contains(settingsProvider.ActiveRunSettings.SettingsXml, "\r\n"); - } - - [TestMethod] - public void InitializeShouldSetActiveRunSettingsForTestSettingsFiles() - { - // Arrange. - var fileName = "C:\\temp\\r.testsettings"; - var settingsXml = ""; - - var settingsProvider = new TestableRunSettingsProvider(); - - var executor = new TestableRunSettingsArgumentExecutor( - CommandLineOptions.Instance, - settingsProvider, - settingsXml); - - // Setup mocks. - var mockFileHelper = new Mock(); - mockFileHelper.Setup(fh => fh.Exists(It.IsAny())).Returns(true); - executor.FileHelper = mockFileHelper.Object; - - // Act. - executor.Initialize(fileName); - - // Assert. - Assert.IsNotNull(settingsProvider.ActiveRunSettings); - StringAssert.Contains(settingsProvider.ActiveRunSettings.SettingsXml, "\r\n \r\n X86\r\n Framework45\r\n \r\n \r\n C:\\temp\\r.testsettings\r\n true\r\n \r\n \r\n \r\n \r\n"); - } - - [TestMethod] - public void InitializeShouldSetActiveRunSettingsWithCodeCoverageData() - { - // Arrange. - var fileName = "C:\\temp\\r.runsettings"; - var settingsXml = ""; - - var settingsProvider = new TestableRunSettingsProvider(); - - CommandLineOptions.Instance.EnableCodeCoverage = true; - var executor = new TestableRunSettingsArgumentExecutor( - CommandLineOptions.Instance, - settingsProvider, - settingsXml); - - // Setup mocks. - var mockFileHelper = new Mock(); - mockFileHelper.Setup(fh => fh.Exists(It.IsAny())).Returns(true); - executor.FileHelper = mockFileHelper.Object; - - // Act. - executor.Initialize(fileName); - - // Assert. - Assert.IsNotNull(settingsProvider.ActiveRunSettings); - StringAssert.Contains(settingsProvider.ActiveRunSettings.SettingsXml, "\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n .*CPPUnitTestFramework.*\r\n .*vstest.console.*\r\n .*microsoft.intellitrace.*\r\n .*vstest.executionengine.*\r\n .*vstest.discoveryengine.*\r\n .*microsoft.teamfoundation.testplatform.*\r\n .*microsoft.visualstudio.testplatform.*\r\n .*microsoft.visualstudio.testwindow.*\r\n .*microsoft.visualstudio.mstest.*\r\n .*microsoft.visualstudio.qualitytools.*\r\n .*microsoft.vssdk.testhostadapter.*\r\n .*microsoft.vssdk.testhostframework.*\r\n .*qtagent32.*\r\n .*msvcr.*dll$\r\n .*msvcp.*dll$\r\n .*clr.dll$\r\n .*clr.ni.dll$\r\n .*clrjit.dll$\r\n .*clrjit.ni.dll$\r\n .*mscoree.dll$\r\n .*mscoreei.dll$\r\n .*mscoreei.ni.dll$\r\n .*mscorlib.dll$\r\n .*mscorlib.ni.dll$\r\n \r\n \r\n True\r\n True\r\n True\r\n false\r\n \r\n \r\n \r\n ^std::.*\r\n ^ATL::.*\r\n .*::__GetTestMethodInfo.*\r\n .*__CxxPureMSILEntry.*\r\n ^Microsoft::VisualStudio::CppCodeCoverageFramework::.*\r\n ^Microsoft::VisualStudio::CppUnitTestFramework::.*\r\n .*::YOU_CAN_ONLY_DESIGNATE_ONE_.*\r\n ^__.*\r\n .*::__.*\r\n \r\n \r\n \r\n \r\n ^System.Diagnostics.DebuggerHiddenAttribute$\r\n ^System.Diagnostics.DebuggerNonUserCodeAttribute$\r\n ^System.Runtime.CompilerServices.CompilerGeneratedAttribute$\r\n ^System.CodeDom.Compiler.GeneratedCodeAttribute$\r\n ^System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverageAttribute$\r\n \r\n \r\n \r\n \r\n .*\\\\atlmfc\\\\.*\r\n .*\\\\vctools\\\\.*\r\n .*\\\\public\\\\sdk\\\\.*\r\n .*\\\\externalapis\\\\.*\r\n .*\\\\microsoft sdks\\\\.*\r\n .*\\\\vc\\\\include\\\\.*\r\n .*\\\\msclr\\\\.*\r\n .*\\\\ucrt\\\\.*\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n"); - } - - #endregion - - #region Testable Implementations - - private class TestableRunSettingsArgumentExecutor : RunSettingsArgumentExecutor - { - private string runSettingsString; - - internal TestableRunSettingsArgumentExecutor( - CommandLineOptions commandLineOptions, - IRunSettingsProvider runSettingsManager, - string runSettings) - : base(commandLineOptions, runSettingsManager) - - { - this.runSettingsString = runSettings; - } - - protected override XmlReader GetReaderForFile(string runSettingsFile) - { - if (this.runSettingsString == null) - { - return null; - } - - var reader = new StringReader(this.runSettingsString); - var xmlReader = XmlReader.Create(reader, XmlRunSettingsUtilities.ReaderSettings); - - return xmlReader; - } - } - - private class TestableRunSettingsProvider : IRunSettingsProvider - { - public RunSettings ActiveRunSettings - { - get; - set; - } - - public void SetActiveRunSettings(RunSettings runSettings) - { - this.ActiveRunSettings = runSettings; - } - } - - #endregion - } -} diff --git a/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs index 6db1cb3665..1bafa2a6de 100644 --- a/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/RunSpecificTestsArgumentProcessorTests.cs @@ -12,11 +12,12 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors using Microsoft.VisualStudio.TestPlatform.Common.Logging; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; - using Microsoft.VisualStudio.TestPlatform.Utilities; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; + using Microsoft.VisualStudio.TestPlatform.Client.RequestHelper; + using vstest.console.UnitTests.Processors; using Moq; - using CoreUtilities.Tracing; using CoreUtilities.Tracing.Interfaces; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; @@ -29,6 +30,13 @@ public class RunSpecificTestsArgumentProcessorTests private Mock mockTestPlatformEventSource; + private static RunSpecificTestsArgumentExecutor GetExecutor(ITestRequestManager testRequestManager) + { + var runSettingsProvider = new TestableRunSettingsProvider(); + runSettingsProvider.AddDefaultRunSettings(); + return new RunSpecificTestsArgumentExecutor(CommandLineOptions.Instance, runSettingsProvider, testRequestManager); + } + public RunSpecificTestsArgumentProcessorTests() { this.mockFileHelper = new Mock(); @@ -45,7 +53,7 @@ public void GetMetadataShouldReturnRunSpecificTestsArgumentProcessorCapabilities } [TestMethod] - public void GetExecutorShouldReturnRunSpecificTestsArgumentProcessorCapabilities() + public void GetExecutorShouldReturnRunSpecificTestsArgumentExecutor() { RunSpecificTestsArgumentProcessor processor = new RunSpecificTestsArgumentProcessor(); @@ -59,7 +67,7 @@ public void CapabilitiesShouldReturnAppropriateProperties() { RunSpecificTestsArgumentProcessorCapabilities capabilities = new RunSpecificTestsArgumentProcessorCapabilities(); Assert.AreEqual("/Tests", capabilities.CommandName); - StringAssert.Contains(capabilities.HelpContentResourceName,"/Tests:\n Run tests with names that match the provided values."); + StringAssert.Contains(capabilities.HelpContentResourceName, "/Tests:\n Run tests with names that match the provided values."); Assert.AreEqual(HelpContentPriority.RunSpecificTestsArgumentProcessorHelpPriority, capabilities.HelpPriority); Assert.AreEqual(true, capabilities.IsAction); @@ -79,7 +87,7 @@ public void ExecutorExecuteForNoSourcesShouldThrowCommandLineException() CommandLineOptions.Instance.Reset(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, new TestPlatform(), TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - RunSpecificTestsArgumentExecutor executor = new RunSpecificTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); Assert.ThrowsException(() => executor.Execute()); } @@ -89,7 +97,7 @@ public void ExecutorExecuteForValidSourceWithTestCaseFilterShouldThrowCommandLin this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, new TestPlatform(), TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunSpecificTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); CommandLineOptions.Instance.TestCaseFilterValue = "Filter"; Assert.ThrowsException(() => executor.Execute()); } @@ -108,7 +116,7 @@ public void ExecutorExecuteShouldCatchTestPlatformExceptionThrownDuringDiscovery this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunSpecificTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); ArgumentProcessorResult argumentProcessorResult = executor.Execute(); Assert.AreEqual(ArgumentProcessorResult.Fail, argumentProcessorResult); @@ -128,7 +136,7 @@ public void ExecutorExecuteShouldCatchInvalidOperationExceptionThrownDuringDisco this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunSpecificTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); ArgumentProcessorResult argumentProcessorResult = executor.Execute(); Assert.AreEqual(ArgumentProcessorResult.Fail, argumentProcessorResult); @@ -148,7 +156,7 @@ public void ExecutorExecuteShouldCatchSettingsExceptionThrownDuringDiscoveryAndR this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunSpecificTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); ArgumentProcessorResult argumentProcessorResult = executor.Execute(); Assert.AreEqual(ArgumentProcessorResult.Fail, argumentProcessorResult); @@ -173,7 +181,7 @@ public void ExecutorExecuteShouldCatchTestPlatformExceptionThrownDuringExecution this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunSpecificTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); executor.Initialize("Test1"); @@ -200,7 +208,7 @@ public void ExecutorExecuteShouldCatchSettingsExceptionThrownDuringExecutionAndR this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunSpecificTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); executor.Initialize("Test1"); @@ -227,7 +235,7 @@ public void ExecutorExecuteShouldCatchInvalidOperationExceptionThrownDuringExecu this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunSpecificTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); executor.Initialize("Test1"); @@ -252,7 +260,7 @@ public void ExecutorExecuteShouldForValidSourcesAndValidSelectedTestsRunsTestsAn mockTestPlatform.Setup(tp => tp.CreateDiscoveryRequest(It.IsAny())).Returns(mockDiscoveryRequest.Object); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunSpecificTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); executor.Initialize("Test1"); diff --git a/test/vstest.console.UnitTests/Processors/RunTestsArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/RunTestsArgumentProcessorTests.cs index 7cec41e3a3..99ae5e1fce 100644 --- a/test/vstest.console.UnitTests/Processors/RunTestsArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/RunTestsArgumentProcessorTests.cs @@ -11,6 +11,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors using Microsoft.VisualStudio.TestPlatform.Client; using Microsoft.VisualStudio.TestPlatform.CommandLine.Internal; using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers; using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework; using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities; @@ -22,6 +23,8 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors using Microsoft.VisualStudio.TestPlatform.Utilities; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Microsoft.VisualStudio.TestPlatform.Client.RequestHelper; + using vstest.console.UnitTests.Processors; using Moq; @@ -65,7 +68,7 @@ public void CapabilitiesShouldReturnAppropriateProperties() { RunTestsArgumentProcessorCapabilities capabilities = new RunTestsArgumentProcessorCapabilities(); Assert.AreEqual("/RunTests", capabilities.CommandName); - Assert.AreEqual("[TestFileNames]\n Run tests from the specified files. Separate multiple test file names\n by spaces.\n Examples: mytestproject.dll\n mytestproject.dll myothertestproject.exe", capabilities.HelpContentResourceName); + Assert.AreEqual("[TestFileNames]" + Environment.NewLine + " Run tests from the specified files. Separate multiple test file names" + Environment.NewLine + " by spaces." + Environment.NewLine + " Examples: mytestproject.dll" + Environment.NewLine + " mytestproject.dll myothertestproject.exe", capabilities.HelpContentResourceName); Assert.AreEqual(HelpContentPriority.RunTestsArgumentProcessorHelpPriority, capabilities.HelpPriority); Assert.AreEqual(true, capabilities.IsAction); @@ -86,11 +89,22 @@ public void ExecutorExecuteForNoSourcesShouldThrowCommandLineException() var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, new TestPlatform(), TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); Assert.ThrowsException(() => executor.Execute()); } + private static RunTestsArgumentExecutor GetExecutor(ITestRequestManager testRequestManager) + { + var runSettingsProvider = new TestableRunSettingsProvider(); + runSettingsProvider.AddDefaultRunSettings(); + var executor = new RunTestsArgumentExecutor( + CommandLineOptions.Instance, + runSettingsProvider, + testRequestManager); + return executor; + } + [TestMethod] public void ExecutorExecuteShouldCatchTestPlatformExceptionAndReturnFail() { @@ -103,7 +117,7 @@ public void ExecutorExecuteShouldCatchTestPlatformExceptionAndReturnFail() this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); ArgumentProcessorResult argumentProcessorResult = executor.Execute(); Assert.AreEqual(ArgumentProcessorResult.Fail, argumentProcessorResult); @@ -121,7 +135,7 @@ public void ExecutorExecuteShouldCatchSettingsExceptionAndReturnFail() this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); ArgumentProcessorResult argumentProcessorResult = executor.Execute(); Assert.AreEqual(ArgumentProcessorResult.Fail, argumentProcessorResult); @@ -139,7 +153,7 @@ public void ExecutorExecuteShouldCatchInvalidOperationExceptionAndReturnFail() this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); ArgumentProcessorResult argumentProcessorResult = executor.Execute(); Assert.AreEqual(ArgumentProcessorResult.Fail, argumentProcessorResult); @@ -157,7 +171,7 @@ public void ExecutorExecuteShouldThrowOtherExceptions() this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); Assert.ThrowsException(() => executor.Execute()); } @@ -212,7 +226,7 @@ private ArgumentProcessorResult RunRunArgumentProcessorExecuteWithMockSetup(ITes this.ResetAndAddSourceToCommandLineOptions(); var testRequestManager = new TestRequestManager(CommandLineOptions.Instance, mockTestPlatform.Object, TestLoggerManager.Instance, TestRunResultAggregator.Instance, this.mockTestPlatformEventSource.Object); - var executor = new RunTestsArgumentExecutor(CommandLineOptions.Instance, null, testRequestManager); + var executor = GetExecutor(testRequestManager); return executor.Execute(); } diff --git a/test/vstest.console.UnitTests/Processors/TestableRunSettingsProvider.cs b/test/vstest.console.UnitTests/Processors/TestableRunSettingsProvider.cs new file mode 100644 index 0000000000..169443da7d --- /dev/null +++ b/test/vstest.console.UnitTests/Processors/TestableRunSettingsProvider.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace vstest.console.UnitTests.Processors +{ + using Microsoft.VisualStudio.TestPlatform.Common; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; + + internal class TestableRunSettingsProvider : IRunSettingsProvider + { + public RunSettings ActiveRunSettings + { + get; + set; + } + + public void SetActiveRunSettings(RunSettings runSettings) + { + this.ActiveRunSettings = runSettings; + } + } +} diff --git a/test/vstest.console.UnitTests/Processors/Utilities/RunSettingsProviderExtensionsTests.cs b/test/vstest.console.UnitTests/Processors/Utilities/RunSettingsProviderExtensionsTests.cs new file mode 100644 index 0000000000..4d6d7463a6 --- /dev/null +++ b/test/vstest.console.UnitTests/Processors/Utilities/RunSettingsProviderExtensionsTests.cs @@ -0,0 +1,174 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors.Utilities +{ + using System; + using System.IO; + + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; + using Microsoft.VisualStudio.TestPlatform.Common; + using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using vstest.console.UnitTests.Processors; + + [TestClass] + public class RunSettingsProviderExtensionsTests + { + private const string DefaultRunSettingsTemplate = + "\r\n \r\n %ResultsDirectory%\r\n X86\r\n %DefaultFramework%\r\n \r\n"; + private IRunSettingsProvider runSettingsProvider; + + [TestInitialize] + public void Init() + { + runSettingsProvider = new TestableRunSettingsProvider(); + } + + [TestCleanup] + public void TestCleanup() + { + CommandLineOptions.Instance.Reset(); + } + + [TestMethod] + public void UpdateRunSettingsShouldUpdateGivenSettingsXml() + { + const string runSettingsXml = "\r\n \r\n X86\r\n \r\n"; + + this.runSettingsProvider.UpdateRunSettings(runSettingsXml); + + StringAssert.Contains(this.runSettingsProvider.ActiveRunSettings.SettingsXml, runSettingsXml); + } + + [TestMethod] + public void UpdateRunSettingsShouldThrownExceptionIfRunSettingsProviderIsNull() + { + Assert.ThrowsException( + () => RunSettingsProviderExtensions.UpdateRunSettings(null, "")); + } + + [TestMethod] + public void UpdateRunSettingsShouldThrownExceptionIfSettingsXmlIsNull() + { + Assert.ThrowsException( + () => this.runSettingsProvider.UpdateRunSettings(null)); + } + + [TestMethod] + public void UpdateRunSettingsShouldThrownExceptionIfSettingsXmlIsEmptyOrWhiteSpace() + { + Assert.ThrowsException( + () => this.runSettingsProvider.UpdateRunSettings(" ")); + } + + [TestMethod] + public void AddDefaultRunSettingsShouldSetDefaultSettingsForEmptySettings() + { + this.runSettingsProvider.AddDefaultRunSettings(); + + var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(this.runSettingsProvider.ActiveRunSettings.SettingsXml); + Assert.AreEqual(runConfiguration.ResultsDirectory, Constants.DefaultResultsDirectory); + Assert.AreEqual(runConfiguration.TargetFrameworkVersion.ToString(), Framework.DefaultFramework.ToString()); + Assert.AreEqual(runConfiguration.TargetPlatform, Constants.DefaultPlatform); + } + + [TestMethod] + public void AddDefaultRunSettingsShouldAddUnspecifiedSettings() + { + this.runSettingsProvider.UpdateRunSettings("\r\n \r\n X86\r\n \r\n"); + + this.runSettingsProvider.AddDefaultRunSettings(); + + var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(this.runSettingsProvider.ActiveRunSettings.SettingsXml); + Assert.AreEqual(runConfiguration.ResultsDirectory, Constants.DefaultResultsDirectory); + Assert.AreEqual(runConfiguration.TargetFrameworkVersion.ToString(), Framework.DefaultFramework.ToString()); + } + + [TestMethod] + public void AddDefaultRunSettingsShouldNotChangeSpecifiedSettings() + { + + this.runSettingsProvider.UpdateRunSettings("\r\n \r\n X64\r\n \r\n"); + + this.runSettingsProvider.AddDefaultRunSettings(); + + var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(this.runSettingsProvider.ActiveRunSettings.SettingsXml); + Assert.AreEqual(runConfiguration.TargetPlatform, Architecture.X64); + } + + [TestMethod] + public void AddDefaultRunSettingsShouldThrowExceptionIfArgumentIsNull() + { + Assert.ThrowsException(() => RunSettingsProviderExtensions.AddDefaultRunSettings(null)); + } + + [TestMethod] + public void UpdateRunSettingsNodeShouldThrowExceptionIfKeyIsNull() + { + Assert.ThrowsException(() => this.runSettingsProvider.UpdateRunSettingsNode(null, "data")); + } + + [TestMethod] + public void UpdateRunSettingsNodeShouldThrowExceptionIfKeyIsEmptyOrWhiteSpace() + { + Assert.ThrowsException(() => this.runSettingsProvider.UpdateRunSettingsNode(" ", "data")); + } + + [TestMethod] + public void UpdateRunSettingsNodeShouldThrowExceptionIfDataIsNull() + { + Assert.ThrowsException(() => this.runSettingsProvider.UpdateRunSettingsNode("Key", null)); + } + + [TestMethod] + public void UpdateRunSettingsNodeShouldThrowExceptionIfRunSettingsProviderIsNull() + { + Assert.ThrowsException(() => RunSettingsProviderExtensions.UpdateRunSettingsNode(null, "Key", "data")); + } + + [TestMethod] + public void UpdateRunSettingsNodeShouldAddNewKeyIfNotPresent() + { + this.runSettingsProvider.UpdateRunSettings(" "); + this.runSettingsProvider.UpdateRunSettingsNode("Key.Path", "data"); + + Assert.AreEqual("data", this.runSettingsProvider.QueryRunSettingsNode("Key.Path")); + } + + [TestMethod] + public void UpdateRunSettingsNodeShouldUpdateKeyIfAlreadyPresent() + { + this.runSettingsProvider.UpdateRunSettings(" 1 "); + this.runSettingsProvider.UpdateRunSettingsNode("RunConfiguration.MaxCpuCount", "0"); + Assert.AreEqual("0", this.runSettingsProvider.QueryRunSettingsNode("RunConfiguration.MaxCpuCount")); + } + + [TestMethod] + public void QueryRunSettingsNodeShouldThrowIfKeyIsNull() + { + Assert.ThrowsException(() => this.runSettingsProvider.QueryRunSettingsNode(null)); + } + + [TestMethod] + public void QueryRunSettingsNodeShouldThrowIfKeyIsEmptyOrWhiteSpace() + { + Assert.ThrowsException(() => this.runSettingsProvider.QueryRunSettingsNode(" ")); + } + + [TestMethod] + public void QueryRunSettingsNodeShouldReturnNullForNotExistKey() + { + Assert.IsNull(this.runSettingsProvider.QueryRunSettingsNode("RunConfiguration.TargetPlatform")); + } + + [TestMethod] + public void QueryRunSettingsNodeShouldReturnCorrectValue() + { + this.runSettingsProvider.UpdateRunSettings(" x86 "); + Assert.AreEqual("x86", this.runSettingsProvider.QueryRunSettingsNode("RunConfiguration.TargetPlatform")); + } + } +} diff --git a/test/vstest.console.UnitTests/Processors/Utilities/RunSettingsUtilitiesTests.cs b/test/vstest.console.UnitTests/Processors/Utilities/RunSettingsUtilitiesTests.cs deleted file mode 100644 index 258ec7e51b..0000000000 --- a/test/vstest.console.UnitTests/Processors/Utilities/RunSettingsUtilitiesTests.cs +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.Processors.Utilities -{ - using System; - using System.IO; - - using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; - using Microsoft.VisualStudio.TestPlatform.Common; - using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; - using Microsoft.VisualStudio.TestTools.UnitTesting; - using ObjectModel; - - [TestClass] - public class RunSettingsUtilitiesTests - { - private const string DefaultRunSettingsTemplate = - "\r\n \r\n %ResultsDirectory%\r\n X86\r\n %DefaultFramework%\r\n \r\n"; - - [TestCleanup] - public void TestCleanup() - { - CommandLineOptions.Instance.Reset(); - } - - [TestMethod] - public void GetRunSettingsShouldReturnDefaultRunSettingsIfProviderIsNull() - { - var runSettings = RunSettingsUtilities.GetRunSettings(null, null); - - Assert.AreEqual(this.GetDefaultRunSettings(), runSettings); - } - - [TestMethod] - public void GetRunSettingsShouldReturnDefaultRunSettingsIfActiveRunSettingsIsNull() - { - var settingsProvider = new TestableRunSettingsProvider(); - settingsProvider.SetActiveRunSettings(runSettings: null); - - var runSettings = RunSettingsUtilities.GetRunSettings(settingsProvider, null); - - Assert.AreEqual(this.GetDefaultRunSettings(), runSettings); - } - - [TestMethod] - public void GetRunSettingsShouldReturnRunSettingsFromTheProvider() - { - var settingsProvider = new TestableRunSettingsProvider(); - - var settings = "\r\n \r\n 432423\r\n \r\n"; - settingsProvider.SetActiveRunSettings(settings); - - var receivedRunSettings = RunSettingsUtilities.GetRunSettings(settingsProvider, null); - - StringAssert.Contains(receivedRunSettings, "432423"); - } - - [TestMethod] - public void GetRunSettingsShouldReturnSettingsWithPlatformSpecifiedInCommandLineOptions() - { - var settingsProvider = new TestableRunSettingsProvider(); - settingsProvider.SetActiveRunSettings(runSettings: null); - - CommandLineOptions.Instance.TargetArchitecture = ObjectModel.Architecture.X64; - - var runSettings = RunSettingsUtilities.GetRunSettings(settingsProvider, CommandLineOptions.Instance); - - var defaultRunSettings = this.GetDefaultRunSettings(); - //Replace with the platform specified. - var expectedSettings = defaultRunSettings.Replace("X86", "X64"); - Assert.AreEqual(expectedSettings, runSettings); - } - - [TestMethod] - public void GetRunSettingsShouldReturnSettingsWithFrameworkSpecifiedInCommandLineOptions() - { - var settingsProvider = new TestableRunSettingsProvider(); - settingsProvider.SetActiveRunSettings(runSettings: null); - - CommandLineOptions.Instance.TargetFrameworkVersion = Framework.FromString(".NETFramework,Version=v3.5"); - - var runSettings = RunSettingsUtilities.GetRunSettings(settingsProvider, CommandLineOptions.Instance); - - var defaultRunSettings = this.GetDefaultRunSettings(); - - //Replace with the framework specified. - var expectedSettings = defaultRunSettings.Replace(Framework.DefaultFramework.Name, ".NETFramework,Version=v3.5"); - Assert.AreEqual(expectedSettings, runSettings); - } - - [TestMethod] - public void GetRunSettingsShouldReturnSettingsWithoutParallelOptionWhenParallelIsOff() - { - var settingsProvider = new TestableRunSettingsProvider(); - settingsProvider.SetActiveRunSettings(runSettings: null); - - // Do not have to explicitly set - but for readability - CommandLineOptions.Instance.Parallel = false; - - var runSettings = RunSettingsUtilities.GetRunSettings(settingsProvider, CommandLineOptions.Instance); - - Assert.IsTrue(!runSettings.Contains("MaxCpuCount"), "MaxCpuCount must not be set if parallel setting is false."); - } - - - [TestMethod] - public void GetRunSettingsShouldReturnSettingsWithParallelOptionWhenParallelIsOn() - { - var settingsProvider = new TestableRunSettingsProvider(); - settingsProvider.SetActiveRunSettings(runSettings: null); - - CommandLineOptions.Instance.Parallel = true; - - var runSettings = RunSettingsUtilities.GetRunSettings(settingsProvider, CommandLineOptions.Instance); - StringAssert.Contains(runSettings, "0", "MaxCpuCount must be set to 0 if Parallel Enabled."); - } - - [TestMethod] - public void GetRunSettingsShouldReturnWithoutChangeIfUserProvidesBothParallelSwitchAndSettings() - { - string settingXml = @"2"; - var settingsProvider = new TestableRunSettingsProvider(); - settingsProvider.SetActiveRunSettings(settingXml); - - CommandLineOptions.Instance.Parallel = true; - - var runSettings = RunSettingsUtilities.GetRunSettings(settingsProvider, CommandLineOptions.Instance); - - var parallelValue = Environment.ProcessorCount; - StringAssert.Contains(runSettings, "2", "RunSettings Parallel value should take precendence over parallel switch."); - } - - - #region Testable Implementations - - private string GetDefaultRunSettings() - { - var defaultResultsDirectory = Path.Combine(Directory.GetCurrentDirectory(), "TestResults"); - return DefaultRunSettingsTemplate.Replace("%ResultsDirectory%", defaultResultsDirectory).Replace("%DefaultFramework%", Framework.DefaultFramework.Name); - } - - private class TestableRunSettingsProvider : IRunSettingsProvider - { - public RunSettings ActiveRunSettings { get; private set; } - - public void SetActiveRunSettings(RunSettings runSettings) - { - this.ActiveRunSettings = runSettings; - } - - public void SetActiveRunSettings(string settingsXml) - { - var runSettings = new RunSettings(); - runSettings.LoadSettingsXml(settingsXml); - - SetActiveRunSettings(runSettings); - } - } - - #endregion - } -}