Latest MSTest framework and assert extension. (microsoft#681)

* Updating to the latest mstest framework version and using an extension based on the new assert extensibility that it provides.

* Fixed signing and removed dependency on internal resource strings.

* Moving a few more tests to use the extension.

* Moving the extension to dependency.props.

scripts/build/TestPlatform.Dependencies.props

@@ -6,9 +6,10 @@
     <!-- Name of the elements must be in sync with test\Microsoft.TestPlatform.TestUtilities\IntegrationTestBase.cs -->
     <NETTestSdkPreviousVersion>15.0.0</NETTestSdkPreviousVersion>
 
-    <MSTestFrameworkVersion>1.1.11</MSTestFrameworkVersion>
-    <MSTestAdapterVersion>1.1.11</MSTestAdapterVersion>
-
+    <MSTestFrameworkVersion>1.1.14</MSTestFrameworkVersion>
+    <MSTestAdapterVersion>1.1.14</MSTestAdapterVersion>
+    <MSTestAssertExtensionVersion>1.0.2-preview</MSTestAssertExtensionVersion>
+
     <XUnitFrameworkVersion>2.2.0</XUnitFrameworkVersion>
     <XUnitAdapterVersion>2.2.0</XUnitAdapterVersion>
 

scripts/build/TestPlatform.Settings.targets

@@ -85,6 +85,9 @@
         <PackageReference Include="Moq">
           <Version>4.6.38-*</Version>
         </PackageReference>
+        <PackageReference Include="MSTest.Assert.Extensions">
+          <Version>$(MSTestAssertExtensionVersion)</Version>
+        </PackageReference>
       </ItemGroup>
     </When>
   </Choose>

test/Microsoft.TestPlatform.AcceptanceTests/Extension/CustomDataTestMethodAttribute.cs

@@ -43,7 +43,7 @@ public override TestResult[] Execute(ITestMethod testMethod)
 
             if (dataRows.Count == 0)
             {
-                return new TestResult[] { new TestResult() { Outcome = UnitTestOutcome.Failed, TestFailureException = new Exception(FrameworkMessages.NoDataRow) } };
+                return new TestResult[] { new TestResult() { Outcome = UnitTestOutcome.Failed, TestFailureException = new Exception("No DataRowAttribute specified. Atleast one DataRowAttribute is required with DataTestMethodAttribute.") } };
             }
 
             return RunDataDrivenTest(testMethod, dataRows.ToArray());
@@ -69,7 +69,7 @@ internal static TestResult[] RunDataDrivenTest(ITestMethod testMethod, DataRowAt
                 }
                 else
                 {
-                    result.DisplayName = string.Format(CultureInfo.CurrentCulture, FrameworkMessages.DataDrivenResultDisplayName, testMethod.TestMethodName, string.Join(",", dataRow.Data));
+                    result.DisplayName = string.Format(CultureInfo.CurrentCulture, "{0} ({1})", testMethod.TestMethodName, string.Join(",", dataRow.Data));
                 }
 
                 results.Add(result);

test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Adapter/RunContextTests.cs

@@ -9,6 +9,7 @@ namespace TestPlatform.CrossPlatEngine.UnitTests.Adapter
     using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Adapter;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
     using Microsoft.VisualStudio.TestTools.UnitTesting;
+    using MSTest.TestFramework.AssertExtensions;
 
     [TestClass]
     public class RunContextTests
@@ -47,9 +48,9 @@ public void GetTestCaseFilterShouldThrowOnInvalidProperties()
         {
             this.runContext.FilterExpressionWrapper = new FilterExpressionWrapper("highlyunlikelyproperty=unused");
 
-            var exception = Assert.ThrowsException<TestPlatformFormatException>(() => this.runContext.GetTestCaseFilter(new List<string> { "TestCategory" }, (s) => { return null; }));
-
-            StringAssert.Contains(exception.Message, "No tests matched the filter because it contains one or more properties that are not valid (highlyunlikelyproperty). Specify filter expression containing valid properties (TestCategory) and try again.");
+            Assert.That.Throws<TestPlatformFormatException>(() => 
+                        this.runContext.GetTestCaseFilter(new List<string> { "TestCategory" }, (s) => { return null; }))
+                    .WithMessage("No tests matched the filter because it contains one or more properties that are not valid (highlyunlikelyproperty). Specify filter expression containing valid properties (TestCategory) and try again.");
         }
 
         [TestMethod]

test/Microsoft.TestPlatform.ObjectModel.UnitTests/RunSettings/RunConfigurationTests.cs

@@ -8,6 +8,7 @@ namespace Microsoft.TestPlatform.ObjectModel.UnitTests
     using Microsoft.VisualStudio.TestPlatform.ObjectModel;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities;
     using Microsoft.VisualStudio.TestTools.UnitTesting;
+    using MSTest.TestFramework.AssertExtensions;
 
     [TestClass]
     public class RunConfigurationTests
@@ -106,10 +107,10 @@ public void RunConfigurationFromXmlThrowsSettingsExceptionIfBatchSizeIsInvalid()
                      </RunConfiguration>
                 </RunSettings>";
 
-            var ex = Assert.ThrowsException<SettingsException>(
-              () => XmlRunSettingsUtilities.GetRunConfigurationNode(settingsXml)
-            );
-            Assert.AreEqual(ex.Message, "Invalid settings 'RunConfiguration'.  Invalid value 'Foo' specified for 'BatchSize'.");
+
+            Assert.That.Throws<SettingsException>(
+                    () => XmlRunSettingsUtilities.GetRunConfigurationNode(settingsXml))
+                    .WithExactMessage("Invalid settings 'RunConfiguration'.  Invalid value 'Foo' specified for 'BatchSize'.");
         }
 
         [TestMethod]
@@ -123,10 +124,9 @@ public void RunConfigurationFromXmlThrowsSettingsExceptionIfBatchSizeIsNegativeI
                      </RunConfiguration>
                 </RunSettings>";
 
-            var ex = Assert.ThrowsException<SettingsException>(
-              () => XmlRunSettingsUtilities.GetRunConfigurationNode(settingsXml)
-            );
-            Assert.AreEqual(ex.Message, "Invalid settings 'RunConfiguration'.  Invalid value '-10' specified for 'BatchSize'.");
+            Assert.That.Throws<SettingsException>(
+              () => XmlRunSettingsUtilities.GetRunConfigurationNode(settingsXml))
+              .WithExactMessage("Invalid settings 'RunConfiguration'.  Invalid value '-10' specified for 'BatchSize'.");
         }
 
         [DataRow(true)]

test/Microsoft.TestPlatform.Utilities.UnitTests/InferRunSettingsHelperTests.cs

@@ -12,6 +12,7 @@ namespace Microsoft.TestPlatform.Utilities.UnitTests
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities;
     using Microsoft.VisualStudio.TestPlatform.Utilities;
     using Microsoft.VisualStudio.TestTools.UnitTesting;
+    using MSTest.TestFramework.AssertExtensions;
 
     [TestClass]
     public class InferRunSettingsHelperTests
@@ -24,10 +25,9 @@ public void UpdateRunSettingsShouldThrowIfRunSettingsNodeDoesNotExist()
 
             Action action = () => InferRunSettingsHelper.UpdateRunSettingsWithUserProvidedSwitches(navigator, Architecture.X86, Framework.DefaultFramework, "temp");
 
-            ExceptionUtilities.ThrowsException<XmlException>(
-                action,
-                "An error occurred while loading the settings.  Error: {0}.",
-                "Could not find 'RunSettings' node.");
+            Assert.That.Throws<XmlException>(action)
+                        .WithMessage(string.Format("An error occurred while loading the settings.  Error: {0}.",
+                                            "Could not find 'RunSettings' node."));
         }
 
         [TestMethod]
@@ -38,10 +38,12 @@ public void UpdateRunSettingsShouldThrowIfPlatformNodeIsInvalid()
 
             Action action = () => InferRunSettingsHelper.UpdateRunSettingsWithUserProvidedSwitches(navigator, Architecture.X86, Framework.DefaultFramework, "temp");
 
-            ExceptionUtilities.ThrowsException<XmlException>(
-                action,
-                "An error occurred while loading the settings.  Error: {0}.",
-                string.Format("Invalid setting '{0}'. Invalid value '{1}' specified for '{2}'", "RunConfiguration", "foo", "TargetPlatform"));
+            Assert.That.Throws<XmlException>(action)
+                        .WithMessage(string.Format("An error occurred while loading the settings.  Error: {0}.",
+                                        string.Format("Invalid setting '{0}'. Invalid value '{1}' specified for '{2}'",
+                                            "RunConfiguration",
+                                            "foo",
+                                            "TargetPlatform")));
         }
 
         [TestMethod]
@@ -52,10 +54,12 @@ public void UpdateRunSettingsShouldThrowIfFrameworkNodeIsInvalid()
 
             Action action = () => InferRunSettingsHelper.UpdateRunSettingsWithUserProvidedSwitches(navigator, Architecture.X86, Framework.DefaultFramework, "temp");
 
-            ExceptionUtilities.ThrowsException<XmlException>(
-                action,
-                "An error occurred while loading the settings.  Error: {0}.",
-                string.Format("Invalid setting '{0}'. Invalid value '{1}' specified for '{2}'", "RunConfiguration", "foo", "TargetFrameworkVersion"));
+            Assert.That.Throws<XmlException>(action)
+                        .WithMessage(string.Format("An error occurred while loading the settings.  Error: {0}.",
+                                        string.Format("Invalid setting '{0}'. Invalid value '{1}' specified for '{2}'",
+                                        "RunConfiguration",
+                                        "foo",
+                                        "TargetFrameworkVersion")));
         }
 
         [TestMethod]
@@ -203,11 +207,11 @@ public void UpdateRunSettingsShouldThrowIfArchitectureSetIsIncompatibleWithCurre
 
             Action action = () => InferRunSettingsHelper.UpdateRunSettingsWithUserProvidedSwitches(navigator, Architecture.ARM, Framework.DefaultFramework, "temp");
 
-            ExceptionUtilities.ThrowsException<SettingsException>(
-                action,
-                "Incompatible Target platform settings '{0}' with system architecture '{1}'.",
-                "ARM",
-                XmlRunSettingsUtilities.OSArchitecture.ToString());
+            Assert.That.Throws<SettingsException>(action)
+                .WithMessage(string.Format(
+                        "Incompatible Target platform settings '{0}' with system architecture '{1}'.",
+                        "ARM",
+                        XmlRunSettingsUtilities.OSArchitecture.ToString()));
         }
 
         [TestMethod]

test/Microsoft.TestPlatform.Utilities.UnitTests/MSTestSettingsUtilitiesTests.cs

@@ -11,6 +11,7 @@ namespace Microsoft.TestPlatform.Utilities.Tests
     using Microsoft.VisualStudio.TestPlatform.Utilities;
     using Microsoft.VisualStudio.TestTools.UnitTesting;
     using VisualStudio.TestPlatform.ObjectModel;
+    using MSTest.TestFramework.AssertExtensions;
 
     [TestClass]
     public class MSTestSettingsUtilitiesTests
@@ -53,7 +54,7 @@ public void ImportShouldThrowIfNotLegacySettingsFile()
                     GetXPathNavigable(xmlDocument),
                     Architecture.X86,
                     FrameworkVersion.Framework45);
-            ExceptionUtilities.ThrowsException<XmlException>(action, "Unexpected settings file specified.");
+            Assert.That.Throws<XmlException>(action).WithMessage("Unexpected settings file specified.");
         }
 
         [TestMethod]
@@ -70,7 +71,7 @@ public void ImportShouldThrowIfDefaultRunSettingsIsIncorrect()
                     GetXPathNavigable(xmlDocument),
                     Architecture.X86,
                     FrameworkVersion.Framework45);
-            ExceptionUtilities.ThrowsException<XmlException>(action, "Could not find 'RunSettings' node.");
+            Assert.That.Throws<XmlException>(action).WithMessage("Could not find 'RunSettings' node.");
         }
 
         [TestMethod]

test/datacollector.UnitTests/DataCollectorConfigTests.cs

@@ -7,6 +7,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector.UnitTests
     using System.Globalization;
 
     using Microsoft.VisualStudio.TestTools.UnitTesting;
+    using MSTest.TestFramework.AssertExtensions;
 
     [TestClass]
     public class DataCollectorConfigTests
@@ -33,46 +34,40 @@ public void ConstructorShouldThrowExceptionIfTypeIsNull()
         [TestMethod]
         public void ConstructorShouldThrowExceptionIfUriIsNotSpecifiedInDataCollector()
         {
-            ThrowsExceptionWithMessage<ArgumentException>(() =>
-            {
+            Assert.That.Throws<ArgumentException>(() =>
+                    {
                         new DataCollectorConfig(typeof(CustomDataCollectorWithoutUri));
-                    },
-                string.Format(
-                    CultureInfo.CurrentCulture,
-                    Resources.Resources.DataCollector_TypeIsNull,
-                    typeof(CustomDataCollectorWithoutUri).FullName));
+                    })
+                    .WithMessage(string.Format(
+                            CultureInfo.CurrentCulture,
+                            Resources.Resources.DataCollector_TypeIsNull,
+                            typeof(CustomDataCollectorWithoutUri).FullName));
         }
 
         [TestMethod]
         public void ConstructorShouldThrowExceptionIfFriendlyNameIsEmpty()
         {
-            ThrowsExceptionWithMessage<ArgumentException>(() =>
-            {
+            Assert.That.Throws<ArgumentException>(() =>
+                    {
                         new DataCollectorConfig(typeof(CustomDataCollectorWithEmptyFriendlyName));
-                    },
-                string.Format(
-                    CultureInfo.CurrentCulture,
-                    Resources.Resources.FriendlyNameIsNullOrEmpty,
-                    typeof(CustomDataCollectorWithEmptyFriendlyName).FullName));
+                    })
+                    .WithMessage(string.Format(
+                            CultureInfo.CurrentCulture,
+                            Resources.Resources.FriendlyNameIsNullOrEmpty,
+                            typeof(CustomDataCollectorWithEmptyFriendlyName).FullName));
         }
 
         [TestMethod]
         public void ConstructorShouldThrowExceptionIfFriendlyNameIsNotSpecified()
         {
-            ThrowsExceptionWithMessage<ArgumentException>(() =>
-            {
+            Assert.That.Throws<ArgumentException>(() =>
+                    {
                         new DataCollectorConfig(typeof(CustomDataCollectorWithoutFriendlyName));
-                    },
-                string.Format(
-                    CultureInfo.CurrentCulture,
-                    Resources.Resources.FriendlyNameIsNullOrEmpty,
-                    typeof(CustomDataCollectorWithoutFriendlyName).FullName));
-        }
-
-        public static void ThrowsExceptionWithMessage<T>(Action action, string message) where T : Exception
-        {
-            var exception = Assert.ThrowsException<T>(action);
-            StringAssert.Contains(exception.Message, message);
+                    }).
+                    WithMessage(string.Format(
+                            CultureInfo.CurrentCulture,
+                            Resources.Resources.FriendlyNameIsNullOrEmpty,
+                            typeof(CustomDataCollectorWithoutFriendlyName).FullName));
         }
     }
 }

test/vstest.console.UnitTests/CommandLine/CommandLineOptionsTests.cs

@@ -11,6 +11,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests.CommandLine
     using Microsoft.VisualStudio.TestTools.UnitTesting;
     using Moq;
     using System.IO;
+    using MSTest.TestFramework.AssertExtensions;
 
     [TestClass]
     public class CommandLineOptionsTests
@@ -86,8 +87,8 @@ public void CommandLineOptionsAddSourceShouldAddSourceThrowExceptionIfDuplicateS
             this.fileHelper.Setup(fh => fh.Exists(testFilePath)).Returns(true);
 
             CommandLineOptions.Instance.AddSource(testFilePath);
-            var ex = Assert.ThrowsException<CommandLineException>(() => CommandLineOptions.Instance.AddSource(testFilePath));
-            Assert.AreEqual("Duplicate source " + testFilePath + " specified.", ex.Message);
+            Assert.That.Throws<CommandLineException>(() => CommandLineOptions.Instance.AddSource(testFilePath))
+                .WithExactMessage("Duplicate source " + testFilePath + " specified.");
         }
 
         [TestMethod]