Skip to content

Commit

Permalink
Show proper warning message no tests to run because testcasefilter (#…
Browse files Browse the repository at this point in the history 
…1656)

* Show proper warning message on no test with testcasefilter

* Show only first 60 character of testcasefilter

* Add acceptance tests for discovery case

* Fix acceptance test

* Fix DiscovererEnumeratorTests nits

* Add unit tests for DiscovererEnumeratorTests

* Add unit tests for ConsoleLogger and RunTestsWithSources

* Fix acceptance tests

* Add resources changes

* Reduce ref usage
  • Loading branch information
@smadala
smadala authored Jun 25, 2018
1 parent f5fa5bc commit e0caba4
Show file tree
Hide file tree
Showing 44 changed files with 977 additions and 550 deletions.
4 changes: 2 additions & 2 deletions scripts/build/TestPlatform.Dependencies.props
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
<!-- Name of the elements must be in sync with test\Microsoft.TestPlatform.TestUtilities\IntegrationTestBase.cs -->
<NETTestSdkPreviousVersion>15.5.0</NETTestSdkPreviousVersion>

<MSTestFrameworkVersion>1.3.2</MSTestFrameworkVersion>
<MSTestAdapterVersion>1.3.2</MSTestAdapterVersion>
<MSTestFrameworkVersion>1.3.1</MSTestFrameworkVersion>
<MSTestAdapterVersion>1.3.1</MSTestAdapterVersion>
<MSTestAssertExtensionVersion>1.0.3-preview</MSTestAssertExtensionVersion>

<XUnitFrameworkVersion>2.3.1</XUnitFrameworkVersion>
Expand Down
233 changes: 149 additions & 84 deletions src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscovererEnumerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions;

using Utilities;
using CrossPlatEngineResources = Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Resources.Resources;

/// <summary>
Expand Down Expand Up @@ -102,14 +102,10 @@ internal void LoadTests(IDictionary<string, IEnumerable<string>> testExtensionSo
Justification = "This methods must invoke all possible discoverers and not fail or crash in any one.")]
private void LoadTestsFromAnExtension(string extensionAssembly, IEnumerable<string> sources, IRunSettings settings, string testCaseFilter, IMessageLogger logger)
{
double totalAdaptersUsed = 0;

// Stopwatch to collect metrics
var timeStart = DateTime.UtcNow;

var discoverersFromDeprecatedLocations = false;

var discovererToSourcesMap = GetDiscovererToSourcesMap(extensionAssembly, sources, logger, this.assemblyProperties);
var discovererToSourcesMap = DiscovererEnumerator.GetDiscovererToSourcesMap(extensionAssembly, sources, logger, this.assemblyProperties);
var totalAdapterLoadTIme = DateTime.UtcNow - timeStart;

// Collecting Data Point for TimeTaken to Load Adapters
Expand All @@ -132,98 +128,167 @@ private void LoadTestsFromAnExtension(string extensionAssembly, IEnumerable<stri

var discoverySink = new TestCaseDiscoverySink(this.discoveryResultCache);
double totalTimeTakenByAdapters = 0;
double totalAdaptersUsed = 0;

foreach (var discoverer in discovererToSourcesMap.Keys)
{
Type discovererType = null;
this.DiscoverTestsFromSingleDiscoverer(discoverer, discovererToSourcesMap, context, discoverySink, logger, ref totalAdaptersUsed, ref totalTimeTakenByAdapters);
}

// See if discoverer can be instantiated successfully else move next.
try
{
discovererType = discoverer.Value.GetType();
}
catch (Exception e)
{
var mesage = string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.DiscovererInstantiationException,
e.Message);
logger.SendMessage(TestMessageLevel.Warning, mesage);
EqtTrace.Error("DiscovererEnumerator.LoadTestsFromAnExtension: {0} ", e);
if (this.discoveryResultCache.TotalDiscoveredTests == 0)
{
DiscovererEnumerator.LogWarningOnNoTestsDiscovered(sources, testCaseFilter, logger);
}

continue;
}
this.CollectTelemetryAtEnd(totalTimeTakenByAdapters, totalAdaptersUsed);
}

private void CollectTelemetryAtEnd(double totalTimeTakenByAdapters, double totalAdaptersUsed)
{
// Collecting Total Time Taken by Adapters
this.requestData.MetricsCollection.Add(TelemetryDataConstants.TimeTakenInSecByAllAdapters,
totalTimeTakenByAdapters);

// Collecting Total Adapters Used to Discover tests
this.requestData.MetricsCollection.Add(TelemetryDataConstants.NumberOfAdapterUsedToDiscoverTests,
totalAdaptersUsed);
}

private void DiscoverTestsFromSingleDiscoverer(
LazyExtension<ITestDiscoverer, ITestDiscovererCapabilities> discoverer,
Dictionary<LazyExtension<ITestDiscoverer, ITestDiscovererCapabilities>, IEnumerable<string>> discovererToSourcesMap,
DiscoveryContext context,
TestCaseDiscoverySink discoverySink,
IMessageLogger logger,
ref double totalAdaptersUsed,
ref double totalTimeTakenByAdapters)
{
if (DiscovererEnumerator.TryToLoadDiscoverer(discoverer, logger, out var discovererType) == false)
{
// Fail to instantiate the discoverer type.
return;
}

// on instantiated successfully, get tests
try
{
EqtTrace.Verbose(
"DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: Loading tests for {0}",
discoverer.Value.GetType().FullName);

var currentTotalTests = this.discoveryResultCache.TotalDiscoveredTests;
var newTimeStart = DateTime.UtcNow;

this.testPlatformEventSource.AdapterDiscoveryStart(discoverer.Metadata.DefaultExecutorUri.AbsoluteUri);
discoverer.Value.DiscoverTests(discovererToSourcesMap[discoverer], context, logger, discoverySink);

var totalAdapterRunTime = DateTime.UtcNow - newTimeStart;

this.testPlatformEventSource.AdapterDiscoveryStop(this.discoveryResultCache.TotalDiscoveredTests -
currentTotalTests);

// Record Total Tests Discovered By each Discoverer.
var totalTestsDiscoveredByCurrentDiscoverer = this.discoveryResultCache.TotalDiscoveredTests - currentTotalTests;
this.requestData.MetricsCollection.Add(
string.Format("{0}.{1}", TelemetryDataConstants.TotalTestsByAdapter,
discoverer.Metadata.DefaultExecutorUri), totalTestsDiscoveredByCurrentDiscoverer);

totalAdaptersUsed++;


EqtTrace.Verbose("DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: Done loading tests for {0}",
discoverer.Value.GetType().FullName);

// if instantiated successfully, get tests
try
var discovererFromDeprecatedLocations = DiscovererEnumerator.IsDiscovererFromDeprecatedLocations(discoverer);
if (discovererFromDeprecatedLocations)
{
if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose(
"DiscoveryContext.LoadTests: Loading tests for {0}",
discoverer.Value.GetType().FullName);
}

var currentTotalTests = this.discoveryResultCache.TotalDiscoveredTests;
var newTimeStart = DateTime.UtcNow;

this.testPlatformEventSource.AdapterDiscoveryStart(discoverer.Metadata.DefaultExecutorUri.AbsoluteUri);
discoverer.Value.DiscoverTests(discovererToSourcesMap[discoverer], context, logger, discoverySink);

var totalAdapterRunTime = DateTime.UtcNow - newTimeStart;

this.testPlatformEventSource.AdapterDiscoveryStop(this.discoveryResultCache.TotalDiscoveredTests - currentTotalTests);

// Collecting Total Tests Discovered By each Adapter.
if (this.discoveryResultCache.TotalDiscoveredTests > currentTotalTests)
{
var totalDiscoveredTests = this.discoveryResultCache.TotalDiscoveredTests - currentTotalTests;
this.requestData.MetricsCollection.Add(string.Format("{0}.{1}", TelemetryDataConstants.TotalTestsByAdapter, discoverer.Metadata.DefaultExecutorUri), totalDiscoveredTests);
if (!CrossPlatEngine.Constants.DefaultAdapters.Contains(discoverer.Metadata.DefaultExecutorUri.ToString(), StringComparer.OrdinalIgnoreCase))
{
var discovererLocation = discoverer.Value.GetType().GetTypeInfo().Assembly.GetAssemblyLocation();

discoverersFromDeprecatedLocations |= Path.GetDirectoryName(discovererLocation).Equals(CrossPlatEngine.Constants.DefaultAdapterLocation, StringComparison.OrdinalIgnoreCase);
}
totalAdaptersUsed++;
}

if (EqtTrace.IsVerboseEnabled)
{
EqtTrace.Verbose(
"DiscoveryContext.LoadTests: Done loading tests for {0}",
discoverer.Value.GetType().FullName);
}

if (discoverersFromDeprecatedLocations)
{
logger.SendMessage(TestMessageLevel.Warning, string.Format(CultureInfo.CurrentCulture, CrossPlatEngineResources.DeprecatedAdapterPath));
}

// Collecting Data Point for Time Taken to Discover Tests by each Adapter
this.requestData.MetricsCollection.Add(string.Format("{0}.{1}", TelemetryDataConstants.TimeTakenToDiscoverTestsByAnAdapter, discoverer.Metadata.DefaultExecutorUri), totalAdapterRunTime.TotalSeconds);
totalTimeTakenByAdapters += totalAdapterRunTime.TotalSeconds;
logger.SendMessage(TestMessageLevel.Warning,
string.Format(CultureInfo.CurrentCulture, CrossPlatEngineResources.DeprecatedAdapterPath));
}
catch (Exception e)
{
var message = string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.ExceptionFromLoadTests,
discovererType.Name,
e.Message);

logger.SendMessage(TestMessageLevel.Error, message);
EqtTrace.Error("DiscovererEnumerator.LoadTestsFromAnExtension: {0} ", e);
}
// Collecting Data Point for Time Taken to Discover Tests by each Adapter
this.requestData.MetricsCollection.Add(
string.Format("{0}.{1}", TelemetryDataConstants.TimeTakenToDiscoverTestsByAnAdapter,
discoverer.Metadata.DefaultExecutorUri), totalAdapterRunTime.TotalSeconds);
totalTimeTakenByAdapters += totalAdapterRunTime.TotalSeconds;
}
catch (Exception e)
{
var message = string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.ExceptionFromLoadTests,
discovererType.Name,
e.Message);

logger.SendMessage(TestMessageLevel.Error, message);
EqtTrace.Error("DiscovererEnumerator.DiscoverTestsFromSingleDiscoverer: {0} ", e);
}
}

// Collecting Total Time Taken by Adapters
this.requestData.MetricsCollection.Add(TelemetryDataConstants.TimeTakenInSecByAllAdapters, totalTimeTakenByAdapters);
private static bool TryToLoadDiscoverer(LazyExtension<ITestDiscoverer, ITestDiscovererCapabilities> discoverer, IMessageLogger logger, out Type discovererType)
{
discovererType = null;

// Collecting Total Adapters Used to Discover tests
this.requestData.MetricsCollection.Add(TelemetryDataConstants.NumberOfAdapterUsedToDiscoverTests, totalAdaptersUsed);
// See if discoverer can be instantiated successfully else move next.
try
{
discovererType = discoverer.Value.GetType();
}
catch (Exception e)
{
var mesage = string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.DiscovererInstantiationException,
e.Message);
logger.SendMessage(TestMessageLevel.Warning, mesage);
EqtTrace.Error("DiscovererEnumerator.LoadTestsFromAnExtension: {0} ", e);

return false;
}

return true;
}

private static bool IsDiscovererFromDeprecatedLocations(
LazyExtension<ITestDiscoverer, ITestDiscovererCapabilities> discoverer)
{
if (CrossPlatEngine.Constants.DefaultAdapters.Contains(discoverer.Metadata.DefaultExecutorUri.ToString(),
StringComparer.OrdinalIgnoreCase))
{
return false;
}

var discovererLocation = discoverer.Value.GetType().GetTypeInfo().Assembly.GetAssemblyLocation();

return Path.GetDirectoryName(discovererLocation)
.Equals(CrossPlatEngine.Constants.DefaultAdapterLocation, StringComparison.OrdinalIgnoreCase);
}

private static void LogWarningOnNoTestsDiscovered(IEnumerable<string> sources, string testCaseFilter, IMessageLogger logger)
{
var sourcesString = string.Join(" ", sources);

// Print warning on no tests.
if (string.IsNullOrEmpty(testCaseFilter) == false)
{
var testCaseFilterToShow = TestCaseFilterDeterminer.ShortenTestCaseFilterIfRequired(testCaseFilter);

logger.SendMessage(
TestMessageLevel.Warning,
string.Format(CrossPlatEngineResources.NoTestsAvailableForGivenTestCaseFilter, testCaseFilterToShow, sourcesString));
}
else
{
logger.SendMessage(
TestMessageLevel.Warning,
string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.TestRunFailed_NoDiscovererFound_NoTestsAreAvailableInTheSources,
sourcesString));
}
}


private void SetAdapterLoggingSettings(IMessageLogger messageLogger, IRunSettings runSettings)
{
var discoveryMessageLogger = messageLogger as TestSessionMessageLogger;
Expand Down
41 changes: 37 additions & 4 deletions src/Microsoft.TestPlatform.CrossPlatEngine/Execution/RunTestsWithSources.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Execution

using ObjectModel.Client;
using ObjectModel.Logging;

using Utilities;
using CrossPlatEngineResources = Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Resources.Resources;

internal class RunTestsWithSources : BaseRunTests
Expand Down Expand Up @@ -62,10 +62,26 @@ protected override void BeforeRaisingTestRunComplete(bool exceptionsHitDuringRun
if (!exceptionsHitDuringRunTests && this.executorUriVsSourceList?.Count > 0 && !this.IsCancellationRequested
&& this.TestRunCache?.TotalExecutedTests <= 0)
{
IEnumerable<string> sources = new List<string>();
var sourcesArray = this.adapterSourceMap.Values.Aggregate(sources, (current, enumerable) => current.Concat(enumerable)).ToArray();
var sourcesString = string.Join(" ", sourcesArray);
this.LogWarningOnNoTestsExecuted();
}
}

private void LogWarningOnNoTestsExecuted()
{
IEnumerable<string> sources = new List<string>();
var sourcesArray = this.adapterSourceMap.Values
.Aggregate(sources, (current, enumerable) => current.Concat(enumerable)).ToArray();
var sourcesString = string.Join(" ", sourcesArray);

if (this.TestExecutionContext.TestCaseFilter != null)
{
var testCaseFilterToShow = TestCaseFilterDeterminer.ShortenTestCaseFilterIfRequired(this.TestExecutionContext.TestCaseFilter);
this.TestRunEventsHandler?.HandleLogMessage(
TestMessageLevel.Warning,
string.Format(CrossPlatEngineResources.NoTestsAvailableForGivenTestCaseFilter, testCaseFilterToShow, sourcesString));
}
else
{
this.TestRunEventsHandler?.HandleLogMessage(
TestMessageLevel.Warning,
string.Format(
Expand Down Expand Up @@ -170,5 +186,22 @@ private Dictionary<Tuple<Uri, string>, IEnumerable<string>> GetExecutorVsSources

return result;
}

private static string TestCaseFilterToShow(string testCaseFilter)
{
var maxTestCaseFilterToShowLength = 63;
string testCaseFilterToShow;

if (testCaseFilter.Length > maxTestCaseFilterToShowLength)
{
testCaseFilterToShow = testCaseFilter.Substring(0, maxTestCaseFilterToShowLength - 3) + "...";
}
else
{
testCaseFilterToShow = testCaseFilter;
}

return testCaseFilterToShow;
}
}
}
2 changes: 1 addition & 1 deletion src/Microsoft.TestPlatform.CrossPlatEngine/Execution/RunTestsWithTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ protected override void BeforeRaisingTestRunComplete(bool exceptionsHitDuringRun
protected override IEnumerable<Tuple<Uri, string>> GetExecutorUriExtensionMap(IFrameworkHandle testExecutorFrameworkHandle, RunContext runContext)
{
this.executorUriVsTestList = this.GetExecutorVsTestCaseList(this.testCases);

Debug.Assert(this.TestExecutionContext.TestCaseFilter == null, "TestCaseFilter should be null for specific tests.");
runContext.FilterExpressionWrapper = null;

Expand Down
5 changes: 4 additions & 1 deletion src/Microsoft.TestPlatform.CrossPlatEngine/Microsoft.TestPlatform.CrossPlatEngine.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@
<!--<EnableCodeAnalysis>true</EnableCodeAnalysis>-->
</PropertyGroup>
<ItemGroup>
<EmbeddedResource Include="Resources\Resources.resx" />
<EmbeddedResource Include="Resources\Resources.resx">
<Generator>ResXFileCodeGenerator</Generator>
<LastGenOutput>Resources.Designer.cs</LastGenOutput>
</EmbeddedResource>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Microsoft.TestPlatform.CommunicationUtilities\Microsoft.TestPlatform.CommunicationUtilities.csproj" />
Expand Down
Loading

0 comments on commit e0caba4

Please sign in to comment.