Enable code coverage reports

Resolves #14427 by limiting reports to a single service directory (CIs) or per test project (dev environments). For CIs, full coverage reports of everything build will be uploaded. For dev environments, a summary HTML file will be output to test projects' TestResults directories.
Azure · Nov 25, 2020 · 9d65733 · 9d65733
1 parent 8f48366
commit 9d65733
Show file tree

Hide file tree

Showing 11 changed files with 106 additions and 34 deletions.
diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json
@@ -0,0 +1,12 @@
+{
+  "version": 1,
+  "isRoot": true,
+  "tools": {
+    "dotnet-reportgenerator-globaltool": {
+      "version": "4.8.0",
+      "commands": [
+        "reportgenerator"
+      ]
+    }
+  }
+}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -55,8 +55,11 @@ Nuget package will be created in root directory under \artifacts\packages\Debug
 ### Using the command line:
 
 Run e.g. `msbuild eng\mgmt.proj /t:"Runtests" /p:Scope=Compute`
-In the above example _RunTests_ will build and run tests for Compute only or you can use command line CLI
-`dotnet test Compute\Microsoft.Azure.Management.Compute\tests\Microsoft.Azure.Management.Tests.csproj`
+In the above example _RunTests_ will build and run tests for Compute only or you can use command line CLI:
+
+```bash
+dotnet test Compute\Microsoft.Azure.Management.Compute\tests\Microsoft.Azure.Management.Tests.csproj
+```
 
 ### Non-Windows command line build
 
@@ -67,6 +70,17 @@ Now you can use the same command on non-windows as above for e.g. on Ubuntu you
 - `dotnet msbuild eng\mgmt.proj /t:CreateNugetPackage /p:scope=Compute`
 - `dotnet msbuild build.proj /t:Util /p:UtilityName=InstallPsModules`
 
+### Code Coverage
+
+If you want to enable code coverage reporting, on the command line pass `/p:CollectCoverage=true` like so:
+
+```bash
+dotnet test /p:CollectCoverage=true
+```
+
+On developers' machines, you can open `index.html` from within the `TestResults` directory in your test projects.
+Coverage reports can also be found in Azure Pipelines on the "Code Coverage" tab after a pull request validation build completes.
+
 ### Update build tools
 
 Build tools are now downloaded as part of a nuget package under `root\restoredPackages\microsoft.internal.netsdkbuild.mgmt.tools`

diff --git a/eng/CodeCoverage.runsettings b/eng/CodeCoverage.runsettings
@@ -1,16 +1,23 @@
 <?xml version="1.0"?>
 <RunSettings>
+  <LoggerRunSettings>
+    <Loggers>
+      <Logger friendlyName="trx" />
+    </Loggers>
+  </LoggerRunSettings>
   <DataCollectionRunSettings>
     <DataCollectors>
-      <DataCollector friendlyName="XPlat code coverage">
+      <DataCollector friendlyName="XPlat Code Coverage" enabled="true">
         <Configuration>
           <Format>cobertura</Format>
-          <ExcludeByAttribute>ExcludeFromCodeCoverage,GeneratedCodeAttribute,Obsolete</ExcludeByAttribute>
-          <SingleHit>false</SingleHit>
+          <ExcludeByAttribute>ExcludeFromCodeCoverageAttribute,GeneratedCodeAttribute,Obsolete</ExcludeByAttribute>
           <IncludeTestAssembly>false</IncludeTestAssembly>
+          <SingleHit>false</SingleHit>
           <SkipAutoProps>true</SkipAutoProps>
         </Configuration>
       </DataCollector>
+      <!-- Enable logging to diagnose test host failures -->
+      <DataCollector friendlyName="blame" enabled="true" />
     </DataCollectors>
   </DataCollectionRunSettings>
 </RunSettings>
diff --git a/eng/CodeCoverage.targets b/eng/CodeCoverage.targets
@@ -1,11 +1,42 @@
 <Project>
-  <ItemGroup Condition="'$(IsTestProject)' == 'true'">
+  <PropertyGroup>
+    <CodeCoverageDirectory Condition="'$(CodeCoverageDirectory)' == ''">$([System.IO.Path]::GetFullPath("$(MSBuildProjectDirectory)\.."))</CodeCoverageDirectory>
+    <SkipCoverageReport Condition="'$(SkipCoverageReport)' == '' and '$(ContinuousIntegrationBuild)' == 'true'">true</SkipCoverageReport>
+    <TestResultsDirectory>$(MSBuildProjectDirectory)\TestResults</TestResultsDirectory>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <!--
+      Use VSTest integration to work around blocking issues
+      https://github.com/coverlet-coverage/coverlet/blob/master/Documentation/VSTestIntegration.md
+    -->
     <PackageReference Include="coverlet.collector">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
   </ItemGroup>
 
+  <!-- Clean up previous TestResults so reports are recent -->
+  <Target Name="CleanPreviousCodeCoverage"
+          BeforeTargets="VSTest">
+    <RemoveDir Directories="$(TestResultsDirectory)" />
+  </Target>
+
+  <!-- Should be similar to what's in the pipelines, though generate a full HTML report -->
+  <Target Name="GenerateCodeCoverageReport"
+          AfterTargets="VSTest"
+          Condition="'$(SkipCoverageReport)' != 'true'">
+    <PropertyGroup>
+      <CoverageReportCommandLine>dotnet tool run reportgenerator --</CoverageReportCommandLine>
+      <CoverageReportCommandLine>$(CoverageReportCommandLine) "-reports:$(TestResultsDirectory)\**\coverage.cobertura.xml"</CoverageReportCommandLine>
+      <CoverageReportCommandLine>$(CoverageReportCommandLine) -reporttypes:Html</CoverageReportCommandLine>
+      <CoverageReportCommandLine>$(CoverageReportCommandLine) "-targetdir:$(TestResultsDirectory)"</CoverageReportCommandLine>
+      <CoverageReportCommandLine>$(CoverageReportCommandLine) "-filefilters:+$(CodeCoverageDirectory)\**"</CoverageReportCommandLine>
+    </PropertyGroup>
+    <Exec Command="$(CoverageReportCommandLine)"
+          IgnoreExitCode="true" />
+  </Target>
+
   <!--
     Allows Collection of Code Coverage for Deterministic Builds
     https://github.com/coverlet-coverage/coverlet/blob/master/Documentation/DeterministicBuild.md
@@ -19,7 +50,8 @@
           Returns="@(_LocalTopLevelSourceRoot)"
           Condition="'$(DeterministicSourcePaths)' == 'true'">
     <ItemGroup>
-      <_LocalTopLevelSourceRoot Include="@(SourceRoot)" Condition="'%(SourceRoot.NestedRoot)' == ''"/>
+      <_LocalTopLevelSourceRoot Include="@(SourceRoot)"
+                                Condition="'%(SourceRoot.NestedRoot)' == ''"/>
     </ItemGroup>
   </Target>
 </Project>
diff --git a/eng/Directory.Build.Data.props b/eng/Directory.Build.Data.props
@@ -83,18 +83,6 @@
     <CodeAnalysisRuleSet>$(RepoEngPath)\CodeAnalysis.ruleset</CodeAnalysisRuleSet>
   </PropertyGroup>
 
-  <!--
-    Code Coverage
-    Enable using MSBuild integration since it's easier to use in build pipelines
-    and likely easier to remember than the VSTest platform integration.
-    Also: https://github.com/coverlet-coverage/coverlet/blob/master/Documentation/KnownIssues.md#1-vstest-stops-process-execution-earlydotnet-test
-  -->
-  <PropertyGroup Condition="'$(CollectCoverage)' == 'true'">
-    <VSTestCollect Condition="'$(VSTestCollect)' == ''">XPlat Code Coverage</VSTestCollect>
-    <!-- Override this property in your projects' or solutions' Directory.Build.props to further customize code coverage. -->
-    <VSTestSetting Condition="'$(VSTestSetting)' == ''">$(MSBuildThisFileDirectory)CodeCoverage.runsettings</VSTestSetting>
-  </PropertyGroup>
-
   <!-- TargetFramework default properties -->
   <PropertyGroup>
     <!-- Client libraries are moving forward to NS 2.0 and Net 4.6.1 as the min supported versions -->
@@ -108,7 +96,7 @@
   <PropertyGroup Condition="'$(IsTestProject)' == 'true' or '$(IsTestSupportProject)' == 'true' or '$(IsSamplesProject)' == 'true' or '$(IsPerfProject)' == 'true' or '$(IsStressProject)' == 'true'">
     <IsPackable>false</IsPackable>
     <RequiredTargetFrameworks>netcoreapp2.1;net5.0</RequiredTargetFrameworks>
-    <RequiredTargetFrameworks Condition="'$(OS)' == 'Windows_NT'">netcoreapp2.1;net5.0;net461</RequiredTargetFrameworks>
+    <RequiredTargetFrameworks Condition="'$(OS)' == 'Windows_NT'">net461;netcoreapp2.1;net5.0</RequiredTargetFrameworks>
   </PropertyGroup>
 
   <Import Project="$(RepoRoot)/sdk/core/Azure.Core/src/Azure.Core.props" Condition="'$(IsMgmtClientLibrary)' == 'true'"/>

diff --git a/eng/Directory.Build.Data.targets b/eng/Directory.Build.Data.targets
@@ -71,8 +71,20 @@
     <None Condition="Exists('$(MSBuildProjectDirectory)/../README.md')" Include="$(MSBuildProjectDirectory)/../README.md" Pack="true" PackagePath=""/>
   </ItemGroup>
 
-  <!-- Collect Code Coverage -->
-  <Import Condition="'$(CollectCoverage)' == 'true'"  Project="$(MSBuildThisFileDirectory)\CodeCoverage.targets" />
+  <!--
+    Code Coverage for Track 2
+    Enable using MSBuild integration since it's easier to use in build pipelines
+    and likely easier to remember than the VSTest platform integration.
+    Also: https://github.com/coverlet-coverage/coverlet/blob/master/Documentation/KnownIssues.md#1-vstest-stops-process-execution-earlydotnet-test
+  -->
+  <PropertyGroup Condition="'$(CollectCoverage)' == 'true' and '$(IsClientLibrary)' == 'true' and '$(IsTestProject)' == 'true' and '$(ExcludeFromCodeCoverage)' != 'true'">
+    <_ImportCodeCoverage>true</_ImportCodeCoverage>
+    <VSTestCollect Condition="'$(VSTestCollect)' == ''">XPlat Code Coverage</VSTestCollect>
+    <!-- Override this property in your projects' or solutions' Directory.Build.props to further customize code coverage. -->
+    <VSTestSetting Condition="'$(VSTestSetting)' == ''">$(MSBuildThisFileDirectory)CodeCoverage.runsettings</VSTestSetting>
+  </PropertyGroup>
+
+  <Import Condition="'$(_ImportCodeCoverage)' == 'true'" Project="$(MSBuildThisFileDirectory)\CodeCoverage.targets" />
 
   <!-- Add StyleCop Analyzers -->
   <ItemGroup Condition="'$(EnableStyleCopAnalyzers)' == 'true'"  >

diff --git a/eng/Packages.Data.props b/eng/Packages.Data.props
@@ -11,7 +11,7 @@
     <PackageReference Update="ApprovalUtilities" Version="3.0.22" />
     <PackageReference Update="AutoRest.CSharp.V3" Version="1.0.0-alpha.20201123.1" />
     <PackageReference Update="Azure.AI.FormRecognizer" Version="3.0.0" />
-		<PackageReference Update="Azure.AI.TextAnalytics" Version="5.0.0" />
+    <PackageReference Update="Azure.AI.TextAnalytics" Version="5.0.0" />
     <PackageReference Update="Azure.Data.AppConfiguration" Version="1.0.0" />
     <PackageReference Update="Azure.Core" Version="1.6.0" />
     <PackageReference Update="Azure.Core.Amqp" Version="1.0.0" />
@@ -180,9 +180,9 @@
     <PackageReference Update="Microsoft.Extensions.PlatformAbstractions" Version="1.1.0" />
 
     <PackageReference Update="Microsoft.Extensions.DependencyInjection" Version="2.1.0" />
-	
+
     <PackageReference Update="Microsoft.Azure.Devices.Client" Version="1.27.0" />
-	
+
     <!-- Mgmt sdk packages-->
     <PackageReference Update="Azure.ResourceManager.Resources" Version="1.0.0-preview.2" />
     <PackageReference Update="Azure.ResourceManager.Compute" Version="1.0.0-preview.2" />

diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml
@@ -150,7 +150,8 @@ jobs:
           --logger "trx;LogFileName=$(TestTargetFramework).trx" --logger:"console;verbosity=normal"
           /p:ServiceDirectory=${{parameters.ServiceToTest}}
           /p:IncludeSrc=false /p:IncludeSamples=false /p:IncludePerf=false /p:IncludeStress=false
-          /p:Configuration=$(BuildConfiguration) $(ConvertToProjectReferenceOption) /p:CollectCoverage=$(CollectCoverage)
+          /p:Configuration=$(BuildConfiguration) $(ConvertToProjectReferenceOption)
+          /p:CollectCoverage=$(CollectCoverage) /p:CodeCoverageDirectory=${{parameters.ServiceDirectory}}
         displayName: "Build & Test ($(TestTargetFramework))"
         env:
           DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
@@ -165,14 +166,16 @@ jobs:
           mergeTestResults: true
       - task: Palmmedia.reportgenerator.reportgenerator-build-release-task.reportgenerator@4
         condition: and(succeededOrFailed(), eq(variables['CollectCoverage'], 'true'))
-        displayName: ReportGenerator
+        displayName: Generate Code Coverage Reports
         inputs:
-          reports: 'sdk/**/coverage.cobertura.xml'
-          targetdir: '$(Build.SourcesDirectory)'
-          reporttypes: 'Cobertura;HtmlSummary'
+          reports: sdk/**/coverage.cobertura.xml
+          targetdir: $(Build.SourcesDirectory)
+          reporttypes: Cobertura;HtmlSummary
+          filefilters: >-
+            +sdk/${{parameters.ServiceDirectory}}
       - task: PublishCodeCoverageResults@1
         condition: and(succeededOrFailed(), eq(variables['CollectCoverage'], 'true'))
-        displayName: 'Publish code coverage report'
+        displayName: Publish Code Coverage Reports
         inputs:
-          codeCoverageTool: 'Cobertura'
-          summaryFileLocation: 'Cobertura.xml'
+          codeCoverageTool: Cobertura
+          summaryFileLocation: Cobertura.xml
diff --git a/eng/service.proj b/eng/service.proj
@@ -9,6 +9,9 @@
     <IncludeStress Condition="'$(IncludeStress)' == ''">true</IncludeStress>
     <IncludeSamplesApplications Condition="'$(IncludeSamplesApplications)' == ''">true</IncludeSamplesApplications>
     <IncludeSamplesApplications Condition="'$(ServiceDirectory)' != '*' or '$(IncludeSamples)' == 'false'">false</IncludeSamplesApplications>
+    <TraversalGlobalProperties>
+      CodeCoverageDirectory=$([System.IO.Path]::GetFullPath("$(CodeCoverageDirectory)", "$(MSBuildThisFileDirectory)..\sdk"));
+    </TraversalGlobalProperties>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/sdk/keyvault/Azure.Security.KeyVault.Certificates/tests/CertificatesTestBase.cs b/sdk/keyvault/Azure.Security.KeyVault.Certificates/tests/CertificatesTestBase.cs
@@ -17,7 +17,7 @@ namespace Azure.Security.KeyVault.Certificates.Tests
         CertificateClientOptions.ServiceVersion.V7_0,
         CertificateClientOptions.ServiceVersion.V7_1)]
     [NonParallelizable]
-    public class CertificatesTestBase : RecordedTestBase<KeyVaultTestEnvironment>
+    public abstract class CertificatesTestBase : RecordedTestBase<KeyVaultTestEnvironment>
     {
         protected readonly TimeSpan PollingInterval = TimeSpan.FromSeconds(5);
         private readonly CertificateClientOptions.ServiceVersion _serviceVersion;

diff --git a/sdk/keyvault/samples/Directory.Build.props b/sdk/keyvault/samples/Directory.Build.props
@@ -3,6 +3,7 @@
     <!-- Signal that samples are building in the repo as opposed to a standalone download from Samples Browser -->
     <IsSample>true</IsSample>
     <IsPackable>false</IsPackable>
+    <ExcludeFromCodeCoverage>true</ExcludeFromCodeCoverage>
     <WarnOnPackingNonPackableProject>false</WarnOnPackingNonPackableProject>
   </PropertyGroup>