[ci] Run nunit tests with stable .NET version

Attempt to improve desktop test execution reliability by running our test assemblies with a stable version of .NET by default. The default value of the `useDotNet` yaml template parameters have been flipped as part of continued efforts to remove classic test runs. .NET 7.0 SDK provisioning has been updated to install the latest GA version now that it is stable.
dotnet · Feb 24, 2023 · e3fe64a · e3fe64a
1 parent c8b2cc0
commit e3fe64a
Show file tree

Hide file tree

Showing 8 changed files with 40 additions and 18 deletions.
diff --git a/build-tools/automation/azure-pipelines-nightly.yaml b/build-tools/automation/azure-pipelines-nightly.yaml
@@ -142,6 +142,7 @@ stages:
         extraBuildArgs: /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) /p:TestAvdType=$(avdType)
         artifactSource: bin/Test$(XA.Build.Configuration)/Mono.Android_Tests-Signed.apk
         artifactFolder: Default
+        useDotNet: false
 
     - task: MSBuild@1
       displayName: shut down emulator

diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml
@@ -132,6 +132,7 @@ stages:
         testResultsFiles: TestResult-Xamarin.Android.JcwGen_Tests-$(ApkTestConfiguration).xml
         artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Android.JcwGen_Tests-Signed.apk
         artifactFolder: Default
+        useDotNet: false
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -142,6 +143,7 @@ stages:
         artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Android.JcwGen_Tests-Signed.apk
         artifactFolder: FastDev_Assemblies_Dexes
         extraBuildArgs: /p:AndroidFastDeploymentType=Assemblies:Dexes
+        useDotNet: false
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -151,6 +153,7 @@ stages:
         testResultsFiles: TestResult-Xamarin.Android.Locale_Tests-$(ApkTestConfiguration).xml
         artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Android.Locale_Tests-Signed.apk
         artifactFolder: Default
+        useDotNet: false
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -161,6 +164,7 @@ stages:
         extraBuildArgs: /p:AotAssemblies=True
         artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Android.Locale_Tests-Signed.apk
         artifactFolder: Aot
+        useDotNet: false
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -171,6 +175,7 @@ stages:
         extraBuildArgs: /p:AotAssemblies=True /p:AndroidEnableProfiledAot=true
         artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Android.Locale_Tests-Signed.apk
         artifactFolder: Profiled-Aot
+        useDotNet: false
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -180,6 +185,7 @@ stages:
         testResultsFiles: TestResult-Xamarin.Android.EmbeddedDSO_Test.nunit-$(ApkTestConfiguration).xml
         artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Android.EmbeddedDSO_Test-Signed.apk
         artifactFolder: Default
+        useDotNet: false
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -189,6 +195,7 @@ stages:
         testResultsFiles: TestResult-apkdiff-Xamarin.Forms_Performance_Integration-Signed-$(ApkTestConfiguration).xml
         artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Forms_Performance_Integration-Signed.apk
         artifactFolder: Default
+        useDotNet: false
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -199,6 +206,7 @@ stages:
         extraBuildArgs: /p:AotAssemblies=true
         artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Forms_Performance_Integration-Signed.apk
         artifactFolder: Aot
+        useDotNet: false
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -209,6 +217,7 @@ stages:
         extraBuildArgs: /p:AotAssemblies=True /p:AndroidEnableProfiledAot=true
         artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Forms_Performance_Integration-Signed.apk
         artifactFolder: Profiled-Aot
+        useDotNet: false
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -219,6 +228,7 @@ stages:
         extraBuildArgs: /p:BundleAssemblies=true
         artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Forms_Performance_Integration-Signed.apk
         artifactFolder: Bundle
+        useDotNet: false
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -228,6 +238,7 @@ stages:
         testResultsFiles: TestResult-apkdiff-com.companyname.vsandroidapp-Signed-$(ApkTestConfiguration).xml
         artifactSource: bin/Test$(ApkTestConfiguration)/com.companyname.vsandroidapp-Signed.apk
         artifactFolder: Default
+        useDotNet: false
 
     - task: MSBuild@1
       displayName: shut down emulator
@@ -347,7 +358,6 @@ stages:
         testResultsFiles: TestResult-Mono.Android.NET_Tests-$(XA.Build.Configuration).xml
         artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.aab
         artifactFolder: $(DotNetTargetFramework)-$(XA.Build.Configuration)
-        useDotNet: true
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -358,7 +368,6 @@ stages:
         testResultsFiles: TestResult-Mono.Android.NET_Tests-Debug.xml
         artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.apk
         artifactFolder: $(DotNetTargetFramework)-Debug
-        useDotNet: true
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -369,7 +378,6 @@ stages:
         extraBuildArgs: -p:TestsFlavor=NoAab -p:AndroidPackageFormat=apk
         artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.apk
         artifactFolder: $(DotNetTargetFramework)-NoAab
-        useDotNet: true
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -380,7 +388,6 @@ stages:
         extraBuildArgs: -p:TestsFlavor=Interpreter -p:UseInterpreter=True
         artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.aab
         artifactFolder: $(DotNetTargetFramework)-Interpreter
-        useDotNet: true
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -391,7 +398,6 @@ stages:
         extraBuildArgs: -p:TestsFlavor=NoAot -p:RunAOTCompilation=false
         artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.aab
         artifactFolder: $(DotNetTargetFramework)-NoAot
-        useDotNet: true
 
     - template: yaml-templates/apk-instrumentation.yaml
       parameters:
@@ -402,7 +408,6 @@ stages:
         extraBuildArgs: -p:TestsFlavor=AotLlvm -p:EnableLLVM=true -p:AndroidEnableProfiledAot=false
         artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.aab
         artifactFolder: $(DotNetTargetFramework)-AotLlvm
-        useDotNet: true
 
     - task: MSBuild@1
       displayName: shut down emulator
@@ -497,7 +502,6 @@ stages:
 
     - template: yaml-templates/run-nunit-tests.yaml
       parameters:
-        useDotNet: true
         testRunTitle: MSBuildDeviceIntegration Smoke - macOS
         testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
         dotNetTestExtraArgs: --filter "TestCategory = SmokeTests $(DotNetNUnitCategories)"
@@ -546,7 +550,6 @@ stages:
 
     - template: yaml-templates/run-nunit-tests.yaml
       parameters:
-        useDotNet: true
         testRunTitle: Xamarin.Android.Build.Tests - Linux .NET 6 Smoke Tests
         testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Build.Tests.dll
         dotNetTestExtraArgs: --filter "TestCategory = SmokeTests $(DotNetNUnitCategories)"
@@ -739,7 +742,6 @@ stages:
 
     - template: yaml-templates/run-nunit-tests.yaml
       parameters:
-        useDotNet: true
         testRunTitle: WearOS On Device - macOS
         testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
         dotNetTestExtraArgs: --filter "TestCategory = WearOS"
@@ -805,6 +807,7 @@ stages:
         testResultsFiles: TestResult-Xamarin.Android.Bcl_Tests.nunit-$(XA.Build.Configuration).xml
         artifactSource: bin/Test$(XA.Build.Configuration)/Xamarin.Android.Bcl_Tests-Signed.apk
         artifactFolder: Default
+        useDotNet: false
 
     - task: PublishTestResults@2
       displayName: publish Xamarin.Android.Bcl-Tests-XUnit results

diff --git a/build-tools/automation/yaml-templates/apk-instrumentation.yaml b/build-tools/automation/yaml-templates/apk-instrumentation.yaml
@@ -9,7 +9,7 @@ parameters:
   testResultsFormat: NUnit
   artifactSource: ""
   artifactFolder: ""
-  useDotNet: false
+  useDotNet: true
   condition: succeeded()
 
 steps:

diff --git a/build-tools/automation/yaml-templates/build-windows.yaml b/build-tools/automation/yaml-templates/build-windows.yaml
@@ -116,14 +116,14 @@ stages:
 
     - template: run-nunit-tests.yaml
       parameters:
+        useDotNet: false
         testRunTitle: Smoke MSBuild Tests - Windows Build Tree
         testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\net472\Xamarin.Android.Build.Tests.dll
         testResultsFile: TestResult-SmokeMSBuildTests-WinBuildTree-$(XA.Build.Configuration).xml
         nunitConsoleExtraArgs: --where "cat == SmokeTests"
 
     - template: run-nunit-tests.yaml
       parameters:
-        useDotNet: true
         testRunTitle: Smoke MSBuild Tests - Windows Dotnet Build
         testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\$(DotNetStableTargetFramework)\Xamarin.Android.Build.Tests.dll
         testResultsFile: TestResult-SmokeMSBuildTests-WinDotnetBuild-$(XA.Build.Configuration).xml

diff --git a/build-tools/automation/yaml-templates/run-localization-tests.yaml b/build-tools/automation/yaml-templates/run-localization-tests.yaml
@@ -36,7 +36,6 @@ jobs:
       parameters:
         testRunTitle: LocalizationTests On Device - macOS
         testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
-        useDotNet: true
         dotNetTestExtraArgs:  --filter "Name~CheckLocalizationIsCorrectNode${{ parameters.node_id }}"
         testResultsFile: TestResult-LocalizationTests-Node${{ parameters.node_id }}-$(XA.Build.Configuration).xml
 

diff --git a/build-tools/automation/yaml-templates/run-nunit-tests.yaml b/build-tools/automation/yaml-templates/run-nunit-tests.yaml
@@ -6,12 +6,15 @@ parameters:
   testResultsFile: TestResult.xml
   nunitConsoleExtraArgs: ''
   dotNetTestExtraArgs: ''
-  useDotNet: false
+  useDotNet: true
+  useDotNetPreview: false
   workers: $(NUnit.NumberOfTestWorkers)
   condition: succeeded()
+  timeoutInMinutes: 0
+  retryCountOnTaskFailure: 0
 
 steps:
-- ${{ if eq(parameters.useDotNet, false) }}:
+- ${{ if and(eq(parameters.useDotNet, false), eq(parameters.useDotNetPreview, false)) }}:
   - powershell: |
       Write-Host '##vso[task.setvariable variable=TestResultsFormat]NUnit'
       if ([Environment]::OSVersion.Platform -eq "Unix") {
@@ -28,7 +31,7 @@ steps:
     condition: ${{ parameters.condition }}
     continueOnError: true
 
-- ${{ if eq(parameters.useDotNet, true) }}:
+- ${{ if and(eq(parameters.useDotNet, true), eq(parameters.useDotNetPreview, true)) }}:
   - powershell: Write-Host '##vso[task.setvariable variable=TestResultsFormat]VSTest'
   - template: run-dotnet-preview.yaml
     parameters:
@@ -43,11 +46,26 @@ steps:
       displayName: run ${{ parameters.testRunTitle }}
       condition: ${{ parameters.condition }}
 
+- ${{ if and(eq(parameters.useDotNet, true), eq(parameters.useDotNetPreview, false)) }}:
+  - task: DotNetCoreCLI@2
+    inputs:
+      command: test
+      projects: ${{ parameters.testAssembly }}
+      arguments: >-
+        ${{ parameters.dotNetTestExtraArgs }} -- NUnit.NumberOfTestWorkers=${{ parameters.workers }}
+      publishTestResults: true
+      testRunTitle: ${{ parameters.testRunTitle }}
+    displayName: run ${{ parameters.testRunTitle }}
+    condition: ${{ parameters.condition }}
+    continueOnError: true
+    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+    retryCountOnTaskFailure: ${{ parameters.retryCountOnTaskFailure }}
+
 - template: kill-processes.yaml
 
 - task: PublishTestResults@2
   inputs:
     testResultsFormat: $(TestResultsFormat)
     testResultsFiles: ${{ parameters.testResultsFile }}
     testRunTitle: ${{ parameters.testRunTitle }}
-  condition: ${{ parameters.condition }}
+  condition: and(${{ parameters.condition }}, or(ne('${{ parameters.useDotNet }}', 'true'), eq('${{ parameters.useDotNetPreview }}', 'true')))
diff --git a/build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml b/build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml
@@ -34,6 +34,7 @@ jobs:
 
     - template: run-nunit-tests.yaml
       parameters:
+        useDotNet: false
         testRunTitle: TimeZoneInfoTests On Device - macOS
         testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/net472/MSBuildDeviceIntegration.dll
         nunitConsoleExtraArgs: --where "test == Xamarin.Android.Build.Tests.DeploymentTest.CheckTimeZoneInfoIsCorrectNode${{ parameters.node_id }}"

diff --git a/build-tools/automation/yaml-templates/variables.yaml b/build-tools/automation/yaml-templates/variables.yaml
@@ -18,9 +18,9 @@ variables:
 - name: NUnit.NumberOfTestWorkers
   value: 4
 - name: DotNetSdkVersion
-  value: 7.0.1xx
+  value: 7.0
 - name: DotNetSdkQuality
-  value: preview
+  value: GA
 - name: GitHub.Token
   value: $(github--pat--vs-mobiletools-engineering-service2)
 - name: HostedMacImage