[ci] Improvements for emulator test jobs (#7731)

The following changes attempt to improve the reliability and performance of our MSBuild test jobs. All nightly tests have been updated to run against our .NET build/test artifacts rather than classic XA. The setup execution time for non-device tests has been improved by moving a handful of adb commands from BaseTest to DeviceTest setup. These would run for nearly every test run and result in a lot of waiting due to some RunProcess failures and timeouts. The output from the shell command we run to check if a device is online is now cached and only refreshed when required by certain tests/asserts. The `AssertHasDevices` check has been moved into `DeviceTest` setup, and removed from individual tests in most cases. Attempts to restart the emulator if a test determines that it is inaccessible have been fixed, and emulator data will not be reset in this case. I played around with a handful of emulator launch settings and found that removing the no-boot-anim and headless UI options improved the reliability of nightly launch tests, and resulted in faster boot times locally. These options are now disabled for those tests. The `DeploymentTest` class has been replaced by `TimeZoneInfoTests` and `LocalizationTests`, and the other tests in that class have been moved to `InstallAndRunTests`. `TimeZoneInfoTests` and `LocalizationTests` will now validate command line output rather than trying to press a button and read a UI element. This should make them more reliable and faster. The `TimeZoneInfoTests` and `LocalizationTests` suites have been moved into separate test stages in the nightly test job. An issue that caused LocalizationTests nodes 11 and 12 to not contain any tests has been fixed. An issue that caused LocalizationTests node 1 to also run tests from node 11 and node 12 has been fixed. Test result attachments have been fixed for `TimeZoneInfoTests` and `LocalizationTests`, ensuring that we capture the right logcat and build files for each test variant. An issue causing binlog files to be overwritten by tests that use multiple build targets has been fixed by naming the binlog file after the build log file. Console output verbosity has been set to normal for dotnet test invocations, which should help with debugging. Test NUnit packages have been updated to their latest versions.
dotnet · Mar 9, 2023 · fa9b99f · fa9b99f
1 parent fe00f48
commit fa9b99f
Show file tree

Hide file tree

Showing 28 changed files with 1,086 additions and 900 deletions.
diff --git a/...Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/StartAndroidEmulator.cs b/...Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/StartAndroidEmulator.cs
@@ -30,11 +30,10 @@ public class StartAndroidEmulator : Task
 		public                  string          AvdManagerHome  {get; set;}
 		public                  string          Port            {get; set;}
 		public                  string          ImageName       {get; set;} = "XamarinAndroidTestRunner64";
-		public			string		Arguments	{get; set;}
+		public                  string          Arguments       {get; set;}
 		public                  string          ToolPath        {get; set;}
 		public                  string          ToolExe         {get; set;}
 		public                  string          LogcatFile      {get; set;}
-		public                  bool            ShowWindow      {get; set;} = true;
 
 		public override bool Execute ()
 		{
@@ -69,10 +68,9 @@ void Run (string emulator)
 			if (emulator == null)
 				return;
 
-			var port = string.IsNullOrEmpty (Port) ? "" : $" -port {Port}";
-			var showWindow = ShowWindow ? "" : " -no-window";
-			var arguments = $"{Arguments ?? string.Empty} -verbose -detect-image-hang -logcat-output \"{LogcatFile}\" -no-boot-anim -no-audio -no-snapshot -cache-size 512 -change-locale en-US -timezone \"Etc/UTC\" {showWindow}{port} -avd {ImageName}";
-			Log.LogMessage (MessageImportance.Low, $"Tool {emulator} execution started with arguments: {arguments}");
+			var port = string.IsNullOrEmpty (Port) ? "" : $"-port {Port}";
+			var arguments = $"{Arguments ?? string.Empty} -verbose -detect-image-hang -logcat-output \"{LogcatFile}\" -no-audio -no-snapshot -cache-size 512 -change-locale en-US -timezone \"Etc/UTC\" {port} -avd {ImageName}";
+			Log.LogMessage ($"Tool {emulator} execution started with arguments: {arguments}");
 			var psi = new ProcessStartInfo () {
 				FileName                = emulator,
 				Arguments               = arguments,

diff --git a/build-tools/automation/azure-pipelines-nightly.yaml b/build-tools/automation/azure-pipelines-nightly.yaml
diff --git a/...n/yaml-templates/run-systemapp-tests.yaml → ...on/yaml-templates/run-emulator-tests.yaml b/...n/yaml-templates/run-systemapp-tests.yaml → ...on/yaml-templates/run-emulator-tests.yaml
@@ -1,12 +1,17 @@
-# Runs TimeZoneInfo tests against an emulator running on macOS
+# Runs a test or set of tests on an emulator running on macOS
+
+parameters:
+  emulatorMSBuildArgs: ''
+  jobName: CheckTimeZoneInfoIsCorrectNode1
+  jobTimeout: 360
+  testSteps: []
 
 jobs:
-  - job: mac_systemapp_tests
-    displayName: System App Emulator Tests
+  - job: mac_${{ parameters.jobName }}_tests
+    displayName: ${{ parameters.jobName }} Emulator Tests
     pool:
       vmImage: $(HostedMacImage)
-    timeoutInMinutes: 120
-    cancelTimeoutInMinutes: 5
+    timeoutInMinutes: ${{ parameters.jobTimeout }}
     workspace:
       clean: all
     steps:
@@ -27,14 +32,9 @@ jobs:
         solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
         configuration: $(XA.Build.Configuration)
         msbuildArguments: >-
-          /t:AcquireAndroidTarget /p:TestEmulatorArguments=-writable-system /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
+          /t:AcquireAndroidTarget ${{ parameters.emulatorMSBuildArgs }} /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
 
-    - template: run-nunit-tests.yaml
-      parameters:
-        testRunTitle: System App On Device - macOS
-        testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/net472/MSBuildDeviceIntegration.dll
-        nunitConsoleExtraArgs: --where "cat == SystemApplication"
-        testResultsFile: TestResult-SystemApp--$(XA.Build.Configuration).xml
+    - ${{ parameters.testSteps }}
 
     - task: MSBuild@1
       displayName: shut down emulator
@@ -48,6 +48,6 @@ jobs:
 
     - template: upload-results.yaml
       parameters:
-        artifactName: Test Results - System App With Emulator - macOS
+        artifactName: Test Results - ${{ parameters.jobName }} With Emulator - macOS
 
-    - template: fail-on-issue.yaml
+    - template: fail-on-issue.yaml
diff --git a/build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml b/build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml
diff --git a/build-tools/scripts/NUnitReferences.projitems b/build-tools/scripts/NUnitReferences.projitems
@@ -4,10 +4,6 @@
     <PackageReference Include="NUnit"               Version="3.13.3" />
     <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleVersion)" />
     <PackageReference Include="NUnit3TestAdapter"   Version="4.3.1" />
-  </ItemGroup>
-  <!-- Required packages for .NET Core -->
-  <ItemGroup Condition=" '$(TargetFramework)' != 'net472' and '$(TargetFramework)' != 'netstandard2.0' ">
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0-preview-20221003-04" />
-    <PackageReference Include="System.Configuration.ConfigurationManager" Version="6.0.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
   </ItemGroup>
 </Project>
diff --git a/build-tools/scripts/TestApks.targets b/build-tools/scripts/TestApks.targets
@@ -21,12 +21,13 @@
     <TestAvdAbi Condition=" '$(TestAvdAbi)' == '' and '$(HostOS)' == 'Darwin' and  '$(HostOSArchitecture)' == 'Arm64' ">arm64-v8a</TestAvdAbi>
     <TestAvdAbi Condition=" '$(TestAvdAbi)' == '' ">x86_64</TestAvdAbi>
     <TestAvdType Condition=" '$(TestAvdType)' == '' ">default</TestAvdType>
+    <TestAvdForceCreation Condition=" '$(TestAvdForceCreation)' == '' ">true</TestAvdForceCreation>
+    <TestAvdShowWindow Condition=" '$(TestAvdShowWindow)' == '' and '$(RunningOnCI)' == 'true' ">false</TestAvdShowWindow>
+    <TestAvdExtraBootArgs Condition=" '$(TestAvdShowWindow)' == 'false' ">-no-window -no-boot-anim $(TestAvdExtraBootArgs)</TestAvdExtraBootArgs>
     <TestDeviceName Condition=" '$(TestDeviceName)' == '' ">pixel_4</TestDeviceName>
     <SdkManagerImageName Condition=" '$(SdkManagerImageName)' == '' ">system-images;android-$(TestAvdApiLevel);$(TestAvdType);$(TestAvdAbi)</SdkManagerImageName>
     <TestAvdName>XamarinAndroidTestRunner$(TestAvdApiLevel)-$(TestAvdAbi)</TestAvdName>
     <_AdbEmulatorPort>5570</_AdbEmulatorPort>
-    <_AdbEmulatorShowWindow Condition=" '$(RunningOnCI)' == 'True' And '$(_AdbEmulatorShowWindow)' == '' ">False</_AdbEmulatorShowWindow>
-    <_AdbEmulatorShowWindow Condition=" '$(_AdbEmulatorShowWindow)' == '' ">True</_AdbEmulatorShowWindow>
     <_ApkSizesReferenceDirectory>$(MSBuildThisFileDirectory)..\..\tests\apk-sizes-reference</_ApkSizesReferenceDirectory>
     <AvdLaunchTimeoutMinutes Condition=" '$(AvdLaunchTimeoutMinutes)' == '' ">10</AvdLaunchTimeoutMinutes>
     <AvdLaunchTimeoutSeconds>$([MSBuild]::Multiply($(AvdLaunchTimeoutMinutes), 60))</AvdLaunchTimeoutSeconds>
@@ -56,7 +57,7 @@
       <Output TaskParameter="IsValidTarget" PropertyName="_ValidAdbTarget"  />
     </Xamarin.Android.Tools.BootstrapTasks.CheckAdbTarget>
     <CreateAndroidEmulator
-        Condition=" '$(_ValidAdbTarget)' != 'True' "
+        Condition=" '$(_ValidAdbTarget)' != 'True' and ( '$(TestAvdForceCreation)' == 'true' or !Exists('$(AvdManagerHome)\.android\avd\$(TestAvdName).avd') ) "
         AndroidAbi="$(TestAvdAbi)"
         AvdManagerHome="$(AvdManagerHome)"
         JavaSdkHome="$(JavaSdkDirectory)"
@@ -74,11 +75,10 @@
         Condition=" '$(_ValidAdbTarget)' != 'True' "
         AndroidSdkDirectory="$(AndroidSdkDirectory)"
         AvdManagerHome="$(AvdManagerHome)"
-        Arguments="$(TestEmulatorArguments)"
+        Arguments="$(TestAvdExtraBootArgs)"
         ImageName="$(TestAvdName)"
         LogcatFile="$(_LogcatFilenameBase)-full.txt"
         Port="$(_AdbEmulatorPort)"
-        ShowWindow="$(_AdbEmulatorShowWindow)"
         ToolExe="$(EmulatorToolExe)"
         ToolPath="$(EmulatorToolPath)">
       <Output TaskParameter="AdbTarget" PropertyName="_AdbTarget" />
@@ -124,6 +124,9 @@
   </Target>
 
   <Target Name="ReleaseAndroidTarget">
+    <PropertyGroup>
+      <_EmuTarget Condition=" '$(_EmuTarget)' == '' ">-s emulator-$(_AdbEmulatorPort)</_EmuTarget>
+    </PropertyGroup>
     <Xamarin.Android.Tools.BootstrapTasks.Adb
         Condition="'@(_FailedComponent)' != ''"
         ContinueOnError="ErrorAndContinue"
@@ -141,7 +144,7 @@
         Timeout="60000"
     />
     <KillProcess
-        Condition=" '$(_EmuTarget)' != '' "
+        Condition=" '$(_EmuPid)' != '' "
         ContinueOnError="ErrorAndContinue"
         ProcessId="$(_EmuPid)"
     />

diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs
@@ -25,16 +25,6 @@ public class BaseTest
 		[SetUpFixture]
 		public class SetUp
 		{
-			public static string DeviceAbi {
-				get;
-				private set;
-			}
-
-			public static int DeviceSdkVersion {
-				get;
-				private set;
-			}
-
 			public static string TestDirectoryRoot {
 				get;
 				private set;
@@ -44,41 +34,6 @@ public static string TestDirectoryRoot {
 			public void BeforeAllTests ()
 			{
 				TestDirectoryRoot = XABuildPaths.TestOutputDirectory;
-
-				try {
-					DeviceSdkVersion = GetSdkVersion ();
-					if (DeviceSdkVersion != -1) {
-						if (DeviceSdkVersion >= 21)
-							DeviceAbi = RunAdbCommand ("shell getprop ro.product.cpu.abilist64").Trim ();
-
-						if (string.IsNullOrEmpty (DeviceAbi))
-							DeviceAbi = RunAdbCommand ("shell getprop ro.product.cpu.abi") ?? RunAdbCommand ("shell getprop ro.product.cpu.abi2");
-
-						if (DeviceAbi.Contains (",")) {
-							DeviceAbi = DeviceAbi.Split (',')[0];
-						}
-					}
-				} catch (Exception ex) {
-					Console.Error.WriteLine ("Failed to determine whether there is Android target emulator or not: " + ex);
-				}
-			}
-
-			int GetSdkVersion ()
-			{
-				var command = $"shell getprop ro.build.version.sdk";
-				var result = RunAdbCommand (command);
-				if (result.Contains ("*")) {
-					// Run the command again, we likely got:
-					// * daemon not running; starting now at tcp:5037
-					// * daemon started successfully
-					// adb.exe: device offline
-					TestContext.WriteLine ($"Retrying:\n{command}\n{result}");
-					result = RunAdbCommand (command);
-				}
-				if (!int.TryParse (result, out var sdkVersion)) {
-					sdkVersion = -1;
-				}
-				return sdkVersion;
 			}
 
 			[OneTimeTearDown]
@@ -208,7 +163,7 @@ protected static string RunAdbCommand (string command, bool ignoreErrors = true,
 			string ext = Environment.OSVersion.Platform != PlatformID.Unix ? ".exe" : "";
 			string adb = Path.Combine (AndroidSdkPath, "platform-tools", "adb" + ext);
 			string adbTarget = Environment.GetEnvironmentVariable ("ADB_TARGET");
-			return RunProcess (adb, $"{adbTarget} {command}");
+			return RunProcess (adb, $"{adbTarget} {command}", timeout);
 		}
 
 		protected static (int code, string stdOutput, string stdError) RunApkDiffCommand (string args)
@@ -225,9 +180,9 @@ protected static (int code, string stdOutput, string stdError) RunApkDiffCommand
 			}
 		}
 
-		protected static string RunProcess (string exe, string args)
+		protected static string RunProcess (string exe, string args, int timeoutInSeconds = 30)
 		{
-			var (_, stdOutput, stdError) = RunProcessWithExitCode (exe, args);
+			var (_, stdOutput, stdError) = RunProcessWithExitCode (exe, args, timeoutInSeconds);
 
 			return stdOutput + stdError;
 		}