[ci] Improvements for emulator test jobs

The setup execution time for non-device tests has been improved by
moving a handful of adb commands from BaseTest to DeviceTest setup.
These would run for nearly every test run and result in a lot of waiting
due to some RunProcess failures and timeouts.

The output from the shell command we run to check if a device is online
is now cached and only refreshed when required by certain tests/asserts.

The AssertHasDevices check has been consolidated into DeviceTest setup,
and removed from individual tests in most cases.

Attempts to restart the emulator if a test determines that it is
inaccessible have been fixed, and emulator data will not be reset in this
case.

I played around with a handful of emulator launch settings and found
that removing the no-boot-anim and headless UI options improved the
reliability of nightly LocalizationTests, and resulted in faster boot
times locally. These options are now disabled for those tests.

The DeploymentTest class has been replaced by TimeZoneInfoTests and
LocalizationTests, and the other tests in that class have been moved to
InstallAndRunTests. TimeZoneInfoTests and LocalizationTests will now
validate command line output rather than trying to press a button and
read a UI element. This should make them more reliable and faster.

The TimeZoneInfoTests and LocalizationTests suites have been moved into
separate test stages in the nightly test job.

All nightly tests now run against our .NET build/test artifacts rather
than classic XA.

An issue that resulted in LocalizationTests nodes 11 and 12 not
containing any tests has been fixed.

An issue that would cause LocalizationTests node 1 to run tests from
node 11 and node 12 has been fixed.

Test result attachments have been fixed for TimeZoneInfoTests and
LocalizationTests, ensuring that we capture the right logcat and build
files for each test variant.

An issue causing binlog files to be overwritten by tests that use
multiple build targets has been fixed by naming the binlog file after
the build log file.

Console output verbosity has been set to normal for dotnet test
invocations, which should help with debugging.

Test NUnit packages have been updated to their latest versions.

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/StartAndroidEmulator.cs

@@ -30,11 +30,10 @@ public class StartAndroidEmulator : Task
 		public                  string          AvdManagerHome  {get; set;}
 		public                  string          Port            {get; set;}
 		public                  string          ImageName       {get; set;} = "XamarinAndroidTestRunner64";
-		public			string		Arguments	{get; set;}
+		public                  string          Arguments       {get; set;}
 		public                  string          ToolPath        {get; set;}
 		public                  string          ToolExe         {get; set;}
 		public                  string          LogcatFile      {get; set;}
-		public                  bool            ShowWindow      {get; set;} = true;
 
 		public override bool Execute ()
 		{
@@ -69,10 +68,9 @@ void Run (string emulator)
 			if (emulator == null)
 				return;
 
-			var port = string.IsNullOrEmpty (Port) ? "" : $" -port {Port}";
-			var showWindow = ShowWindow ? "" : " -no-window";
-			var arguments = $"{Arguments ?? string.Empty} -verbose -detect-image-hang -logcat-output \"{LogcatFile}\" -no-boot-anim -no-audio -no-snapshot -cache-size 512 -change-locale en-US -timezone \"Etc/UTC\" {showWindow}{port} -avd {ImageName}";
-			Log.LogMessage (MessageImportance.Low, $"Tool {emulator} execution started with arguments: {arguments}");
+			var port = string.IsNullOrEmpty (Port) ? "" : $"-port {Port}";
+			var arguments = $"{Arguments ?? string.Empty} -verbose -detect-image-hang -logcat-output \"{LogcatFile}\" -no-audio -no-snapshot -cache-size 512 -change-locale en-US -timezone \"Etc/UTC\" {port} -avd {ImageName}";
+			Log.LogMessage ($"Tool {emulator} execution started with arguments: {arguments}");
 			var psi = new ProcessStartInfo () {
 				FileName                = emulator,
 				Arguments               = arguments,

build-tools/automation/yaml-templates/run-systemapp-tests.yaml → build-tools/automation/yaml-templates/run-emulator-tests.yaml

@@ -1,12 +1,17 @@
-# Runs TimeZoneInfo tests against an emulator running on macOS
+# Runs a test or set of tests on an emulator running on macOS
+
+parameters:
+  emulatorMSBuildArgs: ''
+  jobName: CheckTimeZoneInfoIsCorrectNode1
+  jobTimeout: 360
+  testSteps: []
 
 jobs:
-  - job: mac_systemapp_tests
-    displayName: System App Emulator Tests
+  - job: mac_${{ parameters.jobName }}_tests
+    displayName: ${{ parameters.jobName }} Emulator Tests
     pool:
       vmImage: $(HostedMacImage)
-    timeoutInMinutes: 120
-    cancelTimeoutInMinutes: 5
+    timeoutInMinutes: ${{ parameters.jobTimeout }}
     workspace:
       clean: all
     steps:
@@ -27,14 +32,9 @@ jobs:
         solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
         configuration: $(XA.Build.Configuration)
         msbuildArguments: >-
-          /t:AcquireAndroidTarget /p:TestEmulatorArguments=-writable-system /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
+          /t:AcquireAndroidTarget ${{ parameters.emulatorMSBuildArgs }} /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
 
-    - template: run-nunit-tests.yaml
-      parameters:
-        testRunTitle: System App On Device - macOS
-        testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/net472/MSBuildDeviceIntegration.dll
-        nunitConsoleExtraArgs: --where "cat == SystemApplication"
-        testResultsFile: TestResult-SystemApp--$(XA.Build.Configuration).xml
+    - ${{ parameters.testSteps }}
 
     - task: MSBuild@1
       displayName: shut down emulator
@@ -48,6 +48,6 @@ jobs:
 
     - template: upload-results.yaml
       parameters:
-        artifactName: Test Results - System App With Emulator - macOS
+        artifactName: Test Results - ${{ parameters.jobName }} With Emulator - macOS
 
-    - template: fail-on-issue.yaml
+    - template: fail-on-issue.yaml

build-tools/scripts/NUnitReferences.projitems

@@ -4,10 +4,6 @@
     <PackageReference Include="NUnit"               Version="3.13.3" />
     <PackageReference Include="NUnit.ConsoleRunner" Version="$(NUnitConsoleVersion)" />
     <PackageReference Include="NUnit3TestAdapter"   Version="4.3.1" />
-  </ItemGroup>
-  <!-- Required packages for .NET Core -->
-  <ItemGroup Condition=" '$(TargetFramework)' != 'net472' and '$(TargetFramework)' != 'netstandard2.0' ">
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0-preview-20221003-04" />
-    <PackageReference Include="System.Configuration.ConfigurationManager" Version="6.0.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
   </ItemGroup>
 </Project>

build-tools/scripts/TestApks.targets

@@ -21,12 +21,13 @@
     <TestAvdAbi Condition=" '$(TestAvdAbi)' == '' and '$(HostOS)' == 'Darwin' and  '$(HostOSArchitecture)' == 'Arm64' ">arm64-v8a</TestAvdAbi>
     <TestAvdAbi Condition=" '$(TestAvdAbi)' == '' ">x86_64</TestAvdAbi>
     <TestAvdType Condition=" '$(TestAvdType)' == '' ">default</TestAvdType>
+    <TestAvdForceCreation Condition=" '$(TestAvdForceCreation)' == '' ">true</TestAvdForceCreation>
+    <TestAvdShowWindow Condition=" '$(TestAvdShowWindow)' == '' and '$(RunningOnCI)' == 'true' ">false</TestAvdShowWindow>
+    <TestAvdExtraBootArgs Condition=" '$(TestAvdShowWindow)' == 'false' ">-no-window -no-boot-anim $(TestAvdExtraBootArgs)</TestAvdExtraBootArgs>
     <TestDeviceName Condition=" '$(TestDeviceName)' == '' ">pixel_4</TestDeviceName>
     <SdkManagerImageName Condition=" '$(SdkManagerImageName)' == '' ">system-images;android-$(TestAvdApiLevel);$(TestAvdType);$(TestAvdAbi)</SdkManagerImageName>
     <TestAvdName>XamarinAndroidTestRunner$(TestAvdApiLevel)-$(TestAvdAbi)</TestAvdName>
     <_AdbEmulatorPort>5570</_AdbEmulatorPort>
-    <_AdbEmulatorShowWindow Condition=" '$(RunningOnCI)' == 'True' And '$(_AdbEmulatorShowWindow)' == '' ">False</_AdbEmulatorShowWindow>
-    <_AdbEmulatorShowWindow Condition=" '$(_AdbEmulatorShowWindow)' == '' ">True</_AdbEmulatorShowWindow>
     <_ApkSizesReferenceDirectory>$(MSBuildThisFileDirectory)..\..\tests\apk-sizes-reference</_ApkSizesReferenceDirectory>
     <AvdLaunchTimeoutMinutes Condition=" '$(AvdLaunchTimeoutMinutes)' == '' ">10</AvdLaunchTimeoutMinutes>
     <AvdLaunchTimeoutSeconds>$([MSBuild]::Multiply($(AvdLaunchTimeoutMinutes), 60))</AvdLaunchTimeoutSeconds>
@@ -56,7 +57,7 @@
       <Output TaskParameter="IsValidTarget" PropertyName="_ValidAdbTarget"  />
     </Xamarin.Android.Tools.BootstrapTasks.CheckAdbTarget>
     <CreateAndroidEmulator
-        Condition=" '$(_ValidAdbTarget)' != 'True' "
+        Condition=" '$(_ValidAdbTarget)' != 'True' and ( '$(TestAvdForceCreation)' == 'true' or !Exists('$(AvdManagerHome)\.android\avd\$(TestAvdName).avd') ) "
         AndroidAbi="$(TestAvdAbi)"
         AvdManagerHome="$(AvdManagerHome)"
         JavaSdkHome="$(JavaSdkDirectory)"
@@ -74,11 +75,10 @@
         Condition=" '$(_ValidAdbTarget)' != 'True' "
         AndroidSdkDirectory="$(AndroidSdkDirectory)"
         AvdManagerHome="$(AvdManagerHome)"
-        Arguments="$(TestEmulatorArguments)"
+        Arguments="$(TestAvdExtraBootArgs)"
         ImageName="$(TestAvdName)"
         LogcatFile="$(_LogcatFilenameBase)-full.txt"
         Port="$(_AdbEmulatorPort)"
-        ShowWindow="$(_AdbEmulatorShowWindow)"
         ToolExe="$(EmulatorToolExe)"
         ToolPath="$(EmulatorToolPath)">
       <Output TaskParameter="AdbTarget" PropertyName="_AdbTarget" />
@@ -124,6 +124,9 @@
   </Target>
 
   <Target Name="ReleaseAndroidTarget">
+    <PropertyGroup>
+      <_EmuTarget Condition=" '$(_EmuTarget)' == '' ">-s emulator-$(_AdbEmulatorPort)</_EmuTarget>
+    </PropertyGroup>
     <Xamarin.Android.Tools.BootstrapTasks.Adb
         Condition="'@(_FailedComponent)' != ''"
         ContinueOnError="ErrorAndContinue"
@@ -141,7 +144,7 @@
         Timeout="60000"
     />
     <KillProcess
-        Condition=" '$(_EmuTarget)' != '' "
+        Condition=" '$(_EmuPid)' != '' "
         ContinueOnError="ErrorAndContinue"
         ProcessId="$(_EmuPid)"
     />

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs

@@ -25,16 +25,6 @@ public class BaseTest
 		[SetUpFixture]
 		public class SetUp
 		{
-			public static string DeviceAbi {
-				get;
-				private set;
-			}
-
-			public static int DeviceSdkVersion {
-				get;
-				private set;
-			}
-
 			public static string TestDirectoryRoot {
 				get;
 				private set;
@@ -44,41 +34,6 @@ public static string TestDirectoryRoot {
 			public void BeforeAllTests ()
 			{
 				TestDirectoryRoot = XABuildPaths.TestOutputDirectory;
-
-				try {
-					DeviceSdkVersion = GetSdkVersion ();
-					if (DeviceSdkVersion != -1) {
-						if (DeviceSdkVersion >= 21)
-							DeviceAbi = RunAdbCommand ("shell getprop ro.product.cpu.abilist64").Trim ();
-
-						if (string.IsNullOrEmpty (DeviceAbi))
-							DeviceAbi = RunAdbCommand ("shell getprop ro.product.cpu.abi") ?? RunAdbCommand ("shell getprop ro.product.cpu.abi2");
-
-						if (DeviceAbi.Contains (",")) {
-							DeviceAbi = DeviceAbi.Split (',')[0];
-						}
-					}
-				} catch (Exception ex) {
-					Console.Error.WriteLine ("Failed to determine whether there is Android target emulator or not: " + ex);
-				}
-			}
-
-			int GetSdkVersion ()
-			{
-				var command = $"shell getprop ro.build.version.sdk";
-				var result = RunAdbCommand (command);
-				if (result.Contains ("*")) {
-					// Run the command again, we likely got:
-					// * daemon not running; starting now at tcp:5037
-					// * daemon started successfully
-					// adb.exe: device offline
-					TestContext.WriteLine ($"Retrying:\n{command}\n{result}");
-					result = RunAdbCommand (command);
-				}
-				if (!int.TryParse (result, out var sdkVersion)) {
-					sdkVersion = -1;
-				}
-				return sdkVersion;
 			}
 
 			[OneTimeTearDown]
@@ -208,7 +163,7 @@ protected static string RunAdbCommand (string command, bool ignoreErrors = true,
 			string ext = Environment.OSVersion.Platform != PlatformID.Unix ? ".exe" : "";
 			string adb = Path.Combine (AndroidSdkPath, "platform-tools", "adb" + ext);
 			string adbTarget = Environment.GetEnvironmentVariable ("ADB_TARGET");
-			return RunProcess (adb, $"{adbTarget} {command}");
+			return RunProcess (adb, $"{adbTarget} {command}", timeout);
 		}
 
 		protected static (int code, string stdOutput, string stdError) RunApkDiffCommand (string args)
@@ -225,9 +180,9 @@ protected static (int code, string stdOutput, string stdError) RunApkDiffCommand
 			}
 		}
 
-		protected static string RunProcess (string exe, string args)
+		protected static string RunProcess (string exe, string args, int timeoutInSeconds = 30)
 		{
-			var (_, stdOutput, stdError) = RunProcessWithExitCode (exe, args);
+			var (_, stdOutput, stdError) = RunProcessWithExitCode (exe, args, timeoutInSeconds);
 
 			return stdOutput + stdError;
 		}