From fa9b99fde60163b2525e3de1235de1c94e23b474 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Thu, 9 Mar 2023 14:29:02 -0500 Subject: [PATCH] [ci] Improvements for emulator test jobs (#7731) The following changes attempt to improve the reliability and performance of our MSBuild test jobs. All nightly tests have been updated to run against our .NET build/test artifacts rather than classic XA. The setup execution time for non-device tests has been improved by moving a handful of adb commands from BaseTest to DeviceTest setup. These would run for nearly every test run and result in a lot of waiting due to some RunProcess failures and timeouts. The output from the shell command we run to check if a device is online is now cached and only refreshed when required by certain tests/asserts. The `AssertHasDevices` check has been moved into `DeviceTest` setup, and removed from individual tests in most cases. Attempts to restart the emulator if a test determines that it is inaccessible have been fixed, and emulator data will not be reset in this case. I played around with a handful of emulator launch settings and found that removing the no-boot-anim and headless UI options improved the reliability of nightly launch tests, and resulted in faster boot times locally. These options are now disabled for those tests. The `DeploymentTest` class has been replaced by `TimeZoneInfoTests` and `LocalizationTests`, and the other tests in that class have been moved to `InstallAndRunTests`. `TimeZoneInfoTests` and `LocalizationTests` will now validate command line output rather than trying to press a button and read a UI element. This should make them more reliable and faster. The `TimeZoneInfoTests` and `LocalizationTests` suites have been moved into separate test stages in the nightly test job. An issue that caused LocalizationTests nodes 11 and 12 to not contain any tests has been fixed. An issue that caused LocalizationTests node 1 to also run tests from node 11 and node 12 has been fixed. Test result attachments have been fixed for `TimeZoneInfoTests` and `LocalizationTests`, ensuring that we capture the right logcat and build files for each test variant. An issue causing binlog files to be overwritten by tests that use multiple build targets has been fixed by naming the binlog file after the build log file. Console output verbosity has been set to normal for dotnet test invocations, which should help with debugging. Test NUnit packages have been updated to their latest versions. --- .../StartAndroidEmulator.cs | 10 +- .../automation/azure-pipelines-nightly.yaml | 385 +++++++++++--- ...app-tests.yaml => run-emulator-tests.yaml} | 28 +- .../run-timezoneinfo-tests.yaml | 57 --- build-tools/scripts/NUnitReferences.projitems | 6 +- build-tools/scripts/TestApks.targets | 15 +- .../Utilities/BaseTest.cs | 51 +- .../Utilities/DeviceTest.cs | 122 +++-- .../Xamarin.Android.Build.Tests.csproj | 4 - .../Xamarin.ProjectTools/Common/Builder.cs | 4 +- .../Xamarin.ProjectTools/Common/DotNetCLI.cs | 11 +- .../Common/ProjectBuilder.cs | 2 +- .../Tests/AotProfileTests.cs | 2 - .../Tests/BundleToolTests.cs | 2 - .../Tests/DebuggingTest.cs | 34 +- .../Tests/DeleteBinObjTest.cs | 2 +- .../Tests/DeploymentTest.cs | 470 ------------------ .../Tests/InstallAndRunTests.cs | 231 +++++---- .../Tests/InstallTests.cs | 16 +- .../Tests/InstantRunTest.cs | 16 +- .../Tests/LocalizationTests.cs | 254 ++++++++++ .../Tests/MonoAndroidExportTest.cs | 9 +- .../Tests/PerformanceTest.cs | 4 +- .../Tests/SystemApplicationTests.cs | 1 - .../Tests/TimeZoneInfoTests.cs | 227 +++++++++ .../Tests/UncaughtExceptionTests.cs | 5 - .../Tests/XASdkDeployTests.cs | 6 - .../Mono.Android-Tests/System.Net/SslTest.cs | 12 + 28 files changed, 1086 insertions(+), 900 deletions(-) rename build-tools/automation/yaml-templates/{run-systemapp-tests.yaml => run-emulator-tests.yaml} (56%) delete mode 100644 build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml delete mode 100644 tests/MSBuildDeviceIntegration/Tests/DeploymentTest.cs create mode 100644 tests/MSBuildDeviceIntegration/Tests/LocalizationTests.cs create mode 100644 tests/MSBuildDeviceIntegration/Tests/TimeZoneInfoTests.cs diff --git a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/StartAndroidEmulator.cs b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/StartAndroidEmulator.cs index adb65136049..9fdeb494971 100644 --- a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/StartAndroidEmulator.cs +++ b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/StartAndroidEmulator.cs @@ -30,11 +30,10 @@ public class StartAndroidEmulator : Task public string AvdManagerHome {get; set;} public string Port {get; set;} public string ImageName {get; set;} = "XamarinAndroidTestRunner64"; - public string Arguments {get; set;} + public string Arguments {get; set;} public string ToolPath {get; set;} public string ToolExe {get; set;} public string LogcatFile {get; set;} - public bool ShowWindow {get; set;} = true; public override bool Execute () { @@ -69,10 +68,9 @@ void Run (string emulator) if (emulator == null) return; - var port = string.IsNullOrEmpty (Port) ? "" : $" -port {Port}"; - var showWindow = ShowWindow ? "" : " -no-window"; - var arguments = $"{Arguments ?? string.Empty} -verbose -detect-image-hang -logcat-output \"{LogcatFile}\" -no-boot-anim -no-audio -no-snapshot -cache-size 512 -change-locale en-US -timezone \"Etc/UTC\" {showWindow}{port} -avd {ImageName}"; - Log.LogMessage (MessageImportance.Low, $"Tool {emulator} execution started with arguments: {arguments}"); + var port = string.IsNullOrEmpty (Port) ? "" : $"-port {Port}"; + var arguments = $"{Arguments ?? string.Empty} -verbose -detect-image-hang -logcat-output \"{LogcatFile}\" -no-audio -no-snapshot -cache-size 512 -change-locale en-US -timezone \"Etc/UTC\" {port} -avd {ImageName}"; + Log.LogMessage ($"Tool {emulator} execution started with arguments: {arguments}"); var psi = new ProcessStartInfo () { FileName = emulator, Arguments = arguments, diff --git a/build-tools/automation/azure-pipelines-nightly.yaml b/build-tools/automation/azure-pipelines-nightly.yaml index cc6168b0f35..36cd5864453 100644 --- a/build-tools/automation/azure-pipelines-nightly.yaml +++ b/build-tools/automation/azure-pipelines-nightly.yaml @@ -58,15 +58,16 @@ stages: artifactName: Build Results - Nightly macOS includeBuildResults: true -- stage: test - displayName: Test + +- stage: test_apk + displayName: Test APKs dependsOn: mac_build variables: - - group: Xamarin-Secrets - - group: xamops-azdev-secrets + - group: Xamarin-Secrets + - group: xamops-azdev-secrets jobs: - - job: emulator_tests - displayName: macOS > Tests > APKs (Emulator) + - job: test_apk_monoandroid + displayName: macOS > Test APKs > Mono.Android timeoutInMinutes: 180 strategy: matrix: @@ -136,13 +137,12 @@ stages: - template: yaml-templates/apk-instrumentation.yaml parameters: configuration: $(XA.Build.Configuration) - testName: Mono.Android_Tests-$(avdApiLevel)-$(avdAbi)-$(avdType) - project: tests/Mono.Android-Tests/Mono.Android-Tests.csproj - testResultsFiles: TestResult-Mono.Android_Tests-$(XA.Build.Configuration).xml - extraBuildArgs: /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) /p:TestAvdType=$(avdType) - artifactSource: bin/Test$(XA.Build.Configuration)/Mono.Android_Tests-Signed.apk - artifactFolder: Default - useDotNet: false + testName: Mono.Android.NET_Tests-$(XA.Build.Configuration)-$(avdApiLevel) + project: tests/Mono.Android-Tests/Runtime-Microsoft.Android.Sdk/Mono.Android.NET-Tests.csproj + testResultsFiles: TestResult-Mono.Android.NET_Tests-$(XA.Build.Configuration)$(avdApiLevel).xml + extraBuildArgs: -p:TestsFlavor=$(avdApiLevel) -p:TestAvdApiLevel=$(avdApiLevel) -p:TestAvdAbi=$(avdAbi) -p:TestAvdType=$(avdType) + artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.aab + artifactFolder: Mono.Android-$(XA.Build.Configuration)-$(avdApiLevel) - task: MSBuild@1 displayName: shut down emulator @@ -159,70 +159,317 @@ stages: - template: yaml-templates/fail-on-issue.yaml - # TimeZoneInfo test jobs - - template: yaml-templates/run-timezoneinfo-tests.yaml - parameters: - node_id: 1 - - - template: yaml-templates/run-timezoneinfo-tests.yaml - parameters: - node_id: 2 - - - template: yaml-templates/run-timezoneinfo-tests.yaml - parameters: - node_id: 3 - - - template: yaml-templates/run-timezoneinfo-tests.yaml + - template: yaml-templates/run-emulator-tests.yaml parameters: - node_id: 4 - - # Localization test jobs - - template: yaml-templates/run-localization-tests.yaml - parameters: - node_id: 1 - - - template: yaml-templates/run-localization-tests.yaml - parameters: - node_id: 2 - - - template: yaml-templates/run-localization-tests.yaml + emulatorMSBuildArgs: -p:TestAvdExtraBootArgs=-writable-system + jobName: SystemApplicationTests + testSteps: + - template: run-nunit-tests.yaml + parameters: + testRunTitle: SystemApplicationTests On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=SystemApplicationTests" + testResultsFile: TestResult-SystemApplicationTests-$(XA.Build.Configuration).xml + + +# TimeZoneInfo test jobs +- stage: test_timezoneinfo + displayName: Test TimeZoneInfo + dependsOn: mac_build + variables: + - group: Xamarin-Secrets + - group: xamops-azdev-secrets + jobs: + - template: yaml-templates/run-emulator-tests.yaml parameters: - node_id: 3 - - - template: yaml-templates/run-localization-tests.yaml + jobName: TimeZoneInfoTests1 + emulatorMSBuildArgs: -p:TestAvdShowWindow=true + testSteps: + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode1 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode1" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode1-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode2 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode2" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode2-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode3 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode3" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode3-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + + - template: yaml-templates/run-emulator-tests.yaml parameters: - node_id: 4 - - - template: yaml-templates/run-localization-tests.yaml + jobName: TimeZoneInfoTests2 + emulatorMSBuildArgs: -p:TestAvdShowWindow=true + testSteps: + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode4 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode4" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode4-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode5 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode5" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode5-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode6 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode6" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode6-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + + - template: yaml-templates/run-emulator-tests.yaml parameters: - node_id: 5 - - - template: yaml-templates/run-localization-tests.yaml + jobName: TimeZoneInfoTests3 + emulatorMSBuildArgs: -p:TestAvdShowWindow=true + testSteps: + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode7 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode7" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode7-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode8 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode8" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode8-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode9 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode9" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode9-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + + - template: yaml-templates/run-emulator-tests.yaml parameters: - node_id: 6 - - - template: yaml-templates/run-localization-tests.yaml + jobName: TimeZoneInfoTests4 + emulatorMSBuildArgs: -p:TestAvdShowWindow=true + testSteps: + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode10 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode10" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode10-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode11 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode11" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode11-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode12 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode12" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode12-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + + - template: yaml-templates/run-emulator-tests.yaml parameters: - node_id: 7 - - - template: yaml-templates/run-localization-tests.yaml + jobName: TimeZoneInfoTests5 + emulatorMSBuildArgs: -p:TestAvdShowWindow=true + testSteps: + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode13 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode13" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode13-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode14 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode14" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode14-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckTimeZoneInfoIsCorrectNode15 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode15" + testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode15-$(XA.Build.Configuration).xml + timeoutInMinutes: 75 + retryCountOnTaskFailure: 2 + + +# Localization test jobs +- stage: test_locals + displayName: Test Localization + dependsOn: mac_build + variables: + - group: Xamarin-Secrets + - group: xamops-azdev-secrets + jobs: + - template: yaml-templates/run-emulator-tests.yaml parameters: - node_id: 8 - - - template: yaml-templates/run-localization-tests.yaml + jobName: LocalizationTests1 + emulatorMSBuildArgs: -p:TestAvdShowWindow=true + testSteps: + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode1 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode1" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode1-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode2 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode2" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode2-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode3 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode3" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode3-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + + - template: yaml-templates/run-emulator-tests.yaml parameters: - node_id: 9 - - - template: yaml-templates/run-localization-tests.yaml + jobName: LocalizationTests2 + emulatorMSBuildArgs: -p:TestAvdShowWindow=true + testSteps: + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode4 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode4" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode4-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode5 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode5" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode5-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode6 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode6" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode6-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + + - template: yaml-templates/run-emulator-tests.yaml parameters: - node_id: 10 - - - template: yaml-templates/run-localization-tests.yaml + jobName: LocalizationTests3 + emulatorMSBuildArgs: -p:TestAvdShowWindow=true + testSteps: + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode7 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode7" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode7-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode8 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode8" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode8-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode9 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode9" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode9-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + + - template: yaml-templates/run-emulator-tests.yaml parameters: - node_id: 11 - - - template: yaml-templates/run-localization-tests.yaml + jobName: LocalizationTests4 + emulatorMSBuildArgs: -p:TestAvdShowWindow=true + testSteps: + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode10 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode10" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode10-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode11 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode11" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode11-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode12 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode12" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode12-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + + - template: yaml-templates/run-emulator-tests.yaml parameters: - node_id: 12 - - - template: yaml-templates/run-systemapp-tests.yaml + jobName: LocalizationTests5 + emulatorMSBuildArgs: -p:TestAvdShowWindow=true + testSteps: + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode13 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode13" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode13-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode14 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode14" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode14-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 + - template: run-nunit-tests.yaml + parameters: + testRunTitle: CheckLocalizationIsCorrectNode15 On Device - macOS + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode15" + testResultsFile: TestResult-CheckLocalizationIsCorrectNode15-$(XA.Build.Configuration).xml + retryCountOnTaskFailure: 2 diff --git a/build-tools/automation/yaml-templates/run-systemapp-tests.yaml b/build-tools/automation/yaml-templates/run-emulator-tests.yaml similarity index 56% rename from build-tools/automation/yaml-templates/run-systemapp-tests.yaml rename to build-tools/automation/yaml-templates/run-emulator-tests.yaml index f86c3813c4c..8d1b4f7fd11 100644 --- a/build-tools/automation/yaml-templates/run-systemapp-tests.yaml +++ b/build-tools/automation/yaml-templates/run-emulator-tests.yaml @@ -1,12 +1,17 @@ -# Runs TimeZoneInfo tests against an emulator running on macOS +# Runs a test or set of tests on an emulator running on macOS + +parameters: + emulatorMSBuildArgs: '' + jobName: CheckTimeZoneInfoIsCorrectNode1 + jobTimeout: 360 + testSteps: [] jobs: - - job: mac_systemapp_tests - displayName: System App Emulator Tests + - job: mac_${{ parameters.jobName }}_tests + displayName: ${{ parameters.jobName }} Emulator Tests pool: vmImage: $(HostedMacImage) - timeoutInMinutes: 120 - cancelTimeoutInMinutes: 5 + timeoutInMinutes: ${{ parameters.jobTimeout }} workspace: clean: all steps: @@ -27,14 +32,9 @@ jobs: solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj configuration: $(XA.Build.Configuration) msbuildArguments: >- - /t:AcquireAndroidTarget /p:TestEmulatorArguments=-writable-system /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog + /t:AcquireAndroidTarget ${{ parameters.emulatorMSBuildArgs }} /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog - - template: run-nunit-tests.yaml - parameters: - testRunTitle: System App On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/net472/MSBuildDeviceIntegration.dll - nunitConsoleExtraArgs: --where "cat == SystemApplication" - testResultsFile: TestResult-SystemApp--$(XA.Build.Configuration).xml + - ${{ parameters.testSteps }} - task: MSBuild@1 displayName: shut down emulator @@ -48,6 +48,6 @@ jobs: - template: upload-results.yaml parameters: - artifactName: Test Results - System App With Emulator - macOS + artifactName: Test Results - ${{ parameters.jobName }} With Emulator - macOS - - template: fail-on-issue.yaml \ No newline at end of file + - template: fail-on-issue.yaml diff --git a/build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml b/build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml deleted file mode 100644 index 3f96affb8bf..00000000000 --- a/build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# Runs TimeZoneInfo tests against an emulator running on macOS - -parameters: - node_id: 0 - -jobs: - - job: mac_timezoneinfo_tests_${{ parameters.node_id }} - displayName: TimeZoneInfo Emulator Tests ${{ parameters.node_id }} - pool: - vmImage: $(HostedMacImage) - timeoutInMinutes: 120 - cancelTimeoutInMinutes: 5 - workspace: - clean: all - steps: - - template: setup-test-environment.yaml - - - template: run-xaprepare.yaml - parameters: - arguments: --s=EmulatorTestDependencies - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: $(TestAssembliesArtifactName) - downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration) - - - task: MSBuild@1 - displayName: start emulator - inputs: - solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj - configuration: $(XA.Build.Configuration) - msbuildArguments: >- - /t:AcquireAndroidTarget /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog - - - template: run-nunit-tests.yaml - parameters: - useDotNet: false - testRunTitle: TimeZoneInfoTests On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/net472/MSBuildDeviceIntegration.dll - nunitConsoleExtraArgs: --where "test == Xamarin.Android.Build.Tests.DeploymentTest.CheckTimeZoneInfoIsCorrectNode${{ parameters.node_id }}" - testResultsFile: TestResult-TimeZoneInfoTests-Node${{ parameters.node_id }}-$(XA.Build.Configuration).xml - - - task: MSBuild@1 - displayName: shut down emulator - inputs: - solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj - configuration: $(XA.Build.Configuration) - msbuildArguments: >- - /t:AcquireAndroidTarget,ReleaseAndroidTarget - /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog - condition: always() - - - template: upload-results.yaml - parameters: - artifactName: Test Results - TimeZoneInfo With Emulator - macOS - ${{ parameters.node_id }} - - - template: fail-on-issue.yaml diff --git a/build-tools/scripts/NUnitReferences.projitems b/build-tools/scripts/NUnitReferences.projitems index 38c6801b999..5e6cd1960a4 100644 --- a/build-tools/scripts/NUnitReferences.projitems +++ b/build-tools/scripts/NUnitReferences.projitems @@ -4,10 +4,6 @@ - - - - - + diff --git a/build-tools/scripts/TestApks.targets b/build-tools/scripts/TestApks.targets index f940b08efcc..4592c83a885 100644 --- a/build-tools/scripts/TestApks.targets +++ b/build-tools/scripts/TestApks.targets @@ -21,12 +21,13 @@ arm64-v8a x86_64 default + true + false + -no-window -no-boot-anim $(TestAvdExtraBootArgs) pixel_4 system-images;android-$(TestAvdApiLevel);$(TestAvdType);$(TestAvdAbi) XamarinAndroidTestRunner$(TestAvdApiLevel)-$(TestAvdAbi) <_AdbEmulatorPort>5570 - <_AdbEmulatorShowWindow Condition=" '$(RunningOnCI)' == 'True' And '$(_AdbEmulatorShowWindow)' == '' ">False - <_AdbEmulatorShowWindow Condition=" '$(_AdbEmulatorShowWindow)' == '' ">True <_ApkSizesReferenceDirectory>$(MSBuildThisFileDirectory)..\..\tests\apk-sizes-reference 10 $([MSBuild]::Multiply($(AvdLaunchTimeoutMinutes), 60)) @@ -56,7 +57,7 @@ @@ -124,6 +124,9 @@ + + <_EmuTarget Condition=" '$(_EmuTarget)' == '' ">-s emulator-$(_AdbEmulatorPort) + diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs index a8f678a4d9a..4048d6d3d18 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs @@ -25,16 +25,6 @@ public class BaseTest [SetUpFixture] public class SetUp { - public static string DeviceAbi { - get; - private set; - } - - public static int DeviceSdkVersion { - get; - private set; - } - public static string TestDirectoryRoot { get; private set; @@ -44,41 +34,6 @@ public static string TestDirectoryRoot { public void BeforeAllTests () { TestDirectoryRoot = XABuildPaths.TestOutputDirectory; - - try { - DeviceSdkVersion = GetSdkVersion (); - if (DeviceSdkVersion != -1) { - if (DeviceSdkVersion >= 21) - DeviceAbi = RunAdbCommand ("shell getprop ro.product.cpu.abilist64").Trim (); - - if (string.IsNullOrEmpty (DeviceAbi)) - DeviceAbi = RunAdbCommand ("shell getprop ro.product.cpu.abi") ?? RunAdbCommand ("shell getprop ro.product.cpu.abi2"); - - if (DeviceAbi.Contains (",")) { - DeviceAbi = DeviceAbi.Split (',')[0]; - } - } - } catch (Exception ex) { - Console.Error.WriteLine ("Failed to determine whether there is Android target emulator or not: " + ex); - } - } - - int GetSdkVersion () - { - var command = $"shell getprop ro.build.version.sdk"; - var result = RunAdbCommand (command); - if (result.Contains ("*")) { - // Run the command again, we likely got: - // * daemon not running; starting now at tcp:5037 - // * daemon started successfully - // adb.exe: device offline - TestContext.WriteLine ($"Retrying:\n{command}\n{result}"); - result = RunAdbCommand (command); - } - if (!int.TryParse (result, out var sdkVersion)) { - sdkVersion = -1; - } - return sdkVersion; } [OneTimeTearDown] @@ -208,7 +163,7 @@ protected static string RunAdbCommand (string command, bool ignoreErrors = true, string ext = Environment.OSVersion.Platform != PlatformID.Unix ? ".exe" : ""; string adb = Path.Combine (AndroidSdkPath, "platform-tools", "adb" + ext); string adbTarget = Environment.GetEnvironmentVariable ("ADB_TARGET"); - return RunProcess (adb, $"{adbTarget} {command}"); + return RunProcess (adb, $"{adbTarget} {command}", timeout); } protected static (int code, string stdOutput, string stdError) RunApkDiffCommand (string args) @@ -225,9 +180,9 @@ protected static (int code, string stdOutput, string stdError) RunApkDiffCommand } } - protected static string RunProcess (string exe, string args) + protected static string RunProcess (string exe, string args, int timeoutInSeconds = 30) { - var (_, stdOutput, stdError) = RunProcessWithExitCode (exe, args); + var (_, stdOutput, stdError) = RunProcessWithExitCode (exe, args, timeoutInSeconds); return stdOutput + stdError; } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs index 0f0b2cbe21f..38f38962ea0 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs @@ -17,11 +17,17 @@ public class DeviceTest: BaseTest { public const string GuestUserName = "guest1"; - protected bool HasDevices { - get { - string output = RunAdbCommand ("shell echo OK"); - return output.Contains ("OK"); + protected string DeviceAbi { get; private set; } + + protected int DeviceSdkVersion { get; private set;} + + static string _shellEchoOutput = null; + protected static bool IsDeviceAttached (bool refreshCachedValue = false) + { + if (string.IsNullOrEmpty (_shellEchoOutput) || refreshCachedValue) { + _shellEchoOutput = RunAdbCommand ("shell echo OK", timeout: 15); } + return _shellEchoOutput.Contains ("OK"); } /// @@ -30,7 +36,7 @@ protected bool HasDevices { /// public void AssertHasDevices (bool fail = true) { - if (!HasDevices) { + if (!IsDeviceAttached (refreshCachedValue: true)) { var message = "This test requires an attached device or emulator."; if (fail) { Assert.Fail (message); @@ -40,34 +46,55 @@ public void AssertHasDevices (bool fail = true) } } - protected string DeviceAbi => SetUp.DeviceAbi; - - protected int DeviceSdkVersion => SetUp.DeviceSdkVersion; - [OneTimeSetUp] public void DeviceSetup () { - SetAdbLogcatBufferSize (64); - ClearAdbLogcat (); - CreateGuestUser (GuestUserName); + if (IsDeviceAttached ()) { + try { + DeviceSdkVersion = GetSdkVersion (); + if (DeviceSdkVersion != -1) { + if (DeviceSdkVersion >= 21) + DeviceAbi = RunAdbCommand ("shell getprop ro.product.cpu.abilist64").Trim (); + + if (string.IsNullOrEmpty (DeviceAbi)) + DeviceAbi = RunAdbCommand ("shell getprop ro.product.cpu.abi") ?? RunAdbCommand ("shell getprop ro.product.cpu.abi2"); + + if (DeviceAbi.Contains (",")) { + DeviceAbi = DeviceAbi.Split (',')[0]; + } + } + } catch (Exception ex) { + Console.Error.WriteLine ("Failed to determine whether there is Android target emulator or not: " + ex); + } + SetAdbLogcatBufferSize (64); + CreateGuestUser (GuestUserName); + } } [OneTimeTearDown] public void DeviceTearDown () { - // make sure we are not on a guest user anymore. - SwitchUser (); - DeleteGuestUser(GuestUserName); + if (IsDeviceAttached ()) { + // make sure we are not on a guest user anymore. + SwitchUser (); + DeleteGuestUser (GuestUserName); + + var packages = TestPackageNames.Values.ToArray (); + TestPackageNames.Clear (); + foreach (var package in packages) { + RunAdbCommand ($"uninstall {package}"); + } + } } [SetUp] - public void CheckDevice () + public virtual void SetupTest () { - if (!HasDevices) { + if (!IsDeviceAttached (refreshCachedValue: true)) { // something went wrong with the emulator. // lets restart it. - TestContext.Out.WriteLine ($"{nameof(CheckDevice)} is restarting the emulator."); - RestartDevice (Path.Combine (Root, "Emulator.csproj")); + RestartDevice (); + AssertHasDevices (); } // We want to start with a clean logcat buffer for each test @@ -77,7 +104,7 @@ public void CheckDevice () [TearDown] protected override void CleanupTest () { - if (HasDevices && TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed && + if (TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed && IsDeviceAttached () && TestOutputDirectories.TryGetValue (TestContext.CurrentContext.Test.ID, out string outputDir)) { Directory.CreateDirectory (outputDir); string local = Path.Combine (outputDir, "screenshot.png"); @@ -106,30 +133,35 @@ protected override void CleanupTest () } } - ClearAdbLogcat (); - - base.CleanupTest (); } - [OneTimeTearDown] - protected static void UnInstallTestApp () - { - var packages = TestPackageNames.Values.ToArray (); - TestPackageNames.Clear (); - foreach(var package in packages) { - RunAdbCommand ($"uninstall {package}"); + protected int GetSdkVersion () + { + var command = $"shell getprop ro.build.version.sdk"; + var result = RunAdbCommand (command); + if (result.Contains ("*")) { + // Run the command again, we likely got: + // * daemon not running; starting now at tcp:5037 + // * daemon started successfully + // adb.exe: device offline + TestContext.WriteLine ($"Retrying:\n{command}\n{result}"); + result = RunAdbCommand (command); + } + if (!int.TryParse (result, out var sdkVersion)) { + sdkVersion = -1; } - } + return sdkVersion; + } - public static void RestartDevice (string project) + public static void RestartDevice () { TestContext.Out.WriteLine ($"Trying to restart Emulator"); // shell out to msbuild and start the emulator again - using (var builder = new Builder ()) { - var out1 = RunProcessWithExitCode (builder.BuildTool, $"{(Builder.UseDotNet ? "build" : "")} {project} /restore /t:AcquireAndroidTarget", timeoutInSeconds: 120); - TestContext.Out.WriteLine ($"{out1}"); - } + var dotnet = new DotNetCLI (Path.Combine (XABuildPaths.TopDirectory, "src", "Xamarin.Android.Build.Tasks", "Tests", "Xamarin.Android.Build.Tests", "Emulator.csproj")); + dotnet.ProjectDirectory = XABuildPaths.TestAssemblyOutputDirectory; + Assert.IsTrue (dotnet.Build ("AcquireAndroidTarget", parameters: new string[] { "TestAvdForceCreation=false", $"Configuration={XABuildPaths.Configuration}" }), "Failed to acquire emulator."); + WaitFor ((int)TimeSpan.FromSeconds (5).TotalMilliseconds); } protected static void RunAdbInput (string command, params object [] args) @@ -152,9 +184,23 @@ protected static string ClearDebugProperty () return RunAdbCommand ("shell setprop debug.mono.extra \"\""); } - protected static void AdbStartActivity (string activity) + protected static string AdbStartActivity (string activity) + { + return RunAdbCommand ($"shell am start -S -n \"{activity}\""); + } + + protected static void RunProjectAndAssert (XamarinAndroidApplicationProject proj, ProjectBuilder builder, string logName = "run.log", bool doNotCleanupOnUpdate = false, string [] parameters = null) { - RunAdbCommand ($"shell am start -S -n \"{activity}\""); + if (Builder.UseDotNet) { + builder.BuildLogFile = logName; + Assert.True (builder.RunTarget (proj, "Run", doNotCleanupOnUpdate: doNotCleanupOnUpdate, parameters: parameters), "Project should have run."); + } else if (CommercialBuildAvailable) { + builder.BuildLogFile = logName; + Assert.True (builder.RunTarget (proj, "_Run", doNotCleanupOnUpdate: doNotCleanupOnUpdate, parameters: parameters), "Project should have run."); + } else { + var result = AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); + Assert.IsTrue (result.Contains ("Starting: Intent { cmp="), $"Attempt to start activity failed with:\n{result}"); + } } protected TimeSpan ProfileFor (Func func, TimeSpan? timeout = null) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj index 8fab24d3966..b4596ed2356 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj @@ -51,10 +51,6 @@ ..\Expected\CheckPackageManagerAssemblyOrder.java PreserveNewest - - ..\Emulator.csproj - PreserveNewest - diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs index 5f3d93bce01..334aa28ddfb 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs @@ -186,7 +186,7 @@ protected virtual void Dispose (bool disposing) RegexOptions.Multiline | RegexOptions.Compiled ); - protected bool BuildInternal (string projectOrSolution, string target, string [] parameters = null, Dictionary environmentVariables = null, bool restore = true) + protected bool BuildInternal (string projectOrSolution, string target, string [] parameters = null, Dictionary environmentVariables = null, bool restore = true, string binlogName = "msbuild") { buildLogFullPath = (!string.IsNullOrEmpty (BuildLogFile)) ? Path.GetFullPath (Path.Combine (XABuildPaths.TestOutputDirectory, Path.GetDirectoryName (projectOrSolution), BuildLogFile)) @@ -256,7 +256,7 @@ protected bool BuildInternal (string projectOrSolution, string target, string [] } psi.SetEnvironmentVariable ("MSBUILD", "msbuild"); - sw.WriteLine ($"/bl:\"{Path.GetFullPath (Path.Combine (XABuildPaths.TestOutputDirectory, Path.GetDirectoryName (projectOrSolution), "msbuild.binlog"))}\""); + sw.WriteLine ($"/bl:\"{Path.GetFullPath (Path.Combine (XABuildPaths.TestOutputDirectory, Path.GetDirectoryName (projectOrSolution), $"{binlogName}.binlog"))}\""); if (environmentVariables != null) { foreach (var kvp in environmentVariables) { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs index 9b64f3d7922..c0ba14ad487 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs @@ -14,8 +14,7 @@ public class DotNetCLI public string Verbosity { get; set; } = "diag"; public string AndroidSdkPath { get; set; } = AndroidSdkResolver.GetAndroidSdkPath (); public string JavaSdkPath { get; set; } = AndroidSdkResolver.GetJavaSdkPath (); - - public string ProjectDirectory { get; private set; } + public string ProjectDirectory { get; set; } readonly XASdkProject project; readonly string projectOrSolution; @@ -140,20 +139,16 @@ public IEnumerable LastBuildOutput { List GetDefaultCommandLineArgs (string verb, string target = null, string runtimeIdentifier = null, string [] parameters = null) { - string testDir = Path.GetDirectoryName (projectOrSolution); - if (string.IsNullOrEmpty (ProcessLogFile)) - ProcessLogFile = Path.Combine (testDir, "process.log"); - + string testDir = string.IsNullOrEmpty (ProjectDirectory) ? Path.GetDirectoryName (projectOrSolution) : ProjectDirectory; if (string.IsNullOrEmpty (BuildLogFile)) BuildLogFile = Path.Combine (testDir, "build.log"); - var binlog = string.IsNullOrEmpty (target) ? Path.GetFileNameWithoutExtension (string.IsNullOrEmpty (BuildLogFile) ? "msbuild" : BuildLogFile) : target; var arguments = new List { verb, $"\"{projectOrSolution}\"", "/noconsolelogger", $"/flp1:LogFile=\"{BuildLogFile}\";Encoding=UTF-8;Verbosity={Verbosity}", - $"/bl:\"{Path.Combine (testDir, $"{binlog}.binlog")}\"", + $"/bl:\"{Path.Combine (testDir, $"{(string.IsNullOrEmpty (target) ? "msbuild" : target)}.binlog")}\"", "-m:1", "-nr:false", "/p:_DisableParallelAot=true", diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/ProjectBuilder.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/ProjectBuilder.cs index 0d4f1f3fbcb..6118fc6a0c8 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/ProjectBuilder.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/ProjectBuilder.cs @@ -70,7 +70,7 @@ public bool Build (XamarinProject project, bool doNotCleanupOnUpdate = false, st project.NuGetRestore (Path.Combine (XABuildPaths.TestOutputDirectory, ProjectDirectory), PackagesDirectory); } - bool result = BuildInternal (Path.Combine (ProjectDirectory, project.ProjectFilePath), Target, parameters, environmentVariables, restore: project.ShouldRestorePackageReferences); + bool result = BuildInternal (Path.Combine (ProjectDirectory, project.ProjectFilePath), Target, parameters, environmentVariables, restore: project.ShouldRestorePackageReferences, binlogName: Path.GetFileNameWithoutExtension (BuildLogFile)); built_before = true; if (CleanupAfterSuccessfulBuild) diff --git a/tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs b/tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs index 151ece0ba6c..df4eca72697 100644 --- a/tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs @@ -21,8 +21,6 @@ public class AotProfileTests : DeviceTest [Test] public void BuildBasicApplicationAndAotProfileIt () { - AssertHasDevices (); - var proj = new XamarinAndroidApplicationProject () { IsRelease = true, AotAssemblies = false, diff --git a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs index 1c94a4bc95a..c6976df1e2e 100644 --- a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs @@ -300,8 +300,6 @@ public void AppBundleSigned () [Test, Category ("UsesDevice")] public void ApkSet () { - AssertHasDevices (); - appBuilder.BuildLogFile = "install.log"; Assert.IsTrue (appBuilder.RunTarget (app, "Install"), "App should have installed."); diff --git a/tests/MSBuildDeviceIntegration/Tests/DebuggingTest.cs b/tests/MSBuildDeviceIntegration/Tests/DebuggingTest.cs index 89afd11dae3..ac74bd613fa 100755 --- a/tests/MSBuildDeviceIntegration/Tests/DebuggingTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/DebuggingTest.cs @@ -14,7 +14,8 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] [Category ("UsesDevice")] - public class DebuggingTest : DeviceTest { + public class DebuggingTest : DeviceTest + { [TearDown] public void ClearDebugProperties () { @@ -48,7 +49,6 @@ int FindTextInFile (string file, string text) [Test] public void ApplicationRunsWithoutDebugger ([Values (false, true)] bool isRelease, [Values (false, true)] bool extractNativeLibs, [Values (false, true)] bool useEmbeddedDex) { - AssertHasDevices (); SwitchUser (); var proj = new XamarinFormsAndroidApplicationProject () { @@ -68,13 +68,7 @@ public void ApplicationRunsWithoutDebugger ([Values (false, true)] bool isReleas Assert.True (b.Install (proj), "Project should have installed."); var manifest = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android", "AndroidManifest.xml"); AssertExtractNativeLibs (manifest, extractNativeLibs); - ClearAdbLogcat (); - b.BuildLogFile = "run.log"; - if (CommercialBuildAvailable) - Assert.True (b.RunTarget (proj, "_Run"), "Project should have run."); - else - AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); - + RunProjectAndAssert (proj, b); Assert.True (WaitForActivityToStart (proj.PackageName, "MainActivity", Path.Combine (Root, b.ProjectDirectory, "logcat.log"), 30), "Activity should have started."); b.BuildLogFile = "uninstall.log"; @@ -85,7 +79,6 @@ public void ApplicationRunsWithoutDebugger ([Values (false, true)] bool isReleas [Test] public void ClassLibraryMainLauncherRuns ([Values (true, false)] bool preloadAssemblies) { - AssertHasDevices (); SwitchUser (); var path = Path.Combine ("temp", TestName); @@ -130,13 +123,7 @@ public void ClassLibraryMainLauncherRuns ([Values (true, false)] bool preloadAss SetTargetFrameworkAndManifest (app, appBuilder); Assert.IsTrue (libBuilder.Build (lib), "library build should have succeeded."); Assert.True (appBuilder.Install (app), "app should have installed."); - ClearAdbLogcat (); - appBuilder.BuildLogFile = "run.log"; - if (CommercialBuildAvailable) - Assert.True (appBuilder.RunTarget (app, "_Run"), "Project should have run."); - else - AdbStartActivity ($"{app.PackageName}/{app.JavaPackageName}.MainActivity"); - + RunProjectAndAssert (app, appBuilder); Assert.True (WaitForActivityToStart (app.PackageName, "MainActivity", Path.Combine (Root, appBuilder.ProjectDirectory, "logcat.log"), 30), "Activity should have started."); } @@ -173,7 +160,6 @@ public void ClassLibraryMainLauncherRuns ([Values (true, false)] bool preloadAss public void CustomApplicationRunsWithDebuggerAndBreaks (bool embedAssemblies, string fastDevType, bool activityStarts) { AssertCommercialBuild (); - AssertHasDevices (); SwitchUser (); var path = Path.Combine (Root, "temp", TestName); @@ -246,13 +232,11 @@ public override void OnCreate () EvaluationOptions = EvaluationOptions.DefaultOptions, }; options.EvaluationOptions.UseExternalTypeResolver = true; - ClearAdbLogcat (); - b.BuildLogFile = "run.log"; - Assert.True (b.RunTarget (proj, "_Run", doNotCleanupOnUpdate: true, parameters: new string [] { + RunProjectAndAssert (proj, b, doNotCleanupOnUpdate: true, parameters: new string [] { $"AndroidSdbTargetPort={port}", $"AndroidSdbHostPort={port}", "AndroidAttachDebugger=True", - }), "Project should have run."); + }); session.LogWriter += (isStderr, text) => { Console.WriteLine (text); }; session.OutputWriter += (isStderr, text) => { Console.WriteLine (text); }; @@ -351,7 +335,6 @@ public override void OnCreate () public void ApplicationRunsWithDebuggerAndBreaks (bool embedAssemblies, string fastDevType, bool allowDeltaInstall, string username, string debugType) { AssertCommercialBuild (); - AssertHasDevices (); SwitchUser (); WaitFor (5000); @@ -463,15 +446,12 @@ public Foo () EvaluationOptions = EvaluationOptions.DefaultOptions, }; options.EvaluationOptions.UseExternalTypeResolver = true; - ClearAdbLogcat (); - appBuilder.BuildLogFile = "run.log"; parameters.Add ($"AndroidSdbTargetPort={port}"); parameters.Add ($"AndroidSdbHostPort={port}"); parameters.Add ("AndroidAttachDebugger=True"); - Assert.True (appBuilder.RunTarget (app, "_Run", doNotCleanupOnUpdate: true, - parameters: parameters.ToArray ()), "Project should have run."); + RunProjectAndAssert (app, appBuilder, doNotCleanupOnUpdate: true, parameters: parameters.ToArray ()); session.LogWriter += (isStderr, text) => { TestContext.Out.WriteLine (text); diff --git a/tests/MSBuildDeviceIntegration/Tests/DeleteBinObjTest.cs b/tests/MSBuildDeviceIntegration/Tests/DeleteBinObjTest.cs index 85cecea4c96..532b5abb21e 100644 --- a/tests/MSBuildDeviceIntegration/Tests/DeleteBinObjTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/DeleteBinObjTest.cs @@ -52,7 +52,7 @@ void RunTest (string name, string sln, string csproj, string version, string rev } else { parameters.Add (KnownProperties.AndroidSupportedAbis + "=\"armeabi-v7a;arm64-v8a;x86;x86_64\""); } - if (HasDevices) { + if (IsDeviceAttached ()) { Assert.IsTrue (builder.Install (project, doNotCleanupOnUpdate: true, parameters: parameters.ToArray (), saveProject: false), "Install should have succeeded."); ClearAdbLogcat (); diff --git a/tests/MSBuildDeviceIntegration/Tests/DeploymentTest.cs b/tests/MSBuildDeviceIntegration/Tests/DeploymentTest.cs deleted file mode 100644 index 17e6982a6ea..00000000000 --- a/tests/MSBuildDeviceIntegration/Tests/DeploymentTest.cs +++ /dev/null @@ -1,470 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Xml.Linq; -using System.Reflection; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using System.Xml; -using System.Xml.Linq; -using Microsoft.Build.Framework; -using NUnit.Framework; -using NUnit.Framework.Interfaces; -using Xamarin.ProjectTools; -using System.Xml.XPath; - -namespace Xamarin.Android.Build.Tests -{ - [TestFixture] - [Category ("UsesDevice")] - [NonParallelizable] - public class DeploymentTest : DeviceTest { - - static ProjectBuilder builder; - static XamarinFormsAndroidApplicationProject proj; - - [OneTimeSetUp] - public void BeforeDeploymentTests () - { - AssertHasDevices (); - - string debuggable = RunAdbCommand ("shell getprop ro.debuggable"); - if (debuggable != "1") { - Assert.Ignore ("TimeZone tests need to use `su root` and this device does not support that feature. Try using an emulator."); - } - // Disable auto timezone - RunAdbCommand ("shell settings put global auto_time_zone 0"); - - proj = new XamarinFormsAndroidApplicationProject (); - proj.SetAndroidSupportedAbis ("armeabi-v7a", "x86", "x86_64"); - var mainPage = proj.Sources.First (x => x.Include () == "MainPage.xaml.cs"); - var source = mainPage.TextContent ().Replace ("InitializeComponent ();", @"InitializeComponent (); - Console.WriteLine ($""TimeZoneInfoNative={Java.Util.TimeZone.Default.ID}""); - Console.WriteLine ($""TimeZoneInfo={TimeZoneInfo.Local.DisplayName}""); - Console.WriteLine ($""LocaleNative={Java.Util.Locale.Default.Language}-{Java.Util.Locale.Default.Country}""); - Console.WriteLine ($""CurrentCulture={System.Globalization.CultureInfo.CurrentCulture.Name}""); - Console.WriteLine ($""Strings.SomeString={Strings.SomeString}""); - myLabel.Text = Strings.SomeString; -"); - source = source.Replace ("Console.WriteLine (\"Button was Clicked!\");", @"Console.WriteLine (""Button was Clicked!""); - Console.WriteLine ($""TimeZoneInfoClick={TimeZoneInfo.Local.DisplayName}""); - Console.WriteLine ($""CurrentCultureClick={System.Globalization.CultureInfo.CurrentCulture.Name}""); - Console.WriteLine ($""StringsClick={Strings.SomeString}""); -"); - mainPage.TextContent = () => source; - builder = CreateApkBuilder (Path.Combine ("temp", "DeploymentTests")); - string apiLevel; - proj.TargetFrameworkVersion = builder.LatestTargetFrameworkVersion (out apiLevel); - - proj.PackageName = "Xamarin.TimeZoneTest"; - proj.AndroidManifest = $@" - - - - - "; - InlineData.AddCultureResourcesToProject (proj, "Strings", "SomeString"); - InlineData.AddCultureResourceDesignerToProject (proj, proj.RootNamespace ?? proj.ProjectName, "Strings", "SomeString"); - - Assert.IsTrue (builder.Build (proj), "Build should have succeeded."); - builder.BuildLogFile = "install.log"; - Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); - } - - [OneTimeTearDown] - public void AfterDeploymentTests () - { - if (HasDevices && proj != null) - RunAdbCommand ($"uninstall {proj.PackageName}"); - - if (builder != null) - return; - - string output = Path.Combine (Root, builder.ProjectDirectory); - if (TestContext.CurrentContext.Result.FailCount == 0 && Directory.Exists (output)) { - Directory.Delete (output, recursive: true); - return; - } - - foreach (var file in Directory.GetFiles (output, "*.log", SearchOption.AllDirectories)) { - TestContext.AddTestAttachment (file, Path.GetFileName (output)); - } - } - - [Test] - public void CheckResouceIsOverridden ([Values (true, false)] bool useAapt2) - { - AssertHasDevices (); - AssertAaptSupported (useAapt2); - - var library = new XamarinAndroidLibraryProject () { - ProjectName = "Library1", - AndroidResources = { - new AndroidItem.AndroidResource (() => "Resources\\values\\strings2.xml") { - TextContent = () => @" - - Click Me! One -", - }, - }, - }; - var library2 = new XamarinAndroidLibraryProject () { - ProjectName = "Library2", - AndroidResources = { - new AndroidItem.AndroidResource (() => "Resources\\values\\strings2.xml") { - TextContent = () => @" - - Click Me! Two -", - }, - }, - }; - var app = new XamarinAndroidApplicationProject () { - PackageName = "Xamarin.ResourceTest", - References = { - new BuildItem.ProjectReference ("..\\Library1\\Library1.csproj"), - new BuildItem.ProjectReference ("..\\Library2\\Library2.csproj"), - }, - }; - library.AndroidUseAapt2 = - library2.AndroidUseAapt2 = - app.AndroidUseAapt2 = useAapt2; - app.LayoutMain = app.LayoutMain.Replace ("@string/hello", "@string/hello_me"); - using (var l1 = CreateDllBuilder (Path.Combine ("temp", TestName, library.ProjectName))) - using (var l2 = CreateDllBuilder (Path.Combine ("temp", TestName, library2.ProjectName))) - using (var b = CreateApkBuilder (Path.Combine ("temp", TestName, app.ProjectName))) { - b.ThrowOnBuildFailure = false; - string apiLevel; - app.TargetFrameworkVersion = b.LatestTargetFrameworkVersion (out apiLevel); - - app.AndroidManifest = $@" - - - - - "; - Assert.IsTrue (l1.Build (library, doNotCleanupOnUpdate: true), $"Build of {library.ProjectName} should have suceeded."); - Assert.IsTrue (l2.Build (library2, doNotCleanupOnUpdate: true), $"Build of {library2.ProjectName} should have suceeded."); - b.BuildLogFile = "build1.log"; - Assert.IsTrue (b.Build (app, doNotCleanupOnUpdate: true), $"Build of {app.ProjectName} should have suceeded."); - b.BuildLogFile = "install1.log"; - Assert.IsTrue (b.Install (app, doNotCleanupOnUpdate: true), "Install should have suceeded."); - AdbStartActivity ($"{app.PackageName}/{app.JavaPackageName}.MainActivity"); - WaitForPermissionActivity (Path.Combine (Root, builder.ProjectDirectory, "permission-logcat.log")); - WaitForActivityToStart (app.PackageName, "MainActivity", - Path.Combine (Root, builder.ProjectDirectory, "startup-logcat.log"), 15); - ClearBlockingDialogs (); - XDocument ui = GetUI (); - XElement node = ui.XPathSelectElement ($"//node[contains(@resource-id,'myButton')]"); - Assert.IsNotNull (node , "Could not find `my-Button` in the user interface. Check the screenshot of the test failure."); - StringAssert.AreEqualIgnoringCase ("Click Me! One", node.Attribute ("text").Value, "Text of Button myButton should have been \"Click Me! One\""); - b.BuildLogFile = "clean.log"; - Assert.IsTrue (b.Clean (app, doNotCleanupOnUpdate: true), "Clean should have suceeded."); - - app = new XamarinAndroidApplicationProject () { - PackageName = "Xamarin.ResourceTest", - References = { - new BuildItem.ProjectReference ("..\\Library1\\Library1.csproj"), - new BuildItem.ProjectReference ("..\\Library2\\Library2.csproj"), - }, - }; - - library2.References.Add (new BuildItem.ProjectReference ("..\\Library1\\Library1.csproj")); - app.AndroidUseAapt2 = useAapt2; - app.LayoutMain = app.LayoutMain.Replace ("@string/hello", "@string/hello_me"); - app.TargetFrameworkVersion = b.LatestTargetFrameworkVersion (out apiLevel); - - app.AndroidManifest = $@" - - - - - "; - b.BuildLogFile = "build.log"; - Assert.IsTrue (b.Build (app, doNotCleanupOnUpdate: true), $"Build of {app.ProjectName} should have suceeded."); - b.BuildLogFile = "install.log"; - Assert.IsTrue (b.Install (app, doNotCleanupOnUpdate: true), "Install should have suceeded."); - AdbStartActivity ($"{app.PackageName}/{app.JavaPackageName}.MainActivity"); - WaitForPermissionActivity (Path.Combine (Root, builder.ProjectDirectory, "permission-logcat.log")); - WaitForActivityToStart (app.PackageName, "MainActivity", - Path.Combine (Root, builder.ProjectDirectory, "startup-logcat.log"), 15); - ui = GetUI (); - node = ui.XPathSelectElement ($"//node[contains(@resource-id,'myButton')]"); - StringAssert.AreEqualIgnoringCase ("Click Me! One", node.Attribute ("text").Value, "Text of Button myButton should have been \"Click Me! One\""); - - } - } - - - [Test] - public void CheckXamarinFormsAppDeploysAndAButtonWorks () - { - AssertHasDevices (); - - AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); - WaitForActivityToStart (proj.PackageName, "MainActivity", - Path.Combine (Root, builder.ProjectDirectory, "startup-logcat.log"), 15); - ClearAdbLogcat (); - ClearBlockingDialogs (); - ClickButton (proj.PackageName, "myXFButton", "CLICK ME"); - Assert.IsTrue (MonitorAdbLogcat ((line) => { - return line.Contains ("Button was Clicked!"); - }, Path.Combine (Root, builder.ProjectDirectory, "button-logcat.log")), "Button Should have been Clicked."); - } - - private const int TIMEZONE_NODE_COUNT = 4; - - static object [] GetTimeZoneTestCases (int node) - { - List tests = new List (); - var ignore = new string [] { - "Asia/Qostanay", - "US/Pacific-New" - }; - - foreach (var tz in NodaTime.DateTimeZoneProviders.Tzdb.Ids) { - if (ignore.Contains (tz)) { - TestContext.WriteLine ($"Ignoring {tz} TimeZone Test"); - continue; - } - tests.Add (new object [] { - tz, - }); - } - return tests.Where (p => tests.IndexOf (p) % TIMEZONE_NODE_COUNT == node).ToArray (); - } - - [Test] - [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 0 })] - [Category ("TimeZoneInfo")] - [Retry (2)] - public void CheckTimeZoneInfoIsCorrectNode1 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); - - [Test] - [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 1 })] - [Category ("TimeZoneInfo")] - [Retry (2)] - public void CheckTimeZoneInfoIsCorrectNode2 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); - - [Test] - [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 2 })] - [Category ("TimeZoneInfo")] - [Retry (2)] - public void CheckTimeZoneInfoIsCorrectNode3 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); - - [Test] - [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 3 })] - [Category ("TimeZoneInfo")] - [Retry (2)] - public void CheckTimeZoneInfoIsCorrectNode4 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); - - public void CheckTimeZoneInfoIsCorrect (string timeZone) - { - AssertHasDevices (); - - string currentTimeZone = RunAdbCommand ("shell getprop persist.sys.timezone")?.Trim (); - string deviceTz = string.Empty; - string logFile = Path.Combine (Root, builder.ProjectDirectory, $"startup-logcat-{timeZone.Replace ("/", "-")}.log"); - try { - for (int attempt = 0; attempt < 5; attempt++) { - RunAdbCommand ($"shell su root setprop persist.sys.timezone \"{timeZone}\""); - deviceTz = RunAdbCommand ("shell getprop persist.sys.timezone")?.Trim (); - if (deviceTz == timeZone) { - break; - } - } - Assert.AreEqual (timeZone, deviceTz, $"The command to set the device timezone to {timeZone} failed. Current device timezone is {deviceTz}"); - ClearAdbLogcat (); - RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}"); - RunAdbCommand ($"shell am kill --user all {proj.PackageName}"); - WaitFor ((int)TimeSpan.FromSeconds (2).TotalMilliseconds); - ClearAdbLogcat (); - AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); - Assert.IsTrue (WaitForActivityToStart (proj.PackageName, "MainActivity", logFile), "Activity should have started"); - string line = ""; - string logCatFile = Path.Combine (Root, builder.ProjectDirectory, $"timezone-logcat-{timeZone.Replace ("/", "-")}.log"); - ClickButton (proj.PackageName, "myXFButton", "CLICK ME"); - Assert.IsTrue (MonitorAdbLogcat ((l) => { - if (l.Contains ("TimeZoneInfoClick=")) { - line = l; - return l.Contains ($"{timeZone}"); - } - return false; - }, logCatFile, timeout:30), $"TimeZone should have been {timeZone}. We found : {line}"); - } finally { - RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}"); - RunAdbCommand ($"shell am kill --user all {proj.PackageName}"); - if (!string.IsNullOrEmpty (currentTimeZone)) { - RunAdbCommand ($"shell su root setprop persist.sys.timezone \"{currentTimeZone}\""); - } - if (File.Exists (logFile)) { - TestContext.AddTestAttachment (logFile); - } - } - } - - private const int LOCALIZATION_NODE_COUNT = 12; - - static object [] GetLocalizationTestCases (int node) - { - List tests = new List (); - var ignore = new string [] { - "he-IL", // maps to wi-IL on Android. - "id-ID", // maps to in-ID on Android - }; - foreach (CultureInfo ci in CultureInfo.GetCultures(CultureTypes.SpecificCultures)) { - if (ci.Name.Length > 5) { - TestContext.WriteLine ($"Skipping {ci.Name} Localization Test"); - continue; - } - if (ignore.Contains (ci.Name)) { - TestContext.WriteLine ($"Ignoring {ci.Name} Localization Test"); - continue; - } - tests.Add (new object [] { - ci.Name, - }); - } - - return tests.Where (p => tests.IndexOf (p) % LOCALIZATION_NODE_COUNT == node).ToArray (); - } - - [Test] - [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 0 })] - [Category ("Localization")] - [Retry (2)] - public void CheckLocalizationIsCorrectNode1 (string locale) => CheckLocalizationIsCorrect (locale); - - [Test] - [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 1 })] - [Category ("Localization")] - [Retry (2)] - public void CheckLocalizationIsCorrectNode2 (string locale) => CheckLocalizationIsCorrect (locale); - - [Test] - [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 2 })] - [Category ("Localization")] - [Retry (2)] - public void CheckLocalizationIsCorrectNode3 (string locale) => CheckLocalizationIsCorrect (locale); - - [Test] - [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 3 })] - [Category ("Localization")] - [Retry (2)] - public void CheckLocalizationIsCorrectNode4 (string locale) => CheckLocalizationIsCorrect (locale); - - [Test] - [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 4 })] - [Category ("Localization")] - [Retry (2)] - public void CheckLocalizationIsCorrectNode5 (string locale) => CheckLocalizationIsCorrect (locale); - - [Test] - [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 5 })] - [Category ("Localization")] - [Retry (2)] - public void CheckLocalizationIsCorrectNode6 (string locale) => CheckLocalizationIsCorrect (locale); - - [Test] - [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 6 })] - [Category ("Localization")] - [Retry (2)] - public void CheckLocalizationIsCorrectNode7 (string locale) => CheckLocalizationIsCorrect (locale); - - [Test] - [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 7 })] - [Category ("Localization")] - [Retry (2)] - public void CheckLocalizationIsCorrectNode8 (string locale) => CheckLocalizationIsCorrect (locale); - - [Test] - [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 8 })] - [Category ("Localization")] - [Retry (2)] - public void CheckLocalizationIsCorrectNode9 (string locale) => CheckLocalizationIsCorrect (locale); - - [Test] - [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 9 })] - [Category ("Localization")] - [Retry (2)] - public void CheckLocalizationIsCorrectNode10 (string locale) => CheckLocalizationIsCorrect (locale); - - public void CheckLocalizationIsCorrect (string locale) - { - AssertHasDevices (); - - string currentLocale = RunAdbCommand ("shell getprop persist.sys.locale")?.Trim (); - TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: Current Locale is {currentLocale}"); - string deviceLocale = currentLocale; - string logFile = Path.Combine (Root, builder.ProjectDirectory, $"startup-logcat-{locale.Replace ("/", "-")}.log"); - string monitorLogFile = Path.Combine (Root, builder.ProjectDirectory, $"monitor-logcat-{locale.Replace ("/", "-")}.log"); - try { - TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: Setting Locale to {locale}"); - if (deviceLocale != locale) { - for (int attempt = 0; attempt < 5; attempt++) { - TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: attempt {attempt}"); - RunAdbCommand ($"shell su root setprop persist.sys.locale {locale}"); - RunAdbCommand ("shell su root setprop ctl.restart zygote"); - if (!MonitorAdbLogcat ((l) => { - if (l.Contains ("Finished processing BOOT_COMPLETED for")) - return true; - return false; - }, monitorLogFile, timeout:60)) { - TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: wating for boot to complete failed or timed out."); - } - WaitFor ((int)TimeSpan.FromSeconds (5).TotalMilliseconds); - deviceLocale = RunAdbCommand ("shell getprop persist.sys.locale")?.Trim (); - if (deviceLocale == locale) { - break; - } - } - } - Assert.AreEqual (locale, deviceLocale, $"The command to set the device locale to {locale} failed. Current device locale is {deviceLocale}"); - ClearAdbLogcat (); - RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}"); - RunAdbCommand ($"shell am kill --user all {proj.PackageName}"); - WaitFor ((int)TimeSpan.FromSeconds (2).TotalMilliseconds); - ClearAdbLogcat (); - AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); - Assert.IsTrue (WaitForActivityToStart (proj.PackageName, "MainActivity", logFile, timeout: 120), "Activity should have started"); - string line = ""; - string logCatFile = Path.Combine (Root, builder.ProjectDirectory, $"locale-logcat-{locale.Replace ("/", "-")}.log"); - ClickButton (proj.PackageName, "myXFButton", "CLICK ME"); - Assert.IsTrue (MonitorAdbLogcat ((l) => { - if (l.Contains ("StringsClick=") || l.Contains ("Strings.SomeString=")) { - line = l; - bool result = l.Contains ($"{locale}"); - if (l.Contains ("Strings.SomeString=") && !result) - return false; - return result; - } - return false; - }, logCatFile, timeout:30), $"Locale should have been {locale}. We found : {line}"); - } finally { - RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}"); - RunAdbCommand ($"shell am kill --user all {proj.PackageName}"); - if (!string.IsNullOrEmpty (currentLocale)) { - TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: Setting Locale back to {currentLocale}"); - RunAdbCommand ($"shell su root setprop persist.sys.locale \"{currentLocale}\""); - RunAdbCommand ("shell su root setprop ctl.restart zygote"); - MonitorAdbLogcat ((l) => { - if (l.Contains ("Finished processing BOOT_COMPLETED for")) - return true; - return false; - }, monitorLogFile, timeout:60); - WaitFor ((int)TimeSpan.FromSeconds (2).TotalMilliseconds); - } - if (File.Exists (logFile)) { - TestContext.AddTestAttachment (logFile); - } - if (File.Exists (monitorLogFile)) { - TestContext.AddTestAttachment (monitorLogFile); - } - } - } - } -} diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs index c202c0c264a..bb503c04ec7 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs @@ -1,7 +1,10 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text.RegularExpressions; +using System.Xml.Linq; +using System.Xml.XPath; using NUnit.Framework; using Xamarin.ProjectTools; @@ -17,13 +20,6 @@ public class InstallAndRunTests : DeviceTest [TearDown] public void Teardown () { - if (HasDevices && proj != null) - RunAdbCommand ($"uninstall {proj.PackageName}"); - - if (TestContext.CurrentContext.Result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Passed - && builder != null && Directory.Exists (builder.ProjectDirectory)) - Directory.Delete (builder.ProjectDirectory, recursive: true); - builder?.Dispose (); builder = null; proj = null; @@ -78,8 +74,6 @@ public void NativeAssemblyCacheWithSatelliteAssemblies () [Test] public void GlobalLayoutEvent_ShouldRegisterAndFire_OnActivityLaunch ([Values (false, true)] bool isRelease) { - AssertHasDevices (); - string expectedLogcatOutput = "Bug 29730: GlobalLayout event handler called!"; proj = new XamarinAndroidApplicationProject () { @@ -99,7 +93,6 @@ void Button_ViewTreeObserver_GlobalLayout (object sender, EventArgs e) "); builder = CreateApkBuilder (Path.Combine ("temp", $"Bug29730-{isRelease}")); Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); - ClearAdbLogcat (); AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); Assert.IsTrue (MonitorAdbLogcat ((line) => { return line.Contains (expectedLogcatOutput); @@ -109,8 +102,6 @@ void Button_ViewTreeObserver_GlobalLayout (object sender, EventArgs e) [Test] public void SubscribeToAppDomainUnhandledException () { - AssertHasDevices (); - proj = new XamarinAndroidApplicationProject () { IsRelease = true, }; @@ -124,11 +115,7 @@ public void SubscribeToAppDomainUnhandledException () "); builder = CreateApkBuilder (); Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); - ClearAdbLogcat (); - if (CommercialBuildAvailable) - Assert.True (builder.RunTarget (proj, "_Run"), "Project should have run."); - else - AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); + RunProjectAndAssert (proj, builder); #if NETCOREAPP string expectedLogcatOutput = "# Unhandled Exception: sender=System.Object; e.IsTerminating=True; e.ExceptionObject=System.Exception: CRASH"; @@ -189,8 +176,6 @@ void SymbolicateAndAssert (string symbolArchivePath, string logcatFilePath, IEnu [Test, Category ("MonoSymbolicate")] public void MonoSymbolicateAndroidStackTrace () { - AssertHasDevices (); - proj = new XamarinAndroidApplicationProject () { IsRelease = true, }; @@ -203,12 +188,7 @@ public void MonoSymbolicateAndroidStackTrace () Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); var archivePath = Path.Combine (Root, builder.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}.apk.mSYM"); Assert.IsTrue (Directory.Exists (archivePath), $"Symbol archive path {archivePath} should exist."); - - ClearAdbLogcat (); - if (CommercialBuildAvailable) - Assert.True (builder.RunTarget (proj, "_Run"), "Project should have run."); - else - AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); + RunProjectAndAssert (proj, builder); var logcatPath = Path.Combine (Root, builder.ProjectDirectory, "crash-logcat.log"); MonitorAdbLogcat ((line) => { @@ -229,7 +209,6 @@ public void MonoSymbolicateAndroidStackTrace () [Category ("UsesDevice"), Category ("SmokeTests")] public void SmokeTestBuildAndRunWithSpecialCharacters () { - AssertHasDevices (); var testName = "ใƒ†ใ‚นใƒˆ"; var rootPath = Path.Combine (Root, "temp", TestName); @@ -237,11 +216,11 @@ public void SmokeTestBuildAndRunWithSpecialCharacters () ProjectName = testName, IsRelease = true, }; - proj.SetAndroidSupportedAbis ("armeabi-v7a", "x86", "x86_64"); + proj.SetAndroidSupportedAbis ("arm64-v8a", "x86_64"); proj.SetDefaultTargetDevice (); using (var builder = CreateApkBuilder (Path.Combine (rootPath, proj.ProjectName))){ Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); - Assert.IsTrue (builder.RunTarget (proj, "_Run", doNotCleanupOnUpdate: true), "Project should have run."); + RunProjectAndAssert (proj, builder); var timeoutInSeconds = 120; Assert.IsTrue (WaitForActivityToStart (proj.PackageName, "MainActivity", Path.Combine (Root, builder.ProjectDirectory, "startup-logcat.log"), timeoutInSeconds)); @@ -251,8 +230,6 @@ public void SmokeTestBuildAndRunWithSpecialCharacters () [Test, Category ("MonoSymbolicate")] public void MonoSymbolicateNetStandardStackTrace () { - AssertHasDevices (); - var lib = new DotNetStandard { ProjectName = "Library1", Sdk = "Microsoft.NET.Sdk", @@ -299,12 +276,7 @@ public string GetData() { Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); var archivePath = Path.Combine (Root, builder.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}.apk.mSYM"); Assert.IsTrue (Directory.Exists (archivePath), $"Symbol archive path {archivePath} should exist."); - - ClearAdbLogcat (); - if (CommercialBuildAvailable) - Assert.True (builder.RunTarget (proj, "_Run"), "Project should have run."); - else - AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); + RunProjectAndAssert (proj, builder); var logcatPath = Path.Combine (Root, builder.ProjectDirectory, "crash-logcat.log"); MonitorAdbLogcat ((line) => { @@ -338,7 +310,6 @@ public string GetData() { [Category ("DotNetIgnore")] // TODO: libmono-profiler-log.so is missing in .NET 6 public void ProfilerLogOptions_ShouldCreateMlpdFiles ([ValueSource (nameof (ProfilerOptions))] string profilerOption) { - AssertHasDevices (); AssertCommercialBuild (); proj = new XamarinAndroidApplicationProject () { @@ -350,7 +321,7 @@ public void ProfilerLogOptions_ShouldCreateMlpdFiles ([ValueSource (nameof (Prof File.Delete (mlpdDestination); RunAdbCommand ($"shell setprop debug.mono.profile {profilerOption}"); - Assert.True (builder.RunTarget (proj, "_Run"), "Project should have run."); + RunProjectAndAssert (proj, builder); Assert.True (WaitForActivityToStart (proj.PackageName, "MainActivity", Path.Combine (Root, builder.ProjectDirectory, "logcat.log"), 30), "Activity should have started."); @@ -380,8 +351,6 @@ public void ProfilerLogOptions_ShouldCreateMlpdFiles ([ValueSource (nameof (Prof [Test] public void CustomLinkDescriptionPreserve ([Values (AndroidLinkMode.SdkOnly, AndroidLinkMode.Full)] AndroidLinkMode linkMode) { - AssertHasDevices (); - var lib1 = new XamarinAndroidLibraryProject () { ProjectName = "Library1", Sources = { @@ -528,12 +497,7 @@ public class LinkModeFullClass { builder = CreateApkBuilder (Path.Combine (rootPath, proj.ProjectName)); Assert.IsTrue (builder.Install (proj), "First install should have succeeded."); - - ClearAdbLogcat (); - if (CommercialBuildAvailable) - Assert.True (builder.RunTarget (proj, "_Run"), "Project should have run."); - else - AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); + RunProjectAndAssert (proj, builder); var logcatPath = Path.Combine (Root, builder.ProjectDirectory, "logcat.log"); Assert.IsTrue (MonitorAdbLogcat ((line) => { @@ -552,8 +516,6 @@ public class LinkModeFullClass { [Test] public void JsonDeserializationCreatesJavaHandle ([Values (false, true)] bool isRelease) { - AssertHasDevices (); - proj = new XamarinAndroidApplicationProject () { IsRelease = isRelease, }; @@ -667,7 +629,6 @@ public override Type BindToType (string assemblyName, string typeName) builder = CreateApkBuilder (); Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); - ClearAdbLogcat (); AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); Assert.IsFalse (MonitorAdbLogcat ((line) => { return line.Contains ("TestJsonDeserializationCreatesJavaHandle"); @@ -677,8 +638,6 @@ public override Type BindToType (string assemblyName, string typeName) [Test] public void RunWithInterpreterEnabled ([Values (false, true)] bool isRelease) { - AssertHasDevices (); - proj = new XamarinAndroidApplicationProject () { IsRelease = isRelease, AotAssemblies = false, // Release defaults to Profiled AOT for .NET 6 @@ -697,16 +656,10 @@ public void RunWithInterpreterEnabled ([Values (false, true)] bool isRelease) } } - ClearAdbLogcat (); RunAdbCommand ("shell setprop debug.mono.log all"); var logProp = RunAdbCommand ("shell getprop debug.mono.log")?.Trim (); Assert.AreEqual (logProp, "all", "The debug.mono.log prop was not set correctly."); - - builder.BuildLogFile = "run.log"; - if (CommercialBuildAvailable) - Assert.True (builder.RunTarget (proj, "_Run"), "Project should have run."); - else - AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); + RunProjectAndAssert (proj, builder); Func checkForInterpMessage = line => { return line.Contains ("Enabling Mono Interpreter"); @@ -728,12 +681,10 @@ public void RunWithInterpreterEnabled ([Values (false, true)] bool isRelease) [Test] public void RunWithLLVMEnabled () { - AssertHasDevices (); - var proj = new XamarinAndroidApplicationProject () { IsRelease = true, }; - proj.SetAndroidSupportedAbis ("armeabi-v7a", "x86", "x86_64"); + proj.SetAndroidSupportedAbis ("armeabi-v7a", "arm64-v8a", "x86", "x86_64"); proj.SetProperty ("EnableLLVM", true.ToString ()); if (!Builder.UseDotNet) { proj.AotAssemblies = true; @@ -741,13 +692,7 @@ public void RunWithLLVMEnabled () builder = CreateApkBuilder (); Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); - - if (Builder.UseDotNet) - Assert.True (builder.RunTarget (proj, "Run"), "Project should have run."); - else if (CommercialBuildAvailable) - Assert.True (builder.RunTarget (proj, "_Run"), "Project should have run."); - else - AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); + RunProjectAndAssert (proj, builder); Assert.IsTrue (WaitForActivityToStart (proj.PackageName, "MainActivity", Path.Combine (Root, builder.ProjectDirectory, "startup-logcat.log"))); @@ -756,8 +701,6 @@ public void RunWithLLVMEnabled () [Test] public void ResourceDesignerWithNuGetReference ([Values ("net8.0-android33.0")] string dotnetTargetFramework) { - AssertHasDevices (); - string path = Path.Combine (Root, "temp", TestName); if (!Builder.UseDotNet) { @@ -790,7 +733,7 @@ public void ResourceDesignerWithNuGetReference ([Values ("net8.0-android33.0")] var proj = new XamarinAndroidApplicationProject () { IsRelease = true, }; - proj.SetAndroidSupportedAbis ("armeabi-v7a", "x86", "x86_64"); + proj.SetAndroidSupportedAbis ("arm64-v8a", "x86_64"); proj.OtherBuildItems.Add (new BuildItem ("None", "NuGet.config") { TextContent = () => @" @@ -813,8 +756,6 @@ public void ResourceDesignerWithNuGetReference ([Values ("net8.0-android33.0")] [Test] public void SingleProject_ApplicationId ([Values (false, true)] bool testOnly) { - AssertHasDevices (); - proj = new XamarinAndroidApplicationProject (); proj.SetProperty ("ApplicationId", "com.i.should.get.overridden.by.the.manifest"); if (testOnly) @@ -824,13 +765,7 @@ public void SingleProject_ApplicationId ([Values (false, true)] bool testOnly) proj.SetAndroidSupportedAbis (abis); builder = CreateApkBuilder (); Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); - - if (Builder.UseDotNet) - Assert.True (builder.RunTarget (proj, "Run"), "Project should have run."); - else if (CommercialBuildAvailable) - Assert.True (builder.RunTarget (proj, "_Run"), "Project should have run."); - else - AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); + RunProjectAndAssert (proj, builder); var didStart = WaitForActivityToStart (proj.PackageName, "MainActivity", Path.Combine (Root, builder.ProjectDirectory, "startup-logcat.log")); @@ -840,8 +775,6 @@ public void SingleProject_ApplicationId ([Values (false, true)] bool testOnly) [Test] public void AppWithStyleableUsageRuns ([Values (true, false)] bool isRelease, [Values (true, false)] bool linkResources) { - AssertHasDevices (); - var rootPath = Path.Combine (Root, "temp", TestName); var lib = new XamarinAndroidLibraryProject () { ProjectName = "Styleable.Library" @@ -935,19 +868,139 @@ public MyLayout (Android.Content.Context context, Android.Util.IAttributeSet att Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); - - if (Builder.UseDotNet) - Assert.True (builder.RunTarget (proj, "Run"), "Project should have run."); - else if (CommercialBuildAvailable) - Assert.True (builder.RunTarget (proj, "_Run"), "Project should have run."); - else - AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); + RunProjectAndAssert (proj, builder); var didStart = WaitForActivityToStart (proj.PackageName, "MainActivity", Path.Combine (Root, builder.ProjectDirectory, "startup-logcat.log")); Assert.IsTrue (didStart, "Activity should have started."); } + [Test] + public void CheckXamarinFormsAppDeploysAndAButtonWorks () + { + var proj = new XamarinFormsAndroidApplicationProject (); + proj.SetAndroidSupportedAbis ("arm64-v8a", "x86_64"); + var builder = CreateApkBuilder (); + + Assert.IsTrue (builder.Build (proj), "Build should have succeeded."); + builder.BuildLogFile = "install.log"; + Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); + + AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); + WaitForActivityToStart (proj.PackageName, "MainActivity", + Path.Combine (Root, builder.ProjectDirectory, "startup-logcat.log"), 15); + ClearAdbLogcat (); + ClearBlockingDialogs (); + ClickButton (proj.PackageName, "myXFButton", "CLICK ME"); + Assert.IsTrue (MonitorAdbLogcat ((line) => { + return line.Contains ("Button was Clicked!"); + }, Path.Combine (Root, builder.ProjectDirectory, "button-logcat.log")), "Button Should have been Clicked."); + } + + + [Test] + public void CheckResouceIsOverridden ([Values (true, false)] bool useAapt2) + { + AssertAaptSupported (useAapt2); + + var library = new XamarinAndroidLibraryProject () { + ProjectName = "Library1", + AndroidResources = { + new AndroidItem.AndroidResource (() => "Resources\\values\\strings2.xml") { + TextContent = () => @" + + Click Me! One +", + }, + }, + }; + var library2 = new XamarinAndroidLibraryProject () { + ProjectName = "Library2", + AndroidResources = { + new AndroidItem.AndroidResource (() => "Resources\\values\\strings2.xml") { + TextContent = () => @" + + Click Me! Two +", + }, + }, + }; + var app = new XamarinAndroidApplicationProject () { + PackageName = "Xamarin.ResourceTest", + References = { + new BuildItem.ProjectReference ("..\\Library1\\Library1.csproj"), + new BuildItem.ProjectReference ("..\\Library2\\Library2.csproj"), + }, + }; + library.AndroidUseAapt2 = + library2.AndroidUseAapt2 = + app.AndroidUseAapt2 = useAapt2; + app.LayoutMain = app.LayoutMain.Replace ("@string/hello", "@string/hello_me"); + using (var l1 = CreateDllBuilder (Path.Combine ("temp", TestName, library.ProjectName))) + using (var l2 = CreateDllBuilder (Path.Combine ("temp", TestName, library2.ProjectName))) + using (var b = CreateApkBuilder (Path.Combine ("temp", TestName, app.ProjectName))) { + b.ThrowOnBuildFailure = false; + string apiLevel; + app.TargetFrameworkVersion = b.LatestTargetFrameworkVersion (out apiLevel); + + app.AndroidManifest = $@" + + + + + "; + Assert.IsTrue (l1.Build (library, doNotCleanupOnUpdate: true), $"Build of {library.ProjectName} should have suceeded."); + Assert.IsTrue (l2.Build (library2, doNotCleanupOnUpdate: true), $"Build of {library2.ProjectName} should have suceeded."); + b.BuildLogFile = "build1.log"; + Assert.IsTrue (b.Build (app, doNotCleanupOnUpdate: true), $"Build of {app.ProjectName} should have suceeded."); + b.BuildLogFile = "install1.log"; + Assert.IsTrue (b.Install (app, doNotCleanupOnUpdate: true), "Install should have suceeded."); + AdbStartActivity ($"{app.PackageName}/{app.JavaPackageName}.MainActivity"); + WaitForPermissionActivity (Path.Combine (Root, b.ProjectDirectory, "permission-logcat.log")); + WaitForActivityToStart (app.PackageName, "MainActivity", + Path.Combine (Root, b.ProjectDirectory, "startup-logcat.log"), 15); + ClearBlockingDialogs (); + XDocument ui = GetUI (); + XElement node = ui.XPathSelectElement ($"//node[contains(@resource-id,'myButton')]"); + Assert.IsNotNull (node , "Could not find `my-Button` in the user interface. Check the screenshot of the test failure."); + StringAssert.AreEqualIgnoringCase ("Click Me! One", node.Attribute ("text").Value, "Text of Button myButton should have been \"Click Me! One\""); + b.BuildLogFile = "clean.log"; + Assert.IsTrue (b.Clean (app, doNotCleanupOnUpdate: true), "Clean should have suceeded."); + + app = new XamarinAndroidApplicationProject () { + PackageName = "Xamarin.ResourceTest", + References = { + new BuildItem.ProjectReference ("..\\Library1\\Library1.csproj"), + new BuildItem.ProjectReference ("..\\Library2\\Library2.csproj"), + }, + }; + + library2.References.Add (new BuildItem.ProjectReference ("..\\Library1\\Library1.csproj")); + app.AndroidUseAapt2 = useAapt2; + app.LayoutMain = app.LayoutMain.Replace ("@string/hello", "@string/hello_me"); + app.TargetFrameworkVersion = b.LatestTargetFrameworkVersion (out apiLevel); + + app.AndroidManifest = $@" + + + + + "; + b.BuildLogFile = "build.log"; + Assert.IsTrue (b.Build (app, doNotCleanupOnUpdate: true), $"Build of {app.ProjectName} should have suceeded."); + b.BuildLogFile = "install.log"; + Assert.IsTrue (b.Install (app, doNotCleanupOnUpdate: true), "Install should have suceeded."); + AdbStartActivity ($"{app.PackageName}/{app.JavaPackageName}.MainActivity"); + WaitForPermissionActivity (Path.Combine (Root, b.ProjectDirectory, "permission-logcat.log")); + WaitForActivityToStart (app.PackageName, "MainActivity", + Path.Combine (Root, b.ProjectDirectory, "startup-logcat.log"), 15); + ui = GetUI (); + node = ui.XPathSelectElement ($"//node[contains(@resource-id,'myButton')]"); + StringAssert.AreEqualIgnoringCase ("Click Me! One", node.Attribute ("text").Value, "Text of Button myButton should have been \"Click Me! One\""); + } + } + + DotNetCLI CreateDotNetBuilder (string relativeProjectDir = null) { if (string.IsNullOrEmpty (relativeProjectDir)) { diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs index 582f5d0a1c1..75ffe121ca4 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs @@ -32,7 +32,6 @@ string GetContentFromAllOverrideDirectories (string packageName, bool useRunAsCo public void ReInstallIfUserUninstalled ([Values (false, true)] bool isRelease) { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject () { IsRelease = isRelease, @@ -61,7 +60,6 @@ public void ReInstallIfUserUninstalled ([Values (false, true)] bool isRelease) public void InstallAndUnInstall ([Values (false, true)] bool isRelease) { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject () { IsRelease = isRelease, @@ -95,7 +93,6 @@ public void InstallAndUnInstall ([Values (false, true)] bool isRelease) public void ChangeKeystoreRedeploy () { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject () { PackageName = "com.xamarin.keytest" @@ -125,7 +122,6 @@ public void ChangeKeystoreRedeploy () public void SwitchConfigurationsShouldRedeploy () { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject () { IsRelease = false, @@ -171,7 +167,6 @@ public void SwitchConfigurationsShouldRedeploy () public void InstallWithoutSharedRuntime () { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject () { IsRelease = true, @@ -229,7 +224,6 @@ public void InstallWithoutSharedRuntime () public void InstallErrorCode () { AssertCommercialBuild (); - AssertHasDevices (); //Setup a situation where we get INSTALL_FAILED_NO_MATCHING_ABIS var abi = "armeabi-v7a"; @@ -252,7 +246,6 @@ public void InstallErrorCode () public void ToggleFastDev () { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject { EmbedAssembliesIntoApk = false, @@ -339,7 +332,6 @@ public void ToggleDebugReleaseWithSigning ([Values ("aab", "apk")] string packag public void LoggingPropsShouldCreateOverrideDirForRelease () { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject { IsRelease = true, @@ -357,7 +349,7 @@ public void LoggingPropsShouldCreateOverrideDirForRelease () using (var builder = CreateApkBuilder ()) { Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); RunAdbCommand ("shell setprop debug.mono.log timing"); - Assert.True (builder.RunTarget (proj, "_Run"), "Project should have run."); + RunProjectAndAssert (proj, builder); var didLaunch = WaitForActivityToStart (proj.PackageName, "MainActivity", Path.Combine (Root, builder.ProjectDirectory, "logcat.log"), 30); RunAdbCommand ("shell setprop debug.mono.log \"\""); Assert.True (didLaunch, "Activity should have started."); @@ -371,7 +363,6 @@ public void LoggingPropsShouldCreateOverrideDirForRelease () public void BlankAdbTarget () { AssertCommercialBuild (); - AssertHasDevices (); var serial = GetAttachedDeviceSerial (); var proj = new XamarinAndroidApplicationProject () { @@ -423,7 +414,6 @@ public void BlankAdbTarget () [TestCaseSource (nameof (AndroidStoreKeyTests))] public void TestAndroidStoreKey (bool useApkSigner, bool isRelease, string packageFormat, string androidKeyStore, string password, string expected, bool shouldInstall) { - AssertHasDevices (); if (DeviceSdkVersion >= 30 && !useApkSigner && packageFormat == "apk") { Assert.Ignore ($"Test Skipped. jarsigner and {packageFormat} does not work with API 30 and above"); return; @@ -490,7 +480,6 @@ public void TestAndroidStoreKey (bool useApkSigner, bool isRelease, string packa public void LocalizedAssemblies_ShouldBeFastDeployed () { AssertCommercialBuild (); - AssertHasDevices (); var path = Path.Combine ("temp", TestName); var lib = new XamarinAndroidLibraryProject { @@ -529,7 +518,6 @@ public void LocalizedAssemblies_ShouldBeFastDeployed () public void IncrementalFastDeployment () { AssertCommercialBuild (); - AssertHasDevices (); var class1src = new BuildItem.Source ("Class1.cs") { TextContent = () => "namespace Library1 { public class Class1 { public static int foo = 0; } }" @@ -622,8 +610,6 @@ public void IncrementalFastDeployment () [Test] public void AdbTargetChangesAppBundle () { - AssertHasDevices (); - var proj = new XamarinAndroidApplicationProject { IsRelease = true }; diff --git a/tests/MSBuildDeviceIntegration/Tests/InstantRunTest.cs b/tests/MSBuildDeviceIntegration/Tests/InstantRunTest.cs index daeccc6d9aa..6cbc552c57b 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstantRunTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstantRunTest.cs @@ -14,7 +14,6 @@ public class InstantRunTest : DeviceTest public void InstantRunSimpleBuild () { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinFormsAndroidApplicationProject { AndroidFastDeploymentType = "Assemblies:Dexes", @@ -49,7 +48,6 @@ public void InstantRunSimpleBuild () public void TargetsSkipped ([Values(false, true)] bool useManagedResourceGenerator) { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject () { AndroidFastDeploymentType = "Assemblies:Dexes", @@ -126,7 +124,6 @@ public void TargetsSkipped ([Values(false, true)] bool useManagedResourceGenerat public void SimpleInstallAndUninstall () { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject { AndroidFastDeploymentType = "Assemblies:Dexes", @@ -143,7 +140,6 @@ public void SimpleInstallAndUninstall () public void SkipFastDevAlreadyInstalledFile () { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject { AndroidFastDeploymentType = "Assemblies:Dexes", @@ -183,7 +179,6 @@ public void SkipFastDevAlreadyInstalledFile () public void SkipFastDevAlreadyInstalledResources (Package [] packages, string baseActivityClass) { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject () { AndroidFastDeploymentType = "Assemblies:Dexes", @@ -214,7 +209,6 @@ public void SkipFastDevAlreadyInstalledResources (Package [] packages, string ba public void InstantRunResourceChange () { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject () { AndroidFastDeploymentType = "Assemblies:Dexes", @@ -247,7 +241,6 @@ public void InstantRunResourceChange () public void InstantRunFastDevTypemaps () { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject () { AndroidFastDeploymentType = "Assemblies:Dexes", @@ -273,7 +266,6 @@ public void InstantRunFastDevTypemaps () public void InstantRunNativeLibrary () { AssertCommercialBuild (); - AssertHasDevices (); var nativeLib = new AndroidItem.AndroidNativeLibrary ($"foo\\{DeviceAbi}\\libtest.so") { BinaryContent = () => new byte [10], @@ -316,7 +308,6 @@ public void InstantRunNativeLibrary () public void InstantRunFastDevDexes ([Values (false, true)] bool useEmbeddedDex) { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject () { AndroidFastDeploymentType = "Assemblies:Dexes", @@ -333,12 +324,7 @@ public void InstantRunFastDevDexes ([Values (false, true)] bool useEmbeddedDex) Assert.IsTrue (logLines.Any (l => l.Contains ("Building target \"_Upload\" completely")), "_Upload target should have run"); Assert.IsTrue (logLines.Any (l => l.Contains ("NotifySync CopyFile") && l.Contains ("classes.dex")), "classes.dex should have been uploaded"); ClearAdbLogcat (); - b.BuildLogFile = "run.log"; - if (CommercialBuildAvailable) - Assert.True (b.RunTarget (proj, "_Run"), "Project should have run."); - else - AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); - + RunProjectAndAssert (proj, b); Assert.True (WaitForActivityToStart (proj.PackageName, "MainActivity", Path.Combine (Root, b.ProjectDirectory, "logcat.log"), 30), "Activity should have started."); b.BuildLogFile = "uninstall.log"; diff --git a/tests/MSBuildDeviceIntegration/Tests/LocalizationTests.cs b/tests/MSBuildDeviceIntegration/Tests/LocalizationTests.cs new file mode 100644 index 00000000000..8c0893503e5 --- /dev/null +++ b/tests/MSBuildDeviceIntegration/Tests/LocalizationTests.cs @@ -0,0 +1,254 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using Mono.Unix.Native; +using NUnit.Framework; +using NUnit.Framework.Interfaces; +using Xamarin.ProjectTools; + +namespace Xamarin.Android.Build.Tests +{ + [TestFixture] + [Category ("Localization")] + [NonParallelizable] + public class LocalizationTests : DeviceTest + { + static ProjectBuilder builder; + static XamarinAndroidApplicationProject proj; + string localeFileSuffix; + + [OneTimeSetUp] + public void BeforeAllTests () + { + AssertHasDevices (); + + string debuggable = RunAdbCommand ("shell getprop ro.debuggable"); + if (debuggable != "1") { + Assert.Fail ("LocalizationTests need to use `su root` and this device does not support that feature. Try using an emulator."); + } + + proj = new XamarinAndroidApplicationProject (packageName: "LocalizationTests"); + proj.MainActivity = proj.DefaultMainActivity.Replace ("//${AFTER_ONCREATE}", @"button.Text = $""Strings.SomeString={Strings.SomeString}""; + Console.WriteLine ($""LocaleNative={Java.Util.Locale.Default.Language}-{Java.Util.Locale.Default.Country}""); + Console.WriteLine ($""CurrentCulture={System.Globalization.CultureInfo.CurrentCulture.Name}""); + Console.WriteLine ($""Strings.SomeString={Strings.SomeString}""); +"); + InlineData.AddCultureResourcesToProject (proj, "Strings", "SomeString"); + InlineData.AddCultureResourceDesignerToProject (proj, proj.RootNamespace ?? proj.ProjectName, "Strings", "SomeString"); + + builder = CreateApkBuilder (Path.Combine ("temp", "LocalizationTests")); + builder.BuildLogFile = "onetimesetup-install.log"; + Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); + } + + [SetUp] + public override void SetupTest () + { + var localeParam = TestContext.CurrentContext.Test.Arguments[0] as string; + if (!string.IsNullOrEmpty (localeParam)) { + localeFileSuffix = localeParam.Replace ("/", "-"); + } + + if (!IsDeviceAttached (refreshCachedValue: true)) { + RestartDevice (); + AssertHasDevices (); + } + + // Attempt to reinstall the app that was installed during fixture setup if it is missing + var packageOutput = RunAdbCommand ($"shell pm list packages {proj.PackageName}").Trim (); + var expectedPackageOutput = $"package:{proj.PackageName}"; + if (packageOutput != expectedPackageOutput) { + builder.BuildLogFile = $"setup-install-{localeFileSuffix}.log"; + Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); + } + } + + /// + /// Calling BaseTest.CleanupTest() will cause our shared test directory to be deleted after any test passes + /// This can be problematic for cases that run dozens or hundreds of tests with the same root (CheckTimeZoneInfoIsCorrect, CheckLocalizationIsCorrect) + /// + [TearDown] + protected override void CleanupTest () + { + string output = Path.Combine (Root, builder?.ProjectDirectory); + if (TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed && Directory.Exists (output)) { + foreach (var setupFile in Directory.GetFiles (output, $"*onetimesetup*log", SearchOption.AllDirectories)) { + TestContext.AddTestAttachment (setupFile, Path.GetFileNameWithoutExtension (setupFile)); + } + foreach (var testFile in Directory.GetFiles (output, $"*{localeFileSuffix}*log", SearchOption.AllDirectories)) { + TestContext.AddTestAttachment (testFile, Path.GetFileNameWithoutExtension (testFile)); + } + } + } + + [OneTimeTearDown] + public void AfterAllTests () + { + string output = Path.Combine (Root, builder?.ProjectDirectory); + if (TestContext.CurrentContext.Result.FailCount == 0 && Directory.Exists (output)) { + try { + Directory.Delete (output, recursive: true); + } catch (IOException ex) { + // This happens on CI occasionally, let's not fail the test + TestContext.Out.WriteLine ($"Failed to delete '{output}': {ex}"); + } + } + } + + + const int LOCALIZATION_NODE_COUNT = 15; + const int LOCALIZATION_RETRY_COUNT = 3; + + static object [] GetLocalizationTestCases (int node) + { + List tests = new List (); + var ignore = new string [] { + "he-IL", // maps to wi-IL on Android. + "id-ID", // maps to in-ID on Android + }; + foreach (CultureInfo ci in CultureInfo.GetCultures(CultureTypes.SpecificCultures)) { + if (ci.Name.Length > 5) { + TestContext.WriteLine ($"Skipping {ci.Name} Localization Test"); + continue; + } + if (ignore.Contains (ci.Name)) { + TestContext.WriteLine ($"Ignoring {ci.Name} Localization Test"); + continue; + } + tests.Add (new object [] { + ci.Name, + }); + } + + return tests.Where (p => tests.IndexOf (p) % LOCALIZATION_NODE_COUNT == node).ToArray (); + } + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 0 })] + public void CheckLocalizationIsCorrectNode1 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 1 })] + public void CheckLocalizationIsCorrectNode2 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 2 })] + public void CheckLocalizationIsCorrectNode3 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 3 })] + public void CheckLocalizationIsCorrectNode4 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 4 })] + public void CheckLocalizationIsCorrectNode5 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 5 })] + public void CheckLocalizationIsCorrectNode6 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 6 })] + public void CheckLocalizationIsCorrectNode7 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 7 })] + public void CheckLocalizationIsCorrectNode8 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 8 })] + public void CheckLocalizationIsCorrectNode9 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 9 })] + public void CheckLocalizationIsCorrectNode10 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 10 })] + public void CheckLocalizationIsCorrectNode11 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 11 })] + public void CheckLocalizationIsCorrectNode12 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 12 })] + public void CheckLocalizationIsCorrectNode13 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 13 })] + public void CheckLocalizationIsCorrectNode14 (string locale) => CheckLocalizationIsCorrect (locale); + + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 14 })] + public void CheckLocalizationIsCorrectNode15 (string locale) => CheckLocalizationIsCorrect (locale); + + + public void CheckLocalizationIsCorrect (string locale) + { + var appStartupLogcatFile = Path.Combine (Root, builder.ProjectDirectory, $"startup-logcat-{locale.Replace ("/", "-")}.log"); + string deviceLocale = RunAdbCommand ("shell getprop persist.sys.locale")?.Trim (); + TestContext.Out.WriteLine ($"test value:{locale}, prop value:{deviceLocale}"); + + if (deviceLocale != locale) { + for (int attempt = 0; attempt < 5; attempt++) { + TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: Setting Locale to {locale}, attempt {attempt}..."); + ClearAdbLogcat (); + var rebootLogcatFile = Path.Combine (Root, builder.ProjectDirectory, $"reboot{attempt}-logcat-{locale.Replace ("/", "-")}.log"); + + // https://developer.android.com/guide/topics/resources/localization#changing-the-emulator-locale-from-the-adb-shell + RunAdbCommand ($"shell \"su root setprop persist.sys.locale {locale};su root stop;sleep 5;su root start;\""); + + if (!MonitorAdbLogcat ((l) => { + if (l.Contains ("ActivityManager: Finished processing BOOT_COMPLETED")) + return true; + return false; + }, rebootLogcatFile, timeout: 60)) { + TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: wating for boot to complete failed or timed out."); + } + deviceLocale = RunAdbCommand ("shell getprop persist.sys.locale")?.Trim (); + if (deviceLocale == locale) { + break; + } + } + } + + Assert.AreEqual (locale, deviceLocale, $"The command to set the device locale to {locale} failed. Current device locale is {deviceLocale}"); + ClearAdbLogcat (); + RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}"); + RunAdbCommand ($"shell am kill --user all {proj.PackageName}"); + RunProjectAndAssert (proj, builder, logName: $"run-{locale.Replace ("/", "-")}.log"); + + string logcatSearchString = "Strings.SomeString="; + string expectedLogcatOutput = $"{logcatSearchString}{locale}"; + string logLine = string.Empty; + + Assert.IsTrue (MonitorAdbLogcat ((line) => { + if (line.Contains (logcatSearchString)) { + logLine = line; + return true; + } + return false; + }, appStartupLogcatFile, 45), $"App output did not contain '{logcatSearchString}'"); + + Assert.IsTrue (logLine.Contains (expectedLogcatOutput), $"Line '{logLine}' did not contain '{expectedLogcatOutput}'"); + } + } +} diff --git a/tests/MSBuildDeviceIntegration/Tests/MonoAndroidExportTest.cs b/tests/MSBuildDeviceIntegration/Tests/MonoAndroidExportTest.cs index 3b55b461388..69af71239fd 100644 --- a/tests/MSBuildDeviceIntegration/Tests/MonoAndroidExportTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/MonoAndroidExportTest.cs @@ -13,7 +13,8 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] [Category ("UsesDevice")] - public class MonoAndroidExportTest : DeviceTest { + public class MonoAndroidExportTest : DeviceTest + { #pragma warning disable 414 static object [] MonoAndroidExportTestCases = new object [] { new object[] { @@ -49,7 +50,6 @@ public class MonoAndroidExportTest : DeviceTest { public void MonoAndroidExportReferencedAppStarts (bool embedAssemblies, string fastDevType, bool isRelease) { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject () { IsRelease = isRelease, References = { @@ -118,10 +118,7 @@ protected override void OnCreate (Bundle bundle) "; Assert.True (b.Install (proj), "Project should have installed."); - ClearAdbLogcat (); - b.BuildLogFile = "run.log"; - Assert.True (b.RunTarget (proj, "StartAndroidActivity", doNotCleanupOnUpdate: true), "Project should have run."); - + RunProjectAndAssert (proj, b, doNotCleanupOnUpdate: true); Assert.True (WaitForActivityToStart (proj.PackageName, "MainActivity", Path.Combine (Root, b.ProjectDirectory, "logcat.log"), 30), "Activity should have started."); string expectedLogcatOutput = "ContainsExportedMethods: constructed! Handle="; diff --git a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs index e7e811eea17..0b44e54f68c 100644 --- a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs @@ -65,7 +65,7 @@ void Profile (ProjectBuilder builder, Action action, [CallerMemb double GetDurationFromBinLog (ProjectBuilder builder) { - var binlog = Path.Combine (Root, builder.ProjectDirectory, "msbuild.binlog"); + var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog"); FileAssert.Exists (binlog); var build = BinaryLog.ReadBuild (binlog); @@ -305,7 +305,6 @@ public void Build_XAML_Change (bool produceReferenceAssembly, bool install) { if (install) { AssertCommercialBuild (); // This test will fail without Fast Deployment - AssertHasDevices (); } var path = Path.Combine ("temp", TestName); @@ -381,7 +380,6 @@ public void Build_XAML_Change (bool produceReferenceAssembly, bool install) public void Install_CSharp_Change () { AssertCommercialBuild (); // This test will fail without Fast Deployment - AssertHasDevices (); var proj = CreateApplicationProject (); proj.PackageName = "com.xamarin.install_csharp_change"; diff --git a/tests/MSBuildDeviceIntegration/Tests/SystemApplicationTests.cs b/tests/MSBuildDeviceIntegration/Tests/SystemApplicationTests.cs index 63f4eeb5db5..6d41a39204f 100644 --- a/tests/MSBuildDeviceIntegration/Tests/SystemApplicationTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/SystemApplicationTests.cs @@ -19,7 +19,6 @@ public class SystemApplicationTests : DeviceTest public void SystemApplicationCanInstall () { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XamarinAndroidApplicationProject () { IsRelease = false, diff --git a/tests/MSBuildDeviceIntegration/Tests/TimeZoneInfoTests.cs b/tests/MSBuildDeviceIntegration/Tests/TimeZoneInfoTests.cs new file mode 100644 index 00000000000..cc25697cc69 --- /dev/null +++ b/tests/MSBuildDeviceIntegration/Tests/TimeZoneInfoTests.cs @@ -0,0 +1,227 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using NUnit.Framework; +using NUnit.Framework.Interfaces; +using Xamarin.ProjectTools; + +namespace Xamarin.Android.Build.Tests +{ + [TestFixture] + [Category ("TimeZoneInfo")] + [NonParallelizable] + public class TimeZoneInfoTests : DeviceTest + { + static ProjectBuilder builder; + + [OneTimeSetUp] + public void BeforeAllTests () + { + string debuggable = RunAdbCommand ("shell getprop ro.debuggable"); + if (debuggable != "1") { + Assert.Fail ("TimeZoneInfoTests need to use `su root` and this device does not support that feature. Try using an emulator."); + } + // Disable auto timezone + RunAdbCommand ("shell settings put global auto_time_zone 0"); + + builder = CreateApkBuilder (Path.Combine ("temp", "TimeZoneInfoTests")); + } + + [SetUp] + public override void SetupTest () + { + if (!IsDeviceAttached (refreshCachedValue: true)) { + RestartDevice (); + AssertHasDevices (); + } + } + + /// + /// Calling BaseTest.CleanupTest() will cause our shared test directory to be deleted after any test passes + /// This can be problematic for cases that run dozens or hundreds of tests with the same root (CheckTimeZoneInfoIsCorrect, CheckLocalizationIsCorrect) + /// + [TearDown] + protected override void CleanupTest () + { + var tzParam = TestContext.CurrentContext.Test.Arguments[0] as string; + if (!string.IsNullOrEmpty (tzParam)) { + tzParam = tzParam.Replace ("/", "-"); + } + + string output = Path.Combine (Root, builder?.ProjectDirectory); + if (TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed && Directory.Exists (output)) { + foreach (var testFile in Directory.GetFiles (output, $"*{tzParam}*log", SearchOption.AllDirectories)) { + TestContext.AddTestAttachment (testFile, Path.GetFileNameWithoutExtension (testFile)); + } + } + } + + [OneTimeTearDown] + public void AfterAllTests () + { + string output = Path.Combine (Root, builder?.ProjectDirectory); + if (TestContext.CurrentContext.Result.FailCount == 0 && Directory.Exists (output)) { + try { + Directory.Delete (output, recursive: true); + } catch (IOException ex) { + // This happens on CI occasionally, let's not fail the test + TestContext.Out.WriteLine ($"Failed to delete '{output}': {ex}"); + } + } + } + + + const int TIMEZONE_NODE_COUNT = 15; + const int TIMEZONE_RETRY_COUNT = 3; + + static object [] GetTimeZoneTestCases (int node) + { + List tests = new List (); + var ignore = new string [] { + "Asia/Qostanay", + "US/Pacific-New" + }; + + foreach (var tz in NodaTime.DateTimeZoneProviders.Tzdb.Ids) { + if (ignore.Contains (tz)) { + TestContext.WriteLine ($"Ignoring {tz} TimeZone Test"); + continue; + } + tests.Add (new object [] { + tz, + }); + } + return tests.Where (p => tests.IndexOf (p) % TIMEZONE_NODE_COUNT == node).ToArray (); + } + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 0 })] + public void CheckTimeZoneInfoIsCorrectNode1 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 1 })] + public void CheckTimeZoneInfoIsCorrectNode2 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 2 })] + public void CheckTimeZoneInfoIsCorrectNode3 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 3 })] + public void CheckTimeZoneInfoIsCorrectNode4 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 4 })] + public void CheckTimeZoneInfoIsCorrectNode5 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 5 })] + public void CheckTimeZoneInfoIsCorrectNode6 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 6 })] + public void CheckTimeZoneInfoIsCorrectNode7 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 7 })] + public void CheckTimeZoneInfoIsCorrectNode8 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 8 })] + public void CheckTimeZoneInfoIsCorrectNode9 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 9 })] + public void CheckTimeZoneInfoIsCorrectNode10 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 10 })] + public void CheckTimeZoneInfoIsCorrectNode11 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 11 })] + public void CheckTimeZoneInfoIsCorrectNode12 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 12 })] + public void CheckTimeZoneInfoIsCorrectNode13 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 13 })] + public void CheckTimeZoneInfoIsCorrectNode14 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 14 })] + public void CheckTimeZoneInfoIsCorrectNode15 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + + + public void CheckTimeZoneInfoIsCorrect (string timeZone) + { + var proj = new XamarinAndroidApplicationProject (packageName: "TimeZoneInfoTests"); + if (Builder.UseDotNet) { + proj.MainActivity = proj.DefaultMainActivity.Replace ("//${AFTER_ONCREATE}", @"button.Text = $""TimeZoneInfo={TimeZoneInfo.Local.Id}""; + Console.WriteLine ($""TimeZoneInfoNative={Java.Util.TimeZone.Default.ID}""); + Console.WriteLine ($""TimeZoneInfoTests.TimeZoneInfo={TimeZoneInfo.Local.Id}""); +"); + } else { + proj.MainActivity = proj.DefaultMainActivity.Replace ("//${AFTER_ONCREATE}", @"button.Text = $""TimeZoneInfo={TimeZoneInfo.Local.DisplayName}""; + Console.WriteLine ($""TimeZoneInfoNative={Java.Util.TimeZone.Default.ID}""); + Console.WriteLine ($""TimeZoneInfoTests.TimeZoneInfo={TimeZoneInfo.Local.DisplayName}""); +"); + } + + var appStartupLogcatFile = Path.Combine (Root, builder.ProjectDirectory, $"startup-logcat-{timeZone.Replace ("/", "-")}.log"); + string deviceTz = RunAdbCommand ("shell getprop persist.sys.timezone")?.Trim (); + + if (deviceTz != timeZone) { + for (int attempt = 0; attempt < 5; attempt++) { + TestContext.Out.WriteLine ($"{nameof (CheckTimeZoneInfoIsCorrect)}: Setting TimeZone to {timeZone}, attempt {attempt}..."); + ClearAdbLogcat (); + RunAdbCommand ($"shell su root setprop persist.sys.timezone \"{timeZone}\""); + deviceTz = RunAdbCommand ("shell getprop persist.sys.timezone")?.Trim (); + if (deviceTz == timeZone) { + break; + } + } + } + + Assert.AreEqual (timeZone, deviceTz, $"The command to set the device timezone to {timeZone} failed. Current device timezone is {deviceTz}"); + builder.BuildLogFile = $"install-{timeZone.Replace ("/", "-")}.log"; + Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); + ClearAdbLogcat (); + RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}"); + RunAdbCommand ($"shell am kill --user all {proj.PackageName}"); + RunProjectAndAssert (proj, builder, logName: $"run-{timeZone.Replace ("/", "-")}.log"); + + string logcatSearchString = "TimeZoneInfoTests.TimeZoneInfo="; + string expectedLogcatOutput = $"{logcatSearchString}{timeZone}"; + string logLine = string.Empty; + + Assert.IsTrue (MonitorAdbLogcat ((line) => { + if (line.Contains (logcatSearchString)) { + logLine = line; + return true; + } + return false; + }, appStartupLogcatFile, 45), $"App output did not contain '{logcatSearchString}'"); + + Assert.IsTrue (logLine.Contains (expectedLogcatOutput), $"Line '{logLine}' did not contain '{expectedLogcatOutput}'"); + } + } +} diff --git a/tests/MSBuildDeviceIntegration/Tests/UncaughtExceptionTests.cs b/tests/MSBuildDeviceIntegration/Tests/UncaughtExceptionTests.cs index cbd646f55c8..50a6425c5be 100644 --- a/tests/MSBuildDeviceIntegration/Tests/UncaughtExceptionTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/UncaughtExceptionTests.cs @@ -22,8 +22,6 @@ class LogcatLine [Test] public void EnsureUncaughtExceptionWorks () { - AssertHasDevices (); - var lib = new XamarinAndroidBindingProject { ProjectName = "Scratch.Try", AndroidClassParser = "class-parse", @@ -163,9 +161,6 @@ public void OnCatch (Java.Lang.Throwable t) using (var appBuilder = CreateApkBuilder (Path.Combine (path, app.ProjectName))) { Assert.True (libBuilder.Build (lib), "Library should have built."); Assert.IsTrue (appBuilder.Install (app), "Install should have succeeded."); - - ClearAdbLogcat (); - AdbStartActivity ($"{app.PackageName}/{app.JavaPackageName}.MainActivity"); string logcatPath = Path.Combine (Root, appBuilder.ProjectDirectory, "logcat.log"); diff --git a/tests/MSBuildDeviceIntegration/Tests/XASdkDeployTests.cs b/tests/MSBuildDeviceIntegration/Tests/XASdkDeployTests.cs index ea14979cc75..6b02e4d3b59 100644 --- a/tests/MSBuildDeviceIntegration/Tests/XASdkDeployTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/XASdkDeployTests.cs @@ -71,8 +71,6 @@ public class XASdkDeployTests : DeviceTest [TestCaseSource (nameof (DotNetInstallAndRunSource))] public void DotNetInstallAndRun (bool isRelease, bool xamarinForms, string targetFramework) { - AssertHasDevices (); - XASdkProject proj; if (xamarinForms) { proj = new XamarinFormsXASdkProject { @@ -107,8 +105,6 @@ public void DotNetInstallAndRun (bool isRelease, bool xamarinForms, string targe [Test] public void TypeAndMemberRemapping ([Values (false, true)] bool isRelease) { - AssertHasDevices (); - var proj = new XASdkProject () { IsRelease = isRelease, OtherBuildItems = { @@ -213,7 +209,6 @@ public void SupportDesugaringStaticInterfaceMethods () public void DotNetDebug ([Values("net6.0-android", "net7.0-android")] string targetFramework) { AssertCommercialBuild (); - AssertHasDevices (); var proj = new XASdkProject (); proj.TargetFramework = targetFramework; @@ -256,7 +251,6 @@ public void DotNetDebug ([Values("net6.0-android", "net7.0-android")] string tar EvaluationOptions = EvaluationOptions.DefaultOptions, }; options.EvaluationOptions.UseExternalTypeResolver = true; - ClearAdbLogcat (); dotnet.BuildLogFile = Path.Combine (Root, dotnet.ProjectDirectory, "run.log"); Assert.True (dotnet.Build ("Run", parameters: new [] { $"AndroidSdbTargetPort={port}", diff --git a/tests/Mono.Android-Tests/System.Net/SslTest.cs b/tests/Mono.Android-Tests/System.Net/SslTest.cs index a4ad35df116..37662f646e8 100644 --- a/tests/Mono.Android-Tests/System.Net/SslTest.cs +++ b/tests/Mono.Android-Tests/System.Net/SslTest.cs @@ -71,6 +71,12 @@ public void SslWithinTasksShouldWork () [Test] public void HttpsShouldWork () { +#if NET6_0_OR_GREATER + if (!OperatingSystem.IsAndroidVersionAtLeast (24)) { + Assert.Ignore ("Not supported on API 23 and lower."); + } +#endif // NET6_0_OR_GREATER + RunIgnoringWebException (DoHttpsShouldWork); } @@ -98,6 +104,12 @@ void DoHttpsShouldWork () [Test (Description="Bug https://bugzilla.xamarin.com/show_bug.cgi?id=18962")] public void VerifyTrustedCertificates () { +#if NET6_0_OR_GREATER + if (!OperatingSystem.IsAndroidVersionAtLeast (24)) { + Assert.Ignore ("Not supported on API 23 and lower."); + } +#endif // NET6_0_OR_GREATER + Assert.DoesNotThrow (() => RunIgnoringWebException (DoVerifyTrustedCertificates), "Certificate validation"); }