[Mobile] Add E2E BrowserStack tests for iOS tests

onnxruntime/test/platform/apple/generate_ipa_export_options_plist.py

@@ -0,0 +1,54 @@
+import argparse
+
+plist_file_content = """
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>method</key>
+    <string>development</string>
+    <key>teamID</key>
+    <string>{team_id}</string>
+    <key>provisioningProfiles</key>
+    <dict>
+        <key>ai.onnxruntime.tests.ios-package-test</key>
+        <string>{provisioning_profile_uuid}</string>
+    </dict>
+    <key>signingStyle</key>
+    <string>manual</string>
+</dict>
+</plist>
+"""
+if __name__ == "__main__":
+    # handle cli args
+    parser = argparse.ArgumentParser(
+        "Generates a PList file to the relevant destination. This PList file contains the properties to allow a user to generate an IPA file for the ios-package-test. "
+    )
+
+    parser.add_argument("--dest_file", type=str, help="Path to output the PList file to.", required=True)
+    parser.add_argument(
+        "--apple_team_id",
+        type=str,
+        help="The Team ID associated with the provisioning profile. You should be able to find this from the Apple developer portal under Membership.",
+        required=True,
+    )
+    parser.add_argument(
+        "--provisioning_profile_uuid",
+        type=str,
+        help="The Provisioning Profile UUID, which can be found in the .mobileprovision file. ",
+        required=True,
+    )
+
+    args = parser.parse_args()
+
+    formatted_plist = plist_file_content.format(
+        team_id=args.apple_team_id, provisioning_profile_uuid=args.provisioning_profile_uuid
+    )
+
+    with open(args.dest_file, "w") as file:
+        file.write(formatted_plist)
+
+    print("Wrote plist file to ", args.dest_file)
+    print()
+    print("Contents of file:")
+    print(formatted_plist)

tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml

@@ -77,8 +77,8 @@ jobs:
       pip install requests
       python $(Build.SourcesDirectory)/tools/python/upload_and_run_browserstack_tests.py \
         --test_platform espresso \
-        --app_apk_path "debug/app-debug.apk" \
-        --test_apk_path "androidTest/debug/app-debug-androidTest.apk" \
+        --app_path "debug/app-debug.apk" \
+        --test_path "androidTest/debug/app-debug-androidTest.apk" \
         --devices "Samsung Galaxy S23-13.0" "Google Pixel 3-9.0"
     displayName: Run E2E tests using Browserstack
     workingDirectory: $(Build.BinariesDirectory)/android_test/android/app/build/outputs/apk

tools/ci_build/github/azure-pipelines/templates/stages/mac-ios-packaging-build-stage.yml

@@ -70,8 +70,6 @@ stages:
       parameters:
         xcodeVersion: $(xcodeVersion)
 
-    - template: ../install-appcenter.yml
-
     - script: |
         pip install -r tools/ci_build/github/apple/ios_packaging/requirements.txt
       displayName: "Install Python requirements"
@@ -100,15 +98,15 @@ stages:
           --prepare_test_project_only
       displayName: "Assemble test project for App Center"
 
+    # Xcode tasks require absolute paths because it searches for the paths and files relative to
+    # the root directory and not relative to the working directory
     - task: Xcode@5
       inputs:
         actions: 'build-for-testing'
         configuration: 'Debug'
         xcWorkspacePath: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/apple_package_test.xcworkspace'
         sdk: 'iphoneos'
         scheme: 'ios_package_test'
-        xcodeVersion: 'specifyPath'
-        xcodeDeveloperDir: '/Applications/Xcode_${{ variables.xcodeVersion }}.app/Contents/Developer'
         signingOption: 'manual'
         signingIdentity: '$(APPLE_CERTIFICATE_SIGNING_IDENTITY)'
         provisioningProfileUuid: '$(APPLE_PROV_PROFILE_UUID)'
@@ -117,16 +115,69 @@ stages:
         useXcpretty: false  # xcpretty can hide useful error output so we will disable it
       displayName: 'Build App Center iPhone arm64 tests'
 
+    - script: |
+        zip -r --symlinks $(Build.ArtifactStagingDirectory)/package_tests.zip ios_package_testUITests-Runner.app
+      workingDirectory: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/DerivedData/Build/Products/Debug-iphoneos'
+      displayName: "Create .zip file of the tests"
+
+    - script: |
+        python $(Build.SourcesDirectory)/onnxruntime/test/platform/apple/generate_ipa_export_options_plist.py \
+          --dest_file "exportOptions.plist" \
+          --apple_team_id $(APPLE_TEAM_ID) \
+          --provisioning_profile_uuid $(APPLE_PROV_PROFILE_UUID)
+      workingDirectory: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/'
+      displayName: "Generate .plist file for the .ipa file"
+
+    # Task only generates an .xcarchive file if the plist export options are included, but does
+    # not produce an IPA file.
+    # Source code: https://github.com/microsoft/azure-pipelines-tasks/blob/master/Tasks/XcodeV5/xcode.ts
+    - task: Xcode@5
+      inputs:
+        actions: 'archive'
+        xcWorkspacePath: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/apple_package_test.xcworkspace'
+        packageApp: true
+        archivePath: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/'
+        exportOptions: 'plist'
+        exportOptionsPlist: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/exportOptions.plist'
+        configuration: 'Debug'
+        sdk: 'iphoneos'
+        scheme: 'ios_package_test'
+        args: '-derivedDataPath $(Build.BinariesDirectory)/app_center_test/apple_package_test/DerivedData'
+        workingDirectory: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/'
+        useXcpretty: false
+      displayName: 'Create archive for the .ipa file'
+
+    # Use script step because exporting the .ipa file using the Xcode@5 task was too brittle (Xcode@5 is designed
+    # to handle both the .xcarchive step and the .ipa step in the same step -- ran into countless issues with signing
+    # and the .plist file)
+    - script: |
+        xcodebuild -exportArchive \
+          -archivePath ios_package_test.xcarchive \
+          -exportOptionsPlist exportOptions.plist \
+          -exportPath $(Build.ArtifactStagingDirectory)/test_ipa
+      workingDirectory: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/'
+      displayName: "Create .ipa file"
+
+    # Publish the BrowserStack artifacts first so that if the next step fails, the artifacts will still be published
+    # so that users can attempt to locally debug
+    - publish: "$(Build.ArtifactStagingDirectory)"
+      artifact: "browserstack_test_artifacts"
+      displayName: "Publish BrowserStack test artifacts"
+
     - script: |
         set -e -x
-        appcenter test run xcuitest \
-          --app "AI-Frameworks/ORT-Mobile-iOS" \
-          --devices $(app_center_test_devices) \
-          --test-series "master" \
-          --locale "en_US" \
-          --build-dir $(Build.BinariesDirectory)/app_center_test/apple_package_test/DerivedData/Build/Products/Debug-iphoneos \
-          --token $(app_center_api_token)
-      displayName: "Run E2E tests on App Center"
+        pip install requests
+        python $(Build.SourcesDirectory)/tools/python/upload_and_run_browserstack_tests.py \
+          --test_platform xcuitest \
+          --app_path "$(Build.ArtifactStagingDirectory)/test_ipa/ios_package_test.ipa" \
+          --test_path "$(Build.ArtifactStagingDirectory)/package_tests.zip" \
+          --devices "iPhone 15-17"
+      displayName: Run E2E tests using Browserstack
+      workingDirectory: $(Build.BinariesDirectory)/app_center_test/apple_package_test
+      timeoutInMinutes: 15
+      env:
+        BROWSERSTACK_ID: $(browserstack_username)
+        BROWSERSTACK_TOKEN: $(browserstack_access_key)
 
     - script: |
         set -e -x

tools/python/upload_and_run_browserstack_tests.py

@@ -78,22 +78,24 @@ def build_query_loop(build_id, test_platform, id, token):
         "--test_platform", type=str, help="Testing platform", choices=["espresso", "xcuitest"], required=True
     )
     parser.add_argument(
-        "--app_apk_path",
+        "--app_path",
         type=Path,
         help=(
-            "Path to the app APK. "
-            "Typically, the app APK is in "
+            "Path to the app file. "
+            "For Android, typically, the app file (the APK) is in "
             "{build_output_dir}/android_test/android/app/build/outputs/apk/debug/app-debug.apk"
+            ". For iOS, you will have to build an IPA file from the test app, which is built from the .xcarchive path"
         ),
         required=True,
     )
     parser.add_argument(
-        "--test_apk_path",
+        "--test_path",
         type=Path,
         help=(
-            "Path to the test APK. "
+            "Path to the test suite file. "
             "Typically, the test APK is in "
             "{build_output_dir}/android_test/android/app/build/outputs/apk/androidTest/debug/app-debug-androidTest.apk"
+            ". For iOS, you will have to create a .zip of the tests. After manually building the tests, the tests that you need to zip will be in {{Xcode DerivedData Folder Path}}/Build/Products"
         ),
         required=True,
     )
@@ -102,7 +104,7 @@ def build_query_loop(build_id, test_platform, id, token):
         type=str,
         nargs="+",
         help="List of devices to run the tests on. For more info, "
-        "see https://www.browserstack.com/docs/app-automate/espresso/specify-devices",
+        "see https://www.browserstack.com/docs/app-automate/espresso/specify-devices (Android) or https://www.browserstack.com/docs/app-automate/xcuitest/specify-devices (iOS)",
         required=True,
     )
 
@@ -121,13 +123,13 @@ def build_query_loop(build_id, test_platform, id, token):
     # Upload the app and test suites
     upload_app_json = upload_apk_parse_json(
         f"https://api-cloud.browserstack.com/app-automate/{args.test_platform}/v2/app",
-        args.app_apk_path,
+        args.app_path,
         browserstack_id,
         browserstack_token,
     )
     upload_test_json = upload_apk_parse_json(
         f"https://api-cloud.browserstack.com/app-automate/{args.test_platform}/v2/test-suite",
-        args.test_apk_path,
+        args.test_path,
         browserstack_id,
         browserstack_token,
     )