From 94b77938435693792e57c96d76691d58d7361530 Mon Sep 17 00:00:00 2001 From: Riccardo Cipolleschi Date: Thu, 26 Sep 2024 06:01:28 -0700 Subject: [PATCH] Run Maestro tests also in debug mode (#46573) Summary: This change runs Maestro tests also in Debug mode, by starting Metro in background. ## Changelog: [Internal] - Add E2E tests in Debug mode too Pull Request resolved: https://github.com/facebook/react-native/pull/46573 Test Plan: GHA must be green. Successful run: https://github.com/facebook/react-native/actions/runs/11033322135?pr=46573 Reviewed By: cortinico Differential Revision: D63452169 Pulled By: cipolleschi fbshipit-source-id: e04b87f6a3e7aca8519dc2cb37c982dff3c20100 --- .github/actions/maestro-android/action.yml | 44 +++++--- .github/actions/maestro-ios/action.yml | 33 +++++- .github/actions/test-ios-rntester/action.yml | 2 +- .github/workflow-scripts/maestro-android.js | 108 +++++++++++++++++++ .github/workflows/test-all.yml | 57 +++++++--- 5 files changed, 209 insertions(+), 35 deletions(-) create mode 100644 .github/workflow-scripts/maestro-android.js diff --git a/.github/actions/maestro-android/action.yml b/.github/actions/maestro-android/action.yml index 1b34e6efe0e117..c8882c023b279f 100644 --- a/.github/actions/maestro-android/action.yml +++ b/.github/actions/maestro-android/action.yml @@ -17,6 +17,15 @@ inputs: required: false default: 'true' description: whether this action has to install java 17 or not + flavor: + required: true + description: the flavor we want to run - either debug or release + default: release + working-directory: + required: false + default: "." + description: The directory from which metro should be started + runs: using: composite steps: @@ -25,7 +34,7 @@ runs: run: export MAESTRO_VERSION=1.36.0; curl -Ls "https://get.maestro.mobile.dev" | bash - name: Set up JDK 17 if: ${{ inputs.install-java == 'true' }} - uses: actions/setup-java@v2 + uses: actions/setup-java@v4 with: java-version: '17' distribution: 'zulu' @@ -38,27 +47,32 @@ runs: echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm + - name: Build Codegen + shell: bash + if: ${{ inputs.flavor == 'debug' }} + run: ./packages/react-native-codegen/scripts/oss/build.sh - name: Run e2e tests uses: reactivecircus/android-emulator-runner@v2 with: api-level: 24 arch: x86 - script: | - echo "Install APK from ${{ inputs.app-path }}" - adb install "${{ inputs.app-path }}" - - echo "Start recording to /sdcard/screen.mp4" - adb shell screenrecord /sdcard/screen.mp4 - - echo "Start testing ${{ inputs.maestro-flow }}" - $HOME/.maestro/bin/maestro test ${{ inputs.maestro-flow }} --format junit -e APP_ID=${{ inputs.app-id }} --debug-output /tmp/MaestroLogs - - echo "Stop recording. Saving to screen.mp4" - adb pull /sdcard/screen.mp4 + ram-size: '4096M' + disk-size: '10G' + disable-animations: false + avd-name: e2e_emulator + script: node .github/workflow-scripts/maestro-android.js ${{ inputs.app-path }} ${{ inputs.app-id }} ${{ inputs.maestro-flow }} ${{ inputs.flavor }} ${{ inputs.working-directory }} + - name: Normalize APP_ID + id: normalize-app-id + shell: bash + if: always() + run: | + NORM_APP_ID=$(echo "${{ inputs.app-id }}" | tr '.' '-') + echo "app-id=$NORM_APP_ID" >> $GITHUB_OUTPUT - name: Store tests result uses: actions/upload-artifact@v3 + if: always() with: - name: e2e_android_${{ inputs.app-id }}_report_${{ inputs.jsengine }} + name: e2e_android_${{ steps.normalize-app-id.outputs.app-id }}_report_${{ inputs.jsengine }}_${{ inputs.flavor }} path: | report.xml screen.mp4 @@ -66,5 +80,5 @@ runs: if: failure() && steps.run-tests.outcome == 'failure' uses: actions/upload-artifact@v4.3.4 with: - name: maestro-logs-android-${{ inputs.app-id }}-${{ inputs.jsengine }} + name: maestro-logs-android-${{ steps.normalize-app-id.outputs.app-id }}-${{ inputs.jsengine }}-${{ inputs.flavor }} path: /tmp/MaestroLogs diff --git a/.github/actions/maestro-ios/action.yml b/.github/actions/maestro-ios/action.yml index 8379a6d458f6ab..9a07eb8e7fd6c3 100644 --- a/.github/actions/maestro-ios/action.yml +++ b/.github/actions/maestro-ios/action.yml @@ -13,6 +13,15 @@ inputs: maestro-flow: required: true description: the folder that contains the maestro tests + flavor: + required: true + description: Whether we are building for Debug or Release + default: Release + working-directory: + required: false + default: "." + description: The directory from which metro should be started + runs: using: composite steps: @@ -29,6 +38,12 @@ runs: with: java-version: '17' distribution: 'zulu' + - name: Start Metro in Debug + shell: bash + if: ${{ inputs.flavor == 'Debug' }} + run: | + cd ${{ inputs.working-directory }} + yarn start & - name: Run tests id: run-tests shell: bash @@ -54,17 +69,24 @@ runs: xcrun simctl launch $UDID ${{ inputs.app-id }} echo "Running tests with Maestro" - export MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 # 25 min. CI is extremely slow + export MAESTRO_DRIVER_STARTUP_TIMEOUT=1800000 # 30 min. CI is extremely slow # Add retries for flakyness - MAX_ATTEMPTS=3 + MAX_ATTEMPTS=5 CURR_ATTEMPT=0 RESULT=1 while [[ $CURR_ATTEMPT -lt $MAX_ATTEMPTS ]] && [[ $RESULT -ne 0 ]]; do + if [[ $CURR_ATTEMPT -ne 0 ]]; then + echo "Rebooting simulator for stability" + xcrun simctl boot "iPhone 15 Pro" + fi + CURR_ATTEMPT=$((CURR_ATTEMPT+1)) echo "Attempt number $CURR_ATTEMPT" + + echo "Start video record using pid: video_record_${{ inputs.jsengine }}_$CURR_ATTEMPT.pid" xcrun simctl io booted recordVideo video_record_$CURR_ATTEMPT.mov & echo $! > video_record_${{ inputs.jsengine }}_$CURR_ATTEMPT.pid @@ -75,6 +97,9 @@ runs: # Stop video kill -SIGINT $(cat video_record_${{ inputs.jsengine }}_$CURR_ATTEMPT.pid) + + echo "Shutting down simulator for stability" + xcrun simctl shutdown "iPhone 15 Pro" done exit $RESULT @@ -82,7 +107,7 @@ runs: if: always() uses: actions/upload-artifact@v4.3.4 with: - name: e2e_ios_${{ inputs.app-id }}_report_${{ inputs.jsengine }} + name: e2e_ios_${{ inputs.app-id }}_report_${{ inputs.jsengine }}_${{ inputs.flavor }} path: | video_record_1.mov video_record_2.mov @@ -92,5 +117,5 @@ runs: if: failure() && steps.run-tests.outcome == 'failure' uses: actions/upload-artifact@v4.3.4 with: - name: maestro-logs-${{ inputs.app-id }}-${{ inputs.jsengine }} + name: maestro-logs-${{ inputs.app-id }}-${{ inputs.jsengine }}-${{ inputs.flavor }} path: /tmp/MaestroLogs diff --git a/.github/actions/test-ios-rntester/action.yml b/.github/actions/test-ios-rntester/action.yml index 1ab333089d0901..61c28f6b8b1642 100644 --- a/.github/actions/test-ios-rntester/action.yml +++ b/.github/actions/test-ios-rntester/action.yml @@ -132,7 +132,7 @@ runs: set -o pipefail && xcodebuild \ -scheme "RNTester" \ -workspace packages/rn-tester/RNTesterPods.xcworkspace \ - -configuration "Release" \ + -configuration "${{ inputs.flavor }}" \ -sdk "iphonesimulator" \ -destination "generic/platform=iOS Simulator" \ -derivedDataPath "/tmp/RNTesterBuild" | xcbeautify diff --git a/.github/workflow-scripts/maestro-android.js b/.github/workflow-scripts/maestro-android.js new file mode 100644 index 00000000000000..462ce25ae42007 --- /dev/null +++ b/.github/workflow-scripts/maestro-android.js @@ -0,0 +1,108 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const childProcess = require('child_process'); + +const usage = ` +=== Usage === +node maestro-android.js + +@param {string} appPath - Path to the app APK +@param {string} appId - App ID that needs to be launched +@param {string} maestroFlow - Path to the maestro flow to be executed +@param {string} flavor - Flavor of the app to be launched. Can be 'release' or 'debug' +@param {string} workingDirectory - Working directory from where to run Metro +============== +`; + +const args = process.argv.slice(2); + +if (args.length !== 5) { + throw new Error(`Invalid number of arguments.\n${usage}`); +} + +const APP_PATH = args[0]; +const APP_ID = args[1]; +const MAESTRO_FLOW = args[2]; +const IS_DEBUG = args[3] === 'debug'; +const WORKING_DIRECTORY = args[4]; + +async function main() { + console.info('\n=============================='); + console.info('Running tests for Android with the following parameters:'); + console.info(`APP_PATH: ${APP_PATH}`); + console.info(`APP_ID: ${APP_ID}`); + console.info(`MAESTRO_FLOW: ${MAESTRO_FLOW}`); + console.info(`IS_DEBUG: ${IS_DEBUG}`); + console.info(`WORKING_DIRECTORY: ${WORKING_DIRECTORY}`); + console.info('==============================\n'); + + console.info('Install app'); + childProcess.execSync(`adb install ${APP_PATH}`, {stdio: 'ignore'}); + + let metroProcess = null; + if (IS_DEBUG) { + console.info('Start Metro'); + childProcess.execSync(`cd ${WORKING_DIRECTORY}`, {stdio: 'ignore'}); + metroProcess = childProcess.spawn('yarn', ['start', '&'], { + cwd: WORKING_DIRECTORY, + stdio: 'ignore', + detached: true, + }); + console.info(`- Metro PID: ${metroProcess.pid}`); + } + + console.info('Wait For Metro to Start'); + await sleep(5000); + + console.info('Start the app'); + childProcess.execSync(`adb shell monkey -p ${APP_ID} 1`, {stdio: 'ignore'}); + + console.info('Start recording to /sdcard/screen.mp4'); + childProcess + .exec('adb shell screenrecord /sdcard/screen.mp4', { + stdio: 'ignore', + detached: true, + }) + .unref(); + + console.info(`Start testing ${MAESTRO_FLOW}`); + let error = null; + try { + childProcess.execSync( + `MAESTRO_DRIVER_STARTUP_TIMEOUT=120000 $HOME/.maestro/bin/maestro test ${MAESTRO_FLOW} --format junit -e APP_ID=${APP_ID} --debug-output /tmp/MaestroLogs`, + {stdio: 'inherit'}, + ); + } catch (err) { + error = err; + } finally { + console.info('Stop recording'); + childProcess.execSync('adb pull /sdcard/screen.mp4', {stdio: 'ignore'}); + + if (IS_DEBUG && metroProcess != null) { + const pid = metroProcess.pid; + console.info(`Kill Metro. PID: ${pid}`); + process.kill(-pid); + console.info(`Metro Killed`); + process.exit(); + } + } + + if (error) { + throw error; + } +} + +function sleep(ms) { + return new Promise(resolve => { + setTimeout(resolve, ms); + }); +} + +main(); diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 48db77b226c9f6..eb456dcc3a0589 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -196,6 +196,7 @@ jobs: matrix: jsengine: [Hermes, JSC] architecture: [NewArch] + flavor: [Debug, Release] steps: - name: Checkout uses: actions/checkout@v4 @@ -209,13 +210,15 @@ jobs: hermes-version: ${{ needs.prepare_hermes_workspace.outputs.hermes-version }} react-native-version: ${{ needs.prepare_hermes_workspace.outputs.react-native-version }} run-e2e-tests: "true" + flavor: ${{ matrix.flavor }} - name: Run E2E Tests uses: ./.github/actions/maestro-ios with: - app-path: "/tmp/RNTesterBuild/Build/Products/Release-iphonesimulator/RNTester.app" + app-path: "/tmp/RNTesterBuild/Build/Products/${{ matrix.flavor }}-iphonesimulator/RNTester.app" app-id: com.meta.RNTester.localDevelopment jsengine: ${{ matrix.jsengine }} maestro-flow: ./packages/rn-tester/.maestro/ + flavor: ${{ matrix.flavor }} test_e2e_ios_templateapp: if: ${{ github.ref == 'refs/heads/main' || contains(github.ref, 'stable') || inputs.run-e2e-tests }} @@ -229,6 +232,7 @@ jobs: fail-fast: false matrix: jsengine: [Hermes, JSC] + flavor: [Debug, Release] steps: - name: Checkout uses: actions/checkout@v4 @@ -245,7 +249,7 @@ jobs: - name: Download Hermes uses: actions/download-artifact@v4 with: - name: hermes-darwin-bin-Release + name: hermes-darwin-bin-${{matrix.flavor}} path: /tmp/react-native-tmp - name: Download React Native Package uses: actions/download-artifact@v4 @@ -262,26 +266,35 @@ jobs: HERMES_PATH=$(find /tmp/react-native-tmp -type f -name "*.tar.gz") echo "Hermes path is $HERMES_PATH" - node ./scripts/e2e/init-project-e2e.js --projectName RNTestProject --currentBranch ${{ github.ref_name }} --directory /tmp/RNTestProject --pathToLocalReactNative $REACT_NATIVE_PKG + # For stable branches, we want to use the stable branch of the template + # In all the other cases, we want to use "main" + BRANCH=${{ github.ref_name }} + if ! [[ $BRANCH == *-stable* ]]; then + BRANCH=main + fi + + node ./scripts/e2e/init-project-e2e.js --projectName RNTestProject --currentBranch $BRANCH --directory /tmp/RNTestProject --pathToLocalReactNative $REACT_NATIVE_PKG cd /tmp/RNTestProject/ios bundle install HERMES_ENGINE_TARBALL_PATH=$HERMES_PATH bundle exec pod install - set -o pipefail && xcodebuild \ + xcodebuild \ -scheme "RNTestProject" \ -workspace RNTestProject.xcworkspace \ - -configuration "Release" \ + -configuration "${{ matrix.flavor }}" \ -sdk "iphonesimulator" \ -destination "generic/platform=iOS Simulator" \ - -derivedDataPath "/tmp/RNTestProject" | xcbeautify + -derivedDataPath "/tmp/RNTestProject" - name: Run E2E Tests uses: ./.github/actions/maestro-ios with: - app-path: "/tmp/RNTestProject/Build/Products/Release-iphonesimulator/RNTestProject.app" + app-path: "/tmp/RNTestProject/Build/Products/${{ matrix.flavor }}-iphonesimulator/RNTestProject.app" app-id: org.reactjs.native.example.RNTestProject jsengine: ${{ matrix.jsengine }} maestro-flow: ./scripts/e2e/.maestro/ + flavor: ${{ matrix.flavor }} + working-directory: /tmp/RNTestProject test_e2e_android_templateapp: if: ${{ github.ref == 'refs/heads/main' || contains(github.ref, 'stable') || inputs.run-e2e-tests }} @@ -292,6 +305,7 @@ jobs: fail-fast: false matrix: jsengine: [Hermes, JSC] + flavor: [debug, release] steps: - name: Checkout uses: actions/checkout@v4 @@ -317,6 +331,7 @@ jobs: - name: Print /tmp folder run: ls -lR /tmp/react-native-tmp - name: Prepare artifacts + id: prepare-artifacts run: | REACT_NATIVE_PKG=$(find /tmp/react-native-tmp -type f -name "*.tgz") echo "React Native tgs is $REACT_NATIVE_PKG" @@ -324,7 +339,13 @@ jobs: MAVEN_LOCAL=/tmp/react-native-tmp/maven-local echo "Maven local path is $MAVEN_LOCAL" - node ./scripts/e2e/init-project-e2e.js --projectName RNTestProject --currentBranch ${{ github.ref_name }} --directory /tmp/RNTestProject --pathToLocalReactNative $REACT_NATIVE_PKG + # For stable branches, we want to use the stable branch of the template + # In all the other cases, we want to use "main" + BRANCH=${{ github.ref_name }} + if ! [[ $BRANCH == *-stable* ]]; then + BRANCH=main + fi + node ./scripts/e2e/init-project-e2e.js --projectName RNTestProject --currentBranch $BRANCH --directory /tmp/RNTestProject --pathToLocalReactNative $REACT_NATIVE_PKG echo "Feed maven local to gradle.properties" cd /tmp/RNTestProject @@ -332,15 +353,19 @@ jobs: # Build cd android - ./gradlew assembleRelease --no-daemon -PreactNativeArchitectures=x86 + CAPITALIZED_FLAVOR=$(echo "${{ matrix.flavor }}" | awk '{print toupper(substr($0, 1, 1)) substr($0, 2)}') + ./gradlew assemble$CAPITALIZED_FLAVOR --no-daemon -PreactNativeArchitectures=x86 + - name: Run E2E Tests uses: ./.github/actions/maestro-android with: - app-path: /tmp/RNTestProject/android/app/build/outputs/apk/release/app-release.apk + app-path: /tmp/RNTestProject/android/app/build/outputs/apk/${{ matrix.flavor }}/app-${{ matrix.flavor }}.apk app-id: com.rntestproject jsengine: ${{ matrix.jsengine }} maestro-flow: ./scripts/e2e/.maestro/ install-java: 'false' + flavor: ${{ matrix.flavor }} + working-directory: /tmp/RNTestProject build_hermesc_linux: runs-on: ubuntu-latest @@ -403,6 +428,7 @@ jobs: fail-fast: false matrix: jsengine: [hermes, jsc] + flavor: [debug, release] steps: - name: Checkout uses: actions/checkout@v4 @@ -413,17 +439,18 @@ jobs: - name: Download APK uses: actions/download-artifact@v4 with: - name: rntester-${{ matrix.jsengine }}-release - path: ./packages/rn-tester/android/app/build/outputs/apk/${{ matrix.jsengine }}/release/ + name: rntester-${{ matrix.jsengine }}-${{ matrix.flavor }} + path: ./packages/rn-tester/android/app/build/outputs/apk/${{ matrix.jsengine }}/${{ matrix.flavor }}/ - name: Print folder structure - run: ls -lR ./packages/rn-tester/android/app/build/outputs/apk/${{ matrix.jsengine }}/release/ + run: ls -lR ./packages/rn-tester/android/app/build/outputs/apk/${{ matrix.jsengine }}/${{ matrix.flavor }}/ - name: Run E2E Tests uses: ./.github/actions/maestro-android with: - app-path: ./packages/rn-tester/android/app/build/outputs/apk/${{ matrix.jsengine }}/release/app-${{ matrix.jsengine }}-x86-release.apk + app-path: ./packages/rn-tester/android/app/build/outputs/apk/${{ matrix.jsengine }}/${{ matrix.flavor }}/app-${{ matrix.jsengine }}-x86-${{ matrix.flavor }}.apk app-id: com.facebook.react.uiapp jsengine: ${{ matrix.jsengine }} - maestro-flow: ./packages/rn-tester/.maestro/ + maestro-flow: ./packages/rn-tester/.maestro + flavor: ${{ matrix.flavor }} build_npm_package: runs-on: 8-core-ubuntu