Skip to content

Commit

Permalink
Run Maestro tests also in debug mode (facebook#46573)
Browse files Browse the repository at this point in the history
Summary:
This change runs Maestro tests also in Debug mode, by starting Metro in background.

## Changelog:
[Internal] - Add E2E tests in Debug mode too

Pull Request resolved: facebook#46573

Test Plan:
GHA must be green.
Successful run: https://github.com/facebook/react-native/actions/runs/11033322135?pr=46573

Reviewed By: cortinico

Differential Revision: D63452169

Pulled By: cipolleschi

fbshipit-source-id: e04b87f6a3e7aca8519dc2cb37c982dff3c20100
  • Loading branch information
cipolleschi authored and facebook-github-bot committed Sep 26, 2024
1 parent 19d468f commit 94b7793
Show file tree
Hide file tree
Showing 5 changed files with 209 additions and 35 deletions.
44 changes: 29 additions & 15 deletions .github/actions/maestro-android/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,15 @@ inputs:
required: false
default: 'true'
description: whether this action has to install java 17 or not
flavor:
required: true
description: the flavor we want to run - either debug or release
default: release
working-directory:
required: false
default: "."
description: The directory from which metro should be started

runs:
using: composite
steps:
Expand All @@ -25,7 +34,7 @@ runs:
run: export MAESTRO_VERSION=1.36.0; curl -Ls "https://get.maestro.mobile.dev" | bash
- name: Set up JDK 17
if: ${{ inputs.install-java == 'true' }}
uses: actions/setup-java@v2
uses: actions/setup-java@v4
with:
java-version: '17'
distribution: 'zulu'
Expand All @@ -38,33 +47,38 @@ runs:
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
sudo udevadm control --reload-rules
sudo udevadm trigger --name-match=kvm
- name: Build Codegen
shell: bash
if: ${{ inputs.flavor == 'debug' }}
run: ./packages/react-native-codegen/scripts/oss/build.sh
- name: Run e2e tests
uses: reactivecircus/android-emulator-runner@v2
with:
api-level: 24
arch: x86
script: |
echo "Install APK from ${{ inputs.app-path }}"
adb install "${{ inputs.app-path }}"
echo "Start recording to /sdcard/screen.mp4"
adb shell screenrecord /sdcard/screen.mp4
echo "Start testing ${{ inputs.maestro-flow }}"
$HOME/.maestro/bin/maestro test ${{ inputs.maestro-flow }} --format junit -e APP_ID=${{ inputs.app-id }} --debug-output /tmp/MaestroLogs
echo "Stop recording. Saving to screen.mp4"
adb pull /sdcard/screen.mp4
ram-size: '4096M'
disk-size: '10G'
disable-animations: false
avd-name: e2e_emulator
script: node .github/workflow-scripts/maestro-android.js ${{ inputs.app-path }} ${{ inputs.app-id }} ${{ inputs.maestro-flow }} ${{ inputs.flavor }} ${{ inputs.working-directory }}
- name: Normalize APP_ID
id: normalize-app-id
shell: bash
if: always()
run: |
NORM_APP_ID=$(echo "${{ inputs.app-id }}" | tr '.' '-')
echo "app-id=$NORM_APP_ID" >> $GITHUB_OUTPUT
- name: Store tests result
uses: actions/upload-artifact@v3
if: always()
with:
name: e2e_android_${{ inputs.app-id }}_report_${{ inputs.jsengine }}
name: e2e_android_${{ steps.normalize-app-id.outputs.app-id }}_report_${{ inputs.jsengine }}_${{ inputs.flavor }}
path: |
report.xml
screen.mp4
- name: Store Logs
if: failure() && steps.run-tests.outcome == 'failure'
uses: actions/upload-artifact@v4.3.4
with:
name: maestro-logs-android-${{ inputs.app-id }}-${{ inputs.jsengine }}
name: maestro-logs-android-${{ steps.normalize-app-id.outputs.app-id }}-${{ inputs.jsengine }}-${{ inputs.flavor }}
path: /tmp/MaestroLogs
33 changes: 29 additions & 4 deletions .github/actions/maestro-ios/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,15 @@ inputs:
maestro-flow:
required: true
description: the folder that contains the maestro tests
flavor:
required: true
description: Whether we are building for Debug or Release
default: Release
working-directory:
required: false
default: "."
description: The directory from which metro should be started

runs:
using: composite
steps:
Expand All @@ -29,6 +38,12 @@ runs:
with:
java-version: '17'
distribution: 'zulu'
- name: Start Metro in Debug
shell: bash
if: ${{ inputs.flavor == 'Debug' }}
run: |
cd ${{ inputs.working-directory }}
yarn start &
- name: Run tests
id: run-tests
shell: bash
Expand All @@ -54,17 +69,24 @@ runs:
xcrun simctl launch $UDID ${{ inputs.app-id }}
echo "Running tests with Maestro"
export MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 # 25 min. CI is extremely slow
export MAESTRO_DRIVER_STARTUP_TIMEOUT=1800000 # 30 min. CI is extremely slow
# Add retries for flakyness
MAX_ATTEMPTS=3
MAX_ATTEMPTS=5
CURR_ATTEMPT=0
RESULT=1
while [[ $CURR_ATTEMPT -lt $MAX_ATTEMPTS ]] && [[ $RESULT -ne 0 ]]; do
if [[ $CURR_ATTEMPT -ne 0 ]]; then
echo "Rebooting simulator for stability"
xcrun simctl boot "iPhone 15 Pro"
fi
CURR_ATTEMPT=$((CURR_ATTEMPT+1))
echo "Attempt number $CURR_ATTEMPT"
echo "Start video record using pid: video_record_${{ inputs.jsengine }}_$CURR_ATTEMPT.pid"
xcrun simctl io booted recordVideo video_record_$CURR_ATTEMPT.mov & echo $! > video_record_${{ inputs.jsengine }}_$CURR_ATTEMPT.pid
Expand All @@ -75,14 +97,17 @@ runs:
# Stop video
kill -SIGINT $(cat video_record_${{ inputs.jsengine }}_$CURR_ATTEMPT.pid)
echo "Shutting down simulator for stability"
xcrun simctl shutdown "iPhone 15 Pro"
done
exit $RESULT
- name: Store video record
if: always()
uses: actions/upload-artifact@v4.3.4
with:
name: e2e_ios_${{ inputs.app-id }}_report_${{ inputs.jsengine }}
name: e2e_ios_${{ inputs.app-id }}_report_${{ inputs.jsengine }}_${{ inputs.flavor }}
path: |
video_record_1.mov
video_record_2.mov
Expand All @@ -92,5 +117,5 @@ runs:
if: failure() && steps.run-tests.outcome == 'failure'
uses: actions/upload-artifact@v4.3.4
with:
name: maestro-logs-${{ inputs.app-id }}-${{ inputs.jsengine }}
name: maestro-logs-${{ inputs.app-id }}-${{ inputs.jsengine }}-${{ inputs.flavor }}
path: /tmp/MaestroLogs
2 changes: 1 addition & 1 deletion .github/actions/test-ios-rntester/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ runs:
set -o pipefail && xcodebuild \
-scheme "RNTester" \
-workspace packages/rn-tester/RNTesterPods.xcworkspace \
-configuration "Release" \
-configuration "${{ inputs.flavor }}" \
-sdk "iphonesimulator" \
-destination "generic/platform=iOS Simulator" \
-derivedDataPath "/tmp/RNTesterBuild" | xcbeautify
Expand Down
108 changes: 108 additions & 0 deletions .github/workflow-scripts/maestro-android.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/

const childProcess = require('child_process');

const usage = `
=== Usage ===
node maestro-android.js <path to app> <app_id> <maestro_flow> <flavor> <working_directory>
@param {string} appPath - Path to the app APK
@param {string} appId - App ID that needs to be launched
@param {string} maestroFlow - Path to the maestro flow to be executed
@param {string} flavor - Flavor of the app to be launched. Can be 'release' or 'debug'
@param {string} workingDirectory - Working directory from where to run Metro
==============
`;

const args = process.argv.slice(2);

if (args.length !== 5) {
throw new Error(`Invalid number of arguments.\n${usage}`);
}

const APP_PATH = args[0];
const APP_ID = args[1];
const MAESTRO_FLOW = args[2];
const IS_DEBUG = args[3] === 'debug';
const WORKING_DIRECTORY = args[4];

async function main() {
console.info('\n==============================');
console.info('Running tests for Android with the following parameters:');
console.info(`APP_PATH: ${APP_PATH}`);
console.info(`APP_ID: ${APP_ID}`);
console.info(`MAESTRO_FLOW: ${MAESTRO_FLOW}`);
console.info(`IS_DEBUG: ${IS_DEBUG}`);
console.info(`WORKING_DIRECTORY: ${WORKING_DIRECTORY}`);
console.info('==============================\n');

console.info('Install app');
childProcess.execSync(`adb install ${APP_PATH}`, {stdio: 'ignore'});

let metroProcess = null;
if (IS_DEBUG) {
console.info('Start Metro');
childProcess.execSync(`cd ${WORKING_DIRECTORY}`, {stdio: 'ignore'});
metroProcess = childProcess.spawn('yarn', ['start', '&'], {
cwd: WORKING_DIRECTORY,
stdio: 'ignore',
detached: true,
});
console.info(`- Metro PID: ${metroProcess.pid}`);
}

console.info('Wait For Metro to Start');
await sleep(5000);

console.info('Start the app');
childProcess.execSync(`adb shell monkey -p ${APP_ID} 1`, {stdio: 'ignore'});

console.info('Start recording to /sdcard/screen.mp4');
childProcess
.exec('adb shell screenrecord /sdcard/screen.mp4', {
stdio: 'ignore',
detached: true,
})
.unref();

console.info(`Start testing ${MAESTRO_FLOW}`);
let error = null;
try {
childProcess.execSync(
`MAESTRO_DRIVER_STARTUP_TIMEOUT=120000 $HOME/.maestro/bin/maestro test ${MAESTRO_FLOW} --format junit -e APP_ID=${APP_ID} --debug-output /tmp/MaestroLogs`,
{stdio: 'inherit'},
);
} catch (err) {
error = err;
} finally {
console.info('Stop recording');
childProcess.execSync('adb pull /sdcard/screen.mp4', {stdio: 'ignore'});

if (IS_DEBUG && metroProcess != null) {
const pid = metroProcess.pid;
console.info(`Kill Metro. PID: ${pid}`);
process.kill(-pid);
console.info(`Metro Killed`);
process.exit();
}
}

if (error) {
throw error;
}
}

function sleep(ms) {
return new Promise(resolve => {
setTimeout(resolve, ms);
});
}

main();
Loading

0 comments on commit 94b7793

Please sign in to comment.