chore: update appium test to multi-app

performance-tests/appium/test/configs/appinfo.ts

@@ -0,0 +1,11 @@
+export class AppInfo {
+    startupTimes!: number[]
+
+    constructor(
+        public name: string,
+        public activity: string,
+        public path: string,
+    ) {
+        this.path = this.path.replace(/\\/g, '/')
+    }
+}

performance-tests/appium/test/configs/wdio.android.local.conf.ts

@@ -1,13 +1,24 @@
-import { join } from 'path';
+import * as path from 'path'
+import { AppInfo } from './appinfo';
 import config from './wdio.shared.local.appium.conf';
 
 config.capabilities = [{
     platformName: 'Android',
-    // 'appium:platformVersion': '11',
-    // 'appium:deviceName': 'Android Emulator',
     'appium:automationName': 'UIAutomator2',
-    'appium:app': join(process.cwd(), './test-apps/Android-MyDemoAppRN.apk'),
     'appium:autoLaunch': false,
 }];
 
+config.customApps = [
+    new AppInfo(
+        'io.sentry.java.tests.perf.appplain',
+        'MainActivity',
+        path.join(process.cwd(), '../test-app-plain/app/build/outputs/apk/release/app-release.apk')
+    ),
+    new AppInfo(
+        'io.sentry.java.tests.perf.appsentry',
+        'MainActivity',
+        path.join(process.cwd(), '../test-app-sentry/app/build/outputs/apk/release/app-release.apk')
+    ),
+]
+
 exports.config = config;

performance-tests/appium/test/configs/wdio.android.sauce.conf.ts

@@ -6,9 +6,6 @@ config.capabilities = [{
     'appium:platformVersion': '11',
     // 'appium:deviceName': 'Android GoogleAPI Emulator',
     'appium:automationName': 'UIAutomator2',
-    // The name of the App in the Sauce Labs storage, for more info see
-    // https://docs.saucelabs.com/mobile-apps/app-storage/
-    'appium:app': 'storage:filename=Android-MyDemoAppRN.apk',
     'appium:autoLaunch': false,
     'appium:newCommandTimeout': 240,
     'sauce:options': {
@@ -17,4 +14,8 @@ config.capabilities = [{
     },
 }];
 
+config.customApps = [
+    new AppInfo('com.saucelabs.mydemoapp.rn', 'MainActivity', 'storage:filename=Android-MyDemoAppRN.apk')
+]
+
 exports.config = config;

performance-tests/appium/test/specs/startup.spec.ts

@@ -1,32 +1,48 @@
 import * as ss from 'simple-statistics'
 import * as assert from 'assert'
+import { AppInfo } from '../configs/appinfo'
 
+const appsUnderTest = driver.config.customApps as AppInfo[]
+const runs = 10
 
-const appName = 'com.saucelabs.mydemoapp.rn'
-const activityName = 'MainActivity'
+describe('Apps', () => {
+    // install apps and collect their startup times
+    before(async () => {
+        for (var j = 0; j < appsUnderTest.length; j++) {
+            const app = appsUnderTest[j]
 
-describe('App', () => {
-    it('starts quickly', async () => {
-        const runs = 10
-        for (var i = 0; i < runs; i++) {
-            if (i > 0) {
-                // kill the app and sleep before running the next iteration
-                driver.terminateApp(appName)
+            console.log(`Installing app ${app.name} from ${app.path}`)
+            await driver.installApp(app.path)
+
+            console.log(`Collecting startup times for app ${app.name}`)
+            for (var i = 0; i < runs; i++) {
+                // Note: sleeping before launching the app (instead of after), improves the speed of the first launch.
                 await new Promise(resolve => setTimeout(resolve, 1000))
+
+                // Note: there's also .activateApp() which should be OS independent, but doesn't seem to wait for the activity to start
+                await driver.startActivity(app.name, app.activity)
+
+                // kill the app and sleep before running the next iteration
+                await driver.terminateApp(app.name)
             }
 
-            // NOTE: there's also .activateApp() which should be OS independent, but doesn't seem to wait for the activity to start
-            await driver.startActivity(appName, activityName)
-        }
+            const events = await driver.getEvents([])
+            const offset = j * runs
+            app.startupTimes = events.commands
+                .filter((cmd: any) => cmd.cmd == 'startActivity')
+                .map((cmd: any) => cmd.endTime - cmd.startTime)
+                .slice(offset, offset + runs)
 
-        const events = await driver.getEvents([])
-        const startupTimes = events.commands
-            .filter((cmd: any) => cmd.cmd == 'startActivity')
-            .map((cmd: any) => cmd.endTime - cmd.startTime)
+            assert.equal(app.startupTimes.length, runs)
+        }
+    })
 
-        assert.equal(startupTimes.length, runs)
+    it('starts', async () => {
+        for (const app of appsUnderTest) {
+            console.log(`App ${app.name} launch times: [${app.startupTimes}]`)
+            console.log(`App ${app.name} launch mean: ${ss.mean(app.startupTimes)} ms | stddev: ${ss.standardDeviation(app.startupTimes).toFixed(2)}`)
+        }
 
-        console.log(`App launch times: [${startupTimes}]`)
-        console.log(`App launch mean: ${ss.mean(startupTimes)} ms | stddev: ${ss.standardDeviation(startupTimes).toFixed(2)}`)
+        // TODO compare between the apps
     })
 })