Merge pull request #2065 from bugsnag/je/plat-12353-split-out-anr

[PLAT-12353] Split out ANR tests to BS
bugsnag · Aug 13, 2024 · 4a0bde0 · 4a0bde0
2 parents 3e7d8f4 + 7b3bba1
commit 4a0bde0
Show file tree

Hide file tree

Showing 9 changed files with 657 additions and 135 deletions.
diff --git a/.buildkite/pipeline.full.yml b/.buildkite/pipeline.full.yml
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -16,11 +16,13 @@ steps:
     artifact_paths:
       - "build/fixture-r19.apk"
       - "build/fixture-r19-url.txt"
+      - "build/bs-fixture-r19-url.txt"
       - "build/fixture-r19/*"
     commands:
       - bundle install
       - make fixture-r19
       - bundle exec upload-app --farm=bb --app=./build/fixture-r19.apk --app-id-file=build/fixture-r19-url.txt
+      - bundle exec upload-app --farm=bs --app=./build/fixture-r19.apk --app-id-file=build/bs-fixture-r19-url.txt
     env:
       JAVA_VERSION: 17
 
@@ -32,11 +34,13 @@ steps:
     artifact_paths:
       - "build/fixture-r21.apk"
       - "build/fixture-r21-url.txt"
+      - "build/bs-fixture-r21-url.txt"
       - "build/fixture-r21/*"
     commands:
       - bundle install
       - make fixture-r21
       - bundle exec upload-app --farm=bb --app=./build/fixture-r21.apk --app-id-file=build/fixture-r21-url.txt
+      - bundle exec upload-app --farm=bs --app=./build/fixture-r21.apk --app-id-file=build/bs-fixture-r21-url.txt
     env:
       JAVA_VERSION: 17
 
@@ -116,6 +120,7 @@ steps:
         service-ports: true
         command:
           - "features/smoke_tests"
+          - "--exclude=features/smoke_tests/01_anr.feature"
           - "--app=@build/fixture-r19-url.txt"
           - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
           - "--app-package=com.bugsnag.android.mazerunner"
@@ -131,6 +136,36 @@ steps:
     concurrency_group: 'bitbar'
     concurrency_method: eager
 
+  - label: ':browserstack: Android 7 NDK r19 ANR smoke tests'
+    depends_on: "fixture-r19"
+    timeout_in_minutes: 60
+    plugins:
+      artifacts#v1.9.0:
+        download:
+          - "build/bs-fixture-r19-url.txt"
+          - "build/fixture-r19/*"
+        upload:
+          - "maze_output/failed/**/*"
+          - "maze_output/metrics.csv"
+      docker-compose#v4.7.0:
+        pull: maze-runner
+        run: maze-runner
+        service-ports: true
+        command:
+          - "features/smoke_tests/01_anr.feature"
+          - "--app=@build/bs-fixture-r19-url.txt"
+          - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
+          - "--app-package=com.bugsnag.android.mazerunner"
+          - "--appium-version=1.22.0"
+          - "--farm=bs"
+          - "--device=ANDROID_7"
+          - "--fail-fast"
+    env:
+      TEST_FIXTURE_SYMBOL_DIR: "build/fixture-r19"
+    concurrency: 5
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+
   - label: ':bitbar: Android 8 NDK r19 smoke tests'
     depends_on: "fixture-r19"
     timeout_in_minutes: 60
@@ -146,6 +181,7 @@ steps:
         service-ports: true
         command:
           - "features/smoke_tests"
+          - "--exclude=features/smoke_tests/01_anr.feature"
           - "--app=@build/fixture-r19-url.txt"
           - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
           - "--app-package=com.bugsnag.android.mazerunner"
@@ -161,6 +197,34 @@ steps:
     concurrency_group: 'bitbar'
     concurrency_method: eager
 
+  - label: ':browserstack: Android 8 NDK r19 ANR smoke tests'
+    depends_on: "fixture-r19"
+    timeout_in_minutes: 60
+    plugins:
+      artifacts#v1.9.0:
+        download:
+          - "build/bs-fixture-r19-url.txt"
+          - "build/fixture-r19/*"
+        upload: "maze_output/failed/**/*"
+      docker-compose#v4.7.0:
+        pull: maze-runner
+        run: maze-runner
+        service-ports: true
+        command:
+          - "features/smoke_tests/01_anr.feature"
+          - "--app=@build/bs-fixture-r19-url.txt"
+          - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
+          - "--app-package=com.bugsnag.android.mazerunner"
+          - "--appium-version=1.22.0"
+          - "--farm=bs"
+          - "--device=ANDROID_8"
+          - "--fail-fast"
+    env:
+      TEST_FIXTURE_SYMBOL_DIR: "build/fixture-r19"
+    concurrency: 5
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+
   - label: ':bitbar: Android 9 NDK r21 smoke tests'
     depends_on: "fixture-r21"
     timeout_in_minutes: 60
@@ -178,6 +242,7 @@ steps:
         service-ports: true
         command:
           - "features/smoke_tests"
+          - "--exclude=features/smoke_tests/01_anr.feature"
           - "--app=@build/fixture-r21-url.txt"
           - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
           - "--app-package=com.bugsnag.android.mazerunner"
@@ -193,6 +258,36 @@ steps:
     concurrency_group: 'bitbar'
     concurrency_method: eager
 
+  - label: ':browserstack: Android 9 NDK r21 ANR smoke tests'
+    depends_on: "fixture-r21"
+    timeout_in_minutes: 60
+    plugins:
+      artifacts#v1.9.0:
+        download:
+          - "build/bs-fixture-r21-url.txt"
+          - "build/fixture-r21/*"
+        upload:
+          - "maze_output/failed/**/*"
+          - "maze_output/metrics.csv"
+      docker-compose#v4.7.0:
+        pull: maze-runner
+        run: maze-runner
+        service-ports: true
+        command:
+          - "features/smoke_tests/01_anr.feature"
+          - "--app=@build/bs-fixture-r21-url.txt"
+          - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
+          - "--app-package=com.bugsnag.android.mazerunner"
+          - "--appium-version=1.22.0"
+          - "--farm=bs"
+          - "--device=ANDROID_9"
+          - "--fail-fast"
+    env:
+      TEST_FIXTURE_SYMBOL_DIR: "build/fixture-r21"
+    concurrency: 5
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+
   - label: ':bitbar: Android 10 NDK r21 smoke tests'
     depends_on: "fixture-r21"
     timeout_in_minutes: 60
@@ -210,6 +305,7 @@ steps:
         service-ports: true
         command:
           - "features/smoke_tests"
+          - "--exclude=features/smoke_tests/01_anr.feature"
           - "--app=@build/fixture-r21-url.txt"
           - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
           - "--app-package=com.bugsnag.android.mazerunner"
@@ -225,6 +321,36 @@ steps:
     concurrency_group: 'bitbar'
     concurrency_method: eager
 
+  - label: ':browserstack: Android 10 NDK r21 ANR smoke tests'
+    depends_on: "fixture-r21"
+    timeout_in_minutes: 60
+    plugins:
+      artifacts#v1.9.0:
+        download:
+          - "build/bs-fixture-r21-url.txt"
+          - "build/fixture-r21/*"
+        upload:
+          - "maze_output/failed/**/*"
+          - "maze_output/metrics.csv"
+      docker-compose#v4.7.0:
+        pull: maze-runner
+        run: maze-runner
+        service-ports: true
+        command:
+          - "features/smoke_tests/01_anr.feature"
+          - "--app=@build/bs-fixture-r21-url.txt"
+          - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
+          - "--app-package=com.bugsnag.android.mazerunner"
+          - "--appium-version=1.22.0"
+          - "--farm=bs"
+          - "--device=ANDROID_10"
+          - "--fail-fast"
+    env:
+      TEST_FIXTURE_SYMBOL_DIR: "build/fixture-r21"
+    concurrency: 5
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+
   - label: ':bitbar: Android 11 NDK r21 smoke tests'
     key: 'android-11-smoke'
     depends_on: "fixture-r21"
@@ -243,6 +369,7 @@ steps:
         service-ports: true
         command:
           - "features/smoke_tests"
+          - "--exclude=features/smoke_tests/01_anr.feature"
           - "--app=@build/fixture-r21-url.txt"
           - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
           - "--app-package=com.bugsnag.android.mazerunner"
@@ -258,6 +385,37 @@ steps:
     concurrency_group: 'bitbar'
     concurrency_method: eager
 
+  - label: ':browserstack: Android 11 NDK r21 ANR smoke tests'
+    key: 'android-11-anr-smoke'
+    depends_on: "fixture-r21"
+    timeout_in_minutes: 60
+    plugins:
+      artifacts#v1.9.0:
+        download:
+          - "build/bs-fixture-r21-url.txt"
+          - "build/fixture-r21/*"
+        upload:
+          - "maze_output/failed/**/*"
+          - "maze_output/metrics.csv"
+      docker-compose#v4.7.0:
+        pull: maze-runner
+        run: maze-runner
+        service-ports: true
+        command:
+          - "features/smoke_tests/01_anr.feature"
+          - "--app=@build/bs-fixture-r21-url.txt"
+          - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
+          - "--app-package=com.bugsnag.android.mazerunner"
+          - "--appium-version=1.22.0"
+          - "--farm=bs"
+          - "--device=ANDROID_11"
+          - "--fail-fast"
+    env:
+      TEST_FIXTURE_SYMBOL_DIR: "build/fixture-r21"
+    concurrency: 5
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+
  # Android 11+ devices have the GWP-ASAN tool enabled which randomly samples native memory
   # to detect misuse (such as use-after-free, buffer overflow). If a failure is detected then
   # the device will raise a SIGABRT mentioning GWP-ASAN - this can be investigated further
@@ -281,6 +439,7 @@ steps:
         command:
           - "features/full_tests"
           - "--exclude=features/full_tests/[^a-k].*.feature"
+          - "--exclude=features/full_tests/anr.feature"
           - "--app=@build/fixture-r21-url.txt"
           - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
           - "--app-package=com.bugsnag.android.mazerunner"
@@ -296,6 +455,37 @@ steps:
     concurrency_group: 'bitbar'
     concurrency_method: eager
 
+  - label: ':browserstack: Android 12 NDK r21 end-to-end tests - batch ANR'
+    depends_on:
+      - "fixture-r21"
+    timeout_in_minutes: 60
+    plugins:
+      artifacts#v1.9.0:
+        download:
+          - "build/bs-fixture-r21-url.txt"
+          - "build/fixture-r21/*"
+        upload:
+          - "maze_output/failed/**/*"
+          - "maze_output/metrics.csv"
+      docker-compose#v4.7.0:
+        pull: maze-runner
+        run: maze-runner
+        service-ports: true
+        command:
+          - "features/full_tests/anr.feature"
+          - "--app=@build/bs-fixture-r21-url.txt"
+          - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
+          - "--app-package=com.bugsnag.android.mazerunner"
+          - "--appium-version=1.22.0"
+          - "--farm=bs"
+          - "--device=ANDROID_12"
+          - "--fail-fast"
+    env:
+      TEST_FIXTURE_SYMBOL_DIR: "build/fixture-r21"
+    concurrency: 5
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+
   - label: ':bitbar: Android 12 NDK r21 end-to-end tests - batch 2'
     depends_on:
       - "fixture-r21"
@@ -315,6 +505,7 @@ steps:
         command:
           - "features/full_tests"
           - "--exclude=features/full_tests/[^l-z].*.feature"
+          - "--exclude=features/full_tests/anr.feature"
           - "--app=@build/fixture-r21-url.txt"
           - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
           - "--app-package=com.bugsnag.android.mazerunner"
@@ -347,6 +538,7 @@ steps:
         service-ports: true
         command:
           - "features/smoke_tests"
+          - "--exclude=features/smoke_tests/01_anr.feature"
           - "--app=@build/fixture-r21-url.txt"
           - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
           - "--app-package=com.bugsnag.android.mazerunner"
@@ -362,6 +554,36 @@ steps:
     concurrency_group: 'bitbar'
     concurrency_method: eager
 
+  - label: ':browserstack: Android 13 NDK r21 ANR smoke tests'
+    depends_on: "fixture-r21"
+    timeout_in_minutes: 60
+    plugins:
+      artifacts#v1.9.0:
+        download:
+          - "build/bs-fixture-r21-url.txt"
+          - "build/fixture-r21/*"
+        upload:
+          - "maze_output/failed/**/*"
+          - "maze_output/metrics.csv"
+      docker-compose#v4.7.0:
+        pull: maze-runner
+        run: maze-runner
+        service-ports: true
+        command:
+          - "features/smoke_tests/01_anr.feature"
+          - "--app=@build/bs-fixture-r21-url.txt"
+          - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
+          - "--app-package=com.bugsnag.android.mazerunner"
+          - "--appium-version=1.22.0"
+          - "--farm=bs"
+          - "--device=ANDROID_13"
+          - "--fail-fast"
+    env:
+      TEST_FIXTURE_SYMBOL_DIR: "build/fixture-r21"
+    concurrency: 5
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+
   - label: ':bitbar: Android 14 NDK r21 smoke tests'
     depends_on: "fixture-r21"
     timeout_in_minutes: 60
@@ -379,6 +601,7 @@ steps:
         service-ports: true
         command:
           - "features/smoke_tests"
+          - "--exclude=features/smoke_tests/01_anr.feature"
           - "--app=@build/fixture-r21-url.txt"
           - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
           - "--app-package=com.bugsnag.android.mazerunner"
@@ -394,6 +617,37 @@ steps:
     concurrency_group: 'bitbar'
     concurrency_method: eager
 
+  - label: ':browserstack: Android 14 NDK r21 ANR smoke tests'
+    depends_on: "fixture-r21"
+    timeout_in_minutes: 60
+    plugins:
+      artifacts#v1.9.0:
+        download:
+          - "build/bs-fixture-r21-url.txt"
+          - "build/fixture-r21/*"
+        upload:
+          - "maze_output/failed/**/*"
+          - "maze_output/metrics.csv"
+      docker-compose#v4.7.0:
+        pull: maze-runner
+        run: maze-runner
+        service-ports: true
+        command:
+          - "features/smoke_tests/01_anr.feature"
+          - "--app=@build/bs-fixture-r21-url.txt"
+          - "--app-activity=com.bugsnag.android.mazerunner.MainActivity"
+          - "--app-package=com.bugsnag.android.mazerunner"
+          - "--appium-version=1.22.0"
+          - "--farm=bs"
+          - "--device=ANDROID_14"
+          - "--fail-fast"
+    env:
+      TEST_FIXTURE_SYMBOL_DIR: "build/fixture-r21"
+    concurrency: 5
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+
+
   - label: 'Conditionally include device farms/full tests'
     agents:
       queue: macos-14