perf(ci, emulator): community emulator action / AVD caching / gradle caching

.github/workflows/tests_emulator.yml

@@ -36,109 +36,112 @@ jobs:
   emulator_test:
     name: Android Emulator Test
     runs-on: macos-latest
-    timeout-minutes: 60
-    env:
-      EMULATOR_COMMAND: "-avd TestingAVD -noaudio -gpu swiftshader_indirect -camera-back none -no-snapshot -no-window -no-boot-anim -nojni -memory 2048 -timezone 'Europe/London' -cores 2"
-      EMULATOR_EXECUTABLE: qemu-system-x86_64-headless
+    timeout-minutes: 75
+    strategy:
+      fail-fast: false
+      matrix:
+        # Refactor to make these dynamic with a low/high bracket only on schedule, not push
+        # For now this is just the fastest combo (api/arch/target/snapshot-warm-time) based on testing
+        api-level: [29]
+        arch: [x86_64]
+        target: [google_apis]
+        first-boot-delay: [600]
+        # This is useful for benchmarking, do 0, 1, 2, etc (up to 256 max job-per-matrix limit) for averages
+        iteration: [0]
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 50
 
-      - name: Gradle Cache
-        uses: actions/cache@v3
-        with:
-          path: ~/.gradle/caches
-          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}-v1
-
       - name: Configure JDK 1.11
         uses: actions/setup-java@v3
         with:
           distribution: "adopt"
           java-version: "11" # ubuntu-latest is about to default to 11, force it everywhere
 
-      - name: Verify JDK11
-        # Default JDK varies depending on different runner flavors, make sure we are on 11
-        # Run a check that exits with error unless it is 11 version to future-proof against unexpected upgrades
-        run: java -fullversion 2>&1 | grep '11.0'
-        shell: bash
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+        with:
+          # Only write to the cache for builds on the 'main' branches, stops branches evicting main cache
+          # Builds on other branches will only read from main branch cache writes
+          # Comment this and the with: above out for performance testing on a branch
+          cache-read-only: ${{ github.ref != 'refs/heads/main' }}
 
       - name: Warm Gradle Cache
         # This makes sure we fetch gradle network resources with a retry
         uses: nick-invision/retry@v2
         with:
-          timeout_minutes: 10
+          timeout_minutes: 15
           retry_wait_seconds: 60
           max_attempts: 3
-          command: ./gradlew assembleDebug assembleAndroidTest robolectricSdkDownload
+          command: ./gradlew packagePlayDebug packagePlayDebugAndroidTest
 
-      - name: Download Emulator Image
-        # This can fail on network request, wrap with retry
-        uses: nick-invision/retry@v2
+        # This appears to be 'Cache Size: ~1230 MB (1290026823 B)' based on watching action logs
+        # Repo limit is 10GB; branch caches are independent; branches may read default branch cache.
+        # We don't want branches to evict main branch snapshot, so save on main, read-only all else
+      - name: AVD cache
+        uses: actions/cache@v3
+        id: avd-cache
         with:
-          timeout_minutes: 10
-          retry_wait_seconds: 60
-          max_attempts: 3
-          command: echo "y" | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "system-images;android-30;google_apis;x86_64"
-
-      - name: Create Emulator
-        run: echo "no" | $ANDROID_HOME/cmdline-tools/latest/bin//avdmanager create avd --force --name TestingAVD --device "Nexus 5X" -k 'system-images;android-30;google_apis;x86_64' -g google_apis
-
-      # These Emulator start steps are the current best practice to do retries on multi-line commands with persistent (nohup) processes
-      - name: Start Android Emulator
-        id: emu1
-        timeout-minutes: 5
-        continue-on-error: true
-        run: |
-          echo "Starting emulator"
-          nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND &
-          $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
-
-      - name: Start Android Emulator Retry 1
-        id: emu2
-        if: steps.emu1.outcome=='failure'
-        timeout-minutes: 5
-        continue-on-error: true
-        run: |
-          echo "Starting emulator, second attempt"
-          $ANDROID_HOME/platform-tools/adb devices
-          sudo killall -9 $EMULATOR_EXECUTABLE || true
-          sleep 2
-          nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND &
-          $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
-
-      - name: Start Android Emulator Retry 2
-        id: emu3
-        if: steps.emu2.outcome=='failure'
-        timeout-minutes: 5
-        continue-on-error: true
-        run: |
-          echo "Starting emulator, third attempt"
-          $ANDROID_HOME/platform-tools/adb devices
-          sudo killall -9 $EMULATOR_EXECUTABLE || true
-          sleep 2
-          nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND &
-          $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
-
-      - name: Emulator Status
-        if: always()
-        run: |
-          if ${{ steps.emu1.outcome=='success' || steps.emu2.outcome=='success' || steps.emu3.outcome=='success' }}; then
-             echo "Emulator Started"
-          else
-             exit 1
-          fi
-
-      - name: Emulator Test
-        timeout-minutes: 40
-        run: |
-          $ANDROID_HOME/platform-tools/adb devices
-          $ANDROID_HOME/platform-tools/adb shell settings put global window_animation_scale 0.0
-          $ANDROID_HOME/platform-tools/adb shell settings put global transition_animation_scale 0.0
-          $ANDROID_HOME/platform-tools/adb shell settings put global animator_duration_scale 0.0
-          nohup sh -c "$ANDROID_HOME/platform-tools/adb logcat '*:D' > adb-log.txt" &
-          ./gradlew jacocoAndroidTestReport
-        shell: bash
+          path: |
+            ~/.android/avd/*
+            ~/.android/adb*
+          key: avd-${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-v1-${{ hashFiles('~/.android/avd/**/snapshots/**') }}
+          restore-keys: |
+            avd-${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-v1
+
+      - name: AVD Boot and Snapshot Creation
+        # Only generate a snapshot for saving if we are on main branch with a cache miss
+        # Comment the if out to generate snapshots on branch for performance testing
+        if: steps.avd-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main'
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: ${{ matrix.api-level }}
+          force-avd-creation: false
+          target: ${{ matrix.target }}
+          arch: ${{ matrix.arch }}
+          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          sdcard-path-or-size: 100M
+          disable-animations: true
+          # Give the emulator a little time to run and do first boot stuff before taking snapshot
+          script: echo "Generated AVD snapshot for caching."
+
+        # This step is separate so pure install time may be calculated as a step
+      - name: Emulator Snapshot After Firstboot Warmup
+        # Only generate a snapshot for saving if we are on main branch with a cache miss
+        # Switch the if statements via comment if generating snapshots for performance testing
+        # if: matrix.first-boot-delay != '0'
+        if: steps.avd-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main'
+        env:
+          FIRST_BOOT_DELAY: ${{ matrix.first-boot-delay }}
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: ${{ matrix.api-level }}
+          force-avd-creation: false
+          target: ${{ matrix.target }}
+          arch: ${{ matrix.arch }}
+          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          sdcard-path-or-size: 100M
+          disable-animations: true
+          # Give the emulator a little time to run and do first boot stuff before taking snapshot
+          script: |
+            sleep $FIRST_BOOT_DELAY
+            echo "First boot warmup completed."
+
+      - name: Run Emulator Tests
+        uses: reactivecircus/android-emulator-runner@v2
+        timeout-minutes: 30
+        with:
+          api-level: ${{ matrix.api-level }}
+          force-avd-creation: false
+          target: ${{ matrix.target }}
+          arch: ${{ matrix.arch }}
+          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          sdcard-path-or-size: 100M
+          disable-animations: true
+          script: |
+            $ANDROID_HOME/platform-tools/adb logcat '*:D' > adb-log.txt &
+            ./gradlew uninstallAll jacocoAndroidTestReport
 
       - name: Compress Emulator Log
         if: always()
@@ -149,7 +152,7 @@ jobs:
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: adb_logs
+          name: ${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-${{matrix.first-boot-delay}}-${{matrix.iteration}}-adb_logs
           path: adb-log.txt.gz
 
       - name: Submit Coverage

tools/emulator_performance/.gitignore

@@ -0,0 +1 @@
+emulator_perf_results*

tools/emulator_performance/analyze_emulator_performance.js

@@ -0,0 +1,101 @@
+// Fetch results to parse like this, with the workflow run id you want:
+// curl https://api.github.com/repos/mikehardy/Anki-Android/actions/runs/2210525974/jobs?per_page=100 > emulator_perf_results.json
+
+// Or if you have more than 100 results, you need to page through them and merge them, there is a script
+// ./fetch_workflow_jobs_json.sh 2212862357
+
+function main() {
+    // Read in the results
+    // console.log("Processing results in " + process.argv[2]);
+    var fs = require("fs");
+    var runLog = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
+
+    console.log(
+        '"Android API","Emulator Architecture","Emulator Image","First Boot Warmup Delay","Average AVD Create/Boot Elapsed Seconds","Average AVD Reboot/Test Elapsed Seconds","Average Total Elapsed Seconds","Failure Count"',
+    );
+
+    let averageTimings = {};
+
+    runLog.jobs.forEach(job => {
+        // console.log("analyzing job " + job.name);
+        const matrixVars = job.name.match(/.*\((.*)\)/)[1].split(", ");
+        // console.log("Job name: " + job.name);
+        // console.log("  Android API level: " + matrixVars[0]);
+        // console.log("  Emulator Architecture: " + matrixVars[1]);
+        // console.log("  Emulator Image: " + matrixVars[2]);
+
+        const startTime = new Date(job.started_at);
+        const endTime = new Date(job.completed_at);
+        let jobElapsed = endTime - startTime;
+        jobElapsed = jobElapsed > 0 ? jobElapsed : 0; // some are negative !?
+
+        // console.log("  conclusion: " + job.conclusion);
+        // console.log("  elapsed_time_seconds: " + jobElapsed / 1000);
+
+        let AVDCreateBootElapsedSeconds = -1;
+        let AVDRebootTestElapsedSeconds = -1;
+        let stepFailed = false;
+
+        job.steps.forEach(step => {
+            if (!["success", "skipped"].includes(step.conclusion)) {
+                stepFailed = true;
+                return;
+            }
+            const stepStart = new Date(step.started_at);
+            const stepEnd = new Date(step.completed_at);
+            let stepElapsedSeconds = (stepEnd - stepStart) / 1000;
+            stepElapsedSeconds = stepElapsedSeconds > 0 ? stepElapsedSeconds : 0; // some are negative !?
+
+            switch (step.name) {
+                case "AVD Boot and Snapshot Creation":
+                    AVDCreateBootElapsedSeconds = stepElapsedSeconds;
+                case "Run Emulator Tests":
+                    AVDRebootTestElapsedSeconds = stepElapsedSeconds;
+            }
+        });
+
+        // Get or create aggregate timing entry
+        timingKey = `${matrixVars[0]}_${matrixVars[1]}_${matrixVars[2]}_${matrixVars[3]}`;
+        let currentAverageTiming = averageTimings[timingKey];
+        if (currentAverageTiming === undefined) {
+            currentAverageTiming = {
+                api: matrixVars[0],
+                arch: matrixVars[1],
+                target: matrixVars[2],
+                warmtime: matrixVars[3],
+                totalCreateBootElapsedSecs: 0,
+                totalTestElapsedSecs: 0,
+                runs: 0,
+                failureCount: 0,
+            };
+            averageTimings[timingKey] = currentAverageTiming;
+        }
+
+        // If something failed, set status and skip timing aggregation
+        if (stepFailed) {
+            currentAverageTiming.failureCount++;
+            return;
+        }
+
+        // Update our aggregate timings
+        currentAverageTiming.totalCreateBootElapsedSecs += AVDCreateBootElapsedSeconds;
+        currentAverageTiming.totalTestElapsedSecs += AVDRebootTestElapsedSeconds;
+        currentAverageTiming.runs++;
+    });
+
+    // Print out averages for each non-iteration combo
+    Object.keys(averageTimings).forEach(key => {
+        // console.log("printing timings for key " + key);
+        const timing = averageTimings[key];
+        // console.log("entry is " + JSON.stringify(timing));
+        console.log(
+            `"${timing.api}","${timing.arch}","${timing.target}","${timing.warmtime}","${
+                timing.totalCreateBootElapsedSecs / timing.runs
+            }","${timing.totalTestElapsedSecs / timing.runs}","${
+                (timing.totalCreateBootElapsedSecs + timing.totalTestElapsedSecs) / timing.runs
+            }","${timing.failureCount}"`,
+        );
+    });
+}
+
+main();

tools/emulator_performance/fetch_workflow_jobs_json.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+echo "Fetching jobs JSON for workflow run $1"
+
+rm -f emulator_perf_results_page*.json
+
+REPO_URL=https://api.github.com/repos/mikehardy/Anki-Android
+PER_PAGE=100
+PAGE=1
+curl --silent "$REPO_URL/actions/runs/$1/jobs?per_page=$PER_PAGE&page=$PAGE" > emulator_perf_results_page"$PAGE".json
+
+TOTAL_COUNT=$(jq '.total_count' emulator_perf_results.json)
+LAST_PAGE=$((TOTAL_COUNT / PER_PAGE + 1))
+echo "$TOTAL_COUNT jobs so $LAST_PAGE pages"
+for ((PAGE=2; PAGE <= LAST_PAGE; PAGE++)); do
+  echo "On iteration $PAGE"
+  curl --silent "$REPO_URL/actions/runs/$1/jobs?per_page=$PER_PAGE&page=$PAGE" > emulator_perf_results_page"$PAGE".json
+done
+
+jq -s 'def deepmerge(a;b):
+  reduce b[] as $item (a;
+    reduce ($item | keys_unsorted[]) as $key (.;
+      $item[$key] as $val | ($val | type) as $type | .[$key] = if ($type == "object") then
+        deepmerge({}; [if .[$key] == null then {} else .[$key] end, $val])
+      elif ($type == "array") then
+        (.[$key] + $val | unique)
+      else
+        $val
+      end)
+    );
+  deepmerge({}; .)' emulator_perf_results_page*.json > emulator_perf_results.json
+
+rm -f emulator_perf_results_page*.json