From b453cd8579e0f22040c4f175353e35e9b0a4c72f Mon Sep 17 00:00:00 2001 From: Mike Hardy Date: Fri, 22 Apr 2022 11:29:58 -0500 Subject: [PATCH] perf(ci, emulator): community emulator action / AVD caching / gradle cachine Also adds an emulator run benchmarking suite that works in combo with matrix If branch builds result in cache upload of emulator files the cache files there will need to be scoped for just changes on the snapshot file(s) --- .github/workflows/tests_emulator.yml | 171 +++++++++--------- tools/emulator_performance/.gitignore | 1 + .../analyze_emulator_performance.js | 101 +++++++++++ .../fetch_workflow_jobs_json.sh | 33 ++++ 4 files changed, 222 insertions(+), 84 deletions(-) create mode 100644 tools/emulator_performance/.gitignore create mode 100644 tools/emulator_performance/analyze_emulator_performance.js create mode 100755 tools/emulator_performance/fetch_workflow_jobs_json.sh diff --git a/.github/workflows/tests_emulator.yml b/.github/workflows/tests_emulator.yml index 62bfb539dfc4..7c014045e8b4 100644 --- a/.github/workflows/tests_emulator.yml +++ b/.github/workflows/tests_emulator.yml @@ -36,109 +36,112 @@ jobs: emulator_test: name: Android Emulator Test runs-on: macos-latest - timeout-minutes: 60 - env: - EMULATOR_COMMAND: "-avd TestingAVD -noaudio -gpu swiftshader_indirect -camera-back none -no-snapshot -no-window -no-boot-anim -nojni -memory 2048 -timezone 'Europe/London' -cores 2" - EMULATOR_EXECUTABLE: qemu-system-x86_64-headless + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + # Refactor to make these dynamic with a low/high bracket only on schedule, not push + # For now this is just the fastest combo (api/arch/target/snapshot-warm-time) based on testing + api-level: [29] + arch: [x86_64] + target: [google_apis] + first-boot-delay: [600] + # This is useful for benchmarking, do 0, 1, 2, etc (up to 256 max job-per-matrix limit) for averages + iteration: [0] steps: - uses: actions/checkout@v3 with: fetch-depth: 50 - - name: Gradle Cache - uses: actions/cache@v3 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}-v1 - - name: Configure JDK 1.11 uses: actions/setup-java@v3 with: distribution: "adopt" java-version: "11" # ubuntu-latest is about to default to 11, force it everywhere - - name: Verify JDK11 - # Default JDK varies depending on different runner flavors, make sure we are on 11 - # Run a check that exits with error unless it is 11 version to future-proof against unexpected upgrades - run: java -fullversion 2>&1 | grep '11.0' - shell: bash + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + with: + # Only write to the cache for builds on the 'main' branches, stops branches evicting main cache + # Builds on other branches will only read from main branch cache writes + # Comment this and the with: above out for performance testing on a branch + cache-read-only: ${{ github.ref != 'refs/heads/main' }} - name: Warm Gradle Cache # This makes sure we fetch gradle network resources with a retry uses: nick-invision/retry@v2 with: - timeout_minutes: 10 + timeout_minutes: 15 retry_wait_seconds: 60 max_attempts: 3 - command: ./gradlew assembleDebug assembleAndroidTest robolectricSdkDownload + command: ./gradlew packagePlayDebug packagePlayDebugAndroidTest - - name: Download Emulator Image - # This can fail on network request, wrap with retry - uses: nick-invision/retry@v2 + # This appears to be 'Cache Size: ~1230 MB (1290026823 B)' based on watching action logs + # Repo limit is 10GB; branch caches are independent; branches may read default branch cache. + # We don't want branches to evict main branch snapshot, so save on main, read-only all else + - name: AVD cache + uses: actions/cache@v3 + id: avd-cache with: - timeout_minutes: 10 - retry_wait_seconds: 60 - max_attempts: 3 - command: echo "y" | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "system-images;android-30;google_apis;x86_64" - - - name: Create Emulator - run: echo "no" | $ANDROID_HOME/cmdline-tools/latest/bin//avdmanager create avd --force --name TestingAVD --device "Nexus 5X" -k 'system-images;android-30;google_apis;x86_64' -g google_apis - - # These Emulator start steps are the current best practice to do retries on multi-line commands with persistent (nohup) processes - - name: Start Android Emulator - id: emu1 - timeout-minutes: 5 - continue-on-error: true - run: | - echo "Starting emulator" - nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND & - $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done' - - - name: Start Android Emulator Retry 1 - id: emu2 - if: steps.emu1.outcome=='failure' - timeout-minutes: 5 - continue-on-error: true - run: | - echo "Starting emulator, second attempt" - $ANDROID_HOME/platform-tools/adb devices - sudo killall -9 $EMULATOR_EXECUTABLE || true - sleep 2 - nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND & - $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done' - - - name: Start Android Emulator Retry 2 - id: emu3 - if: steps.emu2.outcome=='failure' - timeout-minutes: 5 - continue-on-error: true - run: | - echo "Starting emulator, third attempt" - $ANDROID_HOME/platform-tools/adb devices - sudo killall -9 $EMULATOR_EXECUTABLE || true - sleep 2 - nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND & - $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done' - - - name: Emulator Status - if: always() - run: | - if ${{ steps.emu1.outcome=='success' || steps.emu2.outcome=='success' || steps.emu3.outcome=='success' }}; then - echo "Emulator Started" - else - exit 1 - fi - - - name: Emulator Test - timeout-minutes: 40 - run: | - $ANDROID_HOME/platform-tools/adb devices - $ANDROID_HOME/platform-tools/adb shell settings put global window_animation_scale 0.0 - $ANDROID_HOME/platform-tools/adb shell settings put global transition_animation_scale 0.0 - $ANDROID_HOME/platform-tools/adb shell settings put global animator_duration_scale 0.0 - nohup sh -c "$ANDROID_HOME/platform-tools/adb logcat '*:D' > adb-log.txt" & - ./gradlew jacocoAndroidTestReport - shell: bash + path: | + ~/.android/avd/* + ~/.android/adb* + key: avd-${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-v1-${{ hashFiles('~/.android/avd/**/snapshots/**') }} + restore-keys: | + avd-${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-v1 + + - name: AVD Boot and Snapshot Creation + # Only generate a snapshot for saving if we are on main branch with a cache miss + # Comment the if out to generate snapshots on branch for performance testing + if: steps.avd-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: ${{ matrix.api-level }} + force-avd-creation: false + target: ${{ matrix.target }} + arch: ${{ matrix.arch }} + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + sdcard-path-or-size: 100M + disable-animations: true + # Give the emulator a little time to run and do first boot stuff before taking snapshot + script: echo "Generated AVD snapshot for caching." + + # This step is separate so pure install time may be calculated as a step + - name: Emulator Snapshot After Firstboot Warmup + # Only generate a snapshot for saving if we are on main branch with a cache miss + # Switch the if statements via comment if generating snapshots for performance testing + # if: matrix.first-boot-delay != '0' + if: steps.avd-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + env: + FIRST_BOOT_DELAY: ${{ matrix.first-boot-delay }} + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: ${{ matrix.api-level }} + force-avd-creation: false + target: ${{ matrix.target }} + arch: ${{ matrix.arch }} + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + sdcard-path-or-size: 100M + disable-animations: true + # Give the emulator a little time to run and do first boot stuff before taking snapshot + script: | + sleep $FIRST_BOOT_DELAY + echo "First boot warmup completed." + + - name: Run Emulator Tests + uses: reactivecircus/android-emulator-runner@v2 + timeout-minutes: 30 + with: + api-level: ${{ matrix.api-level }} + force-avd-creation: false + target: ${{ matrix.target }} + arch: ${{ matrix.arch }} + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + sdcard-path-or-size: 100M + disable-animations: true + script: | + $ANDROID_HOME/platform-tools/adb logcat '*:D' > adb-log.txt & + ./gradlew uninstallAll jacocoAndroidTestReport - name: Compress Emulator Log if: always() @@ -149,7 +152,7 @@ jobs: uses: actions/upload-artifact@v3 if: always() with: - name: adb_logs + name: ${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-${{matrix.first-boot-delay}}-${{matrix.iteration}}-adb_logs path: adb-log.txt.gz - name: Submit Coverage diff --git a/tools/emulator_performance/.gitignore b/tools/emulator_performance/.gitignore new file mode 100644 index 000000000000..8bd6bcf4c52f --- /dev/null +++ b/tools/emulator_performance/.gitignore @@ -0,0 +1 @@ +emulator_perf_results* diff --git a/tools/emulator_performance/analyze_emulator_performance.js b/tools/emulator_performance/analyze_emulator_performance.js new file mode 100644 index 000000000000..121f80a57f4c --- /dev/null +++ b/tools/emulator_performance/analyze_emulator_performance.js @@ -0,0 +1,101 @@ +// Fetch results to parse like this, with the workflow run id you want: +// curl https://api.github.com/repos/mikehardy/Anki-Android/actions/runs/2210525974/jobs?per_page=100 > emulator_perf_results.json + +// Or if you have more than 100 results, you need to page through them and merge them, there is a script +// ./fetch_workflow_jobs_json.sh 2212862357 + +function main() { + // Read in the results + // console.log("Processing results in " + process.argv[2]); + var fs = require("fs"); + var runLog = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); + + console.log( + '"Android API","Emulator Architecture","Emulator Image","First Boot Warmup Delay","Average AVD Create/Boot Elapsed Seconds","Average AVD Reboot/Test Elapsed Seconds","Average Total Elapsed Seconds","Failure Count"', + ); + + let averageTimings = {}; + + runLog.jobs.forEach(job => { + // console.log("analyzing job " + job.name); + const matrixVars = job.name.match(/.*\((.*)\)/)[1].split(", "); + // console.log("Job name: " + job.name); + // console.log(" Android API level: " + matrixVars[0]); + // console.log(" Emulator Architecture: " + matrixVars[1]); + // console.log(" Emulator Image: " + matrixVars[2]); + + const startTime = new Date(job.started_at); + const endTime = new Date(job.completed_at); + let jobElapsed = endTime - startTime; + jobElapsed = jobElapsed > 0 ? jobElapsed : 0; // some are negative !? + + // console.log(" conclusion: " + job.conclusion); + // console.log(" elapsed_time_seconds: " + jobElapsed / 1000); + + let AVDCreateBootElapsedSeconds = -1; + let AVDRebootTestElapsedSeconds = -1; + let stepFailed = false; + + job.steps.forEach(step => { + if (!["success", "skipped"].includes(step.conclusion)) { + stepFailed = true; + return; + } + const stepStart = new Date(step.started_at); + const stepEnd = new Date(step.completed_at); + let stepElapsedSeconds = (stepEnd - stepStart) / 1000; + stepElapsedSeconds = stepElapsedSeconds > 0 ? stepElapsedSeconds : 0; // some are negative !? + + switch (step.name) { + case "AVD Boot and Snapshot Creation": + AVDCreateBootElapsedSeconds = stepElapsedSeconds; + case "Run Emulator Tests": + AVDRebootTestElapsedSeconds = stepElapsedSeconds; + } + }); + + // Get or create aggregate timing entry + timingKey = `${matrixVars[0]}_${matrixVars[1]}_${matrixVars[2]}_${matrixVars[3]}`; + let currentAverageTiming = averageTimings[timingKey]; + if (currentAverageTiming === undefined) { + currentAverageTiming = { + api: matrixVars[0], + arch: matrixVars[1], + target: matrixVars[2], + warmtime: matrixVars[3], + totalCreateBootElapsedSecs: 0, + totalTestElapsedSecs: 0, + runs: 0, + failureCount: 0, + }; + averageTimings[timingKey] = currentAverageTiming; + } + + // If something failed, set status and skip timing aggregation + if (stepFailed) { + currentAverageTiming.failureCount++; + return; + } + + // Update our aggregate timings + currentAverageTiming.totalCreateBootElapsedSecs += AVDCreateBootElapsedSeconds; + currentAverageTiming.totalTestElapsedSecs += AVDRebootTestElapsedSeconds; + currentAverageTiming.runs++; + }); + + // Print out averages for each non-iteration combo + Object.keys(averageTimings).forEach(key => { + // console.log("printing timings for key " + key); + const timing = averageTimings[key]; + // console.log("entry is " + JSON.stringify(timing)); + console.log( + `"${timing.api}","${timing.arch}","${timing.target}","${timing.warmtime}","${ + timing.totalCreateBootElapsedSecs / timing.runs + }","${timing.totalTestElapsedSecs / timing.runs}","${ + (timing.totalCreateBootElapsedSecs + timing.totalTestElapsedSecs) / timing.runs + }","${timing.failureCount}"`, + ); + }); +} + +main(); diff --git a/tools/emulator_performance/fetch_workflow_jobs_json.sh b/tools/emulator_performance/fetch_workflow_jobs_json.sh new file mode 100755 index 000000000000..97bda3b33d03 --- /dev/null +++ b/tools/emulator_performance/fetch_workflow_jobs_json.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +echo "Fetching jobs JSON for workflow run $1" + +rm -f emulator_perf_results_page*.json + +REPO_URL=https://api.github.com/repos/mikehardy/Anki-Android +PER_PAGE=100 +PAGE=1 +curl --silent "$REPO_URL/actions/runs/$1/jobs?per_page=$PER_PAGE&page=$PAGE" > emulator_perf_results_page"$PAGE".json + +TOTAL_COUNT=$(jq '.total_count' emulator_perf_results.json) +LAST_PAGE=$((TOTAL_COUNT / PER_PAGE + 1)) +echo "$TOTAL_COUNT jobs so $LAST_PAGE pages" +for ((PAGE=2; PAGE <= LAST_PAGE; PAGE++)); do + echo "On iteration $PAGE" + curl --silent "$REPO_URL/actions/runs/$1/jobs?per_page=$PER_PAGE&page=$PAGE" > emulator_perf_results_page"$PAGE".json +done + +jq -s 'def deepmerge(a;b): + reduce b[] as $item (a; + reduce ($item | keys_unsorted[]) as $key (.; + $item[$key] as $val | ($val | type) as $type | .[$key] = if ($type == "object") then + deepmerge({}; [if .[$key] == null then {} else .[$key] end, $val]) + elif ($type == "array") then + (.[$key] + $val | unique) + else + $val + end) + ); + deepmerge({}; .)' emulator_perf_results_page*.json > emulator_perf_results.json + +rm -f emulator_perf_results_page*.json \ No newline at end of file