diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 974e767d05e8..a7ac73237417 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -38,7 +38,6 @@ # Infra Top-Level Directories /build_tools/ @ScottTodd -/build_tools/benchmarks/ @antiagainst /build_tools/python_deploy/ @stellaraccident /build_tools/scripts/ @ScottTodd /build_tools/third_party/ @ScottTodd @stellaraccident diff --git a/.github/workflows/ARTIFACT_SUMMARY_TEMPLATE.md b/.github/workflows/ARTIFACT_SUMMARY_TEMPLATE.md deleted file mode 100644 index 69d366dc3d59..000000000000 --- a/.github/workflows/ARTIFACT_SUMMARY_TEMPLATE.md +++ /dev/null @@ -1,47 +0,0 @@ -### :link: Artifact Links - -```sh -# Compiler and host tools archive: -export INSTALL_DIR_URL="${INSTALL_DIR_GCS_ARTIFACT}" -# Directory of e2e test artifacts: -export E2E_TEST_ARTIFACTS_DIR_URL="${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}" -# Directory of benchmark tool binaries: -export BENCHMARK_TOOLS_DIR_URL="${BENCHMARK_TOOLS_GCS_ARTIFACT_DIR}" -# Directory of execution benchmark results: -export EXECUTION_BENCHMARK_RESULTS_DIR_URL="${EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR}" -# Compilation benchmark results: -export COMPILATION_BENCHMARK_RESULTS_URL="${COMPILATION_BENCHMARK_RESULTS_GCS_ARTIFACT}" -``` - -You can list `gs://` directories and download files with [gcloud cli]( -https://cloud.google.com/sdk/gcloud/reference/storage). - -
-Click to show common usages - -```sh -# Get compile and run flags of benchmarks -gcloud storage cp -r "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}/benchmark-flag-dump.txt" /tmp/ -``` - -```sh -# Download MLIR input files and command lines -mkdir /tmp/iree_e2e_test_inputs -gcloud storage cp "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}/*" /tmp/iree_e2e_test_inputs -``` - -```sh -# Download all artifacts (MLIR input files and compiled VMFBs) -mkdir /tmp/iree_e2e_test_artifacts -gcloud storage cp -r "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}/*" /tmp/iree_e2e_test_artifacts -``` - -```sh -# Download execution benchmark results -mkdir /tmp/iree_benchmark_results -gcloud storage cp -r "${EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR}/*" /tmp/iree_benchmark_results -``` -
- -To run benchmarks locally with the CI-built e2e test artifacts, see [IREE Benchmark Suites]( -https://iree.dev/developers/performance/benchmark-suites/#3-fetch-the-benchmark-artifacts). diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index 9b4b6a6db3bc..000000000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,296 +0,0 @@ -# Copyright 2024 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -name: Benchmark - -on: - workflow_dispatch: - pull_request: - push: - branches: - - main - -concurrency: - # A PR number if a pull request and otherwise the commit hash. This cancels - # queued and in-progress runs for the same PR (presubmit) or commit - # (postsubmit). The workflow name is prepended to avoid conflicts between - # different workflows. - group: ${{ github.workflow }}-${{ github.event.number || github.sha }} - cancel-in-progress: true - -env: - # This needs to be in env instead of the outputs of setup because it contains - # the run attempt and we want that to be the current attempt, not whatever - # attempt the setup step last ran in. - GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }} - -jobs: - setup: - uses: ./.github/workflows/setup.yml - - build_for_benchmarks: - needs: setup - if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_for_benchmarks') && needs.setup.outputs.benchmark-presets != '' - uses: ./.github/workflows/build_all.yml - with: - runner-group: ${{ needs.setup.outputs.runner-group }} - runner-env: ${{ needs.setup.outputs.runner-env }} - write-caches: ${{ needs.setup.outputs.write-caches }} - run-tests: false - - build_benchmark_tools: - needs: [setup, build_for_benchmarks] - if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_benchmark_tools') && needs.setup.outputs.benchmark-presets != '' - uses: ./.github/workflows/build_benchmark_tools.yml - with: - runner-group: ${{ needs.setup.outputs.runner-group }} - runner-env: ${{ needs.setup.outputs.runner-env }} - install-dir: ${{ needs.build_for_benchmarks.outputs.install-dir }} - install-dir-archive: ${{ needs.build_for_benchmarks.outputs.install-dir-archive }} - install-dir-gcs-artifact: ${{ needs.build_for_benchmarks.outputs.install-dir-gcs-artifact }} - - build_e2e_test_artifacts: - needs: [setup, build_for_benchmarks] - if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_e2e_test_artifacts') && needs.setup.outputs.benchmark-presets != '' - uses: ./.github/workflows/build_e2e_test_artifacts.yml - with: - runner-group: ${{ needs.setup.outputs.runner-group }} - runner-env: ${{ needs.setup.outputs.runner-env }} - install-dir: ${{ needs.build_for_benchmarks.outputs.install-dir }} - install-dir-archive: ${{ needs.build_for_benchmarks.outputs.install-dir-archive }} - install-dir-gcs-artifact: ${{ needs.build_for_benchmarks.outputs.install-dir-gcs-artifact }} - benchmark-presets: ${{ needs.setup.outputs.benchmark-presets }} - shard-count: "c2-standard-60=2,default=1" - - test_benchmark_suites: - needs: [setup, build_for_benchmarks, build_e2e_test_artifacts] - if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_benchmark_suites') && needs.setup.outputs.benchmark-presets != '' - strategy: - matrix: - target: - - platform: linux - arch: riscv_64 - docker_image: "gcr.io/iree-oss/riscv@sha256:62e87bad3405d691ddba6f9be0ef44eeb60461a467c8d86f0842c81a1f97da79" - run_scripts: "./build_tools/cmake/build_riscv.sh && ./build_tools/cmake/test_riscv.sh" - - platform: linux - arch: riscv_32 - docker_image: "gcr.io/iree-oss/riscv@sha256:62e87bad3405d691ddba6f9be0ef44eeb60461a467c8d86f0842c81a1f97da79" - run_scripts: "./build_tools/cmake/build_riscv.sh && ./build_tools/cmake/test_riscv.sh" - - platform: linux - arch: x86_64 - docker_image: "gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" - run_scripts: "./build_tools/cmake/test_benchmark_suites_on_linux.sh" - # Requires Intel CascadeLake CPU. - host_machine: c2s601t - runs-on: - - self-hosted # must come first - - runner-group=${{ needs.setup.outputs.runner-group }} - - environment=${{ needs.setup.outputs.runner-env }} - - ${{ matrix.target.host_machine || 'cpu' }} # Default to generic x86_64 VM. - env: - PLATFORM: ${{ matrix.target.platform }} - ARCH: ${{ matrix.target.arch }} - DOCKER_IMAGE: ${{ matrix.target.docker_image }} - RUN_SCRIPTS: ${{ matrix.target.run_scripts }} - INSTALL_DIR: ${{ needs.build_for_benchmarks.outputs.install-dir }} - INSTALL_DIR_ARCHIVE: ${{ needs.build_for_benchmarks.outputs.install-dir-archive }} - INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_for_benchmarks.outputs.install-dir-gcs-artifact }} - TARGET_BUILD_DIR: build-${{ matrix.target.platform }}-${{ matrix.target.arch }} - E2E_TEST_ARTIFACTS_DIR: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }} - E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }} - steps: - - name: "Checking out repository" - uses: actions/checkout@v4.1.7 - - name: "Checking out runtime submodules" - run: ./build_tools/scripts/git/update_runtime_submodules.sh - - name: "Downloading install dir archive" - run: gcloud storage cp "${INSTALL_DIR_GCS_ARTIFACT}" "${INSTALL_DIR_ARCHIVE}" - - name: "Extracting install directory" - run: tar -xf "${INSTALL_DIR_ARCHIVE}" - # TODO(#11136): Only download the needed artifacts instead of everything. - - name: "Downloading e2e test artifacts" - run: | - mkdir -p ${E2E_TEST_ARTIFACTS_DIR} - gcloud storage cp -r "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}/*" "${E2E_TEST_ARTIFACTS_DIR}" - - name: "Build iree-run-module and test benchmark suite modules" - run: | - ./build_tools/github_actions/docker_run.sh \ - --env "IREE_TARGET_PLATFORM=${PLATFORM}" \ - --env "IREE_TARGET_ARCH=${ARCH}" \ - --env "IREE_TARGET_BUILD_DIR=${TARGET_BUILD_DIR}" \ - --env "BUILD_PRESET=benchmark-suite-test" \ - --env "IREE_HOST_BIN_DIR=${INSTALL_DIR}/bin" \ - --env "E2E_TEST_ARTIFACTS_DIR=${E2E_TEST_ARTIFACTS_DIR}" \ - "${DOCKER_IMAGE}" \ - bash -euo pipefail -c \ - "${RUN_SCRIPTS}" - - compilation_benchmarks: - needs: [setup, build_e2e_test_artifacts] - if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'compilation_benchmarks') && needs.setup.outputs.benchmark-presets != '' - uses: ./.github/workflows/benchmark_compilation.yml - with: - runner-group: ${{ needs.setup.outputs.runner-group }} - runner-env: ${{ needs.setup.outputs.runner-env }} - e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }} - e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }} - e2e-test-artifacts-build-log: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log }} - e2e-test-artifacts-build-log-gcs-artifact: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log-gcs-artifact }} - - execution_benchmarks: - needs: [setup, build_benchmark_tools, build_e2e_test_artifacts] - if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'execution_benchmarks') && needs.setup.outputs.benchmark-presets != '' - uses: ./.github/workflows/benchmark_execution.yml - with: - runner-group: ${{ needs.setup.outputs.runner-group }} - runner-env: ${{ needs.setup.outputs.runner-env }} - e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }} - e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }} - benchmark-tools-gcs-artifact-dir: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir }} - - process_benchmark_results: - needs: [setup, compilation_benchmarks, execution_benchmarks] - if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'process_benchmark_results') && needs.setup.outputs.benchmark-presets != '' - runs-on: - - self-hosted # must come first - - runner-group=${{ needs.setup.outputs.runner-group }} - - environment=${{ needs.setup.outputs.runner-env }} - - cpu - - os-family=Linux - env: - COMPILE_STATS_RESULTS: ${{ needs.compilation_benchmarks.outputs.compile-stats-results }} - COMPILE_STATS_RESULTS_GCS_ARTIFACT: ${{ needs.compilation_benchmarks.outputs.compile-stats-results-gcs-artifact }} - # Empty if no execution benchmark runs. - EXECUTION_BENCHMARK_RESULTS_DIR: ${{ needs.execution_benchmarks.outputs.benchmark-results-dir }} - # Empty if no execution benchmark runs. - EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR: ${{ needs.execution_benchmarks.outputs.benchmark-results-gcs-artifact-dir }} - steps: - - name: "Checking out repository" - uses: actions/checkout@v4.1.7 - with: - # We need the full history (and main branch) to generate the report. - fetch-depth: 0 - - name: Downloading compilation benchmark results - run: | - gcloud storage cp \ - "${COMPILE_STATS_RESULTS_GCS_ARTIFACT}" \ - "${COMPILE_STATS_RESULTS}" - - name: Downloading execution benchmark results - id: download-execution-results - # Skip the download if there is no execution benchmark results (e.g. no - # benchmark matches the preset/filter). In such case, no benchmark job - # is run in benchmark_execution.yml and the output variables are empty. - if: env.EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR != '' - run: | - gcloud storage cp -r \ - "${EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR}/benchmark-results-*.json" \ - "${EXECUTION_BENCHMARK_RESULTS_DIR}" - echo "execution-benchmark-results-pattern=${EXECUTION_BENCHMARK_RESULTS_DIR}/benchmark-results-*.json" >> "${GITHUB_OUTPUT}" - - name: Generating comment - if: fromJson(needs.setup.outputs.is-pr) - id: generate-comment - env: - # Wildcard pattern to match all execution benchmark results. Empty if - # execution_benchmarks is skipped, which results in no match. - EXECUTION_BENCHMARK_RESULTS_PATTERN: ${{ steps.download-execution-results.outputs.execution-benchmark-results-pattern }} - IREE_BUILD_URL: https://github.com/iree-org/iree/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} - PR_NUMBER: ${{ github.event.pull_request.number }} - BENCHMARK_COMMENT_ARTIFACT: benchmark-comment.json - run: | - build_tools/github_actions/docker_run.sh \ - gcr.io/iree-oss/benchmark-report@sha256:7498c6f32f63f13faf085463cc38656d4297519c824e63e1c99c8c258147f6ff \ - ./build_tools/benchmarks/generate_benchmark_comment.py \ - --verbose \ - --pr_number="${PR_NUMBER}" \ - --pr_committish="${GITHUB_SHA}" \ - --pr_base_branch="origin/${GITHUB_BASE_REF}" \ - --comment_type="benchmark-summary" \ - --build_url="${IREE_BUILD_URL}" \ - --benchmark_files="${EXECUTION_BENCHMARK_RESULTS_PATTERN}" \ - --compile_stats_files="${COMPILE_STATS_RESULTS}" \ - --output="${BENCHMARK_COMMENT_ARTIFACT}" - echo "benchmark-comment-artifact=${BENCHMARK_COMMENT_ARTIFACT}" >> "${GITHUB_OUTPUT}" - - name: Uploading comment artifact - # Due to security reasons, instead of posting the comment to PR, we only - # upload the comment data in presubmit workflow and trigger the posting - # workflow on the main branch. See post_benchmark_comment.yaml - if: fromJson(needs.setup.outputs.is-pr) - env: - BENCHMARK_COMMENT_ARTIFACT: ${{ steps.generate-comment.outputs.benchmark-comment-artifact }} - BENCHMARK_COMMENT_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.generate-comment.outputs.benchmark-comment-artifact }} - run: | - gcloud storage cp \ - "${BENCHMARK_COMMENT_ARTIFACT}" \ - "${BENCHMARK_COMMENT_GCS_ARTIFACT}" - - name: Uploading results to dashboard - if: github.ref_name == 'main' - env: - EXECUTION_BENCHMARK_RESULTS_PATTERN: ${{ steps.download-execution-results.outputs.execution-benchmark-results-pattern }} - IREE_DASHBOARD_API_TOKEN: ${{ secrets.IREE_DASHBOARD_API_TOKEN }} - run: | - build_tools/github_actions/docker_run.sh \ - --env "IREE_DASHBOARD_API_TOKEN=${IREE_DASHBOARD_API_TOKEN}" \ - gcr.io/iree-oss/benchmark-report@sha256:7498c6f32f63f13faf085463cc38656d4297519c824e63e1c99c8c258147f6ff \ - ./build_tools/benchmarks/upload_benchmarks_to_dashboard.py \ - --verbose \ - --benchmark_files="${EXECUTION_BENCHMARK_RESULTS_PATTERN}" \ - --compile_stats_files="${COMPILE_STATS_RESULTS}" - - ############################################################################## - - # Depends on all the other jobs to provide a single anchor that indicates the - # final status. Status reporting will become more sophisticated in the future - # and we can hopefully avoid the need to explicitly list every single job... - benchmark_summary: - # Even if you have an explicit if condition, you still need to override - # GitHub's default behavior of not running if any dependencies failed. - if: always() - runs-on: ubuntu-20.04 - needs: - - setup - - build_for_benchmarks - - # Benchmark pipeline - - build_benchmark_tools - - build_e2e_test_artifacts - - test_benchmark_suites - - compilation_benchmarks - - execution_benchmarks - - process_benchmark_results - steps: - - name: "Checking out repository" - uses: actions/checkout@v4.1.7 - - name: Getting failed jobs - id: failed_jobs - run: | - echo '${{ toJson(needs) }}' - FAILED_JOBS="$(echo '${{ toJson(needs) }}' \ - | jq --raw-output \ - 'map_values(select(.result!="success" and .result!="skipped")) | keys | join(",")' \ - )" - echo "failed-jobs=${FAILED_JOBS}" >> $GITHUB_OUTPUT - if [[ "${FAILED_JOBS}" != "" ]]; then - echo "The following jobs failed: ${FAILED_JOBS}" - exit 1 - fi - - name: Show useful artifact links - if: always() - env: - # If the job of an artifact is skipped or failed, we show "NOT_PRESENT". - INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_for_benchmarks.outputs.install-dir-gcs-artifact || 'NOT_PRESENT' }} - E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir || 'NOT_PRESENT' }} - BENCHMARK_TOOLS_GCS_ARTIFACT_DIR: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir || 'NOT_PRESENT' }} - EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR: ${{ needs.execution_benchmarks.outputs.benchmark-results-gcs-artifact-dir || 'NOT_PRESENT' }} - COMPILATION_BENCHMARK_RESULTS_GCS_ARTIFACT: ${{ needs.compilation_benchmarks.outputs.compile-stats-results-gcs-artifact || 'NOT_PRESENT' }} - run: | - envsubst < ./.github/workflows/ARTIFACT_SUMMARY_TEMPLATE.md >> "${GITHUB_STEP_SUMMARY}" - - name: Posting to Discord - uses: sarisia/actions-status-discord@v1.14.3 - if: failure() && github.ref_name == 'main' - with: - webhook: ${{ secrets.DISCORD_WEBHOOK }} - description: "The following jobs failed: ${{ steps.failed_jobs.outputs.failed-jobs }}" - url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}" diff --git a/.github/workflows/benchmark_compilation.yml b/.github/workflows/benchmark_compilation.yml deleted file mode 100644 index df6e9c333e62..000000000000 --- a/.github/workflows/benchmark_compilation.yml +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Workflow for running compilation benchmarks. -# It is designed to be called from a parent workflow. -# The concurrency of this workflow is controlled by the caller's job. - -name: Benchmark Compilation - -on: - workflow_call: - inputs: - runner-group: - required: true - type: string - runner-env: - required: true - type: string - e2e-test-artifacts-dir: - required: true - type: string - e2e-test-artifacts-gcs-artifact-dir: - required: true - type: string - e2e-test-artifacts-build-log: - required: true - type: string - e2e-test-artifacts-build-log-gcs-artifact: - required: true - type: string - outputs: - compile-stats-results: - description: | - Local path to the compilation benchmark results. - value: ${{ jobs.compilation_benchmarks.outputs.compile-stats-results }} - compile-stats-results-gcs-artifact: - description: | - GCS path to the uploaded compilation benchmark results. - value: ${{ jobs.compilation_benchmarks.outputs.compile-stats-results-gcs-artifact }} - -permissions: - contents: read - -env: - # This duplicates the variable from benchmark.yml. The variable needs to be in env - # instead of the outputs of setup because it contains the run attempt and we - # want that to be the current attempt, not whatever attempt the setup step - # last ran in. It therefore can't be passed in via inputs because the env - # context isn't available there. - GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }} - -jobs: - compilation_benchmarks: - runs-on: - - self-hosted # must come first - - runner-group=${{ inputs.runner-group }} - - environment=${{ inputs.runner-env }} - - cpu - - os-family=Linux - env: - E2E_TEST_ARTIFACTS_DIR: ${{ inputs.e2e-test-artifacts-dir }} - E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR: ${{ inputs.e2e-test-artifacts-gcs-artifact-dir }} - E2E_TEST_ARTIFACTS_BUILD_LOG: ${{ inputs.e2e-test-artifacts-build-log }} - E2E_TEST_ARTIFACTS_BUILD_LOG_GCS_ARTIFACT: ${{ inputs.e2e-test-artifacts-build-log-gcs-artifact }} - outputs: - compile-stats-results: ${{ steps.collect.outputs.compile-stats-results }} - compile-stats-results-gcs-artifact: ${{ steps.upload.outputs.compile-stats-results-gcs-artifact }} - steps: - - name: "Checking out repository" - uses: actions/checkout@v4.1.7 - - name: "Downloading assets" - id: "download-assets" - env: - BENCHMARK_CONFIG: ${{ env.E2E_TEST_ARTIFACTS_DIR }}/compilation-benchmark-config.json - BENCHMARK_CONFIG_GCS_ARTIFACT: ${{ env.E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR }}/compilation-benchmark-config.json - run: | - mkdir -p "${E2E_TEST_ARTIFACTS_DIR}" - gcloud storage cp "${BENCHMARK_CONFIG_GCS_ARTIFACT}" "${BENCHMARK_CONFIG}" - gcloud storage cp \ - "${E2E_TEST_ARTIFACTS_BUILD_LOG_GCS_ARTIFACT}" \ - "${E2E_TEST_ARTIFACTS_BUILD_LOG}" - jq -r \ - --arg GCS_ARTIFACT_DIR "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}" \ - '.module_dir_paths | map("\($GCS_ARTIFACT_DIR)/\(.)") | join("\n")' \ - "${BENCHMARK_CONFIG}" | \ - gcloud storage cp -r --read-paths-from-stdin \ - "${E2E_TEST_ARTIFACTS_DIR}" - echo "benchmark-config=${BENCHMARK_CONFIG}" >> "${GITHUB_OUTPUT}" - - name: "Collecting compilation statistics" - id: collect - env: - BENCHMARK_CONFIG: ${{ steps.download-assets.outputs.benchmark-config }} - COMPILE_STATS_RESULTS: benchmark-results/compile-stats-results.json - run: | - mkdir -p benchmark-results - ./build_tools/benchmarks/collect_compilation_statistics.py \ - --e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR}" \ - --build_log="${E2E_TEST_ARTIFACTS_BUILD_LOG}" \ - --compilation_benchmark_config="${BENCHMARK_CONFIG}" \ - --output="${COMPILE_STATS_RESULTS}" - echo "compile-stats-results=${COMPILE_STATS_RESULTS}" >> "${GITHUB_OUTPUT}" - - name: "Uploading benchmark results" - id: upload - env: - COMPILE_STATS_RESULTS: ${{ steps.collect.outputs.compile-stats-results }} - COMPILE_STATS_RESULTS_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.collect.outputs.compile-stats-results }} - run: | - gcloud storage cp \ - "${COMPILE_STATS_RESULTS}" \ - "${COMPILE_STATS_RESULTS_GCS_ARTIFACT}" - echo "compile-stats-results-gcs-artifact=${COMPILE_STATS_RESULTS_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/benchmark_execution.yml b/.github/workflows/benchmark_execution.yml deleted file mode 100644 index baa1c57abbce..000000000000 --- a/.github/workflows/benchmark_execution.yml +++ /dev/null @@ -1,186 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Workflow for running execution benchmarks. -# It is designed to be called from a parent workflow. -# The concurrency of this workflow is controlled by the caller's job. - -name: Benchmark execution - -on: - workflow_call: - inputs: - runner-group: - required: true - type: string - runner-env: - required: true - type: string - e2e-test-artifacts-dir: - required: true - type: string - e2e-test-artifacts-gcs-artifact-dir: - required: true - type: string - benchmark-tools-gcs-artifact-dir: - required: true - type: string - outputs: - benchmark-results-dir: - description: | - Local path that stores all benchmark results. - Empty if no benchmark runs. - value: ${{ jobs.run_benchmarks.outputs.benchmark-results-dir }} - benchmark-results-gcs-artifact-dir: - description: | - GCS path that stores all benchmark results. - Empty if no benchmark runs. - value: ${{ jobs.run_benchmarks.outputs.benchmark-results-gcs-artifact-dir }} - -permissions: - contents: read - -env: - # This duplicates the variable from benchmark.yml. The variable needs to be in env - # instead of the outputs of setup because it contains the run attempt and we - # want that to be the current attempt, not whatever attempt the setup step - # last ran in. It therefore can't be passed in via inputs because the env - # context isn't available there. - GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }} - BENCHMARK_CONFIG: ${{ inputs.e2e-test-artifacts-dir }}/execution-benchmark-config.json - BENCHMARK_CONFIG_GCS_ARTIFACT: ${{ inputs.e2e-test-artifacts-gcs-artifact-dir }}/execution-benchmark-config.json - -jobs: - generate_matrix: - runs-on: - - self-hosted # must come first - - runner-group=${{ inputs.runner-group }} - - environment=${{ inputs.runner-env }} - - cpu - - os-family=Linux - outputs: - benchmark-matrix: ${{ steps.generate.outputs.benchmark-matrix }} - steps: - - name: "Checking out repository" - uses: actions/checkout@v4.1.7 - - name: "Checking out runtime submodules" - run: ./build_tools/scripts/git/update_runtime_submodules.sh - - name: "Generating benchmark matrix" - id: generate - run: | - gcloud storage cp "${BENCHMARK_CONFIG_GCS_ARTIFACT}" "${BENCHMARK_CONFIG}" - # This jq command takes a benchmark config with the following structure: - # - # { "target_device_name" : { - # "host_environment" : { ... }, - # "shards" : [ { ... }, { ... } ] - # }, - # "other_target_device_name" : { ... } - # - # and turns it into a flat list of benchmark jobs: - # - # [ - # { "device_name" : "target_device_name", - # "host_environment" : { ... }, - # "shard" : { index: 0, count: 2 } - # }, - # { "device_name" : "target_device_name", - # "host_environment" : { ... }, - # "shard" : { index: 1, count: 2 } - # }, - # { "device_name" : "other_target_device_name", - # "host_environment" : { ... }, - # "shard" : { index: 0, count: N } - # }, - # ... - # ] - echo benchmark-matrix="$(jq -c '[ . | to_entries[] - | .key as $device_name - | .value.host_environment as $host_environment - | (.value.shards | length) as $count - | .value.shards[] - | {$device_name, $host_environment, shard: {index, $count}} - ]' "${BENCHMARK_CONFIG}")" >> "${GITHUB_OUTPUT}" - - run_benchmarks: - needs: [generate_matrix] - if: needs.generate_matrix.outputs.benchmark-matrix != '[]' - strategy: - # Don't kill other benchmark jobs if one of them failed. - fail-fast: false - # Matrix is dynamically generated by the job generate_matrix. So we only - # runs the benchmarks specified in ${BENCHMARK_CONFIG}. - # All tasks in matrix are seen as a single job in Github CI and the job - # can output a single set of values. - matrix: - benchmark: ${{ fromJSON(needs.generate_matrix.outputs.benchmark-matrix) }} - runs-on: - - self-hosted # must come first - - runner-group=${{ inputs.runner-group }} - - environment=${{ inputs.runner-env }} - - machine-type=${{ matrix.benchmark.device_name }} - env: - DEVICE_NAME: ${{ matrix.benchmark.device_name }} - SHARD_INDEX: ${{ matrix.benchmark.shard.index }} - SHARD_COUNT: ${{ matrix.benchmark.shard.count }} - PLATFORM_ARCH: ${{ matrix.benchmark.host_environment.platform }}-${{ matrix.benchmark.host_environment.architecture }} - E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR: ${{ inputs.e2e-test-artifacts-gcs-artifact-dir }} - BENCHMARK_RESULTS_DIR: benchmark-results - outputs: - benchmark-results-dir: ${{ env.BENCHMARK_RESULTS_DIR }} - # Ideally this should be defined in env, so it can be used in the upload - # step. But Github CI doesn't allow us to access env.GCS in env. - benchmark-results-gcs-artifact-dir: ${{ env.GCS_DIR }}/${{ env.BENCHMARK_RESULTS_DIR }} - steps: - - name: "Checking out repository" - uses: actions/checkout@v4.1.7 - - name: "Checking out runtime submodules" - run: ./build_tools/scripts/git/update_runtime_submodules.sh - - name: "Downloading benchmark tools" - id: download-tools - env: - # See `build_benchmark_tools` step for the name format of benchmark tools artifacts. - BENCHMARK_TOOLS_ARCHIVE: ${{ env.PLATFORM_ARCH }}-benchmark-tools.tar - BENCHMARK_TOOLS_GCS_ARTIFACT: ${{ inputs.benchmark-tools-gcs-artifact-dir }}/${{ env.PLATFORM_ARCH }}-benchmark-tools.tar - run: | - gcloud storage cp "${BENCHMARK_TOOLS_GCS_ARTIFACT}" "${BENCHMARK_TOOLS_ARCHIVE}" - echo "benchmark-tools-archive=${BENCHMARK_TOOLS_ARCHIVE}" >> "${GITHUB_OUTPUT}" - - name: "Downloading benchmark assets" - id: download-assets - run: | - gcloud storage cp "${BENCHMARK_CONFIG_GCS_ARTIFACT}" "${BENCHMARK_CONFIG}" - echo "benchmark-config=${BENCHMARK_CONFIG}" >> "${GITHUB_OUTPUT}" - - name: "Unpacking benchmark tools" - id: unpack-tools - env: - BENCHMARK_TOOLS_ARCHIVE: ${{ steps.download-tools.outputs.benchmark-tools-archive }} - # See `build_benchmark_tools` step for the name format of benchmark tools directory. - BENCHMARK_TOOLS_DIR: ${{ env.PLATFORM_ARCH }}-benchmark-tools-dir - run: | - tar -xf ${BENCHMARK_TOOLS_ARCHIVE} - echo "benchmark-tools-dir=${BENCHMARK_TOOLS_DIR}/build/tools" >> "${GITHUB_OUTPUT}" - - name: "Determine Shard Suffix" - id: sharding - run: | - if (( SHARD_COUNT > 1 )); then - echo "suffix=$(printf -- "-%02d-of-%02d" "${SHARD_INDEX}" "${SHARD_COUNT}")" >> "${GITHUB_OUTPUT}" - else - echo "suffix=" >> "${GITHUB_OUTPUT}" - fi - - name: "Running benchmarks" - env: - IREE_EXECUTION_BENCHMARK_CONFIG: ${{ steps.download-assets.outputs.benchmark-config }} - IREE_DOCKER_WRAPPER: ./build_tools/github_actions/docker_run.sh - IREE_BENCHMARK_TOOLS_DIR: ${{ steps.unpack-tools.outputs.benchmark-tools-dir }} - IREE_TARGET_DEVICE_NAME: ${{ env.DEVICE_NAME }} - IREE_SHARD_INDEX: ${{ matrix.benchmark.shard.index }} - IREE_BENCHMARK_RESULTS: ${{ env.BENCHMARK_RESULTS_DIR }}/benchmark-results-${{ matrix.benchmark.device_name }}${{ steps.sharding.outputs.suffix }}.json - run: | - mkdir -p ${BENCHMARK_RESULTS_DIR} - export IREE_E2E_TEST_ARTIFACTS_DIR="${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR/gs:\/\//https://storage.googleapis.com/}" - ./build_tools/benchmarks/run_benchmarks.sh - - name: "Uploading benchmark results" - run: gcloud storage cp -r "${BENCHMARK_RESULTS_DIR}" "${GCS_DIR}/" diff --git a/.github/workflows/benchmark_large.yml b/.github/workflows/benchmark_large.yml deleted file mode 100644 index f512c9f6d4ea..000000000000 --- a/.github/workflows/benchmark_large.yml +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -# Executes large benchmarks that are expected to take multple hours. - -name: Benchmark Large - -on: - schedule: - # Scheduled to run at 09:00 UTC and 21:00 UTC. - - cron: "0 09,21 * * *" - workflow_dispatch: - inputs: - shard-count: - description: | - A device-name to integer mapping as a comma separated list. - Allows to assign a distinct shard count for each target device. - The reserved keyword `default` assigns a shard count to all target devices - that are not explicitly listed. - # Please keep this default value in sync with the jobs.build_e2e_test_artifacts.with.shard-count field below - default: a2-highgpu-1g=1,c2-standard-60=2,default=1 - type: string - -concurrency: - # A PR number if a pull request and otherwise the commit hash. This cancels - # queued and in-progress runs for the same PR (presubmit) or commit - # (postsubmit). The workflow name is prepended to avoid conflicts between - # different workflows. - group: ${{ github.workflow }}-${{ github.event.number || github.sha }} - cancel-in-progress: true - -env: - # This needs to be in env instead of the outputs of setup because it contains - # the run attempt and we want that to be the current attempt, not whatever - # attempt the setup step last ran in. - GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }} - -# Jobs are organized into groups and topologically sorted by dependencies -jobs: - setup: - uses: ./.github/workflows/setup.yml - - build_all: - needs: setup - if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_all') - uses: ./.github/workflows/build_all.yml - with: - runner-group: ${{ needs.setup.outputs.runner-group }} - runner-env: ${{ needs.setup.outputs.runner-env }} - write-caches: ${{ needs.setup.outputs.write-caches }} - run-tests: false - - build_benchmark_tools: - needs: [setup, build_all] - if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_benchmark_tools') - uses: ./.github/workflows/build_benchmark_tools.yml - with: - runner-group: ${{ needs.setup.outputs.runner-group }} - runner-env: ${{ needs.setup.outputs.runner-env }} - install-dir: ${{ needs.build_all.outputs.install-dir }} - install-dir-archive: ${{ needs.build_all.outputs.install-dir-archive }} - install-dir-gcs-artifact: ${{ needs.build_all.outputs.install-dir-gcs-artifact }} - - build_e2e_test_artifacts: - needs: [setup, build_all] - if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_e2e_test_artifacts') - uses: ./.github/workflows/build_e2e_test_artifacts.yml - with: - runner-group: ${{ needs.setup.outputs.runner-group }} - runner-env: ${{ needs.setup.outputs.runner-env }} - install-dir: ${{ needs.build_all.outputs.install-dir }} - install-dir-archive: ${{ needs.build_all.outputs.install-dir-archive }} - install-dir-gcs-artifact: ${{ needs.build_all.outputs.install-dir-gcs-artifact }} - benchmark-presets: cuda-large,comp-stats-large,x86_64-large - build-default-benchmark-suites: 0 - # Please keep the shard count default value in sync with on.workflow_dispatch.shard-count.default - shard-count: ${{ inputs && inputs.shard_count || 'a2-highgpu-1g=1,c2-standard-60=2,default=1' }} - - compilation_benchmarks: - needs: [setup, build_e2e_test_artifacts] - if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'compilation_benchmarks') - uses: ./.github/workflows/benchmark_compilation.yml - with: - runner-group: ${{ needs.setup.outputs.runner-group }} - runner-env: ${{ needs.setup.outputs.runner-env }} - e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }} - e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }} - e2e-test-artifacts-build-log: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log }} - e2e-test-artifacts-build-log-gcs-artifact: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log-gcs-artifact }} - - execution_benchmarks: - needs: [setup, build_benchmark_tools, build_e2e_test_artifacts] - if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'execution_benchmarks') - uses: ./.github/workflows/benchmark_execution.yml - with: - # env.GCS_DIR is also duplicated in this workflow. See the note there on - # why this is. - runner-group: ${{ needs.setup.outputs.runner-group }} - runner-env: ${{ needs.setup.outputs.runner-env }} - e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }} - e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }} - benchmark-tools-gcs-artifact-dir: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir }} - - benchmark_summary: - # Even if you have an explicit if condition, you still need to override - # GitHub's default behavior of not running if any dependencies failed. - if: always() - runs-on: ubuntu-20.04 - needs: - - setup - - build_all - - build_benchmark_tools - - build_e2e_test_artifacts - - compilation_benchmarks - - execution_benchmarks - steps: - - name: Getting failed jobs - id: failed_jobs - run: | - echo '${{ toJson(needs) }}' - FAILED_JOBS="$(echo '${{ toJson(needs) }}' \ - | jq --raw-output \ - 'map_values(select(.result!="success" and .result!="skipped")) | keys | join(",")' \ - )" - echo "failed-jobs=${FAILED_JOBS}" >> $GITHUB_OUTPUT - if [[ "${FAILED_JOBS}" != "" ]]; then - echo "The following jobs failed: ${FAILED_JOBS}" - exit 1 - fi - - name: Posting to Discord - uses: sarisia/actions-status-discord@v1.14.3 - if: failure() && github.ref_name == 'main' - with: - webhook: ${{ secrets.DISCORD_WEBHOOK }} - description: "The following jobs failed: ${{ steps.failed_jobs.outputs.failed-jobs }}" diff --git a/.github/workflows/benchmark_trigger.yml b/.github/workflows/benchmark_trigger.yml deleted file mode 100644 index 9fc3de193f5a..000000000000 --- a/.github/workflows/benchmark_trigger.yml +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -name: Benchmark Trigger - -# Trigger benchmark CI workflows when benchmark PR labels are added. -# When a benchmark label is added, this cancels and then re-runs the Benchmark -# workflow. When the workflow is re-run, it will pick up the latest PR -# description and labels. See -# https://github.com/iree-org/iree/issues/10042#issuecomment-1449250094 for -# more background. -# -# This workflow is running on the base branch the PR targets. - -on: - pull_request_target: - types: - - labeled - -env: - PR_NUMBER: ${{ github.event.pull_request.number }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - # Target workflow that runs the benchmarks. - WORKFLOW_NAME: Benchmark - BENCHMARK_LABEL_PREFIX: "benchmarks:" - -jobs: - # Precondition check is a separate step because we can't put the concurrency - # constraint on it; otherwise the irrelevant labeling events will cancel the - # events that meet the preconditions. Even with cancel-in-progress = false, - # the queued events will still be cancelled. - precondition: - runs-on: ubuntu-20.04 - outputs: - found-label: ${{ steps.precondition.outputs.found-label }} - triggered-at: ${{ steps.precondition.outputs.triggered-at }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: "Checking labels" - id: precondition - run: | - echo "found-label=$(jq \ - --arg label_prefix ${BENCHMARK_LABEL_PREFIX} \ - '.label.name | startswith($label_prefix)' \ - ${GITHUB_EVENT_PATH})" >> "${GITHUB_OUTPUT}" - # pull_request_target event doesn't have the workflow start time. Get - # the approximate start time at the beginning. - echo "triggered-at=$(date +%s)" >> "${GITHUB_OUTPUT}" - - trigger: - needs: precondition - if: fromJSON(needs.precondition.outputs.found-label) - runs-on: ubuntu-20.04 - # Required for cancel and rerun APIs. - permissions: - actions: write - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - concurrency: - # Only allows a single trigger to run for a PR concurrently. - # Timestamp-based check below makes sure we don't rerun benchmark twice - # when multiple label events happen the same time. We don't use - # `cancel-in-progress` to avoid that because cancelled jobs are - # considered as failures and show red on Github UI. - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} - steps: - - name: "Checking out repository" - # This checkouts from the base branch instead of the pull request. See - # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target - uses: actions/checkout@v4.1.7 - - - name: "Finding the previous workflow run" - id: find-workflow - env: - RUN_JSON: workflow-run.json - run: | - gh api "/repos/${GITHUB_REPOSITORY}/actions/runs" --method GET -F head_sha="${HEAD_SHA}" \ - | jq --arg name "${WORKFLOW_NAME}" '.workflow_runs | map(select(.name == $name))[0]' \ - > "${RUN_JSON}" - echo "workflow-url=$(jq '.url' ${RUN_JSON})" >> "${GITHUB_OUTPUT}" - echo "run-status=$(jq --raw-output '.status' ${RUN_JSON})" >> "${GITHUB_OUTPUT}" - echo "run-started-at=$(jq --raw-output '.run_started_at' ${RUN_JSON})" >> "${GITHUB_OUTPUT}" - - - name: "Checking if the workflow has been rerun" - id: check - if: fromJSON(steps.find-workflow.outputs.workflow-url) != null - env: - RUN_STARTED_AT: ${{ steps.find-workflow.outputs.run-started-at }} - TRIGGERED_AT: ${{ needs.precondition.outputs.triggered-at }} - run: | - # If the latest workflow run started after the trigger event, it means - # the workflow has been rerun and picked up the new labels. Skip rerun - # in this case. - RUN_STARTED_AT_EPOCH="$(date --date="${RUN_STARTED_AT}" +%s)" - if (( RUN_STARTED_AT_EPOCH < TRIGGERED_AT )); then - SHOULD_RERUN="true" - else - SHOULD_RERUN="false" - fi - echo "should-rerun=${SHOULD_RERUN}" >> "${GITHUB_OUTPUT}" - - cat <> "${GITHUB_OUTPUT}" - - - name: "Rerequesting the workflow" - if: | - fromJSON(steps.check.outputs.should-rerun) && - steps.get-sha.outputs.latest-sha == env.HEAD_SHA - env: - WORKFLOW_RUN_URL: ${{ fromJSON(steps.find-workflow.outputs.workflow-url) }} - run: | - gh api "${WORKFLOW_RUN_URL}/rerun" --method POST diff --git a/.github/workflows/build_benchmark_tools.yml b/.github/workflows/build_benchmark_tools.yml deleted file mode 100644 index 820c2831af51..000000000000 --- a/.github/workflows/build_benchmark_tools.yml +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Workflow for building benchmark tools. -# It is designed to be called from a parent workflow. -# The concurrency of this workflow is controlled by the caller's job. - -name: Build Benchmark Tools - -on: - workflow_call: - inputs: - runner-group: - required: true - type: string - runner-env: - required: true - type: string - install-dir: - required: true - type: string - install-dir-archive: - required: true - type: string - install-dir-gcs-artifact: - required: true - type: string - outputs: - benchmark-tools-gcs-artifact-dir: - description: | - GCS path to benchmark tools archive. - value: ${{ jobs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir }} - -permissions: - contents: read - -env: - # This duplicates the variable from benchmark.yml. The variable needs to be in env - # instead of the outputs of setup because it contains the run attempt and we - # want that to be the current attempt, not whatever attempt the setup step - # last ran in. It therefore can't be passed in via inputs because the env - # context isn't available there. - GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }} - -jobs: - build_benchmark_tools: - runs-on: - - self-hosted # must come first - - runner-group=${{ inputs.runner-group }} - - environment=${{ inputs.runner-env }} - - cpu - - os-family=Linux - outputs: - # We can't collect all outputs from the matrix jobs due to Github's - # limitation (https://github.com/orgs/community/discussions/17245). - # Therefore, the output is the GCS directory that stores all benchmark - # tools archives. The following jobs need to construct the archive names - # by themselves and combine with path of GCS directory here to fetch the - # archives. - benchmark-tools-gcs-artifact-dir: ${{ steps.upload.outputs.benchmark-tools-gcs-artifact-dir }} - strategy: - matrix: - target: - - platform: "linux" - arch: "x86_64" - docker_image: "gcr.io/iree-oss/base-bleeding-edge@sha256:cf2e78194e64fd0166f4141317366261d7a62432b72e9a324cb8c2ff4e1a515a" - # Builds tools on the host and assumes the builder is Linux x86_64. - build_script: "./build_tools/cmake/build_runtime.sh" - - platform: "linux" - arch: "riscv_64" - docker_image: "gcr.io/iree-oss/riscv@sha256:62e87bad3405d691ddba6f9be0ef44eeb60461a467c8d86f0842c81a1f97da79" - build_script: "./build_tools/cmake/build_riscv.sh" - - platform: "android" - arch: "armv8.2-a" - docker_image: "gcr.io/iree-oss/android@sha256:66b92a1c920588a73d03316f26025407ea754bab93e8f9bfe40dbf6ed5fe6c7e" - build_script: "./build_tools/cmake/build_android.sh" - env: - PLATFORM: ${{ matrix.target.platform }} - ARCH: ${{ matrix.target.arch }} - DOCKER_IMAGE: ${{ matrix.target.docker_image }} - BUILD_SCRIPT: ${{ matrix.target.build_script }} - BUILD_TOOLS_DIR: ${{ matrix.target.platform }}-${{ matrix.target.arch }}-benchmark-tools-dir - INSTALL_DIR: ${{ inputs.install-dir }} - INSTALL_DIR_ARCHIVE: ${{ inputs.install-dir-archive }} - INSTALL_DIR_GCS_ARTIFACT: ${{ inputs.install-dir-gcs-artifact }} - steps: - - name: "Checking out repository" - uses: actions/checkout@v4.1.7 - - name: "Checking out runtime submodules" - run: ./build_tools/scripts/git/update_runtime_submodules.sh - - name: "Downloading install dir archive" - run: gcloud storage cp "${INSTALL_DIR_GCS_ARTIFACT}" "${INSTALL_DIR_ARCHIVE}" - - name: "Extracting install directory" - run: tar -xf "${INSTALL_DIR_ARCHIVE}" - - name: "Compiling the benchmark tools" - id: build - run: | - ./build_tools/github_actions/docker_run.sh \ - --env "IREE_TARGET_PLATFORM=${PLATFORM}" \ - --env "IREE_TARGET_ARCH=${ARCH}" \ - --env "BUILD_PRESET=benchmark" \ - --env "IREE_HOST_BIN_DIR=${INSTALL_DIR}/bin" \ - "${DOCKER_IMAGE}" "${BUILD_SCRIPT}" "${BUILD_TOOLS_DIR}/build" - - name: "Compiling the benchmark tools with tracing" - id: build-with-tracing - run: | - ./build_tools/github_actions/docker_run.sh \ - --env "IREE_TARGET_PLATFORM=${PLATFORM}" \ - --env "IREE_TARGET_ARCH=${ARCH}" \ - --env "BUILD_PRESET=benchmark-with-tracing" \ - --env "IREE_HOST_BIN_DIR=${INSTALL_DIR}/bin" \ - "${DOCKER_IMAGE}" "${BUILD_SCRIPT}" "${BUILD_TOOLS_DIR}/build-traced" - - name: "Creating the benchmark tools archive" - id: archive - env: - BENCHMARK_TOOLS_ARCHIVE: ${{ env.PLATFORM }}-${{ env.ARCH }}-benchmark-tools.tar - run: | - tar -cf "${BENCHMARK_TOOLS_ARCHIVE}" \ - "${BUILD_TOOLS_DIR}"/*/tools/iree-benchmark-module \ - "${BUILD_TOOLS_DIR}"/*/tools/build_config.txt - echo "benchmark-tools-archive=${BENCHMARK_TOOLS_ARCHIVE}" >> "${GITHUB_OUTPUT}" - - name: "Uploading the benchmark tools archive" - id: upload - env: - BENCHMARK_TOOLS_ARCHIVE: ${{ steps.archive.outputs.benchmark-tools-archive }} - BENCHMARK_TOOLS_GCS_ARTIFACT_DIR: ${{ env.GCS_DIR }}/benchmark-tools - run: | - gcloud storage cp "${BENCHMARK_TOOLS_ARCHIVE}" "${BENCHMARK_TOOLS_GCS_ARTIFACT_DIR}/" - echo "benchmark-tools-gcs-artifact-dir=${BENCHMARK_TOOLS_GCS_ARTIFACT_DIR}" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/build_e2e_test_artifacts.yml b/.github/workflows/build_e2e_test_artifacts.yml deleted file mode 100644 index 99378b517335..000000000000 --- a/.github/workflows/build_e2e_test_artifacts.yml +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Workflow for building E2E test artifacts. -# It is designed to be called from a parent workflow. -# The concurrency of this workflow is controlled by the caller's job. - -name: Build E2E Test Artifacts - -on: - workflow_call: - inputs: - runner-group: - required: true - type: string - runner-env: - required: true - type: string - install-dir: - required: true - type: string - install-dir-archive: - required: true - type: string - install-dir-gcs-artifact: - required: true - type: string - benchmark-presets: - description: | - A comma-separated string of benchmark presets to build. - required: true - type: string - build-default-benchmark-suites: - description: | - Build default benchmark suites for sanity check and e2e model testing - if set to 1. - default: 1 - type: number - shard-count: - description: | - A device-name to integer mapping as a comma separated list. - Allows to assign a distinct shard count for each target device. - The reserved keyword `default` assigns a shard count to all target devices - that are not explicitly listed. - default: default=1 - type: string - outputs: - e2e-test-artifacts-dir: - description: | - Local path that stores compiled test artifacts. - value: ${{ jobs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }} - e2e-test-artifacts-gcs-artifact-dir: - description: | - GCS path to the uploaded test artifacts. - value: ${{ jobs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }} - e2e-test-artifacts-build-log: - description: | - Local path that contains the build log. - value: ${{ jobs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log }} - e2e-test-artifacts-build-log-gcs-artifact: - description: | - GCS path to the uploaded build log. - value: ${{ jobs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log-gcs-artifact }} - -env: - # This duplicates the variable from benchmark.yml. The variable needs to be in env - # instead of the outputs of setup because it contains the run attempt and we - # want that to be the current attempt, not whatever attempt the setup step - # last ran in. It therefore can't be passed in via inputs because the env - # context isn't available there. - GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }} - -jobs: - build_e2e_test_artifacts: - runs-on: - - self-hosted # must come first - - runner-group=${{ inputs.runner-group }} - - environment=${{ inputs.runner-env }} - - cpu - - os-family=Linux - env: - INSTALL_DIR: ${{ inputs.install-dir }} - INSTALL_DIR_ARCHIVE: ${{ inputs.install-dir-archive }} - INSTALL_DIR_GCS_ARTIFACT: ${{ inputs.install-dir-gcs-artifact }} - outputs: - e2e-test-artifacts-dir: ${{ steps.build.outputs.e2e-test-artifacts-dir }} - e2e-test-artifacts-gcs-artifact-dir: ${{ steps.upload.outputs.e2e-test-artifacts-gcs-artifact-dir }} - e2e-test-artifacts-build-log: ${{ steps.build.outputs.e2e-test-artifacts-build-log }} - e2e-test-artifacts-build-log-gcs-artifact: ${{ steps.upload.outputs.e2e-test-artifacts-build-log-gcs-artifact }} - steps: - - name: "Checking out repository" - uses: actions/checkout@v4.1.7 - - name: "Checking out runtime submodules" - run: ./build_tools/scripts/git/update_runtime_submodules.sh - - name: "Downloading install dir archive" - run: gcloud storage cp "${INSTALL_DIR_GCS_ARTIFACT}" "${INSTALL_DIR_ARCHIVE}" - - name: "Extracting install directory" - run: tar -xf "${INSTALL_DIR_ARCHIVE}" - - name: "Building e2e test artifacts" - id: build - env: - IREE_BENCHMARK_PRESETS: ${{ inputs.benchmark-presets }} - IREE_BUILD_DEFAULT_BENCHMARK_SUITES: ${{ inputs.build-default-benchmark-suites }} - BUILD_E2E_TEST_ARTIFACTS_DIR: build-e2e-test-artifacts - IREE_SHARD_COUNT: ${{ inputs.shard-count }} - run: | - echo "e2e-test-artifacts-dir=${BUILD_E2E_TEST_ARTIFACTS_DIR}/e2e_test_artifacts" >> "${GITHUB_OUTPUT}" - echo "e2e-test-artifacts-build-log=${BUILD_E2E_TEST_ARTIFACTS_DIR}/.ninja_log" >> "${GITHUB_OUTPUT}" - build_tools/github_actions/docker_run.sh \ - --env "IREE_HOST_BIN_DIR=${INSTALL_DIR}/bin" \ - --env "IREE_BENCHMARK_PRESETS=${IREE_BENCHMARK_PRESETS}" \ - --env "IREE_BUILD_DEFAULT_BENCHMARK_SUITES=${IREE_BUILD_DEFAULT_BENCHMARK_SUITES}" \ - --env "IREE_SHARD_COUNT=${IREE_SHARD_COUNT}" \ - gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \ - build_tools/cmake/build_e2e_test_artifacts.sh \ - "${BUILD_E2E_TEST_ARTIFACTS_DIR}" - - name: "Uploading e2e test artifacts" - id: upload - if: ${{ !cancelled() }} - env: - E2E_TEST_ARTIFACTS_DIR: ${{ steps.build.outputs.e2e-test-artifacts-dir }} - E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR: ${{ env.GCS_DIR }}/e2e-test-artifacts - E2E_TEST_ARTIFACTS_BUILD_LOG: ${{ steps.build.outputs.e2e-test-artifacts-build-log }} - E2E_TEST_ARTIFACTS_BUILD_LOG_GCS_ARTIFACT: ${{ env.GCS_DIR }}/e2e-test-artifacts/ninja_log - run: | - # TODO(#13677): MLIRs of the versioned MLIR source models are fetched - # from GCS and they won't change. We can upload links instead of files - # to save space. - # Uploads all artifacts except for the `*.tflite`, which are the - # source models we imported into MLIR, and empty directories (GCS - # doesn't accept them). - # Not archiving the directory to allow fetching each file as needed - # separately. `find` depth is limited at 1 to exclude the directory - # itself and list all files and subdirectories at the first level. Let - # gcloud cli handle the recursive file traversal. - find "${E2E_TEST_ARTIFACTS_DIR}" -mindepth 1 -maxdepth 1 ! -name "*.tflite" ! -empty \ - | gcloud storage cp --read-paths-from-stdin -r \ - "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}" - gcloud storage cp "${E2E_TEST_ARTIFACTS_BUILD_LOG}" \ - "${E2E_TEST_ARTIFACTS_BUILD_LOG_GCS_ARTIFACT}" - echo "e2e-test-artifacts-gcs-artifact-dir=${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}" >> "${GITHUB_OUTPUT}" - echo "e2e-test-artifacts-build-log-gcs-artifact=${E2E_TEST_ARTIFACTS_BUILD_LOG_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5b45e8d30e7..7bbdf67444fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -368,7 +368,6 @@ jobs: # --env "IREE_TARGET_ARCH=${ARCH}" \ # --env "IREE_TARGET_ABI=${ABI}" \ # --env "IREE_TARGET_BUILD_DIR=${TARGET_BUILD_DIR}" \ - # --env "BUILD_PRESET=test" \ # --env "IREE_HOST_BIN_DIR=${INSTALL_DIR}/bin" \ # "${DOCKER_IMAGE}" \ # "${BUILD_SCRIPT}" @@ -379,7 +378,6 @@ jobs: # --env "IREE_TARGET_PLATFORM=${PLATFORM}" \ # --env "IREE_TARGET_ARCH=${ARCH}" \ # --env "IREE_TARGET_BUILD_DIR=${TARGET_BUILD_DIR}" \ - # --env "BUILD_PRESET=test" \ # "${DOCKER_IMAGE}" \ # "${TEST_SCRIPT}" diff --git a/.github/workflows/pkgci_test_android.yml b/.github/workflows/pkgci_test_android.yml index 0fb63efe85c8..b297f0c92390 100644 --- a/.github/workflows/pkgci_test_android.yml +++ b/.github/workflows/pkgci_test_android.yml @@ -78,7 +78,6 @@ jobs: IREE_WRITE_REMOTE_CCACHE: 0 IREE_READ_LOCAL_CCACHE: 1 IREE_WRITE_LOCAL_CCACHE: ${{ inputs.write-caches }} - BUILD_PRESET: test run: build_tools/cmake/build_android.sh # TODO: run tests on physical devices or emulators diff --git a/.github/workflows/post_benchmark_comment.yaml b/.github/workflows/post_benchmark_comment.yaml deleted file mode 100644 index 7d4dfb3c728a..000000000000 --- a/.github/workflows/post_benchmark_comment.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -name: Post benchmark comment - -on: - workflow_run: - # TODO(#17001): switch to just "Benchmark" after finishing refactor and - # pending PRs have been rebased - workflows: ["CI", "Benchmark"] - types: [completed] - -env: - PR_CI_STAGE: ${{ github.event.workflow_run.event == 'pull_request' && 'presubmit' || 'postsubmit' }} - PR_CI_RUN_ID: ${{ github.event.workflow_run.id }} - PR_CI_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - -jobs: - post_comment: - # Only run the workflow if it's triggered from a pull request. - if: github.event.workflow_run.event == 'pull_request' - runs-on: ubuntu-20.04 - permissions: - issues: write - pull-requests: write - steps: - - name: "Checking out repository" - uses: actions/checkout@v4.1.7 - - name: "Checking benchmark processing job" - id: check - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BENCHMARK_PROCESS_JOB: process_benchmark_results - run: | - # Assume the workflow has fewer than 100 jobs. - export CONCLUSION=$(gh api \ - "/repos/${GITHUB_REPOSITORY}/actions/runs/${PR_CI_RUN_ID}/attempts/${PR_CI_RUN_ATTEMPT}/jobs?per_page=100" \ - | jq --raw-output --arg job "${BENCHMARK_PROCESS_JOB}" \ - '.jobs | map(select(.name==$job))[0] | .conclusion') - echo "job-conclusion=${CONCLUSION}" >> "${GITHUB_OUTPUT}" - - name: "Downloading artifacts" - if: steps.check.outputs.job-conclusion == 'success' - id: download - env: - # Currently GitHub managed runner doesn't have GCP credential. Fetch - # from public URL directly. - PR_CI_GCS_URL: https://storage.googleapis.com/iree-github-actions-${{ env.PR_CI_STAGE }}-artifacts/${{ env.PR_CI_RUN_ID }}/${{ env.PR_CI_RUN_ATTEMPT }} - BENCHMARK_COMMENT_ARTIFACT: benchmark-comment.json - run: | - wget -O "${BENCHMARK_COMMENT_ARTIFACT}" \ - "${PR_CI_GCS_URL}/${BENCHMARK_COMMENT_ARTIFACT}" - echo "benchmark-comment-artifact=${BENCHMARK_COMMENT_ARTIFACT}" >> "${GITHUB_OUTPUT}" - - name: Posting comments - if: steps.check.outputs.job-conclusion == 'success' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GIST_BOT_TOKEN: ${{ secrets.GIST_BOT_TOKEN }} - COMMENT_BOT_USER: github-actions[bot] - # Get PR number from the event instead of the untrusted comment - # artifact, to make sure we won't incorrectly update other PRs. - BENCHMARK_COMMENT_ARTIFACT: ${{ steps.download.outputs.benchmark-comment-artifact }} - run: | - ./build_tools/benchmarks/post_benchmark_comment.py \ - --verbose \ - --github_event_json="${GITHUB_EVENT_PATH}" \ - "${BENCHMARK_COMMENT_ARTIFACT}" diff --git a/.github/workflows/setup.yml b/.github/workflows/setup.yml index 67403efe9646..49f7287d8bed 100644 --- a/.github/workflows/setup.yml +++ b/.github/workflows/setup.yml @@ -33,10 +33,6 @@ on: description: | Whether to write to caches. value: ${{ jobs.setup.outputs.write-caches }} - benchmark-presets: - description: | - A comma-separated string of benchmarks to run. - value: ${{ jobs.setup.outputs.benchmark-presets }} permissions: contents: read @@ -54,7 +50,6 @@ jobs: runner-env: ${{ steps.configure.outputs.runner-env }} runner-group: ${{ steps.configure.outputs.runner-group }} write-caches: ${{ steps.configure.outputs.write-caches }} - benchmark-presets: ${{ steps.configure.outputs.benchmark-presets }} steps: - name: "Checking out repository" uses: actions/checkout@v4.1.7 @@ -107,10 +102,7 @@ jobs: - name: "Show enabled options" env: - BENCHMARK_PRESETS: ${{ steps.configure.outputs.benchmark-presets || 'None' }} ENABLED_JOBS: ${{ join(fromJson(steps.configure.outputs.enabled-jobs)) || 'None' }} run: | echo ":green_circle: Enabled jobs: \`${ENABLED_JOBS}\`" \ >> "${GITHUB_STEP_SUMMARY}" - echo ":stopwatch: Enabled benchmarks: \`${BENCHMARK_PRESETS}\`" \ - >> "${GITHUB_STEP_SUMMARY}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da192e1e859e..1d2402e7a9bf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,64 +5,63 @@ exclude: "third_party/" repos: - -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.2.0 hooks: - - id: check-merge-conflict + - id: check-merge-conflict - - id: check-yaml + - id: check-yaml # * Extensions can't be included in the mkdocs schema, so skip checking # https://github.com/squidfunk/mkdocs-material/issues/6378 # * clang-format files use `---` to split for multiple languages, # resulting in errors like `expected a single document in the stream` exclude: "mkdocs.yml|.clang-format" - - id: end-of-file-fixer + - id: end-of-file-fixer exclude_types: ["image", "jupyter"] - - id: trailing-whitespace + - id: trailing-whitespace exclude_types: ["image", "jupyter"] -- repo: https://github.com/psf/black + - repo: https://github.com/psf/black rev: 23.3.0 hooks: - - id: black + - id: black name: Run Black to format Python files -- repo: https://github.com/pre-commit/mirrors-clang-format + - repo: https://github.com/pre-commit/mirrors-clang-format # Loosely track the most recent versions in # * Runner images: https://github.com/actions/runner-images/ # * Editor extensions: https://github.com/microsoft/vscode-cpptools rev: v18.1.3 hooks: - - id: clang-format + - id: clang-format name: Run clang-format on C/C++/etc. files exclude_types: ["jupyter"] -- repo: https://github.com/igorshubovych/markdownlint-cli + - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.41.0 hooks: - - id: markdownlint + - id: markdownlint name: Run markdownlint on .md files args: ["--config", "docs/.markdownlint.yml"] files: "docs/website/.*.md" exclude: "mlir-dialects/!(index).md" -- repo: https://github.com/Lucas-C/pre-commit-hooks + - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.5.5 hooks: - - id: forbid-tabs + - id: forbid-tabs exclude: ".gitmodules|Makefile" -- repo: https://github.com/jlebar/pre-commit-hooks.git + - repo: https://github.com/jlebar/pre-commit-hooks.git rev: f2d115a052860b09b2888b4f104be614bf3b4779 hooks: - - id: do-not-submit + - id: do-not-submit -- repo: local + - repo: local hooks: - - id: buildifier + - id: buildifier name: Run buildifier entry: buildifier language: golang @@ -72,7 +71,10 @@ repos: # of improving it or at least explaining it as hundreds of developers # search for basic support: https://github.com/golang/go/issues/35732. # Docs are technically at https://go.dev/ref/mod#go-install ¯\_(ツ)_/¯ - additional_dependencies: ["github.com/bazelbuild/buildtools/buildifier@1429e15ae755a6762d0edf9198062dc6ed04408d"] + additional_dependencies: + [ + "github.com/bazelbuild/buildtools/buildifier@1429e15ae755a6762d0edf9198062dc6ed04408d", + ] files: '^(.*/)?(BUILD\.bazel|BUILD|WORKSPACE|WORKSPACE\.bazel|WORKSPACE\.bzlmod|MODULE\.bazel)$|\.BUILD$|\.bzl$' # Pin the language version so other system version are _not_ used. # Older go versions used different syntax for "go install" (which is @@ -81,7 +83,7 @@ repos: # probably changed again in a future version, whatever. language_version: "1.16" - - id: bazel_to_cmake + - id: bazel_to_cmake name: Run bazel_to_cmake.py language: python # Note: this passes file names to the tool. The tool can also be run @@ -90,35 +92,19 @@ repos: # Keep the top level directories here in sync with .bazel_to_cmake.cfg.py. files: '^(compiler|runtime|samples|tests|tools)/(.*/)?(BUILD\.bazel|CMakeLists.txt)$' - - id: check_path_lengths + - id: check_path_lengths name: Check for excessively long path lengths language: fail entry: Path lengths relative to the root should be < 75 characters (run ./build_tools/scripts/check_path_lengths.py for detailed output) # The regex includes/excludes here should roughly match the behavior of # the check_path_lengths.py script. files: '^compiler/.{66,}/\w+\.' - exclude: 'test/' + exclude: "test/" - - id: build_file_names + - id: build_file_names name: Check Bazel file names entry: Files should be named BUILD.bazel instead of BUILD language: fail files: "BUILD$" # TODO(scotttodd): mypy type checking for Python (https://mypy-lang.org/) - - - id: generate_cmake_e2e_test_artifacts_suite - name: Run generate_cmake_e2e_test_artifacts_suite.py - language: python - entry: ./build_tools/testing/generate_cmake_e2e_test_artifacts_suite.py - args: ["--output_dir", "./tests/e2e/test_artifacts"] - files: '^(build_tools/python|build_tools/testing|tests/e2e/test_artifacts)/.*' - pass_filenames: false - - - id: generate_cmake_e2e_model_tests - name: Run generate_cmake_e2e_model_tests.py - language: python - entry: ./build_tools/testing/generate_cmake_e2e_model_tests.py - args: ["--output", "./tests/e2e/stablehlo_models/generated_e2e_model_tests.cmake"] - files: '^(build_tools/python|build_tools/testing|tests/e2e/stablehlo_models)/.*' - pass_filenames: false diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c36350e402b..c42e006168a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,15 +102,6 @@ option(IREE_BUILD_BINDINGS_TFLITE_JAVA "Builds the IREE TFLite Java bindings wit option(IREE_BUILD_ALL_CHECK_TEST_MODULES "Builds all modules for iree_check_test, regardless of which would be tested" ON) -option(IREE_BUILD_E2E_TEST_ARTIFACTS "Builds IREE E2E test artifacts suite generated by the e2e test framework." "${IREE_BUILD_EXPERIMENTAL_E2E_TEST_ARTIFACTS}") -set(IREE_E2E_TEST_ARTIFACTS_DIR "" CACHE STRING "Set external prebuilt E2E test artifacts directory") -if(IREE_BUILD_E2E_TEST_ARTIFACTS AND NOT "${IREE_E2E_TEST_ARTIFACTS_DIR}" STREQUAL "") - message(SEND_ERROR - "IREE_E2E_TEST_ARTIFACTS_DIR and IREE_BUILD_E2E_TEST_ARTIFACTS can't be" - " set at the same time." - ) -endif() - option(IREE_ENABLE_COLLECTIVE_RUNTIME_TESTS "Enable runtime tests for collective operations." OFF) # For development, builds LLVM (and in the future) the whole compiler as @@ -568,11 +559,8 @@ include(iree_check_test) include(iree_e2e_generated_runner_test) include(iree_native_test) include(iree_cc_binary_benchmark) -include(iree_benchmark_suite) include(iree_hal_cts_test_suite) include(iree_static_linker_test) -include(iree_fetch_artifact) -include(iree_run_module_test) include(iree_plugin_register) # Default any sub-tree which doesn't provide its own package namespacing @@ -956,23 +944,6 @@ endif() add_subdirectory(build_tools/embed_data/) -if(IREE_BUILD_E2E_TEST_ARTIFACTS) - find_program(IREE_IMPORT_TFLITE_PATH iree-import-tflite) - if(IREE_IMPORT_TFLITE_PATH) - message(STATUS "Found ${IREE_IMPORT_TFLITE_PATH} to generate benchmark artifacts") - else() - message(STATUS "iree-import-tflite not found. Some benchmarks may not configure") - endif() - find_program(IREE_IMPORT_TF_PATH iree-import-tf) - if(IREE_IMPORT_TF_PATH) - message(STATUS "Found ${IREE_IMPORT_TF_PATH} to generate benchmark artifacts") - else() - message(STATUS "iree-import-tf not found. Some benchmarks may not configure") - endif() - - set(IREE_E2E_TEST_ARTIFACTS_DIR "${IREE_BINARY_DIR}/e2e_test_artifacts") -endif() - # tools/ can depend on compiler/ and runtime/. # Note: tools sub directory is added before compiler/ so that phony targets for # files with the same names from different rules are disambiguated towards @@ -1033,9 +1004,7 @@ set(IREE_PUBLIC_INCLUDE_DIRS "${IREE_COMMON_INCLUDE_DIRS}" # IREE build tools #------------------------------------------------------------------------------- -add_subdirectory(build_tools/benchmarks) add_subdirectory(build_tools/github_actions) -add_subdirectory(build_tools/python) #------------------------------------------------------------------------------- # Optional features diff --git a/build_tools/benchmarks/CMakeLists.txt b/build_tools/benchmarks/CMakeLists.txt deleted file mode 100644 index 851f552f2de6..000000000000 --- a/build_tools/benchmarks/CMakeLists.txt +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -set(BENCHMARKS_TOOL_PYTHON_DIR "${CMAKE_CURRENT_SOURCE_DIR}") - -# benchmark_tool_py_test() -# -# CMake function to test benchmark python tools. -# -# Parameters: -# NAME: name of test -# SRC: Test source file -# ARGS: Command line arguments to the Python source file. -# LABELS: Additional labels to apply to the test. The package path is added -# automatically. -function(benchmark_tool_py_test) - cmake_parse_arguments( - _RULE - "" - "NAME;SRC" - "ARGS;LABELS" - ${ARGN} - ) - - iree_build_tools_py_test( - NAME - "${_RULE_NAME}" - SRC - "${_RULE_SRC}" - ARGS - ${_RULE_ARGS} - LABELS - ${_RULE_LABELS} - PACKAGE_DIRS - ${BENCHMARKS_TOOL_PYTHON_DIR} - ) -endfunction() - -add_subdirectory(common) - -benchmark_tool_py_test( - NAME - collect_compilation_statistics_test - SRC - "collect_compilation_statistics_test.py" -) - -benchmark_tool_py_test( - NAME - export_benchmark_config_test - SRC - "export_benchmark_config_test.py" -) - -benchmark_tool_py_test( - NAME - post_benchmark_comment_test - SRC - "post_benchmark_comment_test.py" -) diff --git a/build_tools/benchmarks/README.md b/build_tools/benchmarks/README.md deleted file mode 100644 index 819f5f14898e..000000000000 --- a/build_tools/benchmarks/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# IREE Benchmark Tools - -See -[Benchmark suites](https://iree.dev/developers/performance/benchmark-suites/) -to learn how to run benchmarks locally with the tools under this directory. diff --git a/build_tools/benchmarks/benchmark_helper.py b/build_tools/benchmarks/benchmark_helper.py deleted file mode 100755 index 44c6893cf4be..000000000000 --- a/build_tools/benchmarks/benchmark_helper.py +++ /dev/null @@ -1,291 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Miscellaneous tool to help work with benchmark suite and benchmark CI.""" - -import pathlib -import sys - -# Add build_tools python dir to the search path. -sys.path.insert(0, str(pathlib.Path(__file__).parent.with_name("python"))) - -import argparse -import json -import os -import shlex -import subprocess -from typing import Dict, List, Optional, Sequence -import functools - -from e2e_test_artifacts import model_artifacts, iree_artifacts -from e2e_test_framework import serialization -from e2e_test_framework.definitions import iree_definitions - -IREE_COMPILER_NAME = "iree-compile" - - -def _convert_to_cmd_string(cmds: Sequence[str]) -> str: - if os.name == "nt": - # list2cmdline is an undocumented method for Windows command lines. Python - # doesn't provide an official method for quoting Windows command lines and - # the correct implementation is slightly non-trivial. Use the undocumented - # method for now and can be rewritten with our own implementation later. - # See https://learn.microsoft.com/en-us/archive/blogs/twistylittlepassagesallalike/everyone-quotes-command-line-arguments-the-wrong-way - return subprocess.list2cmdline(cmds) - - return " ".join(shlex.quote(cmd) for cmd in cmds) - - -def _dump_cmds_of_generation_config( - gen_config: iree_definitions.ModuleGenerationConfig, - root_path: pathlib.PurePath = pathlib.PurePath(), -): - imported_model = gen_config.imported_model - imported_model_path = iree_artifacts.get_imported_model_path( - imported_model=imported_model, root_path=root_path - ) - module_dir_path = iree_artifacts.get_module_dir_path( - module_generation_config=gen_config, root_path=root_path - ) - module_path = module_dir_path / iree_artifacts.MODULE_FILENAME - compile_cmds = [ - IREE_COMPILER_NAME, - str(imported_model_path), - "-o", - str(module_path), - ] - compile_cmds += gen_config.materialize_compile_flags( - module_dir_path=module_dir_path - ) - compile_cmd_str = _convert_to_cmd_string(compile_cmds) - - if imported_model.import_config.tool == iree_definitions.ImportTool.NONE: - import_cmd_str = "# (Source model is already in MLIR)" - else: - source_model_path = model_artifacts.get_model_path( - model=imported_model.model, root_path=root_path - ) - import_cmds = [ - imported_model.import_config.tool.value, - str(source_model_path), - "-o", - str(imported_model_path), - ] - import_cmds += imported_model.import_config.materialize_import_flags( - model=imported_model.model - ) - import_cmd_str = _convert_to_cmd_string(import_cmds) - - # Insert a blank line after each command to help read with line wrap. - return ["Compile Module:", compile_cmd_str, "", "Import Model:", import_cmd_str, ""] - - -def _dump_cmds_from_run_config( - run_config: iree_definitions.E2EModelRunConfig, - root_path: pathlib.PurePath = pathlib.PurePath(), -): - gen_config = run_config.module_generation_config - module_path = ( - iree_artifacts.get_module_dir_path( - module_generation_config=gen_config, root_path=root_path - ) - / iree_artifacts.MODULE_FILENAME - ) - - run_cmds = [run_config.tool.value, f"--module={module_path}"] - run_cmds += run_config.materialize_run_flags() - # Insert a blank line after the command to help read with line wrap. - lines = ["Run Module:", _convert_to_cmd_string(run_cmds), ""] - lines += _dump_cmds_of_generation_config(gen_config=gen_config, root_path=root_path) - return lines - - -def _dump_cmds_handler( - e2e_test_artifacts_dir: pathlib.Path, - execution_benchmark_config: Optional[pathlib.Path], - compilation_benchmark_config: Optional[pathlib.Path], - benchmark_id: Optional[str], - **_unused_args, -): - lines = [] - - if execution_benchmark_config is not None: - benchmark_groups = json.loads(execution_benchmark_config.read_text()) - for target_device, benchmark_group in benchmark_groups.items(): - shard_count = len(benchmark_group["shards"]) - for shard in benchmark_group["shards"]: - run_configs = serialization.unpack_and_deserialize( - data=shard["run_configs"], - root_type=List[iree_definitions.E2EModelRunConfig], - ) - for run_config in run_configs: - if ( - benchmark_id is not None - and benchmark_id != run_config.composite_id - ): - continue - - lines.append("################") - lines.append("") - lines.append(f"Execution Benchmark ID: {run_config.composite_id}") - lines.append(f"Name: {run_config}") - lines.append(f"Target Device: {target_device}") - lines.append(f"Shard: {shard['index']} / {shard_count}") - lines.append("") - lines += _dump_cmds_from_run_config( - run_config=run_config, root_path=e2e_test_artifacts_dir - ) - - if compilation_benchmark_config is not None: - benchmark_config = json.loads(compilation_benchmark_config.read_text()) - gen_configs = serialization.unpack_and_deserialize( - data=benchmark_config["generation_configs"], - root_type=List[iree_definitions.ModuleGenerationConfig], - ) - for gen_config in gen_configs: - if benchmark_id is not None and benchmark_id != gen_config.composite_id: - continue - - lines.append("################") - lines.append("") - lines.append(f"Compilation Benchmark ID: {gen_config.composite_id}") - lines.append(f"Name: {gen_config}") - lines.append("") - lines += _dump_cmds_of_generation_config( - gen_config=gen_config, root_path=e2e_test_artifacts_dir - ) - - print(*lines, sep="\n") - - -# Represents a benchmark results file with the data already loaded from a JSON file. -class JSONBackedBenchmarkData: - def __init__(self, source_filepath: pathlib.PurePath, data: Dict): - if not isinstance(data, dict): - raise ValueError( - f"'{source_filepath}' seems not to be a valid benchmark-results-file (No JSON struct as root element)." - ) - if "commit" not in data: - raise ValueError( - f"'{source_filepath}' seems not to be a valid benchmark-results-file ('commit' field not found)." - ) - if "benchmarks" not in data: - raise ValueError( - f"'{source_filepath}' seems not to be a valid benchmark-results-file ('benchmarks' field not found)." - ) - - self.source_filepath: pathlib.PurePath = source_filepath - self.data: Dict = data - - # Parses a JSON benchmark results file and makes some sanity checks - @staticmethod - def load_from_file(filepath: pathlib.Path): - try: - data = json.loads(filepath.read_bytes()) - except json.JSONDecodeError as e: - raise ValueError(f"'{filepath}' seems not to be a valid JSON file: {e.msg}") - return JSONBackedBenchmarkData(filepath, data) - - # A convenience wrapper around `loadFromFile` that accepts a sequence of paths and returns a sequence of JSONBackedBenchmarkData objects as a generator. - @staticmethod - def load_many_from_files(filepaths: Sequence[pathlib.Path]): - return ( - JSONBackedBenchmarkData.load_from_file(filepath) for filepath in filepaths - ) - - -# Merges the benchmark results from `right` into `left` and returns the updated `left` -def _merge_two_resultsets( - left: JSONBackedBenchmarkData, right: JSONBackedBenchmarkData -) -> JSONBackedBenchmarkData: - if left.data["commit"] != right.data["commit"]: - raise ValueError( - f"'{right.source_filepath}' and the previous files are based on different commits ({left.data['commit']} != {right.data['commit']}). Merging not supported." - ) - left.data["benchmarks"].extend(right.data["benchmarks"]) - return left - - -def merge_results(benchmark_results: Sequence[JSONBackedBenchmarkData]): - return functools.reduce(_merge_two_resultsets, benchmark_results) - - -def _merge_results_handler( - benchmark_results_files: Sequence[pathlib.Path], **_unused_args -): - print( - json.dumps( - merge_results( - JSONBackedBenchmarkData.load_many_from_files(benchmark_results_files) - ) - ) - ) - - -def _parse_arguments() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Miscellaneous tool to help work with benchmark suite and benchmark CI." - ) - - subparser = parser.add_subparsers( - required=True, title="operation", dest="operation" - ) - dump_cmds_parser = subparser.add_parser( - "dump-cmds", help="Dump the commands to compile and run benchmarks manually." - ) - dump_cmds_parser.add_argument( - "--e2e_test_artifacts_dir", - type=pathlib.PurePath, - default=pathlib.Path(), - help="E2E test artifacts root path used in the outputs of artifact paths", - ) - dump_cmds_parser.add_argument( - "--benchmark_id", type=str, help="Only dump the benchmark with this id" - ) - dump_cmds_parser.add_argument( - "--execution_benchmark_config", - type=pathlib.Path, - help="Config file exported from export_benchmark_config.py execution", - ) - dump_cmds_parser.add_argument( - "--compilation_benchmark_config", - type=pathlib.Path, - help="Config file exported from export_benchmark_config.py compilation", - ) - dump_cmds_parser.set_defaults(handler=_dump_cmds_handler) - - merge_results_parser = subparser.add_parser( - "merge-results", - help="Merges the results from multiple benchmark results JSON files into a single JSON structure.", - ) - merge_results_parser.add_argument( - "benchmark_results_files", - type=pathlib.Path, - nargs="+", - help="One or more benchmark results JSON file paths", - ) - merge_results_parser.set_defaults(handler=_merge_results_handler) - - args = parser.parse_args() - if ( - args.operation == "dump-cmds" - and args.execution_benchmark_config is None - and args.compilation_benchmark_config is None - ): - parser.error( - "At least one of --execution_benchmark_config or " - "--compilation_benchmark_config must be set." - ) - - return args - - -def main(args: argparse.Namespace): - args.handler(**vars(args)) - - -if __name__ == "__main__": - main(_parse_arguments()) diff --git a/build_tools/benchmarks/benchmark_helper_test.py b/build_tools/benchmarks/benchmark_helper_test.py deleted file mode 100644 index 9086d3c1b81f..000000000000 --- a/build_tools/benchmarks/benchmark_helper_test.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import json -import unittest -import benchmark_helper -import tempfile -import pathlib - - -class BenchmarkHelperTest(unittest.TestCase): - def test_merge_results_simple(self): - first = benchmark_helper.JSONBackedBenchmarkData( - pathlib.Path("first.json"), - { - "commit": "123", - "benchmarks": [{"benchmark_id": "first1"}, {"benchmark_id": "first2"}], - }, - ) - - second = benchmark_helper.JSONBackedBenchmarkData( - pathlib.Path("second.json"), - { - "commit": "123", - "benchmarks": [ - {"benchmark_id": "second1"}, - {"benchmark_id": "second2"}, - ], - }, - ) - - result = benchmark_helper.merge_results([first, second]) - - self.assertEqual( - result.data, - { - "commit": "123", - "benchmarks": [ - {"benchmark_id": "first1"}, - {"benchmark_id": "first2"}, - {"benchmark_id": "second1"}, - {"benchmark_id": "second2"}, - ], - }, - ) - - def test_merge_results_mismatching_commits(self): - first = benchmark_helper.JSONBackedBenchmarkData( - pathlib.Path("first.json"), {"commit": "123", "benchmarks": []} - ) - second = benchmark_helper.JSONBackedBenchmarkData( - pathlib.Path("second.json"), {"commit": "456", "benchmarks": []} - ) - - with self.assertRaisesRegex(ValueError, "based on different commits"): - benchmark_helper.merge_results([first, second]) - - def test_create_json_backed_benchmark_data_success(self): - benchmark_helper.JSONBackedBenchmarkData( - pathlib.Path("first.json"), {"commit": "123", "benchmarks": []} - ) - - def test_create_json_backed_benchmark_data_with_missing_benchmark_list(self): - with self.assertRaisesRegex(ValueError, "'benchmarks' field not found"): - benchmark_helper.JSONBackedBenchmarkData( - pathlib.Path("second.json"), {"commit": "123"} - ) - - def test_load_from_file_success(self): - with tempfile.TemporaryDirectory() as dir: - filepath = pathlib.Path(dir) / "first.json" - contents = {"commit": "123", "benchmarks": []} - filepath.write_text(json.dumps(contents)) - - result = benchmark_helper.JSONBackedBenchmarkData.load_from_file(filepath) - self.assertEqual(result.data, contents) - self.assertEqual(result.source_filepath, filepath) - - def test_load_from_file_invalid_json(self): - with tempfile.TemporaryDirectory() as dir: - filepath = pathlib.Path(dir) / "first.json" - filepath.write_text("bliblablub") - - with self.assertRaisesRegex( - ValueError, "seems not to be a valid JSON file" - ): - benchmark_helper.JSONBackedBenchmarkData.load_from_file(filepath) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/benchmarks/collect_compilation_statistics.py b/build_tools/benchmarks/collect_compilation_statistics.py deleted file mode 100755 index f712749b4564..000000000000 --- a/build_tools/benchmarks/collect_compilation_statistics.py +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Collect compilation statistics from benchmark suites. - -See https://iree.dev/developers/performance/benchmark-suites/ for how to build -the benchmark suites. -""" - -import pathlib -import sys - -# Add build_tools python dir to the search path. -sys.path.insert(0, str(pathlib.Path(__file__).parent.with_name("python"))) - -import argparse -import json -import os -import re -import zipfile - -from dataclasses import asdict, dataclass -from typing import BinaryIO, Dict, List, Optional, TextIO - -from common import benchmark_definition -from common.benchmark_definition import ( - CompilationInfo, - CompilationResults, - CompilationStatistics, - ModuleComponentSizes, - get_git_commit_hash, -) -from common import benchmark_config -from e2e_test_artifacts import iree_artifacts -from e2e_test_framework import serialization -from e2e_test_framework.definitions import iree_definitions - -BENCHMARK_FLAGFILE = "flagfile" -MODULE_DIR = "vmfb" -MODULE_FILE_EXTENSION = "vmfb" -NINJA_LOG_HEADER = "ninja log v5" -NINJA_BUILD_LOG = ".ninja_log" -COMPILATION_STATS_MODULE_SUFFIX = "compile-stats" -E2E_TEST_ARTIFACTS_REL_PATH = "e2e_test_artifacts" - -VM_COMPONENT_NAME = "module.fb" -CONST_COMPONENT_NAME = "_const.bin" -DISPATCH_COMPONENT_PATTERNS = [ - r".+_embedded_elf_.+\.so", - r".+_vulkan_spirv_fb\.fb", - r".+_cuda_nvptx_fb\.fb", - r".+_vmvx_bytecode_fb\.fb", -] - - -@dataclass(frozen=True) -class ModuleInfo(object): - module_path: pathlib.Path - stream_stats_path: pathlib.Path - - -def match_module_cmake_target(module_path: pathlib.PurePath) -> Optional[str]: - if module_path.match( - f"{E2E_TEST_ARTIFACTS_REL_PATH}/iree_*/" f"{iree_artifacts.MODULE_FILENAME}" - ): - # /iree_/ - path_parts = module_path.parts[-3:] - # Join to get the CMake target name. This is *not* a filesystem path, so we - # don't want \ separators on Windows that we would get with os.path.join(). - return "/".join(path_parts) - - return None - - -def parse_compilation_time_from_ninja_log(log: TextIO) -> Dict[str, int]: - """Retrieve the compilation time (ms) from the Ninja build log. - - Returns: - Map of target name and compilation time in ms. - """ - - target_build_time_map = {} - header = log.readline() - if NINJA_LOG_HEADER not in header: - raise NotImplementedError(f"Unsupported ninja log version: {header}") - - for line in log: - start_time, end_time, _, target, _ = line.strip().split("\t") - cmake_target = match_module_cmake_target(pathlib.PurePath(target)) - if cmake_target is None: - continue - - start_time = int(start_time) - end_time = int(end_time) - target_build_time_map[cmake_target] = end_time - start_time - - return target_build_time_map - - -def get_module_component_info( - module: BinaryIO, module_file_bytes: int -) -> ModuleComponentSizes: - with zipfile.ZipFile(module) as module_zipfile: - size_map = dict( - (info.filename, info.file_size) for info in module_zipfile.infolist() - ) - - identified_names = set() - if VM_COMPONENT_NAME in size_map: - vm_component_bytes = size_map[VM_COMPONENT_NAME] - identified_names.add(VM_COMPONENT_NAME) - else: - vm_component_bytes = 0 - - if CONST_COMPONENT_NAME in size_map: - const_component_bytes = size_map[CONST_COMPONENT_NAME] - identified_names.add(CONST_COMPONENT_NAME) - else: - const_component_bytes = 0 - - total_dispatch_component_bytes = 0 - for filename, size in size_map.items(): - for pattern in DISPATCH_COMPONENT_PATTERNS: - if re.match(pattern, filename): - total_dispatch_component_bytes += size - identified_names.add(filename) - break - - actual_key_set = set(size_map.keys()) - if identified_names != actual_key_set: - # With consteval, we invoke the compiler within the compiler, which - # can yield additional temporaries. - print( - f"Ignoring extra components in the module: {actual_key_set-identified_names}", - file=sys.stderr, - ) - - return ModuleComponentSizes( - file_bytes=module_file_bytes, - vm_component_bytes=vm_component_bytes, - const_component_bytes=const_component_bytes, - total_dispatch_component_bytes=total_dispatch_component_bytes, - ) - - -def get_module_map_from_compilation_benchmark_config( - compilation_benchmark_config_data: TextIO, e2e_test_artifacts_dir: pathlib.PurePath -) -> Dict[CompilationInfo, ModuleInfo]: - benchmark_config = json.load(compilation_benchmark_config_data) - gen_configs = serialization.unpack_and_deserialize( - data=benchmark_config["generation_configs"], - root_type=List[iree_definitions.ModuleGenerationConfig], - ) - module_map = {} - for gen_config in gen_configs: - model = gen_config.imported_model.model - compile_config = gen_config.compile_config - target_archs = [] - for compile_target in compile_config.compile_targets: - arch = compile_target.target_architecture - target_archs.append( - ( - f"{arch.type.value}-{arch.architecture}-{arch.microarchitecture}-" - f"{compile_target.target_abi.value}" - ) - ) - compilation_info = CompilationInfo( - name=gen_config.name, - model_name=model.name, - model_tags=tuple(model.tags), - model_source=model.source_type.value, - target_arch=f"[{','.join(target_archs)}]", - compile_tags=tuple(compile_config.tags), - gen_config_id=gen_config.composite_id, - ) - module_dir_path = pathlib.Path( - iree_artifacts.get_module_dir_path( - module_generation_config=gen_config, root_path=e2e_test_artifacts_dir - ) - ) - module_path = module_dir_path / iree_artifacts.MODULE_FILENAME - stream_stats_path = module_dir_path / iree_artifacts.SCHEDULING_STATS_FILENAME - module_map[compilation_info] = ModuleInfo( - module_path=module_path, stream_stats_path=stream_stats_path - ) - - return module_map - - -def _check_dir_path(path_str: str) -> pathlib.Path: - path = pathlib.Path(path_str) - if not path.is_dir(): - raise argparse.ArgumentTypeError(f"{path} is not a directory.") - return path - - -def _check_file_path(path_str: str) -> pathlib.Path: - path = pathlib.Path(path_str) - if not path.is_file(): - raise argparse.ArgumentTypeError(f"{path} is not a file.") - return path - - -def _parse_arguments(): - """Returns an argument parser with common options.""" - - parser = argparse.ArgumentParser( - description="Collect compilation statistics from benchmark suites." - ) - parser.add_argument( - "--compilation_benchmark_config", - type=_check_file_path, - required=True, - help="Exported compilation benchmark config of e2e test artifacts.", - ) - parser.add_argument( - "--build_log", - type=_check_file_path, - required=True, - help="Path to the ninja build log.", - ) - parser.add_argument( - "--e2e_test_artifacts_dir", - type=_check_dir_path, - required=True, - help="Path to the e2e test artifacts directory.", - ) - parser.add_argument("--output", type=pathlib.Path, help="Path to output JSON file.") - - return parser.parse_args() - - -def main(args: argparse.Namespace): - config_data = args.compilation_benchmark_config.open("r") - module_map = get_module_map_from_compilation_benchmark_config( - compilation_benchmark_config_data=config_data, - e2e_test_artifacts_dir=args.e2e_test_artifacts_dir, - ) - build_log_path = args.build_log - - with build_log_path.open("r") as log_file: - target_build_time_map = parse_compilation_time_from_ninja_log(log_file) - - compilation_statistics_list = [] - for compilation_info, module_info in module_map.items(): - module_path = module_info.module_path - with module_path.open("rb") as module_file: - module_component_sizes = get_module_component_info( - module_file, module_path.stat().st_size - ) - - cmake_target = match_module_cmake_target(module_path) - if cmake_target is None: - raise RuntimeError( - f"Module path isn't a module cmake target: {module_path}" - ) - compilation_time_ms = target_build_time_map[cmake_target] - - stream_stats_json = json.loads(module_info.stream_stats_path.read_text()) - exec_stats_json = stream_stats_json["stream-aggregate"]["execution"] - ir_stats = benchmark_definition.IRStatistics( - stream_dispatch_count=exec_stats_json["dispatch-count"] - ) - - compilation_statistics = CompilationStatistics( - compilation_info=compilation_info, - module_component_sizes=module_component_sizes, - compilation_time_ms=compilation_time_ms, - ir_stats=ir_stats, - ) - compilation_statistics_list.append(compilation_statistics) - - commit = get_git_commit_hash("HEAD") - compilation_results = CompilationResults( - commit=commit, compilation_statistics=compilation_statistics_list - ) - - json_output = json.dumps(asdict(compilation_results), indent=2) - if args.output is None: - print(json_output) - else: - args.output.write_text(json_output) - - -if __name__ == "__main__": - main(_parse_arguments()) diff --git a/build_tools/benchmarks/collect_compilation_statistics_test.py b/build_tools/benchmarks/collect_compilation_statistics_test.py deleted file mode 100644 index 44a9753e4dd5..000000000000 --- a/build_tools/benchmarks/collect_compilation_statistics_test.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -from io import BytesIO, StringIO -import json -import pathlib -import unittest -import zipfile - -from common.benchmark_definition import ModuleComponentSizes -from collect_compilation_statistics import ( - CONST_COMPONENT_NAME, - VM_COMPONENT_NAME, - get_module_component_info, - parse_compilation_time_from_ninja_log, -) -from e2e_test_artifacts import iree_artifacts -from e2e_test_framework import serialization -from e2e_test_framework.definitions import common_definitions, iree_definitions -import collect_compilation_statistics -import common.benchmark_definition - - -class CollectCompilationStatistics(unittest.TestCase): - def test_match_module_cmake_target_with_e2e_test_artifacts(self): - target = collect_compilation_statistics.match_module_cmake_target( - pathlib.PurePath("e2e_test_artifacts/iree_abcd/module.vmfb") - ) - - self.assertEqual(target, "e2e_test_artifacts/iree_abcd/module.vmfb") - - def test_match_module_cmake_target_not_match(self): - target = collect_compilation_statistics.match_module_cmake_target( - pathlib.PurePath("other/target.vmfb") - ) - - self.assertIsNone(target) - - def test_parse_compilation_time_from_ninja_log(self): - target1 = "e2e_test_artifacts/iree_deeplabv3/module.vmfb" - target2 = "e2e_test_artifacts/iree_mobilessd/module.vmfb" - ninja_log = StringIO( - "# ninja log v5\n" - f"0\t100\taaa\tbuild/{target1}\taaa\n" - f"130\t200\tbbb\tbuild/{target2}\tbbb\n" - ) - - target_map = parse_compilation_time_from_ninja_log(ninja_log) - - self.assertEqual(target_map, {target1: 100, target2: 70}) - - def test_get_module_component_info(self): - module_file = BytesIO() - with zipfile.ZipFile(module_file, "w") as zip: - zip.writestr(VM_COMPONENT_NAME, b"abcd") - zip.writestr(CONST_COMPONENT_NAME, b"123") - zip.writestr("main_dispatch_0_vulkan_spirv_fb.fb", b"bindata0") - zip.writestr("main_dispatch_1_vulkan_spirv_fb.fb", b"bindata1") - zip.writestr("predict_dispatch_2_cuda_nvptx_fb.fb", b"bindata2") - zip.writestr("dispatch_3_embedded_elf_x86_64.so", b"bindata3") - module_file_data = module_file.getvalue() - - component_sizes = get_module_component_info( - BytesIO(module_file_data), len(module_file_data) - ) - - self.assertEqual( - component_sizes, - ModuleComponentSizes( - file_bytes=len(module_file_data), - vm_component_bytes=4, - const_component_bytes=3, - total_dispatch_component_bytes=32, - ), - ) - - def test_get_module_map_from_compilation_benchmark_config(self): - model_a = common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32"], - ) - imported_model_a = iree_definitions.ImportedModel.from_model(model_a) - compile_config_a = iree_definitions.CompileConfig.build( - id="config_a", - tags=["defaults"], - compile_targets=[ - iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - ], - ) - compile_config_b = iree_definitions.CompileConfig.build( - id="config_b", - tags=["defaults"], - compile_targets=[ - iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - ], - ) - gen_config_a = iree_definitions.ModuleGenerationConfig.build( - imported_model=imported_model_a, compile_config=compile_config_a - ) - gen_config_b = iree_definitions.ModuleGenerationConfig.build( - imported_model=imported_model_a, compile_config=compile_config_b - ) - benchmark_config = dict( - generation_configs=serialization.serialize_and_pack( - [gen_config_a, gen_config_b] - ), - module_dir_paths=["a", "b"], - ) - root_dir = pathlib.PurePath("artifacts_dir") - - module_map = collect_compilation_statistics.get_module_map_from_compilation_benchmark_config( - compilation_benchmark_config_data=StringIO(json.dumps(benchmark_config)), - e2e_test_artifacts_dir=root_dir, - ) - - compile_info_a = common.benchmark_definition.CompilationInfo( - name=gen_config_a.name, - model_name=model_a.name, - model_tags=tuple(model_a.tags), - model_source=model_a.source_type.value, - target_arch=f"[cpu-x86_64-cascadelake-linux-gnu]", - compile_tags=tuple(gen_config_a.compile_config.tags), - gen_config_id=gen_config_a.composite_id, - ) - module_dir_a = pathlib.Path( - iree_artifacts.get_module_dir_path(gen_config_a, root_dir) - ) - module_info_a = collect_compilation_statistics.ModuleInfo( - module_path=module_dir_a / iree_artifacts.MODULE_FILENAME, - stream_stats_path=module_dir_a / iree_artifacts.SCHEDULING_STATS_FILENAME, - ) - compile_info_b = common.benchmark_definition.CompilationInfo( - name=gen_config_b.name, - model_name=model_a.name, - model_tags=tuple(model_a.tags), - model_source=model_a.source_type.value, - target_arch=f"[cpu-riscv_64-generic-linux-gnu]", - compile_tags=tuple(gen_config_a.compile_config.tags), - gen_config_id=gen_config_b.composite_id, - ) - module_dir_b = pathlib.Path( - iree_artifacts.get_module_dir_path(gen_config_b, root_dir) - ) - module_info_b = collect_compilation_statistics.ModuleInfo( - module_path=module_dir_b / iree_artifacts.MODULE_FILENAME, - stream_stats_path=module_dir_b / iree_artifacts.SCHEDULING_STATS_FILENAME, - ) - self.assertEqual( - module_map, {compile_info_a: module_info_a, compile_info_b: module_info_b} - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/benchmarks/common/CMakeLists.txt b/build_tools/benchmarks/common/CMakeLists.txt deleted file mode 100644 index 2934d1841afd..000000000000 --- a/build_tools/benchmarks/common/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -################################################################################ -# Tests -################################################################################ - -benchmark_tool_py_test( - NAME - linux_device_utils_test - SRC - "linux_device_utils_test.py" -) - -benchmark_tool_py_test( - NAME - common_arguments_test - SRC - "common_arguments_test.py" -) - -benchmark_tool_py_test( - NAME - benchmark_config_test - SRC - "benchmark_config_test.py" -) - -benchmark_tool_py_test( - NAME - benchmark_suite_test - SRC - "benchmark_suite_test.py" -) - -benchmark_tool_py_test( - NAME - benchmark_driver_test - SRC - "benchmark_driver_test.py" -) diff --git a/build_tools/benchmarks/common/__init__.py b/build_tools/benchmarks/common/__init__.py deleted file mode 100644 index 7bea5c5b1d72..000000000000 --- a/build_tools/benchmarks/common/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/build_tools/benchmarks/common/android_device_utils.py b/build_tools/benchmarks/common/android_device_utils.py deleted file mode 100644 index 5149da8d1d33..000000000000 --- a/build_tools/benchmarks/common/android_device_utils.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Utils for accessing Android devices.""" - -import json -import re - -from typing import Sequence -from .benchmark_definition import execute_cmd_and_get_stdout, DeviceInfo, PlatformType - - -def get_android_device_model(verbose: bool = False) -> str: - """Returns the Android device model.""" - model = execute_cmd_and_get_stdout( - ["adb", "shell", "getprop", "ro.product.model"], verbose=verbose - ) - model = re.sub(r"\W+", "-", model) - return model - - -def get_android_cpu_abi(verbose: bool = False) -> str: - """Returns the CPU ABI for the Android device.""" - return execute_cmd_and_get_stdout( - ["adb", "shell", "getprop", "ro.product.cpu.abi"], verbose=verbose - ) - - -def get_android_cpu_features(verbose: bool = False) -> Sequence[str]: - """Returns the CPU features for the Android device.""" - cpuinfo = execute_cmd_and_get_stdout( - ["adb", "shell", "cat", "/proc/cpuinfo"], verbose=verbose - ) - features = [] - for line in cpuinfo.splitlines(): - if line.startswith("Features"): - _, features = line.split(":") - return features.strip().split() - return features - - -def get_android_gpu_name(verbose: bool = False) -> str: - """Returns the GPU name for the Android device.""" - vkjson = execute_cmd_and_get_stdout( - ["adb", "shell", "cmd", "gpu", "vkjson"], verbose=verbose - ) - vkjson = json.loads(vkjson) - name = vkjson["devices"][0]["properties"]["deviceName"] - - # Perform some canonicalization: - - # - Adreno GPUs have raw names like "Adreno (TM) 650". - name = name.replace("(TM)", "") - - # Replace all consecutive non-word characters with a single hyphen. - name = re.sub(r"\W+", "-", name) - - return name - - -def get_android_device_info(verbose: bool = False) -> DeviceInfo: - """Returns device info for the Android device.""" - return DeviceInfo( - platform_type=PlatformType.ANDROID, - model=get_android_device_model(verbose), - cpu_abi=get_android_cpu_abi(verbose), - cpu_uarch=None, - cpu_features=get_android_cpu_features(verbose), - gpu_name=get_android_gpu_name(verbose), - ) diff --git a/build_tools/benchmarks/common/benchmark_config.py b/build_tools/benchmarks/common/benchmark_config.py deleted file mode 100644 index c90dd0718a64..000000000000 --- a/build_tools/benchmarks/common/benchmark_config.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# All benchmarks' relative path against root build directory. - -from argparse import Namespace -from dataclasses import dataclass -from typing import Optional -import pathlib -import re - -from common import benchmark_definition - -BENCHMARK_RESULTS_REL_PATH = "benchmark-results" - - -@dataclass -class BenchmarkConfig: - """Represents the settings to run benchmarks. - - tmp_dir: per-commit temporary directory. - root_benchmark_dir: the root directory path/URL containing the built - benchmark suites. - benchmark_results_dir: the directory to store benchmark results files. - git_commit_hash: the git commit hash. - benchmark_tool_dir: the path to the tool directory. - driver_filter: filter benchmarks to those whose driver matches this regex - (or all if this is None). - model_name_filter: filter benchmarks to those whose model name matches this - regex (or all if this is None). - mode_filter: filter benchmarks to those whose benchmarking mode matches this - regex (or all if this is None). - keep_going: whether to proceed if an individual run fails. Exceptions will - logged and returned. - benchmark_min_time: min number of seconds to run the benchmark for, if - specified. Otherwise, the benchmark will be repeated a fixed number of - times. - continue_from_previous: skip the benchmarks if their results are found in - the benchmark_results_dir. - verify: verify the output if model's expected output is available. - """ - - tmp_dir: pathlib.Path - root_benchmark_dir: benchmark_definition.ResourceLocation - benchmark_results_dir: pathlib.Path - git_commit_hash: str - - benchmark_tool_dir: Optional[pathlib.Path] = None - - driver_filter: Optional[str] = None - model_name_filter: Optional[str] = None - mode_filter: Optional[str] = None - use_compatible_filter: bool = False - - keep_going: bool = False - benchmark_min_time: float = 0 - continue_from_previous: bool = False - verify: bool = False - - @staticmethod - def build_from_args(args: Namespace, git_commit_hash: str): - """Build config from command arguments. - - Args: - args: the command arguments. - git_commit_hash: the git commit hash of IREE. - """ - - def real_path_or_none(path: Optional[pathlib.Path]) -> Optional[pathlib.Path]: - return path.resolve() if path else None - - if not args.benchmark_tool_dir: - raise ValueError("--benchmark_tool_dir should be specified.") - - per_commit_tmp_dir: pathlib.Path = (args.tmp_dir / git_commit_hash).resolve() - - root_benchmark_dir = args.e2e_test_artifacts_dir - # Convert the local path into Path object. - if re.match("^[^:]+://", str(root_benchmark_dir)): - root_benchmark_dir = benchmark_definition.ResourceLocation.build_url( - root_benchmark_dir - ) - else: - root_benchmark_dir = benchmark_definition.ResourceLocation.build_local_path( - root_benchmark_dir - ) - - return BenchmarkConfig( - tmp_dir=per_commit_tmp_dir, - root_benchmark_dir=root_benchmark_dir, - benchmark_results_dir=per_commit_tmp_dir / BENCHMARK_RESULTS_REL_PATH, - git_commit_hash=git_commit_hash, - benchmark_tool_dir=real_path_or_none(args.benchmark_tool_dir), - driver_filter=args.driver_filter_regex, - model_name_filter=args.model_name_regex, - mode_filter=args.mode_regex, - use_compatible_filter=args.compatible_only, - keep_going=args.keep_going, - benchmark_min_time=args.benchmark_min_time, - continue_from_previous=args.continue_from_previous, - verify=args.verify, - ) diff --git a/build_tools/benchmarks/common/benchmark_config_test.py b/build_tools/benchmarks/common/benchmark_config_test.py deleted file mode 100644 index ab76800e0895..000000000000 --- a/build_tools/benchmarks/common/benchmark_config_test.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import pathlib -import unittest -import tempfile - -from common import benchmark_config, benchmark_definition, common_arguments - - -class BenchmarkConfigTest(unittest.TestCase): - def setUp(self): - self._tmp_dir_manager = tempfile.TemporaryDirectory() - self.tmp_dir = pathlib.Path(self._tmp_dir_manager.name).resolve() - self._build_dir_manager = tempfile.TemporaryDirectory() - self.build_dir = pathlib.Path(self._build_dir_manager.name).resolve() - self.e2e_test_artifacts_dir = self.build_dir / "e2e_test_artifacts" - self.e2e_test_artifacts_dir.mkdir() - self.normal_tool_dir = self.build_dir / "normal_tool" - self.normal_tool_dir.mkdir() - self.execution_config = self.build_dir / "execution_config.json" - self.execution_config.touch() - - def tearDown(self): - self._build_dir_manager.cleanup() - self._tmp_dir_manager.cleanup() - - def test_build_from_args(self): - args = common_arguments.Parser().parse_args( - [ - f"--tmp_dir={self.tmp_dir}", - f"--benchmark_tool_dir={self.normal_tool_dir}", - f"--driver_filter_regex=a", - f"--model_name_regex=b", - f"--mode_regex=c", - f"--keep_going", - f"--benchmark_min_time=10", - f"--compatible_only", - f"--e2e_test_artifacts_dir={self.e2e_test_artifacts_dir}", - f"--execution_benchmark_config={self.execution_config}", - "--target_device=test", - "--verify", - ] - ) - - config = benchmark_config.BenchmarkConfig.build_from_args( - args=args, git_commit_hash="abcd" - ) - - per_commit_tmp_dir = self.tmp_dir / "abcd" - expected_config = benchmark_config.BenchmarkConfig( - tmp_dir=per_commit_tmp_dir, - root_benchmark_dir=benchmark_definition.ResourceLocation.build_local_path( - self.e2e_test_artifacts_dir - ), - benchmark_results_dir=per_commit_tmp_dir / "benchmark-results", - git_commit_hash="abcd", - benchmark_tool_dir=self.normal_tool_dir, - driver_filter="a", - model_name_filter="b", - mode_filter="c", - keep_going=True, - benchmark_min_time=10, - use_compatible_filter=True, - verify=True, - ) - self.assertEqual(config, expected_config) - - def test_build_from_args_benchmark_only(self): - args = common_arguments.Parser().parse_args( - [ - f"--tmp_dir={self.tmp_dir}", - f"--benchmark_tool_dir={self.normal_tool_dir}", - f"--e2e_test_artifacts_dir={self.e2e_test_artifacts_dir}", - f"--execution_benchmark_config={self.execution_config}", - "--target_device=test", - ] - ) - - config = benchmark_config.BenchmarkConfig.build_from_args( - args=args, git_commit_hash="abcd" - ) - - def test_build_from_args_with_test_artifacts_dir_url(self): - args = common_arguments.Parser().parse_args( - [ - f"--tmp_dir={self.tmp_dir}", - f"--benchmark_tool_dir={self.normal_tool_dir}", - f"--e2e_test_artifacts_dir=https://example.com/testdata", - f"--execution_benchmark_config={self.execution_config}", - "--target_device=test", - ] - ) - - config = benchmark_config.BenchmarkConfig.build_from_args( - args=args, git_commit_hash="abcd" - ) - - self.assertEqual( - config.root_benchmark_dir.get_url(), "https://example.com/testdata" - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/benchmarks/common/benchmark_definition.py b/build_tools/benchmarks/common/benchmark_definition.py deleted file mode 100644 index 484f0a5fb412..000000000000 --- a/build_tools/benchmarks/common/benchmark_definition.py +++ /dev/null @@ -1,720 +0,0 @@ -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Utilities for describing benchmarks. - -This file provides common and structured representation of devices, benchmark -definitions, and benchmark result collections, so that they can be shared -between different stages of the same benchmark pipeline. -""" - -import json -import pathlib -import re -import subprocess -import urllib.parse -import urllib.request - -import dataclasses -from enum import Enum -from typing import Any, Dict, List, Optional, Sequence, Tuple, Union - -from e2e_test_framework.definitions import common_definitions - -# A map from CPU ABI to IREE's benchmark target architecture. -CPU_ABI_TO_TARGET_ARCH_MAP = { - "arm64-v8a": common_definitions.DeviceArchitecture.ARMV8_2_A_GENERIC, - "x86_64-cascadelake": common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, -} - -# A map from GPU name to IREE's benchmark target architecture. -GPU_NAME_TO_TARGET_ARCH_MAP = { - "adreno-640": common_definitions.DeviceArchitecture.QUALCOMM_ADRENO, - "adreno-650": common_definitions.DeviceArchitecture.QUALCOMM_ADRENO, - "adreno-660": common_definitions.DeviceArchitecture.QUALCOMM_ADRENO, - "adreno-730": common_definitions.DeviceArchitecture.QUALCOMM_ADRENO, - "mali-g77": common_definitions.DeviceArchitecture.ARM_VALHALL, - "mali-g78": common_definitions.DeviceArchitecture.ARM_VALHALL, - "tesla-v100-sxm2-16gb": common_definitions.DeviceArchitecture.NVIDIA_PASCAL, - "nvidia-a100-sxm4-40gb": common_definitions.DeviceArchitecture.NVIDIA_AMPERE, - "nvidia-geforce-rtx-3090": common_definitions.DeviceArchitecture.NVIDIA_AMPERE, -} - - -@dataclasses.dataclass(frozen=True) -class ResourceLocation: - """Class to represent either local resource path or an URL.""" - - local_path: Optional[pathlib.Path] - url: Optional[str] - - def get_local_path(self) -> Optional[pathlib.Path]: - """Returns the local path or None if it is an URL.""" - return self.local_path - - def get_url(self) -> Optional[str]: - """Returns the URL or None if it is a local path.""" - return self.url - - def __truediv__(self, sub_path: Union[str, pathlib.PurePath]) -> "ResourceLocation": - """Appends the sub path and returns the new location.""" - local_path = self.get_local_path() - if local_path: - return self.__class__.build_local_path(local_path / sub_path) - url = self.get_url() - assert url is not None - sub_url_path = urllib.request.pathname2url(str(sub_path)) - # urljoin requires the directly URL ended with "/". - return self.__class__.build_url(urllib.parse.urljoin(url + "/", sub_url_path)) - - @classmethod - def build_local_path(cls, path: Union[pathlib.Path, str]) -> "ResourceLocation": - """Build from a local path.""" - return cls(local_path=pathlib.Path(path), url=None) - - @classmethod - def build_url(cls, url: str) -> "ResourceLocation": - """Build from an URL.""" - return cls(local_path=None, url=url) - - -@dataclasses.dataclass -class DriverInfo: - """An object describing a IREE HAL driver. - - It includes the following characteristics: - - pretty_name: the pretty name, e.g., 'IREE-LLVM-CPU' - - device_type: the targeted device type, e.g., 'CPU' - - driver_name: runtime driver flag, e.g., 'local-task' - - loader_name: executable loader name, if used - """ - - pretty_name: str - device_type: str - driver_name: str - loader_name: str - - -# A map for IREE driver names. This allows us to normalize driver names like -# mapping to more friendly ones and detach to keep driver names used in -# benchmark presentation stable. -IREE_DRIVERS_INFOS = { - "iree-llvm-cpu": DriverInfo("IREE-LLVM-CPU", "CPU", "local-task", "embedded-elf"), - "iree-llvm-cpu-sync": DriverInfo( - "IREE-LLVM-CPU-Sync", "CPU", "local-sync", "embedded-elf" - ), - "iree-vmvx": DriverInfo("IREE-VMVX", "CPU", "local-task", "vmvx-module"), - "iree-vmvx-sync": DriverInfo("IREE-VMVX-Sync", "CPU", "local-sync", "vmvx-module"), - "iree-vulkan": DriverInfo("IREE-Vulkan", "GPU", "vulkan", ""), - "iree-cuda": DriverInfo("IREE-CUDA", "GPU", "cuda", ""), -} - -IREE_PRETTY_NAME_TO_DRIVER_NAME = { - v.pretty_name: k for k, v in IREE_DRIVERS_INFOS.items() -} - - -def execute_cmd( - args: Sequence[Any], verbose: bool = False, **kwargs -) -> subprocess.CompletedProcess: - """Executes a command and returns the completed process. - - A thin wrapper around subprocess.run that sets some useful defaults and - optionally prints out the command being run. - - Raises: - CalledProcessError if the command fails. - """ - if verbose: - print(f"cmd: {args}") - try: - return subprocess.run(args, check=True, text=True, **kwargs) - except subprocess.CalledProcessError as exc: - print( - ( - f"\n\nThe following command failed:\n\n{args}" - f"\n\nReturn code: {exc.returncode}\n\n" - ) - ) - if exc.stdout: - print(f"Stdout:\n\n{exc.stdout}\n\n") - if exc.stderr: - print(f"Stderr:\n\n{exc.stderr}\n\n") - raise exc - - -def execute_cmd_and_get_output( - args: Sequence[Any], verbose: bool = False, **kwargs -) -> Tuple[str, str]: - """Executes a command and returns its stdout and stderr - - Same as execute_cmd except captures stdout and stderr. - """ - exc = execute_cmd( - args, verbose=verbose, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs - ) - return exc.stdout.strip(), exc.stderr.strip() - - -def execute_cmd_and_get_stdout( - args: Sequence[Any], verbose: bool = False, **kwargs -) -> str: - """Executes a command and returns its stdout. - - Same as execute_cmd except captures stdout (and not stderr). - """ - stdout, _ = execute_cmd_and_get_output(args, verbose=verbose, **kwargs) - return stdout - - -def get_git_commit_hash(commit: str) -> str: - return execute_cmd_and_get_stdout( - ["git", "rev-parse", commit], cwd=pathlib.Path(__file__).resolve().parent - ) - - -def parse_external_param_url(param_url: str) -> Tuple[str, str, str]: - """Parse `scope=url` into (scope, URL, filename of the URL).""" - scope, url = param_url.split("=", maxsplit=1) - url_path = urllib.parse.urlparse(url).path - filename = pathlib.PurePath(url_path).name - return scope, url, filename - - -def get_iree_benchmark_module_arguments( - driver_info: DriverInfo, - benchmark_min_time: Optional[float] = None, - dump_results: bool = True, -): - """Returns the common arguments to run iree-benchmark-module.""" - - if driver_info.loader_name == "vmvx-module": - # VMVX is very unoptimized for now and can take a long time to run. - # Decrease the repetition for it until it's reasonably fast. - repetitions = 3 - else: - repetitions = 10 - - cmd = [] - if dump_results: - cmd += [ - "--time_unit=ns", - "--benchmark_format=json", - "--benchmark_out_format=json", - "--print_statistics=true", - ] - - if benchmark_min_time: - cmd.extend( - [ - f"--benchmark_min_time={benchmark_min_time}", - ] - ) - else: - cmd.extend( - [ - f"--benchmark_repetitions={repetitions}", - ] - ) - - return cmd - - -def wait_for_iree_benchmark_module_start( - process: subprocess.Popen, verbose: bool = False -) -> None: - """Wait for the start of iree-benchmark module; otherwise will see connection - failure when opening the catpure tool.""" - - while True: - line = process.stdout.readline() # pytype: disable=attribute-error - if line == "" and process.poll() is not None: # Process completed - raise ValueError("Cannot find benchmark result line in the log!") - if verbose: - print(line.strip()) - # Result available - if re.match(r"^BM_.+/real_time", line) is not None: - break - - -class PlatformType(Enum): - ANDROID = "Android" - LINUX = "Linux" - - -@dataclasses.dataclass(frozen=True) -class DeviceInfo: - """An object describing a device. - - It includes the following characteristics: - - platform_type: the OS platform, e.g., 'Android' - - model: the product model, e.g., 'Pixel-4' - - cpu_abi: the CPU ABI, e.g., 'arm64-v8a', 'x86_64' - - cpu_uarch: the CPU microarchitecture, e.g., 'CascadeLake' - - cpu_features: the detailed CPU features, e.g., ['fphp', 'sve'] - - gpu_name: the GPU name, e.g., 'Mali-G77' - """ - - platform_type: PlatformType - model: str - cpu_abi: str - cpu_uarch: Optional[str] - cpu_features: Sequence[str] - gpu_name: str - - def __str__(self): - features = ", ".join(self.cpu_features) - params = [ - f"model='{self.model}'", - f"cpu_abi='{self.cpu_abi}'", - f"cpu_uarch='{self.cpu_uarch}'", - f"gpu_name='{self.gpu_name}'", - f"cpu_features=[{features}]", - ] - params = ", ".join(params) - return f"{self.platform_type.value} device <{params}>" - - def get_cpu_arch(self) -> Optional[common_definitions.DeviceArchitecture]: - name = self.cpu_abi.lower() - if self.cpu_uarch: - name += f"-{self.cpu_uarch.lower()}" - - return CPU_ABI_TO_TARGET_ARCH_MAP.get(name) - - def get_gpu_arch(self) -> Optional[common_definitions.DeviceArchitecture]: - name = self.gpu_name.lower() - return GPU_NAME_TO_TARGET_ARCH_MAP.get(name) - - def get_detailed_cpu_arch_name(self) -> str: - """Returns the detailed architecture name.""" - - if self.cpu_abi == "arm64-v8a": - return self.__get_arm_cpu_arch_revision() - if self.cpu_abi == "x86_64": - return self.__get_x86_detailed_cpu_arch_name() - raise ValueError("Unrecognized CPU ABI; need to update the list") - - def to_json_object(self) -> Dict[str, Any]: - return { - "platform_type": self.platform_type.value, - "model": self.model, - "cpu_abi": self.cpu_abi, - "cpu_uarch": self.cpu_uarch if self.cpu_uarch else "", - "cpu_features": self.cpu_features, - "gpu_name": self.gpu_name, - } - - @staticmethod - def from_json_object(json_object: Dict[str, Any]): - cpu_uarch = json_object.get("cpu_uarch") - return DeviceInfo( - PlatformType(json_object["platform_type"]), - json_object["model"], - json_object["cpu_abi"], - None if cpu_uarch == "" else cpu_uarch, - json_object["cpu_features"], - json_object["gpu_name"], - ) - - def __get_x86_detailed_cpu_arch_name(self) -> str: - """Returns the x86 architecture with microarchitecture name.""" - - if not self.cpu_uarch: - return self.cpu_abi - - return f"{self.cpu_abi}-{self.cpu_uarch}" - - def __get_arm_cpu_arch_revision(self) -> str: - """Returns the ARM architecture revision.""" - - # CPU features for ARMv8 revisions. - # From https://en.wikichip.org/wiki/arm/armv8#ARMv8_Extensions_and_Processor_Features - rev1_features = ["atomics", "asimdrdm"] - rev2_features = [ - "fphp", - "dcpop", - "sha3", - "sm3", - "sm4", - "asimddp", - "sha512", - "sve", - ] - - rev = "ARMv8-A" - if any([f in self.cpu_features for f in rev1_features]): - rev = "ARMv8.1-A" - if any([f in self.cpu_features for f in rev2_features]): - rev = "ARMv8.2-A" - return rev - - -@dataclasses.dataclass(frozen=True) -class BenchmarkInfo: - """An object describing the current benchmark. - - It includes the following benchmark characteristics: - - name: the benchmark name - - model_name: the model name, e.g., 'MobileNetV2' - - model_tags: a list of tags used to describe additional model information, - e.g., ['imagenet'] - - model_source: the source of the model, e.g., 'TensorFlow' - - bench_mode: a list of tags for benchmark mode, - e.g., ['1-thread', 'big-core', 'full-inference'] - - device_info: an DriverInfo object describing the IREE runtime dirver. - - device_info: an DeviceInfo object describing the device where benchmarks run - - compile_tags: an optional list of tags to describe the compile configs, - e.g., ['fuse-padding'] - - runner: which runner is used for benchmarking, e.g., 'iree_vulkan', 'tflite' - - run_config_id: ID of the corresponding iree_definitions.E2EModelRunConfig. - """ - - name: str - model_name: str - model_tags: Sequence[str] - model_source: str - bench_mode: Sequence[str] - driver_info: DriverInfo - device_info: DeviceInfo - compile_tags: Optional[Sequence[str]] = None - run_config_id: Optional[str] = None - - def __str__(self): - return self.name - - def to_json_object(self) -> Dict[str, Any]: - return { - "name": self.name, - "model_name": self.model_name, - "model_tags": self.model_tags, - "model_source": self.model_source, - "bench_mode": self.bench_mode, - "compile_tags": self.compile_tags, - # Get the "iree-*" driver name from the DriverInfo. - "runner": IREE_PRETTY_NAME_TO_DRIVER_NAME[self.driver_info.pretty_name], - "device_info": self.device_info.to_json_object(), - "run_config_id": self.run_config_id, - } - - @staticmethod - def from_json_object(json_object: Dict[str, Any]): - driver_info = IREE_DRIVERS_INFOS.get(json_object["runner"]) - if not driver_info: - raise ValueError(f"Unrecognized runner: {json_object['runner']}") - - return BenchmarkInfo( - name=json_object["name"], - model_name=json_object["model_name"], - model_tags=json_object["model_tags"], - model_source=json_object["model_source"], - bench_mode=json_object["bench_mode"], - compile_tags=json_object.get("compile_tags"), - driver_info=driver_info, - device_info=DeviceInfo.from_json_object(json_object["device_info"]), - run_config_id=json_object.get("run_config_id"), - ) - - -@dataclasses.dataclass(frozen=True) -class BenchmarkLatency: - """Stores latency statistics for a benchmark run.""" - - mean: int - median: int - stddev: int - unit: str - - def to_json_object(self) -> Dict[str, Any]: - return dataclasses.asdict(self) - - @staticmethod - def from_json_object(json_object: Dict[str, Any]): - return BenchmarkLatency(**json_object) - - -def _get_google_benchmark_latencies( - benchmark_json: Dict[str, Any] -) -> Tuple[BenchmarkLatency, BenchmarkLatency]: - """Returns the Google Benchmark aggregate latencies. - - Args: - benchmark_json: The JSON string or object returned by Google Benchmark. - - Returns: - Real time and CPU time BenchmarkLatency. - """ - real_time_object: Dict[str, Any] = dict(unit="ns") - cpu_time_object: Dict[str, Any] = dict(unit="ns") - metrics = ["mean", "median", "stddev"] - for case in benchmark_json["benchmarks"]: - if any(case["name"].endswith(f"real_time_{m}") for m in metrics): - if case["time_unit"] != "ns": - raise ValueError(f"Expected ns as time unit") - metric = case["name"].split("_")[-1] - real_time_object[metric] = int(round(case["real_time"])) - cpu_time_object[metric] = int(round(case["cpu_time"])) - - # from_json_object implicitly validates that all metrics were found. - real_time = BenchmarkLatency.from_json_object(real_time_object) - cpu_time = BenchmarkLatency.from_json_object(cpu_time_object) - return real_time, cpu_time - - -@dataclasses.dataclass(frozen=True) -class BenchmarkMemory: - """Stores memory statistics for a benchmark run.""" - - peak: int - allocated: int - freed: int - live: int - unit: str - - def to_json_object(self) -> Dict[str, int]: - return dataclasses.asdict(self) - - @staticmethod - def from_json_object(json_object: Dict[str, Any]): - return BenchmarkMemory(**json_object) - - -def _get_iree_memory_statistics(benchmark_stderr: str, device: str) -> BenchmarkMemory: - """Extracts IREE's memory statistics for a given device.""" - # The memory statistics for each device are listed on their own line. - pattern = ( - rf"{device}:" - r"\s*(?P\d+)B peak /" - r"\s*(?P\d+)B allocated /" - r"\s*(?P\d+)B freed /" - r"\s*(?P\d+)B live" - ) - match = re.search(pattern, benchmark_stderr) - if match is None: - raise ValueError(f"Unable to find memory statistics in '{benchmark_stderr}'") - return BenchmarkMemory( - peak=int(match["peak"]), - allocated=int(match["allocated"]), - freed=int(match["freed"]), - live=int(match["live"]), - unit="bytes", - ) - - -@dataclasses.dataclass(frozen=True) -class BenchmarkMetrics(object): - """An object describing the results from a single benchmark. - - - real_time: the real time latency statistics returned by the benchmarking - framework. - - cpu_time: the cpu time latency statistics returned by the benchmarking - framework. - - host_memory: the host memory statistics returned by the benchmarking - framework. - - device_memory: the device memory statistics returned by the benchmarking - framework. - - raw_data: additional JSON-compatible raw results returned by the - benchmarking framework. - """ - - real_time: BenchmarkLatency - cpu_time: BenchmarkLatency - host_memory: BenchmarkMemory - device_memory: BenchmarkMemory - raw_data: Dict[str, Any] - - def to_json_object(self) -> Dict[str, Any]: - return { - "real_time": self.real_time.to_json_object(), - "cpu_time": self.cpu_time.to_json_object(), - "host_memory": self.host_memory.to_json_object(), - "device_memory": self.device_memory.to_json_object(), - "raw_data": self.raw_data, - } - - @staticmethod - def from_json_object(json_object: Dict[str, Any]): - return BenchmarkMetrics( - real_time=BenchmarkLatency.from_json_object(json_object["real_time"]), - cpu_time=BenchmarkLatency.from_json_object(json_object["cpu_time"]), - host_memory=BenchmarkMemory.from_json_object(json_object["host_memory"]), - device_memory=BenchmarkMemory.from_json_object( - json_object["device_memory"] - ), - raw_data=json_object["raw_data"], - ) - - -def parse_iree_benchmark_metrics( - benchmark_stdout: str, benchmark_stderr: str -) -> BenchmarkMetrics: - """Extract benchmark metrics from the output of iree-benchmark-module. - - Args: - benchmark_stdout: The stdout of iree-benchmark-module with - --benchmark_format=json. - benchmark_stdout: The stderr of iree-benchmark-module with - --print_statistics=true. - - Returns: - A populated BenchmarkMetrics dataclass. - """ - benchmark_json = json.loads(benchmark_stdout) - real_time, cpu_time = _get_google_benchmark_latencies(benchmark_json) - return BenchmarkMetrics( - real_time=real_time, - cpu_time=cpu_time, - host_memory=_get_iree_memory_statistics(benchmark_stderr, "HOST_LOCAL"), - device_memory=_get_iree_memory_statistics(benchmark_stderr, "DEVICE_LOCAL"), - raw_data=benchmark_json, - ) - - -@dataclasses.dataclass(frozen=True) -class BenchmarkRun(object): - """An object describing a single run of the benchmark binary. - - - info: a BenchmarkInfo object describing the benchmark setup. - - metrics: a BenchmarkMetrics object containing the results of the benchmark. - """ - - info: BenchmarkInfo - metrics: BenchmarkMetrics - - def to_json_object(self) -> Dict[str, Any]: - return { - "info": self.info.to_json_object(), - "metrics": self.metrics.to_json_object(), - } - - @staticmethod - def from_json_object(json_object: Dict[str, Any]): - return BenchmarkRun( - BenchmarkInfo.from_json_object(json_object["info"]), - BenchmarkMetrics.from_json_object(json_object["metrics"]), - ) - - -class BenchmarkResults(object): - """An object describing a set of benchmarks for one particular commit. - - It contains the following fields: - - commit: the commit SHA for this set of benchmarks. - - benchmarks: a list of BenchmarkRun objects - """ - - def __init__(self): - self.commit: str = "" - self.benchmarks: List[BenchmarkRun] = [] - - def set_commit(self, commit: str): - self.commit = commit - - def merge(self, other): - if self.commit != other.commit: - raise ValueError("Inconsistent pull request commit") - self.benchmarks.extend(other.benchmarks) - - def to_json_str(self) -> str: - json_object = {"commit": self.commit, "benchmarks": []} - json_object["benchmarks"] = [b.to_json_object() for b in self.benchmarks] - return json.dumps(json_object, indent=2) - - @staticmethod - def from_json_str(json_str: str): - json_object = json.loads(json_str) - results = BenchmarkResults() - results.set_commit(json_object["commit"]) - results.benchmarks = [ - BenchmarkRun.from_json_object(b) for b in json_object["benchmarks"] - ] - return results - - -@dataclasses.dataclass(frozen=True) -class CompilationInfo(object): - name: str - model_name: str - model_tags: Tuple[str] - model_source: str - target_arch: str - compile_tags: Tuple[str] - gen_config_id: Optional[str] = None - - def __str__(self): - return self.name - - @staticmethod - def from_json_object(json_object: Dict[str, Any]): - return CompilationInfo( - name=json_object["name"], - model_name=json_object["model_name"], - model_tags=tuple(json_object["model_tags"]), - model_source=json_object["model_source"], - target_arch=json_object["target_arch"], - compile_tags=tuple(json_object["compile_tags"]), - gen_config_id=json_object.get("gen_config_id"), - ) - - -@dataclasses.dataclass(frozen=True) -class ModuleComponentSizes(object): - file_bytes: int - vm_component_bytes: int - const_component_bytes: int - total_dispatch_component_bytes: int - - @staticmethod - def from_json_object(json_object: Dict[str, Any]): - return ModuleComponentSizes(**json_object) - - -@dataclasses.dataclass(frozen=True) -class IRStatistics(object): - # Number of cmd.dispatch ops in IR. - stream_dispatch_count: int - - @staticmethod - def from_json_object(json_object: Dict[str, Any]): - return IRStatistics(**json_object) - - -@dataclasses.dataclass(frozen=True) -class CompilationStatistics(object): - compilation_info: CompilationInfo - # Module file and component sizes. - module_component_sizes: ModuleComponentSizes - # Module compilation time in ms. - compilation_time_ms: int - # IR-level statistics - ir_stats: IRStatistics - - @staticmethod - def from_json_object(json_object: Dict[str, Any]): - return CompilationStatistics( - compilation_info=CompilationInfo.from_json_object( - json_object["compilation_info"] - ), - module_component_sizes=ModuleComponentSizes.from_json_object( - json_object["module_component_sizes"] - ), - compilation_time_ms=json_object["compilation_time_ms"], - ir_stats=IRStatistics.from_json_object(json_object["ir_stats"]), - ) - - -@dataclasses.dataclass(frozen=True) -class CompilationResults(object): - commit: str - compilation_statistics: Sequence[CompilationStatistics] - - @staticmethod - def from_json_object(json_object: Dict[str, Any]): - return CompilationResults( - commit=json_object["commit"], - compilation_statistics=[ - CompilationStatistics.from_json_object(obj) - for obj in json_object["compilation_statistics"] - ], - ) diff --git a/build_tools/benchmarks/common/benchmark_driver.py b/build_tools/benchmarks/common/benchmark_driver.py deleted file mode 100644 index e2914833f680..000000000000 --- a/build_tools/benchmarks/common/benchmark_driver.py +++ /dev/null @@ -1,254 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import json -import pathlib -import time -from typing import List, Optional, Sequence, Set, Tuple -from common.benchmark_suite import BenchmarkCase, BenchmarkSuite -from common.benchmark_config import BenchmarkConfig -from common.benchmark_definition import ( - BenchmarkInfo, - BenchmarkResults, - BenchmarkMetrics, - BenchmarkRun, - DeviceInfo, -) - - -class BenchmarkDriver(object): - """Abstract driver runs the whole benchmark flow.""" - - def __init__( - self, - device_info: DeviceInfo, - benchmark_config: BenchmarkConfig, - benchmark_suite: BenchmarkSuite, - benchmark_grace_time: float = 0.0, - verbose: bool = False, - ): - self.device_info = device_info - self.config = benchmark_config - self.benchmark_suite = benchmark_suite - self.benchmark_grace_time = benchmark_grace_time - self.verbose = verbose - self.finished_benchmarks: List[Tuple[BenchmarkInfo, pathlib.Path]] = [] - self.benchmark_errors = [] - self._seen_benchmark_names: Set[str] = set() - - def run_benchmark_case( - self, - benchmark_case: BenchmarkCase, - benchmark_results_filename: Optional[pathlib.Path], - ) -> None: - """Runs the benchmark case and serializes the results. - - Args: - benchmark_case: the benchmark_case. - benchmark_results_filename: the path to store the serialized - BenchmarkMetrics. Benchmarking is required if set. - - Raises: - Exception during benchmarking. - """ - raise NotImplementedError("Should be overwritten by a subclass.") - - def run(self) -> None: - """Execute the benchmark flow. - - It performs the following steps: - 1. Enumerate and filter benchmark cases. - 2. Call 'run_benchmark_case' for each benchmark case. - 3. Collect the benchmark results. - """ - - self.config.benchmark_results_dir.mkdir(parents=True, exist_ok=True) - - cpu_target_arch = self.device_info.get_cpu_arch() - gpu_target_arch = self.device_info.get_gpu_arch() - detected_architectures = [ - arch for arch in [cpu_target_arch, gpu_target_arch] if arch is not None - ] - if self.config.use_compatible_filter: - if cpu_target_arch is None: - print( - "INFO: Detected unsupported CPU architecture in" - f' "{self.device_info}", CPU benchmarking is disabled.' - ) - if gpu_target_arch is None: - print( - "INFO: Detected unsupported GPU architecture in" - f' "{self.device_info}", GPU benchmarking is disabled.' - ) - compatible_arch_filter = detected_architectures - else: - # No compatible filter on the target architectures. - compatible_arch_filter = None - - drivers, loaders = self.__get_available_drivers_and_loaders() - - benchmark_cases = self.benchmark_suite.filter_benchmarks( - available_drivers=drivers, - available_loaders=loaders, - target_architectures=compatible_arch_filter, - driver_filter=self.config.driver_filter, - mode_filter=self.config.mode_filter, - model_name_filter=self.config.model_name_filter, - ) - - for benchmark_case in benchmark_cases: - benchmark_info = self.__get_benchmark_info_from_case( - benchmark_case=benchmark_case - ) - benchmark_name = str(benchmark_info) - - if benchmark_case.target_arch not in detected_architectures: - print( - f"WARNING: Benchmark '{benchmark_name}' may be incompatible" - f" with the detected architectures '{detected_architectures}'" - f" on the device. Pass --compatible-only to skip incompatible" - f" benchmarks." - ) - - # Sanity check for the uniqueness of benchmark names. - if benchmark_name in self._seen_benchmark_names: - raise ValueError( - f"Found duplicate benchmark {benchmark_name} in the suites." - ) - self._seen_benchmark_names.add(benchmark_name) - - results_path = self.__get_output_paths(benchmark_name) - # If we continue from the previous results, check and skip if the result - # files exist. - if self.config.continue_from_previous: - if results_path is not None and results_path.exists(): - self.finished_benchmarks.append((benchmark_info, results_path)) - results_path = None - - # Skip if no need to benchmark. - if results_path is None: - continue - - print(f"--> Benchmark started: {benchmark_name} <--") - - try: - self.run_benchmark_case(benchmark_case, results_path) - except Exception as e: - # Delete unfinished results if they exist. - if results_path is not None: - results_path.unlink(missing_ok=True) - - if not self.config.keep_going: - raise e - - print(f"Processing of benchmark failed with: {e}") - self.benchmark_errors.append(e) - continue - finally: - # Some grace time. - time.sleep(self.benchmark_grace_time) - - print("Benchmark completed") - - if results_path: - self.finished_benchmarks.append((benchmark_info, results_path)) - - def get_benchmark_results(self) -> BenchmarkResults: - """Returns the finished benchmark results.""" - - results = BenchmarkResults() - results.set_commit(self.config.git_commit_hash) - - finished_benchmarks = sorted( - self.finished_benchmarks, key=lambda pair: str(pair[0]) - ) - for info, path in finished_benchmarks: - benchmark_metrics_json_object = json.loads(path.read_text()) - benchmark_run = BenchmarkRun( - info=info, - metrics=BenchmarkMetrics.from_json_object( - benchmark_metrics_json_object - ), - ) - results.benchmarks.append(benchmark_run) - - return results - - def get_benchmark_result_filenames(self) -> Sequence[pathlib.Path]: - """Returns the json file paths of finished benchmarks.""" - return [path for info, path in self.finished_benchmarks] - - def get_benchmark_errors(self): - """Returns the exceptions captured during benchmarking.""" - return self.benchmark_errors - - def __get_output_paths(self, benchmark_name: str): - """Get output path for the results. The path is None if the benchmark doesn't need to be run.""" - - benchmark_results_filename = None - if self.config.benchmark_tool_dir: - benchmark_results_filename = ( - self.config.benchmark_results_dir / f"{benchmark_name}.json" - ) - - return benchmark_results_filename - - def __get_benchmark_info_from_case( - self, benchmark_case: BenchmarkCase - ) -> BenchmarkInfo: - run_config = benchmark_case.run_config - run_tags = run_config.module_execution_config.tags - gen_config = run_config.module_generation_config - model_source = str(gen_config.imported_model.model.source_type) - compile_tags = gen_config.compile_config.tags - return BenchmarkInfo( - name=run_config.name, - model_name=benchmark_case.model_name, - model_tags=benchmark_case.model_tags, - model_source=model_source, - bench_mode=run_tags, - compile_tags=compile_tags, - driver_info=benchmark_case.driver_info, - device_info=self.device_info, - run_config_id=run_config.composite_id, - ) - - def __get_available_drivers_and_loaders( - self, - ) -> Tuple[Sequence[str], Sequence[str]]: - config_txt_file_path = self.config.benchmark_tool_dir / "build_config.txt" - config_txt_file_lines = config_txt_file_path.read_text().splitlines() - - available_drivers = [] - available_loaders = [] - for line in config_txt_file_lines: - name, value = line.strip().split("=") - if value != "ON": - continue - if name == "IREE_HAL_DRIVER_CUDA": - available_drivers.append("cuda") - elif name == "IREE_HAL_DRIVER_LOCAL_SYNC": - available_drivers.append("local-sync") - elif name == "IREE_HAL_DRIVER_LOCAL_TASK": - available_drivers.append("local-task") - elif name == "IREE_HAL_DRIVER_VULKAN": - available_drivers.append("vulkan") - elif name == "IREE_HAL_EXECUTABLE_LOADER_EMBEDDED_ELF": - available_loaders.append("embedded-elf") - elif name == "IREE_HAL_EXECUTABLE_LOADER_SYSTEM_LIBRARY": - available_loaders.append("system-library") - elif name == "IREE_HAL_EXECUTABLE_LOADER_VMVX_MODULE": - available_loaders.append("vmvx-module") - else: - continue - - if self.verbose: - available_drivers_str = ", ".join(available_drivers) - print(f"Available drivers: {available_drivers_str}") - available_loaders_str = ", ".join(available_loaders) - print(f"Available loaders: {available_loaders_str}") - - return available_drivers, available_loaders diff --git a/build_tools/benchmarks/common/benchmark_driver_test.py b/build_tools/benchmarks/common/benchmark_driver_test.py deleted file mode 100644 index d25fdf0a3d0b..000000000000 --- a/build_tools/benchmarks/common/benchmark_driver_test.py +++ /dev/null @@ -1,288 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import dataclasses -import json -import pathlib -import tempfile -from typing import Optional -import unittest - -from common import benchmark_config -from common.benchmark_suite import BenchmarkCase, BenchmarkSuite -from common.benchmark_driver import BenchmarkDriver -from common import benchmark_definition -from common.benchmark_definition import ( - IREE_DRIVERS_INFOS, - DeviceInfo, - PlatformType, - BenchmarkLatency, - BenchmarkMemory, - BenchmarkMetrics, -) -from e2e_test_framework.definitions import common_definitions, iree_definitions - - -class FakeBenchmarkDriver(BenchmarkDriver): - def __init__( - self, *args, raise_exception_on_case: Optional[BenchmarkCase] = None, **kwargs - ): - super().__init__(*args, **kwargs) - self.raise_exception_on_case = raise_exception_on_case - self.run_benchmark_cases = [] - - def run_benchmark_case( - self, - benchmark_case: BenchmarkCase, - benchmark_results_filename: Optional[pathlib.Path], - ) -> None: - if self.raise_exception_on_case == benchmark_case: - raise Exception("fake exception") - - self.run_benchmark_cases.append(benchmark_case) - - if benchmark_results_filename: - fake_benchmark_metrics = BenchmarkMetrics( - real_time=BenchmarkLatency(0, 0, 0, "ns"), - cpu_time=BenchmarkLatency(0, 0, 0, "ns"), - host_memory=BenchmarkMemory(0, 0, 0, 0, "bytes"), - device_memory=BenchmarkMemory(0, 0, 0, 0, "bytes"), - raw_data={}, - ) - benchmark_results_filename.write_text( - json.dumps(fake_benchmark_metrics.to_json_object()) - ) - - -class BenchmarkDriverTest(unittest.TestCase): - def setUp(self): - self._tmp_dir_obj = tempfile.TemporaryDirectory() - self._root_dir_obj = tempfile.TemporaryDirectory() - - self.tmp_dir = pathlib.Path(self._tmp_dir_obj.name) - (self.tmp_dir / "build_config.txt").write_text( - "IREE_HAL_DRIVER_LOCAL_SYNC=ON\n" - "IREE_HAL_DRIVER_LOCAL_TASK=ON\n" - "IREE_HAL_EXECUTABLE_LOADER_EMBEDDED_ELF=ON\n" - ) - - self.benchmark_results_dir = ( - self.tmp_dir / benchmark_config.BENCHMARK_RESULTS_REL_PATH - ) - self.benchmark_results_dir.mkdir() - - self.config = benchmark_config.BenchmarkConfig( - tmp_dir=self.tmp_dir, - root_benchmark_dir=benchmark_definition.ResourceLocation.build_local_path( - self._root_dir_obj.name - ), - benchmark_results_dir=self.benchmark_results_dir, - git_commit_hash="abcd", - benchmark_tool_dir=self.tmp_dir, - use_compatible_filter=True, - ) - - self.device_info = DeviceInfo( - platform_type=PlatformType.LINUX, - model="Unknown", - cpu_abi="x86_64", - cpu_uarch="CascadeLake", - cpu_features=[], - gpu_name="unknown", - ) - - model_tflite = common_definitions.Model( - id="tflite", - name="model_tflite", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="", - entry_function="predict", - input_types=["1xf32"], - ) - device_spec = common_definitions.DeviceSpec.build( - id="dev", - device_name="test_dev", - architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - device_parameters=[], - tags=[], - ) - compile_target = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_architecture=( - common_definitions.DeviceArchitecture.X86_64_CASCADELAKE - ), - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - gen_config = iree_definitions.ModuleGenerationConfig.build( - imported_model=iree_definitions.ImportedModel.from_model(model_tflite), - compile_config=iree_definitions.CompileConfig.build( - id="comp_a", tags=[], compile_targets=[compile_target] - ), - ) - exec_config_a = iree_definitions.ModuleExecutionConfig.build( - id="exec_a", - tags=["sync"], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_SYNC, - ) - run_config_a = iree_definitions.E2EModelRunConfig.build( - module_generation_config=gen_config, - module_execution_config=exec_config_a, - target_device_spec=device_spec, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - exec_config_b = iree_definitions.ModuleExecutionConfig.build( - id="exec_b", - tags=["task"], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_TASK, - ) - run_config_b = iree_definitions.E2EModelRunConfig.build( - module_generation_config=gen_config, - module_execution_config=exec_config_b, - target_device_spec=device_spec, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - self.case1 = BenchmarkCase( - model_name="model_tflite", - model_tags=[], - bench_mode=["sync"], - target_arch=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu-sync"], - module_dir=benchmark_definition.ResourceLocation.build_local_path("case1"), - benchmark_tool_name="tool", - run_config=run_config_a, - ) - self.case2 = BenchmarkCase( - model_name="model_tflite", - model_tags=[], - bench_mode=["task"], - target_arch=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu"], - module_dir=benchmark_definition.ResourceLocation.build_local_path("case2"), - benchmark_tool_name="tool", - run_config=run_config_b, - ) - - compile_target_rv64 = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - gen_config_rv64 = iree_definitions.ModuleGenerationConfig.build( - imported_model=iree_definitions.ImportedModel.from_model(model_tflite), - compile_config=iree_definitions.CompileConfig.build( - id="comp_rv64", tags=[], compile_targets=[compile_target_rv64] - ), - ) - device_spec_rv64 = common_definitions.DeviceSpec.build( - id="rv64_dev", - device_name="rv64_dev", - architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - device_parameters=[], - tags=[], - ) - run_config_incompatible = iree_definitions.E2EModelRunConfig.build( - module_generation_config=gen_config_rv64, - module_execution_config=exec_config_b, - target_device_spec=device_spec_rv64, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - self.incompatible_case = BenchmarkCase( - model_name="model_tflite", - model_tags=[], - bench_mode=["task"], - target_arch=common_definitions.DeviceArchitecture.RV64_GENERIC, - driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu"], - module_dir=benchmark_definition.ResourceLocation.build_local_path( - "incompatible_case" - ), - benchmark_tool_name="tool", - run_config=run_config_incompatible, - ) - self.benchmark_suite = BenchmarkSuite( - [ - self.case1, - self.case2, - self.incompatible_case, - ] - ) - - def tearDown(self) -> None: - self._tmp_dir_obj.cleanup() - self._root_dir_obj.cleanup() - - def test_run(self): - driver = FakeBenchmarkDriver( - self.device_info, self.config, self.benchmark_suite - ) - - driver.run() - - self.assertEqual(driver.get_benchmark_results().commit, "abcd") - self.assertEqual(len(driver.get_benchmark_results().benchmarks), 2) - self.assertEqual( - driver.get_benchmark_results().benchmarks[0].metrics.raw_data, {} - ) - self.assertEqual( - driver.get_benchmark_result_filenames(), - [ - self.benchmark_results_dir / f"{self.case1.run_config}.json", - self.benchmark_results_dir / f"{self.case2.run_config}.json", - ], - ) - self.assertEqual(driver.get_benchmark_errors(), []) - - def test_run_disable_compatible_filter(self): - self.config.use_compatible_filter = False - driver = FakeBenchmarkDriver( - self.device_info, self.config, self.benchmark_suite - ) - - driver.run() - - self.assertEqual(len(driver.get_benchmark_results().benchmarks), 3) - - def test_run_with_exception_and_keep_going(self): - self.config.keep_going = True - driver = FakeBenchmarkDriver( - self.device_info, - self.config, - self.benchmark_suite, - raise_exception_on_case=self.case1, - ) - - driver.run() - - self.assertEqual(len(driver.get_benchmark_errors()), 1) - self.assertEqual(len(driver.get_benchmark_result_filenames()), 1) - - def test_run_with_previous_benchmarks(self): - benchmark_filename = ( - self.benchmark_results_dir / f"{self.case1.run_config}.json" - ) - benchmark_filename.touch() - config = dataclasses.replace(self.config, continue_from_previous=True) - driver = FakeBenchmarkDriver( - device_info=self.device_info, - benchmark_config=config, - benchmark_suite=self.benchmark_suite, - ) - - driver.run() - - self.assertEqual(len(driver.run_benchmark_cases), 1) - self.assertEqual(len(driver.get_benchmark_result_filenames()), 2) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/benchmarks/common/benchmark_presentation.py b/build_tools/benchmarks/common/benchmark_presentation.py deleted file mode 100644 index b0186b8c4693..000000000000 --- a/build_tools/benchmarks/common/benchmark_presentation.py +++ /dev/null @@ -1,726 +0,0 @@ -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import ( - Any, - Callable, - Dict, - Generic, - List, - Optional, - Sequence, - Tuple, - TypeVar, - Union, -) -import pathlib -import dataclasses -import json -import urllib.parse -import markdown_strings as md -import math - -from common import benchmark_definition, benchmark_thresholds -from common.benchmark_thresholds import ( - BENCHMARK_THRESHOLDS, - COMPILATION_TIME_THRESHOLDS, - TOTAL_ARTIFACT_SIZE_THRESHOLDS, - TOTAL_DISPATCH_SIZE_THRESHOLDS, - BenchmarkThreshold, - ThresholdUnit, -) - -GetMetricFunc = Callable[[Any], Tuple[int, Optional[int]]] - -PERFBOARD_SERIES_PREFIX = "https://perf.iree.dev/serie?IREE?" -BENCHMARK_RESULTS_HEADERS = [ - "Benchmark Name", - "Average Latency (ms)", - "Median Latency (ms)", - "Latency Standard Deviation (ms)", -] -# Since We don't have a structural way to store metric data yet, each metric is -# assigned with a fixed id generated from uuid.uuid4(), to identify the series. -COMPILATION_TIME_METRIC_ID = "e54cd682-c079-4c42-b4ad-d92c4bedea13" -COMPILATION_TIME_SERIES_SUFFIX = "compilation:module:compilation-time" -TOTAL_DISPATCH_SIZE_METRIC_ID = "9e15f7e6-383c-47ec-bd38-ecba55a5f10a" -TOTAL_DISPATCH_SIZE_SERIES_SUFFIX = ( - "compilation:module:component-size:total-dispatch-size" -) -TOTAL_ARTIFACT_SIZE_METRIC_ID = "2c8a9198-c01c-45b9-a7da-69c82cf749f7" -TOTAL_ARTIFACT_SIZE_SERIES_SUFFIX = "compilation:module:total-artifact-size" -STREAM_IR_DISPATCH_COUNT_METRIC_ID = "7b72cd9e-43ed-4078-b6d3-20b810f9e4ad" -STREAM_IR_DISPATCH_COUNT_SERIES_SUFFIX = "compilation:ir:stream-dispatch-count" - - -@dataclass -class AggregateBenchmarkLatency: - """An object for describing aggregate latency numbers for a benchmark.""" - - name: str - benchmark_info: benchmark_definition.BenchmarkInfo - mean_time: int - median_time: int - stddev_time: int - # The average latency time for the base commit to compare against. - base_mean_time: Optional[int] = None - - def __str__(self) -> str: - return self.name - - -@dataclass(frozen=True) -class CompilationMetrics: - """An object for describing the summary of statistics and the reference.""" - - name: str - compilation_info: benchmark_definition.CompilationInfo - compilation_time_ms: int - total_dispatch_component_bytes: int - total_artifact_bytes: int - stream_ir_dispatch_count: int - base_compilation_time_ms: Optional[int] = None - base_total_artifact_bytes: Optional[int] = None - base_total_dispatch_component_bytes: Optional[int] = None - base_stream_ir_dispatch_count: Optional[int] = None - - def __str__(self) -> str: - return self.name - - -T = TypeVar("T") - - -class MetricsToTableMapper(ABC, Generic[T]): - """Abstract class to help map benchmark metrics to table. - - It contains a set of methods to help table generator get the required - information for a metric. For example, extract the current and base metric - value, the metric thresholds, the table header of the metrics, ... - """ - - @abstractmethod - def update_base_value(self, obj: T, base_value: Any) -> T: - """Sets the base value and returns the updated metric object.""" - raise NotImplementedError() - - @abstractmethod - def get_current_and_base_value(self, obj: T) -> Tuple[int, Optional[int]]: - """Returns the current and base (can be None) value.""" - raise NotImplementedError() - - def get_series_id(self, benchmark_id: str) -> str: - """Returns the dashboard series id.""" - return f"{benchmark_id}-{self.get_metric_id()}" - - @abstractmethod - def get_metric_id(self) -> str: - """Returns the dashboard series id.""" - raise NotImplementedError() - - @abstractmethod - def get_series_name(self, name: str) -> str: - """Returns the dashboard series name.""" - raise NotImplementedError() - - @abstractmethod - def get_unit(self) -> str: - """Returns the unit of the metric value.""" - raise NotImplementedError() - - @abstractmethod - def get_table_header(self) -> str: - """Returns the header of the table.""" - raise NotImplementedError() - - @staticmethod - @abstractmethod - def get_metric_thresholds() -> Sequence[BenchmarkThreshold]: - raise NotImplementedError() - - @staticmethod - @abstractmethod - def get_table_title() -> str: - raise NotImplementedError() - - -class CompilationTimeToTable(MetricsToTableMapper[CompilationMetrics]): - """Helper to map CompilationMetrics to compilation time column.""" - - def update_base_value( - self, compile_metrics: CompilationMetrics, base_value: Any - ) -> CompilationMetrics: - return dataclasses.replace(compile_metrics, base_compilation_time_ms=base_value) - - def get_current_and_base_value( - self, compile_metrics: CompilationMetrics - ) -> Tuple[int, Optional[int]]: - return ( - compile_metrics.compilation_time_ms, - compile_metrics.base_compilation_time_ms, - ) - - def get_metric_id(self) -> str: - return COMPILATION_TIME_METRIC_ID - - def get_series_name(self, name: str) -> str: - return f"{name} [{COMPILATION_TIME_SERIES_SUFFIX}]" - - def get_unit(self) -> str: - return "ms" - - def get_table_header(self) -> str: - return f"Compilation Time ({self.get_unit()})" - - @staticmethod - def get_metric_thresholds() -> Sequence[BenchmarkThreshold]: - return COMPILATION_TIME_THRESHOLDS - - @staticmethod - def get_table_title() -> str: - return "Compilation Times" - - -class TotalDispatchSizeToTable(MetricsToTableMapper[CompilationMetrics]): - """Helper to map CompilationMetrics to total dispatch size column.""" - - def update_base_value( - self, compile_metrics: CompilationMetrics, base_value: Any - ) -> CompilationMetrics: - return dataclasses.replace( - compile_metrics, base_total_dispatch_component_bytes=base_value - ) - - def get_current_and_base_value( - self, compile_metrics: CompilationMetrics - ) -> Tuple[int, Optional[int]]: - return ( - compile_metrics.total_dispatch_component_bytes, - compile_metrics.base_total_dispatch_component_bytes, - ) - - def get_metric_id(self) -> str: - return TOTAL_DISPATCH_SIZE_METRIC_ID - - def get_series_name(self, name: str) -> str: - return f"{name} [{TOTAL_DISPATCH_SIZE_SERIES_SUFFIX}]" - - def get_unit(self) -> str: - return "bytes" - - def get_table_header(self) -> str: - return f"Total Dispatch Size ({self.get_unit()})" - - @staticmethod - def get_metric_thresholds() -> Sequence[BenchmarkThreshold]: - return TOTAL_DISPATCH_SIZE_THRESHOLDS - - @staticmethod - def get_table_title() -> str: - return "Total Dispatch Sizes" - - -class TotalArtifactSizeToTable(MetricsToTableMapper[CompilationMetrics]): - """Helper to map CompilationMetrics to total artifact size column.""" - - def update_base_value( - self, compile_metrics: CompilationMetrics, base_value: Any - ) -> CompilationMetrics: - return dataclasses.replace( - compile_metrics, base_total_artifact_bytes=base_value - ) - - def get_current_and_base_value( - self, compile_metrics: CompilationMetrics - ) -> Tuple[int, Optional[int]]: - return ( - compile_metrics.total_artifact_bytes, - compile_metrics.base_total_artifact_bytes, - ) - - def get_metric_id(self) -> str: - return TOTAL_ARTIFACT_SIZE_METRIC_ID - - def get_series_name(self, name: str) -> str: - return f"{name} [{TOTAL_ARTIFACT_SIZE_SERIES_SUFFIX}]" - - def get_unit(self) -> str: - return "bytes" - - def get_table_header(self) -> str: - return f"Total Artifact Size ({self.get_unit()})" - - @staticmethod - def get_metric_thresholds() -> Sequence[BenchmarkThreshold]: - return TOTAL_ARTIFACT_SIZE_THRESHOLDS - - @staticmethod - def get_table_title() -> str: - return "Total Artifact Sizes" - - -class StreamIRDispatchCountToTable(MetricsToTableMapper[CompilationMetrics]): - """Helper to map CompilationMetrics to Stream IR Dispatch Count column.""" - - def update_base_value( - self, compile_metrics: CompilationMetrics, base_value: Any - ) -> CompilationMetrics: - return dataclasses.replace( - compile_metrics, base_stream_ir_dispatch_count=base_value - ) - - def get_current_and_base_value( - self, compile_metrics: CompilationMetrics - ) -> Tuple[int, Optional[int]]: - return ( - compile_metrics.stream_ir_dispatch_count, - compile_metrics.base_stream_ir_dispatch_count, - ) - - def get_metric_id(self) -> str: - return STREAM_IR_DISPATCH_COUNT_METRIC_ID - - def get_series_name(self, name: str) -> str: - return f"{name} [{STREAM_IR_DISPATCH_COUNT_SERIES_SUFFIX}]" - - def get_unit(self) -> str: - return "number" - - def get_table_header(self) -> str: - return f"Stream IR Dispatch Count (# of cmd.dispatch ops)" - - @staticmethod - def get_metric_thresholds() -> Sequence[BenchmarkThreshold]: - return benchmark_thresholds.STREAM_IR_DISPATCH_COUNT_THRESHOLDS - - @staticmethod - def get_table_title() -> str: - return "Stream IR Dispatch Count (# of cmd.dispatch ops)" - - -COMPILATION_METRICS_TO_TABLE_MAPPERS: List[MetricsToTableMapper[CompilationMetrics]] = [ - CompilationTimeToTable(), - TotalDispatchSizeToTable(), - TotalArtifactSizeToTable(), - StreamIRDispatchCountToTable(), -] - - -def aggregate_all_benchmarks( - benchmark_files: Sequence[pathlib.Path], expected_pr_commit: Optional[str] = None -) -> Dict[str, AggregateBenchmarkLatency]: - """Aggregates all benchmarks in the given files. - - Args: - - benchmark_files: A list of JSON files, each can be decoded as a - BenchmarkResults. - - expected_pr_commit: An optional Git commit SHA to match against. - - Returns: - - A dict of benchmark names to AggregateBenchmarkLatency numbers. - """ - - aggregate_results = {} - benchmark_names = set() - for benchmark_file in benchmark_files: - file_results = benchmark_definition.BenchmarkResults.from_json_str( - benchmark_file.read_text() - ) - - if (expected_pr_commit is not None) and ( - file_results.commit != expected_pr_commit - ): - raise ValueError("Inconsistent pull request commit") - - for benchmark_index in range(len(file_results.benchmarks)): - benchmark_run = file_results.benchmarks[benchmark_index] - - series_name = str(benchmark_run.info) - # Make sure each benchmark has a unique name. - if series_name in benchmark_names: - raise ValueError(f"Duplicated benchmark name: {series_name}") - benchmark_names.add(series_name) - - series_id = benchmark_run.info.run_config_id - if series_id in aggregate_results: - raise ValueError(f"Duplicated benchmark id: {series_id}") - - aggregate_results[series_id] = AggregateBenchmarkLatency( - name=series_name, - benchmark_info=benchmark_run.info, - mean_time=benchmark_run.metrics.real_time.mean, - median_time=benchmark_run.metrics.real_time.median, - stddev_time=benchmark_run.metrics.real_time.stddev, - ) - - return aggregate_results - - -def collect_all_compilation_metrics( - compile_stats_files: Sequence[pathlib.Path], - expected_pr_commit: Optional[str] = None, -) -> Dict[str, CompilationMetrics]: - """Collects all compilation statistics in the given files. - - Args: - compile_stats_files: A list of JSON files, each can be decoded as a - CompilationResults. - expected_pr_commit: An optional Git commit SHA to match against. - - Returns: - A dict of benchmark names to CompilationMetrics. - """ - compile_metrics = {} - target_names = set() - for compile_stats_file in compile_stats_files: - with compile_stats_file.open("r") as f: - file_results = benchmark_definition.CompilationResults.from_json_object( - json.load(f) - ) - - if (expected_pr_commit is not None) and ( - file_results.commit != expected_pr_commit - ): - raise ValueError("Inconsistent pull request commit") - - for compile_stats in file_results.compilation_statistics: - component_sizes = compile_stats.module_component_sizes - stream_dispatch_count = compile_stats.ir_stats.stream_dispatch_count - - target_name = str(compile_stats.compilation_info) - if target_name in target_names: - raise ValueError(f"Duplicated target name: {target_name}") - target_names.add(target_name) - - target_id = compile_stats.compilation_info.gen_config_id - if target_id in compile_metrics: - raise ValueError(f"Duplicated target id: {target_id}") - - compile_metrics[target_id] = CompilationMetrics( - name=target_name, - compilation_info=compile_stats.compilation_info, - compilation_time_ms=compile_stats.compilation_time_ms, - total_artifact_bytes=component_sizes.file_bytes, - total_dispatch_component_bytes=component_sizes.total_dispatch_component_bytes, - stream_ir_dispatch_count=stream_dispatch_count, - ) - - return compile_metrics - - -def make_series_link(name: str, series_id: str) -> str: - """Add link to the given benchmark name. - - Args: - name: the text to show on the link. - series_id: the dashboard series id. - """ - url = PERFBOARD_SERIES_PREFIX + urllib.parse.quote(series_id, safe="()[]@,") - return md.link(name, url) - - -def _add_header_and_get_markdown_table( - headers: Sequence[str], rows: Sequence[Tuple], size_cut: Optional[int] = None -) -> str: - """Generates a markdown table with headers. - - Args: - headers: list of table headers. - rows: list of rows. Each row is a tuple with the same length as headers. - size_cut: If not None, only show the top N results for each table. - """ - - total_size = len(rows) - if size_cut is not None: - rows = rows[0:size_cut] - - columns = [[header] for header in headers] - for row in rows: - for column, item in zip(columns, row): - column.append(item) - - table_str = md.table(columns) - if size_cut is not None and size_cut < total_size: - table_str += "\n\n" - table_str += md.italics(f"[Top {size_cut} out of {total_size} results showed]") - return table_str - - -T = TypeVar("T") - - -def _categorize_on_single_metric( - metrics_map: Dict[str, T], - metric_func: GetMetricFunc, - thresholds: Sequence[BenchmarkThreshold], - metric_unit: str, -) -> Tuple[Dict[str, T], Dict[str, T], Dict[str, T], Dict[str, T]]: - """Categorize the metrics object into regressed, improved, similar, and the - raw group (the group with no base to compare to). - - Args: - metrics_map: map of (series_id, metrics object). - metric_func: the function returns current and base value of the metric. - thresholds: list of threshold settings to match for categorizing. - Returns: - A tuple of (regressed, improved, similar, raw) groups. - """ - - regressed_map = {} - improved_map = {} - similar_map = {} - raw_map = {} - for series_id, metrics_obj in metrics_map.items(): - current, base = metric_func(metrics_obj) - if base is None: - raw_map[series_id] = metrics_obj - continue - - series_name = str(metrics_obj) - similar_threshold = None - for threshold in thresholds: - if threshold.regex.match(series_name): - similar_threshold = threshold - break - if similar_threshold is None: - raise ValueError(f"No matched threshold setting for: {series_name}") - - if similar_threshold.unit == ThresholdUnit.PERCENTAGE: - ratio = abs(current - base) / base * 100 - elif similar_threshold.unit.value == metric_unit: - ratio = abs(current - base) - else: - raise ValueError( - f"Mismatch between metric unit '{metric_unit}' and threshold unit '{similar_threshold.unit.value}'" - ) - - if ratio <= similar_threshold.threshold: - similar_map[series_id] = metrics_obj - elif current > base: - regressed_map[series_id] = metrics_obj - else: - improved_map[series_id] = metrics_obj - - return (regressed_map, improved_map, similar_map, raw_map) - - -def _get_fixed_point_str(value: Union[int, float], digits=3) -> str: - if isinstance(value, int) or value.is_integer(): - return str(math.floor(value)) - return f"{{:.{digits}f}}".format(value) - - -def _get_compare_text(current: float, base: Optional[int]) -> str: - """Generates the text of comparison between current and base value. Returns - the current value if the base value is None. - """ - # If base is None, don't need to do compare. - if base is None: - return f"{_get_fixed_point_str(current)}" - - ratio = abs(current - base) / base - direction = "↑" if current > base else ("↓" if current < base else "") - return f"{_get_fixed_point_str(current)} (vs. {_get_fixed_point_str(base)}, {ratio:.2%}{direction})" - - -def _sort_benchmarks_and_get_table( - benchmarks: Dict[str, AggregateBenchmarkLatency], size_cut: Optional[int] = None -) -> str: - """Sorts all benchmarks according to the improvement/regression ratio and - returns a markdown table for it. - - Args: - benchmarks_map: map of (series_id, benchmark object). - size_cut: If not None, only show the top N results for each table. - """ - sorted_rows = [] - for series_id, benchmark in benchmarks.items(): - current = benchmark.mean_time / 1e6 - base = benchmark.base_mean_time / 1e6 - ratio = abs(current - base) / base - str_mean = _get_compare_text(current, base) - clickable_name = make_series_link(benchmark.name, series_id) - sorted_rows.append( - ( - ratio, - ( - clickable_name, - str_mean, - f"{_get_fixed_point_str(benchmark.median_time / 1e6)}", - f"{_get_fixed_point_str(benchmark.stddev_time / 1e6)}", - ), - ) - ) - sorted_rows.sort(key=lambda row: row[0], reverse=True) - - return _add_header_and_get_markdown_table( - headers=BENCHMARK_RESULTS_HEADERS, - rows=[row[1] for row in sorted_rows], - size_cut=size_cut, - ) - - -def categorize_benchmarks_into_tables( - benchmarks: Dict[str, AggregateBenchmarkLatency], size_cut: Optional[int] = None -) -> str: - """Splits benchmarks into regressed/improved/similar/raw categories and - returns their markdown tables. - - If size_cut is None, the table includes regressed/improved/similar/raw - categories; otherwise, the table includes regressed/improved/raw categories. - - Args: - benchmarks: A dictionary of benchmark names to its aggregate info. - size_cut: If not None, only show the top N results for each table. - """ - regressed, improved, similar, raw = _categorize_on_single_metric( - benchmarks, - lambda results: (results.mean_time, results.base_mean_time), - BENCHMARK_THRESHOLDS, - "ns", - ) - - tables = [] - if regressed: - tables.append(md.header("Regressed Latencies 🚩", 3)) - tables.append(_sort_benchmarks_and_get_table(regressed, size_cut)) - if improved: - tables.append(md.header("Improved Latencies 🎉", 3)) - tables.append(_sort_benchmarks_and_get_table(improved, size_cut)) - # If we want to abbreviate, similar results won't be interesting. - if similar and size_cut is None: - tables.append(md.header("Similar Latencies", 3)) - tables.append(_sort_benchmarks_and_get_table(similar, size_cut)) - if raw: - tables.append(md.header("Raw Latencies", 3)) - raw_list = [ - ( - make_series_link(name=v.name, series_id=k), - f"{_get_fixed_point_str(v.mean_time / 1e6)}", - f"{_get_fixed_point_str(v.median_time / 1e6)}", - f"{_get_fixed_point_str(v.stddev_time / 1e6)}", - ) - for k, v in raw.items() - ] - tables.append( - _add_header_and_get_markdown_table( - BENCHMARK_RESULTS_HEADERS, raw_list, size_cut=size_cut - ) - ) - return "\n\n".join(tables) - - -def _sort_metrics_objects_and_get_table( - metrics_objs: Dict[str, T], - mapper: MetricsToTableMapper[T], - headers: Sequence[str], - size_cut: Optional[int] = None, -) -> str: - """Sorts all metrics objects according to the improvement/regression ratio and - returns a markdown table for it. - - Args: - metrics_objs: map of (target_id, CompilationMetrics). All objects must - contain base value. - mapper: MetricsToTableMapper for metrics_objs. - headers: list of table headers. - size_cut: If not None, only show the top N results for each table. - """ - sorted_rows = [] - for target_id, metrics_obj in metrics_objs.items(): - current, base = mapper.get_current_and_base_value(metrics_obj) - if base is None: - raise AssertionError("Base can't be None for sorting.") - ratio = abs(current - base) / base - sorted_rows.append( - ( - ratio, - ( - make_series_link(str(metrics_obj), mapper.get_series_id(target_id)), - _get_compare_text(current, base), - ), - ) - ) - sorted_rows.sort(key=lambda row: row[0], reverse=True) - - return _add_header_and_get_markdown_table( - headers=headers, rows=[row[1] for row in sorted_rows], size_cut=size_cut - ) - - -def categorize_compilation_metrics_into_tables( - compile_metrics_map: Dict[str, CompilationMetrics], size_cut: Optional[int] = None -) -> str: - """Splits compilation metrics into regressed/improved/all categories - and returns their markdown tables. - - If size_cut is None, the table includes regressed/improved/all categories; - otherwise, the table includes regressed/improved categories. - - Args: - compile_metrics_map: A dictionary of benchmark names to its compilation - metrics. - size_cut: If not None, only show the top N results for each table. - """ - - tables = [] - for mapper in COMPILATION_METRICS_TO_TABLE_MAPPERS: - regressed, improved, _, _ = _categorize_on_single_metric( - compile_metrics_map, - mapper.get_current_and_base_value, - mapper.get_metric_thresholds(), - mapper.get_unit(), - ) - - table_title = mapper.get_table_title() - table_header = mapper.get_table_header() - if regressed: - tables.append(md.header(f"Regressed {table_title} 🚩", 3)) - tables.append( - _sort_metrics_objects_and_get_table( - metrics_objs=regressed, - mapper=mapper, - headers=["Benchmark Name", table_header], - size_cut=size_cut, - ) - ) - if improved: - tables.append(md.header(f"Improved {table_title} 🎉", 3)) - tables.append( - _sort_metrics_objects_and_get_table( - metrics_objs=improved, - mapper=mapper, - headers=["Benchmark Name", table_header], - size_cut=size_cut, - ) - ) - - # If we want to abbreviate, similar results won't be interesting. - if size_cut is None and compile_metrics_map: - tables.append(md.header("All Compilation Metrics", 3)) - headers = ["Benchmark Name"] + [ - mapper.get_table_header() for mapper in COMPILATION_METRICS_TO_TABLE_MAPPERS - ] - rows = [] - for target_id, metrics in compile_metrics_map.items(): - row = [metrics.name] - for mapper in COMPILATION_METRICS_TO_TABLE_MAPPERS: - current, base = mapper.get_current_and_base_value(metrics) - row.append( - make_series_link( - _get_compare_text(current, base), - mapper.get_series_id(target_id), - ) - ) - rows.append(tuple(row)) - - tables.append( - _add_header_and_get_markdown_table(headers, rows, size_cut=size_cut) - ) - - return "\n\n".join(tables) diff --git a/build_tools/benchmarks/common/benchmark_suite.py b/build_tools/benchmarks/common/benchmark_suite.py deleted file mode 100644 index 551cfec07694..000000000000 --- a/build_tools/benchmarks/common/benchmark_suite.py +++ /dev/null @@ -1,276 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Utilities for handling the benchmark suite. - -See https://iree.dev/developers/performance/benchmark-suites/ for how to build -the benchmark suite. -""" - -import re - -import dataclasses -from dataclasses import dataclass -from typing import Dict, List, Optional, Sequence, Tuple -from common import benchmark_definition -from common.benchmark_definition import IREE_DRIVERS_INFOS, DriverInfo -from e2e_test_artifacts import iree_artifacts -from e2e_test_framework.definitions import common_definitions, iree_definitions -from e2e_test_framework import serialization - - -@dataclass -class BenchmarkCase: - """Represents a benchmark case. - - model_name: the source model, e.g., 'MobileSSD'. - model_tags: the source model tags, e.g., ['f32']. - bench_mode: the benchmark mode, e.g., '1-thread,big-core'. - target_arch: the target CPU/GPU architature. - driver_info: the IREE driver configuration. - benchmark_tool_name: the benchmark tool, e.g., 'iree-benchmark-module'. - run_config: the run config from e2e test framework. - module_dir: path/URL of the module directory. - input_uri: URI to find the input npy. - expected_output_uri: URI to find the expected output npy. - """ - - model_name: str - model_tags: Sequence[str] - bench_mode: Sequence[str] - target_arch: common_definitions.DeviceArchitecture - driver_info: DriverInfo - benchmark_tool_name: str - run_config: iree_definitions.E2EModelRunConfig - module_dir: benchmark_definition.ResourceLocation - input_uri: Optional[str] = None - expected_output_uri: Optional[str] = None - verify_params: List[str] = dataclasses.field(default_factory=list) - external_param_urls: List[str] = dataclasses.field(default_factory=list) - - -# A map from execution config to driver info. This is temporary during migration -# before we can drop the DriverInfo. -EXECUTION_CONFIG_TO_DRIVER_INFO_KEY_MAP: Dict[ - Tuple[iree_definitions.RuntimeDriver, iree_definitions.RuntimeLoader], str -] = { - ( - iree_definitions.RuntimeDriver.LOCAL_TASK, - iree_definitions.RuntimeLoader.EMBEDDED_ELF, - ): "iree-llvm-cpu", - ( - iree_definitions.RuntimeDriver.LOCAL_SYNC, - iree_definitions.RuntimeLoader.EMBEDDED_ELF, - ): "iree-llvm-cpu-sync", - ( - iree_definitions.RuntimeDriver.LOCAL_TASK, - iree_definitions.RuntimeLoader.VMVX_MODULE, - ): "iree-vmvx", - ( - iree_definitions.RuntimeDriver.LOCAL_SYNC, - iree_definitions.RuntimeLoader.VMVX_MODULE, - ): "iree-vmvx-sync", - ( - iree_definitions.RuntimeDriver.VULKAN, - iree_definitions.RuntimeLoader.NONE, - ): "iree-vulkan", - ( - iree_definitions.RuntimeDriver.CUDA, - iree_definitions.RuntimeLoader.NONE, - ): "iree-cuda", -} - - -class BenchmarkSuite(object): - """Represents the benchmarks in benchmark suite directory.""" - - def __init__(self, benchmark_cases: Sequence[BenchmarkCase]): - """Construct a benchmark suite. - - Args: - benchmark_cases: list of benchmark cases. - """ - self.benchmark_cases = list(benchmark_cases) - - def filter_benchmarks( - self, - available_drivers: Optional[Sequence[str]] = None, - available_loaders: Optional[Sequence[str]] = None, - target_architectures: Optional[ - Sequence[common_definitions.DeviceArchitecture] - ] = None, - driver_filter: Optional[str] = None, - mode_filter: Optional[str] = None, - model_name_filter: Optional[str] = None, - ) -> Sequence[BenchmarkCase]: - """Filters benchmarks. - Args: - available_drivers: list of drivers supported by the tools. None means to - match any driver. - available_loaders: list of executable loaders supported by the tools. - None means to match any loader. - target_architectures: list of target architectures to be included. None - means no filter. - driver_filter: driver filter regex. - mode_filter: benchmark mode regex. - model_name_filter: model name regex. - Returns: - A list of matched benchmark cases. - """ - - chosen_cases = [] - for benchmark_case in self.benchmark_cases: - driver_info = benchmark_case.driver_info - - driver_name = driver_info.driver_name - matched_available_driver = ( - available_drivers is None or driver_name in available_drivers - ) - matched_driver_filter = ( - driver_filter is None - or re.match(driver_filter, driver_name) is not None - ) - matched_driver = matched_available_driver and matched_driver_filter - - matched_loader = ( - not driver_info.loader_name - or available_loaders is None - or (driver_info.loader_name in available_loaders) - ) - - if target_architectures is None: - matched_arch = True - else: - matched_arch = benchmark_case.target_arch in target_architectures - - bench_mode = ",".join(benchmark_case.bench_mode) - matched_mode = ( - mode_filter is None or re.match(mode_filter, bench_mode) is not None - ) - - model_name_with_tags = benchmark_case.model_name - if len(benchmark_case.model_tags) > 0: - model_name_with_tags += f"-{','.join(benchmark_case.model_tags)}" - matched_model_name = ( - model_name_filter is None - or re.match(model_name_filter, model_name_with_tags) is not None - ) - - if ( - matched_driver - and matched_loader - and matched_arch - and matched_model_name - and matched_mode - ): - chosen_cases.append(benchmark_case) - - return chosen_cases - - @staticmethod - def load_from_run_configs( - run_configs: Sequence[iree_definitions.E2EModelRunConfig], - root_benchmark_dir: benchmark_definition.ResourceLocation, - ): - """Loads the benchmarks from the run configs. - - Args: - run_configs: list of benchmark run configs. - root_benchmark_dir: path/URL of the root benchmark directory. - Returns: - A benchmark suite. - """ - - benchmark_cases = [] - for run_config in run_configs: - module_gen_config = run_config.module_generation_config - module_exec_config = run_config.module_execution_config - target_device_spec = run_config.target_device_spec - - driver_info_key = EXECUTION_CONFIG_TO_DRIVER_INFO_KEY_MAP.get( - (module_exec_config.driver, module_exec_config.loader) - ) - if driver_info_key is None: - raise ValueError( - f"Can't map execution config to driver info: {module_exec_config}." - ) - driver_info = IREE_DRIVERS_INFOS[driver_info_key] - - target_arch = target_device_spec.architecture - model = module_gen_config.imported_model.model - - module_rel_dir = iree_artifacts.get_module_dir_path(module_gen_config) - module_dir = root_benchmark_dir / module_rel_dir - - benchmark_case = BenchmarkCase( - model_name=model.name, - model_tags=model.tags, - bench_mode=module_exec_config.tags, - target_arch=target_arch, - driver_info=driver_info, - benchmark_tool_name=run_config.tool.value, - module_dir=module_dir, - input_uri=model.input_url, - expected_output_uri=model.expected_output_url, - verify_params=model.verify_params, - external_param_urls=model.external_param_urls, - run_config=run_config, - ) - benchmark_cases.append(benchmark_case) - - return BenchmarkSuite(benchmark_cases=benchmark_cases) - - -def get_run_configs_by_target_and_shard( - benchmark_groups: Dict, target_device_name: str, shard_index: Optional[int] = None -): - """Returns a flat list of run_configs from `benchmark_groups`, filtered by the given `target_device_name`. - If a `shard_index` is given, only the run configs for the given shard are returned, otherwise all the run configs are returned. - """ - benchmark_group = benchmark_groups.get(target_device_name) - if benchmark_group is None: - raise ValueError( - "Target device '{}' not found in the benchmark config.".format( - target_device_name - ) - ) - - if shard_index is None: - # In case no shard index was given we will run ALL benchmarks from ALL shards - packed_run_configs = [ - shard["run_configs"] for shard in benchmark_group["shards"] - ] - else: - # Otherwise we will only run the benchmarks from the given shard - benchmark_shard = next( - ( - shard - for shard in benchmark_group["shards"] - if shard["index"] == shard_index - ), - None, - ) - if benchmark_shard is None: - raise ValueError( - "Given shard (index={}) not found in the benchmark config group. Available indexes: [{}].".format( - shard_index, - ", ".join( - str(shard["index"]) for shard in benchmark_group["shards"] - ), - ) - ) - packed_run_configs = [benchmark_shard["run_configs"]] - - # When no `shard_index` is given we might have more than one shard to process. - # We do this by deserializing the `run_config` field from each shard separately - # and then merge the unpacked flat lists of `E2EModelRunConfig`. - return [ - run_config - for packed_run_config in packed_run_configs - for run_config in serialization.unpack_and_deserialize( - data=packed_run_config, - root_type=List[iree_definitions.E2EModelRunConfig], - ) - ] diff --git a/build_tools/benchmarks/common/benchmark_suite_test.py b/build_tools/benchmarks/common/benchmark_suite_test.py deleted file mode 100644 index b9035e2efba9..000000000000 --- a/build_tools/benchmarks/common/benchmark_suite_test.py +++ /dev/null @@ -1,334 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import pathlib -import unittest -from common import benchmark_definition -from common.benchmark_definition import IREE_DRIVERS_INFOS -from common.benchmark_suite import BenchmarkCase, BenchmarkSuite -from e2e_test_framework.definitions import common_definitions, iree_definitions -from e2e_test_artifacts import iree_artifacts - - -class BenchmarkSuiteTest(unittest.TestCase): - def test_filter_benchmarks(self): - model = common_definitions.Model( - id="model", - name="model", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url="", - entry_function="predict", - input_types=["1xf32"], - ) - exec_config = iree_definitions.ModuleExecutionConfig.build( - id="exec", - tags=[], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_SYNC, - ) - device_spec = common_definitions.DeviceSpec.build( - id="dev", - device_name="dev", - architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - device_parameters=[], - tags=[], - ) - compile_target = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - dummy_run_config = iree_definitions.E2EModelRunConfig.build( - module_generation_config=iree_definitions.ModuleGenerationConfig.build( - imported_model=iree_definitions.ImportedModel.from_model(model), - compile_config=iree_definitions.CompileConfig.build( - id="1", tags=[], compile_targets=[compile_target] - ), - ), - module_execution_config=exec_config, - target_device_spec=device_spec, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - - case1 = BenchmarkCase( - model_name="deepnet", - model_tags=[], - bench_mode=["1-thread", "full-inference"], - target_arch=common_definitions.DeviceArchitecture.ARMV8_2_A_GENERIC, - driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu"], - module_dir=benchmark_definition.ResourceLocation.build_local_path("case1"), - benchmark_tool_name="tool", - run_config=dummy_run_config, - ) - case2 = BenchmarkCase( - model_name="deepnetv2", - model_tags=["f32"], - bench_mode=["full-inference"], - target_arch=common_definitions.DeviceArchitecture.ARM_VALHALL, - driver_info=IREE_DRIVERS_INFOS["iree-vulkan"], - module_dir=benchmark_definition.ResourceLocation.build_local_path("case2"), - benchmark_tool_name="tool", - run_config=dummy_run_config, - ) - case3 = BenchmarkCase( - model_name="deepnetv3", - model_tags=["f32"], - bench_mode=["full-inference"], - target_arch=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu-sync"], - module_dir=benchmark_definition.ResourceLocation.build_local_path("case3"), - benchmark_tool_name="tool", - run_config=dummy_run_config, - ) - suite = BenchmarkSuite([case1, case2, case3]) - - cpu_and_gpu_benchmarks = suite.filter_benchmarks( - available_drivers=["local-task", "vulkan"], - available_loaders=["embedded-elf"], - target_architectures=[ - common_definitions.DeviceArchitecture.ARMV8_2_A_GENERIC, - common_definitions.DeviceArchitecture.ARM_VALHALL, - ], - driver_filter=None, - mode_filter=".*full-inference.*", - model_name_filter="deepnet.*", - ) - gpu_benchmarks = suite.filter_benchmarks( - available_drivers=["local-task", "vulkan"], - available_loaders=["embedded-elf"], - target_architectures=[ - common_definitions.DeviceArchitecture.ARM_VALHALL, - ], - driver_filter="vulkan", - mode_filter=".*full-inference.*", - model_name_filter="deepnet.*", - ) - all_benchmarks = suite.filter_benchmarks( - available_drivers=None, - target_architectures=None, - driver_filter=None, - mode_filter=None, - model_name_filter=None, - ) - - self.assertEqual(cpu_and_gpu_benchmarks, [case1, case2]) - self.assertEqual(gpu_benchmarks, [case2]) - self.assertEqual(all_benchmarks, [case1, case2, case3]) - - def test_load_from_run_configs(self): - model_tflite = common_definitions.Model( - id="tflite", - name="model_tflite", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="", - entry_function="predict", - input_types=["1xf32"], - ) - model_tf = common_definitions.Model( - id="tf", - name="model_tf", - tags=["fp32"], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url="", - entry_function="predict", - input_types=["1xf32"], - input_url="https://abc/inputs_npy.tgz", - expected_output_url="https://abc/outputs_npy.tgz", - ) - exec_config_a = iree_definitions.ModuleExecutionConfig.build( - id="exec_a", - tags=["defaults"], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_SYNC, - ) - exec_config_b = iree_definitions.ModuleExecutionConfig.build( - id="exec_b", - tags=["experimental"], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_TASK, - ) - device_spec_a = common_definitions.DeviceSpec.build( - id="dev_a", - device_name="a", - architecture=common_definitions.DeviceArchitecture.RV32_GENERIC, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - device_parameters=[], - tags=[], - ) - device_spec_b = common_definitions.DeviceSpec.build( - id="dev_b", - device_name="b", - architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - device_parameters=[], - tags=[], - ) - compile_target = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - run_config_a = iree_definitions.E2EModelRunConfig.build( - module_generation_config=iree_definitions.ModuleGenerationConfig.build( - imported_model=iree_definitions.ImportedModel.from_model(model_tflite), - compile_config=iree_definitions.CompileConfig.build( - id="1", tags=[], compile_targets=[compile_target] - ), - ), - module_execution_config=exec_config_a, - target_device_spec=device_spec_a, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - run_config_b = iree_definitions.E2EModelRunConfig.build( - module_generation_config=iree_definitions.ModuleGenerationConfig.build( - imported_model=iree_definitions.ImportedModel.from_model(model_tflite), - compile_config=iree_definitions.CompileConfig.build( - id="2", tags=[], compile_targets=[compile_target] - ), - ), - module_execution_config=exec_config_b, - target_device_spec=device_spec_b, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - run_config_c = iree_definitions.E2EModelRunConfig.build( - module_generation_config=iree_definitions.ModuleGenerationConfig.build( - imported_model=iree_definitions.ImportedModel.from_model(model_tf), - compile_config=iree_definitions.CompileConfig.build( - id="3", tags=[], compile_targets=[compile_target] - ), - ), - module_execution_config=exec_config_a, - target_device_spec=device_spec_a, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - run_configs = [run_config_a, run_config_b, run_config_c] - root_dir = pathlib.Path("root") - - suite = BenchmarkSuite.load_from_run_configs( - run_configs=run_configs, - root_benchmark_dir=benchmark_definition.ResourceLocation.build_local_path( - root_dir - ), - ) - - loaded_run_configs = [case.run_config for case in suite.filter_benchmarks()] - self.assertEqual( - loaded_run_configs, - [ - run_config_a, - run_config_b, - run_config_c, - ], - ) - run_config_c_case_dir = pathlib.Path( - iree_artifacts.get_module_dir_path( - run_config_c.module_generation_config, root_dir - ) - ) - self.assertEqual( - suite.filter_benchmarks( - target_architectures=[ - common_definitions.DeviceArchitecture.RV32_GENERIC - ], - model_name_filter="model_tf.*fp32", - mode_filter="defaults", - ), - [ - BenchmarkCase( - model_name=model_tf.name, - model_tags=model_tf.tags, - bench_mode=exec_config_a.tags, - target_arch=common_definitions.DeviceArchitecture.RV32_GENERIC, - driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu-sync"], - benchmark_tool_name="iree-benchmark-module", - module_dir=benchmark_definition.ResourceLocation.build_local_path( - run_config_c_case_dir - ), - input_uri=model_tf.input_url, - expected_output_uri=model_tf.expected_output_url, - run_config=run_config_c, - ) - ], - ) - - def test_load_from_run_configs_with_root_url(self): - model_tflite = common_definitions.Model( - id="tflite", - name="model", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="", - entry_function="predict", - input_types=["1xf32"], - ) - exec_config_a = iree_definitions.ModuleExecutionConfig.build( - id="exec_a", - tags=["defaults"], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_SYNC, - ) - device_spec_a = common_definitions.DeviceSpec.build( - id="dev_a", - device_name="a", - architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - device_parameters=[], - tags=[], - ) - compile_target = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - run_config_a = iree_definitions.E2EModelRunConfig.build( - module_generation_config=iree_definitions.ModuleGenerationConfig.build( - imported_model=iree_definitions.ImportedModel.from_model(model_tflite), - compile_config=iree_definitions.CompileConfig.build( - id="1", tags=[], compile_targets=[compile_target] - ), - ), - module_execution_config=exec_config_a, - target_device_spec=device_spec_a, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - - suite = BenchmarkSuite.load_from_run_configs( - run_configs=[run_config_a], - root_benchmark_dir=benchmark_definition.ResourceLocation.build_url( - "https://example.com/testdata" - ), - ) - - self.assertEqual( - suite.filter_benchmarks(), - [ - BenchmarkCase( - model_name=model_tflite.name, - model_tags=model_tflite.tags, - bench_mode=exec_config_a.tags, - target_arch=common_definitions.DeviceArchitecture.RV64_GENERIC, - driver_info=IREE_DRIVERS_INFOS["iree-llvm-cpu-sync"], - benchmark_tool_name="iree-benchmark-module", - module_dir=benchmark_definition.ResourceLocation.build_url( - "https://example.com/testdata/iree_module_model_tflite___riscv_64-generic-linux_gnu-llvm_cpu___" - ), - run_config=run_config_a, - ) - ], - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/benchmarks/common/benchmark_thresholds.py b/build_tools/benchmarks/common/benchmark_thresholds.py deleted file mode 100644 index 5d12cb934ea8..000000000000 --- a/build_tools/benchmarks/common/benchmark_thresholds.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""A list of benchmarks and their similarity thresholds.""" - -import re - -from dataclasses import dataclass -from enum import Enum - - -class ThresholdUnit(Enum): - PERCENTAGE = "%" # Percentage - VALUE_NS = "ns" # Absolute value in nanoseconds - - -@dataclass -class BenchmarkThreshold: - """Similarity threshold for benchmarks matching a regular expression.""" - - # A regular expression to match against the benchmark identifier. - regex: re.Pattern - # A threshold for computing the benchmark value average. Benchmark sample - # values from consecutive runs and within the given range will be considered - # as similar (with some noise). They will be used to compute the moving - # average. The number will be interpreted according to the given unit. - # What value to set depends on the noise range of the particular benchmark. - threshold: int - unit: ThresholdUnit - - def get_threshold_str(self): - """Returns a string representation of the threshold.""" - if self.unit == ThresholdUnit.PERCENTAGE: - return f"{self.threshold}%" - return self.threshold - - -# A list of benchmarks and their similarity thresholds. -# Order matters here: if multiple regexes match a single benchmark, the first -# match is used. -BENCHMARK_THRESHOLDS = [ - # Fluctuating benchmarks on ARM64 CPUs. - BenchmarkThreshold( - re.compile(r"^DeepLabV3.*big-core.*LLVM-CPU.* @ Pixel"), - 20, - ThresholdUnit.PERCENTAGE, - ), - BenchmarkThreshold( - re.compile(r"^MobileBertSquad.*big-core.*LLVM-CPU-Sync @ Pixel-4"), - 20, - ThresholdUnit.PERCENTAGE, - ), - BenchmarkThreshold( - re.compile(r"^MobileNetV2.*LLVM-CPU.* @ Pixel"), 15, ThresholdUnit.PERCENTAGE - ), - BenchmarkThreshold( - re.compile(r"^MobileNetV3Small.*LLVM-CPU.* @ Pixel"), - 25, - ThresholdUnit.PERCENTAGE, - ), - BenchmarkThreshold( - re.compile(r"^MobileSSD.*little-core.*LLVM-CPU.* @ Pixel-6"), - 20, - ThresholdUnit.PERCENTAGE, - ), - BenchmarkThreshold( - re.compile(r"^PoseNet.*big-core.*LLVM-CPU.* @ Pixel"), - 15, - ThresholdUnit.PERCENTAGE, - ), - # Benchmarks that complete <= 10ms on X86_64 CPUs; using percentage is not - # suitable anymore. - BenchmarkThreshold( - re.compile(r"^DeepLabV3_fp32.*x86_64"), 1 * 10**6, ThresholdUnit.VALUE_NS - ), - BenchmarkThreshold( - re.compile(r"^EfficientNet_int8.*x86_64"), 1 * 10**6, ThresholdUnit.VALUE_NS - ), - BenchmarkThreshold( - re.compile(r"^MobileNetV1_fp32.*x86_64"), 1 * 10**6, ThresholdUnit.VALUE_NS - ), - BenchmarkThreshold( - re.compile(r"^MobileNetV2_fp32.*x86_64"), 2 * 10**6, ThresholdUnit.VALUE_NS - ), - BenchmarkThreshold( - re.compile(r"^MobileNetV3Small_fp32.*x86_64"), - 1 * 10**6, - ThresholdUnit.VALUE_NS, - ), - BenchmarkThreshold( - re.compile(r"^PersonDetect_int8.*x86_64"), 5 * 10**5, ThresholdUnit.VALUE_NS - ), - BenchmarkThreshold( - re.compile(r"^PoseNet_fp32.*x86_64"), 1 * 10**6, ThresholdUnit.VALUE_NS - ), - # Fluctuating benchmarks on mobile GPUs. - BenchmarkThreshold( - re.compile(r"^MobileBertSquad.*int8.*full-inference.*GPU-Mali"), - 10, - ThresholdUnit.PERCENTAGE, - ), - BenchmarkThreshold( - re.compile(r"^MobileBertSquad.*fp16.*full-inference.*GPU-Mali"), - 10, - ThresholdUnit.PERCENTAGE, - ), - BenchmarkThreshold( - re.compile(r"^MobileNetV3Small.*full-inference.*GPU-Mali"), - 2 * 10**6, - ThresholdUnit.VALUE_NS, - ), - # Benchmarks that complete <= 10ms on GPUs; using percentage is not - # suitable anymore. - BenchmarkThreshold( - re.compile(r"^DeepLabV3.*GPU-Mali"), 1 * 10**6, ThresholdUnit.VALUE_NS - ), - BenchmarkThreshold( - re.compile(r"^PersonDetect.*int8.*GPU-Mali"), - 2 * 10**5, - ThresholdUnit.VALUE_NS, - ), - BenchmarkThreshold( - re.compile(r"^EfficientNet.*int8.*GPU-Mali"), - 15 * 10**5, - ThresholdUnit.VALUE_NS, - ), - BenchmarkThreshold( - re.compile(r"^MobileNet.*GPU"), 1 * 10**6, ThresholdUnit.VALUE_NS - ), - # Default threshold for all ARM64/X86_64 benchmarks: 10%. - BenchmarkThreshold(re.compile(r".*CPU-ARM.*"), 10, ThresholdUnit.PERCENTAGE), - BenchmarkThreshold(re.compile(r".*x86_64.*"), 10, ThresholdUnit.PERCENTAGE), - # Default threshold for all benchmarks: 5%. - BenchmarkThreshold(re.compile(r".*"), 5, ThresholdUnit.PERCENTAGE), -] - -COMPILATION_TIME_THRESHOLDS = [ - # TODO(#11922): Compilation time measurement is very unstable right now. - # Use a large threshold until we make it stable. - BenchmarkThreshold(re.compile(r".*"), 100, ThresholdUnit.PERCENTAGE), -] - -TOTAL_DISPATCH_SIZE_THRESHOLDS = [ - # Default threshold: 5%. - BenchmarkThreshold(re.compile(r".*"), 5, ThresholdUnit.PERCENTAGE), -] - -TOTAL_ARTIFACT_SIZE_THRESHOLDS = [ - # Default threshold: 5%. - BenchmarkThreshold(re.compile(r".*"), 5, ThresholdUnit.PERCENTAGE), -] - -STREAM_IR_DISPATCH_COUNT_THRESHOLDS = [ - # Default threshold: 0%. - # Any change on dispatch count should be reported. - BenchmarkThreshold(re.compile(r".*"), 0, ThresholdUnit.PERCENTAGE), -] diff --git a/build_tools/benchmarks/common/common_arguments.py b/build_tools/benchmarks/common/common_arguments.py deleted file mode 100644 index 5fd389218932..000000000000 --- a/build_tools/benchmarks/common/common_arguments.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import glob -import os -import argparse -import pathlib -from typing import List, Optional, Sequence - - -def _check_dir_path(path): - path = pathlib.Path(path) - if path.is_dir(): - return path - else: - raise argparse.ArgumentTypeError(path) - - -def _check_file_path(path): - path = pathlib.Path(path) - if path.is_file(): - return path - else: - raise argparse.ArgumentTypeError(f"'{path}' is not found") - - -def _check_exe_path(path): - path = pathlib.Path(path) - if os.access(path, os.X_OK): - return path - else: - raise argparse.ArgumentTypeError(f"'{path}' is not an executable") - - -class Parser(argparse.ArgumentParser): - """Argument parser that includes common arguments and does validation.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.add_argument( - "--e2e_test_artifacts_dir", - metavar="", - type=str, - required=True, - help="Path/URL to the IREE e2e test artifacts directory.", - ) - - self.add_argument( - "--benchmark_tool_dir", - "--normal-benchmark-tool-dir", - type=_check_dir_path, - default=None, - help="Path to the normal (non-tracing) iree tool directory", - ) - self.add_argument( - "--driver-filter-regex", - "--driver_filter_regex", - type=str, - default=None, - help="Only run benchmarks matching the given driver regex", - ) - self.add_argument( - "--model-name-regex", - "--model_name_regex", - type=str, - default=None, - help="Only run benchmarks matching the given model name regex", - ) - self.add_argument( - "--mode-regex", - "--mode_regex", - type=str, - default=None, - help="Only run benchmarks matching the given benchmarking mode regex", - ) - self.add_argument( - "--output", - "-o", - default=None, - type=pathlib.Path, - help="Path to the output file", - ) - self.add_argument( - "--no-clean", - action="store_true", - help="Do not clean up the temporary directory used for " - "benchmarking on the Android device", - ) - self.add_argument( - "--verbose", - action="store_true", - help="Print internal information during execution", - ) - self.add_argument( - "--pin-cpu-freq", - "--pin_cpu_freq", - action="store_true", - help="Pin CPU frequency for all cores to the maximum. Requires root", - ) - self.add_argument( - "--pin-gpu-freq", - "--pin_gpu_freq", - action="store_true", - help="Pin GPU frequency to the maximum. Requires root", - ) - self.add_argument( - "--keep_going", - "--keep-going", - action="store_true", - help="Continue running after a failed benchmark. The overall exit status" - " will still indicate failure and all errors will be reported at the end.", - ) - self.add_argument( - "--tmp_dir", - "--tmp-dir", - "--tmpdir", - default=pathlib.Path("/tmp/iree-benchmarks"), - type=_check_dir_path, - help="Base directory in which to store temporary files. A subdirectory" - " with a name matching the git commit hash will be created.", - ) - self.add_argument( - "--continue_from_previous", - "--continue-from-previous", - action="store_true", - help="Previous benchmark results will be used and not " - "rerun if they are found in the benchmark results directory.", - ) - self.add_argument( - "--benchmark_min_time", - "--benchmark-min-time", - default=0, - type=float, - help="If specified, this will be passed as --benchmark_min_time to the" - "iree-benchmark-module (minimum number of seconds to repeat running " - "for). In that case, no --benchmark_repetitions flag will be passed." - " If not specified, a --benchmark_repetitions will be passed " - "instead.", - ) - self.add_argument( - "--compatible_only", - "--compatible-only", - action="store_true", - help="Only run compatible benchmarks based on the detected device " - "information", - ) - self.add_argument( - "--verify", - action="store_true", - help="Verify the output when the expected output is available", - ) - self.add_argument( - "--execution_benchmark_config", - type=_check_file_path, - required=True, - help="JSON config for the execution benchmarks", - ) - self.add_argument( - "--target_device_name", - type=str, - required=True, - help="Target device in benchmark config to run", - ) - self.add_argument( - "--shard_index", - type=int, - default=None, - help="Shard in benchmark config to run", - ) - - -def expand_and_check_file_paths(paths: Sequence[str]) -> List[pathlib.Path]: - """Expands the wildcards in the paths and check if they are files. - Returns: - List of expanded paths. - """ - - expanded_paths = [] - for path in paths: - expanded_paths += [pathlib.Path(path) for path in glob.glob(path)] - - for path in expanded_paths: - if not path.is_file(): - raise ValueError(f"{path} is not a file.") - - return expanded_paths diff --git a/build_tools/benchmarks/common/common_arguments_test.py b/build_tools/benchmarks/common/common_arguments_test.py deleted file mode 100644 index 5d1467bd74a2..000000000000 --- a/build_tools/benchmarks/common/common_arguments_test.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import pathlib -import tempfile -import unittest - -import common.common_arguments - - -class CommonArgumentsTest(unittest.TestCase): - def setUp(self): - self._build_dir_manager = tempfile.TemporaryDirectory() - self.build_dir = pathlib.Path(self._build_dir_manager.name).resolve() - self.e2e_test_artifacts_dir = self.build_dir / "e2e_test_artifacts" - self.e2e_test_artifacts_dir.mkdir() - self.normal_tool_dir = self.build_dir / "normal_tool" - self.normal_tool_dir.mkdir() - self.execution_config = self.build_dir / "execution_config.json" - self.execution_config.touch() - - def tearDown(self): - self._build_dir_manager.cleanup() - - def test_parser(self): - common.common_arguments.Parser().parse_args( - [ - f"--benchmark_tool_dir={self.normal_tool_dir}", - f"--e2e_test_artifacts_dir={self.e2e_test_artifacts_dir}", - f"--execution_benchmark_config={self.execution_config}", - "--target_device=test", - ] - ) - - def test_parser_check_benchmark_tool(self): - arg_parser = common.common_arguments.Parser() - with self.assertRaises(SystemExit): - arg_parser.parse_args( - [ - "--benchmark_tool_dir=nonexistent", - f"--e2e_test_artifacts_dir={self.e2e_test_artifacts_dir}", - f"--execution_benchmark_config={self.execution_config}", - "--target_device=test", - ] - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/benchmarks/common/linux_device_utils.py b/build_tools/benchmarks/common/linux_device_utils.py deleted file mode 100644 index e72b576ef973..000000000000 --- a/build_tools/benchmarks/common/linux_device_utils.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Utils for accessing Linux device information.""" - -import re -from typing import Optional, Sequence - -from .benchmark_definition import execute_cmd_and_get_stdout, DeviceInfo, PlatformType - - -def _get_lscpu_field(lscpu_output: str, field_name: str) -> str: - (value,) = re.findall(f"^{field_name}:\s*(.+)", lscpu_output, re.MULTILINE) - return value - - -def get_linux_cpu_arch(lscpu_output: str) -> str: - """Returns CPU Architecture, e.g., 'x86_64'.""" - return _get_lscpu_field(lscpu_output, "Architecture") - - -def get_linux_cpu_features(lscpu_output: str) -> Sequence[str]: - """Returns CPU feature lists, e.g., ['mmx', 'fxsr', 'sse', 'sse2'].""" - return _get_lscpu_field(lscpu_output, "Flags").split(" ") - - -def canonicalize_gpu_name(gpu_name: str) -> str: - # Replace all consecutive non-word characters with a single hyphen. - return re.sub(r"\W+", "-", gpu_name) - - -def get_linux_device_info( - device_model: str = "Unknown", - cpu_uarch: Optional[str] = None, - gpu_id: str = "0", - verbose: bool = False, -) -> DeviceInfo: - """Returns device info for the Linux device. - - Args: - - device_model: the device model name, e.g., 'ThinkStation P520' - - cpu_uarch: the CPU microarchitecture, e.g., 'CascadeLake' - - gpu_id: the target GPU ID, e.g., '0' or 'GPU-' - """ - lscpu_output = execute_cmd_and_get_stdout(["lscpu"], verbose) - - try: - gpu_name = execute_cmd_and_get_stdout( - [ - "nvidia-smi", - "--query-gpu=name", - "--format=csv,noheader", - f"--id={gpu_id}", - ], - verbose, - ) - except FileNotFoundError: - # Set GPU name to Unknown if the tool "nvidia-smi" doesn't exist. - gpu_name = "Unknown" - - return DeviceInfo( - PlatformType.LINUX, - # Includes CPU model as it is the key factor of the device performance. - model=device_model, - # Currently we only have x86, so CPU ABI = CPU arch. - cpu_abi=get_linux_cpu_arch(lscpu_output), - cpu_uarch=cpu_uarch, - cpu_features=get_linux_cpu_features(lscpu_output), - gpu_name=canonicalize_gpu_name(gpu_name), - ) diff --git a/build_tools/benchmarks/common/linux_device_utils_test.py b/build_tools/benchmarks/common/linux_device_utils_test.py deleted file mode 100644 index 60e76c2e0651..000000000000 --- a/build_tools/benchmarks/common/linux_device_utils_test.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import unittest - -from unittest import mock - -from common.benchmark_definition import DeviceInfo, PlatformType -from common.linux_device_utils import ( - canonicalize_gpu_name, - get_linux_cpu_arch, - get_linux_cpu_features, -) - -LSCPU_OUTPUT = ( - "Architecture: x86_64\n" - "Vendor ID: AuthenticAMD\n" - "Flags: fpu vme de pse tsc\n" -) - - -class LinuxDeviceUtilsTest(unittest.TestCase): - def test_get_linux_cpu_arch(self): - self.assertEqual(get_linux_cpu_arch(LSCPU_OUTPUT), "x86_64") - - def test_get_linux_cpu_features(self): - self.assertEqual( - get_linux_cpu_features(LSCPU_OUTPUT), ["fpu", "vme", "de", "pse", "tsc"] - ) - - def test_canonicalize_gpu_name(self): - self.assertEqual( - canonicalize_gpu_name("Tesla V100-SXM2-16GB"), "Tesla-V100-SXM2-16GB" - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/benchmarks/diff_local_benchmarks.py b/build_tools/benchmarks/diff_local_benchmarks.py deleted file mode 100755 index eeb43339a0b9..000000000000 --- a/build_tools/benchmarks/diff_local_benchmarks.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Diffs two local benchmark result JSON files. - -Example usage: - python3 diff_local_benchmarks.py --base=/path/to/base_benchmarks.json - --target=/path/to/target_benchmarks.json -""" - -import pathlib -import sys - -# Add build_tools python dir to the search path. -sys.path.insert(0, str(pathlib.Path(__file__).parent.with_name("python"))) - -import argparse - -from typing import Optional - -from common.benchmark_presentation import * - - -def get_benchmark_result_markdown( - base_benchmark_file: Optional[pathlib.Path], - target_benchmark_file: Optional[pathlib.Path], - base_compile_stats_file: Optional[pathlib.Path], - target_compile_stats_file: Optional[pathlib.Path], - verbose: bool = False, -) -> str: - """Gets the full markdown summary of all benchmarks in files.""" - base_benchmarks = {} - target_benchmarks = {} - base_compilation_metrics = {} - target_compilation_metrics = {} - if base_benchmark_file and target_benchmark_file: - base_benchmarks = aggregate_all_benchmarks([base_benchmark_file]) - target_benchmarks = aggregate_all_benchmarks([target_benchmark_file]) - if base_compile_stats_file and target_compile_stats_file: - base_compilation_metrics = collect_all_compilation_metrics( - [base_compile_stats_file] - ) - target_compilation_metrics = collect_all_compilation_metrics( - [target_compile_stats_file] - ) - - # Update the target benchmarks with their corresponding base numbers. - for bench in base_benchmarks: - if bench in target_benchmarks: - target_benchmarks[bench].base_mean_time = base_benchmarks[bench].mean_time - - for target_name, base_metrics in base_compilation_metrics.items(): - updated_metrics = base_metrics - for mapper in COMPILATION_METRICS_TO_TABLE_MAPPERS: - metric_key = mapper.get_series_name(target_name) - base_value, _ = mapper.get_current_and_base_value(base_metrics) - updated_metrics = mapper.update_base_value(updated_metrics, base_value) - target_compilation_metrics[target_name] = updated_metrics - - # Compose the full benchmark tables. - full_table = [md.header("Full Benchmark Summary", 2)] - full_table.append(categorize_benchmarks_into_tables(target_benchmarks)) - - # Compose the full compilation metrics tables. - full_table.append( - categorize_compilation_metrics_into_tables(target_compilation_metrics) - ) - - return "\n\n".join(full_table) - - -def parse_arguments(): - """Parses command-line options.""" - - def check_file_path(path): - path = pathlib.Path(path) - if path.is_file(): - return path - else: - raise ValueError(path) - - parser = argparse.ArgumentParser() - parser.add_argument("--base", type=check_file_path, help="Base benchmark results") - parser.add_argument( - "--target", type=check_file_path, help="Target benchmark results" - ) - parser.add_argument( - "--base-compile-stats", type=check_file_path, help="Base compilation statistics" - ) - parser.add_argument( - "--target-compile-stats", - type=check_file_path, - help="Target compilation statistics", - ) - parser.add_argument( - "--verbose", - action="store_true", - help="Print internal information during execution", - ) - args = parser.parse_args() - - return args - - -if __name__ == "__main__": - args = parse_arguments() - if args.base or args.target: - if not args.base or not args.target: - raise ValueError("--base and --target must be used together.") - if args.base_compile_stats or args.target_compile_stats: - if not args.base_compile_stats or not args.target_compile_stats: - raise ValueError( - "--base-compile-stats and --target-compile-stats must " - "be used together." - ) - - print( - get_benchmark_result_markdown( - args.base, - args.target, - args.base_compile_stats, - args.target_compile_stats, - verbose=args.verbose, - ) - ) diff --git a/build_tools/benchmarks/export_benchmark_config.py b/build_tools/benchmarks/export_benchmark_config.py deleted file mode 100755 index 4b5dc1b3659d..000000000000 --- a/build_tools/benchmarks/export_benchmark_config.py +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Exports JSON config for benchmarking and compilation statistics. - -Export type: "execution" outputs: -[ - : { - host_environment: HostEnvironment, - module_dir_paths: [], - run_configs: serialized [E2EModelRunConfig] - }, - ... -] -to be used in build_tools/benchmarks/run_benchmarks_on_*.py - -Export type: "compilation" outputs: -{ - module_dir_paths: [], - generation_configs: serialized [ModuleGenerationConfig] -} -of generation configs defined for compilation statistics, to be used in -build_tools/benchmarks/collect_compilation_statistics.py -""" - -import sys -import pathlib - -# Add build_tools python dir to the search path. -sys.path.insert(0, str(pathlib.Path(__file__).parent.with_name("python"))) - -from typing import Dict, Iterable, List, Optional, Set, Sequence -import argparse -import collections -import dataclasses -import json -import textwrap - -from benchmark_suites.iree import benchmark_collections, benchmark_presets -from e2e_test_artifacts import iree_artifacts -from e2e_test_framework import serialization -from e2e_test_framework.definitions import iree_definitions - - -def filter_and_group_run_configs( - run_configs: List[iree_definitions.E2EModelRunConfig], - target_device_names: Optional[Set[str]] = None, - presets: Optional[Set[str]] = None, -) -> Dict[str, List[iree_definitions.E2EModelRunConfig]]: - """Filters run configs and groups by target device name. - - Args: - run_configs: source e2e model run configs. - target_device_names: list of target device names, includes all if not set. - presets: set of presets, matches all if not set. - - Returns: - A map of e2e model run configs keyed by target device name. - """ - grouped_run_config_map = collections.defaultdict(list) - - for run_config in run_configs: - device_name = run_config.target_device_spec.device_name - if target_device_names is not None and device_name not in target_device_names: - continue - if presets is not None and not presets.intersection(run_config.presets): - continue - grouped_run_config_map[device_name].append(run_config) - - return grouped_run_config_map - - -def _get_distinct_module_dir_paths( - module_generation_configs: Iterable[iree_definitions.ModuleGenerationConfig], - root_path: pathlib.PurePath = pathlib.PurePath(), -) -> List[str]: - module_dir_paths = ( - str(iree_artifacts.get_module_dir_path(config, root_path=root_path)) - for config in module_generation_configs - ) - return sorted(set(module_dir_paths)) - - -def _export_execution_handler( - presets: Optional[Sequence[str]] = None, - target_device_names: Optional[Sequence[str]] = None, - shard_count: Optional[Dict[str, int]] = None, - **_unused_args, -): - _, all_run_configs = benchmark_collections.generate_benchmarks() - target_device_name_set = ( - None if target_device_names is None else set(target_device_names) - ) - grouped_run_config_map = filter_and_group_run_configs( - all_run_configs, - target_device_names=target_device_name_set, - presets=None if presets is None else set(presets), - ) - - shard_count = {} if shard_count is None else shard_count - default_shard_count = shard_count.get("default", 1) - - output_map = {} - for device_name, run_configs in grouped_run_config_map.items(): - host_environments = set( - run_config.target_device_spec.host_environment for run_config in run_configs - ) - if len(host_environments) > 1: - raise ValueError( - "Device specs of the same device should have the same host environment." - ) - host_environment = host_environments.pop() - - current_shard_count = int(shard_count.get(device_name, default_shard_count)) - # This splits the `run_configs` list into `current_shard_count` sub-lists in a round-robin way. - # Example: current_shard_count = 3; run_configs = range(10); assert(sharded_run_configs == [[0, 3, 6, 9], [1, 4, 7], [2, 5, 8]] - sharded_run_configs = [ - run_configs[shard_idx::current_shard_count] - for shard_idx in range(current_shard_count) - ] - - for index, shard in enumerate(sharded_run_configs): - distinct_module_dir_paths = _get_distinct_module_dir_paths( - config.module_generation_config for config in shard - ) - - serialized_run_configs = serialization.serialize_and_pack(shard) - output_map.setdefault( - device_name, - { - "host_environment": dataclasses.asdict(host_environment), - "shards": [], - }, - ) - output_map[device_name]["shards"].append( - { - "index": index, - "module_dir_paths": distinct_module_dir_paths, - "run_configs": serialized_run_configs, - } - ) - - return output_map - - -def _export_compilation_handler( - presets: Optional[Sequence[str]] = None, **_unused_args -): - all_gen_configs, _ = benchmark_collections.generate_benchmarks() - - if presets is None: - presets = benchmark_presets.ALL_COMPILATION_PRESETS - preset_set = set(presets) - - compile_stats_gen_configs = [ - gen_config - for gen_config in all_gen_configs - if preset_set.intersection(gen_config.presets) - ] - - distinct_module_dir_paths = _get_distinct_module_dir_paths( - compile_stats_gen_configs - ) - - return { - "module_dir_paths": distinct_module_dir_paths, - "generation_configs": serialization.serialize_and_pack( - compile_stats_gen_configs - ), - } - - -def _parse_and_strip_list_argument(arg: str) -> List[str]: - return [part.strip() for part in arg.split(",") if part != ""] - - -def _parse_benchmark_presets(arg: str, available_presets: Sequence[str]) -> List[str]: - presets = [] - for preset in _parse_and_strip_list_argument(arg): - if preset not in available_presets: - raise argparse.ArgumentTypeError( - f"Unrecognized benchmark preset: '{preset}'." - ) - presets.append(preset) - return presets - - -def _parse_shard_count(arg: str): - return dict(map(str.strip, el.split("=", 1)) for el in arg.split(",")) - - -def _parse_arguments(): - """Parses command-line options.""" - - # Makes global options come *after* command. - # See https://stackoverflow.com/q/23296695 - subparser_base = argparse.ArgumentParser(add_help=False) - subparser_base.add_argument( - "--output", type=pathlib.Path, help="Path to write the JSON output." - ) - - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent( - """ - Export type: "execution" outputs: - [ - : { - host_environment: HostEnvironment, - module_dir_paths: [], - run_configs: serialized [E2EModelRunConfig] - }, - ... - ] - to be used in build_tools/benchmarks/run_benchmarks_on_*.py - - Export type: "compilation" outputs: - { - module_dir_paths: [], - generation_configs: serialized [ModuleGenerationConfig] - } - of generation configs defined for compilation statistics, to be used in - build_tools/benchmarks/collect_compilation_statistics.py - """ - ), - ) - - subparser = parser.add_subparsers(required=True, title="export type") - execution_parser = subparser.add_parser( - "execution", - parents=[subparser_base], - help="Export execution config to run benchmarks.", - ) - execution_parser.set_defaults(handler=_export_execution_handler) - execution_parser.add_argument( - "--target_device_names", - type=_parse_and_strip_list_argument, - help=( - "Target device names, separated by comma, not specified means " - "including all devices." - ), - ) - execution_parser.add_argument( - "--presets", - "--benchmark_presets", - type=lambda arg: _parse_benchmark_presets( - arg, benchmark_presets.ALL_EXECUTION_PRESETS - ), - help=( - "Presets that select a bundle of benchmarks, separated by comma, " - "multiple presets will be union. Available options: " - f"{','.join(benchmark_presets.ALL_EXECUTION_PRESETS)}" - ), - ) - execution_parser.add_argument( - "--shard_count", - type=_parse_shard_count, - default={}, - help="Accepts a comma-separated list of device-name to shard-count mappings. Use reserved keyword 'default' for setting a default shard count: c2-standard-60=3,default=2", - ) - - compilation_parser = subparser.add_parser( - "compilation", - parents=[subparser_base], - help=( - "Export serialized list of module generation configs defined for " - "compilation statistics." - ), - ) - compilation_parser.set_defaults(handler=_export_compilation_handler) - compilation_parser.add_argument( - "--presets", - "--benchmark_presets", - type=lambda arg: _parse_benchmark_presets( - arg, benchmark_presets.ALL_COMPILATION_PRESETS - ), - help=( - "Presets `comp-stats*` that select a bundle of compilation" - " benchmarks, separated by comma, multiple presets will be union." - " Available options: " - f"{','.join(benchmark_presets.ALL_COMPILATION_PRESETS)}" - ), - ) - - return parser.parse_args() - - -def main(args: argparse.Namespace): - output_obj = args.handler(**vars(args)) - json_data = json.dumps(output_obj, indent=2) - if args.output is None: - print(json_data) - else: - args.output.write_text(json_data) - - -if __name__ == "__main__": - main(_parse_arguments()) diff --git a/build_tools/benchmarks/export_benchmark_config_test.py b/build_tools/benchmarks/export_benchmark_config_test.py deleted file mode 100644 index 88fff2fdfc60..000000000000 --- a/build_tools/benchmarks/export_benchmark_config_test.py +++ /dev/null @@ -1,290 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import unittest - -from e2e_test_framework.definitions import common_definitions, iree_definitions -import export_benchmark_config - -COMMON_MODEL = common_definitions.Model( - id="tflite", - name="model_tflite", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="", - entry_function="predict", - input_types=["1xf32"], -) -COMMON_GEN_CONFIG = iree_definitions.ModuleGenerationConfig.build( - imported_model=iree_definitions.ImportedModel.from_model(COMMON_MODEL), - compile_config=iree_definitions.CompileConfig.build( - id="1", - tags=[], - compile_targets=[ - iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - ], - ), -) -COMMON_EXEC_CONFIG = iree_definitions.ModuleExecutionConfig.build( - id="exec", - tags=[], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_SYNC, -) - - -class ExportBenchmarkConfigTest(unittest.TestCase): - def test_filter_and_group_run_configs_set_all_filters(self): - device_spec_a = common_definitions.DeviceSpec.build( - id="dev_a_cpu", - device_name="dev_a_cpu", - architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=[], - ) - device_spec_b = common_definitions.DeviceSpec.build( - id="dev_a_gpu", - device_name="dev_a_gpu", - architecture=common_definitions.DeviceArchitecture.ARM_VALHALL, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=[], - ) - device_spec_c = common_definitions.DeviceSpec.build( - id="dev_c", - device_name="dev_c", - architecture=common_definitions.DeviceArchitecture.CUDA_SM80, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - tags=[], - ) - matched_run_config_a = iree_definitions.E2EModelRunConfig.build( - module_generation_config=COMMON_GEN_CONFIG, - module_execution_config=COMMON_EXEC_CONFIG, - target_device_spec=device_spec_a, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - presets=["preset_x"], - ) - unmatched_run_config_b = iree_definitions.E2EModelRunConfig.build( - module_generation_config=COMMON_GEN_CONFIG, - module_execution_config=COMMON_EXEC_CONFIG, - target_device_spec=device_spec_b, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - presets=["preset_y"], - ) - matched_run_config_c = iree_definitions.E2EModelRunConfig.build( - module_generation_config=COMMON_GEN_CONFIG, - module_execution_config=COMMON_EXEC_CONFIG, - target_device_spec=device_spec_c, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - presets=["preset_y", "preset_z"], - ) - - run_config_map = export_benchmark_config.filter_and_group_run_configs( - run_configs=[ - matched_run_config_a, - unmatched_run_config_b, - matched_run_config_c, - ], - target_device_names={"dev_a_cpu", "dev_c"}, - presets={"preset_x", "preset_y"}, - ) - - self.assertEqual( - run_config_map, - { - "dev_a_cpu": [matched_run_config_a], - "dev_c": [matched_run_config_c], - }, - ) - - def test_filter_and_group_run_configs_include_all(self): - device_spec_a = common_definitions.DeviceSpec.build( - id="dev_a_cpu", - device_name="dev_a_cpu", - architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=[], - ) - device_spec_b = common_definitions.DeviceSpec.build( - id="dev_a_gpu", - device_name="dev_a_gpu", - architecture=common_definitions.DeviceArchitecture.ARM_VALHALL, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=[], - ) - device_spec_c = common_definitions.DeviceSpec.build( - id="dev_a_second_gpu", - device_name="dev_a_gpu", - architecture=common_definitions.DeviceArchitecture.QUALCOMM_ADRENO, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=[], - ) - run_config_a = iree_definitions.E2EModelRunConfig.build( - module_generation_config=COMMON_GEN_CONFIG, - module_execution_config=COMMON_EXEC_CONFIG, - target_device_spec=device_spec_a, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - run_config_b = iree_definitions.E2EModelRunConfig.build( - module_generation_config=COMMON_GEN_CONFIG, - module_execution_config=COMMON_EXEC_CONFIG, - target_device_spec=device_spec_b, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - run_config_c = iree_definitions.E2EModelRunConfig.build( - module_generation_config=COMMON_GEN_CONFIG, - module_execution_config=COMMON_EXEC_CONFIG, - target_device_spec=device_spec_c, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - - run_config_map = export_benchmark_config.filter_and_group_run_configs( - run_configs=[run_config_a, run_config_b, run_config_c] - ) - - self.maxDiff = 100000 - - self.assertEqual( - run_config_map, - { - "dev_a_cpu": [run_config_a], - "dev_a_gpu": [run_config_b, run_config_c], - }, - ) - - def test_filter_and_group_run_configs_set_target_device_names(self): - device_spec_a = common_definitions.DeviceSpec.build( - id="dev_a", - device_name="dev_a", - architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=[], - ) - device_spec_b = common_definitions.DeviceSpec.build( - id="dev_b", - device_name="dev_b", - architecture=common_definitions.DeviceArchitecture.ARM_VALHALL, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=[], - ) - run_config_a = iree_definitions.E2EModelRunConfig.build( - module_generation_config=COMMON_GEN_CONFIG, - module_execution_config=COMMON_EXEC_CONFIG, - target_device_spec=device_spec_a, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - run_config_b = iree_definitions.E2EModelRunConfig.build( - module_generation_config=COMMON_GEN_CONFIG, - module_execution_config=COMMON_EXEC_CONFIG, - target_device_spec=device_spec_b, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - - run_config_map = export_benchmark_config.filter_and_group_run_configs( - run_configs=[run_config_a, run_config_b], - target_device_names={"dev_a", "dev_b"}, - ) - - self.assertEqual( - run_config_map, - { - "dev_a": [run_config_a], - "dev_b": [run_config_b], - }, - ) - - def test_filter_and_group_run_configs_set_presets(self): - small_model = common_definitions.Model( - id="small_model", - name="small_model", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="", - entry_function="predict", - input_types=["1xf32"], - ) - big_model = common_definitions.Model( - id="big_model", - name="big_model", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="", - entry_function="predict", - input_types=["1xf32"], - ) - compile_target = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - compile_config = iree_definitions.CompileConfig.build( - id="1", tags=[], compile_targets=[compile_target] - ) - small_gen_config = iree_definitions.ModuleGenerationConfig.build( - imported_model=iree_definitions.ImportedModel.from_model(small_model), - compile_config=compile_config, - ) - big_gen_config = iree_definitions.ModuleGenerationConfig.build( - imported_model=iree_definitions.ImportedModel.from_model(big_model), - compile_config=compile_config, - ) - device_spec_a = common_definitions.DeviceSpec.build( - id="dev_a", - device_name="dev_a", - architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=[], - ) - device_spec_b = common_definitions.DeviceSpec.build( - id="dev_b", - device_name="dev_b", - architecture=common_definitions.DeviceArchitecture.ARM_VALHALL, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=[], - ) - run_config_a = iree_definitions.E2EModelRunConfig.build( - module_generation_config=small_gen_config, - module_execution_config=COMMON_EXEC_CONFIG, - target_device_spec=device_spec_a, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - presets=["preset_x"], - ) - run_config_b = iree_definitions.E2EModelRunConfig.build( - module_generation_config=big_gen_config, - module_execution_config=COMMON_EXEC_CONFIG, - target_device_spec=device_spec_b, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - presets=["preset_y"], - ) - - run_config_map = export_benchmark_config.filter_and_group_run_configs( - run_configs=[run_config_a, run_config_b], presets={"preset_x"} - ) - - self.assertEqual( - run_config_map, - { - "dev_a": [run_config_a], - }, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/benchmarks/generate_benchmark_comment.py b/build_tools/benchmarks/generate_benchmark_comment.py deleted file mode 100755 index 2dcce352c2f8..000000000000 --- a/build_tools/benchmarks/generate_benchmark_comment.py +++ /dev/null @@ -1,448 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Generates benchmark results as pull request comments. - -This script is meant to be used by CI and uses pip package "markdown_strings". -""" - -import sys -import pathlib - -# Add build_tools python dir to the search path. -sys.path.insert(0, str(pathlib.Path(__file__).parent.with_name("python"))) - -import argparse -import collections -import dataclasses -import json -from typing import Any, Dict, Optional, Set, Tuple - -import markdown_strings as md -import requests - -from common import benchmark_definition, benchmark_presentation, common_arguments -from reporting import benchmark_comment - -GITHUB_IREE_REPO_PREFIX = "https://github.com/iree-org/iree" -IREE_DASHBOARD_URL = "https://perf.iree.dev/apis/v2" -IREE_PROJECT_ID = "IREE" -# The maximal numbers of trials when querying base commit benchmark results. -MAX_BASE_COMMIT_QUERY_COUNT = 10 -# The max number of rows to show per table. -TABLE_SIZE_CUT = 3 -THIS_DIRECTORY = pathlib.Path(__file__).resolve().parent - - -@dataclasses.dataclass(frozen=True) -class CommentDef(object): - title: str - type_id: str - - -# Map from comment type to comment definition. -COMMENT_DEF_MAP = { - "android-benchmark-summary": CommentDef( - title="Abbreviated Android Benchmark Summary", - type_id="bf8cdf94-a992-466d-b11c-778cbd805a22", - ), - "linux-benchmark-summary": CommentDef( - title="Abbreviated Linux Benchmark Summary", - type_id="37549014-3c67-4e74-8d88-8e929231abe3", - ), - "benchmark-summary": CommentDef( - title="Abbreviated Benchmark Summary", - type_id="5b42cbfe-26a0-4164-a51c-07f06762e2dc", - ), -} - - -def get_git_total_commit_count(commit: str, verbose: bool = False) -> int: - """Gets the total commit count in history ending with the given commit.""" - # TODO(#11703): Should use --first-parent here. See issue for the required - # work. - count = benchmark_definition.execute_cmd_and_get_stdout( - ["git", "rev-list", "--count", commit], cwd=THIS_DIRECTORY, verbose=verbose - ) - return int(count) - - -def get_from_dashboard( - url: str, payload: Dict[str, Any], verbose: bool = False -) -> Dict[str, Dict[str, Any]]: - headers = {"Content-type": "application/json"} - data = json.dumps(payload) - - if verbose: - print(f"API request payload: {data}") - - response = requests.get(url, data=data, headers=headers) - code = response.status_code - if code != 200: - raise requests.RequestException( - f"Failed to get from dashboard server with status code {code}" - ) - - data = response.json() - if verbose: - print(f"Queried base benchmark data: {data}") - return data - - -BenchmarkQueryResults = Dict[str, Dict[str, Any]] - - -def query_base_benchmark_results( - commit: str, verbose: bool = False -) -> BenchmarkQueryResults: - """Queries the benchmark results for the given commit.""" - build_id = get_git_total_commit_count(commit, verbose) - payload = {"projectId": IREE_PROJECT_ID, "buildId": build_id} - return get_from_dashboard( - f"{IREE_DASHBOARD_URL}/getBuild", payload, verbose=verbose - ) - - -@dataclasses.dataclass(frozen=True) -class ComparableBenchmarkResults(object): - commit_sha: str - benchmark_results: BenchmarkQueryResults - - -def _find_comparable_benchmark_results( - start_commit: str, required_benchmark_keys: Set[str], verbose: bool = False -) -> Optional[ComparableBenchmarkResults]: - cmds = [ - "git", - "rev-list", - "--first-parent", - f"--max-count={MAX_BASE_COMMIT_QUERY_COUNT}", - start_commit, - ] - output = benchmark_definition.execute_cmd_and_get_stdout( - cmds, cwd=THIS_DIRECTORY, verbose=verbose - ) - previous_commits = output.splitlines() - # Try to query some base benchmark to diff against, from the top of the - # tree. Bail out if the maximal trial number is exceeded. - for base_commit in previous_commits: - base_benchmarks = query_base_benchmark_results( - commit=base_commit, verbose=verbose - ) - base_benchmark_keys = set(base_benchmarks.keys()) - if required_benchmark_keys <= base_benchmark_keys: - return ComparableBenchmarkResults( - commit_sha=base_commit, benchmark_results=base_benchmarks - ) - - return None - - -def _get_git_commit_hash(ref: str, verbose: bool = False) -> str: - """Gets the commit hash for the given commit.""" - return benchmark_definition.execute_cmd_and_get_stdout( - ["git", "rev-parse", ref], cwd=THIS_DIRECTORY, verbose=verbose - ) - - -def _get_git_merge_base_commit( - pr_commit: str, target_branch: str, verbose: bool = False -) -> str: - return benchmark_definition.execute_cmd_and_get_stdout( - args=["git", "merge-base", target_branch, pr_commit], - cwd=THIS_DIRECTORY, - verbose=verbose, - ) - - -def _get_experimental_dt_comparison_markdown( - execution_benchmarks: Dict[str, benchmark_presentation.AggregateBenchmarkLatency], -) -> Optional[str]: - """Get the comparison table to compare different data-tiling options.""" - - dt_tags = {"no-dt": "No-DT (baseline)", "dt-only": "DT-Only", "dt-uk": "DT-UK"} - latency_map = collections.defaultdict(dict) - for bench_id, latency in execution_benchmarks.items(): - dt_tag = next((tag for tag in dt_tags if tag in latency.name), None) - if dt_tag is None: - continue - # See build_tools/python/e2e_test_framework/definitions/iree_definitions.py - # for how benchmark names are constructed. - # Format: model_name gen_tags exec_tags ... - model, gen_tags, remaining = latency.name.split(" ", maxsplit=2) - # Format: [compile targets][tags] - compile_targets = gen_tags.split("][")[0] + "]" - key_name = " ".join([model, compile_targets, remaining]) - latency_map[key_name][dt_tag] = (bench_id, latency.mean_time / 1e6) - - if len(latency_map) == 0: - return None - - # Compute speedup vs. the baseline. - table = {} - for key_name, data in latency_map.items(): - baseline = data.get("no-dt") - baseline = None if baseline is None else baseline[1] - row = {} - for dt_tag in dt_tags: - pair = data.get(dt_tag) - if pair is None: - continue - bench_id, mean_time = pair - text = f"{mean_time:.03f}" - if baseline is not None: - text += f" ({(baseline / mean_time):.01f}X)" - row[dt_tag] = (bench_id, text) - table[key_name] = row - - table_columns = [["Name"] + list(table.keys())] - for dt_tag, dt_name in dt_tags.items(): - column = [dt_name] - for key_name, data in table.items(): - pair = data.get(dt_tag) - if pair is None: - column.append("N/A") - continue - bench_id, text = pair - column.append(benchmark_presentation.make_series_link(text, bench_id)) - table_columns.append(column) - - return md.table(table_columns) - - -def _get_benchmark_result_markdown( - execution_benchmarks: Dict[str, benchmark_presentation.AggregateBenchmarkLatency], - compilation_metrics: Dict[str, benchmark_presentation.CompilationMetrics], - pr_url: str, - build_url: str, - comment_def: CommentDef, - commit_info_md: str, -) -> Tuple[str, str]: - """Gets the full/abbreviated markdown summary of all benchmarks in files.""" - - pr_info = md.link("Pull request", pr_url) - build_info = md.link("Build", build_url) - - # Compose the full benchmark tables. - full_table = [md.header("Full Benchmark Summary", 2)] - full_table.append(md.unordered_list([commit_info_md, pr_info, build_info])) - - # Compose the abbreviated benchmark tables. - abbr_table = [md.header(comment_def.title, 2)] - abbr_table.append(commit_info_md) - - # The temporary table to help compare different data-tiling options. - dt_cmp_table = _get_experimental_dt_comparison_markdown( - execution_benchmarks=execution_benchmarks - ) - if dt_cmp_table is not None: - dt_cmp_header = md.header("Data-Tiling Comparison Table", 3) - full_table += [dt_cmp_header, dt_cmp_table] - abbr_table += [ - dt_cmp_header, - "
", - "Click to show", - dt_cmp_table, - "
", - ] - - if len(execution_benchmarks) > 0: - full_table.append( - benchmark_presentation.categorize_benchmarks_into_tables( - execution_benchmarks - ) - ) - - abbr_benchmarks_tables = ( - benchmark_presentation.categorize_benchmarks_into_tables( - execution_benchmarks, TABLE_SIZE_CUT - ) - ) - if len(abbr_benchmarks_tables) == 0: - abbr_table.append("No improved or regressed benchmarks 🏖️") - else: - abbr_table.append(abbr_benchmarks_tables) - - # Compose the full compilation metrics tables. - if len(compilation_metrics) > 0: - full_table.append( - benchmark_presentation.categorize_compilation_metrics_into_tables( - compilation_metrics - ) - ) - - abbr_compilation_metrics_tables = ( - benchmark_presentation.categorize_compilation_metrics_into_tables( - compilation_metrics, TABLE_SIZE_CUT - ) - ) - if len(abbr_compilation_metrics_tables) == 0: - abbr_table.append("No improved or regressed compilation metrics 🏖️") - else: - abbr_table.append(abbr_compilation_metrics_tables) - - abbr_table.append("For more information:") - # We don't know until a Gist is really created. Use a placeholder for now and - # replace later. - full_result_info = md.link( - "Full benchmark result tables", benchmark_comment.GIST_LINK_PLACEHORDER - ) - abbr_table.append(md.unordered_list([full_result_info, build_info])) - - # Append the unique comment type id to help identify and update the existing - # comment. - abbr_table.append(f"") - - return "\n\n".join(full_table), "\n\n".join(abbr_table) - - -def parse_arguments(): - """Parses command-line options.""" - - parser = argparse.ArgumentParser() - parser.add_argument( - "--benchmark_files", - metavar="", - default=[], - action="append", - help=( - "Paths to the JSON files containing benchmark results, " "accepts wildcards" - ), - ) - parser.add_argument( - "--compile_stats_files", - metavar="", - default=[], - action="append", - help=( - "Paths to the JSON files containing compilation statistics, " - "accepts wildcards" - ), - ) - parser.add_argument("--pr_number", required=True, type=int, help="PR number") - parser.add_argument( - "--pr_committish", type=str, default="HEAD", help="PR commit hash or ref" - ) - parser.add_argument( - "--pr_base_branch", type=str, default=None, help="Base branch to merge the PR." - ) - parser.add_argument( - "--comment_type", - required=True, - choices=COMMENT_DEF_MAP.keys(), - help="Type of summary comment", - ) - parser.add_argument( - "--build_url", - required=True, - type=str, - help="CI build page url to show in the report", - ) - parser.add_argument("--output", type=pathlib.Path, default=None) - parser.add_argument( - "--verbose", - action="store_true", - help="Print internal information during execution", - ) - - return parser.parse_args() - - -def main(args): - benchmark_files = common_arguments.expand_and_check_file_paths(args.benchmark_files) - compile_stats_files = common_arguments.expand_and_check_file_paths( - args.compile_stats_files - ) - - pr_commit = _get_git_commit_hash(ref=args.pr_committish, verbose=args.verbose) - execution_benchmarks = benchmark_presentation.aggregate_all_benchmarks( - benchmark_files=benchmark_files, expected_pr_commit=pr_commit - ) - compilation_metrics = benchmark_presentation.collect_all_compilation_metrics( - compile_stats_files=compile_stats_files, expected_pr_commit=pr_commit - ) - - if args.pr_base_branch is None: - pr_base_commit = None - else: - pr_base_commit = _get_git_merge_base_commit( - pr_commit=pr_commit, target_branch=args.pr_base_branch, verbose=args.verbose - ) - - if pr_base_commit is None: - comparable_results = None - else: - required_benchmark_keys = set(execution_benchmarks.keys()) - for target_id in compilation_metrics: - for mapper in benchmark_presentation.COMPILATION_METRICS_TO_TABLE_MAPPERS: - required_benchmark_keys.add(mapper.get_series_id(target_id)) - - comparable_results = _find_comparable_benchmark_results( - start_commit=pr_base_commit, - required_benchmark_keys=required_benchmark_keys, - verbose=args.verbose, - ) - - if comparable_results is None: - comparable_commit = None - else: - comparable_commit = comparable_results.commit_sha - # Update the execution benchmarks with base numbers. - for bench in execution_benchmarks: - base_benchmark = comparable_results.benchmark_results[bench] - if base_benchmark["sampleUnit"] != "ns": - raise ValueError("Only support nanoseconds for latency sample.") - execution_benchmarks[bench].base_mean_time = base_benchmark["sample"] - - # Update the compilation metrics with base numbers. - for target_id, metrics in compilation_metrics.items(): - updated_metrics = metrics - for mapper in benchmark_presentation.COMPILATION_METRICS_TO_TABLE_MAPPERS: - base_benchmark = comparable_results.benchmark_results[ - mapper.get_series_id(target_id) - ] - if base_benchmark["sampleUnit"] != mapper.get_unit(): - raise ValueError("Unit of the queried sample is mismatched.") - updated_metrics = mapper.update_base_value( - updated_metrics, base_benchmark["sample"] - ) - compilation_metrics[target_id] = updated_metrics - - pr_commit_link = md.link(pr_commit, f"{GITHUB_IREE_REPO_PREFIX}/commit/{pr_commit}") - commit_info_md = f"@ commit {pr_commit_link}" - if comparable_commit is not None: - baseline_commit_link = md.link( - comparable_commit, f"{GITHUB_IREE_REPO_PREFIX}/commit/{comparable_commit}" - ) - commit_info_md += f" (vs. base {baseline_commit_link})" - elif pr_base_commit is not None: - commit_info_md += " (no previous benchmark results to compare)" - - comment_def = COMMENT_DEF_MAP[args.comment_type] - full_md, abbr_md = _get_benchmark_result_markdown( - execution_benchmarks=execution_benchmarks, - compilation_metrics=compilation_metrics, - pr_url=f"{GITHUB_IREE_REPO_PREFIX}/pull/{args.pr_number}", - build_url=args.build_url, - comment_def=comment_def, - commit_info_md=commit_info_md, - ) - - comment_data = benchmark_comment.CommentData( - type_id=comment_def.type_id, - abbr_md=abbr_md, - full_md=full_md, - unverified_pr_number=args.pr_number, - ) - comment_json_data = json.dumps(dataclasses.asdict(comment_data), indent=2) - if args.output is None: - print(comment_json_data) - else: - args.output.write_text(comment_json_data) - - -if __name__ == "__main__": - main(parse_arguments()) diff --git a/build_tools/benchmarks/post_benchmark_comment.py b/build_tools/benchmarks/post_benchmark_comment.py deleted file mode 100755 index a6e880b4b02f..000000000000 --- a/build_tools/benchmarks/post_benchmark_comment.py +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -""" -Posts benchmark results to gist and comments on pull requests. - -Requires the environment variables: - -- GITHUB_TOKEN: token from GitHub action that has write access on issues. See - https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token -- COMMENT_BOT_USER: user name that posts the comment. Note this can be different - from the user creates the gist. -- GIST_BOT_TOKEN: token that has write access to gist. Gist will be posted as - the owner of the token. See - https://docs.github.com/en/rest/overview/permissions-required-for-fine-grained-personal-access-tokens#gists -""" - -import sys -import pathlib - -# Add build_tools python dir to the search path. -sys.path.insert(0, str(pathlib.Path(__file__).parent.with_name("python"))) - -import argparse -import http.client -import json -import os -import requests -from typing import Any, Optional - -from reporting import benchmark_comment - -GITHUB_IREE_API_PREFIX = "https://api.github.com/repos/iree-org/iree" -GITHUB_GIST_API = "https://api.github.com/gists" -GITHUB_API_VERSION = "2022-11-28" - - -class APIRequester(object): - """REST API client that injects proper GitHub authentication headers.""" - - def __init__(self, github_token: str): - self._api_headers = { - "Accept": "application/vnd.github+json", - "Authorization": f"token {github_token}", - "X-GitHub-Api-Version": GITHUB_API_VERSION, - } - self._session = requests.session() - - def get(self, endpoint: str, payload: Any = {}) -> requests.Response: - return self._session.get( - endpoint, data=json.dumps(payload), headers=self._api_headers - ) - - def post(self, endpoint: str, payload: Any = {}) -> requests.Response: - return self._session.post( - endpoint, data=json.dumps(payload), headers=self._api_headers - ) - - def patch(self, endpoint: str, payload: Any = {}) -> requests.Response: - return self._session.patch( - endpoint, data=json.dumps(payload), headers=self._api_headers - ) - - -class GithubClient(object): - """Helper to call Github REST APIs.""" - - def __init__(self, requester: APIRequester): - self._requester = requester - - def post_to_gist(self, filename: str, content: str, verbose: bool = False) -> str: - """Posts the given content to a new GitHub Gist and returns the URL to it.""" - - response = self._requester.post( - endpoint=GITHUB_GIST_API, - payload={"public": True, "files": {filename: {"content": content}}}, - ) - if response.status_code != http.client.CREATED: - raise RuntimeError( - f"Failed to create on gist; error code: {response.status_code} - {response.text}" - ) - - response = response.json() - if verbose: - print(f"Gist posting response: {response}") - - if response["truncated"]: - raise RuntimeError(f"Content is too large and was truncated") - - return response["html_url"] - - def get_previous_comment_on_pr( - self, - pr_number: int, - comment_bot_user: str, - comment_type_id: str, - query_comment_per_page: int = 100, - max_pages_to_search: int = 10, - verbose: bool = False, - ) -> Optional[int]: - """Gets the previous comment's id from GitHub.""" - - for page in range(1, max_pages_to_search + 1): - response = self._requester.get( - endpoint=f"{GITHUB_IREE_API_PREFIX}/issues/{pr_number}/comments", - payload={ - "per_page": query_comment_per_page, - "page": page, - "sort": "updated", - "direction": "desc", - }, - ) - if response.status_code != http.client.OK: - raise RuntimeError( - f"Failed to get PR comments from GitHub; error code: {response.status_code} - {response.text}" - ) - - comments = response.json() - if verbose: - print(f"Previous comment query response on page {page}: {comments}") - - # Find the most recently updated comment that matches. - for comment in comments: - if ( - comment["user"]["login"] == comment_bot_user - and comment_type_id in comment["body"] - ): - return comment["id"] - - if len(comments) < query_comment_per_page: - break - - return None - - def update_comment_on_pr(self, comment_id: int, content: str): - """Updates the content of the given comment id.""" - - response = self._requester.patch( - endpoint=f"{GITHUB_IREE_API_PREFIX}/issues/comments/{comment_id}", - payload={"body": content}, - ) - if response.status_code != http.client.OK: - raise RuntimeError( - f"Failed to comment on GitHub; error code: {response.status_code} - {response.text}" - ) - - def create_comment_on_pr(self, pr_number: int, content: str): - """Posts the given content as comments to the current pull request.""" - - response = self._requester.post( - endpoint=f"{GITHUB_IREE_API_PREFIX}/issues/{pr_number}/comments", - payload={"body": content}, - ) - if response.status_code != http.client.CREATED: - raise RuntimeError( - f"Failed to comment on GitHub; error code: {response.status_code} - {response.text}" - ) - - def get_pull_request_head_commit(self, pr_number: int) -> str: - """Get pull request head commit SHA.""" - - response = self._requester.get( - endpoint=f"{GITHUB_IREE_API_PREFIX}/pulls/{pr_number}" - ) - if response.status_code != http.client.OK: - raise RuntimeError( - f"Failed to fetch the pull request: {pr_number}; " - f"error code: {response.status_code} - {response.text}" - ) - - return response.json()["head"]["sha"] - - -def _parse_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument("comment_json", type=pathlib.Path) - parser.add_argument("--verbose", action="store_true") - verification_parser = parser.add_mutually_exclusive_group(required=True) - verification_parser.add_argument("--github_event_json", type=pathlib.Path) - return parser.parse_args() - - -def main(args: argparse.Namespace): - github_token = os.environ.get("GITHUB_TOKEN") - if github_token is None: - raise ValueError("GITHUB_TOKEN must be set.") - - comment_bot_user = os.environ.get("COMMENT_BOT_USER") - if comment_bot_user is None: - raise ValueError("COMMENT_BOT_USER must be set.") - - gist_bot_token = os.environ.get("GIST_BOT_TOKEN") - if gist_bot_token is None: - raise ValueError("GIST_BOT_TOKEN must be set.") - - comment_data = benchmark_comment.CommentData( - **json.loads(args.comment_json.read_text()) - ) - # Sanitize the pr number to make sure it is an integer. - pr_number = int(comment_data.unverified_pr_number) - - pr_client = GithubClient(requester=APIRequester(github_token=github_token)) - if args.github_event_json is None: - github_event = None - else: - github_event = json.loads(args.github_event_json.read_text()) - workflow_run_sha = github_event["workflow_run"]["head_sha"] - pr_head_sha = pr_client.get_pull_request_head_commit(pr_number=pr_number) - # We can't get the trusted PR number of a workflow run from GitHub API. So we - # take the untrusted PR number from presubmit workflow and verify if the PR's - # current head SHA matches the commit SHA in the workflow run. It assumes - # that to generate the malicious comment data, attacker must modify the code - # and has a new commit SHA. So if the PR head commit matches the workflow - # run with attacker's commit, either the PR is created by the attacker or - # other's PR has the malicious commit. In both cases posting malicious - # comment is acceptable. - # - # Note that the collision of a target SHA1 is possible but GitHub has some - # protections (https://github.blog/2017-03-20-sha-1-collision-detection-on-github-com/). - # The assumption also only holds if files in GCS can't be overwritten (so the - # comment data can't be modified without changing the code). - # The check will also fail if the PR author pushes the new commit after the - # workflow is triggered. But pushing the new commit means to cancel the - # current CI run including the benchmarking. So it will unlikely fail for - # that reason. - if workflow_run_sha != pr_head_sha: - raise ValueError( - f"Workflow run SHA: {workflow_run_sha} does not match " - f"the head SHA: {pr_head_sha} of the pull request: {pr_number}." - ) - - gist_client = GithubClient(requester=APIRequester(github_token=gist_bot_token)) - gist_url = gist_client.post_to_gist( - filename=f"iree-full-benchmark-results-{pr_number}.md", - content=comment_data.full_md, - verbose=args.verbose, - ) - - previous_comment_id = pr_client.get_previous_comment_on_pr( - pr_number=pr_number, - comment_bot_user=comment_bot_user, - comment_type_id=comment_data.type_id, - verbose=args.verbose, - ) - - abbr_md = comment_data.abbr_md.replace( - benchmark_comment.GIST_LINK_PLACEHORDER, gist_url - ) - if github_event is not None: - abbr_md += ( - f'\n\n[Source Workflow Run]({github_event["workflow_run"]["html_url"]})' - ) - if previous_comment_id is not None: - pr_client.update_comment_on_pr(comment_id=previous_comment_id, content=abbr_md) - else: - pr_client.create_comment_on_pr(pr_number=pr_number, content=abbr_md) - - -if __name__ == "__main__": - main(_parse_arguments()) diff --git a/build_tools/benchmarks/post_benchmark_comment_test.py b/build_tools/benchmarks/post_benchmark_comment_test.py deleted file mode 100644 index d54fb8de6def..000000000000 --- a/build_tools/benchmarks/post_benchmark_comment_test.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import http.client -import requests -import unittest -from unittest import mock -from typing import Any - -import post_benchmark_comment - - -class GithubClientTest(unittest.TestCase): - def setUp(self): - self._mock_response = mock.create_autospec(requests.Response) - self._mock_requester = mock.create_autospec(post_benchmark_comment.APIRequester) - self._mock_requester.get.return_value = self._mock_response - self._mock_requester.post.return_value = self._mock_response - self._mock_requester.patch.return_value = self._mock_response - - def test_post_to_gist(self): - gist_url = "https://example.com/123455/1234.md" - self._mock_response.status_code = http.client.CREATED - self._mock_response.json.return_value = { - "html_url": gist_url, - "truncated": False, - } - client = post_benchmark_comment.GithubClient(self._mock_requester) - - url = client.post_to_gist(filename="1234.md", content="xyz") - - self.assertEqual(url, gist_url) - self._mock_requester.post.assert_called_once_with( - endpoint=post_benchmark_comment.GITHUB_GIST_API, - payload={"public": True, "files": {"1234.md": {"content": "xyz"}}}, - ) - - def test_post_to_gist_truncated(self): - gist_url = "example.com/123455/1234.md" - self._mock_response.status_code = http.client.CREATED - self._mock_response.json.return_value = { - "html_url": gist_url, - "truncated": True, - } - client = post_benchmark_comment.GithubClient(self._mock_requester) - - with self.assertRaises(RuntimeError) as _: - client.post_to_gist(filename="1234.md", content="xyz") - - def test_get_previous_comment_on_pr(self): - first_mock_response = mock.create_autospec(requests.Response) - first_mock_response.status_code = http.client.OK - first_mock_response.json.return_value = [ - {"id": 1, "user": {"login": "bot"}, "body": "comment id: abcd"}, - {"id": 2, "user": {"login": "user"}, "body": "comment id: 1234"}, - ] - second_mock_response = mock.create_autospec(requests.Response) - second_mock_response.status_code = http.client.OK - second_mock_response.json.return_value = [ - {"id": 3, "user": {"login": "bot"}, "body": "comment id: 1234"} - ] - mock_requester = mock.create_autospec(post_benchmark_comment.APIRequester) - mock_requester.get.side_effect = [first_mock_response, second_mock_response] - client = post_benchmark_comment.GithubClient(mock_requester) - - comment_id = client.get_previous_comment_on_pr( - pr_number=23, - comment_bot_user="bot", - comment_type_id="1234", - query_comment_per_page=2, - max_pages_to_search=10, - ) - - self.assertEqual(comment_id, 3) - self.assertEqual(mock_requester.get.call_count, 2) - endpoint_url = ( - f"{post_benchmark_comment.GITHUB_IREE_API_PREFIX}/issues/23/comments" - ) - mock_requester.get.assert_any_call( - endpoint=endpoint_url, - payload={"per_page": 2, "page": 1, "sort": "updated", "direction": "desc"}, - ) - mock_requester.get.assert_any_call( - endpoint=endpoint_url, - payload={"per_page": 2, "page": 2, "sort": "updated", "direction": "desc"}, - ) - - def test_get_previous_comment_on_pr_not_found(self): - mock_response = mock.create_autospec(requests.Response) - mock_response.status_code = http.client.OK - mock_response.json.return_value = [ - {"id": 1, "user": {"login": "bot"}, "body": "comment id: 5678"} - ] - mock_requester = mock.create_autospec(post_benchmark_comment.APIRequester) - mock_requester.get.side_effect = [mock_response] * 10 - client = post_benchmark_comment.GithubClient(mock_requester) - - comment_id = client.get_previous_comment_on_pr( - pr_number=23, - comment_bot_user="bot", - comment_type_id="1234", - query_comment_per_page=1, - max_pages_to_search=10, - ) - - self.assertIsNone(comment_id) - self.assertEqual(mock_requester.get.call_count, 10) - endpoint_url = ( - f"{post_benchmark_comment.GITHUB_IREE_API_PREFIX}/issues/23/comments" - ) - mock_requester.get.assert_any_call( - endpoint=endpoint_url, - payload={"per_page": 1, "page": 1, "sort": "updated", "direction": "desc"}, - ) - mock_requester.get.assert_any_call( - endpoint=endpoint_url, - payload={"per_page": 1, "page": 10, "sort": "updated", "direction": "desc"}, - ) - - def test_update_comment_on_pr(self): - self._mock_response.status_code = http.client.OK - client = post_benchmark_comment.GithubClient(self._mock_requester) - - client.update_comment_on_pr(comment_id=123, content="xyz") - - self._mock_requester.patch.assert_called_once_with( - endpoint=f"{post_benchmark_comment.GITHUB_IREE_API_PREFIX}/issues/comments/123", - payload={"body": "xyz"}, - ) - - def test_create_comment_on_pr(self): - self._mock_response.status_code = http.client.CREATED - client = post_benchmark_comment.GithubClient(self._mock_requester) - - client.create_comment_on_pr(pr_number=1234, content="xyz") - - self._mock_requester.post.assert_called_once_with( - endpoint=f"{post_benchmark_comment.GITHUB_IREE_API_PREFIX}/issues/1234/comments", - payload={"body": "xyz"}, - ) - - def test_get_pull_request_head_commit(self): - self._mock_response.status_code = http.client.OK - self._mock_response.json.return_value = {"head": {"sha": "sha123"}} - client = post_benchmark_comment.GithubClient(self._mock_requester) - - commit_sha = client.get_pull_request_head_commit(pr_number=123) - - self.assertEqual(commit_sha, "sha123") - self._mock_requester.get.assert_called_once_with( - endpoint=f"{post_benchmark_comment.GITHUB_IREE_API_PREFIX}/pulls/123" - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/benchmarks/run_benchmarks.sh b/build_tools/benchmarks/run_benchmarks.sh deleted file mode 100755 index 888d02012120..000000000000 --- a/build_tools/benchmarks/run_benchmarks.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Script to run benchmarks on CI with the proper docker image and benchmark tool -# based on the IREE_DEVICE_NAME. This script can also run locally, but some -# devices require docker to run benchmarks. By default it uses the wrapper -# build_tools/docker/docker_run.sh if IREE_DOCKER_WRAPPER is not specified. See -# the script to learn about the required setup. -# -# IREE_BENCHMARK_TOOLS_DIR needs to point to a directory contains IREE -# benchmark tools. See benchmarks/README.md for more information. -# -# Command line arguments: -# 1. The path of e2e test artifacts directory -# 2. The path of IREE benchmark run config -# 3. The target device name -# 4. The shard index -# 5. The path to write benchmark results - -set -euo pipefail - -DOCKER_WRAPPER="${IREE_DOCKER_WRAPPER:-./build_tools/docker/docker_run.sh}" -BENCHMARK_TOOLS_DIR="${IREE_BENCHMARK_TOOLS_DIR}" -E2E_TEST_ARTIFACTS_DIR="${1:-${IREE_E2E_TEST_ARTIFACTS_DIR}}" -EXECUTION_BENCHMARK_CONFIG="${2:-${IREE_EXECUTION_BENCHMARK_CONFIG}}" -TARGET_DEVICE_NAME="${3:-${IREE_TARGET_DEVICE_NAME}}" -SHARD_INDEX="${4:-${IREE_SHARD_INDEX}}" -BENCHMARK_RESULTS="${5:-${IREE_BENCHMARK_RESULTS}}" - -if [[ "${TARGET_DEVICE_NAME}" == "a2-highgpu-1g" ]]; then - ${DOCKER_WRAPPER} \ - --gpus all \ - --env NVIDIA_DRIVER_CAPABILITIES=all \ - gcr.io/iree-oss/nvidia-bleeding-edge@sha256:a3b7b78c66c6fe3383b01c7662982c63f708637af1f92a7890b3256c2f178aa7 \ - ./build_tools/benchmarks/run_benchmarks_on_linux.py \ - --benchmark_tool_dir="${BENCHMARK_TOOLS_DIR}" \ - --e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR}" \ - --execution_benchmark_config="${EXECUTION_BENCHMARK_CONFIG}" \ - --target_device_name="${TARGET_DEVICE_NAME}" \ - --shard_index="${SHARD_INDEX}" \ - --output="${BENCHMARK_RESULTS}" \ - --verbose -elif [[ "${TARGET_DEVICE_NAME}" == "c2-standard-60" ]]; then - ${DOCKER_WRAPPER} \ - gcr.io/iree-oss/base-bleeding-edge@sha256:cf2e78194e64fd0166f4141317366261d7a62432b72e9a324cb8c2ff4e1a515a \ - ./build_tools/benchmarks/run_benchmarks_on_linux.py \ - --benchmark_tool_dir="${BENCHMARK_TOOLS_DIR}" \ - --e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR}" \ - --execution_benchmark_config="${EXECUTION_BENCHMARK_CONFIG}" \ - --target_device_name="${TARGET_DEVICE_NAME}" \ - --shard_index="${SHARD_INDEX}" \ - --output="${BENCHMARK_RESULTS}" \ - --device_model="GCP-${TARGET_DEVICE_NAME}" \ - --cpu_uarch=CascadeLake \ - --verbose -elif [[ "${TARGET_DEVICE_NAME}" =~ ^(pixel-4|pixel-6-pro|moto-edge-x30)$ ]]; then - ./build_tools/benchmarks/run_benchmarks_on_android.py \ - --benchmark_tool_dir="${BENCHMARK_TOOLS_DIR}" \ - --e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR}" \ - --execution_benchmark_config="${EXECUTION_BENCHMARK_CONFIG}" \ - --target_device_name="${TARGET_DEVICE_NAME}" \ - --shard_index="${SHARD_INDEX}" \ - --output="${BENCHMARK_RESULTS}" \ - --pin-cpu-freq \ - --pin-gpu-freq \ - --verbose -else - echo "${TARGET_DEVICE_NAME} is not supported yet." - exit 1 -fi diff --git a/build_tools/benchmarks/run_benchmarks_on_android.py b/build_tools/benchmarks/run_benchmarks_on_android.py deleted file mode 100755 index 5b1277db4eaa..000000000000 --- a/build_tools/benchmarks/run_benchmarks_on_android.py +++ /dev/null @@ -1,550 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Runs all matched benchmark suites on an Android device. - -This script probes the Android phone via `adb` and uses the device information -to filter and run suitable benchmarks on the Android device. - -It expects that `adb` is installed, and there is iree tools cross-compiled -towards Android. - -Example usage: - - python3 run_benchmarks.py \ - --benchmark_tool_dir=/path/to/normal/android/target/tools/dir \ - /path/to/host/build/dir - -""" - -import sys -import pathlib - -# Add build_tools python dir to the search path. -sys.path.insert(0, str(pathlib.Path(__file__).parent.with_name("python"))) - -import atexit -import json -import requests -import shutil -import socket -import struct -import subprocess -import tarfile -import time -from typing import Any, Optional, Sequence, Tuple - -from common import benchmark_suite as benchmark_suite_module -from common.benchmark_config import BenchmarkConfig -from common.benchmark_driver import BenchmarkDriver -from common import benchmark_definition -from common.benchmark_definition import ( - execute_cmd, - execute_cmd_and_get_stdout, - execute_cmd_and_get_output, - get_git_commit_hash, - get_iree_benchmark_module_arguments, - wait_for_iree_benchmark_module_start, - parse_iree_benchmark_metrics, -) -from common.benchmark_suite import BenchmarkCase, BenchmarkSuite -from common.android_device_utils import ( - get_android_device_model, - get_android_device_info, - get_android_gpu_name, -) -import common.common_arguments -from e2e_test_artifacts import iree_artifacts -from e2e_test_framework.definitions import iree_definitions - -# Root directory to perform benchmarks in on the Android device. -ANDROID_TMPDIR = pathlib.PurePosixPath("/data/local/tmp/iree-benchmarks") -ADB_SERVER_ADDR = ("localhost", 5037) -ANDROID_NORMAL_TOOL_DIR = ANDROID_TMPDIR / "normal-tools" -ANDROID_TRACY_PORT = 8086 - - -def adb_push_file( - source: pathlib.Path, - dest: pathlib.PurePosixPath, - verbose: bool = False, -) -> pathlib.PurePosixPath: - """Pushes content onto the Android device. - - Args: - source: the path to the source file. - dest: the full dest path on the device. - - Returns: - The full path to the content on the Android device. - """ - # When the output is a TTY, keep the default progress info output. - # In other cases, redirect progress info to null to avoid bloating log files. - stdout_redirect = None if sys.stdout.isatty() else subprocess.DEVNULL - execute_cmd( - ["adb", "push", source.resolve(), dest], - verbose=verbose, - stdout=stdout_redirect, - ) - return dest - - -def adb_execute_and_get_output( - cmd_args: Sequence[str], - cwd: pathlib.PurePosixPath = ANDROID_TMPDIR, - verbose: bool = False, -) -> Tuple[str, str]: - """Executes command with adb shell. - - Switches to `cwd` before executing. Waits for completion and returns the - command stdout. - - Args: - cmd_args: a list containing the command to execute and its parameters - cwd: the directory to execute the command in - - Returns: - Strings for stdout and stderr. - """ - cmd = ["adb", "shell", "cd", cwd, "&&"] - cmd.extend(cmd_args) - return execute_cmd_and_get_output(cmd, verbose=verbose) - - -def adb_execute( - cmd_args: Sequence[str], - cwd: pathlib.PurePosixPath = ANDROID_TMPDIR, - verbose: bool = False, -) -> subprocess.CompletedProcess: - """Executes command with adb shell. - - Switches to `cwd` before executing. Waits for completion. Output is streamed - to the terminal. - - Args: - cmd_args: a list containing the command to execute and its parameters - cwd: the directory to execute the command in - - Returns: - The completed process. - """ - cmd = ["adb", "shell", "cd", cwd, "&&"] - cmd.extend(cmd_args) - return execute_cmd(cmd, verbose=verbose) - - -def is_magisk_su(): - """Returns true if the Android device has a Magisk SU binary.""" - stdout, _ = adb_execute_and_get_output(["su", "--help"]) - return "MagiskSU" in stdout - - -def adb_execute_as_root(cmd_args: Sequence[Any]) -> subprocess.CompletedProcess: - """Executes the given command as root.""" - cmd = ["su", "-c" if is_magisk_su() else "root"] - cmd.extend(cmd_args) - return adb_execute(cmd) - - -def adb_start_cmd( - cmd_args: Sequence[str], - cwd: pathlib.PurePosixPath = ANDROID_TMPDIR, - verbose: bool = False, -) -> subprocess.Popen: - """Executes command with adb shell in a directory and returns the handle - without waiting for completion. - - Args: - cmd_args: a list containing the command to execute and its parameters - cwd: the directory to execute the command in - - Returns: - A Popen object for the started command. - """ - cmd = ["adb", "shell", "cd", cwd, "&&"] - cmd.extend(cmd_args) - - if verbose: - print(f"cmd: {cmd}") - return subprocess.Popen(cmd, stdout=subprocess.PIPE, text=True) - - -def adb_path_exists(android_path: pathlib.PurePosixPath, verbose: bool = False): - """Run stat to check if the path exists.""" - proc = adb_start_cmd(["stat", str(android_path)], verbose=verbose) - return proc.wait() == 0 - - -def adb_fetch_and_push_file( - source: benchmark_definition.ResourceLocation, - dest: pathlib.PurePosixPath, - verbose: bool = False, -): - """Fetch file from the path/URL and stream to the device. - - In the case of fetching, this method avoids the temporary file on the host - and reduces the overhead when the file is large. - - Args: - source: path/URL to fetch the file. - dest: the full dest path on the device. - verbose: output verbose message. - - Returns: - File path on the device. - """ - - if adb_path_exists(dest, verbose): - return dest - - # If the source is a local file, push directly. - local_path = source.get_local_path() - if local_path: - return adb_push_file(local_path, dest, verbose=verbose) - - if verbose: - print(f"Streaming file {source} to {dest}.") - - url = source.get_url() - assert url is not None - req = requests.get(url, stream=True, timeout=60) - if not req.ok: - raise RuntimeError(f"Failed to fetch {source}: {req.status_code} - {req.text}") - - # Implement the ADB sync protocol to stream file chunk to the device, since - # the adb client tool doesn't support it. - # - # Alternatively we can use thrid-party library such as - # https://github.com/JeffLIrion/adb_shell. But the protocol we need is - # simple and fairly stable. This part can be replaced with other solutions - # if needed. - # - # To understand the details of the protocol, see - # https://cs.android.com/android/_/android/platform/packages/modules/adb/+/93c8e3c26e4de3a2b767a2394200bc0721bb1e24:OVERVIEW.TXT - - def wait_ack_ok(sock: socket.socket): - buf = bytearray() - while len(buf) < 4: - data = sock.recv(4 - len(buf)) - if not data: - break - buf += data - - if buf.decode("utf-8") != "OKAY": - raise RuntimeError(f"ADB communication error: {buf}") - - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.connect(ADB_SERVER_ADDR) - # Connect to any device (the first 4 hexadecimals is the following text - # command length). - sock.sendall(b"0012host:transport-any") - wait_ack_ok(sock) - # Switch to sync mode. - sock.sendall(b"0005sync:") - wait_ack_ok(sock) - # Send the dest file path and file permissions 0644 (rw-r-r). - file_attr = f"{dest},{0o644}".encode("utf-8") - sock.sendall(b"SEND" + struct.pack("I", len(file_attr)) + file_attr) - # Stream the file chunks. 64k bytes is the max chunk size for adb. - for data in req.iter_content(chunk_size=64 * 1024): - sock.sendall(b"DATA" + struct.pack("I", len(data)) + data) - # End the file stream and set the creation time. - sock.sendall(b"DONE" + struct.pack("I", int(time.time()))) - wait_ack_ok(sock) - - return dest - - -class AndroidBenchmarkDriver(BenchmarkDriver): - """Android benchmark driver.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.already_pushed_files = {} - - def run_benchmark_case( - self, - benchmark_case: BenchmarkCase, - benchmark_results_filename: Optional[pathlib.Path], - ) -> None: - module_rel_dir = iree_artifacts.get_module_dir_path( - benchmark_case.run_config.module_generation_config - ) - android_case_dir = ANDROID_TMPDIR / module_rel_dir - - module_path = benchmark_case.module_dir / iree_artifacts.MODULE_FILENAME - module_device_path = adb_fetch_and_push_file( - source=module_path, - dest=android_case_dir / iree_artifacts.MODULE_FILENAME, - verbose=self.verbose, - ) - - inputs_dir = None - if benchmark_case.input_uri: - inputs_dir = self.__fetch_and_unpack_npy( - url=benchmark_case.input_uri, - device_dir=android_case_dir / "inputs_npy", - ) - expected_outputs_dir = None - if self.config.verify and benchmark_case.expected_output_uri: - expected_outputs_dir = self.__fetch_and_unpack_npy( - url=benchmark_case.expected_output_uri, - device_dir=android_case_dir / "expected_outputs_npy", - ) - external_params = [] - if benchmark_case.external_param_urls: - for param_url in benchmark_case.external_param_urls: - scope, url, filename = benchmark_definition.parse_external_param_url( - param_url - ) - param_path = adb_fetch_and_push_file( - source=benchmark_definition.ResourceLocation.build_url(url), - dest=android_case_dir / filename, - verbose=self.verbose, - ) - param_arg = f"{scope}={param_path}" if scope else param_path - external_params.append(param_arg) - - run_config = benchmark_case.run_config - # TODO(#15452): Change to `--task_topology_cpu_ids` once we figure out - # the right mapping. - taskset = self.__deduce_taskset_from_run_config(run_config) - run_args = run_config.materialize_run_flags( - inputs_dir=inputs_dir, external_params=external_params - ) - run_args.append(f"--module={module_device_path}") - - if benchmark_results_filename is not None: - if self.config.benchmark_tool_dir is None: - raise ValueError("benchmark_tool_dir can't be None.") - if expected_outputs_dir: - self.__run_verify( - host_tool_dir=self.config.benchmark_tool_dir, - run_args=run_args, - expected_outputs_dir=expected_outputs_dir, - verify_params=benchmark_case.verify_params, - taskset=taskset, - ) - - self.__run_benchmark( - host_tool_dir=self.config.benchmark_tool_dir, - benchmark_case=benchmark_case, - run_args=run_args, - results_filename=benchmark_results_filename, - taskset=taskset, - ) - - def __run_verify( - self, - host_tool_dir: pathlib.Path, - run_args: Sequence[str], - expected_outputs_dir: pathlib.PurePosixPath, - verify_params: Sequence[str], - taskset: str, - ): - device_tool = self.__check_and_push_file( - host_tool_dir / "iree-run-module", ANDROID_NORMAL_TOOL_DIR - ) - cmd = ["taskset", taskset, device_tool] - cmd += run_args - # Currently only support single output. - cmd.append(f'--expected_output=@{expected_outputs_dir / "output_0.npy"}') - cmd += verify_params - adb_execute(cmd, verbose=self.verbose) - - def __run_benchmark( - self, - host_tool_dir: pathlib.Path, - benchmark_case: BenchmarkCase, - run_args: Sequence[str], - results_filename: pathlib.Path, - taskset: str, - ): - tool_name = benchmark_case.benchmark_tool_name - device_tool = self.__check_and_push_file( - host_tool_dir / tool_name, ANDROID_NORMAL_TOOL_DIR - ) - cmd = ["taskset", taskset, device_tool] - cmd += run_args - if tool_name == "iree-benchmark-module": - cmd += get_iree_benchmark_module_arguments( - driver_info=benchmark_case.driver_info, - benchmark_min_time=self.config.benchmark_min_time, - ) - - benchmark_stdout, benchmark_stderr = adb_execute_and_get_output( - cmd, verbose=self.verbose - ) - benchmark_metrics = parse_iree_benchmark_metrics( - benchmark_stdout, benchmark_stderr - ) - if self.verbose: - print(benchmark_metrics) - results_filename.write_text(json.dumps(benchmark_metrics.to_json_object())) - - def __deduce_taskset_from_run_config( - self, run_config: iree_definitions.E2EModelRunConfig - ) -> str: - """Deduces the CPU mask according to device and execution config.""" - - cpu_params = run_config.target_device_spec.device_parameters.cpu_params - if not cpu_params: - # Assume the mobile CPUs have <= 16 cores. - return "ffff" - - exec_config = run_config.module_execution_config - pinned_cores = cpu_params.pinned_cores - # Use the fastest cores in the spec for single-thread benchmarks. - if ( - exec_config.driver == iree_definitions.RuntimeDriver.LOCAL_SYNC - or "1-thread" in exec_config.tags - ): - pinned_cores = pinned_cores[-1:] - - cpu_mask = sum(1 << core_id for core_id in cpu_params.pinned_cores) - return f"{cpu_mask:04x}" - - def __check_and_push_file( - self, host_path: pathlib.Path, device_dir: pathlib.PurePosixPath - ): - """Checks if the file has been pushed and pushes it if not.""" - android_path = self.already_pushed_files.get(host_path) - if android_path is not None: - return android_path - - android_path = adb_push_file( - host_path, - device_dir / host_path.name, - verbose=self.verbose, - ) - self.already_pushed_files[host_path] = android_path - return android_path - - def __fetch_and_unpack_npy(self, url: str, device_dir: pathlib.PurePosixPath): - if adb_path_exists(device_dir, verbose=self.verbose): - return device_dir - - archive_path = adb_fetch_and_push_file( - source=benchmark_definition.ResourceLocation.build_url(url), - dest=device_dir.with_suffix(".tgz"), - ) - adb_execute( - ["mkdir", "-p", str(device_dir)] - + ["&&", "tar", "-xvf", str(archive_path), "-C", str(device_dir)], - verbose=self.verbose, - ) - return device_dir - - -def set_cpu_frequency_scaling_governor(governor: str): - git_root = execute_cmd_and_get_stdout(["git", "rev-parse", "--show-toplevel"]) - cpu_script = ( - pathlib.Path(git_root) - / "build_tools" - / "benchmarks" - / "set_android_scaling_governor.sh" - ) - android_path = adb_push_file(cpu_script, ANDROID_TMPDIR / cpu_script.name) - adb_execute_as_root([android_path, governor]) - - -def set_gpu_frequency_scaling_policy(policy: str): - git_root = execute_cmd_and_get_stdout(["git", "rev-parse", "--show-toplevel"]) - device_model = get_android_device_model() - gpu_name = get_android_gpu_name() - benchmarks_tool_dir = pathlib.Path(git_root) / "build_tools" / "benchmarks" - if device_model == "Pixel-6" or device_model == "Pixel-6-Pro": - gpu_script = benchmarks_tool_dir / "set_pixel6_gpu_scaling_policy.sh" - elif gpu_name.lower().startswith("adreno"): - gpu_script = benchmarks_tool_dir / "set_adreno_gpu_scaling_policy.sh" - else: - raise RuntimeError( - f"Unsupported device '{device_model}' for setting GPU scaling policy" - ) - android_path = adb_push_file(gpu_script, ANDROID_TMPDIR / gpu_script.name) - adb_execute_as_root([android_path, policy]) - - -def add_port_forwarding(port: int, verbose: bool): - """Add adb port forwarding.""" - execute_cmd_and_get_stdout( - ["adb", "forward", f"tcp:{port}", f"tcp:{port}"], verbose=verbose - ) - atexit.register( - execute_cmd_and_get_stdout, - ["adb", "forward", "--remove", f"tcp:{port}"], - verbose=verbose, - ) - - -def main(args): - device_info = get_android_device_info(args.verbose) - if args.verbose: - print(device_info) - - commit = get_git_commit_hash("HEAD") - benchmark_config = BenchmarkConfig.build_from_args(args, commit) - benchmark_groups = json.loads(args.execution_benchmark_config.read_text()) - run_configs = benchmark_suite_module.get_run_configs_by_target_and_shard( - benchmark_groups, args.target_device_name, args.shard_index - ) - - benchmark_suite = BenchmarkSuite.load_from_run_configs( - run_configs=run_configs, root_benchmark_dir=benchmark_config.root_benchmark_dir - ) - - benchmark_driver = AndroidBenchmarkDriver( - device_info=device_info, - benchmark_config=benchmark_config, - benchmark_suite=benchmark_suite, - benchmark_grace_time=1.0, - verbose=args.verbose, - ) - - if args.pin_cpu_freq: - set_cpu_frequency_scaling_governor("performance") - atexit.register(set_cpu_frequency_scaling_governor, "schedutil") - if args.pin_gpu_freq: - set_gpu_frequency_scaling_policy("performance") - atexit.register(set_gpu_frequency_scaling_policy, "default") - - # Clear the benchmark directory on the Android device first just in case - # there are leftovers from manual or failed runs. - adb_execute( - ["rm", "-rf", str(ANDROID_TMPDIR), "&&", "mkdir", "-p", str(ANDROID_TMPDIR)], - cwd=pathlib.PurePosixPath("/"), - verbose=args.verbose, - ) - - if not args.no_clean: - # Clear the benchmark directory on the Android device. - atexit.register( - execute_cmd_and_get_stdout, - ["adb", "shell", "rm", "-rf", ANDROID_TMPDIR], - verbose=args.verbose, - ) - # Also clear temporary directory on the host device. - atexit.register(shutil.rmtree, args.tmp_dir) - - benchmark_driver.run() - - benchmark_results = benchmark_driver.get_benchmark_results() - if args.output is not None: - with open(args.output, "w") as f: - f.write(benchmark_results.to_json_str()) - - if args.verbose: - print(benchmark_results.commit) - print(benchmark_results.benchmarks) - - benchmark_errors = benchmark_driver.get_benchmark_errors() - if benchmark_errors: - print("Benchmarking completed with errors", file=sys.stderr) - raise RuntimeError(benchmark_errors) - - -if __name__ == "__main__": - main(common.common_arguments.Parser().parse_args()) diff --git a/build_tools/benchmarks/run_benchmarks_on_linux.py b/build_tools/benchmarks/run_benchmarks_on_linux.py deleted file mode 100755 index 0adc89a18f64..000000000000 --- a/build_tools/benchmarks/run_benchmarks_on_linux.py +++ /dev/null @@ -1,291 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Runs all matched benchmark suites on a Linux device.""" - -import sys -import pathlib - -# Add build_tools python dir to the search path. -sys.path.insert(0, str(pathlib.Path(__file__).parent.with_name("python"))) - -from typing import Any, List, Optional, Sequence -import atexit -import json -import requests -import shutil -import subprocess -import tarfile - -from common import benchmark_suite as benchmark_suite_module -from common.benchmark_driver import BenchmarkDriver -from common.benchmark_suite import BenchmarkCase, BenchmarkSuite -from common.benchmark_config import BenchmarkConfig -from common import benchmark_definition -from common.benchmark_definition import ( - execute_cmd, - execute_cmd_and_get_output, - get_git_commit_hash, - get_iree_benchmark_module_arguments, - wait_for_iree_benchmark_module_start, - parse_iree_benchmark_metrics, -) -from common.linux_device_utils import get_linux_device_info -from e2e_test_artifacts import iree_artifacts - -import common.common_arguments - - -class LinuxBenchmarkDriver(BenchmarkDriver): - """Linux benchmark driver.""" - - def __init__(self, gpu_id: str, *args, **kwargs): - self.gpu_id = gpu_id - super().__init__(*args, **kwargs) - - def run_benchmark_case( - self, - benchmark_case: BenchmarkCase, - benchmark_results_filename: Optional[pathlib.Path], - ) -> None: - module_dir = benchmark_case.module_dir - local_module_dir = module_dir.get_local_path() - if local_module_dir: - case_tmp_dir = local_module_dir - module_path = local_module_dir / iree_artifacts.MODULE_FILENAME - else: - module_rel_dir = iree_artifacts.get_module_dir_path( - benchmark_case.run_config.module_generation_config - ) - case_tmp_dir = self.config.tmp_dir / module_rel_dir - case_tmp_dir.mkdir(parents=True, exist_ok=True) - module_url = (module_dir / iree_artifacts.MODULE_FILENAME).get_url() - assert module_url is not None - module_path = self.__fetch_file( - uri=module_url, dest=case_tmp_dir / iree_artifacts.MODULE_FILENAME - ) - - inputs_dir = None - expected_output_dir = None - if benchmark_case.input_uri: - inputs_dir = self.__fetch_and_unpack_npy( - uri=benchmark_case.input_uri, dest_dir=case_tmp_dir / "inputs_npy" - ) - if benchmark_case.expected_output_uri: - expected_output_dir = self.__fetch_and_unpack_npy( - uri=benchmark_case.expected_output_uri, - dest_dir=case_tmp_dir / "expected_outputs_npy", - ) - external_params = [] - if benchmark_case.external_param_urls: - for param_url in benchmark_case.external_param_urls: - scope, url, filename = benchmark_definition.parse_external_param_url( - param_url - ) - param_path = self.__fetch_file(uri=url, dest=case_tmp_dir / filename) - param_arg = f"{scope}={param_path}" if scope else param_path - external_params.append(param_arg) - - if benchmark_results_filename: - if self.config.benchmark_tool_dir is None: - raise ValueError("benchmark_tool_dir can't be None.") - - if self.config.verify and expected_output_dir: - if not inputs_dir: - raise ValueError(f"Input data is missing for {benchmark_case}.") - self.__run_verify( - tool_dir=self.config.benchmark_tool_dir, - benchmark_case=benchmark_case, - module_path=module_path, - inputs_dir=inputs_dir, - expected_outputs_dir=expected_output_dir, - external_params=external_params, - ) - - self.__run_benchmark( - tool_dir=self.config.benchmark_tool_dir, - benchmark_case=benchmark_case, - module_path=module_path, - results_filename=benchmark_results_filename, - external_params=external_params, - ) - - def __build_tool_cmds( - self, - benchmark_case: BenchmarkCase, - tool_path: pathlib.Path, - module_path: pathlib.Path, - inputs_dir: Optional[pathlib.Path] = None, - external_params: Sequence[str] = (), - ) -> List[Any]: - run_config = benchmark_case.run_config - cmds = [tool_path, f"--module={module_path}"] - cmds += run_config.materialize_run_flags( - gpu_id=self.gpu_id, - inputs_dir=inputs_dir, - external_params=external_params, - ) - cpu_params = run_config.target_device_spec.device_parameters.cpu_params - if cpu_params: - raise ValueError("CPU pinning is not supported yet.") - return cmds - - def __fetch_and_unpack_npy(self, uri: str, dest_dir: pathlib.Path) -> pathlib.Path: - out_dir = self.__unpack_file( - src=self.__fetch_file( - uri=uri, - dest=dest_dir.with_suffix(".tgz"), - ), - dest=dest_dir, - ) - return out_dir.absolute() - - def __fetch_file(self, uri: str, dest: pathlib.Path) -> pathlib.Path: - """Check and fetch file if needed.""" - if dest.exists(): - return dest - req = requests.get(uri, stream=True, timeout=60) - if not req.ok: - raise RuntimeError(f"Failed to fetch {uri}: {req.status_code} - {req.text}") - with dest.open("wb") as dest_file: - for data in req.iter_content(chunk_size=64 * 1024 * 1024): - dest_file.write(data) - return dest - - def __unpack_file(self, src: pathlib.Path, dest: pathlib.Path) -> pathlib.Path: - """Unpack tar with/without compression.""" - if dest.exists(): - return dest - with tarfile.open(src) as tar_file: - tar_file.extractall(dest) - return dest - - def __run_verify( - self, - tool_dir: pathlib.Path, - benchmark_case: BenchmarkCase, - module_path: pathlib.Path, - inputs_dir: pathlib.Path, - expected_outputs_dir: pathlib.Path, - external_params: Sequence[str] = (), - ): - cmd = self.__build_tool_cmds( - benchmark_case=benchmark_case, - tool_path=tool_dir / "iree-run-module", - module_path=module_path, - inputs_dir=inputs_dir, - external_params=external_params, - ) - # Currently only support single output. - cmd.append(f'--expected_output=@{expected_outputs_dir / "output_0.npy"}') - cmd += benchmark_case.verify_params - execute_cmd_and_get_output(cmd, verbose=self.verbose) - - def __run_benchmark( - self, - tool_dir: pathlib.Path, - benchmark_case: BenchmarkCase, - module_path: pathlib.Path, - results_filename: pathlib.Path, - external_params: Sequence[str] = (), - ): - tool_name = benchmark_case.benchmark_tool_name - cmd = self.__build_tool_cmds( - benchmark_case=benchmark_case, - tool_path=tool_dir / tool_name, - module_path=module_path, - external_params=external_params, - ) - - if tool_name == "iree-benchmark-module": - cmd.extend( - get_iree_benchmark_module_arguments( - driver_info=benchmark_case.driver_info, - benchmark_min_time=self.config.benchmark_min_time, - ) - ) - - benchmark_stdout, benchmark_stderr = execute_cmd_and_get_output( - cmd, verbose=self.verbose - ) - benchmark_metrics = parse_iree_benchmark_metrics( - benchmark_stdout, benchmark_stderr - ) - if self.verbose: - print(benchmark_metrics) - results_filename.write_text(json.dumps(benchmark_metrics.to_json_object())) - - -def main(args): - device_info = get_linux_device_info( - args.device_model, args.cpu_uarch, args.gpu_id, args.verbose - ) - if args.verbose: - print(device_info) - - commit = get_git_commit_hash("HEAD") - benchmark_config = BenchmarkConfig.build_from_args(args, commit) - - benchmark_groups = json.loads(args.execution_benchmark_config.read_text()) - run_configs = benchmark_suite_module.get_run_configs_by_target_and_shard( - benchmark_groups, args.target_device_name, args.shard_index - ) - - benchmark_suite = BenchmarkSuite.load_from_run_configs( - run_configs=run_configs, root_benchmark_dir=benchmark_config.root_benchmark_dir - ) - - benchmark_driver = LinuxBenchmarkDriver( - gpu_id=args.gpu_id, - device_info=device_info, - benchmark_config=benchmark_config, - benchmark_suite=benchmark_suite, - benchmark_grace_time=1.0, - verbose=args.verbose, - ) - - if args.pin_cpu_freq: - raise NotImplementedError("CPU freq pinning is not supported yet.") - if args.pin_gpu_freq: - raise NotImplementedError("GPU freq pinning is not supported yet.") - if not args.no_clean: - atexit.register(shutil.rmtree, args.tmp_dir) - - benchmark_driver.run() - - benchmark_results = benchmark_driver.get_benchmark_results() - if args.output is not None: - with args.output.open("w") as f: - f.write(benchmark_results.to_json_str()) - - if args.verbose: - print(benchmark_results.commit) - print(benchmark_results.benchmarks) - - benchmark_errors = benchmark_driver.get_benchmark_errors() - if benchmark_errors: - print("Benchmarking completed with errors", file=sys.stderr) - raise RuntimeError(benchmark_errors) - - -def parse_argument(): - arg_parser = common.common_arguments.Parser() - arg_parser.add_argument("--device_model", default="Unknown", help="Device model") - arg_parser.add_argument( - "--cpu_uarch", default=None, help="CPU microarchitecture, e.g., CascadeLake" - ) - arg_parser.add_argument( - "--gpu_id", - type=str, - default="0", - help="GPU ID to run the benchmark, e.g., '0' or 'GPU-'", - ) - - return arg_parser.parse_args() - - -if __name__ == "__main__": - main(parse_argument()) diff --git a/build_tools/benchmarks/set_adreno_gpu_scaling_policy.sh b/build_tools/benchmarks/set_adreno_gpu_scaling_policy.sh deleted file mode 100755 index d2aca56b88e8..000000000000 --- a/build_tools/benchmarks/set_adreno_gpu_scaling_policy.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/sh - -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Runs on a phone with Adreno GPU to set the GPU frequency scaling policy. - -################################### WARNING #################################### -# This will overheat the phone if it's not on a cooling plate, resulting in # -# thermal throttling. To prevent anything catching on fire, the actual GPU # -# frequencies will be throttled to below the maximum, skewing your results. # -################################################################################ - -set -euo pipefail - -POLICY="${1:-performance}" - -readonly ADRENO_GPU_PATH="/sys/class/kgsl/kgsl-3d0" - -# Available frequencies are sorted, either in ascending or descending order. -readonly ADRENO_MIN_FREQ=$(cat "${ADRENO_GPU_PATH}/devfreq/available_frequencies" | tr " " "\n" | sort -u -n | head -1) -readonly ADRENO_MAX_FREQ=$(cat "${ADRENO_GPU_PATH}/devfreq/available_frequencies" | tr " " "\n" | sort -u -n | tail -1) - -# Power levels match available freqencies. -readonly ADRENO_MAX_PWRLEVEL=0 -(( ADRENO_MIN_PWRLEVEL = $(cat "${ADRENO_GPU_PATH}/num_pwrlevels") - 1 )) -readonly ADRENO_MIN_PWRLEVEL - -# Idle timers affect governor change and frequncy reset. -readonly ADRENO_DEFAULT_IDLE_TIMER=80 # ms -readonly ADRENO_1HOUR_IDLE_TIMER=3600000 # ms - -echo "GPU info (before changing frequency scaling policy):" -echo 'model\t\tcur\t\tmin\t\tmax' -echo "---------------------------------------------------------" -paste \ - "${ADRENO_GPU_PATH}/gpu_model" \ - "${ADRENO_GPU_PATH}/devfreq/cur_freq" \ - "${ADRENO_GPU_PATH}/devfreq/min_freq" \ - "${ADRENO_GPU_PATH}/devfreq/max_freq" - -echo "Setting GPU frequency scaling policy to ${POLICY}" - -case "$POLICY" in - performance) - echo 1 > "${ADRENO_GPU_PATH}/force_clk_on" - echo ${ADRENO_1HOUR_IDLE_TIMER} > "${ADRENO_GPU_PATH}/idle_timer" - - # Some devices only expose the msm-adreno-tz governor, so allow the - # following to fail. - echo performance > "${ADRENO_GPU_PATH}/devfreq/governor" || true - - echo ${ADRENO_MAX_FREQ} > "${ADRENO_GPU_PATH}/gpuclk" - echo ${ADRENO_MAX_FREQ} > "${ADRENO_GPU_PATH}/devfreq/max_freq" - echo ${ADRENO_MAX_FREQ} > "${ADRENO_GPU_PATH}/devfreq/min_freq" - - echo ${ADRENO_MAX_PWRLEVEL} > "${ADRENO_GPU_PATH}/max_pwrlevel" - echo ${ADRENO_MAX_PWRLEVEL} > "${ADRENO_GPU_PATH}/min_pwrlevel" - ;; - default) - echo 0 > "${ADRENO_GPU_PATH}/force_clk_on" - echo ${ADRENO_DEFAULT_IDLE_TIMER} > "${ADRENO_GPU_PATH}/idle_timer" - - # msm-adreno-tz is the default governor for Adreno GPUs. - echo msm-adreno-tz > "${ADRENO_GPU_PATH}/devfreq/governor" - - echo ${ADRENO_MAX_FREQ} > "${ADRENO_GPU_PATH}/devfreq/max_freq" - echo ${ADRENO_MIN_FREQ} > "${ADRENO_GPU_PATH}/devfreq/min_freq" - - echo ${ADRENO_MAX_PWRLEVEL} > "${ADRENO_GPU_PATH}/max_pwrlevel" - echo ${ADRENO_MIN_PWRLEVEL} > "${ADRENO_GPU_PATH}/min_pwrlevel" - ;; - *) - echo "Unknown frequency scaling policy: ${POLICY}" - exit 1 - ;; -esac - -echo "GPU info (after changing frequency scaling policy):" -echo 'model\t\tcur\t\tmin\t\tmax' -echo "---------------------------------------------------------" -paste \ - "${ADRENO_GPU_PATH}/gpu_model" \ - "${ADRENO_GPU_PATH}/devfreq/cur_freq" \ - "${ADRENO_GPU_PATH}/devfreq/min_freq" \ - "${ADRENO_GPU_PATH}/devfreq/max_freq" diff --git a/build_tools/benchmarks/set_android_scaling_governor.sh b/build_tools/benchmarks/set_android_scaling_governor.sh deleted file mode 100755 index e262926f4a13..000000000000 --- a/build_tools/benchmarks/set_android_scaling_governor.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh - -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Runs on an android device itself to set the frequency scaling governor for all -# CPUs (default performance). - -################################### WARNING #################################### -# This will overheat the phone if it's not on a cooling plate, resulting in # -# thermal throttling. To prevent anything catching on fire, the actual CPU # -# frequencies will be throttled to below the maximum, skewing your results. # -################################################################################ - -set -euo pipefail - -GOVERNOR="${1:-performance}" - -echo "CPU info (before changing governor):" -echo 'cpu\tgovernor\tcur\tmin\tmax' -echo "------------------------------------------------" -for i in `cat /sys/devices/system/cpu/present | tr '-' ' ' | xargs seq`; do \ - echo "cpu${i}" | paste \ - - \ - "/sys/devices/system/cpu/cpu${i}/cpufreq/scaling_governor" \ - "/sys/devices/system/cpu/cpu${i}/cpufreq/cpuinfo_cur_freq" \ - "/sys/devices/system/cpu/cpu${i}/cpufreq/cpuinfo_min_freq" \ - "/sys/devices/system/cpu/cpu${i}/cpufreq/cpuinfo_max_freq"; \ -done - -echo "Setting CPU frequency governor to ${GOVERNOR}" - -for i in `cat /sys/devices/system/cpu/present | tr '-' ' ' | xargs seq`; do \ - echo "${GOVERNOR}" > \ - "/sys/devices/system/cpu/cpu${i?}/cpufreq/scaling_governor"; \ -done - -echo "CPU info (after changing governor):" -echo 'cpu\tgovernor\tcur\tmin\tmax' -echo "------------------------------------------------" -for i in `cat /sys/devices/system/cpu/present | tr '-' ' ' | xargs seq`; do \ - echo "cpu${i}" | paste \ - - \ - "/sys/devices/system/cpu/cpu${i}/cpufreq/scaling_governor" \ - "/sys/devices/system/cpu/cpu${i}/cpufreq/cpuinfo_cur_freq" \ - "/sys/devices/system/cpu/cpu${i}/cpufreq/cpuinfo_min_freq" \ - "/sys/devices/system/cpu/cpu${i}/cpufreq/cpuinfo_max_freq"; \ -done diff --git a/build_tools/benchmarks/set_pixel6_gpu_scaling_policy.sh b/build_tools/benchmarks/set_pixel6_gpu_scaling_policy.sh deleted file mode 100755 index da8ba367ec0f..000000000000 --- a/build_tools/benchmarks/set_pixel6_gpu_scaling_policy.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh - -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Runs on a Pixel 6 device itself to set the GPU frequency scaling policy. - -################################### WARNING #################################### -# This will overheat the phone if it's not on a cooling plate, resulting in # -# thermal throttling. To prevent anything catching on fire, the actual GPU # -# frequencies will be throttled to below the maximum, skewing your results. # -################################################################################ - -set -euo pipefail - -POLICY="${1:-performance}" - -readonly MALI_GPU_PATH="/sys/devices/platform/1c500000.mali" - -echo "GPU info (before changing frequency scaling policy):" -echo 'policy\t\t\t\t\tcur\tmin\tmax' -echo "--------------------------------------------------------------" -paste \ - "${MALI_GPU_PATH}/power_policy" \ - "${MALI_GPU_PATH}/cur_freq" \ - "${MALI_GPU_PATH}/min_freq" \ - "${MALI_GPU_PATH}/max_freq" - -echo "Setting GPU frequency scaling policy to ${POLICY}" - -case "$POLICY" in - performance) - echo "always_on" > "${MALI_GPU_PATH}/power_policy" - cat "${MALI_GPU_PATH}/max_freq" > "${MALI_GPU_PATH}/scaling_max_freq" - cat "${MALI_GPU_PATH}/max_freq" > "${MALI_GPU_PATH}/scaling_min_freq" - ;; - default) - echo "coarse_demand" > "${MALI_GPU_PATH}/power_policy" - cat "${MALI_GPU_PATH}/max_freq" > "${MALI_GPU_PATH}/scaling_max_freq" - cat "${MALI_GPU_PATH}/min_freq" > "${MALI_GPU_PATH}/scaling_min_freq" - ;; - *) - echo "Unknown frequency scaling policy: ${POLICY}" - exit 1 - ;; -esac - -echo "GPU info (after changing frequency scaling policy):" -echo 'policy\t\t\t\t\tcur\tmin\tmax' -echo "--------------------------------------------------------------" -paste \ - "${MALI_GPU_PATH}/power_policy" \ - "${MALI_GPU_PATH}/cur_freq" \ - "${MALI_GPU_PATH}/min_freq" \ - "${MALI_GPU_PATH}/max_freq" diff --git a/build_tools/benchmarks/upload_benchmarks_to_dashboard.py b/build_tools/benchmarks/upload_benchmarks_to_dashboard.py deleted file mode 100755 index 70ab3ff77330..000000000000 --- a/build_tools/benchmarks/upload_benchmarks_to_dashboard.py +++ /dev/null @@ -1,444 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Upload benchmark results to IREE Benchmark Dashboards. - -This script is meant to be used by GitHub Actions workflows for automation. - -Example usage: - # Export necessary environment variables: - export IREE_DASHBOARD_API_TOKEN=... - # Then run the script: - python3 upload_benchmarks.py /path/to/benchmark/json/file -""" - -import pathlib -import sys - -# Add build_tools python dir to the search path. -sys.path.insert(0, str(pathlib.Path(__file__).parent.with_name("python"))) - -import argparse -import json -import os -import requests - -from typing import Any, Dict, Optional, Union - -from common.common_arguments import expand_and_check_file_paths -from common import benchmark_definition, benchmark_presentation, benchmark_thresholds - -IREE_DASHBOARD_URL = "https://perf.iree.dev" -IREE_GITHUB_COMMIT_URL_PREFIX = "https://github.com/iree-org/iree/commit" -IREE_PROJECT_ID = "IREE" -THIS_DIRECTORY = pathlib.Path(__file__).resolve().parent - -COMMON_DESCRIPTION = """ -
-For the graph, the x axis is the Git commit index, and the y axis is the -measured metrics. The unit for the numbers is shown in the "Unit" dropdown. -
-See -https://iree.dev/developers/performance/benchmark-suites/ - for benchmark philosophy, specification, and definitions. -""" - -# A non-exhaustive list of models and their source URLs. -# For models listed here we can provide a nicer description for them on -# webpage. -IREE_TF_MODEL_SOURCE_URL = { - "MobileBertSquad": "https://github.com/google-research/google-research/tree/master/mobilebert", - "MobileNetV2": "https://www.tensorflow.org/api_docs/python/tf/keras/applications/MobileNetV2", - "MobileNetV3Small": "https://www.tensorflow.org/api_docs/python/tf/keras/applications/MobileNetV3Small", -} - -IREE_TFLITE_MODEL_SOURCE_URL = { - "DeepLabV3": "https://tfhub.dev/tensorflow/lite-model/deeplabv3/1/default/1", - "MobileSSD": "https://www.tensorflow.org/lite/performance/gpu#demo_app_tutorials", - "PoseNet": "https://tfhub.dev/tensorflow/lite-model/posenet/mobilenet/float/075/1/default/1", -} - - -def get_model_description(model_name: str, model_source: str) -> Optional[str]: - """Gets the model description for the given benchmark.""" - url = None - if model_source == "TensorFlow": - url = IREE_TF_MODEL_SOURCE_URL.get(model_name) - elif model_source == "TFLite": - url = IREE_TFLITE_MODEL_SOURCE_URL.get(model_name) - if url is not None: - description = f'{model_name} from {url}.' - return description - return None - - -def get_git_commit_hash(commit: str, verbose: bool = False) -> str: - """Gets the commit hash for the given commit.""" - return benchmark_definition.execute_cmd_and_get_stdout( - ["git", "rev-parse", commit], cwd=THIS_DIRECTORY, verbose=verbose - ) - - -def get_git_total_commit_count(commit: str, verbose: bool = False) -> int: - """Gets the total commit count in history ending with the given commit.""" - count = benchmark_definition.execute_cmd_and_get_stdout( - ["git", "rev-list", "--count", commit], cwd=THIS_DIRECTORY, verbose=verbose - ) - return int(count) - - -def get_git_commit_info(commit: str, verbose: bool = False) -> Dict[str, str]: - """Gets commit information dictionary for the given commit.""" - cmd = ["git", "show", "--format=%H:::%h:::%an:::%ae:::%s", "--no-patch", commit] - info = benchmark_definition.execute_cmd_and_get_stdout( - cmd, cwd=THIS_DIRECTORY, verbose=verbose - ) - segments = info.split(":::") - return { - "hash": segments[0], - "abbrevHash": segments[1], - "authorName": segments[2], - "authorEmail": segments[3], - "subject": segments[4], - } - - -def compose_series_payload( - project_id: str, - series_id: str, - series_unit: str, - series_name: Optional[str] = None, - series_description: Optional[str] = None, - average_range: Union[int, str] = "5%", - average_min_count: int = 3, - better_criterion: str = "smaller", - override: bool = False, -) -> Dict[str, Any]: - """Composes the payload dictionary for a series.""" - payload = { - "projectId": project_id, - "serieId": series_id, - "serieUnit": series_unit, - "serieName": series_name, - "analyse": { - "benchmark": { - "range": average_range, - "required": average_min_count, - "trend": better_criterion, - } - }, - "override": override, - } - if series_description is not None: - payload["description"] = series_description - return payload - - -def compose_build_payload( - project_id: str, - project_github_commit_url: str, - build_id: int, - commit: str, - override: bool = False, -) -> Dict[str, Any]: - """Composes the payload dictionary for a build.""" - commit_info = get_git_commit_info(commit) - commit_info["url"] = f'{project_github_commit_url}/{commit_info["hash"]}' - return { - "projectId": project_id, - "build": { - "buildId": build_id, - "infos": commit_info, - }, - "override": override, - } - - -def compose_sample_payload( - project_id: str, - series_id: str, - build_id: int, - sample_unit: str, - sample_value: int, - override: bool = False, -) -> Dict[str, Any]: - """Composes the payload dictionary for a sample.""" - return { - "projectId": project_id, - "serieId": series_id, - "sampleUnit": sample_unit, - "sample": {"buildId": build_id, "value": sample_value}, - "override": override, - } - - -def get_required_env_var(var: str) -> str: - """Gets the value for a required environment variable.""" - value = os.getenv(var) - if value is None: - raise RuntimeError(f'Missing environment variable "{var}"') - return value - - -def post_to_dashboard( - url: str, payload: Dict[str, Any], dry_run: bool = False, verbose: bool = False -): - data = json.dumps(payload) - - if dry_run or verbose: - print(f"API request payload: {data}") - - if dry_run: - return - - api_token = get_required_env_var("IREE_DASHBOARD_API_TOKEN") - headers = { - "Content-type": "application/json", - "Authorization": f"Bearer {api_token}", - } - - response = requests.post(url, data=data, headers=headers) - code = response.status_code - if code != 200: - raise requests.RequestException( - f"Failed to post to dashboard server with {code} - {response.text}" - ) - - -def add_new_iree_series( - series_id: str, - series_unit: str, - series_name: str, - series_description: Optional[str] = None, - average_range: Optional[Union[str, int]] = None, - override: bool = False, - dry_run: bool = False, - verbose: bool = False, -): - """Posts a new series to the dashboard.""" - if average_range is None: - raise ValueError(f"no matched threshold setting for benchmark: {series_id}") - - payload = compose_series_payload( - IREE_PROJECT_ID, - series_id, - series_unit, - series_name, - series_description, - average_range=average_range, - override=override, - ) - post_to_dashboard( - f"{IREE_DASHBOARD_URL}/apis/v2/addSerie", - payload, - dry_run=dry_run, - verbose=verbose, - ) - - -def add_new_iree_build( - build_id: int, - commit: str, - override: bool = False, - dry_run: bool = False, - verbose: bool = False, -): - """Posts a new build to the dashboard.""" - payload = compose_build_payload( - IREE_PROJECT_ID, IREE_GITHUB_COMMIT_URL_PREFIX, build_id, commit, override - ) - post_to_dashboard( - f"{IREE_DASHBOARD_URL}/apis/addBuild", payload, dry_run=dry_run, verbose=verbose - ) - - -def add_new_sample( - series_id: str, - build_id: int, - sample_unit: str, - sample_value: int, - override: bool = False, - dry_run: bool = False, - verbose: bool = False, -): - """Posts a new sample to the dashboard.""" - payload = compose_sample_payload( - IREE_PROJECT_ID, series_id, build_id, sample_unit, sample_value, override - ) - post_to_dashboard( - f"{IREE_DASHBOARD_URL}/apis/v2/addSample", - payload, - dry_run=dry_run, - verbose=verbose, - ) - - -def parse_arguments(): - """Parses command-line options.""" - - parser = argparse.ArgumentParser() - parser.add_argument( - "--benchmark_files", - metavar="", - default=[], - action="append", - help=( - "Paths to the JSON files containing benchmark results, " "accepts wildcards" - ), - ) - parser.add_argument( - "--compile_stats_files", - metavar="", - default=[], - action="append", - help=( - "Paths to the JSON files containing compilation statistics, " - "accepts wildcards" - ), - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="Print the comment instead of posting to dashboard", - ) - parser.add_argument( - "--verbose", - action="store_true", - help="Print internal information during execution", - ) - args = parser.parse_args() - - return args - - -def main(args): - benchmark_files = expand_and_check_file_paths(args.benchmark_files) - compile_stats_files = expand_and_check_file_paths(args.compile_stats_files) - - if len(benchmark_files) > 0: - committish = benchmark_definition.BenchmarkResults.from_json_str( - benchmark_files[0].read_text() - ).commit - elif len(compile_stats_files) > 0: - committish = benchmark_definition.CompilationResults.from_json_object( - json.loads(compile_stats_files[0].read_text()) - ).commit - else: - raise ValueError("No benchmark/compilation results.") - - # Register a new build for the current commit. - commit_hash = get_git_commit_hash(committish, verbose=args.verbose) - commit_count = get_git_total_commit_count(commit_hash, verbose=args.verbose) - - aggregate_results = benchmark_presentation.aggregate_all_benchmarks( - benchmark_files=benchmark_files, expected_pr_commit=commit_hash - ) - - all_compilation_metrics = benchmark_presentation.collect_all_compilation_metrics( - compile_stats_files=compile_stats_files, expected_pr_commit=commit_hash - ) - - # Allow override to support uploading data for the same build in - # different batches. - add_new_iree_build( - commit_count, - commit_hash, - override=True, - dry_run=args.dry_run, - verbose=args.verbose, - ) - - # Upload benchmark results to the dashboard. - for series_id, benchmark_latency in aggregate_results.items(): - series_name = benchmark_latency.name - benchmark_info = benchmark_latency.benchmark_info - description = get_model_description( - benchmark_info.model_name, benchmark_info.model_source - ) - if description is None: - description = "" - description += COMMON_DESCRIPTION - - threshold = next( - ( - threshold - for threshold in benchmark_thresholds.BENCHMARK_THRESHOLDS - if threshold.regex.match(series_name) - ), - None, - ) - average_range = threshold.get_threshold_str() if threshold is not None else None - - # Override by default to allow updates to the series. - add_new_iree_series( - series_id=series_id, - series_unit="ns", - series_name=benchmark_latency.name, - series_description=description, - average_range=average_range, - override=True, - dry_run=args.dry_run, - verbose=args.verbose, - ) - add_new_sample( - series_id=series_id, - build_id=commit_count, - sample_unit="ns", - sample_value=benchmark_latency.mean_time, - dry_run=args.dry_run, - verbose=args.verbose, - ) - - for target_id, compile_metrics in all_compilation_metrics.items(): - description = get_model_description( - compile_metrics.compilation_info.model_name, - compile_metrics.compilation_info.model_source, - ) - if description is None: - description = "" - description += COMMON_DESCRIPTION - - for mapper in benchmark_presentation.COMPILATION_METRICS_TO_TABLE_MAPPERS: - sample_value, _ = mapper.get_current_and_base_value(compile_metrics) - series_unit = mapper.get_unit() - series_id = mapper.get_series_id(target_id) - series_name = mapper.get_series_name(compile_metrics.name) - - threshold = next( - ( - threshold - for threshold in mapper.get_metric_thresholds() - if threshold.regex.match(series_name) - ), - None, - ) - average_range = ( - threshold.get_threshold_str() if threshold is not None else None - ) - - # Override by default to allow updates to the series. - add_new_iree_series( - series_id=series_id, - series_unit=series_unit, - series_name=series_name, - series_description=description, - average_range=average_range, - override=True, - dry_run=args.dry_run, - verbose=args.verbose, - ) - add_new_sample( - series_id=series_id, - build_id=commit_count, - sample_unit=series_unit, - sample_value=sample_value, - dry_run=args.dry_run, - verbose=args.verbose, - ) - - -if __name__ == "__main__": - main(parse_arguments()) diff --git a/build_tools/cmake/build_android.sh b/build_tools/cmake/build_android.sh index 4318b298600a..f78e125a9afd 100755 --- a/build_tools/cmake/build_android.sh +++ b/build_tools/cmake/build_android.sh @@ -10,9 +10,7 @@ # # The required IREE_HOST_BIN_DIR environment variable indicates the location # of the precompiled IREE binaries. Also requires that ANDROID_NDK variables be -# set. The BUILD_PRESET environment variable indicates how the project should be -# configured: "test", "benchmark", "benchmark-with-tracing", or -# "benchmark-suite-test". Defaults to "test". +# set. # # The desired build directory can be passed as the first argument. Otherwise, it # uses the environment variable IREE_TARGET_BUILD_DIR, defaulting to @@ -30,8 +28,6 @@ set -xeuo pipefail BUILD_DIR="${1:-${IREE_TARGET_BUILD_DIR:-build-android}}" ANDROID_ABI="${IREE_ANDROID_ABI:-arm64-v8a}" IREE_HOST_BIN_DIR="$(realpath ${IREE_HOST_BIN_DIR})" -E2E_TEST_ARTIFACTS_DIR="${E2E_TEST_ARTIFACTS_DIR:-build-e2e-test-artifacts/e2e_test_artifacts}" -BUILD_PRESET="${BUILD_PRESET:-test}" source build_tools/cmake/setup_build.sh source build_tools/cmake/setup_ccache.sh @@ -45,44 +41,13 @@ declare -a args=( -DANDROID_ABI="${ANDROID_ABI}" -DANDROID_PLATFORM=android-29 -DIREE_HOST_BIN_DIR="${IREE_HOST_BIN_DIR}" + -DIREE_ENABLE_ASSERTIONS=ON -DIREE_BUILD_COMPILER=OFF -DIREE_BUILD_TESTS=ON -DIREE_BUILD_ALL_CHECK_TEST_MODULES=OFF -DIREE_BUILD_SAMPLES=OFF ) -case "${BUILD_PRESET}" in - test) - args+=( - -DIREE_ENABLE_ASSERTIONS=ON - ) - ;; - benchmark) - args+=( - -DIREE_ENABLE_ASSERTIONS=OFF - -DIREE_BUILD_TESTS=OFF - ) - ;; - benchmark-with-tracing) - args+=( - -DIREE_ENABLE_ASSERTIONS=OFF - -DIREE_BUILD_TESTS=OFF - -DIREE_ENABLE_RUNTIME_TRACING=ON - ) - ;; - benchmark-suite-test) - E2E_TEST_ARTIFACTS_DIR="$(realpath ${E2E_TEST_ARTIFACTS_DIR})" - args+=( - -DIREE_ENABLE_ASSERTIONS=ON - -DIREE_E2E_TEST_ARTIFACTS_DIR="${E2E_TEST_ARTIFACTS_DIR}" - ) - ;; - *) - echo "Unknown build preset: ${BUILD_PRESET}" - exit 1 - ;; -esac - # Configure towards 64-bit Android 10, then build. "${CMAKE_BIN}" "${args[@]}" diff --git a/build_tools/cmake/build_e2e_test_artifacts.sh b/build_tools/cmake/build_e2e_test_artifacts.sh deleted file mode 100755 index b4f625be2f29..000000000000 --- a/build_tools/cmake/build_e2e_test_artifacts.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash - -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Build e2e test artifacts using a host tools directory. -# -# The required IREE_HOST_BIN_DIR environment variable indicates the location -# of the precompiled IREE binaries. The IREE_BENCHMARK_PRESETS environment -# variable can be set to build required artifacts for the comma-separated -# benchmark presets. By default `iree-benchmark-suites` is built for sanity -# check and e2e model testing. It can be disabled with the environment variable -# `IREE_BUILD_DEFAULT_BENCHMARK_SUITES=0`. -# -# Designed for CI, but can be run locally. The desired build directory can be -# passed as the first argument. Otherwise, it uses the environment variable -# IREE_BUILD_E2E_TEST_ARTIFACTS_DIR, defaulting to "build-e2e-test-artifacts". -# It reuses the build directory if it already exists. Expects to be run from the -# root of the IREE repository. - - -set -xeuo pipefail - -BUILD_DIR="${1:-${IREE_BUILD_E2E_TEST_ARTIFACTS_DIR:-build-e2e-test-artifacts}}" -IREE_HOST_BIN_DIR="$(realpath ${IREE_HOST_BIN_DIR})" -BENCHMARK_PRESETS="${IREE_BENCHMARK_PRESETS:-}" -BUILD_DEFAULT_BENCHMARK_SUITES="${IREE_BUILD_DEFAULT_BENCHMARK_SUITES:-1}" -SHARD_COUNT="${IREE_SHARD_COUNT:-default=1}" - -source build_tools/cmake/setup_build.sh -source build_tools/scripts/setup_tf_python.sh - -declare -a BUILD_TARGETS - -if (( "${BUILD_DEFAULT_BENCHMARK_SUITES}" == 1 )); then - BUILD_TARGETS+=("iree-benchmark-suites") -fi - -# Separate the presets into the execution and compilation benchmark presets to -# export different configs with export_benchmark_config.py. -COMPILATION_PRESETS="" -EXECUTION_PRESETS="" -if [[ -n "${BENCHMARK_PRESETS}" ]]; then - IFS=, read -r -a PRESET_ARRAY <<< "${BENCHMARK_PRESETS}" - for PRESET in "${PRESET_ARRAY[@]}"; do - case "${PRESET}" in - comp-stats) - BUILD_TARGETS+=(iree-benchmark-suites-comp-stats) - COMPILATION_PRESETS="${COMPILATION_PRESETS},${PRESET}" - ;; - comp-stats-large) - BUILD_TARGETS+=(iree-benchmark-suites-comp-stats-large) - COMPILATION_PRESETS="${COMPILATION_PRESETS},${PRESET}" - ;; - *-large) - BUILD_TARGETS+=(iree-benchmark-suites-large) - EXECUTION_PRESETS="${EXECUTION_PRESETS},${PRESET}" - ;; - *) - # Build target of the default preset has been added above. - EXECUTION_PRESETS="${EXECUTION_PRESETS},${PRESET}" - ;; - esac - done - COMPILATION_PRESETS="${COMPILATION_PRESETS#,}" - EXECUTION_PRESETS="${EXECUTION_PRESETS#,}" -fi - -if (( "${#BUILD_TARGETS[@]}" == 0 )); then - echo "No target to build." - exit 1 -fi - -echo "Configuring to build e2e test artifacts" -"${CMAKE_BIN}" -B "${BUILD_DIR}" \ - -G Ninja \ - -DPython3_EXECUTABLE="${IREE_PYTHON3_EXECUTABLE}" \ - -DPYTHON_EXECUTABLE="${IREE_PYTHON3_EXECUTABLE}" \ - -DIREE_HOST_BIN_DIR="${IREE_HOST_BIN_DIR}" \ - -DIREE_BUILD_E2E_TEST_ARTIFACTS=ON \ - -DIREE_BUILD_COMPILER=OFF \ - -DIREE_BUILD_SAMPLES=OFF \ - -DIREE_BUILD_TESTS=OFF - -echo "Building e2e test artifacts" -"${CMAKE_BIN}" \ - --build "${BUILD_DIR}" \ - --target "${BUILD_TARGETS[@]}" \ - -- -k 0 - -E2E_TEST_ARTIFACTS_DIR="${BUILD_DIR}/e2e_test_artifacts" -COMPILATION_CONFIG="${E2E_TEST_ARTIFACTS_DIR}/compilation-benchmark-config.json" -EXECUTION_CONFIG="${E2E_TEST_ARTIFACTS_DIR}/execution-benchmark-config.json" -FLAG_DUMP="${E2E_TEST_ARTIFACTS_DIR}/benchmark-flag-dump.txt" -./build_tools/benchmarks/export_benchmark_config.py \ - compilation \ - --benchmark_presets="${COMPILATION_PRESETS}" \ - --output="${COMPILATION_CONFIG}" -./build_tools/benchmarks/export_benchmark_config.py \ - execution \ - --benchmark_presets="${EXECUTION_PRESETS}" \ - --output="${EXECUTION_CONFIG}" \ - --shard_count="${SHARD_COUNT}" -./build_tools/benchmarks/benchmark_helper.py dump-cmds \ - --execution_benchmark_config="${EXECUTION_CONFIG}" \ - --compilation_benchmark_config="${COMPILATION_CONFIG}" \ - > "${FLAG_DUMP}" diff --git a/build_tools/cmake/build_riscv.sh b/build_tools/cmake/build_riscv.sh index 9feb52b31d10..51e9e746874e 100755 --- a/build_tools/cmake/build_riscv.sh +++ b/build_tools/cmake/build_riscv.sh @@ -9,9 +9,7 @@ # Cross-compile the runtime using CMake targeting RISC-V # # The required IREE_HOST_BIN_DIR environment variable indicates the location -# of the precompiled IREE binaries. The BUILD_PRESET environment variable -# indicates how the project should be configured: "test", "benchmark", -# "benchmark-with-tracing", or "benchmark-suite-test". Defaults to "test". +# of the precompiled IREE binaries. # # The desired build directory can be passed as the first argument. Otherwise, it # uses the environment variable IREE_TARGET_BUILD_DIR, defaulting to @@ -27,8 +25,6 @@ RISCV_PLATFORM="${IREE_TARGET_PLATFORM:-linux}" RISCV_ARCH="${IREE_TARGET_ARCH:-riscv_64}" RISCV_COMPILER_FLAGS="${RISCV_COMPILER_FLAGS:--O3}" IREE_HOST_BIN_DIR="$(realpath ${IREE_HOST_BIN_DIR})" -E2E_TEST_ARTIFACTS_DIR="${E2E_TEST_ARTIFACTS_DIR:-build-e2e-test-artifacts/e2e_test_artifacts}" -BUILD_PRESET="${BUILD_PRESET:-test}" source build_tools/cmake/setup_build.sh @@ -46,6 +42,8 @@ args=( "-DIREE_HOST_BIN_DIR=${IREE_HOST_BIN_DIR}" "-DRISCV_CPU=${RISCV_PLATFORM_ARCH}" "-DRISCV_COMPILER_FLAGS=${RISCV_COMPILER_FLAGS}" + "-DIREE_ENABLE_ASSERTIONS=ON" + "-DIREE_BUILD_SAMPLES=ON" "-DIREE_BUILD_COMPILER=OFF" "-DIREE_BUILD_ALL_CHECK_TEST_MODULES=OFF" # CPU info doesn't work on RISCV @@ -69,59 +67,11 @@ else return -1 fi -case "${BUILD_PRESET}" in - test) - args+=( - -DIREE_ENABLE_ASSERTIONS=ON - -DIREE_BUILD_SAMPLES=ON - ) - ;; - benchmark) - args+=( - -DIREE_ENABLE_ASSERTIONS=OFF - -DIREE_BUILD_SAMPLES=OFF - -DIREE_BUILD_TESTS=OFF - -DRISCV_LINKER_FLAGS_EXE="-static" - ) - ;; - benchmark-with-tracing) - args+=( - -DIREE_ENABLE_ASSERTIONS=OFF - -DIREE_BUILD_SAMPLES=OFF - -DIREE_BUILD_TESTS=OFF - -DIREE_ENABLE_RUNTIME_TRACING=ON - -DRISCV_LINKER_FLAGS_EXE="-static" - ) - ;; - benchmark-suite-test) - E2E_TEST_ARTIFACTS_DIR="$(realpath ${E2E_TEST_ARTIFACTS_DIR})" - args+=( - -DIREE_E2E_TEST_ARTIFACTS_DIR="${E2E_TEST_ARTIFACTS_DIR}" - ) - ;; - *) - echo "Unknown build preset: ${BUILD_PRESET}" - exit 1 - ;; -esac - "${CMAKE_BIN}" "${args[@]}" -if [[ "${BUILD_PRESET}" == "benchmark-suite-test" ]]; then - if [[ "${RISCV_PLATFORM}" == "linux" ]]; then - echo "Building iree-run-module and run-module-test deps for RISC-V" - echo "------------------------------------------------------------" - "${CMAKE_BIN}" --build "${BUILD_DIR}" --target iree-run-module \ - iree-run-module-test-deps -- -k 0 - else - echo "benchmark-suite-test on ${RISCV_PLATFORM_ARCH} not supported yet" - return -1 - fi -else - "${CMAKE_BIN}" --build "${BUILD_DIR}" -- -k 0 - if [[ "${RISCV_PLATFORM}" == "linux" ]]; then - echo "Building test deps for RISC-V" - echo "-----------------------------" - "${CMAKE_BIN}" --build "${BUILD_DIR}" --target iree-test-deps -- -k 0 - fi +"${CMAKE_BIN}" --build "${BUILD_DIR}" -- -k 0 +if [[ "${RISCV_PLATFORM}" == "linux" ]]; then + echo "Building test deps for RISC-V" + echo "-----------------------------" + "${CMAKE_BIN}" --build "${BUILD_DIR}" --target iree-test-deps -- -k 0 fi diff --git a/build_tools/cmake/build_runtime.sh b/build_tools/cmake/build_runtime.sh index c18babc0748a..545f813c07d3 100755 --- a/build_tools/cmake/build_runtime.sh +++ b/build_tools/cmake/build_runtime.sh @@ -16,7 +16,6 @@ set -xeuo pipefail BUILD_DIR="${1:-${IREE_TARGET_BUILD_DIR:-build-runtime}}" -BUILD_PRESET="${BUILD_PRESET:-test}" CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-RelWithDebInfo}" IREE_BUILD_PYTHON_BINDINGS="${IREE_BUILD_PYTHON_BINDINGS:-ON}" @@ -28,8 +27,10 @@ args=( "-G" "Ninja" "-B" "${BUILD_DIR}" - "-DIREE_BUILD_COMPILER=OFF" "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}" + "-DIREE_BUILD_COMPILER=OFF" + "-DIREE_ENABLE_ASSERTIONS=ON" + "-DIREE_BUILD_SAMPLES=ON" # Use `lld` for faster linking. "-DIREE_ENABLE_LLD=ON" @@ -38,49 +39,9 @@ args=( "-DPython3_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}" "-DPYTHON_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}" ) - -case "${BUILD_PRESET}" in - test) - args+=( - -DIREE_ENABLE_ASSERTIONS=ON - -DIREE_BUILD_SAMPLES=ON - ) - ;; - benchmark) - args+=( - -DIREE_ENABLE_ASSERTIONS=OFF - -DIREE_BUILD_SAMPLES=OFF - -DIREE_BUILD_TESTS=OFF - ) - ;; - benchmark-with-tracing) - args+=( - -DIREE_ENABLE_ASSERTIONS=OFF - -DIREE_BUILD_SAMPLES=OFF - -DIREE_BUILD_TESTS=OFF - -DIREE_ENABLE_RUNTIME_TRACING=ON - ) - ;; - *) - echo "Unknown build preset: ${BUILD_PRESET}" - exit 1 - ;; -esac - "${CMAKE_BIN}" "${args[@]}" -case "${BUILD_PRESET}" in - test) - "${CMAKE_BIN}" --build "${BUILD_DIR}" -- -k 0 - ;; - benchmark|benchmark-with-tracing) - "${CMAKE_BIN}" --build "${BUILD_DIR}" --target iree-benchmark-module -- -k 0 - ;; - *) - echo "Unknown build preset: ${BUILD_PRESET}" - exit 1 - ;; -esac +"${CMAKE_BIN}" --build "${BUILD_DIR}" -- -k 0 if (( IREE_USE_CCACHE == 1 )); then ccache --show-stats diff --git a/build_tools/cmake/iree_benchmark_suite.cmake b/build_tools/cmake/iree_benchmark_suite.cmake deleted file mode 100644 index b15afb66ab36..000000000000 --- a/build_tools/cmake/iree_benchmark_suite.cmake +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# iree_import_tflite_model() -# -# Generates MLIR file from a TFLite model file. The generated target will be -# also added to the iree-benchmark-import-models. -# -# Parameters: -# TARGET_NAME: The target name to be created for this module. -# SOURCE: Source TF model direcotry -# IMPORT_FLAGS: Flags to include in the import command. -# OUTPUT_MLIR_FILE: The path to output the generated MLIR file. -function(iree_import_tflite_model) - cmake_parse_arguments( - PARSE_ARGV 0 - _RULE - "" - "TARGET_NAME;SOURCE;OUTPUT_MLIR_FILE" - "IMPORT_FLAGS" - ) - iree_validate_required_arguments( - _RULE - "TARGET_NAME;SOURCE;OUTPUT_MLIR_FILE" - "" - ) - - if(NOT IREE_IMPORT_TFLITE_PATH) - message(SEND_ERROR "Benchmarks of ${_RULE_SOURCE} require" - " that iree-import-tflite be available " - " (either on PATH or via IREE_IMPORT_TFLITE_PATH). " - " Install from a release with " - " `python -m pip install iree-tools-tflite -f https://iree.dev/pip-release-links.html`") - endif() - - if(NOT TARGET "${_RULE_TARGET_NAME}") - cmake_path(GET _RULE_SOURCE FILENAME _MODEL_BASENAME) - add_custom_command( - OUTPUT "${_RULE_OUTPUT_MLIR_FILE}" - COMMAND - "${IREE_IMPORT_TFLITE_PATH}" - "${_RULE_SOURCE}" - "-o=${_RULE_OUTPUT_MLIR_FILE}" - ${_RULE_IMPORT_FLAGS} - DEPENDS - "${_RULE_SOURCE}" - COMMENT "Importing TFLite model ${_MODEL_BASENAME}" - VERBATIM - ) - add_custom_target("${_RULE_TARGET_NAME}" - DEPENDS - "${_RULE_OUTPUT_MLIR_FILE}" - COMMENT - "Importing ${_MODEL_BASENAME} into MLIR" - ) - endif() -endfunction() - -# iree_import_tf_model() -# -# Generates MLIR file from a TensorFlow SavedModel. The generated target will -# be also added to the iree-benchmark-import-models. -# -# Parameters: -# TARGET_NAME: The target name to be created for this module. -# SOURCE: Source TF model direcotry -# IMPORT_FLAGS: Flags to include in the import command. -# OUTPUT_MLIR_FILE: The path to output the generated MLIR file. -function(iree_import_tf_model) - cmake_parse_arguments( - PARSE_ARGV 0 - _RULE - "" - "TARGET_NAME;SOURCE;OUTPUT_MLIR_FILE" - "IMPORT_FLAGS" - ) - iree_validate_required_arguments( - _RULE - "TARGET_NAME;SOURCE;OUTPUT_MLIR_FILE" - "" - ) - - if(NOT IREE_IMPORT_TF_PATH) - message(SEND_ERROR "Benchmarks of ${_RULE_SOURCE} require" - " that iree-import-tf be available " - " (either on PATH or via IREE_IMPORT_TF_PATH). " - " Install from a release with " - " `python -m pip install iree-tools-tf -f https://iree.dev/pip-release-links.html`") - endif() - - if(NOT TARGET "${_RULE_TARGET_NAME}") - cmake_path(GET _RULE_SOURCE FILENAME _MODEL_BASENAME) - add_custom_command( - OUTPUT "${_RULE_OUTPUT_MLIR_FILE}" - COMMAND - "${IREE_IMPORT_TF_PATH}" - "${_RULE_SOURCE}" - "-o=${_RULE_OUTPUT_MLIR_FILE}" - ${_RULE_IMPORT_FLAGS} - DEPENDS - "${_RULE_SOURCE}" - COMMENT "Importing TF model ${_MODEL_BASENAME}" - VERBATIM - ) - add_custom_target("${_RULE_TARGET_NAME}" - DEPENDS - "${_RULE_OUTPUT_MLIR_FILE}" - COMMENT - "Importing ${_MODEL_BASENAME} into MLIR" - ) - endif() -endfunction() diff --git a/build_tools/cmake/iree_fetch_artifact.cmake b/build_tools/cmake/iree_fetch_artifact.cmake deleted file mode 100644 index b282b7db3e6d..000000000000 --- a/build_tools/cmake/iree_fetch_artifact.cmake +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# iree_fetch_artifact() -# -# Download file from URL. NEVER Use this rule to download from untrusted -# sources, it doesn't unpack the file safely. -# -# Parameters: -# NAME: Name of target (see Note). -# SOURCE_URL: Source URL to donwload the file. -# OUTPUT: Path to the output file or directory to unpack. -# UNPACK: When added, it will try to unpack the archive if supported. -# -# Note: -# By default, it will create a target named ${_PACKAGE_NAME}_${_RULE_NAME}. -function(iree_fetch_artifact) - cmake_parse_arguments( - _RULE - "UNPACK" - "NAME;SOURCE_URL;OUTPUT" - "" - ${ARGN} - ) - - set(_ARGS "${IREE_ROOT_DIR}/build_tools/scripts/download_file.py") - list(APPEND _ARGS "${_RULE_SOURCE_URL}") - list(APPEND _ARGS "-o") - list(APPEND _ARGS "${_RULE_OUTPUT}") - - if(_RULE_UNPACK) - list(APPEND _ARGS "--unpack") - endif() - - # TODO: CMake built-in file command can replace the python script. But python - # script also provides streaming unpack (doesn't use double space when - # unpacking). Need to evaluate if we want to replace. - add_custom_command( - OUTPUT "${_RULE_OUTPUT}" - COMMAND - "${Python3_EXECUTABLE}" - ${_ARGS} - DEPENDS - "${IREE_ROOT_DIR}/build_tools/scripts/download_file.py" - COMMENT "Downloading ${_RULE_SOURCE_URL}" - ) - - iree_package_name(_PACKAGE_NAME) - add_custom_target("${_PACKAGE_NAME}_${_RULE_NAME}" - DEPENDS - "${_RULE_OUTPUT}" - ) -endfunction() diff --git a/build_tools/cmake/iree_run_module_test.cmake b/build_tools/cmake/iree_run_module_test.cmake deleted file mode 100644 index b16e2db1e080..000000000000 --- a/build_tools/cmake/iree_run_module_test.cmake +++ /dev/null @@ -1,329 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Utility function to return the platform name in crosscompile. -# List of CMAKE_SYSTEM_NAME values: -# https://gitlab.kitware.com/cmake/cmake/-/issues/21489#note_1077167 -# Examples: arm_64-Android arm_32-Linux x86_64-Windows arm_64-iOS riscv_64-Linux -function(iree_get_platform PLATFORM) - set(${PLATFORM} "${IREE_ARCH}-${CMAKE_SYSTEM_NAME}" PARENT_SCOPE) -endfunction() - -# iree_run_module_test() -# -# Creates a test using iree-run-module to run an IREE module (vmfb). -# -# The function is unimplemented in Bazel because it is not used there. -# -# Parameters: -# NAME: Name of the target -# MODULE_SRC: IREE module (vmfb) file. -# DRIVER: Driver to run the module with. -# RUNNER_ARGS: additional args to pass to iree-run-module. The driver -# and input file are passed automatically. -# EXPECTED_OUTPUT: A string representing the expected output from executing -# the module in the format accepted by `iree-run-module` or a file -# containing the same. Can also be an HTTPS URL to download large npy. -# LABELS: Additional labels to apply to the test. The package path, -# "test-type=run-module-test", and "driver=${DRIVER}" labels are added -# automatically. -# XFAIL_PLATFORMS: List of platforms (see iree_get_platform) for which the -# test is expected to fail. The test pass/fail status is inverted. -# UNSUPPORTED_PLATFORMS: List of platforms (see iree_get_platform) for which -# the test is skipped entirely. -# DEPS: (Optional) List of targets to build the test artifacts. -# TIMEOUT: (optional) Test timeout. -# -# Examples: -# -# iree_run_module_test( -# NAME -# iree_run_module_correctness_test -# MODULE_SRC -# "iree_run_module_bytecode_module_llvm_cpu.vmfb" -# DRIVER -# "local-sync" -# RUNNER_ARGS -# "--function=abs" -# "--input=f32=-10" -# EXPECTED_OUTPUT -# "f32=10" -# ) -# -# iree_run_module_test( -# NAME -# mobilenet_v1_fp32_correctness_test -# MODULE_SRC -# "mobilenet_v1_fp32.vmfb" -# DRIVER -# "local-sync" -# RUNNER_ARGS -# "--function=main" -# "--input=1x224x224x3xf32=0" -# EXPECTED_OUTPUT -# "mobilenet_v1_fp32_expected_output.txt" -# UNSUPPORTED_PLATFORMS -# "arm_64-Android" -# "riscv_32-Linux" -# ) - -function(iree_run_module_test) - if(NOT IREE_BUILD_TESTS) - return() - endif() - - cmake_parse_arguments( - _RULE - "" - "NAME;MODULE_SRC;DRIVER;EXPECTED_OUTPUT;TIMEOUT" - "RUNNER_ARGS;LABELS;XFAIL_PLATFORMS;UNSUPPORTED_PLATFORMS;DEPS" - ${ARGN} - ) - - iree_get_platform(_PLATFORM) - if(_PLATFORM IN_LIST _RULE_UNSUPPORTED_PLATFORMS) - return() - endif() - - if(NOT DEFINED _RULE_DRIVER) - message(SEND_ERROR "The DRIVER argument is required.") - endif() - - iree_package_path(_PACKAGE_PATH) - - # All the file paths referred in the _RUNNER_FILE_ARGS are absolute paths and - # the portability is handled by `iree_native_test`. - list(APPEND _RUNNER_FILE_ARGS "--module={{${_RULE_MODULE_SRC}}}") - - # A target specifically for the test. - iree_package_name(_PACKAGE_NAME) - set(_TARGET_NAME "${_PACKAGE_NAME}_${_RULE_NAME}") - add_custom_target("${_TARGET_NAME}" ALL) - - if(_RULE_EXPECTED_OUTPUT) - # Get the extension (.xyz) from either a file path or an URL. - # This may be a file or a literal output. In the latter case, the - # extension variable will be empty. - string(REGEX MATCH "\\.[A-Za-z0-9_]+$" _OUTPUT_FILE_TYPE "${_RULE_EXPECTED_OUTPUT}") - if(NOT _OUTPUT_FILE_TYPE) # The expected output is listed in the field. - list(APPEND _RULE_RUNNER_ARGS "--expected_output=\"${_RULE_EXPECTED_OUTPUT}\"") - elseif(_OUTPUT_FILE_TYPE STREQUAL ".txt") - file(REAL_PATH "${_RULE_EXPECTED_OUTPUT}" _OUTPUT_FILE_ABS_PATH) - # Process the text input to remove the line breaks. - file(READ "${_OUTPUT_FILE_ABS_PATH}" _EXPECTED_OUTPUT) - string(REPLACE "\n" " " _EXPECTED_OUTPUT_STR "${_EXPECTED_OUTPUT}") - set(_EXPECTED_OUTPUT_STR "--expected_output=\"${_EXPECTED_OUTPUT_STR}\"") - list(APPEND _RULE_RUNNER_ARGS ${_EXPECTED_OUTPUT_STR}) - elseif(_OUTPUT_FILE_TYPE STREQUAL ".npy") - # Large npy files are not stored in the codebase. Need to download them - # from the URL first and store them in the following possible paths. - if(_RULE_EXPECTED_OUTPUT MATCHES "^https://.+/([^/]+)$") - set(_EXPECTED_OUTPUT_URL "${_RULE_EXPECTED_OUTPUT}") - set(_RULE_EXPECTED_OUTPUT "${CMAKE_MATCH_1}") - - cmake_path(ABSOLUTE_PATH - _RULE_EXPECTED_OUTPUT - BASE_DIRECTORY "${IREE_E2E_TEST_ARTIFACTS_DIR}" - NORMALIZE - OUTPUT_VARIABLE _OUTPUT_FILE_ABS_PATH) - - set(_FETCH_NAME "model-expected-output-${_RULE_NAME}") - iree_fetch_artifact( - NAME "${_FETCH_NAME}" - SOURCE_URL "${_EXPECTED_OUTPUT_URL}" - OUTPUT "${_OUTPUT_FILE_ABS_PATH}" - ) - add_dependencies(${_TARGET_NAME} - "${_PACKAGE_NAME}_${_FETCH_NAME}" - ) - else() - find_file(_OUTPUT_FILE_ABS_PATH - NAME - "${_RULE_EXPECTED_OUTPUT}" - PATHS - "${CMAKE_CURRENT_SOURCE_DIR}" - "${CMAKE_CURRENT_BINARY_DIR}" - "${IREE_E2E_TEST_ARTIFACTS_DIR}" - NO_CACHE - NO_DEFAULT_PATH - ) - if(NOT _OUTPUT_FILE_ABS_PATH) - message(SEND_ERROR "${_RULE_EXPECTED_OUTPUT} is not found in\n\ - ${CMAKE_CURRENT_SOURCE_DIR}\n\ - ${CMAKE_CURRENT_BINARY_DIR}\n\ - ${IREE_E2E_TEST_ARTIFACTS_DIR}") - endif() - endif() - list(APPEND _RUNNER_FILE_ARGS - "--expected_output=@{{${_OUTPUT_FILE_ABS_PATH}}}") - else() - message(SEND_ERROR "Unsupported expected output file type: ${_RULE_EXPECTED_OUTPUT}") - endif(NOT _OUTPUT_FILE_TYPE) - endif(_RULE_EXPECTED_OUTPUT) - - # Dump the flags into a flag file to avoid CMake's naive handling of spaces - # in expected output. `--module` is coded separatedly to make it portable. - if(_RULE_RUNNER_ARGS) - # Write each argument in a new line. - string(REPLACE ";" "\n" _OUTPUT_FLAGS "${_RULE_RUNNER_ARGS}") - file(CONFIGURE - OUTPUT - "${_RULE_NAME}_flagfile" - CONTENT - "${_OUTPUT_FLAGS}" - ) - list(APPEND _RUNNER_FILE_ARGS - "--flagfile={{${CMAKE_CURRENT_BINARY_DIR}/${_RULE_NAME}_flagfile}}") - endif() - - # Set expect failure cases. - set(_TEST_XFAIL FALSE) - if(_PLATFORM IN_LIST _RULE_XFAIL_PLATFORMS OR - _RULE_XFAIL_PLATFORMS STREQUAL "all") - set(_TEST_XFAIL TRUE) - endif() - - set(_RUNNER_TARGET "iree-run-module") - - iree_native_test( - NAME - "${_RULE_NAME}" - DRIVER - "${_RULE_DRIVER}" - SRC - "${_RUNNER_TARGET}" - ARGS - ${_RUNNER_FILE_ARGS} - WILL_FAIL - ${_TEST_XFAIL} - LABELS - "test-type=run-module-test" - ${_RULE_LABELS} - TIMEOUT - ${_RULE_TIMEOUT} - ) - - if(_RULE_DEPS) - add_dependencies(${_TARGET_NAME} - ${_RULE_DEPS} - ) - endif() - - add_dependencies(iree-test-deps "${_TARGET_NAME}") - add_dependencies(iree-run-module-test-deps "${_TARGET_NAME}") -endfunction() - -# iree_benchmark_suite_module_test() -# -# The function is unimplemented in Bazel because it is not used there. -# -# Creates a test using iree-run-module to run a benchmark suite module. -# -# Parameters: -# NAME: Name of the target -# DRIVER: Driver to run the module with. -# MODULES: Platform-module path list (relative to IREE_E2E_TEST_ARTIFACTS_DIR) -# for the supported platforms. Each item is in the format: -# "${PLATFORM}=${MODULE_PATH}" -# RUNNER_ARGS: additional args to pass to iree-run-module. The driver -# and input file are passed automatically. -# EXPECTED_OUTPUT: A file of expected output to compare with the output from -# iree-run-module -# LABELS: Additional labels to apply to the test. The package path and -# "driver=${DRIVER}" are added automatically. -# XFAIL_PLATFORMS: List of platforms (see iree_get_platform) for which the -# test is expected to fail. The test pass/fail status is inverted. -# UNSUPPORTED_PLATFORMS: List of platforms (see iree_get_platform) for which -# the test is skipped entirely. -# TIMEOUT: (optional) Test timeout. -# -# Example: -# -# iree_benchmark_suite_module_test( -# NAME -# mobilenet_v1_fp32_correctness_test -# DRIVER -# "local-sync" -# MODULES -# "riscv32-Linux=iree_module_EfficientNet_int8_riscv32/module.vmfb" -# "x86_64=iree_module_EfficientNet_int8_x86_64/module.vmfb" -# RUNNER_ARGS -# "--function=main" -# "--input=1x224x224x3xf32=0" -# EXPECTED_OUTPUT -# "mobilenet_v1_fp32_expected_output.txt" -# ) -function(iree_benchmark_suite_module_test) - if(NOT IREE_BUILD_TESTS) - return() - endif() - - cmake_parse_arguments( - _RULE - "" - "NAME;DRIVER;EXPECTED_OUTPUT;TIMEOUT" - "MODULES;RUNNER_ARGS;LABELS;XFAIL_PLATFORMS;UNSUPPORTED_PLATFORMS" - ${ARGN} - ) - - # Benchmark artifacts needs to be stored at the location of - # `IREE_E2E_TEST_ARTIFACTS_DIR` or the test target is bypassed. - if("${IREE_E2E_TEST_ARTIFACTS_DIR}" STREQUAL "") - return() - endif() - - iree_get_platform(_PLATFORM) - - foreach(_PLATFORM_MODULE_PAIR IN LISTS _RULE_MODULES) - string(REGEX MATCH "^${_PLATFORM}=(.+)$" _MATCHED_PAIR "${_PLATFORM_MODULE_PAIR}") - if (_MATCHED_PAIR) - set(_MODULE_PATH "${CMAKE_MATCH_1}") - break() - endif() - endforeach() - - # Platform is not in the supported module list, skip the test. - if (NOT _MODULE_PATH) - return() - endif() - - # Build module locally if IREE_BUILD_E2E_TEST_ARTIFACTS is set. - if (IREE_BUILD_E2E_TEST_ARTIFACTS) - # The module path follows the format: - # iree_module_abc/module.vmfb - # Ane the corresponding build target from e2e test artifacts is: - # iree-module-abc - cmake_path(GET _MODULE_PATH PARENT_PATH _MODULE_DIR) - string(REGEX REPLACE "^iree_module_" "iree-module-" _DEP_NAME "${_MODULE_DIR}") - # Append the prefix of package name. - set(_MODULE_DEP_TARGET "iree_tests_e2e_test_artifacts_${_DEP_NAME}") - else() - set(_MODULE_DEP_TARGET "") - endif() - - iree_run_module_test( - NAME - "${_RULE_NAME}" - MODULE_SRC - "${IREE_E2E_TEST_ARTIFACTS_DIR}/${_MODULE_PATH}" - DRIVER - "${_RULE_DRIVER}" - EXPECTED_OUTPUT - "${_RULE_EXPECTED_OUTPUT}" - RUNNER_ARGS - ${_RULE_RUNNER_ARGS} - XFAIL_PLATFORMS - ${_RULE_XFAIL_PLATFORMS} - UNSUPPORTED_PLATFORMS - ${_RULE_UNSUPPORTED_PLATFORMS} - LABELS - ${_RULE_LABELS} - TIMEOUT - ${_RULE_TIMEOUT} - DEPS - "${_MODULE_DEP_TARGET}" - ) -endfunction() diff --git a/build_tools/cmake/test_benchmark_suites_on_linux.sh b/build_tools/cmake/test_benchmark_suites_on_linux.sh deleted file mode 100755 index 913ecda34090..000000000000 --- a/build_tools/cmake/test_benchmark_suites_on_linux.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Build tests based on benchmark suites for IREE. -# -# The desired build directory can be passed as the first argument. Otherwise, it -# uses the environment variable IREE_TARGET_BUILD_DIR, defaulting to -# "build-benchmark-suites-test". Designed for CI, but can be run manually. It -# reuses the build directory if it already exists. Expects to be run from the -# root of the IREE repository. -set -xeuo pipefail - -BUILD_DIR="${1:-${IREE_TARGET_BUILD_DIR:-build-benchmark-suites-test}}" -E2E_TEST_ARTIFACTS_DIR="$(realpath ${E2E_TEST_ARTIFACTS_DIR:-build-e2e-test-artifacts/e2e_test_artifacts})" -IREE_HOST_BIN_DIR="$(realpath ${IREE_HOST_BIN_DIR})" - -source build_tools/cmake/setup_build.sh - -# Configure, build, test -declare -a CMAKE_ARGS=( - "-G" "Ninja" - "-B" "${BUILD_DIR}" - "-DPython3_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}" - "-DPYTHON_EXECUTABLE=${IREE_PYTHON3_EXECUTABLE}" - "-DCMAKE_BUILD_TYPE=RelWithDebInfo" - "-DIREE_BUILD_COMPILER=OFF" - "-DIREE_HOST_BIN_DIR=${IREE_HOST_BIN_DIR}" - "-DIREE_BUILD_TESTS=ON" - "-DIREE_BUILD_SAMPLES=OFF" - "-DIREE_E2E_TEST_ARTIFACTS_DIR=${E2E_TEST_ARTIFACTS_DIR}" -) - -echo "Configuring to build tests for benchmark suites" -"${CMAKE_BIN}" "${CMAKE_ARGS[@]}" - -echo "Building tests artifacts" -"${CMAKE_BIN}" --build "${BUILD_DIR}" \ - --target iree-run-module iree-run-module-test-deps -- -k 0 - -ctest_args=( - "--timeout 900" - "--output-on-failure" - "--no-tests=error" -) - -declare -a label_args=( - "^test-type=run-module-test$" -) -label_include_regex="($(IFS="|" ; echo "${label_args[*]}"))" - -echo "******** Running run-module CTest ********" -ctest --test-dir ${BUILD_DIR} ${ctest_args[@]} \ - --label-regex ${label_include_regex} diff --git a/build_tools/cmake/test_riscv.sh b/build_tools/cmake/test_riscv.sh index da20b8719c15..b78f5f730a73 100755 --- a/build_tools/cmake/test_riscv.sh +++ b/build_tools/cmake/test_riscv.sh @@ -18,7 +18,6 @@ set -xeuo pipefail BUILD_DIR="${1:-${IREE_TARGET_BUILD_DIR:-build-riscv}}" RISCV_PLATFORM="${IREE_TARGET_PLATFORM:-linux}" RISCV_ARCH="${IREE_TARGET_ARCH:-riscv_64}" -BUILD_PRESET="${BUILD_PRESET:-test}" # Environment variable used by the emulator. export RISCV_TOOLCHAIN_ROOT="${RISCV_RV64_LINUX_TOOLCHAIN_ROOT}" @@ -48,17 +47,6 @@ declare -a test_exclude_args=( "regression_llvm-cpu_lowering_config" ) -if [[ "${BUILD_PRESET}" == "benchmark-suite-test" ]]; then - declare -a label_args=( - "^test-type=run-module-test$" - ) - label_include_regex="($(IFS="|" ; echo "${label_args[*]}"))" - echo "******** Running run-module CTest ********" - ctest --test-dir ${BUILD_DIR} ${ctest_args[@]} \ - --label-regex ${label_include_regex} - exit 0 -fi - # Test runtime unit tests runtime_label_exclude_regex="($(IFS="|" ; echo "${label_exclude_args[*]}"))" runtime_ctest_args=( diff --git a/build_tools/github_actions/cancel_workflow_and_wait.sh b/build_tools/github_actions/cancel_workflow_and_wait.sh deleted file mode 100755 index a87145a4985c..000000000000 --- a/build_tools/github_actions/cancel_workflow_and_wait.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -# Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Perform a best-effort attempt to cancel a running workflow (started before -# this script is run) and wait for the cancellation to complete. -# -# Due to limitations in GitHub's APIs, there are race conditions cases that -# this script can't be sure if the run is cancelled. It will detect and fail -# fast in such cases. - -set -euo pipefail - -MAX_WAIT_IN_SECS="${IREE_MAX_WAIT_IN_SECS:-180}" -WORKFLOW_RUN_URL="${1:-${IREE_WORKFLOW_RUN_URL}}" - -RUN_ATTEMPT_BEFORE="$(gh api ${WORKFLOW_RUN_URL} \ - | jq --raw-output '.run_attempt')" - -# Unfortunately Github API doesn't allow us to specify which run attempt to -# cancel. So there is a chance that we cancel a newer run attempt. -CANCEL_RESPONSE="$(gh api ${WORKFLOW_RUN_URL}/cancel --method POST \ - | jq --raw-output '.message' \ -)" || true - -if [[ "${CANCEL_RESPONSE}" == "Cannot cancel a workflow run that is completed." ]]; then - exit 0 -elif [[ "${CANCEL_RESPONSE}" != "null" ]]; then - echo "Failed to cancel the workflow: ${CANCEL_RESPONSE}" - exit 1 -fi - -RUN_ATTEMPT_AFTER="$(gh api ${WORKFLOW_RUN_URL} \ - | jq --raw-output '.run_attempt')" -# If the run attempt is changed after the cancel request, We don't know whether -# ${RUN_ATTEMPT_BEFORE} or ${RUN_ATTEMPT_AFTER} is cancelled. Instead of waiting -# on the wrong run attempt potentially and timeout, fail fast in this case. -if [[ "${RUN_ATTEMPT_BEFORE}" != "${RUN_ATTEMPT_AFTER}" ]]; then - echo "Uncertain on which run attempt we cancelled." - exit 1 -fi - -START_TIME="$(date +%s)" -while true; do - # Wait on the specific run attempt so we won't wait on a new attempt that was - # not cancelled. - WORKFLOW_STATUS="$(gh api ${WORKFLOW_RUN_URL}/attempts/${RUN_ATTEMPT_BEFORE} \ - | jq --raw-output '.status')" - echo "Waiting for the workflow to stop: ${WORKFLOW_STATUS}." - if [[ "${WORKFLOW_STATUS}" == "completed" ]]; then - break - fi - if (( "$(date +%s)" - "${START_TIME}" > "${MAX_WAIT_IN_SECS}" )); then - echo "Timeout on waiting for the workflow." - exit 1 - fi - sleep 10 -done diff --git a/build_tools/github_actions/configure_ci.py b/build_tools/github_actions/configure_ci.py index 90dc1c4a46a8..b32dbdcc8522 100755 --- a/build_tools/github_actions/configure_ci.py +++ b/build_tools/github_actions/configure_ci.py @@ -52,8 +52,6 @@ # Add build_tools python dir to the search path. sys.path.insert(0, str(pathlib.Path(__file__).parent.with_name("python"))) -from benchmark_suites.iree import benchmark_presets - # We don't get StrEnum till Python 3.11 @enum.unique @@ -65,9 +63,6 @@ class Trailer(str, enum.Enum): EXTRA_JOBS = "ci-extra" EXACTLY_JOBS = "ci-exactly" RUNNER_ENV = "runner-env" - BENCHMARK_EXTRA = "benchmark-extra" - # Trailer to prevent benchmarks from always running on LLVM integration PRs. - SKIP_LLVM_INTEGRATE_BENCHMARK = "skip-llvm-integrate-benchmark" # Before Python 3.12, it the native __contains__ doesn't work for checking # member values like this and it's not possible to easily override this. @@ -169,32 +164,6 @@ def contains(cls, val): # ("test_amd_w7900", AMDGPU_PATHS), ] -# Default presets enabled in CI. -DEFAULT_BENCHMARK_PRESET_GROUP = [ - preset - for preset in benchmark_presets.DEFAULT_PRESETS - # RISC-V benchmarks haven't been supported in CI workflow. - if preset not in [benchmark_presets.RISCV] - # Disabled while iree-pixel-6-pro-1 lab Android presubmit device is offline. - # The iree-pixel-6-pro-2 postsubmit device may still be online, so this - # could be adapted to keep the Android benchmarks enabled on postsubmit. - and preset - not in [ - benchmark_presets.ANDROID_CPU, - benchmark_presets.ANDROID_CPU_DT_ONLY, - benchmark_presets.ANDROID_GPU, - ] - # CUDA benchmarks on CI depend on low availability a100 runners. - and preset not in [benchmark_presets.CUDA] -] + ["comp-stats"] -DEFAULT_BENCHMARK_PRESET = "default" -LARGE_BENCHMARK_PRESET_GROUP = benchmark_presets.LARGE_PRESETS -# All available benchmark preset options including experimental presets. -BENCHMARK_PRESET_OPTIONS = ( - benchmark_presets.ALL_EXECUTION_PRESETS + benchmark_presets.ALL_COMPILATION_PRESETS -) -BENCHMARK_LABEL_PREFIX = "benchmarks" - PR_DESCRIPTION_TEMPLATE = string.Template("${title}\n\n${body}") # Patterns to detect "LLVM integration" PRs, i.e. changes that update the @@ -534,76 +503,6 @@ def get_enabled_jobs( return (enabled_jobs | extra_jobs) - skip_jobs -def get_benchmark_presets( - trailers: Mapping[str, str], - labels: Sequence[str], - is_pr: bool, - is_llvm_integrate_pr: bool, -) -> str: - """Parses and validates the benchmark presets from trailers. - - Args: - trailers: trailers from PR description. - labels: list of PR labels. - is_pr: is pull request event. - is_llvm_integrate_pr: is LLVM integration PR. - - Returns: - A comma separated preset string, which later will be parsed by - build_tools/benchmarks/export_benchmark_config.py. - """ - - skip_llvm_integrate_benchmark = Trailer.SKIP_LLVM_INTEGRATE_BENCHMARK in trailers - if skip_llvm_integrate_benchmark: - print( - f"Skipping default benchmarking on LLVM integration because PR " - f"description has '{Trailer.SKIP_LLVM_INTEGRATE_BENCHMARK}'" - f" trailer." - ) - - if not is_pr: - preset_options = {DEFAULT_BENCHMARK_PRESET} - print(f"Using benchmark presets '{preset_options}' for non-PR run") - elif is_llvm_integrate_pr and not skip_llvm_integrate_benchmark: - # Run all benchmark presets for LLVM integration PRs. - preset_options = {DEFAULT_BENCHMARK_PRESET} - print(f"Using benchmark preset '{preset_options}' for LLVM integration PR") - else: - preset_options = set( - label.split(":", maxsplit=1)[1] - for label in labels - if label.startswith(BENCHMARK_LABEL_PREFIX + ":") - ) - trailer = trailers.get(Trailer.BENCHMARK_EXTRA) - if trailer is not None: - preset_options = preset_options.union( - option.strip() for option in trailer.split(",") - ) - print( - f"Using benchmark preset '{preset_options}' from trailers" - f" and labels" - ) - - if DEFAULT_BENCHMARK_PRESET in preset_options: - preset_options.remove(DEFAULT_BENCHMARK_PRESET) - preset_options.update(DEFAULT_BENCHMARK_PRESET_GROUP) - - if preset_options.intersection(DEFAULT_BENCHMARK_PRESET_GROUP): - # The is a sugar to run the compilation benchmarks when any default - # benchmark preset is present. - preset_options.add("comp-stats") - - preset_options = sorted(preset_options) - for preset_option in preset_options: - if preset_option not in BENCHMARK_PRESET_OPTIONS: - raise ValueError( - f"Unknown benchmark preset option: '{preset_option}'.\n" - f"Available options: '{BENCHMARK_PRESET_OPTIONS}'." - ) - - return ",".join(preset_options) - - def main(): is_pr = os.environ["GITHUB_EVENT_NAME"] == "pull_request" trailers, labels = get_trailers_and_labels(is_pr) @@ -618,9 +517,6 @@ def main(): base_ref = os.environ["BASE_REF"] try: - benchmark_presets = get_benchmark_presets( - trailers, labels, is_pr, is_llvm_integrate_pr - ) all_jobs = parse_jobs_from_workflow_file(workflow_file) enabled_jobs = get_enabled_jobs( trailers, @@ -638,7 +534,6 @@ def main(): "runner-env": get_runner_env(trailers), "runner-group": "presubmit" if is_pr else "postsubmit", "write-caches": "0" if is_pr else "1", - "benchmark-presets": benchmark_presets, } set_output(output) diff --git a/build_tools/github_actions/configure_ci_test.py b/build_tools/github_actions/configure_ci_test.py index b9a6c0738566..c089be67daaa 100644 --- a/build_tools/github_actions/configure_ci_test.py +++ b/build_tools/github_actions/configure_ci_test.py @@ -11,100 +11,8 @@ import configure_ci -SORTED_DEFAULT_BENCHMARK_PRESETS_STR = ",".join( - sorted(configure_ci.DEFAULT_BENCHMARK_PRESET_GROUP) -) - class ConfigureCITest(unittest.TestCase): - def test_get_benchmark_presets_no_preset(self): - presets_str = configure_ci.get_benchmark_presets( - trailers={}, - labels=["unrelated-labels"], - is_pr=True, - is_llvm_integrate_pr=False, - ) - - self.assertEqual(presets_str, "") - - def test_get_benchmark_presets_from_pr_labels(self): - presets_str = configure_ci.get_benchmark_presets( - trailers={}, - labels=["benchmarks:x86_64", "benchmarks:cuda"], - is_pr=True, - is_llvm_integrate_pr=False, - ) - - self.assertEqual(presets_str, "comp-stats,cuda,x86_64") - - def test_get_benchmark_presets_from_trailers_and_labels(self): - presets_str = configure_ci.get_benchmark_presets( - trailers={"benchmark-extra": "android-cpu,cuda-large,x86_64-large"}, - labels=["benchmarks:vulkan-nvidia"], - is_pr=True, - is_llvm_integrate_pr=False, - ) - - self.assertEqual( - presets_str, "android-cpu,comp-stats,cuda-large,vulkan-nvidia,x86_64-large" - ) - - def test_get_benchmark_presets_from_default_group(self): - presets_str = configure_ci.get_benchmark_presets( - trailers={"benchmark-extra": "default"}, - labels=[], - is_pr=True, - is_llvm_integrate_pr=False, - ) - - self.assertEqual(presets_str, SORTED_DEFAULT_BENCHMARK_PRESETS_STR) - # Sanity check to ensure no `*-large` preset in the default group. - self.assertNotIn("-large", presets_str) - - def test_get_benchmark_presets_for_non_pr(self): - presets_str = configure_ci.get_benchmark_presets( - trailers={}, labels=[], is_pr=False, is_llvm_integrate_pr=False - ) - - self.assertEqual(presets_str, SORTED_DEFAULT_BENCHMARK_PRESETS_STR) - - def test_get_benchmark_presets_for_llvm_integrate_pr(self): - presets_str = configure_ci.get_benchmark_presets( - trailers={}, labels=[], is_pr=True, is_llvm_integrate_pr=True - ) - - self.assertEqual(presets_str, SORTED_DEFAULT_BENCHMARK_PRESETS_STR) - - # Sample PR description: - # ``` - # PR Title - # - # PR body... - # - # skip-llvm-integrate-benchmark: some good reasons - # ``` - # Result: No benchmark is automatically enabled on the LLVM integrate PR. - def test_get_benchmark_presets_skip_llvm_integrate_benchmark(self): - presets_str = configure_ci.get_benchmark_presets( - trailers={"skip-llvm-integrate-benchmark": "some good reasons"}, - labels=[], - is_pr=True, - is_llvm_integrate_pr=True, - ) - - self.assertEqual(presets_str, "") - - def test_get_benchmark_presets_unknown_preset(self): - self.assertRaises( - ValueError, - lambda: configure_ci.get_benchmark_presets( - trailers={"benchmark-extra": "unknown"}, - labels=[], - is_pr=True, - is_llvm_integrate_pr=False, - ), - ) - def test_parse_jobs_trailer(self): trailers = {"key": "job1,job2"} key = "key" diff --git a/build_tools/python/CMakeLists.txt b/build_tools/python/CMakeLists.txt deleted file mode 100644 index 1e70c9dcf47f..000000000000 --- a/build_tools/python/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -iree_add_all_subdirs() diff --git a/build_tools/python/__init__.py b/build_tools/python/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/build_tools/python/benchmark_suites/CMakeLists.txt b/build_tools/python/benchmark_suites/CMakeLists.txt deleted file mode 100644 index 1e70c9dcf47f..000000000000 --- a/build_tools/python/benchmark_suites/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -iree_add_all_subdirs() diff --git a/build_tools/python/benchmark_suites/__init__.py b/build_tools/python/benchmark_suites/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/build_tools/python/benchmark_suites/iree/CMakeLists.txt b/build_tools/python/benchmark_suites/iree/CMakeLists.txt deleted file mode 100644 index 1930c7188ce8..000000000000 --- a/build_tools/python/benchmark_suites/iree/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -iree_build_tools_py_test( - NAME - benchmark_collections_test - SRC - "benchmark_collections_test.py" -) diff --git a/build_tools/python/benchmark_suites/iree/README.md b/build_tools/python/benchmark_suites/iree/README.md deleted file mode 100644 index 3754b1a1bbad..000000000000 --- a/build_tools/python/benchmark_suites/iree/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# IREE Benchmark Suites Configurations - -This directory contains the Python scripts that define the benchmark -configrations. To run the benchmark suites, see -[IREE Benchmark Suites](https://iree.dev/developers/performance/benchmark-suites/). - -## Updating Benchmarks - -1. Modify the Python scripts of benchmark generators under - [build_tools/python/benchmark_suites/iree](/build_tools/python/benchmark_suites/iree). -2. Follow - [tests/e2e/test_artifacts](https://github.com/iree-org/iree/tree/main/tests/e2e/test_artifacts) - to regenerate the cmake files that will build the benchmarks. - -To add a new source model, see -[Adding a new model](/build_tools/python/e2e_test_framework/models/README.md#adding-a-new-model) - -## Updating TF/TFLite Importer in Benchmark CI - -For TF and TFLite source models, benchmark CI installs `iree-import-tf/tflite` -from -[integrations/tensorflow/python_projects](/integrations/tensorflow/python_projects) -to import models. See -[Updating Tensorflow Importers in CI](/integrations/tensorflow/README.md#updating-tensorflow-importers-in-ci) -to learn about the update procedure. - -## Benchmark Suites Design - -> TODO(#12215): Explain the design and the end-to-end flow. diff --git a/build_tools/python/benchmark_suites/iree/__init__.py b/build_tools/python/benchmark_suites/iree/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/build_tools/python/benchmark_suites/iree/adreno_benchmarks.py b/build_tools/python/benchmark_suites/iree/adreno_benchmarks.py deleted file mode 100644 index 52e242c7b36c..000000000000 --- a/build_tools/python/benchmark_suites/iree/adreno_benchmarks.py +++ /dev/null @@ -1,84 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines IREE Adreno GPU benchmarks.""" - -from typing import List - -from benchmark_suites.iree import benchmark_presets, module_execution_configs, utils -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions, iree_definitions -from e2e_test_framework.models import tflite_models, tf_models -from e2e_test_framework.device_specs import device_collections - - -class Android_Adreno_Benchmarks(object): - """Benchmarks on Android devices with Adreno GPU.""" - - ADRENO_GPU_COMPILE_TARGET = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.VULKAN_SPIRV, - target_architecture=common_definitions.DeviceArchitecture.QUALCOMM_ADRENO, - target_abi=iree_definitions.TargetABI.VULKAN_ANDROID31, - ) - DEFAULT_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_ANDROID_QUALCOMM_ADRENO_DEFAULTS, - tags=["default-flags"], - compile_targets=[ADRENO_GPU_COMPILE_TARGET], - ) - FUSE_PADDING_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_ANDROID_QUALCOMM_ADRENO_FUSE_PADDING, - tags=["experimental-flags", "fuse-padding"], - compile_targets=[ADRENO_GPU_COMPILE_TARGET], - extra_flags=["--iree-flow-enable-fuse-padding-into-linalg-consumer-ops"], - ) - FUSE_PADDING_REPEATED_KERNEL_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_ANDROID_QUALCOMM_ADRENO_FUSE_PADDING_REPEATED_KERNEL, - tags=["experimental-flags", "fuse-padding", "repeated-kernel"], - compile_targets=[ADRENO_GPU_COMPILE_TARGET], - extra_flags=FUSE_PADDING_COMPILE_CONFIG.extra_flags - + ["--iree-hal-benchmark-dispatch-repeat-count=16"], - ) - - def generate( - self, - ) -> List[iree_definitions.E2EModelRunConfig]: - default_models = [ - tflite_models.MOBILEBERT_FP32, - ] - default_gen_configs = [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=self.DEFAULT_COMPILE_CONFIG, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in default_models - ] - fuse_padding_gen_configs = [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=self.FUSE_PADDING_COMPILE_CONFIG, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in default_models - ] - - adreno_devices = ( - device_collections.DEFAULT_DEVICE_COLLECTION.query_device_specs( - architecture=common_definitions.DeviceArchitecture.QUALCOMM_ADRENO, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - ) - ) - run_configs = utils.generate_e2e_model_run_configs( - module_generation_configs=default_gen_configs, - module_execution_configs=[module_execution_configs.VULKAN_CONFIG], - device_specs=adreno_devices, - presets=[benchmark_presets.ANDROID_GPU], - ) - run_configs += utils.generate_e2e_model_run_configs( - module_generation_configs=fuse_padding_gen_configs, - module_execution_configs=[module_execution_configs.VULKAN_CONFIG], - device_specs=adreno_devices, - presets=[benchmark_presets.ANDROID_GPU], - ) - - return run_configs diff --git a/build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py b/build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py deleted file mode 100644 index 25bae6707452..000000000000 --- a/build_tools/python/benchmark_suites/iree/armv8_a_benchmarks.py +++ /dev/null @@ -1,180 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines IREE ARMv8-A benchmarks.""" - -from typing import List, Sequence - -from benchmark_suites.iree import benchmark_presets, module_execution_configs, utils -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions, iree_definitions -from e2e_test_framework.device_specs import device_collections -from e2e_test_framework.models import tflite_models, tf_models, matmul - - -class Android_ARMv8_A_Benchmarks(object): - """Benchmarks on ARMv8-A Android devices.""" - - MODELS = [ - tflite_models.DEEPLABV3_FP32, - tflite_models.MOBILEBERT_FP32, - tf_models.GPT2_117M_1x4_FP32_TF, - tf_models.GPT2_117M_1x1_FP32_TF, - tflite_models.MOBILEBERT_INT8, - tflite_models.VIT_INT8_TFL, - ] - - MATMULS = [ - matmul.MATMUL_1x256x2048_I8xI4_MLIR, - matmul.MATMUL_256x256x2048_I8xI4_MLIR, - matmul.MATMUL_1x256x2048_I8xI8_MLIR, - matmul.MATMUL_256x256x2048_I8xI8_MLIR, - ] - - ARMV8_A_CPU_TARGET = iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.ARMV8_2_A_GENERIC, - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_abi=iree_definitions.TargetABI.LINUX_ANDROID29, - ) - - NO_DT_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_ANDROID_ARMV8_2_A_GENERIC_NO_DT, - tags=["experimental-flags", "no-dt"], - compile_targets=[ARMV8_A_CPU_TARGET], - extra_flags=[ - "--iree-opt-data-tiling=false", - "--iree-llvmcpu-target-cpu-features=+dotprod", - ], - ) - DT_ONLY_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_ANDROID_ARMV8_2_A_GENERIC_DT_ONLY, - tags=["experimental-flags", "dt-only"], - compile_targets=[ARMV8_A_CPU_TARGET], - extra_flags=[ - "--iree-opt-data-tiling=true", - "--iree-llvmcpu-enable-ukernels=none", - "--iree-llvmcpu-target-cpu-features=+dotprod", - ], - ) - DT_UK_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_ANDROID_ARMV8_2_A_GENERIC_DT_UK, - tags=["default-flags", "dt-uk"], - compile_targets=[ARMV8_A_CPU_TARGET], - extra_flags=[ - "--iree-opt-data-tiling=true", - "--iree-llvmcpu-enable-ukernels=all", - "--iree-llvmcpu-target-cpu-features=+dotprod", - ], - ) - - def _build_run_configs( - self, - gen_configs: Sequence[iree_definitions.ModuleGenerationConfig], - exec_configs: Sequence[iree_definitions.ModuleExecutionConfig], - device_specs: Sequence[common_definitions.DeviceSpec], - presets: Sequence[str], - ) -> List[iree_definitions.E2EModelRunConfig]: - return utils.generate_e2e_model_run_configs( - module_generation_configs=gen_configs, - module_execution_configs=exec_configs, - device_specs=device_specs, - presets=presets, - ) - - def _generate_model_configs( - self, device_specs: List[common_definitions.DeviceSpec] - ) -> List[iree_definitions.E2EModelRunConfig]: - """Generates IREE compile and run configs for list of models.""" - local_sync_execution_configs = [module_execution_configs.ELF_LOCAL_SYNC_CONFIG] - local_task_execution_configs = [ - module_execution_configs.get_elf_system_scheduling_local_task_config( - thread_num - ) - for thread_num in [1, 2] - ] - no_dt_gen_configs = [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=self.NO_DT_COMPILE_CONFIG, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in self.MODELS - ] - dt_only_gen_configs = [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=self.DT_ONLY_COMPILE_CONFIG, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in self.MODELS - ] - dt_uk_gen_configs = [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=self.DT_UK_COMPILE_CONFIG, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in self.MODELS - ] - return self._build_run_configs( - no_dt_gen_configs + dt_uk_gen_configs, - local_sync_execution_configs + local_task_execution_configs, - device_specs, - [benchmark_presets.ANDROID_CPU], - ) + self._build_run_configs( - dt_only_gen_configs, - local_sync_execution_configs + local_task_execution_configs, - device_specs, - [benchmark_presets.ANDROID_CPU_DT_ONLY], - ) - - def _generate_matmul_configs( - self, device_specs: List[common_definitions.DeviceSpec] - ) -> List[iree_definitions.E2EModelRunConfig]: - no_dt_gen_configs = [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=self.NO_DT_COMPILE_CONFIG, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in self.MATMULS - ] - dt_only_gen_configs = [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=self.DT_ONLY_COMPILE_CONFIG, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in self.MATMULS - ] - dt_uk_gen_configs = [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=self.DT_UK_COMPILE_CONFIG, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in self.MATMULS - ] - return self._build_run_configs( - no_dt_gen_configs + dt_uk_gen_configs, - [module_execution_configs.ELF_LOCAL_SYNC_CONFIG], - device_specs, - [benchmark_presets.ANDROID_CPU], - ) + self._build_run_configs( - dt_only_gen_configs, - [module_execution_configs.ELF_LOCAL_SYNC_CONFIG], - device_specs, - [benchmark_presets.ANDROID_CPU_DT_ONLY], - ) - - def generate( - self, - ) -> List[iree_definitions.E2EModelRunConfig]: - """Generates IREE compile and run configs.""" - big_cores_devices = ( - device_collections.DEFAULT_DEVICE_COLLECTION.query_device_specs( - architecture=common_definitions.DeviceArchitecture.ARMV8_2_A_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=["big-cores"], - ) - ) - - return self._generate_model_configs( - big_cores_devices - ) + self._generate_matmul_configs(big_cores_devices) diff --git a/build_tools/python/benchmark_suites/iree/benchmark_collections.py b/build_tools/python/benchmark_suites/iree/benchmark_collections.py deleted file mode 100644 index f309843d7964..000000000000 --- a/build_tools/python/benchmark_suites/iree/benchmark_collections.py +++ /dev/null @@ -1,175 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Generates all benchmarks.""" - -import collections -import re -from typing import List, Tuple, Sequence - -from e2e_test_artifacts import iree_artifacts -from e2e_test_framework.definitions import iree_definitions -from benchmark_suites.iree import ( - benchmark_presets, - riscv_benchmarks, - x86_64_benchmarks, - adreno_benchmarks, - armv8_a_benchmarks, - cuda_benchmarks, - mali_benchmarks, - vulkan_nvidia_benchmarks, - vmvx_benchmarks, -) - -COMPILE_STATS_ID_SUFFIX = "-compile-stats" -ALLOWED_NAME_FORMAT = re.compile(r"^[0-9a-zA-Z.,\-_()\[\] @]+$") - - -def validate_gen_configs( - gen_configs: Sequence[iree_definitions.ModuleGenerationConfig], -): - """Check the uniqueness and name format of module generation configs.""" - - ids_to_configs = {} - names_to_configs = {} - for gen_config in gen_configs: - if not ALLOWED_NAME_FORMAT.match(gen_config.name): - raise ValueError( - f"Module generation config name: '{gen_config.name}' doesn't" - f" follow the format '{ALLOWED_NAME_FORMAT.pattern}'" - ) - - if gen_config.composite_id in ids_to_configs: - raise ValueError( - "Two module generation configs have the same ID:\n\n" - f"{repr(gen_config)}\n\n" - f"{repr(ids_to_configs[gen_config.composite_id])}" - ) - ids_to_configs[gen_config.composite_id] = gen_config - - if gen_config.name in names_to_configs: - raise ValueError( - "Two module generation configs have the same name:\n\n" - f"{repr(gen_config)}\n\n" - f"{repr(names_to_configs[gen_config.name])}" - ) - names_to_configs[gen_config.name] = gen_config - - -def validate_run_configs( - run_configs: Sequence[iree_definitions.E2EModelRunConfig], -): - """Check the uniqueness and name format of E2E model run configs.""" - - ids_to_configs = {} - names_to_configs = {} - for run_config in run_configs: - if not ALLOWED_NAME_FORMAT.match(run_config.name): - raise ValueError( - f"E2E model run config name: '{run_config.name}' doesn't" - f" follow the format '{ALLOWED_NAME_FORMAT.pattern}'" - ) - - if run_config.composite_id in ids_to_configs: - raise ValueError( - "Two e2e model run configs have the same ID:\n\n" - f"{repr(run_config)}\n\n" - f"{repr(ids_to_configs[run_config.composite_id])}" - ) - ids_to_configs[run_config.composite_id] = run_config - - if run_config.name in names_to_configs: - raise ValueError( - "Two e2e model run configs have the same name:\n\n" - f"{repr(run_config)}\n\n" - f"{repr(names_to_configs[run_config.name])}" - ) - names_to_configs[run_config.name] = run_config - - -def generate_benchmarks() -> ( - Tuple[ - List[iree_definitions.ModuleGenerationConfig], - List[iree_definitions.E2EModelRunConfig], - ] -): - """Generate the benchmark suite.""" - - benchmarks = [ - x86_64_benchmarks.Linux_x86_64_Benchmarks(), - cuda_benchmarks.Linux_CUDA_Benchmarks(), - riscv_benchmarks.Linux_RV64_Benchmarks(), - riscv_benchmarks.Linux_RV32_Benchmarks(), - armv8_a_benchmarks.Android_ARMv8_A_Benchmarks(), - adreno_benchmarks.Android_Adreno_Benchmarks(), - mali_benchmarks.Android_Mali_Benchmarks(), - vulkan_nvidia_benchmarks.Linux_Vulkan_NVIDIA_Benchmarks(), - vmvx_benchmarks.VMVX_Benchmarks(), - ] - all_run_configs: List[iree_definitions.E2EModelRunConfig] = [] - for benchmark in benchmarks: - run_configs = benchmark.generate() - all_run_configs += run_configs - - # Collect all module generation configs in run configs. - all_gen_configs = {} - gen_config_dependents = collections.defaultdict(list) - for run_config in all_run_configs: - gen_config = run_config.module_generation_config - all_gen_configs[gen_config.composite_id] = gen_config - gen_config_dependents[gen_config.composite_id].append(run_config) - - all_gen_configs = list(all_gen_configs.values()) - - validate_gen_configs(all_gen_configs) - validate_run_configs(all_run_configs) - - compile_stats_gen_configs = [] - # For now we simply track compilation statistics of all modules. - for gen_config in all_gen_configs: - compile_config = gen_config.compile_config - # Use POSIX path, see the comment of iree_definitions.MODULE_DIR_VARIABLE. - scheduling_stats_path = f"{iree_definitions.MODULE_DIR_VARIABLE}/{iree_artifacts.SCHEDULING_STATS_FILENAME}" - compile_stats_config = iree_definitions.CompileConfig.build( - id=compile_config.id + COMPILE_STATS_ID_SUFFIX, - tags=compile_config.tags + ["compile-stats"], - compile_targets=compile_config.compile_targets, - extra_flags=compile_config.extra_flags - + [ - # Enable zip polyglot to provide component sizes. - "--iree-vm-emit-polyglot-zip=true", - # Disable debug symbols to provide correct component sizes. - "--iree-llvmcpu-debug-symbols=false", - # Dump scheduling statistics - "--iree-scheduling-dump-statistics-format=json", - f"--iree-scheduling-dump-statistics-file={scheduling_stats_path}", - ], - ) - - dependents = gen_config_dependents[gen_config.composite_id] - compile_stats_presets = set() - for dependent in dependents: - dep_presets = set(dependent.presets) - # Assign compilation benchmark presets based on the size of the - # original benchmarks. - # A benchmark can be in default and large presets at the same time, - # for example, batch-1 benchmark is added to default for sanity - # check. So check both cases. - if dep_presets.intersection(benchmark_presets.DEFAULT_PRESETS): - compile_stats_presets.add(benchmark_presets.COMP_STATS) - if dep_presets.intersection(benchmark_presets.LARGE_PRESETS): - compile_stats_presets.add(benchmark_presets.COMP_STATS_LARGE) - - compile_stats_gen_configs.append( - iree_definitions.ModuleGenerationConfig.build( - imported_model=gen_config.imported_model, - compile_config=compile_stats_config, - presets=sorted(compile_stats_presets), - tags=gen_config.tags, - ) - ) - all_gen_configs += compile_stats_gen_configs - - return (all_gen_configs, all_run_configs) diff --git a/build_tools/python/benchmark_suites/iree/benchmark_collections_test.py b/build_tools/python/benchmark_suites/iree/benchmark_collections_test.py deleted file mode 100644 index 8c2d95fb59b3..000000000000 --- a/build_tools/python/benchmark_suites/iree/benchmark_collections_test.py +++ /dev/null @@ -1,266 +0,0 @@ -## Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import unittest - -from benchmark_suites.iree import benchmark_collections -from e2e_test_framework.definitions import common_definitions, iree_definitions - -MODEL = common_definitions.Model( - id="dummy-model-1234", - name="dummy-model", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url="https://example.com/xyz.mlir", - entry_function="main", - input_types=["1xf32"], -) -IMPORTED_MODEL = iree_definitions.ImportedModel.from_model(MODEL) -COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id="dummy-config-1234", - compile_targets=[ - iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - ], -) -EXEC_CONFIG = iree_definitions.ModuleExecutionConfig.build( - id="dummy-exec-1234", - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_SYNC, -) -DEVICE_SPEC = common_definitions.DeviceSpec.build( - id="dummy-device-1234", - device_name="dummy-device", - architecture=common_definitions.DeviceArchitecture.CUDA_SM80, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, -) - - -class BenchmarkCollectionsTest(unittest.TestCase): - def test_validate_gen_configs(self): - config_a = iree_definitions.ModuleGenerationConfig( - composite_id="a", - name="model-name (A.RCH)[tag_0,tag_1]", - tags=[], - presets=["default"], - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - compile_flags=[], - ) - config_b = iree_definitions.ModuleGenerationConfig( - composite_id="b", - name="name-b", - tags=[], - presets=["default"], - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - compile_flags=[], - ) - - benchmark_collections.validate_gen_configs([config_a, config_b]) - - def test_validate_gen_configs_duplicate_name(self): - config_a = iree_definitions.ModuleGenerationConfig( - composite_id="a", - name="name", - tags=[], - presets=["default"], - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - compile_flags=[], - ) - config_b = iree_definitions.ModuleGenerationConfig( - composite_id="b", - name="name", - tags=[], - presets=["default"], - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - compile_flags=[], - ) - - self.assertRaises( - ValueError, - lambda: benchmark_collections.validate_gen_configs([config_a, config_b]), - ) - - def test_validate_gen_configs_disallowed_characters(self): - config_a = iree_definitions.ModuleGenerationConfig( - composite_id="a", - name="name+a", - tags=[], - presets=["default"], - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - compile_flags=[], - ) - - self.assertRaises( - ValueError, - lambda: benchmark_collections.validate_gen_configs([config_a]), - ) - - def test_validate_gen_configs_duplicate_id(self): - config_a = iree_definitions.ModuleGenerationConfig( - composite_id="x", - name="name-a", - tags=[], - presets=["default"], - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - compile_flags=[], - ) - config_b = iree_definitions.ModuleGenerationConfig( - composite_id="x", - name="name-b", - tags=[], - presets=["default"], - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - compile_flags=[], - ) - - self.assertRaises( - ValueError, - lambda: benchmark_collections.validate_gen_configs([config_a, config_b]), - ) - - def test_validate_run_configs(self): - config_a = iree_definitions.E2EModelRunConfig( - composite_id="a", - name="model-name (A.RCH)[tag_0,tag_1] @ device", - tags=[], - presets=["default"], - module_generation_config=iree_definitions.ModuleGenerationConfig.build( - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - ), - module_execution_config=EXEC_CONFIG, - target_device_spec=DEVICE_SPEC, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - run_flags=[], - ) - config_b = iree_definitions.E2EModelRunConfig( - composite_id="b", - name="name-b", - tags=[], - presets=["default"], - module_generation_config=iree_definitions.ModuleGenerationConfig.build( - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - ), - module_execution_config=EXEC_CONFIG, - target_device_spec=DEVICE_SPEC, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - run_flags=[], - ) - - benchmark_collections.validate_run_configs([config_a, config_b]) - - def test_validate_run_configs_duplicate_name(self): - config_a = iree_definitions.E2EModelRunConfig( - composite_id="a", - name="name", - tags=[], - presets=["default"], - module_generation_config=iree_definitions.ModuleGenerationConfig.build( - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - ), - module_execution_config=EXEC_CONFIG, - target_device_spec=DEVICE_SPEC, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - run_flags=[], - ) - config_b = iree_definitions.E2EModelRunConfig( - composite_id="b", - name="name", - tags=[], - presets=["default"], - module_generation_config=iree_definitions.ModuleGenerationConfig.build( - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - ), - module_execution_config=EXEC_CONFIG, - target_device_spec=DEVICE_SPEC, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - run_flags=[], - ) - - self.assertRaises( - ValueError, - lambda: benchmark_collections.validate_run_configs([config_a, config_b]), - ) - - def test_validate_run_configs_duplicate_id(self): - config_a = iree_definitions.E2EModelRunConfig( - composite_id="x", - name="name-a", - tags=[], - presets=["default"], - module_generation_config=iree_definitions.ModuleGenerationConfig.build( - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - ), - module_execution_config=EXEC_CONFIG, - target_device_spec=DEVICE_SPEC, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - run_flags=[], - ) - config_b = iree_definitions.E2EModelRunConfig( - composite_id="x", - name="name-b", - tags=[], - presets=["default"], - module_generation_config=iree_definitions.ModuleGenerationConfig.build( - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - ), - module_execution_config=EXEC_CONFIG, - target_device_spec=DEVICE_SPEC, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - run_flags=[], - ) - - self.assertRaises( - ValueError, - lambda: benchmark_collections.validate_run_configs([config_a, config_b]), - ) - - def test_validate_run_configs_disallowed_characters(self): - config = iree_definitions.E2EModelRunConfig( - composite_id="x", - name="name+a", - tags=[], - presets=["default"], - module_generation_config=iree_definitions.ModuleGenerationConfig.build( - imported_model=IMPORTED_MODEL, - compile_config=COMPILE_CONFIG, - ), - module_execution_config=EXEC_CONFIG, - target_device_spec=DEVICE_SPEC, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - run_flags=[], - ) - - self.assertRaises( - ValueError, - lambda: benchmark_collections.validate_run_configs([config]), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/python/benchmark_suites/iree/benchmark_presets.py b/build_tools/python/benchmark_suites/iree/benchmark_presets.py deleted file mode 100644 index 863e23ef284b..000000000000 --- a/build_tools/python/benchmark_suites/iree/benchmark_presets.py +++ /dev/null @@ -1,55 +0,0 @@ -## Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Presets to group benchmarks by their characteristics. - -To put a benchmark into a preset, add the preset to its `presets` field. -""" - -# Default Android CPU execution benchmarks. -ANDROID_CPU = "android-cpu" -# Android CPU execution benchmarks with data-tiling only. -ANDROID_CPU_DT_ONLY = "android-cpu-dt-only" -# Default Android GPU execution benchmarks. -ANDROID_GPU = "android-gpu" -# Default CUDA execution benchmarks. -CUDA = "cuda" -# Large CUDA execution benchmarks. -CUDA_LARGE = "cuda-large" -# Default RISC-V execution benchamrks. -RISCV = "riscv" -# Default Vulkan NVIDIA execution benchamrks. -VULKAN_NVIDIA = "vulkan-nvidia" -# Default x86_64 execution benchmarks. -X86_64 = "x86_64" -# x86_64 execution benchmarks with data-tiling only. -X86_64_DT_ONLY = "x86_64-dt-only" -# Large x86_64 execution benchmarks. -X86_64_LARGE = "x86_64-large" - -# Default compilation benchmark preset. -COMP_STATS = "comp-stats" -# Large compilation benchmark preset. -COMP_STATS_LARGE = "comp-stats-large" - -# Default execution benchmark presets. -DEFAULT_PRESETS = [ - ANDROID_CPU, - ANDROID_CPU_DT_ONLY, - ANDROID_GPU, - CUDA, - RISCV, - VULKAN_NVIDIA, - X86_64, - X86_64_DT_ONLY, -] -# Large execution benchmark presets. -LARGE_PRESETS = [ - CUDA_LARGE, - X86_64_LARGE, -] - -ALL_EXECUTION_PRESETS = DEFAULT_PRESETS + LARGE_PRESETS -ALL_COMPILATION_PRESETS = [COMP_STATS, COMP_STATS_LARGE] diff --git a/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py b/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py deleted file mode 100644 index 62847d5268ec..000000000000 --- a/build_tools/python/benchmark_suites/iree/cuda_benchmarks.py +++ /dev/null @@ -1,97 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines IREE CUDA benchmarks.""" - -from typing import List, Sequence - -from benchmark_suites.iree import benchmark_presets, module_execution_configs, utils -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions, iree_definitions -from e2e_test_framework.device_specs import device_collections -from e2e_test_framework.models import model_groups - - -class Linux_CUDA_Benchmarks(object): - """Benchmarks on CUDA Linux devices.""" - - SM_80_GPU_TARGET = iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.CUDA_SM80, - target_backend=iree_definitions.TargetBackend.CUDA, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - SM_80_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_LINUX_CUDA_SM80_DEFAULTS, - tags=["default-flags"], - compile_targets=[SM_80_GPU_TARGET], - ) - SM_80_UBENCH_MATMUL_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_LINUX_CUDA_SM80_MATMUL_UBENCH, - tags=["ukernel", "matmul"], - compile_targets=[SM_80_GPU_TARGET], - extra_flags=["--iree-hal-benchmark-dispatch-repeat-count=100"], - ) - SM_80_UBENCH_MATMUL_SPLITK_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_LINUX_CUDA_SM80_MATMUL_SPLITK_UBENCH, - tags=["ukernel", "matmul", "splitk"], - compile_targets=[SM_80_GPU_TARGET], - extra_flags=[ - "--iree-hal-benchmark-dispatch-repeat-count=100", - "--iree-flow-split-matmul-reduction=4", - "--iree-codegen-llvmgpu-use-wmma", - ], - ) - - def _generate_configs( - self, - models: Sequence[common_definitions.Model], - compile_config: iree_definitions.CompileConfig, - execution_config: iree_definitions.ModuleExecutionConfig, - presets: Sequence[str], - ) -> List[iree_definitions.E2EModelRunConfig]: - gen_configs = [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=compile_config, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in models - ] - sm80_devices = device_collections.DEFAULT_DEVICE_COLLECTION.query_device_specs( - architecture=common_definitions.DeviceArchitecture.NVIDIA_AMPERE, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - ) - run_module_configs = utils.generate_e2e_model_run_configs( - module_generation_configs=gen_configs, - module_execution_configs=[execution_config], - device_specs=sm80_devices, - presets=presets, - ) - - return run_module_configs - - def generate( - self, - ) -> List[iree_definitions.E2EModelRunConfig]: - """Generates IREE compile and run configs.""" - # The CUDA tag is required to put them into the CUDA benchmark preset. - run_configs = self._generate_configs( - model_groups.CUDA_MODELS, - self.SM_80_COMPILE_CONFIG, - execution_config=module_execution_configs.CUDA_CONFIG, - presets=[benchmark_presets.CUDA], - ) - ubench_run_configs = self._generate_configs( - model_groups.MICRO_MATMUL, - self.SM_80_UBENCH_MATMUL_COMPILE_CONFIG, - execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG, - presets=[benchmark_presets.CUDA], - ) - ubench_splitk_run_configs = self._generate_configs( - model_groups.MICRO_MATMUL_SPLITK, - self.SM_80_UBENCH_MATMUL_SPLITK_COMPILE_CONFIG, - execution_config=module_execution_configs.CUDA_BATCH_SIZE_100_CONFIG, - presets=[benchmark_presets.CUDA], - ) - return run_configs + ubench_run_configs + ubench_splitk_run_configs diff --git a/build_tools/python/benchmark_suites/iree/mali_benchmarks.py b/build_tools/python/benchmark_suites/iree/mali_benchmarks.py deleted file mode 100644 index b0fffe2bd739..000000000000 --- a/build_tools/python/benchmark_suites/iree/mali_benchmarks.py +++ /dev/null @@ -1,147 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines IREE Mali GPU benchmarks.""" - -from typing import List, Sequence - -from benchmark_suites.iree import benchmark_presets, module_execution_configs, utils -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions, iree_definitions -from e2e_test_framework.models import tflite_models, tf_models -from e2e_test_framework.device_specs import device_collections - - -class Android_Mali_Benchmarks(object): - """Benchmarks on Android devices with Mali GPU.""" - - ARM_VALHALL_GPU_TARGET = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.VULKAN_SPIRV, - target_architecture=common_definitions.DeviceArchitecture.ARM_VALHALL, - target_abi=iree_definitions.TargetABI.VULKAN_ANDROID31, - ) - DEFAULT_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_ANDROID_ARM_VALHALL_DEFAULTS, - tags=["default-flags"], - compile_targets=[ARM_VALHALL_GPU_TARGET], - ) - EXPERIMENTAL_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_ANDROID_ARM_VALHALL_EXPERIMENTAL, - tags=["experimental-flags", "fuse-padding", "max-concurrency"], - compile_targets=[ARM_VALHALL_GPU_TARGET], - extra_flags=[ - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops", - "--iree-stream-partitioning-favor=max-concurrency", - ], - ) - # Kernel execution - # Note that for kernel-execution benchmarks batch_size/repeat-count need to be - # low enough that the whole dispatch completes within an OS-specific timeout. - # Otherwise you'll get error like: - # ``` - # INTERNAL; VK_ERROR_DEVICE_LOST; vkQueueSubmit; while invoking native function - # hal.fence.await; while calling import; - # ``` - EXPERIMENTAL_REPEATED_KERNEL_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_ANDROID_ARM_VALHALL_EXPERIMENTAL_REPEATED_KERNEL, - tags=[ - "experimental-flags", - "fuse-padding", - "max-concurrency", - "repeated-kernel", - ], - compile_targets=[ARM_VALHALL_GPU_TARGET], - extra_flags=EXPERIMENTAL_COMPILE_CONFIG.extra_flags - + ["--iree-hal-benchmark-dispatch-repeat-count=32"], - ) - EXPERIMENTAL_REPEATED_KERNEL_RUN_FLAGS = ["--batch_size=32"] - - FP32_MODELS = [ - tflite_models.MOBILEBERT_FP32, - ] - FP16_MODELS = [tflite_models.MOBILEBERT_FP16] - QUANT_MODELS = [ - tflite_models.MOBILEBERT_INT8, - ] - - def generate( - self, - ) -> List[iree_definitions.E2EModelRunConfig]: - default_gen_configs = self._get_module_generation_configs( - compile_config=self.DEFAULT_COMPILE_CONFIG, - fp32_models=self.FP32_MODELS, - fp16_models=self.FP16_MODELS, - quant_models=self.QUANT_MODELS, - ) - experimental_gen_configs = self._get_module_generation_configs( - compile_config=self.EXPERIMENTAL_COMPILE_CONFIG, - fp32_models=self.FP32_MODELS, - fp16_models=self.FP16_MODELS, - quant_models=self.QUANT_MODELS, - ) - experimental_repeated_kernel_gen_configs = self._get_module_generation_configs( - compile_config=self.EXPERIMENTAL_REPEATED_KERNEL_COMPILE_CONFIG, - fp32_models=self.FP32_MODELS, - fp16_models=self.FP16_MODELS, - quant_models=self.QUANT_MODELS, - ) - - mali_devices = device_collections.DEFAULT_DEVICE_COLLECTION.query_device_specs( - architecture=common_definitions.DeviceArchitecture.ARM_VALHALL, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - ) - run_configs = utils.generate_e2e_model_run_configs( - module_generation_configs=default_gen_configs + experimental_gen_configs, - module_execution_configs=[module_execution_configs.VULKAN_CONFIG], - device_specs=mali_devices, - presets=[benchmark_presets.ANDROID_GPU], - ) - run_configs += utils.generate_e2e_model_run_configs( - module_generation_configs=experimental_repeated_kernel_gen_configs, - module_execution_configs=[ - module_execution_configs.VULKAN_BATCH_SIZE_32_CONFIG - ], - device_specs=mali_devices, - presets=[benchmark_presets.ANDROID_GPU], - ) - - return run_configs - - def _get_module_generation_configs( - self, - compile_config: iree_definitions.CompileConfig, - fp32_models: Sequence[common_definitions.Model], - fp16_models: Sequence[common_definitions.Model], - quant_models: Sequence[common_definitions.Model], - ) -> List[iree_definitions.ModuleGenerationConfig]: - demote_compile_config = iree_definitions.CompileConfig.build( - id=compile_config.id + "-demote-f32-to-16", - tags=compile_config.tags + ["demote-f32-to-f16"], - compile_targets=compile_config.compile_targets, - extra_flags=compile_config.extra_flags + ["--iree-input-demote-f32-to-f16"], - ) - return ( - [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=compile_config, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in fp32_models - ] - + [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=demote_compile_config, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in fp16_models - ] - + [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=compile_config, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in quant_models - ] - ) diff --git a/build_tools/python/benchmark_suites/iree/module_execution_configs.py b/build_tools/python/benchmark_suites/iree/module_execution_configs.py deleted file mode 100644 index 2d076b42a09c..000000000000 --- a/build_tools/python/benchmark_suites/iree/module_execution_configs.py +++ /dev/null @@ -1,111 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines ModuleExecutionConfig for benchmarks.""" - -from typing import List, Optional, Sequence - -from e2e_test_framework.definitions import iree_definitions -from e2e_test_framework import unique_ids - - -def _with_caching_allocator( - id: str, - tags: List[str], - loader: iree_definitions.RuntimeLoader, - driver: iree_definitions.RuntimeDriver, - extra_flags: Optional[Sequence[str]] = None, -) -> iree_definitions.ModuleExecutionConfig: - extra_flags = [] if extra_flags is None else list(extra_flags) - return iree_definitions.ModuleExecutionConfig.build( - id=id, - tags=tags, - loader=loader, - driver=driver, - extra_flags=["--device_allocator=caching"] + extra_flags, - ) - - -ELF_LOCAL_SYNC_CONFIG = _with_caching_allocator( - id=unique_ids.IREE_MODULE_EXECUTION_CONFIG_LOCAL_SYNC, - tags=["full-inference", "default-flags"], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_SYNC, -) - -CUDA_CONFIG = _with_caching_allocator( - id=unique_ids.IREE_MODULE_EXECUTION_CONFIG_CUDA, - tags=["full-inference", "default-flags"], - loader=iree_definitions.RuntimeLoader.NONE, - driver=iree_definitions.RuntimeDriver.CUDA, -) - -CUDA_BATCH_SIZE_100_CONFIG = _with_caching_allocator( - id=unique_ids.IREE_MODULE_EXECUTION_CONFIG_CUDA, - tags=["full-inference", "default-flags"], - loader=iree_definitions.RuntimeLoader.NONE, - driver=iree_definitions.RuntimeDriver.CUDA, - extra_flags=["--batch_size=100"], -) - -VULKAN_CONFIG = _with_caching_allocator( - id=unique_ids.IREE_MODULE_EXECUTION_CONFIG_VULKAN, - tags=["full-inference", "default-flags"], - loader=iree_definitions.RuntimeLoader.NONE, - driver=iree_definitions.RuntimeDriver.VULKAN, -) - -VULKAN_BATCH_SIZE_16_CONFIG = _with_caching_allocator( - id=unique_ids.IREE_MODULE_EXECUTION_CONFIG_VULKAN_BATCH_SIZE_16, - tags=["full-inference", "experimental-flags"], - loader=iree_definitions.RuntimeLoader.NONE, - driver=iree_definitions.RuntimeDriver.VULKAN, - extra_flags=["--batch_size=16"], -) - -VULKAN_BATCH_SIZE_32_CONFIG = _with_caching_allocator( - id=unique_ids.IREE_MODULE_EXECUTION_CONFIG_VULKAN_BATCH_SIZE_32, - tags=["full-inference", "experimental-flags"], - loader=iree_definitions.RuntimeLoader.NONE, - driver=iree_definitions.RuntimeDriver.VULKAN, - extra_flags=["--batch_size=32"], -) - - -def get_elf_local_task_config(thread_num: int): - config_id = ( - f"{unique_ids.IREE_MODULE_EXECUTION_CONFIG_LOCAL_TASK_BASE}-{thread_num}" - ) - return _with_caching_allocator( - id=config_id, - tags=[f"{thread_num}-thread", "full-inference", "default-flags"], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_TASK, - extra_flags=[f"--task_topology_max_group_count={thread_num}"], - ) - - -def get_vmvx_local_task_config(thread_num: int): - config_id = ( - f"{unique_ids.IREE_MODULE_EXECUTION_CONFIG_VMVX_LOCAL_TASK_BASE}-{thread_num}" - ) - return _with_caching_allocator( - id=config_id, - tags=[f"{thread_num}-thread", "full-inference", "default-flags"], - loader=iree_definitions.RuntimeLoader.VMVX_MODULE, - driver=iree_definitions.RuntimeDriver.LOCAL_TASK, - extra_flags=[f"--task_topology_max_group_count={thread_num}"], - ) - - -def get_elf_system_scheduling_local_task_config(thread_num: int): - config_id = f"{unique_ids.IREE_MODULE_EXECUTION_CONFIG_SYS_SCHED_LOCAL_TASK_BASE}-{thread_num}" - return _with_caching_allocator( - id=config_id, - tags=[f"{thread_num}-thread", "full-inference", "system-scheduling"], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_TASK, - extra_flags=[f"--task_topology_group_count={thread_num}"], - ) diff --git a/build_tools/python/benchmark_suites/iree/riscv_benchmarks.py b/build_tools/python/benchmark_suites/iree/riscv_benchmarks.py deleted file mode 100644 index 16d3f98bd5bb..000000000000 --- a/build_tools/python/benchmark_suites/iree/riscv_benchmarks.py +++ /dev/null @@ -1,105 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines IREE RISC-V benchmarks.""" - -from typing import List - -from benchmark_suites.iree import benchmark_presets, module_execution_configs, utils -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions, iree_definitions -from e2e_test_framework.device_specs import riscv_specs -from e2e_test_framework.models import tflite_models, tf_models, torch_models - - -class Linux_RV64_Benchmarks(object): - """Benchmarks RV64 on Linux devices.""" - - RV64_CPU_TARGET = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - DEFAULT_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_LINUX_RV64_GENERIC_DEFAULTS, - tags=["default-flags"], - compile_targets=[RV64_CPU_TARGET], - ) - MODELS = [ - tf_models.MINILM_L12_H384_UNCASED_INT32_SEQLEN128, - tflite_models.DEEPLABV3_FP32, - tflite_models.EFFICIENTNET_INT8, - tflite_models.MOBILEBERT_FP32, - tflite_models.MOBILEBERT_INT8, - tflite_models.MOBILENET_V1, - tflite_models.MOBILENET_V2_INT8, - tflite_models.PERSON_DETECT_INT8, - # PyTorch model are disabled due to https://github.com/iree-org/iree/issues/14993. - # torch_models.MODEL_CLIP_TEXT_SEQLEN64_FP32_TORCH, - ] - - def generate( - self, - ) -> List[iree_definitions.E2EModelRunConfig]: - """Generates IREE compile and run configs.""" - gen_configs = [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=self.DEFAULT_COMPILE_CONFIG, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in self.MODELS - ] - run_configs = utils.generate_e2e_model_run_configs( - module_generation_configs=gen_configs, - module_execution_configs=[ - module_execution_configs.ELF_LOCAL_SYNC_CONFIG, - module_execution_configs.get_elf_system_scheduling_local_task_config( - thread_num=2 - ), - ], - device_specs=[riscv_specs.EMULATOR_RISCV_64], - presets=[benchmark_presets.RISCV], - ) - return run_configs - - -class Linux_RV32_Benchmarks(object): - """Benchmarks RV32 on Linux devices.""" - - RV32_CPU_TARGET = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_architecture=common_definitions.DeviceArchitecture.RV32_GENERIC, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - DEFAULT_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_LINUX_RV32_GENERIC_DEFAULTS, - tags=["default-flags"], - compile_targets=[RV32_CPU_TARGET], - ) - MODELS = [ - tflite_models.EFFICIENTNET_INT8, - tflite_models.MOBILEBERT_INT8, - tflite_models.PERSON_DETECT_INT8, - tflite_models.MOBILENET_V2_INT8, - ] - - def generate( - self, - ) -> List[iree_definitions.E2EModelRunConfig]: - """Generates IREE compile and run configs.""" - gen_configs = [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=self.DEFAULT_COMPILE_CONFIG, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in self.MODELS - ] - run_configs = utils.generate_e2e_model_run_configs( - module_generation_configs=gen_configs, - module_execution_configs=[module_execution_configs.ELF_LOCAL_SYNC_CONFIG], - device_specs=[riscv_specs.EMULATOR_RISCV_32], - presets=[benchmark_presets.RISCV], - ) - return run_configs diff --git a/build_tools/python/benchmark_suites/iree/utils.py b/build_tools/python/benchmark_suites/iree/utils.py deleted file mode 100644 index 17a47a4233ac..000000000000 --- a/build_tools/python/benchmark_suites/iree/utils.py +++ /dev/null @@ -1,35 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import itertools -from typing import List, Sequence - -from e2e_test_framework.definitions import common_definitions, iree_definitions - - -def generate_e2e_model_run_configs( - module_generation_configs: Sequence[iree_definitions.ModuleGenerationConfig], - module_execution_configs: Sequence[iree_definitions.ModuleExecutionConfig], - device_specs: Sequence[common_definitions.DeviceSpec], - presets: Sequence[str], - tags: Sequence[str] = (), - tool: iree_definitions.E2EModelRunTool = iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, -) -> List[iree_definitions.E2EModelRunConfig]: - """Generates the run specs from the product of compile specs and run configs.""" - return [ - iree_definitions.E2EModelRunConfig.build( - module_generation_config=module_generation_config, - module_execution_config=module_execution_config, - target_device_spec=device_spec, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=tool, - tags=tags, - presets=presets, - ) - for module_generation_config, module_execution_config, device_spec in itertools.product( - module_generation_configs, module_execution_configs, device_specs - ) - ] diff --git a/build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py b/build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py deleted file mode 100644 index 6a5b7c16ca54..000000000000 --- a/build_tools/python/benchmark_suites/iree/vmvx_benchmarks.py +++ /dev/null @@ -1,59 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines IREE VMVX benchmarks.""" - -from typing import List - -from benchmark_suites.iree import benchmark_presets, module_execution_configs, utils -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions, iree_definitions -from e2e_test_framework.device_specs import device_collections -from e2e_test_framework.models import tflite_models - - -class VMVX_Benchmarks(object): - """Benchmarks with VMVX backend on different platforms.""" - - VMVX_CPU_TARGET = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.VMVX, - target_architecture=common_definitions.DeviceArchitecture.VMVX_GENERIC, - target_abi=iree_definitions.TargetABI.VMVX, - ) - EXPERIMENTAL_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_VMVX_GENERIC_EXPERIMENTAL, - tags=["experimental-flags"], - compile_targets=[VMVX_CPU_TARGET], - ) - - def generate( - self, - ) -> List[iree_definitions.E2EModelRunConfig]: - """Generates IREE compile and run configs.""" - - gen_configs = [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=self.EXPERIMENTAL_COMPILE_CONFIG, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in [tflite_models.MOBILENET_V2, tflite_models.MOBILENET_V3SMALL] - ] - local_task_exec_configs = [ - module_execution_configs.get_vmvx_local_task_config(thread_num=8) - ] - cascadelake_devices = ( - device_collections.DEFAULT_DEVICE_COLLECTION.query_device_specs( - architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - ) - ) - run_configs = utils.generate_e2e_model_run_configs( - module_generation_configs=gen_configs, - module_execution_configs=local_task_exec_configs, - device_specs=cascadelake_devices, - presets=[benchmark_presets.X86_64], - ) - - return run_configs diff --git a/build_tools/python/benchmark_suites/iree/vulkan_nvidia_benchmarks.py b/build_tools/python/benchmark_suites/iree/vulkan_nvidia_benchmarks.py deleted file mode 100644 index 6788052dfab1..000000000000 --- a/build_tools/python/benchmark_suites/iree/vulkan_nvidia_benchmarks.py +++ /dev/null @@ -1,111 +0,0 @@ -## Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines IREE Vulkan NVIDIA benchmarks.""" - -from typing import List, Sequence - -from benchmark_suites.iree import benchmark_presets, module_execution_configs, utils -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions, iree_definitions -from e2e_test_framework.device_specs import device_collections -from e2e_test_framework.models import model_groups - - -def _get_compile_flag(): - preprocess_passes = [ - "iree-global-opt-detach-elementwise-from-named-ops", - "iree-preprocessing-convert-conv2d-to-img2col", - "iree-global-opt-convert-1x1-filter-conv2d-to-matmul", - "iree-preprocessing-pad-linalg-ops{pad-size=32}", - ] - preprocess_flag_template = ( - "--iree-preprocessing-pass-pipeline=builtin.module(func.func({}))" - ) - return [ - preprocess_flag_template.format(",".join(preprocess_passes)), - ] - - -class Linux_Vulkan_NVIDIA_Benchmarks(object): - """Benchmarks on Linux Vulkan NVIDIA devices.""" - - AMPERE_TARGET = iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.NVIDIA_AMPERE, - target_backend=iree_definitions.TargetBackend.VULKAN_SPIRV, - target_abi=iree_definitions.TargetABI.VULKAN_LINUX, - ) - PASCAL_TARGET = iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.NVIDIA_PASCAL, - target_backend=iree_definitions.TargetBackend.VULKAN_SPIRV, - target_abi=iree_definitions.TargetABI.VULKAN_LINUX, - ) - - SIMT_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_LINUX_VULKAN_SD_SIMT, - tags=["experimental-flags", "simt"], - compile_targets=[PASCAL_TARGET], - extra_flags=_get_compile_flag(), - ) - TENSORCORE_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_LINUX_VULKAN_SD_TENSORCORE, - tags=["experimental-flags", "tensorcore"], - compile_targets=[AMPERE_TARGET], - extra_flags=_get_compile_flag(), - ) - - def _generate_configs( - self, - models: Sequence[common_definitions.Model], - compile_config: iree_definitions.CompileConfig, - execution_config: iree_definitions.ModuleExecutionConfig, - presets: Sequence[str], - ) -> List[iree_definitions.E2EModelRunConfig]: - gen_configs = [ - iree_definitions.ModuleGenerationConfig.build( - compile_config=compile_config, - imported_model=iree_definitions.ImportedModel.from_model(model), - ) - for model in models - ] - # We use the same NVIDIA Ampere GPU for benchmarking code generated for - # both Pascal and Ampere architectures. What we care is not exactly these - # two architectures per se; they represent SIMT and tensorcore CodeGen - # paths that we would want both to work. Ampere is able to run both SIMT - # and tensorcore cases. - ampere_devices = ( - device_collections.DEFAULT_DEVICE_COLLECTION.query_device_specs( - architecture=common_definitions.DeviceArchitecture.NVIDIA_AMPERE, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - ) - ) - run_module_configs = utils.generate_e2e_model_run_configs( - module_generation_configs=gen_configs, - module_execution_configs=[execution_config], - device_specs=ampere_devices, - presets=presets, - ) - - return run_module_configs - - def generate( - self, - ) -> List[iree_definitions.E2EModelRunConfig]: - """Generates IREE compile and run configs.""" - # The `vulkan-nvidia`` tag is required to put them into the Vulkan NVIDIA - # benchmark preset. - tensorcore_run_configs = self._generate_configs( - model_groups.VULKAN_MODELS, - self.TENSORCORE_COMPILE_CONFIG, - execution_config=module_execution_configs.VULKAN_CONFIG, - presets=[benchmark_presets.VULKAN_NVIDIA], - ) - simt_run_configs = self._generate_configs( - model_groups.VULKAN_MODELS, - self.SIMT_COMPILE_CONFIG, - execution_config=module_execution_configs.VULKAN_CONFIG, - presets=[benchmark_presets.VULKAN_NVIDIA], - ) - return tensorcore_run_configs + simt_run_configs diff --git a/build_tools/python/benchmark_suites/iree/x86_64_benchmarks.py b/build_tools/python/benchmark_suites/iree/x86_64_benchmarks.py deleted file mode 100644 index 14f640830b50..000000000000 --- a/build_tools/python/benchmark_suites/iree/x86_64_benchmarks.py +++ /dev/null @@ -1,140 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines IREE x86_64 benchmarks.""" - -from typing import List, Sequence - -from benchmark_suites.iree import benchmark_presets, module_execution_configs, utils -from e2e_test_framework.definitions import common_definitions, iree_definitions -from e2e_test_framework.device_specs import device_collections -from e2e_test_framework.models import model_groups -from e2e_test_framework import unique_ids - - -class Linux_x86_64_Benchmarks(object): - """Benchmarks on x86_64 linux devices.""" - - CASCADELAKE_CPU_TARGET = iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - - CASCADELAKE_NO_DT_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_LINUX_CASCADELAKE_NO_DT, - tags=["experimental-flags", "no-dt"], - compile_targets=[CASCADELAKE_CPU_TARGET], - extra_flags=[ - "--iree-opt-data-tiling=false", - ], - ) - CASCADELAKE_DT_ONLY_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_LINUX_CASCADELAKE_DT_ONLY, - tags=["experimental-flags", "dt-only"], - compile_targets=[CASCADELAKE_CPU_TARGET], - extra_flags=[ - "--iree-opt-data-tiling=true", - "--iree-llvmcpu-enable-ukernels=none", - ], - ) - CASCADELAKE_DT_UK_COMPILE_CONFIG = iree_definitions.CompileConfig.build( - id=unique_ids.IREE_COMPILE_CONFIG_LINUX_CASCADELAKE_DT_UK, - tags=["default-flags", "dt-uk"], - compile_targets=[CASCADELAKE_CPU_TARGET], - extra_flags=[ - "--iree-opt-data-tiling=true", - "--iree-llvmcpu-enable-ukernels=all", - ], - ) - - def _generate( - self, - benchmark_configs: List[common_definitions.CpuBenchmarkConfig], - compile_config: iree_definitions.CompileConfig, - device_specs: List[common_definitions.DeviceSpec], - presets: Sequence[str], - ) -> List[iree_definitions.E2EModelRunConfig]: - run_configs_all = [] - # We avoid the full combinatorial explosion of testing all models with all - # thread counts and instead test each model with a number of threads - # appropriate for its size and configurations we're interested in. - for config in benchmark_configs: - gen_config = iree_definitions.ModuleGenerationConfig.build( - compile_config=compile_config, - imported_model=iree_definitions.ImportedModel.from_model(config.model), - ) - - execution_configs = [] - for thread in config.threads: - if thread == 0: - execution_configs.append( - module_execution_configs.ELF_LOCAL_SYNC_CONFIG - ) - else: - execution_configs.append( - module_execution_configs.get_elf_local_task_config(thread) - ) - - run_configs = utils.generate_e2e_model_run_configs( - module_generation_configs=[gen_config], - module_execution_configs=execution_configs, - device_specs=device_specs, - presets=presets, - ) - - run_configs_all.extend(run_configs) - - return run_configs_all - - def generate( - self, - ) -> List[iree_definitions.E2EModelRunConfig]: - """Generates IREE compile and run configs.""" - - cascadelake_devices = ( - device_collections.DEFAULT_DEVICE_COLLECTION.query_device_specs( - architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - ) - ) - - # The X86_64 tag is required to put them into the X86_64 benchmark preset. - no_dt_run_configs = self._generate( - model_groups.X86_64_BENCHMARK_CONFIG, - self.CASCADELAKE_NO_DT_COMPILE_CONFIG, - cascadelake_devices, - presets=[benchmark_presets.X86_64], - ) - dt_only_run_configs = self._generate( - model_groups.X86_64_BENCHMARK_CONFIG, - self.CASCADELAKE_DT_ONLY_COMPILE_CONFIG, - cascadelake_devices, - presets=[benchmark_presets.X86_64_DT_ONLY], - ) - dt_uk_run_configs = self._generate( - model_groups.X86_64_BENCHMARK_CONFIG, - self.CASCADELAKE_DT_UK_COMPILE_CONFIG, - cascadelake_devices, - presets=[benchmark_presets.X86_64], - ) - large_run_configs = self._generate( - model_groups.X86_64_BENCHMARK_CONFIG_LARGE, - self.CASCADELAKE_DT_UK_COMPILE_CONFIG, - cascadelake_devices, - presets=[benchmark_presets.X86_64_LARGE], - ) - - return ( - no_dt_run_configs - + dt_only_run_configs - + dt_uk_run_configs - + large_run_configs - ) - - -def generate() -> List[iree_definitions.E2EModelRunConfig]: - """Generates all compile and run configs for IREE benchmarks.""" - return Linux_x86_64_Benchmarks().generate() diff --git a/build_tools/python/cmake_builder/CMakeLists.txt b/build_tools/python/cmake_builder/CMakeLists.txt deleted file mode 100644 index 2a6f53ee0af9..000000000000 --- a/build_tools/python/cmake_builder/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -iree_build_tools_py_test( - NAME - rules_test - SRC - "rules_test.py" -) diff --git a/build_tools/python/cmake_builder/__init__.py b/build_tools/python/cmake_builder/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/build_tools/python/cmake_builder/rules.py b/build_tools/python/cmake_builder/rules.py deleted file mode 100644 index 33d152414e9b..000000000000 --- a/build_tools/python/cmake_builder/rules.py +++ /dev/null @@ -1,246 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Helpers that build CMake rules. - -Each function takes a list of parameters and returns a string ready to be -included in a CMakeLists.txt file. Builder functions handle optional arguments, -lists, formatting, etc. - -For example: - -build_iree_fetch_artifact( - target_name="abcd", - source_url="https://example.com/abcd.tflite", - output="./abcd.tflite", - unpack=True) - -Outputs: - -iree_fetch_artifact( - NAME - "abcd" - SOURCE_URL - "https://example.com/abcd.tflite" - OUTPUT - "./abcd.tflite" - UNPACK -) -""" - -from typing import Dict, List, Optional, Sequence - -INDENT_SPACES = " " * 2 - - -def _get_string_list(values: Sequence[str], quote: bool = True) -> List[str]: - if quote: - return [f'"{value}"' for value in values] - return list(values) - - -def _get_block_body(body: List[str]) -> List[str]: - return [INDENT_SPACES + line for line in body] - - -def _get_string_arg_block( - keyword: str, value: Optional[str], quote: bool = True -) -> List[str]: - if value is None: - return [] - if quote: - value = f'"{value}"' - return [f"{keyword} {value}"] - - -def _get_string_list_arg_block( - keyword: str, values: Sequence[str], quote: bool = True -) -> List[str]: - if len(values) == 0: - return [] - body = _get_string_list(values, quote) - return [keyword] + _get_block_body(body) - - -def _get_option_arg_block(keyword: str, value: Optional[bool]) -> List[str]: - if value is True: - return [keyword] - return [] - - -def _build_call_rule( - rule_name: str, parameter_blocks: Sequence[List[str]] -) -> List[str]: - output = [f"{rule_name}("] - for block in parameter_blocks: - if len(block) == 0: - continue - output.extend(_get_block_body(block)) - output.append(")") - return output - - -def _convert_block_to_string(block: List[str]) -> str: - # Hack to append the terminating newline and only copies the list instead of - # the whole string. - return "\n".join(block + [""]) - - -def build_iree_bytecode_module( - target_name: str, - src: str, - module_name: str, - flags: List[str] = [], - compile_tool_target: Optional[str] = None, - c_identifier: Optional[str] = None, - static_lib_path: Optional[str] = None, - deps: List[str] = [], - friendly_name: Optional[str] = None, - testonly: bool = False, - public: bool = True, -) -> str: - name_block = _get_string_arg_block("NAME", target_name) - src_block = _get_string_arg_block("SRC", src) - module_name_block = _get_string_arg_block("MODULE_FILE_NAME", module_name) - c_identifier_block = _get_string_arg_block("C_IDENTIFIER", c_identifier) - static_lib_block = _get_string_arg_block("STATIC_LIB_PATH", static_lib_path) - compile_tool_target_block = _get_string_arg_block( - "COMPILE_TOOL", compile_tool_target - ) - flags_block = _get_string_list_arg_block("FLAGS", flags) - deps_block = _get_string_list_arg_block("DEPS", deps) - friendly_name_block = _get_string_arg_block("FRIENDLY_NAME", friendly_name) - testonly_block = _get_option_arg_block("TESTONLY", testonly) - public_block = _get_option_arg_block("PUBLIC", public) - return _convert_block_to_string( - _build_call_rule( - rule_name="iree_bytecode_module", - parameter_blocks=[ - name_block, - src_block, - module_name_block, - c_identifier_block, - compile_tool_target_block, - static_lib_block, - flags_block, - friendly_name_block, - deps_block, - testonly_block, - public_block, - ], - ) - ) - - -def build_iree_fetch_artifact( - target_name: str, source_url: str, output: str, unpack: bool -) -> str: - name_block = _get_string_arg_block("NAME", target_name) - source_url_block = _get_string_arg_block("SOURCE_URL", source_url) - output_block = _get_string_arg_block("OUTPUT", output) - unpack_block = _get_option_arg_block("UNPACK", unpack) - return _convert_block_to_string( - _build_call_rule( - rule_name="iree_fetch_artifact", - parameter_blocks=[name_block, source_url_block, output_block, unpack_block], - ) - ) - - -def build_iree_import_tf_model( - target_path: str, source: str, import_flags: List[str], output_mlir_file: str -) -> str: - target_name_block = _get_string_arg_block("TARGET_NAME", target_path) - source_block = _get_string_arg_block("SOURCE", source) - import_flags_block = _get_string_list_arg_block("IMPORT_FLAGS", import_flags) - output_mlir_file_block = _get_string_arg_block("OUTPUT_MLIR_FILE", output_mlir_file) - return _convert_block_to_string( - _build_call_rule( - rule_name="iree_import_tf_model", - parameter_blocks=[ - target_name_block, - source_block, - import_flags_block, - output_mlir_file_block, - ], - ) - ) - - -def build_iree_import_tflite_model( - target_path: str, source: str, import_flags: List[str], output_mlir_file: str -) -> str: - target_name_block = _get_string_arg_block("TARGET_NAME", target_path) - source_block = _get_string_arg_block("SOURCE", source) - import_flags_block = _get_string_list_arg_block("IMPORT_FLAGS", import_flags) - output_mlir_file_block = _get_string_arg_block("OUTPUT_MLIR_FILE", output_mlir_file) - return _convert_block_to_string( - _build_call_rule( - rule_name="iree_import_tflite_model", - parameter_blocks=[ - target_name_block, - source_block, - import_flags_block, - output_mlir_file_block, - ], - ) - ) - - -def build_iree_benchmark_suite_module_test( - target_name: str, - driver: str, - expected_output: str, - platform_module_map: Dict[str, str], - runner_args: Sequence[str], - timeout_secs: Optional[int] = None, - labels: Sequence[str] = [], - xfail_platforms: Sequence[str] = [], -) -> str: - name_block = _get_string_arg_block("NAME", target_name) - driver_block = _get_string_arg_block("DRIVER", driver) - expected_output_block = _get_string_arg_block("EXPECTED_OUTPUT", expected_output) - modules_block = _get_string_list_arg_block( - "MODULES", - [f"{platform}={path}" for platform, path in platform_module_map.items()], - ) - timeout_block = _get_string_arg_block( - "TIMEOUT", str(timeout_secs) if timeout_secs is not None else None - ) - runner_args_block = _get_string_list_arg_block("RUNNER_ARGS", runner_args) - labels_block = _get_string_list_arg_block("LABELS", labels) - xfail_platforms_block = _get_string_list_arg_block( - "XFAIL_PLATFORMS", xfail_platforms - ) - return _convert_block_to_string( - _build_call_rule( - rule_name="iree_benchmark_suite_module_test", - parameter_blocks=[ - name_block, - driver_block, - expected_output_block, - timeout_block, - modules_block, - runner_args_block, - labels_block, - xfail_platforms_block, - ], - ) - ) - - -def build_add_dependencies(target: str, deps: List[str]) -> str: - if len(deps) == 0: - raise ValueError("Target dependencies can't be empty.") - deps_list = _get_string_list(deps, quote=False) - return _convert_block_to_string( - [f"add_dependencies({target}"] + _get_block_body(deps_list) + [")"] - ) - - -def build_set(variable_name: str, value: str) -> str: - return _convert_block_to_string( - [f"set({variable_name}"] + _get_block_body([value]) + [")"] - ) diff --git a/build_tools/python/cmake_builder/rules_test.py b/build_tools/python/cmake_builder/rules_test.py deleted file mode 100644 index ad44e7e77be6..000000000000 --- a/build_tools/python/cmake_builder/rules_test.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python3 -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import textwrap -import unittest -import cmake_builder.rules - - -class RulesTest(unittest.TestCase): - def test_build_iree_bytecode_module(self): - rule = cmake_builder.rules.build_iree_bytecode_module( - target_name="abcd", - src="abcd.mlir", - module_name="abcd.vmfb", - flags=["--backend=cpu", "--opt=3"], - compile_tool_target="iree_iree-compile2", - c_identifier="abcd.c", - static_lib_path="libx.a", - deps=["iree_libx", "iree_liby"], - testonly=True, - public=False, - ) - - self.assertEqual( - rule, - textwrap.dedent( - """\ - iree_bytecode_module( - NAME "abcd" - SRC "abcd.mlir" - MODULE_FILE_NAME "abcd.vmfb" - C_IDENTIFIER "abcd.c" - COMPILE_TOOL "iree_iree-compile2" - STATIC_LIB_PATH "libx.a" - FLAGS - "--backend=cpu" - "--opt=3" - DEPS - "iree_libx" - "iree_liby" - TESTONLY - ) - """ - ), - ) - - def test_build_iree_bytecode_module_with_defaults(self): - rule = cmake_builder.rules.build_iree_bytecode_module( - target_name="abcd", - src="abcd.mlir", - module_name="abcd.vmfb", - flags=["--backend=cpu", "--opt=3"], - ) - - self.assertEqual( - rule, - textwrap.dedent( - """\ - iree_bytecode_module( - NAME "abcd" - SRC "abcd.mlir" - MODULE_FILE_NAME "abcd.vmfb" - FLAGS - "--backend=cpu" - "--opt=3" - PUBLIC - ) - """ - ), - ) - - def test_build_iree_fetch_artifact(self): - rule = cmake_builder.rules.build_iree_fetch_artifact( - target_name="abcd", - source_url="https://example.com/abcd.tflite", - output="./abcd.tflite", - unpack=True, - ) - - self.assertEqual( - rule, - textwrap.dedent( - """\ - iree_fetch_artifact( - NAME "abcd" - SOURCE_URL "https://example.com/abcd.tflite" - OUTPUT "./abcd.tflite" - UNPACK - ) - """ - ), - ) - - def test_build_iree_import_tf_model(self): - rule = cmake_builder.rules.build_iree_import_tf_model( - target_path="pkg_abcd", - source="abcd/model", - import_flags=[ - "--tf-savedmodel-exported-names=main", - "--tf-import-type=savedmodel_v1", - ], - output_mlir_file="abcd.mlir", - ) - - self.assertEqual( - rule, - textwrap.dedent( - """\ - iree_import_tf_model( - TARGET_NAME "pkg_abcd" - SOURCE "abcd/model" - IMPORT_FLAGS - "--tf-savedmodel-exported-names=main" - "--tf-import-type=savedmodel_v1" - OUTPUT_MLIR_FILE "abcd.mlir" - ) - """ - ), - ) - - def test_build_iree_import_tflite_model(self): - rule = cmake_builder.rules.build_iree_import_tflite_model( - target_path="pkg_abcd", - source="abcd.tflite", - import_flags=["--fake-flag=abcd"], - output_mlir_file="abcd.mlir", - ) - - self.assertEqual( - rule, - textwrap.dedent( - """\ - iree_import_tflite_model( - TARGET_NAME "pkg_abcd" - SOURCE "abcd.tflite" - IMPORT_FLAGS - "--fake-flag=abcd" - OUTPUT_MLIR_FILE "abcd.mlir" - ) - """ - ), - ) - - def test_build_iree_benchmark_suite_module_test(self): - rule = cmake_builder.rules.build_iree_benchmark_suite_module_test( - target_name="model_test", - driver="LOCAL_TASK", - expected_output="xyz", - platform_module_map={"x86_64": "a.vmfb", "arm": "b.vmfb"}, - runner_args=["--x=0", "--y=1"], - timeout_secs=10, - labels=["defaults", "e2e"], - xfail_platforms=["arm_64-Android", "riscv_32-Linux"], - ) - - self.assertEqual( - rule, - textwrap.dedent( - """\ - iree_benchmark_suite_module_test( - NAME "model_test" - DRIVER "LOCAL_TASK" - EXPECTED_OUTPUT "xyz" - TIMEOUT "10" - MODULES - "x86_64=a.vmfb" - "arm=b.vmfb" - RUNNER_ARGS - "--x=0" - "--y=1" - LABELS - "defaults" - "e2e" - XFAIL_PLATFORMS - "arm_64-Android" - "riscv_32-Linux" - ) - """ - ), - ) - - def test_build_add_dependencies(self): - rule = cmake_builder.rules.build_add_dependencies( - target="iree_mlir_suites", deps=["pkg_abcd", "pkg_efgh"] - ) - - self.assertEqual( - rule, - textwrap.dedent( - """\ - add_dependencies(iree_mlir_suites - pkg_abcd - pkg_efgh - ) - """ - ), - ) - - def test_build_set(self): - rule = cmake_builder.rules.build_set(variable_name="_ABC", value="123") - - self.assertEqual( - rule, - textwrap.dedent( - """\ - set(_ABC - 123 - ) - """ - ), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/python/e2e_model_tests/__init__.py b/build_tools/python/e2e_model_tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/build_tools/python/e2e_model_tests/cmake_generator.py b/build_tools/python/e2e_model_tests/cmake_generator.py deleted file mode 100644 index 32744246a03b..000000000000 --- a/build_tools/python/e2e_model_tests/cmake_generator.py +++ /dev/null @@ -1,77 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Generates CMake rules for e2e model tests.""" - -from typing import List - -from e2e_model_tests import test_definitions -from e2e_test_artifacts import iree_artifacts -from e2e_test_framework.definitions import iree_definitions -import cmake_builder.rules - - -def generate_rules( - module_generation_configs: List[iree_definitions.ModuleGenerationConfig], -) -> List[str]: - """Generates CMake rules for e2e model tests.""" - - # ModelTestConfig uses (imported_model, compile_config (mapped from platform)) - # to define the required module. Collect module paths indexed by the pair. - all_module_path_map = {} - for gen_config in module_generation_configs: - module_path = ( - iree_artifacts.get_module_dir_path(gen_config) - / iree_artifacts.MODULE_FILENAME - ).as_posix() - all_module_path_map[ - (gen_config.imported_model.composite_id, gen_config.compile_config.id) - ] = module_path - - cmake_rules = [] - for test_config in test_definitions.TEST_CONFIGS: - imported_model = test_config.imported_model - platform_module_map = {} - for platform in test_definitions.CMakePlatform: - if platform in test_config.unsupported_platforms: - continue - - compile_config = test_definitions.PLATFORM_COMPILE_CONFIG_MAP[platform] - module_path = all_module_path_map.get( - (imported_model.composite_id, compile_config.id) - ) - if module_path is None: - raise ValueError( - f"Module for {test_config.name} on {platform} not found." - ) - platform_module_map[platform.value] = module_path - - # TODO(#11136): Currently the DRIVER is a separate field in the CMake rule ( - # and has effect on test labels). Rules should be generated in another way - # to avoid that. Generates the flags without the driver for now. - runner_args = ( - iree_definitions.generate_run_flags( - imported_model=imported_model, - module_execution_config=test_config.execution_config, - with_driver=False, - ) - + test_config.extra_test_flags - ) - runner_args += [ - f"--input={input_type}=0" for input_type in imported_model.model.input_types - ] - cmake_rule = cmake_builder.rules.build_iree_benchmark_suite_module_test( - target_name=test_config.name, - driver=test_config.execution_config.driver.value, - expected_output=test_config.expected_output, - platform_module_map=platform_module_map, - runner_args=runner_args, - xfail_platforms=[ - platform.value for platform in test_config.xfail_platforms - ], - ) - cmake_rules.append(cmake_rule) - - return cmake_rules diff --git a/build_tools/python/e2e_model_tests/test_definitions.py b/build_tools/python/e2e_model_tests/test_definitions.py deleted file mode 100644 index 9052f2b334ad..000000000000 --- a/build_tools/python/e2e_model_tests/test_definitions.py +++ /dev/null @@ -1,113 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines e2e model tests on benchmark models.""" - -from typing import List -from dataclasses import dataclass -import dataclasses -import enum - -from e2e_test_framework.definitions import common_definitions, iree_definitions -from e2e_test_framework.models import tflite_models -from benchmark_suites.iree import ( - riscv_benchmarks, - x86_64_benchmarks, - armv8_a_benchmarks, - module_execution_configs, -) - - -class CMakePlatform(enum.Enum): - """Enum of CMake system platform string.""" - - ANDROID_ARMV8_A = "arm_64-Android" - LINUX_RISCV32 = "riscv_32-Linux" - LINUX_RISCV64 = "riscv_64-Linux" - LINUX_X86_64 = "x86_64-Linux" - - -# Compile config used for each CMake system platform. -PLATFORM_COMPILE_CONFIG_MAP = { - CMakePlatform.ANDROID_ARMV8_A: armv8_a_benchmarks.Android_ARMv8_A_Benchmarks.DT_UK_COMPILE_CONFIG, - CMakePlatform.LINUX_RISCV32: riscv_benchmarks.Linux_RV32_Benchmarks.DEFAULT_COMPILE_CONFIG, - CMakePlatform.LINUX_RISCV64: riscv_benchmarks.Linux_RV64_Benchmarks.DEFAULT_COMPILE_CONFIG, - CMakePlatform.LINUX_X86_64: x86_64_benchmarks.Linux_x86_64_Benchmarks.CASCADELAKE_DT_UK_COMPILE_CONFIG, -} - - -@dataclass(frozen=True) -class ModelTestConfig(object): - """Defines an e2e model test to run by iree-run-module.""" - - # Test name shown in the test rule. - name: str - imported_model: iree_definitions.ImportedModel - execution_config: iree_definitions.ModuleExecutionConfig - - # Either a string literal or a file path. - expected_output: str - - # Platforms to ignore this test. - unsupported_platforms: List[CMakePlatform] = dataclasses.field(default_factory=list) - # Platforms to expect this test failed. - xfail_platforms: List[CMakePlatform] = dataclasses.field(default_factory=list) - # Extra flags for `iree-run-module`. - extra_test_flags: List[str] = dataclasses.field(default_factory=list) - - -TEST_CONFIGS = [ - # mobilenet_v1_fp32_correctness_test - ModelTestConfig( - name="mobilenet_v1_fp32_correctness_test", - imported_model=iree_definitions.ImportedModel.from_model( - tflite_models.MOBILENET_V1 - ), - execution_config=module_execution_configs.ELF_LOCAL_SYNC_CONFIG, - expected_output="mobilenet_v1_fp32_expected_output.txt", - unsupported_platforms=[ - CMakePlatform.LINUX_RISCV32, - CMakePlatform.ANDROID_ARMV8_A, - ], - ), - # efficientnet_int8_correctness_test - ModelTestConfig( - name="efficientnet_int8_correctness_test", - imported_model=iree_definitions.ImportedModel.from_model( - tflite_models.EFFICIENTNET_INT8 - ), - execution_config=module_execution_configs.ELF_LOCAL_SYNC_CONFIG, - expected_output="efficientnet_int8_expected_output.txt", - unsupported_platforms=[ - CMakePlatform.ANDROID_ARMV8_A, - CMakePlatform.LINUX_RISCV32, - CMakePlatform.LINUX_RISCV64, - ], - ), - # deeplab_v3_fp32_correctness_test - ModelTestConfig( - name="deeplab_v3_fp32_correctness_test", - imported_model=iree_definitions.ImportedModel.from_model( - tflite_models.DEEPLABV3_FP32 - ), - execution_config=module_execution_configs.ELF_LOCAL_SYNC_CONFIG, - expected_output="https://storage.googleapis.com/iree-model-artifacts/deeplab_v3_fp32_input_0_expected_output.npy", - extra_test_flags=["--expected_f32_threshold=0.001"], - unsupported_platforms=[ - CMakePlatform.LINUX_RISCV32, - CMakePlatform.LINUX_RISCV64, - ], - ), - # person_detect_int8_correctness_test - ModelTestConfig( - name="person_detect_int8_correctness_test", - imported_model=iree_definitions.ImportedModel.from_model( - tflite_models.PERSON_DETECT_INT8 - ), - execution_config=module_execution_configs.ELF_LOCAL_SYNC_CONFIG, - expected_output="1x2xi8=[72 -72]", - unsupported_platforms=[CMakePlatform.ANDROID_ARMV8_A], - ), -] diff --git a/build_tools/python/e2e_test_artifacts/CMakeLists.txt b/build_tools/python/e2e_test_artifacts/CMakeLists.txt deleted file mode 100644 index 17c2ac31cc5c..000000000000 --- a/build_tools/python/e2e_test_artifacts/CMakeLists.txt +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -iree_build_tools_py_test( - NAME - model_artifacts_test - SRC - "model_artifacts_test.py" -) - -iree_build_tools_py_test( - NAME - iree_artifacts_test - SRC - "iree_artifacts_test.py" -) - -iree_build_tools_py_test( - NAME - utils_test - SRC - "utils_test.py" -) - - -add_subdirectory(cmake_generator) diff --git a/build_tools/python/e2e_test_artifacts/__init__.py b/build_tools/python/e2e_test_artifacts/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/build_tools/python/e2e_test_artifacts/cmake_generator/CMakeLists.txt b/build_tools/python/e2e_test_artifacts/cmake_generator/CMakeLists.txt deleted file mode 100644 index 60feee0a3ad3..000000000000 --- a/build_tools/python/e2e_test_artifacts/cmake_generator/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -iree_build_tools_py_test( - NAME - model_rule_generator_test - SRC - "model_rule_generator_test.py" -) - -iree_build_tools_py_test( - NAME - iree_rule_generator_test - SRC - "iree_rule_generator_test.py" -) diff --git a/build_tools/python/e2e_test_artifacts/cmake_generator/__init__.py b/build_tools/python/e2e_test_artifacts/cmake_generator/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py b/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py deleted file mode 100644 index fbf4c5375cfa..000000000000 --- a/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator.py +++ /dev/null @@ -1,248 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Generates CMake rules to build IREE artifacts.""" - -import collections -from dataclasses import dataclass -from typing import Dict, List, Sequence -import pathlib - -from benchmark_suites.iree import benchmark_presets -from e2e_test_artifacts import iree_artifacts, utils -from e2e_test_artifacts.cmake_generator import model_rule_generator -from e2e_test_framework.definitions import iree_definitions -import cmake_builder.rules - -# Imported models for default benchmarks. -BENCHMARK_IMPORT_MODELS_CMAKE_TARGET = "iree-benchmark-import-models" -# Imported models for large benchmarks. -LARGE_BENCHMARK_IMPORT_MODELS_CMAKE_TARGET = "iree-benchmark-import-models-large" -# Prefix of benchmark suite cmake targets. -BENCHMARK_SUITES_CMAKE_TARGET_PREFIX = "iree-benchmark-suites-" - - -@dataclass(frozen=True) -class IreeModelImportRule(object): - target_name: str - output_file_path: pathlib.PurePath - cmake_rules: List[str] - - -@dataclass(frozen=True) -class IreeModuleCompileRule(object): - target_name: str - output_module_path: pathlib.PurePath - cmake_rules: List[str] - - -class IreeRuleBuilder(object): - """Builder to generate IREE CMake rules.""" - - _package_name: str - - def __init__(self, package_name: str): - self._package_name = package_name - - def build_model_import_rule( - self, - source_model_rule: model_rule_generator.ModelRule, - imported_model: iree_definitions.ImportedModel, - output_file_path: pathlib.PurePath, - ) -> IreeModelImportRule: - model = imported_model.model - import_config = imported_model.import_config - if import_config.tool == iree_definitions.ImportTool.NONE: - if source_model_rule.file_path != output_file_path: - raise ValueError( - f"Separate path for MLIR model isn't supported yet: " - f"('{source_model_rule.file_path }' != '{output_file_path}')" - ) - return IreeModelImportRule( - target_name=source_model_rule.target_name, - output_file_path=output_file_path, - cmake_rules=[], - ) - - # Import target name: iree-imported-model- - target_name = utils.get_safe_name(f"iree-imported-model-{imported_model.name}") - - import_flags = import_config.materialize_import_flags(model) - if import_config.tool == iree_definitions.ImportTool.TFLITE_IMPORTER: - cmake_rules = [ - cmake_builder.rules.build_iree_import_tflite_model( - target_path=self.build_target_path(target_name), - source=str(source_model_rule.file_path.as_posix()), - import_flags=import_flags, - output_mlir_file=str(output_file_path.as_posix()), - ) - ] - elif import_config.tool == iree_definitions.ImportTool.TF_IMPORTER: - cmake_rules = [ - cmake_builder.rules.build_iree_import_tf_model( - target_path=self.build_target_path(target_name), - source=str(source_model_rule.file_path.as_posix()), - import_flags=import_flags, - output_mlir_file=str(output_file_path.as_posix()), - ) - ] - else: - raise ValueError( - f"Unsupported import tool '{import_config.tool}' of the model '{model.id}'." - ) - - return IreeModelImportRule( - target_name=target_name, - output_file_path=output_file_path, - cmake_rules=cmake_rules, - ) - - def build_module_compile_rule( - self, - model_import_rule: IreeModelImportRule, - module_generation_config: iree_definitions.ModuleGenerationConfig, - output_file_path: pathlib.PurePath, - ) -> IreeModuleCompileRule: - compile_flags = module_generation_config.materialize_compile_flags( - module_dir_path=output_file_path.parent - ) - - # Module target name: iree-module- - target_name = utils.get_safe_name( - f"iree-module-{module_generation_config.name}" - ) - - cmake_rules = [ - cmake_builder.rules.build_iree_bytecode_module( - target_name=target_name, - src=str(model_import_rule.output_file_path.as_posix()), - module_name=str(output_file_path.as_posix()), - flags=compile_flags, - friendly_name=str(module_generation_config), - ) - ] - - # TODO(#10155): Dump the compile flags from iree_bytecode_module into a flagfile. - - return IreeModuleCompileRule( - target_name=target_name, - output_module_path=output_file_path, - cmake_rules=cmake_rules, - ) - - def build_target_path(self, target_name: str): - """Returns the full target path by combining the package name and the target - name. - """ - return f"{self._package_name}_{target_name}" - - -def generate_rules( - package_name: str, - root_path: pathlib.PurePath, - gen_configs: Sequence[iree_definitions.ModuleGenerationConfig], - run_configs: Sequence[iree_definitions.E2EModelRunConfig], - model_rule_map: Dict[str, model_rule_generator.ModelRule], -) -> List[str]: - """Generates all rules to build IREE artifacts. - - Args: - package_name: CMake package name for rules. - root_path: path of the root artifact directory. - gen_configs: full list of IREE module generation configs of both - compilation and execution benchmarks. - run_configs: full list of IREE E2E model run configs to calculate the - artifact dependencies of exectuion benchmarks. - model_rule_map: map of generated model rules keyed by model id, it must - cover all model referenced in gen_configs. - Returns: - List of cmake rules. - """ - - rule_builder = IreeRuleBuilder(package_name=package_name) - - all_imported_models = dict( - (config.imported_model.composite_id, config.imported_model) - for config in gen_configs - ) - - cmake_rules = [] - model_import_rule_map = {} - for imported_model_id, imported_model in all_imported_models.items(): - model_rule = model_rule_map.get(imported_model.model.id) - if model_rule is None: - raise ValueError(f"Model rule not found for {imported_model.model.id}.") - - imported_model_path = iree_artifacts.get_imported_model_path( - imported_model=imported_model, root_path=root_path - ) - model_import_rule = rule_builder.build_model_import_rule( - source_model_rule=model_rule, - imported_model=imported_model, - output_file_path=imported_model_path, - ) - model_import_rule_map[imported_model_id] = model_import_rule - cmake_rules.extend(model_import_rule.cmake_rules) - - gen_config_to_presets = collections.defaultdict(set) - for gen_config in gen_configs: - gen_config_to_presets[gen_config.composite_id].update(gen_config.presets) - # Include the presets from dependent run configs, so they are built for - # execution benchmark presets. - for run_config in run_configs: - gen_config = run_config.module_generation_config - gen_config_to_presets[gen_config.composite_id].update(run_config.presets) - - suites_to_deps = collections.defaultdict(set) - for gen_config in gen_configs: - model_import_rule = model_import_rule_map[ - gen_config.imported_model.composite_id - ] - module_dir_path = iree_artifacts.get_module_dir_path( - module_generation_config=gen_config, root_path=root_path - ) - module_compile_rule = rule_builder.build_module_compile_rule( - model_import_rule=model_import_rule, - module_generation_config=gen_config, - output_file_path=module_dir_path / iree_artifacts.MODULE_FILENAME, - ) - cmake_rules.extend(module_compile_rule.cmake_rules) - - presets = gen_config_to_presets[gen_config.composite_id] - # A benchmark can be in default and large presets at the same time. For - # example, batch-1 benchmark is added to default for sanity check. So - # check both cases. - if presets.intersection(benchmark_presets.DEFAULT_PRESETS): - presets.add("default") - if presets.intersection(benchmark_presets.LARGE_PRESETS): - presets.add("large") - - for preset in presets: - preset_target = f"{BENCHMARK_SUITES_CMAKE_TARGET_PREFIX}{preset}" - suites_to_deps[preset_target].add(module_compile_rule.target_name) - - import_target = model_import_rule.target_name - if "default" in presets: - suites_to_deps[BENCHMARK_IMPORT_MODELS_CMAKE_TARGET].add(import_target) - if "large" in presets: - suites_to_deps[LARGE_BENCHMARK_IMPORT_MODELS_CMAKE_TARGET].add( - import_target - ) - - # The dict suites_to_deps is inserted in a quite arbitrary order. Sort it by - # the key first. - for suite_target in sorted(suites_to_deps.keys()): - target_names = suites_to_deps[suite_target] - cmake_rules.append( - cmake_builder.rules.build_add_dependencies( - target=suite_target, - deps=[ - rule_builder.build_target_path(target_name) - for target_name in sorted(target_names) - ], - ) - ) - - return cmake_rules diff --git a/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator_test.py b/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator_test.py deleted file mode 100644 index 3353ebaa167f..000000000000 --- a/build_tools/python/e2e_test_artifacts/cmake_generator/iree_rule_generator_test.py +++ /dev/null @@ -1,242 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import pathlib -import unittest - -from e2e_test_artifacts import utils -from e2e_test_artifacts.cmake_generator import model_rule_generator, iree_rule_generator -from e2e_test_framework.definitions import common_definitions, iree_definitions - - -class IreeRuleBuilderTest(unittest.TestCase): - def setUp(self): - self._builder = iree_rule_generator.IreeRuleBuilder(package_name="${package}") - - def test_build_model_import_rule_tflite(self): - tflite_model = common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32"], - ) - tflite_imported_model = iree_definitions.ImportedModel.from_model(tflite_model) - model_rule = model_rule_generator.ModelRule( - target_name="model-1234", - file_path=pathlib.PurePath("root/models/x.tflite"), - cmake_rules=["abc"], - ) - output_file_path = pathlib.PurePath( - "root", "iree", tflite_model.id, f"{tflite_model.name}.mlir" - ) - - rule = self._builder.build_model_import_rule( - source_model_rule=model_rule, - imported_model=tflite_imported_model, - output_file_path=output_file_path, - ) - - self.assertEqual( - rule.target_name, - utils.get_safe_name(f"iree-imported-model-{tflite_imported_model.name}"), - ) - self.assertEqual(rule.output_file_path, output_file_path) - - def test_build_model_import_rule_linalg(self): - linalg_model = common_definitions.Model( - id="9012", - name="linalg_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://example.com/xyz.mlir", - entry_function="main", - input_types=["3xf32"], - ) - linalg_imported_model = iree_definitions.ImportedModel.from_model(linalg_model) - model_rule = model_rule_generator.ModelRule( - target_name="model-5678", - file_path=pathlib.PurePath("root/models/y.mlir"), - cmake_rules=["abc"], - ) - - rule = self._builder.build_model_import_rule( - source_model_rule=model_rule, - imported_model=linalg_imported_model, - output_file_path=pathlib.PurePath(model_rule.file_path), - ) - - self.assertEqual(rule.target_name, model_rule.target_name) - self.assertEqual( - pathlib.PurePath(rule.output_file_path), - pathlib.PurePath(model_rule.file_path), - ) - - def test_build_module_compile_rule(self): - model = common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32"], - ) - imported_model = iree_definitions.ImportedModel.from_model(model) - compile_config = iree_definitions.CompileConfig.build( - id="config_a", - tags=["defaults"], - compile_targets=[ - iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - ], - ) - gen_config = iree_definitions.ModuleGenerationConfig.build( - imported_model=imported_model, compile_config=compile_config - ) - model_import_rule = iree_rule_generator.IreeModelImportRule( - target_name=f"iree-import-model-abcd", - output_file_path=pathlib.PurePath("root/iree/abcd/1234.mlir"), - cmake_rules=["abc"], - ) - output_file_path = pathlib.PurePath("root/iree/test_output") - - rule = self._builder.build_module_compile_rule( - model_import_rule=model_import_rule, - module_generation_config=gen_config, - output_file_path=output_file_path, - ) - - self.assertEqual( - rule.target_name, utils.get_safe_name(f"iree-module-{gen_config.name}") - ) - self.assertEqual(rule.output_module_path, output_file_path) - - def test_build_target_path(self): - builder = iree_rule_generator.IreeRuleBuilder(package_name="xyz") - - path = builder.build_target_path("target-abc") - - self.assertEqual(path, f"xyz_target-abc") - - -class IreeGeneratorTest(unittest.TestCase): - def test_generate_rules(self): - model_a = common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32"], - ) - model_b = common_definitions.Model( - id="5678", - name="stablehlo_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url="https://example.com/xyz_stablehlo.mlir", - entry_function="predict", - input_types=["2xf32"], - ) - imported_model_a = iree_definitions.ImportedModel.from_model(model_a) - imported_model_b = iree_definitions.ImportedModel.from_model(model_b) - compile_config_a = iree_definitions.CompileConfig.build( - id="config_a", - tags=["defaults"], - compile_targets=[ - iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - ], - ) - compile_config_b = iree_definitions.CompileConfig.build( - id="config_b", - tags=["defaults"], - compile_targets=[ - iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - ], - ) - gen_config_a = iree_definitions.ModuleGenerationConfig.build( - imported_model=imported_model_a, compile_config=compile_config_a - ) - gen_config_b = iree_definitions.ModuleGenerationConfig.build( - imported_model=imported_model_b, compile_config=compile_config_a - ) - gen_config_c = iree_definitions.ModuleGenerationConfig.build( - imported_model=imported_model_b, compile_config=compile_config_b - ) - run_config_a = iree_definitions.E2EModelRunConfig.build( - module_generation_config=gen_config_a, - module_execution_config=iree_definitions.ModuleExecutionConfig.build( - id="exec_a", - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_SYNC, - ), - target_device_spec=common_definitions.DeviceSpec.build( - id="dev_a", - device_name="dev_a", - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - ), - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - presets=["test"], - ) - model_rule_map = { - model_a.id: model_rule_generator.ModelRule( - target_name="model-x", - file_path=pathlib.PurePath("x.tflite"), - cmake_rules=["abc"], - ), - model_b.id: model_rule_generator.ModelRule( - target_name=f"model-y", - file_path=pathlib.PurePath("root/model_stablehlo_m.mlir"), - cmake_rules=["efg"], - ), - } - - cmake_rules = iree_rule_generator.generate_rules( - package_name="${package}", - root_path=pathlib.PurePath("root"), - gen_configs=[gen_config_a, gen_config_b, gen_config_c], - run_configs=[run_config_a], - model_rule_map=model_rule_map, - ) - - concated_cmake_rules = "\n".join(cmake_rules) - self.assertRegex( - concated_cmake_rules, - utils.get_safe_name(f"iree-imported-model-{imported_model_a.name}"), - ) - self.assertRegex( - concated_cmake_rules, - utils.get_safe_name(f"iree-module-{gen_config_a.name}"), - ) - self.assertRegex( - concated_cmake_rules, - utils.get_safe_name(f"iree-module-{gen_config_b.name}"), - ) - self.assertRegex( - concated_cmake_rules, - utils.get_safe_name(f"iree-module-{gen_config_c.name}"), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/python/e2e_test_artifacts/cmake_generator/model_rule_generator.py b/build_tools/python/e2e_test_artifacts/cmake_generator/model_rule_generator.py deleted file mode 100644 index e121e094e641..000000000000 --- a/build_tools/python/e2e_test_artifacts/cmake_generator/model_rule_generator.py +++ /dev/null @@ -1,53 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Generates CMake rules to fetch model artifacts.""" - -from dataclasses import dataclass -from typing import Dict, Iterable, List -import pathlib -import urllib.parse - -from e2e_test_artifacts import model_artifacts, utils -from e2e_test_framework.definitions import common_definitions -import cmake_builder.rules - - -@dataclass -class ModelRule(object): - target_name: str - file_path: pathlib.PurePath - cmake_rules: List[str] - - -def generate_model_rule_map( - root_path: pathlib.PurePath, models: Iterable[common_definitions.Model] -) -> Dict[str, ModelRule]: - """Returns the model rules keyed by model id in an ordered map.""" - - model_rules = {} - for model in models: - # Model target: -model- - target_name = utils.get_safe_name(f"model-{model.name}") - model_path = model_artifacts.get_model_path(model=model, root_path=root_path) - - model_url = urllib.parse.urlparse(model.source_url) - if model_url.scheme == "https": - cmake_rules = [ - cmake_builder.rules.build_iree_fetch_artifact( - target_name=target_name, - source_url=model.source_url, - output=str(model_path.as_posix()), - unpack=True, - ) - ] - else: - raise ValueError("Unsupported model url: {model.source_url}.") - - model_rules[model.id] = ModelRule( - target_name=target_name, file_path=model_path, cmake_rules=cmake_rules - ) - - return model_rules diff --git a/build_tools/python/e2e_test_artifacts/cmake_generator/model_rule_generator_test.py b/build_tools/python/e2e_test_artifacts/cmake_generator/model_rule_generator_test.py deleted file mode 100644 index efc39c1b5b51..000000000000 --- a/build_tools/python/e2e_test_artifacts/cmake_generator/model_rule_generator_test.py +++ /dev/null @@ -1,61 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import pathlib -import unittest - -from e2e_test_artifacts import model_artifacts, utils -from e2e_test_artifacts.cmake_generator import model_rule_generator -from e2e_test_framework.definitions import common_definitions - - -class CommonGeneratorsTest(unittest.TestCase): - def test_generate_model_rule_map(self): - model_a = common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32"], - ) - model_b = common_definitions.Model( - id="5678", - name="tf_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url="https://example.com/xyz_mlir", - entry_function="predict", - input_types=["2xf32"], - ) - root_path = pathlib.PurePath("model_root") - - rule_map = model_rule_generator.generate_model_rule_map( - root_path=root_path, models=[model_a, model_b] - ) - - self.assertEqual(list(rule_map.keys()), [model_a.id, model_b.id]) - self.assertEqual( - rule_map[model_a.id].target_name, - utils.get_safe_name(f"model-{model_a.name}"), - ) - self.assertEqual( - rule_map[model_a.id].file_path, - model_artifacts.get_model_path(model=model_a, root_path=root_path), - ) - self.assertEqual( - rule_map[model_b.id].target_name, - utils.get_safe_name(f"model-{model_b.name}"), - ) - self.assertEqual( - rule_map[model_b.id].file_path, - model_artifacts.get_model_path(model=model_b, root_path=root_path), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/python/e2e_test_artifacts/iree_artifacts.py b/build_tools/python/e2e_test_artifacts/iree_artifacts.py deleted file mode 100644 index 7c7f4551f32f..000000000000 --- a/build_tools/python/e2e_test_artifacts/iree_artifacts.py +++ /dev/null @@ -1,74 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Helpers that generate paths for IREE artifacts.""" - -from typing import Dict, Iterable -import pathlib - -from e2e_test_artifacts import model_artifacts, utils -from e2e_test_framework.definitions import common_definitions, iree_definitions - -IREE_ARTIFACT_PREFIX = "iree" -MODULE_FILENAME = "module.vmfb" -SCHEDULING_STATS_FILENAME = "scheduling_stats.json" - - -def get_imported_model_path( - imported_model: iree_definitions.ImportedModel, - root_path: pathlib.PurePath = pathlib.PurePath(), -) -> pathlib.PurePath: - """Returns the path of an IREE imported MLIR model. If the source model is - in MLIR format, returns the path of source model. - - Args: - imported_model: IREE model importing config. - root_path: path of the root artifact directory, on which the returned path - will base. - Returns: - Path of the imported model file. - """ - model = imported_model.model - if model.source_type in [ - common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - ]: - # Uses the MLIR model directly. - return model_artifacts.get_model_path(model=model, root_path=root_path) - - # Imported model path: /_.mlir - filename = utils.get_safe_name(f"{IREE_ARTIFACT_PREFIX}_{imported_model.name}.mlir") - return root_path / filename - - -def get_module_dir_path( - module_generation_config: iree_definitions.ModuleGenerationConfig, - root_path: pathlib.PurePath = pathlib.PurePath(), -) -> pathlib.PurePath: - """Returns the path of an IREE module directory, which contains the compiled - module and related flag files. - - Args: - module_generation_config: IREE module generation config. - root_path: path of the root artifact directory, on which the returned path - will base. - Returns: - Path of the module directory. - """ - # Module dir path: /_module_ - filename = utils.get_safe_name( - f"{IREE_ARTIFACT_PREFIX}_module_{module_generation_config.name}" - ) - return root_path / filename - - -def get_dependent_model_map( - module_generation_configs: Iterable[iree_definitions.ModuleGenerationConfig], -) -> Dict[str, common_definitions.Model]: - """Returns an ordered map of the dependent models keyed by model id.""" - return dict( - (config.imported_model.model.id, config.imported_model.model) - for config in module_generation_configs - ) diff --git a/build_tools/python/e2e_test_artifacts/iree_artifacts_test.py b/build_tools/python/e2e_test_artifacts/iree_artifacts_test.py deleted file mode 100644 index 14709fea777c..000000000000 --- a/build_tools/python/e2e_test_artifacts/iree_artifacts_test.py +++ /dev/null @@ -1,161 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import pathlib -import unittest - -from e2e_test_framework.definitions import common_definitions, iree_definitions -from e2e_test_artifacts import model_artifacts, iree_artifacts, utils - - -class IreeArtifactsTest(unittest.TestCase): - def test_get_imported_model_path(self): - model = common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32"], - ) - imported_model = iree_definitions.ImportedModel.from_model(model) - root_path = pathlib.PurePath("root") - - path = iree_artifacts.get_imported_model_path( - imported_model=imported_model, root_path=root_path - ) - - filename = utils.get_safe_name( - f"{iree_artifacts.IREE_ARTIFACT_PREFIX}_{imported_model.name}.mlir" - ) - self.assertEqual( - path, - root_path / filename, - ) - - def test_get_imported_model_path_with_mlir_model(self): - model = common_definitions.Model( - id="9012", - name="linalg_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://example.com/xyz.mlir", - entry_function="main", - input_types=["3xf32"], - ) - imported_model = iree_definitions.ImportedModel.from_model(model) - root_path = pathlib.PurePath("root") - - path = iree_artifacts.get_imported_model_path( - imported_model=imported_model, root_path=root_path - ) - - self.assertEqual( - path, model_artifacts.get_model_path(model=model, root_path=root_path) - ) - - def test_get_module_dir_path(self): - model = common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32"], - ) - imported_model = iree_definitions.ImportedModel.from_model(model) - compile_config = iree_definitions.CompileConfig.build( - id="config_a", - tags=["defaults"], - compile_targets=[ - iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - ], - ) - gen_config = iree_definitions.ModuleGenerationConfig.build( - imported_model=imported_model, compile_config=compile_config - ) - root_path = pathlib.PurePath("root") - - path = iree_artifacts.get_module_dir_path( - module_generation_config=gen_config, root_path=root_path - ) - - filename = utils.get_safe_name( - f"{iree_artifacts.IREE_ARTIFACT_PREFIX}_module_{gen_config.name}" - ) - self.assertEqual( - path, - root_path / filename, - ) - - def test_get_dependent_model_map(self): - model_a = common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32"], - ) - model_b = common_definitions.Model( - id="9012", - name="linalg_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://example.com/xyz.mlir", - entry_function="main", - input_types=["3xf32"], - ) - imported_model_a = iree_definitions.ImportedModel.from_model(model_a) - imported_model_b = iree_definitions.ImportedModel.from_model(model_b) - compile_config_a = iree_definitions.CompileConfig.build( - id="config_a", - tags=["defaults"], - compile_targets=[ - iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - ], - ) - compile_config_b = iree_definitions.CompileConfig.build( - id="config_b", - tags=["defaults"], - compile_targets=[ - iree_definitions.CompileTarget( - target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - ], - ) - gen_config_a = iree_definitions.ModuleGenerationConfig.build( - imported_model=imported_model_a, compile_config=compile_config_a - ) - gen_config_b = iree_definitions.ModuleGenerationConfig.build( - imported_model=imported_model_b, compile_config=compile_config_a - ) - gen_config_c = iree_definitions.ModuleGenerationConfig.build( - imported_model=imported_model_b, compile_config=compile_config_b - ) - - models = iree_artifacts.get_dependent_model_map( - module_generation_configs=[gen_config_a, gen_config_b, gen_config_c] - ) - - self.assertEqual(models, {model_a.id: model_a, model_b.id: model_b}) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/python/e2e_test_artifacts/model_artifacts.py b/build_tools/python/e2e_test_artifacts/model_artifacts.py deleted file mode 100644 index 3fc705e111c8..000000000000 --- a/build_tools/python/e2e_test_artifacts/model_artifacts.py +++ /dev/null @@ -1,40 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Helpers that generate paths for model artifacts.""" - -import pathlib -import urllib.parse - -from e2e_test_artifacts import utils -from e2e_test_framework.definitions import common_definitions - -MODEL_ARTIFACT_PREFIX = "model" -# Archive extensions used to pack models. -ARCHIVE_FILE_EXTENSIONS = [".tar", ".gz"] - - -def get_model_path( - model: common_definitions.Model, root_path: pathlib.PurePath = pathlib.PurePath() -) -> pathlib.PurePath: - """Returns the path of an model artifact file or directory. - - Args: - model: source model. - root_path: path of the root artifact directory, on which the returned path - will base. - Returns: - Path of the model artifact. - """ - model_url = urllib.parse.urlparse(model.source_url) - # Drop the archive extensions. - file_exts = pathlib.PurePath(model_url.path).suffixes - while len(file_exts) > 0 and file_exts[-1] in ARCHIVE_FILE_EXTENSIONS: - file_exts.pop() - model_ext = "".join(file_exts) - - # Model path: /_ - filename = utils.get_safe_name(model.name + model_ext) - return root_path / f"{MODEL_ARTIFACT_PREFIX}_{filename}" diff --git a/build_tools/python/e2e_test_artifacts/model_artifacts_test.py b/build_tools/python/e2e_test_artifacts/model_artifacts_test.py deleted file mode 100644 index affea06cc4a8..000000000000 --- a/build_tools/python/e2e_test_artifacts/model_artifacts_test.py +++ /dev/null @@ -1,61 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import pathlib -import unittest - -from e2e_test_artifacts import model_artifacts, utils -from e2e_test_framework.definitions import common_definitions - - -class ModelArtifactsTest(unittest.TestCase): - def test_get_model_path_with_tflite_model(self): - tflite_model = common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32"], - ) - root_path = pathlib.PurePath("root") - - path = model_artifacts.get_model_path(model=tflite_model, root_path=root_path) - - filename = utils.get_safe_name( - f"{model_artifacts.MODEL_ARTIFACT_PREFIX}_{tflite_model.name}.tflite" - ) - self.assertEqual( - path, - root_path / filename, - ) - - def test_get_model_path_with_tf_model(self): - tf_model = common_definitions.Model( - id="5678", - name="tf_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url="https://example.com/xyz_stablehlo.mlir", - entry_function="predict", - input_types=["2xf32"], - ) - root_path = pathlib.PurePath("root") - - path = model_artifacts.get_model_path(model=tf_model, root_path=root_path) - - filename = utils.get_safe_name( - f"{model_artifacts.MODEL_ARTIFACT_PREFIX}_{tf_model.name}.mlir" - ) - self.assertEqual( - path, - root_path / filename, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/python/e2e_test_artifacts/utils.py b/build_tools/python/e2e_test_artifacts/utils.py deleted file mode 100644 index 274edb58fd55..000000000000 --- a/build_tools/python/e2e_test_artifacts/utils.py +++ /dev/null @@ -1,19 +0,0 @@ -## Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Utilities that help generate artifacts.""" - -import re - -# The characters unsafe for CMake and file systems. It's the negation of the -# allowed list, which is derived from the join of -# https://cmake.org/cmake/help/v3.27/policy/CMP0037.html and -# https://www.mtu.edu/umc/services/websites/writing/characters-avoid/. -UNSAFE_CHARACTERS = re.compile(r"[^0-9a-zA-Z\-_.]") - - -def get_safe_name(name: str) -> str: - """Replace unsafe characters for CMake and file systems with `_`.""" - return UNSAFE_CHARACTERS.sub("_", name) diff --git a/build_tools/python/e2e_test_artifacts/utils_test.py b/build_tools/python/e2e_test_artifacts/utils_test.py deleted file mode 100644 index cb3a1ca5773f..000000000000 --- a/build_tools/python/e2e_test_artifacts/utils_test.py +++ /dev/null @@ -1,27 +0,0 @@ -## Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import unittest - -from e2e_test_artifacts import utils - - -class UtilsTest(unittest.TestCase): - def test_get_safe_name_with_disallowed_characters(self): - sanitized_name = utils.get_safe_name("test(abc) [x,y,z]") - - self.assertEqual(sanitized_name, "test_abc___x_y_z_") - - def test_get_safe_name_with_all_allowed_characters(self): - safe_name = "123_AB-C.test" - - sanitized_name = utils.get_safe_name(safe_name) - - self.assertEqual(sanitized_name, safe_name) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/python/e2e_test_framework/CMakeLists.txt b/build_tools/python/e2e_test_framework/CMakeLists.txt deleted file mode 100644 index 54a1a950f997..000000000000 --- a/build_tools/python/e2e_test_framework/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -iree_build_tools_py_test( - NAME - serialization_test - SRC - "serialization_test.py" -) - -iree_build_tools_py_test( - NAME - unique_ids_test - SRC - "unique_ids_test.py" -) - -iree_add_all_subdirs() diff --git a/build_tools/python/e2e_test_framework/__init__.py b/build_tools/python/e2e_test_framework/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/build_tools/python/e2e_test_framework/definitions/CMakeLists.txt b/build_tools/python/e2e_test_framework/definitions/CMakeLists.txt deleted file mode 100644 index 1ff547ce6373..000000000000 --- a/build_tools/python/e2e_test_framework/definitions/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -iree_build_tools_py_test( - NAME - iree_definitions_test - SRC - "iree_definitions_test.py" -) - -iree_build_tools_py_test( - NAME - utils_test - SRC - "utils_test.py" -) diff --git a/build_tools/python/e2e_test_framework/definitions/__init__.py b/build_tools/python/e2e_test_framework/definitions/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/build_tools/python/e2e_test_framework/definitions/common_definitions.py b/build_tools/python/e2e_test_framework/definitions/common_definitions.py deleted file mode 100644 index c862dbde90f5..000000000000 --- a/build_tools/python/e2e_test_framework/definitions/common_definitions.py +++ /dev/null @@ -1,242 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Common classes for benchmark definitions.""" - -import dataclasses -from dataclasses import dataclass -from enum import Enum -from typing import List, Optional, Sequence - -from e2e_test_framework import serialization, unique_ids - - -class ArchitectureType(Enum): - """Type of architecture.""" - - CPU = "cpu" - GPU = "gpu" - - -@dataclass(frozen=True) -class _ArchitectureInfo(object): - """Architecture information.""" - - type: ArchitectureType - architecture: str - microarchitecture: str = "" - vendor: str = "" - - -class DeviceArchitecture(_ArchitectureInfo, Enum): - """Predefined architecture/microarchitecture.""" - - # VMVX virtual machine - VMVX_GENERIC = (ArchitectureType.CPU, "vmvx", "generic") - - # x86_64 CPUs - X86_64_CASCADELAKE = (ArchitectureType.CPU, "x86_64", "cascadelake") - - # ARM CPUs - ARMV8_2_A_GENERIC = (ArchitectureType.CPU, "armv8.2-a", "generic") - ARMV9_A_GENERIC = (ArchitectureType.CPU, "armv9-a", "generic") - - # RISC-V CPUs - RV64_GENERIC = (ArchitectureType.CPU, "riscv_64", "generic") - RV32_GENERIC = (ArchitectureType.CPU, "riscv_32", "generic") - - # Vulkan GPUs - QUALCOMM_ADRENO = (ArchitectureType.GPU, "adreno", "", "qualcomm") - ARM_VALHALL = (ArchitectureType.GPU, "valhall", "", "arm") - NVIDIA_AMPERE = (ArchitectureType.GPU, "ampere", "", "nvidia") - NVIDIA_PASCAL = (ArchitectureType.GPU, "pascal", "", "nvidia") - - # CUDA GPUs - CUDA_SM70 = (ArchitectureType.GPU, "cuda", "sm_70") - CUDA_SM80 = (ArchitectureType.GPU, "cuda", "sm_80") - - # Starting from 3.11, enum members are defined before the subclasses (don't - # follow MRO, see https://docs.python.org/3/whatsnew/3.11.html#enum). - # Therefore __str__ is defined here instead of in _ArchitectureInfo to - # override the default one. - def __str__(self): - parts = [self.vendor, self.architecture, self.microarchitecture] - return "-".join(part for part in parts if part != "") - - -@dataclass(frozen=True) -class _HostEnvironmentInfo(object): - """Environment information of a host. - - The definitions and terms here matches the macros in - `runtime/src/iree/base/target_platform.h`. - - Note that this is the environment where the runtime "runs". For example: - ``` - { - "platform": "linux", - "architecture": "x86_64" - } - ``` - means the runtime will run on a Linux x86_64 host. The runtime might dispatch - the workloads on GPU or it can be a VM to run workloads compiled in another - ISA, but those are irrelevant to the information here. - """ - - platform: str - architecture: str - - -class HostEnvironment(_HostEnvironmentInfo, Enum): - """Predefined host environment.""" - - LINUX_X86_64 = ("linux", "x86_64") - LINUX_RISCV_64 = ("linux", "riscv_64") - LINUX_RISCV_32 = ("linux", "riscv_32") - ANDROID_ARMV8_2_A = ("android", "armv8.2-a") - - -class ModelSourceType(Enum): - """Type of model source.""" - - # Exported Linalg MLIR file. - EXPORTED_LINALG_MLIR = "exported_linalg_mlir" - # Exported Stable HLO file. - EXPORTED_STABLEHLO_MLIR = "exported_stablehlo_mlir" - # Exported TFLite model file. - EXPORTED_TFLITE = "exported_tflite" - - -@serialization.serializable -@dataclass(frozen=True) -class CPUParameters: - """Describes CPU related parameters.""" - - # CPU cores to pin at, ordered from the slowest to the fastest. - pinned_cores: List[int] - - -@serialization.serializable -@dataclass(frozen=True) -class DeviceParameters: - """Describes device parameters.""" - - cpu_params: Optional[CPUParameters] = None - - -@serialization.serializable(type_key="device_specs") -@dataclass(frozen=True) -class DeviceSpec(object): - """Benchmark device specification.""" - - id: str - - # Unique name of the device spec. - name: str - - # Device name. E.g., Pixel-6. - device_name: str - - # Tags to describe the device spec. - tags: List[str] - - # Host environment where the IREE runtime is running. For CPU device type, - # this is usually the same as the device that workloads are dispatched to. - # With a separate device, such as a GPU, however, the runtime and dispatched - # workloads will run on different platforms. - host_environment: HostEnvironment - - # Architecture of the target device. - architecture: DeviceArchitecture - - # Device-specific parameters. E.g., 2-big-cores, 4-little-cores. - # This is for modeling the spec of a heterogeneous processor. Depending on - # which cores you run, the device has a different spec. Benchmark machines use - # these parameters to set up the devices. E.g. set CPU mask. - device_parameters: DeviceParameters - - def __str__(self): - return self.name - - @classmethod - def build( - cls, - id: str, - device_name: str, - host_environment: HostEnvironment, - architecture: DeviceArchitecture, - device_parameters: DeviceParameters = DeviceParameters(), - tags: Sequence[str] = (), - ): - tag_part = ",".join(tags) - # Format: [,...] - name = f"{device_name}[{tag_part}]" - return cls( - id=id, - name=name, - tags=list(tags), - device_name=device_name, - host_environment=host_environment, - architecture=architecture, - device_parameters=device_parameters, - ) - - -@serialization.serializable(type_key="models") -@dataclass(frozen=True) -class Model(object): - """Model to be benchmarked.""" - - id: str - # Friendly unique name. - name: str - # Tags that describe the model characteristics. - tags: List[str] - source_type: ModelSourceType - source_url: str - entry_function: str - # Input types. E.g., ["100x100xf32", "200x200x5xf32"]. - input_types: List[str] - # URL to fetch input data tgz. The archive should contain - # "input_{0,1,...}.npy" for each input. - input_url: Optional[str] = None - # URL to fetch expected output tgz. The archive should contain - # "ouptut_0.npy". - expected_output_url: Optional[str] = None - # Parameters for iree-run-module to control the tolerance. - # For example: --expected_f32_threshold=0.0001 - verify_params: List[str] = dataclasses.field(default_factory=list) - # Model parameter files (/docs/website/docs/guides/parameters.md). - # Format: `scope=parameter file url` (scoped) or `=parameter file url` (unscoped) - external_param_urls: List[str] = dataclasses.field(default_factory=list) - - def __str__(self): - return self.name - - -@serialization.serializable(type_key="model_input_data") -@dataclass(frozen=True) -class ModelInputData(object): - """Input data to benchmark the model.""" - - id: str - # Associated model. - name: str - - def __str__(self): - return self.name - - -# Get input from model input_url if available; otherwise use all zeros. -DEFAULT_INPUT_DATA = ModelInputData( - id=unique_ids.MODEL_INPUT_DATA_ZEROS, - name="default", -) - - -@dataclass(frozen=True) -class CpuBenchmarkConfig(object): - model: Model - threads: List[int] diff --git a/build_tools/python/e2e_test_framework/definitions/iree_definitions.py b/build_tools/python/e2e_test_framework/definitions/iree_definitions.py deleted file mode 100644 index ebf318c19377..000000000000 --- a/build_tools/python/e2e_test_framework/definitions/iree_definitions.py +++ /dev/null @@ -1,519 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Classes for IREE compilation and run definitions.""" - -import dataclasses -from dataclasses import dataclass -from enum import Enum -import pathlib -from typing import List, Optional, Sequence - -from e2e_test_framework.definitions import common_definitions, utils -from e2e_test_framework import serialization, unique_ids - - -class TargetBackend(Enum): - """IREE target backend.""" - - LLVM_CPU = "llvm-cpu" - CUDA = "cuda" - ROCM = "rocm" - VMVX = "vmvx" - METAL_SPIRV = "metal-spirv" - VULKAN_SPIRV = "vulkan-spirv" - - -class TargetABI(Enum): - VMVX = "vmvx" - LINUX_GNU = "linux-gnu" - LINUX_ANDROID29 = "linux-android29" - # IREE defined OS name for vulkan target. See: - # compiler/src/iree/compiler/Dialect/Vulkan/IR/VulkanBase.td - VULKAN_ANDROID30 = "android30" - VULKAN_ANDROID31 = "android31" - VULKAN_LINUX = "linux" - - -class RuntimeLoader(Enum): - """IREE runtime loader.""" - - # For target that doesn't support loader configuration. - NONE = "none" - EMBEDDED_ELF = "embedded-elf" - VMVX_MODULE = "vmvx-module" - SYSTEM_LIBRARY = "system-library" - - -class RuntimeDriver(Enum): - """IREE runtime driver.""" - - LOCAL_SYNC = "local-sync" - LOCAL_TASK = "local-task" - CUDA = "cuda" - VULKAN = "vulkan" - - -@serialization.serializable -@dataclass(frozen=True) -class CompileTarget(object): - """Describes a target device to build for.""" - - target_backend: TargetBackend - target_architecture: common_definitions.DeviceArchitecture - target_abi: TargetABI - - def __str__(self): - return ( - f"{self.target_architecture}-" - f"{self.target_abi.name}-" - f"{self.target_backend.name}" - ).lower() - - -@serialization.serializable(type_key="iree_compile_configs") -@dataclass(frozen=True) -class CompileConfig(object): - """Describes the options to build a module.""" - - id: str - name: str - tags: List[str] - compile_targets: List[CompileTarget] - extra_flags: List[str] = dataclasses.field(default_factory=list) - - def __str__(self): - return self.name - - @classmethod - def build( - cls, - id: str, - compile_targets: Sequence[CompileTarget], - extra_flags: Optional[Sequence[str]] = None, - tags: Sequence[str] = (), - ): - target_part = ",".join(str(target) for target in compile_targets) - tag_part = ",".join(tags) - # Format: [,...][,...] - name = f"[{target_part}][{tag_part}]" - extra_flags = extra_flags or [] - return cls( - id=id, - name=name, - tags=list(tags), - compile_targets=list(compile_targets), - extra_flags=list(extra_flags), - ) - - -@serialization.serializable(type_key="iree_module_execution_configs") -@dataclass(frozen=True) -class ModuleExecutionConfig(object): - """Describes the options to run a module.""" - - id: str - name: str - tags: List[str] - loader: RuntimeLoader - driver: RuntimeDriver - extra_flags: List[str] = dataclasses.field(default_factory=list) - - def __str__(self): - return self.name - - @classmethod - def build( - cls, - id: str, - loader: RuntimeLoader, - driver: RuntimeDriver, - extra_flags: Sequence[str] = (), - tags: Sequence[str] = (), - ): - runtime_part = f"{driver.name}({loader.name})".lower() - tag_part = ",".join(tags) - # Format: ()[,...] - name = f"{runtime_part}[{tag_part}]" - return cls( - id=id, - name=name, - tags=list(tags), - loader=loader, - driver=driver, - extra_flags=list(extra_flags), - ) - - -class ImportTool(Enum): - """Iree model import tool.""" - - NONE = "none" - TF_IMPORTER = "iree-import-tf" - TFLITE_IMPORTER = "iree-import-tflite" - - -# Value should be the name of an IREE supported input type (--iree-input-type). -class MLIRDialectType(Enum): - """Imported MLIR dialect type.""" - - NONE = "none" - TOSA = "tosa" - STABLEHLO = "stablehlo" - - -@serialization.serializable(type_key="iree_import_configs") -@dataclass(frozen=True) -class ImportConfig(object): - """Config to import the model.""" - - id: str - name: str - tool: ImportTool - dialect_type: MLIRDialectType - import_flags: List[str] = dataclasses.field(default_factory=list) - - def __str__(self): - return self.name - - def materialize_import_flags(self, model: common_definitions.Model) -> List[str]: - """Materialize flags with dependent values.""" - return utils.substitute_flag_vars( - flags=self.import_flags, ENTRY_FUNCTION=model.entry_function - ) - - -DEFAULT_TFLITE_IMPORT_CONFIG = ImportConfig( - id=unique_ids.IREE_MODEL_IMPORT_TFLITE_DEFAULT, - name="tflite", - tool=ImportTool.TFLITE_IMPORTER, - dialect_type=MLIRDialectType.TOSA, -) - -DEFAULT_LINALG_MLIR_IMPORT_CONFIG = ImportConfig( - id=unique_ids.IREE_MODEL_IMPORT_LINALG_MLIR_DEFAULT, - name="linalg", - tool=ImportTool.NONE, - dialect_type=MLIRDialectType.NONE, -) - -DEFAULT_STABLEHLO_MLIR_IMPORT_CONFIG = ImportConfig( - id=unique_ids.IREE_MODEL_IMPORT_STABLEHLO_MLIR_DEFAULT, - name="stablehlo", - tool=ImportTool.NONE, - dialect_type=MLIRDialectType.STABLEHLO, -) - -MODEL_SOURCE_TO_DEFAULT_IMPORT_CONFIG_MAP = { - common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR: DEFAULT_LINALG_MLIR_IMPORT_CONFIG, - common_definitions.ModelSourceType.EXPORTED_TFLITE: DEFAULT_TFLITE_IMPORT_CONFIG, - common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR: DEFAULT_STABLEHLO_MLIR_IMPORT_CONFIG, -} - - -@serialization.serializable(type_key="iree_imported_models", id_field="composite_id") -@dataclass(frozen=True) -class ImportedModel(object): - """Describes an imported MLIR model.""" - - composite_id: str - name: str - model: common_definitions.Model - import_config: ImportConfig - - def __str__(self): - return self.name - - @classmethod - def from_model(cls, model: common_definitions.Model): - config = MODEL_SOURCE_TO_DEFAULT_IMPORT_CONFIG_MAP.get(model.source_type) - if config is None: - raise ValueError(f"Unsupported model source type: {model.source_type}.") - - composite_id = unique_ids.hash_composite_id([model.id, config.id]) - # Format: () - name = f"{model}({config})" - return cls( - composite_id=composite_id, name=name, model=model, import_config=config - ) - - -# Variable in flags to be replaced with module dir path. The whole path should -# be written in the POSIX format and starts with the variable. -MODULE_DIR_VARIABLE = r"${MODULE_DIR}" - - -@serialization.serializable( - type_key="iree_module_generation_configs", id_field="composite_id" -) -@dataclass(frozen=True) -class ModuleGenerationConfig(object): - """Describes a compile target to generate the module.""" - - composite_id: str - name: str - tags: List[str] - presets: List[str] - imported_model: ImportedModel - compile_config: CompileConfig - # Full list of flags to compile with, derived from sub-components, with - # unmaterialized placeholders. Allows the compile flags to be persisted and - # decouple from the generation code. Also serves as useful information in the - # serialized JSON. - compile_flags: List[str] - - def __str__(self): - return self.name - - def materialize_compile_flags(self, module_dir_path: pathlib.PurePath): - """Materialize flags with dependent values.""" - - def _replace_module_dir_placeholder(value: str) -> str: - """Replaces ${MODULE_DIR} in a POSIX path and returns the new path.""" - parts = pathlib.PurePosixPath(value).parts - if MODULE_DIR_VARIABLE not in parts: - return value - if parts[0] != MODULE_DIR_VARIABLE: - raise ValueError( - f"'{MODULE_DIR_VARIABLE}' needs to be the head of flag value" - f" if present, but got '{value}'." - ) - return str(module_dir_path.joinpath(*parts[1:]).as_posix()) - - return utils.transform_flags( - flags=self.compile_flags, map_funcs=[_replace_module_dir_placeholder] - ) - - @classmethod - def build( - cls, - imported_model: ImportedModel, - compile_config: CompileConfig, - tags: Sequence[str] = (), - presets: Sequence[str] = (), - ): - composite_id = unique_ids.hash_composite_id( - [imported_model.composite_id, compile_config.id] - ) - # Format: - name = f"{imported_model} {compile_config}" - compile_flags = _generate_compile_flags( - compile_config, imported_model.import_config.dialect_type - ) - return cls( - composite_id=composite_id, - name=name, - tags=list(tags), - presets=list(presets), - imported_model=imported_model, - compile_config=compile_config, - compile_flags=compile_flags, - ) - - -class E2EModelRunTool(Enum): - """Tool to run a module.""" - - IREE_BENCHMARK_MODULE = "iree-benchmark-module" - - -@serialization.serializable( - type_key="iree_e2e_model_run_configs", id_field="composite_id" -) -@dataclass(frozen=True) -class E2EModelRunConfig(object): - """Describes an e2e run.""" - - composite_id: str - name: str - tags: List[str] - presets: List[str] - module_generation_config: ModuleGenerationConfig - module_execution_config: ModuleExecutionConfig - target_device_spec: common_definitions.DeviceSpec - input_data: common_definitions.ModelInputData - # Full list of flags to run with, derived from sub-components, with - # unmaterialized placeholders. Allows the run flags to be persisted and - # decouple from the generation code. Also serves as useful information in the - # serialized JSON. - run_flags: List[str] - tool: E2EModelRunTool - - def __str__(self): - return self.name - - def materialize_run_flags( - self, - gpu_id: str = "0", - inputs_dir: Optional[pathlib.PurePath] = None, - external_params: Sequence[str] = (), - ) -> List[str]: - """Materialize flags with dependent values. - - Args: - gpu_id: gpu id to use. - inputs_dir: directory contains input_{0,1,...}.npy for each input. - - Returns: - List of flags - """ - flags = utils.substitute_flag_vars(flags=self.run_flags, GPU_ID=gpu_id) - - model = self.module_generation_config.imported_model.model - if inputs_dir: - input_npys = [ - inputs_dir / f"input_{idx}.npy" for idx in range(len(model.input_types)) - ] - flags += [f"--input=@{npy}" for npy in input_npys] - else: - flags += [f"--input={input_type}=0" for input_type in model.input_types] - - flags += [f"--parameters={param}" for param in external_params] - - return flags - - @classmethod - def build( - cls, - module_generation_config: ModuleGenerationConfig, - module_execution_config: ModuleExecutionConfig, - target_device_spec: common_definitions.DeviceSpec, - input_data: common_definitions.ModelInputData, - tool: E2EModelRunTool, - tags: Sequence[str] = (), - presets: Sequence[str] = (), - ): - composite_id = unique_ids.hash_composite_id( - [ - module_generation_config.composite_id, - module_execution_config.id, - target_device_spec.id, - input_data.id, - ] - ) - # Format: with @ - name = f"{module_generation_config} {module_execution_config} with {input_data} @ {target_device_spec}" - run_flags = generate_run_flags( - imported_model=module_generation_config.imported_model, - module_execution_config=module_execution_config, - gpu_id=r"${GPU_ID}", - ) - return cls( - composite_id=composite_id, - name=name, - tags=list(tags), - presets=list(presets), - module_generation_config=module_generation_config, - module_execution_config=module_execution_config, - target_device_spec=target_device_spec, - input_data=input_data, - run_flags=run_flags, - tool=tool, - ) - - -def generate_run_flags( - imported_model: ImportedModel, - module_execution_config: ModuleExecutionConfig, - gpu_id: str = "0", - with_driver: bool = True, -) -> List[str]: - """Returns the IREE run module flags of the input model and execution config. - Args: - model: source model. - input_data: model input data. - module_execution_config: execution config. - gpu_id: target gpu id, if runs on GPUs. - with_driver: populate the driver flags if true. False can be used for - generating flags for some CMake rules with a separate DRIVER arg. - Returns: - List of flags. - """ - - model = imported_model.model - run_flags = [f"--function={model.entry_function}"] - - exec_config = module_execution_config - run_flags += exec_config.extra_flags.copy() - if with_driver: - driver = exec_config.driver - if driver == RuntimeDriver.CUDA: - run_flags.append(f"--device=cuda://{gpu_id}") - else: - run_flags.append(f"--device={driver.value}") - - return run_flags - - -def _generate_compile_flags( - compile_config: CompileConfig, dialect_type: MLIRDialectType -) -> List[str]: - if len(compile_config.compile_targets) != 1: - raise ValueError( - f"Only one compile target is supported. Got:" - f" {compile_config.compile_targets}" - ) - - compile_target = compile_config.compile_targets[0] - flags = [ - f"--iree-hal-target-backends={compile_target.target_backend.value}", - f"--iree-input-type={dialect_type.value}", - ] - flags += _generate_compile_target_flags(compile_target) - flags += compile_config.extra_flags - return flags - - -def _generate_compile_target_flags(target: CompileTarget) -> List[str]: - arch_info = target.target_architecture - if target.target_backend == TargetBackend.VULKAN_SPIRV: - gpu_arch = ( - arch_info.microarchitecture - if len(arch_info.microarchitecture) != 0 - else arch_info.architecture - ) - return [ - f"--iree-vulkan-target-triple={gpu_arch}-unknown-{target.target_abi.value}", - ] - - if arch_info.architecture == "x86_64": - flags = [ - f"--iree-llvmcpu-target-triple=x86_64-unknown-{target.target_abi.value}", - f"--iree-llvmcpu-target-cpu={arch_info.microarchitecture.lower()}", - ] - elif arch_info.architecture == "riscv_64": - flags = [ - f"--iree-llvmcpu-target-triple=riscv64-pc-{target.target_abi.value}", - "--iree-llvmcpu-target-cpu=generic-rv64", - "--iree-llvmcpu-target-abi=lp64d", - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v", - "--riscv-v-fixed-length-vector-lmul-max=8", - ] - elif arch_info.architecture == "riscv_32": - # TODO(llvm-project/60463): Replace 'zve32f' with 'zve32x'. - flags = [ - f"--iree-llvmcpu-target-triple=riscv32-pc-{target.target_abi.value}", - "--iree-llvmcpu-target-cpu=generic-rv32", - "--iree-llvmcpu-target-abi=ilp32", - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+zvl512b,+zve32f", - "--riscv-v-fixed-length-vector-lmul-max=8", - ] - elif arch_info.architecture == "armv8.2-a": - flags = [ - f"--iree-llvmcpu-target-triple=aarch64-none-{target.target_abi.value}", - ] - elif arch_info.architecture == "cuda": - if target.target_abi != TargetABI.LINUX_GNU: - raise ValueError( - f"Unsupported target ABI for CUDA backend: `{target.target_abi}`" - ) - flags = [ - f"--iree-hal-cuda-llvm-target-arch={arch_info.microarchitecture}", - ] - elif arch_info.architecture == "vmvx": - flags = [] - else: - raise ValueError(f"Unsupported architecture: '{arch_info.architecture}'") - return flags diff --git a/build_tools/python/e2e_test_framework/definitions/iree_definitions_test.py b/build_tools/python/e2e_test_framework/definitions/iree_definitions_test.py deleted file mode 100644 index bf4bdfb325ae..000000000000 --- a/build_tools/python/e2e_test_framework/definitions/iree_definitions_test.py +++ /dev/null @@ -1,240 +0,0 @@ -## Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import pathlib -import unittest - -from e2e_test_framework.definitions import common_definitions, iree_definitions - - -class IreeDefinitionsTest(unittest.TestCase): - def test_generate_run_flags(self): - imported_model = iree_definitions.ImportedModel.from_model( - common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32", "2x2xf32"], - ) - ) - execution_config = iree_definitions.ModuleExecutionConfig.build( - id="123", - tags=["test"], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_TASK, - extra_flags=["--task=10"], - ) - - flags = iree_definitions.generate_run_flags( - imported_model=imported_model, - module_execution_config=execution_config, - ) - - self.assertEqual( - flags, - [ - "--function=main", - "--task=10", - "--device=local-task", - ], - ) - - def test_generate_run_flags_with_cuda(self): - imported_model = iree_definitions.ImportedModel.from_model( - common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32"], - ) - ) - execution_config = iree_definitions.ModuleExecutionConfig.build( - id="123", - tags=["test"], - loader=iree_definitions.RuntimeLoader.NONE, - driver=iree_definitions.RuntimeDriver.CUDA, - extra_flags=[], - ) - - flags = iree_definitions.generate_run_flags( - imported_model=imported_model, - module_execution_config=execution_config, - gpu_id="3", - ) - - self.assertEqual(flags, ["--function=main", "--device=cuda://3"]) - - def test_generate_run_flags_without_driver(self): - imported_model = iree_definitions.ImportedModel.from_model( - common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32"], - ) - ) - execution_config = iree_definitions.ModuleExecutionConfig.build( - id="123", - tags=["test"], - loader=iree_definitions.RuntimeLoader.EMBEDDED_ELF, - driver=iree_definitions.RuntimeDriver.LOCAL_TASK, - extra_flags=["--task=10"], - ) - - flags = iree_definitions.generate_run_flags( - imported_model=imported_model, - module_execution_config=execution_config, - with_driver=False, - ) - - self.assertEqual(flags, ["--function=main", "--task=10"]) - - def test_materialize_run_flags(self): - imported_model = iree_definitions.ImportedModel.from_model( - common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32", "2x2xf32"], - ) - ) - compile_target = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.CUDA, - target_architecture=common_definitions.DeviceArchitecture.CUDA_SM80, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - compile_config = iree_definitions.CompileConfig( - id="compile_config_a", - name="compile_config_a", - tags=["test"], - compile_targets=[compile_target], - ) - gen_config = iree_definitions.ModuleGenerationConfig.build( - imported_model=imported_model, compile_config=compile_config - ) - exec_config = iree_definitions.ModuleExecutionConfig.build( - id="123", - tags=["test"], - loader=iree_definitions.RuntimeLoader.NONE, - driver=iree_definitions.RuntimeDriver.CUDA, - ) - device_spec = common_definitions.DeviceSpec.build( - id="test_dev", - device_name="test_model", - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - architecture=common_definitions.DeviceArchitecture.CUDA_SM80, - ) - run_config = iree_definitions.E2EModelRunConfig.build( - gen_config, - exec_config, - device_spec, - input_data=common_definitions.DEFAULT_INPUT_DATA, - tool=iree_definitions.E2EModelRunTool.IREE_BENCHMARK_MODULE, - ) - - inputs_dir = pathlib.PurePath("inputs_dir") - flags = run_config.materialize_run_flags(gpu_id="10", inputs_dir=inputs_dir) - - self.assertIn("--device=cuda://10", flags) - first_input = f'--input=@{inputs_dir / "input_0.npy"}' - self.assertIn(first_input, flags) - first_input_idx = flags.index(first_input) - self.assertEqual( - flags[first_input_idx + 1], f'--input=@{inputs_dir/"input_1.npy"}' - ) - - -class ModuleGenerationConfigTest(unittest.TestCase): - def test_materialize_compile_flags(self): - imported_model = iree_definitions.ImportedModel.from_model( - common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32"], - ) - ) - compile_target = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - compile_config = iree_definitions.CompileConfig( - id="compile_config_a", - name="compile_config_a", - tags=["test"], - compile_targets=[compile_target], - extra_flags=[r"--test=${MODULE_DIR}/test.json"], - ) - gen_config = iree_definitions.ModuleGenerationConfig.build( - imported_model=imported_model, compile_config=compile_config - ) - - flags = gen_config.materialize_compile_flags( - module_dir_path=pathlib.Path("abc") - ) - - expected_path = pathlib.Path("abc", "test.json").as_posix() - self.assertIn(f"--test={expected_path}", flags) - - def test_materialize_compile_flags_invalid_module_dir_position(self): - imported_model = iree_definitions.ImportedModel.from_model( - common_definitions.Model( - id="1234", - name="tflite_m", - tags=[], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://example.com/xyz.tflite", - entry_function="main", - input_types=["1xf32"], - ) - ) - compile_target = iree_definitions.CompileTarget( - target_backend=iree_definitions.TargetBackend.LLVM_CPU, - target_architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - target_abi=iree_definitions.TargetABI.LINUX_GNU, - ) - compile_config = iree_definitions.CompileConfig( - id="compile_config_a", - name="compile_config_a", - tags=["test"], - compile_targets=[compile_target], - extra_flags=[r"--test=prefix/${MODULE_DIR}/test.json"], - ) - gen_config = iree_definitions.ModuleGenerationConfig.build( - imported_model=imported_model, compile_config=compile_config - ) - expected_error = ( - r"^'\${MODULE_DIR}' needs to be the head of flag value if present," - r" but got 'prefix/\${MODULE_DIR}/test.json'.$" - ) - - self.assertRaisesRegex( - ValueError, - expected_error, - lambda: gen_config.materialize_compile_flags( - module_dir_path=pathlib.Path("abc") - ), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/python/e2e_test_framework/definitions/utils.py b/build_tools/python/e2e_test_framework/definitions/utils.py deleted file mode 100644 index 5079cc6679b1..000000000000 --- a/build_tools/python/e2e_test_framework/definitions/utils.py +++ /dev/null @@ -1,52 +0,0 @@ -## Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Utils that help construct definitions.""" - -import string -from typing import Any, Callable, List, Sequence - -MAX_SUBSTITUTION_ITERATIONS = 10 - - -def transform_flags( - flags: Sequence[str], map_funcs: Sequence[Callable[[str], str]] -) -> List[str]: - """Call map functions to transform flag values, e.g., replace placeholders - that were unknown when the flag was constructed. - - It parses and extracts the flag values from both keyword and positional flags, - transforms them, and returns the updated flags with transformed values. - - Each flag value is transformed only once by each map function in order. - - Args: - flags: list of flags. - map_funcs: list of map functions to map flag value. - Returns: - List of transformed flags. - """ - - transformed_flags = [] - for flag in flags: - keyword, separator, value = ("", "", flag) - if flag.startswith("-"): - keyword, separator, value = flag.partition("=") - - if value: - for map_func in map_funcs: - value = map_func(value) - - transformed_flags.append(f"{keyword}{separator}{value}") - - return transformed_flags - - -def substitute_flag_vars(flags: Sequence[str], **mapping: Any) -> List[str]: - """Sugar of transform_flags to substitute variables in string.Template format.""" - return transform_flags( - flags=flags, - map_funcs=[lambda value: string.Template(value).substitute(mapping)], - ) diff --git a/build_tools/python/e2e_test_framework/definitions/utils_test.py b/build_tools/python/e2e_test_framework/definitions/utils_test.py deleted file mode 100644 index 13ca5f619899..000000000000 --- a/build_tools/python/e2e_test_framework/definitions/utils_test.py +++ /dev/null @@ -1,53 +0,0 @@ -## Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import unittest - -from e2e_test_framework.definitions import utils - - -class UtilsTest(unittest.TestCase): - def test_transform_flags(self): - flags = utils.transform_flags( - flags=[ - r"${HOLDER_A} ${HOLDER_B}", - r"--key=${HOLDER_A}", - "--no-value-key", - r"--filter=x=${HOLDER_A}", - ], - map_funcs=[ - lambda value: value.replace(r"${HOLDER_A}", "val_a"), - lambda value: value.replace(r"${HOLDER_B}", "val_b"), - ], - ) - - self.assertEqual( - flags, ["val_a val_b", "--key=val_a", "--no-value-key", "--filter=x=val_a"] - ) - - def test_substitute_flag_vars(self): - raw_flags = [ - r"${HOLDER_A}", - r"--key=${HOLDER_B}", - ] - - flags = utils.substitute_flag_vars(flags=raw_flags, HOLDER_A=1, HOLDER_B="b") - - self.assertEqual(flags, ["1", "--key=b"]) - - def test_substitute_flag_vars_missing_variable(self): - raw_flags = [ - r"${HOLDER_A}", - r"--key=${HOLDER_B}", - ] - - self.assertRaises( - KeyError, lambda: utils.substitute_flag_vars(flags=raw_flags, HOLDER_A=1) - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/python/e2e_test_framework/device_specs/CMakeLists.txt b/build_tools/python/e2e_test_framework/device_specs/CMakeLists.txt deleted file mode 100644 index 46dca219e9d9..000000000000 --- a/build_tools/python/e2e_test_framework/device_specs/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -iree_build_tools_py_test( - NAME - device_collections_test - SRC - "device_collections_test.py" -) diff --git a/build_tools/python/e2e_test_framework/device_specs/README.md b/build_tools/python/e2e_test_framework/device_specs/README.md deleted file mode 100644 index bc9dbfc86e81..000000000000 --- a/build_tools/python/e2e_test_framework/device_specs/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# IREE Benchmark Device Specs - -This direcotry contains the specifications of all target devices we run -benchmarks. Definitions can be found in the `*_specs.py` files. - -## Adding a new device spec - -1. Register a unique device spec ID in - [build_tools/python/e2e_test_framework/unique_ids.py](/build_tools/python/e2e_test_framework/unique_ids.py). -2. Define the new device spec in an existing python module or create a new one. diff --git a/build_tools/python/e2e_test_framework/device_specs/__init__.py b/build_tools/python/e2e_test_framework/device_specs/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/build_tools/python/e2e_test_framework/device_specs/device_collections.py b/build_tools/python/e2e_test_framework/device_specs/device_collections.py deleted file mode 100644 index ef09b09c9dbf..000000000000 --- a/build_tools/python/e2e_test_framework/device_specs/device_collections.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines the collections of device specs and provides query methods.""" - -from typing import List, Sequence, Set -from e2e_test_framework.definitions import common_definitions -from e2e_test_framework.device_specs import ( - gcp_specs, - moto_edge_x30_specs, - pixel_6_pro_specs, - riscv_specs, -) - - -class DeviceCollection(object): - """Class to collect and query device specs.""" - - def __init__(self, device_specs: Sequence[common_definitions.DeviceSpec]): - self.device_specs = device_specs - - def query_device_specs( - self, - architecture: common_definitions.DeviceArchitecture, - host_environment: common_definitions.HostEnvironment, - tags: Sequence[str] = (), - ) -> List[common_definitions.DeviceSpec]: - """Query the device specs. - - Args: - architecture: device architecture to match. - platform: device platform to match. - tags: tags that devices need to have. - Returns: - List of matched device specs. - """ - - matched_device_specs = [] - tag_set = set(tags) - for device_spec in self.device_specs: - if device_spec.architecture != architecture: - continue - if device_spec.host_environment != host_environment: - continue - if not tag_set.issubset(device_spec.tags): - continue - matched_device_specs.append(device_spec) - - return matched_device_specs - - -ALL_DEVICE_SPECS = [ - # Pixel 6 Pro - pixel_6_pro_specs.BIG_CORES, - pixel_6_pro_specs.ALL_CORES, - pixel_6_pro_specs.GPU, - # Moto Edge X30 - moto_edge_x30_specs.GPU, - # GCP machines - gcp_specs.GCP_C2_STANDARD_60, - gcp_specs.GCP_A2_HIGHGPU_1G, - # RISCV emulators - riscv_specs.EMULATOR_RISCV_32, - riscv_specs.EMULATOR_RISCV_64, -] -DEFAULT_DEVICE_COLLECTION = DeviceCollection(ALL_DEVICE_SPECS) diff --git a/build_tools/python/e2e_test_framework/device_specs/device_collections_test.py b/build_tools/python/e2e_test_framework/device_specs/device_collections_test.py deleted file mode 100644 index 123c4461919b..000000000000 --- a/build_tools/python/e2e_test_framework/device_specs/device_collections_test.py +++ /dev/null @@ -1,89 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import unittest -from e2e_test_framework.definitions import common_definitions -from e2e_test_framework.device_specs import device_collections - - -class DeviceCollectionTest(unittest.TestCase): - def test_query_device_specs(self): - linux_x86_device_spec = common_definitions.DeviceSpec.build( - id="linux_x86", - device_name="a", - architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - tags=[], - ) - android_x86_device_spec = common_definitions.DeviceSpec.build( - id="android_x86", - device_name="b", - architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=[], - ) - little_cores_device_spec = common_definitions.DeviceSpec.build( - id="android_little", - device_name="c", - architecture=common_definitions.DeviceArchitecture.ARMV9_A_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=["little-cores"], - ) - big_cores_device_spec = common_definitions.DeviceSpec.build( - id="android_big", - device_name="d", - architecture=common_definitions.DeviceArchitecture.ARMV9_A_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=["big-cores"], - ) - devices = device_collections.DeviceCollection( - device_specs=[ - linux_x86_device_spec, - android_x86_device_spec, - little_cores_device_spec, - big_cores_device_spec, - ] - ) - - linux_x86_devices = devices.query_device_specs( - architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - ) - android_x86_devices = devices.query_device_specs( - architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - ) - little_cores_devices = devices.query_device_specs( - architecture=common_definitions.DeviceArchitecture.ARMV9_A_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=["little-cores"], - ) - big_cores_devices = devices.query_device_specs( - architecture=common_definitions.DeviceArchitecture.ARMV9_A_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=["big-cores"], - ) - all_arm_devices = devices.query_device_specs( - architecture=common_definitions.DeviceArchitecture.ARMV9_A_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - ) - no_matched_device = devices.query_device_specs( - architecture=common_definitions.DeviceArchitecture.ARMV9_A_GENERIC, - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - ) - - self.assertEqual(linux_x86_devices, [linux_x86_device_spec]) - self.assertEqual(android_x86_devices, [android_x86_device_spec]) - self.assertEqual(little_cores_devices, [little_cores_device_spec]) - self.assertEqual(big_cores_devices, [big_cores_device_spec]) - self.assertEqual( - all_arm_devices, [little_cores_device_spec, big_cores_device_spec] - ) - self.assertEqual(no_matched_device, []) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/python/e2e_test_framework/device_specs/gcp_specs.py b/build_tools/python/e2e_test_framework/device_specs/gcp_specs.py deleted file mode 100644 index 752ca0b09f42..000000000000 --- a/build_tools/python/e2e_test_framework/device_specs/gcp_specs.py +++ /dev/null @@ -1,25 +0,0 @@ -## copyright 2022 the iree authors -# -# licensed under the apache license v2.0 with llvm exceptions. -# see https://llvm.org/license.txt for license information. -# spdx-license-identifier: apache-2.0 with llvm-exception -"""Defines device specs for GCP machines.""" - -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions - -GCP_C2_STANDARD_60 = common_definitions.DeviceSpec.build( - id=unique_ids.DEVICE_SPEC_GCP_C2_STANDARD_60, - device_name="c2-standard-60", - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - architecture=common_definitions.DeviceArchitecture.X86_64_CASCADELAKE, - tags=["cpu"], -) - -GCP_A2_HIGHGPU_1G = common_definitions.DeviceSpec.build( - id=unique_ids.DEVICE_SPEC_GCP_A2_HIGHGPU_1G, - device_name="a2-highgpu-1g", - host_environment=common_definitions.HostEnvironment.LINUX_X86_64, - architecture=common_definitions.DeviceArchitecture.NVIDIA_AMPERE, - tags=["gpu"], -) diff --git a/build_tools/python/e2e_test_framework/device_specs/moto_edge_x30_specs.py b/build_tools/python/e2e_test_framework/device_specs/moto_edge_x30_specs.py deleted file mode 100644 index abb1e392a785..000000000000 --- a/build_tools/python/e2e_test_framework/device_specs/moto_edge_x30_specs.py +++ /dev/null @@ -1,23 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines device specs of Moto Edge X30.""" - -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions - -DEVICE_NAME = "moto-edge-x30" - -GPU = common_definitions.DeviceSpec.build( - id=unique_ids.DEVICE_SPEC_MOBILE_MOTO_EDGE_X30 + "-gpu", - device_name=DEVICE_NAME, - architecture=common_definitions.DeviceArchitecture.QUALCOMM_ADRENO, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - device_parameters=common_definitions.DeviceParameters( - # Pin on the fastest CPU core. - cpu_params=common_definitions.CPUParameters(pinned_cores=[7]) - ), - tags=["gpu"], -) diff --git a/build_tools/python/e2e_test_framework/device_specs/pixel_6_pro_specs.py b/build_tools/python/e2e_test_framework/device_specs/pixel_6_pro_specs.py deleted file mode 100644 index 4eaef97daad6..000000000000 --- a/build_tools/python/e2e_test_framework/device_specs/pixel_6_pro_specs.py +++ /dev/null @@ -1,41 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines device specs of Pixel 6 Pro.""" - -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions - -DEVICE_NAME = "pixel-6-pro" - -BIG_CORES = common_definitions.DeviceSpec.build( - id=unique_ids.DEVICE_SPEC_MOBILE_PIXEL_6_PRO + "-big-core", - device_name=DEVICE_NAME, - architecture=common_definitions.DeviceArchitecture.ARMV8_2_A_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - device_parameters=common_definitions.DeviceParameters( - # Cortex-X1 - cpu_params=common_definitions.CPUParameters(pinned_cores=[6, 7]) - ), - tags=["big-cores"], -) -ALL_CORES = common_definitions.DeviceSpec.build( - id=unique_ids.DEVICE_SPEC_MOBILE_PIXEL_6_PRO + "-all-core", - device_name=DEVICE_NAME, - architecture=common_definitions.DeviceArchitecture.ARMV8_2_A_GENERIC, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - tags=["all-cores"], -) -GPU = common_definitions.DeviceSpec.build( - id=unique_ids.DEVICE_SPEC_MOBILE_PIXEL_6_PRO + "-gpu", - device_name=DEVICE_NAME, - architecture=common_definitions.DeviceArchitecture.ARM_VALHALL, - host_environment=common_definitions.HostEnvironment.ANDROID_ARMV8_2_A, - device_parameters=common_definitions.DeviceParameters( - # Pin on the fastest CPU core. - cpu_params=common_definitions.CPUParameters(pinned_cores=[7]) - ), - tags=["gpu"], -) diff --git a/build_tools/python/e2e_test_framework/device_specs/riscv_specs.py b/build_tools/python/e2e_test_framework/device_specs/riscv_specs.py deleted file mode 100644 index 5c8bbc5705b2..000000000000 --- a/build_tools/python/e2e_test_framework/device_specs/riscv_specs.py +++ /dev/null @@ -1,26 +0,0 @@ -## Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines device specs of RISC-V devices.""" - -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions - - -EMULATOR_RISCV_64 = common_definitions.DeviceSpec.build( - id=unique_ids.DEVICE_SPEC_EMULATOR_RISCV_64, - device_name="emulator-riscv_64", - host_environment=common_definitions.HostEnvironment.LINUX_RISCV_64, - architecture=common_definitions.DeviceArchitecture.RV64_GENERIC, - tags=["cpu"], -) - -EMULATOR_RISCV_32 = common_definitions.DeviceSpec.build( - id=unique_ids.DEVICE_SPEC_EMULATOR_RISCV_32, - device_name="emulator-riscv_32", - host_environment=common_definitions.HostEnvironment.LINUX_RISCV_32, - architecture=common_definitions.DeviceArchitecture.RV32_GENERIC, - tags=["cpu"], -) diff --git a/build_tools/python/e2e_test_framework/models/CMakeLists.txt b/build_tools/python/e2e_test_framework/models/CMakeLists.txt deleted file mode 100644 index 075a0c0fc558..000000000000 --- a/build_tools/python/e2e_test_framework/models/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -iree_build_tools_py_test( - NAME - utils_test - SRC - "utils_test.py" -) diff --git a/build_tools/python/e2e_test_framework/models/README.md b/build_tools/python/e2e_test_framework/models/README.md deleted file mode 100644 index b438da1b6ac5..000000000000 --- a/build_tools/python/e2e_test_framework/models/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# IREE E2E Test Artifacts Models - -This directory contains the definitions of source models for e2e tests -(including the benchmark suites). Each source type of models is defined in -`_models.py` - -## Adding a new model - -1. Upload the source model file to the GCS bucket gs://iree-model-artifacts. - - You can ask IREE team members for help if you don't have access. -2. Register a unique model ID in - [build_tools/python/e2e_test_framework/unique_ids.py](/build_tools/python/e2e_test_framework/unique_ids.py). - - Run `uuid.uuid4()`, or `uuidgen` on Linux, to generate a new uuid for - the new model. The ID is a totally arbitrary artificial key to - distinguish different models. -3. Define a new model with GCS URL and model ID in `_models.py`. -4. Optionally add the model to a model group in - [build_tools/python/e2e_test_framework/models/model_groups.py](/build_tools/python/e2e_test_framework/models/model_groups.py). diff --git a/build_tools/python/e2e_test_framework/models/__init__.py b/build_tools/python/e2e_test_framework/models/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/build_tools/python/e2e_test_framework/models/jax_models.py b/build_tools/python/e2e_test_framework/models/jax_models.py deleted file mode 100644 index 1a8ce4bc8d5d..000000000000 --- a/build_tools/python/e2e_test_framework/models/jax_models.py +++ /dev/null @@ -1,88 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines JAX models.""" - -import string - -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions -import e2e_test_framework.models.utils as model_utils - -GCS_ARTIFACT_ROOT_DIR = "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.25_1709787220" - -ID_FORMAT = string.Template("${model_id}-batch${batch_size}") -NAME_FORMAT = string.Template("${name}_BATCH${batch_size}") -SOURCE_URL_FORMAT = string.Template( - GCS_ARTIFACT_ROOT_DIR + "/${directory}_BATCH${batch_size}/stablehlo.mlirbc" -) - -# Derived from https://huggingface.co/docs/transformers/model_doc/resnet#transformers.FlaxResNetModel. -RESNET50_TAGS = ["fp32", "cnn", "resnet"] - -RESNET50_FP32_JAX_3X224X224XF32_BATCHES = model_utils.generate_batch_models( - id_template=model_utils.partial_template_substitute( - ID_FORMAT, model_id=unique_ids.MODEL_RESNET50_FP32_JAX_3X224X224XF32 - ), - name_template=model_utils.partial_template_substitute( - NAME_FORMAT, name="RESNET50_FP32_JAX_3X224X224XF32" - ), - tags=RESNET50_TAGS, - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url_template=model_utils.partial_template_substitute( - SOURCE_URL_FORMAT, directory="RESNET50_FP32_JAX_3X224X224XF32" - ), - entry_function="main", - input_type_templates=[string.Template("${batch_size}x3x224x224xf32")], - batch_sizes=[1, 8, 64, 128, 256, 2048], -) - -# Derived from https://huggingface.co/docs/transformers/model_doc/bert#transformers.FlaxBertModel. -BERT_LARGE_TAGS = ["fp32", "seqlen384", "jax", "bert-variant"] - -BERT_LARGE_FP32_JAX_384XI32_BATCHES = model_utils.generate_batch_models( - id_template=model_utils.partial_template_substitute( - ID_FORMAT, model_id=unique_ids.MODEL_BERT_LARGE_FP32_JAX_384XI32 - ), - name_template=model_utils.partial_template_substitute( - NAME_FORMAT, name="BERT_LARGE_JAX_384XI32" - ), - tags=BERT_LARGE_TAGS, - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url_template=model_utils.partial_template_substitute( - SOURCE_URL_FORMAT, directory="BERT_LARGE_FP32_JAX_384XI32" - ), - entry_function="main", - input_type_templates=[ - string.Template("${batch_size}x384xi32"), - string.Template("${batch_size}x384xi32"), - ], - batch_sizes=[1, 16, 24, 32, 48, 64, 512, 1024, 1280], -) - -# Derived from https://huggingface.co/docs/transformers/model_doc/t5#transformers.FlaxT5Model -T5_TAGS = ["fp32", "transformer-encoder", "transformer-decoder", "t5"] - -T5_LARGE_FP32_JAX_512XI32_BATCHES = model_utils.generate_batch_models( - id_template=model_utils.partial_template_substitute( - ID_FORMAT, model_id=unique_ids.MODEL_T5_LARGE_FP32_JAX_512XI32 - ), - name_template=model_utils.partial_template_substitute( - NAME_FORMAT, name="T5_LARGE_FP32_JAX_512XI32" - ), - tags=T5_TAGS, - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url_template=model_utils.partial_template_substitute( - SOURCE_URL_FORMAT, directory="T5_LARGE_FP32_JAX_512XI32" - ), - entry_function="main", - input_type_templates=[ - string.Template("${batch_size}x512xi32"), - string.Template("${batch_size}x512xi32"), - string.Template("${batch_size}x512xi32"), - string.Template("${batch_size}x512xi32"), - ], - batch_sizes=[1, 16, 24, 32, 48, 64, 512], -) diff --git a/build_tools/python/e2e_test_framework/models/matmul.py b/build_tools/python/e2e_test_framework/models/matmul.py deleted file mode 100644 index 7714c4c34662..000000000000 --- a/build_tools/python/e2e_test_framework/models/matmul.py +++ /dev/null @@ -1,149 +0,0 @@ -## Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines Matmul microbenchmarks.""" - -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions - -MATMUL_3456X1024X2048_FP16_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_3456X1024X2048_FP16_MLIR, - name="matmul_3456x1024x2048_f16t_tile_config_default", - tags=["fp16", "ubench", "matmul"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230410_1681181224/matmul_3456x1024x2048_f16t_f16t_f16t_tile_config_default.mlirbc", - entry_function="matmul_3456x1024x2048_f16t_f16t_f16t_tile_config_default", - input_types=["3456x2048xf16", "2048x1024xf16"], -) - -MATMUL_3456X1024X2048_FP32_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_3456X1024X2048_FP32_MLIR, - name="matmul_3456x1024x2048_f32t_tile_config_default", - tags=["fp32", "ubench", "matmul"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230410_1681181224/matmul_3456x1024x2048_f32t_f32t_f32t_tile_config_default.mlirbc", - entry_function="matmul_3456x1024x2048_f32t_f32t_f32t_tile_config_default", - input_types=["3456x2048xf32", "2048x1024xf32"], -) - -MATMUL_2560X2560X2560_FP16_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_2560X2560X2560_FP16_MLIR, - name="matmul_2560x2560x2560_f16t_tile_config_default", - tags=["fp16", "ubench", "matmul"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230410_1681181224/matmul_2560x2560x2560_f16t_f16t_f16t_tile_config_default.mlirbc", - entry_function="matmul_2560x2560x2560_f16t_f16t_f16t_tile_config_default", - input_types=["2560x2560xf16", "2560x2560xf16"], -) - -MATMUL_2560X2560X2560_FP32_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_2560X2560X2560_FP32_MLIR, - name="matmul_2560x2560x2560_f32t_tile_config_default", - tags=["fp32", "ubench", "matmul"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230410_1681181224/matmul_2560x2560x2560_f32t_f32t_f32t_tile_config_default.mlirbc", - entry_function="matmul_2560x2560x2560_f32t_f32t_f32t_tile_config_default", - input_types=["2560x2560xf32", "2560x2560xf32"], -) - -MATMUL_128X256X8192_FP16_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_128X256X8192_FP16_MLIR, - name="matmul_128x256x8192_f16t_tile_config_default", - tags=["fp16", "ubench", "matmul", "splitk"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230410_1681181224/matmul_128x256x8192_f16t_f16t_f16t_tile_config_default.mlirbc", - entry_function="matmul_128x256x8192_f16t_f16t_f16t_tile_config_default", - input_types=["128x8192xf16", "8192x256xf16"], -) - -MATMUL_128X256X8192_FP32_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_128X256X8192_FP32_MLIR, - name="matmul_128x256x8192_f32t_tile_config_default", - tags=["fp32", "ubench", "matmul", "splitk"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230410_1681181224/matmul_128x256x8192_f32t_f32t_f32t_tile_config_default.mlirbc", - entry_function="matmul_128x256x8192_f32t_f32t_f32t_tile_config_default", - input_types=["128x8192xf32", "8192x256xf32"], -) - -MATMUL_2564x2564x2564_FP32_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_2564x2564x2564_FP32_MLIR, - name="matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default", - tags=["fp32", "ubench", "matmul"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230525_1685058259/matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default.mlirbc", - entry_function="matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default", - input_types=["2564x2564xf32", "2564x2564xf32", "2564x2564xf32"], -) - -MATMUL_2562x2564x2562_FP32_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_2562x2564x2562_FP32_MLIR, - name="matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default", - tags=["fp32", "ubench", "matmul"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230525_1685058259/matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default.mlirbc", - entry_function="matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default", - input_types=["2562x2562xf32", "2562x2564xf32", "2562x2564xf32"], -) - -MATMUL_2562x2561x2561_FP32_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_2562x2561x2561_FP32_MLIR, - name="matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default", - tags=["fp32", "ubench", "matmul"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230525_1685058259/matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc", - entry_function="matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default", - input_types=["2562x2561xf32", "2561x2561xf32", "2562x2561xf32"], -) - -MATMUL_123x2561x2561_FP32_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_123x2561x2561_FP32_MLIR, - name="matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default", - tags=["fp32", "ubench", "matmul"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230612_1686563210/matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc", - entry_function="matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default", - input_types=["123x2561xf32", "2561x2561xf32", "123x2561xf32"], -) - -MATMUL_1x256x2048_I8xI4_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_1x256x2048_I8xI4_MLIR, - name="matmul_1x256x2048_i8_i4_i32_tile_config_default", - tags=["i8i4", "ubench", "matmul"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.30_1719457850/DOT_PRODUCT_JAX_1X256X2048XI8I4/linalg.mlirbc", - entry_function="main", - input_types=["1x256xi8"], -) - -MATMUL_256x256x2048_I8xI4_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_256x256x2048_I8xI4_MLIR, - name="matmul_256x256x2048_i8_i4_i32_tile_config_default", - tags=["i8i4", "ubench", "matmul"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.30_1719457850/DOT_PRODUCT_JAX_256X256X2048XI8I4/linalg.mlirbc", - entry_function="main", - input_types=["256x256xi8"], -) - -MATMUL_1x256x2048_I8xI8_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_1x256x2048_I8xI8_MLIR, - name="matmul_1x256x2048_i8_i8_i32_tile_config_default", - tags=["i8i8", "ubench", "matmul"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.30_1719457850/DOT_PRODUCT_JAX_1X256X2048XI8I8/linalg.mlirbc", - entry_function="main", - input_types=["1x256xi8"], -) - -MATMUL_256x256x2048_I8xI8_MLIR = common_definitions.Model( - id=unique_ids.MICRO_MATMUL_256x256x2048_I8xI8_MLIR, - name="matmul_256x256x2048_i8_i8_i32_tile_config_default", - tags=["i8i8", "ubench", "matmul"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.30_1719457850/DOT_PRODUCT_JAX_256X256X2048XI8I8/linalg.mlirbc", - entry_function="main", - input_types=["256x256xi8"], -) diff --git a/build_tools/python/e2e_test_framework/models/model_groups.py b/build_tools/python/e2e_test_framework/models/model_groups.py deleted file mode 100644 index 151f21121bfd..000000000000 --- a/build_tools/python/e2e_test_framework/models/model_groups.py +++ /dev/null @@ -1,176 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines the groups of models.""" - -from e2e_test_framework.definitions import common_definitions -from e2e_test_framework.models import ( - matmul, - tflite_models, - torch_models, - tf_models, - jax_models, -) - -# x86 models, single batch. - -# A list of models with thread configurations. -# Note `0` represents sync execution. -X86_64_BENCHMARK_CONFIG = [ - # Microbenchmarks. - common_definitions.CpuBenchmarkConfig( - model=matmul.MATMUL_1x256x2048_I8xI4_MLIR, threads=[0] - ), - common_definitions.CpuBenchmarkConfig( - model=matmul.MATMUL_256x256x2048_I8xI4_MLIR, threads=[0] - ), - common_definitions.CpuBenchmarkConfig( - model=matmul.MATMUL_1x256x2048_I8xI8_MLIR, threads=[0] - ), - common_definitions.CpuBenchmarkConfig( - model=matmul.MATMUL_256x256x2048_I8xI8_MLIR, threads=[0] - ), - # Tiny models. - common_definitions.CpuBenchmarkConfig( - model=tflite_models.PERSON_DETECT_INT8, threads=[0, 1] - ), - common_definitions.CpuBenchmarkConfig( - model=tflite_models.MOBILENET_V3SMALL, threads=[0, 1] - ), - # Small models. - common_definitions.CpuBenchmarkConfig( - model=tflite_models.DEEPLABV3_FP32, threads=[1, 8] - ), - common_definitions.CpuBenchmarkConfig( - model=tflite_models.EFFICIENTNET_INT8, threads=[1, 8] - ), - common_definitions.CpuBenchmarkConfig( - model=tflite_models.MOBILENET_V1, threads=[1, 8] - ), - common_definitions.CpuBenchmarkConfig( - model=tflite_models.MOBILENET_V2, threads=[1, 8] - ), - common_definitions.CpuBenchmarkConfig( - model=tflite_models.MOBILENET_V2_INT8, threads=[1, 8] - ), - common_definitions.CpuBenchmarkConfig( - model=tflite_models.MOBILESSD_FP32, threads=[1, 8] - ), - common_definitions.CpuBenchmarkConfig( - model=tflite_models.POSENET_FP32, threads=[1, 8] - ), - # Medium models. - common_definitions.CpuBenchmarkConfig( - model=tflite_models.MOBILEBERT_FP16, threads=[1, 15] - ), - common_definitions.CpuBenchmarkConfig( - model=tflite_models.MOBILEBERT_FP32, threads=[1, 15] - ), - common_definitions.CpuBenchmarkConfig( - model=tflite_models.MOBILEBERT_INT8, threads=[1, 15] - ), - common_definitions.CpuBenchmarkConfig( - model=tf_models.EFFICIENTNET_V2_S_FP32, threads=[1, 15] - ), - common_definitions.CpuBenchmarkConfig( - model=tf_models.MINILM_L12_H384_UNCASED_INT32_SEQLEN128, threads=[1, 15] - ), - common_definitions.CpuBenchmarkConfig( - model=tf_models.GPT2_117M_1x4_FP32_TF, threads=[1, 15] - ), - common_definitions.CpuBenchmarkConfig( - model=tf_models.GPT2_117M_1x1_FP32_TF, threads=[1, 15] - ), - # Large models. - common_definitions.CpuBenchmarkConfig( - model=tf_models.BERT_FOR_MASKED_LM_FP32_SEQLEN512, threads=[30] - ), - common_definitions.CpuBenchmarkConfig( - model=tf_models.BERT_LARGE_TF_FP32_SEQLEN384, threads=[30] - ), - # TODO(#17344): regenerate .mlirbc files - # common_definitions.CpuBenchmarkConfig( - # model=torch_models.BERT_LARGE_384_FP32_TORCH_BATCHES[1], threads=[30] - # ), - # common_definitions.CpuBenchmarkConfig( - # model=torch_models.FALCON7B_1X100XI64_GPTQ_TORCH, threads=[30] - # ), - # common_definitions.CpuBenchmarkConfig( - # model=torch_models.FALCON7B_INT4_1X100XI64_GPTQ_TORCH, threads=[30] - # ), -] - -X86_64_BENCHMARK_CONFIG_LARGE = [ - common_definitions.CpuBenchmarkConfig( - model=jax_models.BERT_LARGE_FP32_JAX_384XI32_BATCHES[1], threads=[30] - ), - common_definitions.CpuBenchmarkConfig( - model=jax_models.BERT_LARGE_FP32_JAX_384XI32_BATCHES[32], threads=[30] - ), - common_definitions.CpuBenchmarkConfig( - model=jax_models.BERT_LARGE_FP32_JAX_384XI32_BATCHES[64], threads=[30] - ), - # TODO(#17344): regenerate .mlirbc files - # common_definitions.CpuBenchmarkConfig( - # model=torch_models.BERT_LARGE_384_FP32_TORCH_BATCHES[24], threads=[30] - # ), - # common_definitions.CpuBenchmarkConfig( - # model=torch_models.BERT_LARGE_384_FP32_TORCH_BATCHES[48], threads=[30] - # ), - common_definitions.CpuBenchmarkConfig( - model=jax_models.RESNET50_FP32_JAX_3X224X224XF32_BATCHES[1], threads=[30] - ), - common_definitions.CpuBenchmarkConfig( - model=jax_models.RESNET50_FP32_JAX_3X224X224XF32_BATCHES[64], threads=[30] - ), - common_definitions.CpuBenchmarkConfig( - model=jax_models.RESNET50_FP32_JAX_3X224X224XF32_BATCHES[128], threads=[30] - ), - common_definitions.CpuBenchmarkConfig( - model=jax_models.T5_LARGE_FP32_JAX_512XI32_BATCHES[1], threads=[30] - ), - common_definitions.CpuBenchmarkConfig( - model=jax_models.T5_LARGE_FP32_JAX_512XI32_BATCHES[16], threads=[30] - ), - common_definitions.CpuBenchmarkConfig( - model=jax_models.T5_LARGE_FP32_JAX_512XI32_BATCHES[32], threads=[30] - ), -] - -# Microkernels. - -MICRO_MATMUL = [ - matmul.MATMUL_3456X1024X2048_FP16_MLIR, - matmul.MATMUL_3456X1024X2048_FP32_MLIR, - matmul.MATMUL_2560X2560X2560_FP16_MLIR, - matmul.MATMUL_2560X2560X2560_FP32_MLIR, - matmul.MATMUL_2564x2564x2564_FP32_MLIR, - matmul.MATMUL_2562x2564x2562_FP32_MLIR, - matmul.MATMUL_2562x2561x2561_FP32_MLIR, - matmul.MATMUL_123x2561x2561_FP32_MLIR, -] - -MICRO_MATMUL_SPLITK = [ - matmul.MATMUL_128X256X8192_FP16_MLIR, - matmul.MATMUL_128X256X8192_FP32_MLIR, -] - -# GPU model groups. - -CUDA_MODELS = [ - tf_models.EFFICIENTNET_V2_S_FP32, - tf_models.MINILM_L12_H384_UNCASED_INT32_SEQLEN128, - tf_models.BERT_FOR_MASKED_LM_FP32_SEQLEN512, - tf_models.BERT_LARGE_TF_FP32_SEQLEN384, - # PyTorch model are disabled due to https://github.com/iree-org/iree/issues/14993. - # torch_models.MODEL_CLIP_TEXT_SEQLEN64_FP32_TORCH, - # torch_models.MODEL_UNET_2D_FP32_TORCH, -] - -VULKAN_MODELS = [ - # PyTorch model are disabled due to https://github.com/iree-org/iree/issues/14993. - # torch_models.MODEL_CLIP_TEXT_SEQLEN64_FP32_TORCH, - # torch_models.MODEL_UNET_2D_FP32_TORCH, -] diff --git a/build_tools/python/e2e_test_framework/models/tf_models.py b/build_tools/python/e2e_test_framework/models/tf_models.py deleted file mode 100644 index c616aa168e72..000000000000 --- a/build_tools/python/e2e_test_framework/models/tf_models.py +++ /dev/null @@ -1,150 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines Tensorflow models.""" - -import string - -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions -import e2e_test_framework.models.utils as model_utils - -TF_MODELS_MANUAL_ROOT_DIR = ( - "https://storage.googleapis.com/iree-model-artifacts/tensorflow/manual" -) - -MINILM_L12_H384_UNCASED_INT32_SEQLEN128 = common_definitions.Model( - id=unique_ids.MODEL_MINILM_L12_H384_UNCASED_INT32_SEQLEN128, - name="MiniLML12H384Uncased", - tags=["int32", "seqlen128"], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - # Converted from https://huggingface.co/microsoft/MiniLM-L12-H384-uncased/commit/44acabbec0ef496f6dbc93adadea57f376b7c0ec - source_url=f"{TF_MODELS_MANUAL_ROOT_DIR}/MiniLML12H384Uncased_5aed9c3c3dfe8247ce76b74d518fa570b94dc0c3732631734d02ad70e4c74867.mlirbc", - entry_function="predict", - input_types=["1x128xi32", "1x128xi32", "1x128xi32"], -) - -BERT_FOR_MASKED_LM_FP32_SEQLEN512 = common_definitions.Model( - id=unique_ids.MODEL_BERT_FOR_MASKED_LM_FP32_SEQLEN512_TF, - name="BertForMaskedLMTF", - tags=["fp32", "seqlen512", "tensorflow"], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - # Converted from https://huggingface.co/transformers/v3.0.2/model_doc/bert.html#tfbertformaskedlm - source_url=f"{TF_MODELS_MANUAL_ROOT_DIR}/BertForMaskedLMTF_e757a10b24f6ff83aaae0ceb5bb05d4efe9ff3e9931f8e9a29f12bc5c2e42b5e.mlirbc", - entry_function="forward", - input_types=["1x512xi32", "1x512xi32"], -) - -EFFICIENTNET_V2_S_FP32 = common_definitions.Model( - id=unique_ids.MODEL_EFFICIENTNET_V2_S_FP32_TF, - name="EfficientNetV2STF", - tags=["fp32", "cnn", "tensorflow"], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - # Converted from https://github.com/keras-team/keras/blob/v2.10.0/keras/applications/efficientnet_v2.py - source_url=f"{TF_MODELS_MANUAL_ROOT_DIR}/EfficientNetV2STF_1af8c88f4e64e388a0c87bbeddcfb888084059df30cd631340d51794a0796e0f.mlirbc", - entry_function="forward", - input_types=["1x384x384x3xf32"], -) - -# This is the model used in the MLPerf Inference Suite. -BERT_LARGE_TF_FP32_SEQLEN384 = common_definitions.Model( - id=unique_ids.MODEL_BERT_LARGE_TF_FP32_SEQLEN384, - name="BertLargeTF", - tags=["fp32", "seqlen384", "tensorflow", "bert-variant", "batch-1"], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - # Derived from https://github.com/mlcommons/inference/tree/master/language/bert - # Instructions on how to regenerate the model: https://gist.github.com/mariecwhite/e61ccebd979d98d097946ac7725bcc29 - source_url=f"{TF_MODELS_MANUAL_ROOT_DIR}/BertLargeTF_000793afb016fb3afc559304bcb3ba6cdb2df1825e8976ca236c07c12e4f65fa.mlirbc", - entry_function="serving_default", - input_types=["1x384xi32", "1x384xi32", "1x384xi32"], -) - -GPT2_117M_1x4_FP32_TF = common_definitions.Model( - id=unique_ids.MODEL_GPT2_117M_1x4_FP32_TF, - name="GPT2_117M_TF_1X4XI32", - tags=["fp32", "tensorflow", "gpt2", "batch-1"], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url="https://storage.googleapis.com/iree-shared-files/tf_gpt2/static_input_seqlen5/stablehlo.mlir", - entry_function="forward", - input_types=["1x4xi32", "1x4xi32"], -) - -GPT2_117M_1x1_FP32_TF = common_definitions.Model( - id=unique_ids.MODEL_GPT2_117M_1x1_FP32_TF, - name="GPT2_117M_TF_1X1XI32", - tags=["fp32", "tensorflow", "gpt2", "batch-1"], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url="https://storage.googleapis.com/iree-shared-files/tf_gpt2/static_input_seqlen1/stablehlo.mlir", - entry_function="forward", - input_types=["1x1xi32", "12x2x1x12x4x64xf32"], -) - -TF_MODELS_ROOT_DIR = "https://storage.googleapis.com/iree-model-artifacts/tensorflow/tf_models_2.15.0.dev20230817_1692333975j" - -ID_FORMAT = string.Template("${model_id}-batch-${batch_size}") -NAME_FORMAT = string.Template("${name}Batch${batch_size}") -SOURCE_URL_FORMAT = string.Template( - TF_MODELS_ROOT_DIR + "/${directory}_BATCH${batch_size}/stablehlo.mlirbc" -) - -# Derived from https://huggingface.co/docs/transformers/model_doc/bert#transformers.TFBertModel. -BERT_LARGE_384_FP32_TF_BATCHES = model_utils.generate_batch_models( - id_template=model_utils.partial_template_substitute( - ID_FORMAT, model_id=unique_ids.MODEL_BERT_LARGE_384_FP32_TF - ), - name_template=model_utils.partial_template_substitute( - NAME_FORMAT, name="BertLargeTF" - ), - tags=["fp32", "seqlen384", "tensorflow", "bert-variant"], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url_template=model_utils.partial_template_substitute( - SOURCE_URL_FORMAT, directory="BERT_LARGE_FP32_TF_384XI32" - ), - entry_function="forward", - input_type_templates=[ - string.Template("${batch_size}x384xi32"), - string.Template("${batch_size}x384xi32"), - ], - batch_sizes=[1, 16, 24, 32, 48, 64, 512, 1024, 1280], -) - -# Converted from https://www.tensorflow.org/api_docs/python/tf/keras/applications/resnet50/ResNet50 -RESNET50_3X224X224_FP32_TF_BATCHES = model_utils.generate_batch_models( - id_template=model_utils.partial_template_substitute( - ID_FORMAT, model_id=unique_ids.MODEL_RESNET50_3X224X224_FP32_TF - ), - name_template=model_utils.partial_template_substitute( - NAME_FORMAT, name="Resnet50TF" - ), - tags=["fp32", "cnn"], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url_template=model_utils.partial_template_substitute( - SOURCE_URL_FORMAT, directory="RESNET50_FP32_TF_224X224X3XF32" - ), - entry_function="forward", - input_type_templates=[string.Template("${batch_size}x3x224x224xf32")], - batch_sizes=[1, 8, 64, 128, 256, 2048], -) - -# Derived from https://huggingface.co/transformers/v3.0.2/model_doc/t5.html#tft5model. -T5_LARGE_512_FP32_TF_BATCHES = model_utils.generate_batch_models( - id_template=model_utils.partial_template_substitute( - ID_FORMAT, model_id=unique_ids.MODEL_T5_LARGE_512_FP32_TF - ), - name_template=model_utils.partial_template_substitute( - NAME_FORMAT, name="T5LargeTF" - ), - tags=["fp32", "seqlen512", "tensorflow"], - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - source_url_template=model_utils.partial_template_substitute( - SOURCE_URL_FORMAT, directory="T5_LARGE_FP32_TF_512XI32" - ), - entry_function="forward", - input_type_templates=[ - string.Template("${batch_size}x512xi32"), - string.Template("${batch_size}x512xi32"), - ], - batch_sizes=[1, 16, 24, 32, 48, 64, 512], -) diff --git a/build_tools/python/e2e_test_framework/models/tflite_models.py b/build_tools/python/e2e_test_framework/models/tflite_models.py deleted file mode 100644 index 77a9f2fb1eb8..000000000000 --- a/build_tools/python/e2e_test_framework/models/tflite_models.py +++ /dev/null @@ -1,152 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines TFLite models.""" - -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions - -DEEPLABV3_FP32 = common_definitions.Model( - id=unique_ids.MODEL_DEEPLABV3_FP32, - name="DeepLabV3_fp32", - tags=["fp32"], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - # Mirror of https://tfhub.dev/tensorflow/lite-model/deeplabv3/1/default/1 - source_url="https://storage.googleapis.com/iree-model-artifacts/deeplabv3.tflite", - entry_function="main", - input_types=["1x257x257x3xf32"], -) - -MOBILESSD_FP32 = common_definitions.Model( - id=unique_ids.MODEL_MOBILESSD_FP32, - name="MobileSSD_fp32", - tags=["fp32"], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - # Mirror of https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/mobile_ssd_v2_float_coco.tflite - source_url="https://storage.googleapis.com/iree-model-artifacts/mobile_ssd_v2_float_coco.tflite", - entry_function="main", - input_types=["1x320x320x3xf32"], -) - -POSENET_FP32 = common_definitions.Model( - id=unique_ids.MODEL_POSENET_FP32, - name="PoseNet_fp32", - tags=["fp32"], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - # Mirror of https://tfhub.dev/tensorflow/lite-model/posenet/mobilenet/float/075/1/default/1 - source_url="https://storage.googleapis.com/iree-model-artifacts/posenet.tflite", - entry_function="main", - input_types=["1x353x257x3xf32"], -) - -MOBILEBERT_FP32 = common_definitions.Model( - id=unique_ids.MODEL_MOBILEBERT_FP32, - name="MobileBertSquad_fp32", - tags=["fp32"], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - # Mirror of https://tfhub.dev/iree/lite-model/mobilebert/fp32/1 - source_url="https://storage.googleapis.com/iree-model-artifacts/mobilebert-baseline-tf2-float.tflite", - entry_function="main", - input_types=["1x384xi32", "1x384xi32", "1x384xi32"], -) - -MOBILEBERT_INT8 = common_definitions.Model( - id=unique_ids.MODEL_MOBILEBERT_INT8, - name="MobileBertSquad_int8", - tags=["int8"], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - # Mirror of https://tfhub.dev/iree/lite-model/mobilebert/int8/1 - source_url="https://storage.googleapis.com/iree-model-artifacts/mobilebert-baseline-tf2-quant.tflite", - entry_function="main", - input_types=["1x384xi32", "1x384xi32", "1x384xi32"], -) - -MOBILEBERT_FP16 = common_definitions.Model( - id=unique_ids.MODEL_MOBILEBERT_FP16, - name="MobileBertSquad_fp16", - tags=["fp16"], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - # Mirror of https://tfhub.dev/tensorflow/lite-model/mobilebert/1/default/1 - source_url="https://storage.googleapis.com/iree-model-artifacts/mobilebertsquad.tflite", - entry_function="main", - input_types=["1x384xi32", "1x384xi32", "1x384xi32"], -) - -MOBILENET_V1 = common_definitions.Model( - id=unique_ids.MODEL_MOBILENET_V1, - name="MobileNetV1_fp32", - tags=["fp32", "imagenet"], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - # Mirror of https://tfhub.dev/iree/lite-model/mobilenet_v1_100_224/fp32/1 - source_url="https://storage.googleapis.com/iree-model-artifacts/mobilenet_v1_224_1.0_float.tflite", - entry_function="main", - input_types=["1x224x224x3xf32"], -) - -MOBILENET_V2 = common_definitions.Model( - id=unique_ids.MODEL_MOBILENET_V2, - name="MobileNetV2_fp32", - tags=["fp32", "imagenet"], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - # Mirror of https://github.com/tensorflow/tflite-support/blob/master/tensorflow_lite_support/metadata/python/tests/testdata/image_classifier/mobilenet_v2_1.0_224.tflite - source_url="https://storage.googleapis.com/iree-model-artifacts/mobilenet_v2_1.0_224.tflite", - entry_function="main", - input_types=["1x224x224x3xf32"], -) - -MOBILENET_V3SMALL = common_definitions.Model( - id=unique_ids.MODEL_MOBILENET_V3SMALL, - name="MobileNetV3Small_fp32", - tags=["fp32", "imagenet"], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - # https://tfhub.dev/google/imagenet/mobilenet_v3_small_100_224/classification/5 - # Manually exported to tflite with static batch dimension - source_url="https://storage.googleapis.com/iree-model-artifacts/MobileNetV3SmallStaticBatch.tflite", - entry_function="main", - input_types=["1x224x224x3xf32"], -) - -PERSON_DETECT_INT8 = common_definitions.Model( - id=unique_ids.MODEL_PERSON_DETECT_INT8, - name="PersonDetect_int8", - tags=["int8"], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - # Mirror of https://github.com/tensorflow/tflite-micro/raw/aeac6f39e5c7475cea20c54e86d41e3a38312546/tensorflow/lite/micro/models/person_detect.tflite - source_url="https://storage.googleapis.com/iree-model-artifacts/person_detect.tflite", - entry_function="main", - input_types=["1x96x96x1xi8"], -) - -EFFICIENTNET_INT8 = common_definitions.Model( - id=unique_ids.MODEL_EFFICIENTNET_INT8, - name="EfficientNet_int8", - tags=["int8"], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - # Mirror of https://tfhub.dev/tensorflow/lite-model/efficientnet/lite0/int8/2 - source_url="https://storage.googleapis.com/iree-model-artifacts/efficientnet_lite0_int8_2.tflite", - entry_function="main", - input_types=["1x224x224x3xui8"], -) - -MOBILENET_V2_INT8 = common_definitions.Model( - name="MobileNetV2_int8", - id=unique_ids.MODEL_MOBILENET_V2_INT8, - tags=["int8", "imagenet"], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - # Mirror of https://tfhub.dev/tensorflow/lite-model/mobilenet_v2_1.0_224_quantized/1/default/1 - source_url="https://storage.googleapis.com/iree-model-artifacts/mobilenet_v2_1.0_224_quantized.tflite", - entry_function="main", - input_types=["1x224x224x3xui8"], -) - -VIT_INT8_TFL = common_definitions.Model( - name="Vit_int8", - id=unique_ids.MODEL_VIT_INT8_TFL, - tags=["int8", "vision-transformer"], - source_type=common_definitions.ModelSourceType.EXPORTED_TFLITE, - source_url="https://storage.googleapis.com/iree-model-artifacts/tflite/tflite_models_1698315913/VIT_CLASSIFICATION_INT8_TFLITE_3X224X224XINT8/model_int8.tflite", - entry_function="main", - input_types=["1x3x224x224xi8"], -) diff --git a/build_tools/python/e2e_test_framework/models/torch_models.py b/build_tools/python/e2e_test_framework/models/torch_models.py deleted file mode 100644 index 59cd48d79288..000000000000 --- a/build_tools/python/e2e_test_framework/models/torch_models.py +++ /dev/null @@ -1,141 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Defines PyTorch models.""" - -import string - -from e2e_test_framework import unique_ids -from e2e_test_framework.definitions import common_definitions -import e2e_test_framework.models.utils as model_utils - -# Implementations of the models listed below can be found in -# https://github.com/iree-org/iree-experimental/tree/main/iree-torch/library/models. -# We import the PyTorch models offline and make the .mlir available here for benchmarking. -# If the mlir artifacts need to be updated, please run -# https://github.com/iree-org/iree-experimental/blob/main/iree-torch/library/import_torch_models.sh -# Then update the `source_url` below with the new paths. - -# `ClipTextModel` encodes text into an embedding. -# -# Used in Stable Diffusion to convert a text prompt into an embedding for input to the `Unet2d` model. -# -# Converted from https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel -MODEL_CLIP_TEXT_SEQLEN64_FP32_TORCH = common_definitions.Model( - id=unique_ids.MODEL_CLIP_TEXT_SEQLEN64_FP32_TORCH, - name="ClipTextSeqLen64PT", - tags=["fp32", "seqlen64"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230307.103_1678163233/SD_CLIP_TEXT_MODEL_SEQLEN64/linalg.mlir", - entry_function="forward", - input_types=["1x77xi64", "1x77xi64"], -) - -# `Unet2d` consists of `ResNet` encoder and decoder blocks with cross-attention layers. -# -# Takes 2 inputs: -# i) a text embedding generated from a language model like `ClipTextModel` -# ii) an image seed (either random noise or an actual image) in the latent space (compressed representation of an image). -# Outputs an image in the latent space. -# -# Used in Stable Diffusion to gradually subtract noise in the latent space. -# Usually run over multiple steps until the image is sufficiently de-noised. -# -# Once complete, output is upsampled using a Variation Auto Encoder (`VAE`). -# -# Converted from https://huggingface.co/docs/diffusers/api/models#diffusers.UNet2DConditionModel -MODEL_UNET_2D_FP32_TORCH = common_definitions.Model( - id=unique_ids.MODEL_UNET_2D_FP32_TORCH, - name="Unet2dPT", - tags=["fp32"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230307.103_1678163233/SD_UNET_MODEL/linalg.mlir", - entry_function="forward", - input_types=["1x4x64x64xf32", "1x77x768xf32"], -) - -# Converted from https://pytorch.org/vision/stable/models/generated/torchvision.models.efficientnet_v2_s.html#torchvision.models.efficientnet_v2_s -EFFICIENTNET_V2_S_FP32_TORCH = common_definitions.Model( - id=unique_ids.MODEL_EFFICIENTNET_V2_S_FP32_TORCH, - name="EfficientNetV2SPT", - tags=["fp32", "cnn", "depthwise-conv"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/EFFICIENTNET_V2_S/batch_1/linalg.mlir", - entry_function="forward", - input_types=["1x3x384x384xf32"], -) - -# FP16 -EFFICIENTNET_V2_S_FP16_TORCH = common_definitions.Model( - id=unique_ids.MODEL_EFFICIENTNET_V2_S_FP16_TORCH, - name="EfficientNetV2Sfp16PT", - tags=["fp16", "cnn", "depthwise-conv"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230522.846_1684831160/EFFICIENTNET_V2_S_FP16/batch_1/linalg.mlir", - entry_function="forward", - input_types=["1x3x384x384xf16"], -) - -# Converted from https://pytorch.org/vision/stable/models/generated/torchvision.models.efficientnet_b7.html#torchvision.models.efficientnet_b7 -EFFICIENTNET_B7_FP32_TORCH = common_definitions.Model( - id=unique_ids.MODEL_EFFICIENTNET_B7_FP32_TORCH, - name="EfficientNetB7PT", - tags=["fp32", "cnn", "depthwise-conv"], - source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, - source_url="https://storage.googleapis.com/iree-model-artifacts/pytorch/torch_models_20230321.784_1679461251/EFFICIENTNET_B7/batch_1/linalg.mlir", - entry_function="forward", - input_types=["1x3x600x600xf32"], -) - -ID_FORMAT = string.Template("${model_id}-batch-${batch_size}") -NAME_FORMAT = string.Template("${name}Batch${batch_size}") -BERT_LARGE_FP32_URL = string.Template( - "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20240124.1093_1706139741/BERT_LARGE_FP32_PT_384XI32_BATCH${batch_size}/linalg.mlirbc" -) -BERT_LARGE_FP16_URL = string.Template( - "https://storage.googleapis.com/iree-model-artifacts/pytorch/pt_models_20230816.932_1692245822/BERT_LARGE_FP16_PT_384XI32_BATCH${batch_size}/linalg.mlirbc" -) - -# Converted from https://huggingface.co/docs/transformers/v4.27.2/en/model_doc/bert#transformers.BertModel -# TODO(#17344): regenerate .mlirbc files -# BERT_LARGE_384_FP32_TORCH_BATCHES = model_utils.generate_batch_models( -# id_template=model_utils.partial_template_substitute( -# ID_FORMAT, model_id=unique_ids.MODEL_BERT_LARGE_384_FP32_TORCH -# ), -# name_template=model_utils.partial_template_substitute( -# NAME_FORMAT, name="BertLargePT" -# ), -# tags=["fp32", "transformer", "seqlen384"], -# source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, -# source_url_template=BERT_LARGE_FP32_URL, -# entry_function="forward", -# input_type_templates=[ -# string.Template("${batch_size}x384xi64"), -# string.Template("${batch_size}x384xi64"), -# ], -# batch_sizes=[1, 16, 24, 32, 48, 64, 512, 1024, 1280], -# ) - -# TODO(#17344): regenerate .mlirbc files -# FALCON7B_1X100XI64_GPTQ_TORCH = common_definitions.Model( -# id=unique_ids.MODEL_FALCON7B_1X100XI64_GPTQ_TORCH, -# name="Falcon7bGptqPT", -# tags=["gptq", "transformer"], -# source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, -# source_url="https://storage.googleapis.com/iree-model-artifacts/pytorch/manual/falcon_7b_gptq_linalg_1702432230.mlirbc", -# entry_function="forward", -# input_types=["1x100xi64"], -# ) - -# TODO(#17344): regenerate .mlirbc files -# FALCON7B_INT4_1X100XI64_GPTQ_TORCH = common_definitions.Model( -# id=unique_ids.MODEL_FALCON7B_INT4_1X100XI64_GPTQ_TORCH, -# name="Falcon7bInt4GptqPT", -# tags=["gptq", "transformer", "int4"], -# source_type=common_definitions.ModelSourceType.EXPORTED_LINALG_MLIR, -# source_url="https://storage.googleapis.com/iree-model-artifacts/pytorch/manual/falcon_7b_gptq_linalg_int4_1702863828.mlirbc", -# entry_function="forward", -# input_types=["1x100xi64"], -# ) diff --git a/build_tools/python/e2e_test_framework/models/utils.py b/build_tools/python/e2e_test_framework/models/utils.py deleted file mode 100644 index 53f96b38d890..000000000000 --- a/build_tools/python/e2e_test_framework/models/utils.py +++ /dev/null @@ -1,56 +0,0 @@ -## Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Utilities to help define models.""" - -import string -from typing import Dict, Sequence - -from e2e_test_framework.definitions import common_definitions - - -def partial_template_substitute( - template: string.Template, **substitutions -) -> string.Template: - """Partially substitutes keywords in the template and returns a template.""" - return string.Template(template.safe_substitute(**substitutions)) - - -def generate_batch_models( - id_template: string.Template, - name_template: string.Template, - tags: Sequence[str], - source_type: common_definitions.ModelSourceType, - source_url_template: string.Template, - entry_function: str, - input_type_templates: Sequence[string.Template], - batch_sizes: Sequence[int], -) -> Dict[int, common_definitions.Model]: - """Generate model definitions for different batch sizes by substituting - ${batch_size}` in the template strings. - - Only `*_template` parameters will be treated as templates and substituted. A - `batch-` tag will be appended to the tags in each returned model. - - Returns: - Map of batch size to model. - """ - model_map = {} - for batch_size in batch_sizes: - substituted_input_types = [ - input_type.substitute(batch_size=batch_size) - for input_type in input_type_templates - ] - model_map[batch_size] = common_definitions.Model( - id=id_template.substitute(batch_size=batch_size), - name=name_template.substitute(batch_size=batch_size), - tags=list(tags) + [f"batch-{batch_size}"], - source_type=source_type, - source_url=source_url_template.substitute(batch_size=batch_size), - entry_function=entry_function, - input_types=substituted_input_types, - ) - - return model_map diff --git a/build_tools/python/e2e_test_framework/models/utils_test.py b/build_tools/python/e2e_test_framework/models/utils_test.py deleted file mode 100644 index 6d6034d4a80a..000000000000 --- a/build_tools/python/e2e_test_framework/models/utils_test.py +++ /dev/null @@ -1,86 +0,0 @@ -## Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import string -import unittest - -from e2e_test_framework.definitions import common_definitions -import e2e_test_framework.models.utils as model_utils - - -class UtilsTest(unittest.TestCase): - def test_partial_template_substitute(self): - template = string.Template("${name}-${batch_size}") - - result = model_utils.partial_template_substitute(template, name="xyz") - - self.assertEqual(result.substitute(batch_size=10), "xyz-10") - - def test_generate_batch_models(self): - models = model_utils.generate_batch_models( - id_template=string.Template("1234-${batch_size}"), - name_template=string.Template("model-batch-${batch_size}"), - tags=["abc"], - source_url_template=string.Template( - "https://example.com/x/${batch_size}.mlir" - ), - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - entry_function="forward", - input_type_templates=[ - string.Template("${batch_size}x128"), - string.Template("${batch_size}x256"), - ], - batch_sizes=[1, 4], - ) - - self.assertEqual( - models, - { - 1: common_definitions.Model( - id="1234-1", - name="model-batch-1", - tags=["abc", "batch-1"], - source_url="https://example.com/x/1.mlir", - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - entry_function="forward", - input_types=["1x128", "1x256"], - ), - 4: common_definitions.Model( - id="1234-4", - name="model-batch-4", - tags=["abc", "batch-4"], - source_url="https://example.com/x/4.mlir", - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - entry_function="forward", - input_types=["4x128", "4x256"], - ), - }, - ) - - def test_generate_batch_models_missing_substitution(self): - id_template_with_unknown = string.Template("1234-${unknown}-${batch_size}") - - self.assertRaises( - KeyError, - lambda: model_utils.generate_batch_models( - id_template=id_template_with_unknown, - name_template=string.Template("model-batch-${batch_size}"), - tags=["abc"], - source_url_template=string.Template( - "https://example.com/x/${batch_size}.mlir" - ), - source_type=common_definitions.ModelSourceType.EXPORTED_STABLEHLO_MLIR, - entry_function="forward", - input_type_templates=[ - string.Template("${batch_size}x128"), - ], - batch_sizes=[1, 4], - ), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/python/e2e_test_framework/serialization.py b/build_tools/python/e2e_test_framework/serialization.py deleted file mode 100644 index b62078e11b03..000000000000 --- a/build_tools/python/e2e_test_framework/serialization.py +++ /dev/null @@ -1,253 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Helpers to serialize/deserialize objects.""" - -from enum import Enum -from typing import Any, Dict, Optional, Sequence, Tuple, Type, TypeVar, Union -import dataclasses -import typing - -# types.NoneType is only added after Python 3.10. -NONE_TYPE = type(None) -SERIALIZE_FUNC_NAME = "__serialize__" -DESERIALIZE_FUNC_NAME = "__deserialize__" -SUPPORTED_DICT_KEY_TYPES = {str, int, float, bool} -SUPPORTED_PRIMITIVE_TYPES = {str, int, float, bool, NONE_TYPE} - - -def serialize_and_pack( - obj, root_obj_field_name="root_obj", keyed_obj_map_field_name="keyed_obj_map" -): - """Converts and packs the object into a serializable object. - - Args: - obj: object to be serialized. - root_obj_field_name: field name of the top-level object in the return dict. - keyed_obj_map_field_name: field name of the keyed object map in the return - dict. - Returns - A serializable dict. - """ - - if root_obj_field_name == keyed_obj_map_field_name: - raise ValueError(f"root_obj and keyed_obj_map can't have the same field name.") - - keyed_obj_map = {} - root_obj = _serialize(obj=obj, keyed_obj_map=keyed_obj_map) - return {root_obj_field_name: root_obj, keyed_obj_map_field_name: keyed_obj_map} - - -T = TypeVar("T") - - -def unpack_and_deserialize( - data, - root_type: Type[T], - root_obj_field_name="root_obj", - keyed_obj_map_field_name="keyed_obj_map", -) -> T: - """Unpacks and deserializes the data back to the typed object. - - Args: - data: serialized data dict. - root_type: top-level object type of the data. - root_obj_field_name: field name of the top-level object in the dict. - keyed_obj_map_field_name: field name of the keyed object map in the dict. - Returns: - A deserialized object. - """ - obj = _deserialize( - data=data[root_obj_field_name], - obj_type=root_type, - keyed_obj_map=data[keyed_obj_map_field_name], - ) - return typing.cast(root_type, obj) - - -def _serialize(obj, keyed_obj_map: Dict[str, Any]): - """Converts the object into a serializable object. - - Args: - obj: object to be serialized. - keyed_obj_map: mutable container to store the keyed serializable object. - Returns - A serializable object. - """ - - serialize_func = getattr(obj, SERIALIZE_FUNC_NAME, None) - if serialize_func is not None: - return serialize_func(keyed_obj_map) - - elif isinstance(obj, list): - return [_serialize(value, keyed_obj_map) for value in obj] - - elif isinstance(obj, Enum): - return obj.name - - elif isinstance(obj, dict): - result_dict = {} - for key, value in obj.items(): - if type(key) not in SUPPORTED_DICT_KEY_TYPES: - raise ValueError(f"Unsupported key {key} in the dict {obj}.") - result_dict[key] = _serialize(value, keyed_obj_map) - return result_dict - - elif type(obj) in SUPPORTED_PRIMITIVE_TYPES: - return obj - - raise ValueError(f"Unsupported object: {obj}.") - - -def _deserialize( - data, obj_type: Type, keyed_obj_map: Dict[str, Any], obj_cache: Dict[str, Any] = {} -): - """Deserializes the data back to the typed object. - - Args: - data: serialized data. - obj_type: type of the data. - keyed_obj_map: container of the keyed serializable object. - Returns: - A deserialized object. - """ - - deserialize_func = getattr(obj_type, DESERIALIZE_FUNC_NAME, None) - if deserialize_func is not None: - return deserialize_func(data, keyed_obj_map, obj_cache) - - elif typing.get_origin(obj_type) == list: - (subtype,) = typing.get_args(obj_type) - return [_deserialize(item, subtype, keyed_obj_map, obj_cache) for item in data] - - elif typing.get_origin(obj_type) == dict: - _, value_type = typing.get_args(obj_type) - return dict( - (key, _deserialize(value, value_type, keyed_obj_map, obj_cache)) - for key, value in data.items() - ) - - elif typing.get_origin(obj_type) == Union: - subtypes = typing.get_args(obj_type) - if len(subtypes) != 2 or NONE_TYPE not in subtypes: - raise ValueError(f"Unsupported union type: {obj_type}.") - if data is None: - return None - subtype = subtypes[0] if subtypes[1] == NONE_TYPE else subtypes[1] - return _deserialize(data, subtype, keyed_obj_map, obj_cache) - - elif issubclass(obj_type, Enum): - for member in obj_type: - if data == member.name: - return member - raise ValueError(f"Member {data} not found in the enum {obj_type}.") - - return data - - -def serializable(cls=None, type_key: Optional[str] = None, id_field: str = "id"): - """Decorator to make a dataclass serializable. - - Args: - type_key: string defines the object type and indeicates that the class is a - keyed object, which is unique per id and will only have one copy in the - serialization per id. - id_field: field name of the id field of a keyed object. - - Example: - @serializable - @dataclass - class A(object): - ... - - @serialzable(type_key="obj_b") - @dataclass - class B(object): - id: str - """ - - if type_key is not None and ":" in type_key: - raise ValueError("':' is the reserved character in type_key.") - - def wrap(cls): - if not dataclasses.is_dataclass(cls): - raise ValueError(f"{cls} is not a dataclass.") - - fields = dataclasses.fields(cls) - if type_key is not None and all(field.name != id_field for field in fields): - raise ValueError(f'Id field "{id_field}" not found in the class {cls}.') - - def serialize(self, keyed_obj_map: Dict[str, Any]): - if type_key is None: - return _fields_to_dict(self, fields, keyed_obj_map) - - obj_id = getattr(self, id_field) - obj_key = f"{type_key}:{obj_id}" - if obj_key in keyed_obj_map: - # If the value in the map is None, it means we have visited this object - # before but not yet finished serializing it. This will only happen if - # there is a circular reference. - if keyed_obj_map[obj_key] is None: - raise ValueError(f"Circular reference is not supported: {obj_key}.") - return obj_id - - # Populate the keyed_obj_map with None first to detect circular reference. - keyed_obj_map[obj_key] = None - obj_dict = _fields_to_dict(self, fields, keyed_obj_map) - keyed_obj_map[obj_key] = obj_dict - return obj_id - - def deserialize(data, keyed_obj_map: Dict[str, Any], obj_cache: Dict[str, Any]): - if type_key is None: - field_value_map = _dict_to_fields( - data, fields, keyed_obj_map, obj_cache - ) - return cls(**field_value_map) - - obj_id = data - obj_key = f"{type_key}:{obj_id}" - if obj_key in obj_cache: - return obj_cache[obj_key] - - field_value_map = _dict_to_fields( - keyed_obj_map[obj_key], fields, keyed_obj_map, obj_cache - ) - derialized_obj = cls(**field_value_map) - obj_cache[obj_key] = derialized_obj - return derialized_obj - - setattr(cls, SERIALIZE_FUNC_NAME, serialize) - setattr(cls, DESERIALIZE_FUNC_NAME, deserialize) - return cls - - # Trick to allow the decoration with `@serializable(...)`. In that case, - # `serializable` is called without cls and should return a decorator. - if cls is None: - return wrap - return wrap(cls) - - -def _fields_to_dict( - obj, fields: Sequence[dataclasses.Field], keyed_obj_map: Dict[str, Any] -) -> Dict[str, Any]: - return dict( - (field.name, _serialize(getattr(obj, field.name), keyed_obj_map)) - for field in fields - ) - - -def _dict_to_fields( - obj_dict, - fields: Sequence[dataclasses.Field], - keyed_obj_map: Dict[str, Any], - obj_cache: Dict[str, Any], -) -> Dict[str, Any]: - return dict( - ( - field.name, - _deserialize(obj_dict[field.name], field.type, keyed_obj_map, obj_cache), - ) - for field in fields - ) diff --git a/build_tools/python/e2e_test_framework/serialization_test.py b/build_tools/python/e2e_test_framework/serialization_test.py deleted file mode 100644 index ba687b267d7b..000000000000 --- a/build_tools/python/e2e_test_framework/serialization_test.py +++ /dev/null @@ -1,185 +0,0 @@ -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -from dataclasses import dataclass -from typing import List, Optional -import json -import pathlib -import enum -import typing -import unittest - -from e2e_test_framework import serialization - - -class EnumX(enum.Enum): - OPTION_A = "a" - OPTION_B = "b" - OPTION_C = "c" - - -@serialization.serializable -@dataclass -class TestC(object): - float_val: float - - -@serialization.serializable(type_key="test_b", id_field="key") -@dataclass -class TestB(object): - key: str - int_val: int - - -@serialization.serializable -@dataclass -class TestA(object): - b_list: List[TestB] - c_obj: TestC - str_val: Optional[str] - enum_val: EnumX - optional_val: Optional[TestB] - - -@serialization.serializable -@dataclass -class TestUnsupported(object): - path: pathlib.PurePath - - -@serialization.serializable(type_key="test_circular") -@dataclass -class TestCircularReference(object): - id: str - child: Optional["TestCircularReference"] - - -class SerializationTest(unittest.TestCase): - def test_serialize_and_pack(self): - b_obj_a = TestB(key="id_a", int_val=10) - b_obj_b = TestB(key="id_b", int_val=20) - test_objs = [ - TestA( - b_list=[b_obj_a, b_obj_b], - c_obj=TestC(float_val=0.1), - str_val="test1", - enum_val=EnumX.OPTION_B, - optional_val=None, - ), - TestA( - b_list=[b_obj_a], - c_obj=TestC(float_val=0.2), - str_val=None, - enum_val=EnumX.OPTION_C, - optional_val=b_obj_b, - ), - ] - - results = serialization.serialize_and_pack( - test_objs, - root_obj_field_name="main_obj", - keyed_obj_map_field_name="obj_map", - ) - - self.maxDiff = None - self.assertEqual( - results, - { - "main_obj": [ - dict( - b_list=["id_a", "id_b"], - c_obj=dict(float_val=0.1), - str_val="test1", - enum_val="OPTION_B", - optional_val=None, - ), - dict( - b_list=["id_a"], - c_obj=dict(float_val=0.2), - str_val=None, - enum_val="OPTION_C", - optional_val="id_b", - ), - ], - "obj_map": { - "test_b:id_a": dict(key="id_a", int_val=10), - "test_b:id_b": dict(key="id_b", int_val=20), - }, - }, - ) - - def test_serialize_and_pack_with_unsupported_type(self): - self.assertRaises( - ValueError, - lambda: serialization.serialize_and_pack( - TestUnsupported(path=pathlib.PurePath("abc")) - ), - ) - - def test_serialize_and_pack_with_unsupported_dict_key(self): - self.assertRaises( - ValueError, lambda: serialization.serialize_and_pack({(0, 0): "test"}) - ) - - def test_serialize_and_pack_with_circular_reference(self): - obj_a = TestCircularReference(id="0", child=None) - obj_b = TestCircularReference(id="1", child=obj_a) - obj_a.child = obj_b - - self.assertRaises(ValueError, lambda: serialization.serialize_and_pack(obj_a)) - - def test_roundtrip(self): - b_obj_a = TestB(key="id_a", int_val=10) - b_obj_b = TestB(key="id_b", int_val=20) - test_objs = [ - TestA( - b_list=[b_obj_a, b_obj_b], - c_obj=TestC(float_val=0.1), - str_val="test1", - enum_val=EnumX.OPTION_B, - optional_val=None, - ), - TestA( - b_list=[b_obj_a], - c_obj=TestC(float_val=0.2), - str_val=None, - enum_val=EnumX.OPTION_C, - optional_val=None, - ), - TestA( - b_list=[b_obj_b], - c_obj=TestC(float_val=0.3), - str_val="test3", - enum_val=EnumX.OPTION_A, - optional_val=b_obj_a, - ), - ] - - results = serialization.unpack_and_deserialize( - serialization.serialize_and_pack(test_objs), typing.List[TestA] - ) - - self.assertEqual(results, test_objs) - - def test_roundtrip_with_json(self): - b_obj_a = TestB(key="id_a", int_val=10) - b_obj_b = TestB(key="id_b", int_val=20) - - objs = { - "x": b_obj_a, - "y": b_obj_b, - } - - json_str = json.dumps(serialization.serialize_and_pack(objs)) - results = serialization.unpack_and_deserialize( - json.loads(json_str), typing.Dict[str, TestB] - ) - - self.assertEqual(results, objs) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/python/e2e_test_framework/unique_ids.py b/build_tools/python/e2e_test_framework/unique_ids.py deleted file mode 100644 index 0bc49e62e2b9..000000000000 --- a/build_tools/python/e2e_test_framework/unique_ids.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""List of unique random IDs in the framework and id utilities. - -Each ID should be generated from uuid.uuid4(). -""" - -import hashlib -from typing import Sequence - -# Special id which will be ignored when calculating the composite id. -# -# It should only be used when adding a new field to a composite object while -# we want to maintain the same id on the existing composite objects. -# -# In such case, you need to create a "default config" for the new field with -# this id and populate that config to the fields of the existing objects. The -# composite id computing function will ignore this id and keep the output -# unchanged. -TRANSPARENT_ID = "00000000-0000-0000-0000-000000000000" - - -def hash_composite_id(keys: Sequence[str]) -> str: - """Computes the composite hash id from string keys. - - String keys are the component ids that compose this composite object. We hash - the composite id since the id isn't designed to be inspected and insufficient - to reconstruct the original composite object. - - Note that the output is sensitive to the order of the keys, and any key == - TRANSPARENT_ID will be skipped. When adding a new key to the keys, the new key - should be always appended to the end. In this way, the composite id can be - unchanged for the existing composite object if they use TRANSPARENT_ID on the - new keyed field. - - The composite id is computed in the following steps: - 1. Index each key with its position in the list from 0. - 2. Remove any key == TRANSPARENT_ID - 3. Get the SHA256 hex digest of "0-key_0:1-key_1:..." - - Step 1 is needed to avoid the ambiguity between: - ["key_abc", TRANSPARENT_ID] and [TRANSPARENT_ID, "key_abc"] - since after removing TRANSPARENT_ID, they both become ["key_abc"] without the - position index. - - Args: - keys: list of string keys. - - Returns: - Unique composite id. - """ - trimmed_indexed_key = [ - f"{index}-{key}" for index, key in enumerate(keys) if key != TRANSPARENT_ID - ] - return hashlib.sha256(":".join(trimmed_indexed_key).encode("utf-8")).hexdigest() - - -# To generate an id, run `uuid.uuid4()`. - -# Models. -# TFLite. -MODEL_DEEPLABV3_FP32 = "c36c63b0-220a-4d78-8ade-c45ce47d89d3" -MODEL_MOBILESSD_FP32 = "0e466f69-91d6-4e50-b62b-a82b6213a231" -MODEL_POSENET_FP32 = "5afc3014-d29d-4e88-a840-fbaf678acf2b" -MODEL_MOBILEBERT_FP32 = "cc69d69f-6d1f-4a1a-a31e-e021888d0d28" -MODEL_MOBILEBERT_INT8 = "e3997104-a3d2-46b4-9fbf-39069906d123" -MODEL_MOBILEBERT_FP16 = "73a0402e-271b-4aa8-a6a5-ac05839ca569" -MODEL_MOBILENET_V1 = "78eab9e5-9ff1-4769-9b55-933c81cc9a0f" -MODEL_MOBILENET_V2 = "7d45f8e5-bb5e-48d0-928d-8f125104578f" -MODEL_MOBILENET_V3SMALL = "58855e40-eba9-4a71-b878-6b35e3460244" -MODEL_PERSON_DETECT_INT8 = "bc1338be-e3df-44fd-82e4-40ba9560a073" -MODEL_EFFICIENTNET_INT8 = "4a6f545e-1b4e-41a5-9236-792aa578184b" -MODEL_VIT_INT8_TFL = "227cf5aa-5e5c-411b-8c7d-ecc83e6128dd" - -# Tensorflow. -MODEL_MINILM_L12_H384_UNCASED_INT32_SEQLEN128 = "ecf5c970-ee97-49f0-a4ed-df1f34e9d493" -MODEL_BERT_FOR_MASKED_LM_FP32_SEQLEN512_TF = "39d157ad-f0ec-4a76-963b-d783beaed60f" -MODEL_EFFICIENTNET_V2_S_FP32_TF = "ebe7897f-5613-435b-a330-3cb967704e5e" -MODEL_BERT_LARGE_TF_FP32_SEQLEN384 = "8871f602-571c-4eb8-b94d-554cc8ceec5a" -MODEL_MOBILENET_V2_INT8 = "3dd5a95e-92a9-4486-9062-9a33224f28db" - -MODEL_RESNET50_3X224X224_FP32_TF = "9a5a8b8c-6e7a-4b51-bb4f-84e738957238" -MODEL_BERT_LARGE_384_FP32_TF = "5f3de3b3-fd00-4582-a97e-b70ff5edab07" -MODEL_T5_LARGE_512_FP32_TF = "587e595d-2adf-4e41-9617-43178a133725" - -MODEL_GPT2_117M_1x4_FP32_TF = "52af9a47-e92c-4fbb-bdbf-6e1ee1a97e3a" -MODEL_GPT2_117M_1x1_FP32_TF = "3f7398e6-bdb6-41ac-886e-0b3f17da603c" - -# PyTorch. -MODEL_CLIP_TEXT_SEQLEN64_FP32_TORCH = "9a9515c7-cb68-4c34-b1d2-0e8c0a3620b8" -MODEL_UNET_2D_FP32_TORCH = "340553d1-e6fe-41b6-b2c7-687c74ccec56" -MODEL_EFFICIENTNET_V2_S_FP32_TORCH = "cc474102-7d2f-4ec1-92ae-84e83ba0f390" -MODEL_EFFICIENTNET_V2_S_FP16_TORCH = "271ea7a0-68e7-45b6-91f4-f39d5ce9e29c" -MODEL_EFFICIENTNET_B7_FP32_TORCH = "68caa96e-b8bb-48a2-bb08-a3044981a370" - -MODEL_BERT_LARGE_384_FP32_TORCH = "cbc5e400-7c93-4844-aca8-bce8f1bf9948" -MODEL_BERT_LARGE_384_FP16_TORCH = "c1be9a9d-64c3-4d88-8551-89a8b4f305ba" -MODEL_RESNET50_3X224X224_FP32_TORCH = "fd05da43-5e37-4fa0-88f8-3ceec1682345" -MODEL_RESNET50_3X224X224_FP16_TORCH = "5e779dd2-f115-4a1c-9aab-87b7ae4a568d" - -MODEL_FALCON7B_1X100XI64_GPTQ_TORCH = "d532c6f9-0670-4441-ad1c-73dd9bab920a" -MODEL_FALCON7B_INT4_1X100XI64_GPTQ_TORCH = "2fca8c51-3182-4a48-b605-b092b48b38ee" - -# JAX. We use same ids as what is defined in the OpenXLA model library. -MODEL_RESNET50 = "aff75509-4420-40a8-844e-dbfc48494fe6-MODEL_RESNET50" -MODEL_RESNET50_FP32 = f"{MODEL_RESNET50}-fp32" -MODEL_RESNET50_FP32_JAX = f"{MODEL_RESNET50_FP32}-JAX" -MODEL_RESNET50_FP32_JAX_3X224X224XF32 = f"{MODEL_RESNET50_FP32_JAX}-3x224x224xf32" - -MODEL_BERT_LARGE = "47cb0d3a-5eb7-41c7-9d7c-97aae7023ecf-MODEL_BERT_LARGE" -MODEL_BERT_LARGE_FP32 = f"{MODEL_BERT_LARGE}-fp32" -MODEL_BERT_LARGE_FP32_JAX = f"{MODEL_BERT_LARGE_FP32}-JAX" -MODEL_BERT_LARGE_FP32_JAX_384XI32 = f"{MODEL_BERT_LARGE_FP32_JAX}-384xi32" - -MODEL_T5_LARGE = "173c7180-bad4-4b91-8423-4beeb13d2b0a-MODEL_T5_LARGE" -MODEL_T5_LARGE_FP32 = f"{MODEL_T5_LARGE}-fp32" -MODEL_T5_LARGE_FP32_JAX = f"{MODEL_T5_LARGE_FP32}-JAX" -MODEL_T5_LARGE_FP32_JAX_512XI32 = f"{MODEL_T5_LARGE_FP32_JAX}-512xi32" - -# Microbenchmarks. UB is shorthand for microbenchmark. -MICRO_MATMUL_3456X1024X2048_FP16_MLIR = "50a7aece-73f9-47f4-a93a-4a1178f45407" -MICRO_MATMUL_3456X1024X2048_FP32_MLIR = "a55afe1c-9410-47a6-b417-04b0d75ee5f4" -MICRO_MATMUL_2560X2560X2560_FP16_MLIR = "81cebaaf-e23d-4a32-89dc-9fc7adc37a8f" -MICRO_MATMUL_2560X2560X2560_FP32_MLIR = "579b77ea-76bd-4eb3-bd85-067c25a89eff" -MICRO_MATMUL_128X256X8192_FP16_MLIR = "699fd533-02a9-49f0-bf26-1902d8dbb5af" -MICRO_MATMUL_128X256X8192_FP32_MLIR = "a6c2b812-0a71-45e7-9ea5-f3d8529213ef" -MICRO_MATMUL_2564x2564x2564_FP32_MLIR = "4e75ff72-f807-49f6-b740-febca1794334" -MICRO_MATMUL_2562x2564x2562_FP32_MLIR = "8d6be288-9b88-41bd-bc5a-5644df0481bb" -MICRO_MATMUL_2562x2561x2561_FP32_MLIR = "0a3d952b-41ca-43d2-9ec3-ccb8dde20ce3" -MICRO_MATMUL_123x2561x2561_FP32_MLIR = "9b81a2f5-bab2-4f6f-b8b5-13cc4b3e9413" -MICRO_MATMUL_1x256x2048_I8xI4_MLIR = "89cc6a83-82f3-47d7-9874-fb2881d7bde5" -MICRO_MATMUL_256x256x2048_I8xI4_MLIR = "a38a79b3-c144-4f5f-b4f1-18ebb5b94cad" -MICRO_MATMUL_1x256x2048_I8xI8_MLIR = "203dfc5e-4305-44ea-9204-df5c9170df4f" -MICRO_MATMUL_256x256x2048_I8xI8_MLIR = "169f3fe6-9f27-4193-a1a3-dd897f198592" - -# Model input data -MODEL_INPUT_DATA_ZEROS = "8d4a034e-944d-4725-8402-d6f6e61be93c" - -# Devices -DEVICE_SPEC_GCP_C2_STANDARD_60 = "9a4804f1-b1b9-46cd-b251-7f16a655f782" -DEVICE_SPEC_GCP_A2_HIGHGPU_1G = "78c56b95-2d7d-44b5-b5fd-8e47aa961108" -DEVICE_SPEC_MOBILE_PIXEL_4 = "fc901efc-ddf8-44c0-b009-8eecb8286521" -DEVICE_SPEC_MOBILE_PIXEL_6_PRO = "0f624778-dc50-43eb-8060-7f4aea9634e1" -DEVICE_SPEC_MOBILE_MOTO_EDGE_X30 = "dae12e6a-4169-419d-adc9-4f63a2ff1997" -DEVICE_SPEC_MOBILE_SM_G980F = "0247f709-a300-4d12-b986-bdb0c15c2653" -DEVICE_SPEC_EMULATOR_RISCV_32 = "5e4277a5-993e-4a0d-96c3-e9ccc75c5c54" -DEVICE_SPEC_EMULATOR_RISCV_64 = "38ca19e5-a82d-4ac8-ae6f-bcb8c3fea2a6" - -# IREE benchmarks -IREE_COMPILE_CONFIG_VMVX_GENERIC_EXPERIMENTAL = "75336abd-8108-462c-9ce3-15443e3f32f4" -IREE_COMPILE_CONFIG_LINUX_CASCADELAKE_NO_DT = "e7e18b0f-c72d-4f1c-89b1-5afee70df6e9" -IREE_COMPILE_CONFIG_LINUX_CASCADELAKE_DT_ONLY = "bbb022c4-e7ee-4216-9c04-4f97a217da53" -IREE_COMPILE_CONFIG_LINUX_CASCADELAKE_DT_UK = "6d0d5716-5525-44ad-b71d-8075ee1583a6" -IREE_COMPILE_CONFIG_LINUX_RV64_GENERIC_DEFAULTS = "cdf579a9-5446-403b-a991-802a6c702e65" -IREE_COMPILE_CONFIG_LINUX_RV32_GENERIC_DEFAULTS = "6d9ce240-ec14-4d8f-a8e4-1b20aa17b4e4" -IREE_COMPILE_CONFIG_LINUX_CUDA_SM80_DEFAULTS = "09cb5300-7f73-45cf-9f68-e114c77ca030" -IREE_COMPILE_CONFIG_LINUX_CUDA_SM80_MATMUL_UBENCH = ( - "3f66ba98-5716-4d30-9a87-50bc78e5f714" -) -IREE_COMPILE_CONFIG_LINUX_CUDA_SM80_MATMUL_SPLITK_UBENCH = ( - "54cf2ec3-d073-4281-9561-b6c1280bd0eb" -) -IREE_COMPILE_CONFIG_LINUX_VULKAN_SD_SIMT = "da0ea6e6-719b-43ee-bfec-72eb3b1173bf" -IREE_COMPILE_CONFIG_LINUX_VULKAN_SD_TENSORCORE = "97790694-4f0f-4d83-bc52-d74e019c1df9" -IREE_COMPILE_CONFIG_ANDROID_ARM_VALHALL_DEFAULTS = ( - "8da35f2b-a042-4b7d-9dcf-5ebbc1728765" -) -IREE_COMPILE_CONFIG_ANDROID_ARM_VALHALL_EXPERIMENTAL = ( - "32a56c8d-cc6c-41b8-8620-1f8eda0b8223" -) -IREE_COMPILE_CONFIG_ANDROID_ARM_VALHALL_EXPERIMENTAL_REPEATED_KERNEL = ( - "6b601a8d-4824-42e0-bcc6-500c0c3fa346" -) -IREE_COMPILE_CONFIG_ANDROID_ARMV8_2_A_GENERIC_NO_DT = ( - "1f2adf49-282e-4aff-9d4f-e63b1621f1e8" -) -IREE_COMPILE_CONFIG_ANDROID_ARMV8_2_A_GENERIC_DT_ONLY = ( - "3d630c19-2d54-4029-b203-b4346855680f" -) -IREE_COMPILE_CONFIG_ANDROID_ARMV8_2_A_GENERIC_DT_UK = ( - "d463322c-24e6-4685-85ca-d541b41a405f" -) -IREE_COMPILE_CONFIG_ANDROID_QUALCOMM_ADRENO_DEFAULTS = ( - "c7eea358-d8d2-4199-9d75-bb741c399b1b" -) -IREE_COMPILE_CONFIG_ANDROID_QUALCOMM_ADRENO_FUSE_PADDING = ( - "d3038b95-c889-456a-bff6-5cbabd10f1ad" -) -IREE_COMPILE_CONFIG_ANDROID_QUALCOMM_ADRENO_FUSE_PADDING_REPEATED_KERNEL = ( - "70b823ca-2807-4531-8c00-e02af7d70466" -) -IREE_MODULE_EXECUTION_CONFIG_LOCAL_SYNC = "13fc65a9-e5dc-4cbb-9c09-25b0b08f4c03" -IREE_MODULE_EXECUTION_CONFIG_LOCAL_TASK_BASE = "c7c4a15e-b20c-4898-bb4a-864f34ff34b2" -IREE_MODULE_EXECUTION_CONFIG_SYS_SCHED_LOCAL_TASK_BASE = ( - "0dfb6b03-bd15-45a9-b82a-345c03f1fea6" -) -IREE_MODULE_EXECUTION_CONFIG_CUDA = "f7c0ec98-f028-436a-b05a-7d35cf18ce2d" -IREE_MODULE_EXECUTION_CONFIG_CUDA_BATCH_SIZE_100 = ( - "ce15c338-b1d1-4ee3-b876-22d3cc5a831d" -) -IREE_MODULE_EXECUTION_CONFIG_VULKAN = "34ae13f0-d6d9-43f7-befb-15d024e88e89" -IREE_MODULE_EXECUTION_CONFIG_VULKAN_BATCH_SIZE_16 = ( - "b10737a8-5da4-4052-9b7a-5b07f21e02d0" -) -IREE_MODULE_EXECUTION_CONFIG_VULKAN_BATCH_SIZE_32 = ( - "c59f6ed8-ef78-4ddd-93ea-f173c5e4d6b8" -) -IREE_MODULE_EXECUTION_CONFIG_VMVX_LOCAL_TASK_BASE = ( - "953183e2-1e84-4a51-a43c-9b869bdc2218" -) -IREE_MODULE_EXECUTION_CONFIG_VMVX_SYS_SCHED_LOCAL_TASK_BASE = ( - "a1a9795e-2fc5-4d95-abc0-b0fb41b07557" -) -IREE_MODEL_IMPORT_STABLEHLO_MLIR_DEFAULT = "8b2df698-f3ba-4207-8696-6c909776eac4" -IREE_MODEL_IMPORT_TFLITE_DEFAULT = "16280d67-7ce0-4807-ab4b-0cb3c771d206" -IREE_MODEL_IMPORT_LINALG_MLIR_DEFAULT = "8afc4561-e84d-4a91-af55-2b1917465fcc" diff --git a/build_tools/python/e2e_test_framework/unique_ids_test.py b/build_tools/python/e2e_test_framework/unique_ids_test.py deleted file mode 100644 index 475f599e66b3..000000000000 --- a/build_tools/python/e2e_test_framework/unique_ids_test.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2023 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import hashlib -import unittest - -import unique_ids - - -class UniqueIdsTest(unittest.TestCase): - def test_hash_composite_id(self): - output = unique_ids.hash_composite_id(["abc", "123"]) - - self.assertEqual( - output, hashlib.sha256(f"0-abc:1-123".encode("utf-8")).hexdigest() - ) - - def test_hash_composite_id_diff_keys(self): - ids = [ - unique_ids.hash_composite_id([]), - unique_ids.hash_composite_id(["abc", "123"]), - unique_ids.hash_composite_id(["123", "abc"]), - unique_ids.hash_composite_id(["123", unique_ids.TRANSPARENT_ID]), - unique_ids.hash_composite_id(["123", "abc", "xyz"]), - unique_ids.hash_composite_id(["123", unique_ids.TRANSPARENT_ID, "xyz"]), - ] - - # Check if they are all distinct. - self.assertCountEqual(set(ids), ids) - - def test_hash_composite_id_unchanged_with_transparent_id(self): - existing_id = unique_ids.hash_composite_id(["abc"]) - new_id_a = unique_ids.hash_composite_id(["abc", unique_ids.TRANSPARENT_ID]) - new_id_b = unique_ids.hash_composite_id( - ["abc", unique_ids.TRANSPARENT_ID, unique_ids.TRANSPARENT_ID] - ) - - self.assertEqual(existing_id, new_id_a) - self.assertEqual(existing_id, new_id_b) - - def test_hash_composite_id_with_transparent_ids_in_diff_pos(self): - id_a = unique_ids.hash_composite_id([unique_ids.TRANSPARENT_ID, "abc"]) - id_b = unique_ids.hash_composite_id(["abc", unique_ids.TRANSPARENT_ID]) - - self.assertNotEqual(id_a, id_b) - - -if __name__ == "__main__": - unittest.main() diff --git a/build_tools/python/reporting/__init__.py b/build_tools/python/reporting/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/build_tools/python/reporting/benchmark_comment.py b/build_tools/python/reporting/benchmark_comment.py deleted file mode 100644 index ce6135d3a149..000000000000 --- a/build_tools/python/reporting/benchmark_comment.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -from dataclasses import dataclass - -GIST_LINK_PLACEHORDER = "<>" - - -@dataclass(frozen=True) -class CommentData(object): - """Benchmark comment data.""" - - # Unique id to identify the same kind of comment. - type_id: str - # Abbreviated markdown to post as a comment. - abbr_md: str - # Abbreviated markdown to post on gist. - full_md: str - # Unverified PR number. - unverified_pr_number: int diff --git a/build_tools/python/reporting/common/__init__.py b/build_tools/python/reporting/common/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/build_tools/python/reporting/common/html_utils.py b/build_tools/python/reporting/common/html_utils.py deleted file mode 100644 index 871953bddd41..000000000000 --- a/build_tools/python/reporting/common/html_utils.py +++ /dev/null @@ -1,173 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -import pandas as pd - -_LEGEND_0 = ">10.0x better" -_LEGEND_1 = ">2.0x, <=10.0x better" -_LEGEND_2 = ">=1.0x, <=2.0x better" -_LEGEND_3 = ">1.0x, <1.15x worse" -_LEGEND_4 = ">=1.15x, <2.0x worse" -_LEGEND_5 = ">=2.0x, <10.x worse" -_LEGEND_6 = ">=10.0x worse" - - -def get_table_css(): - styles = [ - dict(selector="tr:hover", props=[("background", "#f4f4f4")]), - dict(selector="tbody tr", props=[("background-color", "#ffffff")]), - dict(selector="tbody td", props=[("border", "1px solid #dddfe1")]), - dict( - selector="th", - props=[ - ("background-color", "#54585d"), - ("color", "#ffffff"), - ("font-weight", "bold"), - ("border", "1px solid #54585d"), - ("padding", "10px"), - ], - ), - dict(selector="td", props=[("padding", "10px")]), - dict( - selector="", - props=[ - ("border-collapse", "collapse"), - ("font-family", "Tahoma, Geneva, sans-serif"), - ], - ), - dict( - selector="caption", - props=[ - ("text-align", "center"), - ("padding", "10px"), - ("font-weight", "bold"), - ("font-size", "1.2em"), - ("color", "#636363"), - ], - ), - ] - return styles - - -def style_legend(v): - if _LEGEND_0 in v: - props = "background-color: #0277BD;" - elif _LEGEND_1 in v: - props = "background-color: #2E7D32;" - elif _LEGEND_2 in v: - props = "background-color: #66BB6A;" - elif _LEGEND_3 in v: - props = "background-color: #FBC02D;" - elif _LEGEND_4 in v: - props = "background-color: #E57373;" - elif _LEGEND_5 in v: - props = "background-color: #C62828;" - else: - props = "background-color: #880E4F" - return props - - -def generate_header_and_legend(version_html): - html = "" - html = html + version_html - - legend = pd.DataFrame(columns=[""]) - legend.loc[len(legend)] = [_LEGEND_0] - legend.loc[len(legend)] = [_LEGEND_1] - legend.loc[len(legend)] = [_LEGEND_2] - legend.loc[len(legend)] = [_LEGEND_3] - legend.loc[len(legend)] = [_LEGEND_4] - legend.loc[len(legend)] = [_LEGEND_5] - legend.loc[len(legend)] = [_LEGEND_6] - - styled_legend = legend.style.set_table_styles(get_table_css()) - styled_legend.set_caption("Legend") - styled_legend = styled_legend.set_properties(**{"color": "#ffffff"}) - styled_legend = styled_legend.set_properties(**{"width": "200px"}) - styled_legend = styled_legend.applymap(style_legend) - styled_legend = styled_legend.hide(axis="index") - styled_legend = styled_legend.hide(axis="columns") - html = html + styled_legend.to_html() + "
" - return html - - -def style_speedup(v): - if v > 10.0: - props = "background-color: #0277BD;" - elif v > 2.0: - props = "background-color: #2E7D32;" - elif v >= 1.0: - props = "background-color: #66BB6A;" - else: - props = "background-color: #FBC02D;" - return props - - -def style_slowdown(v): - if v >= 10.0: - props = "background-color: #880E4F" - elif v >= 2.0: - props = "background-color: #C62828;" - elif v > 1.15: - props = "background-color: #E57373;" - else: - props = "background-color: #FBC02D;" - return props - - -def style_performance(v): - if "faster" in v: - return style_speedup(float(v.split("x")[0])) - else: - return style_slowdown(float(v.split("x")[0])) - - -def style_latency(v): - if v == "nan": - return "color: #636363" - if "faster" in v: - return style_speedup(float(v.split("x")[0])) - else: - return style_slowdown(float(v.split("x")[0])) - - -def style_memory(v): - if v == "nan": - return "color: #636363" - if "smaller" in v: - return style_speedup(float(v.split("x")[0])) - else: - return style_slowdown(float(v.split("x")[0])) - - -def format_latency_comparison(iree_latency, baseline_latency): - if iree_latency == 0 or baseline_latency == 0: - return "nan" - - speedup = baseline_latency / iree_latency - slowdown = iree_latency / baseline_latency - faster_label = "{:.2f}x faster" - slower_label = "{:.2f}x slower" - latency = ( - faster_label.format(speedup) - if speedup >= 1.0 - else slower_label.format(slowdown) - ) - return latency - - -def format_memory_comparison(iree_memory, baseline_memory): - if iree_memory == 0 or baseline_memory == 0: - return "nan" - - smaller = baseline_memory / iree_memory - larger = iree_memory / baseline_memory - smaller_label = "{:.2f}x smaller" - larger_label = "{:0.2f}x larger" - memory = ( - smaller_label.format(smaller) if smaller >= 1.0 else larger_label.format(larger) - ) - return memory diff --git a/build_tools/scripts/download_file.py b/build_tools/scripts/download_file.py deleted file mode 100755 index ffa422077528..000000000000 --- a/build_tools/scripts/download_file.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Downloads a file from the web and decompresses it if necessary. NEVER Use - this tool to download from untrusted sources, it doesn't unpack the file - safely. -""" - -import argparse -import gzip -import os -import shutil -import tarfile -import urllib.request -import urllib.error -import logging -import time - -DEFAULT_MAX_TRIES = 3 -RETRY_COOLDOWN_TIME = 1.0 - - -def parse_arguments(): - """Parses command line arguments.""" - parser = argparse.ArgumentParser( - description="Downloads a file from the web " - "and decompresses it if necessary. NEVER Use this tool to download from " - "untrusted sources, it doesn't unpack the file safely." - ) - parser.add_argument( - "source_url", type=str, metavar="", help="Source URL to download" - ) - parser.add_argument( - "-o", - "--output", - type=str, - required=True, - metavar="", - help="Output file path", - ) - parser.add_argument( - "--unpack", - action="store_true", - default=False, - help="Unpack the downloaded file if it's an archive", - ) - parser.add_argument( - "--max-tries", - metavar="", - type=int, - default=DEFAULT_MAX_TRIES, - help="Number of tries before giving up", - ) - return parser.parse_args() - - -def download_and_extract(source_url: str, output: str, unpack: bool): - # Open the URL and get the file-like streaming object. - with urllib.request.urlopen(source_url) as response: - if response.status != 200: - raise RuntimeError( - f"Failed to download file with status {response.status} {response.msg}" - ) - - if unpack: - if source_url.endswith(".tar.gz"): - # Open tar.gz in the streaming mode. - with tarfile.open(fileobj=response, mode="r|*") as tar_file: - if os.path.exists(output): - shutil.rmtree(output) - os.makedirs(output) - tar_file.extractall(output) - return - elif source_url.endswith(".gz"): - # Open gzip from a file-like object, which will be in the streaming mode. - with gzip.open(filename=response, mode="rb") as input_file: - with open(output, "wb") as output_file: - shutil.copyfileobj(input_file, output_file) - return - - # Fallback to download the file only. - with open(output, "wb") as output_file: - # Streaming copy. - shutil.copyfileobj(response, output_file) - - -def main(args): - output_dir = os.path.dirname(args.output) - - if not os.path.isdir(output_dir): - os.makedirs(output_dir) - - remaining_tries = args.max_tries - while remaining_tries > 0: - try: - download_and_extract(args.source_url, args.output, args.unpack) - break - except (ConnectionResetError, ConnectionRefusedError, urllib.error.URLError): - remaining_tries -= 1 - if remaining_tries == 0: - raise - else: - logging.warning( - f"Connection error, remaining {remaining_tries} tries", - exc_info=True, - ) - time.sleep(RETRY_COOLDOWN_TIME) - - -if __name__ == "__main__": - main(parse_arguments()) diff --git a/build_tools/scripts/generate_cmake_files.sh b/build_tools/scripts/generate_cmake_files.sh deleted file mode 100755 index 23da394f4c84..000000000000 --- a/build_tools/scripts/generate_cmake_files.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# Run all CMake file generators with proper output paths. - -set -euo pipefail - -ROOT_DIR=$(git rev-parse --show-toplevel) -cd "${ROOT_DIR}" - -./build_tools/testing/generate_cmake_e2e_test_artifacts_suite.py \ - --output_dir ./tests/e2e/test_artifacts - -./build_tools/testing/generate_cmake_e2e_model_tests.py \ - --output ./tests/e2e/stablehlo_models/generated_e2e_model_tests.cmake diff --git a/build_tools/scripts/generate_compilation_flagfile.py b/build_tools/scripts/generate_compilation_flagfile.py deleted file mode 100755 index adda56ed2beb..000000000000 --- a/build_tools/scripts/generate_compilation_flagfile.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Generates a compilation flagfile for iree-compiler. - -This tool is added due to CMake's incapabilities on generating files with -multiple lines. CMake's configure_file doesn't work in our case as it can't be -triggered from a target. -""" - -import argparse - - -def parse_arguments(): - """Parses command line arguments.""" - parser = argparse.ArgumentParser() - parser.add_argument( - "--output", type=str, required=True, help="output file to write to" - ) - parser.add_argument( - "compilation_flags", - metavar="", - nargs="*", - help="list of compilation flags", - ) - return parser.parse_args() - - -def main(args): - with open(args.output, "w") as f: - f.write("\n".join(args.compilation_flags) + "\n") - - -if __name__ == "__main__": - main(parse_arguments()) diff --git a/build_tools/scripts/generate_flagfile.py b/build_tools/scripts/generate_flagfile.py deleted file mode 100755 index fb1effd12686..000000000000 --- a/build_tools/scripts/generate_flagfile.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Generates a flagfile for iree-benchmark-module.""" - -import argparse -import os - - -def parse_arguments(): - """Parses command line arguments.""" - parser = argparse.ArgumentParser() - parser.add_argument( - "--module", - type=str, - required=True, - metavar="", - help="The name of the module file", - ) - parser.add_argument( - "--device", - type=str, - required=True, - metavar="", - help="The name of the HAL device", - ) - parser.add_argument( - "--function", - type=str, - required=True, - metavar="", - help="The name of the entry function", - ) - parser.add_argument( - "--inputs", - type=str, - required=True, - metavar="", - help="A list of comma-separated function inputs", - ) - parser.add_argument( - "--additional_args", - type=str, - required=True, - metavar="", - help="Additional command-line arguments", - ) - parser.add_argument( - "-o", - "--output", - type=str, - required=True, - metavar="", - help="Output file to write to", - ) - return parser.parse_args() - - -def main(args): - lines = [ - f"--device={args.device}", - f"--module={args.module}", - f"--function={args.function}", - ] - lines.extend([("--input=" + e) for e in args.inputs.split(",")]) - lines.extend(args.additional_args.split(";")) - content = "\n".join(lines) + "\n" - - with open(args.output, "w") as f: - f.writelines(content) - - -if __name__ == "__main__": - main(parse_arguments()) diff --git a/build_tools/scripts/update_tflite_models.py b/build_tools/scripts/update_tflite_models.py deleted file mode 100644 index 2af25fb0c409..000000000000 --- a/build_tools/scripts/update_tflite_models.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2021 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -# This tool handles mirroring tflite testing files from their source to the -# the iree-model-artifacts test bucket. This avoids taking dependency on -# external test files that may change or no longer be available. -# -# To update all files: -# python update_tflite_models.py --file all -# -# To update a specific file: -# python update_tflite_models.py --file posenet_i8_input.jpg -# -# Note you must have write permission to the iree-model-artifacts GCS bucket -# with local gcloud authentication. - -from absl import app -from absl import flags -from google.cloud import storage -from google_auth_oauthlib import flow - -import tempfile -import urllib - -FLAGS = flags.FLAGS -flags.DEFINE_string("file", "", "file to update") - -file_dict = dict( - { - "mobilenet_v1.tflite": "https://tfhub.dev/tensorflow/lite-model/mobilenet_v1_1.0_160/1/default/1?lite-format=tflite", - "posenet_i8.tflite": "https://tfhub.dev/google/lite-model/movenet/singlepose/lightning/tflite/int8/4?lite-format=tflite", - "posenet_i8_input.jpg": "https://github.com/tensorflow/examples/raw/master/lite/examples/pose_estimation/raspberry_pi/test_data/image3.jpeg", - } -) - -BUCKET_NAME = "iree-model-artifacts" -FOLDER_NAME = "tflite-integration-tests" - - -def upload_model(source, destination, tmpfile): - """Uploads a file to the bucket.""" - urllib.request.urlretrieve(source, tmpfile) - - storage_client = storage.Client() - bucket = storage_client.get_bucket(BUCKET_NAME) - blob = bucket.blob("/".join([FOLDER_NAME, destination])) - blob.upload_from_filename(tmpfile) - - -def main(argv): - tf = tempfile.NamedTemporaryFile() - - items = file_dict.items() - - if FLAGS.file in file_dict: - items = [(FLAGS.file, file_dict[FLAGS.file])] - elif FLAGS.file != "all": - print("Unknown file to upload: ", '"' + FLAGS.file + '"') - exit() - - for dst, src in items: - upload_model(src, dst, tf.name) - - -if __name__ == "__main__": - app.run(main) diff --git a/build_tools/testing/generate_cmake_e2e_model_tests.py b/build_tools/testing/generate_cmake_e2e_model_tests.py deleted file mode 100755 index bd3153ce0fc4..000000000000 --- a/build_tools/testing/generate_cmake_e2e_model_tests.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Generates a CMake file to define e2e mdoel tests.""" - -import string -import sys -import pathlib -import argparse - -# Add build_tools python dir to the search path. -sys.path.insert(0, str(pathlib.Path(__file__).parent / ".." / "python")) - -import benchmark_suites.iree.benchmark_collections -import e2e_model_tests.cmake_generator - -TEMPLATE_DIR = pathlib.Path(__file__).parent -GENERATED_E2E_MODEL_TESTS_CMAKE_TEMPLATE = string.Template( - (TEMPLATE_DIR / "generated_e2e_model_tests_template.cmake").read_text() -) - - -def parse_arguments(): - """Parses command-line options.""" - - parser = argparse.ArgumentParser() - parser.add_argument( - "--output", required=True, help="Path to write the generated cmake file." - ) - - return parser.parse_args() - - -def main(args: argparse.Namespace): - (gen_configs, _) = benchmark_suites.iree.benchmark_collections.generate_benchmarks() - cmake_rules = e2e_model_tests.cmake_generator.generate_rules( - module_generation_configs=gen_configs - ) - output = GENERATED_E2E_MODEL_TESTS_CMAKE_TEMPLATE.substitute( - __TEST_RULES="\n".join(cmake_rules) - ) - output = output.rstrip() + "\n" # Trim any extra newlines. - with open(args.output, "w") as output_file: - output_file.write(output) - - -if __name__ == "__main__": - main(parse_arguments()) diff --git a/build_tools/testing/generate_cmake_e2e_test_artifacts_suite.py b/build_tools/testing/generate_cmake_e2e_test_artifacts_suite.py deleted file mode 100755 index 0e90dc140e1d..000000000000 --- a/build_tools/testing/generate_cmake_e2e_test_artifacts_suite.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 -## Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -"""Generates a CMake file to build test artifacts.""" - -import sys -import pathlib - -# Add build_tools python dir to the search path. -sys.path.insert(0, str(pathlib.Path(__file__).parent.with_name("python"))) - -import argparse -import itertools - -from e2e_test_artifacts.cmake_generator import model_rule_generator, iree_rule_generator -from e2e_test_artifacts import iree_artifacts -import benchmark_suites.iree.benchmark_collections - -# CMake variable name to store IREE package name. -PACKAGE_NAME_CMAKE_VARIABLE = "PACKAGE_NAME" -ROOT_ARTIFACTS_DIR_CMAKE_VARIABLE = "ROOT_ARTIFACTS_DIR" - -GENERATED_E2E_TEST_FETCH_MODELS_CMAKE_FILENAMAE = ( - "generated_e2e_test_fetch_models.cmake" -) -GENERATED_E2E_TEST_IREE_ARTIFACTS_CMAKE_FILENAME = ( - "generated_e2e_test_iree_artifacts.cmake" -) - - -def parse_arguments(): - """Parses command-line options.""" - - parser = argparse.ArgumentParser() - parser.add_argument( - "--output_dir", - required=True, - help="Dir path to write the generated cmake files.", - ) - - return parser.parse_args() - - -def main(args: argparse.Namespace): - # Currently IREE benchmark suite is the only source of configs. - ( - gen_configs, - run_configs, - ) = benchmark_suites.iree.benchmark_collections.generate_benchmarks() - - dependent_model_map = iree_artifacts.get_dependent_model_map(gen_configs) - - root_path = pathlib.PurePath("${%s}" % ROOT_ARTIFACTS_DIR_CMAKE_VARIABLE) - model_rule_map = model_rule_generator.generate_model_rule_map( - root_path=root_path, models=dependent_model_map.values() - ) - - output_dir = pathlib.Path(args.output_dir) - fetch_models_cmake_file = ( - output_dir / GENERATED_E2E_TEST_FETCH_MODELS_CMAKE_FILENAMAE - ) - model_cmake_rules = itertools.chain.from_iterable( - rule.cmake_rules for rule in model_rule_map.values() - ) - fetch_models_cmake_file.write_text("\n".join(model_cmake_rules)) - - package_name = "${%s}" % PACKAGE_NAME_CMAKE_VARIABLE - iree_cmake_rules = iree_rule_generator.generate_rules( - package_name=package_name, - root_path=root_path, - gen_configs=gen_configs, - run_configs=run_configs, - model_rule_map=model_rule_map, - ) - - (output_dir / GENERATED_E2E_TEST_IREE_ARTIFACTS_CMAKE_FILENAME).write_text( - "\n".join(iree_cmake_rules) - ) - - -if __name__ == "__main__": - main(parse_arguments()) diff --git a/build_tools/testing/generated_e2e_model_tests_template.cmake b/build_tools/testing/generated_e2e_model_tests_template.cmake deleted file mode 100644 index 2cfeaaf8ffc0..000000000000 --- a/build_tools/testing/generated_e2e_model_tests_template.cmake +++ /dev/null @@ -1,7 +0,0 @@ -################################################################################ -# Autogenerated by build_tools/testing/generate_cmake_e2e_model_tests.py # -# To update the tests, modify definitions in the generator and regenerate this # -# file. # -################################################################################ - -$__TEST_RULES diff --git a/docs/website/docs/developers/general/contributing.md b/docs/website/docs/developers/general/contributing.md index e8f5e7f2eda1..186ec2eff4f5 100644 --- a/docs/website/docs/developers/general/contributing.md +++ b/docs/website/docs/developers/general/contributing.md @@ -317,8 +317,6 @@ ci-skip: jobs,to,skip ci-extra: extra,jobs,to,run ci-exactly: exact,set,of,jobs,to,run skip-ci: free form reason -skip-llvm-integrate-benchmark: free form reason -benchmark-extra: extra,benchmarks,to,run runner-env: [testing|prod] ``` @@ -356,25 +354,6 @@ runner-env: [testing|prod] dependencies are satisfied. Thus, if you request skipping the `build_all` job, all the jobs that depend on it will fail, not be skipped. -??? info - "Using `benchmark-extra`, `skip-llvm-integrate-benchmark`" - - ``` text - benchmark-extra: extra,benchmarks,to,run - skip-llvm-integrate-benchmark: free form reason - ``` - - Benchmarks don't run by default on PRs, and must be specifically requested. - - The `benchmark-extra` option allows specifying additional benchmark presets - to run as part of benchmarking. It accepts a comma-separated list of - benchmark presets. This combines with labels added to the PR (which are a - more limited set of options). See the - [benchmark suites documentation](../performance/benchmark-suites.md). - - Benchmarks *do* run by default on PRs detected to be an integration of LLVM - into IREE, but this behavior can be disabled with - `skip-llvm-integrate-benchmark`. - ??? info - "Using `runner-env`" The `runner-env` option controls which runner environment to target for our diff --git a/docs/website/docs/developers/general/testing-guide.md b/docs/website/docs/developers/general/testing-guide.md index 1135cf81381f..a6d2c048cf01 100644 --- a/docs/website/docs/developers/general/testing-guide.md +++ b/docs/website/docs/developers/general/testing-guide.md @@ -224,13 +224,6 @@ to running CTest (from the build directory): cmake --build . --target iree-test-deps ``` -To run e2e model tests in -[generated_e2e_model_tests.cmake](https://github.com/iree-org/iree/blob/main/tests/e2e/stablehlo_models/generated_e2e_model_tests.cmake), -because of their dependencies, `-DIREE_BUILD_E2E_TEST_ARTIFACTS=ON` needs to be -set when configuring CMake. Also see -[IREE Benchmark Suite Prerequisites](../performance/benchmark-suites.md#prerequisites) -for required packages. - ### Running a Test For the test diff --git a/docs/website/docs/developers/performance/benchmark-suites.md b/docs/website/docs/developers/performance/benchmark-suites.md deleted file mode 100644 index d637c0022cff..000000000000 --- a/docs/website/docs/developers/performance/benchmark-suites.md +++ /dev/null @@ -1,308 +0,0 @@ ---- -icon: simple/speedtest ---- - -# Benchmark suites - -IREE Benchmarks Suites is a collection of benchmarks for IREE developers to -track performance improvements/regressions during development. - -The benchmark suites are run for each commit on the main branch and the results -are uploaded to for regression analysis (for the current -supported targets). On pull requests, users can add labels `benchmarks:*` to -trigger the benchmark runs. The results will be compared with - and post in the comments. - -Information about the definitions of the benchmark suites can be found in the -[IREE Benchmark Suites Configurations](https://github.com/iree-org/iree/blob/main/build_tools/python/benchmark_suites/iree/README.md). - -## Running benchmark suites locally - -### Prerequisites - -Install `iree-import-tf` and `iree-import-tflite` in your Python environment -(see -[Tensorflow Integration](https://iree.dev/guides/ml-frameworks/tensorflow/) -and -[TFLite Integration](https://iree.dev/guides/ml-frameworks/tflite/)). - -### Choose benchmark presets - -IREE Benchmark Suites contain many benchmarks for different devices and model -sizes, which can take lots of space and time to build all of them. So benchmarks -are grouped into presets to allow building and running only a subset of them. -The available presets are: - -Execution benchmarks: - -- `android-cpu`: benchmarks for mobile CPUs -- `android-gpu`: benchmarks for mobile GPUs -- `cuda`: benchmarks for CUDA with a small model set -- `cuda-large`: benchmarks for CUDA with a large model set -- `vulkan-nvidia`: benchmarks for Vulkan on NVIDIA graphics cards -- `x86_64`: benchmarks for x86_64 CPUs with a small model set -- `x86_64-large`: benchmarks for x86_64 with a large model set - -Compilation benchmarks (to collect compilation statistics, such as module -sizes): - -- `comp-stats`: compilation benchmarks with a small model set -- `comp-stats-large`: compilation benchmark with a large model set - -Note that `*-large` presets will download and build a few hundreds GBs of -artifacts. - -Set the environment variables of benchmark presets for the steps below, for -example: - -```sh -export EXECUTION_BENCHMARK_PRESETS="cuda,x86_64" -export COMPILATION_BENCHMARK_PRESETS="comp-stats" -``` - -### Build benchmark suites - -Configure IREE with `-DIREE_BUILD_E2E_TEST_ARTIFACTS=ON`: - -```sh -cmake -GNinja -B "${IREE_BUILD_DIR?}" -S "${IREE_REPO?}" \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DIREE_ENABLE_LLD=ON \ - -DIREE_BUILD_E2E_TEST_ARTIFACTS=ON -``` - -If you only need the imported MLIR models: - -```sh -cmake --build "${IREE_BUILD_DIR?}" --target \ - iree-benchmark-import-models - # For large benchmarks (this will take > 100G disk space) - # iree-benchmark-import-models-large -``` - -Otherwise, compile the benchmark suites and tools for benchmarking: - -```sh -cmake --build "${IREE_BUILD_DIR?}" --target \ - iree-benchmark-suites \ - # If any *-large preset is enabled, also build this target: - # iree-benchmark-suites-large \ - iree-benchmark-module -export E2E_TEST_ARTIFACTS_DIR="${IREE_BUILD_DIR?}/e2e_test_artifacts" -``` - -### Run benchmarks - -Export the execution benchmark config: - -```sh -build_tools/benchmarks/export_benchmark_config.py execution \ - --benchmark_presets="${EXECUTION_BENCHMARK_PRESETS?}" \ - > "${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json" -``` - -Run benchmarks (currently only support running on a Linux host): - -```sh -build_tools/benchmarks/run_benchmarks_on_linux.py \ - --benchmark_tool_dir="${IREE_BUILD_DIR?}/tools" \ - --e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR?}" \ - --execution_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json" \ - --target_device_name="" \ - --output="${E2E_TEST_ARTIFACTS_DIR?}/benchmark_results.json" \ - --verbose \ - --cpu_uarch="" -``` - -Note that: - -- `` selects a benchmark group targets a specific device: - - Common options: - - `c2-standard-60` for x86_64 CPU benchmarks. - - `a2-highgpu-1g` for NVIDIA GPU benchmarks. - - All device names are defined under - [build_tools/python/e2e_test_framework/device_specs](https://github.com/iree-org/iree/tree/main/build_tools/python/e2e_test_framework/device_specs). -- To run x86_64 benchmarks, right now `--cpu_uarch` needs to be provided and - only `CascadeLake` is available currently. - -Filters can be used to select the benchmarks: - -```sh -build_tools/benchmarks/run_benchmarks_on_linux.py \ - --benchmark_tool_dir="${IREE_BUILD_DIR?}/tools" \ - --e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR?}" \ - --execution_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json" \ - --target_device_name="c2-standard-60" \ - --output="${E2E_TEST_ARTIFACTS_DIR?}/benchmark_results.json" \ - --verbose \ - --cpu_uarch="CascadeLake" \ - --model_name_regex="MobileBert*" \ - --driver_filter_regex='local-task' \ - --mode_regex="4-thread" -``` - -### Generate compilation statistics (compilation benchmarks) - -Export the compilation benchmark config: - -```sh -build_tools/benchmarks/export_benchmark_config.py compilation \ - --benchmark_presets="${COMPILATION_BENCHMARK_PRESETS?}" \ - > "${E2E_TEST_ARTIFACTS_DIR?}/comp_config.json" -``` - -Generate the compilation statistics: - -```sh -build_tools/benchmarks/collect_compilation_statistics.py \ - --compilation_benchmark_config=comp_config.json \ - --e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR?}" \ - --build_log="${IREE_BUILD_DIR?}/.ninja_log" \ - --output="${E2E_TEST_ARTIFACTS_DIR?}/compile_stats_results.json" -``` - -Note that you need to use [Ninja](https://ninja-build.org/) to build the -benchmark suites as the tool collects information from its build log. - -### Show execution / compilation benchmark results - -If you want to generate a comparison report locally, you can use -[diff_local_benchmarks.py](https://github.com/iree-org/iree/blob/main/build_tools/benchmarks/diff_local_benchmarks.py) -script to compare two result json files and generate the report. For example: - -```sh -build_tools/benchmarks/diff_local_benchmarks.py \ - --base "${E2E_TEST_ARTIFACTS_DIR?}/before_benchmark_results.json" \ - --target "${E2E_TEST_ARTIFACTS_DIR?}/after_benchmark_results.json" \ - > report.md -``` - -An example that compares compilation statistics: - -```sh -build_tools/benchmarks/diff_local_benchmarks.py \ - --base-compile-stats "${E2E_TEST_ARTIFACTS_DIR?}/before_compile_stats_results.json" \ - --target-compile-stats "${E2E_TEST_ARTIFACTS_DIR?}/after_compile_stats_results.json" \ - > report.md -``` - -### Find compile and run commands to reproduce benchmarks - -Each benchmark has its benchmark ID in the benchmark suites, you will see a -benchmark ID at: - -- In the serie's URL of - - Execution benchmark: `https://perf.iree.dev/serie?IREE?` - - Compilation benchmark: - `https://perf.iree.dev/serie?IREE?-` -- In `benchmark_results.json` and `compile_stats_results.json` - - Execution benchmark result has a field `run_config_id` - - Compilation benchmark result has a field `gen_config_id` -- In PR benchmark summary or the markdown generated by - `diff_local_benchmarks.py`, each benchmark has the link to its - URL, which includes the benchmark ID. - -If you don't have artifacts locally, see -[Fetching Benchmark Artifacts from CI](#fetching-benchmark-artifacts-from-ci) to -find the GCS directory of the CI artifacts. Then fetch the needed files: - -```sh -# Get ${E2E_TEST_ARTIFACTS_DIR_URL} from "Fetching Benchmark Artifacts from CI". -export E2E_TEST_ARTIFACTS_DIR="e2e_test_artifacts" - -# Download all artifacts -mkdir "${E2E_TEST_ARTIFACTS_DIR?}" -gcloud storage cp -r "${E2E_TEST_ARTIFACTS_DIR_URL?}" "${E2E_TEST_ARTIFACTS_DIR?}" -``` - -Run the helper tool to dump benchmark commands from benchmark configs: - -```sh -build_tools/benchmarks/benchmark_helper.py dump-cmds \ - --execution_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/execution-benchmark-config.json" \ - --compilation_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/compilation-benchmark-config.json" \ - --e2e_test_artifacts_dir="${E2E_TEST_ARTIFACTS_DIR?}" \ - --benchmark_id="" -``` - -### Get full list of benchmarks - -The commands below output the full list of execution and compilation benchmarks, -including the benchmark names and their flags: - -```sh -build_tools/benchmarks/export_benchmark_config.py execution > "${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json" -build_tools/benchmarks/export_benchmark_config.py compilation > "${E2E_TEST_ARTIFACTS_DIR?}/comp_config.json" -build_tools/benchmarks/benchmark_helper.py dump-cmds \ - --execution_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json" \ - --compilation_benchmark_config="${E2E_TEST_ARTIFACTS_DIR?}/comp_config.json" -``` - -## Fetching benchmark Artifacts from CI - -### 1. Find the corresponding CI workflow run - -On the commit of the benchmark run, you can find the list of the workflow jobs -by clicking the green check mark. Click any job starts with `CI /`: - -![image](https://user-images.githubusercontent.com/2104162/234647960-3df9d0f0-a34a-47ad-bda8-095ae44de865.png) - -### 2. Get URLs of GCS artifacts - -On the CI page, click `Summary` on the top-left to open the summary page. Scroll -down and the links to artifacts are listed in a section titled "Artifact Links". -Paste the content in your shell to define all needed variables for the following -steps: - -![image](https://user-images.githubusercontent.com/2104162/234716421-3a69b6ad-211d-4e39-8f9e-a4f22f91739d.png) - -### 3. Fetch the benchmark artifacts - -To fetch files from the GCS URL, the gcloud CLI tool -() can list the directory contents and -download files (see for -more usages). If you want to use CI artifacts to reproduce benchmarks locally, -see -[Find Compile and Run Commands to Reproduce Benchmarks](#find-compile-and-run-commands-to-reproduce-benchmarks). - -Assume you get the GCS URL variables from -[Get URLs of GCS artifacts](#2-get-urls-of-gcs-artifacts). - -Download artifacts: - -```sh -# The GCS directory has the same structure as your local ${IREE_BUILD_DIR?}/e2e_test_artifacts. -gcloud storage ls "${E2E_TEST_ARTIFACTS_DIR_URL?}" - -# Download all source and imported MLIR files: -gcloud storage cp "${E2E_TEST_ARTIFACTS_DIR_URL?}/*.mlir" "" -``` - -Execution and compilation benchmark configs can be downloaded at: - -```sh -# Execution benchmark config: -gcloud storage cp \ - "${E2E_TEST_ARTIFACTS_DIR_URL?}/execution-benchmark-config.json" \ - "${E2E_TEST_ARTIFACTS_DIR?}/exec_config.json" - -# Compilation benchmark config: -gcloud storage cp \ - "${E2E_TEST_ARTIFACTS_DIR_URL?}/compilation-benchmark-config.json" \ - "${E2E_TEST_ARTIFACTS_DIR?}/comp_config.json" -``` - -Benchmark raw results can be downloaded at: - -```sh -# Execution benchmark raw results -gcloud storage cp "${EXECUTION_BENCHMARK_RESULTS_DIR_URL?}/benchmark-results-*.json" . - -# Optional: Merge raw results into a single file -build_tools/benchmarks/benchmark_helper.py merge-results benchmark-results-*.json > benchmark_results.json - -# Compilation benchmark results -gcloud storage cp "${COMPILATION_BENCHMARK_RESULTS_URL?}" . -``` diff --git a/docs/website/docs/developers/performance/benchmarking.md b/docs/website/docs/developers/performance/benchmarking.md index cd1f7449e950..0d7a70986a26 100644 --- a/docs/website/docs/developers/performance/benchmarking.md +++ b/docs/website/docs/developers/performance/benchmarking.md @@ -314,79 +314,3 @@ $ for i in `cat /sys/devices/system/cpu/present | tr '-' ' ' | xargs seq`; do \ "/sys/devices/system/cpu/cpu${i?}/cpufreq/scaling_governor"; \ done ``` - -#### Android Scripts - -We provide a few scripts to set clockspeeds on Android (under -`build_tools/benchmarks`). These are somewhat device-specific: - -* The `set_android_scaling_governor.sh` work on all CPUs, but the default - governor name may be different across devices. -* The `set_*_gpu_scaling_policy.sh` script used should match the actual GPU on - your device. - -Sample configuration steps for Pixel 6: - -1. Copy all scripts to the device: - - ```shell - adb push build_tools/benchmarks/*.sh /data/local/tmp - ``` - -1. Launch interactive adb shell as super user: - - ```shell - adb shell - oriole:/ # su - oriole:/ # cd /data/local/tmp - ``` - -1. Pin frequencies (high clockspeeds): - - ```shell - oriole:/ # ./set_android_scaling_governor.sh - CPU info (before changing governor): - cpu governor cur min max - ------------------------------------------------ - cpu0 sched_pixel 1098000 300000 1803000 - cpu1 sched_pixel 1598000 300000 1803000 - cpu2 sched_pixel 1598000 300000 1803000 - cpu3 sched_pixel 1098000 300000 1803000 - cpu4 sched_pixel 400000 400000 2253000 - cpu5 sched_pixel 400000 400000 2253000 - cpu6 sched_pixel 500000 500000 2802000 - cpu7 sched_pixel 500000 500000 2802000 - Setting CPU frequency governor to performance - CPU info (after changing governor): - cpu governor cur min max - ------------------------------------------------ - cpu0 performance 1803000 300000 1803000 - cpu1 performance 1803000 300000 1803000 - cpu2 performance 1803000 300000 1803000 - cpu3 performance 1803000 300000 1803000 - cpu4 performance 2253000 400000 2253000 - cpu5 performance 2253000 400000 2253000 - cpu6 performance 2802000 500000 2802000 - cpu7 performance 2802000 500000 2802000 - oriole:/data/local/tmp # ./set_pixel6_gpu_scaling_policy.sh - GPU info (before changing frequency scaling policy): - policy cur min max - -------------------------------------------------------------- - coarse_demand [adaptive] always_on 251000 151000 848000 - Setting GPU frequency scaling policy to performance - GPU info (after changing frequency scaling policy): - policy cur min max - -------------------------------------------------------------- - coarse_demand adaptive [always_on] 848000 151000 848000 - ``` - -1. Restore default frequencies: - - ```shell - oriole:/ # ./set_android_scaling_governor.sh sched_pixel - ... - oriole:/ # ./set_pixel6_gpu_scaling_policy.sh default - ... - ``` - -TODO(scotttodd): Windows instructions diff --git a/docs/website/docs/developers/usage-best-practices.md b/docs/website/docs/developers/usage-best-practices.md index 8f8c6b23b60a..b15ec8d9e9a0 100644 --- a/docs/website/docs/developers/usage-best-practices.md +++ b/docs/website/docs/developers/usage-best-practices.md @@ -53,16 +53,6 @@ TODO: which compiler targets to use (try both CUDA and Vulkan?) TODO: use the most specific LLVM target triple you can? -### Tuning compilation heuristics - -IREE runs its own suite of benchmarks continuously using the definitions at -. The flags set for these -benchmarks represent the latest manually tuned values for workloads we track -closely and referencing them may help with your own search for peak performance. -You can use these flags in your own explorations, but note that as compiler -performance matures, the existing flags will gradually be replaced with -attributes for autotuning or command line options for experimental features. - ## Practices for runtime use TODO: sample code, profile numbers diff --git a/docs/website/mkdocs.yml b/docs/website/mkdocs.yml index e683cc31bddc..964229d22682 100644 --- a/docs/website/mkdocs.yml +++ b/docs/website/mkdocs.yml @@ -205,7 +205,6 @@ nav: - "developers/debugging/sanitizers.md" - "Performance": - "developers/performance/benchmarking.md" - - "developers/performance/benchmark-suites.md" - "developers/performance/profiling.md" - "developers/performance/profiling-cpu-events.md" - "developers/performance/profiling-gpu-vulkan.md" diff --git a/experimental/web/generate_web_metrics.sh b/experimental/web/generate_web_metrics.sh index 0bbe785ff679..8553a79c3470 100644 --- a/experimental/web/generate_web_metrics.sh +++ b/experimental/web/generate_web_metrics.sh @@ -84,9 +84,6 @@ wget -nc https://storage.googleapis.com/iree-model-artifacts/MobileNetV3SmallSta # Import programs into MLIR # ############################################################################### -# Note: you can also download imported programs from CI runs: -# https://iree.dev/developers/performance/benchmark-suites/#fetching-benchmark-artifacts-from-ci - IREE_IMPORT_TFLITE_PATH=iree-import-tflite # import_program helper diff --git a/tests/e2e/stablehlo_models/CMakeLists.txt b/tests/e2e/stablehlo_models/CMakeLists.txt index 8db14fbe70bd..f12f2fa970f2 100644 --- a/tests/e2e/stablehlo_models/CMakeLists.txt +++ b/tests/e2e/stablehlo_models/CMakeLists.txt @@ -96,5 +96,3 @@ iree_static_linker_test( "--iree-vm-target-index-bits=32" EMITC ) - -include(generated_e2e_model_tests.cmake) diff --git a/tests/e2e/stablehlo_models/generated_e2e_model_tests.cmake b/tests/e2e/stablehlo_models/generated_e2e_model_tests.cmake deleted file mode 100644 index 2f940a66beb7..000000000000 --- a/tests/e2e/stablehlo_models/generated_e2e_model_tests.cmake +++ /dev/null @@ -1,58 +0,0 @@ -################################################################################ -# Autogenerated by build_tools/testing/generate_cmake_e2e_model_tests.py # -# To update the tests, modify definitions in the generator and regenerate this # -# file. # -################################################################################ - -iree_benchmark_suite_module_test( - NAME "mobilenet_v1_fp32_correctness_test" - DRIVER "local-sync" - EXPECTED_OUTPUT "mobilenet_v1_fp32_expected_output.txt" - MODULES - "riscv_64-Linux=iree_module_MobileNetV1_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - "x86_64-Linux=iree_module_MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - RUNNER_ARGS - "--function=main" - "--device_allocator=caching" - "--input=1x224x224x3xf32=0" -) - -iree_benchmark_suite_module_test( - NAME "efficientnet_int8_correctness_test" - DRIVER "local-sync" - EXPECTED_OUTPUT "efficientnet_int8_expected_output.txt" - MODULES - "x86_64-Linux=iree_module_EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - RUNNER_ARGS - "--function=main" - "--device_allocator=caching" - "--input=1x224x224x3xui8=0" -) - -iree_benchmark_suite_module_test( - NAME "deeplab_v3_fp32_correctness_test" - DRIVER "local-sync" - EXPECTED_OUTPUT "https://storage.googleapis.com/iree-model-artifacts/deeplab_v3_fp32_input_0_expected_output.npy" - MODULES - "arm_64-Android=iree_module_DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_/module.vmfb" - "x86_64-Linux=iree_module_DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - RUNNER_ARGS - "--function=main" - "--device_allocator=caching" - "--expected_f32_threshold=0.001" - "--input=1x257x257x3xf32=0" -) - -iree_benchmark_suite_module_test( - NAME "person_detect_int8_correctness_test" - DRIVER "local-sync" - EXPECTED_OUTPUT "1x2xi8=[72 -72]" - MODULES - "riscv_32-Linux=iree_module_PersonDetect_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - "riscv_64-Linux=iree_module_PersonDetect_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - "x86_64-Linux=iree_module_PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - RUNNER_ARGS - "--function=main" - "--device_allocator=caching" - "--input=1x96x96x1xi8=0" -) diff --git a/tests/e2e/test_artifacts/CMakeLists.txt b/tests/e2e/test_artifacts/CMakeLists.txt deleted file mode 100644 index 5f8e005e763c..000000000000 --- a/tests/e2e/test_artifacts/CMakeLists.txt +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2022 The IREE Authors -# -# Licensed under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -################################################################################ -# # -# E2E Test Artifacts Suite # -# # -# Includes the generated cmake files to build the e2e test artifacts. # -# The cmake files "generated_*.cmake" are generated by the python script: # -# "build_tools/testing/generate_cmake_e2e_test_artifacts_suite.py" and sould # -# not be edited manually. Please read README.md for the instructions to update # -# the test artifacts. # -# # -################################################################################ - -if(NOT IREE_BUILD_E2E_TEST_ARTIFACTS) - return() -endif() - -################################################################################ -# Defines the required variables. # -################################################################################ -iree_package_name(PACKAGE_NAME) -set(ROOT_ARTIFACTS_DIR "${IREE_BINARY_DIR}/e2e_test_artifacts") - -################################################################################ -# Defines the benchmark suite targets # -# # -# iree-benchmark-suites* fully prepares benchmark models for benchmarking: # -# * importing from source formats to IREE-compatible .mlir files # -# * compiling from .mlir files to benchmark-ready .vmfb files # -# # -# Each benchmark preset has a corresponding target # -# `iree-benchmark-suites-`. The full preset list can be found in: # -# build_tools/python/benchmark_suites/iree/benchmark_presets.py # -# # -################################################################################ - -# Size-based execution benchmark preset targets. -add_custom_target(iree-benchmark-suites-default) -add_custom_target(iree-benchmark-suites-large) - -# Architecture-based execution benchmark preset targets. -add_custom_target(iree-benchmark-suites-android-cpu) -add_custom_target(iree-benchmark-suites-android-cpu-dt-only) -add_custom_target(iree-benchmark-suites-android-gpu) -add_custom_target(iree-benchmark-suites-cuda) -add_custom_target(iree-benchmark-suites-cuda-large) -add_custom_target(iree-benchmark-suites-riscv) -add_custom_target(iree-benchmark-suites-vulkan-nvidia) -add_custom_target(iree-benchmark-suites-x86_64) -add_custom_target(iree-benchmark-suites-x86_64-dt-only) -add_custom_target(iree-benchmark-suites-x86_64-large) - -# Compilation benchmark preset targets. -add_custom_target(iree-benchmark-suites-comp-stats) -add_custom_target(iree-benchmark-suites-comp-stats-large) - -# iree-e2e-test-artifacts builds all e2e test artifacts. -add_custom_target(iree-e2e-test-artifacts) -add_dependencies(iree-e2e-test-artifacts - iree-benchmark-suites-default - iree-benchmark-suites-large - iree-benchmark-suites-comp-stats - iree-benchmark-suites-comp-stats-large -) - -# Map the legacy, intuitive target to default benchmark preset. -add_custom_target(iree-benchmark-suites) -add_dependencies(iree-benchmark-suites - iree-benchmark-suites-default -) - -# iree-benchmark-import-models* imports benchmark models from their source -# formats, such as .tflite flatbuffers, to IREE-compatible .mlir files. - -# Imported models used in iree-benchmark-suites-default + iree-benchmark-suites-comp-stats-default -add_custom_target(iree-benchmark-import-models) -# Imported models used in iree-benchmark-suites-large + iree-benchmark-suites-comp-stats-large -add_custom_target(iree-benchmark-import-models-large) - -# Import the generated benchmark suite cmake files. -include(generated_e2e_test_fetch_models.cmake) - -include(generated_e2e_test_iree_artifacts.cmake) diff --git a/tests/e2e/test_artifacts/README.md b/tests/e2e/test_artifacts/README.md deleted file mode 100644 index 50f6e585240d..000000000000 --- a/tests/e2e/test_artifacts/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# IREE E2E Test Artifacts Suite - -IREE E2E test artifacts suite is a collection of artifacts for e2e tests and -benchmarking, which usually depend on external models and module compilations. - -## Updating Test Artifacts - -All the test artifacts are defined by the python modules. To add/remove/update -these artifacts, modify the related python modules and regenerate the CMake -files with the command below: - -```sh -build_tools/scripts/generate_cmake_files.sh -``` - -Here are the places to find the definitions of the artifacts: - -- Model sources: - [build_tools/python/e2e_test_framework/models](/build_tools/python/e2e_test_framework/models) -- Benchmarks: - [build_tools/python/benchmark_suites](/build_tools/python/benchmark_suites) - -## Debugging Build Failures - -When an IREE module is failed to build in e2e test artifacts, the error message -will be like: - -``` -[1/1] Generating /.../e2e_test_artifacts/iree_MobileNetV3Small_fp32_module_baec9d4086496a94853f349354f87acb8397bf36169134d3269d5803888dcf49/module.vmfb from MobileNetV3Small_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags] -FAILED: e2e_test_artifacts/iree_MobileNetV3Small_fp32_module_baec9d4086496a94853f349354f87acb8397bf36169134d3269d5803888dcf49/module.vmfb /.../e2e_test_artifacts/iree_MobileNetV3Small_fp32_module_baec9d4086496a94853f349354f87acb8397bf36169134d3269d5803888dcf49/module.vmfb -cd /.../tests/e2e/test_artifacts && /.../tools/iree-compile --output-format=vm-bytecode ... -ninja: build stopped: subcommand failed. -``` - -The first line shows `Generating ... from `. You can find the compile flags in -[tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake](/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake) -by searching with it. - -> Note that some texts might be truncated from a long output line when shown on -> the console. You can pipe the CMake output to a log file to get full texts. - -The model name, architecture, and tags should also lead you to the source that -generates the build rule under -[build_tools/python/benchmark_suites](/build_tools/python/benchmark_suites). diff --git a/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake b/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake deleted file mode 100644 index d9d36517e224..000000000000 --- a/tests/e2e/test_artifacts/generated_e2e_test_fetch_models.cmake +++ /dev/null @@ -1,293 +0,0 @@ -iree_fetch_artifact( - NAME "model-matmul_1x256x2048_i8_i4_i32_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.30_1719457850/DOT_PRODUCT_JAX_1X256X2048XI8I4/linalg.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i4_i32_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-matmul_256x256x2048_i8_i4_i32_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.30_1719457850/DOT_PRODUCT_JAX_256X256X2048XI8I4/linalg.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i4_i32_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-matmul_1x256x2048_i8_i8_i32_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.30_1719457850/DOT_PRODUCT_JAX_1X256X2048XI8I8/linalg.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i8_i32_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-matmul_256x256x2048_i8_i8_i32_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.30_1719457850/DOT_PRODUCT_JAX_256X256X2048XI8I8/linalg.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i8_i32_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-PersonDetect_int8" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/person_detect.tflite" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_PersonDetect_int8.tflite" - UNPACK -) - -iree_fetch_artifact( - NAME "model-MobileNetV3Small_fp32" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/MobileNetV3SmallStaticBatch.tflite" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_MobileNetV3Small_fp32.tflite" - UNPACK -) - -iree_fetch_artifact( - NAME "model-DeepLabV3_fp32" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/deeplabv3.tflite" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_DeepLabV3_fp32.tflite" - UNPACK -) - -iree_fetch_artifact( - NAME "model-EfficientNet_int8" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/efficientnet_lite0_int8_2.tflite" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_EfficientNet_int8.tflite" - UNPACK -) - -iree_fetch_artifact( - NAME "model-MobileNetV1_fp32" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/mobilenet_v1_224_1.0_float.tflite" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_MobileNetV1_fp32.0_float.tflite" - UNPACK -) - -iree_fetch_artifact( - NAME "model-MobileNetV2_fp32" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/mobilenet_v2_1.0_224.tflite" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_MobileNetV2_fp32.0_224.tflite" - UNPACK -) - -iree_fetch_artifact( - NAME "model-MobileNetV2_int8" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/mobilenet_v2_1.0_224_quantized.tflite" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_MobileNetV2_int8.0_224_quantized.tflite" - UNPACK -) - -iree_fetch_artifact( - NAME "model-MobileSSD_fp32" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/mobile_ssd_v2_float_coco.tflite" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_MobileSSD_fp32.tflite" - UNPACK -) - -iree_fetch_artifact( - NAME "model-PoseNet_fp32" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/posenet.tflite" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_PoseNet_fp32.tflite" - UNPACK -) - -iree_fetch_artifact( - NAME "model-MobileBertSquad_fp16" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/mobilebertsquad.tflite" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_MobileBertSquad_fp16.tflite" - UNPACK -) - -iree_fetch_artifact( - NAME "model-MobileBertSquad_fp32" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/mobilebert-baseline-tf2-float.tflite" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_MobileBertSquad_fp32.tflite" - UNPACK -) - -iree_fetch_artifact( - NAME "model-MobileBertSquad_int8" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/mobilebert-baseline-tf2-quant.tflite" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_MobileBertSquad_int8.tflite" - UNPACK -) - -iree_fetch_artifact( - NAME "model-EfficientNetV2STF" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/tensorflow/manual/EfficientNetV2STF_1af8c88f4e64e388a0c87bbeddcfb888084059df30cd631340d51794a0796e0f.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_EfficientNetV2STF.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-MiniLML12H384Uncased" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/tensorflow/manual/MiniLML12H384Uncased_5aed9c3c3dfe8247ce76b74d518fa570b94dc0c3732631734d02ad70e4c74867.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_MiniLML12H384Uncased.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-GPT2_117M_TF_1X4XI32" - SOURCE_URL "https://storage.googleapis.com/iree-shared-files/tf_gpt2/static_input_seqlen5/stablehlo.mlir" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X4XI32.mlir" - UNPACK -) - -iree_fetch_artifact( - NAME "model-GPT2_117M_TF_1X1XI32" - SOURCE_URL "https://storage.googleapis.com/iree-shared-files/tf_gpt2/static_input_seqlen1/stablehlo.mlir" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X1XI32.mlir" - UNPACK -) - -iree_fetch_artifact( - NAME "model-BertForMaskedLMTF" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/tensorflow/manual/BertForMaskedLMTF_e757a10b24f6ff83aaae0ceb5bb05d4efe9ff3e9931f8e9a29f12bc5c2e42b5e.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BertForMaskedLMTF.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-BertLargeTF" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/tensorflow/manual/BertLargeTF_000793afb016fb3afc559304bcb3ba6cdb2df1825e8976ca236c07c12e4f65fa.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BertLargeTF.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-BERT_LARGE_JAX_384XI32_BATCH1" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.25_1709787220/BERT_LARGE_FP32_JAX_384XI32_BATCH1/stablehlo.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BERT_LARGE_JAX_384XI32_BATCH1.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-BERT_LARGE_JAX_384XI32_BATCH32" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.25_1709787220/BERT_LARGE_FP32_JAX_384XI32_BATCH32/stablehlo.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BERT_LARGE_JAX_384XI32_BATCH32.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-BERT_LARGE_JAX_384XI32_BATCH64" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.25_1709787220/BERT_LARGE_FP32_JAX_384XI32_BATCH64/stablehlo.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_BERT_LARGE_JAX_384XI32_BATCH64.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-RESNET50_FP32_JAX_3X224X224XF32_BATCH1" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.25_1709787220/RESNET50_FP32_JAX_3X224X224XF32_BATCH1/stablehlo.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_RESNET50_FP32_JAX_3X224X224XF32_BATCH1.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-RESNET50_FP32_JAX_3X224X224XF32_BATCH64" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.25_1709787220/RESNET50_FP32_JAX_3X224X224XF32_BATCH64/stablehlo.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_RESNET50_FP32_JAX_3X224X224XF32_BATCH64.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-RESNET50_FP32_JAX_3X224X224XF32_BATCH128" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.25_1709787220/RESNET50_FP32_JAX_3X224X224XF32_BATCH128/stablehlo.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_RESNET50_FP32_JAX_3X224X224XF32_BATCH128.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-T5_LARGE_FP32_JAX_512XI32_BATCH1" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.25_1709787220/T5_LARGE_FP32_JAX_512XI32_BATCH1/stablehlo.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_T5_LARGE_FP32_JAX_512XI32_BATCH1.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-T5_LARGE_FP32_JAX_512XI32_BATCH16" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.25_1709787220/T5_LARGE_FP32_JAX_512XI32_BATCH16/stablehlo.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_T5_LARGE_FP32_JAX_512XI32_BATCH16.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-T5_LARGE_FP32_JAX_512XI32_BATCH32" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/jax/jax_models_0.4.25_1709787220/T5_LARGE_FP32_JAX_512XI32_BATCH32/stablehlo.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_T5_LARGE_FP32_JAX_512XI32_BATCH32.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-matmul_3456x1024x2048_f16t_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230410_1681181224/matmul_3456x1024x2048_f16t_f16t_f16t_tile_config_default.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_3456x1024x2048_f16t_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-matmul_3456x1024x2048_f32t_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230410_1681181224/matmul_3456x1024x2048_f32t_f32t_f32t_tile_config_default.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_3456x1024x2048_f32t_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-matmul_2560x2560x2560_f16t_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230410_1681181224/matmul_2560x2560x2560_f16t_f16t_f16t_tile_config_default.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_2560x2560x2560_f16t_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-matmul_2560x2560x2560_f32t_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230410_1681181224/matmul_2560x2560x2560_f32t_f32t_f32t_tile_config_default.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_2560x2560x2560_f32t_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230525_1685058259/matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230525_1685058259/matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230525_1685058259/matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230612_1686563210/matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-matmul_128x256x8192_f16t_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230410_1681181224/matmul_128x256x8192_f16t_f16t_f16t_tile_config_default.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_128x256x8192_f16t_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-matmul_128x256x8192_f32t_tile_config_default" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/microbenchmarks/matmul/20230410_1681181224/matmul_128x256x8192_f32t_f32t_f32t_tile_config_default.mlirbc" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_matmul_128x256x8192_f32t_tile_config_default.mlirbc" - UNPACK -) - -iree_fetch_artifact( - NAME "model-Vit_int8" - SOURCE_URL "https://storage.googleapis.com/iree-model-artifacts/tflite/tflite_models_1698315913/VIT_CLASSIFICATION_INT8_TFLITE_3X224X224XINT8/model_int8.tflite" - OUTPUT "${ROOT_ARTIFACTS_DIR}/model_Vit_int8.tflite" - UNPACK -) diff --git a/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake b/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake deleted file mode 100644 index df6dc8367f61..000000000000 --- a/tests/e2e/test_artifacts/generated_e2e_test_iree_artifacts.cmake +++ /dev/null @@ -1,5349 +0,0 @@ -iree_import_tflite_model( - TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-PersonDetect_int8_tflite_" - SOURCE "${ROOT_ARTIFACTS_DIR}/model_PersonDetect_int8.tflite" - OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_PersonDetect_int8_tflite_.mlir" -) - -iree_import_tflite_model( - TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-MobileNetV3Small_fp32_tflite_" - SOURCE "${ROOT_ARTIFACTS_DIR}/model_MobileNetV3Small_fp32.tflite" - OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV3Small_fp32_tflite_.mlir" -) - -iree_import_tflite_model( - TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-DeepLabV3_fp32_tflite_" - SOURCE "${ROOT_ARTIFACTS_DIR}/model_DeepLabV3_fp32.tflite" - OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" -) - -iree_import_tflite_model( - TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-EfficientNet_int8_tflite_" - SOURCE "${ROOT_ARTIFACTS_DIR}/model_EfficientNet_int8.tflite" - OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_EfficientNet_int8_tflite_.mlir" -) - -iree_import_tflite_model( - TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-MobileNetV1_fp32_tflite_" - SOURCE "${ROOT_ARTIFACTS_DIR}/model_MobileNetV1_fp32.0_float.tflite" - OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV1_fp32_tflite_.mlir" -) - -iree_import_tflite_model( - TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-MobileNetV2_fp32_tflite_" - SOURCE "${ROOT_ARTIFACTS_DIR}/model_MobileNetV2_fp32.0_224.tflite" - OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_fp32_tflite_.mlir" -) - -iree_import_tflite_model( - TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-MobileNetV2_int8_tflite_" - SOURCE "${ROOT_ARTIFACTS_DIR}/model_MobileNetV2_int8.0_224_quantized.tflite" - OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_int8_tflite_.mlir" -) - -iree_import_tflite_model( - TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-MobileSSD_fp32_tflite_" - SOURCE "${ROOT_ARTIFACTS_DIR}/model_MobileSSD_fp32.tflite" - OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_MobileSSD_fp32_tflite_.mlir" -) - -iree_import_tflite_model( - TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-PoseNet_fp32_tflite_" - SOURCE "${ROOT_ARTIFACTS_DIR}/model_PoseNet_fp32.tflite" - OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_PoseNet_fp32_tflite_.mlir" -) - -iree_import_tflite_model( - TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-MobileBertSquad_fp16_tflite_" - SOURCE "${ROOT_ARTIFACTS_DIR}/model_MobileBertSquad_fp16.tflite" - OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp16_tflite_.mlir" -) - -iree_import_tflite_model( - TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-MobileBertSquad_fp32_tflite_" - SOURCE "${ROOT_ARTIFACTS_DIR}/model_MobileBertSquad_fp32.tflite" - OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" -) - -iree_import_tflite_model( - TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-MobileBertSquad_int8_tflite_" - SOURCE "${ROOT_ARTIFACTS_DIR}/model_MobileBertSquad_int8.tflite" - OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" -) - -iree_import_tflite_model( - TARGET_NAME "${PACKAGE_NAME}_iree-imported-model-Vit_int8_tflite_" - SOURCE "${ROOT_ARTIFACTS_DIR}/model_Vit_int8.tflite" - OUTPUT_MLIR_FILE "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir" -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "matmul_1x256x2048_i8_i4_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "matmul_256x256x2048_i8_i4_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "matmul_1x256x2048_i8_i8_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "matmul_256x256x2048_i8_i8_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PersonDetect_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "PersonDetect_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV3Small_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "MobileNetV3Small_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_EfficientNet_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "EfficientNet_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV1_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "MobileNetV1_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "MobileNetV2_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "MobileNetV2_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileSSD_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "MobileSSD_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PoseNet_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "PoseNet_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp16_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "MobileBertSquad_fp16(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_EfficientNetV2STF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "EfficientNetV2STF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_MiniLML12H384Uncased.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "MiniLML12H384Uncased(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X4XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "GPT2_117M_TF_1X4XI32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X1XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "GPT2_117M_TF_1X1XI32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertForMaskedLMTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "BertForMaskedLMTF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertLargeTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - FRIENDLY_NAME "BertLargeTF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "matmul_1x256x2048_i8_i4_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "matmul_256x256x2048_i8_i4_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "matmul_1x256x2048_i8_i8_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "matmul_256x256x2048_i8_i8_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PersonDetect_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "PersonDetect_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV3Small_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "MobileNetV3Small_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_EfficientNet_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "EfficientNet_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV1_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "MobileNetV1_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "MobileNetV2_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "MobileNetV2_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileSSD_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "MobileSSD_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PoseNet_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "PoseNet_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp16_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "MobileBertSquad_fp16(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_EfficientNetV2STF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "EfficientNetV2STF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_MiniLML12H384Uncased.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "MiniLML12H384Uncased(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X4XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "GPT2_117M_TF_1X4XI32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X1XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "GPT2_117M_TF_1X1XI32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertForMaskedLMTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "BertForMaskedLMTF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertLargeTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - FRIENDLY_NAME "BertLargeTF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "matmul_1x256x2048_i8_i4_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "matmul_256x256x2048_i8_i4_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "matmul_1x256x2048_i8_i8_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "matmul_256x256x2048_i8_i8_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PersonDetect_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "PersonDetect_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV3Small_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "MobileNetV3Small_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_EfficientNet_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "EfficientNet_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV1_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "MobileNetV1_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "MobileNetV2_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "MobileNetV2_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileSSD_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "MobileSSD_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PoseNet_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "PoseNet_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp16_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "MobileBertSquad_fp16(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_EfficientNetV2STF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "EfficientNetV2STF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_MiniLML12H384Uncased.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "MiniLML12H384Uncased(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X4XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "GPT2_117M_TF_1X4XI32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X1XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "GPT2_117M_TF_1X1XI32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertForMaskedLMTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "BertForMaskedLMTF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertLargeTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "BertLargeTF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BERT_LARGE_JAX_384XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BERT_LARGE_JAX_384XI32_BATCH1.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BERT_LARGE_JAX_384XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "BERT_LARGE_JAX_384XI32_BATCH1(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BERT_LARGE_JAX_384XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BERT_LARGE_JAX_384XI32_BATCH32.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BERT_LARGE_JAX_384XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "BERT_LARGE_JAX_384XI32_BATCH32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BERT_LARGE_JAX_384XI32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BERT_LARGE_JAX_384XI32_BATCH64.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BERT_LARGE_JAX_384XI32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "BERT_LARGE_JAX_384XI32_BATCH64(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_RESNET50_FP32_JAX_3X224X224XF32_BATCH1.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_RESNET50_FP32_JAX_3X224X224XF32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "RESNET50_FP32_JAX_3X224X224XF32_BATCH1(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_RESNET50_FP32_JAX_3X224X224XF32_BATCH64.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_RESNET50_FP32_JAX_3X224X224XF32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "RESNET50_FP32_JAX_3X224X224XF32_BATCH64(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH128_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_RESNET50_FP32_JAX_3X224X224XF32_BATCH128.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_RESNET50_FP32_JAX_3X224X224XF32_BATCH128_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "RESNET50_FP32_JAX_3X224X224XF32_BATCH128(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_T5_LARGE_FP32_JAX_512XI32_BATCH1.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_T5_LARGE_FP32_JAX_512XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "T5_LARGE_FP32_JAX_512XI32_BATCH1(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH16_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_T5_LARGE_FP32_JAX_512XI32_BATCH16.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_T5_LARGE_FP32_JAX_512XI32_BATCH16_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "T5_LARGE_FP32_JAX_512XI32_BATCH16(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_T5_LARGE_FP32_JAX_512XI32_BATCH32.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_T5_LARGE_FP32_JAX_512XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - FRIENDLY_NAME "T5_LARGE_FP32_JAX_512XI32_BATCH32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNetV2STF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/model_EfficientNetV2STF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNetV2STF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=stablehlo" - "--iree-hal-cuda-llvm-target-arch=sm_80" - FRIENDLY_NAME "EfficientNetV2STF(stablehlo) [cuda-sm_80-linux_gnu-cuda][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MiniLML12H384Uncased_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/model_MiniLML12H384Uncased.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=stablehlo" - "--iree-hal-cuda-llvm-target-arch=sm_80" - FRIENDLY_NAME "MiniLML12H384Uncased(stablehlo) [cuda-sm_80-linux_gnu-cuda][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertForMaskedLMTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertForMaskedLMTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertForMaskedLMTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=stablehlo" - "--iree-hal-cuda-llvm-target-arch=sm_80" - FRIENDLY_NAME "BertForMaskedLMTF(stablehlo) [cuda-sm_80-linux_gnu-cuda][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertLargeTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertLargeTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=stablehlo" - "--iree-hal-cuda-llvm-target-arch=sm_80" - FRIENDLY_NAME "BertLargeTF(stablehlo) [cuda-sm_80-linux_gnu-cuda][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_3456x1024x2048_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_3456x1024x2048_f16t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_3456x1024x2048_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - FRIENDLY_NAME "matmul_3456x1024x2048_f16t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_3456x1024x2048_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_3456x1024x2048_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_3456x1024x2048_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - FRIENDLY_NAME "matmul_3456x1024x2048_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_2560x2560x2560_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_2560x2560x2560_f16t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2560x2560x2560_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - FRIENDLY_NAME "matmul_2560x2560x2560_f16t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_2560x2560x2560_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_2560x2560x2560_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2560x2560x2560_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - FRIENDLY_NAME "matmul_2560x2560x2560_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - FRIENDLY_NAME "matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - FRIENDLY_NAME "matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - FRIENDLY_NAME "matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - FRIENDLY_NAME "matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_128x256x8192_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_128x256x8192_f16t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_128x256x8192_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - "--iree-flow-split-matmul-reduction=4" - "--iree-codegen-llvmgpu-use-wmma" - FRIENDLY_NAME "matmul_128x256x8192_f16t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul,splitk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_128x256x8192_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_128x256x8192_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_128x256x8192_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - "--iree-flow-split-matmul-reduction=4" - "--iree-codegen-llvmgpu-use-wmma" - FRIENDLY_NAME "matmul_128x256x8192_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul,splitk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MiniLML12H384Uncased_stablehlo___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/model_MiniLML12H384Uncased.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - FRIENDLY_NAME "MiniLML12H384Uncased(stablehlo) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNet_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_EfficientNet_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - FRIENDLY_NAME "EfficientNet_int8(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV1_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV1_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV1_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - FRIENDLY_NAME "MobileNetV1_fp32(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - FRIENDLY_NAME "MobileNetV2_int8(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PersonDetect_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PersonDetect_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - FRIENDLY_NAME "PersonDetect_int8(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNet_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_EfficientNet_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv32-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv32" - "--iree-llvmcpu-target-abi=ilp32" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+zvl512b,+zve32f" - "--riscv-v-fixed-length-vector-lmul-max=8" - FRIENDLY_NAME "EfficientNet_int8(tflite) [riscv_32-generic-linux_gnu-llvm_cpu][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv32-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv32" - "--iree-llvmcpu-target-abi=ilp32" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+zvl512b,+zve32f" - "--riscv-v-fixed-length-vector-lmul-max=8" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [riscv_32-generic-linux_gnu-llvm_cpu][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PersonDetect_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PersonDetect_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv32-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv32" - "--iree-llvmcpu-target-abi=ilp32" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+zvl512b,+zve32f" - "--riscv-v-fixed-length-vector-lmul-max=8" - FRIENDLY_NAME "PersonDetect_int8(tflite) [riscv_32-generic-linux_gnu-llvm_cpu][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv32-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv32" - "--iree-llvmcpu-target-abi=ilp32" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+zvl512b,+zve32f" - "--riscv-v-fixed-length-vector-lmul-max=8" - FRIENDLY_NAME "MobileNetV2_int8(tflite) [riscv_32-generic-linux_gnu-llvm_cpu][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X4XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "GPT2_117M_TF_1X4XI32(stablehlo) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X1XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "GPT2_117M_TF_1X1XI32(stablehlo) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "Vit_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X4XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "GPT2_117M_TF_1X4XI32(stablehlo) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X1XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "GPT2_117M_TF_1X1XI32(stablehlo) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "Vit_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X4XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "GPT2_117M_TF_1X4XI32(stablehlo) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X1XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "GPT2_117M_TF_1X1XI32(stablehlo) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "Vit_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "matmul_1x256x2048_i8_i4_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "matmul_256x256x2048_i8_i4_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "matmul_1x256x2048_i8_i8_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "matmul_256x256x2048_i8_i8_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "matmul_1x256x2048_i8_i4_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "matmul_256x256x2048_i8_i4_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "matmul_1x256x2048_i8_i8_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "matmul_256x256x2048_i8_i8_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "matmul_1x256x2048_i8_i4_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "matmul_256x256x2048_i8_i4_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "matmul_1x256x2048_i8_i8_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - FRIENDLY_NAME "matmul_256x256x2048_i8_i8_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=adreno-unknown-android31" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [qualcomm-adreno-vulkan_android31-vulkan_spirv][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=adreno-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [qualcomm-adreno-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_demote-f32-to-f16_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp16_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_demote-f32-to-f16_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-input-demote-f32-to-f16" - FRIENDLY_NAME "MobileBertSquad_fp16(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][default-flags,demote-f32-to-f16]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][default-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - "--iree-stream-partitioning-favor=max-concurrency" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding,max-concurrency]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_demote-f32-to-f16_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp16_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_demote-f32-to-f16_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - "--iree-stream-partitioning-favor=max-concurrency" - "--iree-input-demote-f32-to-f16" - FRIENDLY_NAME "MobileBertSquad_fp16(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding,max-concurrency,demote-f32-to-f16]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - "--iree-stream-partitioning-favor=max-concurrency" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding,max-concurrency]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - "--iree-stream-partitioning-favor=max-concurrency" - "--iree-hal-benchmark-dispatch-repeat-count=32" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding,max-concurrency,repeated-kernel]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_demote-f32-to-f16_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp16_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_demote-f32-to-f16_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - "--iree-stream-partitioning-favor=max-concurrency" - "--iree-hal-benchmark-dispatch-repeat-count=32" - "--iree-input-demote-f32-to-f16" - FRIENDLY_NAME "MobileBertSquad_fp16(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding,max-concurrency,repeated-kernel,demote-f32-to-f16]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - "--iree-stream-partitioning-favor=max-concurrency" - "--iree-hal-benchmark-dispatch-repeat-count=32" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding,max-concurrency,repeated-kernel]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vmvx" - "--iree-input-type=tosa" - FRIENDLY_NAME "MobileNetV2_fp32(tflite) [vmvx-generic-vmvx-vmvx][experimental-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV3Small_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV3Small_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV3Small_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vmvx" - "--iree-input-type=tosa" - FRIENDLY_NAME "MobileNetV3Small_fp32(tflite) [vmvx-generic-vmvx-vmvx][experimental-flags]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_1x256x2048_i8_i4_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_256x256x2048_i8_i4_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_1x256x2048_i8_i8_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_256x256x2048_i8_i8_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PersonDetect_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "PersonDetect_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV3Small_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV3Small_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_EfficientNet_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "EfficientNet_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV1_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV1_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV2_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV2_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileSSD_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileSSD_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PoseNet_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "PoseNet_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp16_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp16(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_EfficientNetV2STF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "EfficientNetV2STF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_MiniLML12H384Uncased.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MiniLML12H384Uncased(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X4XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "GPT2_117M_TF_1X4XI32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X1XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "GPT2_117M_TF_1X1XI32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertForMaskedLMTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "BertForMaskedLMTF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertLargeTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=false" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "BertLargeTF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_1x256x2048_i8_i4_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_256x256x2048_i8_i4_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_1x256x2048_i8_i8_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_256x256x2048_i8_i8_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PersonDetect_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "PersonDetect_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV3Small_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV3Small_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_EfficientNet_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "EfficientNet_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV1_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV1_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV2_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV2_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileSSD_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileSSD_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PoseNet_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "PoseNet_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp16_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp16(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_EfficientNetV2STF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "EfficientNetV2STF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_MiniLML12H384Uncased.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MiniLML12H384Uncased(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X4XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "GPT2_117M_TF_1X4XI32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X1XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "GPT2_117M_TF_1X1XI32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertForMaskedLMTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "BertForMaskedLMTF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertLargeTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "BertLargeTF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_1x256x2048_i8_i4_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_256x256x2048_i8_i4_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_1x256x2048_i8_i8_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_256x256x2048_i8_i8_i32_tile_config_default(linalg) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PersonDetect_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "PersonDetect_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV3Small_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV3Small_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_EfficientNet_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "EfficientNet_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV1_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV1_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV2_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV2_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileSSD_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileSSD_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PoseNet_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "PoseNet_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp16_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp16(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_EfficientNetV2STF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "EfficientNetV2STF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_MiniLML12H384Uncased.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MiniLML12H384Uncased(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X4XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "GPT2_117M_TF_1X4XI32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X1XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "GPT2_117M_TF_1X1XI32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertForMaskedLMTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "BertForMaskedLMTF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertLargeTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "BertLargeTF(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BERT_LARGE_JAX_384XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BERT_LARGE_JAX_384XI32_BATCH1.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BERT_LARGE_JAX_384XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_BERT_LARGE_JAX_384XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "BERT_LARGE_JAX_384XI32_BATCH1(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BERT_LARGE_JAX_384XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BERT_LARGE_JAX_384XI32_BATCH32.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BERT_LARGE_JAX_384XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_BERT_LARGE_JAX_384XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "BERT_LARGE_JAX_384XI32_BATCH32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BERT_LARGE_JAX_384XI32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BERT_LARGE_JAX_384XI32_BATCH64.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BERT_LARGE_JAX_384XI32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_BERT_LARGE_JAX_384XI32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "BERT_LARGE_JAX_384XI32_BATCH64(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_RESNET50_FP32_JAX_3X224X224XF32_BATCH1.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_RESNET50_FP32_JAX_3X224X224XF32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_RESNET50_FP32_JAX_3X224X224XF32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "RESNET50_FP32_JAX_3X224X224XF32_BATCH1(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_RESNET50_FP32_JAX_3X224X224XF32_BATCH64.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_RESNET50_FP32_JAX_3X224X224XF32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_RESNET50_FP32_JAX_3X224X224XF32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "RESNET50_FP32_JAX_3X224X224XF32_BATCH64(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH128_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_RESNET50_FP32_JAX_3X224X224XF32_BATCH128.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_RESNET50_FP32_JAX_3X224X224XF32_BATCH128_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_RESNET50_FP32_JAX_3X224X224XF32_BATCH128_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "RESNET50_FP32_JAX_3X224X224XF32_BATCH128(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_T5_LARGE_FP32_JAX_512XI32_BATCH1.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_T5_LARGE_FP32_JAX_512XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_T5_LARGE_FP32_JAX_512XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "T5_LARGE_FP32_JAX_512XI32_BATCH1(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH16_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_T5_LARGE_FP32_JAX_512XI32_BATCH16.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_T5_LARGE_FP32_JAX_512XI32_BATCH16_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_T5_LARGE_FP32_JAX_512XI32_BATCH16_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "T5_LARGE_FP32_JAX_512XI32_BATCH16(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_T5_LARGE_FP32_JAX_512XI32_BATCH32.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_T5_LARGE_FP32_JAX_512XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=x86_64-unknown-linux-gnu" - "--iree-llvmcpu-target-cpu=cascadelake" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_T5_LARGE_FP32_JAX_512XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "T5_LARGE_FP32_JAX_512XI32_BATCH32(stablehlo) [x86_64-cascadelake-linux_gnu-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNetV2STF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_EfficientNetV2STF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNetV2STF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=stablehlo" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNetV2STF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "EfficientNetV2STF(stablehlo) [cuda-sm_80-linux_gnu-cuda][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MiniLML12H384Uncased_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_MiniLML12H384Uncased.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=stablehlo" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MiniLML12H384Uncased(stablehlo) [cuda-sm_80-linux_gnu-cuda][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertForMaskedLMTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertForMaskedLMTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertForMaskedLMTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=stablehlo" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_BertForMaskedLMTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "BertForMaskedLMTF(stablehlo) [cuda-sm_80-linux_gnu-cuda][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-BertLargeTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_BertLargeTF.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=stablehlo" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_BertLargeTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "BertLargeTF(stablehlo) [cuda-sm_80-linux_gnu-cuda][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_3456x1024x2048_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_3456x1024x2048_f16t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_3456x1024x2048_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_3456x1024x2048_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_3456x1024x2048_f16t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_3456x1024x2048_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_3456x1024x2048_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_3456x1024x2048_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_3456x1024x2048_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_3456x1024x2048_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_2560x2560x2560_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_2560x2560x2560_f16t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2560x2560x2560_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2560x2560x2560_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_2560x2560x2560_f16t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_2560x2560x2560_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_2560x2560x2560_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2560x2560x2560_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2560x2560x2560_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_2560x2560x2560_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_128x256x8192_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_128x256x8192_f16t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_128x256x8192_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - "--iree-flow-split-matmul-reduction=4" - "--iree-codegen-llvmgpu-use-wmma" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_128x256x8192_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_128x256x8192_f16t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul,splitk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_128x256x8192_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_128x256x8192_f32t_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_128x256x8192_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=cuda" - "--iree-input-type=none" - "--iree-hal-cuda-llvm-target-arch=sm_80" - "--iree-hal-benchmark-dispatch-repeat-count=100" - "--iree-flow-split-matmul-reduction=4" - "--iree-codegen-llvmgpu-use-wmma" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_128x256x8192_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_128x256x8192_f32t_tile_config_default(linalg) [cuda-sm_80-linux_gnu-cuda][ukernel,matmul,splitk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MiniLML12H384Uncased_stablehlo___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_MiniLML12H384Uncased.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MiniLML12H384Uncased_stablehlo___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MiniLML12H384Uncased(stablehlo) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNet_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_EfficientNet_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "EfficientNet_int8(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV1_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV1_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV1_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV1_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV1_fp32(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV2_int8(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PersonDetect_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PersonDetect_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv64-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv64" - "--iree-llvmcpu-target-abi=lp64d" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+d,+zvl512b,+v" - "--riscv-v-fixed-length-vector-lmul-max=8" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "PersonDetect_int8(tflite) [riscv_64-generic-linux_gnu-llvm_cpu][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-EfficientNet_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_EfficientNet_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv32-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv32" - "--iree-llvmcpu-target-abi=ilp32" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+zvl512b,+zve32f" - "--riscv-v-fixed-length-vector-lmul-max=8" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_EfficientNet_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "EfficientNet_int8(tflite) [riscv_32-generic-linux_gnu-llvm_cpu][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv32-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv32" - "--iree-llvmcpu-target-abi=ilp32" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+zvl512b,+zve32f" - "--riscv-v-fixed-length-vector-lmul-max=8" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [riscv_32-generic-linux_gnu-llvm_cpu][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-PersonDetect_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_PersonDetect_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv32-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv32" - "--iree-llvmcpu-target-abi=ilp32" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+zvl512b,+zve32f" - "--riscv-v-fixed-length-vector-lmul-max=8" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_PersonDetect_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "PersonDetect_int8(tflite) [riscv_32-generic-linux_gnu-llvm_cpu][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=riscv32-pc-linux-gnu" - "--iree-llvmcpu-target-cpu=generic-rv32" - "--iree-llvmcpu-target-abi=ilp32" - "--iree-llvmcpu-target-cpu-features=+m,+a,+f,+zvl512b,+zve32f" - "--riscv-v-fixed-length-vector-lmul-max=8" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV2_int8(tflite) [riscv_32-generic-linux_gnu-llvm_cpu][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X4XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "GPT2_117M_TF_1X4XI32(stablehlo) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X1XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "GPT2_117M_TF_1X1XI32(stablehlo) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "Vit_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X4XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "GPT2_117M_TF_1X4XI32(stablehlo) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X1XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "GPT2_117M_TF_1X1XI32(stablehlo) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "Vit_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_DeepLabV3_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "DeepLabV3_fp32(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X4XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "GPT2_117M_TF_1X4XI32(stablehlo) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_GPT2_117M_TF_1X1XI32.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=stablehlo" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "GPT2_117M_TF_1X1XI32(stablehlo) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_Vit_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=tosa" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "Vit_int8(tflite) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_1x256x2048_i8_i4_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_256x256x2048_i8_i4_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_1x256x2048_i8_i8_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=false" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_256x256x2048_i8_i8_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,no-dt,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_1x256x2048_i8_i4_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_256x256x2048_i8_i4_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_1x256x2048_i8_i8_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=all" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_256x256x2048_i8_i8_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][default-flags,dt-uk,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_1x256x2048_i8_i4_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i4_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_256x256x2048_i8_i4_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_1x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_1x256x2048_i8_i8_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/model_matmul_256x256x2048_i8_i8_i32_tile_config_default.mlirbc" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=llvm-cpu" - "--iree-input-type=none" - "--iree-llvmcpu-target-triple=aarch64-none-linux-android29" - "--iree-opt-data-tiling=true" - "--iree-llvmcpu-enable-ukernels=none" - "--iree-llvmcpu-target-cpu-features=+dotprod" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "matmul_256x256x2048_i8_i8_i32_tile_config_default(linalg) [armv8.2-a-generic-linux_android29-llvm_cpu][experimental-flags,dt-only,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=adreno-unknown-android31" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [qualcomm-adreno-vulkan_android31-vulkan_spirv][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=adreno-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [qualcomm-adreno-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_demote-f32-to-f16_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp16_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_demote-f32-to-f16_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-input-demote-f32-to-f16" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_demote-f32-to-f16_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp16(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][default-flags,demote-f32-to-f16,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][default-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - "--iree-stream-partitioning-favor=max-concurrency" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding,max-concurrency,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_demote-f32-to-f16_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp16_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_demote-f32-to-f16_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - "--iree-stream-partitioning-favor=max-concurrency" - "--iree-input-demote-f32-to-f16" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_demote-f32-to-f16_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp16(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding,max-concurrency,demote-f32-to-f16,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - "--iree-stream-partitioning-favor=max-concurrency" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding,max-concurrency,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - "--iree-stream-partitioning-favor=max-concurrency" - "--iree-hal-benchmark-dispatch-repeat-count=32" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp32(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding,max-concurrency,repeated-kernel,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_demote-f32-to-f16_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_fp16_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_demote-f32-to-f16_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - "--iree-stream-partitioning-favor=max-concurrency" - "--iree-hal-benchmark-dispatch-repeat-count=32" - "--iree-input-demote-f32-to-f16" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_demote-f32-to-f16_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_fp16(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding,max-concurrency,repeated-kernel,demote-f32-to-f16,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileBertSquad_int8_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vulkan-spirv" - "--iree-input-type=tosa" - "--iree-vulkan-target-triple=valhall-unknown-android31" - "--iree-flow-enable-fuse-padding-into-linalg-consumer-ops" - "--iree-stream-partitioning-favor=max-concurrency" - "--iree-hal-benchmark-dispatch-repeat-count=32" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileBertSquad_int8(tflite) [arm-valhall-vulkan_android31-vulkan_spirv][experimental-flags,fuse-padding,max-concurrency,repeated-kernel,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV2_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV2_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vmvx" - "--iree-input-type=tosa" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV2_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV2_fp32(tflite) [vmvx-generic-vmvx-vmvx][experimental-flags,compile-stats]" - PUBLIC -) - -iree_bytecode_module( - NAME "iree-module-MobileNetV3Small_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_compile-stats_" - SRC "${ROOT_ARTIFACTS_DIR}/iree_MobileNetV3Small_fp32_tflite_.mlir" - MODULE_FILE_NAME "${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV3Small_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_compile-stats_/module.vmfb" - FLAGS - "--iree-hal-target-backends=vmvx" - "--iree-input-type=tosa" - "--iree-vm-emit-polyglot-zip=true" - "--iree-llvmcpu-debug-symbols=false" - "--iree-scheduling-dump-statistics-format=json" - "--iree-scheduling-dump-statistics-file=${ROOT_ARTIFACTS_DIR}/iree_module_MobileNetV3Small_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_compile-stats_/scheduling_stats.json" - FRIENDLY_NAME "MobileNetV3Small_fp32(tflite) [vmvx-generic-vmvx-vmvx][experimental-flags,compile-stats]" - PUBLIC -) - -add_dependencies(iree-benchmark-import-models - ${PACKAGE_NAME}_iree-imported-model-DeepLabV3_fp32_tflite_ - ${PACKAGE_NAME}_iree-imported-model-EfficientNet_int8_tflite_ - ${PACKAGE_NAME}_iree-imported-model-MobileBertSquad_fp16_tflite_ - ${PACKAGE_NAME}_iree-imported-model-MobileBertSquad_fp32_tflite_ - ${PACKAGE_NAME}_iree-imported-model-MobileBertSquad_int8_tflite_ - ${PACKAGE_NAME}_iree-imported-model-MobileNetV1_fp32_tflite_ - ${PACKAGE_NAME}_iree-imported-model-MobileNetV2_fp32_tflite_ - ${PACKAGE_NAME}_iree-imported-model-MobileNetV2_int8_tflite_ - ${PACKAGE_NAME}_iree-imported-model-MobileNetV3Small_fp32_tflite_ - ${PACKAGE_NAME}_iree-imported-model-MobileSSD_fp32_tflite_ - ${PACKAGE_NAME}_iree-imported-model-PersonDetect_int8_tflite_ - ${PACKAGE_NAME}_iree-imported-model-PoseNet_fp32_tflite_ - ${PACKAGE_NAME}_iree-imported-model-Vit_int8_tflite_ - ${PACKAGE_NAME}_model-BertForMaskedLMTF - ${PACKAGE_NAME}_model-BertLargeTF - ${PACKAGE_NAME}_model-EfficientNetV2STF - ${PACKAGE_NAME}_model-GPT2_117M_TF_1X1XI32 - ${PACKAGE_NAME}_model-GPT2_117M_TF_1X4XI32 - ${PACKAGE_NAME}_model-MiniLML12H384Uncased - ${PACKAGE_NAME}_model-matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default - ${PACKAGE_NAME}_model-matmul_128x256x8192_f16t_tile_config_default - ${PACKAGE_NAME}_model-matmul_128x256x8192_f32t_tile_config_default - ${PACKAGE_NAME}_model-matmul_1x256x2048_i8_i4_i32_tile_config_default - ${PACKAGE_NAME}_model-matmul_1x256x2048_i8_i8_i32_tile_config_default - ${PACKAGE_NAME}_model-matmul_2560x2560x2560_f16t_tile_config_default - ${PACKAGE_NAME}_model-matmul_2560x2560x2560_f32t_tile_config_default - ${PACKAGE_NAME}_model-matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default - ${PACKAGE_NAME}_model-matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default - ${PACKAGE_NAME}_model-matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default - ${PACKAGE_NAME}_model-matmul_256x256x2048_i8_i4_i32_tile_config_default - ${PACKAGE_NAME}_model-matmul_256x256x2048_i8_i8_i32_tile_config_default - ${PACKAGE_NAME}_model-matmul_3456x1024x2048_f16t_tile_config_default - ${PACKAGE_NAME}_model-matmul_3456x1024x2048_f32t_tile_config_default -) - -add_dependencies(iree-benchmark-import-models-large - ${PACKAGE_NAME}_model-BERT_LARGE_JAX_384XI32_BATCH1 - ${PACKAGE_NAME}_model-BERT_LARGE_JAX_384XI32_BATCH32 - ${PACKAGE_NAME}_model-BERT_LARGE_JAX_384XI32_BATCH64 - ${PACKAGE_NAME}_model-RESNET50_FP32_JAX_3X224X224XF32_BATCH1 - ${PACKAGE_NAME}_model-RESNET50_FP32_JAX_3X224X224XF32_BATCH128 - ${PACKAGE_NAME}_model-RESNET50_FP32_JAX_3X224X224XF32_BATCH64 - ${PACKAGE_NAME}_model-T5_LARGE_FP32_JAX_512XI32_BATCH1 - ${PACKAGE_NAME}_model-T5_LARGE_FP32_JAX_512XI32_BATCH16 - ${PACKAGE_NAME}_model-T5_LARGE_FP32_JAX_512XI32_BATCH32 -) - -add_dependencies(iree-benchmark-suites-android-cpu - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ -) - -add_dependencies(iree-benchmark-suites-android-cpu-dt-only - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ -) - -add_dependencies(iree-benchmark-suites-android-gpu - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_demote-f32-to-f16_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_demote-f32-to-f16_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_demote-f32-to-f16_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_ -) - -add_dependencies(iree-benchmark-suites-comp-stats - ${PACKAGE_NAME}_iree-module-BertForMaskedLMTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-BertLargeTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-EfficientNetV2STF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_demote-f32-to-f16_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_demote-f32-to-f16_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_demote-f32-to-f16_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV1_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV3Small_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_compile-stats_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_128x256x8192_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_128x256x8192_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_2560x2560x2560_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_2560x2560x2560_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_3456x1024x2048_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_ - ${PACKAGE_NAME}_iree-module-matmul_3456x1024x2048_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_compile-stats_ -) - -add_dependencies(iree-benchmark-suites-comp-stats-large - ${PACKAGE_NAME}_iree-module-BERT_LARGE_JAX_384XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-BERT_LARGE_JAX_384XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-BERT_LARGE_JAX_384XI32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH128_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH16_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ - ${PACKAGE_NAME}_iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_compile-stats_ -) - -add_dependencies(iree-benchmark-suites-cuda - ${PACKAGE_NAME}_iree-module-BertForMaskedLMTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_ - ${PACKAGE_NAME}_iree-module-BertLargeTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_ - ${PACKAGE_NAME}_iree-module-EfficientNetV2STF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_ - ${PACKAGE_NAME}_iree-module-matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_128x256x8192_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_ - ${PACKAGE_NAME}_iree-module-matmul_128x256x8192_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_ - ${PACKAGE_NAME}_iree-module-matmul_2560x2560x2560_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_2560x2560x2560_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_3456x1024x2048_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_3456x1024x2048_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ -) - -add_dependencies(iree-benchmark-suites-default - ${PACKAGE_NAME}_iree-module-BertForMaskedLMTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_ - ${PACKAGE_NAME}_iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-BertLargeTF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_ - ${PACKAGE_NAME}_iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-EfficientNetV2STF_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_ - ${PACKAGE_NAME}_iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___cuda-sm_80-linux_gnu-cuda__default-flags_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_demote-f32-to-f16_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_demote-f32-to-f16_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_demote-f32-to-f16_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___qualcomm-adreno-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___arm-valhall-vulkan_android31-vulkan_spirv__experimental-flags_fuse-padding_max-concurrency_repeated-kernel_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileNetV1_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileNetV3Small_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_ - ${PACKAGE_NAME}_iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-Vit_int8_tflite___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_123x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_128x256x8192_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_ - ${PACKAGE_NAME}_iree-module-matmul_128x256x8192_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_splitk_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_2560x2560x2560_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_2560x2560x2560_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_2562x2561x2561_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_2562x2564x2562_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_2564x2564x2564_f32t_f32t_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___armv8.2-a-generic-linux_android29-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_3456x1024x2048_f16t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ - ${PACKAGE_NAME}_iree-module-matmul_3456x1024x2048_f32t_tile_config_default_linalg___cuda-sm_80-linux_gnu-cuda__ukernel_matmul_ -) - -add_dependencies(iree-benchmark-suites-large - ${PACKAGE_NAME}_iree-module-BERT_LARGE_JAX_384XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-BERT_LARGE_JAX_384XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-BERT_LARGE_JAX_384XI32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH128_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH16_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ -) - -add_dependencies(iree-benchmark-suites-riscv - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileNetV1_fp32_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___riscv_32-generic-linux_gnu-llvm_cpu__default-flags_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___riscv_64-generic-linux_gnu-llvm_cpu__default-flags_ -) - -add_dependencies(iree-benchmark-suites-x86_64 - ${PACKAGE_NAME}_iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileNetV3Small_fp32_tflite___vmvx-generic-vmvx-vmvx__experimental-flags_ - ${PACKAGE_NAME}_iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_no-dt_ -) - -add_dependencies(iree-benchmark-suites-x86_64-dt-only - ${PACKAGE_NAME}_iree-module-BertForMaskedLMTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-BertLargeTF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-DeepLabV3_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-EfficientNetV2STF_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-EfficientNet_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X1XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-GPT2_117M_TF_1X4XI32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MiniLML12H384Uncased_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp16_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileBertSquad_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileNetV1_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileNetV2_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileNetV3Small_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-MobileSSD_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-PersonDetect_int8_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-PoseNet_fp32_tflite___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_1x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i4_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ - ${PACKAGE_NAME}_iree-module-matmul_256x256x2048_i8_i8_i32_tile_config_default_linalg___x86_64-cascadelake-linux_gnu-llvm_cpu__experimental-flags_dt-only_ -) - -add_dependencies(iree-benchmark-suites-x86_64-large - ${PACKAGE_NAME}_iree-module-BERT_LARGE_JAX_384XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-BERT_LARGE_JAX_384XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-BERT_LARGE_JAX_384XI32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH128_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-RESNET50_FP32_JAX_3X224X224XF32_BATCH64_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH16_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH1_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ - ${PACKAGE_NAME}_iree-module-T5_LARGE_FP32_JAX_512XI32_BATCH32_stablehlo___x86_64-cascadelake-linux_gnu-llvm_cpu__default-flags_dt-uk_ -) diff --git a/tools/test/CMakeLists.txt b/tools/test/CMakeLists.txt index a866548ddb5a..952b8947a960 100644 --- a/tools/test/CMakeLists.txt +++ b/tools/test/CMakeLists.txt @@ -58,34 +58,3 @@ iree_lit_test_suite( ) ### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ### - -if(IREE_BUILD_TESTS AND - (IREE_TARGET_BACKEND_VMVX OR IREE_HOST_BIN_DIR) AND - IREE_HAL_DRIVER_LOCAL_SYNC) - iree_bytecode_module( - NAME - iree_run_module_bytecode_module_vmvx - SRC - "iree-run-module.mlir" - MODULE_FILE_NAME - "iree_run_module_bytecode_module_vmvx.vmfb" - FLAGS - "--iree-hal-target-backends=vmvx" - ) - - iree_run_module_test( - NAME - iree_run_module_correctness_test - MODULE_SRC - "${CMAKE_CURRENT_BINARY_DIR}/iree_run_module_bytecode_module_vmvx.vmfb" - DRIVER - "local-sync" - RUNNER_ARGS - "--function=abs" - "--input=2xf32=-10 -5" - EXPECTED_OUTPUT - "2xf32=10 5" - DEPS - iree_tools_test_iree_run_module_bytecode_module_vmvx - ) -endif()