Drop Tracy from CI benchmarks. (iree-org#17383)

This drops support for capturing traces as part of CI benchmarking to
fix iree-org#16856. This PR is a synced
and updated version of iree-org#16857.

While traces are invaluable in analyzing performance,
* This implementation is difficult to maintain (updating IREE's Tracy
version can't be performed without also building for multiple operating
systems and uploading binary files to a cloud bucket with limited
permissions)
* Trace collection nearly doubles CI time for every benchmark run (i.e.
10m->20m or 20m->40m), leading to occasionally long queueing (2h+)
* Trace collection contributes to cloud storage and network usage (there
are larger offenders: iree-org#16396, but
we still need to trim costs)

---------

Co-authored-by: Benoit Jacob <jacob.benoit.1@gmail.com>
Signed-off-by: Lubo Litchev <lubol@google.com>

.github/workflows/ARTIFACT_SUMMARY_TEMPLATE.md

@@ -7,7 +7,7 @@ export INSTALL_DIR_URL="${INSTALL_DIR_GCS_ARTIFACT}"
 export E2E_TEST_ARTIFACTS_DIR_URL="${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}"
 # Directory of benchmark tool binaries:
 export BENCHMARK_TOOLS_DIR_URL="${BENCHMARK_TOOLS_GCS_ARTIFACT_DIR}"
-# Directory of execution benchmark results and traces:
+# Directory of execution benchmark results:
 export EXECUTION_BENCHMARK_RESULTS_DIR_URL="${EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR}"
 # Compilation benchmark results:
 export COMPILATION_BENCHMARK_RESULTS_URL="${COMPILATION_BENCHMARK_RESULTS_GCS_ARTIFACT}"
@@ -37,7 +37,7 @@ gcloud storage cp -r "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}/*" /tmp/iree_e2e_te
 ```
 
 ```sh
-# Download execution benchmark results and traces
+# Download execution benchmark results
 mkdir /tmp/iree_benchmark_results
 gcloud storage cp -r "${EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR}/*" /tmp/iree_benchmark_results
 ```

.github/workflows/benchmark_execution.yml

@@ -163,9 +163,7 @@ jobs:
           BENCHMARK_TOOLS_DIR: ${{ env.PLATFORM_ARCH }}-benchmark-tools-dir
         run: |
           tar -xf ${BENCHMARK_TOOLS_ARCHIVE}
-          echo "normal-benchmark-tools-dir=${BENCHMARK_TOOLS_DIR}/build/tools" >> "${GITHUB_OUTPUT}"
-          echo "traced-benchmark-tools-dir=${BENCHMARK_TOOLS_DIR}/build-traced/tools" >> "${GITHUB_OUTPUT}"
-          echo "tracy-capture-tool=${BENCHMARK_TOOLS_DIR}/build-traced/tracy-capture" >> "${GITHUB_OUTPUT}"
+          echo "benchmark-tools-dir=${BENCHMARK_TOOLS_DIR}/build/tools" >> "${GITHUB_OUTPUT}"
       - name: "Determine Shard Suffix"
         id: sharding
         run: |
@@ -178,13 +176,10 @@ jobs:
         env:
           IREE_EXECUTION_BENCHMARK_CONFIG: ${{ steps.download-assets.outputs.benchmark-config }}
           IREE_DOCKER_WRAPPER: ./build_tools/github_actions/docker_run.sh
-          IREE_NORMAL_BENCHMARK_TOOLS_DIR: ${{ steps.unpack-tools.outputs.normal-benchmark-tools-dir }}
-          IREE_TRACED_BENCHMARK_TOOLS_DIR: ${{ steps.unpack-tools.outputs.traced-benchmark-tools-dir }}
-          IREE_TRACY_CAPTURE_TOOL: ${{ steps.unpack-tools.outputs.tracy-capture-tool }}
+          IREE_BENCHMARK_TOOLS_DIR: ${{ steps.unpack-tools.outputs.benchmark-tools-dir }}
           IREE_TARGET_DEVICE_NAME: ${{ env.DEVICE_NAME }}
           IREE_SHARD_INDEX: ${{ matrix.benchmark.shard.index }}
           IREE_BENCHMARK_RESULTS: ${{ env.BENCHMARK_RESULTS_DIR }}/benchmark-results-${{ matrix.benchmark.device_name }}${{ steps.sharding.outputs.suffix }}.json
-          IREE_BENCHMARK_TRACES: ${{ env.BENCHMARK_RESULTS_DIR }}/benchmark-traces-${{ matrix.benchmark.device_name }}${{ steps.sharding.outputs.suffix }}.tar.gz
         run: |
           mkdir -p ${BENCHMARK_RESULTS_DIR}
           export IREE_E2E_TEST_ARTIFACTS_DIR="${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR/gs:\/\//https://storage.googleapis.com/}"

.github/workflows/build_benchmark_tools.yml

@@ -69,17 +69,14 @@ jobs:
             docker_image: "gcr.io/iree-oss/base-bleeding-edge@sha256:c5f28883e6c570c20128fb37d7af3a00a25df3ce4e2b3a24c3a8dcd183182a27"
             # Builds tools on the host and assumes the builder is Linux x86_64.
             build_script: "./build_tools/cmake/build_runtime.sh"
-            tracy_capture: "gs://iree-shared-files/tracy-capture-linux-x86_64-af73dba7"
           - platform: "linux"
             arch: "riscv_64"
             docker_image: "gcr.io/iree-oss/riscv@sha256:62e87bad3405d691ddba6f9be0ef44eeb60461a467c8d86f0842c81a1f97da79"
             build_script: "./build_tools/cmake/build_riscv.sh"
-            tracy_capture: "gs://iree-shared-files/tracy-capture-linux-x86_64-af73dba7"
           - platform: "android"
             arch: "armv8.2-a"
             docker_image: "gcr.io/iree-oss/android@sha256:66b92a1c920588a73d03316f26025407ea754bab93e8f9bfe40dbf6ed5fe6c7e"
             build_script: "./build_tools/cmake/build_android.sh"
-            tracy_capture: "gs://iree-shared-files/tracy-capture-linux-arm_64-af73dba7"
     env:
       PLATFORM: ${{ matrix.target.platform }}
       ARCH: ${{ matrix.target.arch }}
@@ -116,15 +113,6 @@ jobs:
             --env "BUILD_PRESET=benchmark-with-tracing" \
             --env "IREE_HOST_BIN_DIR=${INSTALL_DIR}/bin" \
             "${DOCKER_IMAGE}" "${BUILD_SCRIPT}" "${BUILD_TOOLS_DIR}/build-traced"
-      - name: "Downloading pre-built tracy capture tool"
-        id: download-tracy-capture
-        env:
-          TRACY_CAPTURE_GCS_ARTIFACT: ${{ matrix.target.tracy_capture }}
-          TRACY_CAPTURE: ${{ env.BUILD_TOOLS_DIR }}/build-traced/tracy-capture
-        run: |
-          gcloud storage cp "${TRACY_CAPTURE_GCS_ARTIFACT}" "${TRACY_CAPTURE}"
-          chmod +x "${TRACY_CAPTURE}"
-          echo "tracy-capture=${TRACY_CAPTURE}" >> "${GITHUB_OUTPUT}"
       - name: "Creating the benchmark tools archive"
         # Here we pack a tracy-capture binary (~7MB) into each benchmark tools
         # archive. This could be wasteful because multiple benchmark tools
@@ -141,13 +129,11 @@ jobs:
         # into each benchmark tools archive.
         id: archive
         env:
-          TRACY_CAPTURE: ${{ steps.download-tracy-capture.outputs.tracy-capture }}
           BENCHMARK_TOOLS_ARCHIVE: ${{ env.PLATFORM }}-${{ env.ARCH }}-benchmark-tools.tar
         run: |
           tar -cf "${BENCHMARK_TOOLS_ARCHIVE}" \
             "${BUILD_TOOLS_DIR}"/*/tools/iree-benchmark-module \
-            "${BUILD_TOOLS_DIR}"/*/tools/build_config.txt \
-            "${TRACY_CAPTURE}"
+            "${BUILD_TOOLS_DIR}"/*/tools/build_config.txt
           echo "benchmark-tools-archive=${BENCHMARK_TOOLS_ARCHIVE}" >> "${GITHUB_OUTPUT}"
       - name: "Uploading the benchmark tools archive"
         id: upload

build_tools/benchmarks/common/benchmark_config.py

@@ -14,24 +14,6 @@
 from common import benchmark_definition
 
 BENCHMARK_RESULTS_REL_PATH = "benchmark-results"
-CAPTURES_REL_PATH = "captures"
-
-
-@dataclass
-class TraceCaptureConfig:
-    """Represents the settings for capturing traces during benchamrking.
-
-    traced_benchmark_tool_dir: the path to the tracing-enabled benchmark tool
-      directory.
-    trace_capture_tool: the path to the tool for collecting captured traces.
-    capture_tarball: the path of capture tar archive.
-    capture_tmp_dir: the temporary directory to store captured traces.
-    """
-
-    traced_benchmark_tool_dir: pathlib.Path
-    trace_capture_tool: pathlib.Path
-    capture_tarball: pathlib.Path
-    capture_tmp_dir: pathlib.Path
 
 
 @dataclass
@@ -43,10 +25,7 @@ class BenchmarkConfig:
       benchmark suites.
     benchmark_results_dir: the directory to store benchmark results files.
     git_commit_hash: the git commit hash.
-    normal_benchmark_tool_dir: the path to the non-traced benchmark tool
-      directory.
-    trace_capture_config: the config for capturing traces. Set if and only if
-      the traces need to be captured.
+    benchmark_tool_dir: the path to the tool directory.
     driver_filter: filter benchmarks to those whose driver matches this regex
       (or all if this is None).
     model_name_filter: filter benchmarks to those whose model name matches this
@@ -68,8 +47,7 @@ class BenchmarkConfig:
     benchmark_results_dir: pathlib.Path
     git_commit_hash: str
 
-    normal_benchmark_tool_dir: Optional[pathlib.Path] = None
-    trace_capture_config: Optional[TraceCaptureConfig] = None
+    benchmark_tool_dir: Optional[pathlib.Path] = None
 
     driver_filter: Optional[str] = None
     model_name_filter: Optional[str] = None
@@ -93,31 +71,11 @@ def build_from_args(args: Namespace, git_commit_hash: str):
         def real_path_or_none(path: Optional[pathlib.Path]) -> Optional[pathlib.Path]:
             return path.resolve() if path else None
 
-        if not args.normal_benchmark_tool_dir and not args.traced_benchmark_tool_dir:
-            raise ValueError(
-                "At least one of --normal_benchmark_tool_dir or --traced_benchmark_tool_dir should be specified."
-            )
-        if not (
-            (args.traced_benchmark_tool_dir is None)
-            == (args.trace_capture_tool is None)
-            == (args.capture_tarball is None)
-        ):
-            raise ValueError(
-                "The following 3 flags should be simultaneously all specified or all unspecified: --traced_benchmark_tool_dir, --trace_capture_tool, --capture_tarball"
-            )
+        if not args.benchmark_tool_dir:
+            raise ValueError("--benchmark_tool_dir should be specified.")
 
         per_commit_tmp_dir: pathlib.Path = (args.tmp_dir / git_commit_hash).resolve()
 
-        if args.traced_benchmark_tool_dir is None:
-            trace_capture_config = None
-        else:
-            trace_capture_config = TraceCaptureConfig(
-                traced_benchmark_tool_dir=args.traced_benchmark_tool_dir.resolve(),
-                trace_capture_tool=args.trace_capture_tool.resolve(),
-                capture_tarball=args.capture_tarball.resolve(),
-                capture_tmp_dir=per_commit_tmp_dir / CAPTURES_REL_PATH,
-            )
-
         root_benchmark_dir = args.e2e_test_artifacts_dir
         # Convert the local path into Path object.
         if re.match("^[^:]+://", str(root_benchmark_dir)):
@@ -134,8 +92,7 @@ def real_path_or_none(path: Optional[pathlib.Path]) -> Optional[pathlib.Path]:
             root_benchmark_dir=root_benchmark_dir,
             benchmark_results_dir=per_commit_tmp_dir / BENCHMARK_RESULTS_REL_PATH,
             git_commit_hash=git_commit_hash,
-            normal_benchmark_tool_dir=real_path_or_none(args.normal_benchmark_tool_dir),
-            trace_capture_config=trace_capture_config,
+            benchmark_tool_dir=real_path_or_none(args.benchmark_tool_dir),
             driver_filter=args.driver_filter_regex,
             model_name_filter=args.model_name_regex,
             mode_filter=args.mode_regex,

build_tools/benchmarks/common/benchmark_config_test.py

@@ -22,11 +22,6 @@ def setUp(self):
         self.e2e_test_artifacts_dir.mkdir()
         self.normal_tool_dir = self.build_dir / "normal_tool"
         self.normal_tool_dir.mkdir()
-        self.traced_tool_dir = self.build_dir / "traced_tool"
-        self.traced_tool_dir.mkdir()
-        self.trace_capture_tool = self.build_dir / "tracy_capture"
-        # Create capture tool with executable file mode.
-        self.trace_capture_tool.touch(mode=0o755)
         self.execution_config = self.build_dir / "execution_config.json"
         self.execution_config.touch()
 
@@ -38,10 +33,7 @@ def test_build_from_args(self):
         args = common_arguments.Parser().parse_args(
             [
                 f"--tmp_dir={self.tmp_dir}",
-                f"--normal_benchmark_tool_dir={self.normal_tool_dir}",
-                f"--traced_benchmark_tool_dir={self.traced_tool_dir}",
-                f"--trace_capture_tool={self.trace_capture_tool}",
-                f"--capture_tarball=capture.tar",
+                f"--benchmark_tool_dir={self.normal_tool_dir}",
                 f"--driver_filter_regex=a",
                 f"--model_name_regex=b",
                 f"--mode_regex=c",
@@ -60,21 +52,14 @@ def test_build_from_args(self):
         )
 
         per_commit_tmp_dir = self.tmp_dir / "abcd"
-        expected_trace_capture_config = benchmark_config.TraceCaptureConfig(
-            traced_benchmark_tool_dir=self.traced_tool_dir,
-            trace_capture_tool=pathlib.Path(self.trace_capture_tool).resolve(),
-            capture_tarball=pathlib.Path("capture.tar").resolve(),
-            capture_tmp_dir=per_commit_tmp_dir / "captures",
-        )
         expected_config = benchmark_config.BenchmarkConfig(
             tmp_dir=per_commit_tmp_dir,
             root_benchmark_dir=benchmark_definition.ResourceLocation.build_local_path(
                 self.e2e_test_artifacts_dir
             ),
             benchmark_results_dir=per_commit_tmp_dir / "benchmark-results",
             git_commit_hash="abcd",
-            normal_benchmark_tool_dir=self.normal_tool_dir,
-            trace_capture_config=expected_trace_capture_config,
+            benchmark_tool_dir=self.normal_tool_dir,
             driver_filter="a",
             model_name_filter="b",
             mode_filter="c",
@@ -89,7 +74,7 @@ def test_build_from_args_benchmark_only(self):
         args = common_arguments.Parser().parse_args(
             [
                 f"--tmp_dir={self.tmp_dir}",
-                f"--normal_benchmark_tool_dir={self.normal_tool_dir}",
+                f"--benchmark_tool_dir={self.normal_tool_dir}",
                 f"--e2e_test_artifacts_dir={self.e2e_test_artifacts_dir}",
                 f"--execution_benchmark_config={self.execution_config}",
                 "--target_device=test",
@@ -100,13 +85,11 @@ def test_build_from_args_benchmark_only(self):
             args=args, git_commit_hash="abcd"
         )
 
-        self.assertIsNone(config.trace_capture_config)
-
     def test_build_from_args_with_test_artifacts_dir_url(self):
         args = common_arguments.Parser().parse_args(
             [
                 f"--tmp_dir={self.tmp_dir}",
-                f"--normal_benchmark_tool_dir={self.normal_tool_dir}",
+                f"--benchmark_tool_dir={self.normal_tool_dir}",
                 f"--e2e_test_artifacts_dir=https://example.com/testdata",
                 f"--execution_benchmark_config={self.execution_config}",
                 "--target_device=test",
@@ -121,25 +104,6 @@ def test_build_from_args_with_test_artifacts_dir_url(self):
             config.root_benchmark_dir.get_url(), "https://example.com/testdata"
         )
 
-    def test_build_from_args_invalid_capture_args(self):
-        args = common_arguments.Parser().parse_args(
-            [
-                f"--tmp_dir={self.tmp_dir}",
-                f"--normal_benchmark_tool_dir={self.normal_tool_dir}",
-                f"--traced_benchmark_tool_dir={self.traced_tool_dir}",
-                f"--e2e_test_artifacts_dir={self.e2e_test_artifacts_dir}",
-                f"--execution_benchmark_config={self.execution_config}",
-                "--target_device=test",
-            ]
-        )
-
-        self.assertRaises(
-            ValueError,
-            lambda: benchmark_config.BenchmarkConfig.build_from_args(
-                args=args, git_commit_hash="abcd"
-            ),
-        )
-
 
 if __name__ == "__main__":
     unittest.main()

build_tools/benchmarks/common/benchmark_definition.py

@@ -187,18 +187,13 @@ def get_iree_benchmark_module_arguments(
     driver_info: DriverInfo,
     benchmark_min_time: Optional[float] = None,
     dump_results: bool = True,
-    capture_mode: bool = False,
 ):
     """Returns the common arguments to run iree-benchmark-module."""
 
     if driver_info.loader_name == "vmvx-module":
         # VMVX is very unoptimized for now and can take a long time to run.
         # Decrease the repetition for it until it's reasonably fast.
         repetitions = 3
-    elif capture_mode:
-        # Capture mode is slower and we just need enough repetition to collect
-        # trace after the warmup phase.
-        repetitions = 4
     else:
         repetitions = 10