Extract unittest scripts and add Buck mode (#8493)

Fixes #8419

.ci/scripts/unittest-buck2.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+# TODO: expand this to //...
+buck2 query //runtime/...
+
+# TODO: expand the covered scope of Buck targets.
+buck2 build //runtime/core/portable_type/...
+buck2 test //runtime/core/portable_type/...

.ci/scripts/unittest-linux-cmake.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+# Run pytest with coverage
+pytest -n auto --cov=./ --cov-report=xml
+# Run gtest
+LLVM_PROFDATA=llvm-profdata-12 LLVM_COV=llvm-cov-12 \
+test/run_oss_cpp_tests.sh

.ci/scripts/unittest-linux.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+BUILD_TOOL=$1
+if [[ $BUILD_TOOL =~ ^(cmake|buck2)$ ]]; then
+    echo "Running unittests for ${BUILD_TOOL} ..."
+else
+  echo "Missing build tool (require buck2 or cmake), exiting..."
+  exit 1
+fi
+
+# The generic Linux job chooses to use base env, not the one setup by the image
+eval "$(conda shell.bash hook)"
+CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+conda activate "${CONDA_ENV}"
+
+# Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate
+source .ci/scripts/setup-vulkan-linux-deps.sh
+
+PYTHON_EXECUTABLE=python \
+EXECUTORCH_BUILD_PYBIND=ON \
+CMAKE_ARGS="-DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \
+.ci/scripts/setup-linux.sh "$BUILD_TOOL"
+
+# Install llama3_2_vision dependencies.
+PYTHON_EXECUTABLE=python ./examples/models/llama3_2_vision/install_requirements.sh
+
+if [[ "$BUILD_TOOL" == "cmake" ]]; then
+    .ci/scripts/unittest-linux-cmake.sh
+elif [[ "$BUILD_TOOL" == "buck2" ]]; then
+    .ci/scripts/unittest-buck2.sh
+else
+    echo "Unknown build tool $BUILD_TOOL"
+    exit 1
+fi

.ci/scripts/unittest-macos-cmake.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+# Run pytest with coverage
+${CONDA_RUN} pytest -n auto --cov=./ --cov-report=xml
+# Run gtest
+LLVM_PROFDATA="xcrun llvm-profdata" LLVM_COV="xcrun llvm-cov" \
+${CONDA_RUN} test/run_oss_cpp_tests.sh

.ci/scripts/unittest-macos.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+BUILD_TOOL=$1
+if [[ $BUILD_TOOL =~ ^(cmake|buck2)$ ]]; then
+    echo "Running unittests for ${BUILD_TOOL} ..."
+else
+  echo "Missing build tool (require buck2 or cmake), exiting..."
+  exit 1
+fi
+
+bash .ci/scripts/setup-conda.sh
+eval "$(conda shell.bash hook)"
+
+# Create temp directory for sccache shims
+export TMP_DIR=$(mktemp -d)
+export PATH="${TMP_DIR}:$PATH"
+trap 'rm -rfv ${TMP_DIR}' EXIT
+
+# Setup MacOS dependencies as there is no Docker support on MacOS atm
+PYTHON_EXECUTABLE=python \
+EXECUTORCH_BUILD_PYBIND=ON \
+CMAKE_ARGS="-DEXECUTORCH_BUILD_COREML=ON -DEXECUTORCH_BUILD_MPS=ON -DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \
+${CONDA_RUN} --no-capture-output \
+.ci/scripts/setup-macos.sh cmake
+
+# Install llama3_2_vision dependencies.
+PYTHON_EXECUTABLE=python ./examples/models/llama3_2_vision/install_requirements.sh
+
+if [[ "$BUILD_TOOL" == "cmake" ]]; then
+    .ci/scripts/unittest-macos-cmake.sh
+elif [[ "$BUILD_TOOL" == "buck2" ]]; then
+    .ci/scripts/unittest-buck2.sh
+else
+    echo "Unknown build tool $BUILD_TOOL"
+    exit 1
+fi

.github/workflows/_unittest.yml

@@ -7,6 +7,10 @@ on:
         required: true
         type: string
         description: Name of the docker image to use.
+      build-tool:
+        required: true
+        type: string
+        description: Build tool to use, cmake or buck2.
       python-version:
         required: false
         type: string
@@ -26,28 +30,7 @@ jobs:
       timeout: 90
       script: |
         set -eux
-
-        # The generic Linux job chooses to use base env, not the one setup by the image
-        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
-        conda activate "${CONDA_ENV}"
-
-        # Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate
-        source .ci/scripts/setup-vulkan-linux-deps.sh
-
-        # Setup MacOS dependencies as there is no Docker support on MacOS atm
-        PYTHON_EXECUTABLE=python \
-        EXECUTORCH_BUILD_PYBIND=ON \
-        CMAKE_ARGS="-DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \
-        .ci/scripts/setup-linux.sh cmake
-
-        # Install llama3_2_vision dependencies.
-        PYTHON_EXECUTABLE=python ./examples/models/llama3_2_vision/install_requirements.sh
-
-        # Run pytest with coverage
-        pytest -n auto --cov=./ --cov-report=xml
-        # Run gtest
-        LLVM_PROFDATA=llvm-profdata-12 LLVM_COV=llvm-cov-12 \
-        test/run_oss_cpp_tests.sh
+        .ci/scripts/unittest-linux.sh "${{ inputs.build-tool }}"
 
   macos:
     uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
@@ -58,27 +41,4 @@ jobs:
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       script: |
         set -eux
-
-        bash .ci/scripts/setup-conda.sh
-
-        # Create temp directory for sccache shims
-        export TMP_DIR=$(mktemp -d)
-        export PATH="${TMP_DIR}:$PATH"
-        trap 'rm -rfv ${TMP_DIR}' EXIT
-
-        # Setup MacOS dependencies as there is no Docker support on MacOS atm
-        PYTHON_EXECUTABLE=python \
-        EXECUTORCH_BUILD_PYBIND=ON \
-        CMAKE_ARGS="-DEXECUTORCH_BUILD_COREML=ON -DEXECUTORCH_BUILD_MPS=ON -DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \
-        ${CONDA_RUN} --no-capture-output \
-        .ci/scripts/setup-macos.sh cmake
-
-        # Install llama3_2_vision dependencies.
-        PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \
-        ./examples/models/llama3_2_vision/install_requirements.sh
-
-        # Run pytest with coverage
-        ${CONDA_RUN} pytest -n auto --cov=./ --cov-report=xml
-        # Run gtest
-        LLVM_PROFDATA="xcrun llvm-profdata" LLVM_COV="xcrun llvm-cov" \
-        ${CONDA_RUN} test/run_oss_cpp_tests.sh
+        .ci/scripts/unittest-macos.sh "${{ inputs.build-tool }}"

.github/workflows/pull.yml

@@ -367,6 +367,16 @@ jobs:
       id-token: write
       contents: read
     with:
+      build-tool: cmake
+      docker-image: executorch-ubuntu-22.04-clang12
+
+  unittest-buck:
+    uses: ./.github/workflows/_unittest.yml
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      build-tool: buck2
       docker-image: executorch-ubuntu-22.04-clang12
 
   unittest-arm: