diff --git a/.bazelrc b/.bazelrc index f678853..f5f11e6 100644 --- a/.bazelrc +++ b/.bazelrc @@ -23,3 +23,24 @@ build:noninteractive --keep_going build:release --config=noninteractive build:release --stamp + +build:engflow_common --jobs=40 +build:engflow_common --define=EXECUTOR=remote +build:engflow_common --experimental_inmemory_dotd_files +build:engflow_common --experimental_inmemory_jdeps_files +build:engflow_common --incompatible_strict_action_env=true +build:engflow_common --remote_timeout=600 +build:engflow_common --legacy_important_outputs=false +build:engflow_common --grpc_keepalive_time=30s +build:engflow_common --experimental_remote_cache_compression=true +build:engflow_common --remote_instance_name=auth +build:engflow_common --bes_instance_name=auth +build:engflow_common --bes_lifecycle_events + +build:opal --config=engflow_common +build:opal --bes_backend=grpcs://opal.cluster.engflow.com +build:opal --bes_results_url=https://opal.cluster.engflow.com/invocations/auth/ +# TODO(CUS-349): enable remote execution + +# Load authentication flags for the remote service, if any. +try-import .bazelrc.user diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2ec371f..d1b1670 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,9 +24,12 @@ on: concurrency: group: ${{ github.workflow }}.${{ github.ref }} -# Recommended here: https://github.com/bazelbuild/bazelisk/issues/88#issuecomment-625178467 env: + # Recommended here: https://github.com/bazelbuild/bazelisk/issues/88#issuecomment-625178467 BAZELISK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CLUSTER_HOST: opal.cluster.engflow.com + CLUSTER_CONFIG: opal + CRED_HELPER_TOKEN: ${{ secrets.OPAL_CRED_HELPER_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPAL_RPC_CREDENTIALS: ${{ secrets.OPAL_RPC_CREDENTIALS }} @@ -43,13 +46,24 @@ jobs: - "engflow-runner-id=${{ github.repository_id }}_bazel-builder_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" timeout-minutes: 10 + env: + ARCH: "x64" + OS: "linux" + steps: - uses: actions/checkout@v4 + - name: Log in + run: infra/login.sh + - name: Run all tests + if: success() run: | # TODO(CUS-345): Enable remote execution - bazel test --config=noninteractive //... + bazel test --config=noninteractive --config="${CLUSTER_CONFIG}" //... + + - name: Log out + run: infra/logout.sh golang-builder: runs-on: @@ -76,14 +90,28 @@ jobs: os_version: "2022" revision: "d04e89854b3931f4aaced77aa3a2fcad5834b3a6" + env: + ARCH: "${{ matrix.arch }}" + OS: "${{ matrix.os }}" + steps: - uses: actions/checkout@v4 + - name: Log in + shell: bash + run: infra/login.sh + - name: Run all tests + if: success() + shell: bash run: | # TODO(CUS-345): Enable remote execution - bazel run --config=noninteractive @rules_go//go -- test ./... - bazel run --config=noninteractive @rules_go//go -- clean -cache -modcache + bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" @rules_go//go -- test ./... + bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" @rules_go//go -- clean -cache -modcache + + - name: Log out + shell: bash + run: infra/logout.sh golang-builder-ci-runners: runs-on: @@ -96,15 +124,26 @@ jobs: - "engflow-runner-id=${{ github.repository_id }}_golang-builder-ci-runners_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" timeout-minutes: 10 + env: + ARCH: "x64" + OS: "linux" + steps: - uses: actions/checkout@v4 + - name: Log in + run: infra/login.sh + - name: Run all tests + if: success() run: | # TODO(CUS-345): Enable remote execution bazel run --config=noninteractive @rules_go//go -- test ./... bazel run --config=noninteractive @rules_go//go -- clean -cache -modcache + - name: Log out + run: infra/logout.sh + copyright-headers-check: runs-on: - self-hosted @@ -116,9 +155,20 @@ jobs: - "engflow-runner-id=${{ github.repository_id }}_copyright-headers-check_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" timeout-minutes: 10 + env: + ARCH: "x64" + OS: "linux" + steps: - uses: actions/checkout@v4 + - name: Log in + run: infra/login.sh + - name: Check copyright headers + if: success() run: | - bazel run --config=noninteractive //infra/internal/check_copyright_headers + bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" //infra/internal/check_copyright_headers + + - name: Log out + run: infra/logout.sh diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml index b4e4501..bfe3352 100644 --- a/.github/workflows/presubmit.yml +++ b/.github/workflows/presubmit.yml @@ -26,11 +26,13 @@ concurrency: group: ${{ github.workflow }}.${{ github.ref }} cancel-in-progress: true -# Recommended here: https://github.com/bazelbuild/bazelisk/issues/88#issuecomment-625178467 env: + # Recommended here: https://github.com/bazelbuild/bazelisk/issues/88#issuecomment-625178467 BAZELISK_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CLUSTER_HOST: opal.cluster.engflow.com + CLUSTER_CONFIG: opal + CRED_HELPER_TOKEN: ${{ secrets.OPAL_CRED_HELPER_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPAL_RPC_CREDENTIALS: ${{ secrets.OPAL_RPC_CREDENTIALS }} jobs: # TODO(OND-616): use remote execution and caching for all CI Runner jobs. @@ -45,13 +47,24 @@ jobs: - "engflow-runner-id=${{ github.repository_id }}_bazel-builder_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" timeout-minutes: 10 + env: + ARCH: "x64" + OS: "linux" + steps: - uses: actions/checkout@v4 + - name: Log in + run: infra/login.sh + - name: Run all tests + if: success() run: | # TODO(CUS-345): Enable remote execution - bazel test --config=noninteractive //... + bazel test --config=noninteractive --config="${CLUSTER_CONFIG}" //... + + - name: Log out + run: infra/logout.sh golang-builder: runs-on: @@ -78,14 +91,28 @@ jobs: os_version: "2022" revision: "d04e89854b3931f4aaced77aa3a2fcad5834b3a6" + env: + ARCH: "${{ matrix.arch }}" + OS: "${{ matrix.os }}" + steps: - uses: actions/checkout@v4 + - name: Log in + shell: bash + run: infra/login.sh + - name: Run all tests + if: success() + shell: bash run: | # TODO(CUS-345): Enable remote execution - bazel run --config=noninteractive @rules_go//go -- test ./... - bazel run --config=noninteractive @rules_go//go -- clean -cache -modcache + bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" @rules_go//go -- test ./... + bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" @rules_go//go -- clean -cache -modcache + + - name: Log out + shell: bash + run: infra/logout.sh golang-builder-ci-runners: runs-on: @@ -98,14 +125,25 @@ jobs: - "engflow-runner-id=${{ github.repository_id }}_golang-builder-ci-runners_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" timeout-minutes: 10 + env: + ARCH: "x64" + OS: "linux" + steps: - uses: actions/checkout@v4 + - name: Log in + run: infra/login.sh + - name: Run all tests + if: success() run: | # TODO(CUS-345): Enable remote execution - bazel run --config=noninteractive @rules_go//go -- test ./... - bazel run --config=noninteractive @rules_go//go -- clean -cache -modcache + bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" @rules_go//go -- test ./... + bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" @rules_go//go -- clean -cache -modcache + + - name: Log out + run: infra/logout.sh copyright-headers-check: runs-on: @@ -118,9 +156,20 @@ jobs: - "engflow-runner-id=${{ github.repository_id }}_copyright-headers-check_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}" timeout-minutes: 10 + env: + ARCH: "x64" + OS: "linux" + steps: - uses: actions/checkout@v4 + - name: Log in + run: infra/login.sh + - name: Check copyright headers + if: success() run: | - bazel run --config=noninteractive //infra/internal/check_copyright_headers + bazel run --config=noninteractive --config="${CLUSTER_CONFIG}" //infra/internal/check_copyright_headers + + - name: Log out + run: infra/logout.sh diff --git a/infra/login.sh b/infra/login.sh new file mode 100755 index 0000000..4ea2bfe --- /dev/null +++ b/infra/login.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +# Copyright 2024 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# login.sh configures the build environment to authenticate with a +# remote execution service. It should be run in CI before any bazel command. +# logout.sh should be run afterward to remove stored credentials. + +set -o nounset -o pipefail -o errexit +[[ "${SCRIPT_DEBUG:-"off"}" == "on" ]] && set -o xtrace + +if [[ -z "${ARCH:-}" ]]; then + echo "ARCH not set" + exit 1 +fi +if [[ -z "${CLUSTER_HOST:-}" ]]; then + echo "CLUSTER_HOST not set" + exit 1 +fi +if [[ -z "${CRED_HELPER_TOKEN:-}" ]]; then + echo "CRED_HELPER_TOKEN not set" + exit 1 +fi +if [[ -z "${OS:-}" ]]; then + echo "OS not set" + exit 1 +fi + +# Download a recent version of engflow_auth to a local directory, +# then use it to import the credential. +readonly ENGFLOW_AUTH_VERSION=v0.0.6 +readonly TOOLS_DIR=$(pwd)/_tools +readonly ENGFLOW_AUTH_URL="https://github.com/EngFlow/auth/releases/download/${ENGFLOW_AUTH_VERSION}/engflow_auth_${OS}_${ARCH}" +if [[ "${OS}" == "windows" ]]; then + # On Windows, ensure engflow_auth has an .exe extension. Use an absolute + # Windows path with forward slashes (C:/a/b), NOT a cygwin path (/c/a/b), + # and NOT backslashes (C:\a\b). Bazel only accepts the first form. + readonly ENGFLOW_AUTH_PATH="$(cygpath --mixed "${TOOLS_DIR}/engflow_auth.exe")" +else + readonly ENGFLOW_AUTH_PATH="${TOOLS_DIR}/engflow_auth" +fi +mkdir -p "${TOOLS_DIR}" +if ! curl --fail-with-body --location --output "${ENGFLOW_AUTH_PATH}" "${ENGFLOW_AUTH_URL}"; then + cat "${ENGFLOW_AUTH_PATH}" >&2 + exit 1 +fi +chmod +x "${ENGFLOW_AUTH_PATH}" + +# Import the credential. +"${ENGFLOW_AUTH_PATH}" import -store=file <<<"${CRED_HELPER_TOKEN}" + +# Configure Bazel to use the credential. +cat >.bazelrc.user <