From 78d6c45f101dabf97c76a0316d11baa5fedb12d8 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 26 Jun 2023 13:56:31 -0700 Subject: [PATCH] [ci] cleanup install-bazel.sh (#36729) --- ci/ci.sh | 2 - ci/env/install-bazel.sh | 101 ++++++++++------------------------------ 2 files changed, 25 insertions(+), 78 deletions(-) diff --git a/ci/ci.sh b/ci/ci.sh index 5a2681a05760..a6952f72093b 100755 --- a/ci/ci.sh +++ b/ci/ci.sh @@ -432,8 +432,6 @@ build_wheels_and_jars() { -v "${HOME}/ray-bazel-cache":/root/ray-bazel-cache -e "TRAVIS=true" -e "TRAVIS_PULL_REQUEST=${TRAVIS_PULL_REQUEST:-false}" - -e "encrypted_1c30b31fe1ee_key=${encrypted_1c30b31fe1ee_key-}" - -e "encrypted_1c30b31fe1ee_iv=${encrypted_1c30b31fe1ee_iv-}" -e "TRAVIS_COMMIT=${TRAVIS_COMMIT}" -e "CI=${CI}" -e "RAY_INSTALL_JAVA=${RAY_INSTALL_JAVA:-}" diff --git a/ci/env/install-bazel.sh b/ci/env/install-bazel.sh index 437044417bda..6b0909b90577 100755 --- a/ci/env/install-bazel.sh +++ b/ci/env/install-bazel.sh @@ -28,34 +28,33 @@ case "${OSTYPE}" in exit 1 esac -architecture="${HOSTTYPE}" -echo "Architecture is $architecture" +echo "Architecture(HOSTTYPE) is ${HOSTTYPE}" -if [ "${BAZEL_CONFIG_ONLY-}" != "1" ]; then +if [[ "${BAZEL_CONFIG_ONLY-}" != "1" ]]; then # Sanity check: Verify we have symlinks where we expect them, or Bazel can produce weird "missing input file" errors. # This is most likely to occur on Windows, where symlinks are sometimes disabled by default. { git ls-files -s 2>/dev/null || true; } | ( set +x missing_symlinks=() while read -r mode _ _ path; do - if [ "${mode}" = 120000 ]; then + if [[ "${mode}" == 120000 ]]; then test -L "${path}" || missing_symlinks+=("${path}") fi done - if [ ! 0 -eq "${#missing_symlinks[@]}" ]; then + if [[ ! 0 -eq "${#missing_symlinks[@]}" ]]; then echo "error: expected symlink: ${missing_symlinks[*]}" 1>&2 echo "For a correct build, please run 'git config --local core.symlinks true' and re-run git checkout." 1>&2 false fi ) - if [ "${OSTYPE}" = "msys" ]; then + if [[ "${OSTYPE}" == "msys" ]]; then target="${MINGW_DIR-/usr}/bin/bazel.exe" mkdir -p "${target%/*}" curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-amd64" else # Buildkite mac instances - if [[ -n "${BUILDKITE-}" ]] && [ "${platform}" = "darwin" ]; then + if [[ -n "${BUILDKITE-}" && "${platform}" == "darwin" ]]; then mkdir -p "$HOME/bin" # Add bazel to the path. # shellcheck disable=SC2016 @@ -64,7 +63,7 @@ if [ "${BAZEL_CONFIG_ONLY-}" != "1" ]; then source ~/.zshenv INSTALL_USER=1 # Buildkite linux instance - elif [ "${CI-}" = true ] || [ "${arg1-}" = "--system" ]; then + elif [[ "${CI-}" == true || "${arg1-}" == "--system" ]]; then INSTALL_USER=0 # User else @@ -73,17 +72,17 @@ if [ "${BAZEL_CONFIG_ONLY-}" != "1" ]; then export PATH=$PATH:"$HOME/bin" fi - if [ "${architecture}" = "aarch64" ] || [ "${architecture}" = "arm64" ]; then + if [[ "${HOSTTYPE}" == "aarch64" || "${HOSTTYPE}" = "arm64" ]]; then # architecture is "aarch64", but the bazel tag is "arm64" url="https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-${platform}-arm64" - elif [ "${architecture}" = "x86_64" ]; then + elif [ "${HOSTTYPE}" = "x86_64" ]; then url="https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-${platform}-amd64" else - echo "Could not found matching bazelisk URL for platform ${platform} and architecture ${architecture}" + echo "Could not found matching bazelisk URL for platform ${platform} and architecture ${HOSTTYPE}" exit 1 fi - if [ "$INSTALL_USER" = "1" ]; then + if [[ "$INSTALL_USER" == "1" ]]; then target="$HOME/bin/bazel" curl -f -s -L -R -o "${target}" "${url}" chmod +x "${target}" @@ -100,10 +99,7 @@ bazel --version # clear bazelrc echo > ~/.bazelrc -for bazel_cfg in ${BAZEL_CONFIG-}; do - echo "build --config=${bazel_cfg}" >> ~/.bazelrc -done -if [ "${TRAVIS-}" = true ]; then +if [[ "${TRAVIS-}" == true ]]; then echo "build --config=ci-travis" >> ~/.bazelrc # If we are in Travis, most of the compilation result will be cached. @@ -116,82 +112,35 @@ if [ "${TRAVIS-}" = true ]; then echo "build --jobs=50" >> ~/.bazelrc fi -if [ "${BUILDKITE-}" = "true" ]; then +if [[ "${BUILDKITE-}" == "true" ]]; then cp "${ROOT_DIR}"/../../.bazeliskrc ~/.bazeliskrc fi -if [ "${GITHUB_ACTIONS-}" = true ]; then +if [[ "${GITHUB_ACTIONS-}" == "true" ]]; then echo "build --config=ci-github" >> ~/.bazelrc echo "build --jobs="$(($(nproc)+2)) >> ~/.bazelrc fi -if [ "${CI-}" = true ]; then - +if [[ "${CI-}" == "true" ]]; then # Ask bazel to anounounce the config it finds in bazelrcs, which makes # understanding how to reproduce bazel easier. echo "build --announce_rc" >> ~/.bazelrc - echo "build --config=ci" >> ~/.bazelrc # In Windows CI we want to use this to avoid long path issue # https://docs.bazel.build/versions/main/windows.html#avoid-long-path-issues - if [ "${OSTYPE}" = msys ]; then + if [[ "${OSTYPE}" == msys ]]; then echo "startup --output_user_root=c:/tmp" >> ~/.bazelrc fi - - # If we are in master build, we can write to the cache as well. - upload=0 - if [ "${TRAVIS_PULL_REQUEST-false}" = false ]; then - # shellcheck disable=SC2154 - if [ -n "${BAZEL_CACHE_CREDENTIAL_B64:+x}" ]; then - { - printf "%s" "${BAZEL_CACHE_CREDENTIAL_B64}" | base64 -d - >> "${HOME}/bazel_cache_credential.json" - } 2>&- # avoid printing secrets - upload=1 - elif [ -n "${encrypted_1c30b31fe1ee_key:+x}" ]; then - { - # shellcheck disable=SC2154 - openssl aes-256-cbc -K "${encrypted_1c30b31fe1ee_key}" \ - -iv "${encrypted_1c30b31fe1ee_iv}" \ - -in "${ROOT_DIR}/bazel_cache_credential.json.enc" \ - -out "${HOME}/bazel_cache_credential.json" -d - } 2>&- # avoid printing secrets - # shellcheck disable=SC2181 - if [ 0 -eq $? ]; then - upload=1 - fi - fi - fi - if [ 0 -ne "${upload}" ]; then - translated_path=~/bazel_cache_credential.json - if [ "${OSTYPE}" = msys ]; then # On Windows, we need path translation - translated_path="$(cygpath -m -- "${translated_path}")" - fi - cat <> ~/.bazelrc -build --google_credentials="${translated_path}" -EOF - elif [ -n "${BUILDKITE-}" ]; then - - if [ "${platform}" = "darwin" ]; then - echo "Using local disk cache on mac" - cat <> ~/.bazelrc -build --disk_cache=/tmp/bazel-cache -build --repository_cache=/tmp/bazel-repo-cache -EOF - else - echo "Using buildkite secret store to communicate with cache address" - cat <> ~/.bazelrc -build --remote_cache=${BUILDKITE_BAZEL_CACHE_URL} -EOF - if [ "${BUILDKITE_PULL_REQUEST}" != "false" ]; then - echo "build --remote_upload_local_results=false" >> ~/.bazelrc - fi - fi - + + if [[ "${platform}" == darwin ]]; then + echo "Using local disk cache on mac" + echo "build --disk_cache=/tmp/bazel-cache" >> ~/.bazelrc + echo "build --repository_cache=/tmp/bazel-repo-cache" >> ~/.bazelrc else - echo "Using remote build cache in read-only mode." 1>&2 - cat <> ~/.bazelrc -build --remote_upload_local_results=false -EOF + echo "build --remote_cache=${BUILDKITE_BAZEL_CACHE_URL}" >> ~/.bazelrc + if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then + echo "build --remote_upload_local_results=false" >> ~/.bazelrc + fi fi fi