diff --git a/ci/scripts/bootstrap_local_ci.sh b/ci/scripts/bootstrap_local_ci.sh new file mode 100755 index 0000000000..d2f3cee716 --- /dev/null +++ b/ci/scripts/bootstrap_local_ci.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export WORKSPACE_TMP="$(pwd)/ws_tmp" +mkdir -p ${WORKSPACE_TMP} +git clone ${GIT_URL} Morpheus +cd Morpheus/ +git checkout ${GIT_BRANCH} +git pull + +export MORPHEUS_ROOT=$(pwd) +export WORKSPACE=${MORPHEUS_ROOT} +export LOCAL_CI=1 +unset CMAKE_CUDA_COMPILER_LAUNCHER +unset CMAKE_CXX_COMPILER_LAUNCHER +unset CMAKE_C_COMPILER_LAUNCHER + +if [[ "${STAGE}" != "bash" ]]; then + ${MORPHEUS_ROOT}/ci/scripts/github/${STAGE}.sh +fi diff --git a/ci/scripts/github/build.sh b/ci/scripts/github/build.sh index 3e5242ba6c..8aec506af0 100755 --- a/ci/scripts/github/build.sh +++ b/ci/scripts/github/build.sh @@ -28,20 +28,26 @@ cmake --version ninja --version sccache --version -rapids-logger "Configuring cmake for Morpheus" git submodule update --init --recursive -cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} \ - -DCCACHE_PROGRAM_PATH=$(which sccache) \ - -DMORPHEUS_PYTHON_BUILD_WHEEL=ON \ - -DMORPHEUS_PYTHON_BUILD_STUBS=OFF \ - -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON . +CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}" +CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_WHEEL=ON" +CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF" +CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON" +if [[ "${LOCAL_CI}" == "" ]]; then + CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)" +fi + +rapids-logger "Configuring cmake for Morpheus with ${CMAKE_FLAGS}" +cmake -B build -G Ninja ${CMAKE_FLAGS} . rapids-logger "Building Morpheus" cmake --build build --parallel ${PARALLEL_LEVEL} -rapids-logger "sccache usage for morpheus build:" -sccache --show-stats +if [[ "${LOCAL_CI}" == "" ]]; then + rapids-logger "sccache usage for morpheus build:" + sccache --show-stats +fi rapids-logger "Archiving results" tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" build/dist @@ -54,9 +60,9 @@ CPP_TESTS=($(find ${MORPHEUS_ROOT}/build/morpheus/_lib/tests -name "*.x" -exec r tar cfj "${WORKSPACE_TMP}/cpp_tests.tar.bz" "${CPP_TESTS[@]}" rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}" -aws s3 cp --no-progress "${WORKSPACE_TMP}/wheel.tar.bz" "${ARTIFACT_URL}/wheel.tar.bz" -aws s3 cp --no-progress "${WORKSPACE_TMP}/morhpeus_libs.tar.bz" "${ARTIFACT_URL}/morhpeus_libs.tar.bz" -aws s3 cp --no-progress "${WORKSPACE_TMP}/cpp_tests.tar.bz" "${ARTIFACT_URL}/cpp_tests.tar.bz" +upload_artifact "${WORKSPACE_TMP}/wheel.tar.bz" +upload_artifact "${WORKSPACE_TMP}/morhpeus_libs.tar.bz" +upload_artifact "${WORKSPACE_TMP}/cpp_tests.tar.bz" rapids-logger "Success" exit 0 diff --git a/ci/scripts/github/checks.sh b/ci/scripts/github/checks.sh index c87fa0b449..9f985e9db1 100755 --- a/ci/scripts/github/checks.sh +++ b/ci/scripts/github/checks.sh @@ -31,13 +31,19 @@ ${MORPHEUS_ROOT}/ci/scripts/python_checks.sh git submodule update --init --recursive rapids-logger "Configuring cmake for Morpheus" -cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} -DCCACHE_PROGRAM_PATH=$(which sccache) . +CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}" +if [[ "${LOCAL_CI}" == "" ]]; then + CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)" +fi +cmake -B build -G Ninja ${CMAKE_FLAGS} . rapids-logger "Building targets that generate source code" cmake --build build --target morpheus_style_checks --parallel ${PARALLEL_LEVEL} -rapids-logger "sccache usage for source build:" -sccache --show-stats +if [[ "${LOCAL_CI}" == "" ]]; then + rapids-logger "sccache usage for source build:" + sccache --show-stats +fi rapids-logger "Checking versions" ${MORPHEUS_ROOT}/ci/scripts/version_checks.sh diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh index cc431abcf3..0c7709de14 100644 --- a/ci/scripts/github/common.sh +++ b/ci/scripts/github/common.sh @@ -41,17 +41,17 @@ rapids-logger "Memory" rapids-logger "User Info" id -# For PRs, $GIT_BRANCH is like: pull-request/989 -REPO_NAME=$(basename "${GITHUB_REPOSITORY}") -ORG_NAME="${GITHUB_REPOSITORY_OWNER}" -PR_NUM="${GITHUB_REF_NAME##*/}" - # S3 vars export S3_URL="s3://rapids-downloads/ci/morpheus" export DISPLAY_URL="https://downloads.rapids.ai/ci/morpheus" export ARTIFACT_ENDPOINT="/pull-request/${PR_NUM}/${GIT_COMMIT}/${NVARCH}" export ARTIFACT_URL="${S3_URL}${ARTIFACT_ENDPOINT}" -export DISPLAY_ARTIFACT_URL="${DISPLAY_URL}${ARTIFACT_ENDPOINT}/" + +if [[ "${LOCAL_CI}" == "1" ]]; then + export DISPLAY_ARTIFACT_URL="${LOCAL_CI_TMP}" +else + export DISPLAY_ARTIFACT_URL="${DISPLAY_URL}${ARTIFACT_ENDPOINT}/" +fi # Set sccache env vars export SCCACHE_S3_KEY_PREFIX=morpheus-${NVARCH} @@ -106,7 +106,12 @@ function update_conda_env() { show_conda_info } -function fetch_base_branch() { +function fetch_base_branch_gh_api() { + # For PRs, $GIT_BRANCH is like: pull-request/989 + REPO_NAME=$(basename "${GITHUB_REPOSITORY}") + ORG_NAME="${GITHUB_REPOSITORY_OWNER}" + PR_NUM="${GITHUB_REF_NAME##*/}" + rapids-logger "Retrieving base branch from GitHub API" [[ -n "$GH_TOKEN" ]] && CURL_HEADERS=('-H' "Authorization: token ${GH_TOKEN}") RESP=$( @@ -116,11 +121,29 @@ function fetch_base_branch() { "${GITHUB_API_URL}/repos/${ORG_NAME}/${REPO_NAME}/pulls/${PR_NUM}" ) - BASE_BRANCH=$(echo "${RESP}" | jq -r '.base.ref') + export BASE_BRANCH=$(echo "${RESP}" | jq -r '.base.ref') # Change target is the branch name we are merging into but due to the weird way jenkins does # the checkout it isn't recognized by git without the origin/ prefix export CHANGE_TARGET="origin/${BASE_BRANCH}" +} + +function fetch_base_branch_local() { + rapids-logger "Retrieving base branch from git" + git remote add upstream ${GIT_UPSTREAM_URL} + git fetch upstream --tags + source ${MORPHEUS_ROOT}/ci/scripts/common.sh + export BASE_BRANCH=$(get_base_branch) + export CHANGE_TARGET="upstream/${BASE_BRANCH}" +} + +function fetch_base_branch() { + if [[ "${LOCAL_CI}" == "1" ]]; then + fetch_base_branch_local + else + fetch_base_branch_gh_api + fi + rapids-logger "Base branch: ${BASE_BRANCH}" } @@ -131,3 +154,24 @@ function show_conda_info() { conda config --show-sources conda list --show-channel-urls } + +function upload_artifact() { + FILE_NAME=$1 + BASE_NAME=$(basename "${FILE_NAME}") + rapids-logger "Uploading artifact: ${BASE_NAME}" + if [[ "${LOCAL_CI}" == "1" ]]; then + cp ${FILE_NAME} "${LOCAL_CI_TMP}/${BASE_NAME}" + else + aws s3 cp --only-show-errors "${FILE_NAME}" "${ARTIFACT_URL}/${BASE_NAME}" + fi +} + +function download_artifact() { + ARTIFACT=$1 + rapids-logger "Downloading ${ARTIFACT} from ${DISPLAY_ARTIFACT_URL}" + if [[ "${LOCAL_CI}" == "1" ]]; then + cp "${LOCAL_CI_TMP}/${ARTIFACT}" "${WORKSPACE_TMP}/${ARTIFACT}" + else + aws s3 cp --only-show-errors "${ARTIFACT_URL}/${ARTIFACT}" "${WORKSPACE_TMP}/${ARTIFACT}" + fi +} diff --git a/ci/scripts/github/docs.sh b/ci/scripts/github/docs.sh index f3073dd28c..6175aef585 100755 --- a/ci/scripts/github/docs.sh +++ b/ci/scripts/github/docs.sh @@ -20,7 +20,7 @@ source ${WORKSPACE}/ci/scripts/github/common.sh update_conda_env -aws s3 cp --no-progress "${ARTIFACT_URL}/wheel.tar.bz" "${WORKSPACE_TMP}/wheel.tar.bz" +download_artifact "wheel.tar.bz" tar xf "${WORKSPACE_TMP}/wheel.tar.bz" @@ -44,7 +44,7 @@ rapids-logger "Archiving the docs" tar cfj "${WORKSPACE_TMP}/docs.tar.bz" build/docs/html rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}" -aws s3 cp --no-progress "${WORKSPACE_TMP}/docs.tar.bz" "${ARTIFACT_URL}/docs.tar.bz" +upload_artifact "${WORKSPACE_TMP}/docs.tar.bz" rapids-logger "Success" exit 0 diff --git a/ci/scripts/github/test.sh b/ci/scripts/github/test.sh index edefe6e4c2..1fad54e560 100755 --- a/ci/scripts/github/test.sh +++ b/ci/scripts/github/test.sh @@ -21,9 +21,9 @@ source ${WORKSPACE}/ci/scripts/github/common.sh update_conda_env -aws s3 cp --no-progress "${ARTIFACT_URL}/wheel.tar.bz" "${WORKSPACE_TMP}/wheel.tar.bz" -aws s3 cp --no-progress "${ARTIFACT_URL}/cpp_tests.tar.bz" "${WORKSPACE_TMP}/cpp_tests.tar.bz" -aws s3 cp --no-progress "${ARTIFACT_URL}/morhpeus_libs.tar.bz" "${WORKSPACE_TMP}/morhpeus_libs.tar.bz" +download_artifact "wheel.tar.bz" +download_artifact "cpp_tests.tar.bz" +download_artifact "morhpeus_libs.tar.bz" tar xf "${WORKSPACE_TMP}/wheel.tar.bz" tar xf "${WORKSPACE_TMP}/morhpeus_libs.tar.bz" @@ -86,6 +86,6 @@ cd $(dirname ${REPORTS_DIR}) tar cfj ${WORKSPACE_TMP}/test_reports.tar.bz $(basename ${REPORTS_DIR}) rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}" -aws s3 cp ${WORKSPACE_TMP}/test_reports.tar.bz "${ARTIFACT_URL}/test_reports.tar.bz" +upload_artifact ${WORKSPACE_TMP}/test_reports.tar.bz exit ${TEST_RESULTS} diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh new file mode 100755 index 0000000000..761e215ed4 --- /dev/null +++ b/ci/scripts/run_ci_local.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +case "$1" in + "" ) + STAGES=("bash") + ;; + "all" ) + STAGES=("checks" "build" "docs" "test") + ;; + "checks" | "build" | "docs" | "test" | "bash" ) + STAGES=("$1") + ;; + * ) + echo "Error: Invalid argument \"$1\" provided. Expected values: \"all\", \"checks\", \"build\", \"docs\", \"test\", or \"bash\"" + exit 1 + ;; +esac + +# CI image doesn't contain ssh, need to use https +function git_ssh_to_https() +{ + local url=$1 + echo $url | sed -e 's|^git@github\.com:|https://github.com/|' +} + +MORPHEUS_ROOT=${MORPHEUS_ROOT:-$(git rev-parse --show-toplevel)} + +GIT_URL=$(git remote get-url origin) +GIT_URL=$(git_ssh_to_https ${GIT_URL}) + +GIT_UPSTREAM_URL=$(git remote get-url upstream) +GIT_UPSTREAM_URL=$(git_ssh_to_https ${GIT_UPSTREAM_URL}) + +GIT_BRANCH=$(git branch --show-current) +GIT_COMMIT=$(git log -n 1 --pretty=format:%H) + +LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp} +CONTAINER_VER=${CONTAINER_VER:-230510} +CUDA_VER=${CUDA_VER:-11.8} +DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} + +BUILD_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-${CONTAINER_VER}" +TEST_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-${CONTAINER_VER}" + +ENV_LIST="--env LOCAL_CI_TMP=/ci_tmp" +ENV_LIST="${ENV_LIST} --env GIT_URL=${GIT_URL}" +ENV_LIST="${ENV_LIST} --env GIT_UPSTREAM_URL=${GIT_UPSTREAM_URL}" +ENV_LIST="${ENV_LIST} --env GIT_BRANCH=${GIT_BRANCH}" +ENV_LIST="${ENV_LIST} --env GIT_COMMIT=${GIT_COMMIT}" +ENV_LIST="${ENV_LIST} --env PARALLEL_LEVEL=$(nproc)" +ENV_LIST="${ENV_LIST} --env CUDA_VER=${CUDA_VER}" + +mkdir -p ${LOCAL_CI_TMP} +cp ${MORPHEUS_ROOT}/ci/scripts/bootstrap_local_ci.sh ${LOCAL_CI_TMP} + +for STAGE in "${STAGES[@]}"; do + DOCKER_RUN_ARGS="--rm -ti --net=host -v "${LOCAL_CI_TMP}":/ci_tmp ${ENV_LIST} --env STAGE=${STAGE}" + if [[ "${STAGE}" == "test" || "${USE_GPU}" == "1" ]]; then + CONTAINER="${TEST_CONTAINER}" + DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia --gpus all" + if [[ "${STAGE}" == "test" ]]; then + DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --env MERGE_EXAMPLES_YAML=1 --cap-add=sys_nice" + fi + else + CONTAINER="${BUILD_CONTAINER}" + DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=runc" + if [[ "${STAGE}" == "docs" ]]; then + DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --env MERGE_DOCS_YAML=1" + fi + fi + + if [[ "${STAGE}" == "bash" ]]; then + DOCKER_RUN_CMD="bash --init-file /ci_tmp/bootstrap_local_ci.sh" + else + DOCKER_RUN_CMD="/ci_tmp/bootstrap_local_ci.sh" + fi + + echo "Running ${STAGE} stage in ${CONTAINER}" + docker run ${DOCKER_RUN_ARGS} ${DOCKER_EXTRA_ARGS} ${CONTAINER} ${DOCKER_RUN_CMD} + + STATUS=$? + if [[ ${STATUS} -ne 0 ]]; then + echo "Error: docker exited with a non-zero status code for ${STAGE} of ${STATUS}" + exit ${STATUS} + fi +done