forked from nv-morpheus/Morpheus
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Helper scripts for running CI locally (nv-morpheus#949)
Usage looks like: ``` ci/scripts/run_ci_local.sh checks ci/scripts/run_ci_local.sh build ci/scripts/run_ci_local.sh docs ci/scripts/run_ci_local.sh test ``` Build artifacts are stored locally in `.tmp/local_ci_tmp` or can be overridden by defining `LOCAL_CI_TMP` To run the entire CI pipeline: ``` ci/scripts/run_ci_local.sh all ``` To run bash inside of the CI containers: ``` # Runs in the build container ci/scripts/run_ci_local.sh # Runs in the test container USE_GPU=1 ci/scripts/run_ci_local.sh ``` Includes changes from nv-morpheus#904 Fixes nv-morpheus#948 Authors: - David Gardner (https://github.com/dagardner-nv) Approvers: - Michael Demoret (https://github.com/mdemoret-nv) URL: nv-morpheus#949
- Loading branch information
1 parent
e595793
commit 0c99762
Showing
7 changed files
with
217 additions
and
28 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| #!/bin/bash | ||
| # SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| export WORKSPACE_TMP="$(pwd)/ws_tmp" | ||
| mkdir -p ${WORKSPACE_TMP} | ||
| git clone ${GIT_URL} Morpheus | ||
| cd Morpheus/ | ||
| git checkout ${GIT_BRANCH} | ||
| git pull | ||
|
|
||
| export MORPHEUS_ROOT=$(pwd) | ||
| export WORKSPACE=${MORPHEUS_ROOT} | ||
| export LOCAL_CI=1 | ||
| unset CMAKE_CUDA_COMPILER_LAUNCHER | ||
| unset CMAKE_CXX_COMPILER_LAUNCHER | ||
| unset CMAKE_C_COMPILER_LAUNCHER | ||
|
|
||
| if [[ "${STAGE}" != "bash" ]]; then | ||
| ${MORPHEUS_ROOT}/ci/scripts/github/${STAGE}.sh | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| #!/bin/bash | ||
| # SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| case "$1" in | ||
| "" ) | ||
| STAGES=("bash") | ||
| ;; | ||
| "all" ) | ||
| STAGES=("checks" "build" "docs" "test") | ||
| ;; | ||
| "checks" | "build" | "docs" | "test" | "bash" ) | ||
| STAGES=("$1") | ||
| ;; | ||
| * ) | ||
| echo "Error: Invalid argument \"$1\" provided. Expected values: \"all\", \"checks\", \"build\", \"docs\", \"test\", or \"bash\"" | ||
| exit 1 | ||
| ;; | ||
| esac | ||
|
|
||
| # CI image doesn't contain ssh, need to use https | ||
| function git_ssh_to_https() | ||
| { | ||
| local url=$1 | ||
| echo $url | sed -e 's|^git@github\.com:|https://github.com/|' | ||
| } | ||
|
|
||
| MORPHEUS_ROOT=${MORPHEUS_ROOT:-$(git rev-parse --show-toplevel)} | ||
|
|
||
| GIT_URL=$(git remote get-url origin) | ||
| GIT_URL=$(git_ssh_to_https ${GIT_URL}) | ||
|
|
||
| GIT_UPSTREAM_URL=$(git remote get-url upstream) | ||
| GIT_UPSTREAM_URL=$(git_ssh_to_https ${GIT_UPSTREAM_URL}) | ||
|
|
||
| GIT_BRANCH=$(git branch --show-current) | ||
| GIT_COMMIT=$(git log -n 1 --pretty=format:%H) | ||
|
|
||
| LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp} | ||
| CONTAINER_VER=${CONTAINER_VER:-230510} | ||
| CUDA_VER=${CUDA_VER:-11.8} | ||
| DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} | ||
|
|
||
| BUILD_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-${CONTAINER_VER}" | ||
| TEST_CONTAINER="nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-${CONTAINER_VER}" | ||
|
|
||
| ENV_LIST="--env LOCAL_CI_TMP=/ci_tmp" | ||
| ENV_LIST="${ENV_LIST} --env GIT_URL=${GIT_URL}" | ||
| ENV_LIST="${ENV_LIST} --env GIT_UPSTREAM_URL=${GIT_UPSTREAM_URL}" | ||
| ENV_LIST="${ENV_LIST} --env GIT_BRANCH=${GIT_BRANCH}" | ||
| ENV_LIST="${ENV_LIST} --env GIT_COMMIT=${GIT_COMMIT}" | ||
| ENV_LIST="${ENV_LIST} --env PARALLEL_LEVEL=$(nproc)" | ||
| ENV_LIST="${ENV_LIST} --env CUDA_VER=${CUDA_VER}" | ||
|
|
||
| mkdir -p ${LOCAL_CI_TMP} | ||
| cp ${MORPHEUS_ROOT}/ci/scripts/bootstrap_local_ci.sh ${LOCAL_CI_TMP} | ||
|
|
||
| for STAGE in "${STAGES[@]}"; do | ||
| DOCKER_RUN_ARGS="--rm -ti --net=host -v "${LOCAL_CI_TMP}":/ci_tmp ${ENV_LIST} --env STAGE=${STAGE}" | ||
| if [[ "${STAGE}" == "test" || "${USE_GPU}" == "1" ]]; then | ||
| CONTAINER="${TEST_CONTAINER}" | ||
| DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia --gpus all" | ||
| if [[ "${STAGE}" == "test" ]]; then | ||
| DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --env MERGE_EXAMPLES_YAML=1 --cap-add=sys_nice" | ||
| fi | ||
| else | ||
| CONTAINER="${BUILD_CONTAINER}" | ||
| DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=runc" | ||
| if [[ "${STAGE}" == "docs" ]]; then | ||
| DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --env MERGE_DOCS_YAML=1" | ||
| fi | ||
| fi | ||
|
|
||
| if [[ "${STAGE}" == "bash" ]]; then | ||
| DOCKER_RUN_CMD="bash --init-file /ci_tmp/bootstrap_local_ci.sh" | ||
| else | ||
| DOCKER_RUN_CMD="/ci_tmp/bootstrap_local_ci.sh" | ||
| fi | ||
|
|
||
| echo "Running ${STAGE} stage in ${CONTAINER}" | ||
| docker run ${DOCKER_RUN_ARGS} ${DOCKER_EXTRA_ARGS} ${CONTAINER} ${DOCKER_RUN_CMD} | ||
|
|
||
| STATUS=$? | ||
| if [[ ${STATUS} -ne 0 ]]; then | ||
| echo "Error: docker exited with a non-zero status code for ${STAGE} of ${STATUS}" | ||
| exit ${STATUS} | ||
| fi | ||
| done |