From 01e3186ff902455f1874838dc608ff630d1e572b Mon Sep 17 00:00:00 2001 From: "tier4-autoware-private-bot[bot]" <87871706+tier4-autoware-private-bot[bot]@users.noreply.github.com> Date: Wed, 15 Mar 2023 16:29:49 +0900 Subject: [PATCH] chore: sync upstream (#402) * chore: add reviewer/assignee of sync-awf-latest-reference-design's PR (#594) Signed-off-by: Takayuki Murooka Signed-off-by: Takayuki Murooka * ci(slack-send): remove unused workflow (#600) Signed-off-by: Tomohito Ando Signed-off-by: Tomohito Ando * ci(sync-files): sync .fdignore and .rgignore (#601) * chore: sync files (#588) Signed-off-by: GitHub Co-authored-by: kenji-miyake * feat: rename launch branch (#604) Signed-off-by: Takayuki Murooka * feat: remove sync-awf-latest-.yaml (#608) Signed-off-by: Takayuki Murooka * chore: sync files (#602) Signed-off-by: GitHub Co-authored-by: kenji-miyake --------- Signed-off-by: Takayuki Murooka Signed-off-by: Tomohito Ando Signed-off-by: GitHub Co-authored-by: Takayuki Murooka Co-authored-by: Tomohito ANDO Co-authored-by: Kenji Miyake <31987104+kenji-miyake@users.noreply.github.com> Co-authored-by: tier4-autoware-private-bot[bot] <87871706+tier4-autoware-private-bot[bot]@users.noreply.github.com> Co-authored-by: kenji-miyake Co-authored-by: Hiroki OTA --- .fdignore | 1 + .github/sync-files.yaml | 2 + .github/workflows/pre-commit.yaml | 2 +- .github/workflows/slack-send.yaml | 1 - .github/workflows/sync-awf-latest-s1.yaml | 31 ---------- .github/workflows/sync-awf-latest-x1.yaml | 31 ---------- .github/workflows/sync-awf-latest-x2.yaml | 31 ---------- .github/workflows/sync-awf-latest-xx1.yaml | 31 ---------- .pre-commit-config.yaml | 11 ++-- .rgignore | 1 + amd64.env | 1 + ansible-galaxy-requirements.yaml | 2 +- autoware.repos | 2 +- docker/build.sh | 17 +++++- docker/pilot-auto/Dockerfile | 67 ++++++++++++++++++++-- setup-dev-env.sh | 16 ++++-- 16 files changed, 101 insertions(+), 146 deletions(-) create mode 100644 .fdignore delete mode 100644 .github/workflows/sync-awf-latest-s1.yaml delete mode 100644 .github/workflows/sync-awf-latest-x1.yaml delete mode 100644 .github/workflows/sync-awf-latest-x2.yaml delete mode 100644 .github/workflows/sync-awf-latest-xx1.yaml create mode 100644 .rgignore diff --git a/.fdignore b/.fdignore new file mode 100644 index 0000000000000..fc25c6acba9dc --- /dev/null +++ b/.fdignore @@ -0,0 +1 @@ +!/src/ diff --git a/.github/sync-files.yaml b/.github/sync-files.yaml index ab723cfda89af..57ceb4f5538ef 100644 --- a/.github/sync-files.yaml +++ b/.github/sync-files.yaml @@ -16,6 +16,7 @@ - source: .github/workflows/sync-files.yaml - source: .clang-format - source: .clang-tidy + - source: .fdignore - source: .gitignore - source: .markdownlint.yaml - source: .pre-commit-config.yaml @@ -23,6 +24,7 @@ - source: .pre-commit-config-optional.yaml - source: .prettierignore - source: .prettierrc.yaml + - source: .rgignore - source: .shellcheckrc - source: .yamllint.yaml - source: CPPLINT.cfg diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 7fd1cc90fe97d..bda722c87ef09 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -5,7 +5,7 @@ on: jobs: pre-commit: - if: ${{ github.event.repository.private }} + if: ${{ github.event.repository.private }} # Use pre-commit.ci for public repositories runs-on: ubuntu-latest steps: - name: Generate token diff --git a/.github/workflows/slack-send.yaml b/.github/workflows/slack-send.yaml index 0928d43b7e958..87b99f2c07d7d 100644 --- a/.github/workflows/slack-send.yaml +++ b/.github/workflows/slack-send.yaml @@ -3,7 +3,6 @@ on: workflow_run: workflows: - build-main - - build-humble types: - completed diff --git a/.github/workflows/sync-awf-latest-s1.yaml b/.github/workflows/sync-awf-latest-s1.yaml deleted file mode 100644 index 1b909510bc164..0000000000000 --- a/.github/workflows/sync-awf-latest-s1.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: sync-awf-latest-s1 - -on: - schedule: - - cron: 0 0 * * * - workflow_dispatch: - -jobs: - sync-awf-latest-s1: - runs-on: ubuntu-latest - steps: - - name: Generate token - id: generate-token - uses: tibdex/github-app-token@v1 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.PRIVATE_KEY }} - - - name: Run sync-branches - uses: autowarefoundation/autoware-github-actions/sync-branches@v1 - with: - token: ${{ steps.generate-token.outputs.token }} - base-branch: awf-latest-s1 - sync-pr-branch: sync-awf-latest-s1 - sync-target-repository: https://github.com/tier4/pilot-auto - sync-target-branch: awf-latest - pr-title: "chore: sync awf-latest-s1" - pr-labels: | - bot - sync-awf-latest-s1 - auto-merge-method: merge diff --git a/.github/workflows/sync-awf-latest-x1.yaml b/.github/workflows/sync-awf-latest-x1.yaml deleted file mode 100644 index 341404fad5dc2..0000000000000 --- a/.github/workflows/sync-awf-latest-x1.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: sync-awf-latest-x1 - -on: - schedule: - - cron: 0 0 * * * - workflow_dispatch: - -jobs: - sync-awf-latest-x1: - runs-on: ubuntu-latest - steps: - - name: Generate token - id: generate-token - uses: tibdex/github-app-token@v1 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.PRIVATE_KEY }} - - - name: Run sync-branches - uses: autowarefoundation/autoware-github-actions/sync-branches@v1 - with: - token: ${{ steps.generate-token.outputs.token }} - base-branch: awf-latest-x1 - sync-pr-branch: sync-awf-latest-x1 - sync-target-repository: https://github.com/tier4/pilot-auto - sync-target-branch: awf-latest - pr-title: "chore: sync awf-latest-x1" - pr-labels: | - bot - sync-awf-latest-x1 - auto-merge-method: merge diff --git a/.github/workflows/sync-awf-latest-x2.yaml b/.github/workflows/sync-awf-latest-x2.yaml deleted file mode 100644 index dcdec72cf37cc..0000000000000 --- a/.github/workflows/sync-awf-latest-x2.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: sync-awf-latest-x2 - -on: - schedule: - - cron: 0 0 * * * - workflow_dispatch: - -jobs: - sync-awf-latest-x2: - runs-on: ubuntu-latest - steps: - - name: Generate token - id: generate-token - uses: tibdex/github-app-token@v1 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.PRIVATE_KEY }} - - - name: Run sync-branches - uses: autowarefoundation/autoware-github-actions/sync-branches@v1 - with: - token: ${{ steps.generate-token.outputs.token }} - base-branch: awf-latest-x2 - sync-pr-branch: sync-awf-latest-x2 - sync-target-repository: https://github.com/tier4/pilot-auto - sync-target-branch: awf-latest - pr-title: "chore: sync awf-latest-x2" - pr-labels: | - bot - sync-awf-latest-x2 - auto-merge-method: merge diff --git a/.github/workflows/sync-awf-latest-xx1.yaml b/.github/workflows/sync-awf-latest-xx1.yaml deleted file mode 100644 index ed6bed54f0c13..0000000000000 --- a/.github/workflows/sync-awf-latest-xx1.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: sync-awf-latest-xx1 - -on: - schedule: - - cron: 0 0 * * * - workflow_dispatch: - -jobs: - sync-awf-latest-xx1: - runs-on: ubuntu-latest - steps: - - name: Generate token - id: generate-token - uses: tibdex/github-app-token@v1 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.PRIVATE_KEY }} - - - name: Run sync-branches - uses: autowarefoundation/autoware-github-actions/sync-branches@v1 - with: - token: ${{ steps.generate-token.outputs.token }} - base-branch: awf-latest-xx1 - sync-pr-branch: sync-awf-latest-xx1 - sync-target-repository: https://github.com/tier4/pilot-auto - sync-target-branch: awf-latest - pr-title: "chore: sync awf-latest-xx1" - pr-labels: | - bot - sync-awf-latest-xx1 - auto-merge-method: merge diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dbff7a236fe70..e309548ac617c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,5 @@ ci: autofix_commit_msg: "style(pre-commit): autofix" - autoupdate_commit_msg: "ci(pre-commit): autoupdate" repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -18,7 +17,7 @@ repos: args: [--markdown-linebreak-ext=md] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.32.2 + rev: v0.33.0 hooks: - id: markdownlint args: [-c, .markdownlint.yaml, --fix] @@ -29,23 +28,23 @@ repos: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.28.0 + rev: v1.29.0 hooks: - id: yamllint - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.8.0.4 + rev: v0.9.0.2 hooks: - id: shellcheck - repo: https://github.com/scop/pre-commit-shfmt - rev: v3.5.1-1 + rev: v3.6.0-1 hooks: - id: shfmt args: [-w, -s, -i=4] - repo: https://github.com/AleksaC/hadolint-py - rev: v2.10.0 + rev: v2.12.0.2 hooks: - id: hadolint diff --git a/.rgignore b/.rgignore new file mode 100644 index 0000000000000..fc25c6acba9dc --- /dev/null +++ b/.rgignore @@ -0,0 +1 @@ +!/src/ diff --git a/amd64.env b/amd64.env index 1366bd83c917a..afa1725503416 100644 --- a/amd64.env +++ b/amd64.env @@ -2,6 +2,7 @@ rosdistro=humble rmw_implementation=rmw_cyclonedds_cpp base_image=ubuntu:22.04 cuda_base_image=ubuntu:22.04 +prebuilt_base_image=ubuntu:22.04 cuda_version=11.6 cudnn_version=8.4.1.50-1+cuda11.6 tensorrt_version=8.4.2-1+cuda11.6 diff --git a/ansible-galaxy-requirements.yaml b/ansible-galaxy-requirements.yaml index 510e80f10fef8..d32f95cd337a1 100644 --- a/ansible-galaxy-requirements.yaml +++ b/ansible-galaxy-requirements.yaml @@ -4,4 +4,4 @@ collections: - name: https://github.com/autowarefoundation/autoware.git#/ansible type: git - version: 9da9b5e25d40b5c9d4d7141ef807325de2077bcf + version: dfefdef4d308a4418097a6d6b9b67b588c061fe8 diff --git a/autoware.repos b/autoware.repos index e34a1b62f5438..a63922086be2c 100644 --- a/autoware.repos +++ b/autoware.repos @@ -11,7 +11,7 @@ repositories: autoware/launcher: type: git url: git@github.com:tier4/autoware_launch.x1.eve.git - version: tier4/universe + version: tier4/main autoware/aip_launcher: type: git url: https://github.com/tier4/aip_launcher.git diff --git a/docker/build.sh b/docker/build.sh index b58eeed9c7728..32215b8d56d25 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -16,6 +16,9 @@ while [ "$1" != "" ]; do option_platform="$2" shift ;; + --no-prebuilt) + option_no_prebuilt=true + ;; *) args+=("$1") ;; @@ -32,6 +35,14 @@ else image_name_suffix="-cuda" fi +# Set prebuilt options +if [ "$option_no_prebuilt" = "true" ]; then + targets=("devel") +else + # default targets include devel and prebuilt + targets=() +fi + # Set platform if [ -n "$option_platform" ]; then platform="$option_platform" @@ -52,13 +63,15 @@ fi export BUILDKIT_STEP_LOG_MAX_SIZE=10000000 set -x -docker buildx bake --load --progress=plain -f "$SCRIPT_DIR/pilot-auto/docker-bake.hcl" \ +docker buildx bake --no-cache --load --progress=plain -f "$SCRIPT_DIR/pilot-auto/docker-bake.hcl" \ --set "*.context=$WORKSPACE_ROOT" \ --set "*.ssh=default" \ --set "*.platform=$platform" \ --set "*.args.ROS_DISTRO=$rosdistro" \ --set "*.args.BASE_IMAGE=$base_image" \ + --set "*.args.PREBUILT_BASE_IMAGE=$prebuilt_base_image" \ --set "*.args.SETUP_ARGS=$setup_args" \ --set "devel.tags=ghcr.io/tier4/pilot-auto:$rosdistro-latest$image_name_suffix" \ - --set "prebuilt.tags=ghcr.io/tier4/pilot-auto:$rosdistro-latest-prebuilt$image_name_suffix" + --set "prebuilt.tags=ghcr.io/tier4/pilot-auto:$rosdistro-latest-prebuilt$image_name_suffix" \ + "${targets[@]}" set +x diff --git a/docker/pilot-auto/Dockerfile b/docker/pilot-auto/Dockerfile index 156ec4dcd0d82..d89b260b25458 100644 --- a/docker/pilot-auto/Dockerfile +++ b/docker/pilot-auto/Dockerfile @@ -1,4 +1,6 @@ +# Image args should come at the beginning. ARG BASE_IMAGE +ARG PREBUILT_BASE_IMAGE # hadolint ignore=DL3006 FROM $BASE_IMAGE as devel SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -43,10 +45,10 @@ RUN rm -rf \ /etc/apt/sources.list.d/nvidia-docker.list ## Register Vulkan GPU vendors -ADD "https://gitlab.com/nvidia/container-images/vulkan/raw/dc389b0445c788901fda1d85be96fd1cb9410164/nvidia_icd.json" /etc/vulkan/icd.d/nvidia_icd.json -RUN chmod 644 /etc/vulkan/icd.d/nvidia_icd.json -ADD "https://gitlab.com/nvidia/container-images/opengl/raw/5191cf205d3e4bb1150091f9464499b076104354/glvnd/runtime/10_nvidia.json" /etc/glvnd/egl_vendor.d/10_nvidia.json -RUN chmod 644 /etc/glvnd/egl_vendor.d/10_nvidia.json +RUN curl https://gitlab.com/nvidia/container-images/vulkan/raw/dc389b0445c788901fda1d85be96fd1cb9410164/nvidia_icd.json -o /etc/vulkan/icd.d/nvidia_icd.json \ + && chmod 644 /etc/vulkan/icd.d/nvidia_icd.json +RUN curl https://gitlab.com/nvidia/container-images/opengl/raw/5191cf205d3e4bb1150091f9464499b076104354/glvnd/runtime/10_nvidia.json -o /etc/glvnd/egl_vendor.d/10_nvidia.json \ + && chmod 644 /etc/glvnd/egl_vendor.d/10_nvidia.json ## Register OpenCL GPU vendors RUN mkdir -p /etc/OpenCL/vendors \ @@ -58,7 +60,7 @@ RUN mkdir -p /etc/OpenCL/vendors \ RUN echo "source /opt/ros/${ROS_DISTRO}/setup.bash" > /etc/bash.bashrc CMD ["/bin/bash"] -FROM devel as prebuilt +FROM devel as builder SHELL ["/bin/bash", "-o", "pipefail", "-c"] ## Build and change permission for runtime data conversion @@ -66,6 +68,61 @@ RUN source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release \ && find /autoware/install -type d -exec chmod 777 {} \; +# hadolint ignore=DL3006 +FROM $PREBUILT_BASE_IMAGE as prebuilt + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ARG ROS_DISTRO +ARG SETUP_ARGS + +## Install apt packages +# hadolint ignore=DL3008 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \ + git \ + ssh \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +## Copy files +COPY autoware.repos setup-dev-env.sh ansible-galaxy-requirements.yaml amd64.env arm64.env /autoware/ +COPY ansible/ /autoware/ansible/ +WORKDIR /autoware +RUN ls /autoware + +## Add GitHub to known hosts for private repositories +RUN mkdir -p ~/.ssh \ + && ssh-keyscan github.com >> ~/.ssh/known_hosts + +## Set up runtime environment +RUN --mount=type=ssh \ + ./setup-dev-env.sh -y $SETUP_ARGS --no-cuda-drivers --runtime local_dev_env \ + && pip uninstall -y ansible ansible-core \ + && mkdir src \ + && vcs import src < autoware.repos \ + && rosdep update \ + && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro "$ROS_DISTRO" \ + && rm -rf src \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +## Copy install folder from builder +COPY --from=builder /autoware/install/ /autoware/install/ + +## Clean up unnecessary files +RUN rm -rf \ + "$HOME"/.cache \ + /etc/apt/sources.list.d/cuda*.list \ + /etc/apt/sources.list.d/docker.list \ + /etc/apt/sources.list.d/nvidia-docker.list + +## Register Vulkan GPU vendors +ADD "https://gitlab.com/nvidia/container-images/vulkan/raw/dc389b0445c788901fda1d85be96fd1cb9410164/nvidia_icd.json" /etc/vulkan/icd.d/nvidia_icd.json +RUN chmod 644 /etc/vulkan/icd.d/nvidia_icd.json +ADD "https://gitlab.com/nvidia/container-images/opengl/raw/5191cf205d3e4bb1150091f9464499b076104354/glvnd/runtime/10_nvidia.json" /etc/glvnd/egl_vendor.d/10_nvidia.json +RUN chmod 644 /etc/glvnd/egl_vendor.d/10_nvidia.json + ## Create entrypoint +# hadolint ignore=DL3059 RUN echo "source /autoware/install/setup.bash" > /etc/bash.bashrc CMD ["/bin/bash"] diff --git a/setup-dev-env.sh b/setup-dev-env.sh index e637fe8776aa0..b27abf06a0f21 100755 --- a/setup-dev-env.sh +++ b/setup-dev-env.sh @@ -27,9 +27,9 @@ while [ "$1" != "" ]; do # Disable installation of 'cuda-drivers' in the role 'cuda'. option_no_cuda_drivers=true ;; - --no-dev) - # Disable installation dev packages . - option_no_dev=true + --runtime) + # Disable installation dev package of role 'cuda' and 'tensorrt'. + option_runtime=true ;; *) args+=("$1") @@ -82,8 +82,9 @@ if [ "$option_no_cuda_drivers" = "true" ]; then fi # Check installation of dev package -if [ "$option_no_dev" = "true" ]; then +if [ "$option_runtime" = "true" ]; then ansible_args+=("--extra-vars" "install_devel=false") + # ROS installation type, default "desktop" ansible_args+=("--extra-vars" "installation_type=ros-base") else ansible_args+=("--extra-vars" "install_devel=true") @@ -113,10 +114,15 @@ if ! (command -v git >/dev/null 2>&1); then sudo apt-get -y install git fi +# Install pip for ansible +if ! (python3 -m pip --version >/dev/null 2>&1); then + sudo apt-get -y update + sudo apt-get -y install python3-pip python3-venv +fi + # Install pipx for ansible if ! (python3 -m pipx --version >/dev/null 2>&1); then sudo apt-get -y update - sudo apt-get -y install python3-pip python3-venv python3 -m pip install --user pipx fi