From 7c0b751a173b49bd5f402219c752b83c9e8fa45b Mon Sep 17 00:00:00 2001 From: Ceyhun Onur Date: Mon, 6 Jan 2025 12:34:07 +0300 Subject: [PATCH 1/7] add multiatch docker build to support arm --- .github/workflows/publish_docker.yml | 20 +++++--- .github/workflows/publish_docker_image.sh | 35 -------------- scripts/build_docker_image.sh | 57 +++++++++++++++++++++-- 3 files changed, 67 insertions(+), 45 deletions(-) delete mode 100755 .github/workflows/publish_docker_image.sh diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index 5a16349cf5..e9d7713a37 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -4,8 +4,8 @@ on: workflow_dispatch: inputs: vm_id: - description: 'The ID of the VM (binary dst in Docker image)' - default: '' + description: "The ID of the VM (binary dst in Docker image)" + default: "" required: false type: string @@ -17,13 +17,21 @@ on: jobs: publish_docker_image: - name: Publish Docker Image - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Publish image to Dockerhub + - name: Install qemu (required for cross-platform builds) + run: | + sudo apt update + sudo apt -y install qemu-system qemu-user-static + sudo systemctl restart docker + - name: Create multiplatform docker builder + run: docker buildx create --use + - name: Build and publish images to DockerHub env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASS: ${{ secrets.DOCKER_PASS }} DOCKER_REPO: "avaplatform/subnet-evm" - run: .github/workflows/publish_docker_image.sh ${{ inputs.vm_id }} + PUBLISH: 1 + PLATFORMS: "linux/amd64,linux/arm64" + run: scripts/build_docker_image.sh ${{ inputs.vm_id }} diff --git a/.github/workflows/publish_docker_image.sh b/.github/workflows/publish_docker_image.sh deleted file mode 100755 index 71bce26c12..0000000000 --- a/.github/workflows/publish_docker_image.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o nounset -set -o pipefail - -# If this is not a trusted build (Docker Credentials are not set) -if [[ -z "$DOCKER_USERNAME" ]]; then - exit 0; -fi - -# Avalanche root directory -SUBNET_EVM_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd ../.. && pwd ) - -# Load the versions -source "$SUBNET_EVM_PATH"/scripts/versions.sh - -# Set the vm id if provided -if [[ $# -eq 1 ]]; then - VM_ID=$1 -fi - -# Buld the docker image -source "$SUBNET_EVM_PATH"/scripts/build_docker_image.sh - -if [[ $CURRENT_BRANCH == "master" ]]; then - echo "Tagging current image as $DOCKERHUB_REPO:latest" - docker tag "$DOCKERHUB_REPO:$BUILD_IMAGE_ID" "$DOCKERHUB_REPO:latest" -fi - -echo "Pushing $DOCKERHUB_REPO:$BUILD_IMAGE_ID" - -echo "$DOCKER_PASS" | docker login --username "$DOCKER_USERNAME" --password-stdin - -docker push -a "$DOCKERHUB_REPO" diff --git a/scripts/build_docker_image.sh b/scripts/build_docker_image.sh index 9de30cd7e7..1c713fc04f 100755 --- a/scripts/build_docker_image.sh +++ b/scripts/build_docker_image.sh @@ -2,8 +2,25 @@ set -euo pipefail +# If set to non-empty, prompts the building of a multi-arch image when the image +# name indicates use of a registry. +# +# A registry is required to build a multi-arch image since a multi-arch image is +# not really an image at all. A multi-arch image (also called a manifest) is +# basically a list of arch-specific images available from the same registry that +# hosts the manifest. Manifests are not supported for local images. +# +# Reference: https://docs.docker.com/build/building/multi-platform/ +PLATFORMS="${PLATFORMS:-}" + +# If set to non-empty, the image will be published to the registry. +PUBLISH="${PUBLISH:-}" + # Directory above this script -SUBNET_EVM_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) +SUBNET_EVM_PATH=$( + cd "$(dirname "${BASH_SOURCE[0]}")" + cd .. && pwd +) # Load the constants source "$SUBNET_EVM_PATH"/scripts/constants.sh @@ -11,11 +28,38 @@ source "$SUBNET_EVM_PATH"/scripts/constants.sh # Load the versions source "$SUBNET_EVM_PATH"/scripts/versions.sh +# Set the vm id if provided +if [[ $# -eq 1 ]]; then + VM_ID=$1 +fi + +# buildx (BuildKit) improves the speed and UI of builds over the legacy builder and +# simplifies creation of multi-arch images. +# +# Reference: https://docs.docker.com/build/buildkit/ +DOCKER_CMD="docker buildx build ${*}" + +if [[ -n "${PUBLISH}" ]]; then + DOCKER_CMD="${DOCKER_CMD} --push" + + echo "Pushing $DOCKERHUB_REPO:$BUILD_IMAGE_ID" + + # A populated DOCKER_USERNAME env var triggers login + if [[ -n "${DOCKER_USERNAME:-}" ]]; then + echo "$DOCKER_PASS" | docker login --username "$DOCKER_USERNAME" --password-stdin + fi +fi + +# Build a multi-arch image if requested +if [[ -n "${PLATFORMS}" ]]; then + DOCKER_CMD="${DOCKER_CMD} --platform=${PLATFORMS}" +fi + # WARNING: this will use the most recent commit even if there are un-committed changes present BUILD_IMAGE_ID=${BUILD_IMAGE_ID:-"${CURRENT_BRANCH}"} VM_ID=${VM_ID:-"${DEFAULT_VM_ID}"} -if [[ "${VM_ID}" != "${DEFAULT_VM_ID}" ]]; then +if [[ "${VM_ID}" != "${DEFAULT_VM_ID}" ]]; then DOCKERHUB_TAG="${VM_ID}-${DOCKERHUB_TAG}" fi @@ -23,9 +67,14 @@ fi AVALANCHEGO_NODE_IMAGE="${AVALANCHEGO_NODE_IMAGE:-${AVALANCHEGO_IMAGE_NAME}:${AVALANCHE_VERSION}}" echo "Building Docker Image: $DOCKERHUB_REPO:$BUILD_IMAGE_ID based of AvalancheGo@$AVALANCHE_VERSION" -docker build -t "$DOCKERHUB_REPO:$BUILD_IMAGE_ID" -t "$DOCKERHUB_REPO:${DOCKERHUB_TAG}" \ - "$SUBNET_EVM_PATH" -f "$SUBNET_EVM_PATH/Dockerfile" \ +${DOCKER_CMD} -t "$DOCKERHUB_REPO:$BUILD_IMAGE_ID" -t "$DOCKERHUB_REPO:${DOCKERHUB_TAG}" \ + "$SUBNET_EVM_PATH" -f "$SUBNET_EVM_PATH/Dockerfile" \ --build-arg AVALANCHEGO_NODE_IMAGE="$AVALANCHEGO_NODE_IMAGE" \ --build-arg SUBNET_EVM_COMMIT="$SUBNET_EVM_COMMIT" \ --build-arg CURRENT_BRANCH="$CURRENT_BRANCH" \ --build-arg VM_ID="$VM_ID" + +if [[ -n "${PUBLISH}" && $CURRENT_BRANCH == "master" ]]; then + echo "Tagging current image as $DOCKERHUB_REPO:latest" + docker buildx imagetools create -t "$DOCKERHUB_REPO:latest" "$DOCKERHUB_REPO:$BUILD_IMAGE_ID" +fi From d9bb69b782d6a4981b42dea9f2802e35abed8170 Mon Sep 17 00:00:00 2001 From: Ceyhun Onur Date: Mon, 6 Jan 2025 12:57:16 +0300 Subject: [PATCH 2/7] fix build img id --- scripts/build_docker_image.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_docker_image.sh b/scripts/build_docker_image.sh index 1c713fc04f..370001138f 100755 --- a/scripts/build_docker_image.sh +++ b/scripts/build_docker_image.sh @@ -33,6 +33,9 @@ if [[ $# -eq 1 ]]; then VM_ID=$1 fi +# WARNING: this will use the most recent commit even if there are un-committed changes present +BUILD_IMAGE_ID=${BUILD_IMAGE_ID:-"${CURRENT_BRANCH}"} + # buildx (BuildKit) improves the speed and UI of builds over the legacy builder and # simplifies creation of multi-arch images. # @@ -55,9 +58,6 @@ if [[ -n "${PLATFORMS}" ]]; then DOCKER_CMD="${DOCKER_CMD} --platform=${PLATFORMS}" fi -# WARNING: this will use the most recent commit even if there are un-committed changes present -BUILD_IMAGE_ID=${BUILD_IMAGE_ID:-"${CURRENT_BRANCH}"} - VM_ID=${VM_ID:-"${DEFAULT_VM_ID}"} if [[ "${VM_ID}" != "${DEFAULT_VM_ID}" ]]; then DOCKERHUB_TAG="${VM_ID}-${DOCKERHUB_TAG}" From fd1e6cbbb08cf6fdc8e2ec874da12f4cde61da8e Mon Sep 17 00:00:00 2001 From: Ceyhun Onur Date: Mon, 6 Jan 2025 13:28:24 +0300 Subject: [PATCH 3/7] Update .github/workflows/publish_docker.yml Co-authored-by: Quentin McGaw Signed-off-by: Ceyhun Onur --- .github/workflows/publish_docker.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index e9d7713a37..0eb0bee954 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -20,11 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Install qemu (required for cross-platform builds) - run: | - sudo apt update - sudo apt -y install qemu-system qemu-user-static - sudo systemctl restart docker + - uses: docker/setup-qemu-action@v3 - name: Create multiplatform docker builder run: docker buildx create --use - name: Build and publish images to DockerHub From 83bc5aa4de5490213370e690be9b7cc3c173b7d9 Mon Sep 17 00:00:00 2001 From: Ceyhun Onur Date: Mon, 6 Jan 2025 13:28:30 +0300 Subject: [PATCH 4/7] Update .github/workflows/publish_docker.yml Co-authored-by: Quentin McGaw Signed-off-by: Ceyhun Onur --- .github/workflows/publish_docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index 0eb0bee954..a57db13f1e 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -21,8 +21,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: docker/setup-qemu-action@v3 - - name: Create multiplatform docker builder - run: docker buildx create --use + - uses: docker/setup-buildx-action@v3 - name: Build and publish images to DockerHub env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} From d82344c9881622042750cf741c2bfff31f175b09 Mon Sep 17 00:00:00 2001 From: Ceyhun Onur Date: Mon, 6 Jan 2025 14:45:57 +0300 Subject: [PATCH 5/7] fix vm id --- .github/workflows/publish_docker.yml | 3 ++- scripts/build_docker_image.sh | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index a57db13f1e..2be9c7d8d7 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -27,6 +27,7 @@ jobs: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASS: ${{ secrets.DOCKER_PASS }} DOCKER_REPO: "avaplatform/subnet-evm" + VM_ID: ${{ inputs.vm_id }} PUBLISH: 1 PLATFORMS: "linux/amd64,linux/arm64" - run: scripts/build_docker_image.sh ${{ inputs.vm_id }} + run: scripts/build_docker_image.sh diff --git a/scripts/build_docker_image.sh b/scripts/build_docker_image.sh index 370001138f..09a941513a 100755 --- a/scripts/build_docker_image.sh +++ b/scripts/build_docker_image.sh @@ -29,9 +29,7 @@ source "$SUBNET_EVM_PATH"/scripts/constants.sh source "$SUBNET_EVM_PATH"/scripts/versions.sh # Set the vm id if provided -if [[ $# -eq 1 ]]; then - VM_ID=$1 -fi +VM_ID="${VM_ID:-"${DEFAULT_VM_ID}"}" # WARNING: this will use the most recent commit even if there are un-committed changes present BUILD_IMAGE_ID=${BUILD_IMAGE_ID:-"${CURRENT_BRANCH}"} @@ -40,7 +38,7 @@ BUILD_IMAGE_ID=${BUILD_IMAGE_ID:-"${CURRENT_BRANCH}"} # simplifies creation of multi-arch images. # # Reference: https://docs.docker.com/build/buildkit/ -DOCKER_CMD="docker buildx build ${*}" +DOCKER_CMD="docker buildx build" if [[ -n "${PUBLISH}" ]]; then DOCKER_CMD="${DOCKER_CMD} --push" @@ -67,7 +65,7 @@ fi AVALANCHEGO_NODE_IMAGE="${AVALANCHEGO_NODE_IMAGE:-${AVALANCHEGO_IMAGE_NAME}:${AVALANCHE_VERSION}}" echo "Building Docker Image: $DOCKERHUB_REPO:$BUILD_IMAGE_ID based of AvalancheGo@$AVALANCHE_VERSION" -${DOCKER_CMD} -t "$DOCKERHUB_REPO:$BUILD_IMAGE_ID" -t "$DOCKERHUB_REPO:${DOCKERHUB_TAG}" \ +echo ${DOCKER_CMD} -t "$DOCKERHUB_REPO:$BUILD_IMAGE_ID" -t "$DOCKERHUB_REPO:${DOCKERHUB_TAG}" \ "$SUBNET_EVM_PATH" -f "$SUBNET_EVM_PATH/Dockerfile" \ --build-arg AVALANCHEGO_NODE_IMAGE="$AVALANCHEGO_NODE_IMAGE" \ --build-arg SUBNET_EVM_COMMIT="$SUBNET_EVM_COMMIT" \ From 74bcd4aa901ece6e9c9493ec4f09a43345e05583 Mon Sep 17 00:00:00 2001 From: Ceyhun Onur Date: Mon, 6 Jan 2025 14:48:35 +0300 Subject: [PATCH 6/7] remove debug echo --- scripts/build_docker_image.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/build_docker_image.sh b/scripts/build_docker_image.sh index 09a941513a..93474e4a36 100755 --- a/scripts/build_docker_image.sh +++ b/scripts/build_docker_image.sh @@ -28,9 +28,6 @@ source "$SUBNET_EVM_PATH"/scripts/constants.sh # Load the versions source "$SUBNET_EVM_PATH"/scripts/versions.sh -# Set the vm id if provided -VM_ID="${VM_ID:-"${DEFAULT_VM_ID}"}" - # WARNING: this will use the most recent commit even if there are un-committed changes present BUILD_IMAGE_ID=${BUILD_IMAGE_ID:-"${CURRENT_BRANCH}"} @@ -65,7 +62,7 @@ fi AVALANCHEGO_NODE_IMAGE="${AVALANCHEGO_NODE_IMAGE:-${AVALANCHEGO_IMAGE_NAME}:${AVALANCHE_VERSION}}" echo "Building Docker Image: $DOCKERHUB_REPO:$BUILD_IMAGE_ID based of AvalancheGo@$AVALANCHE_VERSION" -echo ${DOCKER_CMD} -t "$DOCKERHUB_REPO:$BUILD_IMAGE_ID" -t "$DOCKERHUB_REPO:${DOCKERHUB_TAG}" \ +${DOCKER_CMD} -t "$DOCKERHUB_REPO:$BUILD_IMAGE_ID" -t "$DOCKERHUB_REPO:${DOCKERHUB_TAG}" \ "$SUBNET_EVM_PATH" -f "$SUBNET_EVM_PATH/Dockerfile" \ --build-arg AVALANCHEGO_NODE_IMAGE="$AVALANCHEGO_NODE_IMAGE" \ --build-arg SUBNET_EVM_COMMIT="$SUBNET_EVM_COMMIT" \ From d9ed8405351ada5d8d774f0631f4e35bd1602ba5 Mon Sep 17 00:00:00 2001 From: Ceyhun Onur Date: Mon, 6 Jan 2025 15:16:46 +0300 Subject: [PATCH 7/7] revert format change --- .github/workflows/publish_docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index 2be9c7d8d7..d1764edc66 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -4,8 +4,8 @@ on: workflow_dispatch: inputs: vm_id: - description: "The ID of the VM (binary dst in Docker image)" - default: "" + description: 'The ID of the VM (binary dst in Docker image)' + default: '' required: false type: string