Docker build and push - part 1 (#56)

Co-authored-by: Age Manning <Age@AgeManning.com>

.github/scripts/toml_reader.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# TOML Reader - A script to read values from simple TOML files
+# Usage: ./toml_reader.sh <file_path> <section> <key>
+
+get_section() {
+    # Function to get the section from a TOML file
+    # Parameters:
+    # $1 - TOML file path
+    # $2 - section name
+    local file="$1"
+    local section="$2"
+
+    sed -n "/^\[$section\]/,/^\[/p" "$file" | sed '$d'
+}
+
+get_toml_value() {
+    # Function to get a value from a TOML file
+    # Parameters:
+    # $1 - TOML file path
+    # $2 - section
+    # $3 - key
+    local file="$1"
+    local section="$2"
+    local key="$3"
+
+    get_section "$file" "$section" | grep "^$key " | cut -d "=" -f2- | tr -d ' "'
+}
+
+# Show usage if script is called directly
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    if [ "$#" -ne 3 ]; then
+        echo "Error: Incorrect number of arguments"
+        echo "Usage: $0 <file_path> <section> <key>"
+        echo "Example: $0 ./config.toml server_b domain"
+        exit 1
+    fi
+
+    # Check if file exists
+    if [ ! -f "$1" ]; then
+        echo "Error: File '$1' does not exist"
+        exit 1
+    fi
+
+    # Get the value
+    result=$(get_toml_value "$1" "$2" "$3")
+
+    # Check if value was found
+    if [ -z "$result" ]; then
+        echo "Error: No value found for section '[$2]' and key '$3'"
+        exit 1
+    fi
+
+    echo "$result"
+fi

.github/workflows/book.yml

@@ -11,6 +11,7 @@ concurrency:
 
 jobs:
   build-and-upload-to-s3:
+    if: github.repository_owner == 'sigp'
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4
@@ -33,4 +34,4 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: 'ap-southeast-2'
-          SOURCE_DIR: 'book/book'
+          SOURCE_DIR: 'book/book'

.github/workflows/docker.yml

@@ -1,163 +1,134 @@
-# TODO: Setup docker
-# name: docker
-#
-# on:
-#     push:
-#         branches:
-#             - stable
-#         tags:
-#             - v*
-#
-# concurrency:
-#   group: ${{ github.workflow }}-${{ github.ref }}
-#   cancel-in-progress: true
-#
-# env:
-#     DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-#     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-#     # Enable self-hosted runners for the sigp repo only.
-#     SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/anchor' }}
-#
-# jobs:
-#     # Extract the VERSION which is either `latest` or `vX.Y.Z`, and the VERSION_SUFFIX
-#     # which is either empty or `-unstable`.
-#     #
-#     # It would be nice if the arch didn't get spliced into the version between `latest` and
-#     # `unstable`, but for now we keep the two parts of the version separate for backwards
-#     # compatibility.
-#     extract-version:
-#         runs-on: ubuntu-22.04
-#         steps:
-#             - name: Extract version (if stable)
-#               if: github.event.ref == 'refs/heads/stable'
-#               run: |
-#                     echo "VERSION=latest" >> $GITHUB_ENV
-#                     echo "VERSION_SUFFIX=" >> $GITHUB_ENV
-#             - name: Extract version (if unstable)
-#               if: github.event.ref == 'refs/heads/unstable'
-#               run: |
-#                     echo "VERSION=latest" >> $GITHUB_ENV
-#                     echo "VERSION_SUFFIX=-unstable" >> $GITHUB_ENV
-#             - name: Extract version (if tagged release)
-#               if: startsWith(github.event.ref, 'refs/tags')
-#               run: |
-#                     echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV
-#                     echo "VERSION_SUFFIX=" >> $GITHUB_ENV
-#         outputs:
-#             VERSION: ${{ env.VERSION }}
-#             VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }}
-#     build-docker-single-arch:
-#         name: build-docker-${{ matrix.binary }}-${{ matrix.cpu_arch }}${{ matrix.features.version_suffix }}
-#         # Use self-hosted runners only on the sigp repo.
-#         runs-on: ${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "linux", "release"]') || 'ubuntu-22.04'  }}
-#         strategy:
-#             matrix:
-#                 binary:   [anchor]
-#                 cpu_arch: [aarch64, x86_64]
-#                 include:
-#                     - profile: maxperf
-#
-#         needs: [extract-version]
-#         env:
-#             VERSION: ${{ needs.extract-version.outputs.VERSION }}
-#             VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
-#         steps:
-#             - uses: actions/checkout@v4
-#             - name: Update Rust
-#               if: env.SELF_HOSTED_RUNNERS == 'false'
-#               run: rustup update stable
-#             - name: Dockerhub login
-#               run: |
-#                   echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
-#
-#             - name: Sets env vars for Anchor
-#               if: startsWith(matrix.binary, 'anchor')
-#               run: |
-#                 echo "CROSS_FEATURES=gnosis,spec-minimal,slasher-lmdb,jemalloc" >> $GITHUB_ENV
-#
-#             - name: Set `make` command for anchor
-#               if: startsWith(matrix.binary, 'anchor')
-#               run: |
-#                 echo "MAKE_CMD=build-${{ matrix.cpu_arch }}" >> $GITHUB_ENV
-#
-#             - name: Set `make` command for lcli
-#               if: startsWith(matrix.binary, 'lcli')
-#               run: |
-#                 echo "MAKE_CMD=build-lcli-${{ matrix.cpu_arch }}" >> $GITHUB_ENV
-#
-#             - name: Cross build binaries
-#               run: |
-#                   cargo install cross
-#                   env CROSS_PROFILE=${{ matrix.profile }} CROSS_FEATURES=${{ env.CROSS_FEATURES }} make ${{ env.MAKE_CMD }}
-#
-#             - name: Make bin dir
-#               run: mkdir ./bin
-#
-#             - name: Move cross-built binary into Docker scope
-#               run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/${{ matrix.profile }}/${{ matrix.binary }} ./bin
-#
-#             - name: Map aarch64 to arm64 short arch
-#               if: startsWith(matrix.cpu_arch, 'aarch64')
-#               run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV
-#
-#             - name: Map x86_64 to amd64 short arch
-#               if: startsWith(matrix.cpu_arch, 'x86_64')
-#               run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV;
-#
-#             - name: Install QEMU
-#               if: env.SELF_HOSTED_RUNNERS == 'false'
-#               run: sudo apt-get update && sudo apt-get install -y qemu-user-static
-#
-#             - name: Set up Docker Buildx
-#               if: env.SELF_HOSTED_RUNNERS == 'false'
-#               uses: docker/setup-buildx-action@v3
-#
-#             - name: Build and push (Anchor)
-#               if: startsWith(matrix.binary, 'anchor')
-#               uses: docker/build-push-action@v5
-#               with:
-#                 file: ./Dockerfile.cross
-#                 context: .
-#                 platforms: linux/${{ env.SHORT_ARCH }}
-#                 push: true
-#                 tags: |
-#                   ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}
-#
-#             - name: Build and push (lcli)
-#               if: startsWith(matrix.binary, 'lcli')
-#               uses: docker/build-push-action@v5
-#               with:
-#                 file: ./lcli/Dockerfile.cross
-#                 context: .
-#                 platforms: linux/${{ env.SHORT_ARCH }}
-#                 push: true
-#
-#                 tags: |
-#                   ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}
-#
-#
-#     build-docker-multiarch:
-#         name: build-docker-${{ matrix.binary }}-multiarch
-#         runs-on: ubuntu-22.04
-#         strategy:
-#             matrix:
-#                 binary: [anchor,
-#                          lcli]
-#         needs: [build-docker-single-arch, extract-version]
-#         env:
-#             VERSION: ${{ needs.extract-version.outputs.VERSION }}
-#             VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
-#         steps:
-#             - name: Set up Docker Buildx
-#               uses: docker/setup-buildx-action@v3
-#
-#             - name: Dockerhub login
-#               run: |
-#                   echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
-#
-#             - name: Create and push multiarch manifests
-#               run: |
-#                   docker buildx imagetools create -t ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}${VERSION_SUFFIX} \
-#                       ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}-arm64${VERSION_SUFFIX} \
-#                       ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}-amd64${VERSION_SUFFIX};
-#
+name: Docker build and push
+
+on:
+  push:
+    branches:
+      - stable
+    tags:
+      - v*
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+    # Enable self-hosted runners for the sigp repo only.
+    SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/anchor' }}
+    RUST_VERSION: '1.80.0'
+    SHORT_ARCH: 'amd64'
+    # Deny warnings in CI
+    RUSTFLAGS: "-D warnings -C debuginfo=0"
+    # Prevent Github API rate limiting
+    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+    extract-version:
+        uses: ./.github/workflows/extract-version.yml
+
+
+    build-docker-multi-arch:
+        name: build-docker-anchor-${{ matrix.cpu_arch }}
+        runs-on: ${{ github.repository == 'sigp/anchor' && fromJson('["self-hosted", "linux", "release"]') || 'ubuntu-22.04'  }}
+        strategy:
+            matrix:
+                cpu_arch: [aarch64, x86_64]
+                include:
+                    - profile: maxperf
+
+        needs: [extract-version]
+        env:
+            VERSION: ${{ needs.extract-version.outputs.VERSION }}
+            VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
+        steps:
+
+            - name: Map aarch64 to arm64 short arch
+              if: startsWith(matrix.cpu_arch, 'aarch64')
+              run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV
+
+            - name: Map x86_64 to amd64 short arch
+              if: startsWith(matrix.cpu_arch, 'x86_64')
+              run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV
+
+
+            - name: Install Rust and Cargo
+              run: |
+                curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+                source $HOME/.cargo/env
+                echo "PATH=$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV
+
+                
+            - name: cargo install cross
+              run: |
+                    cargo install cross
+
+            # - uses: actions/checkout@v4
+            # - name: Update Rust
+            #   if: env.SELF_HOSTED_RUNNERS == 'false'
+            #   run: rustup update stable
+            - name: Checkout sources
+              uses: actions/checkout@v4
+
+            - name: Get rust-version 
+              id: get-rust-version 
+              run: |
+                echo "RUST_VERSION=$(./.github/scripts/toml_reader.sh ./anchor/Cargo.toml package rust-version)" >> $GITHUB_ENV
+          
+            - name: Get latest version of stable Rust
+              run: echo "rust version is ${{ env.RUST_VERSION }}"
+
+            - name: Retrieve Docker credentials from Vault
+              uses: hashicorp/vault-action@v2
+              with:
+                url: https://vault.sigp.io
+                method: github
+                githubToken: ${{ secrets.GH_TOKEN }}
+                secrets: |
+                    spesi_kv/data/dev/docker/anchor DOCKER_USERNAME ;
+                    spesi_kv/data/dev/docker/anchor DOCKER_PASSWORD
+
+            - name: Dockerhub login
+              run: |
+                  echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+
+            - name: Build binary
+              run: |
+                cargo install cross
+                env CROSS_PROFILE=${{ matrix.profile }} make build-${{ matrix.cpu_arch }}
+
+
+            - name: Set `make` command for anchor
+              run: |
+                  echo "MAKE_CMD=build-${{ matrix.cpu_arch }}" >> $GITHUB_ENV                
+
+            - name: Make bin dir
+              run: mkdir ./bin
+
+            - name: Move built binary into Docker scope
+              run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/${{ matrix.profile }}/anchor ./bin
+
+            - name: Install QEMU
+              if: env.SELF_HOSTED_RUNNERS == 'false'
+              run: sudo apt-get update && sudo apt-get install -y qemu-user-static
+
+            - name: Set up Docker Buildx
+              if: env.SELF_HOSTED_RUNNERS == 'false'
+              uses: docker/setup-buildx-action@v3
+
+            - name: Build and push
+              uses: docker/build-push-action@v5
+              with:
+                file: ./anchor/Dockerfile.cross
+                context: .
+                platforms: linux/${{ env.SHORT_ARCH }}
+                labels: |
+                    git.revision=${{ github.sha }}
+                    git.branch=${{ github.ref }}
+                    git.tag=${{ github.ref }}
+                    git.repository=${{ github.repository }}
+                push: true
+                tags: |
+                  ${{ github.repository_owner}}/anchor:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}
+                build-args: |
+                  RUST_VERSION=${{ env.RUST_VERSION }}
+                  TARGETPLATFORM=linux/${{ env.SHORT_ARCH }}
+