github/workflows: add ARM macos build binaries job (#6427)

This PR adds the required changes to release `polkadot`, `polkadot-parachain` and `polkadot-omni-node` binaries built on Apple Sillicon macos. This addresses requests from the community for such binaries: #802, and they should be part of the Github release page. Test on paritytech-stg solely focused on macos binaries: https://github.com/paritytech-stg/polkadot-sdk/actions/runs/11824692766/job/32946793308, except the steps related to `pgpkms` (which need AWS credentials, missing from paritytech-stg). The binary names don't have a `darwin-arm` identifier, and conflict with the existing x86_64-linux binaries. I haven't tested building everything on `paritytech-stg` because the x86_64-linux builds run on `unbutu-latest-m` which isn't enabled on `pairtytech-stg` (and I haven't asked CI team to enable one), so testing how to go around naming conflicts should be covered next. - [x] Test the workflow start to end (especially the last bits related to uploading the binaries on S3 and ensuring the previous binaries and the new ones coexist harmoniously on S3/action artifacts storage without naming conflicts) @EgorPopelyaev - [x] Publish the arm binaries on the Github release page - to clarify what's needed @iulianbarbu . Current practice is to manually publish the binaries built via `release-build-rc.yml` workflow, taken from S3. Would be great to have the binaries there in the first place before working on automating this, but I would also do it in a follow up PR. - [ ] unify the binaries building under `release-30_publish_release_draft.yml` maybe? - [ ] automate binary artifacts upload to S3 in `release-30_publish_release_draft.yml` --------- Signed-off-by: Iulian Barbu <iulian.barbu@parity.io> Co-authored-by: EgorPopelyaev <egor@parity.io>
paritytech · Nov 22, 2024 · 3fa6a2d · 3fa6a2d
1 parent 4c64049
commit 3fa6a2d
Show file tree

Hide file tree

Showing 6 changed files with 850 additions and 0 deletions.
diff --git a/.github/scripts/release/build-linux-release.sh b/.github/scripts/release/build-linux-release.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# This is used to build our binaries:
+# - polkadot
+# - polkadot-parachain
+# - polkadot-omni-node 
+#
+# set -e
+
+BIN=$1
+PACKAGE=${2:-$BIN}
+
+PROFILE=${PROFILE:-production}
+ARTIFACTS=/artifacts/$BIN
+VERSION=$(git tag -l --contains HEAD | grep -E "^v.*")
+
+echo "Artifacts will be copied into $ARTIFACTS"
+mkdir -p "$ARTIFACTS"
+
+git log --pretty=oneline -n 1
+time cargo build --profile $PROFILE --locked --verbose --bin $BIN --package $PACKAGE
+
+echo "Artifact target: $ARTIFACTS"
+
+cp ./target/$PROFILE/$BIN "$ARTIFACTS"
+pushd "$ARTIFACTS" > /dev/null
+sha256sum "$BIN" | tee "$BIN.sha256"
+
+EXTRATAG="$($ARTIFACTS/$BIN --version |
+    sed -n -r 's/^'$BIN' ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
+
+EXTRATAG="${VERSION}-${EXTRATAG}-$(cut -c 1-8 $ARTIFACTS/$BIN.sha256)"
+
+echo "$BIN version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
+echo -n ${VERSION} > "$ARTIFACTS/VERSION"
+echo -n ${EXTRATAG} > "$ARTIFACTS/EXTRATAG"
diff --git a/.github/scripts/release/build-macos-release.sh b/.github/scripts/release/build-macos-release.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# This is used to build our binaries:
+# - polkadot
+# - polkadot-parachain
+# - polkadot-omni-node
+# set -e
+
+BIN=$1
+PACKAGE=${2:-$BIN}
+
+PROFILE=${PROFILE:-production}
+# parity-macos runner needs a path where it can
+# write, so make it relative to github workspace.
+ARTIFACTS=$GITHUB_WORKSPACE/artifacts/$BIN
+VERSION=$(git tag -l --contains HEAD | grep -E "^v.*")
+
+echo "Artifacts will be copied into $ARTIFACTS"
+mkdir -p "$ARTIFACTS"
+
+git log --pretty=oneline -n 1
+time cargo build --profile $PROFILE --locked --verbose --bin $BIN --package $PACKAGE
+
+echo "Artifact target: $ARTIFACTS"
+
+cp ./target/$PROFILE/$BIN "$ARTIFACTS"
+pushd "$ARTIFACTS" > /dev/null
+sha256sum "$BIN" | tee "$BIN.sha256"
+
+EXTRATAG="$($ARTIFACTS/$BIN --version |
+    sed -n -r 's/^'$BIN' ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
+
+EXTRATAG="${VERSION}-${EXTRATAG}-$(cut -c 1-8 $ARTIFACTS/$BIN.sha256)"
+
+echo "$BIN version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
+echo -n ${VERSION} > "$ARTIFACTS/VERSION"
+echo -n ${EXTRATAG} > "$ARTIFACTS/EXTRATAG"
diff --git a/.github/scripts/release/release_lib.sh b/.github/scripts/release/release_lib.sh
@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+# Set the new version by replacing the value of the constant given as pattern
+# in the file.
+#
+# input: pattern, version, file
+#output: none
+set_version() {
+    pattern=$1
+    version=$2
+    file=$3
+
+    sed -i "s/$pattern/\1\"${version}\"/g" $file
+    return 0
+}
+
+# Commit changes to git with specific message.
+# "|| true" does not let script to fail with exit code 1,
+# in case there is nothing to commit.
+#
+# input: MESSAGE (any message which should be used for the commit)
+# output: none
+commit_with_message() {
+    MESSAGE=$1
+    git commit -a -m "$MESSAGE" || true
+}
+
+# Retun list of the runtimes filterd
+# input: none
+# output: list of filtered runtimes
+get_filtered_runtimes_list() {
+    grep_filters=("runtime.*" "test|template|starters|substrate")
+
+    git grep spec_version: | grep .rs: | grep -e "${grep_filters[0]}" | grep "lib.rs" | grep -vE "${grep_filters[1]}" | cut -d: -f1
+}
+
+# Sets provided spec version
+# input: version
+set_spec_versions() {
+  NEW_VERSION=$1
+  runtimes_list=(${@:2})
+
+  printf "Setting spec_version to $NEW_VERSION\n"
+
+  for f in ${runtimes_list[@]}; do
+      printf "  processing $f"
+      sed -ri "s/spec_version: [0-9]+_[0-9]+_[0-9]+,/spec_version: $NEW_VERSION,/" $f
+  done
+
+  commit_with_message "Bump spec_version to $NEW_VERSION"
+
+  git_show_log 'spec_version'
+}
+
+# Displays formated results of the git log command
+# for the given pattern which needs to be found in logs
+# input: pattern, count (optional, default is 10)
+git_show_log() {
+  PATTERN="$1"
+  COUNT=${2:-10}
+  git log --pretty=format:"%h %ad | %s%d [%an]" --graph --date=iso-strict | \
+      head -n $COUNT | grep -iE "$PATTERN" --color=always -z
+}
+
+# Get a spec_version number from the crate version
+#
+# ## inputs
+#  - v1.12.0 or 1.12.0
+#
+# ## output:
+# 1_012_000 or 1_012_001 if SUFFIX is set
+function get_spec_version() {
+    INPUT=$1
+    SUFFIX=${SUFFIX:-000} #this variable makes it possible to set a specific ruuntime version like 93826 it can be intialised as sestem variable
+    [[ $INPUT =~ .*([0-9]+\.[0-9]+\.[0-9]{1,2}).* ]]
+    VERSION="${BASH_REMATCH[1]}"
+    MATCH="${BASH_REMATCH[0]}"
+    if [ -z $MATCH ]; then
+        return 1
+    else
+        SPEC_VERSION="$(sed -e "s/\./_0/g" -e "s/_[^_]*\$/_$SUFFIX/" <<< $VERSION)"
+        echo "$SPEC_VERSION"
+        return 0
+    fi
+}
+
+# Reorganize the prdoc files for the release
+#
+# input: VERSION (e.g. v1.0.0)
+# output: none
+reorder_prdocs() {
+    VERSION="$1"
+
+    printf "[+] ℹ️ Reordering prdocs:"
+
+    VERSION=$(sed -E 's/^v([0-9]+\.[0-9]+\.[0-9]+).*$/\1/' <<< "$VERSION") #getting reed of the 'v' prefix
+    mkdir -p "prdoc/$VERSION"
+    mv prdoc/pr_*.prdoc prdoc/$VERSION
+    git add -A
+    commit_with_message "Reordering prdocs for the release $VERSION"
+}
+
+# Bump the binary version of the polkadot-parachain binary with the
+# new bumped version and commit changes.
+#
+# input: version e.g. 1.16.0
+set_polkadot_parachain_binary_version() {
+    bumped_version="$1"
+    cargo_toml_file="$2"
+
+    set_version "\(^version = \)\".*\"" $bumped_version $cargo_toml_file
+
+    cargo update --workspace --offline # we need this to update Cargo.loc with the new versions as well
+
+    MESSAGE="Bump versions in: ${cargo_toml_file}"
+    commit_with_message "$MESSAGE"
+    git_show_log "$MESSAGE"
+}
+
+
+upload_s3_release() {
+    alias aws='podman run --rm -it docker.io/paritytech/awscli -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_BUCKET aws'
+
+    product=$1
+    version=$2
+    target=$3
+
+    echo "Working on product:  $product "
+    echo "Working on version:  $version "
+    echo "Working on platform: $target "
+
+    echo "Current content, should be empty on new uploads:"
+    aws s3 ls "s3://releases.parity.io/${product}/${version}/${target}" --recursive --human-readable --summarize || true
+    echo "Content to be uploaded:"
+    artifacts="artifacts/$product/"
+    ls "$artifacts"
+    aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/${product}/${version}/${target}"
+    echo "Uploaded files:"
+    aws s3 ls "s3://releases.parity.io/${product}/${version}/${target}" --recursive --human-readable --summarize
+    echo "✅ The release should be at https://releases.parity.io/${product}/${version}/${target}"
+}
diff --git a/.github/workflows/release-build-rc.yml b/.github/workflows/release-build-rc.yml
@@ -0,0 +1,173 @@
+name: Release - Build node release candidate
+
+on:
+  workflow_dispatch:
+    inputs:
+      binary:
+        description: Binary to be build for the release
+        default: all
+        type: choice
+        options:
+          - polkadot
+          - polkadot-parachain
+          - polkadot-omni-node
+          - all
+
+      release_tag:
+        description: Tag matching the actual release candidate with the format stableYYMM-rcX or stableYYMM
+        type: string
+
+jobs:
+  check-synchronization:
+    uses: paritytech-release/sync-workflows/.github/workflows/check-syncronization.yml@main
+
+  validate-inputs:
+    needs: [check-synchronization]
+    if: ${{ needs.check-synchronization.outputs.checks_passed }} == 'true'
+    runs-on: ubuntu-latest
+    outputs:
+        release_tag: ${{ steps.validate_inputs.outputs.release_tag }}
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+      - name: Validate inputs
+        id: validate_inputs
+        run: |
+          . ./.github/scripts/common/lib.sh
+
+          RELEASE_TAG=$(validate_stable_tag ${{ inputs.release_tag }})
+          echo "release_tag=${RELEASE_TAG}" >> $GITHUB_OUTPUT
+
+  build-polkadot-binary:
+    needs: [validate-inputs]
+    if: ${{ inputs.binary == 'polkadot' || inputs.binary == 'all' }}
+    uses: "./.github/workflows/release-reusable-rc-buid.yml"
+    with:
+      binary: '["polkadot", "polkadot-prepare-worker", "polkadot-execute-worker"]'
+      package: polkadot
+      release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+      target: x86_64-unknown-linux-gnu
+    secrets:
+      PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
+      PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
+      AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION:  ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
+
+  build-polkadot-parachain-binary:
+    needs: [validate-inputs]
+    if: ${{ inputs.binary == 'polkadot-parachain' || inputs.binary == 'all' }}
+    uses: "./.github/workflows/release-reusable-rc-buid.yml"
+    with:
+      binary: '["polkadot-parachain"]'
+      package: "polkadot-parachain-bin"
+      release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+      target: x86_64-unknown-linux-gnu
+    secrets:
+      PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
+      PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
+      AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION:  ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
+
+  build-polkadot-omni-node-binary:
+    needs: [validate-inputs]
+    if: ${{ inputs.binary == 'polkadot-omni-node' || inputs.binary == 'all' }}
+    uses: "./.github/workflows/release-reusable-rc-buid.yml"
+    with:
+      binary: '["polkadot-omni-node"]'
+      package: "polkadot-omni-node"
+      release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+      target: x86_64-unknown-linux-gnu
+    secrets:
+      PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
+      PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
+      AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION:  ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
+
+  build-polkadot-macos-binary:
+    needs: [validate-inputs]
+    if: ${{ inputs.binary == 'polkadot' || inputs.binary == 'all' }}
+    uses: "./.github/workflows/release-reusable-rc-buid.yml"
+    with:
+      binary: '["polkadot", "polkadot-prepare-worker", "polkadot-execute-worker"]'
+      package: polkadot
+      release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+      target: aarch64-apple-darwin
+    secrets:
+      PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
+      PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
+      AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION:  ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
+
+  build-polkadot-parachain-macos-binary:
+    needs: [validate-inputs]
+    if: ${{ inputs.binary == 'polkadot-parachain' || inputs.binary == 'all' }}
+    uses: "./.github/workflows/release-reusable-rc-buid.yml"
+    with:
+      binary: '["polkadot-parachain"]'
+      package: "polkadot-parachain-bin"
+      release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+      target: aarch64-apple-darwin
+    secrets:
+      PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
+      PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
+      AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION:  ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
+
+  build-polkadot-omni-node-macos-binary:
+    needs: [validate-inputs]
+    if: ${{ inputs.binary == 'polkadot-omni-node' || inputs.binary == 'all' }}
+    uses: "./.github/workflows/release-reusable-rc-buid.yml"
+    with:
+      binary: '["polkadot-omni-node"]'
+      package: "polkadot-omni-node"
+      release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+      target: aarch64-apple-darwin
+    secrets:
+      PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
+      PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
+      AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION:  ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read