Skip to content

Commit

Permalink
feat(ci)!: implement reusable workflows for image building (#4173)
Browse files Browse the repository at this point in the history
* refactor(ci): test building in a separate workflow

* force a change

* force a change

* fix(ci): send the correct variables to the reusable build

* fix(ci): variables are not allowed

* fix(ci): conditions are not allowed as input

* fix(ci): use expected value

* refactor(build): simplify the use of other dockerfiles

* fix(cd): depend on docker build yml

* fix(cd): use main branch as image name

* imp(actions): remove uneeded variable repetition

* imp(build): remove unused variables

* imp(actions): rename the image building workflow

Not all images are for zebra execution as we also have one for zcash-params

* fix(ci): add dependable workflow in paths filters

* docs(ci): remove TODO as this won't be needed at least an issue arises

* docs(ci): CARGO_INCREMENTAL can decrease build time when running from a cache

* fix: revert forced changes

* fix(build): remove unused build inputs in zcash-params

* imp(cd): as this is the production image, use the executable name

* imp(ci): reduce log level to improve speed

Co-authored-by: teor <teor@riseup.net>

* imp(ci): use the correct name for the workflow

Co-authored-by: teor <teor@riseup.net>

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: teor <teor@riseup.net>
  • Loading branch information
3 people authored Apr 28, 2022
1 parent 597f553 commit 83d2689
Show file tree
Hide file tree
Showing 7 changed files with 179 additions and 385 deletions.
109 changes: 10 additions & 99 deletions .github/workflows/cd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,113 +16,24 @@ on:
- main

env:
CARGO_INCREMENTAL: '1'
NETWORK: Mainnet
PROJECT_ID: zealous-zebra
GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra
GCR_BASE: gcr.io/zealous-zebra
REGION: us-central1
ZONE: us-central1-a
MACHINE_TYPE: c2-standard-4

jobs:
build:
name: Build images
# TODO: remove timeout until we have an average build time
# timeout-minutes: 180
runs-on: ubuntu-latest
permissions:
contents: 'read'
id-token: 'write'

steps:
- uses: actions/checkout@v3.0.2
with:
persist-credentials: false

- name: Inject slug/short variables
uses: rlespinasse/github-slug-action@v4
with:
short-length: 7

# Automatic tag management and OCI Image Format Specification for labels
- name: Docker meta
id: meta
uses: docker/metadata-action@v3.7.0
with:
# list of Docker images to use as base name for tags
images: |
${{ env.GAR_BASE }}/${{ env.GITHUB_REF_SLUG_URL }}
${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.GITHUB_REF_SLUG_URL }}
# generate Docker tags based on the following events/attributes
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=sha
# Setup gcloud CLI
- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@v0.7.1
with:
workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com'
token_format: 'access_token'

- name: Set up QEMU
id: qemu
uses: docker/setup-qemu-action@v1
with:
image: tonistiigi/binfmt:latest
platforms: all

# Setup Docker Buildx to allow use of docker cache layers from GH
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1

- name: Login to Google Artifact Registry
uses: docker/login-action@v1.14.1
with:
registry: us-docker.pkg.dev
username: oauth2accesstoken
password: ${{ steps.auth.outputs.access_token }}

- name: Login to Google Container Registry
uses: docker/login-action@v1.14.1
with:
registry: gcr.io
username: oauth2accesstoken
password: ${{ steps.auth.outputs.access_token }}

# Build and push image to Google Artifact Registry
- name: Build & push
id: docker_build
uses: docker/build-push-action@v2.10.0
with:
target: runtime
context: .
file: ./docker/Dockerfile
# TODO: building crates is taking too long with arm64 and it's timing out on GHA
# platforms: |
# linux/amd64
# linux/arm64
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
NETWORK=${{ github.event.inputs.network || env.NETWORK }}
SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
RUST_BACKTRACE=1
ZEBRA_SKIP_IPV6_TESTS="1"
CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
push: true
cache-from: type=registry,ref=${{ env.GAR_BASE }}/${{ env.GITHUB_REF_SLUG_URL }}:buildcache
cache-to: type=registry,ref=${{ env.GAR_BASE }}/${{ env.GITHUB_REF_SLUG_URL }}:buildcache,mode=max
uses: ./.github/workflows/docker-image-build.yml
with:
dockerfile_path: ./docker/Dockerfile
dockerfile_target: runtime
image_name: zebrad
network: Mainnet
checkpoint_sync: true
rust_backtrace: '1'
zebra_skip_ipv6_tests: '1'
rust_log: info

deploy-nodes:
name: Deploy Mainnet nodes
Expand Down
125 changes: 125 additions & 0 deletions .github/workflows/docker-image-build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
name: Build docker image

on:
workflow_call:
inputs:
network:
required: false
type: string
checkpoint_sync:
required: false
type: boolean
image_name:
required: true
type: string
dockerfile_path:
required: true
type: string
dockerfile_target:
required: true
type: string
short_sha:
required: false
type: string
rust_backtrace:
required: false
type: string
rust_lib_backtrace:
required: false
type: string
colorbt_show_hidden:
required: false
type: string
zebra_skip_ipv6_tests:
required: false
type: string
rust_log:
required: false
type: string

jobs:
build:
name: Build images
timeout-minutes: 210
runs-on: ubuntu-latest
permissions:
contents: 'read'
id-token: 'write'
steps:
- uses: actions/checkout@v3.0.2
with:
persist-credentials: false

- name: Inject slug/short variables
uses: rlespinasse/github-slug-action@v4
with:
short-length: 7

# Automatic tag management and OCI Image Format Specification for labels
- name: Docker meta
id: meta
uses: docker/metadata-action@v3.7.0
with:
# list of Docker images to use as base name for tags
images: |
us-docker.pkg.dev/zealous-zebra/zebra/${{ inputs.image_name }}
gcr.io/zealous-zebra/zcashfoundation-zebra/${{ inputs.image_name }}
# generate Docker tags based on the following events/attributes
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=sha
# Setup Docker Buildx to allow use of docker cache layers from GH
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1

- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@v0.7.1
with:
workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com'
token_format: 'access_token'

- name: Login to Google Artifact Registry
uses: docker/login-action@v1.14.1
with:
registry: us-docker.pkg.dev
username: oauth2accesstoken
password: ${{ steps.auth.outputs.access_token }}

- name: Login to Google Container Registry
uses: docker/login-action@v1.14.1
with:
registry: gcr.io
username: oauth2accesstoken
password: ${{ steps.auth.outputs.access_token }}

# Build and push image to Google Artifact Registry
- name: Build & push
id: docker_build
uses: docker/build-push-action@v2.10.0
with:
target: ${{ inputs.dockerfile_target }}
context: .
file: ${{ inputs.dockerfile_path }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
NETWORK=${{ inputs.network }}
SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
RUST_BACKTRACE=${{ inputs.rust_backtrace }}
RUST_LIB_BACKTRACE=${{ inputs.rust_lib_backtrace }}
COLORBT_SHOW_HIDDEN=${{ inputs.colorbt_show_hidden }}
ZEBRA_SKIP_IPV6_TESTS=${{ inputs.zebra_skip_ipv6_tests }}
CHECKPOINT_SYNC=${{ inputs.checkpoint_sync }}
RUST_LOG=${{ inputs.rust_log }}
push: true
cache-from: type=registry,ref=us-docker.pkg.dev/zealous-zebra/zebra/${{ inputs.image_name }}:${{ env.GITHUB_REF_SLUG_URL }}-buildcache
cache-to: type=registry,ref=us-docker.pkg.dev/zealous-zebra/zebra/${{ inputs.image_name }}:${{ env.GITHUB_REF_SLUG_URL }}-buildcache,mode=max
108 changes: 15 additions & 93 deletions .github/workflows/test-full-sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,113 +43,35 @@ on:
# workflow definitions
- 'docker/**'
- '.github/workflows/test-full-sync.yml'
- '.github/workflows/docker-image-build.yml'

env:
CARGO_INCREMENTAL: '1'
ZEBRA_SKIP_IPV6_TESTS: '1'
RUST_BACKTRACE: full
RUST_LIB_BACKTRACE: full
COLORBT_SHOW_HIDDEN: '1'
NETWORK: Mainnet
PROJECT_ID: zealous-zebra
IMAGE_NAME: zebrad-test
GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra
GCR_BASE: gcr.io/zealous-zebra
REGION: us-central1
ZONE: us-central1-a
MACHINE_TYPE: c2d-standard-16
IMAGE_NAME: zebrad-test

jobs:
build:
# TODO add `startsWith(github.head_ref, 'mergify/merge-queue/')` to the condition to
# only run on Mergify head branches, and on manual dispatch:
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1
if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
name: Build images
timeout-minutes: 210
runs-on: ubuntu-latest
permissions:
contents: 'read'
id-token: 'write'
steps:
- uses: actions/checkout@v3.0.2
with:
persist-credentials: false

- name: Inject slug/short variables
uses: rlespinasse/github-slug-action@v4
with:
short-length: 7

# Automatic tag management and OCI Image Format Specification for labels
- name: Docker meta
id: meta
uses: docker/metadata-action@v3.7.0
with:
# list of Docker images to use as base name for tags
images: |
${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}
${{ env.GCR_BASE }}/${{ env.GITHUB_REPOSITORY_SLUG_URL }}/${{ env.IMAGE_NAME }}
# generate Docker tags based on the following events/attributes
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=sha
# Setup Docker Buildx to allow use of docker cache layers from GH
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1

- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@v0.7.1
with:
workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com'
token_format: 'access_token'

- name: Login to Google Artifact Registry
uses: docker/login-action@v1.14.1
with:
registry: us-docker.pkg.dev
username: oauth2accesstoken
password: ${{ steps.auth.outputs.access_token }}

- name: Login to Google Container Registry
uses: docker/login-action@v1.14.1
with:
registry: gcr.io
username: oauth2accesstoken
password: ${{ steps.auth.outputs.access_token }}

# Build and push image to Google Artifact Registry
- name: Build & push
id: docker_build
uses: docker/build-push-action@v2.10.0
with:
target: tester
context: .
file: ./docker/Dockerfile
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
NETWORK=${{ github.event.inputs.network || env.NETWORK }}
SHORT_SHA=${{ env.GITHUB_SHA_SHORT }}
RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}
RUST_LIB_BACKTRACE=${{ env.RUST_LIB_BACKTRACE }}
COLORBT_SHOW_HIDDEN=${{ env.COLORBT_SHOW_HIDDEN }}
ZEBRA_SKIP_NETWORK_TESTS="1"
CHECKPOINT_SYNC=${{ github.event.inputs.checkpoint_sync || true }}
RUST_LOG=debug
SENTRY_DSN=${{ secrets.SENTRY_ENDPOINT }}
push: true
cache-from: type=registry,ref=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_REF_SLUG_URL }}-buildcache
cache-to: type=registry,ref=${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:${{ env.GITHUB_REF_SLUG_URL }}-buildcache,mode=max
uses: ./.github/workflows/docker-image-build.yml
with:
dockerfile_path: ./docker/Dockerfile
dockerfile_target: tester
image_name: zebrad-test
network: Mainnet
checkpoint_sync: true
rust_backtrace: full
rust_lib_backtrace: full
colorbt_show_hidden: '1'
zebra_skip_ipv6_tests: '1'
rust_log: info

# Test that Zebra can run a full mainnet sync after a PR is approved
test-full-sync:
Expand Down Expand Up @@ -203,7 +125,7 @@ jobs:
--container-restart-policy=never \
--container-stdin \
--container-tty \
--container-env=ZEBRA_SKIP_IPV6_TESTS=1,TEST_FULL_SYNC=1,ZEBRA_FORCE_USE_COLOR=1,FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 \
--container-env=TEST_FULL_SYNC=1,ZEBRA_FORCE_USE_COLOR=1,FULL_SYNC_MAINNET_TIMEOUT_MINUTES=600 \
--machine-type ${{ env.MACHINE_TYPE }} \
--scopes cloud-platform \
--metadata=google-monitoring-enabled=true,google-logging-enabled=true \
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/test.patch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ on:
- '**/Cargo.lock'
- 'docker/**'
- '.github/workflows/test.yml'
- '.github/workflows/docker-image-build.yml'

jobs:
build:
Expand Down
Loading

0 comments on commit 83d2689

Please sign in to comment.