Replace actions/cache with a more robust strategy (#11)

* Adding new workflow * Fixing new action * Adding args * Making context info dump default * Using the inspect * Not trusting the cache... * Adding context information * Adding version and context information to post-artifact * Tossing out actions/cache * Adding info about how cache works * Forgot to use the outputs * Adding output comment to the results * Correcting the dependency graph of the two-step-container test * Correcting ifelse statement and adding check * Correcting test * Correcting check and adding an explicit explaination of what happens with caching * Adding a test regarding the args (formal test) * Capturing the context in a txt * Lite context info * Fixing usage of ARGs
CDCgov · Jan 22, 2025 · 9390790 · 9390790
1 parent 3e3f6dd
commit 9390790
Show file tree

Hide file tree

Showing 6 changed files with 237 additions and 32 deletions.
diff --git a/.github/workflows/test-twostep-container-build.yml b/.github/workflows/test-twostep-container-build.yml
@@ -14,6 +14,7 @@ on:
   workflow_dispatch:
 
 jobs:
+  # This job is used to build and push the image.
   test-no-args:
     runs-on: ubuntu-latest
 
@@ -35,11 +36,52 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
           container-file-1: twostep-container-build/examples/Containerfile.dependencies
           container-file-2: twostep-container-build/examples/Containerfile
-          first-step-cache-key: ${{ hashFiles('twostep-container-build/examples/Containerfile.dependencies') }}
+          first-step-cache-key: no-args-${{ hashFiles('twostep-container-build/examples/Containerfile.dependencies') }}
           image: cdcgov/cfa-actions
+
+  # This job is used to rerun the first step of the build
+  # to ensure that caching is working as expected.
+  test-no-args-rerun:
+    runs-on: ubuntu-latest
+    needs: test-no-args
+
+    permissions:
+      contents: read
+      packages: write
+      pull-requests: write
 
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout code
+
+      - name: Two-step build
+        uses: ./twostep-container-build
+        id: twostep-1
+        with:
+          registry: ghcr.io/
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          container-file-1: twostep-container-build/examples/Containerfile.dependencies
+          container-file-2: twostep-container-build/examples/Containerfile
+          first-step-cache-key: no-args-${{ hashFiles('twostep-container-build/examples/Containerfile.dependencies') }}
+          image: cdcgov/cfa-actions
+          push-image-1: false
+          push-image-2: false
+
+      - name: Check the output
+        run: |
+          if [ "${{ steps.twostep-1.outputs.summary }}" == "cached" ]; then
+            echo "Using the cached version (OK)"
+          else
+            echo "This was supposed to use the cache version"
+            exit 1
+          fi
+
+  # This job is used to test the action with arguments.
+  # Caching should also be triggered here.
   test-with-args:
     runs-on: ubuntu-latest
+    needs: test-no-args
 
     permissions:
       contents: read
@@ -60,15 +102,29 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
           container-file-1: twostep-container-build/examples/Containerfile.dependencies
           container-file-2: twostep-container-build/examples/Containerfile
-          first-step-cache-key: ${{ hashFiles('twostep-container-build/examples/Containerfile.dependencies') }}
-          image: cdcgov/cfa-actions-with-args
+          first-step-cache-key: with-args-${{ hashFiles('twostep-container-build/examples/Containerfile.dependencies') }}
+          image: cdcgov/cfa-actions
           build-args-2: |
             GH_SHA=${{ github.sha }}
           push-image-1: false
           push-image-2: false
 
       - name: Listing the labels from the image
         run: |
-          docker inspect ghcr.io/cdcgov/cfa-actions-with-args:${{ steps.twostep-2.outputs.tag }} \
-            --format='{{json .Config.Labels}}' | jq .
-        
+          docker inspect ghcr.io/cdcgov/cfa-actions:${{ steps.twostep-2.outputs.tag }} \
+            --format='{{ index .Config.Labels "GH_SHA" }}' > _${{ github.sha }}_labels.json
+
+          if [ "$(cat _${{ github.sha }}_labels.json)" != "${{ github.sha }}" ]; then
+            echo "The argument GH_SHA does not match the expected value."
+            exit 1
+          fi
+
+      - name: Check the output
+        run: |
+          if [ "${{ steps.twostep-2.outputs.summary }}" == "rebuilt" ]; then
+            echo "Using a re-built version (OK)"
+          else
+            echo "This was supposed to use the cache version"
+            exit 1
+          fi
+    
diff --git a/post-artifact/README.md b/post-artifact/README.md
@@ -35,7 +35,7 @@ Here are the contents of a job that (i) uploads an artifact using `actions/uploa
 
       # Post the artifact pulling the id from the `readme` step.
       - name: Post the artifact
-        uses: CDCgov/cfa-actions/post-artifact@main
+        uses: CDCgov/cfa-actions/post-artifact@1.2.0
         if: ${{ github.event_name == 'pull_request' }}
         with:
           artifact-name: readme

diff --git a/post-artifact/action.yml b/post-artifact/action.yml
@@ -33,6 +33,42 @@ runs:
 
   steps:
 
+    - name: Getting the context information
+      shell: bash
+      run: |
+        echo "::group::GitHub context"
+        echo "github.*"
+        echo "action              : ${{ github.action }}"
+        echo "action_path         : ${{ github.action_path }}"
+        echo "action_ref          : ${{ github.action_ref }}"
+        echo "action_repository   : ${{ github.action_repository }}"
+        echo "action_status       : ${{ github.action_status }}"
+        echo "actor               : ${{ github.actor }}"
+        echo "actor_id            : ${{ github.actor_id }}"
+        echo "base_ref            : ${{ github.base_ref }}"
+        echo "env                 : ${{ github.env }}"
+        echo "event_name          : ${{ github.event_name }}"
+        echo "event_path          : ${{ github.event_path }}"
+        echo "graphql_url         : ${{ github.graphql_url }}"
+        echo "head_ref            : ${{ github.head_ref }}"
+        echo "job                 : ${{ github.job }}"
+        echo "path                : ${{ github.path }}"
+        echo "ref                 : ${{ github.ref }}"
+        echo "ref_name            : ${{ github.ref_name }}"
+        echo "ref_protected       : ${{ github.ref_protected }}"
+        echo "ref_type            : ${{ github.ref_type }}"
+        echo "repository          : ${{ github.repository }}"
+        echo "repository_id       : ${{ github.repository_id }}"
+        echo "repository_owner    : ${{ github.repository_owner }}"
+        echo "repository_owner_id : ${{ github.repository_owner_id }}"
+        echo "repositoryUrl       : ${{ github.repositoryUrl }}"
+        echo "retention_days      : ${{ github.retention_days }}"
+        echo "run_id              : ${{ github.run_id }}"
+        echo "run_number          : ${{ github.run_number }}"
+        echo "run_attempt         : ${{ github.run_attempt }}"
+        echo "secret_source       : ${{ github.secret_source }}"
+        echo "::endgroup::"
+
     - name: Check this is a PR
       if: ${{ github.event_name != 'pull_request' }}
       run: 

diff --git a/twostep-container-build/README.md b/twostep-container-build/README.md
@@ -9,7 +9,22 @@ flowchart LR
   Containerfile2-->|Generates|Image2
 ```
 
-Caching is done using the [actions/cache](https://github.com/actions/cache/tree/v4) (lookup only) and [docker/build-push-action@v6](https://github.com/docker/build-push-action/tree/v6) actions. Users have to explicitly provide the cache key for the first step. For example, if you are dealing with an R package, you can cache the dependencies by passing the key `${{ hashFiles('DESCRIPTION') }}` to the `first-step-cache-key` input. That way, the first step will only be executed if the dependencies change.
+The bulk of the action is dealing with whether the first image is available or not. If available, the action will check if the cache key matches the hash of the container file. If it does, the action will skip the first step and use the cached image. If the cache key does not match, the action will build the image and push it to the registry (if `push-image-1` is set to `true`):
+
+```mermaid
+flowchart TB
+  subgraph "Caching"
+    start((Start)) --> tag_exists
+    tag_exists{Tag exists}-->|Yes|pull_image
+    pull_image[Pull image<br>& inspect<br>labels]-->check_label
+    check_label{Label<br>matches<br>hash}-->|Yes|done((Done))
+    check_label-->|No|build_image["Build image<br>& push (optional)"]
+    tag_exists-->|No|build_image
+    build_image-->done
+  end
+```
+
+Caching is done by storing the cache-key as a label in the image (`TWO_STEP_BUILD_CACHE_KEY`); and the build and push process is done using the [docker/build-push-action@v6](https://github.com/docker/build-push-action/tree/v6) action. Users have to explicitly provide the cache key for the first step. For example, if you are dealing with an R package, you can cache the dependencies by passing the key `${{ hashFiles('DESCRIPTION') }}` to the `first-step-cache-key` input. That way, the first step will only be executed if the dependencies change.
 
 ## Inputs and Outputs
 
@@ -33,13 +48,16 @@ The following are arguments passed to the [docker/build-push-action@v6](https://
 | `push-image-2` | Push the image created during the second step | false | `false` |
 | `build-args-1` | Build arguments for the first step | false | |
 | `build-args-2` | Build arguments for the second step | false | |
+| `labels-1` | Labels for the first step | false | |
+| `labels-2` | Labels for the second step | false | |
 
 The action has the following outputs:
 
 | Field | Description |
 |-------|-------------|
 | `tag` | Container tag of the built image |
 | `branch` | Branch name |
+| `summary` | A summary of the action: (`built`, `re-built`, or `cached`) |
 
 
 ## Example: Using ghcr.io
@@ -71,7 +89,7 @@ jobs:
         name: Checkout code
 
       - name: Two-step build
-        uses: CDCgov/cfa-actions/twostep-container-build@v1.1.0
+        uses: CDCgov/cfa-actions/twostep-container-build@v1.2.0
         with:
           # Login information
           registry: ghcr.io/
@@ -106,12 +124,18 @@ CMD ["bash"]
 [`Containerfile`](examples/Containerfile)
 
 ```Containerfile
+# Collection of ARGs
 ARG TAG=dependencies-latest
+ARG IMAGE=ghcr.io/cdcgov/cfa-actions
+ARG GH_SHA=default_var
 
-FROM ghcr.io/cdcgov/cfa-actions:${TAG}
+FROM ${IMAGE}:${TAG}
 
-COPY twostep-container-build/example/Containerfile /app/.
+# Re-declaring the ARGs here is necessary to use them in the LABEL
+ARG GH_SHA
+LABEL GH_SHA=${GH_SHA}
 
+COPY . /app/.
 CMD ["bash"]
 ```
 

diff --git a/twostep-container-build/action.yml b/twostep-container-build/action.yml
@@ -66,6 +66,14 @@ inputs:
     description: |
       The build arguments to use for the second image.
     required: false
+  build-labels-1:
+    description: |
+      The labels to use for the first image.
+    required: false
+  build-labels-2:
+    description: |
+      The labels to use for the second image.
+    required: false
 outputs:
   tag:
     description: |
@@ -75,21 +83,51 @@ outputs:
     description: |
       The branch name.
     value: ${{ steps.branch-name.outputs.branch }}
+  summary:
+    description: |
+      The result of the build.
+    value: ${{ steps.final.outputs.result }}
 
 runs:
   using: 'composite'
 
   steps:
 
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-
-    - name: Getting the commit message
-      id: commit-message
-      run: echo "message=$(git log -1 --pretty=%s HEAD)" >> $GITHUB_OUTPUT
+    - name: Getting the context information
       shell: bash
+      run: |
+        echo "::group::GitHub context"
+        echo "github.*"
+        echo "action              : ${{ github.action }}"
+        echo "action_path         : ${{ github.action_path }}"
+        echo "action_ref          : ${{ github.action_ref }}"
+        echo "action_repository   : ${{ github.action_repository }}"
+        echo "action_status       : ${{ github.action_status }}"
+        echo "actor               : ${{ github.actor }}"
+        echo "actor_id            : ${{ github.actor_id }}"
+        echo "base_ref            : ${{ github.base_ref }}"
+        echo "env                 : ${{ github.env }}"
+        echo "event_name          : ${{ github.event_name }}"
+        echo "event_path          : ${{ github.event_path }}"
+        echo "graphql_url         : ${{ github.graphql_url }}"
+        echo "head_ref            : ${{ github.head_ref }}"
+        echo "job                 : ${{ github.job }}"
+        echo "path                : ${{ github.path }}"
+        echo "ref                 : ${{ github.ref }}"
+        echo "ref_name            : ${{ github.ref_name }}"
+        echo "ref_protected       : ${{ github.ref_protected }}"
+        echo "ref_type            : ${{ github.ref_type }}"
+        echo "repository          : ${{ github.repository }}"
+        echo "repository_id       : ${{ github.repository_id }}"
+        echo "repository_owner    : ${{ github.repository_owner }}"
+        echo "repository_owner_id : ${{ github.repository_owner_id }}"
+        echo "repositoryUrl       : ${{ github.repositoryUrl }}"
+        echo "retention_days      : ${{ github.retention_days }}"
+        echo "run_id              : ${{ github.run_id }}"
+        echo "run_number          : ${{ github.run_number }}"
+        echo "run_attempt         : ${{ github.run_attempt }}"
+        echo "secret_source       : ${{ github.secret_source }}"
+        echo "::endgroup::"
 
     - name: Checking out the latest (may be merge if PR)
       uses: actions/checkout@v4
@@ -114,15 +152,6 @@ runs:
           echo "tag=${{ steps.branch-name.outputs.branch }}" >> $GITHUB_OUTPUT
         fi
 
-    - name: Check cache for base image
-      uses: actions/cache@v4
-      id: cache
-      with:
-        key: ${{ inputs.first-step-cache-key }}
-        lookup-only: true
-        path:
-          ${{ inputs.container-file-1 }}
-
     - name: Login to the Container Registry
       if: inputs.registry != ''
       uses: docker/login-action@v3
@@ -131,8 +160,44 @@ runs:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
 
+    - name: Checking if the image exists
+      id: pull-image
+      shell: bash
+      run: |
+        # Ensuring we can pull the image, if cache exists
+        docker pull ${{ inputs.registry }}${{ inputs.image }}:dependencies-${{ steps.image-tag.outputs.tag }} || \
+          export IMAGE_NOT_FOUND=tue
+
+        if [ -n "$IMAGE_NOT_FOUND" ]; then
+          echo "Image was not found, it will be built."
+          echo "image-found=false" >> $GITHUB_OUTPUT
+        else
+          echo "Image found, we will inspect the labels to check for cache."
+          echo "image-found=true" >> $GITHUB_OUTPUT
+        fi
+    
+    - name: Checking if matches cache
+      if: steps.pull-image.outputs.image-found == 'true'
+      id: cache
+      shell: bash
+      run: |
+
+        # Inspecting the image
+        docker inspect \
+          ${{ inputs.registry }}${{ inputs.image }}:dependencies-${{ steps.image-tag.outputs.tag }} \
+          --format='{{ index .Config.Labels "TWO_STEP_BUILD_CACHE_KEY" }}' > \
+          ${{ github.sha }}_cache_hash
+        
+        if [ "$(cat ${{ github.sha }}_cache_hash)" != "${{ inputs.first-step-cache-key }}" ]; then
+          echo "Cache hash does not match. Rebuilding the image..."
+          echo "cache-hit=false" >> $GITHUB_OUTPUT
+        else
+          echo "Cache hash matches (all good!)."
+          echo "cache-hit=true" >> $GITHUB_OUTPUT
+        fi
+
     - name: Build and push
-      if: steps.cache.outputs.cache-hit != 'true'
+      if: steps.cache.outputs.cache-hit != 'true' || steps.pull-image.outputs.image-found != 'true'
       uses: docker/build-push-action@v6
       with:
         no-cache: true
@@ -141,6 +206,9 @@ runs:
           ${{ inputs.registry }}${{ inputs.image }}:dependencies-${{ steps.image-tag.outputs.tag }}
         file: ${{ inputs.container-file-1 }}
         build-args: ${{ inputs.build-args-1 }}
+        labels: |
+          ${{ inputs.build-labels-1 }}
+          TWO_STEP_BUILD_CACHE_KEY=${{ inputs.first-step-cache-key }}
 
 
     - name: Build and push the main image
@@ -154,4 +222,23 @@ runs:
         file: ${{ inputs.container-file-2 }}
         build-args: |
           ${{ inputs.build-args-2 }}
-          TAG=dependencies-${{ steps.image-tag.outputs.tag }}
+          TAG=dependencies-${{ steps.image-tag.outputs.tag }}
+        labels: |
+          ${{ inputs.build-labels-2 }}
+          TWO_STEP_BUILD_CACHE_KEY=${{ inputs.first-step-cache-key }}
+
+    - name: Set the final result
+      id: final
+      shell: bash
+      run: |
+        # Three possible cases:
+        # - The image did not exist, so we built it
+        # - The image existed, but the cache did not match, so we rebuilt it
+        # - The image existed, and the cache matched, so we did not rebuild it
+        if [ "${{ steps.cache.outputs.cache-hit }}" == "" ]; then
+          echo "result=built" >> $GITHUB_OUTPUT
+        elif [ "${{ steps.cache.outputs.cache-hit }}" == "false" ]; then
+          echo "result=rebuilt" >> $GITHUB_OUTPUT
+        else
+          echo "result=cached" >> $GITHUB_OUTPUT
+        fi