fix: image tagging consistency (#313)

SwissDataScienceCenter · Mar 9, 2023 · e6c1294 · e6c1294
1 parent d9ccb5e
commit e6c1294
Show file tree

Hide file tree

Showing 9 changed files with 111 additions and 168 deletions.
diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml
@@ -26,11 +26,10 @@ name: Renku Docker Image CI
 on: [push]
 
 env:
-  DOCKER_NAME: "renku/renkulab"
+  DOCKER_PREFIX: renku/renkulab
   DEFAULT_PYTHON_VERSION: "3.10"
 
 jobs:
-
   lint:
     strategy:
       fail-fast: true
@@ -75,8 +74,6 @@ jobs:
     - uses: actions/checkout@v3
     - name: Build renku project python-based docker images
       id: build-image
-      env:
-        DOCKER_PREFIX: renku/renkulab    
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
           export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
@@ -94,10 +91,10 @@ jobs:
         # needed by the makefile - these are generated dynamically
         # and hence they are not part of the env setup above
         export BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }}
-        export DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}"
+        export PY_DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}"
         export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
         make py 
-        echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${DOCKER_LABEL}" >> $GITHUB_OUTPUT
+        echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${PY_DOCKER_LABEL}" >> $GITHUB_OUTPUT
 
     - name: Image Acceptance Tests
       uses: cypress-io/github-action@v4
@@ -115,8 +112,8 @@ jobs:
 
         # on master push latest image - the lab-* image is tagged latest
         if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then
-          docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_NAME-py:latest
-          docker push $DOCKER_NAME-py:latest
+          docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-py:latest
+          docker push $DOCKER_PREFIX-py:latest
         fi
 
   build-cuda:
@@ -163,30 +160,26 @@ jobs:
         username: ${{ secrets.RENKU_DOCKER_USERNAME }}
         password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v3  
     - name: Build renku project python-based docker image extensions
       id: build-image
-      env:
-        DOCKER_PREFIX: "renku/renkulab"    
       run: |
-        # cd docker/cuda
-
         if [[ ${{ github.ref }} == refs/tags* ]]; then
           export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
         else
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
+        docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
 
         # these are set dynamically so not part of env
-        export DOCKER_LABEL="${{ matrix.CUDA_VERSION }}-${LABEL}"
-        export CUDA_BASE_IMAGE="renku/renkulab-py:$DEFAULT_PYTHON_VERSION-$LABEL" 
+        export CUDA_COMPAT_PACKAGE="${{ matrix.CUDA_COMPAT_PACKAGE }}" 
+        export CUDA_CUDART_PACKAGE="${{ matrix.CUDA_CUDART_PACKAGE }}" 
         export CUDA_VERSION="${{ matrix.CUDA_VERSION }}" 
+        export CUDA_DOCKER_LABEL="${{ matrix.CUDA_VERSION }}-${LABEL}"
         export EXTRA_LIBRARIES="${{ matrix.EXTRA_LIBRARIES }}" 
-        export CUDA_CUDART_PACKAGE="${{ matrix.CUDA_CUDART_PACKAGE }}" 
-        export CUDA_COMPAT_PACKAGE="${{ matrix.CUDA_COMPAT_PACKAGE }}" 
         export LIBCUDNN_PACKAGE="${{ matrix.LIBCUDNN_PACKAGE }}" 
         make cuda
-        echo "IMAGE_NAME=$DOCKER_NAME-cuda:$DOCKER_LABEL" >> $GITHUB_OUTPUT
+        echo "IMAGE_NAME=$DOCKER_PREFIX-cuda:$CUDA_DOCKER_LABEL" >> $GITHUB_OUTPUT
         
     - name: Image Acceptance Tests
       uses: cypress-io/github-action@v4
@@ -202,9 +195,9 @@ jobs:
         docker push ${{ steps.build-image.outputs.IMAGE_NAME }}
 
         # on master push latest image
-        if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.CUDA_VERSION }}" == "11.5" ]; then
-          docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_NAME-cuda:latest
-          docker push $DOCKER_NAME-cuda:latest
+        if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.CUDA_VERSION }}" == "11.7" ]; then
+          docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-cuda:latest
+          docker push $DOCKER_PREFIX-cuda:latest
         fi
 
   build-py-ext:
@@ -216,6 +209,7 @@ jobs:
       matrix:
         EXTENSIONS:
           - vnc
+          - batch
 
     steps:
     - name: Docker Login
@@ -227,20 +221,19 @@ jobs:
     - uses: actions/checkout@v3
     - name: Build renku project python-based docker image extensions
       env:
-        DOCKER_PREFIX: renku/renkulab    
         EXTENSION: ${{ matrix.EXTENSIONS }}
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
           export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
         else
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
-
+        docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
+        
         # needed by the makefile - for these images, we only tag with commit sha
-        export DOCKER_LABEL="${LABEL}"
-        export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
-        make vnc 
-        docker push $DOCKER_NAME-$EXTENSION:$LABEL
+        export EXTRA_DOCKER_LABEL="${LABEL}"
+        make $EXTENSION 
+        docker push $DOCKER_PREFIX-$EXTENSION:$LABEL
 
   build-vnc-ext:
     needs: build-py-ext
@@ -263,54 +256,20 @@ jobs:
     - uses: actions/checkout@v3
     - name: Build renku project python-based docker image extensions
       env:
-        DOCKER_PREFIX: renku/renkulab    
         EXTENSION: ${{ matrix.VNC-EXTENSIONS }}
-        DOCKER_NAME: "renku/renkulab"
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
           export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
         else
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
+        docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
+        docker pull ${DOCKER_PREFIX}-vnc:${LABEL}
 
-        # needed by the makefile
-        export DOCKER_LABEL="${LABEL}"
-        export GIT_MASTER_HEAD_SHA=$(git rev-parse --short=7 --verify HEAD)
-        # the mainfile contains targets for each of these extensions; if we
-        # add another type of vnc based images, we need to add a target to
-        # the makrfile accordingly
+        # needed by the Makefile
+        export EXTRA_DOCKER_LABEL="${LABEL}"
         make vnc-$EXTENSION
-        docker push $DOCKER_NAME-$EXTENSION:$LABEL
-
-  build-py-batch:
-    needs: build-py
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Docker Login
-      uses: Azure/docker-login@v1
-      with:
-        username: ${{ secrets.RENKU_DOCKER_USERNAME }}
-        password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
-
-    - uses: actions/checkout@v3
-    - name: Build renku project python-based docker image for batch execution
-      env:
-        DOCKER_PREFIX: renku/renkulab    
-      run: |
-        if [[ ${{ github.ref }} == refs/tags* ]]; then
-          export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
-        else
-          export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
-        fi
-
-        # needed by the makefile
-        export DOCKER_LABEL="$LABEL"
-        export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
-        export RENKU_BASE="$DOCKER_PREFIX-py:3.10-$LABEL"
-        export BASE_IMAGE="python:3.10-slim-buster" 
-        make batch
-        echo "IMAGE_NAME=$DOCKER_NAME-batch:$DOCKER_LABEL" >> $GITHUB_OUTPUT
+        docker push $DOCKER_PREFIX-$EXTENSION:$LABEL
 
   build-julia-ext:
     needs: build-py
@@ -332,22 +291,20 @@ jobs:
     - uses: actions/checkout@v3
     - name: Build renku project julia docker image extensions
       id: build-image
-      env:
-        DOCKER_PREFIX: renku/renkulab    
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
           export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
         else
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
+        docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
 
         # needed by the makefile
-        export DOCKER_LABEL="${{ matrix.JULIAVERSIONS }}-$LABEL"
-        export JULIAVERSION="${{ matrix.JULIAVERSIONS }}"
+        export JULIA_DOCKER_LABEL="${{ matrix.JULIAVERSIONS }}-$LABEL"
         export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
-        export BASE_IMAGE="$DOCKER_NAME-py:3.10-$LABEL"
+        export JULIAVERSION="${{ matrix.JULIAVERSIONS }}"
         make julia
-        echo "IMAGE_NAME=$DOCKER_NAME-julia:$DOCKER_LABEL" >> $GITHUB_OUTPUT
+        echo "IMAGE_NAME=$DOCKER_PREFIX-julia:$JULIA_DOCKER_LABEL" >> $GITHUB_OUTPUT
         
     - name: Image Acceptance Tests
       uses: cypress-io/github-action@v4
@@ -361,7 +318,7 @@ jobs:
     - name: Push to docker
       run: |
         docker push ${{ steps.build-image.outputs.IMAGE_NAME }}
-
+        
   build-r-ubuntu:
     needs: build-py
     runs-on: ubuntu-latest
@@ -390,24 +347,21 @@ jobs:
     - uses: actions/checkout@v3
     - name: Build renku project rocker docker images
       id: build-image
-      env:
-        DOCKER_PREFIX: renku/renkulab    
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
           export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
         else
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
+        docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
 
         # needed by the makefile
         export RVERSION="${{ matrix.RVERSION }}"
-        export DOCKER_LABEL="${{ matrix.RVERSION }}-$LABEL"
-        export RENKU_PYTHON_BASE_IMAGE_TAG="3.10"
-        export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
-        export BASE_IMAGE="rocker/verse:${{ matrix.RVERSION }}" 
+        export R_DOCKER_LABEL="${{ matrix.RVERSION }}-$LABEL"
+        export RSTUDIO_BASE_IMAGE="rocker/verse:${{ matrix.RVERSION }}" 
         export RSTUDIO_VERSION_OVERRIDE="${{ matrix.RSTUDIO_VERSION }}"
         make r
-        echo "IMAGE_NAME=$DOCKER_PREFIX-r:$DOCKER_LABEL" >> $GITHUB_OUTPUT
+        echo "IMAGE_NAME=$DOCKER_PREFIX-r:$R_DOCKER_LABEL" >> $GITHUB_OUTPUT
 
     - name: Image Acceptance Tests
       uses: cypress-io/github-action@v4
@@ -427,8 +381,8 @@ jobs:
         # on master push latest image
         if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RVERSION }}" == "4.2.0" ]
         then
-          docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_NAME-r:latest
-          docker push $DOCKER_NAME-r:latest
+          docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-r:latest
+          docker push $DOCKER_PREFIX-r:latest
         fi
 
   build-bioc:
@@ -451,26 +405,24 @@ jobs:
 
     - uses: actions/checkout@v3
     - name: Build renku project bioconductor docker images
-      env:
-        DOCKER_PREFIX: renku/renkulab    
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
           export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
         else
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
+        docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
 
         # needed by the makefile
-        export DOCKER_LABEL="${{ matrix.RELEASE }}-$LABEL"
-        export RENKU_BASE="$DOCKER_PREFIX-py:3.10-$LABEL"
+        export BIOC_DOCKER_LABEL="${{ matrix.RELEASE }}-$LABEL"
         export BIOC_VERSION="${{ matrix.RELEASE }}"
         make bioc
-        docker push $DOCKER_PREFIX-bioc:$DOCKER_LABEL
+        docker push $DOCKER_PREFIX-bioc:$BIOC_DOCKER_LABEL
 
         # on master push latest image
         if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RELEASE }}" == "devel" ]
         then
-          docker tag $DOCKER_NAME-bioc:$DOCKER_TAG $DOCKER_NAME-bioc:latest
-          docker push $DOCKER_NAME-bioc:latest
+          docker tag $DOCKER_PREFIX-bioc:$BIOC_DOCKER_LABEL $DOCKER_PREFIX-bioc:latest
+          docker push $DOCKER_PREFIX-bioc:latest
         fi