Implement backup containers health checks

.deploy/backup-retention-policy/.env

@@ -1 +1,2 @@
 BACKUP_RETENTION_POLICY_SCHEDULE="<CRONJOB-SCHEDULE>"
+BACKUP_RETENTION_POLICY_INTERVAL="<CRONJOB-INTERVAL>"

.deploy/backup-retention-policy/Dockerfile

@@ -7,12 +7,27 @@ RUN apk add --no-cache \
 
 COPY .deploy/backup-retention-policy/script.sh /script.sh
 COPY .deploy/backup-retention-policy/entrypoint.sh /entrypoint.sh
+COPY .deploy/backup-retention-policy/healthcheck.sh /healthcheck.sh
 COPY .backup/keep.py /keep.py
 COPY scripts/cronjob /cronjob
+COPY scripts/cronjob-healthcheck /cronjob-healthcheck
 
 ENV \
   NOW="" \
-  BACKUP_RETENTION_POLICY_SCHEDULE=""
+  BACKUP_RETENTION_POLICY_SCHEDULE="" \
+  BACKUP_RETENTION_POLICY_INTERVAL="" \
+  BACKUP_RETENTION_POLICY_CMD="bash /script.sh /keep.py /backups"
+
+HEALTHCHECK \
+  --interval=1h \
+  --timeout=3s \
+  --start-period=15s \
+  --start-interval=5s \
+  --retries=2 \
+  CMD [ \
+    "bash", \
+    "/healthcheck.sh" \
+  ]
 
 CMD [ \
   "bash", \

.deploy/backup-retention-policy/compose.yaml

@@ -7,3 +7,4 @@ services:
       - ${BACKUP_DIR?error}:/backups
     environment:
       - BACKUP_RETENTION_POLICY_SCHEDULE=${BACKUP_RETENTION_POLICY_SCHEDULE?error}
+      - BACKUP_RETENTION_POLICY_INTERVAL=${BACKUP_RETENTION_POLICY_INTERVAL?error}

.deploy/backup-retention-policy/entrypoint.sh

@@ -2,4 +2,4 @@
 
 export CRONJOB_SCHEDULE="${BACKUP_RETENTION_POLICY_SCHEDULE}"
 
-exec bash /cronjob "bash /script.sh /keep.py /backups"
+exec bash /cronjob "${BACKUP_RETENTION_POLICY_CMD}"

.deploy/backup-retention-policy/healthcheck.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+export CRONJOB_SCHEDULE="${BACKUP_RETENTION_POLICY_SCHEDULE}"
+export CRONJOB_INTERVAL="${BACKUP_RETENTION_POLICY_INTERVAL}"
+
+exec bash /cronjob-healthcheck "${BACKUP_RETENTION_POLICY_CMD}" '+ exit 0'

.deploy/backup-retention-policy/script.sh

@@ -16,3 +16,5 @@ DELETE="$(comm -23 <(echo "${ALL}") <(echo "${KEEP}"))"
 for item in ${DELETE} ; do
   rm "${BACKUP_DIR}/${item}"
 done
+
+exit 0

.deploy/backup/.env

@@ -1,3 +1,4 @@
 BACKUP_KEY="<AGE-PUBLIC-KEY>"
 BACKUP_DIR="/path/to/backup/dir"
 BACKUP_SCHEDULE="<CRONJOB-SCHEDULE>"
+BACKUP_INTERVAL="<BACKUP-INTERVAL>"

.deploy/backup/Dockerfile

@@ -7,14 +7,29 @@
   age
 
 COPY .deploy/backup/entrypoint.sh /entrypoint.sh
+COPY .deploy/backup/healthcheck.sh /healthcheck.sh
 COPY .deploy/backup/script.sh /script.sh
 COPY scripts/cronjob /cronjob
+COPY scripts/cronjob-healthcheck /cronjob-healthcheck
 
 ENV \
   MONGO_URL="" \
   BACKUP_SCHEDULE="" \
+  BACKUP_INTERVAL="" \
+  BACKUP_CMD="bash /script.sh /backups" \
   BACKUP_KEY=""
 
+HEALTHCHECK \
+  --interval=1h \
+  --timeout=3s \
+  --start-period=15s \
+  --start-interval=5s \
+  --retries=2 \
+  CMD [ \
+    "bash", \
+    "/healthcheck.sh" \
+  ]
+
 CMD [ \
   "bash", \
   "/entrypoint.sh" \

.deploy/backup/compose.yaml

@@ -6,6 +6,7 @@ services:
     environment:
       - MONGO_URL=${MONGO_URL:-mongodb://patient-db:27017/meteor}
       - BACKUP_SCHEDULE=${BACKUP_SCHEDULE?error}
+      - BACKUP_INTERVAL=${BACKUP_INTERVAL?error}
       - BACKUP_KEY=${BACKUP_KEY?error}
     depends_on:
       patient-db:

.deploy/backup/entrypoint.sh

@@ -2,4 +2,4 @@
 
 export CRONJOB_SCHEDULE="${BACKUP_SCHEDULE}"
 
-exec bash /cronjob "bash /script.sh /backups"
+exec bash /cronjob "${BACKUP_CMD}"

.deploy/backup/healthcheck.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+export CRONJOB_SCHEDULE="${BACKUP_SCHEDULE}"
+export CRONJOB_INTERVAL="${BACKUP_INTERVAL}"
+
+exec bash /cronjob-healthcheck "${BACKUP_CMD}" '+ exit 0'

.deploy/backup/script.sh

@@ -24,6 +24,8 @@ mongodump --uri "${MONGO_URL}" --archive --gzip |
 
 if [ "$?" -eq 0 ] ; then
   echo 'SUCCESS!!'
+  exit 0
 else
   echo 'FAILURE...'
+  exit 1
 fi

.github/actions/docker/container/is-healthy/action.yml

@@ -15,6 +15,11 @@ inputs:
     required: true
     default: 1
 
+  debug:
+    type: boolean
+    default: false
+
+
 runs:
 
   using: composite
@@ -33,3 +38,15 @@ runs:
       run: |
         timeout "${TIMEOUT}" bash -c \
          'until docker inspect --format "{{json .State.Health}}" "${NAME_OR_ID}" | jq -e ".Status == \"healthy\"" ; do sleep "${POLLING_INTERVAL}"; done'
+
+    - name: Debug HEALTHCHECK
+
+      if: fromJSON(inputs.debug) && failure()
+
+      env:
+        NAME_OR_ID: ${{ inputs.container }}
+
+      shell: bash
+
+      run: |
+         docker inspect --format "{{json .State.Health}}" "${NAME_OR_ID}" | jq

.github/workflows/ci:build:image:backup-retention-policy.yml

@@ -70,6 +70,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           sparse-checkout: |
+            .github/actions/docker/container/is-healthy/action.yml
             .github/actions/docker/container/is-running/action.yml
           sparse-checkout-cone-mode: false
 
@@ -114,12 +115,14 @@ jobs:
         env:
           NOW: 2023-02-10_12:34:56
           BACKUP_RETENTION_POLICY_SCHEDULE: ${{ steps.configure.outputs.schedule }}
+          BACKUP_RETENTION_POLICY_INTERVAL: 120
         run: |
           docker container run \
             --init \
             --detach \
             --env "NOW=${NOW}" \
             --env "BACKUP_RETENTION_POLICY_SCHEDULE=${BACKUP_RETENTION_POLICY_SCHEDULE}" \
+            --env "BACKUP_RETENTION_POLICY_INTERVAL=${BACKUP_RETENTION_POLICY_INTERVAL}" \
             --mount type=bind,src="${GITHUB_WORKSPACE}/backups/",dst=/backups \
             --name backup-retention-policy-container \
             ${{ steps.image-url.outputs.url }}
@@ -130,6 +133,13 @@ jobs:
           container: backup-retention-policy-container
           timeout: 60
 
+      - name: Wait for container to be healthy
+        uses: ./.github/actions/docker/container/is-healthy
+        with:
+          container: backup-retention-policy-container
+          timeout: 15
+          debug: true
+
       - name: Show docker containers
         if: always()
         run: |

.github/workflows/ci:build:image:backup.yml

@@ -140,6 +140,7 @@ jobs:
         env:
           MONGO_URL: mongodb://mongodb:27017/meteor
           BACKUP_SCHEDULE: ${{ steps.configure.outputs.schedule }}
+          BACKUP_INTERVAL: 120
           BACKUP_KEY: ${{ steps.configure.outputs.backup-key }}
         run: |
           docker container run \
@@ -148,6 +149,7 @@ jobs:
             --network patient-backup-network \
             --env "MONGO_URL=${MONGO_URL}" \
             --env "BACKUP_SCHEDULE=${BACKUP_SCHEDULE}" \
+            --env "BACKUP_INTERVAL=${BACKUP_INTERVAL}" \
             --env "BACKUP_KEY=${BACKUP_KEY}" \
             --mount type=bind,src="${GITHUB_WORKSPACE}/backups/",dst=/backups \
             --name backup-container \
@@ -159,6 +161,13 @@ jobs:
           container: backup-container
           timeout: 60
 
+      - name: Wait for backup container to be healthy
+        uses: ./.github/actions/docker/container/is-healthy
+        with:
+          container: backup-container
+          timeout: 15
+          debug: true
+
       - name: Show docker containers
         if: always()
         run: |

scripts/cronjob

@@ -12,12 +12,12 @@ if [ -z "${CMD}" ] ; then
 	exit 1
 fi
 
-LOGS="/var/log/cron.log"
+LOG="/var/log/cron.log"
 
-touch "${LOGS}"
+touch "${LOG}"
 
 printenv | grep -v 'no_proxy' >> /etc/environment
 
-echo "${CRONJOB_SCHEDULE} ${CMD} >> ${LOGS} 2>&1" > /etc/crontabs/root
+echo "${CRONJOB_SCHEDULE} ${CMD} >> ${LOG} 2>&1" > /etc/crontabs/root
 
-crond && tail -f "${LOGS}"
+crond && tail -f "${LOG}"

scripts/cronjob-healthcheck

@@ -0,0 +1,103 @@
+#!/usr/bin/env sh
+
+if [ -z "${CRONJOB_INTERVAL}" ] ; then
+	 >&2 printf '%s: %s\n' "${0}" 'CRONJOB_INTERVAL not set.'
+	exit 1
+fi
+
+if [ -z "${CRONJOB_SCHEDULE}" ] ; then
+	 >&2 printf '%s: %s\n' "${0}" 'CRONJOB_SCHEDULE not set.'
+	exit 1
+fi
+
+CMD="${1}"
+
+if [ -z "${CMD}" ] ; then
+	 >&2 printf '%s: %s\n' "${0}" 'CMD not set.'
+	exit 1
+fi
+
+CMD_SUCCESS_LOG_LINE="${2}"
+
+if [ -z "${CMD_SUCCESS_LOG_LINE}" ] ; then
+	 >&2 printf '%s: %s\n' "${0}" 'CMD_SUCCESS_LOG_LINE not set.'
+	exit 1
+fi
+
+LOG="/var/log/cron.log"
+PID_FILE="/var/run/crond.pid"
+
+
+# NOTE: Check that crond is configured properly.
+
+CRONTAB="$(crontab -l | head -n2)"
+CRONTAB_ENTRIES="$(wc -l <<< "${CRONTAB}")"
+
+if [ "${CRONTAB_ENTRIES}" -ne 1 ] ; then
+	>&2 printf 'Expected 1 crontab entry, got %d:\n' "${CRONTAB_ENTRIES}"
+	>&2 cat <<< "${CRONTAB}"
+	exit 1
+fi
+
+EXPECTED_ENTRY="${CRONJOB_SCHEDULE} ${CMD} >> ${LOG} 2>&1"
+
+if [ "${CRONTAB}" != "${EXPECTED_ENTRY}" ] ; then
+	>&2 printf '%s\n' 'Expected crontab to be:'
+	>&2 printf '%s\n' "${EXPECTED_ENTRY}"
+	>&2 printf '%s\n' 'Got:'
+	>&2 cat <<< "${CRONTAB}"
+	exit 1
+fi
+
+
+# NOTE: Check that crond is running.
+
+PID="$(pgrep crond)"
+
+if [ -z "${PID}" ] ; then
+	>&2 printf '%s\n' 'Could not find crond process.'
+	exit 1
+fi
+
+LAST_PID="$(cat "${PID_FILE}")"
+
+if [ -z "${LAST_PID}" ] ; then
+	>&2 printf '%s\n' 'Could not read pid file for crond process.'
+	exit 1
+fi
+
+if [ "${PID}" -ne "${LAST_PID}" ] ; then
+	>&2 printf 'Expected pid file to contain %d, got %d.\n' "${PID}" "${LAST_PID}"
+	exit 1
+fi
+
+
+# NOTE: Check that cronjob has run recently enough.
+
+NOW="$(date +'%s')"
+LAST_MODIFIED_AT="$(stat "${LOG}" -c %Y)"
+ELAPSED="$((${NOW} - ${LAST_MODIFIED_AT}))"
+
+if [ "${ELAPSED}" -ge "${CRONJOB_INTERVAL}" ] ; then
+	>&2 printf 'Time is now: %s.\n' "$(date -d@"${NOW}")"
+	>&2 printf 'Cronjob last ran %d seconds ago, at %s.\n' "${ELAPSED}" "$(date -d@"${LAST_MODIFIED_AT}")"
+	>&2 printf 'Expected cronjob to run every %d seconds.\n' "${CRONJOB_INTERVAL}"
+	exit 1
+fi
+
+
+# NOTE: Check that cronjob is not erroring.
+
+LAST_LOG_LINE="$(tail -1 "${LOG}")"
+LOG_LINES_COUNT="$(tail -1 "${LOG}" | wc -l)"
+
+if [ "${LOG_LINES_COUNT}" -ge 1 ] && [ "${LAST_LOG_LINE}" != "${CMD_SUCCESS_LOG_LINE}" ] ; then
+	>&2 printf '%s\n' 'Expected last log line to be:'
+	>&2 printf '%s\n' "${CMD_SUCCESS_LOG_LINE}"
+	>&2 printf '%s\n' 'Got:'
+	>&2 printf '%s\n' "${LAST_LOG_LINE}"
+	exit 1
+fi
+
+
+exit 0