[minor] add Dockerfile healthchecks to some core services

activemq/Dockerfile

@@ -36,3 +36,9 @@ COPY --link rootfs /
 
 RUN create-service-user.sh --name activemq && \
     cleanup.sh
+
+HEALTHCHECK CMD curl -s \
+    -u admin:"$(cat /var/run/s6/container_environment/ACTIVEMQ_WEB_ADMIN_PASSWORD)" \
+    -H origin:localhost \
+    "http://localhost:8161/api/jolokia/read/org.apache.activemq:type=Broker,brokerName=localhost,service=Health/CurrentStatus" \
+    | jq .value | grep -q Good

activemq/tests/ServiceHealthcheck/build.gradle.kts

@@ -0,0 +1,54 @@
+import plugins.TestsPlugin.DockerComposeUp
+import plugins.TestsPlugin.DockerComposeUp.Companion.pool
+import java.lang.Thread.sleep
+import java.time.Duration.ofSeconds
+import java.util.concurrent.CompletableFuture.supplyAsync
+import java.io.ByteArrayOutputStream
+
+tasks.named<DockerComposeUp>("test") {
+    doFirst {
+        supplyAsync(
+            {
+                val maxAttempts = 10
+                val delayBetweenAttempts = 5000L // 5 seconds in milliseconds
+                var attempt = 0
+                var foundHealthyService = false
+
+                while (attempt < maxAttempts) {
+                    attempt++
+                    val outputStream = ByteArrayOutputStream()
+                    project.exec {
+                        commandLine = baseArguments + listOf("ps", "--all")
+                        standardOutput = outputStream
+                        workingDir = project.projectDir
+                    }
+                    val output = outputStream.toString()
+
+                    val healthyServicePattern = """(?m)^.+\s+.+\s+Up \d+ seconds \(healthy\).*$""".toRegex()
+                    foundHealthyService = output.lines().any { line ->
+                        healthyServicePattern.matches(line)
+                    }
+
+                    if (foundHealthyService) {
+                        project.exec {
+                            commandLine = baseArguments + listOf("stop")
+                            standardOutput = outputStream
+                            workingDir = project.projectDir
+                        }
+                        break
+                    }
+
+                    if (attempt < maxAttempts) {
+                        println("No healthy service found. Retrying in ${delayBetweenAttempts / 1000} seconds...")
+                        sleep(delayBetweenAttempts)
+                    }
+                }
+
+                // Throw an exception if no healthy service was found after all attempts
+                if (!foundHealthyService) {
+                    throw GradleException("No service is marked as healthy in docker compose ps output after $maxAttempts attempts.")
+                }
+            }, pool
+        )
+    }
+}

activemq/tests/ServiceHealthcheck/docker-compose.yml

@@ -0,0 +1,14 @@
+---
+
+# Common to all services
+x-common: &common
+  restart: "no"
+
+services:
+  activemq:
+    <<: *common
+    image: ${ACTIVEMQ:-islandora/activemq:local}
+    volumes:
+      - ./test.sh:/test.sh # Test to run.
+    command:
+      - /test.sh # Run test and exit.

activemq/tests/ServiceHealthcheck/test.sh

@@ -0,0 +1,13 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+
+on_terminate() {
+    echo "Termination signal received. Exiting..."
+    exit 0
+}
+trap 'on_terminate' SIGTERM
+
+sleep 60
+
+# The kotlin check should be stopping this container
+exit 1

cantaloupe/Dockerfile

@@ -83,9 +83,9 @@ ENV \
     CANTALOUPE_ENDPOINT_ADMIN_ENABLED="false" \
     CANTALOUPE_ENDPOINT_ADMIN_SECRET= \
     CANTALOUPE_ENDPOINT_ADMIN_USERNAME="admin" \
-    CANTALOUPE_ENDPOINT_API_ENABLED="false" \
+    CANTALOUPE_ENDPOINT_API_ENABLED="true" \
     CANTALOUPE_ENDPOINT_API_SECRET= \
-    CANTALOUPE_ENDPOINT_API_USERNAME= \
+    CANTALOUPE_ENDPOINT_API_USERNAME=islandora \
     CANTALOUPE_ENDPOINT_HEALTH_DEPENDENCY_CHECK="false" \
     CANTALOUPE_ENDPOINT_IIIF_1_ENABLED="false" \
     CANTALOUPE_ENDPOINT_IIIF_2_ENABLED="true" \
@@ -271,3 +271,7 @@ ENV \
     CANTALOUPE_TEMP_PATHNAME=
 
 COPY --link rootfs /
+
+HEALTHCHECK CMD curl -s \
+  -u "${CANTALOUPE_ENDPOINT_API_USERNAME}:$(cat /var/run/s6/container_environment/CANTALOUPE_ENDPOINT_API_SECRET)" \
+  http://localhost:8182/health | jq .color | grep -q GREEN

cantaloupe/README.md

@@ -65,9 +65,9 @@ additional settings, volumes, ports, etc.
 | CANTALOUPE_ENDPOINT_ADMIN_ENABLED                                                      | "false"                                                          |
 | CANTALOUPE_ENDPOINT_ADMIN_SECRET                                                       |                                                                  |
 | CANTALOUPE_ENDPOINT_ADMIN_USERNAME                                                     | "admin"                                                          |
-| CANTALOUPE_ENDPOINT_API_ENABLED                                                        | "false"                                                          |
-| CANTALOUPE_ENDPOINT_API_SECRET                                                         |                                                                  |
-| CANTALOUPE_ENDPOINT_API_USERNAME                                                       |                                                                  |
+| CANTALOUPE_ENDPOINT_API_ENABLED                                                        | "true"                                                           |
+| CANTALOUPE_ENDPOINT_API_SECRET                                                         | random 16 char string                                            |
+| CANTALOUPE_ENDPOINT_API_USERNAME                                                       | "islandora"                                                      |
 | CANTALOUPE_ENDPOINT_HEALTH_DEPENDENCY_CHECK                                            | "false"                                                          |
 | CANTALOUPE_ENDPOINT_IIIF_1_ENABLED                                                     | "false"                                                          |
 | CANTALOUPE_ENDPOINT_IIIF_2_ENABLED                                                     | "true"                                                           |

cantaloupe/rootfs/etc/cont-init.d/01-set-api-secret.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/with-contenv bash
+set -e
+
+ENV_FILE="/var/run/s6/container_environment/CANTALOUPE_ENDPOINT_API_SECRET"
+if [ ! -s "$ENV_FILE" ]; then
+    openssl rand -hex 16 > "$ENV_FILE"
+    echo "CANTALOUPE_ENDPOINT_API_SECRET was empty. Set to a new random value."
+fi

cantaloupe/tests/ServiceHealthcheck/build.gradle.kts

@@ -0,0 +1,54 @@
+import plugins.TestsPlugin.DockerComposeUp
+import plugins.TestsPlugin.DockerComposeUp.Companion.pool
+import java.lang.Thread.sleep
+import java.time.Duration.ofSeconds
+import java.util.concurrent.CompletableFuture.supplyAsync
+import java.io.ByteArrayOutputStream
+
+tasks.named<DockerComposeUp>("test") {
+    doFirst {
+        supplyAsync(
+            {
+                val maxAttempts = 10
+                val delayBetweenAttempts = 5000L // 5 seconds in milliseconds
+                var attempt = 0
+                var foundHealthyService = false
+
+                while (attempt < maxAttempts) {
+                    attempt++
+                    val outputStream = ByteArrayOutputStream()
+                    project.exec {
+                        commandLine = baseArguments + listOf("ps", "--all")
+                        standardOutput = outputStream
+                        workingDir = project.projectDir
+                    }
+                    val output = outputStream.toString()
+
+                    val healthyServicePattern = """(?m)^.+\s+.+\s+Up \d+ seconds \(healthy\).*$""".toRegex()
+                    foundHealthyService = output.lines().any { line ->
+                        healthyServicePattern.matches(line)
+                    }
+
+                    if (foundHealthyService) {
+                        project.exec {
+                            commandLine = baseArguments + listOf("stop")
+                            standardOutput = outputStream
+                            workingDir = project.projectDir
+                        }
+                        break
+                    }
+
+                    if (attempt < maxAttempts) {
+                        println("No healthy service found. Retrying in ${delayBetweenAttempts / 1000} seconds...")
+                        sleep(delayBetweenAttempts)
+                    }
+                }
+
+                // Throw an exception if no healthy service was found after all attempts
+                if (!foundHealthyService) {
+                    throw GradleException("No service is marked as healthy in docker compose ps output after $maxAttempts attempts.")
+                }
+            }, pool
+        )
+    }
+}

cantaloupe/tests/ServiceHealthcheck/docker-compose.yml

@@ -0,0 +1,14 @@
+---
+
+# Common to all services
+x-common: &common
+  restart: "no"
+
+services:
+  cantaloupe:
+    <<: *common
+    image: ${CANTALOUPE:-islandora/cantaloupe:local}
+    volumes:
+      - ./test.sh:/test.sh # Test to run.
+    command:
+      - /test.sh # Run test and exit.

cantaloupe/tests/ServiceHealthcheck/test.sh

@@ -0,0 +1,13 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+
+on_terminate() {
+    echo "Termination signal received. Exiting..."
+    exit 0
+}
+trap 'on_terminate' SIGTERM
+
+sleep 60
+
+# The kotlin check should be stopping this container
+exit 1

ci/healthcheck.gradle.kts

@@ -0,0 +1,55 @@
+import plugins.TestsPlugin.DockerComposeUp
+import plugins.TestsPlugin.DockerComposeUp.Companion.pool
+import java.lang.Thread.sleep
+import java.time.Duration.ofSeconds
+import java.util.concurrent.CompletableFuture.supplyAsync
+import java.io.ByteArrayOutputStream
+
+tasks.named<DockerComposeUp>("test") {
+    doFirst {
+        supplyAsync(
+            {
+                val maxAttempts = 10
+                val delayBetweenAttempts = 5000L // 5 seconds in milliseconds
+                var attempt = 0
+                var foundHealthyService = false
+
+                while (attempt < maxAttempts) {
+                    attempt++
+                    val outputStream = ByteArrayOutputStream()
+                    project.exec {
+                        commandLine = baseArguments + listOf("ps", "--all")
+                        standardOutput = outputStream
+                        workingDir = project.projectDir
+                    }
+                    val output = outputStream.toString()
+
+                    val healthyServicePattern = """(?m)^.+\s+.+\s+Up \d+ seconds \(healthy\).*$""".toRegex()
+                    foundHealthyService = output.lines().any { line ->
+                        healthyServicePattern.matches(line)
+                    }
+
+                    if (foundHealthyService) {
+                        println("Service is healthy. Exiting test...")
+                        project.exec {
+                            commandLine = baseArguments + listOf("stop")
+                            standardOutput = outputStream
+                            workingDir = project.projectDir
+                        }
+                        break
+                    }
+
+                    if (attempt < maxAttempts) {
+                        println("No healthy service found. Retrying in ${delayBetweenAttempts / 1000} seconds...")
+                        sleep(delayBetweenAttempts)
+                    }
+                }
+
+                // Throw an exception if no healthy service was found after all attempts
+                if (!foundHealthyService) {
+                    throw GradleException("No service is marked as healthy in docker compose ps output after $maxAttempts attempts.")
+                }
+            }, pool
+        )
+    }
+}

fcrepo6/Dockerfile

@@ -70,3 +70,5 @@ ENV \
 COPY --link rootfs /
 
 RUN chown -R tomcat:tomcat /opt/tomcat
+
+HEALTHCHECK CMD curl -s http://localhost:8080/fcrepo/rest/fcr:systeminfo | jq -e .version

fcrepo6/tests/ServiceHealthcheck/build.gradle.kts

@@ -0,0 +1,54 @@
+import plugins.TestsPlugin.DockerComposeUp
+import plugins.TestsPlugin.DockerComposeUp.Companion.pool
+import java.lang.Thread.sleep
+import java.time.Duration.ofSeconds
+import java.util.concurrent.CompletableFuture.supplyAsync
+import java.io.ByteArrayOutputStream
+
+tasks.named<DockerComposeUp>("test") {
+    doFirst {
+        supplyAsync(
+            {
+                val maxAttempts = 10
+                val delayBetweenAttempts = 5000L // 5 seconds in milliseconds
+                var attempt = 0
+                var foundHealthyService = false
+
+                while (attempt < maxAttempts) {
+                    attempt++
+                    val outputStream = ByteArrayOutputStream()
+                    project.exec {
+                        commandLine = baseArguments + listOf("ps", "--all")
+                        standardOutput = outputStream
+                        workingDir = project.projectDir
+                    }
+                    val output = outputStream.toString()
+
+                    val healthyServicePattern = """(?m)^.+\s+.+\s+Up \d+ seconds \(healthy\).*$""".toRegex()
+                    foundHealthyService = output.lines().any { line ->
+                        healthyServicePattern.matches(line)
+                    }
+
+                    if (foundHealthyService) {
+                        project.exec {
+                            commandLine = baseArguments + listOf("stop")
+                            standardOutput = outputStream
+                            workingDir = project.projectDir
+                        }
+                        break
+                    }
+
+                    if (attempt < maxAttempts) {
+                        println("No healthy service found. Retrying in ${delayBetweenAttempts / 1000} seconds...")
+                        sleep(delayBetweenAttempts)
+                    }
+                }
+
+                // Throw an exception if no healthy service was found after all attempts
+                if (!foundHealthyService) {
+                    throw GradleException("No service is marked as healthy in docker compose ps output after $maxAttempts attempts.")
+                }
+            }, pool
+        )
+    }
+}

fcrepo6/tests/ServiceHealthcheck/docker-compose.yml

@@ -0,0 +1,18 @@
+---
+
+# Common to all services
+x-common: &common
+  restart: "no"
+
+services:
+  activemq:
+    <<: *common
+    image: ${ACTIVEMQ:-islandora/activemq:local}
+  fcrepo6:
+    volumes:
+      - ./test.sh:/test.sh # Test to run.
+    command:
+      - /test.sh # Run test and exit.
+    image: ${FCREPO6:-islandora/fcrepo6:local}
+    depends_on:
+      - activemq

fcrepo6/tests/ServiceHealthcheck/test.sh

@@ -0,0 +1,13 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+
+on_terminate() {
+    echo "Termination signal received. Exiting..."
+    exit 0
+}
+trap 'on_terminate' SIGTERM
+
+sleep 60
+
+# The kotlin check should be stopping this container
+exit 1

mariadb/Dockerfile

@@ -33,3 +33,5 @@ ENV \
     MYSQL_TRANSACTION_ISOLATION=READ-COMMITTED
 
 COPY --link rootfs /
+
+HEALTHCHECK CMD mysqladmin ping --socket=/var/run/mysqld/mysqld.sock | grep -q alive

mariadb/tests/ServiceHealthcheck/build.gradle.kts

@@ -0,0 +1 @@
+../../../ci/healthcheck.gradle.kts

mariadb/tests/ServiceHealthcheck/docker-compose.yml

@@ -0,0 +1,14 @@
+---
+
+# Common to all services
+x-common: &common
+  restart: "no"
+
+services:
+  mariadb:
+    <<: *common
+    image: ${MARIADB:-islandora/mariadb:local}
+    volumes:
+      - ./test.sh:/test.sh # Test to run.
+    command:
+      - /test.sh # Run test and exit.