From 070675d8d61cb6075af01929ecb57efd57a680af Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Fri, 28 Jun 2024 19:21:57 +0100 Subject: [PATCH 1/6] Make `HealthCheckWaitStrategy` event based --- ...ker-compose-with-healthcheck-unhealthy.yml | 12 ++++++ .../docker-compose-environment.test.ts | 8 ++++ .../health-check-wait-strategy.ts | 43 ++++++++++++------- 3 files changed, 48 insertions(+), 15 deletions(-) create mode 100644 packages/testcontainers/fixtures/docker-compose/docker-compose-with-healthcheck-unhealthy.yml diff --git a/packages/testcontainers/fixtures/docker-compose/docker-compose-with-healthcheck-unhealthy.yml b/packages/testcontainers/fixtures/docker-compose/docker-compose-with-healthcheck-unhealthy.yml new file mode 100644 index 000000000..bc847e816 --- /dev/null +++ b/packages/testcontainers/fixtures/docker-compose/docker-compose-with-healthcheck-unhealthy.yml @@ -0,0 +1,12 @@ +version: "3.5" + +services: + container: + image: cristianrgreco/testcontainer:1.1.14 + ports: + - 8080 + healthcheck: + test: "curl -f http://localhost:8081/hello-world || exit 1" + interval: 1s + timeout: 3s + retries: 10 diff --git a/packages/testcontainers/src/docker-compose-environment/docker-compose-environment.test.ts b/packages/testcontainers/src/docker-compose-environment/docker-compose-environment.test.ts index ed79a0e24..68cf14417 100644 --- a/packages/testcontainers/src/docker-compose-environment/docker-compose-environment.test.ts +++ b/packages/testcontainers/src/docker-compose-environment/docker-compose-environment.test.ts @@ -123,6 +123,14 @@ describe("DockerComposeEnvironment", () => { await startedEnvironment.down(); }); + it("should support failing health check wait strategy", async () => { + await expect( + new DockerComposeEnvironment(fixtures, "docker-compose-with-healthcheck-unhealthy.yml") + .withWaitStrategy(await composeContainerName("container"), Wait.forHealthCheck()) + .up() + ).rejects.toThrow(`Health check failed: unhealthy`); + }); + if (!process.env.CI_PODMAN) { it("should stop the container when the health check wait strategy times out", async () => { await expect( diff --git a/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts b/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts index bc2bad018..11024075b 100644 --- a/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts +++ b/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts @@ -1,31 +1,44 @@ import Dockerode from "dockerode"; import { AbstractWaitStrategy } from "./wait-strategy"; -import { IntervalRetry, log } from "../common"; +import { log } from "../common"; import { getContainerRuntimeClient } from "../container-runtime"; export class HealthCheckWaitStrategy extends AbstractWaitStrategy { public async waitUntilReady(container: Dockerode.Container): Promise { log.debug(`Waiting for health check...`, { containerId: container.id }); + const client = await getContainerRuntimeClient(); + const dockerode = client.container.dockerode; + const events = await dockerode.getEvents({ + filters: { + type: ["container"], + container: [container.id], + event: ["health_status"], + }, + }); - const status = await new IntervalRetry(100).retryUntil( - async () => (await client.container.inspect(container)).State.Health?.Status, - (healthCheckStatus) => healthCheckStatus === "healthy" || healthCheckStatus === "unhealthy", - () => { + return new Promise((resolve, reject) => { + setTimeout(() => { const timeout = this.startupTimeout; const message = `Health check not healthy after ${timeout}ms`; log.error(message, { containerId: container.id }); - throw new Error(message); - }, - this.startupTimeout - ); + reject(new Error(message)); + }, this.startupTimeout); + + events.on("data", (data) => { + const parsedData = JSON.parse(data); + const status = parsedData.status.split(":").pop().trim(); - if (status !== "healthy") { - const message = `Health check failed: ${status}`; - log.error(message, { containerId: container.id }); - throw new Error(message); - } + if (status === "healthy") { + resolve(); + } else { + const message = `Health check failed: ${status}`; + log.error(message, { containerId: container.id }); + reject(new Error(message)); + } - log.debug(`Health check wait strategy complete`, { containerId: container.id }); + log.debug(`Health check wait strategy complete`, { containerId: container.id }); + }); + }); } } From aac5ca2e8424b396c92724b4be91d0e900fb7ccd Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Thu, 18 Jul 2024 13:12:31 +0100 Subject: [PATCH 2/6] Cancel timeout and events on completion --- .../wait-strategies/health-check-wait-strategy.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts b/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts index 11024075b..0f0ca8c48 100644 --- a/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts +++ b/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts @@ -2,6 +2,7 @@ import Dockerode from "dockerode"; import { AbstractWaitStrategy } from "./wait-strategy"; import { log } from "../common"; import { getContainerRuntimeClient } from "../container-runtime"; +import { Readable } from "stream"; export class HealthCheckWaitStrategy extends AbstractWaitStrategy { public async waitUntilReady(container: Dockerode.Container): Promise { @@ -9,19 +10,19 @@ export class HealthCheckWaitStrategy extends AbstractWaitStrategy { const client = await getContainerRuntimeClient(); const dockerode = client.container.dockerode; - const events = await dockerode.getEvents({ + const events = (await dockerode.getEvents({ filters: { type: ["container"], container: [container.id], event: ["health_status"], }, - }); + })) as Readable; return new Promise((resolve, reject) => { - setTimeout(() => { - const timeout = this.startupTimeout; - const message = `Health check not healthy after ${timeout}ms`; + const timeout = setTimeout(() => { + const message = `Health check not healthy after ${this.startupTimeout}ms`; log.error(message, { containerId: container.id }); + events.destroy(); reject(new Error(message)); }, this.startupTimeout); @@ -37,6 +38,8 @@ export class HealthCheckWaitStrategy extends AbstractWaitStrategy { reject(new Error(message)); } + clearTimeout(timeout); + events.destroy(); log.debug(`Health check wait strategy complete`, { containerId: container.id }); }); }); From dd6fba9f9f2d313c7311526f17018a1bb9492fd7 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Thu, 18 Jul 2024 13:24:20 +0100 Subject: [PATCH 3/6] Add support for fetching event streams to the container runtime --- .../clients/container/container-client.ts | 1 + .../clients/container/docker-container-client.ts | 15 ++++++++++++++- .../health-check-wait-strategy.ts | 16 ++++------------ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/packages/testcontainers/src/container-runtime/clients/container/container-client.ts b/packages/testcontainers/src/container-runtime/clients/container/container-client.ts index e19eab593..9786c68c4 100644 --- a/packages/testcontainers/src/container-runtime/clients/container/container-client.ts +++ b/packages/testcontainers/src/container-runtime/clients/container/container-client.ts @@ -24,6 +24,7 @@ export interface ContainerClient { logs(container: Container, opts?: ContainerLogsOptions): Promise; exec(container: Container, command: string[], opts?: Partial): Promise; restart(container: Container, opts?: { timeout: number }): Promise; + events(container: Container, eventNames: string[]): Promise; remove(container: Container, opts?: { removeVolumes: boolean }): Promise; connectToNetwork(container: Container, network: Network, networkAliases: string[]): Promise; } diff --git a/packages/testcontainers/src/container-runtime/clients/container/docker-container-client.ts b/packages/testcontainers/src/container-runtime/clients/container/docker-container-client.ts index 0f560fc8e..9b413c36d 100644 --- a/packages/testcontainers/src/container-runtime/clients/container/docker-container-client.ts +++ b/packages/testcontainers/src/container-runtime/clients/container/docker-container-client.ts @@ -12,7 +12,7 @@ import { IncomingMessage } from "http"; import { ExecOptions, ExecResult } from "./types"; import byline from "byline"; import { ContainerClient } from "./container-client"; -import { log, execLog, streamToString } from "../../../common"; +import { execLog, log, streamToString } from "../../../common"; export class DockerContainerClient implements ContainerClient { constructor(public readonly dockerode: Dockerode) {} @@ -248,6 +248,19 @@ export class DockerContainerClient implements ContainerClient { } } + async events(container: Container, eventNames: string[]): Promise { + log.debug(`Fetching event stream...`, { containerId: container.id }); + const stream = (await this.dockerode.getEvents({ + filters: { + type: ["container"], + container: [container.id], + event: eventNames, + }, + })) as Readable; + log.debug(`Fetched event stream...`, { containerId: container.id }); + return stream; + } + protected async demuxStream(containerId: string, stream: Readable): Promise { try { log.debug(`Demuxing stream...`, { containerId }); diff --git a/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts b/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts index 0f0ca8c48..212ba46da 100644 --- a/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts +++ b/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts @@ -2,31 +2,23 @@ import Dockerode from "dockerode"; import { AbstractWaitStrategy } from "./wait-strategy"; import { log } from "../common"; import { getContainerRuntimeClient } from "../container-runtime"; -import { Readable } from "stream"; export class HealthCheckWaitStrategy extends AbstractWaitStrategy { public async waitUntilReady(container: Dockerode.Container): Promise { log.debug(`Waiting for health check...`, { containerId: container.id }); const client = await getContainerRuntimeClient(); - const dockerode = client.container.dockerode; - const events = (await dockerode.getEvents({ - filters: { - type: ["container"], - container: [container.id], - event: ["health_status"], - }, - })) as Readable; + const containerEvents = await client.container.events(container, ["health_status"]); return new Promise((resolve, reject) => { const timeout = setTimeout(() => { const message = `Health check not healthy after ${this.startupTimeout}ms`; log.error(message, { containerId: container.id }); - events.destroy(); + containerEvents.destroy(); reject(new Error(message)); }, this.startupTimeout); - events.on("data", (data) => { + containerEvents.on("data", (data) => { const parsedData = JSON.parse(data); const status = parsedData.status.split(":").pop().trim(); @@ -39,7 +31,7 @@ export class HealthCheckWaitStrategy extends AbstractWaitStrategy { } clearTimeout(timeout); - events.destroy(); + containerEvents.destroy(); log.debug(`Health check wait strategy complete`, { containerId: container.id }); }); }); From 23cedbf47ff7f734bb87ca1ab2543df80ec207a9 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Thu, 18 Jul 2024 14:26:29 +0100 Subject: [PATCH 4/6] Fixes for Podman --- .../src/wait-strategies/health-check-wait-strategy.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts b/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts index 212ba46da..0c558ff16 100644 --- a/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts +++ b/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts @@ -20,7 +20,11 @@ export class HealthCheckWaitStrategy extends AbstractWaitStrategy { containerEvents.on("data", (data) => { const parsedData = JSON.parse(data); - const status = parsedData.status.split(":").pop().trim(); + + const status = + parsedData.status.split(":").length === 2 + ? parsedData.status.split(":").pop().trim() // Docker + : parsedData.HealthStatus; // Podman if (status === "healthy") { resolve(); From af68ee29be199a9ab30eab779997cdebc48e365c Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Thu, 18 Jul 2024 14:27:20 +0100 Subject: [PATCH 5/6] Minor improvement --- .../src/wait-strategies/health-check-wait-strategy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts b/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts index 0c558ff16..51e734569 100644 --- a/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts +++ b/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts @@ -23,7 +23,7 @@ export class HealthCheckWaitStrategy extends AbstractWaitStrategy { const status = parsedData.status.split(":").length === 2 - ? parsedData.status.split(":").pop().trim() // Docker + ? parsedData.status.split(":")[1].trim() // Docker : parsedData.HealthStatus; // Podman if (status === "healthy") { From b1da2dbc02527c2906117bae0b09ac58d8fb743d Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Thu, 18 Jul 2024 20:04:30 +0100 Subject: [PATCH 6/6] Podman emits events for `starting` --- .../docker-compose-with-healthcheck-unhealthy.yml | 4 ++-- .../wait-strategies/health-check-wait-strategy.ts | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/testcontainers/fixtures/docker-compose/docker-compose-with-healthcheck-unhealthy.yml b/packages/testcontainers/fixtures/docker-compose/docker-compose-with-healthcheck-unhealthy.yml index bc847e816..4e33bdbe2 100644 --- a/packages/testcontainers/fixtures/docker-compose/docker-compose-with-healthcheck-unhealthy.yml +++ b/packages/testcontainers/fixtures/docker-compose/docker-compose-with-healthcheck-unhealthy.yml @@ -7,6 +7,6 @@ services: - 8080 healthcheck: test: "curl -f http://localhost:8081/hello-world || exit 1" - interval: 1s timeout: 3s - retries: 10 + interval: 1s + retries: 0 diff --git a/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts b/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts index 51e734569..ced92d562 100644 --- a/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts +++ b/packages/testcontainers/src/wait-strategies/health-check-wait-strategy.ts @@ -18,6 +18,12 @@ export class HealthCheckWaitStrategy extends AbstractWaitStrategy { reject(new Error(message)); }, this.startupTimeout); + const onTerminalState = () => { + clearTimeout(timeout); + containerEvents.destroy(); + log.debug(`Health check wait strategy complete`, { containerId: container.id }); + }; + containerEvents.on("data", (data) => { const parsedData = JSON.parse(data); @@ -28,15 +34,13 @@ export class HealthCheckWaitStrategy extends AbstractWaitStrategy { if (status === "healthy") { resolve(); - } else { + onTerminalState(); + } else if (status === "unhealthy") { const message = `Health check failed: ${status}`; log.error(message, { containerId: container.id }); reject(new Error(message)); + onTerminalState(); } - - clearTimeout(timeout); - containerEvents.destroy(); - log.debug(`Health check wait strategy complete`, { containerId: container.id }); }); }); }