From aa1f95ab4b05bf764c236559782229b677ffcdee Mon Sep 17 00:00:00 2001 From: tomco Date: Wed, 15 Jun 2022 21:16:28 +0200 Subject: [PATCH 1/4] Add Experimental LogMessageWaitStrategy This wait strategy has following characteristics - Validates the regex which has been configured with the entire log history. It does this by using a StringBuffer (thread safety) to concat everything together. - Logic for "x amount of times" or complex setups such as "(a OR b) AND c" is fully to be implemented by regex. - "times" is removed from the class interface. Risks: - Slower for extremely long logs lines. --- .../HoggingLogMessageWaitStrategy.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 core/src/main/java/org/testcontainers/containers/wait/experimental/HoggingLogMessageWaitStrategy.java diff --git a/core/src/main/java/org/testcontainers/containers/wait/experimental/HoggingLogMessageWaitStrategy.java b/core/src/main/java/org/testcontainers/containers/wait/experimental/HoggingLogMessageWaitStrategy.java new file mode 100644 index 00000000000..6d908b8ad75 --- /dev/null +++ b/core/src/main/java/org/testcontainers/containers/wait/experimental/HoggingLogMessageWaitStrategy.java @@ -0,0 +1,76 @@ +package org.testcontainers.containers.wait.experimental; + +import com.github.dockerjava.api.command.LogContainerCmd; +import lombok.SneakyThrows; +import org.testcontainers.containers.ContainerLaunchException; +import org.testcontainers.containers.output.FrameConsumerResultCallback; +import org.testcontainers.containers.output.OutputFrame; +import org.testcontainers.containers.output.WaitingConsumer; +import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; + +/* + This wait strategy has following characteristics + - Validates the regex which has been configured with the entire log history. It does this by using a StringBuffer (thread safety) to concat everything together. + - Logic for "x amount of times" or complex setups such as "(a OR b) AND c" is fully to be implemented by regex. + - "times" is removed from the class interface. + + Risks: + - Slower for extremely long logs lines. + */ + +public class HoggingLogMessageWaitStrategy extends AbstractWaitStrategy { + + // Hardcoded, as we assume the regEx will check the amount of times for this implementation. + private static final int CHECK_TIMES = 1; + + private String regEx; + + @Override + @SneakyThrows(IOException.class) + protected void waitUntilReady() { + WaitingConsumer waitingConsumer = new WaitingConsumer(); + + LogContainerCmd cmd = waitStrategyTarget + .getDockerClient() + .logContainerCmd(waitStrategyTarget.getContainerId()) + .withFollowStream(true) + .withSince(0) + .withStdOut(true) + .withStdErr(true); + + try (FrameConsumerResultCallback callback = new FrameConsumerResultCallback()) { + callback.addConsumer(OutputFrame.OutputType.STDOUT, waitingConsumer); + callback.addConsumer(OutputFrame.OutputType.STDERR, waitingConsumer); + + cmd.exec(callback); + + //region Specific Implementation + + final StringBuffer builder = new StringBuffer(); + + Predicate waitPredicate = outputFrame -> { + builder.append(outputFrame.getUtf8String()); + // (?s) enables line terminator matching (equivalent to Pattern.DOTALL) + return builder.toString().matches("(?s)" + regEx); + }; + + //endregion + + try { + waitingConsumer.waitUntil(waitPredicate, startupTimeout.getSeconds(), TimeUnit.SECONDS, CHECK_TIMES); + } catch (TimeoutException e) { + throw new ContainerLaunchException("Timed out waiting for log output matching '" + regEx + "'"); + } + } + } + + public HoggingLogMessageWaitStrategy withRegEx(String regEx) { + this.regEx = regEx; + return this; + } +} From 69a2da070f4c69a3d911cd4e43147e49cb010b9a Mon Sep 17 00:00:00 2001 From: tomco Date: Wed, 15 Jun 2022 21:30:57 +0200 Subject: [PATCH 2/4] Add Experimental MultiLogMessageWaitStrategy This wait strategy has following characteristics - Allows multiple regex to be added, which will be validated in order. - Logic for "x amount of times" or complex setups such as "(a OR b) AND c" is fully to be implemented by regex (multiple if needed). - "times" is removed from the class interface. --- .../MultiLogMessageWaitStrategy.java | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 core/src/main/java/org/testcontainers/containers/wait/experimental/MultiLogMessageWaitStrategy.java diff --git a/core/src/main/java/org/testcontainers/containers/wait/experimental/MultiLogMessageWaitStrategy.java b/core/src/main/java/org/testcontainers/containers/wait/experimental/MultiLogMessageWaitStrategy.java new file mode 100644 index 00000000000..8c787cfdb52 --- /dev/null +++ b/core/src/main/java/org/testcontainers/containers/wait/experimental/MultiLogMessageWaitStrategy.java @@ -0,0 +1,77 @@ +package org.testcontainers.containers.wait.experimental; + +import com.github.dockerjava.api.command.LogContainerCmd; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.testcontainers.containers.ContainerLaunchException; +import org.testcontainers.containers.output.FrameConsumerResultCallback; +import org.testcontainers.containers.output.OutputFrame; +import org.testcontainers.containers.output.WaitingConsumer; +import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; + +import java.io.IOException; +import java.util.Arrays; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; + +/* + This wait strategy has following characteristics + - Allows multiple regex to be added, which will be validated in order. + - Logic for "x amount of times" or complex setups such as "(a OR b) AND c" is fully to be implemented by regex (multiple if needed). + - "times" is removed from the class interface. + */ + +@Slf4j +public class MultiLogMessageWaitStrategy extends AbstractWaitStrategy { + + private static final int CHECK_TIMES = 1; + + private LinkedBlockingDeque regEx; + + @Override + @SneakyThrows(IOException.class) + protected void waitUntilReady() { + WaitingConsumer waitingConsumer = new WaitingConsumer(); + + LogContainerCmd cmd = waitStrategyTarget + .getDockerClient() + .logContainerCmd(waitStrategyTarget.getContainerId()) + .withFollowStream(true) + .withSince(0) + .withStdOut(true) + .withStdErr(true); + + try (FrameConsumerResultCallback callback = new FrameConsumerResultCallback()) { + callback.addConsumer(OutputFrame.OutputType.STDOUT, waitingConsumer); + callback.addConsumer(OutputFrame.OutputType.STDERR, waitingConsumer); + + cmd.exec(callback); + + Predicate waitPredicate = outputFrame -> { + String nextExpectedRegex = regEx.peek(); + // (?s) enables line terminator matching (equivalent to Pattern.DOTALL) + if(outputFrame.getUtf8String().matches("(?s)" + nextExpectedRegex)) { + // remove the matched item from the collection. + regEx.pop(); + log.info("Regex {} encountered. Waiting for {} more regex statements.", nextExpectedRegex, regEx.size()); + } + // If collection is now empty, we are finished. + return regEx.isEmpty(); + }; + try { + waitingConsumer.waitUntil(waitPredicate, startupTimeout.getSeconds(), TimeUnit.SECONDS, CHECK_TIMES); + } catch (TimeoutException e) { + throw new ContainerLaunchException("Timed out waiting for log output matching '" + regEx + "'"); + } + } + } + + public MultiLogMessageWaitStrategy withRegex(String... regEx) { + // TODO, add validation that we have at least one regex. :-) + this.regEx = new LinkedBlockingDeque<>(Arrays.asList(regEx)); + return this; + } + +} From bdd1aed30035a6f193898a498346ecfe4f5f8c08 Mon Sep 17 00:00:00 2001 From: tomco Date: Tue, 21 Jun 2022 20:23:09 +0200 Subject: [PATCH 3/4] Extend from an AbstractLogMessageWaitStrategy --- .../MultiLogMessageWaitStrategy.java | 56 +++++++------------ .../AbstractLogMessageWaitStrategy.java | 49 ++++++++++++++++ .../wait/strategy/LogMessageWaitStrategy.java | 40 ++++--------- 3 files changed, 78 insertions(+), 67 deletions(-) create mode 100644 core/src/main/java/org/testcontainers/containers/wait/strategy/AbstractLogMessageWaitStrategy.java diff --git a/core/src/main/java/org/testcontainers/containers/wait/experimental/MultiLogMessageWaitStrategy.java b/core/src/main/java/org/testcontainers/containers/wait/experimental/MultiLogMessageWaitStrategy.java index 8c787cfdb52..76fecf4286f 100644 --- a/core/src/main/java/org/testcontainers/containers/wait/experimental/MultiLogMessageWaitStrategy.java +++ b/core/src/main/java/org/testcontainers/containers/wait/experimental/MultiLogMessageWaitStrategy.java @@ -7,6 +7,7 @@ import org.testcontainers.containers.output.FrameConsumerResultCallback; import org.testcontainers.containers.output.OutputFrame; import org.testcontainers.containers.output.WaitingConsumer; +import org.testcontainers.containers.wait.strategy.AbstractLogMessageWaitStrategy; import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; import java.io.IOException; @@ -15,6 +16,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Predicate; +import java.util.stream.Collectors; /* This wait strategy has following characteristics @@ -24,48 +26,28 @@ */ @Slf4j -public class MultiLogMessageWaitStrategy extends AbstractWaitStrategy { - - private static final int CHECK_TIMES = 1; +public class MultiLogMessageWaitStrategy extends AbstractLogMessageWaitStrategy { private LinkedBlockingDeque regEx; @Override - @SneakyThrows(IOException.class) - protected void waitUntilReady() { - WaitingConsumer waitingConsumer = new WaitingConsumer(); - - LogContainerCmd cmd = waitStrategyTarget - .getDockerClient() - .logContainerCmd(waitStrategyTarget.getContainerId()) - .withFollowStream(true) - .withSince(0) - .withStdOut(true) - .withStdErr(true); - - try (FrameConsumerResultCallback callback = new FrameConsumerResultCallback()) { - callback.addConsumer(OutputFrame.OutputType.STDOUT, waitingConsumer); - callback.addConsumer(OutputFrame.OutputType.STDERR, waitingConsumer); - - cmd.exec(callback); - - Predicate waitPredicate = outputFrame -> { - String nextExpectedRegex = regEx.peek(); - // (?s) enables line terminator matching (equivalent to Pattern.DOTALL) - if(outputFrame.getUtf8String().matches("(?s)" + nextExpectedRegex)) { - // remove the matched item from the collection. - regEx.pop(); - log.info("Regex {} encountered. Waiting for {} more regex statements.", nextExpectedRegex, regEx.size()); - } - // If collection is now empty, we are finished. - return regEx.isEmpty(); - }; - try { - waitingConsumer.waitUntil(waitPredicate, startupTimeout.getSeconds(), TimeUnit.SECONDS, CHECK_TIMES); - } catch (TimeoutException e) { - throw new ContainerLaunchException("Timed out waiting for log output matching '" + regEx + "'"); + protected Predicate waitPredicate() { + return outputFrame -> { + String nextExpectedRegex = regEx.peek(); + // (?s) enables line terminator matching (equivalent to Pattern.DOTALL) + if(outputFrame.getUtf8String().matches("(?s)" + nextExpectedRegex)) { + // remove the matched item from the collection. + regEx.pop(); + log.info("Regex {} encountered. Waiting for {} more regex statements.", nextExpectedRegex, regEx.size()); } - } + // If collection is now empty, we are finished. + return regEx.isEmpty(); + }; + } + + @Override + protected String timeoutErrorMessage() { + return "Timed out waiting for log output. Still expecting following log lines: '" + String.join(",", regEx) + "'"; } public MultiLogMessageWaitStrategy withRegex(String... regEx) { diff --git a/core/src/main/java/org/testcontainers/containers/wait/strategy/AbstractLogMessageWaitStrategy.java b/core/src/main/java/org/testcontainers/containers/wait/strategy/AbstractLogMessageWaitStrategy.java new file mode 100644 index 00000000000..ccee1ebc6cb --- /dev/null +++ b/core/src/main/java/org/testcontainers/containers/wait/strategy/AbstractLogMessageWaitStrategy.java @@ -0,0 +1,49 @@ +package org.testcontainers.containers.wait.strategy; + +import com.github.dockerjava.api.command.LogContainerCmd; +import lombok.SneakyThrows; +import org.testcontainers.containers.ContainerLaunchException; +import org.testcontainers.containers.output.FrameConsumerResultCallback; +import org.testcontainers.containers.output.OutputFrame; +import org.testcontainers.containers.output.WaitingConsumer; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; + +public abstract class AbstractLogMessageWaitStrategy extends AbstractWaitStrategy { + + protected int times = 1; + + @Override + @SneakyThrows(IOException.class) + protected void waitUntilReady() { + WaitingConsumer waitingConsumer = new WaitingConsumer(); + + LogContainerCmd cmd = waitStrategyTarget + .getDockerClient() + .logContainerCmd(waitStrategyTarget.getContainerId()) + .withFollowStream(true) + .withSince(0) + .withStdOut(true) + .withStdErr(true); + + try (FrameConsumerResultCallback callback = new FrameConsumerResultCallback()) { + callback.addConsumer(OutputFrame.OutputType.STDOUT, waitingConsumer); + callback.addConsumer(OutputFrame.OutputType.STDERR, waitingConsumer); + + cmd.exec(callback); + + try { + waitingConsumer.waitUntil(waitPredicate(), startupTimeout.getSeconds(), TimeUnit.SECONDS, times); + } catch (TimeoutException e) { + throw new ContainerLaunchException(timeoutErrorMessage()); + } + } + } + + protected abstract Predicate waitPredicate(); + protected abstract String timeoutErrorMessage(); + +} diff --git a/core/src/main/java/org/testcontainers/containers/wait/strategy/LogMessageWaitStrategy.java b/core/src/main/java/org/testcontainers/containers/wait/strategy/LogMessageWaitStrategy.java index 7a095d888f2..f3c02554970 100644 --- a/core/src/main/java/org/testcontainers/containers/wait/strategy/LogMessageWaitStrategy.java +++ b/core/src/main/java/org/testcontainers/containers/wait/strategy/LogMessageWaitStrategy.java @@ -12,41 +12,21 @@ import java.util.concurrent.TimeoutException; import java.util.function.Predicate; -public class LogMessageWaitStrategy extends AbstractWaitStrategy { +public class LogMessageWaitStrategy extends AbstractLogMessageWaitStrategy { private String regEx; - private int times = 1; + @Override + protected Predicate waitPredicate() { + return outputFrame -> { + // (?s) enables line terminator matching (equivalent to Pattern.DOTALL) + return outputFrame.getUtf8String().matches("(?s)" + regEx); + }; + } @Override - @SneakyThrows(IOException.class) - protected void waitUntilReady() { - WaitingConsumer waitingConsumer = new WaitingConsumer(); - - LogContainerCmd cmd = waitStrategyTarget - .getDockerClient() - .logContainerCmd(waitStrategyTarget.getContainerId()) - .withFollowStream(true) - .withSince(0) - .withStdOut(true) - .withStdErr(true); - - try (FrameConsumerResultCallback callback = new FrameConsumerResultCallback()) { - callback.addConsumer(OutputFrame.OutputType.STDOUT, waitingConsumer); - callback.addConsumer(OutputFrame.OutputType.STDERR, waitingConsumer); - - cmd.exec(callback); - - Predicate waitPredicate = outputFrame -> { - // (?s) enables line terminator matching (equivalent to Pattern.DOTALL) - return outputFrame.getUtf8String().matches("(?s)" + regEx); - }; - try { - waitingConsumer.waitUntil(waitPredicate, startupTimeout.getSeconds(), TimeUnit.SECONDS, times); - } catch (TimeoutException e) { - throw new ContainerLaunchException("Timed out waiting for log output matching '" + regEx + "'"); - } - } + protected String timeoutErrorMessage() { + return "Timed out waiting for log output matching '" + regEx + "'"; } public LogMessageWaitStrategy withRegEx(String regEx) { From 58ae898e7d659c7dcbb2f55e6a86e71f3d3266fd Mon Sep 17 00:00:00 2001 From: tomco Date: Mon, 17 Jul 2023 22:03:55 +0200 Subject: [PATCH 4/4] Create a Complex Wait Strategy and include it in the PostgreSQLContainer --- .../experimental/ComplexWaitStrategy.java | 41 +++++++++++++++++++ .../MultiLogMessageWaitStrategy.java | 1 - .../containers/PostgreSQLContainer.java | 16 ++++++-- .../postgresql/SimplePostgreSQLTest.java | 20 ++++++++- 4 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 core/src/main/java/org/testcontainers/containers/wait/experimental/ComplexWaitStrategy.java diff --git a/core/src/main/java/org/testcontainers/containers/wait/experimental/ComplexWaitStrategy.java b/core/src/main/java/org/testcontainers/containers/wait/experimental/ComplexWaitStrategy.java new file mode 100644 index 00000000000..dadbe0e88e0 --- /dev/null +++ b/core/src/main/java/org/testcontainers/containers/wait/experimental/ComplexWaitStrategy.java @@ -0,0 +1,41 @@ +package org.testcontainers.containers.wait.experimental; + +import org.rnorth.ducttape.unreliables.Unreliables; +import org.testcontainers.containers.output.WaitingConsumer; +import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; +import org.testcontainers.containers.wait.strategy.WaitStrategy; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + + +public class ComplexWaitStrategy extends AbstractWaitStrategy { + + private final List waitStrategyList = new ArrayList<>(); + + @Override + protected void waitUntilReady() { + ExecutorService service = Executors.newFixedThreadPool(waitStrategyList.size()); + List> futures = waitStrategyList.stream() + .map(waitStrategy -> service.submit(() -> waitStrategy.waitUntilReady(waitStrategyTarget))) + .collect(Collectors.toList()); + + Unreliables.retryUntilTrue( + (int) startupTimeout.getSeconds(), + TimeUnit.SECONDS, + () -> futures.stream().anyMatch(Future::isDone) + ); + } + + public ComplexWaitStrategy with(WaitStrategy waitStrategy) { + this.waitStrategyList.add(waitStrategy); + return this; + } +} diff --git a/core/src/main/java/org/testcontainers/containers/wait/experimental/MultiLogMessageWaitStrategy.java b/core/src/main/java/org/testcontainers/containers/wait/experimental/MultiLogMessageWaitStrategy.java index 76fecf4286f..b8f31c0cd9d 100644 --- a/core/src/main/java/org/testcontainers/containers/wait/experimental/MultiLogMessageWaitStrategy.java +++ b/core/src/main/java/org/testcontainers/containers/wait/experimental/MultiLogMessageWaitStrategy.java @@ -51,7 +51,6 @@ protected String timeoutErrorMessage() { } public MultiLogMessageWaitStrategy withRegex(String... regEx) { - // TODO, add validation that we have at least one regex. :-) this.regEx = new LinkedBlockingDeque<>(Arrays.asList(regEx)); return this; } diff --git a/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLContainer.java b/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLContainer.java index d8f974ad2d5..75ec6050d00 100644 --- a/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLContainer.java +++ b/modules/postgresql/src/main/java/org/testcontainers/containers/PostgreSQLContainer.java @@ -1,6 +1,8 @@ package org.testcontainers.containers; import org.jetbrains.annotations.NotNull; +import org.testcontainers.containers.wait.experimental.ComplexWaitStrategy; +import org.testcontainers.containers.wait.experimental.MultiLogMessageWaitStrategy; import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; import org.testcontainers.utility.DockerImageName; @@ -49,10 +51,16 @@ public PostgreSQLContainer(final DockerImageName dockerImageName) { dockerImageName.assertCompatibleWith(DEFAULT_IMAGE_NAME); this.waitStrategy = - new LogMessageWaitStrategy() - .withRegEx(".*database system is ready to accept connections.*\\s") - .withTimes(2) - .withStartupTimeout(Duration.of(60, ChronoUnit.SECONDS)); + new ComplexWaitStrategy() + .with(new LogMessageWaitStrategy() + .withRegEx(".*database system is ready to accept connections.*\\s") + .withTimes(2) + .withStartupTimeout(Duration.of(60, ChronoUnit.SECONDS))) + .with(new MultiLogMessageWaitStrategy() + .withRegex(".*PostgreSQL Database directory appears to contain a database; Skipping initialization.*", + ".*database system is ready to accept connections.*\\s") + .withStartupTimeout(Duration.of(60, ChronoUnit.SECONDS))) +; this.setCommand("postgres", "-c", FSYNC_OFF_OPTION); addExposedPort(POSTGRESQL_PORT); diff --git a/modules/postgresql/src/test/java/org/testcontainers/junit/postgresql/SimplePostgreSQLTest.java b/modules/postgresql/src/test/java/org/testcontainers/junit/postgresql/SimplePostgreSQLTest.java index 84beb779fa4..c8b81ae8f9d 100644 --- a/modules/postgresql/src/test/java/org/testcontainers/junit/postgresql/SimplePostgreSQLTest.java +++ b/modules/postgresql/src/test/java/org/testcontainers/junit/postgresql/SimplePostgreSQLTest.java @@ -4,6 +4,7 @@ import org.testcontainers.PostgreSQLTestImages; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.db.AbstractContainerDatabaseTest; +import org.testcontainers.utility.DockerImageName; import java.sql.ResultSet; import java.sql.SQLException; @@ -11,6 +12,7 @@ import java.util.logging.LogManager; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; public class SimplePostgreSQLTest extends AbstractContainerDatabaseTest { static { @@ -20,7 +22,23 @@ public class SimplePostgreSQLTest extends AbstractContainerDatabaseTest { @Test public void testSimple() throws SQLException { - try (PostgreSQLContainer postgres = new PostgreSQLContainer<>(PostgreSQLTestImages.POSTGRES_TEST_IMAGE)) { + try (PostgreSQLContainer postgres = new PostgreSQLContainer<>(DockerImageName.parse("postgres:14"))) { + postgres.start(); + + ResultSet resultSet = performQuery(postgres, "SELECT 1"); + int resultSetInt = resultSet.getInt(1); + assertEquals("A basic SELECT query succeeds", 1, resultSetInt); + } + } + + @Test + public void testSimpleWithData() throws SQLException { + DockerImageName IMAGE = DockerImageName.parse("tomcools/postgres:main") + .asCompatibleSubstituteFor("postgres"); + try (PostgreSQLContainer postgres = new PostgreSQLContainer<>(IMAGE) + .withDatabaseName("testcontainer") + .withUsername("sa") + .withPassword("sa")) { postgres.start(); ResultSet resultSet = performQuery(postgres, "SELECT 1");