diff --git a/core/src/main/java/org/testcontainers/UnstableAPI.java b/core/src/main/java/org/testcontainers/UnstableAPI.java new file mode 100644 index 00000000000..21fd2c3176c --- /dev/null +++ b/core/src/main/java/org/testcontainers/UnstableAPI.java @@ -0,0 +1,21 @@ +package org.testcontainers; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marks that the annotated API is a subject to change and SHOULD NOT be considered + * a stable API. + */ +@Retention(RetentionPolicy.SOURCE) +@Target({ + ElementType.TYPE, + ElementType.METHOD, + ElementType.FIELD, +}) +@Documented +public @interface UnstableAPI { +} diff --git a/core/src/main/java/org/testcontainers/containers/GenericContainer.java b/core/src/main/java/org/testcontainers/containers/GenericContainer.java index 14ce129c9a5..5b34a6e40c9 100644 --- a/core/src/main/java/org/testcontainers/containers/GenericContainer.java +++ b/core/src/main/java/org/testcontainers/containers/GenericContainer.java @@ -1,5 +1,9 @@ package org.testcontainers.containers; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.CreateContainerCmd; import com.github.dockerjava.api.command.InspectContainerResponse; @@ -12,13 +16,15 @@ import com.github.dockerjava.api.model.PortBinding; import com.github.dockerjava.api.model.Volume; import com.github.dockerjava.api.model.VolumesFrom; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import lombok.AccessLevel; import lombok.Data; -import lombok.EqualsAndHashCode; import lombok.NonNull; import lombok.Setter; import lombok.SneakyThrows; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.utils.IOUtils; @@ -33,6 +39,7 @@ import org.rnorth.visibleassertions.VisibleAssertions; import org.slf4j.Logger; import org.testcontainers.DockerClientFactory; +import org.testcontainers.UnstableAPI; import org.testcontainers.containers.output.OutputFrame; import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy; import org.testcontainers.containers.startupcheck.MinimumDurationRunningStartupCheckStrategy; @@ -62,11 +69,14 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Method; import java.nio.charset.Charset; import java.nio.file.Path; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -100,6 +110,8 @@ public class GenericContainer> public static final String INTERNAL_HOST_HOSTNAME = "host.testcontainers.internal"; + static final String HASH_LABEL = "org.testcontainers.hash"; + /* * Default settings */ @@ -168,11 +180,12 @@ public class GenericContainer> protected final Set dependencies = new HashSet<>(); - /* + /** * Unique instance of DockerClient for use by this container object. + * We use {@link LazyDockerClient} here to avoid eager client creation */ @Setter(AccessLevel.NONE) - protected DockerClient dockerClient = DockerClientFactory.instance().client(); + protected DockerClient dockerClient = LazyDockerClient.INSTANCE; /* * Info about the Docker server; lazily fetched. @@ -222,6 +235,8 @@ public class GenericContainer> @Nullable private Map tmpFsMapping; + @Setter(AccessLevel.NONE) + private boolean shouldBeReused = false; public GenericContainer() { this(TestcontainersConfiguration.getInstance().getTinyImage()); @@ -276,13 +291,15 @@ protected void doStart() { try { configure(); + Instant startedAt = Instant.now(); + logger().debug("Starting container: {}", getDockerImageName()); logger().debug("Trying to start container: {}", image.get()); AtomicInteger attempt = new AtomicInteger(0); Unreliables.retryUntilSuccess(startupAttempts, () -> { logger().debug("Trying to start container: {} (attempt {}/{})", image.get(), attempt.incrementAndGet(), startupAttempts); - tryStart(); + tryStart(startedAt); return true; }); @@ -291,7 +308,25 @@ protected void doStart() { } } - private void tryStart() { + @UnstableAPI + @SneakyThrows + protected boolean canBeReused() { + for (Class type = getClass(); type != GenericContainer.class; type = type.getSuperclass()) { + try { + Method method = type.getDeclaredMethod("containerIsCreated", String.class); + if (method.getDeclaringClass() != GenericContainer.class) { + logger().warn("{} can't be reused because it overrides {}", getClass(), method.getName()); + return false; + } + } catch (NoSuchMethodException e) { + // ignore + } + } + + return true; + } + + private void tryStart(Instant startedAt) { try { String dockerImageName = image.get(); logger().debug("Starting container: {}", dockerImageName); @@ -300,16 +335,49 @@ private void tryStart() { CreateContainerCmd createCommand = dockerClient.createContainerCmd(dockerImageName); applyConfiguration(createCommand); - containerId = createCommand.exec().getId(); + createCommand.getLabels().put(DockerClientFactory.TESTCONTAINERS_LABEL, "true"); - connectToPortForwardingNetwork(createCommand.getNetworkMode()); + boolean reused = false; + if (shouldBeReused) { + if (!canBeReused()) { + throw new IllegalStateException("This container does not support reuse"); + } - copyToFileContainerPathMap.forEach(this::copyFileToContainer); + if (TestcontainersConfiguration.getInstance().environmentSupportsReuse()) { + String hash = hash(createCommand); - containerIsCreated(containerId); + containerId = findContainerForReuse(hash).orElse(null); - logger().info("Starting container with ID: {}", containerId); - dockerClient.startContainerCmd(containerId).exec(); + if (containerId != null) { + logger().info("Reusing container with ID: {} and hash: {}", containerId, hash); + reused = true; + } else { + logger().debug("Can't find a reusable running container with hash: {}", hash); + + createCommand.getLabels().put(HASH_LABEL, hash); + } + } else { + logger().info("Reuse was requested but the environment does not support the reuse of containers"); + } + } else { + createCommand.getLabels().put(DockerClientFactory.TESTCONTAINERS_SESSION_ID_LABEL, DockerClientFactory.SESSION_ID); + } + + if (!reused) { + containerId = createCommand.exec().getId(); + + // TODO add to the hash + copyToFileContainerPathMap.forEach(this::copyFileToContainer); + } + + connectToPortForwardingNetwork(createCommand.getNetworkMode()); + + if (!reused) { + containerIsCreated(containerId); + + logger().info("Starting container with ID: {}", containerId); + dockerClient.startContainerCmd(containerId).exec(); + } logger().info("Container {} is starting: {}", dockerImageName, containerId); @@ -331,7 +399,7 @@ private void tryStart() { // Wait until the process within the container has become ready for use (e.g. listening on network, log message emitted, etc). waitUntilContainerStarted(); - logger().info("Container {} started", dockerImageName); + logger().info("Container {} started in {}", dockerImageName, Duration.between(startedAt, Instant.now())); containerIsStarted(containerInfo); } catch (Exception e) { logger().error("Could not start container", e); @@ -351,6 +419,31 @@ private void tryStart() { } } + @UnstableAPI + @SneakyThrows(JsonProcessingException.class) + final String hash(CreateContainerCmd createCommand) { + // TODO add Testcontainers' version to the hash + String commandJson = new ObjectMapper() + .enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY) + .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS) + .writeValueAsString(createCommand); + + return DigestUtils.sha1Hex(commandJson); + } + + @VisibleForTesting + Optional findContainerForReuse(String hash) { + // TODO locking + return dockerClient.listContainersCmd() + .withLabelFilter(ImmutableMap.of(HASH_LABEL, hash)) + .withLimit(1) + .withStatusFilter(Arrays.asList("running")) + .exec() + .stream() + .findAny() + .map(it -> it.getId()); + } + /** * Set any custom settings for the create command such as shared memory size. */ @@ -613,7 +706,6 @@ private void applyConfiguration(CreateContainerCmd createCommand) { if (createCommand.getLabels() != null) { combinedLabels.putAll(createCommand.getLabels()); } - combinedLabels.putAll(DockerClientFactory.DEFAULT_LABELS); createCommand.withLabels(combinedLabels); } @@ -1214,7 +1306,7 @@ public SELF withStartupAttempts(int attempts) { } /** - * Allow low level modifications of {@link CreateContainerCmd} after it was pre-configured in {@link #tryStart()}. + * Allow low level modifications of {@link CreateContainerCmd} after it was pre-configured in {@link #tryStart(Instant)}. * Invocation happens eagerly on a moment when container is created. * Warning: this does expose the underlying docker-java API so might change outside of our control. * @@ -1246,6 +1338,12 @@ public SELF withTmpFs(Map mapping) { return self(); } + @UnstableAPI + public SELF withReuse(boolean reusable) { + this.shouldBeReused = reusable; + return self(); + } + @Override public boolean equals(Object o) { return this == o; diff --git a/core/src/main/java/org/testcontainers/containers/LazyDockerClient.java b/core/src/main/java/org/testcontainers/containers/LazyDockerClient.java new file mode 100644 index 00000000000..b1341b52755 --- /dev/null +++ b/core/src/main/java/org/testcontainers/containers/LazyDockerClient.java @@ -0,0 +1,15 @@ +package org.testcontainers.containers; + +import com.github.dockerjava.api.DockerClient; +import lombok.experimental.Delegate; +import org.testcontainers.DockerClientFactory; + +enum LazyDockerClient implements DockerClient { + + INSTANCE; + + @Delegate + final DockerClient getDockerClient() { + return DockerClientFactory.instance().client(); + } +} diff --git a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java index b7ef772fcf5..1187d1b3a44 100644 --- a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java +++ b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java @@ -2,6 +2,7 @@ import lombok.*; import lombok.extern.slf4j.Slf4j; +import org.testcontainers.UnstableAPI; import java.io.*; import java.net.MalformedURLException; @@ -85,12 +86,17 @@ public boolean isDisableChecks() { return Boolean.parseBoolean((String) environmentProperties.getOrDefault("checks.disable", "false")); } + @UnstableAPI + public boolean environmentSupportsReuse() { + return Boolean.parseBoolean((String) environmentProperties.getOrDefault("testcontainers.reuse.enable", "false")); + } + public String getDockerClientStrategyClassName() { return (String) environmentProperties.get("docker.client.strategy"); } /** - * + * * @deprecated we no longer have different transport types */ @Deprecated diff --git a/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java b/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java new file mode 100644 index 00000000000..b05db5b5077 --- /dev/null +++ b/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java @@ -0,0 +1,200 @@ +package org.testcontainers.containers; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.*; +import com.github.dockerjava.api.model.Container; +import com.github.dockerjava.core.command.CreateContainerCmdImpl; +import com.github.dockerjava.core.command.InspectContainerCmdImpl; +import com.github.dockerjava.core.command.ListContainersCmdImpl; +import com.github.dockerjava.core.command.StartContainerCmdImpl; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import org.junit.Assume; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.junit.runners.BlockJUnit4ClassRunner; +import org.junit.runners.Parameterized; +import org.mockito.Mockito; +import org.mockito.stubbing.Answer; +import org.rnorth.visibleassertions.VisibleAssertions; +import org.testcontainers.containers.startupcheck.StartupCheckStrategy; +import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy; +import org.testcontainers.containers.wait.strategy.WaitStrategy; +import org.testcontainers.utility.TestcontainersConfiguration; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@RunWith(Enclosed.class) +public class ReusabilityUnitTests { + + static final CompletableFuture IMAGE_FUTURE = CompletableFuture.completedFuture( + TestcontainersConfiguration.getInstance().getTinyImage() + ); + + @RunWith(Parameterized.class) + @RequiredArgsConstructor + @FieldDefaults(makeFinal = true) + public static class CanBeReusedTest { + + @Parameterized.Parameters(name = "{0}") + public static Object[][] data() { + return new Object[][] { + { "generic", new GenericContainer(IMAGE_FUTURE), true }, + { "anonymous generic", new GenericContainer(IMAGE_FUTURE) {}, true }, + { "custom", new CustomContainer(), true }, + { "anonymous custom", new CustomContainer() {}, true }, + { "custom with containerIsCreated", new CustomContainerWithContainerIsCreated(), false }, + }; + } + + String name; + + GenericContainer container; + + boolean reusable; + + @Test + public void shouldBeReusable() { + if (reusable) { + VisibleAssertions.assertTrue("Is reusable", container.canBeReused()); + } else { + VisibleAssertions.assertFalse("Is not reusable", container.canBeReused()); + } + } + + static class CustomContainer extends GenericContainer { + CustomContainer() { + super(IMAGE_FUTURE); + } + } + + static class CustomContainerWithContainerIsCreated extends GenericContainer { + + CustomContainerWithContainerIsCreated() { + super(IMAGE_FUTURE); + } + + @Override + protected void containerIsCreated(String containerId) { + super.containerIsCreated(containerId); + } + } + } + + @RunWith(BlockJUnit4ClassRunner.class) + @FieldDefaults(makeFinal = true) + public static class HashTest { + + StartupCheckStrategy startupCheckStrategy = new StartupCheckStrategy() { + @Override + public StartupStatus checkStartupState(DockerClient dockerClient, String containerId) { + return StartupStatus.SUCCESSFUL; + } + }; + + WaitStrategy waitStrategy = new AbstractWaitStrategy() { + @Override + protected void waitUntilReady() { + + } + }; + + DockerClient client = Mockito.mock(DockerClient.class); + + GenericContainer container = new GenericContainer(IMAGE_FUTURE) + .withNetworkMode("none") // to disable the port forwarding + .withStartupCheckStrategy(startupCheckStrategy) + .waitingFor(waitStrategy) + .withReuse(true); + + { + container.dockerClient = client; + } + + @Test + public void shouldStartIfListReturnsEmpty() { + String containerId = randomContainerId(); + when(client.createContainerCmd(any())).then(createContainerAnswer(containerId)); + when(client.listContainersCmd()).then(listContainersAnswer()); + when(client.startContainerCmd(containerId)).then(startContainerAnswer()); + when(client.inspectContainerCmd(containerId)).then(inspectContainerAnswer()); + + container.start(); + + Mockito.verify(client, Mockito.atLeastOnce()).startContainerCmd(containerId); + } + + @Test + public void shouldReuseIfListReturnsID() { + // TODO mock TestcontainersConfiguration + Assume.assumeTrue("supports reuse", TestcontainersConfiguration.getInstance().environmentSupportsReuse()); + when(client.createContainerCmd(any())).then(createContainerAnswer(randomContainerId())); + String existingContainerId = randomContainerId(); + when(client.listContainersCmd()).then(listContainersAnswer(existingContainerId)); + when(client.inspectContainerCmd(existingContainerId)).then(inspectContainerAnswer()); + + container.start(); + + Mockito.verify(client, Mockito.never()).startContainerCmd(existingContainerId); + } + + private String randomContainerId() { + return UUID.randomUUID().toString(); + } + + private Answer listContainersAnswer(String... ids) { + return invocation -> { + ListContainersCmd.Exec exec = command -> { + return new ObjectMapper().convertValue( + Stream.of(ids) + .map(id -> Collections.singletonMap("Id", id)) + .collect(Collectors.toList()), + new TypeReference>() {} + ); + }; + return new ListContainersCmdImpl(exec); + }; + } + + private Answer createContainerAnswer(String containerId) { + return invocation -> { + CreateContainerCmd.Exec exec = command -> { + CreateContainerResponse response = new CreateContainerResponse(); + response.setId(containerId); + return response; + }; + return new CreateContainerCmdImpl(exec, null, "image:latest"); + }; + } + + private Answer startContainerAnswer() { + return invocation -> { + StartContainerCmd.Exec exec = command -> { + return null; + }; + return new StartContainerCmdImpl(exec, invocation.getArgument(0)); + }; + } + + private Answer inspectContainerAnswer() { + return invocation -> { + InspectContainerCmd.Exec exec = command -> { + return new InspectContainerResponse(); + }; + return new InspectContainerCmdImpl(exec, invocation.getArgument(0)); + }; + } + + } +} diff --git a/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java b/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java index ad2a0be6cd3..49a378554f1 100644 --- a/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java +++ b/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java @@ -37,6 +37,17 @@ public void shouldReadDockerClientStrategyFromEnvironmentOnly() { assertEquals("Docker client strategy is changed", "foo", newConfig().getDockerClientStrategyClassName()); } + @Test + public void shouldReadReuseFromEnvironmentOnly() { + assertFalse("no reuse by default", newConfig().environmentSupportsReuse()); + + classpathProperties.setProperty("testcontainers.reuse.enable", "true"); + assertFalse("reuse is not affected by classpath properties", newConfig().environmentSupportsReuse()); + + environmentProperties.setProperty("testcontainers.reuse.enable", "true"); + assertTrue("reuse enabled", newConfig().environmentSupportsReuse()); + } + private TestcontainersConfiguration newConfig() { return new TestcontainersConfiguration(environmentProperties, classpathProperties); } diff --git a/modules/jdbc/src/main/java/org/testcontainers/containers/JdbcDatabaseContainerProvider.java b/modules/jdbc/src/main/java/org/testcontainers/containers/JdbcDatabaseContainerProvider.java index f2ebde857c5..f8694306384 100644 --- a/modules/jdbc/src/main/java/org/testcontainers/containers/JdbcDatabaseContainerProvider.java +++ b/modules/jdbc/src/main/java/org/testcontainers/containers/JdbcDatabaseContainerProvider.java @@ -45,11 +45,14 @@ public JdbcDatabaseContainer newInstance() { * @return Instance of {@link JdbcDatabaseContainer} */ public JdbcDatabaseContainer newInstance(ConnectionUrl url) { + final JdbcDatabaseContainer result; if (url.getImageTag().isPresent()) { - return newInstance(url.getImageTag().get()); + result = newInstance(url.getImageTag().get()); } else { - return newInstance(); + result = newInstance(); } + result.withReuse(url.isReusable()); + return result; } protected JdbcDatabaseContainer newInstanceFromConnectionUrl(ConnectionUrl connectionUrl, final String userParamName, final String pwdParamName) { @@ -59,7 +62,7 @@ protected JdbcDatabaseContainer newInstanceFromConnectionUrl(ConnectionUrl conne final String user = connectionUrl.getQueryParameters().getOrDefault(userParamName, "test"); final String password = connectionUrl.getQueryParameters().getOrDefault(pwdParamName, "test"); - final JdbcDatabaseContainer instance; + final JdbcDatabaseContainer instance; if (connectionUrl.getImageTag().isPresent()) { instance = newInstance(connectionUrl.getImageTag().get()); } else { @@ -67,6 +70,7 @@ protected JdbcDatabaseContainer newInstanceFromConnectionUrl(ConnectionUrl conne } return instance + .withReuse(connectionUrl.isReusable()) .withDatabaseName(databaseName) .withUsername(user) .withPassword(password); diff --git a/modules/jdbc/src/main/java/org/testcontainers/jdbc/ConnectionUrl.java b/modules/jdbc/src/main/java/org/testcontainers/jdbc/ConnectionUrl.java index e236d723c82..4c590474bf7 100644 --- a/modules/jdbc/src/main/java/org/testcontainers/jdbc/ConnectionUrl.java +++ b/modules/jdbc/src/main/java/org/testcontainers/jdbc/ConnectionUrl.java @@ -13,6 +13,7 @@ import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; +import org.testcontainers.UnstableAPI; import static java.util.stream.Collectors.toMap; @@ -49,6 +50,9 @@ public class ConnectionUrl { private Optional initScriptPath = Optional.empty(); + @UnstableAPI + private boolean reusable = false; + private Optional initFunction = Optional.empty(); private Optional queryString; @@ -132,6 +136,8 @@ private void parseUrl() { initScriptPath = Optional.ofNullable(containerParameters.get("TC_INITSCRIPT")); + reusable = Boolean.parseBoolean(containerParameters.get("TC_REUSABLE")); + Matcher funcMatcher = Patterns.INITFUNCTION_MATCHING_PATTERN.matcher(this.getUrl()); if (funcMatcher.matches()) { initFunction = Optional.of(new InitFunctionDef(funcMatcher.group(2), funcMatcher.group(4))); diff --git a/modules/jdbc/src/main/java/org/testcontainers/jdbc/ContainerDatabaseDriver.java b/modules/jdbc/src/main/java/org/testcontainers/jdbc/ContainerDatabaseDriver.java index 6d3df6c7cc5..c5413fe7efb 100644 --- a/modules/jdbc/src/main/java/org/testcontainers/jdbc/ContainerDatabaseDriver.java +++ b/modules/jdbc/src/main/java/org/testcontainers/jdbc/ContainerDatabaseDriver.java @@ -151,7 +151,7 @@ public synchronized Connection connect(String url, final Properties info) throws */ private Connection wrapConnection(final Connection connection, final JdbcDatabaseContainer container, final ConnectionUrl connectionUrl) { - final boolean isDaemon = connectionUrl.isInDaemonMode(); + final boolean isDaemon = connectionUrl.isInDaemonMode() || connectionUrl.isReusable(); Set connections = containerConnections.computeIfAbsent(container.getContainerId(), k -> new HashSet<>());