Merge branch 'main' into ms/issue-946-v2

* main:
  chore(deps): combine and bump compose dependencies (testcontainers#1787)
  feat: support for replacing images with custom substitutions (testcontainers#1719)

.github/workflows/ci.yml

@@ -107,7 +107,9 @@ jobs:
         go-version: [1.20.x, 1.x]
         platform: [ubuntu-latest, macos-latest]
         module: [artemis, cassandra, clickhouse, compose, couchbase, elasticsearch, gcloud, k3s, k6, kafka, localstack, mariadb, mongodb, mysql, nats, neo4j, postgres, pulsar, rabbitmq, redis, redpanda, vault]
-
+        exclude:
+          - go-version: 1.20.x
+            module: compose
     uses: ./.github/workflows/ci-test-go.yml
     with:
       go-version: ${{ matrix.go-version }}

container.go

@@ -98,6 +98,7 @@ type ContainerFile struct {
 type ContainerRequest struct {
 	FromDockerfile
 	Image                   string
+	ImageSubstitutors       []ImageSubstitutor
 	Entrypoint              []string
 	Env                     map[string]string
 	ExposedPorts            []string // allow specifying protocol info

container_test.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/testcontainers/testcontainers-go/wait"
 )
@@ -324,6 +325,113 @@ func Test_GetLogsFromFailedContainer(t *testing.T) {
 	}
 }
 
+type dockerImageSubstitutor struct{}
+
+func (s dockerImageSubstitutor) Description() string {
+	return "DockerImageSubstitutor (prepends docker.io)"
+}
+
+func (s dockerImageSubstitutor) Substitute(image string) (string, error) {
+	return "docker.io/" + image, nil
+}
+
+// noopImageSubstitutor {
+type NoopImageSubstitutor struct{}
+
+// Description returns a description of what is expected from this Substitutor,
+// which is used in logs.
+func (s NoopImageSubstitutor) Description() string {
+	return "NoopImageSubstitutor (noop)"
+}
+
+// Substitute returns the original image, without any change
+func (s NoopImageSubstitutor) Substitute(image string) (string, error) {
+	return image, nil
+}
+
+// }
+
+type errorSubstitutor struct{}
+
+var errSubstitution = errors.New("substitution error")
+
+// Description returns a description of what is expected from this Substitutor,
+// which is used in logs.
+func (s errorSubstitutor) Description() string {
+	return "errorSubstitutor"
+}
+
+// Substitute returns the original image, but returns an error
+func (s errorSubstitutor) Substitute(image string) (string, error) {
+	return image, errSubstitution
+}
+
+func TestImageSubstitutors(t *testing.T) {
+	tests := []struct {
+		name          string
+		image         string // must be a valid image, as the test will try to create a container from it
+		substitutors  []ImageSubstitutor
+		expectedImage string
+		expectedError error
+	}{
+		{
+			name:          "No substitutors",
+			image:         "alpine",
+			expectedImage: "alpine",
+		},
+		{
+			name:          "Noop substitutor",
+			image:         "alpine",
+			substitutors:  []ImageSubstitutor{NoopImageSubstitutor{}},
+			expectedImage: "alpine",
+		},
+		{
+			name:          "Prepend namespace",
+			image:         "alpine",
+			substitutors:  []ImageSubstitutor{dockerImageSubstitutor{}},
+			expectedImage: "docker.io/alpine",
+		},
+		{
+			name:          "Substitution with error",
+			image:         "alpine",
+			substitutors:  []ImageSubstitutor{errorSubstitutor{}},
+			expectedImage: "alpine",
+			expectedError: errSubstitution,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := context.Background()
+			req := ContainerRequest{
+				Image:             test.image,
+				ImageSubstitutors: test.substitutors,
+			}
+
+			container, err := GenericContainer(ctx, GenericContainerRequest{
+				ContainerRequest: req,
+				Started:          true,
+			})
+			if test.expectedError != nil {
+				require.ErrorIs(t, err, test.expectedError)
+				return
+			}
+
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer func() {
+				terminateContainerOnEnd(t, ctx, container)
+			}()
+
+			// enforce the concrete type, as GenericContainer returns an interface,
+			// which will be changed in future implementations of the library
+			dockerContainer := container.(*DockerContainer)
+			assert.Equal(t, test.expectedImage, dockerContainer.Image)
+		})
+	}
+}
+
 func TestShouldStartContainersInParallel(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
 	t.Cleanup(cancel)

docker.go

@@ -919,6 +919,8 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque
 		}
 	}
 
+	imageName := req.Image
+
 	env := []string{}
 	for envKey, envVar := range req.Env {
 		env = append(env, envKey+"="+envVar)
@@ -939,7 +941,7 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque
 
 	var termSignal chan bool
 	// the reaper does not need to start a reaper for itself
-	isReaperContainer := strings.EqualFold(req.Image, reaperImage(reaperOpts.ImageName))
+	isReaperContainer := strings.EqualFold(imageName, reaperImage(reaperOpts.ImageName))
 	if !tcConfig.RyukDisabled && !isReaperContainer {
 		r, err := reuseOrCreateReaper(context.WithValue(ctx, testcontainersdocker.DockerHostContextKey, p.host), testcontainerssession.SessionID(), p, req.ReaperOptions...)
 		if err != nil {
@@ -962,17 +964,24 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque
 		return nil, err
 	}
 
-	var tag string
+	for _, is := range req.ImageSubstitutors {
+		modifiedTag, err := is.Substitute(imageName)
+		if err != nil {
+			return nil, fmt.Errorf("failed to substitute image %s with %s: %w", imageName, is.Description(), err)
+		}
+
+		p.Logger.Printf("✍🏼 Replacing image with %s. From: %s to %s\n", is.Description(), imageName, modifiedTag)
+		imageName = modifiedTag
+	}
+
 	var platform *specs.Platform
 
 	if req.ShouldBuildImage() {
-		tag, err = p.BuildImage(ctx, &req)
+		imageName, err = p.BuildImage(ctx, &req)
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		tag = req.Image
-
 		if req.ImagePlatform != "" {
 			p, err := platforms.Parse(req.ImagePlatform)
 			if err != nil {
@@ -986,7 +995,7 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque
 		if req.AlwaysPullImage {
 			shouldPullImage = true // If requested always attempt to pull image
 		} else {
-			image, _, err := p.client.ImageInspectWithRaw(ctx, tag)
+			image, _, err := p.client.ImageInspectWithRaw(ctx, imageName)
 			if err != nil {
 				if client.IsErrNotFound(err) {
 					shouldPullImage = true
@@ -1004,20 +1013,20 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque
 				Platform: req.ImagePlatform, // may be empty
 			}
 
-			registry, imageAuth, err := DockerImageAuth(ctx, req.Image)
+			registry, imageAuth, err := DockerImageAuth(ctx, imageName)
 			if err != nil {
-				p.Logger.Printf("Failed to get image auth for %s. Setting empty credentials for the image: %s. Error is:%s", registry, req.Image, err)
+				p.Logger.Printf("Failed to get image auth for %s. Setting empty credentials for the image: %s. Error is:%s", registry, imageName, err)
 			} else {
 				// see https://github.com/docker/docs/blob/e8e1204f914767128814dca0ea008644709c117f/engine/api/sdk/examples.md?plain=1#L649-L657
 				encodedJSON, err := json.Marshal(imageAuth)
 				if err != nil {
-					p.Logger.Printf("Failed to marshal image auth. Setting empty credentials for the image: %s. Error is:%s", req.Image, err)
+					p.Logger.Printf("Failed to marshal image auth. Setting empty credentials for the image: %s. Error is:%s", imageName, err)
 				} else {
 					pullOpt.RegistryAuth = base64.URLEncoding.EncodeToString(encodedJSON)
 				}
 			}
 
-			if err := p.attemptToPullImage(ctx, tag, pullOpt); err != nil {
+			if err := p.attemptToPullImage(ctx, imageName, pullOpt); err != nil {
 				return nil, err
 			}
 		}
@@ -1032,7 +1041,7 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque
 
 	dockerInput := &container.Config{
 		Entrypoint: req.Entrypoint,
-		Image:      tag,
+		Image:      imageName,
 		Env:        env,
 		Labels:     req.Labels,
 		Cmd:        req.Cmd,
@@ -1128,7 +1137,7 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque
 	c := &DockerContainer{
 		ID:                resp.ID,
 		WaitingFor:        req.WaitingFor,
-		Image:             tag,
+		Image:             imageName,
 		imageWasBuilt:     req.ShouldBuildImage(),
 		sessionID:         testcontainerssession.SessionID(),
 		provider:          p,

docs/features/common_functional_options.md

@@ -1,3 +1,11 @@
+#### Image Substitutions
+
+- Not available until the next release of testcontainers-go <a href="https://github.com/testcontainers/testcontainers-go"><span class="tc-version">:material-tag: main</span></a>
+
+{% include "./image_name_substitution.md" %}
+
+Using the `WithImageSubstitutors` options, you could define your own substitutions to the container images. E.g. adding a prefix to the images so that they can be pulled from a Docker registry other than Docker Hub. This is the usual mechanism for using Docker image proxies, caches, etc.
+
 #### Wait Strategies
 
 If you need to set a different wait strategy for the container, you can use `testcontainers.WithWaitStrategy` with a valid wait strategy.

docs/features/docker_compose.md

@@ -10,23 +10,12 @@ dependent upon.
 ## Using `docker-compose` directly
 
 !!!warning
-	The minimal version of Go required to use this module is **1.20**.
+	The minimal version of Go required to use this module is **1.21**.
 
 ```
 go get github.com/testcontainers/testcontainers-go/modules/compose
 ```
 
-!!!warning
-
-	Given the version includes the Compose dependency, and the Docker folks added a replace directive in their [go.mod](https://github.com/docker/compose/blob/v2/go.mod#L175-L188),
-	we were forced to add it as well. As a result, users of _Testcontainers for Go_ need to add the following replace directive to their `go.mod` files.
-
-	```
-	replace (
-		github.com/cucumber/godog => github.com/laurazard/godog v0.0.0-20220922095256-4c4b17abdae7
-	)
-	```
-
 Because `docker-compose` v2 is implemented in Go it's possible for _Testcontainers for Go_ to
 use [`github.com/docker/compose`](https://github.com/docker/compose) directly and skip any process execution/_docker-compose-in-a-container_ scenario.
 The `ComposeStack` API exposes this variant of using `docker-compose` in an easy way.

docs/features/image_name_substitution.md

@@ -0,0 +1,10 @@
+In more locked down / secured environments, it can be problematic to pull images from Docker Hub and run them without additional precautions.
+
+An image name substitutor converts a Docker image name, as may be specified in code, to an alternative name. This is intended to provide a way to override image names, for example to enforce pulling of images from a private registry.
+
+_Testcontainers for Go_ exposes an interface to perform this operations: `ImageSubstitutor`, and a No-operation implementation to be used as reference for custom implementations:
+
+<!--codeinclude-->
+[Image Substitutor Interface](../../generic.go) inside_block:imageSubstitutor
+[Noop Image Substitutor](../../container_test.go) inside_block:noopImageSubstitutor
+<!--/codeinclude-->

generic.go

@@ -60,6 +60,24 @@ func WithImage(image string) CustomizeRequestOption {
 	}
 }
 
+// imageSubstitutor {
+// ImageSubstitutor represents a way to substitute container image names
+type ImageSubstitutor interface {
+	// Description returns the name of the type and a short description of how it modifies the image.
+	// Useful to be printed in logs
+	Description() string
+	Substitute(image string) (string, error)
+}
+
+// }
+
+// WithImageSubstitutors sets the image substitutors for a container
+func WithImageSubstitutors(fn ...ImageSubstitutor) CustomizeRequestOption {
+	return func(req *GenericContainerRequest) {
+		req.ImageSubstitutors = fn
+	}
+}
+
 // WithConfigModifier allows to override the default container config
 func WithConfigModifier(modifier func(config *container.Config)) CustomizeRequestOption {
 	return func(req *GenericContainerRequest) {

mkdocs.yml

@@ -40,6 +40,7 @@ nav:
     - Quickstart: quickstart.md
     - Features:
         - features/creating_container.md
+        - features/image_name_substitution.md
         - features/files_and_mounts.md
         - features/creating_networks.md
         - features/configuration.md

modulegen/_template/ci.yml.tmpl

@@ -107,6 +107,9 @@ jobs:
         go-version: [1.20.x, 1.x]
         platform: [ubuntu-latest, macos-latest]
         module: [{{ .Modules }}]
+        exclude:
+          - go-version: 1.20.x
+            module: compose
     uses: ./.github/workflows/ci-test-go.yml
     with:
       go-version: {{ "${{ matrix.go-version }}" }}

modulegen/main_test.go

@@ -485,7 +485,7 @@ func assertModuleGithubWorkflowContent(t *testing.T, module context.Testcontaine
 
 	examplesList, err := ctx.GetExamples()
 	assert.Nil(t, err)
-	assert.Equal(t, "        module: ["+strings.Join(examplesList, ", ")+"]", data[125])
+	assert.Equal(t, "        module: ["+strings.Join(examplesList, ", ")+"]", data[128])
 }
 
 // assert content go.mod

modules/compose/compose_api.go

@@ -216,6 +216,9 @@ func (d *dockerCompose) Up(ctx context.Context, opts ...StackUpOption) (err erro
 
 	err = d.composeService.Up(ctx, d.project, api.UpOptions{
 		Create: api.CreateOptions{
+			Build: &api.BuildOptions{
+				Services: upOptions.Services,
+			},
 			Services:             upOptions.Services,
 			Recreate:             upOptions.Recreate,
 			RecreateDependencies: upOptions.RecreateDependencies,