feat(wait): for file (#2731)

* feat(wait): for file

Add the ability to wait for a file and optionally its contents.

This leverages generated mocks using mockery and testify/mock to make
testing easier.

* debugging: rate limit config dump

Add output of the docker config if we hit rate limiting to aid in
debugging why this is happening on github actions.

Don't use logger as that's no-op when verbose is not in effect.

* docs: list the file wait strategy in docs

---------

Co-authored-by: Manuel de la Peña <mdelapenya@gmail.com>

.mockery.yaml

@@ -0,0 +1,11 @@
+quiet: True
+disable-version-string: True
+with-expecter: True
+mockname: "mock{{.InterfaceName}}"
+filename: "{{ .InterfaceName | lower }}_mock_test.go"
+outpkg: "{{.PackageName}}_test"
+dir: "{{.InterfaceDir}}"
+packages:
+  github.com/testcontainers/testcontainers-go/wait:
+    interfaces:
+      StrategyTarget:

docs/features/wait/file.md

@@ -0,0 +1,14 @@
+# File Wait Strategy
+
+File Wait Strategy waits for a file to exist in the container, and allows to set the following conditions:
+
+- the file to wait for.
+- a matcher which reads the file content, no-op if nil or not set.
+- the startup timeout to be used in seconds, default is 60 seconds.
+- the poll interval to be used in milliseconds, default is 100 milliseconds.
+
+## Waiting for file to exist and extract the content
+
+<!--codeinclude-->
+[Waiting for file to exist and extract the content](../../../wait/file_test.go) inside_block:waitForFileWithMatcher
+<!--/codeinclude-->

docs/features/wait/introduction.md

@@ -8,6 +8,7 @@ Below you can find a list of the available wait strategies that you can use:
 
 - [Exec](./exec.md)
 - [Exit](./exit.md)
+- [File](./file.md)
 - [Health](./health.md)
 - [HostPort](./host_port.md)
 - [HTTP](./http.md)

generic.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"sync"
 
 	"github.com/testcontainers/testcontainers-go/internal/core"
@@ -74,6 +75,14 @@ func GenericContainer(ctx context.Context, req GenericContainerRequest) (Contain
 	}
 	if err != nil {
 		// At this point `c` might not be nil. Give the caller an opportunity to call Destroy on the container.
+		// TODO: Remove this debugging.
+		if strings.Contains(err.Error(), "toomanyrequests") {
+			// Debugging information for rate limiting.
+			cfg, err := getDockerConfig()
+			if err == nil {
+				fmt.Printf("XXX: too many requests: %+v", cfg)
+			}
+		}
 		return c, fmt.Errorf("create container: %w", err)
 	}
 

go.mod

@@ -47,6 +47,7 @@ require (
 	github.com/rogpeppe/go-internal v1.8.1 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect

go.sum

@@ -102,6 +102,8 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=

mkdocs.yml

@@ -57,6 +57,7 @@ nav:
             - Introduction: features/wait/introduction.md
             - Exec: features/wait/exec.md
             - Exit: features/wait/exit.md
+            - File: features/wait/file.md
             - Health: features/wait/health.md
             - HostPort: features/wait/host_port.md
             - HTTP: features/wait/http.md

wait/exec_test.go

@@ -102,6 +102,10 @@ func (st mockExecTarget) State(_ context.Context) (*types.ContainerState, error)
 	return nil, errors.New("not implemented")
 }
 
+func (st mockExecTarget) CopyFileFromContainer(_ context.Context, _ string) (io.ReadCloser, error) {
+	return nil, errors.New("not implemented")
+}
+
 func TestExecStrategyWaitUntilReady(t *testing.T) {
 	target := mockExecTarget{}
 	wg := wait.NewExecStrategy([]string{"true"}).

wait/exit_test.go

@@ -2,6 +2,7 @@ package wait
 
 import (
 	"context"
+	"errors"
 	"io"
 	"testing"
 	"time"
@@ -45,6 +46,10 @@ func (st exitStrategyTarget) State(ctx context.Context) (*types.ContainerState,
 	return &types.ContainerState{Running: st.isRunning}, nil
 }
 
+func (st exitStrategyTarget) CopyFileFromContainer(context.Context, string) (io.ReadCloser, error) {
+	return nil, errors.New("not implemented")
+}
+
 func TestWaitForExit(t *testing.T) {
 	target := exitStrategyTarget{
 		isRunning: false,

wait/file.go

@@ -0,0 +1,112 @@
+package wait
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/docker/docker/errdefs"
+)
+
+var (
+	_ Strategy        = (*FileStrategy)(nil)
+	_ StrategyTimeout = (*FileStrategy)(nil)
+)
+
+// FileStrategy waits for a file to exist in the container.
+type FileStrategy struct {
+	timeout      *time.Duration
+	file         string
+	pollInterval time.Duration
+	matcher      func(io.Reader) error
+}
+
+// NewFileStrategy constructs an FileStrategy strategy.
+func NewFileStrategy(file string) *FileStrategy {
+	return &FileStrategy{
+		file:         file,
+		pollInterval: defaultPollInterval(),
+	}
+}
+
+// WithStartupTimeout can be used to change the default startup timeout
+func (ws *FileStrategy) WithStartupTimeout(startupTimeout time.Duration) *FileStrategy {
+	ws.timeout = &startupTimeout
+	return ws
+}
+
+// WithPollInterval can be used to override the default polling interval of 100 milliseconds
+func (ws *FileStrategy) WithPollInterval(pollInterval time.Duration) *FileStrategy {
+	ws.pollInterval = pollInterval
+	return ws
+}
+
+// WithMatcher can be used to consume the file content.
+// The matcher can return an errdefs.ErrNotFound to indicate that the file is not ready.
+// Any other error will be considered a failure.
+// Default: nil, will only wait for the file to exist.
+func (ws *FileStrategy) WithMatcher(matcher func(io.Reader) error) *FileStrategy {
+	ws.matcher = matcher
+	return ws
+}
+
+// ForFile is a convenience method to assign FileStrategy
+func ForFile(file string) *FileStrategy {
+	return NewFileStrategy(file)
+}
+
+// Timeout returns the timeout for the strategy
+func (ws *FileStrategy) Timeout() *time.Duration {
+	return ws.timeout
+}
+
+// WaitUntilReady waits until the file exists in the container and copies it to the target.
+func (ws *FileStrategy) WaitUntilReady(ctx context.Context, target StrategyTarget) error {
+	timeout := defaultStartupTimeout()
+	if ws.timeout != nil {
+		timeout = *ws.timeout
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	timer := time.NewTicker(ws.pollInterval)
+	defer timer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-timer.C:
+			if err := ws.matchFile(ctx, target); err != nil {
+				if errdefs.IsNotFound(err) {
+					// Not found, continue polling.
+					continue
+				}
+
+				return fmt.Errorf("copy from container: %w", err)
+			}
+			return nil
+		}
+	}
+}
+
+// matchFile tries to copy the file from the container and match it.
+func (ws *FileStrategy) matchFile(ctx context.Context, target StrategyTarget) error {
+	rc, err := target.CopyFileFromContainer(ctx, ws.file)
+	if err != nil {
+		return fmt.Errorf("copy from container: %w", err)
+	}
+	defer rc.Close() //nolint: errcheck // Read close error can't tell us anything useful.
+
+	if ws.matcher == nil {
+		// No matcher, just check if the file exists.
+		return nil
+	}
+
+	if err = ws.matcher(rc); err != nil {
+		return fmt.Errorf("matcher: %w", err)
+	}
+
+	return nil
+}

wait/file_test.go

@@ -0,0 +1,104 @@
+package wait_test
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/errdefs"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/wait"
+)
+
+const testFilename = "/tmp/file"
+
+var anyContext = mock.AnythingOfType("*context.timerCtx")
+
+// newRunningTarget creates a new mockStrategyTarget that is running.
+func newRunningTarget() *mockStrategyTarget {
+	target := &mockStrategyTarget{}
+	target.EXPECT().State(anyContext).
+		Return(&types.ContainerState{Running: true}, nil)
+
+	return target
+}
+
+// testForFile creates a new FileStrategy for testing.
+func testForFile() *wait.FileStrategy {
+	return wait.ForFile(testFilename).
+		WithStartupTimeout(time.Millisecond * 50).
+		WithPollInterval(time.Millisecond)
+}
+
+func TestForFile(t *testing.T) {
+	errNotFound := errdefs.NotFound(errors.New("file not found"))
+	ctx := context.Background()
+
+	t.Run("not-found", func(t *testing.T) {
+		target := newRunningTarget()
+		target.EXPECT().CopyFileFromContainer(anyContext, testFilename).Return(nil, errNotFound)
+		err := testForFile().WaitUntilReady(ctx, target)
+		require.EqualError(t, err, context.DeadlineExceeded.Error())
+	})
+
+	t.Run("other-error", func(t *testing.T) {
+		otherErr := errors.New("other error")
+		target := newRunningTarget()
+		target.EXPECT().CopyFileFromContainer(anyContext, testFilename).Return(nil, otherErr)
+		err := testForFile().WaitUntilReady(ctx, target)
+		require.ErrorIs(t, err, otherErr)
+	})
+
+	t.Run("valid", func(t *testing.T) {
+		data := "my content\nwibble"
+		file := bytes.NewBufferString(data)
+		target := newRunningTarget()
+		target.EXPECT().CopyFileFromContainer(anyContext, testFilename).Once().Return(nil, errNotFound)
+		target.EXPECT().CopyFileFromContainer(anyContext, testFilename).Return(io.NopCloser(file), nil)
+		var out bytes.Buffer
+		err := testForFile().WithMatcher(func(r io.Reader) error {
+			if _, err := io.Copy(&out, r); err != nil {
+				return fmt.Errorf("copy: %w", err)
+			}
+			return nil
+		}).WaitUntilReady(ctx, target)
+		require.NoError(t, err)
+		require.Equal(t, data, out.String())
+	})
+}
+
+func TestFileStrategyWaitUntilReady_WithMatcher(t *testing.T) {
+	// waitForFileWithMatcher {
+	var out bytes.Buffer
+	dockerReq := testcontainers.ContainerRequest{
+		Image: "docker.io/nginx:latest",
+		WaitingFor: wait.ForFile("/etc/nginx/nginx.conf").
+			WithStartupTimeout(time.Second * 10).
+			WithPollInterval(time.Second).
+			WithMatcher(func(r io.Reader) error {
+				if _, err := io.Copy(&out, r); err != nil {
+					return fmt.Errorf("copy: %w", err)
+				}
+				return nil
+			}),
+	}
+	// }
+
+	ctx := context.Background()
+	container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ContainerRequest: dockerReq, Started: true})
+	if container != nil {
+		t.Cleanup(func() {
+			require.NoError(t, container.Terminate(context.Background()))
+		})
+	}
+	require.NoError(t, err)
+	require.Contains(t, out.String(), "worker_processes")
+}

wait/health_test.go

@@ -2,6 +2,7 @@ package wait
 
 import (
 	"context"
+	"errors"
 	"io"
 	"sync"
 	"testing"
@@ -60,6 +61,10 @@ func (st *healthStrategyTarget) setState(health *types.Health) {
 	st.state.Health = health
 }
 
+func (st *healthStrategyTarget) CopyFileFromContainer(_ context.Context, _ string) (io.ReadCloser, error) {
+	return nil, errors.New("not implemented")
+}
+
 // TestWaitForHealthTimesOutForUnhealthy confirms that an unhealthy container will eventually
 // time out.
 func TestWaitForHealthTimesOutForUnhealthy(t *testing.T) {

wait/nop.go

@@ -75,3 +75,7 @@ func (st NopStrategyTarget) Exec(_ context.Context, _ []string, _ ...exec.Proces
 func (st NopStrategyTarget) State(_ context.Context) (*types.ContainerState, error) {
 	return &st.ContainerState, nil
 }
+
+func (st NopStrategyTarget) CopyFileFromContainer(context.Context, string) (io.ReadCloser, error) {
+	return st.ReaderCloser, nil
+}