diff --git a/util/testutil/integration/util.go b/util/testutil/integration/util.go index 5f865dab966f2..85b52b572ce8d 100644 --- a/util/testutil/integration/util.go +++ b/util/testutil/integration/util.go @@ -7,6 +7,7 @@ import ( "io" "os" "os/exec" + "strings" "sync" "syscall" "testing" @@ -102,7 +103,25 @@ func StartCmd(cmd *exec.Cmd, logs map[string]*bytes.Buffer) (func() error, error // On Linux this socket is typically a Unix socket, // while on Windows this will be a named pipe. func WaitSocket(address string, d time.Duration, cmd *exec.Cmd) error { - return waitSocket(address, d, cmd) + address = strings.TrimPrefix(address, socketScheme) + step := 50 * time.Millisecond + i := 0 + for { + if cmd != nil && cmd.ProcessState != nil { + return errors.Errorf("process exited: %s", cmd.String()) + } + + if conn, err := dialPipe(address); err == nil { + conn.Close() + break + } + i++ + if time.Duration(i)*step > d { + return errors.Errorf("failed dialing: %s", address) + } + time.Sleep(step) + } + return nil } func LookupBinary(name string) error { diff --git a/util/testutil/integration/util_unix.go b/util/testutil/integration/util_unix.go index 7cbc6d1e0efaf..f096f954779ee 100644 --- a/util/testutil/integration/util_unix.go +++ b/util/testutil/integration/util_unix.go @@ -5,36 +5,19 @@ package integration import ( "net" - "os/exec" - "strings" - "time" "github.com/pkg/errors" ) -func waitSocket(address string, d time.Duration, cmd *exec.Cmd) error { - address = strings.TrimPrefix(address, "unix://") +var socketScheme = "unix://" + +// abstracted function to handle pipe dialing on unix. +// some simplification has been made to discard +// laddr for unix -- left as nil. +func dialPipe(address string) (net.Conn, error) { addr, err := net.ResolveUnixAddr("unix", address) if err != nil { - return errors.Wrapf(err, "failed resolving unix addr: %s", address) - } - - step := 50 * time.Millisecond - i := 0 - for { - if cmd != nil && cmd.ProcessState != nil { - return errors.Errorf("process exited: %s", cmd.String()) - } - - if conn, err := net.DialUnix("unix", nil, addr); err == nil { - conn.Close() - break - } - i++ - if time.Duration(i)*step > d { - return errors.Errorf("failed dialing: %s", address) - } - time.Sleep(step) + return nil, errors.Wrapf(err, "failed resolving unix addr: %s", address) } - return nil + return net.DialUnix("unix", nil, addr) } diff --git a/util/testutil/integration/util_windows.go b/util/testutil/integration/util_windows.go index 32f235772f699..eef0e37f85e8f 100644 --- a/util/testutil/integration/util_windows.go +++ b/util/testutil/integration/util_windows.go @@ -1,33 +1,15 @@ package integration import ( - "os/exec" - "strings" - "time" + "net" "github.com/Microsoft/go-winio" - "github.com/pkg/errors" ) -func waitSocket(address string, d time.Duration, cmd *exec.Cmd) error { - address = strings.TrimPrefix(address, "npipe://") - step := 50 * time.Millisecond - i := 0 +var socketScheme = "npipe://" - for { - if cmd != nil && cmd.ProcessState != nil { - return errors.Errorf("process exited: %s", cmd.String()) - } - - if conn, err := winio.DialPipe(address, nil); err == nil { - conn.Close() - break - } - i++ - if time.Duration(i)*step > d { - return errors.Errorf("failed dialing: %s", address) - } - time.Sleep(step) - } - return nil +// abstracted function to handle pipe dialing on windows. +// some simplification has been made to discard timeout param. +func dialPipe(address string) (net.Conn, error) { + return winio.DialPipe(address, nil) } diff --git a/util/testutil/workers/containerd.go b/util/testutil/workers/containerd.go index d670645215e7c..ae5d2448a593a 100644 --- a/util/testutil/workers/containerd.go +++ b/util/testutil/workers/containerd.go @@ -7,7 +7,6 @@ import ( "os" "os/exec" "path/filepath" - "runtime" "strconv" "strings" "time" @@ -108,7 +107,7 @@ func (c *Containerd) New(ctx context.Context, cfg *integration.BackendConfig) (b }() rootless := false - if runtime.GOOS != "windows" && c.UID != 0 { + if c.UID != 0 { if c.GID == 0 { return nil, nil, errors.Errorf("unsupported id pair: uid=%d, gid=%d", c.UID, c.GID) } @@ -197,10 +196,22 @@ disabled_plugins = ["cri"] } deferF.Append(ctdStop) - buildkitdArgs := append(getBuildkitdArgs(address), snBuildkitdArgs...) + // handles only windows case, no effect on unix + address = normalizeAddress(address) + buildkitdArgs := []string{ + "buildkitd", + "--containerd-worker-gc=false", + "--containerd-worker=true", + "--containerd-worker-addr", address, + "--containerd-worker-labels=org.mobyproject.buildkit.worker.sandbox=true", // Include use of --containerd-worker-labels to trigger https://github.com/moby/buildkit/pull/603 + } + if ociWorkerFlag != "" { + buildkitdArgs = append(buildkitdArgs, ociWorkerFlag) + } + buildkitdArgs = append(buildkitdArgs, snBuildkitdArgs...) - if runtime.GOOS != "windows" && c.Snapshotter != "native" { - c.ExtraEnv = append(c.ExtraEnv, "BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF=true") + if envForceOverlayDiff != "" && c.Snapshotter != "native" { + c.ExtraEnv = append(c.ExtraEnv, envForceOverlayDiff) } if rootless { pidStr, err := os.ReadFile(filepath.Join(rootlessKitState, "child_pid")) diff --git a/util/testutil/workers/sysprocattr_unix.go b/util/testutil/workers/sysprocattr_unix.go deleted file mode 100644 index 406982cd02e6b..0000000000000 --- a/util/testutil/workers/sysprocattr_unix.go +++ /dev/null @@ -1,41 +0,0 @@ -//go:build !windows -// +build !windows - -package workers - -import ( - "path/filepath" - "syscall" -) - -func getSysProcAttr() *syscall.SysProcAttr { - return &syscall.SysProcAttr{ - Setsid: true, // stretch sudo needs this for sigterm - } -} - -func getBuildkitdAddr(tmpdir string) string { - return "unix://" + filepath.Join(tmpdir, "buildkitd.sock") -} - -func getTraceSocketPath(tmpdir string) string { - return filepath.Join(tmpdir, "otel-grpc.sock") -} - -func getContainerdSock(tmpdir string) string { - return filepath.Join(tmpdir, "containerd.sock") -} - -func getContainerdDebugSock(tmpdir string) string { - return filepath.Join(tmpdir, "debug.sock") -} - -func getBuildkitdArgs(address string) []string { - return []string{"buildkitd", - "--oci-worker=false", - "--containerd-worker-gc=false", - "--containerd-worker=true", - "--containerd-worker-addr", address, - "--containerd-worker-labels=org.mobyproject.buildkit.worker.sandbox=true", // Include use of --containerd-worker-labels to trigger https://github.com/moby/buildkit/pull/603 - } -} diff --git a/util/testutil/workers/sysprocattr_windows.go b/util/testutil/workers/sysprocattr_windows.go deleted file mode 100644 index 1654c48a2ddbc..0000000000000 --- a/util/testutil/workers/sysprocattr_windows.go +++ /dev/null @@ -1,43 +0,0 @@ -//go:build windows -// +build windows - -package workers - -import ( - "path/filepath" - "strings" - "syscall" -) - -func getSysProcAttr() *syscall.SysProcAttr { - return &syscall.SysProcAttr{} -} - -func getBuildkitdAddr(tmpdir string) string { - return "npipe:////./pipe/buildkitd-" + filepath.Base(tmpdir) -} - -func getTraceSocketPath(tmpdir string) string { - return `\\.\pipe\buildkit-otel-grpc-` + filepath.Base(tmpdir) -} - -func getContainerdSock(tmpdir string) string { - return `\\.\pipe\containerd-` + filepath.Base(tmpdir) -} - -func getContainerdDebugSock(tmpdir string) string { - return `\\.\pipe\containerd-` + filepath.Base(tmpdir) + `debug` -} - -func getBuildkitdArgs(address string) []string { - address = filepath.ToSlash(address) - if !strings.HasPrefix(address, "npipe://") { - address = "npipe://" + address - } - return []string{"buildkitd", - "--containerd-worker-gc=false", - "--containerd-worker=true", - "--containerd-worker-addr", address, - "--containerd-worker-labels=org.mobyproject.buildkit.worker.sandbox=true", // Include use of --containerd-worker-labels to trigger https://github.com/moby/buildkit/pull/603 - } -} diff --git a/util/testutil/workers/util.go b/util/testutil/workers/util.go index c9505c1ba8e33..3b9179c98ed59 100644 --- a/util/testutil/workers/util.go +++ b/util/testutil/workers/util.go @@ -1,7 +1,13 @@ package workers import ( + "bytes" + "context" "fmt" + "os" + "os/exec" + "path/filepath" + "time" "github.com/moby/buildkit/util/testutil/integration" ) @@ -19,3 +25,77 @@ func (osp otelSocketPath) UpdateConfigFile(in string) string { socketPath = %q `, in, osp) } + +func runBuildkitd( + ctx context.Context, + conf *integration.BackendConfig, + args []string, + logs map[string]*bytes.Buffer, + uid, gid int, + extraEnv []string, +) (address string, cl func() error, err error) { + deferF := &integration.MultiCloser{} + cl = deferF.F() + + defer func() { + if err != nil { + deferF.F()() + cl = nil + } + }() + + tmpdir, err := os.MkdirTemp("", "bktest_buildkitd") + if err != nil { + return "", nil, err + } + + if err := chown(tmpdir, uid, gid); err != nil { + return "", nil, err + } + + if err := os.MkdirAll(filepath.Join(tmpdir, "tmp"), 0711); err != nil { + return "", nil, err + } + + if err := chown(filepath.Join(tmpdir, "tmp"), uid, gid); err != nil { + return "", nil, err + } + deferF.Append(func() error { return os.RemoveAll(tmpdir) }) + + cfgfile, err := integration.WriteConfig( + append(conf.DaemonConfig, withOTELSocketPath(getTraceSocketPath(tmpdir)))) + if err != nil { + return "", nil, err + } + deferF.Append(func() error { + return os.RemoveAll(filepath.Dir(cfgfile)) + }) + + args = append(args, "--config="+cfgfile) + address = getBuildkitdAddr(tmpdir) + + args = append(args, "--root", tmpdir, "--addr", address, "--debug") + cmd := exec.Command(args[0], args[1:]...) //nolint:gosec // test utility + cmd.Env = append( + os.Environ(), + "BUILDKIT_DEBUG_EXEC_OUTPUT=1", + "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", + "TMPDIR="+filepath.Join(tmpdir, "tmp")) + cmd.Env = append(cmd.Env, extraEnv...) + cmd.SysProcAttr = getSysProcAttr() + + stop, err := integration.StartCmd(cmd, logs) + if err != nil { + return "", nil, err + } + deferF.Append(stop) + + if err := integration.WaitSocket(address, 15*time.Second, cmd); err != nil { + return "", nil, err + } + + // separated out since it's not required in windows + mountInfo(deferF, tmpdir) + + return address, cl, err +} diff --git a/util/testutil/workers/util_unix.go b/util/testutil/workers/util_unix.go index 40d6fbd0031e5..88851a0fbfc81 100644 --- a/util/testutil/workers/util_unix.go +++ b/util/testutil/workers/util_unix.go @@ -5,18 +5,20 @@ package workers import ( "bufio" - "bytes" - "context" "os" - "os/exec" "path/filepath" "strings" - "time" + "syscall" "github.com/moby/buildkit/util/testutil/integration" "github.com/pkg/errors" ) +var ( + ociWorkerFlag = "--oci-worker=false" + envForceOverlayDiff = "BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF=true" +) + func requireRoot() error { if os.Getuid() != 0 { return errors.Wrap(integration.ErrRequirements, "requires root") @@ -24,74 +26,29 @@ func requireRoot() error { return nil } -func runBuildkitd( - ctx context.Context, - conf *integration.BackendConfig, - args []string, - logs map[string]*bytes.Buffer, - uid, gid int, - extraEnv []string, -) (address string, cl func() error, err error) { - deferF := &integration.MultiCloser{} - cl = deferF.F() - - defer func() { - if err != nil { - deferF.F()() - cl = nil - } - }() - - tmpdir, err := os.MkdirTemp("", "bktest_buildkitd") - if err != nil { - return "", nil, err - } - - if err := os.Chown(tmpdir, uid, gid); err != nil { - return "", nil, err - } - - if err := os.MkdirAll(filepath.Join(tmpdir, "tmp"), 0711); err != nil { - return "", nil, err - } - - if err := os.Chown(filepath.Join(tmpdir, "tmp"), uid, gid); err != nil { - return "", nil, err - } - deferF.Append(func() error { return os.RemoveAll(tmpdir) }) - - cfgfile, err := integration.WriteConfig( - append(conf.DaemonConfig, withOTELSocketPath(getTraceSocketPath(tmpdir)))) - if err != nil { - return "", nil, err +func getSysProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{ + Setsid: true, // stretch sudo needs this for sigterm } - deferF.Append(func() error { - return os.RemoveAll(filepath.Dir(cfgfile)) - }) +} - args = append(args, "--config="+cfgfile) - address = getBuildkitdAddr(tmpdir) +func getBuildkitdAddr(tmpdir string) string { + return "unix://" + filepath.Join(tmpdir, "buildkitd.sock") +} - args = append(args, "--root", tmpdir, "--addr", address, "--debug") - cmd := exec.Command(args[0], args[1:]...) //nolint:gosec // test utility - cmd.Env = append( - os.Environ(), - "BUILDKIT_DEBUG_EXEC_OUTPUT=1", - "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", - "TMPDIR="+filepath.Join(tmpdir, "tmp")) - cmd.Env = append(cmd.Env, extraEnv...) - cmd.SysProcAttr = getSysProcAttr() +func getTraceSocketPath(tmpdir string) string { + return filepath.Join(tmpdir, "otel-grpc.sock") +} - stop, err := integration.StartCmd(cmd, logs) - if err != nil { - return "", nil, err - } - deferF.Append(stop) +func getContainerdSock(tmpdir string) string { + return filepath.Join(tmpdir, "containerd.sock") +} - if err := integration.WaitSocket(address, 15*time.Second, cmd); err != nil { - return "", nil, err - } +func getContainerdDebugSock(tmpdir string) string { + return filepath.Join(tmpdir, "debug.sock") +} +func mountInfo(deferF *integration.MultiCloser, tmpdir string) { deferF.Append(func() error { f, err := os.Open("/proc/self/mountinfo") if err != nil { @@ -106,6 +63,15 @@ func runBuildkitd( } return s.Err() }) +} + +// moved here since os.Chown is not supported on Windows. +// see no-op counterpart in util_windows.go +func chown(name string, uid, gid int) error { + return os.Chown(name, uid, gid) +} - return address, cl, err +func normalizeAddress(address string) string { + // for parity with windows, no effect for unix + return address } diff --git a/util/testutil/workers/util_windows.go b/util/testutil/workers/util_windows.go index 452f0a78bde4c..466537a1f4f57 100644 --- a/util/testutil/workers/util_windows.go +++ b/util/testutil/workers/util_windows.go @@ -1,79 +1,56 @@ package workers import ( - "bytes" - "context" - "os" - "os/exec" "path/filepath" - "time" + "strings" + "syscall" "github.com/moby/buildkit/util/testutil/integration" ) +var ( + ociWorkerFlag = "" + envForceOverlayDiff = "" +) + func requireRoot() error { return nil } -func runBuildkitd( - ctx context.Context, - conf *integration.BackendConfig, - args []string, - logs map[string]*bytes.Buffer, - uid, gid int, - extraEnv []string, -) (address string, cl func() error, err error) { - deferF := &integration.MultiCloser{} - cl = deferF.F() - - defer func() { - if err != nil { - deferF.F()() - cl = nil - } - }() +func getSysProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{} +} - tmpdir, err := os.MkdirTemp("", "bktest_buildkitd") - if err != nil { - return "", nil, err - } +func getBuildkitdAddr(tmpdir string) string { + return "npipe:////./pipe/buildkitd-" + filepath.Base(tmpdir) +} - if err := os.MkdirAll(filepath.Join(tmpdir, "tmp"), 0711); err != nil { - return "", nil, err - } - deferF.Append(func() error { return os.RemoveAll(tmpdir) }) +func getTraceSocketPath(tmpdir string) string { + return `\\.\pipe\buildkit-otel-grpc-` + filepath.Base(tmpdir) +} - cfgfile, err := integration.WriteConfig( - append(conf.DaemonConfig, withOTELSocketPath(getTraceSocketPath(tmpdir)))) - if err != nil { - return "", nil, err - } - deferF.Append(func() error { - return os.RemoveAll(filepath.Dir(cfgfile)) - }) +func getContainerdSock(tmpdir string) string { + return `\\.\pipe\containerd-` + filepath.Base(tmpdir) +} - args = append(args, "--config="+cfgfile) - address = getBuildkitdAddr(tmpdir) +func getContainerdDebugSock(tmpdir string) string { + return `\\.\pipe\containerd-` + filepath.Base(tmpdir) + `debug` +} - args = append(args, "--root", tmpdir, "--addr", address, "--debug") - cmd := exec.Command(args[0], args[1:]...) //nolint:gosec // test utility - cmd.Env = append( - os.Environ(), - "BUILDKIT_DEBUG_EXEC_OUTPUT=1", - "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", - "TMPDIR="+filepath.Join(tmpdir, "tmp")) - cmd.Env = append(cmd.Env, extraEnv...) - cmd.SysProcAttr = getSysProcAttr() +//lint:ignore U1000 for parity with unix +func mountInfo(deferF *integration.MultiCloser, tmpdir string) { + // no-op +} - stop, err := integration.StartCmd(cmd, logs) - if err != nil { - return "", nil, err - } - deferF.Append(stop) +func chown(name string, uid, gid int) error { + // Chown not supported on Windows + return nil +} - if err := integration.WaitSocket(address, 15*time.Second, cmd); err != nil { - return "", nil, err +func normalizeAddress(address string) string { + address = filepath.ToSlash(address) + if !strings.HasPrefix(address, "npipe://") { + address = "npipe://" + address } - - return address, cl, err + return address }