[Carry 2535] rootless: support detach-netns mode

When RootlessKit v2.0 (rootless-containers/rootlesskit PR 379) is
installed, `containerd-rootless.sh` launches it with `--detach-netns`
so as to run the daemon in the host network namespace.

This will enable:
- Accelerated (and deflaked) `nerdctl pull`, `nerdctl push`, `nerdctl build`, etc
- Proper support for `nerdctl pull 127.0.0.1:.../...`
- Proper support for `nerdctl run --net=host`

Replaces Fahed Dorgaa's PR 2535

Co-authored-by: fahed dorgaa <fahed.dorgaa@gmail.com>
Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

.github/workflows/test.yml

@@ -158,31 +158,40 @@ jobs:
         include:
           - ubuntu: 20.04
             containerd: v1.6.26
+            rootlesskit: v1.1.1
             target: test-integration-rootless
           - ubuntu: 20.04
             containerd: v1.7.11
+            rootlesskit: v2.0.0-beta.0
             target: test-integration-rootless
           - ubuntu: 22.04
             containerd: v1.7.11
+            rootlesskit: v1.1.1
             target: test-integration-rootless
           - ubuntu: 22.04
             containerd: main
+            rootlesskit: v2.0.0-beta.0
             target: test-integration-rootless
           - ubuntu: 20.04
             containerd: v1.6.26
+            rootlesskit: v1.1.1
             target: test-integration-rootless-port-slirp4netns
           - ubuntu: 20.04
             containerd: v1.7.11
+            rootlesskit: v2.0.0-beta.0
             target: test-integration-rootless-port-slirp4netns
           - ubuntu: 22.04
             containerd: v1.7.11
+            rootlesskit: v1.1.1
             target: test-integration-rootless-port-slirp4netns
           - ubuntu: 22.04
             containerd: main
+            rootlesskit: v2.0.0-beta.0
             target: test-integration-rootless-port-slirp4netns
     env:
       UBUNTU_VERSION: "${{ matrix.ubuntu }}"
       CONTAINERD_VERSION: "${{ matrix.containerd }}"
+      ROOTLESSKIT_VERSION: "${{ matrix.rootlesskit }}"
       TEST_TARGET: "${{ matrix.target }}"
     steps:
       - uses: actions/checkout@v4.1.1
@@ -191,7 +200,7 @@ jobs:
       - name: "Register QEMU (tonistiigi/binfmt)"
         run: docker run --privileged --rm tonistiigi/binfmt --install all
       - name: "Prepare (network driver=slirp4netns, port driver=builtin)"
-        run: DOCKER_BUILDKIT=1 docker build -t ${TEST_TARGET} --target ${TEST_TARGET} --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} .
+        run: DOCKER_BUILDKIT=1 docker build -t ${TEST_TARGET} --target ${TEST_TARGET} --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} --build-arg ROOTLESSKIT_VERSION=${ROOTLESSKIT_VERSION} .
       - name: "Test (network driver=slirp4netns, port driver=builtin)"
         run: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=1 ${TEST_TARGET}
 

Dockerfile

@@ -29,7 +29,7 @@ ARG STARGZ_SNAPSHOTTER_VERSION=v0.15.1
 # Extra deps: Encryption
 ARG IMGCRYPT_VERSION=v1.1.9
 # Extra deps: Rootless
-ARG ROOTLESSKIT_VERSION=v1.1.1
+ARG ROOTLESSKIT_VERSION=v2.0.0-beta.0
 ARG SLIRP4NETNS_VERSION=v1.2.2
 # Extra deps: bypass4netns
 ARG BYPASS4NETNS_VERSION=v0.3.0

Dockerfile.d/SHA256SUMS.d/rootlesskit-v2.0.0-alpha.2

@@ -0,0 +1,6 @@
+7c8c07c159aef32b5e68f5b8dc508dc422499744f61fa916c246bfae16a0d85c  rootlesskit-aarch64.tar.gz
+683ba2c34bfa4a3477f9c50508a233c94b71a6eeaa0ee080abd1ebc2c09a8b9c  rootlesskit-armv7l.tar.gz
+844b6297021d914be9f266e341ff77da4b98f43074504fe9cc020ae5c61a791d  rootlesskit-ppc64le.tar.gz
+d317e9c519d862508d6659083f9e1773573e899aa6e48e89d121211e5e823b6a  rootlesskit-riscv64.tar.gz
+720b425c608f8ab4326354582bc92825031d9d8c40865df155c2c7cb8368f115  rootlesskit-s390x.tar.gz
+d29edd2e3d903974754edb14b251ef19bfa9317e6626436fac760d1213879e8d  rootlesskit-x86_64.tar.gz

cmd/nerdctl/container_run_network_linux_test.go

@@ -494,6 +494,11 @@ func TestRunContainerWithMACAddress(t *testing.T) {
 }
 
 func TestHostsFileMounts(t *testing.T) {
+	if rootlessutil.IsRootless() {
+		if detachedNetNS, _ := rootlessutil.DetachedNetNS(); detachedNetNS != "" {
+			t.Skip("/etc/hosts is not writable")
+		}
+	}
 	base := testutil.NewBase(t)
 
 	base.Cmd("run", "--rm", testutil.CommonImage,

cmd/nerdctl/ipfs_linux_test.go

@@ -73,14 +73,21 @@ func TestIPFSAddress(t *testing.T) {
 
 func runIPFSDaemonContainer(t *testing.T, base *testutil.Base) (ipfsAddress string, done func()) {
 	name := "test-ipfs-address"
-	base.Cmd("run", "-d", "--name", name, "--entrypoint=/bin/sh", testutil.KuboImage, "-c", "ipfs init && ipfs config Addresses.API /ip4/0.0.0.0/tcp/5001 && ipfs daemon --offline").AssertOK()
-	iplines := base.Cmd("inspect", name, "-f", "'{{json .NetworkSettings.IPAddress}}'").OutLines()
-	t.Logf("IPAddress=%v", iplines)
-	assert.Equal(t, len(iplines), 2)
-	matches := iplineRegexp.FindStringSubmatch(iplines[0])
-	t.Logf("ip address matches=%v", matches)
-	assert.Equal(t, len(matches), 2)
-	ipfsaddr := fmt.Sprintf("/ip4/%s/tcp/5001", matches[1])
+	var ipfsaddr string
+	if detachedNetNS, _ := rootlessutil.DetachedNetNS(); detachedNetNS != "" {
+		// detached-netns mode can't use .NetworkSettings.IPAddress, because the daemon and CNI has different network namespaces
+		base.Cmd("run", "-d", "-p", "127.0.0.1:5999:5999", "--name", name, "--entrypoint=/bin/sh", testutil.KuboImage, "-c", "ipfs init && ipfs config Addresses.API /ip4/0.0.0.0/tcp/5999 && ipfs daemon --offline").AssertOK()
+		ipfsaddr = "/ip4/127.0.0.1/tcp/5999"
+	} else {
+		base.Cmd("run", "-d", "--name", name, "--entrypoint=/bin/sh", testutil.KuboImage, "-c", "ipfs init && ipfs config Addresses.API /ip4/0.0.0.0/tcp/5001 && ipfs daemon --offline").AssertOK()
+		iplines := base.Cmd("inspect", name, "-f", "'{{json .NetworkSettings.IPAddress}}'").OutLines()
+		t.Logf("IPAddress=%v", iplines)
+		assert.Equal(t, len(iplines), 2)
+		matches := iplineRegexp.FindStringSubmatch(iplines[0])
+		t.Logf("ip address matches=%v", matches)
+		assert.Equal(t, len(matches), 2)
+		ipfsaddr = fmt.Sprintf("/ip4/%s/tcp/5001", matches[1])
+	}
 	return ipfsaddr, func() {
 		base.Cmd("kill", "test-ipfs-address").AssertOK()
 		base.Cmd("rm", "test-ipfs-address").AssertOK()

docs/rootless.md

@@ -130,6 +130,35 @@ $ nerdctl run -it --rm -p 8080:80 --label nerdctl/bypass4netns=true alpine
 
 More detail is available at [https://github.com/rootless-containers/bypass4netns/blob/master/README.md](https://github.com/rootless-containers/bypass4netns/blob/master/README.md)
 
+## Configuring RootlessKit
+
+Rootless containerd recognizes the following environment variables to configure the behavior of [RootlessKit](https://github.com/rootless-containers/rootlesskit):
+
+* `CONTAINERD_ROOTLESS_ROOTLESSKIT_STATE_DIR=DIR`: the rootlesskit state dir. Defaults to `$XDG_RUNTIME_DIR/containerd-rootless`.
+* `CONTAINERD_ROOTLESS_ROOTLESSKIT_NET=(slirp4netns|vpnkit|lxc-user-nic)`: the rootlesskit network driver. Defaults to "slirp4netns" if slirp4netns (>= v0.4.0) is installed. Otherwise defaults to "vpnkit".
+* `CONTAINERD_ROOTLESS_ROOTLESSKIT_MTU=NUM`: the MTU value for the rootlesskit network driver. Defaults to 65520 for slirp4netns, 1500 for other drivers.
+* `CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=(builtin|slirp4netns)`: the rootlesskit port driver. Defaults to "builtin".
+* `CONTAINERD_ROOTLESS_ROOTLESSKIT_SLIRP4NETNS_SANDBOX=(auto|true|false)`: whether to protect slirp4netns with a dedicated mount namespace. Defaults to "auto".
+* `CONTAINERD_ROOTLESS_ROOTLESSKIT_SLIRP4NETNS_SECCOMP=(auto|true|false)`: whether to protect slirp4netns with seccomp. Defaults to "auto".
+* `CONTAINERD_ROOTLESS_ROOTLESSKIT_DETACH_NETNS=(auto|true|false)`: whether to launch rootlesskit with the "detach-netns" mode.
+  Defaults to "auto", which is resolved to "true" if RootlessKit >= 2.0 is installed.
+  The "detached-netns" mode accelerates `nerdctl (pull|push|build)` and enables `nerdctl run --net=host`,
+  however, there is a relatively minor drawback with the current version of BuildKit (planned to be fixed in [BuildKit v0.13](https://github.com/moby/buildkit/pull/4546)):
+  the host loopback IP address (127.0.0.1) is exposed to Dockerfile's "RUN" instructions during `nerdctl build` (not `nerdctl run`).
+  If you want to hide 127.0.0.1 from "RUN" instructions, you should set `CONTAINERD_ROOTLESS_ROOTLESSKIT_DETACH_NETNS` to "false".
+
+To set these variables, create `~/.config/systemd/user/containerd.service.d/override.conf` as follows:
+```ini
+[Service]
+Environment=CONTAINERD_ROOTLESS_ROOTLESSKIT_DETACH_NETNS="false"
+```
+
+And then run the following commands:
+```bash
+systemctl --user daemon-reload
+systemctl --user restart containerd
+```
+
 ## Troubleshooting
 
 ### Hint to Fedora users

extras/rootless/containerd-rootless-setuptool.sh

@@ -143,7 +143,12 @@ cmd_entrypoint_check() {
 cmd_entrypoint_nsenter() {
 	# No need to call init()
 	pid=$(cat "$XDG_RUNTIME_DIR/containerd-rootless/child_pid")
-	exec nsenter --no-fork --wd="$(pwd)" --preserve-credentials -m -n -U -t "$pid" -- "$@"
+	n=""
+	# If RootlessKit is running with `--detach-netns` mode, we do NOT enter the detached netns here
+	if [ ! -e "$XDG_RUNTIME_DIR/containerd-rootless/netns" ]; then
+		n="-n"
+	fi
+	exec nsenter --no-fork --wd="$(pwd)" --preserve-credentials -m $n -U -t "$pid" -- "$@"
 }
 
 show_systemd_error() {

extras/rootless/containerd-rootless.sh

@@ -28,7 +28,7 @@
 # External dependencies:
 # * newuidmap and newgidmap needs to be installed.
 # * /etc/subuid and /etc/subgid needs to be configured for the current user.
-# * RootlessKit (>= v0.10.0) needs to be installed. RootlessKit >= v0.14.1 is recommended.
+# * RootlessKit (>= v0.10.0) needs to be installed. RootlessKit >= v2.0.0 is recommended.
 # * Either one of slirp4netns (>= v0.4.0), VPNKit, lxc-user-nic needs to be installed. slirp4netns >= v1.1.7 is recommended.
 #
 # Recognized environment variables:
@@ -38,6 +38,14 @@
 # * CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=(builtin|slirp4netns): the rootlesskit port driver. Defaults to "builtin".
 # * CONTAINERD_ROOTLESS_ROOTLESSKIT_SLIRP4NETNS_SANDBOX=(auto|true|false): whether to protect slirp4netns with a dedicated mount namespace. Defaults to "auto".
 # * CONTAINERD_ROOTLESS_ROOTLESSKIT_SLIRP4NETNS_SECCOMP=(auto|true|false): whether to protect slirp4netns with seccomp. Defaults to "auto".
+# * CONTAINERD_ROOTLESS_ROOTLESSKIT_DETACH_NETNS=(auto|true|false): whether to launch rootlesskit with the "detach-netns" mode.
+#   Defaults to "auto", which is resolved to "true" if RootlessKit >= 2.0 is installed.
+#   The "detached-netns" mode accelerates `nerdctl (pull|push|build)` and enables `nerdctl run --net=host`,
+#   however, there is a relatively minor drawback with the current version of BuildKit (planned to be fixed in BuildKit v0.13):
+#   the host loopback IP address (127.0.0.1) is exposed to Dockerfile's "RUN" instructions during `nerdctl build` (not `nerdctl run`).
+#   If you want to hide 127.0.0.1 from "RUN" instructions, you should set CONTAINERD_ROOTLESS_ROOTLESSKIT_DETACH_NETNS to "false".
+
+# See also: https://github.com/containerd/nerdctl/blob/main/docs/rootless.md#configuring-rootlesskit
 
 set -e
 if ! [ -w $XDG_RUNTIME_DIR ]; then
@@ -69,6 +77,7 @@ if [ -z $_CONTAINERD_ROOTLESS_CHILD ]; then
 	: "${CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER:=builtin}"
 	: "${CONTAINERD_ROOTLESS_ROOTLESSKIT_SLIRP4NETNS_SANDBOX:=auto}"
 	: "${CONTAINERD_ROOTLESS_ROOTLESSKIT_SLIRP4NETNS_SECCOMP:=auto}"
+	: "${CONTAINERD_ROOTLESS_ROOTLESSKIT_DETACH_NETNS:=auto}"
 	net=$CONTAINERD_ROOTLESS_ROOTLESSKIT_NET
 	mtu=$CONTAINERD_ROOTLESS_ROOTLESSKIT_MTU
 	if [ -z $net ]; then
@@ -107,6 +116,25 @@ if [ -z $_CONTAINERD_ROOTLESS_CHILD ]; then
 			export _CONTAINERD_ROOTLESS_SELINUX
 		fi
 	fi
+
+	case "$CONTAINERD_ROOTLESS_ROOTLESSKIT_DETACH_NETNS" in
+	auto)
+		if rootlesskit --help | grep -qw -- "--detach-netns"; then
+			CONTAINERD_ROOTLESS_ROOTLESSKIT_FLAGS=--detach-netns $CONTAINERD_ROOTLESS_ROOTLESSKIT_FLAGS
+		fi
+		;;
+	1 | true)
+		CONTAINERD_ROOTLESS_ROOTLESSKIT_FLAGS=--detach-netns $CONTAINERD_ROOTLESS_ROOTLESSKIT_FLAGS
+		;;
+	0 | false)
+		# NOP
+		;;
+	*)
+		echo "Unknown CONTAINERD_ROOTLESS_ROOTLESSKIT_DETACH_NETNS value: $CONTAINERD_ROOTLESS_ROOTLESSKIT_DETACH_NETNS"
+		exit 1
+		;;
+	esac
+
 	# Re-exec the script via RootlessKit, so as to create unprivileged {user,mount,network} namespaces.
 	#
 	# --copy-up allows removing/creating files in the directories by creating tmpfs and symlinks

pkg/cmd/container/create.go

@@ -50,6 +50,7 @@ import (
 	"github.com/containerd/nerdctl/pkg/namestore"
 	"github.com/containerd/nerdctl/pkg/platformutil"
 	"github.com/containerd/nerdctl/pkg/referenceutil"
+	"github.com/containerd/nerdctl/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/pkg/strutil"
 	dockercliopts "github.com/docker/cli/opts"
 	dockeropts "github.com/docker/docker/opts"
@@ -418,6 +419,29 @@ func GenerateLogURI(dataStore string) (*url.URL, error) {
 }
 
 func withNerdctlOCIHook(cmd string, args []string) (oci.SpecOpts, error) {
+	if rootlessutil.IsRootless() {
+		detachedNetNS, err := rootlessutil.DetachedNetNS()
+		if err != nil {
+			return nil, fmt.Errorf("failed to check whether RootlessKit is running with --detach-netns: %w", err)
+		}
+		if detachedNetNS != "" {
+			// Rewrite {cmd, args} if RootlessKit with running with --detach-netns, so that the hook can gain
+			// CAP_NET_ADMIN in the namespaces.
+			//   - Old:
+			//     - cmd:  "/usr/local/bin/nerdctl"
+			//     - args: {"--data-root=/foo", "internal", "oci-hook"}
+			//   - New:
+			//     - cmd:  "/usr/bin/nsenter"
+			//     - args: {"-n/run/user/1000/containerd-rootless/netns", "-F", "--", "/usr/local/bin/nerdctl", "--data-root=/foo", "internal", "oci-hook"}
+			oldCmd, oldArgs := cmd, args
+			cmd, err = exec.LookPath("nsenter")
+			if err != nil {
+				return nil, err
+			}
+			args = append([]string{"-n" + detachedNetNS, "-F", "--", oldCmd}, oldArgs...)
+		}
+	}
+
 	args = append([]string{cmd}, append(args, "internal", "oci-hook")...)
 	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
 		if s.Hooks == nil {

pkg/containerutil/container_network_manager.go

@@ -38,6 +38,7 @@ import (
 	"github.com/containerd/nerdctl/pkg/mountutil"
 	"github.com/containerd/nerdctl/pkg/netutil"
 	"github.com/containerd/nerdctl/pkg/netutil/nettype"
+	"github.com/containerd/nerdctl/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/pkg/strutil"
 	"github.com/opencontainers/runtime-spec/specs-go"
 )
@@ -461,9 +462,49 @@ func (m *hostNetworkManager) ContainerNetworkingOpts(_ context.Context, containe
 		}
 	}
 
+	if rootlessutil.IsRootless() {
+		detachedNetNS, err := rootlessutil.DetachedNetNS()
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to check whether RootlessKit is running with --detach-netns: %w", err)
+		}
+		if detachedNetNS != "" {
+			// For rootless + host netns, we can't mount sysfs.
+			// We can't (non-recursively) bind mount /sys, either.
+			//
+			// TODO: consider to just rbind /sys from the host with rro,
+			// when rro is available (kernel >= 5.12, runc >= 1.1).
+			//
+			// Relevant: https://github.com/moby/buildkit/blob/v0.12.4/util/rootless/specconv/specconv_linux.go#L15-L34
+			specs = append(specs, withRemoveSysfs)
+		}
+	}
+
 	return specs, cOpts, nil
 }
 
+func withRemoveSysfs(_ context.Context, _ oci.Client, c *containers.Container, s *oci.Spec) error {
+	var hasSysfs bool
+	for _, mount := range s.Mounts {
+		if mount.Type == "sysfs" {
+			hasSysfs = true
+			break
+		}
+	}
+	if !hasSysfs {
+		// NOP, as the user has specified a custom /sys mount
+		return nil
+	}
+	var mounts []specs.Mount // nolint: prealloc
+	for _, mount := range s.Mounts {
+		if strings.HasPrefix(mount.Destination, "/sys") {
+			continue
+		}
+		mounts = append(mounts, mount)
+	}
+	s.Mounts = mounts
+	return nil
+}
+
 // types.NetworkOptionsManager implementation for CNI networking settings.
 // This is a more specialized and OS-dependendant networking model so this
 // struct provides different implementations on different platforms.

pkg/defaults/defaults_linux.go

@@ -138,6 +138,7 @@ func HostsDirs() []string {
 
 // HostGatewayIP returns the non-loop-back host ip if available and returns empty string if running into error.
 func HostGatewayIP() string {
+	// no need to use [rootlessutil.WithDetachedNetNSIfAny] here
 	addrs, err := net.InterfaceAddrs()
 	if err != nil {
 		return ""

pkg/netutil/netutil_unix.go

@@ -31,6 +31,7 @@ import (
 	"github.com/Masterminds/semver/v3"
 	"github.com/containerd/log"
 	"github.com/containerd/nerdctl/pkg/defaults"
+	"github.com/containerd/nerdctl/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/pkg/strutil"
 	"github.com/containerd/nerdctl/pkg/systemutil"
 	"github.com/mitchellh/mapstructure"
@@ -322,11 +323,13 @@ func guessFirewallPluginVersion(stderr string) (*semver.Version, error) {
 }
 
 func removeBridgeNetworkInterface(netIf string) error {
-	link, err := netlink.LinkByName(netIf)
-	if err == nil {
-		if err := netlink.LinkDel(link); err != nil {
-			return fmt.Errorf("failed to remove network interface %s: %v", netIf, err)
+	return rootlessutil.WithDetachedNetNSIfAny(func() error {
+		link, err := netlink.LinkByName(netIf)
+		if err == nil {
+			if err := netlink.LinkDel(link); err != nil {
+				return fmt.Errorf("failed to remove network interface %s: %v", netIf, err)
+			}
 		}
-	}
-	return nil
+		return nil
+	})
 }

pkg/netutil/subnet/subnet.go

@@ -19,11 +19,17 @@ package subnet
 import (
 	"fmt"
 	"net"
+
+	"github.com/containerd/nerdctl/pkg/rootlessutil"
 )
 
 func GetLiveNetworkSubnets() ([]*net.IPNet, error) {
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
+	var addrs []net.Addr
+	if err := rootlessutil.WithDetachedNetNSIfAny(func() error {
+		var err2 error
+		addrs, err2 = net.InterfaceAddrs()
+		return err2
+	}); err != nil {
 		return nil, err
 	}
 	nets := make([]*net.IPNet, 0, len(addrs))

pkg/rootlessutil/parent_linux.go

@@ -77,6 +77,16 @@ func ParentMain(hostGatewayIP string) error {
 		return err
 	}
 
+	detachedNetNSPath, err := detachedNetNS(stateDir)
+	if err != nil {
+		return err
+	}
+	detachNetNSMode := detachedNetNSPath != ""
+	log.L.Debugf("RootlessKit detach-netns mode: %v", detachNetNSMode)
+	if err != nil {
+		return err
+	}
+
 	wd, err := os.Getwd()
 	if err != nil {
 		return err
@@ -92,10 +102,13 @@ func ParentMain(hostGatewayIP string) error {
 		"-r/",     // root dir (busybox nsenter wants this to be explicitly specified),
 		"-w" + wd, // work dir
 		"--preserve-credentials",
-		"-m", "-n", "-U",
+		"-m", "-U",
 		"-t", strconv.Itoa(childPid),
 		"-F", // no fork
 	}
+	if !detachNetNSMode {
+		args = append(args, "-n")
+	}
 	args = append(args, os.Args...)
 	log.L.Debugf("rootless parent main: executing %q with %v", arg0, args)