net: switch to socket_vmnet (#459)

* net: fix vmnet build script

* net: modify vmnet build script

* net: add socket_vmnet binary

* misc: remove inefficient code

* net: fix vmnet build script

* net: port to socket_vmnet

* net: fix vmnet_socket

* net: fix network socket when gvproxy is enabled

app/app.go

@@ -312,6 +312,13 @@ func (c colimaApp) Status() error {
 	if conf, err := limautil.InstanceConfig(); err == nil {
 		log.Println("mountType:", conf.MountType)
 	}
+
+	// ip address
+	if ipAddress := limautil.IPAddress(config.CurrentProfile().ID); ipAddress != "127.0.0.1" {
+		log.Println("address:", ipAddress)
+	}
+
+	// docker socket
 	if currentRuntime == docker.Name {
 		log.Println("socket:", "unix://"+docker.HostSocketFile())
 	}

cmd/colima/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"net"
 	"os"
 	"os/exec"
@@ -14,6 +15,7 @@ import (
 	"github.com/abiosoft/colima/cmd/root"
 	"github.com/abiosoft/colima/config"
 	"github.com/abiosoft/colima/daemon/process/gvproxy"
+	"github.com/abiosoft/colima/daemon/process/vmnet"
 	"github.com/sirupsen/logrus"
 )
 
@@ -32,36 +34,52 @@ func qemuWrapper(qemu string) {
 		config.SetProfile(profile)
 	}
 
-	info := gvproxy.Info()
+	gvproxyInfo := gvproxy.Info()
+	vmnetInfo := vmnet.Info()
 
 	// check if qemu is meant to run by lima
 	// decided by -pidfile flag
 	qemuRunning := false
-	for i := 0; i < len(os.Args)-1; i++ {
-		if os.Args[i] == "-pidfile" {
+	for _, arg := range os.Args {
+		if arg == "-pidfile" {
 			qemuRunning = true
 			break
 		}
 	}
 
 	args := os.Args[1:] // forward all args
-	var fd *os.File
+	var extraFiles []*os.File
 
 	gvproxyEnabled, _ := strconv.ParseBool(os.Getenv(gvproxy.SubProcessEnvVar))
+	vmnetEnabled, _ := strconv.ParseBool(os.Getenv(vmnet.SubProcessEnvVar))
 
 	if qemuRunning && gvproxyEnabled {
-		conn, err := net.Dial("unix", info.Socket.File())
-		if err != nil {
-			logrus.Fatal(err)
+		// vmnet should come first as it would be added by Lima and would have the fd 3
+
+		// vmnet
+		if vmnetEnabled {
+			fd := os.NewFile(3, vmnetInfo.Socket.File())
+			extraFiles = append(extraFiles, fd)
 		}
-		fd, err = conn.(*net.UnixConn).File()
-		if err != nil {
-			logrus.Fatal(err)
+
+		// gvproxy
+		{
+			conn, err := net.Dial("unix", gvproxyInfo.Socket.File())
+			if err != nil {
+				logrus.Fatal(fmt.Errorf("error connecting to gvproxy socket: %w", err))
+			}
+			fd, err := conn.(*net.UnixConn).File()
+			if err != nil {
+				logrus.Fatal(fmt.Errorf("error retrieving fd for gvproxy socket: %w", err))
+			}
+			extraFiles = append(extraFiles, fd)
 		}
 
+		// gvproxy fd
+		fd := strconv.Itoa(2 + len(extraFiles))
 		args = append(args,
-			"-netdev", "socket,id=vlan,fd=3",
-			"-device", "virtio-net-pci,netdev=vlan,mac="+info.MacAddress,
+			"-netdev", "socket,id=vlan,fd="+fd,
+			"-device", "virtio-net-pci,netdev=vlan,mac="+gvproxyInfo.MacAddress,
 		)
 	}
 
@@ -71,8 +89,8 @@ func qemuWrapper(qemu string) {
 	cmd.Stderr = os.Stderr
 	cmd.Stdin = os.Stdin
 
-	if fd != nil {
-		cmd.ExtraFiles = append(cmd.ExtraFiles, fd)
+	if len(extraFiles) > 0 {
+		cmd.ExtraFiles = append(cmd.ExtraFiles, extraFiles...)
 	}
 
 	err := cmd.Run()

cmd/start.go

@@ -98,6 +98,8 @@ const (
 	defaultNetworkDriver     = gvproxy.Name
 )
 
+var defaultKubernetesDisable = []string{"traefik"}
+
 var startCmdArgs struct {
 	config.Config
 
@@ -148,7 +150,7 @@ func init() {
 	startCmd.Flags().BoolVarP(&startCmdArgs.Kubernetes.Enabled, "kubernetes", "k", false, "start with Kubernetes")
 	startCmd.Flags().BoolVar(&startCmdArgs.Flags.LegacyKubernetes, "with-kubernetes", false, "start with Kubernetes")
 	startCmd.Flags().StringVar(&startCmdArgs.Kubernetes.Version, "kubernetes-version", defaultKubernetesVersion, "must match a k3s version https://github.com/k3s-io/k3s/releases")
-	startCmd.Flags().StringSliceVar(&startCmdArgs.Kubernetes.Disable, "kubernetes-disable", nil, "components to disable for k3s e.g. traefik,servicelb")
+	startCmd.Flags().StringSliceVar(&startCmdArgs.Kubernetes.Disable, "kubernetes-disable", defaultKubernetesDisable, "components to disable for k3s e.g. traefik,servicelb")
 	startCmd.Flag("with-kubernetes").Hidden = true
 
 	// layer

daemon/process/gvproxy/gvproxy.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/abiosoft/colima/cli"
 	"github.com/abiosoft/colima/daemon/process"
+	"github.com/abiosoft/colima/util/osutil"
 	"github.com/abiosoft/colima/util/shautil"
 	"github.com/containers/gvisor-tap-vsock/pkg/transport"
 	"github.com/containers/gvisor-tap-vsock/pkg/types"
@@ -27,20 +28,15 @@ func New() process.Process {
 	return &gvproxyProcess{}
 }
 
-type Socket string
-
-func (s Socket) Unix() string { return "unix://" + s.File() }
-func (s Socket) File() string { return strings.TrimPrefix(string(s), "unix://") }
-
 func Info() struct {
-	Socket     Socket
+	Socket     osutil.Socket
 	MacAddress string
 } {
 	return struct {
-		Socket     Socket
+		Socket     osutil.Socket
 		MacAddress string
 	}{
-		Socket:     Socket(filepath.Join(process.Dir(), socketFileName)),
+		Socket:     osutil.Socket(filepath.Join(process.Dir(), socketFileName)),
 		MacAddress: MacAddress(),
 	}
 }
@@ -130,7 +126,7 @@ func configuration() types.Configuration {
 	}
 }
 
-func run(ctx context.Context, qemuSocket Socket) error {
+func run(ctx context.Context, qemuSocket osutil.Socket) error {
 	if _, err := os.Stat(qemuSocket.File()); err == nil {
 		if err := os.Remove(qemuSocket.File()); err != nil {
 			return fmt.Errorf("error removing existing qemu socket: %w", err)

daemon/process/vmnet/deps.go

@@ -57,8 +57,8 @@ func (s sudoerFile) Install(host environment.HostActions) error {
 
 var _ process.Dependency = vmnetFile{}
 
-const BinaryPath = "/opt/colima/bin/vde_vmnet"
-const LibraryPath = "/opt/colima/lib/libvdeplug.3.dylib"
+const BinaryPath = "/opt/colima/bin/socket_vmnet"
+const ClientBinaryPath = "/opt/colima/bin/socket_vmnet_client"
 
 type vmnetFile struct{}
 
@@ -73,7 +73,7 @@ func (v vmnetFile) Installed() bool {
 }
 
 func (v vmnetFile) bins() []string {
-	return []string{BinaryPath, LibraryPath}
+	return []string{BinaryPath, ClientBinaryPath}
 }
 func (v vmnetFile) Install(host environment.HostActions) error {
 	arch := "x86_64"
@@ -106,7 +106,7 @@ func (v vmnetFile) Install(host environment.HostActions) error {
 	if err := host.RunInteractive("sudo", "mkdir", "-p", dir); err != nil {
 		return fmt.Errorf("error preparing colima privileged dir: %w", err)
 	}
-	if err := host.RunInteractive("sudo", "sh", "-c", fmt.Sprintf("cd %s && tar xfz %s", dir, f.Name())); err != nil {
+	if err := host.RunInteractive("sudo", "sh", "-c", fmt.Sprintf("cd %s && tar xfz %s 2>/dev/null", dir, f.Name())); err != nil {
 		return fmt.Errorf("error extracting vmnet archive: %w", err)
 	}
 

daemon/process/vmnet/vmnet.go

@@ -3,18 +3,23 @@ package vmnet
 import (
 	"context"
 	"fmt"
+	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
 
 	"github.com/abiosoft/colima/cli"
 	"github.com/abiosoft/colima/config"
 	"github.com/abiosoft/colima/daemon/process"
+	"github.com/abiosoft/colima/util/osutil"
+	"github.com/sirupsen/logrus"
 )
 
 const Name = "vmnet"
 
 const (
+	SubProcessEnvVar = "COLIMA_VMNET"
+
 	NetGateway   = "192.168.106.1"
 	NetDHCPEnd   = "192.168.106.254"
 	NetInterface = "col0"
@@ -29,7 +34,7 @@ type vmnetProcess struct{}
 func (*vmnetProcess) Alive(ctx context.Context) error {
 	info := Info()
 	pidFile := info.PidFile
-	ptpFile := info.PTPFile
+	socketFile := info.Socket.File()
 
 	if _, err := os.Stat(pidFile); err == nil {
 		cmd := exec.CommandContext(ctx, "sudo", "pkill", "-0", "-F", pidFile)
@@ -38,8 +43,15 @@ func (*vmnetProcess) Alive(ctx context.Context) error {
 		}
 	}
 
-	if _, err := os.Stat(ptpFile); err != nil {
-		return fmt.Errorf("vmnet ptp file error: %w", err)
+	if _, err := os.Stat(socketFile); err != nil {
+		return fmt.Errorf("vmnet socket file not found error: %w", err)
+	}
+	if n, err := net.Dial("unix", socketFile); err != nil {
+		return fmt.Errorf("vmnet socket file error: %w", err)
+	} else {
+		if err := n.Close(); err != nil {
+			logrus.Debugln(fmt.Errorf("error closing ping socket connection: %w", err))
+		}
 	}
 
 	return nil
@@ -51,27 +63,31 @@ func (*vmnetProcess) Name() string { return Name }
 // Start implements process.BgProcess
 func (*vmnetProcess) Start(ctx context.Context) error {
 	info := Info()
-	ptp := info.PTPFile
+	socket := info.Socket.File()
 	pid := info.PidFile
 
 	// delete existing sockets if exist
 	// errors ignored on purpose
-	_ = forceDeleteFileIfExists(ptp)
-	_ = forceDeleteFileIfExists(ptp + "+") // created by running qemu instance
+	_ = forceDeleteFileIfExists(socket)
 
 	done := make(chan error, 1)
 
 	go func() {
 		// rootfully start the vmnet daemon
 		command := cli.CommandInteractive("sudo", BinaryPath,
 			"--vmnet-mode", "shared",
-			"--vde-group", "staff",
+			"--socket-group", "staff",
 			"--vmnet-gateway", NetGateway,
 			"--vmnet-dhcp-end", NetDHCPEnd,
 			"--pidfile", pid,
-			ptp+"[]",
+			socket,
 		)
 
+		if cli.Settings.Verbose {
+			command.Env = append(command.Env, os.Environ()...)
+			command.Env = append(command.Env, "DEBUG=1")
+		}
+
 		done <- command.Run()
 	}()
 
@@ -118,13 +134,13 @@ func forceDeleteFileIfExists(name string) error {
 
 func Info() struct {
 	PidFile string
-	PTPFile string
+	Socket  osutil.Socket
 } {
 	return struct {
 		PidFile string
-		PTPFile string
+		Socket  osutil.Socket
 	}{
 		PidFile: filepath.Join(runDir(), "vmnet-"+config.CurrentProfile().ShortName+".pid"),
-		PTPFile: filepath.Join(process.Dir(), "vmnet.ptp"),
+		Socket:  osutil.Socket(filepath.Join(process.Dir(), "vmnet.sock")),
 	}
 }

embedded/defaults/colima.yaml

@@ -36,8 +36,8 @@ kubernetes:
   # EXAMPLE - disable traefik and metrics-server
   # disable: [traefik, metrics-server]
   #
-  # Default: []
-  disable: []
+  # Default: [traefik]
+  disable: [traefik]
 
 # Auto-activate on the Host for client access.
 # Setting to true does the following on startup

embedded/network/sudo.txt

@@ -1,5 +1,5 @@
 # starting vmnet daemon
-%staff ALL=(root:wheel) NOPASSWD:NOSETENV: /opt/colima/bin/vde_vmnet --vmnet-mode shared --vde-group staff --vmnet-gateway 192.168.106.1 --vmnet-dhcp-end 192.168.106.254 *
+%staff ALL=(root:wheel) NOPASSWD:NOSETENV: /opt/colima/bin/socket_vmnet --vmnet-mode shared --socket-group staff --vmnet-gateway 192.168.106.1 --vmnet-dhcp-end 192.168.106.254 *
 # terminating vmnet daemon
 %staff ALL=(root:wheel) NOPASSWD:NOSETENV: /usr/bin/pkill -F /opt/colima/run/*.pid
 # validating vmnet daemon

environment/container/kubernetes/kubernetes.go

@@ -18,7 +18,7 @@ import (
 
 const (
 	Name           = "kubernetes"
-	DefaultVersion = "v1.25.2+k3s1"
+	DefaultVersion = "v1.25.3+k3s1"
 
 	configKey = "kubernetes_config"
 )

environment/vm/lima/lima.go

@@ -176,11 +176,17 @@ func (l *limaVM) startDaemon(ctx context.Context, conf config.Config) (context.C
 		}()
 	}
 
+	// preserve vmnet context
+	if vmnetEnabled, _ := ctx.Value(daemon.CtxKey(vmnet.Name)).(bool); vmnetEnabled {
+		// env var for subprocess to detect vmnet
+		l.host = l.host.WithEnv(vmnet.SubProcessEnvVar + "=1")
+	}
+
 	// preserve gvproxy context
 	if gvproxyEnabled, _ := ctx.Value(daemon.CtxKey(gvproxy.Name)).(bool); gvproxyEnabled {
 		var envs []string
 
-		// env var for subproxy to detect gvproxy
+		// env var for subprocess to detect gvproxy
 		envs = append(envs, gvproxy.SubProcessEnvVar+"=1")
 		// use qemu wrapper for Lima by specifying wrapper binaries via env var
 		envs = append(envs, qemu.LimaDir().BinsEnvVar()...)

environment/vm/lima/limautil/limautil.go

@@ -38,6 +38,7 @@ func InstanceConfig() (config.Config, error) {
 // IPAddress returns the ip address for profile.
 // It returns the PTP address if networking is enabled or falls back to 127.0.0.1.
 // It is guaranteed to return a value.
+//
 // TODO: unnecessary round-trip is done to get instance details from Lima.
 func IPAddress(profileID string) string {
 	// profile = toUserFriendlyName(profile)