From 7841aea2db39fe550108cd300fb542464093fcb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 25 Oct 2020 08:45:19 +0100 Subject: [PATCH 1/5] Optionally use ssh for docker-env instead of tcp Since Docker 18.09, it supports ssh: as well as tcp: for connecting to the docker daemon (in $DOCKER_HOST). This avoids setting up certificates, but leaves key handling (host keys and identity keys) to ssh config. --- cmd/minikube/cmd/docker-env.go | 62 ++++++++++++++++++--- cmd/minikube/cmd/docker-env_test.go | 13 +++++ site/content/en/docs/commands/docker-env.md | 1 + 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index 5f34398ddd17..a9391b2cc8a8 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -43,7 +43,7 @@ import ( "k8s.io/minikube/pkg/minikube/sysinit" ) -var dockerEnvTmpl = fmt.Sprintf("{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerCertPath }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubeDockerdProfile }}{{ .Suffix }}{{ if .NoProxyVar }}{{ .Prefix }}{{ .NoProxyVar }}{{ .Delimiter }}{{ .NoProxyValue }}{{ .Suffix }}{{end}}{{ .UsageHint }}", constants.DockerTLSVerifyEnv, constants.DockerHostEnv, constants.DockerCertPathEnv, constants.MinikubeActiveDockerdEnv) +var dockerEnvTmpl = fmt.Sprintf("{{ if .DockerTLSVerify }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}{{ end }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}{{ if .DockerCertPath }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerCertPath }}{{ .Suffix }}{{ end }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubeDockerdProfile }}{{ .Suffix }}{{ if .NoProxyVar }}{{ .Prefix }}{{ .NoProxyVar }}{{ .Delimiter }}{{ .NoProxyValue }}{{ .Suffix }}{{ end }}{{ .UsageHint }}", constants.DockerTLSVerifyEnv, constants.DockerHostEnv, constants.DockerCertPathEnv, constants.MinikubeActiveDockerdEnv) // DockerShellConfig represents the shell config for Docker type DockerShellConfig struct { @@ -58,6 +58,7 @@ type DockerShellConfig struct { var ( noProxy bool + sshHost bool dockerUnset bool defaultNoProxyGetter NoProxyGetter ) @@ -75,12 +76,19 @@ func dockerShellCfgSet(ec DockerEnvConfig, envMap map[string]string) *DockerShel profile := ec.profile const usgPlz = "To point your shell to minikube's docker-daemon, run:" usgCmd := fmt.Sprintf("minikube -p %s docker-env", profile) + if ec.ssh { + usgCmd += " --ssh-host" + } s := &DockerShellConfig{ Config: *shell.CfgSet(ec.EnvConfig, usgPlz, usgCmd), } - s.DockerCertPath = envMap[constants.DockerCertPathEnv] + if !ec.ssh { + s.DockerCertPath = envMap[constants.DockerCertPathEnv] + } s.DockerHost = envMap[constants.DockerHostEnv] - s.DockerTLSVerify = envMap[constants.DockerTLSVerifyEnv] + if !ec.ssh { + s.DockerTLSVerify = envMap[constants.DockerTLSVerifyEnv] + } s.MinikubeDockerdProfile = envMap[constants.MinikubeActiveDockerdEnv] if ec.noProxy { @@ -162,11 +170,13 @@ var dockerEnvCmd = &cobra.Command{ out.V{"runtime": co.Config.KubernetesConfig.ContainerRuntime}) } - if ok := isDockerActive(co.CP.Runner); !ok { + r := co.CP.Runner + if ok := isDockerActive(r); !ok { klog.Warningf("dockerd is not active will try to restart it...") - mustRestartDocker(cname, co.CP.Runner) + mustRestartDocker(cname, r) } + d := co.CP.Host.Driver var err error port := constants.DockerDaemonPort if driver.NeedsPortForward(driverName) { @@ -176,14 +186,30 @@ var dockerEnvCmd = &cobra.Command{ } } + hostname, err := d.GetSSHHostname() + if err != nil { + exit.Error(reason.IfSSHClient, "Error getting ssh client", err) + } + + sshport, err := d.GetSSHPort() + if err != nil { + exit.Error(reason.IfSSHClient, "Error getting ssh client", err) + } + + hostIP := co.CP.IP.String() ec := DockerEnvConfig{ EnvConfig: sh, profile: cname, driver: driverName, - hostIP: co.CP.IP.String(), + ssh: sshHost, + hostIP: hostIP, port: port, certsDir: localpath.MakeMiniPath("certs"), noProxy: noProxy, + username: d.GetSSHUsername(), + hostname: hostname, + sshport: sshport, + keypath: d.GetSSHKeyPath(), } if ec.Shell == "" { @@ -219,10 +245,15 @@ type DockerEnvConfig struct { shell.EnvConfig profile string driver string + ssh bool hostIP string port int certsDir string noProxy bool + username string + hostname string + sshport int + keypath string } // dockerSetScript writes out a shell-compatible 'docker-env' script @@ -255,15 +286,31 @@ func dockerURL(ip string, port int) string { return fmt.Sprintf("tcp://%s", net.JoinHostPort(ip, strconv.Itoa(port))) } +// sshURL returns the docker endpoint URL when using socket over ssh. +func sshURL(username string, hostname string, port int) string { + // assumes standard /var/run/docker.sock as the path (not possible to set it at the moment) + return fmt.Sprintf("ssh://%s@%s", username, net.JoinHostPort(hostname, strconv.Itoa(port))) +} + // dockerEnvVars gets the necessary docker env variables to allow the use of minikube's docker daemon func dockerEnvVars(ec DockerEnvConfig) map[string]string { - env := map[string]string{ + envTCP := map[string]string{ constants.DockerTLSVerifyEnv: "1", constants.DockerHostEnv: dockerURL(ec.hostIP, ec.port), constants.DockerCertPathEnv: ec.certsDir, constants.MinikubeActiveDockerdEnv: ec.profile, } + envSSH := map[string]string{ + constants.DockerHostEnv: sshURL(ec.username, ec.hostname, ec.sshport), + constants.MinikubeActiveDockerdEnv: ec.profile, + } + var env map[string]string + if ec.ssh { + env = envSSH + } else { + env = envTCP + } return env } @@ -288,6 +335,7 @@ func tryDockerConnectivity(bin string, ec DockerEnvConfig) ([]byte, error) { func init() { defaultNoProxyGetter = &EnvNoProxyGetter{} dockerEnvCmd.Flags().BoolVar(&noProxy, "no-proxy", false, "Add machine IP to NO_PROXY environment variable") + dockerEnvCmd.Flags().BoolVar(&sshHost, "ssh-host", false, "Use SSH connection instead of HTTPS (port 2376)") dockerEnvCmd.Flags().StringVar(&shell.ForceShell, "shell", "", "Force environment to be configured for a specified shell: [fish, cmd, powershell, tcsh, bash, zsh], default is auto-detect") dockerEnvCmd.Flags().BoolVarP(&dockerUnset, "unset", "u", false, "Unset variables instead of setting them") } diff --git a/cmd/minikube/cmd/docker-env_test.go b/cmd/minikube/cmd/docker-env_test.go index aaa1d97f5ba1..fbb1c03daf69 100644 --- a/cmd/minikube/cmd/docker-env_test.go +++ b/cmd/minikube/cmd/docker-env_test.go @@ -51,6 +51,19 @@ export MINIKUBE_ACTIVE_DOCKERD="dockerdriver" # To point your shell to minikube's docker-daemon, run: # eval $(minikube -p dockerdriver docker-env) +`, + `unset DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH MINIKUBE_ACTIVE_DOCKERD +`, + }, + { + "bash", + DockerEnvConfig{profile: "dockerdriver", driver: "docker", ssh: true, username: "root", hostname: "host", sshport: 22}, + nil, + `export DOCKER_HOST="ssh://root@host:22" +export MINIKUBE_ACTIVE_DOCKERD="dockerdriver" + +# To point your shell to minikube's docker-daemon, run: +# eval $(minikube -p dockerdriver docker-env --ssh-host) `, `unset DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH MINIKUBE_ACTIVE_DOCKERD `, diff --git a/site/content/en/docs/commands/docker-env.md b/site/content/en/docs/commands/docker-env.md index 7572c9ab0dc9..5cf577041382 100644 --- a/site/content/en/docs/commands/docker-env.md +++ b/site/content/en/docs/commands/docker-env.md @@ -23,6 +23,7 @@ minikube docker-env [flags] -h, --help help for docker-env --no-proxy Add machine IP to NO_PROXY environment variable --shell string Force environment to be configured for a specified shell: [fish, cmd, powershell, tcsh, bash, zsh], default is auto-detect + --ssh-host Use SSH connection instead of HTTPS (port 2376) -u, --unset Unset variables instead of setting them ``` From b57bf9abcf2aa29867257bb775b5e6503a4e5705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 1 Nov 2020 15:07:02 +0100 Subject: [PATCH 2/5] Refactor dockerUnsetScript with a helper function --- cmd/minikube/cmd/docker-env.go | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index a9391b2cc8a8..4d421158f70b 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -264,20 +264,7 @@ func dockerSetScript(ec DockerEnvConfig, w io.Writer) error { // dockerSetScript writes out a shell-compatible 'docker-env unset' script func dockerUnsetScript(ec DockerEnvConfig, w io.Writer) error { - vars := []string{ - constants.DockerTLSVerifyEnv, - constants.DockerHostEnv, - constants.DockerCertPathEnv, - constants.MinikubeActiveDockerdEnv, - } - - if ec.noProxy { - k, _ := defaultNoProxyGetter.GetNoProxyVar() - if k != "" { - vars = append(vars, k) - } - } - + vars := dockerEnvNames(ec) return shell.UnsetScript(ec.EnvConfig, w, vars) } @@ -314,6 +301,24 @@ func dockerEnvVars(ec DockerEnvConfig) map[string]string { return env } +// dockerEnvNames gets the necessary docker env variables to reset after using minikube's docker daemon +func dockerEnvNames(ec DockerEnvConfig) []string { + vars := []string{ + constants.DockerTLSVerifyEnv, + constants.DockerHostEnv, + constants.DockerCertPathEnv, + constants.MinikubeActiveDockerdEnv, + } + + if ec.noProxy { + k, _ := defaultNoProxyGetter.GetNoProxyVar() + if k != "" { + vars = append(vars, k) + } + } + return vars +} + // dockerEnvVarsList gets the necessary docker env variables to allow the use of minikube's docker daemon to be used in a exec.Command func dockerEnvVarsList(ec DockerEnvConfig) []string { return []string{ From 274be7f52df1156535a08a7eb71254d9eb4d54a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 1 Nov 2020 15:07:19 +0100 Subject: [PATCH 3/5] Split template for docker tcp and ssh in two vars --- cmd/minikube/cmd/docker-env.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index 4d421158f70b..639d80561e40 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -43,7 +43,9 @@ import ( "k8s.io/minikube/pkg/minikube/sysinit" ) -var dockerEnvTmpl = fmt.Sprintf("{{ if .DockerTLSVerify }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}{{ end }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}{{ if .DockerCertPath }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerCertPath }}{{ .Suffix }}{{ end }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubeDockerdProfile }}{{ .Suffix }}{{ if .NoProxyVar }}{{ .Prefix }}{{ .NoProxyVar }}{{ .Delimiter }}{{ .NoProxyValue }}{{ .Suffix }}{{ end }}{{ .UsageHint }}", constants.DockerTLSVerifyEnv, constants.DockerHostEnv, constants.DockerCertPathEnv, constants.MinikubeActiveDockerdEnv) +var dockerEnvTCPTmpl = fmt.Sprintf("{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerCertPath }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubeDockerdProfile }}{{ .Suffix }}{{ if .NoProxyVar }}{{ .Prefix }}{{ .NoProxyVar }}{{ .Delimiter }}{{ .NoProxyValue }}{{ .Suffix }}{{end}}{{ .UsageHint }}", constants.DockerTLSVerifyEnv, constants.DockerHostEnv, constants.DockerCertPathEnv, constants.MinikubeActiveDockerdEnv) + +var dockerEnvSSHTmpl = fmt.Sprintf("{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubeDockerdProfile }}{{ .Suffix }}{{ .UsageHint }}", constants.DockerHostEnv, constants.MinikubeActiveDockerdEnv) // DockerShellConfig represents the shell config for Docker type DockerShellConfig struct { @@ -258,6 +260,12 @@ type DockerEnvConfig struct { // dockerSetScript writes out a shell-compatible 'docker-env' script func dockerSetScript(ec DockerEnvConfig, w io.Writer) error { + var dockerEnvTmpl string + if ec.ssh { + dockerEnvTmpl = dockerEnvSSHTmpl + } else { + dockerEnvTmpl = dockerEnvTCPTmpl + } envVars := dockerEnvVars(ec) return shell.SetScript(ec.EnvConfig, w, dockerEnvTmpl, dockerShellCfgSet(ec, envVars)) } From d2e2ecaf401527b799cca5a92146acc293827a87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 8 Nov 2020 15:14:39 +0100 Subject: [PATCH 4/5] Add option to add the ssh key to the ssh agent --- cmd/minikube/cmd/docker-env.go | 16 ++++++++++++++++ site/content/en/docs/commands/docker-env.md | 1 + 2 files changed, 17 insertions(+) diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index 639d80561e40..4b6f1ff01846 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -61,6 +61,7 @@ type DockerShellConfig struct { var ( noProxy bool sshHost bool + sshAdd bool dockerUnset bool defaultNoProxyGetter NoProxyGetter ) @@ -239,6 +240,20 @@ var dockerEnvCmd = &cobra.Command{ if err := dockerSetScript(ec, os.Stdout); err != nil { exit.Error(reason.InternalDockerScript, "Error generating set output", err) } + + if sshAdd { + klog.Infof("Adding %v", d.GetSSHKeyPath()) + + path, err := exec.LookPath("ssh-add") + if err != nil { + exit.Error(reason.IfSSHClient, "Error with ssh-add", err) + } + + err = exec.Command(path, d.GetSSHKeyPath()).Run() + if err != nil { + exit.Error(reason.IfSSHClient, "Error with ssh-add", err) + } + } }, } @@ -349,6 +364,7 @@ func init() { defaultNoProxyGetter = &EnvNoProxyGetter{} dockerEnvCmd.Flags().BoolVar(&noProxy, "no-proxy", false, "Add machine IP to NO_PROXY environment variable") dockerEnvCmd.Flags().BoolVar(&sshHost, "ssh-host", false, "Use SSH connection instead of HTTPS (port 2376)") + dockerEnvCmd.Flags().BoolVar(&sshAdd, "ssh-add", false, "Add SSH identity key to SSH authentication agent") dockerEnvCmd.Flags().StringVar(&shell.ForceShell, "shell", "", "Force environment to be configured for a specified shell: [fish, cmd, powershell, tcsh, bash, zsh], default is auto-detect") dockerEnvCmd.Flags().BoolVarP(&dockerUnset, "unset", "u", false, "Unset variables instead of setting them") } diff --git a/site/content/en/docs/commands/docker-env.md b/site/content/en/docs/commands/docker-env.md index 5cf577041382..690137583c6d 100644 --- a/site/content/en/docs/commands/docker-env.md +++ b/site/content/en/docs/commands/docker-env.md @@ -23,6 +23,7 @@ minikube docker-env [flags] -h, --help help for docker-env --no-proxy Add machine IP to NO_PROXY environment variable --shell string Force environment to be configured for a specified shell: [fish, cmd, powershell, tcsh, bash, zsh], default is auto-detect + --ssh-add Add SSH identity key to SSH authentication agent --ssh-host Use SSH connection instead of HTTPS (port 2376) -u, --unset Unset variables instead of setting them ``` From 88b884378ecb7e0346b0bf74f7934a7e5c9facb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 8 Nov 2020 15:28:00 +0100 Subject: [PATCH 5/5] Show result from running the ssh-add command --- cmd/minikube/cmd/docker-env.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index 4b6f1ff01846..000d3f93a0eb 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -249,7 +249,9 @@ var dockerEnvCmd = &cobra.Command{ exit.Error(reason.IfSSHClient, "Error with ssh-add", err) } - err = exec.Command(path, d.GetSSHKeyPath()).Run() + cmd := exec.Command(path, d.GetSSHKeyPath()) + cmd.Stderr = os.Stderr + err = cmd.Run() if err != nil { exit.Error(reason.IfSSHClient, "Error with ssh-add", err) }