Bind mount the cache directory on Linux KIC

pkg/drivers/kic/oci/oci.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/detect"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/util/retry"
 )
@@ -140,6 +141,34 @@ func hasMemorySwapCgroup() bool {
 	return memcgSwap
 }
 
+func hasCPUCfsPeriod() bool {
+	cpuCfsPeriod := true
+	if runtime.GOOS == "linux" {
+		if _, err := os.Stat("/sys/fs/cgroup/cpu/cpu.cfs_period_us"); os.IsNotExist(err) {
+			cpuCfsPeriod = false
+		}
+		if !cpuCfsPeriod {
+			// requires CONFIG_CFS_BANDWIDTH
+			klog.Warning("Your kernel does not support CPU cfs period or the cgroup is not mounted.")
+		}
+	}
+	return cpuCfsPeriod
+}
+
+func hasCPUCfsQuota() bool {
+	cpuCfsQuota := true
+	if runtime.GOOS == "linux" {
+		if _, err := os.Stat("/sys/fs/cgroup/cpu/cpu.cfs_quota_us"); os.IsNotExist(err) {
+			cpuCfsQuota = false
+		}
+		if !cpuCfsQuota {
+			// requires CONFIG_CFS_BANDWIDTH
+			klog.Warning("Your kernel does not support CPU cfs quota or the cgroup is not mounted.")
+		}
+	}
+	return cpuCfsQuota
+}
+
 // CreateContainerNode creates a new container node
 func CreateContainerNode(p CreateParams) error {
 	// on windows os, if docker desktop is using Windows Containers. Exit early with error
@@ -185,56 +214,44 @@ func CreateContainerNode(p CreateParams) error {
 		runArgs = append(runArgs, "--ip", p.IP)
 	}
 
-	memcgSwap := hasMemorySwapCgroup()
-	memcg := HasMemoryCgroup()
+	if runtime.GOOS == "linux" && !IsExternalDaemonHost(p.OCIBinary) {
+		// make sure directory exists, to avoid docker creating it as the root user
+		if err := os.MkdirAll(detect.ImageCacheDir(), 0755); err != nil {
+			return errors.Wrap(err, "mkdir cache")
+		}
+
+		// bind-mount the image cache, for faster loading of cached images (without scp)
+		runArgs = append(runArgs, "-v", fmt.Sprintf("%s:/cache", detect.ImageCacheDir()))
+	}
 
 	// https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
 	var virtualization string
 	if p.OCIBinary == Podman { // enable execing in /var
 		// podman mounts var/lib with no-exec by default  https://github.com/containers/libpod/issues/5103
 		runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var:exec", p.Name))
 
-		if memcgSwap {
-			runArgs = append(runArgs, fmt.Sprintf("--memory-swap=%s", p.Memory))
-		}
-
-		if memcg {
-			runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory))
-		}
-
 		virtualization = "podman" // VIRTUALIZATION_PODMAN
 	}
 	if p.OCIBinary == Docker {
 		runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name))
 		// ignore apparmore github actions docker: https://github.com/kubernetes/minikube/issues/7624
 		runArgs = append(runArgs, "--security-opt", "apparmor=unconfined")
 
-		if memcg {
-			runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory))
-		}
-		if memcgSwap {
-			// Disable swap by setting the value to match
-			runArgs = append(runArgs, fmt.Sprintf("--memory-swap=%s", p.Memory))
-		}
-
 		virtualization = "docker" // VIRTUALIZATION_DOCKER
 	}
 
-	cpuCfsPeriod := true
-	cpuCfsQuota := true
-	if runtime.GOOS == "linux" {
-		if _, err := os.Stat("/sys/fs/cgroup/cpu/cpu.cfs_period_us"); os.IsNotExist(err) {
-			cpuCfsPeriod = false
-		}
-		if _, err := os.Stat("/sys/fs/cgroup/cpu/cpu.cfs_quota_us"); os.IsNotExist(err) {
-			cpuCfsQuota = false
-		}
-		if !cpuCfsPeriod || !cpuCfsQuota {
-			// requires CONFIG_CFS_BANDWIDTH
-			klog.Warning("Your kernel does not support CPU cfs period/quota or the cgroup is not mounted.")
-		}
+	memcgSwap := hasMemorySwapCgroup()
+	memcg := HasMemoryCgroup()
+	if memcg {
+		runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory))
+	}
+	if memcgSwap {
+		// Disable swap by setting the value to match
+		runArgs = append(runArgs, fmt.Sprintf("--memory-swap=%s", p.Memory))
 	}
 
+	cpuCfsPeriod := hasCPUCfsPeriod()
+	cpuCfsQuota := hasCPUCfsQuota()
 	if cpuCfsPeriod && cpuCfsQuota {
 		runArgs = append(runArgs, fmt.Sprintf("--cpus=%s", p.CPUs))
 	}

pkg/minikube/driver/driver.go

@@ -130,6 +130,11 @@ func IsKIC(name string) bool {
 	return name == Docker || name == Podman
 }
 
+// ISKIC checks if the driver is using an external host
+func IsExternal(name string) bool {
+	return oci.IsExternalDaemonHost(name)
+}
+
 // IsDocker checks if the driver docker
 func IsDocker(name string) bool {
 	return name == Docker

pkg/minikube/machine/cache_images.go

@@ -21,8 +21,10 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"os/user"
 	"path"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -43,6 +45,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/detect"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/image"
 	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -52,6 +55,9 @@ import (
 // loadRoot is where images should be loaded from within the guest VM
 var loadRoot = path.Join(vmpath.GuestPersistentDir, "images")
 
+// cacheRoot is where images should be loaded from within the guest container
+var cacheRoot = "/cache"
+
 // loadImageLock is used to serialize image loads to avoid overloading the guest VM
 var loadImageLock sync.Mutex
 
@@ -113,6 +119,10 @@ func LoadCachedImages(cc *config.ClusterConfig, runner command.Runner, images []
 			if err == nil {
 				return nil
 			}
+			if driver.IsKIC(cc.Driver) && runtime.GOOS == "linux" && !driver.IsExternal(cc.Driver) {
+				klog.Infof("%q needs load: %v", image, err)
+				return volumeLoadCachedImage(runner, cc.KubernetesConfig, image, cacheDir)
+			}
 			klog.Infof("%q needs transfer: %v", image, err)
 			return transferAndLoadCachedImage(runner, cc.KubernetesConfig, image, cacheDir)
 		})
@@ -316,6 +326,48 @@ func transferAndLoadImage(cr command.Runner, k8s config.KubernetesConfig, src st
 	return nil
 }
 
+// volumeLoadCachedImage loads a single image directly from the mounted cache
+func volumeLoadCachedImage(cr command.Runner, k8s config.KubernetesConfig, imgName string, cacheDir string) error {
+	src := filepath.Join(cacheDir, imgName)
+	src = localpath.SanitizeCacheDir(src)
+	return volumeLoadImage(cr, k8s, src, imgName, cacheDir)
+}
+
+// volumeLoadImage loads the image directly from the mounted cache directory
+func volumeLoadImage(cr command.Runner, k8s config.KubernetesConfig, src string, imgName string, cacheDir string) error {
+	r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime, Runner: cr})
+	if err != nil {
+		return errors.Wrap(err, "runtime")
+	}
+
+	if err := removeExistingImage(r, src, imgName); err != nil {
+		return err
+	}
+
+	klog.Infof("Loading image from: %s", src)
+	if _, err := os.Stat(src); err != nil {
+		return err
+	}
+	relative, err := filepath.Rel(cacheDir, src)
+	if err != nil {
+		return err
+	}
+
+	// host cacheDir is mounted at cacheRoot
+	dst := path.Join(cacheRoot, relative)
+
+	loadImageLock.Lock()
+	defer loadImageLock.Unlock()
+
+	err = r.LoadImage(dst)
+	if err != nil {
+		return errors.Wrapf(err, "%s load %s", r.Name(), dst)
+	}
+
+	klog.Infof("Loaded %s from cache", src)
+	return nil
+}
+
 func removeExistingImage(r cruntime.Manager, src string, imgName string) error {
 	// if loading an image from tar, skip deleting as we don't have the actual image name
 	// ie. imgName = "C:\this_is_a_dir\image.tar.gz"
@@ -350,6 +402,9 @@ func SaveCachedImages(cc *config.ClusterConfig, runner command.Runner, images []
 	for _, image := range images {
 		image := image
 		g.Go(func() error {
+			if driver.IsKIC(cc.Driver) && runtime.GOOS == "linux" && !driver.IsExternal(cc.Driver) {
+				return volumeSaveCachedImage(runner, cc.KubernetesConfig, image, cacheDir)
+			}
 			return transferAndSaveCachedImage(runner, cc.KubernetesConfig, image, cacheDir)
 		})
 	}
@@ -507,6 +562,54 @@ func transferAndSaveImage(cr command.Runner, k8s config.KubernetesConfig, dst st
 	return nil
 }
 
+// volumeSaveCachedImage saves a single image from the cache
+func volumeSaveCachedImage(cr command.Runner, k8s config.KubernetesConfig, imgName string, cacheDir string) error {
+	dst := filepath.Join(cacheDir, imgName)
+	dst = localpath.SanitizeCacheDir(dst)
+	return volumeSaveImage(cr, k8s, dst, imgName, cacheDir)
+}
+
+// volumeSaveImage saves a single image
+func volumeSaveImage(cr command.Runner, k8s config.KubernetesConfig, dst string, imgName string, cacheDir string) error {
+	r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime, Runner: cr})
+	if err != nil {
+		return errors.Wrap(err, "runtime")
+	}
+
+	if !r.ImageExists(imgName, "") {
+		return errors.Errorf("image %s not found", imgName)
+	}
+
+	klog.Infof("Saving image to: %s", dst)
+	filename := filepath.Base(dst)
+
+	src := path.Join(cacheRoot, filename)
+	args := append([]string{"rm", "-f"}, src)
+	if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil {
+		return err
+	}
+	err = r.SaveImage(imgName, src)
+	if err != nil {
+		return errors.Wrapf(err, "%s save %s", r.Name(), src)
+	}
+
+	user, err := user.Current()
+	if err != nil {
+		return err
+	}
+	args = append([]string{"chown", user.Uid}, src)
+	if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil {
+		return err
+	}
+	args = append([]string{"chmod", "0644"}, src)
+	if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil {
+		return err
+	}
+
+	klog.Infof("Saved %s to cache", dst)
+	return nil
+}
+
 // pullImages pulls images to the container run time
 func pullImages(cruntime cruntime.Manager, images []string) error {
 	klog.Infof("PullImages start: %s", images)