Use cgroup root for kubelet

See #1614 and https://d2iq.com/blog/running-kind-inside-a-kubernetes-cluster-for-continuous-integration

images/base/files/usr/local/bin/entrypoint

@@ -62,6 +62,30 @@ fix_mount() {
   mount --make-rshared /
 }
 
+mount_kubelet_cgroup_root() {
+  cgroup_root=$1
+  subsystem=$2
+
+  if [ ! -z ${cgroup_root} ]; then
+    # This is because we set Kubelet's cgroup-root to `/kubelet` by
+    # default. We have to do that because otherwise, it'll collide
+    # with the cgroups used by the Kubelet running on the host if we
+    # run kind cluster within a Kubernetes pod, resulting in random
+    # processes to be killed.
+    mkdir -p "${subsystem}/${cgroup_root}"
+    if [ "${subsystem}" == "/sys/fs/cgroup/cpuset" ]; then
+      # This is needed. Otherwise, assigning process to the cgroup
+      # (or any nested cgroup) would result in ENOSPC.
+      cat "${subsystem}/cpuset.cpus" > "${subsystem}/${cgroup_root}/cpuset.cpus"
+      cat "${subsystem}/cpuset.mems" > "${subsystem}/${cgroup_root}/cpuset.mems"
+    fi
+    # We need to perform a self bind mount here because otherwise,
+    # systemd might delete the cgroup unintentionally before the
+    # kubelet starts.
+    mount --bind "${subsystem}/${cgroup_root}" "${subsystem}/${cgroup_root}"
+  fi
+}
+
 fix_cgroup() {
   echo 'INFO: fix cgroup mounts for all subsystems'
   # For each cgroup subsystem, Docker does a bind mount from the current
@@ -85,10 +109,11 @@ fix_cgroup() {
     while IFS= read -r subsystem; do
       mkdir -p "${subsystem}${docker_cgroup}"
       mount --bind "${subsystem}" "${subsystem}${docker_cgroup}"
+      mount_kubelet_cgroup_root "${CGROUP_ROOT}" "${subsystem}"
     done
   fi
   local podman_cgroup_mounts
-  podman_cgroup_mounts=$(grep /sys/fs/cgroup /proc/self/mountinfo | grep libpod_parent || true)
+  podman_cgroup_mounts=$(grep /sys/fs/cgroup /proc/self/mountinfo | grep libpod || true)
   if [[ -n "${podman_cgroup_mounts}" ]]; then
     local podman_cgroup cgroup_subsystems subsystem
     podman_cgroup=$(echo "${podman_cgroup_mounts}" | head -n 1 | cut -d' ' -f 4)
@@ -97,6 +122,7 @@ fix_cgroup() {
     while IFS= read -r subsystem; do
       mkdir -p "${subsystem}${podman_cgroup}"
       mount --bind "${subsystem}" "${subsystem}${podman_cgroup}"
+      mount_kubelet_cgroup_root "${CGROUP_ROOT}" "${subsystem}"
     done
   fi
 }

pkg/cluster/internal/create/actions/config/config.go

@@ -74,6 +74,7 @@ func (a *Action) Execute(ctx *actions.ActionContext) error {
 		IPv6:                 ctx.Config.Networking.IPFamily == "ipv6",
 		FeatureGates:         ctx.Config.FeatureGates,
 		RuntimeConfig:        ctx.Config.RuntimeConfig,
+		CgroupRoot:           ctx.Config.CgroupRoot,
 	}
 
 	kubeadmConfigPlusPatches := func(node nodes.Node, data kubeadm.ConfigData) func() error {

pkg/cluster/internal/kubeadm/config.go

@@ -70,6 +70,9 @@ type ConfigData struct {
 	// IPv4 values take precedence over IPv6 by default, if true set IPv6 default values
 	IPv6 bool
 
+	// CgroupRoot specifies the cgroup root to configure the kubelet to use
+	CgroupRoot   string
+
 	// DerivedConfigData is populated by Derive()
 	// These auto-generated fields are available to Config templates,
 	// but not meant to be set by hand
@@ -195,6 +198,7 @@ nodeRegistration:
     fail-swap-on: "false"
     node-ip: "{{ .NodeAddress }}"
     provider-id: "kind://{{.NodeProvider}}/{{.ClusterName}}/{{.NodeName}}"
+    cgroup-root: "{{ .CgroupRoot }}"
 ---
 # no-op entry that exists solely so it can be patched
 apiVersion: kubeadm.k8s.io/v1beta1
@@ -213,6 +217,7 @@ nodeRegistration:
     fail-swap-on: "false"
     node-ip: "{{ .NodeAddress }}"
     provider-id: "kind://{{.NodeProvider}}/{{.ClusterName}}/{{.NodeName}}"
+    cgroup-root: "{{ .CgroupRoot }}"
 discovery:
   bootstrapToken:
     apiServerEndpoint: "{{ .ControlPlaneEndpoint }}"
@@ -315,6 +320,7 @@ nodeRegistration:
     fail-swap-on: "false"
     node-ip: "{{ .NodeAddress }}"
     provider-id: "kind://{{.NodeProvider}}/{{.ClusterName}}/{{.NodeName}}"
+    cgroup-root: "{{ .CgroupRoot }}"
 ---
 # no-op entry that exists solely so it can be patched
 apiVersion: kubeadm.k8s.io/v1beta2
@@ -333,6 +339,7 @@ nodeRegistration:
     fail-swap-on: "false"
     node-ip: "{{ .NodeAddress }}"
     provider-id: "kind://{{.NodeProvider}}/{{.ClusterName}}/{{.NodeName}}"
+    cgroup-root: "{{ .CgroupRoot }}"
 discovery:
   bootstrapToken:
     apiServerEndpoint: "{{ .ControlPlaneEndpoint }}"

pkg/cluster/internal/providers/docker/provision.go

@@ -199,6 +199,8 @@ func commonArgs(cluster string, cfg *config.Cluster, networkName string, nodeNam
 		args = append(args, "-e", fmt.Sprintf("%s=%s", key, val))
 	}
 
+	args = append(args, "-e", fmt.Sprintf("CGROUP_ROOT=%s", cfg.CgroupRoot))
+
 	// handle hosts that have user namespace remapping enabled
 	if usernsRemap() {
 		args = append(args, "--userns=host")

pkg/cluster/internal/providers/podman/provision.go

@@ -159,6 +159,8 @@ func commonArgs(cfg *config.Cluster, networkName string) ([]string, error) {
 		args = append(args, "-e", fmt.Sprintf("%s=%s", key, val))
 	}
 
+	args = append(args, "-e", fmt.Sprintf("CGROUP_ROOT=%s", cfg.CgroupRoot))
+
 	return args, nil
 }
 

pkg/internal/apis/config/types.go

@@ -69,6 +69,9 @@ type Cluster struct {
 	// in the order listed.
 	// These should be YAML or JSON formatting RFC 6902 JSON patches
 	ContainerdConfigPatchesJSON6902 []string
+
+	// CgroupRoot specifies the cgroup root for kubelet to use.
+	CgroupRoot string
 }
 
 // Node contains settings for a node in the `kind` Cluster.