Support for Kubernetes v1.24 (gardener#6023)

* Extend docs to support kubernetes v1.24 and allow client creation

* Adapt kubernetes feature gates

./hack/compare-k8s-feature-gates.sh 1.23 1.24                                                                                                    ✔
Feature gates added in 1.24 compared to 1.23:
CSIMigrationRBD
CronJobTimeZone
LegacyServiceAccountTokenNoAutoGeneration
MaxUnavailableStatefulSet
MinDomainsInPodTopologySpread
NetworkPolicyStatus
NodeOutOfServiceVolumeDetach
ServiceIPStaticSubrange

Feature gates removed in 1.24 compared to 1.23:
HugePageStorageMediumSize
ImmutableEphemeralVolumes
MigrationRBD
NamespaceDefaultLabelName
RuntimeClass
SetHostnameAsFQDN
StreamingProxyRedirects
ValidateProxyRedirects
WarningHeaders

Feature gates locked to default in 1.24 compared to 1.23:
CSIMigrationOpenStack
CSIStorageCapacity
CSRDuration
ControllerManagerLeaderMigration
DefaultPodTopologySpread
EfficientWatchResumption
IndexedJob
NonPreemptingPriority
PodAffinityNamespaceSelector
PodOverhead
PreferNominatedNode
RemoveSelfLink
ServiceLBNodePortControl
ServiceLoadBalancerClass
SuspendJob

* Use 1.24 for local shoot

* Drop removed flag --insecure-port for v1.24

ref kubernetes/kubernetes#106859

* Drop removed flag --port for v1.24

ref kubernetes/kubernetes#106860

* Remove deprecated usages of metadata.Selflink

* Use 1.24 e2e test

* Bump kindest/node image to v1.24

* Adapt changes for with k/k v1.24 Secret API objects containing service account tokens are no longer auto-generated for every ServiceAccount

ref kubernetes/kubernetes#108309

* Add unit test

docs/extensions/controlplane-webhooks.md

@@ -41,7 +41,7 @@ The `command` field of the `kube-apiserver` container **shall** contain the [kub
 * admission plugins (`--enable-admission-plugins`, `--disable-admission-plugins`)
 * secure communications (`--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, ...)
 * audit log (`--audit-log-*`)
-* ports (`--insecure-port`, `--secure-port`)
+* ports (`--secure-port`)
 
 The kube-apiserver command line **shall not** contain any provider-specific flags, such as:
 

docs/usage/supported_k8s_versions.md

@@ -14,7 +14,7 @@ It also enables the Gardener code base to leverage more advanced Kubernetes feat
 
 ## Shoot cluster versions
 
-Gardener itself is capable of spinning up clusters with Kubernetes versions **`1.17`** up to **`1.23`**.
+Gardener itself is capable of spinning up clusters with Kubernetes versions **`1.17`** up to **`1.24`**.
 However, the concrete versions that can be used for shoot clusters depend on the installed provider extension.
 Consequently, please consult the documentation of your provider extension to see which Kubernetes versions are supported for shoot clusters.
 

example/gardener-local/kind/cluster-local.yaml

@@ -2,7 +2,7 @@ apiVersion: kind.x-k8s.io/v1alpha4
 kind: Cluster
 nodes:
 - role: control-plane
-  image: kindest/node:v1.21.1
+  image: kindest/node:v1.24.0
   extraPortMappings:
   # istio-ingressgateway
   - containerPort: 30443

example/gardener-local/kind/cluster-skaffold.yaml

@@ -2,7 +2,7 @@ apiVersion: kind.x-k8s.io/v1alpha4
 kind: Cluster
 nodes:
 - role: control-plane
-  image: kindest/node:v1.21.1
+  image: kindest/node:v1.24.0
   extraPortMappings:
   # istio-ingressgateway
   - containerPort: 30443

example/provider-local/base/cloudprofile.yaml

@@ -8,6 +8,7 @@ spec:
   - name: local
   kubernetes:
     versions:
+    - version: 1.24.0
     - version: 1.23.6
     - version: 1.22.0
     - version: 1.21.0

example/provider-local/shoot.yaml

@@ -31,4 +31,4 @@ spec:
       maxSurge: 1
       maxUnavailable: 0
   kubernetes:
-    version: 1.23.6
+    version: 1.24.0

hack/local-development/common/helpers

@@ -174,6 +174,6 @@ preferences: {}
 users:
 - name: cluster
   user:
-    token: $(kubectl -n garden get secret "$(kubectl -n garden get sa "$1" -o jsonpath={.secrets[0].name})" -o jsonpath={.data.token} | base64 -d)
+    token: $(kubectl create -n garden token "$1")
 EOF
 }

hack/local-development/start-gardenlet

@@ -164,7 +164,7 @@ SEED_KUBECONFIG_GARDENLET_TOKEN="$DEV_DIR/gardenlet-seed-$SEED_NAME-token.conf"
 cp "$SEED_KUBECONFIG" "$SEED_KUBECONFIG_GARDENLET_TOKEN"
 
 # use the token of the gardenlet service account to authenticated against the seed API
-token=$(kubectl --kubeconfig="$SEED_KUBECONFIG" --namespace garden get secret $(kubectl --kubeconfig="$SEED_KUBECONFIG" --namespace garden get serviceaccount gardenlet -o jsonpath='{.secrets[0].name}') -o jsonpath='{.data.token}' | base64 --decode)
+token=$(kubectl --kubeconfig="$SEED_KUBECONFIG" --namespace garden create token gardenlet)
 user=$(kubectl --kubeconfig="$SEED_KUBECONFIG" config view --raw --minify -o json | yq eval '.["current-context"] as $currentContext | .contexts[] | select(.name == $currentContext) | .context.user' -)
 USER_VALUE=$user TOKEN_VALUE=$token yq eval --inplace '.users = [{"name": strenv(USER_VALUE), "user": {"token": strenv(TOKEN_VALUE)}}]' $SEED_KUBECONFIG_GARDENLET_TOKEN
 

hack/tools.mk

@@ -55,7 +55,7 @@ GO_APIDIFF                 := $(TOOLS_BIN_DIR)/go-apidiff
 DOCFORGE_VERSION ?= v0.28.0
 GOLANGCI_LINT_VERSION ?= v1.45.2
 HELM_VERSION ?= v3.6.3
-KIND_VERSION ?= v0.11.1
+KIND_VERSION ?= v0.14.0
 SKAFFOLD_VERSION ?= v1.35.0
 YQ_VERSION ?= v4.9.6
 GO_APIDIFF_VERSION ?= v0.3.0

pkg/client/kubernetes/client.go

@@ -240,6 +240,7 @@ var supportedKubernetesVersions = []string{
 	"1.21",
 	"1.22",
 	"1.23",
+	"1.24",
 }
 
 func checkIfSupportedKubernetesVersion(gitVersion string) error {

pkg/operation/botanist/component/kubeapiserver/deployment.go

@@ -27,6 +27,7 @@ import (
 	"github.com/gardener/gardener/pkg/utils"
 	kutil "github.com/gardener/gardener/pkg/utils/kubernetes"
 	"github.com/gardener/gardener/pkg/utils/secrets"
+	"github.com/gardener/gardener/pkg/utils/version"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -513,7 +514,10 @@ func (k *kubeAPIServer) computeKubeAPIServerCommand() []string {
 		out = append(out, kutil.FeatureGatesToCommandLineParameter(k.values.FeatureGates))
 	}
 
-	out = append(out, "--insecure-port=0")
+	if version.ConstraintK8sLess124.Check(k.values.Version) {
+		out = append(out, "--insecure-port=0")
+	}
+
 	out = append(out, "--kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP")
 	out = append(out, fmt.Sprintf("--kubelet-client-certificate=%s/%s", volumeMountPathKubeAPIServerToKubelet, secrets.DataKeyCertificate))
 	out = append(out, fmt.Sprintf("--kubelet-client-key=%s/%s", volumeMountPathKubeAPIServerToKubelet, secrets.DataKeyPrivateKey))

pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager.go

@@ -546,7 +546,13 @@ func (k *kubeControllerManager) computeCommand(port int32) []string {
 		fmt.Sprintf("--service-account-private-key-file=%s/%s", volumeMountPathServiceAccountKey, secrets.DataKeyRSAPrivateKey),
 		fmt.Sprintf("--service-cluster-ip-range=%s", k.serviceNetwork.String()),
 		fmt.Sprintf("--secure-port=%d", port),
-		"--port=0",
+	)
+
+	if version.ConstraintK8sLess124.Check(k.version) {
+		command = append(command, "--port=0")
+	}
+
+	command = append(command,
 		fmt.Sprintf("--horizontal-pod-autoscaler-downscale-stabilization=%s", defaultHorizontalPodAutoscalerConfig.DownscaleStabilization.Duration.String()),
 		fmt.Sprintf("--horizontal-pod-autoscaler-initial-readiness-delay=%s", defaultHorizontalPodAutoscalerConfig.InitialReadinessDelay.Duration.String()),
 		fmt.Sprintf("--horizontal-pod-autoscaler-cpu-initialization-period=%s", defaultHorizontalPodAutoscalerConfig.CPUInitializationPeriod.Duration.String()),

pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager_test.go

@@ -908,7 +908,13 @@ func commandForKubernetesVersion(
 		"--service-account-private-key-file=/srv/kubernetes/service-account-key/id_rsa",
 		fmt.Sprintf("--service-cluster-ip-range=%s", serviceNetwork.String()),
 		fmt.Sprintf("--secure-port=%d", port),
-		"--port=0",
+	)
+
+	if versionutils.ConstraintK8sLess124.Check(semver.MustParse(version)) {
+		command = append(command, "--port=0")
+	}
+
+	command = append(command,
 		fmt.Sprintf("--horizontal-pod-autoscaler-downscale-stabilization=%s", horizontalPodAutoscalerConfig.DownscaleStabilization.Duration.String()),
 		fmt.Sprintf("--horizontal-pod-autoscaler-initial-readiness-delay=%s", horizontalPodAutoscalerConfig.InitialReadinessDelay.Duration.String()),
 		fmt.Sprintf("--horizontal-pod-autoscaler-cpu-initialization-period=%s", horizontalPodAutoscalerConfig.CPUInitializationPeriod.Duration.String()),

pkg/operation/botanist/component/resourcemanager/resource_manager.go

@@ -34,7 +34,9 @@ import (
 	"github.com/gardener/gardener/pkg/utils/retry"
 	"github.com/gardener/gardener/pkg/utils/secrets"
 	secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager"
+	"github.com/gardener/gardener/pkg/utils/version"
 
+	"github.com/Masterminds/semver"
 	admissionv1 "k8s.io/api/admission/v1"
 	admissionv1beta1 "k8s.io/api/admission/v1beta1"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
@@ -124,6 +126,9 @@ var (
 			Verbs:         []string{"get", "watch", "update"},
 		},
 	}
+
+	rootCAVolumeSourceName string
+	volumeProjection       corev1.VolumeProjection
 )
 
 // Interface contains functions for a gardener-resource-manager deployer.
@@ -203,6 +208,8 @@ type Values struct {
 	// WatchedNamespace restricts the gardener-resource-manager to only watch ManagedResources in the defined namespace.
 	// If not set the gardener-resource-manager controller watches for ManagedResources in all namespaces
 	WatchedNamespace *string
+	// Version is the Kubernetes version for the Kubernetes components.
+	Version *semver.Version
 	// VPA contains information for configuring VerticalPodAutoscaler settings for the gardener-resource-manager deployment.
 	VPA *VPAConfig
 }
@@ -448,9 +455,38 @@ func (r *resourceManager) getRootCAVolumeSourceName(ctx context.Context) (string
 func (r *resourceManager) ensureDeployment(ctx context.Context) error {
 	deployment := r.emptyDeployment()
 
-	rootCAVolumeSourceName, err := r.getRootCAVolumeSourceName(ctx)
-	if err != nil {
-		return err
+	if version.ConstraintK8sLess124.Check(r.values.Version) {
+		var err error
+		rootCAVolumeSourceName, err = r.getRootCAVolumeSourceName(ctx)
+		if err != nil {
+			return err
+		}
+
+		volumeProjection = corev1.VolumeProjection{
+			Secret: &corev1.SecretProjection{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: rootCAVolumeSourceName,
+				},
+				Items: []corev1.KeyToPath{{
+					Key:  "ca.crt",
+					Path: "ca.crt",
+				}},
+			},
+		}
+	} else {
+		rootCAVolumeSourceName = "kube-root-ca.crt"
+
+		volumeProjection = corev1.VolumeProjection{
+			ConfigMap: &corev1.ConfigMapProjection{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: rootCAVolumeSourceName,
+				},
+				Items: []corev1.KeyToPath{{
+					Key:  "ca.crt",
+					Path: "ca.crt",
+				}},
+			},
+		}
 	}
 
 	secretServer, err := r.secretsManager.Generate(ctx, &secrets.CertificateSecretConfig{
@@ -567,17 +603,7 @@ func (r *resourceManager) ensureDeployment(ctx context.Context) error {
 											Path:              "token",
 										},
 									},
-									{
-										Secret: &corev1.SecretProjection{
-											LocalObjectReference: corev1.LocalObjectReference{
-												Name: rootCAVolumeSourceName,
-											},
-											Items: []corev1.KeyToPath{{
-												Key:  "ca.crt",
-												Path: "ca.crt",
-											}},
-										},
-									},
+									volumeProjection,
 									{
 										DownwardAPI: &corev1.DownwardAPIProjection{
 											Items: []corev1.DownwardAPIVolumeFile{{