From 80491c129d1129c8ec4d098fa55df85f82b369d3 Mon Sep 17 00:00:00 2001 From: Sonu Kumar Singh Date: Fri, 27 May 2022 14:41:45 +0530 Subject: [PATCH] Support for Kubernetes v1.24 (#6023) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Extend docs to support kubernetes v1.24 and allow client creation * Adapt kubernetes feature gates ./hack/compare-k8s-feature-gates.sh 1.23 1.24  ✔ Feature gates added in 1.24 compared to 1.23: CSIMigrationRBD CronJobTimeZone LegacyServiceAccountTokenNoAutoGeneration MaxUnavailableStatefulSet MinDomainsInPodTopologySpread NetworkPolicyStatus NodeOutOfServiceVolumeDetach ServiceIPStaticSubrange Feature gates removed in 1.24 compared to 1.23: HugePageStorageMediumSize ImmutableEphemeralVolumes MigrationRBD NamespaceDefaultLabelName RuntimeClass SetHostnameAsFQDN StreamingProxyRedirects ValidateProxyRedirects WarningHeaders Feature gates locked to default in 1.24 compared to 1.23: CSIMigrationOpenStack CSIStorageCapacity CSRDuration ControllerManagerLeaderMigration DefaultPodTopologySpread EfficientWatchResumption IndexedJob NonPreemptingPriority PodAffinityNamespaceSelector PodOverhead PreferNominatedNode RemoveSelfLink ServiceLBNodePortControl ServiceLoadBalancerClass SuspendJob * Use 1.24 for local shoot * Drop removed flag --insecure-port for v1.24 ref https://github.com/kubernetes/kubernetes/pull/106859 * Drop removed flag --port for v1.24 ref https://github.com/kubernetes/kubernetes/pull/106860 * Remove deprecated usages of metadata.Selflink * Use 1.24 e2e test * Bump kindest/node image to v1.24 * Adapt changes for with k/k v1.24 Secret API objects containing service account tokens are no longer auto-generated for every ServiceAccount ref https://github.com/kubernetes/kubernetes/pull/108309 * Add unit test --- README.md | 18 +- docs/extensions/controlplane-webhooks.md | 2 +- docs/usage/supported_k8s_versions.md | 2 +- .../gardener-local/kind/cluster-local.yaml | 2 +- .../gardener-local/kind/cluster-skaffold.yaml | 2 +- example/provider-local/base/cloudprofile.yaml | 1 + example/provider-local/shoot.yaml | 2 +- hack/local-development/common/helpers | 2 +- hack/local-development/start-gardenlet | 2 +- hack/tools.mk | 2 +- pkg/client/kubernetes/client.go | 1 + .../component/kubeapiserver/deployment.go | 6 +- .../kube_controller_manager.go | 8 +- .../kube_controller_manager_test.go | 8 +- .../resourcemanager/resource_manager.go | 54 +- .../resourcemanager/resource_manager_test.go | 554 ++++++++++++------ pkg/operation/botanist/resource_manager.go | 2 + pkg/operation/seed/components.go | 3 +- pkg/operation/seed/seed.go | 2 +- .../backupbucket/storage/tableconvertor.go | 2 - .../backupentry/storage/tableconvertor.go | 2 - .../cloudprofile/storage/tableconvertor.go | 2 - .../storage/tableconvertor.go | 2 - .../storage/tableconvertor.go | 2 - .../storage/tableconvertor.go | 2 - .../exposureclass/storage/tableconvertor.go | 2 - .../core/plant/storage/tableconvertor.go | 2 - .../core/project/storage/tableconvertor.go | 2 - .../core/quota/storage/tableconvertor.go | 2 - .../secretbinding/storage/tableconvertor.go | 2 - .../core/seed/storage/tableconvertor.go | 2 - .../core/shoot/storage/tableconvertor.go | 2 - .../core/shootstate/storage/tableconvertor.go | 2 - .../bastion/storage/tableconvertor.go | 2 - .../managedseed/storage/tableconvertor.go | 2 - .../managedseedset/storage/tableconvertor.go | 2 - .../storage/tableconvertor.go | 2 - .../storage/tableconvertor.go | 2 - pkg/utils/kubernetes/unstructured/object.go | 2 +- pkg/utils/validation/features/featuregates.go | 56 +- pkg/utils/version/version.go | 4 + test/e2e/shoot/common.go | 2 +- 42 files changed, 502 insertions(+), 273 deletions(-) diff --git a/README.md b/README.md index fb9a74e87381..f8dbb09fedca 100644 --- a/README.md +++ b/README.md @@ -38,15 +38,15 @@ Gardener takes part in the [Certified Kubernetes Conformance Program](https://ww Continuous conformance test results of the latest stable Gardener release are uploaded regularly to the CNCF test grid: -| Provider/K8s | v1.23 | v1.22 | v1.21 | v1.20 | v1.19 | v1.18 | v1.17 | -| ------------ | ------------ | ------------ | ------------ | ------------ | ---------- | ----------- | ----------- | -| **AWS** | [![Gardener v1.23 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.23%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.23%20AWS) | [![Gardener v1.22 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.22%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.22%20AWS) | [![Gardener v1.21 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.21%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.21%20AWS) | [![Gardener v1.20 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.20%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.20%20AWS) | [![Gardener v1.19 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.19%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.19%20AWS) | [![Gardener v1.18 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.18%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.18%20AWS) | [![Gardener v1.17 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.17%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.17%20AWS) | -| **Azure** | [![Gardener v1.23 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.23%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.23%20Azure) | [![Gardener v1.22 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.22%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.22%20Azure) | [![Gardener v1.21 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.21%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.21%20Azure) | [![Gardener v1.20 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.20%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.20%20Azure) | [![Gardener v1.19 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.19%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.19%20Azure) | [![Gardener v1.18 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.18%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.18%20Azure) | [![Gardener v1.17 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.17%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.17%20Azure) | -| **GCP** | [![Gardener v1.23 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.23%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.23%20GCE) | [![Gardener v1.22 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.22%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.22%20GCE) | [![Gardener v1.21 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.21%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.21%20GCE) | [![Gardener v1.20 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.20%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.20%20GCE) | [![Gardener v1.19 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.19%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.19%20GCE) | [![Gardener v1.18 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.18%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.18%20GCE) | [![Gardener v1.17 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.17%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.17%20GCE) | -| **OpenStack** | [![Gardener v1.23 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.23%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.23%20OpenStack) | [![Gardener v1.22 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.22%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.22%20OpenStack) | [![Gardener v1.21 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.21%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.21%20OpenStack) | [![Gardener v1.20 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.20%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.20%20OpenStack) | [![Gardener v1.19 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.19%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.19%20OpenStack) | [![Gardener v1.18 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.18%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.18%20OpenStack) | [![Gardener v1.17 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.17%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.17%20OpenStack) | -| **Alicloud** | [![Gardener v1.23 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.23%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.23%20Alibaba%20Cloud) | [![Gardener v1.22 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.22%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.22%20Alibaba%20Cloud) | [![Gardener v1.21 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.21%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.21%20Alibaba%20Cloud) | [![Gardener v1.20 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.20%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.20%20Alibaba%20Cloud) | [![Gardener v1.19 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.19%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.19%20Alibaba%20Cloud) | [![Gardener v1.18 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.18%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.18%20Alibaba%20Cloud) | [![Gardener v1.17 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.17%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.17%20Alibaba%20Cloud) | -| **Equinix Metal** | N/A | N/A | N/A | N/A | N/A | N/A | N/A | -| **vSphere** | N/A | N/A | N/A | N/A | N/A | N/A | N/A | +| Provider/K8s | v1.24 | v1.23 | v1.22 | v1.21 | v1.20 | v1.19 | v1.18 | v1.17 | +| ------------ | ------------ | ------------ | ------------ | ------------ | ---------- | ----------- | ----------- | ----------- | +| **AWS** | N/A | [![Gardener v1.23 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.23%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.23%20AWS) | [![Gardener v1.22 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.22%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.22%20AWS) | [![Gardener v1.21 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.21%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.21%20AWS) | [![Gardener v1.20 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.20%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.20%20AWS) | [![Gardener v1.19 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.19%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.19%20AWS) | [![Gardener v1.18 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.18%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.18%20AWS) | [![Gardener v1.17 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.17%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.17%20AWS) | +| **Azure** | N/A | [![Gardener v1.23 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.23%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.23%20Azure) | [![Gardener v1.22 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.22%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.22%20Azure) | [![Gardener v1.21 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.21%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.21%20Azure) | [![Gardener v1.20 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.20%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.20%20Azure) | [![Gardener v1.19 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.19%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.19%20Azure) | [![Gardener v1.18 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.18%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.18%20Azure) | [![Gardener v1.17 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.17%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.17%20Azure) | +| **GCP** | N/A | [![Gardener v1.23 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.23%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.23%20GCE) | [![Gardener v1.22 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.22%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.22%20GCE) | [![Gardener v1.21 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.21%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.21%20GCE) | [![Gardener v1.20 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.20%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.20%20GCE) | [![Gardener v1.19 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.19%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.19%20GCE) | [![Gardener v1.18 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.18%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.18%20GCE) | [![Gardener v1.17 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.17%20GCE/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.17%20GCE) | +| **OpenStack** | N/A | [![Gardener v1.23 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.23%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.23%20OpenStack) | [![Gardener v1.22 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.22%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.22%20OpenStack) | [![Gardener v1.21 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.21%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.21%20OpenStack) | [![Gardener v1.20 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.20%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.20%20OpenStack) | [![Gardener v1.19 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.19%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.19%20OpenStack) | [![Gardener v1.18 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.18%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.18%20OpenStack) | [![Gardener v1.17 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.17%20OpenStack/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.17%20OpenStack) | +| **Alicloud** | N/A | [![Gardener v1.23 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.23%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.23%20Alibaba%20Cloud) | [![Gardener v1.22 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.22%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.22%20Alibaba%20Cloud) | [![Gardener v1.21 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.21%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.21%20Alibaba%20Cloud) | [![Gardener v1.20 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.20%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.20%20Alibaba%20Cloud) | [![Gardener v1.19 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.19%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.19%20Alibaba%20Cloud) | [![Gardener v1.18 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.18%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.18%20Alibaba%20Cloud) | [![Gardener v1.17 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.17%20Alibaba%20Cloud/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.17%20Alibaba%20Cloud) | +| **Equinix Metal** | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | +| **vSphere** | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | [1] Conformance tests are still executed and validated, unfortunately [no longer shown in TestGrid](https://github.com/kubernetes/test-infra/pull/18509#issuecomment-668204180). diff --git a/docs/extensions/controlplane-webhooks.md b/docs/extensions/controlplane-webhooks.md index cfcb81306ead..e6e8f06ce9fb 100644 --- a/docs/extensions/controlplane-webhooks.md +++ b/docs/extensions/controlplane-webhooks.md @@ -41,7 +41,7 @@ The `command` field of the `kube-apiserver` container **shall** contain the [kub * admission plugins (`--enable-admission-plugins`, `--disable-admission-plugins`) * secure communications (`--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, ...) * audit log (`--audit-log-*`) -* ports (`--insecure-port`, `--secure-port`) +* ports (`--secure-port`) The kube-apiserver command line **shall not** contain any provider-specific flags, such as: diff --git a/docs/usage/supported_k8s_versions.md b/docs/usage/supported_k8s_versions.md index 7c9dbf51136d..7fb99ca2e4cd 100644 --- a/docs/usage/supported_k8s_versions.md +++ b/docs/usage/supported_k8s_versions.md @@ -14,7 +14,7 @@ It also enables the Gardener code base to leverage more advanced Kubernetes feat ## Shoot cluster versions -Gardener itself is capable of spinning up clusters with Kubernetes versions **`1.17`** up to **`1.23`**. +Gardener itself is capable of spinning up clusters with Kubernetes versions **`1.17`** up to **`1.24`**. However, the concrete versions that can be used for shoot clusters depend on the installed provider extension. Consequently, please consult the documentation of your provider extension to see which Kubernetes versions are supported for shoot clusters. diff --git a/example/gardener-local/kind/cluster-local.yaml b/example/gardener-local/kind/cluster-local.yaml index 9f96ae265b74..8d8cf7b9eb49 100644 --- a/example/gardener-local/kind/cluster-local.yaml +++ b/example/gardener-local/kind/cluster-local.yaml @@ -2,7 +2,7 @@ apiVersion: kind.x-k8s.io/v1alpha4 kind: Cluster nodes: - role: control-plane - image: kindest/node:v1.21.1 + image: kindest/node:v1.24.0 extraPortMappings: # istio-ingressgateway - containerPort: 30443 diff --git a/example/gardener-local/kind/cluster-skaffold.yaml b/example/gardener-local/kind/cluster-skaffold.yaml index 24cb77fe6b60..24edb453af67 100644 --- a/example/gardener-local/kind/cluster-skaffold.yaml +++ b/example/gardener-local/kind/cluster-skaffold.yaml @@ -2,7 +2,7 @@ apiVersion: kind.x-k8s.io/v1alpha4 kind: Cluster nodes: - role: control-plane - image: kindest/node:v1.21.1 + image: kindest/node:v1.24.0 extraPortMappings: # istio-ingressgateway - containerPort: 30443 diff --git a/example/provider-local/base/cloudprofile.yaml b/example/provider-local/base/cloudprofile.yaml index 347b9434e856..9d3c33749de1 100644 --- a/example/provider-local/base/cloudprofile.yaml +++ b/example/provider-local/base/cloudprofile.yaml @@ -8,6 +8,7 @@ spec: - name: local kubernetes: versions: + - version: 1.24.0 - version: 1.23.6 - version: 1.22.0 - version: 1.21.0 diff --git a/example/provider-local/shoot.yaml b/example/provider-local/shoot.yaml index 1ebff4fe770e..7b2922c5b3ed 100644 --- a/example/provider-local/shoot.yaml +++ b/example/provider-local/shoot.yaml @@ -31,4 +31,4 @@ spec: maxSurge: 1 maxUnavailable: 0 kubernetes: - version: 1.23.6 + version: 1.24.0 diff --git a/hack/local-development/common/helpers b/hack/local-development/common/helpers index 86fbe89b8e6d..851fbeddff2f 100755 --- a/hack/local-development/common/helpers +++ b/hack/local-development/common/helpers @@ -174,6 +174,6 @@ preferences: {} users: - name: cluster user: - token: $(kubectl -n garden get secret "$(kubectl -n garden get sa "$1" -o jsonpath={.secrets[0].name})" -o jsonpath={.data.token} | base64 -d) + token: $(kubectl create -n garden token "$1") EOF } diff --git a/hack/local-development/start-gardenlet b/hack/local-development/start-gardenlet index 40a2aa28fd2c..64def96452d1 100755 --- a/hack/local-development/start-gardenlet +++ b/hack/local-development/start-gardenlet @@ -164,7 +164,7 @@ SEED_KUBECONFIG_GARDENLET_TOKEN="$DEV_DIR/gardenlet-seed-$SEED_NAME-token.conf" cp "$SEED_KUBECONFIG" "$SEED_KUBECONFIG_GARDENLET_TOKEN" # use the token of the gardenlet service account to authenticated against the seed API -token=$(kubectl --kubeconfig="$SEED_KUBECONFIG" --namespace garden get secret $(kubectl --kubeconfig="$SEED_KUBECONFIG" --namespace garden get serviceaccount gardenlet -o jsonpath='{.secrets[0].name}') -o jsonpath='{.data.token}' | base64 --decode) +token=$(kubectl --kubeconfig="$SEED_KUBECONFIG" --namespace garden create token gardenlet) user=$(kubectl --kubeconfig="$SEED_KUBECONFIG" config view --raw --minify -o json | yq eval '.["current-context"] as $currentContext | .contexts[] | select(.name == $currentContext) | .context.user' -) USER_VALUE=$user TOKEN_VALUE=$token yq eval --inplace '.users = [{"name": strenv(USER_VALUE), "user": {"token": strenv(TOKEN_VALUE)}}]' $SEED_KUBECONFIG_GARDENLET_TOKEN diff --git a/hack/tools.mk b/hack/tools.mk index 53995b297011..5606330629d9 100644 --- a/hack/tools.mk +++ b/hack/tools.mk @@ -55,7 +55,7 @@ GO_APIDIFF := $(TOOLS_BIN_DIR)/go-apidiff DOCFORGE_VERSION ?= v0.28.0 GOLANGCI_LINT_VERSION ?= v1.45.2 HELM_VERSION ?= v3.6.3 -KIND_VERSION ?= v0.11.1 +KIND_VERSION ?= v0.14.0 SKAFFOLD_VERSION ?= v1.35.0 YQ_VERSION ?= v4.9.6 GO_APIDIFF_VERSION ?= v0.3.0 diff --git a/pkg/client/kubernetes/client.go b/pkg/client/kubernetes/client.go index 2f529f773b26..494054914062 100644 --- a/pkg/client/kubernetes/client.go +++ b/pkg/client/kubernetes/client.go @@ -240,6 +240,7 @@ var supportedKubernetesVersions = []string{ "1.21", "1.22", "1.23", + "1.24", } func checkIfSupportedKubernetesVersion(gitVersion string) error { diff --git a/pkg/operation/botanist/component/kubeapiserver/deployment.go b/pkg/operation/botanist/component/kubeapiserver/deployment.go index bf5616a1557e..ff7b516209b2 100644 --- a/pkg/operation/botanist/component/kubeapiserver/deployment.go +++ b/pkg/operation/botanist/component/kubeapiserver/deployment.go @@ -27,6 +27,7 @@ import ( "github.com/gardener/gardener/pkg/utils" kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/secrets" + "github.com/gardener/gardener/pkg/utils/version" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -513,7 +514,10 @@ func (k *kubeAPIServer) computeKubeAPIServerCommand() []string { out = append(out, kutil.FeatureGatesToCommandLineParameter(k.values.FeatureGates)) } - out = append(out, "--insecure-port=0") + if version.ConstraintK8sLess124.Check(k.values.Version) { + out = append(out, "--insecure-port=0") + } + out = append(out, "--kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP") out = append(out, fmt.Sprintf("--kubelet-client-certificate=%s/%s", volumeMountPathKubeAPIServerToKubelet, secrets.DataKeyCertificate)) out = append(out, fmt.Sprintf("--kubelet-client-key=%s/%s", volumeMountPathKubeAPIServerToKubelet, secrets.DataKeyPrivateKey)) diff --git a/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager.go b/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager.go index 198392450237..3ad4ee5b342a 100644 --- a/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager.go +++ b/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager.go @@ -546,7 +546,13 @@ func (k *kubeControllerManager) computeCommand(port int32) []string { fmt.Sprintf("--service-account-private-key-file=%s/%s", volumeMountPathServiceAccountKey, secrets.DataKeyRSAPrivateKey), fmt.Sprintf("--service-cluster-ip-range=%s", k.serviceNetwork.String()), fmt.Sprintf("--secure-port=%d", port), - "--port=0", + ) + + if version.ConstraintK8sLess124.Check(k.version) { + command = append(command, "--port=0") + } + + command = append(command, fmt.Sprintf("--horizontal-pod-autoscaler-downscale-stabilization=%s", defaultHorizontalPodAutoscalerConfig.DownscaleStabilization.Duration.String()), fmt.Sprintf("--horizontal-pod-autoscaler-initial-readiness-delay=%s", defaultHorizontalPodAutoscalerConfig.InitialReadinessDelay.Duration.String()), fmt.Sprintf("--horizontal-pod-autoscaler-cpu-initialization-period=%s", defaultHorizontalPodAutoscalerConfig.CPUInitializationPeriod.Duration.String()), diff --git a/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager_test.go b/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager_test.go index 739fd73099bf..d545b821a44f 100644 --- a/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager_test.go +++ b/pkg/operation/botanist/component/kubecontrollermanager/kube_controller_manager_test.go @@ -908,7 +908,13 @@ func commandForKubernetesVersion( "--service-account-private-key-file=/srv/kubernetes/service-account-key/id_rsa", fmt.Sprintf("--service-cluster-ip-range=%s", serviceNetwork.String()), fmt.Sprintf("--secure-port=%d", port), - "--port=0", + ) + + if versionutils.ConstraintK8sLess124.Check(semver.MustParse(version)) { + command = append(command, "--port=0") + } + + command = append(command, fmt.Sprintf("--horizontal-pod-autoscaler-downscale-stabilization=%s", horizontalPodAutoscalerConfig.DownscaleStabilization.Duration.String()), fmt.Sprintf("--horizontal-pod-autoscaler-initial-readiness-delay=%s", horizontalPodAutoscalerConfig.InitialReadinessDelay.Duration.String()), fmt.Sprintf("--horizontal-pod-autoscaler-cpu-initialization-period=%s", horizontalPodAutoscalerConfig.CPUInitializationPeriod.Duration.String()), diff --git a/pkg/operation/botanist/component/resourcemanager/resource_manager.go b/pkg/operation/botanist/component/resourcemanager/resource_manager.go index 707d5741bfe2..084f86bb620a 100644 --- a/pkg/operation/botanist/component/resourcemanager/resource_manager.go +++ b/pkg/operation/botanist/component/resourcemanager/resource_manager.go @@ -34,7 +34,9 @@ import ( "github.com/gardener/gardener/pkg/utils/retry" "github.com/gardener/gardener/pkg/utils/secrets" secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" + "github.com/gardener/gardener/pkg/utils/version" + "github.com/Masterminds/semver" admissionv1 "k8s.io/api/admission/v1" admissionv1beta1 "k8s.io/api/admission/v1beta1" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" @@ -124,6 +126,9 @@ var ( Verbs: []string{"get", "watch", "update"}, }, } + + rootCAVolumeSourceName string + volumeProjection corev1.VolumeProjection ) // Interface contains functions for a gardener-resource-manager deployer. @@ -203,6 +208,8 @@ type Values struct { // WatchedNamespace restricts the gardener-resource-manager to only watch ManagedResources in the defined namespace. // If not set the gardener-resource-manager controller watches for ManagedResources in all namespaces WatchedNamespace *string + // Version is the Kubernetes version for the Kubernetes components. + Version *semver.Version // VPA contains information for configuring VerticalPodAutoscaler settings for the gardener-resource-manager deployment. VPA *VPAConfig } @@ -448,9 +455,38 @@ func (r *resourceManager) getRootCAVolumeSourceName(ctx context.Context) (string func (r *resourceManager) ensureDeployment(ctx context.Context) error { deployment := r.emptyDeployment() - rootCAVolumeSourceName, err := r.getRootCAVolumeSourceName(ctx) - if err != nil { - return err + if version.ConstraintK8sLess124.Check(r.values.Version) { + var err error + rootCAVolumeSourceName, err = r.getRootCAVolumeSourceName(ctx) + if err != nil { + return err + } + + volumeProjection = corev1.VolumeProjection{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: rootCAVolumeSourceName, + }, + Items: []corev1.KeyToPath{{ + Key: "ca.crt", + Path: "ca.crt", + }}, + }, + } + } else { + rootCAVolumeSourceName = "kube-root-ca.crt" + + volumeProjection = corev1.VolumeProjection{ + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: rootCAVolumeSourceName, + }, + Items: []corev1.KeyToPath{{ + Key: "ca.crt", + Path: "ca.crt", + }}, + }, + } } secretServer, err := r.secretsManager.Generate(ctx, &secrets.CertificateSecretConfig{ @@ -567,17 +603,7 @@ func (r *resourceManager) ensureDeployment(ctx context.Context) error { Path: "token", }, }, - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: rootCAVolumeSourceName, - }, - Items: []corev1.KeyToPath{{ - Key: "ca.crt", - Path: "ca.crt", - }}, - }, - }, + volumeProjection, { DownwardAPI: &corev1.DownwardAPIProjection{ Items: []corev1.DownwardAPIVolumeFile{{ diff --git a/pkg/operation/botanist/component/resourcemanager/resource_manager_test.go b/pkg/operation/botanist/component/resourcemanager/resource_manager_test.go index 4657998b603c..38d58cca0d41 100644 --- a/pkg/operation/botanist/component/resourcemanager/resource_manager_test.go +++ b/pkg/operation/botanist/component/resourcemanager/resource_manager_test.go @@ -30,7 +30,9 @@ import ( fakesecretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager/fake" "github.com/gardener/gardener/pkg/utils/test" . "github.com/gardener/gardener/pkg/utils/test/matchers" + versionutils "github.com/gardener/gardener/pkg/utils/version" + "github.com/Masterminds/semver" "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -71,6 +73,7 @@ var _ = Describe("ResourceManager", func() { healthPort int32 = 8081 metricsPort int32 = 8080 serverPort = 10250 + version = semver.MustParse("1.22.1") // optional configuration clusterIdentity = "foo" @@ -111,6 +114,7 @@ var _ = Describe("ResourceManager", func() { cmd []string cmdWithoutWatchedNamespace []string deployment *appsv1.Deployment + deploymentFor func(kubernetesVersion *semver.Version) *appsv1.Deployment defaultLabels map[string]string roleBinding *rbacv1.RoleBinding role *rbacv1.Role @@ -125,6 +129,7 @@ var _ = Describe("ResourceManager", func() { managedResourceSecret *corev1.Secret managedResource *resourcesv1alpha1.ManagedResource networkPolicy *networkingv1.NetworkPolicy + volumeProjection corev1.VolumeProjection ) BeforeEach(func() { @@ -285,6 +290,7 @@ var _ = Describe("ResourceManager", func() { SyncPeriod: &syncPeriod, TargetDiffersFromSourceCluster: true, TargetDisableCache: &targetDisableCache, + Version: version, WatchedNamespace: &watchedNamespace, VPA: &VPAConfig{ MinAllowed: corev1.ResourceList{ @@ -355,211 +361,256 @@ var _ = Describe("ResourceManager", func() { }, AutomountServiceAccountToken: pointer.Bool(false), } - deployment = &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: v1beta1constants.DeploymentNameGardenerResourceManager, - Namespace: deployNamespace, - Labels: defaultLabels, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: pointer.Int32(1), - RevisionHistoryLimit: pointer.Int32(1), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "gardener-resource-manager", + + if versionutils.ConstraintK8sLess124.Check(cfg.Version) { + volumeProjection = corev1.VolumeProjection{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "root-ca", }, + Items: []corev1.KeyToPath{{ + Key: "ca.crt", + Path: "ca.crt", + }}, }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "projected-token-mount.resources.gardener.cloud/skip": "true", - "networking.gardener.cloud/to-dns": "allowed", - "networking.gardener.cloud/to-seed-apiserver": "allowed", - "networking.gardener.cloud/from-prometheus": "allowed", - "networking.gardener.cloud/to-shoot-apiserver": "allowed", - "networking.gardener.cloud/from-shoot-apiserver": "allowed", - v1beta1constants.GardenRole: v1beta1constants.GardenRoleControlPlane, - v1beta1constants.LabelApp: "gardener-resource-manager", + } + } else { + volumeProjection = corev1.VolumeProjection{ + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "kube-root-ca.crt", + }, + Items: []corev1.KeyToPath{{ + Key: "ca.crt", + Path: "ca.crt", + }}, + }, + } + } + + deploymentFor = func(kubernetesVersion *semver.Version) *appsv1.Deployment { + if versionutils.ConstraintK8sLess124.Check(kubernetesVersion) { + volumeProjection = corev1.VolumeProjection{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: serviceAccountSecretName, + }, + Items: []corev1.KeyToPath{{ + Key: "ca.crt", + Path: "ca.crt", + }}, + }, + } + } else { + volumeProjection = corev1.VolumeProjection{ + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "kube-root-ca.crt", + }, + Items: []corev1.KeyToPath{{ + Key: "ca.crt", + Path: "ca.crt", + }}, + }, + } + } + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: v1beta1constants.DeploymentNameGardenerResourceManager, + Namespace: deployNamespace, + Labels: defaultLabels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: pointer.Int32(1), + RevisionHistoryLimit: pointer.Int32(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "gardener-resource-manager", }, }, - Spec: corev1.PodSpec{ - Affinity: &corev1.Affinity{ - PodAntiAffinity: &corev1.PodAntiAffinity{ - PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ - { - Weight: 100, - PodAffinityTerm: corev1.PodAffinityTerm{ - TopologyKey: corev1.LabelHostname, - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - v1beta1constants.GardenRole: v1beta1constants.GardenRoleControlPlane, - v1beta1constants.LabelApp: "gardener-resource-manager", + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "projected-token-mount.resources.gardener.cloud/skip": "true", + "networking.gardener.cloud/to-dns": "allowed", + "networking.gardener.cloud/to-seed-apiserver": "allowed", + "networking.gardener.cloud/from-prometheus": "allowed", + "networking.gardener.cloud/to-shoot-apiserver": "allowed", + "networking.gardener.cloud/from-shoot-apiserver": "allowed", + v1beta1constants.GardenRole: v1beta1constants.GardenRoleControlPlane, + v1beta1constants.LabelApp: "gardener-resource-manager", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ + { + Weight: 100, + PodAffinityTerm: corev1.PodAffinityTerm{ + TopologyKey: corev1.LabelHostname, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + v1beta1constants.GardenRole: v1beta1constants.GardenRoleControlPlane, + v1beta1constants.LabelApp: "gardener-resource-manager", + }, }, }, }, }, }, }, - }, - ServiceAccountName: "gardener-resource-manager", - Containers: []corev1.Container{ - { - Command: cmd, - Image: image, - ImagePullPolicy: corev1.PullIfNotPresent, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/healthz", - Scheme: "HTTP", - Port: intstr.FromInt(int(healthPort)), + ServiceAccountName: "gardener-resource-manager", + Containers: []corev1.Container{ + { + Command: cmd, + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Scheme: "HTTP", + Port: intstr.FromInt(int(healthPort)), + }, }, + InitialDelaySeconds: 30, + FailureThreshold: 5, + PeriodSeconds: 10, + SuccessThreshold: 1, + TimeoutSeconds: 5, }, - InitialDelaySeconds: 30, - FailureThreshold: 5, - PeriodSeconds: 10, - SuccessThreshold: 1, - TimeoutSeconds: 5, - }, - Name: "gardener-resource-manager", - Ports: []corev1.ContainerPort{ - { - Name: "metrics", - ContainerPort: metricsPort, - Protocol: corev1.ProtocolTCP, - }, - { - Name: "health", - ContainerPort: healthPort, - Protocol: corev1.ProtocolTCP, - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/readyz", - Scheme: "HTTP", - Port: intstr.FromInt(int(healthPort)), + Name: "gardener-resource-manager", + Ports: []corev1.ContainerPort{ + { + Name: "metrics", + ContainerPort: metricsPort, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "health", + ContainerPort: healthPort, + Protocol: corev1.ProtocolTCP, }, }, - InitialDelaySeconds: 10, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("23m"), - corev1.ResourceMemory: resource.MustParse("47Mi"), - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - MountPath: secretMountPathAPIAccess, - Name: "kube-api-access-gardener", - ReadOnly: true, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Scheme: "HTTP", + Port: intstr.FromInt(int(healthPort)), + }, + }, + InitialDelaySeconds: 10, }, - { - MountPath: secretMountPathServer, - Name: "tls", - ReadOnly: true, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("23m"), + corev1.ResourceMemory: resource.MustParse("47Mi"), + }, }, + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: secretMountPathAPIAccess, + Name: "kube-api-access-gardener", + ReadOnly: true, + }, + { + MountPath: secretMountPathServer, + Name: "tls", + ReadOnly: true, + }, - { - MountPath: secretMountPathRootCA, - Name: "root-ca", - ReadOnly: true, - }, - { - Name: "kubeconfig", - MountPath: "/var/run/secrets/gardener.cloud/shoot/generic-kubeconfig", - ReadOnly: true, + { + MountPath: secretMountPathRootCA, + Name: "root-ca", + ReadOnly: true, + }, + { + Name: "kubeconfig", + MountPath: "/var/run/secrets/gardener.cloud/shoot/generic-kubeconfig", + ReadOnly: true, + }, }, }, }, - }, - Volumes: []corev1.Volume{ - { - Name: "kube-api-access-gardener", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - DefaultMode: pointer.Int32(420), - Sources: []corev1.VolumeProjection{ - { - ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ - ExpirationSeconds: pointer.Int64(43200), - Path: "token", - }, - }, - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: serviceAccountSecretName, + Volumes: []corev1.Volume{ + { + Name: "kube-api-access-gardener", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: pointer.Int32(420), + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + ExpirationSeconds: pointer.Int64(43200), + Path: "token", }, - Items: []corev1.KeyToPath{{ - Key: "ca.crt", - Path: "ca.crt", - }}, }, - }, - { - DownwardAPI: &corev1.DownwardAPIProjection{ - Items: []corev1.DownwardAPIVolumeFile{{ - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.namespace", - }, - Path: "namespace", - }}, + volumeProjection, + { + DownwardAPI: &corev1.DownwardAPIProjection{ + Items: []corev1.DownwardAPIVolumeFile{{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.namespace", + }, + Path: "namespace", + }}, + }, }, }, }, }, }, - }, - { - Name: "tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: secretNameServer, - DefaultMode: pointer.Int32(420), + { + Name: "tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secretNameServer, + DefaultMode: pointer.Int32(420), + }, }, }, - }, - { - Name: "root-ca", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "ca", - DefaultMode: pointer.Int32(420), + { + Name: "root-ca", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "ca", + DefaultMode: pointer.Int32(420), + }, }, }, - }, - { - Name: "kubeconfig", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - DefaultMode: pointer.Int32(420), - Sources: []corev1.VolumeProjection{ - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: genericTokenKubeconfigSecretName, + { + Name: "kubeconfig", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: pointer.Int32(420), + Sources: []corev1.VolumeProjection{ + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: genericTokenKubeconfigSecretName, + }, + Items: []corev1.KeyToPath{{ + Key: "kubeconfig", + Path: "kubeconfig", + }}, + Optional: pointer.Bool(false), }, - Items: []corev1.KeyToPath{{ - Key: "kubeconfig", - Path: "kubeconfig", - }}, - Optional: pointer.Bool(false), }, - }, - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "shoot-access-gardener-resource-manager", + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "shoot-access-gardener-resource-manager", + }, + Items: []corev1.KeyToPath{{ + Key: resourcesv1alpha1.DataKeyToken, + Path: resourcesv1alpha1.DataKeyToken, + }}, + Optional: pointer.Bool(false), }, - Items: []corev1.KeyToPath{{ - Key: resourcesv1alpha1.DataKeyToken, - Path: resourcesv1alpha1.DataKeyToken, - }}, - Optional: pointer.Bool(false), }, }, }, @@ -569,7 +620,9 @@ var _ = Describe("ResourceManager", func() { }, }, }, - }, + } + + return deployment } vpa = &vpaautoscalingv1.VerticalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ @@ -858,9 +911,10 @@ subjects: }) Describe("#Deploy", func() { - Context("target cluster != source cluster; watched namespace is set", func() { + Context("target cluster != source cluster; watched namespace is set and kubernetes version <1.24", func() { BeforeEach(func() { role.Namespace = watchedNamespace + deployment = deploymentFor(cfg.Version) resourceManager = New(c, deployNamespace, sm, image, cfg) resourceManager.SetSecrets(secrets) }) @@ -1231,11 +1285,161 @@ subjects: }) }) + Context("target cluster != source cluster; watched namespace is set and kubernetes version >=1.24", func() { + BeforeEach(func() { + role.Namespace = watchedNamespace + cfg.Version = semver.MustParse("1.24.0") + deployment = deploymentFor(cfg.Version) + resourceManager = New(c, deployNamespace, sm, image, cfg) + resourceManager.SetSecrets(secrets) + }) + + It("should successfully deploy all resources (w/ shoot access secret)", func() { + gomock.InOrder( + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, secret.Name), gomock.AssignableToTypeOf(&corev1.Secret{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&corev1.Secret{}), gomock.Any()). + Do(func(ctx context.Context, obj client.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(secret)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "gardener-resource-manager"), gomock.AssignableToTypeOf(&corev1.ServiceAccount{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&corev1.ServiceAccount{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(serviceAccount)) + }), + c.EXPECT().Get(ctx, kutil.Key(watchedNamespace, "gardener-resource-manager"), gomock.AssignableToTypeOf(&rbacv1.Role{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&rbacv1.Role{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(role)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "gardener-resource-manager"), gomock.AssignableToTypeOf(&rbacv1.RoleBinding{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&rbacv1.RoleBinding{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(roleBinding)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "gardener-resource-manager"), gomock.AssignableToTypeOf(&corev1.Service{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&corev1.Service{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(service)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "gardener-resource-manager"), gomock.AssignableToTypeOf(&appsv1.Deployment{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&appsv1.Deployment{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(deployment)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, pdb.Name), gomock.AssignableToTypeOf(&policyv1beta1.PodDisruptionBudget{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&policyv1beta1.PodDisruptionBudget{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(pdb)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "gardener-resource-manager-vpa"), gomock.AssignableToTypeOf(&vpaautoscalingv1.VerticalPodAutoscaler{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&vpaautoscalingv1.VerticalPodAutoscaler{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(vpa)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "managedresource-shoot-core-gardener-resource-manager"), gomock.AssignableToTypeOf(&corev1.Secret{})), + c.EXPECT().Update(ctx, gomock.AssignableToTypeOf(&corev1.Secret{})).Do(func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) { + Expect(obj).To(DeepEqual(managedResourceSecret)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "shoot-core-gardener-resource-manager"), gomock.AssignableToTypeOf(&resourcesv1alpha1.ManagedResource{})), + c.EXPECT().Update(ctx, gomock.AssignableToTypeOf(&resourcesv1alpha1.ManagedResource{})).Do(func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) { + Expect(obj).To(DeepEqual(managedResource)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "allow-kube-apiserver-to-gardener-resource-manager"), gomock.AssignableToTypeOf(&networkingv1.NetworkPolicy{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&networkingv1.NetworkPolicy{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(networkPolicy)) + }), + c.EXPECT().Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: deployNamespace, Name: "gardener-resource-manager-server"}}), + ) + + Expect(resourceManager.Deploy(ctx)).To(Succeed()) + }) + + It("should successfully deploy all resources (w/ bootstrap kubeconfig)", func() { + secretNameBootstrapKubeconfig := "bootstrap-kubeconfig" + + secrets.BootstrapKubeconfig = &component.Secret{Name: secretNameBootstrapKubeconfig} + resourceManager = New(c, deployNamespace, sm, image, cfg) + resourceManager.SetSecrets(secrets) + + gomock.InOrder( + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, secret.Name), gomock.AssignableToTypeOf(&corev1.Secret{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&corev1.Secret{}), gomock.Any()). + Do(func(ctx context.Context, obj client.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(secret)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "gardener-resource-manager"), gomock.AssignableToTypeOf(&corev1.ServiceAccount{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&corev1.ServiceAccount{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(serviceAccount)) + }), + c.EXPECT().Get(ctx, kutil.Key(watchedNamespace, "gardener-resource-manager"), gomock.AssignableToTypeOf(&rbacv1.Role{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&rbacv1.Role{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(role)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "gardener-resource-manager"), gomock.AssignableToTypeOf(&rbacv1.RoleBinding{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&rbacv1.RoleBinding{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(roleBinding)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "gardener-resource-manager"), gomock.AssignableToTypeOf(&corev1.Service{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&corev1.Service{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(service)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "gardener-resource-manager"), gomock.AssignableToTypeOf(&appsv1.Deployment{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&appsv1.Deployment{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + deployment.Spec.Template.Spec.Containers[0].VolumeMounts[len(deployment.Spec.Template.Spec.Containers[0].VolumeMounts)-1].Name = "kubeconfig-bootstrap" + deployment.Spec.Template.Spec.Volumes[len(deployment.Spec.Template.Spec.Volumes)-1] = corev1.Volume{ + Name: "kubeconfig-bootstrap", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secretNameBootstrapKubeconfig, + DefaultMode: pointer.Int32(420), + }, + }, + } + + Expect(obj).To(DeepEqual(deployment)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, pdb.Name), gomock.AssignableToTypeOf(&policyv1beta1.PodDisruptionBudget{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&policyv1beta1.PodDisruptionBudget{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(pdb)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "gardener-resource-manager-vpa"), gomock.AssignableToTypeOf(&vpaautoscalingv1.VerticalPodAutoscaler{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&vpaautoscalingv1.VerticalPodAutoscaler{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(vpa)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "managedresource-shoot-core-gardener-resource-manager"), gomock.AssignableToTypeOf(&corev1.Secret{})), + c.EXPECT().Update(ctx, gomock.AssignableToTypeOf(&corev1.Secret{})).Do(func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) { + Expect(obj).To(DeepEqual(managedResourceSecret)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "shoot-core-gardener-resource-manager"), gomock.AssignableToTypeOf(&resourcesv1alpha1.ManagedResource{})), + c.EXPECT().Update(ctx, gomock.AssignableToTypeOf(&resourcesv1alpha1.ManagedResource{})).Do(func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) { + Expect(obj).To(DeepEqual(managedResource)) + }), + c.EXPECT().Get(ctx, kutil.Key(deployNamespace, "allow-kube-apiserver-to-gardener-resource-manager"), gomock.AssignableToTypeOf(&networkingv1.NetworkPolicy{})), + c.EXPECT().Patch(ctx, gomock.AssignableToTypeOf(&networkingv1.NetworkPolicy{}), gomock.Any()). + Do(func(ctx context.Context, obj runtime.Object, _ client.Patch, _ ...client.PatchOption) { + Expect(obj).To(DeepEqual(networkPolicy)) + }), + c.EXPECT().Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: deployNamespace, Name: "gardener-resource-manager-server"}}), + ) + + Expect(resourceManager.Deploy(ctx)).To(Succeed()) + }) + }) + Context("target cluster != source cluster, watched namespace is nil", func() { BeforeEach(func() { clusterRole.Rules = allowManagedResources cfg.TargetDiffersFromSourceCluster = true cfg.WatchedNamespace = nil + deployment = deploymentFor(cfg.Version) deployment.Spec.Template.Spec.Containers[0].Command = cmdWithoutWatchedNamespace resourceManager = New(c, deployNamespace, sm, image, cfg) @@ -1338,6 +1542,7 @@ subjects: Context("target cluster = source cluster", func() { BeforeEach(func() { clusterRole.Rules = allowAll + deployment = deploymentFor(cfg.Version) for i, cmd := range deployment.Spec.Template.Spec.Containers[0].Command { if strings.HasPrefix(cmd, "--root-ca-file=") { @@ -1426,6 +1631,7 @@ subjects: Describe("#Destroy", func() { Context("target differs from source cluster", func() { BeforeEach(func() { + deployment = deploymentFor(cfg.Version) resourceManager = New(c, deployNamespace, sm, image, cfg) }) @@ -1585,6 +1791,7 @@ subjects: BeforeEach(func() { cfg.TargetDiffersFromSourceCluster = false cfg.WatchedNamespace = nil + deployment = deploymentFor(cfg.Version) deployment.Spec.Template.Spec.Containers[0].Command = cmdWithoutWatchedNamespace resourceManager = New(c, deployNamespace, sm, image, cfg) }) @@ -1608,6 +1815,7 @@ subjects: Describe("#Wait", func() { BeforeEach(func() { + deployment = deploymentFor(cfg.Version) resourceManager = New(fakeClient, deployNamespace, nil, image, cfg) }) diff --git a/pkg/operation/botanist/resource_manager.go b/pkg/operation/botanist/resource_manager.go index c5404fb2426c..4c5f8c4b867b 100644 --- a/pkg/operation/botanist/resource_manager.go +++ b/pkg/operation/botanist/resource_manager.go @@ -37,6 +37,7 @@ import ( secretutils "github.com/gardener/gardener/pkg/utils/secrets" secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" + "github.com/Masterminds/semver" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -73,6 +74,7 @@ func (b *Botanist) DefaultResourceManager() (resourcemanager.Interface, error) { SyncPeriod: utils.DurationPtr(time.Minute), TargetDiffersFromSourceCluster: true, TargetDisableCache: pointer.Bool(true), + Version: semver.MustParse(b.K8sSeedClient.Version()), WatchedNamespace: pointer.String(b.Shoot.SeedNamespace), VPA: &resourcemanager.VPAConfig{ MinAllowed: corev1.ResourceList{ diff --git a/pkg/operation/seed/components.go b/pkg/operation/seed/components.go index b3eaf3bcddfb..c4e57b1faecc 100644 --- a/pkg/operation/seed/components.go +++ b/pkg/operation/seed/components.go @@ -104,7 +104,7 @@ func defaultGardenerSeedAdmissionController(c client.Client, imageVector imageve return seedadmissioncontroller.New(c, v1beta1constants.GardenNamespace, secretsManager, image.String()), nil } -func defaultGardenerResourceManager(c client.Client, imageVector imagevector.ImageVector, secretsManager secretsmanager.Interface) (component.DeployWaiter, error) { +func defaultGardenerResourceManager(c client.Client, seedClientVersion string, imageVector imagevector.ImageVector, secretsManager secretsmanager.Interface) (component.DeployWaiter, error) { image, err := imageVector.FindImage(images.ImageNameGardenerResourceManager) if err != nil { return nil, err @@ -125,6 +125,7 @@ func defaultGardenerResourceManager(c client.Client, imageVector imagevector.Ima ResourceClass: pointer.String(v1beta1constants.SeedResourceManagerClass), SecretNameServerCA: v1beta1constants.SecretNameCASeed, SyncPeriod: utils.DurationPtr(time.Hour), + Version: semver.MustParse(seedClientVersion), VPA: &resourcemanager.VPAConfig{ MinAllowed: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("20m"), diff --git a/pkg/operation/seed/seed.go b/pkg/operation/seed/seed.go index 6b930c9fc22b..28dddb8c9de8 100644 --- a/pkg/operation/seed/seed.go +++ b/pkg/operation/seed/seed.go @@ -393,7 +393,7 @@ func RunReconcileSeedFlow( // Deploy gardener-resource-manager first since it serves central functionality (e.g., projected token mount webhook) // which is required for all other components to start-up. - gardenerResourceManager, err := defaultGardenerResourceManager(seedClient, imageVector, secretsManager) + gardenerResourceManager, err := defaultGardenerResourceManager(seedClient, seedClientSet.Version(), imageVector, secretsManager) if err != nil { return err } diff --git a/pkg/registry/core/backupbucket/storage/tableconvertor.go b/pkg/registry/core/backupbucket/storage/tableconvertor.go index e064f3fe7af8..1f7a3acac7d9 100644 --- a/pkg/registry/core/backupbucket/storage/tableconvertor.go +++ b/pkg/registry/core/backupbucket/storage/tableconvertor.go @@ -56,12 +56,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, obj runtime.Object, tabl if m, err := meta.ListAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/core/backupentry/storage/tableconvertor.go b/pkg/registry/core/backupentry/storage/tableconvertor.go index 96d66ee4743f..bcd2dd261d11 100644 --- a/pkg/registry/core/backupentry/storage/tableconvertor.go +++ b/pkg/registry/core/backupentry/storage/tableconvertor.go @@ -56,12 +56,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, obj runtime.Object, tabl if m, err := meta.ListAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/core/cloudprofile/storage/tableconvertor.go b/pkg/registry/core/cloudprofile/storage/tableconvertor.go index 047ff192e18e..4b7ee37af90a 100644 --- a/pkg/registry/core/cloudprofile/storage/tableconvertor.go +++ b/pkg/registry/core/cloudprofile/storage/tableconvertor.go @@ -54,12 +54,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, obj runtime.Object, tabl if m, err := meta.ListAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/core/controllerdeployment/storage/tableconvertor.go b/pkg/registry/core/controllerdeployment/storage/tableconvertor.go index 0282ff0451dc..4f1b58c5acef 100644 --- a/pkg/registry/core/controllerdeployment/storage/tableconvertor.go +++ b/pkg/registry/core/controllerdeployment/storage/tableconvertor.go @@ -53,12 +53,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, o runtime.Object, tableO if m, err := meta.ListAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/core/controllerinstallation/storage/tableconvertor.go b/pkg/registry/core/controllerinstallation/storage/tableconvertor.go index 46dba36c4602..f1fca53a08a9 100644 --- a/pkg/registry/core/controllerinstallation/storage/tableconvertor.go +++ b/pkg/registry/core/controllerinstallation/storage/tableconvertor.go @@ -59,12 +59,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, o runtime.Object, tableO if m, err := meta.ListAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/core/controllerregistration/storage/tableconvertor.go b/pkg/registry/core/controllerregistration/storage/tableconvertor.go index fae3664378d0..dc2c0db46b31 100644 --- a/pkg/registry/core/controllerregistration/storage/tableconvertor.go +++ b/pkg/registry/core/controllerregistration/storage/tableconvertor.go @@ -56,12 +56,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, o runtime.Object, tableO if m, err := meta.ListAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/core/exposureclass/storage/tableconvertor.go b/pkg/registry/core/exposureclass/storage/tableconvertor.go index 990991733354..dcfbc402251c 100644 --- a/pkg/registry/core/exposureclass/storage/tableconvertor.go +++ b/pkg/registry/core/exposureclass/storage/tableconvertor.go @@ -54,12 +54,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, obj runtime.Object, tabl if m, err := meta.ListAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/core/plant/storage/tableconvertor.go b/pkg/registry/core/plant/storage/tableconvertor.go index 174606b999f1..d46e1e421c32 100644 --- a/pkg/registry/core/plant/storage/tableconvertor.go +++ b/pkg/registry/core/plant/storage/tableconvertor.go @@ -59,12 +59,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, o runtime.Object, tableO if m, err := meta.ListAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/core/project/storage/tableconvertor.go b/pkg/registry/core/project/storage/tableconvertor.go index 9544d53635d2..f1d11b493341 100644 --- a/pkg/registry/core/project/storage/tableconvertor.go +++ b/pkg/registry/core/project/storage/tableconvertor.go @@ -57,12 +57,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, obj runtime.Object, tabl if m, err := meta.ListAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/core/quota/storage/tableconvertor.go b/pkg/registry/core/quota/storage/tableconvertor.go index ea3f2bc9264b..1d02a8a1189d 100644 --- a/pkg/registry/core/quota/storage/tableconvertor.go +++ b/pkg/registry/core/quota/storage/tableconvertor.go @@ -56,12 +56,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, obj runtime.Object, tabl if m, err := meta.ListAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/core/secretbinding/storage/tableconvertor.go b/pkg/registry/core/secretbinding/storage/tableconvertor.go index 0efb53ad8591..5de65082687d 100644 --- a/pkg/registry/core/secretbinding/storage/tableconvertor.go +++ b/pkg/registry/core/secretbinding/storage/tableconvertor.go @@ -55,12 +55,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, obj runtime.Object, tabl if m, err := meta.ListAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/core/seed/storage/tableconvertor.go b/pkg/registry/core/seed/storage/tableconvertor.go index db4961bcd62a..90f4fa660fe6 100644 --- a/pkg/registry/core/seed/storage/tableconvertor.go +++ b/pkg/registry/core/seed/storage/tableconvertor.go @@ -59,12 +59,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, obj runtime.Object, tabl if m, err := meta.ListAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/core/shoot/storage/tableconvertor.go b/pkg/registry/core/shoot/storage/tableconvertor.go index 5aa371dac97c..07acd410f9fb 100644 --- a/pkg/registry/core/shoot/storage/tableconvertor.go +++ b/pkg/registry/core/shoot/storage/tableconvertor.go @@ -71,12 +71,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, obj runtime.Object, tabl if m, err := meta.ListAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/core/shootstate/storage/tableconvertor.go b/pkg/registry/core/shootstate/storage/tableconvertor.go index 19d36d03750c..830c45665d83 100644 --- a/pkg/registry/core/shootstate/storage/tableconvertor.go +++ b/pkg/registry/core/shootstate/storage/tableconvertor.go @@ -55,12 +55,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, o runtime.Object, tableO if m, err := meta.ListAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/operations/bastion/storage/tableconvertor.go b/pkg/registry/operations/bastion/storage/tableconvertor.go index 169c6b6ac63a..a468c3316dc8 100644 --- a/pkg/registry/operations/bastion/storage/tableconvertor.go +++ b/pkg/registry/operations/bastion/storage/tableconvertor.go @@ -61,12 +61,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, obj runtime.Object, tabl if m, err := meta.ListAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/seedmanagement/managedseed/storage/tableconvertor.go b/pkg/registry/seedmanagement/managedseed/storage/tableconvertor.go index 4b9e55455ea2..5e5b9c4ed99e 100644 --- a/pkg/registry/seedmanagement/managedseed/storage/tableconvertor.go +++ b/pkg/registry/seedmanagement/managedseed/storage/tableconvertor.go @@ -59,12 +59,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, obj runtime.Object, tabl if m, err := meta.ListAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/seedmanagement/managedseedset/storage/tableconvertor.go b/pkg/registry/seedmanagement/managedseedset/storage/tableconvertor.go index 34a98f1baac1..b7445cf8b0ee 100644 --- a/pkg/registry/seedmanagement/managedseedset/storage/tableconvertor.go +++ b/pkg/registry/seedmanagement/managedseedset/storage/tableconvertor.go @@ -56,12 +56,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, obj runtime.Object, tabl if m, err := meta.ListAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(obj); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/settings/clusteropenidconnectpreset/storage/tableconvertor.go b/pkg/registry/settings/clusteropenidconnectpreset/storage/tableconvertor.go index 9debc651f0d4..40a31f49e108 100644 --- a/pkg/registry/settings/clusteropenidconnectpreset/storage/tableconvertor.go +++ b/pkg/registry/settings/clusteropenidconnectpreset/storage/tableconvertor.go @@ -55,12 +55,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, o runtime.Object, tableO if m, err := meta.ListAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/registry/settings/openidconnectpreset/storage/tableconvertor.go b/pkg/registry/settings/openidconnectpreset/storage/tableconvertor.go index f5aa92f3b204..3013f5298680 100644 --- a/pkg/registry/settings/openidconnectpreset/storage/tableconvertor.go +++ b/pkg/registry/settings/openidconnectpreset/storage/tableconvertor.go @@ -55,12 +55,10 @@ func (c *convertor) ConvertToTable(ctx context.Context, o runtime.Object, tableO if m, err := meta.ListAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() table.Continue = m.GetContinue() } else { if m, err := meta.CommonAccessor(o); err == nil { table.ResourceVersion = m.GetResourceVersion() - table.SelfLink = m.GetSelfLink() } } diff --git a/pkg/utils/kubernetes/unstructured/object.go b/pkg/utils/kubernetes/unstructured/object.go index f54bebdeae9e..f8fedb5c90a8 100644 --- a/pkg/utils/kubernetes/unstructured/object.go +++ b/pkg/utils/kubernetes/unstructured/object.go @@ -27,7 +27,7 @@ import ( "github.com/gardener/gardener/pkg/utils" ) -var systemMetadataFields = []string{"ownerReferences", "uid", "resourceVersion", "generation", "selfLink", "creationTimestamp", "deletionTimestamp", "deletionGracePeriodSeconds", "managedFields"} +var systemMetadataFields = []string{"ownerReferences", "uid", "resourceVersion", "generation", "creationTimestamp", "deletionTimestamp", "deletionGracePeriodSeconds", "managedFields"} // GetObjectByRef returns the object with the given reference and namespace using the given client. // The full content of the object is returned as map[string]interface{}, except for system metadata fields. diff --git a/pkg/utils/validation/features/featuregates.go b/pkg/utils/validation/features/featuregates.go index d9328eb1ce3a..7d0a4a55c51e 100644 --- a/pkg/utils/validation/features/featuregates.go +++ b/pkg/utils/validation/features/featuregates.go @@ -55,6 +55,7 @@ var featureGateVersionRanges = map[string]*FeatureGateVersionRange{ "CPUManagerPolicyBetaOptions": {AddedInVersion: "1.23"}, "CPUManagerPolicyOptions": {AddedInVersion: "1.22"}, "CRIContainerLogRotation": {RemovedInVersion: "1.22"}, + "CronJobTimeZone": {AddedInVersion: "1.24"}, "CSIBlockVolume": {RemovedInVersion: "1.21"}, "CSIDriverRegistry": {RemovedInVersion: "1.21"}, "CSIInlineVolume": {}, @@ -67,20 +68,21 @@ var featureGateVersionRanges = map[string]*FeatureGateVersionRange{ "CSIMigrationAzureFileComplete": {AddedInVersion: "1.17", RemovedInVersion: "1.21"}, "CSIMigrationGCE": {}, "CSIMigrationGCEComplete": {AddedInVersion: "1.17", RemovedInVersion: "1.21"}, - "CSIMigrationOpenStack": {}, + "CSIMigrationOpenStack": {Default: true, AddedInVersion: "1.14", LockedToDefaultInVersion: "1.24"}, "CSIMigrationOpenStackComplete": {AddedInVersion: "1.17", RemovedInVersion: "1.21"}, "CSIMigrationPortworx": {AddedInVersion: "1.23"}, + "CSIMigrationRBD": {AddedInVersion: "1.24"}, "CSIMigrationvSphere": {AddedInVersion: "1.19"}, "CSIMigrationvSphereComplete": {AddedInVersion: "1.19", RemovedInVersion: "1.22"}, "CSINodeInfo": {RemovedInVersion: "1.21"}, "CSIPersistentVolume": {RemovedInVersion: "1.16"}, "CSIServiceAccountToken": {Default: true, AddedInVersion: "1.20", LockedToDefaultInVersion: "1.22"}, - "CSIStorageCapacity": {AddedInVersion: "1.19"}, + "CSIStorageCapacity": {Default: true, AddedInVersion: "1.19", LockedToDefaultInVersion: "1.24"}, "CSIVolumeFSGroupPolicy": {Default: true, AddedInVersion: "1.19", LockedToDefaultInVersion: "1.23"}, "CSIVolumeHealth": {AddedInVersion: "1.21"}, - "CSRDuration": {AddedInVersion: "1.22"}, + "CSRDuration": {Default: true, AddedInVersion: "1.22", LockedToDefaultInVersion: "1.24"}, "ConfigurableFSGroupPolicy": {Default: true, AddedInVersion: "1.18", LockedToDefaultInVersion: "1.23"}, - "ControllerManagerLeaderMigration": {AddedInVersion: "1.21"}, // Missing from docu? + "ControllerManagerLeaderMigration": {Default: true, AddedInVersion: "1.21", LockedToDefaultInVersion: "1.24"}, // Missing from docu? "CronJobControllerV2": {AddedInVersion: "1.20", RemovedInVersion: "1.23"}, "CustomCPUCFSQuotaPeriod": {}, "CustomPodDNS": {RemovedInVersion: "1.16"}, @@ -93,7 +95,7 @@ var featureGateVersionRanges = map[string]*FeatureGateVersionRange{ "DaemonSetUpdateSurge": {AddedInVersion: "1.21"}, // Missing from docu? "DebugContainers": {RemovedInVersion: "1.16"}, // Missing from docu? "DefaultIngressClass": {AddedInVersion: "1.18", RemovedInVersion: "1.20"}, // Missing from docu? - "DefaultPodTopologySpread": {AddedInVersion: "1.19"}, + "DefaultPodTopologySpread": {Default: true, AddedInVersion: "1.19", LockedToDefaultInVersion: "1.24"}, "DelegateFSGroupToCSIDriver": {AddedInVersion: "1.22"}, "DevicePlugins": {}, "DisableAcceleratorUsageMetrics": {AddedInVersion: "1.19"}, @@ -103,7 +105,7 @@ var featureGateVersionRanges = map[string]*FeatureGateVersionRange{ "DryRun": {}, "DynamicAuditing": {RemovedInVersion: "1.19"}, "DynamicKubeletConfig": {}, - "EfficientWatchResumption": {AddedInVersion: "1.20"}, + "EfficientWatchResumption": {Default: true, AddedInVersion: "1.20", LockedToDefaultInVersion: "1.24"}, "EnableAggregatedDiscoveryTimeout": {AddedInVersion: "1.16", RemovedInVersion: "1.17"}, "EndpointSlice": {Default: true, AddedInVersion: "1.16", LockedToDefaultInVersion: "1.21"}, "EndpointSliceNodeName": {Default: true, AddedInVersion: "1.20", LockedToDefaultInVersion: "1.21"}, @@ -127,12 +129,12 @@ var featureGateVersionRanges = map[string]*FeatureGateVersionRange{ "HonorPVReclaimPolicy": {AddedInVersion: "1.23"}, "HPAContainerMetrics": {AddedInVersion: "1.20"}, "HPAScaleToZero": {AddedInVersion: "1.16"}, - "HugePageStorageMediumSize": {Default: true, AddedInVersion: "1.18", LockedToDefaultInVersion: "1.22"}, + "HugePageStorageMediumSize": {Default: true, AddedInVersion: "1.18", LockedToDefaultInVersion: "1.22", RemovedInVersion: "1.24"}, "HugePages": {RemovedInVersion: "1.16"}, "HyperVContainer": {RemovedInVersion: "1.21"}, "IPv6DualStack": {Default: true, AddedInVersion: "1.16", LockedToDefaultInVersion: "1.23"}, "IdentifyPodOS": {AddedInVersion: "1.23"}, - "ImmutableEphemeralVolumes": {Default: true, AddedInVersion: "1.18", LockedToDefaultInVersion: "1.21"}, + "ImmutableEphemeralVolumes": {Default: true, AddedInVersion: "1.18", LockedToDefaultInVersion: "1.21", RemovedInVersion: "1.24"}, "InTreePluginAWSUnregister": {AddedInVersion: "1.21"}, // Missing from docu? "InTreePluginAzureDiskUnregister": {AddedInVersion: "1.21"}, // Missing from docu? "InTreePluginAzureFileUnregister": {AddedInVersion: "1.21"}, // Missing from docu? @@ -141,7 +143,7 @@ var featureGateVersionRanges = map[string]*FeatureGateVersionRange{ "InTreePluginPortworxUnregister": {AddedInVersion: "1.23"}, "InTreePluginRBDUnregister": {AddedInVersion: "1.23"}, "InTreePluginvSphereUnregister": {AddedInVersion: "1.21"}, // Missing from docu? - "IndexedJob": {AddedInVersion: "1.21"}, + "IndexedJob": {Default: true, AddedInVersion: "1.21", LockedToDefaultInVersion: "1.24"}, "IngressClassNamespacedParams": {Default: true, AddedInVersion: "1.21", LockedToDefaultInVersion: "1.23"}, "JobMutableNodeSchedulingDirectives": {AddedInVersion: "1.23"}, "JobReadyPods": {AddedInVersion: "1.23"}, @@ -152,33 +154,38 @@ var featureGateVersionRanges = map[string]*FeatureGateVersionRange{ "KubeletPodResources": {}, "KubeletPodResourcesGetAllocatable": {AddedInVersion: "1.21"}, "LegacyNodeRoleBehavior": {AddedInVersion: "1.16", RemovedInVersion: "1.22"}, + "LegacyServiceAccountTokenNoAutoGeneration": {AddedInVersion: "1.24"}, "LocalStorageCapacityIsolation": {}, "LocalStorageCapacityIsolationFSQuotaMonitoring": {}, "LogarithmicScaleDown": {AddedInVersion: "1.21"}, + "MaxUnavailableStatefulSet": {AddedInVersion: "1.24"}, "MemoryManager": {AddedInVersion: "1.21"}, // Missing from docu? "MemoryQoS": {AddedInVersion: "1.22"}, - "MigrationRBD": {AddedInVersion: "1.23"}, + "MigrationRBD": {AddedInVersion: "1.23", RemovedInVersion: "1.24"}, + "MinDomainsInPodTopologySpread": {AddedInVersion: "1.24"}, "MixedProtocolLBService": {AddedInVersion: "1.20"}, "MountContainers": {RemovedInVersion: "1.17"}, - "NamespaceDefaultLabelName": {Default: true, AddedInVersion: "1.21", LockedToDefaultInVersion: "1.22"}, + "NamespaceDefaultLabelName": {Default: true, AddedInVersion: "1.21", LockedToDefaultInVersion: "1.22", RemovedInVersion: "1.24"}, "NetworkPolicyEndPort": {AddedInVersion: "1.21"}, + "NetworkPolicyStatus": {AddedInVersion: "1.24"}, "NodeDisruptionExclusion": {AddedInVersion: "1.16", RemovedInVersion: "1.22"}, "NodeLease": {RemovedInVersion: "1.23"}, - "NonPreemptingPriority": {}, + "NodeOutOfServiceVolumeDetach": {AddedInVersion: "1.24"}, + "NonPreemptingPriority": {Default: true, LockedToDefaultInVersion: "1.24"}, "NodeSwap": {AddedInVersion: "1.22"}, "OpenAPIEnums": {AddedInVersion: "1.23"}, "OpenAPIV3": {AddedInVersion: "1.23"}, "PersistentLocalVolumes": {RemovedInVersion: "1.17"}, "PodAndContainerStatsFromCRI": {AddedInVersion: "1.23"}, - "PodAffinityNamespaceSelector": {AddedInVersion: "1.21"}, + "PodAffinityNamespaceSelector": {Default: true, AddedInVersion: "1.21", LockedToDefaultInVersion: "1.24"}, "PodDeletionCost": {AddedInVersion: "1.21"}, "PodDisruptionBudget": {Default: true, AddedInVersion: "1.17", LockedToDefaultInVersion: "1.21"}, // Docu says 1.3? - "PodOverhead": {AddedInVersion: "1.16"}, + "PodOverhead": {Default: true, AddedInVersion: "1.16", LockedToDefaultInVersion: "1.24"}, "PodPriority": {RemovedInVersion: "1.18"}, "PodReadinessGates": {RemovedInVersion: "1.16"}, "PodSecurity": {AddedInVersion: "1.22"}, "PodShareProcessNamespace": {RemovedInVersion: "1.19"}, - "PreferNominatedNode": {AddedInVersion: "1.21"}, // Missing from docu? + "PreferNominatedNode": {Default: true, AddedInVersion: "1.21", LockedToDefaultInVersion: "1.24"}, // Missing from docu? "ProbeTerminationGracePeriod": {AddedInVersion: "1.21"}, "ProcMountType": {}, "ProxyTerminatingEndpoints": {AddedInVersion: "1.22"}, @@ -186,7 +193,7 @@ var featureGateVersionRanges = map[string]*FeatureGateVersionRange{ "ReadWriteOncePod": {AddedInVersion: "1.22"}, "RecoverVolumeExpansionFailure": {AddedInVersion: "1.23"}, "RemainingItemCount": {}, - "RemoveSelfLink": {AddedInVersion: "1.16"}, + "RemoveSelfLink": {Default: true, AddedInVersion: "1.16", LockedToDefaultInVersion: "1.24"}, "RequestManagement": {RemovedInVersion: "1.17"}, "ResourceLimitsPriorityFunction": {RemovedInVersion: "1.19"}, "ResourceQuotaScopeSelectors": {RemovedInVersion: "1.18"}, @@ -194,7 +201,7 @@ var featureGateVersionRanges = map[string]*FeatureGateVersionRange{ "RotateKubeletClientCertificate": {RemovedInVersion: "1.21"}, "RotateKubeletServerCertificate": {}, "RunAsGroup": {RemovedInVersion: "1.22"}, - "RuntimeClass": {Default: true, LockedToDefaultInVersion: "1.20"}, + "RuntimeClass": {Default: true, LockedToDefaultInVersion: "1.20", RemovedInVersion: "1.24"}, "SCTPSupport": {RemovedInVersion: "1.22"}, "ScheduleDaemonSetPods": {RemovedInVersion: "1.18"}, "SeccompDefault": {AddedInVersion: "1.22"}, @@ -204,12 +211,13 @@ var featureGateVersionRanges = map[string]*FeatureGateVersionRange{ "ServiceAccountIssuerDiscovery": {AddedInVersion: "1.18", RemovedInVersion: "1.23"}, "ServiceAppProtocol": {AddedInVersion: "1.18", RemovedInVersion: "1.22"}, "ServiceInternalTrafficPolicy": {AddedInVersion: "1.21"}, - "ServiceLBNodePortControl": {AddedInVersion: "1.20"}, - "ServiceLoadBalancerClass": {AddedInVersion: "1.21"}, + "ServiceIPStaticSubrange": {AddedInVersion: "1.24"}, + "ServiceLBNodePortControl": {Default: true, AddedInVersion: "1.20", LockedToDefaultInVersion: "1.24"}, + "ServiceLoadBalancerClass": {Default: true, AddedInVersion: "1.21", LockedToDefaultInVersion: "1.24"}, "ServiceLoadBalancerFinalizer": {RemovedInVersion: "1.20"}, "ServiceNodeExclusion": {RemovedInVersion: "1.22"}, "ServiceTopology": {AddedInVersion: "1.17", RemovedInVersion: "1.22"}, - "SetHostnameAsFQDN": {Default: true, AddedInVersion: "1.19", LockedToDefaultInVersion: "1.22"}, + "SetHostnameAsFQDN": {Default: true, AddedInVersion: "1.19", LockedToDefaultInVersion: "1.22", RemovedInVersion: "1.24"}, "SizeMemoryBackedVolumes": {AddedInVersion: "1.20"}, "StartupProbe": {AddedInVersion: "1.16", RemovedInVersion: "1.23"}, "StatefulSetAutoDeletePVC": {AddedInVersion: "1.23"}, @@ -217,11 +225,11 @@ var featureGateVersionRanges = map[string]*FeatureGateVersionRange{ "StorageObjectInUseProtection": {Default: true, LockedToDefaultInVersion: "1.23"}, "StorageVersionAPI": {AddedInVersion: "1.20"}, "StorageVersionHash": {}, - "StreamingProxyRedirects": {}, + "StreamingProxyRedirects": {RemovedInVersion: "1.24"}, "SupportIPVSProxyMode": {RemovedInVersion: "1.20"}, "SupportNodePidsLimit": {RemovedInVersion: "1.23"}, "SupportPodPidsLimit": {RemovedInVersion: "1.23"}, - "SuspendJob": {AddedInVersion: "1.21"}, + "SuspendJob": {Default: true, AddedInVersion: "1.21", LockedToDefaultInVersion: "1.24"}, "Sysctls": {RemovedInVersion: "1.23"}, "TTLAfterFinished": {Default: true, LockedToDefaultInVersion: "1.23"}, "TaintBasedEvictions": {RemovedInVersion: "1.20"}, @@ -230,14 +238,14 @@ var featureGateVersionRanges = map[string]*FeatureGateVersionRange{ "TokenRequestProjection": {RemovedInVersion: "1.21"}, "TopologyAwareHints": {AddedInVersion: "1.21"}, "TopologyManager": {AddedInVersion: "1.16"}, - "ValidateProxyRedirects": {}, + "ValidateProxyRedirects": {RemovedInVersion: "1.24"}, "VolumeCapacityPriority": {AddedInVersion: "1.21"}, "VolumePVCDataSource": {RemovedInVersion: "1.21"}, "VolumeScheduling": {RemovedInVersion: "1.16"}, "VolumeSnapshotDataSource": {RemovedInVersion: "1.22"}, "VolumeSubpath": {}, "VolumeSubpathEnvExpansion": {RemovedInVersion: "1.19"}, - "WarningHeaders": {Default: true, AddedInVersion: "1.19", LockedToDefaultInVersion: "1.22"}, + "WarningHeaders": {Default: true, AddedInVersion: "1.19", LockedToDefaultInVersion: "1.22", RemovedInVersion: "1.24"}, "WatchBookmark": {Default: true, LockedToDefaultInVersion: "1.17"}, "WinDSR": {}, "WinOverlay": {}, diff --git a/pkg/utils/version/version.go b/pkg/utils/version/version.go index 31dfeb6c0c14..5e4da858d539 100644 --- a/pkg/utils/version/version.go +++ b/pkg/utils/version/version.go @@ -55,6 +55,8 @@ var ( ConstraintK8sEqual123 *semver.Constraints // ConstraintK8sGreaterEqual123 is a version constraint for versions >= 1.23. ConstraintK8sGreaterEqual123 *semver.Constraints + // ConstraintK8sLess124 is a version constraint for versions < 1.24. + ConstraintK8sLess124 *semver.Constraints ) func init() { @@ -92,6 +94,8 @@ func init() { utilruntime.Must(err) ConstraintK8sGreaterEqual123, err = semver.NewConstraint(">= 1.23") utilruntime.Must(err) + ConstraintK8sLess124, err = semver.NewConstraint("< 1.24") + utilruntime.Must(err) } // CompareVersions returns true if the constraint compared by to diff --git a/test/e2e/shoot/common.go b/test/e2e/shoot/common.go index 9e2b9f480d89..92cf3d44c292 100644 --- a/test/e2e/shoot/common.go +++ b/test/e2e/shoot/common.go @@ -67,7 +67,7 @@ func defaultShoot(generateName string) *gardencorev1beta1.Shoot { CloudProfileName: "local", SeedName: pointer.String("local"), Kubernetes: gardencorev1beta1.Kubernetes{ - Version: "1.23.6", + Version: "1.24.0", }, Networking: gardencorev1beta1.Networking{ Type: "calico",