From f9b330698aef416669984ced34b23682dca9ba60 Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Thu, 5 Jan 2023 12:34:34 +0100 Subject: [PATCH] KCP: block upgrade to versions with old registry, improve registry handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stefan Büringer buringerst@vmware.com --- .../kubeadm/api/v1beta1/kubeadm_types.go | 13 +- ...strap.cluster.x-k8s.io_kubeadmconfigs.yaml | 20 +- ...uster.x-k8s.io_kubeadmconfigtemplates.yaml | 23 +- .../v1beta1/kubeadm_control_plane_types.go | 5 + .../v1beta1/kubeadm_control_plane_webhook.go | 31 ++- .../kubeadm_control_plane_webhook_test.go | 201 ++++++++++++++---- ...cluster.x-k8s.io_kubeadmcontrolplanes.yaml | 25 ++- ...x-k8s.io_kubeadmcontrolplanetemplates.yaml | 21 +- .../kubeadm/internal/workload_cluster.go | 34 ++- .../internal/workload_cluster_coredns.go | 20 +- .../internal/workload_cluster_coredns_test.go | 42 +++- internal/util/kubeadm/kubeadm.go | 72 +++++++ internal/util/kubeadm/kubeadm_test.go | 68 ++++++ test/framework/daemonset_helpers.go | 13 +- .../docker/internal/docker/machine.go | 4 +- 15 files changed, 467 insertions(+), 125 deletions(-) create mode 100644 internal/util/kubeadm/kubeadm.go create mode 100644 internal/util/kubeadm/kubeadm_test.go diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go b/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go index f8ea593ec0ac..dc2bb60876f9 100644 --- a/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go +++ b/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go @@ -125,9 +125,16 @@ type ClusterConfiguration struct { CertificatesDir string `json:"certificatesDir,omitempty"` // ImageRepository sets the container registry to pull images from. - // If empty, `registry.k8s.io` will be used by default; in case of kubernetes version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`) - // `gcr.io/k8s-staging-ci-images` will be used as a default for control plane components and for kube-proxy, while `registry.k8s.io` - // will be used for all the other images. + // * If not set, the default registry of kubeadm will be used, i.e. + // * registry.k8s.io (new registry): >= v1.22.17, >= v1.23.15, >= v1.24.9, >= v1.25.0 + // * k8s.gcr.io (old registry): all older versions + // Please note that when imageRepository is not set we don't allow upgrades to + // versions >= v1.22.0 which use the old registry (k8s.gcr.io). Please use + // a newer patch version with the new registry instead (i.e. >= v1.22.17, + // >= v1.23.15, >= v1.24.9, >= v1.25.0). + // * If the version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`) + // `gcr.io/k8s-staging-ci-images` will be used as a default for control plane components + // and for kube-proxy, while `registry.k8s.io` will be used for all the other images. // +optional ImageRepository string `json:"imageRepository,omitempty"` diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml index 71c60711ac0a..1ae748175bdb 100644 --- a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml +++ b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml @@ -2243,13 +2243,19 @@ spec: description: FeatureGates enabled by the user. type: object imageRepository: - description: ImageRepository sets the container registry to pull - images from. If empty, `registry.k8s.io` will be used by default; - in case of kubernetes version is a CI build (kubernetes version - starts with `ci/` or `ci-cross/`) `gcr.io/k8s-staging-ci-images` - will be used as a default for control plane components and for - kube-proxy, while `registry.k8s.io` will be used for all the - other images. + description: 'ImageRepository sets the container registry to pull + images from. * If not set, the default registry of kubeadm will + be used, i.e. * registry.k8s.io (new registry): >= v1.22.17, + >= v1.23.15, >= v1.24.9, >= v1.25.0 * k8s.gcr.io (old registry): + all older versions Please note that when imageRepository is + not set we don''t allow upgrades to versions >= v1.22.0 which + use the old registry (k8s.gcr.io). Please use a newer patch + version with the new registry instead (i.e. >= v1.22.17, >= + v1.23.15, >= v1.24.9, >= v1.25.0). * If the version is a CI + build (kubernetes version starts with `ci/` or `ci-cross/`) + `gcr.io/k8s-staging-ci-images` will be used as a default for + control plane components and for kube-proxy, while `registry.k8s.io` + will be used for all the other images.' type: string kind: description: 'Kind is a string value representing the REST resource diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml index 85493b712abd..b9b33cb1fac5 100644 --- a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml +++ b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml @@ -2246,14 +2246,21 @@ spec: description: FeatureGates enabled by the user. type: object imageRepository: - description: ImageRepository sets the container registry - to pull images from. If empty, `registry.k8s.io` will - be used by default; in case of kubernetes version is - a CI build (kubernetes version starts with `ci/` or - `ci-cross/`) `gcr.io/k8s-staging-ci-images` will be - used as a default for control plane components and for - kube-proxy, while `registry.k8s.io` will be used for - all the other images. + description: 'ImageRepository sets the container registry + to pull images from. * If not set, the default registry + of kubeadm will be used, i.e. * registry.k8s.io (new + registry): >= v1.22.17, >= v1.23.15, >= v1.24.9, >= + v1.25.0 * k8s.gcr.io (old registry): all older versions + Please note that when imageRepository is not set we + don''t allow upgrades to versions >= v1.22.0 which use + the old registry (k8s.gcr.io). Please use a newer patch + version with the new registry instead (i.e. >= v1.22.17, + >= v1.23.15, >= v1.24.9, >= v1.25.0). * If the version + is a CI build (kubernetes version starts with `ci/` + or `ci-cross/`) `gcr.io/k8s-staging-ci-images` will + be used as a default for control plane components and + for kube-proxy, while `registry.k8s.io` will be used + for all the other images.' type: string kind: description: 'Kind is a string value representing the diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go index 8f07c1dd91c6..823fb123f679 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go @@ -60,6 +60,11 @@ type KubeadmControlPlaneSpec struct { Replicas *int32 `json:"replicas,omitempty"` // Version defines the desired Kubernetes version. + // Please note that if kubeadmConfigSpec.ClusterConfiguration.imageRepository is not set + // we don't allow upgrades to versions >= v1.22.0 for which kubeadm uses the old registry (k8s.gcr.io). + // Please use a newer patch version with the new registry instead. The default registries of kubeadm are: + // * registry.k8s.io (new registry): >= v1.22.17, >= v1.23.15, >= v1.24.9, >= v1.25.0 + // * k8s.gcr.io (old registry): all older versions Version string `json:"version"` // MachineTemplate contains information about how machines diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go index e8f4a68367cb..dd512fe26967 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + "sigs.k8s.io/cluster-api/internal/util/kubeadm" "sigs.k8s.io/cluster-api/util/container" "sigs.k8s.io/cluster-api/util/version" ) @@ -601,7 +602,10 @@ func (in *KubeadmControlPlane) validateVersion(previousVersion string) (allErrs return allErrs } - // Since upgrades to the next minor version are allowed, irrespective of the patch version. + // Validate that the update is upgrading at most one minor version. + // Note: Skipping a minor version is not allowed. + // Note: Checking against this ceilVersion allows upgrading to the next minor + // version irrespective of the patch version. ceilVersion := semver.Version{ Major: fromVersion.Major, Minor: fromVersion.Minor + 2, @@ -616,6 +620,31 @@ func (in *KubeadmControlPlane) validateVersion(previousVersion string) (allErrs ) } + // The Kubernetes ecosystem has been requested to move users to the new registry due to cost issues. + // This validation enforces the move to the new registry by forcing users to upgrade to kubeadm versions + // with the new registry. + // NOTE: This only affects users relying on the community maintained registry. + // NOTE: Pinning to the upstream registry is not recommended because it could lead to issues + // given how the migration has been implemented in kubeadm. + // + // Block if imageRepository is not set (i.e. the default registry should be used), + if (in.Spec.KubeadmConfigSpec.ClusterConfiguration == nil || + in.Spec.KubeadmConfigSpec.ClusterConfiguration.ImageRepository == "") && + // the version changed (i.e. we have an upgrade), + toVersion.NE(fromVersion) && + // the version is >= v1.22.0 and < v1.26.0 + toVersion.GTE(kubeadm.MinKubernetesVersionImageRegistryMigration) && + toVersion.LT(kubeadm.NextKubernetesVersionImageRegistryMigration) && + // and the default registry of the new Kubernetes/kubeadm version is the old default registry. + kubeadm.GetDefaultRegistry(toVersion) == kubeadm.OldDefaultImageRepository { + allErrs = append(allErrs, + field.Forbidden( + field.NewPath("spec", "version"), + "cannot upgrade to a Kubernetes/kubeadm version which is using the old default registry. Please use a newer Kubernetes patch release which is using the new default registry (>= v1.22.17, >= v1.23.15, >= v1.24.9)", + ), + ) + } + return allErrs } diff --git a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go index f13d34b36cb9..3ead11553c28 100644 --- a/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go +++ b/controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go @@ -451,14 +451,6 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) { kubernetesVersion := before.DeepCopy() kubernetesVersion.Spec.KubeadmConfigSpec.ClusterConfiguration.KubernetesVersion = "some kubernetes version" - prevKCPWithVersion := func(version string) *KubeadmControlPlane { - prev := before.DeepCopy() - prev.Spec.Version = version - return prev - } - skipMinorControlPlaneVersion := prevKCPWithVersion("v1.18.1") - emptyControlPlaneVersion := prevKCPWithVersion("") - controlPlaneEndpoint := before.DeepCopy() controlPlaneEndpoint.Spec.KubeadmConfigSpec.ClusterConfiguration.ControlPlaneEndpoint = "some control plane endpoint" @@ -611,13 +603,6 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) { DataDir: "/data", } - disallowedUpgrade118Prev := prevKCPWithVersion("v1.18.8") - disallowedUpgrade119Version := before.DeepCopy() - disallowedUpgrade119Version.Spec.Version = "v1.19.0" - - disallowedUpgrade120AlphaVersion := before.DeepCopy() - disallowedUpgrade120AlphaVersion.Spec.Version = "v1.20.0-alpha.0.734_ba502ee555924a" - updateNTPServers := before.DeepCopy() updateNTPServers.Spec.KubeadmConfigSpec.NTP.Servers = []string{"new-server"} @@ -925,36 +910,6 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) { before: withoutClusterConfiguration, kcp: afterEtcdLocalDirAddition, }, - { - name: "should fail when skipping control plane minor versions", - expectErr: true, - before: before, - kcp: skipMinorControlPlaneVersion, - }, - { - name: "should fail when no control plane version is passed", - expectErr: true, - before: before, - kcp: emptyControlPlaneVersion, - }, - { - name: "should pass if control plane version is the same", - expectErr: false, - before: before, - kcp: before.DeepCopy(), - }, - { - name: "should return error when trying to upgrade to v1.19.0", - expectErr: true, - before: disallowedUpgrade118Prev, - kcp: disallowedUpgrade119Version, - }, - { - name: "should return error when trying to upgrade two minor versions", - expectErr: true, - before: disallowedUpgrade118Prev, - kcp: disallowedUpgrade120AlphaVersion, - }, { name: "should not return an error when maxSurge value is updated to 0", expectErr: false, @@ -1051,6 +1006,162 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) { } } +func TestValidateVersion(t *testing.T) { + tests := []struct { + name string + clusterConfiguration *bootstrapv1.ClusterConfiguration + oldVersion string + newVersion string + expectErr bool + }{ + // Basic validation of old and new version. + { + name: "error when old version is empty", + oldVersion: "", + newVersion: "v1.16.6", + expectErr: true, + }, + { + name: "error when old version is invalid", + oldVersion: "invalid-version", + newVersion: "v1.18.1", + expectErr: true, + }, + { + name: "error when new version is empty", + oldVersion: "v1.16.6", + newVersion: "", + expectErr: true, + }, + { + name: "error when new version is invalid", + oldVersion: "v1.18.1", + newVersion: "invalid-version", + expectErr: true, + }, + // Validation that we block upgrade to v1.19.0. + // Note: Upgrading to v1.19.0 is not supported, because of issues in v1.19.0, + // see: https://github.com/kubernetes-sigs/cluster-api/issues/3564 + { + name: "error when upgrading to v1.19.0", + oldVersion: "v1.18.8", + newVersion: "v1.19.0", + expectErr: true, + }, + { + name: "pass when both versions are v1.19.0", + oldVersion: "v1.19.0", + newVersion: "v1.19.0", + expectErr: false, + }, + // Validation for skip-level upgrades. + { + name: "error when upgrading two minor versions", + oldVersion: "v1.18.8", + newVersion: "v1.20.0-alpha.0.734_ba502ee555924a", + expectErr: true, + }, + { + name: "pass when upgrading one minor version", + oldVersion: "v1.20.1", + newVersion: "v1.21.18", + expectErr: false, + }, + // Validation for usage of the old registry. + // Notes: + // * kubeadm versions < v1.22 are always using the old registry. + // * kubeadm versions >= v1.25.0 are always using the new registry. + // * kubeadm versions in between are using the new registry + // starting with certain patch versions. + // This test validates that we don't block upgrades for < v1.22.0 and >= v1.25.0 + // and block upgrades to kubeadm versions in between with the old registry. + { + name: "pass when imageRepository is set", + clusterConfiguration: &bootstrapv1.ClusterConfiguration{ + ImageRepository: "k8s.gcr.io", + }, + oldVersion: "v1.21.1", + newVersion: "v1.22.16", + expectErr: false, + }, + { + name: "pass when version didn't change", + oldVersion: "v1.22.16", + newVersion: "v1.22.16", + expectErr: false, + }, + { + name: "pass when new version is < v1.22.0", + oldVersion: "v1.20.10", + newVersion: "v1.21.5", + expectErr: false, + }, + { + name: "error when new version is using old registry (v1.22.0 <= version <= v1.22.16)", + oldVersion: "v1.21.1", + newVersion: "v1.22.16", // last patch release using old registry + expectErr: true, + }, + { + name: "pass when new version is using new registry (>= v1.22.17)", + oldVersion: "v1.21.1", + newVersion: "v1.22.17", // first patch release using new registry + expectErr: false, + }, + { + name: "error when new version is using old registry (v1.23.0 <= version <= v1.23.14)", + oldVersion: "v1.22.17", + newVersion: "v1.23.14", // last patch release using old registry + expectErr: true, + }, + { + name: "pass when new version is using new registry (>= v1.23.15)", + oldVersion: "v1.22.17", + newVersion: "v1.23.15", // first patch release using new registry + expectErr: false, + }, + { + name: "error when new version is using old registry (v1.24.0 <= version <= v1.24.8)", + oldVersion: "v1.23.1", + newVersion: "v1.24.8", // last patch release using old registry + expectErr: true, + }, + { + name: "pass when new version is using new registry (>= v1.24.9)", + oldVersion: "v1.23.1", + newVersion: "v1.24.9", // first patch release using new registry + expectErr: false, + }, + { + name: "pass when new version is using new registry (>= v1.25.0)", + oldVersion: "v1.24.8", + newVersion: "v1.25.0", // uses new registry + expectErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + kcp := KubeadmControlPlane{ + Spec: KubeadmControlPlaneSpec{ + KubeadmConfigSpec: bootstrapv1.KubeadmConfigSpec{ + ClusterConfiguration: tt.clusterConfiguration, + }, + Version: tt.newVersion, + }, + } + + allErrs := kcp.validateVersion(tt.oldVersion) + if tt.expectErr { + g.Expect(allErrs).ToNot(HaveLen(0)) + } else { + g.Expect(allErrs).To(HaveLen(0)) + } + }) + } +} func TestKubeadmControlPlaneValidateUpdateAfterDefaulting(t *testing.T) { before := &KubeadmControlPlane{ ObjectMeta: metav1.ObjectMeta{ diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml index 533231db6ce0..cdb28bee52a4 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml @@ -2701,13 +2701,20 @@ spec: description: FeatureGates enabled by the user. type: object imageRepository: - description: ImageRepository sets the container registry to - pull images from. If empty, `registry.k8s.io` will be used - by default; in case of kubernetes version is a CI build - (kubernetes version starts with `ci/` or `ci-cross/`) `gcr.io/k8s-staging-ci-images` + description: 'ImageRepository sets the container registry + to pull images from. * If not set, the default registry + of kubeadm will be used, i.e. * registry.k8s.io (new registry): + >= v1.22.17, >= v1.23.15, >= v1.24.9, >= v1.25.0 * k8s.gcr.io + (old registry): all older versions Please note that when + imageRepository is not set we don''t allow upgrades to versions + >= v1.22.0 which use the old registry (k8s.gcr.io). Please + use a newer patch version with the new registry instead + (i.e. >= v1.22.17, >= v1.23.15, >= v1.24.9, >= v1.25.0). + * If the version is a CI build (kubernetes version starts + with `ci/` or `ci-cross/`) `gcr.io/k8s-staging-ci-images` will be used as a default for control plane components and for kube-proxy, while `registry.k8s.io` will be used for - all the other images. + all the other images.' type: string kind: description: 'Kind is a string value representing the REST @@ -3639,7 +3646,13 @@ spec: type: string type: object version: - description: Version defines the desired Kubernetes version. + description: 'Version defines the desired Kubernetes version. Please + note that if kubeadmConfigSpec.ClusterConfiguration.imageRepository + is not set we don''t allow upgrades to versions >= v1.22.0 for which + kubeadm uses the old registry (k8s.gcr.io). Please use a newer patch + version with the new registry instead. The default registries of + kubeadm are: * registry.k8s.io (new registry): >= v1.22.17, >= v1.23.15, + >= v1.24.9, >= v1.25.0 * k8s.gcr.io (old registry): all older versions' type: string required: - kubeadmConfigSpec diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml index 52663f04e22a..ef9869f54f35 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml @@ -1466,14 +1466,21 @@ spec: description: FeatureGates enabled by the user. type: object imageRepository: - description: ImageRepository sets the container registry - to pull images from. If empty, `registry.k8s.io` - will be used by default; in case of kubernetes version - is a CI build (kubernetes version starts with `ci/` - or `ci-cross/`) `gcr.io/k8s-staging-ci-images` will - be used as a default for control plane components + description: 'ImageRepository sets the container registry + to pull images from. * If not set, the default registry + of kubeadm will be used, i.e. * registry.k8s.io + (new registry): >= v1.22.17, >= v1.23.15, >= v1.24.9, + >= v1.25.0 * k8s.gcr.io (old registry): all older + versions Please note that when imageRepository is + not set we don''t allow upgrades to versions >= + v1.22.0 which use the old registry (k8s.gcr.io). + Please use a newer patch version with the new registry + instead (i.e. >= v1.22.17, >= v1.23.15, >= v1.24.9, + >= v1.25.0). * If the version is a CI build (kubernetes + version starts with `ci/` or `ci-cross/`) `gcr.io/k8s-staging-ci-images` + will be used as a default for control plane components and for kube-proxy, while `registry.k8s.io` will - be used for all the other images. + be used for all the other images.' type: string kind: description: 'Kind is a string value representing diff --git a/controlplane/kubeadm/internal/workload_cluster.go b/controlplane/kubeadm/internal/workload_cluster.go index 64e9477bb452..99f3fc1b3524 100644 --- a/controlplane/kubeadm/internal/workload_cluster.go +++ b/controlplane/kubeadm/internal/workload_cluster.go @@ -47,6 +47,7 @@ import ( kubeadmtypes "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" "sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/proxy" + "sigs.k8s.io/cluster-api/internal/util/kubeadm" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/certs" containerutil "sigs.k8s.io/cluster-api/util/container" @@ -85,14 +86,6 @@ var ( // NOTE: The following assumes that kubeadm version equals to Kubernetes version. minVerUnversionedKubeletConfig = semver.MustParse("1.24.0") - // minKubernetesVersionImageRegistryMigration is first kubernetes version where - // the default image registry is registry.k8s.io instead of k8s.gcr.io. - minKubernetesVersionImageRegistryMigration = semver.MustParse("1.22.0") - - // nextKubernetesVersionImageRegistryMigration is the next minor version after - // the default image registry changed to registry.k8s.io. - nextKubernetesVersionImageRegistryMigration = semver.MustParse("1.26.0") - // ErrControlPlaneMinNodes signals that a cluster doesn't meet the minimum required nodes // to remove an etcd member. ErrControlPlaneMinNodes = errors.New("cluster has fewer than 2 control plane nodes; removing an etcd member is not supported") @@ -623,12 +616,15 @@ func yamlToUnstructured(rawYAML []byte) (*unstructured.Unstructured, error) { } // ImageRepositoryFromClusterConfig returns the image repository to use. It returns: -// * clusterConfig.ImageRepository if set. -// * "registry.k8s.io" if v1.22 <= version < v1.26 to migrate to the new registry -// * "" otherwise. -// Beginning with kubernetes v1.22, the default registry for kubernetes is registry.k8s.io -// instead of k8s.gcr.io which is why references should get migrated when upgrading to v1.22. -// The migration follows the behavior of `kubeadm upgrade`. +// - clusterConfig.ImageRepository if set. +// - else either k8s.gcr.io or registry.k8s.io depending on the default registry of the kubeadm +// binary of the given kubernetes version. This is only done for Kubernetes versions >= v1.22.0 +// and < v1.26.0 because in this version range the default registry was changed. +// +// Note: Please see the following issue for more context: https://github.com/kubernetes-sigs/cluster-api/issues/7833 +// tl;dr is that the imageRepository must be in sync with the default registry of kubeadm. +// Otherwise kubeadm preflight checks will fail because kubeadm is trying to pull the CoreDNS image +// from the wrong repository (/coredns instead of /coredns/coredns). func ImageRepositoryFromClusterConfig(clusterConfig *bootstrapv1.ClusterConfiguration, kubernetesVersion semver.Version) string { // If ImageRepository is explicitly specified, return early. if clusterConfig != nil && @@ -636,11 +632,11 @@ func ImageRepositoryFromClusterConfig(clusterConfig *bootstrapv1.ClusterConfigur return clusterConfig.ImageRepository } - // If v1.22 <= version < v1.26 return the default Kubernetes image repository to - // migrate to the new location and not cause changes else. - if kubernetesVersion.GTE(minKubernetesVersionImageRegistryMigration) && - kubernetesVersion.LT(nextKubernetesVersionImageRegistryMigration) { - return kubernetesImageRepository + // If v1.22.0 <= version < v1.26.0 return the default registry of the + // corresponding kubeadm binary. + if kubernetesVersion.GTE(kubeadm.MinKubernetesVersionImageRegistryMigration) && + kubernetesVersion.LT(kubeadm.NextKubernetesVersionImageRegistryMigration) { + return kubeadm.GetDefaultRegistry(kubernetesVersion) } // Use defaulting or current values otherwise. diff --git a/controlplane/kubeadm/internal/workload_cluster_coredns.go b/controlplane/kubeadm/internal/workload_cluster_coredns.go index a105a61a89a4..9ebf7eda9950 100644 --- a/controlplane/kubeadm/internal/workload_cluster_coredns.go +++ b/controlplane/kubeadm/internal/workload_cluster_coredns.go @@ -35,6 +35,7 @@ import ( bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + "sigs.k8s.io/cluster-api/internal/util/kubeadm" containerutil "sigs.k8s.io/cluster-api/util/container" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/cluster-api/util/version" @@ -47,11 +48,8 @@ const ( coreDNSVolumeKey = "config-volume" coreDNSClusterRoleName = "system:coredns" - // kubernetesImageRepository is the default Kubernetes image repository for build artifacts. - kubernetesImageRepository = "registry.k8s.io" - oldKubernetesImageRepository = "k8s.gcr.io" - oldCoreDNSImageName = "coredns" - coreDNSImageName = "coredns/coredns" + oldCoreDNSImageName = "coredns" + coreDNSImageName = "coredns/coredns" oldControlPlaneTaint = "node-role.kubernetes.io/master" // Deprecated: https://github.com/kubernetes/kubeadm/issues/2200 controlPlaneTaint = "node-role.kubernetes.io/control-plane" @@ -200,11 +198,11 @@ func (w *Workload) getCoreDNSInfo(ctx context.Context, clusterConfig *bootstrapv toImageRepository := parsedImage.Repository // Overwrite the image repository if a value was explicitly set or an upgrade is required. if imageRegistryRepository := ImageRepositoryFromClusterConfig(clusterConfig, version); imageRegistryRepository != "" { - if imageRegistryRepository == kubernetesImageRepository { - // Only patch to KubernetesImageRepository if oldKubernetesImageRepository is set as prefix. - if strings.HasPrefix(toImageRepository, oldKubernetesImageRepository) { - // Ensure to keep the repository subpaths when patching from oldKubernetesImageRepository to new KubernetesImageRepository. - toImageRepository = strings.TrimSuffix(imageRegistryRepository+strings.TrimPrefix(toImageRepository, oldKubernetesImageRepository), "/") + if imageRegistryRepository == kubeadm.DefaultImageRepository { + // Only patch to DefaultImageRepository if OldDefaultImageRepository is set as prefix. + if strings.HasPrefix(toImageRepository, kubeadm.OldDefaultImageRepository) { + // Ensure to keep the repository subpaths when patching from OldDefaultImageRepository to new DefaultImageRepository. + toImageRepository = strings.TrimSuffix(imageRegistryRepository+strings.TrimPrefix(toImageRepository, kubeadm.OldDefaultImageRepository), "/") } } else { toImageRepository = strings.TrimSuffix(imageRegistryRepository, "/") @@ -235,7 +233,7 @@ func (w *Workload) getCoreDNSInfo(ctx context.Context, clusterConfig *bootstrapv // * "registry.k8s.io/coredns" to "registry.k8s.io/coredns/coredns" or // * "k8s.gcr.io/coredns" to "k8s.gcr.io/coredns/coredns" toImageName := parsedImage.Name - if (toImageRepository == oldKubernetesImageRepository || toImageRepository == kubernetesImageRepository) && + if (toImageRepository == kubeadm.OldDefaultImageRepository || toImageRepository == kubeadm.DefaultImageRepository) && toImageName == oldCoreDNSImageName && targetMajorMinorPatch.GTE(semver.MustParse("1.8.0")) { toImageName = coreDNSImageName } diff --git a/controlplane/kubeadm/internal/workload_cluster_coredns_test.go b/controlplane/kubeadm/internal/workload_cluster_coredns_test.go index 2436bc024b5e..bac6124e733b 100644 --- a/controlplane/kubeadm/internal/workload_cluster_coredns_test.go +++ b/controlplane/kubeadm/internal/workload_cluster_coredns_test.go @@ -1213,8 +1213,10 @@ func TestGetCoreDNSInfo(t *testing.T) { }, }, { - name: "rename to coredns/coredns when upgrading to coredns=1.8.0 and kubernetesVersion=1.21.0", - objs: []client.Object{newCoreDNSInfoDeploymentWithimage(image162), cm}, + name: "rename to coredns/coredns when upgrading to coredns=1.8.0 and kubernetesVersion=1.22.16", + // 1.22.16 uses k8s.gcr.io as default registry. Thus the registry doesn't get changed as + // FromImage is already using k8s.gcr.io. + objs: []client.Object{newCoreDNSInfoDeploymentWithimage("k8s.gcr.io/coredns:1.6.2"), cm}, clusterConfig: &bootstrapv1.ClusterConfiguration{ DNS: bootstrapv1.DNS{ ImageMeta: bootstrapv1.ImageMeta{ @@ -1222,18 +1224,42 @@ func TestGetCoreDNSInfo(t *testing.T) { }, }, }, - kubernetesVersion: semver.MustParse("1.21.0"), + kubernetesVersion: semver.MustParse("1.22.16"), expectedInfo: coreDNSInfo{ CurrentMajorMinorPatch: "1.6.2", FromImageTag: "1.6.2", TargetMajorMinorPatch: "1.8.0", - FromImage: image162, + FromImage: "k8s.gcr.io/coredns:1.6.2", ToImage: "k8s.gcr.io/coredns/coredns:1.8.0", ToImageTag: "1.8.0", }, }, + { + name: "rename to coredns/coredns when upgrading to coredns=1.8.0 and kubernetesVersion=1.22.17", + // 1.22.17 has registry.k8s.io as default registry. Thus the registry gets changed as + // FromImage is using k8s.gcr.io. + objs: []client.Object{newCoreDNSInfoDeploymentWithimage("k8s.gcr.io/coredns:1.6.2"), cm}, + clusterConfig: &bootstrapv1.ClusterConfiguration{ + DNS: bootstrapv1.DNS{ + ImageMeta: bootstrapv1.ImageMeta{ + ImageTag: "1.8.0", + }, + }, + }, + kubernetesVersion: semver.MustParse("1.22.17"), + expectedInfo: coreDNSInfo{ + CurrentMajorMinorPatch: "1.6.2", + FromImageTag: "1.6.2", + TargetMajorMinorPatch: "1.8.0", + FromImage: "k8s.gcr.io/coredns:1.6.2", + ToImage: "registry.k8s.io/coredns/coredns:1.8.0", + ToImageTag: "1.8.0", + }, + }, { name: "rename to coredns/coredns when upgrading to coredns=1.8.0 and kubernetesVersion=1.26.0", + // 1.26.0 uses registry.k8s.io as default registry. Thus the registry doesn't get changed as + // FromImage is already using registry.k8s.io. objs: []client.Object{newCoreDNSInfoDeploymentWithimage("registry.k8s.io/coredns:1.6.2"), cm}, clusterConfig: &bootstrapv1.ClusterConfiguration{ DNS: bootstrapv1.DNS{ @@ -1242,7 +1268,7 @@ func TestGetCoreDNSInfo(t *testing.T) { }, }, }, - kubernetesVersion: semver.MustParse("1.24.0"), + kubernetesVersion: semver.MustParse("1.26.0"), expectedInfo: coreDNSInfo{ CurrentMajorMinorPatch: "1.6.2", FromImageTag: "1.6.2", @@ -1253,7 +1279,9 @@ func TestGetCoreDNSInfo(t *testing.T) { }, }, { - name: "patches ImageRepository to registry.k8s.io if it's set on neither global nor DNS-level and kubernetesVersion >= v1.22 and rename to coredns/coredns", + name: "patches ImageRepository to registry.k8s.io if it's set on neither global nor DNS-level and kubernetesVersion >= v1.22.17 and rename to coredns/coredns", + // 1.22.17 has registry.k8s.io as default registry. Thus the registry gets changed as + // FromImage is using k8s.gcr.io. objs: []client.Object{newCoreDNSInfoDeploymentWithimage(image162), cm}, clusterConfig: &bootstrapv1.ClusterConfiguration{ DNS: bootstrapv1.DNS{ @@ -1262,7 +1290,7 @@ func TestGetCoreDNSInfo(t *testing.T) { }, }, }, - kubernetesVersion: semver.MustParse("1.22.0"), + kubernetesVersion: semver.MustParse("1.22.17"), expectedInfo: coreDNSInfo{ CurrentMajorMinorPatch: "1.6.2", FromImageTag: "1.6.2", diff --git a/internal/util/kubeadm/kubeadm.go b/internal/util/kubeadm/kubeadm.go new file mode 100644 index 000000000000..bd69027114fa --- /dev/null +++ b/internal/util/kubeadm/kubeadm.go @@ -0,0 +1,72 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kubeadm contains utils related to kubeadm. +package kubeadm + +import "github.com/blang/semver" + +const ( + // DefaultImageRepository is the new default Kubernetes image registry. + DefaultImageRepository = "registry.k8s.io" + // OldDefaultImageRepository is the old default Kubernetes image registry. + OldDefaultImageRepository = "k8s.gcr.io" +) + +var ( + // MinKubernetesVersionImageRegistryMigration is the first Kubernetes minor version which + // has patch versions where the default image registry in kubeadm is registry.k8s.io instead of k8s.gcr.io. + MinKubernetesVersionImageRegistryMigration = semver.MustParse("1.22.0") + + // NextKubernetesVersionImageRegistryMigration is the next minor version after + // the default image registry in kubeadm changed to registry.k8s.io. + NextKubernetesVersionImageRegistryMigration = semver.MustParse("1.26.0") +) + +// GetDefaultRegistry returns the default registry of the given kubeadm version. +func GetDefaultRegistry(version semver.Version) string { + // If version <= v1.22.16 return k8s.gcr.io + if version.LTE(semver.MustParse("1.22.16")) { + return OldDefaultImageRepository + } + + // If v1.22.17 <= version < v1.23.0 return registry.k8s.io + if version.GTE(semver.MustParse("1.22.17")) && + version.LT(semver.MustParse("1.23.0")) { + return DefaultImageRepository + } + + // If v1.23.0 <= version <= v1.23.14 return k8s.gcr.io + if version.GTE(semver.MustParse("1.23.0")) && + version.LTE(semver.MustParse("1.23.14")) { + return OldDefaultImageRepository + } + + // If v1.23.15 <= version < v1.24.0 return registry.k8s.io + if version.GTE(semver.MustParse("1.23.15")) && + version.LT(semver.MustParse("1.24.0")) { + return DefaultImageRepository + } + + // If v1.24.0 <= version <= v1.24.8 return k8s.gcr.io + if version.GTE(semver.MustParse("1.24.0")) && + version.LTE(semver.MustParse("1.24.8")) { + return OldDefaultImageRepository + } + + // If v1.24.9 <= version return registry.k8s.io + return DefaultImageRepository +} diff --git a/internal/util/kubeadm/kubeadm_test.go b/internal/util/kubeadm/kubeadm_test.go new file mode 100644 index 000000000000..4e1f7c81f12a --- /dev/null +++ b/internal/util/kubeadm/kubeadm_test.go @@ -0,0 +1,68 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeadm + +import ( + "fmt" + "testing" + + "github.com/blang/semver" + . "github.com/onsi/gomega" +) + +func TestGetDefaultRegistry(t *testing.T) { + tests := []struct { + version string + expectedRegistry string + }{ + // < v1.22 + {version: "1.19.1", expectedRegistry: OldDefaultImageRepository}, + {version: "1.20.5", expectedRegistry: OldDefaultImageRepository}, + {version: "1.21.85", expectedRegistry: OldDefaultImageRepository}, + + // v1.22 + {version: "1.22.0", expectedRegistry: OldDefaultImageRepository}, + {version: "1.22.16", expectedRegistry: OldDefaultImageRepository}, + {version: "1.22.17", expectedRegistry: DefaultImageRepository}, + {version: "1.22.99", expectedRegistry: DefaultImageRepository}, + + // v1.23 + {version: "1.23.0", expectedRegistry: OldDefaultImageRepository}, + {version: "1.23.14", expectedRegistry: OldDefaultImageRepository}, + {version: "1.23.15", expectedRegistry: DefaultImageRepository}, + {version: "1.23.99", expectedRegistry: DefaultImageRepository}, + + // v1.24 + {version: "1.24.0", expectedRegistry: OldDefaultImageRepository}, + {version: "1.24.8", expectedRegistry: OldDefaultImageRepository}, + {version: "1.24.9", expectedRegistry: DefaultImageRepository}, + {version: "1.24.99", expectedRegistry: DefaultImageRepository}, + + // > v1.24 + {version: "1.25.0", expectedRegistry: DefaultImageRepository}, + {version: "1.26.1", expectedRegistry: DefaultImageRepository}, + {version: "1.27.2", expectedRegistry: DefaultImageRepository}, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("%s => %s", tt.version, tt.expectedRegistry), func(t *testing.T) { + g := NewWithT(t) + + g.Expect(GetDefaultRegistry(semver.MustParse(tt.version))).To(Equal(tt.expectedRegistry)) + }) + } +} diff --git a/test/framework/daemonset_helpers.go b/test/framework/daemonset_helpers.go index 2a4dca7e0020..40510638e93d 100644 --- a/test/framework/daemonset_helpers.go +++ b/test/framework/daemonset_helpers.go @@ -26,6 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/cluster-api/internal/util/kubeadm" containerutil "sigs.k8s.io/cluster-api/util/container" ) @@ -42,15 +43,9 @@ func WaitForKubeProxyUpgrade(ctx context.Context, input WaitForKubeProxyUpgradeI parsedVersion, err := semver.ParseTolerant(input.KubernetesVersion) Expect(err).ToNot(HaveOccurred()) - // Beginning with kubernetes v1.22, kubernetes images including kube-proxy get published to registry.k8s.io. - // This ensures that the imageRepository setting gets patched to registry.k8s.io when upgrading from v1.21 or lower, - // but only if there was no imageRepository explicitly set at the KubeadmControlPlanes ClusterConfiguration. - // This follows the behavior of `kubeadm upgrade`. - wantKubeProxyRegistry := "registry.k8s.io" - if parsedVersion.LT(semver.Version{Major: 1, Minor: 22, Patch: 0, Pre: []semver.PRVersion{{VersionStr: "alpha"}}}) { - wantKubeProxyRegistry = "k8s.gcr.io" - } - wantKubeProxyImage := wantKubeProxyRegistry + "/kube-proxy:" + containerutil.SemverToOCIImageTag(input.KubernetesVersion) + // Validate the kube-proxy image according to how KCP sets the image in the kube-proxy DaemonSet. + // KCP does this based on the default registry of the Kubernetes/kubeadm version. + wantKubeProxyImage := kubeadm.GetDefaultRegistry(parsedVersion) + "/kube-proxy:" + containerutil.SemverToOCIImageTag(input.KubernetesVersion) Eventually(func() (bool, error) { ds := &appsv1.DaemonSet{} diff --git a/test/infrastructure/docker/internal/docker/machine.go b/test/infrastructure/docker/internal/docker/machine.go index 132abbb867f7..054483264a9d 100644 --- a/test/infrastructure/docker/internal/docker/machine.go +++ b/test/infrastructure/docker/internal/docker/machine.go @@ -254,7 +254,7 @@ func (m *Machine) Create(ctx context.Context, image string, role string, version if err != nil { log.Info("Failed running command", "command", "crictl ps") logContainerDebugInfo(ctx, log, m.ContainerName()) - return errors.Wrap(errors.WithStack(err), "failed to run crictl ps") + return errors.Wrap(err, "failed to run crictl ps") } return nil } @@ -381,7 +381,7 @@ func (m *Machine) CheckForBootstrapSuccess(ctx context.Context, logResult bool) if logResult { log.Info("Failed running command", "command", "test -f /run/cluster-api/bootstrap-success.complete", "stdout", outStd.String(), "stderr", outErr.String()) } - return errors.Wrap(errors.WithStack(err), "failed to run bootstrap check") + return errors.Wrap(err, "failed to run bootstrap check") } return nil }