Skip to content

Commit

Permalink
Merge pull request #7870 from sbueringer/pr-kcp-registry-fix-1.2
Browse files Browse the repository at this point in the history 
[release-1.2] ⚠️ KCP: block upgrade to versions with old registry, improve registry handling
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Jan 10, 2023
2 parents 9bf2773 + 88d20ce commit 3d2a0a4
Show file tree
Hide file tree
Showing 15 changed files with 467 additions and 125 deletions.
13 changes: 10 additions & 3 deletions bootstrap/kubeadm/api/v1beta1/kubeadm_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,16 @@ type ClusterConfiguration struct {
CertificatesDir string `json:"certificatesDir,omitempty"`

// ImageRepository sets the container registry to pull images from.
// If empty, `registry.k8s.io` will be used by default; in case of kubernetes version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`)
// `gcr.io/k8s-staging-ci-images` will be used as a default for control plane components and for kube-proxy, while `registry.k8s.io`
// will be used for all the other images.
// * If not set, the default registry of kubeadm will be used, i.e.
// * registry.k8s.io (new registry): >= v1.22.17, >= v1.23.15, >= v1.24.9, >= v1.25.0
// * k8s.gcr.io (old registry): all older versions
// Please note that when imageRepository is not set we don't allow upgrades to
// versions >= v1.22.0 which use the old registry (k8s.gcr.io). Please use
// a newer patch version with the new registry instead (i.e. >= v1.22.17,
// >= v1.23.15, >= v1.24.9, >= v1.25.0).
// * If the version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`)
// `gcr.io/k8s-staging-ci-images` will be used as a default for control plane components
// and for kube-proxy, while `registry.k8s.io` will be used for all the other images.
// +optional
ImageRepository string `json:"imageRepository,omitempty"`

Expand Down
20 changes: 13 additions & 7 deletions bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2243,13 +2243,19 @@ spec:
description: FeatureGates enabled by the user.
type: object
imageRepository:
description: ImageRepository sets the container registry to pull
images from. If empty, `registry.k8s.io` will be used by default;
in case of kubernetes version is a CI build (kubernetes version
starts with `ci/` or `ci-cross/`) `gcr.io/k8s-staging-ci-images`
will be used as a default for control plane components and for
kube-proxy, while `registry.k8s.io` will be used for all the
other images.
description: 'ImageRepository sets the container registry to pull
images from. * If not set, the default registry of kubeadm will
be used, i.e. * registry.k8s.io (new registry): >= v1.22.17,
>= v1.23.15, >= v1.24.9, >= v1.25.0 * k8s.gcr.io (old registry):
all older versions Please note that when imageRepository is
not set we don''t allow upgrades to versions >= v1.22.0 which
use the old registry (k8s.gcr.io). Please use a newer patch
version with the new registry instead (i.e. >= v1.22.17, >=
v1.23.15, >= v1.24.9, >= v1.25.0). * If the version is a CI
build (kubernetes version starts with `ci/` or `ci-cross/`)
`gcr.io/k8s-staging-ci-images` will be used as a default for
control plane components and for kube-proxy, while `registry.k8s.io`
will be used for all the other images.'
type: string
kind:
description: 'Kind is a string value representing the REST resource
Expand Down
23 changes: 15 additions & 8 deletions bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2246,14 +2246,21 @@ spec:
description: FeatureGates enabled by the user.
type: object
imageRepository:
description: ImageRepository sets the container registry
to pull images from. If empty, `registry.k8s.io` will
be used by default; in case of kubernetes version is
a CI build (kubernetes version starts with `ci/` or
`ci-cross/`) `gcr.io/k8s-staging-ci-images` will be
used as a default for control plane components and for
kube-proxy, while `registry.k8s.io` will be used for
all the other images.
description: 'ImageRepository sets the container registry
to pull images from. * If not set, the default registry
of kubeadm will be used, i.e. * registry.k8s.io (new
registry): >= v1.22.17, >= v1.23.15, >= v1.24.9, >=
v1.25.0 * k8s.gcr.io (old registry): all older versions
Please note that when imageRepository is not set we
don''t allow upgrades to versions >= v1.22.0 which use
the old registry (k8s.gcr.io). Please use a newer patch
version with the new registry instead (i.e. >= v1.22.17,
>= v1.23.15, >= v1.24.9, >= v1.25.0). * If the version
is a CI build (kubernetes version starts with `ci/`
or `ci-cross/`) `gcr.io/k8s-staging-ci-images` will
be used as a default for control plane components and
for kube-proxy, while `registry.k8s.io` will be used
for all the other images.'
type: string
kind:
description: 'Kind is a string value representing the
Expand Down
5 changes: 5 additions & 0 deletions controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,11 @@ type KubeadmControlPlaneSpec struct {
Replicas *int32 `json:"replicas,omitempty"`

// Version defines the desired Kubernetes version.
// Please note that if kubeadmConfigSpec.ClusterConfiguration.imageRepository is not set
// we don't allow upgrades to versions >= v1.22.0 for which kubeadm uses the old registry (k8s.gcr.io).
// Please use a newer patch version with the new registry instead. The default registries of kubeadm are:
// * registry.k8s.io (new registry): >= v1.22.17, >= v1.23.15, >= v1.24.9, >= v1.25.0
// * k8s.gcr.io (old registry): all older versions
Version string `json:"version"`

// MachineTemplate contains information about how machines
Expand Down
31 changes: 30 additions & 1 deletion controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/webhook"

bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
"sigs.k8s.io/cluster-api/internal/util/kubeadm"
"sigs.k8s.io/cluster-api/util/container"
"sigs.k8s.io/cluster-api/util/version"
)
Expand Down Expand Up @@ -575,7 +576,10 @@ func (in *KubeadmControlPlane) validateVersion(previousVersion string) (allErrs
return allErrs
}

// Since upgrades to the next minor version are allowed, irrespective of the patch version.
// Validate that the update is upgrading at most one minor version.
// Note: Skipping a minor version is not allowed.
// Note: Checking against this ceilVersion allows upgrading to the next minor
// version irrespective of the patch version.
ceilVersion := semver.Version{
Major: fromVersion.Major,
Minor: fromVersion.Minor + 2,
Expand All @@ -590,6 +594,31 @@ func (in *KubeadmControlPlane) validateVersion(previousVersion string) (allErrs
)
}

// The Kubernetes ecosystem has been requested to move users to the new registry due to cost issues.
// This validation enforces the move to the new registry by forcing users to upgrade to kubeadm versions
// with the new registry.
// NOTE: This only affects users relying on the community maintained registry.
// NOTE: Pinning to the upstream registry is not recommended because it could lead to issues
// given how the migration has been implemented in kubeadm.
//
// Block if imageRepository is not set (i.e. the default registry should be used),
if (in.Spec.KubeadmConfigSpec.ClusterConfiguration == nil ||
in.Spec.KubeadmConfigSpec.ClusterConfiguration.ImageRepository == "") &&
// the version changed (i.e. we have an upgrade),
toVersion.NE(fromVersion) &&
// the version is >= v1.22.0 and < v1.26.0
toVersion.GTE(kubeadm.MinKubernetesVersionImageRegistryMigration) &&
toVersion.LT(kubeadm.NextKubernetesVersionImageRegistryMigration) &&
// and the default registry of the new Kubernetes/kubeadm version is the old default registry.
kubeadm.GetDefaultRegistry(toVersion) == kubeadm.OldDefaultImageRepository {
allErrs = append(allErrs,
field.Forbidden(
field.NewPath("spec", "version"),
"cannot upgrade to a Kubernetes/kubeadm version which is using the old default registry. Please use a newer Kubernetes patch release which is using the new default registry (>= v1.22.17, >= v1.23.15, >= v1.24.9)",
),
)
}

return allErrs
}

Expand Down
201 changes: 156 additions & 45 deletions controlplane/kubeadm/api/v1beta1/kubeadm_control_plane_webhook_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -426,14 +426,6 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) {
kubernetesVersion := before.DeepCopy()
kubernetesVersion.Spec.KubeadmConfigSpec.ClusterConfiguration.KubernetesVersion = "some kubernetes version"

prevKCPWithVersion := func(version string) *KubeadmControlPlane {
prev := before.DeepCopy()
prev.Spec.Version = version
return prev
}
skipMinorControlPlaneVersion := prevKCPWithVersion("v1.18.1")
emptyControlPlaneVersion := prevKCPWithVersion("")

controlPlaneEndpoint := before.DeepCopy()
controlPlaneEndpoint.Spec.KubeadmConfigSpec.ClusterConfiguration.ControlPlaneEndpoint = "some control plane endpoint"

Expand Down Expand Up @@ -586,13 +578,6 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) {
DataDir: "/data",
}

disallowedUpgrade118Prev := prevKCPWithVersion("v1.18.8")
disallowedUpgrade119Version := before.DeepCopy()
disallowedUpgrade119Version.Spec.Version = "v1.19.0"

disallowedUpgrade120AlphaVersion := before.DeepCopy()
disallowedUpgrade120AlphaVersion.Spec.Version = "v1.20.0-alpha.0.734_ba502ee555924a"

updateNTPServers := before.DeepCopy()
updateNTPServers.Spec.KubeadmConfigSpec.NTP.Servers = []string{"new-server"}

Expand Down Expand Up @@ -879,36 +864,6 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) {
before: withoutClusterConfiguration,
kcp: afterEtcdLocalDirAddition,
},
{
name: "should fail when skipping control plane minor versions",
expectErr: true,
before: before,
kcp: skipMinorControlPlaneVersion,
},
{
name: "should fail when no control plane version is passed",
expectErr: true,
before: before,
kcp: emptyControlPlaneVersion,
},
{
name: "should pass if control plane version is the same",
expectErr: false,
before: before,
kcp: before.DeepCopy(),
},
{
name: "should return error when trying to upgrade to v1.19.0",
expectErr: true,
before: disallowedUpgrade118Prev,
kcp: disallowedUpgrade119Version,
},
{
name: "should return error when trying to upgrade two minor versions",
expectErr: true,
before: disallowedUpgrade118Prev,
kcp: disallowedUpgrade120AlphaVersion,
},
{
name: "should not return an error when maxSurge value is updated to 0",
expectErr: false,
Expand Down Expand Up @@ -981,6 +936,162 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) {
}
}

func TestValidateVersion(t *testing.T) {
tests := []struct {
name string
clusterConfiguration *bootstrapv1.ClusterConfiguration
oldVersion string
newVersion string
expectErr bool
}{
// Basic validation of old and new version.
{
name: "error when old version is empty",
oldVersion: "",
newVersion: "v1.16.6",
expectErr: true,
},
{
name: "error when old version is invalid",
oldVersion: "invalid-version",
newVersion: "v1.18.1",
expectErr: true,
},
{
name: "error when new version is empty",
oldVersion: "v1.16.6",
newVersion: "",
expectErr: true,
},
{
name: "error when new version is invalid",
oldVersion: "v1.18.1",
newVersion: "invalid-version",
expectErr: true,
},
// Validation that we block upgrade to v1.19.0.
// Note: Upgrading to v1.19.0 is not supported, because of issues in v1.19.0,
// see: https://github.com/kubernetes-sigs/cluster-api/issues/3564
{
name: "error when upgrading to v1.19.0",
oldVersion: "v1.18.8",
newVersion: "v1.19.0",
expectErr: true,
},
{
name: "pass when both versions are v1.19.0",
oldVersion: "v1.19.0",
newVersion: "v1.19.0",
expectErr: false,
},
// Validation for skip-level upgrades.
{
name: "error when upgrading two minor versions",
oldVersion: "v1.18.8",
newVersion: "v1.20.0-alpha.0.734_ba502ee555924a",
expectErr: true,
},
{
name: "pass when upgrading one minor version",
oldVersion: "v1.20.1",
newVersion: "v1.21.18",
expectErr: false,
},
// Validation for usage of the old registry.
// Notes:
// * kubeadm versions < v1.22 are always using the old registry.
// * kubeadm versions >= v1.25.0 are always using the new registry.
// * kubeadm versions in between are using the new registry
// starting with certain patch versions.
// This test validates that we don't block upgrades for < v1.22.0 and >= v1.25.0
// and block upgrades to kubeadm versions in between with the old registry.
{
name: "pass when imageRepository is set",
clusterConfiguration: &bootstrapv1.ClusterConfiguration{
ImageRepository: "k8s.gcr.io",
},
oldVersion: "v1.21.1",
newVersion: "v1.22.16",
expectErr: false,
},
{
name: "pass when version didn't change",
oldVersion: "v1.22.16",
newVersion: "v1.22.16",
expectErr: false,
},
{
name: "pass when new version is < v1.22.0",
oldVersion: "v1.20.10",
newVersion: "v1.21.5",
expectErr: false,
},
{
name: "error when new version is using old registry (v1.22.0 <= version <= v1.22.16)",
oldVersion: "v1.21.1",
newVersion: "v1.22.16", // last patch release using old registry
expectErr: true,
},
{
name: "pass when new version is using new registry (>= v1.22.17)",
oldVersion: "v1.21.1",
newVersion: "v1.22.17", // first patch release using new registry
expectErr: false,
},
{
name: "error when new version is using old registry (v1.23.0 <= version <= v1.23.14)",
oldVersion: "v1.22.17",
newVersion: "v1.23.14", // last patch release using old registry
expectErr: true,
},
{
name: "pass when new version is using new registry (>= v1.23.15)",
oldVersion: "v1.22.17",
newVersion: "v1.23.15", // first patch release using new registry
expectErr: false,
},
{
name: "error when new version is using old registry (v1.24.0 <= version <= v1.24.8)",
oldVersion: "v1.23.1",
newVersion: "v1.24.8", // last patch release using old registry
expectErr: true,
},
{
name: "pass when new version is using new registry (>= v1.24.9)",
oldVersion: "v1.23.1",
newVersion: "v1.24.9", // first patch release using new registry
expectErr: false,
},
{
name: "pass when new version is using new registry (>= v1.25.0)",
oldVersion: "v1.24.8",
newVersion: "v1.25.0", // uses new registry
expectErr: false,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)

kcp := KubeadmControlPlane{
Spec: KubeadmControlPlaneSpec{
KubeadmConfigSpec: bootstrapv1.KubeadmConfigSpec{
ClusterConfiguration: tt.clusterConfiguration,
},
Version: tt.newVersion,
},
}

allErrs := kcp.validateVersion(tt.oldVersion)
if tt.expectErr {
g.Expect(allErrs).ToNot(HaveLen(0))
} else {
g.Expect(allErrs).To(HaveLen(0))
}
})
}
}
func TestKubeadmControlPlaneValidateUpdateAfterDefaulting(t *testing.T) {
before := &KubeadmControlPlane{
ObjectMeta: metav1.ObjectMeta{
Expand Down
Loading

0 comments on commit 3d2a0a4

Please sign in to comment.