From bb668796302cd83583daae19d6978f2b279fea14 Mon Sep 17 00:00:00 2001 From: Plamen Kokanov <35485709+plkokanov@users.noreply.github.com> Date: Fri, 27 May 2022 11:34:51 +0300 Subject: [PATCH] Set `SeedSystemComponentsHealthy` and `Bootstrapped` seed conditions to `Progressing` on seed reconciliations. (#5995) * Remove unsued variables * Make SeedSystemComponentsHealthy Failed or Progressing if a resource cannot be found * Add seed health check unit test * Set SeedSystemComponentsHealthy and Bootstrap conditions to progressing on seed reconciliation * Increase seed controller syncPeriod to 15m * Apply review comments * Apply review comments * Revert seed controller syncPeriod to 1m --- charts/gardener/gardenlet/values.yaml | 2 + example/20-componentconfig-gardenlet.yaml | 2 + .../controller/seed/seed_care_control.go | 10 - .../controller/seed/seed_care_control_test.go | 2 - .../seed/seed_care_control_types.go | 2 +- pkg/gardenlet/controller/seed/seed_control.go | 15 +- pkg/operation/care/seed_health.go | 7 +- pkg/operation/care/seed_health_test.go | 318 ++++++++++++++++++ pkg/utils/kubernetes/health/seed.go | 6 +- 9 files changed, 345 insertions(+), 19 deletions(-) create mode 100644 pkg/operation/care/seed_health_test.go diff --git a/charts/gardener/gardenlet/values.yaml b/charts/gardener/gardenlet/values.yaml index 424d2703244..c68dc8af8fb 100644 --- a/charts/gardener/gardenlet/values.yaml +++ b/charts/gardener/gardenlet/values.yaml @@ -79,6 +79,8 @@ global: # - production seed: concurrentSyncs: 5 + # TODO (plkokanov): the sync period is currently set to 1m because of the way that SNI detection is done during seed reconciliations. + # ref: https://github.com/gardener/gardener/issues/6036 syncPeriod: 1m # leaseResyncSeconds: 2 # leaseResyncMissThreshold: 10 diff --git a/example/20-componentconfig-gardenlet.yaml b/example/20-componentconfig-gardenlet.yaml index 4fde738eb29..7e9879921f8 100644 --- a/example/20-componentconfig-gardenlet.yaml +++ b/example/20-componentconfig-gardenlet.yaml @@ -69,6 +69,8 @@ controllers: syncPeriod: 30s seed: concurrentSyncs: 5 + # TODO (plkokanov): the sync period is currently set to 1m because of the way that SNI detection is done during seed reconciliations. + # ref: https://github.com/gardener/gardener/issues/6036 syncPeriod: 1m # leaseResyncSeconds: 2 # leaseResyncMissThreshold: 10 diff --git a/pkg/gardenlet/controller/seed/seed_care_control.go b/pkg/gardenlet/controller/seed/seed_care_control.go index db68c3363cf..8f7b6b90d18 100644 --- a/pkg/gardenlet/controller/seed/seed_care_control.go +++ b/pkg/gardenlet/controller/seed/seed_care_control.go @@ -108,20 +108,10 @@ func (r *careReconciler) care(ctx context.Context, gardenClientSet client.Client return nil // We do not want to run in the exponential backoff for the condition checks. } - seedObj, err := NewSeed(careCtx, seed) - if err != nil { - log.Error(err, "SeedObj cannot be constructed") - if err := careSetupFailure(ctx, gardenClientSet, seed, "seedObj cannot be constructed", conditions); err != nil { - log.Error(err, "Unable to create error condition") - } - return nil - } - // Trigger health check seedHealth := NewHealthCheck(seed, seedClient.Client()) updatedConditions := seedHealth.CheckSeed( careCtx, - seedObj, conditions, r.conditionThresholdsToProgressingMapping(), ) diff --git a/pkg/gardenlet/controller/seed/seed_care_control_test.go b/pkg/gardenlet/controller/seed/seed_care_control_test.go index c3909de2113..1eb851aa8e0 100644 --- a/pkg/gardenlet/controller/seed/seed_care_control_test.go +++ b/pkg/gardenlet/controller/seed/seed_care_control_test.go @@ -26,7 +26,6 @@ import ( fakeclientset "github.com/gardener/gardener/pkg/client/kubernetes/fake" "github.com/gardener/gardener/pkg/gardenlet/apis/config" . "github.com/gardener/gardener/pkg/gardenlet/controller/seed" - seedpkg "github.com/gardener/gardener/pkg/operation/seed" kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/test" @@ -249,7 +248,6 @@ func healthCheckFunc(fn resultingConditionFunc) NewHealthCheckFunc { } func (c resultingConditionFunc) CheckSeed(_ context.Context, - seed *seedpkg.Seed, conditions []gardencorev1beta1.Condition, thresholdMappings map[gardencorev1beta1.ConditionType]time.Duration) []gardencorev1beta1.Condition { return c(conditions) diff --git a/pkg/gardenlet/controller/seed/seed_care_control_types.go b/pkg/gardenlet/controller/seed/seed_care_control_types.go index 1e3e9f7cc53..90afffe327c 100644 --- a/pkg/gardenlet/controller/seed/seed_care_control_types.go +++ b/pkg/gardenlet/controller/seed/seed_care_control_types.go @@ -39,5 +39,5 @@ var defaultNewHealthCheck NewHealthCheckFunc = func(seed *gardencorev1beta1.Seed // HealthCheck is an interface used to perform health checks. type HealthCheck interface { - CheckSeed(ctx context.Context, seed *seedpkg.Seed, condition []gardencorev1beta1.Condition, thresholdMappings map[gardencorev1beta1.ConditionType]time.Duration) []gardencorev1beta1.Condition + CheckSeed(ctx context.Context, condition []gardencorev1beta1.Condition, thresholdMappings map[gardencorev1beta1.ConditionType]time.Duration) []gardencorev1beta1.Condition } diff --git a/pkg/gardenlet/controller/seed/seed_control.go b/pkg/gardenlet/controller/seed/seed_control.go index 85dcc1a99c9..a7bf3eedc82 100644 --- a/pkg/gardenlet/controller/seed/seed_control.go +++ b/pkg/gardenlet/controller/seed/seed_control.go @@ -319,6 +319,11 @@ func (r *reconciler) reconcile(ctx context.Context, gardenClient client.Client, return err } + conditionSeedBootstrapped = gardencorev1beta1helper.UpdatedCondition(conditionSeedBootstrapped, gardencorev1beta1.ConditionProgressing, "BootstrapProgressing", "Seed cluster is currently being bootstrapped.") + if err = r.patchSeedStatus(ctx, gardenClient, log, seed, seedKubernetesVersion, capacity, allocatable, conditionSeedBootstrapped); err != nil { + return fmt.Errorf("could not update status of %s condition to %s: %w", conditionSeedBootstrapped.Type, gardencorev1beta1.ConditionProgressing, err) + } + // Bootstrap the Seed cluster. if err := seedpkg.RunReconcileSeedFlow(ctx, gardenClient, seedClientSet, seedObj, gardenSecrets, r.imageVector, r.componentImageVectors, r.config.DeepCopy(), log); err != nil { conditionSeedBootstrapped = gardencorev1beta1helper.UpdatedCondition(conditionSeedBootstrapped, gardencorev1beta1.ConditionFalse, "BootstrappingFailed", err.Error()) @@ -327,8 +332,16 @@ func (r *reconciler) reconcile(ctx context.Context, gardenClient client.Client, return err } + // Set the status of SeedSystemComponentsHealthy condition to Progressing so that the Seed does not immediately become ready + // after being successfully bootstrapped in case the system components got updated. The SeedSystemComponentsHealthy condition + // will be set to either True, False or Progressing by the seed care reconciler depending on the health of the system components + // after the necessary checks are completed. + conditionSeedSystemComponentsHealthy := gardencorev1beta1helper.GetOrInitCondition(seed.Status.Conditions, gardencorev1beta1.SeedSystemComponentsHealthy) + conditionSeedSystemComponentsHealthy = gardencorev1beta1helper.UpdatedCondition(conditionSeedSystemComponentsHealthy, gardencorev1beta1.ConditionProgressing, "SystemComponentsCheckProgressing", "Pending health check of system components after successful bootstrap of seed cluster.") conditionSeedBootstrapped = gardencorev1beta1helper.UpdatedCondition(conditionSeedBootstrapped, gardencorev1beta1.ConditionTrue, "BootstrappingSucceeded", "Seed cluster has been bootstrapped successfully.") - _ = r.patchSeedStatus(ctx, gardenClient, log, seed, seedKubernetesVersion, capacity, allocatable, conditionSeedBootstrapped) + if err = r.patchSeedStatus(ctx, gardenClient, log, seed, seedKubernetesVersion, capacity, allocatable, conditionSeedBootstrapped, conditionSeedSystemComponentsHealthy); err != nil { + return fmt.Errorf("could not update status of %s condition to %s and %s conditions to %s: %w", conditionSeedBootstrapped.Type, gardencorev1beta1.ConditionTrue, conditionSeedSystemComponentsHealthy.Type, gardencorev1beta1.ConditionProgressing, err) + } if seed.Spec.Backup != nil { // This should be post updating the seed is available. Since, scheduler will then mostly use diff --git a/pkg/operation/care/seed_health.go b/pkg/operation/care/seed_health.go index 20ef633641a..8a280e55f92 100644 --- a/pkg/operation/care/seed_health.go +++ b/pkg/operation/care/seed_health.go @@ -30,9 +30,9 @@ import ( "github.com/gardener/gardener/pkg/operation/botanist/component/nginxingress" "github.com/gardener/gardener/pkg/operation/botanist/component/seedadmissioncontroller" "github.com/gardener/gardener/pkg/operation/botanist/component/vpa" - seedpkg "github.com/gardener/gardener/pkg/operation/seed" kutil "github.com/gardener/gardener/pkg/utils/kubernetes" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -62,7 +62,6 @@ func NewHealthForSeed(seed *gardencorev1beta1.Seed, seedClient client.Client) *S // CheckSeed conducts the health checks on all the given conditions. func (h *SeedHealth) CheckSeed(ctx context.Context, - seed *seedpkg.Seed, conditions []gardencorev1beta1.Condition, thresholdMappings map[gardencorev1beta1.ConditionType]time.Duration) []gardencorev1beta1.Condition { @@ -101,6 +100,10 @@ func (h *SeedHealth) checkSeedSystemComponents( for _, name := range managedResources { mr := &resourcesv1alpha1.ManagedResource{} if err := h.seedClient.Get(ctx, kutil.Key(v1beta1constants.GardenNamespace, name), mr); err != nil { + if apierrors.IsNotFound(err) { + exitCondition := checker.FailedCondition(condition, "ResourceNotFound", err.Error()) + return &exitCondition, nil + } return nil, err } diff --git a/pkg/operation/care/seed_health_test.go b/pkg/operation/care/seed_health_test.go new file mode 100644 index 00000000000..480c288c8c1 --- /dev/null +++ b/pkg/operation/care/seed_health_test.go @@ -0,0 +1,318 @@ +// Copyright (c) 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package care_test + +import ( + "context" + "time" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1" + "github.com/gardener/gardener/pkg/client/kubernetes" + "github.com/gardener/gardener/pkg/operation/botanist/component/clusterautoscaler" + "github.com/gardener/gardener/pkg/operation/botanist/component/clusteridentity" + "github.com/gardener/gardener/pkg/operation/botanist/component/dependencywatchdog" + "github.com/gardener/gardener/pkg/operation/botanist/component/etcd" + "github.com/gardener/gardener/pkg/operation/botanist/component/networkpolicies" + "github.com/gardener/gardener/pkg/operation/botanist/component/nginxingress" + "github.com/gardener/gardener/pkg/operation/botanist/component/seedadmissioncontroller" + "github.com/gardener/gardener/pkg/operation/botanist/component/vpa" + "github.com/gardener/gardener/pkg/operation/care" + "github.com/gardener/gardener/pkg/utils/test" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + "github.com/onsi/gomega/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var ( + requiredManagedResources = []string{ + etcd.Druid, + seedadmissioncontroller.Name, + networkpolicies.ManagedResourceControlName, + clusteridentity.ManagedResourceControlName, + clusterautoscaler.ManagedResourceControlName, + vpa.ManagedResourceControlName, + } + + optionalManagedResources = []string{ + dependencywatchdog.ManagedResourceDependencyWatchdogEndpoint, + dependencywatchdog.ManagedResourceDependencyWatchdogProbe, + nginxingress.ManagedResourceName, + } +) + +var _ = Describe("Seed health", func() { + var ( + ctx context.Context + c client.Client + + seed *gardencorev1beta1.Seed + + seedSystemComponentsHealthyCondition gardencorev1beta1.Condition + ) + + BeforeEach(func() { + ctx = context.TODO() + c = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build() + + seed = &gardencorev1beta1.Seed{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: gardencorev1beta1.SeedSpec{ + Ingress: &gardencorev1beta1.Ingress{ + Controller: gardencorev1beta1.IngressController{ + Kind: "nginx", + }, + }, + Settings: &gardencorev1beta1.SeedSettings{ + ShootDNS: &gardencorev1beta1.SeedSettingShootDNS{ + Enabled: true, + }, + DependencyWatchdog: &gardencorev1beta1.SeedSettingDependencyWatchdog{ + Endpoint: &gardencorev1beta1.SeedSettingDependencyWatchdogEndpoint{ + Enabled: true, + }, + Probe: &gardencorev1beta1.SeedSettingDependencyWatchdogProbe{ + Enabled: true, + }, + }, + }, + }, + } + + seedSystemComponentsHealthyCondition = gardencorev1beta1.Condition{ + Type: gardencorev1beta1.SeedSystemComponentsHealthy, + } + }) + + Describe("#CheckSeed", func() { + Context("When all managed resources are deployed successfully", func() { + JustBeforeEach(func() { + for _, name := range append(requiredManagedResources, optionalManagedResources...) { + Expect(c.Create(ctx, healthyManagedResource(name))).To(Succeed()) + } + }) + + It("should set SeedSystemComponentsHealthy condition to true", func() { + healthCheck := care.NewHealthForSeed(seed, c) + updatedConditions := healthCheck.CheckSeed(ctx, []gardencorev1beta1.Condition{seedSystemComponentsHealthyCondition}, nil) + Expect(len(updatedConditions)).ToNot(BeZero()) + Expect(updatedConditions[0]).To(beConditionWithStatusReasonAndMessage(gardencorev1beta1.ConditionTrue, "SystemComponentsRunning", "All system components are healthy.")) + }) + }) + + Context("When optional managed resources are turned off in the seed specification, and required resources are deployed successfully", func() { + JustBeforeEach(func() { + seed.Spec.Ingress.Controller.Kind = "foo" + seed.Spec.Settings.DependencyWatchdog.Endpoint.Enabled = false + seed.Spec.Settings.DependencyWatchdog.Probe.Enabled = false + + for _, name := range requiredManagedResources { + Expect(c.Create(ctx, healthyManagedResource(name))).To(Succeed()) + } + }) + + It("should set SeedSystemComponentsHealthy condition to true", func() { + healthCheck := care.NewHealthForSeed(seed, c) + updatedConditions := healthCheck.CheckSeed(ctx, []gardencorev1beta1.Condition{seedSystemComponentsHealthyCondition}, nil) + Expect(len(updatedConditions)).ToNot(BeZero()) + Expect(updatedConditions[0]).To(beConditionWithStatusReasonAndMessage(gardencorev1beta1.ConditionTrue, "SystemComponentsRunning", "All system components are healthy.")) + }) + }) + + Context("When there are issues with seed managed resources", func() { + var ( + now time.Time + + tests = func(reason, message string) { + It("should set SeedSystemComponentsHealthy condition to False if there is no Progressing threshold duration mapping", func() { + healthCheck := care.NewHealthForSeed(seed, c) + updatedConditions := healthCheck.CheckSeed(ctx, []gardencorev1beta1.Condition{seedSystemComponentsHealthyCondition}, nil) + + Expect(len(updatedConditions)).ToNot(BeZero()) + Expect(updatedConditions[0]).To(beConditionWithStatusReasonAndMessage(gardencorev1beta1.ConditionFalse, reason, message)) + }) + + It("should set SeedSystemComponentsHealthy condition to Progressing if time is within threshold duration and condition is currently False", func() { + defer test.WithVars( + &care.Now, func() time.Time { return now.Add(30 * time.Second) }, + )() + seedSystemComponentsHealthyCondition.Status = gardencorev1beta1.ConditionFalse + + healthCheck := care.NewHealthForSeed(seed, c) + updatedConditions := healthCheck.CheckSeed( + ctx, + []gardencorev1beta1.Condition{seedSystemComponentsHealthyCondition}, + map[gardencorev1beta1.ConditionType]time.Duration{gardencorev1beta1.SeedSystemComponentsHealthy: time.Minute}, + ) + + Expect(len(updatedConditions)).ToNot(BeZero()) + Expect(updatedConditions[0]).To(beConditionWithStatusReasonAndMessage(gardencorev1beta1.ConditionProgressing, reason, message)) + }) + + It("should set SeedSystemComponentsHealthy condition to Progressing if time is within threshold duration and condition is currently True", func() { + defer test.WithVars( + &care.Now, func() time.Time { return now.Add(30 * time.Second) }, + )() + seedSystemComponentsHealthyCondition.Status = gardencorev1beta1.ConditionTrue + + healthCheck := care.NewHealthForSeed(seed, c) + updatedConditions := healthCheck.CheckSeed( + ctx, + []gardencorev1beta1.Condition{seedSystemComponentsHealthyCondition}, + map[gardencorev1beta1.ConditionType]time.Duration{gardencorev1beta1.SeedSystemComponentsHealthy: time.Minute}, + ) + + Expect(len(updatedConditions)).ToNot(BeZero()) + Expect(updatedConditions[0]).To(beConditionWithStatusReasonAndMessage(gardencorev1beta1.ConditionProgressing, reason, message)) + }) + + It("should set SeedSystemComponentsHealthy condition to false if Progressing threshold duration has expired", func() { + defer test.WithVars( + &care.Now, func() time.Time { return now.Add(90 * time.Second) }, + )() + + seedSystemComponentsHealthyCondition.Status = gardencorev1beta1.ConditionProgressing + + healthCheck := care.NewHealthForSeed(seed, c) + updatedConditions := healthCheck.CheckSeed( + ctx, + []gardencorev1beta1.Condition{seedSystemComponentsHealthyCondition}, + map[gardencorev1beta1.ConditionType]time.Duration{gardencorev1beta1.SeedSystemComponentsHealthy: time.Minute}, + ) + + Expect(len(updatedConditions)).ToNot(BeZero()) + Expect(updatedConditions[0]).To(beConditionWithStatusReasonAndMessage(gardencorev1beta1.ConditionFalse, reason, message)) + }) + } + ) + + Context("When optional managed resources are enabled in seed settings but not deployed", func() { + JustBeforeEach(func() { + for _, name := range requiredManagedResources { + Expect(c.Create(ctx, healthyManagedResource(name))).To(Succeed()) + } + }) + + tests("ResourceNotFound", "not found") + }) + + Context("When required managed resources are not deployed", func() { + JustBeforeEach(func() { + for _, name := range optionalManagedResources { + Expect(c.Create(ctx, healthyManagedResource(name))).To(Succeed()) + } + }) + + tests("ResourceNotFound", "not found") + }) + + Context("When all managed resources are deployed, but not healthy", func() { + JustBeforeEach(func() { + for _, name := range append(requiredManagedResources, optionalManagedResources...) { + Expect(c.Create(ctx, notHealthyManagedResource(name))).To(Succeed()) + } + }) + + tests("NotHealthy", "Resources are not healthy") + }) + + Context("When all managed resources are deployed but their resources are not applied", func() { + JustBeforeEach(func() { + for _, name := range append(requiredManagedResources, optionalManagedResources...) { + Expect(c.Create(ctx, notAppliedManagedResource(name))).To(Succeed()) + } + }) + + tests("NotApplied", "Resources are not applied") + }) + }) + }) +}) + +func beConditionWithStatusReasonAndMessage(status gardencorev1beta1.ConditionStatus, reason, message string) types.GomegaMatcher { + return MatchFields(IgnoreExtras, Fields{ + "Status": Equal(status), + "Reason": Equal(reason), + "Message": ContainSubstring(message), + }) +} +func healthyManagedResource(name string) *resourcesv1alpha1.ManagedResource { + return managedResource( + name, + []gardencorev1beta1.Condition{ + { + Type: resourcesv1alpha1.ResourcesApplied, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesHealthy, + Status: gardencorev1beta1.ConditionTrue, + }, + }) +} + +func notHealthyManagedResource(name string) *resourcesv1alpha1.ManagedResource { + return managedResource( + name, + []gardencorev1beta1.Condition{ + { + Type: resourcesv1alpha1.ResourcesApplied, + Status: gardencorev1beta1.ConditionTrue, + }, + { + Type: resourcesv1alpha1.ResourcesHealthy, + Reason: "NotHealthy", + Message: "Resources are not healthy", + Status: gardencorev1beta1.ConditionFalse, + }, + }) +} + +func notAppliedManagedResource(name string) *resourcesv1alpha1.ManagedResource { + return managedResource( + name, + []gardencorev1beta1.Condition{ + { + Type: resourcesv1alpha1.ResourcesApplied, + Reason: "NotApplied", + Message: "Resources are not applied", + Status: gardencorev1beta1.ConditionFalse, + }, + { + Type: resourcesv1alpha1.ResourcesHealthy, + Status: gardencorev1beta1.ConditionTrue, + }, + }) +} + +func managedResource(name string, conditions []gardencorev1beta1.Condition) *resourcesv1alpha1.ManagedResource { + return &resourcesv1alpha1.ManagedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: v1beta1constants.GardenNamespace, + }, + Status: resourcesv1alpha1.ManagedResourceStatus{ + Conditions: conditions, + }, + } +} diff --git a/pkg/utils/kubernetes/health/seed.go b/pkg/utils/kubernetes/health/seed.go index 2a21914904a..02a7fc73074 100644 --- a/pkg/utils/kubernetes/health/seed.go +++ b/pkg/utils/kubernetes/health/seed.go @@ -37,7 +37,7 @@ func CheckSeed(seed *gardencorev1beta1.Seed, identity *gardencorev1beta1.Gardene return fmt.Errorf("observing Gardener version not up to date (%v/%v)", seed.Status.Gardener, identity) } - return checkSeed(seed, identity) + return checkSeed(seed) } // CheckSeedForMigration checks if the Seed is up-to-date (comparing only the versions) and if its extensions have been successfully bootstrapped. @@ -46,11 +46,11 @@ func CheckSeedForMigration(seed *gardencorev1beta1.Seed, identity *gardencorev1b return fmt.Errorf("observing Gardener version not up to date (%s/%s)", seed.Status.Gardener.Version, identity.Version) } - return checkSeed(seed, identity) + return checkSeed(seed) } // checkSeed checks if the seed.Status.ObservedGeneration ObservedGeneration is not outdated and if its extensions have been successfully bootstrapped. -func checkSeed(seed *gardencorev1beta1.Seed, identity *gardencorev1beta1.Gardener) error { +func checkSeed(seed *gardencorev1beta1.Seed) error { if seed.Status.ObservedGeneration < seed.Generation { return fmt.Errorf("observed generation outdated (%d/%d)", seed.Status.ObservedGeneration, seed.Generation) }