Skip to content

Commit

Permalink
update syncRequiredMachineConfigPools to check if any pool is degraded
Browse files Browse the repository at this point in the history
  • Loading branch information
kikisdeliveryservice committed Nov 21, 2020
1 parent f16c53e commit 35c4cb9
Showing 1 changed file with 28 additions and 19 deletions.
47 changes: 28 additions & 19 deletions pkg/operator/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/cache"

Expand Down Expand Up @@ -556,10 +557,6 @@ func (optr *Operator) syncMachineConfigServer(config *renderConfig) error {
// have updated to the latest configuration.
func (optr *Operator) syncRequiredMachineConfigPools(_ *renderConfig) error {
glog.Infof("syncing Required MachineConfigPools")
sel, err := metav1.LabelSelectorAsSelector(metav1.AddLabelToSelector(&metav1.LabelSelector{}, requiredForUpgradeMachineConfigPoolLabelKey, ""))
if err != nil {
return err
}

var lastErr error
if err := wait.Poll(time.Second, 10*time.Minute, func() (bool, error) {
Expand All @@ -580,29 +577,41 @@ func (optr *Operator) syncRequiredMachineConfigPools(_ *renderConfig) error {
return false, nil
}
}
pools, err := optr.mcpLister.List(sel)

pools, err := optr.mcpLister.List(labels.Everything())
if err != nil {
lastErr = err
return false, nil
}
for _, pool := range pools {
if err := isMachineConfigPoolConfigurationValid(pool, version.Hash, optr.mcLister.Get); err != nil {
lastErr = fmt.Errorf("pool %s has not progressed to latest configuration: %v, retrying", pool.Name, err)
return false, nil
}
degraded := isPoolStatusConditionTrue(pool, mcfgv1.MachineConfigPoolDegraded)
if pool.Generation <= pool.Status.ObservedGeneration &&
isPoolStatusConditionTrue(pool, mcfgv1.MachineConfigPoolUpdated) &&
!degraded {
continue
if degraded {
lastErr = fmt.Errorf("error pool %s is not ready, retrying. Status: (pool degraded: %v total: %d, ready %d, updated: %d, unavailable: %d)", pool.Name, degraded, pool.Status.MachineCount, pool.Status.ReadyMachineCount, pool.Status.UpdatedMachineCount, pool.Status.UnavailableMachineCount)
glog.Errorf("Error syncing Required MachineConfigPools: %q", lastErr)
syncerr := optr.syncUpgradeableStatus()
if syncerr != nil {
glog.Errorf("Error syncingUpgradeableStatus: %q", syncerr)
}
return false, nil
}
lastErr = fmt.Errorf("error pool %s is not ready, retrying. Status: (pool degraded: %v total: %d, ready %d, updated: %d, unavailable: %d)", pool.Name, degraded, pool.Status.MachineCount, pool.Status.ReadyMachineCount, pool.Status.UpdatedMachineCount, pool.Status.UnavailableMachineCount)
glog.Errorf("Error syncing Required MachineConfigPools: %q", lastErr)
syncerr := optr.syncUpgradeableStatus()
if syncerr != nil {
glog.Errorf("Error syncingUpgradeableStatus: %q", syncerr)

_, hasRequiredPoolLabel := pool.Labels[requiredForUpgradeMachineConfigPoolLabelKey]

if hasRequiredPoolLabel {
if err := isMachineConfigPoolConfigurationValid(pool, version.Hash, optr.mcLister.Get); err != nil {
lastErr = fmt.Errorf("pool %s has not progressed to latest configuration: %v, retrying", pool.Name, err)
return false, nil
}

if pool.Generation <= pool.Status.ObservedGeneration &&
isPoolStatusConditionTrue(pool, mcfgv1.MachineConfigPoolUpdated) {
continue
}
}
return false, nil
}
syncstatuserr := optr.syncUpgradeableStatus()
if syncstatuserr != nil {
glog.Errorf("Error syncingUpgradeableStatus: %q", syncstatuserr)
}
return true, nil
}); err != nil {
Expand Down

0 comments on commit 35c4cb9

Please sign in to comment.