fix cloneSet controller block caused by scale expectation leakage

Signed-off-by: liheng.zms <liheng.zms@alibaba-inc.com>

pkg/controller/cloneset/cloneset_controller.go

@@ -24,6 +24,7 @@ import (
 
 	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
 	kruiseclient "github.com/openkruise/kruise/pkg/client"
+	kubeClient "github.com/openkruise/kruise/pkg/client"
 	clonesetcore "github.com/openkruise/kruise/pkg/controller/cloneset/core"
 	revisioncontrol "github.com/openkruise/kruise/pkg/controller/cloneset/revision"
 	synccontrol "github.com/openkruise/kruise/pkg/controller/cloneset/sync"
@@ -41,6 +42,7 @@ import (
 	"github.com/openkruise/kruise/pkg/util/refmanager"
 	"github.com/openkruise/kruise/pkg/util/volumeclaimtemplate"
 
+	"github.com/prometheus/client_golang/prometheus"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -59,13 +61,16 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
 func init() {
 	flag.IntVar(&concurrentReconciles, "cloneset-workers", concurrentReconciles, "Max concurrent workers for CloneSet controller.")
+	// register prometheus
+	metrics.Registry.MustRegister(CloneSetScaleExpectationLeakageMetrics)
 }
 
 var (
@@ -75,6 +80,16 @@ var (
 	minimumReplicasToPreDownloadImage int32 = 3
 )
 
+var (
+	CloneSetScaleExpectationLeakageMetrics = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "cloneset_scale_expectation_leakage",
+			Help: "CloneSet Scale Expectation Leakage Metrics",
+			// cloneSet namespace, name
+		}, []string{"namespace", "name"},
+	)
+)
+
 // Add creates a new CloneSet Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
 func Add(mgr manager.Manager) error {
@@ -229,8 +244,27 @@ func (r *ReconcileCloneSet) doReconcile(request reconcile.Request) (res reconcil
 	// If scaling expectations have not satisfied yet, just skip this reconcile.
 	if scaleSatisfied, unsatisfiedDuration, scaleDirtyPods := clonesetutils.ScaleExpectations.SatisfiedExpectations(request.String()); !scaleSatisfied {
 		if unsatisfiedDuration >= expectations.ExpectationTimeout {
+			// In some extreme scenarios, if the Pod is created and then quickly deleted, there may be event loss.
+			// Therefore, a touting mechanism is needed to ensure that clonesets can continue to work.
 			klog.InfoS("Expectation unsatisfied overtime", "cloneSet", request, "scaleDirtyPods", scaleDirtyPods, "overTime", unsatisfiedDuration)
-			return reconcile.Result{}, nil
+			CloneSetScaleExpectationLeakageMetrics.WithLabelValues(request.Namespace, request.Name).Add(1)
+			for _, pods := range scaleDirtyPods {
+				for _, name := range pods {
+					_, err = kubeClient.GetGenericClient().KubeClient.CoreV1().Pods(request.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
+					if err == nil {
+						klog.Warningf("CloneSet(%s/%s) ScaleExpectations leakage, but Pod(%s) already exist", request.Namespace, request.Name, name)
+						return reconcile.Result{RequeueAfter: 30 * time.Second}, nil
+					} else if !errors.IsNotFound(err) {
+						klog.ErrorS(err, "Failed to get Pod", "cloneSet", request, "pod", name)
+						return reconcile.Result{RequeueAfter: 3 * time.Second}, nil
+					}
+				}
+			}
+			klog.InfoS("CloneSet ScaleExpectation DirtyPods no longer exists, and delete ScaleExpectation", "cloneSet", request)
+			clonesetutils.ScaleExpectations.DeleteExpectations(request.String())
+			// In order to avoid the scale expectation timeout,
+			// there is no subsequent Pod, CloneSet event causing CloneSet not to be scheduled
+			return reconcile.Result{RequeueAfter: 10 * time.Second}, nil
 		}
 		klog.V(4).InfoS("Not satisfied scale", "cloneSet", request, "scaleDirtyPods", scaleDirtyPods)
 		return reconcile.Result{RequeueAfter: expectations.ExpectationTimeout - unsatisfiedDuration}, nil