support fix containerrecreaterequest

apis/apps/v1alpha1/containerrecreaterequest_types.go

@@ -37,6 +37,9 @@ const (
 	// ContainerRecreateRequestUnreadyAcquiredKey indicates the Pod has been forced to not-ready.
 	// It is required if the unreadyGracePeriodSeconds is set in ContainerRecreateRequests.
 	ContainerRecreateRequestUnreadyAcquiredKey = "crr.apps.kruise.io/unready-acquired"
+
+	// It is a status key for containers which are be killed by crr
+	ContainerRecreateRequestForceKilledContainerStatusesKey = "crr.apps.kruise.io/force-killed-container-statuses"
 )
 
 // ContainerRecreateRequestSpec defines the desired state of ContainerRecreateRequest
@@ -106,6 +109,8 @@ type ContainerRecreateRequestStrategy struct {
 	FailurePolicy ContainerRecreateRequestFailurePolicyType `json:"failurePolicy,omitempty"`
 	// OrderedRecreate indicates whether to recreate the next container only if the previous one has recreated completely.
 	OrderedRecreate bool `json:"orderedRecreate,omitempty"`
+	// ForceRecreate indicates whether to force kill the container even if the previous container is not running.
+	ForceRecreate bool `json:"forceRecreate,omitempty"`
 	// TerminationGracePeriodSeconds is the optional duration in seconds to wait the container terminating gracefully.
 	// Value must be non-negative integer. The value zero indicates delete immediately.
 	// If this value is nil, we will use pod.Spec.TerminationGracePeriodSeconds as default value.
@@ -172,6 +177,16 @@ type ContainerRecreateRequestSyncContainerStatus struct {
 	ContainerID string `json:"containerID,omitempty"`
 }
 
+// ContainerRecreateRequestKilledContainerStatus contains the state of the last killed container.
+type ContainerRecreateRequestForceKilledContainerStatus struct {
+	// Name of the container.
+	Name string `json:"name"`
+	// Container is killed by force
+	IsForceKilled bool `json:"restartCount"`
+	// Container's ID in the format 'docker://<container_id>'.
+	ContainerID string `json:"containerID,omitempty"`
+}
+
 // +genclient
 // +k8s:openapi-gen=true
 // +kubebuilder:object:root=true

config/rbac/daemon_role.yaml

@@ -38,6 +38,7 @@ rules:
   - get
   - list
   - watch
+  - patch
 - apiGroups:
   - apps.kruise.io
   resources:

pkg/daemon/containerrecreate/crr_daemon_controller.go

@@ -323,6 +323,7 @@ func (c *Controller) manage(crr *appsv1alpha1.ContainerRecreateRequest) error {
 		return c.patchCRRContainerRecreateStates(crr, newCRRContainerRecreateStates)
 	}
 
+	var newForceKilledContainerStatuses []appsv1alpha1.ContainerRecreateRequestForceKilledContainerStatus
 	var completedCount int
 	for i := range newCRRContainerRecreateStates {
 		state := &newCRRContainerRecreateStates[i]
@@ -362,10 +363,25 @@ func (c *Controller) manage(crr *appsv1alpha1.ContainerRecreateRequest) error {
 			}
 			return c.patchCRRContainerRecreateStates(crr, newCRRContainerRecreateStates)
 		}
+
+		if crr.Spec.Strategy.ForceRecreate {
+			newForceKilledContainerStatuses = append(newForceKilledContainerStatuses, appsv1alpha1.ContainerRecreateRequestForceKilledContainerStatus{
+				Name:          state.Name,
+				IsForceKilled: true,
+				ContainerID:   kubeContainerStatus.ID.String(),
+			})
+		}
 		state.Phase = appsv1alpha1.ContainerRecreateRequestRecreating
 		break
 	}
 
+	// record the containers status which are be killed
+	if len(newForceKilledContainerStatuses) > 0 {
+		if err := c.patchForceKilledContainerStatuses(crr, newForceKilledContainerStatuses); err != nil {
+			return err
+		}
+	}
+
 	if !reflect.DeepEqual(crr.Status.ContainerRecreateStates, newCRRContainerRecreateStates) {
 		return c.patchCRRContainerRecreateStates(crr, newCRRContainerRecreateStates)
 	}
@@ -394,6 +410,24 @@ func (c *Controller) patchCRRContainerRecreateStates(crr *appsv1alpha1.Container
 	return c.runtimeClient.Status().Patch(context.TODO(), crr, runtimeclient.RawPatch(types.MergePatchType, []byte(body)))
 }
 
+type patchMetadata struct {
+	Metadata patchMetadataBody `json:"metadata"`
+}
+
+type patchMetadataBody struct {
+	Annotations map[string]string `json:"annotations"`
+}
+
+func (c *Controller) patchForceKilledContainerStatuses(crr *appsv1alpha1.ContainerRecreateRequest, status []appsv1alpha1.ContainerRecreateRequestForceKilledContainerStatus) error {
+	statusStr := util.DumpJSON(status)
+	klog.V(3).Infof("CRR %s/%s patch ForceKilledContainerStatues: %v", crr.Namespace, crr.Name, statusStr)
+	if crr.Annotations[appsv1alpha1.ContainerRecreateRequestForceKilledContainerStatusesKey] != statusStr {
+		body := util.DumpJSON(patchMetadata{Metadata: patchMetadataBody{Annotations: map[string]string{appsv1alpha1.ContainerRecreateRequestForceKilledContainerStatusesKey: statusStr}}})
+		return c.runtimeClient.Patch(context.TODO(), crr, runtimeclient.RawPatch(types.MergePatchType, []byte(body)))
+	}
+	return nil
+}
+
 func (c *Controller) updateCRRPhase(crr *appsv1alpha1.ContainerRecreateRequest, phase appsv1alpha1.ContainerRecreateRequestPhase) error {
 	crr = crr.DeepCopy()
 	crr.Status.Phase = phase

pkg/daemon/containerrecreate/crr_daemon_util.go

@@ -69,6 +69,8 @@ func getCurrentCRRContainersRecreateStates(
 	}
 
 	syncContainerStatuses := getCRRSyncContainerStatuses(crr)
+	// the statuses of force killed containers for this crr resource
+	getForceKillContainerStatuses := getCRRForceKillContainersStatuses(crr)
 	var statuses []appsv1alpha1.ContainerRecreateRequestContainerRecreateState
 
 	for i := range crr.Spec.Containers {
@@ -84,6 +86,8 @@ func getCurrentCRRContainersRecreateStates(
 		syncContainerStatus := syncContainerStatuses[c.Name]
 		kubeContainerStatus := podStatus.FindContainerStatusByName(c.Name)
 
+		killedContainerStatus := getForceKillContainerStatuses[c.Name]
+
 		var currentState appsv1alpha1.ContainerRecreateRequestContainerRecreateState
 		if kubeContainerStatus == nil {
 			// not found the real container
@@ -114,7 +118,9 @@ func getCurrentCRRContainersRecreateStates(
 				syncContainerStatus.Ready {
 				currentState.Phase = appsv1alpha1.ContainerRecreateRequestSucceeded
 			}
-
+			if crr.Spec.Strategy.ForceRecreate && (getForceKillContainerStatuses == nil || killedContainerStatus.ContainerID == "") {
+				currentState.Phase = appsv1alpha1.ContainerRecreateRequestPending
+			}
 		} else {
 			currentState = appsv1alpha1.ContainerRecreateRequestContainerRecreateState{
 				Name:  c.Name,
@@ -157,6 +163,25 @@ func getCRRSyncContainerStatuses(crr *appsv1alpha1.ContainerRecreateRequest) map
 	return statuses
 }
 
+func getCRRForceKillContainersStatuses(crr *appsv1alpha1.ContainerRecreateRequest) map[string]*appsv1alpha1.ContainerRecreateRequestForceKilledContainerStatus {
+	strRaw := crr.Annotations[appsv1alpha1.ContainerRecreateRequestForceKilledContainerStatusesKey]
+	if strRaw == "" {
+		return nil
+	}
+	var forceKillContainerStatuses []appsv1alpha1.ContainerRecreateRequestForceKilledContainerStatus
+	if err := json.Unmarshal([]byte(strRaw), &forceKillContainerStatuses); err != nil {
+		klog.Errorf("Failed to unmarshal CRR %v/%v forceKillContainerStatuses %v, err: %v", crr.Namespace, crr.Name, strRaw, err)
+		return nil
+	}
+
+	forceKillStatuses := make(map[string]*appsv1alpha1.ContainerRecreateRequestForceKilledContainerStatus, len(forceKillContainerStatuses))
+	for i := range forceKillContainerStatuses {
+		c := &forceKillContainerStatuses[i]
+		forceKillStatuses[c.Name] = c
+	}
+	return forceKillStatuses
+}
+
 func convertCRRToPod(crr *appsv1alpha1.ContainerRecreateRequest) *v1.Pod {
 	podName := crr.Spec.PodName
 	podUID := types.UID(crr.Labels[appsv1alpha1.ContainerRecreateRequestPodUIDKey])