Various scale fixes

examples/simple-cluster.yaml

@@ -4,7 +4,3 @@ metadata:
   name: "example-simple-cluster"
 spec:
   mode: cluster
-  dbservers:
-    count: 3
-  coordinators:
-    count: 2

pkg/apis/arangodb/v1alpha/conditions.go

@@ -33,6 +33,8 @@ type ConditionType string
 const (
 	// ConditionTypeReady indicates that the member or entire deployment is ready and running normally.
 	ConditionTypeReady ConditionType = "Ready"
+	// ConditionTypeTerminated indicates that the member has terminated and will not restart.
+	ConditionTypeTerminated ConditionType = "Terminated"
 )
 
 // Condition represents one current condition of a deployment or deployment member.

pkg/deployment/plan_executor.go

@@ -29,6 +29,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	api "github.com/arangodb/k8s-operator/pkg/apis/arangodb/v1alpha"
+	"github.com/arangodb/k8s-operator/pkg/util/k8sutil"
 )
 
 // executePlan tries to execute the plan as far as possible.
@@ -100,6 +101,7 @@ func (d *Deployment) executePlan(ctx context.Context) (bool, error) {
 // the start time needs to be recorded and a ready condition needs to be checked.
 func (d *Deployment) startAction(ctx context.Context, action api.Action) (bool, error) {
 	log := d.deps.Log
+	ns := d.apiObject.GetNamespace()
 
 	switch action.Type {
 	case api.ActionTypeAddMember:
@@ -113,10 +115,25 @@ func (d *Deployment) startAction(ctx context.Context, action api.Action) (bool,
 		}
 		return true, nil
 	case api.ActionTypeRemoveMember:
-		if err := d.status.Members.RemoveByID(action.MemberID, action.Group); api.IsNotFound(err) {
+		m, _, ok := d.status.Members.ElementByID(action.MemberID)
+		if !ok {
 			// We wanted to remove and it is already gone. All ok
 			return true, nil
-		} else if err != nil {
+		}
+		// Remove the pod (if any)
+		if err := d.deps.KubeCli.Core().Pods(ns).Delete(m.PodName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) {
+			log.Debug().Err(err).Str("pod", m.PodName).Msg("Failed to remove pod")
+			return false, maskAny(err)
+		}
+		// Remove the pvc (if any)
+		if m.PersistentVolumeClaimName != "" {
+			if err := d.deps.KubeCli.Core().PersistentVolumeClaims(ns).Delete(m.PersistentVolumeClaimName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) {
+				log.Debug().Err(err).Str("pod", m.PodName).Msg("Failed to remove pvc")
+				return false, maskAny(err)
+			}
+		}
+		// Remove member
+		if err := d.status.Members.RemoveByID(action.MemberID, action.Group); err != nil {
 			log.Debug().Err(err).Str("group", action.Group.AsRole()).Msg("Failed to remove member")
 			return false, maskAny(err)
 		}
@@ -207,12 +224,17 @@ func (d *Deployment) checkActionProgress(ctx context.Context, action api.Action)
 		// Cleanout completed
 		return true, nil
 	case api.ActionTypeShutdownMember:
-		if d.status.Members.ContainsID(action.MemberID) {
-			// Member still exists, retry soon
-			return false, nil
+		m, _, ok := d.status.Members.ElementByID(action.MemberID)
+		if !ok {
+			// Member not long exists
+			return true, nil
 		}
-		// Member is gone, shutdown is done
-		return true, nil
+		if m.Conditions.IsTrue(api.ConditionTypeTerminated) {
+			// Shutdown completed
+			return true, nil
+		}
+		// Member still not shutdown, retry soon
+		return false, nil
 	default:
 		return false, maskAny(fmt.Errorf("Unknown action type"))
 	}

pkg/deployment/pod_inspector.go

@@ -65,12 +65,30 @@ func (d *Deployment) inspectPods() error {
 		}
 
 		// Update state
-		var readyStatus bool
-		reason := "Pod Not Ready"
-		if readyStatus = k8sutil.IsPodReady(&p); readyStatus {
-			reason = "Pod Ready"
+		updateMemberStatusNeeded := false
+		if k8sutil.IsPodSucceeded(&p) {
+			// Pod has terminated with exit code 0.
+			if memberStatus.Conditions.Update(api.ConditionTypeTerminated, true, "Pod Succeeded", "") {
+				updateMemberStatusNeeded = true
+			}
+		} else if k8sutil.IsPodFailed(&p) {
+			// Pod has terminated with at least 1 container with a non-zero exit code.
+			if memberStatus.Conditions.Update(api.ConditionTypeTerminated, true, "Pod Failed", "") {
+				updateMemberStatusNeeded = true
+			}
+		}
+		if k8sutil.IsPodReady(&p) {
+			// Pod is now ready
+			if memberStatus.Conditions.Update(api.ConditionTypeReady, true, "Pod Ready", "") {
+				updateMemberStatusNeeded = true
+			}
+		} else {
+			// Pod is not ready
+			if memberStatus.Conditions.Update(api.ConditionTypeReady, false, "Pod Not Ready", "") {
+				updateMemberStatusNeeded = true
+			}
 		}
-		if memberStatus.Conditions.Update(api.ConditionTypeReady, readyStatus, reason, "") {
+		if updateMemberStatusNeeded {
 			log.Debug().Str("pod-name", p.GetName()).Msg("Updated member status member for pod")
 			if err := d.status.Members.UpdateMemberStatus(memberStatus, group); err != nil {
 				return maskAny(err)
@@ -96,13 +114,7 @@ func (d *Deployment) inspectPods() error {
 					case api.MemberStateNone:
 						// Do nothing
 					case api.MemberStateShuttingDown:
-						// Remove member
-						if m, found := members.ElementByPodName(podName); found {
-							if err := members.RemoveByID(m.ID); err != nil {
-								return maskAny(err)
-							}
-							events = append(events, k8sutil.NewMemberRemoveEvent(podName, group.AsRole(), d.apiObject))
-						}
+						// Shutdown was intended, so not need to do anything here
 					default:
 						m.State = api.MemberStateNone // This is trigger a recreate of the pod.
 						// Create event

pkg/util/k8sutil/pods.go

@@ -36,15 +36,27 @@ const (
 // IsPodReady returns true if the PodReady condition on
 // the given pod is set to true.
 func IsPodReady(pod *v1.Pod) bool {
-	condition := getPodReadyCondition(&pod.Status)
+	condition := getPodCondition(&pod.Status, v1.PodReady)
 	return condition != nil && condition.Status == v1.ConditionTrue
 }
 
-// getPodReadyCondition returns the PodReady condition in the given status.
+// IsPodSucceeded returns true if all containers of the pod
+// have terminated with exit code 0.
+func IsPodSucceeded(pod *v1.Pod) bool {
+	return pod.Status.Phase == v1.PodSucceeded
+}
+
+// IsPodFailed returns true if all containers of the pod
+// have terminated and at least one of them wih a non-zero exit code.
+func IsPodFailed(pod *v1.Pod) bool {
+	return pod.Status.Phase == v1.PodFailed
+}
+
+// getPodCondition returns the condition of given type in the given status.
 // If not found, nil is returned.
-func getPodReadyCondition(status *v1.PodStatus) *v1.PodCondition {
+func getPodCondition(status *v1.PodStatus, condType v1.PodConditionType) *v1.PodCondition {
 	for i := range status.Conditions {
-		if status.Conditions[i].Type == v1.PodReady {
+		if status.Conditions[i].Type == condType {
 			return &status.Conditions[i]
 		}
 	}