add affinity to pump/drainer (#741)

pingcap · Aug 9, 2019 · 84e4c71 · 84e4c71
1 parent d8be966
commit 84e4c71
Show file tree

Hide file tree

Showing 5 changed files with 101 additions and 5 deletions.
diff --git a/charts/tidb-cluster/templates/drainer-statefulset.yaml b/charts/tidb-cluster/templates/drainer-statefulset.yaml
@@ -30,6 +30,14 @@ spec:
         app.kubernetes.io/managed-by: tidb-operator
         app.kubernetes.io/component: drainer
     spec:
+    {{- if .Values.binlog.drainer.affinity }}
+      affinity:
+{{ toYaml .Values.binlog.drainer.affinity | indent 8 }}
+    {{- end }}
+    {{- if .Values.binlog.drainer.tolerations }}
+      tolerations:
+{{ toYaml .Values.binlog.drainer.tolerations | indent 8 }}
+    {{- end }}
       containers:
       - name: drainer
         image: {{ .Values.binlog.drainer.image }}

diff --git a/charts/tidb-cluster/templates/pump-statefulset.yaml b/charts/tidb-cluster/templates/pump-statefulset.yaml
@@ -30,6 +30,14 @@ spec:
         app.kubernetes.io/managed-by: tidb-operator
         app.kubernetes.io/component: pump
     spec:
+    {{- if .Values.binlog.pump.affinity }}
+      affinity:
+{{ toYaml .Values.binlog.pump.affinity | indent 8 }}
+    {{- end }}
+    {{- if .Values.binlog.pump.tolerations }}
+      tolerations:
+{{ toYaml .Values.binlog.pump.tolerations | indent 8 }}
+    {{- end }}
       containers:
       - name: pump
         image: {{ .Values.binlog.pump.image }}

diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml
@@ -410,6 +410,12 @@ binlog:
     # refer to https://kubernetes.io/docs/concepts/storage/storage-classes
     storageClassName: local-storage
     storage: 20Gi
+    # affinity for pump pod assignment, default: empty
+    # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+    affinity: {}
+    # tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints.
+    # refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration
+    tolerations: []
     syncLog: true
     # a integer value to control expiry date of the binlog data, indicates for how long (in days) the binlog data would be stored.
     # must bigger than 0
@@ -428,6 +434,12 @@ binlog:
     # refer to https://kubernetes.io/docs/concepts/storage/storage-classes
     storageClassName: local-storage
     storage: 10Gi
+    # affinity for drainer pod assignment, default: empty
+    # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+    affinity: {}
+    # tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints.
+    # refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration
+    tolerations: []
     # the number of the concurrency of the downstream for synchronization. The bigger the value,
     # the better throughput performance of the concurrency (16 by default)
     workerCount: 16

diff --git a/tests/actions.go b/tests/actions.go
@@ -2333,7 +2333,11 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD
 
 		listOps := metav1.ListOptions{
 			LabelSelector: labels.SelectorFromSet(
-				pumpStatefulSet.Labels,
+				map[string]string{
+					label.ComponentLabelKey: "pump",
+					label.InstanceLabelKey:  pumpStatefulSet.Labels[label.InstanceLabelKey],
+					label.NameLabelKey:      "tidb-cluster",
+				},
 			).String(),
 		}
 
@@ -2345,8 +2349,23 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD
 
 		for _, pod := range pods.Items {
 			if !oa.pumpHealth(info, pod.Spec.Hostname) {
-				glog.Errorf("some pods is not health %s ,%v", pumpStatefulSetName, err)
-				return false, nil
+				glog.Errorf("some pods is not health %s", pumpStatefulSetName)
+				// return false, nil
+			}
+			glog.Info(pod.Spec.Affinity)
+			if len(pod.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 {
+				return true, fmt.Errorf("pump pod %s/%s should have affinity set", pod.Namespace, pod.Name)
+			}
+			glog.Info(pod.Spec.Tolerations)
+			foundKey := false
+			for _, tor := range pod.Spec.Tolerations {
+				if tor.Key == "node-role" {
+					foundKey = true
+					break
+				}
+			}
+			if !foundKey {
+				return true, fmt.Errorf("pump pod %s/%s should have tolerations set", pod.Namespace, pod.Name)
 			}
 		}
 
@@ -2367,7 +2386,11 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD
 
 		listOps = metav1.ListOptions{
 			LabelSelector: labels.SelectorFromSet(
-				drainerStatefulSet.Labels,
+				map[string]string{
+					label.ComponentLabelKey: "pump",
+					label.InstanceLabelKey:  drainerStatefulSet.Labels[label.InstanceLabelKey],
+					label.NameLabelKey:      "tidb-cluster",
+				},
 			).String(),
 		}
 
@@ -2377,7 +2400,23 @@ func (oa *operatorActions) CheckIncrementalBackup(info *TidbClusterConfig, withD
 		}
 		for _, pod := range pods.Items {
 			if !oa.drainerHealth(info, pod.Spec.Hostname) {
-				return false, nil
+				glog.Errorf("some pods is not health %s", drainerStatefulSetName)
+				// return false, nil
+			}
+			glog.Info(pod.Spec.Affinity)
+			if len(pod.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 {
+				return true, fmt.Errorf("drainer pod %s/%s should have spec.affinity set", pod.Namespace, pod.Name)
+			}
+			glog.Info(pod.Spec.Tolerations)
+			foundKey := false
+			for _, tor := range pod.Spec.Tolerations {
+				if tor.Key == "node-role" {
+					foundKey = true
+					break
+				}
+			}
+			if !foundKey {
+				return true, fmt.Errorf("drainer pod %s/%s should have tolerations set", pod.Namespace, pod.Name)
 			}
 		}
 

diff --git a/tests/util.go b/tests/util.go
@@ -104,6 +104,35 @@ var affinityTemp string = `{{.Kind}}:
           topologyKey: {{.TopologyKey}}
           namespaces:
           - {{.Namespace}}
+binlog:
+  pump:
+    tolerations:
+    - key: node-role
+      operator: Equal
+      value: tidb
+      effect: "NoSchedule"
+    affinity:
+      podAntiAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 50
+          podAffinityTerm:
+            topologyKey: {{.TopologyKey}}
+            namespaces:
+            - {{.Namespace}}
+  drainer:
+    tolerations:
+    - key: node-role
+      operator: Equal
+      value: tidb
+      effect: "NoSchedule"
+    affinity:
+      podAntiAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 50
+          podAffinityTerm:
+            topologyKey: {{.TopologyKey}}
+            namespaces:
+            - {{.Namespace}}
 `
 
 type AffinityInfo struct {