From 3f69ebceb21e61e044717b58d20e2a3eb1a00924 Mon Sep 17 00:00:00 2001 From: "chentao.cht" Date: Mon, 20 Nov 2023 14:39:55 +0800 Subject: [PATCH] feat(scheduler): rsp support noSchedule taint --- .../scheduler/framework/plugins/rsp/rsp.go | 43 +- .../framework/plugins/rsp/rsp_test.go | 456 ++++++++++++++++++ 2 files changed, 484 insertions(+), 15 deletions(-) diff --git a/pkg/controllers/scheduler/framework/plugins/rsp/rsp.go b/pkg/controllers/scheduler/framework/plugins/rsp/rsp.go index cee6c439..f0e05e99 100644 --- a/pkg/controllers/scheduler/framework/plugins/rsp/rsp.go +++ b/pkg/controllers/scheduler/framework/plugins/rsp/rsp.go @@ -98,21 +98,6 @@ func (pl *ClusterCapacityWeight) ReplicaScheduling( schedulingWeights = su.Weights } - clusterPreferences := map[string]planner.ClusterPreferences{} - for _, cluster := range clusters { - pref := planner.ClusterPreferences{ - Weight: schedulingWeights[cluster.Name], - MinReplicas: su.MinReplicas[cluster.Name], - MaxReplicas: nil, - } - - if maxReplicas, exists := su.MaxReplicas[cluster.Name]; exists { - pref.MaxReplicas = pointer.Int64Ptr(maxReplicas) - } - - clusterPreferences[cluster.Name] = pref - } - totalReplicas := int64(0) if su.DesiredReplicas != nil { totalReplicas = *su.DesiredReplicas @@ -127,6 +112,34 @@ func (pl *ClusterCapacityWeight) ReplicaScheduling( currentReplicas[cluster] = totalReplicas } + clusterPreferences := map[string]planner.ClusterPreferences{} + for _, cluster := range clusters { + pref := planner.ClusterPreferences{ + Weight: schedulingWeights[cluster.Name], + MinReplicas: su.MinReplicas[cluster.Name], + MaxReplicas: nil, + } + + if maxReplicas, exists := su.MaxReplicas[cluster.Name]; exists { + pref.MaxReplicas = pointer.Int64(maxReplicas) + } + + // if member cluster has untolerated NoSchedule taint, no new replicas will be scheduled to this cluster + if _, isUntolerated := framework.FindMatchingUntoleratedTaint( + cluster.Spec.Taints, + su.Tolerations, + func(t *corev1.Taint) bool { + return t.Effect == corev1.TaintEffectNoSchedule + }, + ); isUntolerated { + if pref.MaxReplicas == nil || currentReplicas[cluster.Name] < *pref.MaxReplicas { + pref.MaxReplicas = pointer.Int64(currentReplicas[cluster.Name]) + } + } + + clusterPreferences[cluster.Name] = pref + } + estimatedCapacity := map[string]int64{} keepUnschedulableReplicas := false if autoMigration := su.AutoMigration; autoMigration != nil { diff --git a/pkg/controllers/scheduler/framework/plugins/rsp/rsp_test.go b/pkg/controllers/scheduler/framework/plugins/rsp/rsp_test.go index bb586532..59a7d8d6 100644 --- a/pkg/controllers/scheduler/framework/plugins/rsp/rsp_test.go +++ b/pkg/controllers/scheduler/framework/plugins/rsp/rsp_test.go @@ -40,6 +40,15 @@ func NewFederatedCluster(name string) *fedcorev1a1.FederatedCluster { } } +func addTaint(cluster *fedcorev1a1.FederatedCluster, key, value string, effect corev1.TaintEffect) *fedcorev1a1.FederatedCluster { + cluster.Spec.Taints = []corev1.Taint{ + { + Key: key, Value: value, Effect: effect, + }, + } + return cluster +} + func TestExtractClusterNames(t *testing.T) { clusters := []*fedcorev1a1.FederatedCluster{} names := []string{"foo", "bar"} @@ -548,6 +557,363 @@ func TestClusterWeights(t *testing.T) { } } +func TestNoScheduleTaint(t *testing.T) { + tests := []struct { + name string + schedulingUnit framework.SchedulingUnit + clusters []*fedcorev1a1.FederatedCluster + expectedReplicasList framework.ClusterReplicasList + expectedResult *framework.Result + }{ + // scaling up + { + name: "Static scheduling with weights specified, noScheduleTaint should be respected when scaling up", + schedulingUnit: framework.SchedulingUnit{ + DesiredReplicas: pointer.Int64(18), + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + ClusterNames: map[string]struct{}{ + "cluster1": {}, + "cluster2": {}, + "cluster3": {}, + }, + CurrentClusters: map[string]*int64{ + "cluster1": pointer.Int64(2), + }, + Weights: map[string]int64{ + "cluster1": 2, + "cluster2": 3, + "cluster3": 5, + }, + }, + clusters: []*fedcorev1a1.FederatedCluster{ + addTaint(NewFederatedCluster("cluster1"), "a", "b", corev1.TaintEffectNoSchedule), + NewFederatedCluster("cluster2"), + NewFederatedCluster("cluster3"), + }, + expectedReplicasList: framework.ClusterReplicasList{ + { + Cluster: addTaint(NewFederatedCluster("cluster1"), "a", "b", corev1.TaintEffectNoSchedule), + Replicas: 2, + }, + { + Cluster: NewFederatedCluster("cluster2"), + Replicas: 6, + }, + { + Cluster: NewFederatedCluster("cluster3"), + Replicas: 10, + }, + }, + expectedResult: framework.NewResult(framework.Success), + }, + { + name: "Dynamic scheduling with no weights specified, noScheduleTaint should be respected when scaling up", + schedulingUnit: framework.SchedulingUnit{ + DesiredReplicas: pointer.Int64(18), + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + CurrentClusters: map[string]*int64{ + "cluster1": pointer.Int64(2), + }, + ClusterNames: map[string]struct{}{ + "cluster1": {}, + "cluster2": {}, + "cluster3": {}, + }, + }, + clusters: []*fedcorev1a1.FederatedCluster{ + addTaint( + makeClusterWithCPU("cluster1", 200, 200), + "a", "b", corev1.TaintEffectNoSchedule, + ), + makeClusterWithCPU("cluster2", 300, 300), + makeClusterWithCPU("cluster3", 500, 500), + }, + expectedReplicasList: framework.ClusterReplicasList{ + { + Cluster: addTaint( + makeClusterWithCPU("cluster1", 200, 200), + "a", "b", corev1.TaintEffectNoSchedule, + ), + Replicas: 2, + }, + { + Cluster: makeClusterWithCPU("cluster2", 300, 300), + Replicas: 6, + }, + { + Cluster: makeClusterWithCPU("cluster3", 500, 500), + Replicas: 10, + }, + }, + expectedResult: framework.NewResult(framework.Success), + }, + { + name: "Static scheduling with some weights specified, noScheduleTaint should be respected when scaling up", + schedulingUnit: framework.SchedulingUnit{ + DesiredReplicas: pointer.Int64(10), + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + ClusterNames: map[string]struct{}{ + "cluster1": {}, + "cluster2": {}, + "cluster3": {}, + }, + CurrentClusters: map[string]*int64{ + "cluster1": pointer.Int64(2), + }, + Weights: map[string]int64{ + "cluster1": 2, + "cluster2": 3, + }, + }, + clusters: []*fedcorev1a1.FederatedCluster{ + addTaint( + NewFederatedCluster("cluster1"), + "a", "b", corev1.TaintEffectNoSchedule, + ), + NewFederatedCluster("cluster2"), + NewFederatedCluster("cluster3"), + }, + expectedReplicasList: framework.ClusterReplicasList{ + { + Cluster: addTaint( + NewFederatedCluster("cluster1"), + "a", "b", corev1.TaintEffectNoSchedule, + ), + Replicas: 2, + }, + { + Cluster: NewFederatedCluster("cluster2"), + Replicas: 8, + }, + }, + expectedResult: framework.NewResult(framework.Success), + }, + { + name: "Static scheduling with weights specified(no currentClusters), noScheduleTaint should be respected when scaling up", + schedulingUnit: framework.SchedulingUnit{ + DesiredReplicas: pointer.Int64(8), + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + ClusterNames: map[string]struct{}{ + "cluster1": {}, + "cluster2": {}, + "cluster3": {}, + }, + Weights: map[string]int64{ + "cluster1": 2, + "cluster2": 3, + "cluster3": 5, + }, + }, + clusters: []*fedcorev1a1.FederatedCluster{ + addTaint(NewFederatedCluster("cluster1"), "a", "b", corev1.TaintEffectNoSchedule), + NewFederatedCluster("cluster2"), + NewFederatedCluster("cluster3"), + }, + expectedReplicasList: framework.ClusterReplicasList{ + { + Cluster: NewFederatedCluster("cluster2"), + Replicas: 3, + }, + { + Cluster: NewFederatedCluster("cluster3"), + Replicas: 5, + }, + }, + expectedResult: framework.NewResult(framework.Success), + }, + // scaling down + { + name: "Static scheduling with weights specified, noScheduleTaint should be ignored when scaling down", + schedulingUnit: framework.SchedulingUnit{ + DesiredReplicas: pointer.Int64(10), + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + ClusterNames: map[string]struct{}{ + "cluster1": {}, + "cluster2": {}, + "cluster3": {}, + }, + CurrentClusters: map[string]*int64{ + "cluster1": pointer.Int64(4), + "cluster2": pointer.Int64(6), + "cluster3": pointer.Int64(10), + }, + Weights: map[string]int64{ + "cluster1": 2, + "cluster2": 3, + "cluster3": 5, + }, + }, + clusters: []*fedcorev1a1.FederatedCluster{ + addTaint(NewFederatedCluster("cluster1"), "a", "b", corev1.TaintEffectNoSchedule), + NewFederatedCluster("cluster2"), + NewFederatedCluster("cluster3"), + }, + expectedReplicasList: framework.ClusterReplicasList{ + { + Cluster: addTaint(NewFederatedCluster("cluster1"), "a", "b", corev1.TaintEffectNoSchedule), + Replicas: 2, + }, + { + Cluster: NewFederatedCluster("cluster2"), + Replicas: 3, + }, + { + Cluster: NewFederatedCluster("cluster3"), + Replicas: 5, + }, + }, + expectedResult: framework.NewResult(framework.Success), + }, + { + name: "Dynamic scheduling with no weights specified, noScheduleTaint should be ignored when scaling down", + schedulingUnit: framework.SchedulingUnit{ + DesiredReplicas: pointer.Int64(10), + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + CurrentClusters: map[string]*int64{ + "cluster1": pointer.Int64(4), + "cluster2": pointer.Int64(6), + "cluster3": pointer.Int64(10), + }, + ClusterNames: map[string]struct{}{ + "cluster1": {}, + "cluster2": {}, + "cluster3": {}, + }, + }, + clusters: []*fedcorev1a1.FederatedCluster{ + addTaint( + makeClusterWithCPU("cluster1", 200, 200), + "a", "b", corev1.TaintEffectNoSchedule, + ), + makeClusterWithCPU("cluster2", 300, 300), + makeClusterWithCPU("cluster3", 500, 500), + }, + expectedReplicasList: framework.ClusterReplicasList{ + { + Cluster: addTaint( + makeClusterWithCPU("cluster1", 200, 200), + "a", "b", corev1.TaintEffectNoSchedule, + ), + Replicas: 2, + }, + { + Cluster: makeClusterWithCPU("cluster2", 300, 300), + Replicas: 3, + }, + { + Cluster: makeClusterWithCPU("cluster3", 500, 500), + Replicas: 5, + }, + }, + expectedResult: framework.NewResult(framework.Success), + }, + { + name: "Static scheduling with some weights specified, noScheduleTaint should be ignored when scaling down", + schedulingUnit: framework.SchedulingUnit{ + DesiredReplicas: pointer.Int64(5), + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + ClusterNames: map[string]struct{}{ + "cluster1": {}, + "cluster2": {}, + "cluster3": {}, + }, + CurrentClusters: map[string]*int64{ + "cluster1": pointer.Int64(4), + "cluster2": pointer.Int64(6), + "cluster3": pointer.Int64(5), + }, + Weights: map[string]int64{ + "cluster1": 2, + "cluster2": 3, + }, + }, + clusters: []*fedcorev1a1.FederatedCluster{ + addTaint( + NewFederatedCluster("cluster1"), + "a", "b", corev1.TaintEffectNoSchedule, + ), + NewFederatedCluster("cluster2"), + NewFederatedCluster("cluster3"), + }, + expectedReplicasList: framework.ClusterReplicasList{ + { + Cluster: addTaint( + NewFederatedCluster("cluster1"), + "a", "b", corev1.TaintEffectNoSchedule, + ), + Replicas: 2, + }, + { + Cluster: NewFederatedCluster("cluster2"), + Replicas: 3, + }, + }, + expectedResult: framework.NewResult(framework.Success), + }, + // tolerate taint + { + name: "Static scheduling with weights specified, noScheduleTaint may be tolerated when scaling up", + schedulingUnit: framework.SchedulingUnit{ + DesiredReplicas: pointer.Int64(17), + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + ClusterNames: map[string]struct{}{ + "cluster1": {}, + "cluster2": {}, + "cluster3": {}, + }, + CurrentClusters: map[string]*int64{ + "cluster1": pointer.Int64(2), + "cluster2": pointer.Int64(3), + }, + Weights: map[string]int64{ + "cluster1": 2, + "cluster2": 3, + "cluster3": 5, + }, + Tolerations: []corev1.Toleration{ + {Key: "a", Operator: corev1.TolerationOpEqual, Value: "b", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + clusters: []*fedcorev1a1.FederatedCluster{ + addTaint(NewFederatedCluster("cluster1"), "a", "b", corev1.TaintEffectNoSchedule), + addTaint(NewFederatedCluster("cluster2"), "c", "d", corev1.TaintEffectNoSchedule), + NewFederatedCluster("cluster3"), + }, + expectedReplicasList: framework.ClusterReplicasList{ + { + Cluster: addTaint(NewFederatedCluster("cluster1"), "a", "b", corev1.TaintEffectNoSchedule), + Replicas: 4, + }, + { + Cluster: addTaint(NewFederatedCluster("cluster2"), "c", "d", corev1.TaintEffectNoSchedule), + Replicas: 3, + }, + { + Cluster: NewFederatedCluster("cluster3"), + Replicas: 10, + }, + }, + expectedResult: framework.NewResult(framework.Success), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rspPlugin := &ClusterCapacityWeight{} + + replicasList, res := rspPlugin.ReplicaScheduling(context.Background(), &tt.schedulingUnit, tt.clusters) + assert.Equalf( + t, + tt.expectedReplicasList, + replicasList, + "unexpected replicas list, want: %v got %v", + tt.expectedReplicasList, + replicasList, + ) + assert.Equalf(t, tt.expectedResult, res, "unexpected result, want: %v got %v", tt.expectedResult, res) + }) + } +} + func TestMinReplicas(t *testing.T) { tests := []struct { name string @@ -667,6 +1033,96 @@ func TestMaxReplicas(t *testing.T) { }, expectedResult: framework.NewResult(framework.Success), }, + { + name: "MaxReplicas < CurrentReplicas, MaxReplicas and noScheduleTaint are effective at the same time", + schedulingUnit: framework.SchedulingUnit{ + DesiredReplicas: pointer.Int64(10), + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + ClusterNames: map[string]struct{}{ + "cluster1": {}, + "cluster2": {}, + "cluster3": {}, + }, + MaxReplicas: map[string]int64{ + "cluster1": 1, + "cluster2": 1, + "cluster3": 1, + }, + CurrentClusters: map[string]*int64{ + "cluster1": pointer.Int64(2), + }, + Weights: map[string]int64{ + "cluster1": 2, + "cluster2": 3, + "cluster3": 5, + }, + }, + clusters: []*fedcorev1a1.FederatedCluster{ + addTaint(NewFederatedCluster("cluster1"), "a", "b", corev1.TaintEffectNoSchedule), + NewFederatedCluster("cluster2"), + NewFederatedCluster("cluster3"), + }, + expectedReplicasList: framework.ClusterReplicasList{ + { + Cluster: addTaint(NewFederatedCluster("cluster1"), "a", "b", corev1.TaintEffectNoSchedule), + Replicas: 1, + }, + { + Cluster: NewFederatedCluster("cluster2"), + Replicas: 1, + }, + { + Cluster: NewFederatedCluster("cluster3"), + Replicas: 1, + }, + }, + expectedResult: framework.NewResult(framework.Success), + }, + { + name: "MaxReplicas > CurrentReplicas, MaxReplicas and noScheduleTaint are effective at the same time", + schedulingUnit: framework.SchedulingUnit{ + DesiredReplicas: pointer.Int64(12), + SchedulingMode: fedcorev1a1.SchedulingModeDivide, + ClusterNames: map[string]struct{}{ + "cluster1": {}, + "cluster2": {}, + "cluster3": {}, + }, + CurrentClusters: map[string]*int64{ + "cluster1": pointer.Int64(2), + }, + Weights: map[string]int64{ + "cluster1": 1, + "cluster2": 1, + "cluster3": 1, + }, + MaxReplicas: map[string]int64{ + "cluster1": 3, + "cluster2": 3, + "cluster3": 3, + }, + }, + clusters: []*fedcorev1a1.FederatedCluster{ + addTaint(NewFederatedCluster("cluster1"), "a", "b", corev1.TaintEffectNoSchedule), + NewFederatedCluster("cluster2"), + NewFederatedCluster("cluster3"), + }, + expectedReplicasList: framework.ClusterReplicasList{ + { + Cluster: addTaint(NewFederatedCluster("cluster1"), "a", "b", corev1.TaintEffectNoSchedule), + Replicas: 2, + }, + { + Cluster: NewFederatedCluster("cluster2"), + Replicas: 3, + }, + { + Cluster: NewFederatedCluster("cluster3"), + Replicas: 3, + }, + }, + expectedResult: framework.NewResult(framework.Success), + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {