Skip to content

Commit

Permalink
[clusteragent/autoscaling] Only activate local recommender when load …
Browse files Browse the repository at this point in the history 
…store is activated (#34449)
  • Loading branch information
@jennchenn
jennchenn authored Feb 26, 2025
1 parent 706d8bc commit 2c8d60b
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 60 deletions.
117 changes: 59 additions & 58 deletions pkg/clusteragent/autoscaling/workload/local/replica_calculator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,8 @@ func TestCalculateUtilizationPodResource(t *testing.T) {
PodName: "pod-name2",
ContainerValues: map[string][]loadstore.EntityValue{
"container-1": {
*newEntityValue(time.Now().Unix(), 2e8),
*newEntityValue(time.Now().Unix()-15, 3e8),
*newEntityValue(testTime.Unix(), 2e8),
*newEntityValue(testTime.Unix()-15, 3e8),
},
},
},
Expand Down Expand Up @@ -203,8 +203,8 @@ func TestCalculateUtilizationPodResource(t *testing.T) {
PodName: "pod-name1",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2.5e8),
*newEntityValue(time.Now().Unix()-30, 3e8),
*newEntityValue(testTime.Unix()-15, 2.5e8),
*newEntityValue(testTime.Unix()-30, 3e8),
},
},
},
Expand Down Expand Up @@ -259,12 +259,12 @@ func TestCalculateUtilizationPodResource(t *testing.T) {
PodName: "pod-name1",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2e8),
*newEntityValue(time.Now().Unix()-30, 3e8),
*newEntityValue(testTime.Unix()-15, 2e8),
*newEntityValue(testTime.Unix()-30, 3e8),
},
"container-name2": {
*newEntityValue(time.Now().Unix()-15, 2e8),
*newEntityValue(time.Now().Unix()-30, 4e8),
*newEntityValue(testTime.Unix()-15, 2e8),
*newEntityValue(testTime.Unix()-30, 4e8),
},
},
},
Expand Down Expand Up @@ -331,17 +331,17 @@ func TestCalculateUtilizationPodResource(t *testing.T) {
PodName: "pod-name1",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2e8),
*newEntityValue(time.Now().Unix()-30, 3e8),
*newEntityValue(testTime.Unix()-15, 2e8),
*newEntityValue(testTime.Unix()-30, 3e8),
},
},
},
{
PodName: "pod-name2",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name2": {
*newEntityValue(time.Now().Unix()-15, 2e8),
*newEntityValue(time.Now().Unix()-30, 4e8),
*newEntityValue(testTime.Unix()-15, 2e8),
*newEntityValue(testTime.Unix()-30, 4e8),
},
},
},
Expand Down Expand Up @@ -409,8 +409,8 @@ func TestCalculateUtilizationPodResource(t *testing.T) {
PodName: "pod-name1",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2e8),
*newEntityValue(time.Now().Unix()-30, 3e8),
*newEntityValue(testTime.Unix()-15, 2e8),
*newEntityValue(testTime.Unix()-30, 3e8),
},
},
},
Expand Down Expand Up @@ -528,8 +528,8 @@ func TestCalculateUtilizationContainerResource(t *testing.T) {
PodName: "pod-name2",
ContainerValues: map[string][]loadstore.EntityValue{
"container-1": {
*newEntityValue(time.Now().Unix(), 2e8),
*newEntityValue(time.Now().Unix()-15, 3e8),
*newEntityValue(testTime.Unix(), 2e8),
*newEntityValue(testTime.Unix()-15, 3e8),
},
},
},
Expand Down Expand Up @@ -569,8 +569,8 @@ func TestCalculateUtilizationContainerResource(t *testing.T) {
PodName: "pod-name1",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2e8),
*newEntityValue(time.Now().Unix()-30, 3e8),
*newEntityValue(testTime.Unix()-15, 2e8),
*newEntityValue(testTime.Unix()-30, 3e8),
},
},
},
Expand Down Expand Up @@ -625,12 +625,12 @@ func TestCalculateUtilizationContainerResource(t *testing.T) {
PodName: "pod-name1",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2e8),
*newEntityValue(time.Now().Unix()-30, 3e8),
*newEntityValue(testTime.Unix()-15, 2e8),
*newEntityValue(testTime.Unix()-30, 3e8),
},
"container-name2": {
*newEntityValue(time.Now().Unix()-15, 2e8),
*newEntityValue(time.Now().Unix()-30, 4e8),
*newEntityValue(testTime.Unix()-15, 2e8),
*newEntityValue(testTime.Unix()-30, 4e8),
},
},
},
Expand Down Expand Up @@ -697,17 +697,17 @@ func TestCalculateUtilizationContainerResource(t *testing.T) {
PodName: "pod-name1",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2e8),
*newEntityValue(time.Now().Unix()-30, 3e8),
*newEntityValue(testTime.Unix()-15, 2e8),
*newEntityValue(testTime.Unix()-30, 3e8),
},
},
},
{
PodName: "pod-name2",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2e8),
*newEntityValue(time.Now().Unix()-30, 4e8),
*newEntityValue(testTime.Unix()-15, 2e8),
*newEntityValue(testTime.Unix()-30, 4e8),
},
},
},
Expand Down Expand Up @@ -775,8 +775,8 @@ func TestCalculateUtilizationContainerResource(t *testing.T) {
PodName: "pod-name1",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2e8),
*newEntityValue(time.Now().Unix()-30, 3e8),
*newEntityValue(testTime.Unix()-15, 2e8),
*newEntityValue(testTime.Unix()-30, 3e8),
},
},
},
Expand Down Expand Up @@ -947,8 +947,8 @@ func TestRecommend(t *testing.T) {
PodName: "pod-name2",
ContainerValues: map[string][]loadstore.EntityValue{
"container-1": {
*newEntityValue(time.Now().Unix(), 2e8),
*newEntityValue(time.Now().Unix()-15, 3e8),
*newEntityValue(testTime.Unix(), 2e8),
*newEntityValue(testTime.Unix()-15, 3e8),
},
},
},
Expand Down Expand Up @@ -1057,8 +1057,8 @@ func TestRecommend(t *testing.T) {
PodName: "pod-name1",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 1e8),
*newEntityValue(time.Now().Unix()-30, 1.23e8),
*newEntityValue(testTime.Unix()-15, 1e8),
*newEntityValue(testTime.Unix()-30, 1.23e8),
},
"container-name2": {
*newEntityValue(testTime.Unix()-15, 1.4e8),
Expand All @@ -1070,26 +1070,26 @@ func TestRecommend(t *testing.T) {
PodName: "pod-name2",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 1e8),
*newEntityValue(time.Now().Unix()-30, 1.1e8),
*newEntityValue(testTime.Unix()-15, 1e8),
*newEntityValue(testTime.Unix()-30, 1.1e8),
},
},
},
{
PodName: "pod-name3",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 1.1e8),
*newEntityValue(time.Now().Unix()-30, 1.1e8),
*newEntityValue(testTime.Unix()-15, 1.1e8),
*newEntityValue(testTime.Unix()-30, 1.1e8),
},
},
},
{
PodName: "pod-name4",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 1.2e8),
*newEntityValue(time.Now().Unix()-30, 1.2e8),
*newEntityValue(testTime.Unix()-15, 1.2e8),
*newEntityValue(testTime.Unix()-30, 1.2e8),
},
},
},
Expand Down Expand Up @@ -1208,39 +1208,39 @@ func TestRecommend(t *testing.T) {
PodName: "pod-name1",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2.4e8),
*newEntityValue(time.Now().Unix()-30, 2.3e8),
*newEntityValue(testTime.Unix()-15, 2.4e8),
*newEntityValue(testTime.Unix()-30, 2.3e8),
},
"container-name2": {
*newEntityValue(time.Now().Unix()-15, 2.44e8),
*newEntityValue(time.Now().Unix()-30, 2.3e8),
*newEntityValue(testTime.Unix()-15, 2.44e8),
*newEntityValue(testTime.Unix()-30, 2.3e8),
},
},
},
{
PodName: "pod-name2",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2.4e8),
*newEntityValue(time.Now().Unix()-30, 2.2e8),
*newEntityValue(testTime.Unix()-15, 2.4e8),
*newEntityValue(testTime.Unix()-30, 2.2e8),
},
},
},
{
PodName: "pod-name3",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2.3e8),
*newEntityValue(time.Now().Unix()-30, 2.4e8),
*newEntityValue(testTime.Unix()-15, 2.3e8),
*newEntityValue(testTime.Unix()-30, 2.4e8),
},
},
},
{
PodName: "pod-name4",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2.4e8),
*newEntityValue(time.Now().Unix()-30, 2.4e8),
*newEntityValue(testTime.Unix()-15, 2.4e8),
*newEntityValue(testTime.Unix()-30, 2.4e8),
},
},
},
Expand Down Expand Up @@ -1351,17 +1351,17 @@ func TestRecommend(t *testing.T) {
PodName: "pod-name1",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2.4e8),
*newEntityValue(time.Now().Unix()-30, 2.3e8),
*newEntityValue(testTime.Unix()-15, 2.4e8),
*newEntityValue(testTime.Unix()-30, 2.3e8),
},
},
},
{
PodName: "pod-name3",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 2.4e8),
*newEntityValue(time.Now().Unix()-30, 2.3e8),
*newEntityValue(testTime.Unix()-15, 2.4e8),
*newEntityValue(testTime.Unix()-30, 2.3e8),
},
},
},
Expand Down Expand Up @@ -1472,17 +1472,17 @@ func TestRecommend(t *testing.T) {
PodName: "pod-name1",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 0.7e8),
*newEntityValue(time.Now().Unix()-30, 0.5e8),
*newEntityValue(testTime.Unix()-15, 0.7e8),
*newEntityValue(testTime.Unix()-30, 0.5e8),
},
},
},
{
PodName: "pod-name4",
ContainerValues: map[string][]loadstore.EntityValue{
"container-name1": {
*newEntityValue(time.Now().Unix()-15, 0.6e8),
*newEntityValue(time.Now().Unix()-30, 0.7e8),
*newEntityValue(testTime.Unix()-15, 0.6e8),
*newEntityValue(testTime.Unix()-30, 0.7e8),
},
},
},
Expand Down Expand Up @@ -1531,6 +1531,7 @@ func TestRecommend(t *testing.T) {
}

func TestCalculateHorizontalRecommendationsScaleUp(t *testing.T) {
testTime := time.Now()
deploymentName := "deploymentName"
ns := "default"

Expand All @@ -1550,9 +1551,9 @@ func TestCalculateHorizontalRecommendationsScaleUp(t *testing.T) {
lStore := loadstore.GetWorkloadMetricStore(context.TODO())
entities := make(map[*loadstore.Entity]*loadstore.EntityValue)
entity := newEntity("container.cpu.usage", ns, deploymentName, "pod1", "container-name1")
entities[entity] = newEntityValue(time.Now().Unix()-30, 2.4e8)
entities[entity] = newEntityValue(testTime.Unix()-30, 2.4e8)
lStore.SetEntitiesValues(entities)
entities[entity] = newEntityValue(time.Now().Unix()-15, 2.45e8)
entities[entity] = newEntityValue(testTime.Unix()-15, 2.45e8)
lStore.SetEntitiesValues(entities)
queryResult := lStore.GetMetricsRaw("container.cpu.usage", ns, deploymentName, "")
assert.Len(t, queryResult.Results, 1)
Expand Down
9 changes: 7 additions & 2 deletions pkg/clusteragent/autoscaling/workload/provider/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/DataDog/datadog-agent/pkg/clusteragent/autoscaling/workload"
"github.com/DataDog/datadog-agent/pkg/clusteragent/autoscaling/workload/local"
"github.com/DataDog/datadog-agent/pkg/clusteragent/autoscaling/workload/model"
pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
"github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver"
)

Expand All @@ -48,7 +49,6 @@ func StartWorkloadAutoscaling(
store := autoscaling.NewStore[model.PodAutoscalerInternal]()
podPatcher := workload.NewPodPatcher(store, isLeaderFunc, apiCl.DynamicCl, eventRecorder)
podWatcher := workload.NewPodWatcher(wlm, podPatcher)
localRecommender := local.NewRecommender(podWatcher, store)

_, err := workload.NewConfigRetriever(store, isLeaderFunc, rcClient)
if err != nil {
Expand Down Expand Up @@ -76,7 +76,12 @@ func StartWorkloadAutoscaling(
// TODO: Wait POD Watcher sync before running the controller
go podWatcher.Run(ctx)
go controller.Run(ctx)
go localRecommender.Run(ctx)

// Only start the local recommender if failover metrics collection is enabled
if pkgconfigsetup.Datadog().GetBool("autoscaling.failover.enabled") {
localRecommender := local.NewRecommender(podWatcher, store)
go localRecommender.Run(ctx)
}

return podPatcher, nil
}

0 comments on commit 2c8d60b

Please sign in to comment.