From 30a2f453e0f7ad5d2f8c8edec0ead2de8551d657 Mon Sep 17 00:00:00 2001 From: Daniel Turner Date: Fri, 9 Aug 2019 16:06:42 -0700 Subject: [PATCH 1/3] Add VPA Metrics --- .../kubernetes_state/kubernetes_state.py | 6 ++++++ kubernetes_state/metadata.csv | 5 +++++ kubernetes_state/tests/fixtures/prometheus.txt | 18 ++++++++++++++++++ .../tests/test_kubernetes_state.py | 14 ++++++++++---- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py b/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py index 42a23ec8bc131..d4e458799bd5a 100644 --- a/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py +++ b/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py @@ -194,6 +194,11 @@ def _create_kubernetes_state_prometheus_instance(self, instance): 'kube_statefulset_status_replicas_current': 'statefulset.replicas_current', 'kube_statefulset_status_replicas_ready': 'statefulset.replicas_ready', 'kube_statefulset_status_replicas_updated': 'statefulset.replicas_updated', + 'kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound' : 'vpa.lower_bound', + 'kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target' : 'vpa.target', + 'kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget' : 'vpa.uncapped_target', + 'kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound' : 'vpa.upperbound', + 'kube_verticalpodautoscaler_spec_updatepolicy_updatemode' : 'vpa.update_mode', } ], 'ignore_metrics': [ @@ -257,6 +262,7 @@ def _create_kubernetes_state_prometheus_instance(self, instance): 'kube_job_status_active', 'kube_job_status_completion_time', # We could compute the duration=completion-start as a gauge 'kube_job_status_start_time', + 'kube_verticalpodautoscaler_labels' ], 'label_joins': { 'kube_pod_info': {'label_to_match': 'pod', 'labels_to_get': ['node']}, diff --git a/kubernetes_state/metadata.csv b/kubernetes_state/metadata.csv index 955df0ed039a1..2b8012e793ed6 100644 --- a/kubernetes_state/metadata.csv +++ b/kubernetes_state/metadata.csv @@ -104,3 +104,8 @@ kubernetes_state.telemetry.metrics.input.count,counter,,,,The number of metrics kubernetes_state.telemetry.metrics.blacklist.count,counter,,,,The number of metrics blacklisted by the check,0,kubernetes,k8s_state.telemetry.metrics.blacklist.count kubernetes_state.telemetry.metrics.ignored.count,counter,,,,The number of metrics ignored by the check,0,kubernetes,k8s_state.telemetry.metrics.ignored.count kubernetes_state.telemetry.collector.metrics.count,counter,,,,The number of metrics by collector (kubernetes object kind) by kubernetes namespaces,0,kubernetes,k8s_state.telemetry.collector.metrics.count +kubernetes_state.vpa.lower_bound,,,,The vpa lower bound recommendation,0,kubernetes,k8s_state.vpa.lower_bound +kubernetes_state.vpa.target,,,,The vpa target recommendation,0,kubernetes,k8s_state.vpa.target +kubernetes_state.vpa.uncapped_target,,,,The vpa uncapped recommendation recommendation,0,kubernetes,k8s_state.vpa.uncapped_target +kubernetes_state.vpa.upperbound,,,,The vpa upper bound recommendation,0,kubernetes,k8s_state.vpa.upper_bound +kubernetes_state.vpa.update_mode,,,,The vpa update mode,0,kubernetes,k8s_state.vpa.update_mode diff --git a/kubernetes_state/tests/fixtures/prometheus.txt b/kubernetes_state/tests/fixtures/prometheus.txt index 73c4fdc99698b..81ab8f149ee87 100644 --- a/kubernetes_state/tests/fixtures/prometheus.txt +++ b/kubernetes_state/tests/fixtures/prometheus.txt @@ -874,3 +874,21 @@ kube_hpa_status_condition{namespace="default",hpa="myhpa",condition="unknown",st kube_hpa_status_condition{namespace="default",hpa="myhpa",condition="true",status="ScalingLimited"} 1 kube_hpa_status_condition{namespace="default",hpa="myhpa",condition="false",status="ScalingLimited"} 0 kube_hpa_status_condition{namespace="default",hpa="myhpa",condition="unknown",status="ScalingLimited"} 0 +# HELP kube_verticalpodautoscaler_labels Kubernetes labels converted to Prometheus labels. +# TYPE kube_verticalpodautoscaler_labels gauge +kube_verticalpodautoscaler_labels{label_app="foobar",namespace="ns1",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",verticalpodautoscaler="vpa1"} 1 +# HELP kube_verticalpodautoscaler_spec_updatepolicy_updatemode Update mode of the VPA. +# TYPE kube_verticalpodautoscaler_spec_updatepolicy_updatemode gauge +kube_verticalpodautoscaler_spec_updatepolicy_updatemode{namespace="default",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",update_mode="Recreate",verticalpodautoscaler="vpa1"} 1 +# HELP kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound Minimum resources the container can use before the VPA updater evicts it. +# TYPE kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound gauge +kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound{container="container1",namespace="default",resource="cpu",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 1 +# HELP kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound Maximum resources the container can use before the VPA updater evicts it. +# TYPE kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound gauge +kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound{container="container1",namespace="default",resource="cpu",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 4 +# HELP kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target Target resources the VPA recommends for the container. +# TYPE kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target gauge +kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target{container="container1",namespace="default",resource="cpu",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 3 +# HELP kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget Target resources the VPA recommends for the container ignoring bounds. +# TYPE kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget gauge +kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget{container="container1",namespace="default",resource="cpu",target_api_version="extensions/v1beta1",target_kind="Deployment",target_name="deployment1",unit="core",verticalpodautoscaler="vpa1"} 6 diff --git a/kubernetes_state/tests/test_kubernetes_state.py b/kubernetes_state/tests/test_kubernetes_state.py index d40019edaeda1..8701dd20d34ff 100644 --- a/kubernetes_state/tests/test_kubernetes_state.py +++ b/kubernetes_state/tests/test_kubernetes_state.py @@ -102,6 +102,12 @@ # jobs NAMESPACE + '.job.failed', NAMESPACE + '.job.succeeded', + # vpa + NAMESPACE + '.vpa.lower_bound', + NAMESPACE + '.vpa.target', + NAMESPACE + '.vpa.uncapped_target', + NAMESPACE + '.vpa.upperbound', + NAMESPACE + '.vpa.update_mode', ] TAGS = { @@ -394,11 +400,11 @@ def test_telemetry(aggregator, instance): for _ in range(2): check.check(instance) - aggregator.assert_metric(NAMESPACE + '.telemetry.payload.size', tags=['optional:tag1'], value=87416.0) - aggregator.assert_metric(NAMESPACE + '.telemetry.metrics.processed.count', tags=['optional:tag1'], value=956.0) - aggregator.assert_metric(NAMESPACE + '.telemetry.metrics.input.count', tags=['optional:tag1'], value=1270.0) + aggregator.assert_metric(NAMESPACE + '.telemetry.payload.size', tags=['optional:tag1'], value=90270.0) + aggregator.assert_metric(NAMESPACE + '.telemetry.metrics.processed.count', tags=['optional:tag1'], value=966.0) + aggregator.assert_metric(NAMESPACE + '.telemetry.metrics.input.count', tags=['optional:tag1'], value=1282.0) aggregator.assert_metric(NAMESPACE + '.telemetry.metrics.blacklist.count', tags=['optional:tag1'], value=24.0) - aggregator.assert_metric(NAMESPACE + '.telemetry.metrics.ignored.count', tags=['optional:tag1'], value=314.0) + aggregator.assert_metric(NAMESPACE + '.telemetry.metrics.ignored.count', tags=['optional:tag1'], value=316.0) aggregator.assert_metric( NAMESPACE + '.telemetry.collector.metrics.count', tags=['resource_name:pod', 'resource_namespace:default', 'optional:tag1'], From c13cfc977cea966b151ba319e8ca8e1d03dda814 Mon Sep 17 00:00:00 2001 From: Danny Turner Date: Wed, 21 Aug 2019 09:26:06 -0400 Subject: [PATCH 2/3] Apply suggestions from code review PR feedback Co-Authored-By: Ofek Lev --- .../kubernetes_state/kubernetes_state.py | 18 +++++++++++++----- kubernetes_state/metadata.csv | 10 +++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py b/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py index d4e458799bd5a..19be5db319a8a 100644 --- a/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py +++ b/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py @@ -194,10 +194,18 @@ def _create_kubernetes_state_prometheus_instance(self, instance): 'kube_statefulset_status_replicas_current': 'statefulset.replicas_current', 'kube_statefulset_status_replicas_ready': 'statefulset.replicas_ready', 'kube_statefulset_status_replicas_updated': 'statefulset.replicas_updated', - 'kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound' : 'vpa.lower_bound', - 'kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target' : 'vpa.target', - 'kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget' : 'vpa.uncapped_target', - 'kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound' : 'vpa.upperbound', + 'kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound': ( + 'vpa.lower_bound' + ), + 'kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target': ( + 'vpa.target' + ), + 'kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget': ( + 'vpa.uncapped_target' + ), + 'kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound': ( + 'vpa.upperbound' + ), 'kube_verticalpodautoscaler_spec_updatepolicy_updatemode' : 'vpa.update_mode', } ], @@ -262,7 +270,7 @@ def _create_kubernetes_state_prometheus_instance(self, instance): 'kube_job_status_active', 'kube_job_status_completion_time', # We could compute the duration=completion-start as a gauge 'kube_job_status_start_time', - 'kube_verticalpodautoscaler_labels' + 'kube_verticalpodautoscaler_labels', ], 'label_joins': { 'kube_pod_info': {'label_to_match': 'pod', 'labels_to_get': ['node']}, diff --git a/kubernetes_state/metadata.csv b/kubernetes_state/metadata.csv index 2b8012e793ed6..7ba76c1624412 100644 --- a/kubernetes_state/metadata.csv +++ b/kubernetes_state/metadata.csv @@ -104,8 +104,8 @@ kubernetes_state.telemetry.metrics.input.count,counter,,,,The number of metrics kubernetes_state.telemetry.metrics.blacklist.count,counter,,,,The number of metrics blacklisted by the check,0,kubernetes,k8s_state.telemetry.metrics.blacklist.count kubernetes_state.telemetry.metrics.ignored.count,counter,,,,The number of metrics ignored by the check,0,kubernetes,k8s_state.telemetry.metrics.ignored.count kubernetes_state.telemetry.collector.metrics.count,counter,,,,The number of metrics by collector (kubernetes object kind) by kubernetes namespaces,0,kubernetes,k8s_state.telemetry.collector.metrics.count -kubernetes_state.vpa.lower_bound,,,,The vpa lower bound recommendation,0,kubernetes,k8s_state.vpa.lower_bound -kubernetes_state.vpa.target,,,,The vpa target recommendation,0,kubernetes,k8s_state.vpa.target -kubernetes_state.vpa.uncapped_target,,,,The vpa uncapped recommendation recommendation,0,kubernetes,k8s_state.vpa.uncapped_target -kubernetes_state.vpa.upperbound,,,,The vpa upper bound recommendation,0,kubernetes,k8s_state.vpa.upper_bound -kubernetes_state.vpa.update_mode,,,,The vpa update mode,0,kubernetes,k8s_state.vpa.update_mode +kubernetes_state.vpa.lower_bound,gauge,,,,The vpa lower bound recommendation,0,kubernetes,k8s_state.vpa.lower_bound +kubernetes_state.vpa.target,gauge,,,,The vpa target recommendation,0,kubernetes,k8s_state.vpa.target +kubernetes_state.vpa.uncapped_target,gauge,,,,The vpa uncapped recommendation recommendation,0,kubernetes,k8s_state.vpa.uncapped_target +kubernetes_state.vpa.upperbound,gauge,,,,The vpa upper bound recommendation,0,kubernetes,k8s_state.vpa.upper_bound +kubernetes_state.vpa.update_mode,gauge,,,,The vpa update mode,0,kubernetes,k8s_state.vpa.update_mode From 999e2af0cff30a440ace82b2de8786028acaec93 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Wed, 21 Aug 2019 12:14:46 -0400 Subject: [PATCH 3/3] style --- .../datadog_checks/kubernetes_state/kubernetes_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py b/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py index 19be5db319a8a..257986e5ff8a1 100644 --- a/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py +++ b/kubernetes_state/datadog_checks/kubernetes_state/kubernetes_state.py @@ -206,7 +206,7 @@ def _create_kubernetes_state_prometheus_instance(self, instance): 'kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound': ( 'vpa.upperbound' ), - 'kube_verticalpodautoscaler_spec_updatepolicy_updatemode' : 'vpa.update_mode', + 'kube_verticalpodautoscaler_spec_updatepolicy_updatemode': 'vpa.update_mode', } ], 'ignore_metrics': [