Support dynamic bearer tokens (Bound Service Account Token Volume)

datadog_checks_base/datadog_checks/base/checks/openmetrics/mixins.py

@@ -4,6 +4,7 @@
 from __future__ import division
 
 import copy
+import time
 from fnmatch import translate
 from math import isinf, isnan
 from os.path import isfile
@@ -366,6 +367,13 @@ def _get_setting(name, default):
 
         # The service account bearer token to be used for authentication
         config['_bearer_token'] = self._get_bearer_token(config['bearer_token_auth'], config['bearer_token_path'])
+        config['_bearer_token_last_refresh'] = time.time()
+
+        # Refresh the bearer token every 60 seconds by default.
+        # Ref https://github.com/DataDog/datadog-agent/pull/11686
+        config['bearer_token_refresh_interval'] = instance.get(
+            'bearer_token_refresh_interval', default_instance.get('bearer_token_refresh_interval', 60)
+        )
 
         config['telemetry'] = is_affirmative(instance.get('telemetry', default_instance.get('telemetry', False)))
 
@@ -396,9 +404,11 @@ def get_http_handler(self, scraper_config):
         """
         Get http handler for a specific scraper config.
         The http handler is cached using `prometheus_url` as key.
+        The http handler doesn't use the cache if a bearer token is used to allow refreshing it.
         """
         prometheus_url = scraper_config['prometheus_url']
-        if prometheus_url in self._http_handlers:
+        bearer_token = scraper_config['_bearer_token']
+        if prometheus_url in self._http_handlers and bearer_token is None:
             return self._http_handlers[prometheus_url]
 
         # TODO: Deprecate this behavior in Agent 8
@@ -557,6 +567,9 @@ def process(self, scraper_config, metric_transformers=None):
         if not scraper_config['_flush_first_value'] and scraper_config['use_process_start_time']:
             agent_start_time = datadog_agent.get_process_start_time()
 
+        if scraper_config['bearer_token_auth']:
+            self._refresh_bearer_token(scraper_config)
+
         for metric in self.scrape_metrics(scraper_config):
             if agent_start_time is not None:
                 if metric.name == 'process_start_time_seconds' and metric.samples:
@@ -1192,6 +1205,17 @@ def _get_bearer_token(self, bearer_token_auth, bearer_token_path):
             self.log.error("Cannot get bearer token from path: %s - error: %s", path, err)
             raise
 
+    def _refresh_bearer_token(self, scraper_config):
+        """
+        Refreshes the bearer token if the refresh interval is elapsed.
+        """
+        now = time.time()
+        if now - scraper_config['_bearer_token_last_refresh'] > scraper_config['bearer_token_refresh_interval']:
+            scraper_config['_bearer_token'] = self._get_bearer_token(
+                scraper_config['bearer_token_auth'], scraper_config['bearer_token_path']
+            )
+            scraper_config['_bearer_token_last_refresh'] = now
+
     def _histogram_convert_values(self, metric_name, converter):
         def _convert(metric, scraper_config=None):
             for index, sample in enumerate(metric.samples):

datadog_checks_base/tests/base/checks/openmetrics/test_openmetrics.py

@@ -9,10 +9,12 @@
 import math
 import os
 import re
+import time
 
 import mock
 import pytest
 import requests
+from mock import patch
 from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily, HistogramMetricFamily, SummaryMetricFamily
 from prometheus_client.samples import Sample
 from six import iteritems
@@ -24,6 +26,7 @@
 
 text_content_type = 'text/plain; version=0.0.4'
 FIXTURE_PATH = os.path.abspath(os.path.join(get_here(), '..', '..', '..', 'fixtures', 'prometheus'))
+TOKENS_PATH = os.path.abspath(os.path.join(get_here(), '..', '..', '..', 'fixtures', 'bearer_tokens'))
 
 FAKE_ENDPOINT = 'http://fake.endpoint:10055/metrics'
 
@@ -2988,3 +2991,23 @@ def test_use_process_start_time(
             )
 
         expect_first_flush = True
+
+
+def test_refresh_bearer_token(text_data, mocked_openmetrics_check_factory):
+    instance = {
+        'prometheus_url': 'https://fake.endpoint:10055/metrics',
+        'metrics': [{'process_virtual_memory_bytes': 'process.vm.bytes'}],
+        "namespace": "default_namespace",
+        "bearer_token_auth": True,
+        "bearer_token_refresh_interval": 1,
+    }
+
+    with patch.object(OpenMetricsBaseCheck, 'KUBERNETES_TOKEN_PATH', os.path.join(TOKENS_PATH, 'default_token')):
+        check = mocked_openmetrics_check_factory(instance)
+        check.poll = mock.MagicMock(return_value=MockResponse(text_data, headers={'Content-Type': text_content_type}))
+        instance = check.get_scraper_config(instance)
+        assert instance['_bearer_token'] == 'my default token'
+        time.sleep(1.5)
+        with patch.object(OpenMetricsBaseCheck, 'KUBERNETES_TOKEN_PATH', os.path.join(TOKENS_PATH, 'custom_token')):
+            check.check(instance)
+            assert instance['_bearer_token'] == 'my custom token'

datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/instances/openmetrics_legacy_base.yaml

@@ -161,6 +161,12 @@
   value:
     example: <TOKEN_PATH>
     type: string
+- name: bearer_token_refresh_interval
+  description: |
+    The refresh interval to keep the bearer token up-to-date.
+  value:
+    example: 60
+    type: integer
 - name: ignore_metrics
   description: |
     A list of metrics to ignore, the "*" wildcard can be used to match multiple metric names.