Merge branch 'master' into vivek-datadog/AGENT-10479

datadog_checks_base/changelog.d/16089.fixed

@@ -0,0 +1 @@
+Add the ability to assert the `external_tags` when running tests

datadog_checks_base/datadog_checks/base/stubs/datadog_agent.py

@@ -22,6 +22,7 @@ def __init__(self):
         self._config = self.get_default_config()
         self._hostname = 'stubbed.hostname'
         self._process_start_time = 0
+        self._external_tags = []
 
     def get_default_config(self):
         return {'enable_metadata_collection': True, 'disable_unsafe_yaml': True}
@@ -31,6 +32,7 @@ def reset(self):
         self._cache.clear()
         self._config = self.get_default_config()
         self._process_start_time = 0
+        self._external_tags = []
 
     def assert_metadata(self, check_id, data):
         actual = {}
@@ -46,6 +48,24 @@ def assert_metadata_count(self, count):
             count, metadata_items, repr(self._metadata)
         )
 
+    def assert_external_tags(self, hostname, external_tags):
+        for h, tags in self._external_tags:
+            if h == hostname:
+                assert (
+                    external_tags == tags
+                ), 'Expected {} external tags for hostname {}, found {}. Submitted external tags: {}'.format(
+                    external_tags, hostname, tags, repr(self._external_tags)
+                )
+                return
+
+        raise AssertionError('Hostname {} not found in external tags {}'.format(hostname, repr(self._external_tags)))
+
+    def assert_external_tags_count(self, count):
+        tags_count = len(self._external_tags)
+        assert tags_count == count, 'Expected {} external tags items, found {}. Submitted external tags: {}'.format(
+            count, tags_count, repr(self._external_tags)
+        )
+
     def get_hostname(self):
         return self._hostname
 
@@ -67,8 +87,8 @@ def log(self, *args, **kwargs):
     def set_check_metadata(self, check_id, name, value):
         self._metadata[(check_id, name)] = value
 
-    def set_external_tags(self, *args, **kwargs):
-        pass
+    def set_external_tags(self, external_tags):
+        self._external_tags = external_tags
 
     def tracemalloc_enabled(self, *args, **kwargs):
         return False

datadog_checks_base/tests/base/checks/test_agent_check.py

@@ -13,7 +13,6 @@
 
 from datadog_checks.base import AgentCheck, to_native_string
 from datadog_checks.base import __version__ as base_package_version
-from datadog_checks.base.checks.base import datadog_agent
 from datadog_checks.dev.testing import requires_py3
 
 
@@ -623,15 +622,17 @@ def test_external_host_tag_normalization(self):
             check.set_external_tags(external_host_tags)
             assert external_host_tags == [('hostname', {'src_name': ['normalize:tag']})]
 
-    def test_external_hostname(self):
+    def test_external_hostname(self, datadog_agent):
         check = AgentCheck()
         external_host_tags = [(u'hostnam\xe9', {'src_name': ['key1:val1']})]
-        with mock.patch.object(datadog_agent, 'set_external_tags') as set_external_tags:
-            check.set_external_tags(external_host_tags)
-            if PY3:
-                set_external_tags.assert_called_with([(u'hostnam\xe9', {'src_name': ['key1:val1']})])
-            else:
-                set_external_tags.assert_called_with([('hostnam\xc3\xa9', {'src_name': ['key1:val1']})])
+        check.set_external_tags(external_host_tags)
+
+        if PY3:
+            datadog_agent.assert_external_tags(u'hostnam\xe9', {'src_name': ['key1:val1']})
+        else:
+            datadog_agent.assert_external_tags('hostnam\xc3\xa9', {'src_name': ['key1:val1']})
+
+        datadog_agent.assert_external_tags_count(1)
 
     @pytest.mark.parametrize(
         "disable_generic_tags, expected_tags",

datadog_checks_base/tests/stubs/test_datadog_agent.py

@@ -0,0 +1,49 @@
+# (C) Datadog, Inc. 2023-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import pytest
+
+DEFAULT_EXTERNAL_TAGS = [
+    ('hostname1', {'src1_name': ['test1:t1']}),
+    ('hostname2', {'src2_name': ['test2:t2']}),
+    ('hostname3', {'src3_name': ['test3:t3']}),
+]
+
+
+@pytest.mark.parametrize(
+    'hostname, tags, raise_exception',
+    [
+        pytest.param('hostname1', {'src1_name': ['test1:t1']}, False, id="hostname1 and tags found"),
+        pytest.param('hostname2', {'src2_name': ['test2:t2']}, False, id="hostname2 and tags found"),
+        pytest.param('hostname3', {'src3_name': ['test3:t3']}, False, id="hostname3 and tags found"),
+        pytest.param('hostname4', {'src4_name': ['test4:t4']}, True, id="hostname4 and tags not found"),
+        pytest.param('hostname1', {'src2_name': ['test2:t2']}, True, id="hostname1 found and tags are wrong"),
+    ],
+)
+def test_assert_external_tags(datadog_agent, hostname, tags, raise_exception):
+    datadog_agent.set_external_tags(DEFAULT_EXTERNAL_TAGS)
+
+    try:
+        datadog_agent.assert_external_tags(hostname, tags)
+    except AssertionError:
+        if not raise_exception:
+            raise
+
+
+@pytest.mark.parametrize(
+    'external_tags, count, raise_exception',
+    [
+        pytest.param(DEFAULT_EXTERNAL_TAGS, 3, False, id="correct count"),
+        pytest.param([], 0, False, id="no tags"),
+        pytest.param([('hostname1', {'src1_name': ['test1:t1']})], 1, False, id="one tag"),
+        pytest.param([('hostname1', {'src1_name': ['test1:t1']})], 2, True, id="wrong count"),
+    ],
+)
+def test_assert_external_tags_count(datadog_agent, external_tags, count, raise_exception):
+    datadog_agent.set_external_tags(external_tags)
+
+    try:
+        datadog_agent.assert_external_tags_count(count)
+    except AssertionError:
+        if not raise_exception:
+            raise

kafka/assets/dashboards/kafka_dashboard.json

@@ -1064,12 +1064,12 @@
                                         {
                                             "data_source": "metrics",
                                             "name": "query1",
-                                            "query": "sum:kafka.consumer.lag{$datacenter,$consumer_group} by {host,consumer_group}"
+                                            "query": "sum:kafka.consumer_lag{$datacenter,$consumer_group} by {host,consumer_group}"
                                         },
                                         {
                                             "data_source": "metrics",
                                             "name": "query2",
-                                            "query": "sum:kafka.consumer.offset{$datacenter,$consumer_group} by {host,consumer_group}"
+                                            "query": "sum:kafka.consumer_offset{$datacenter,$consumer_group} by {host,consumer_group}"
                                         }
                                     ],
                                     "response_format": "timeseries",

kafka_consumer/CHANGELOG.md

@@ -8,10 +8,6 @@
 
 * Add ability to cache offsets and close admin client ([#15960](https://github.com/DataDog/integrations-core/pull/15960))
 
-***Fixed***:
-
-* Attempt client connection before fetching consumer offsets ([#15951](https://github.com/DataDog/integrations-core/pull/15951))
-
 ## 4.1.2 / 2023-09-04 / Agent 7.48.0
 
 ***Fixed***:

kafka_consumer/changelog.d/15951.fixed

@@ -0,0 +1 @@
+Attempt client connection before fetching consumer offsets