Merge branch 'master' into AI-2904/kafka-consumer-revamp

LICENSE-3rdparty.csv

@@ -9,7 +9,7 @@ Pyro4,PyPI,MIT,Copyright (c) 2016 Irmen de Jong
 aerospike,PyPI,Apache-2.0,"Copyright Aerospike, Inc."
 aws-requests-auth,PyPI,BSD-3-Clause,Copyright (c) David Muller.
 beautifulsoup4,PyPI,MIT,Copyright (c) 2004-2017 Leonard Richardson
-beautifulsoup4,PyPI,MIT,Copyright (c) 2004-2022 Leonard Richardson
+beautifulsoup4,PyPI,MIT,Copyright (c) Leonard Richardson
 binary,PyPI,Apache-2.0,Copyright 2018 Ofek Lev
 binary,PyPI,MIT,Copyright 2018 Ofek Lev
 boto,PyPI,MIT,Copyright (c) 2010 Mitch Garnaat
@@ -41,7 +41,7 @@ immutables,PyPI,Apache-2.0,Copyright 2018-present Contributors to the immutables
 importlib-metadata,PyPI,Apache-2.0,"Copyright 2017-2019 Jason R. Coombs, Barry Warsaw"
 in-toto,PyPI,Apache-2.0,Copyright 2018 New York University
 ipaddress,PyPI,PSF,Copyright (c) 2013 Philipp Hagemeister
-jellyfish,PyPI,BSD-3-Clause,"Copyright (c) 2015, James Turk"
+jellyfish,PyPI,MIT,Copyright (c) 2015 James Turk
 kubernetes,PyPI,Apache-2.0,Copyright 2014 The Kubernetes Authors.
 ldap3,PyPI,LGPL-3.0-only,Copyright 2013 - 2020 Giovanni Cannata
 lxml,PyPI,BSD-3-Clause,Copyright (c) 2004 Infrae. All rights reserved.

amazon_msk/pyproject.toml

@@ -42,7 +42,7 @@ text = "BSD-3-Clause"
 [project.optional-dependencies]
 deps = [
     "boto3==1.17.112; python_version < '3.0'",
-    "boto3==1.26.51; python_version > '3.0'",
+    "boto3==1.26.111; python_version > '3.0'",
 ]
 
 [project.urls]

datadog_checks_base/datadog_checks/base/data/agent_requirements.in

@@ -1,16 +1,16 @@
 aerospike==4.0.0; sys_platform != 'win32' and sys_platform != 'darwin' and python_version < '3.0'
 aerospike==7.1.1; sys_platform != 'win32' and sys_platform != 'darwin' and python_version > '3.0'
 aws-requests-auth==0.4.3
-beautifulsoup4==4.11.1; python_version > '3.0'
+beautifulsoup4==4.12.2; python_version > '3.0'
 beautifulsoup4==4.9.3; python_version < '3.0'
 binary==1.0.0
 boto3==1.17.112; python_version < '3.0'
-boto3==1.26.51; python_version > '3.0'
+boto3==1.26.111; python_version > '3.0'
 boto==2.49.0
 botocore==1.20.112; python_version < '3.0'
-botocore==1.29.51; python_version > '3.0'
+botocore==1.29.111; python_version > '3.0'
 cachetools==3.1.1; python_version < '3.0'
-cachetools==5.2.1; python_version > '3.0'
+cachetools==5.3.0; python_version > '3.0'
 clickhouse-cityhash==1.0.2.3; python_version < '3.0'
 clickhouse-cityhash==1.0.2.4; python_version > '3.0'
 clickhouse-driver==0.2.0; python_version < '3.0'
@@ -32,7 +32,7 @@ importlib-metadata==2.1.3; python_version < '3.8'
 in-toto==1.0.1; python_version > '3.0'
 ipaddress==1.0.23; python_version < '3.0'
 jaydebeapi==1.2.3
-jellyfish==0.9.0; python_version > '3.0'
+jellyfish==0.11.2; python_version > '3.0'
 jpype1==1.4.1; python_version > '3.0'
 kubernetes==18.20.0; python_version < '3.0'
 kubernetes==26.1.0; python_version > '3.0'
@@ -41,17 +41,18 @@ lxml==4.9.2
 lz4==2.2.1; python_version < '3.0'
 lz4==3.1.3; python_version > '3.0'
 mmh3==2.5.1; python_version < '3.0'
-mmh3==3.0.0; python_version > '3.0'
+mmh3==3.1.0; python_version > '3.0'
 oauthlib==3.1.0; python_version < '3.0'
 oauthlib==3.2.2; python_version > '3.0'
 openstacksdk==1.0.1; python_version > '3.0'
-oracledb==1.2.1; python_version >= '3.8'
-orjson==3.8.5; python_version > '3.0'
+oracledb==1.3.0; python_version >= '3.8'
+orjson==3.8.10; python_version > '3.0'
 packaging==23.0; python_version > '3.0'
-paramiko==2.12.0
+paramiko==2.12.0; python_version < '3.0'
+paramiko==3.1.0; python_version > '3.0'
 ply==3.11
 prometheus-client==0.12.0; python_version < '3.0'
-prometheus-client==0.15.0; python_version > '3.0'
+prometheus-client==0.16.0; python_version > '3.0'
 protobuf==3.17.3; python_version < '3.0'
 protobuf==3.20.2; python_version > '3.0'
 psutil==5.9.0

datadog_checks_base/pyproject.toml

@@ -37,12 +37,12 @@ text = "BSD-3-Clause"
 [project.optional-dependencies]
 db = [
     "mmh3==2.5.1; python_version < '3.0'",
-    "mmh3==3.0.0; python_version > '3.0'",
+    "mmh3==3.1.0; python_version > '3.0'",
 ]
 deps = [
     "binary==1.0.0",
     "cachetools==3.1.1; python_version < '3.0'",
-    "cachetools==5.2.1; python_version > '3.0'",
+    "cachetools==5.3.0; python_version > '3.0'",
     "contextlib2==0.6.0.post1; python_version < '3.0'",
     "cryptography==3.3.2; python_version < '3.0'",
     "cryptography==39.0.1; python_version > '3.0'",
@@ -52,9 +52,9 @@ deps = [
     "immutables==0.19; python_version > '3.0'",
     "importlib-metadata==2.1.3; python_version < '3.8'",
     "ipaddress==1.0.23; python_version < '3.0'",
-    "jellyfish==0.9.0; python_version > '3.0'",
+    "jellyfish==0.11.2; python_version > '3.0'",
     "prometheus-client==0.12.0; python_version < '3.0'",
-    "prometheus-client==0.15.0; python_version > '3.0'",
+    "prometheus-client==0.16.0; python_version > '3.0'",
     "protobuf==3.17.3; python_version < '3.0'",
     "protobuf==3.20.2; python_version > '3.0'",
     "pydantic==1.10.4; python_version > '3.0'",
@@ -76,7 +76,7 @@ deps = [
 http = [
     "aws-requests-auth==0.4.3",
     "botocore==1.20.112; python_version < '3.0'",
-    "botocore==1.29.51; python_version > '3.0'",
+    "botocore==1.29.111; python_version > '3.0'",
     "oauthlib==3.1.0; python_version < '3.0'",
     "oauthlib==3.2.2; python_version > '3.0'",
     "pyjwt==1.7.1; python_version < '3.0'",
@@ -89,7 +89,7 @@ http = [
     "win-inet-pton==1.1.0; sys_platform == 'win32' and python_version < '3.0'",
 ]
 json = [
-    "orjson==3.8.5; python_version > '3.0'",
+    "orjson==3.8.10; python_version > '3.0'",
 ]
 kube = [
     "kubernetes==18.20.0; python_version < '3.0'",

datadog_cluster_agent/datadog_checks/datadog_cluster_agent/check.py

@@ -38,6 +38,8 @@
     'datadog_requests': 'datadog.requests',
     'endpoint_checks_configs_dispatched': 'endpoint_checks.configs_dispatched',
     'external_metrics': 'external_metrics',
+    'external_metrics_api_elapsed': 'external_metrics.api_elapsed',
+    'external_metrics_api_requests': 'external_metrics.api_requests',
     'external_metrics_datadog_metrics': 'external_metrics.datadog_metrics',
     'external_metrics_delay_seconds': 'external_metrics.delay_seconds',
     'external_metrics_processed_value': 'external_metrics.processed_value',

datadog_cluster_agent/metadata.csv

@@ -38,6 +38,9 @@ datadog.cluster_agent.datadog.rate_limit_queries.reset,gauge,,second,,Number of
 datadog.cluster_agent.datadog.requests,count,,request,,Requests made to Datadog by status,0,datadog_cluster_agent,datadog requests,
 datadog.cluster_agent.endpoint_checks.configs_dispatched,gauge,,,,Number of endpoint-check configurations dispatched by node,0,datadog_cluster_agent,endpoint check configs dispatched,
 datadog.cluster_agent.external_metrics,gauge,,,,Number of external metrics tagged,0,datadog_cluster_agent,external metrics,
+datadog.cluster_agent.external_metrics.api_elapsed.count,count,,,,Count of API Requests received,0,datadog_cluster_agent,external metrics api elapsed count,
+datadog.cluster_agent.external_metrics.api_elapsed.sum,count,,,,Count of API Requests received,0,datadog_cluster_agent,external metrics api elapsed sum,
+datadog.cluster_agent.external_metrics.api_requests,gauge,,,,Count of API Requests received,0,datadog_cluster_agent,external metrics api requests,
 datadog.cluster_agent.external_metrics.datadog_metrics,gauge,,,,"The label valid is true if the DatadogMetric CR is valid, false otherwise",0,datadog_cluster_agent,external metrics datadog metrics,
 datadog.cluster_agent.external_metrics.delay_seconds,gauge,,second,,Freshness of the metric evaluated from querying Datadog,0,datadog_cluster_agent,external metrics delay,
 datadog.cluster_agent.external_metrics.processed_value,gauge,,,,Value processed from querying Datadog by metric,0,datadog_cluster_agent,external metrics processed,

datadog_cluster_agent/tests/fixtures/metrics.txt

@@ -102,6 +102,40 @@ datadog_requests{join_leader="true",status="success"} 13006
 # HELP external_metrics Number of external metrics tagged.
 # TYPE external_metrics gauge
 external_metrics{join_leader="true"} 100
+# HELP external_metrics_api_elapsed Count of API Requests received
+# TYPE external_metrics_api_elapsed histogram
+external_metrics_api_elapsed_bucket{handler="get",in_error="true",namespace="datadog",le="0.005"} 0
+external_metrics_api_elapsed_bucket{handler="get",in_error="true",namespace="datadog",le="0.01"} 0
+external_metrics_api_elapsed_bucket{handler="get",in_error="true",namespace="datadog",le="0.025"} 0
+external_metrics_api_elapsed_bucket{handler="get",in_error="true",namespace="datadog",le="0.05"} 0
+external_metrics_api_elapsed_bucket{handler="get",in_error="true",namespace="datadog",le="0.1"} 0
+external_metrics_api_elapsed_bucket{handler="get",in_error="true",namespace="datadog",le="0.25"} 0
+external_metrics_api_elapsed_bucket{handler="get",in_error="true",namespace="datadog",le="0.5"} 0
+external_metrics_api_elapsed_bucket{handler="get",in_error="true",namespace="datadog",le="1"} 0
+external_metrics_api_elapsed_bucket{handler="get",in_error="true",namespace="datadog",le="2.5"} 0
+external_metrics_api_elapsed_bucket{handler="get",in_error="true",namespace="datadog",le="5"} 0
+external_metrics_api_elapsed_bucket{handler="get",in_error="true",namespace="datadog",le="10"} 0
+external_metrics_api_elapsed_bucket{handler="get",in_error="true",namespace="datadog",le="+Inf"} 24
+external_metrics_api_elapsed_sum{handler="get",in_error="true",namespace="datadog"} 1.696529e+07
+external_metrics_api_elapsed_count{handler="get",in_error="true",namespace="datadog"} 24
+external_metrics_api_elapsed_bucket{handler="list",in_error="false",namespace="",le="0.005"} 0
+external_metrics_api_elapsed_bucket{handler="list",in_error="false",namespace="",le="0.01"} 0
+external_metrics_api_elapsed_bucket{handler="list",in_error="false",namespace="",le="0.025"} 0
+external_metrics_api_elapsed_bucket{handler="list",in_error="false",namespace="",le="0.05"} 0
+external_metrics_api_elapsed_bucket{handler="list",in_error="false",namespace="",le="0.1"} 0
+external_metrics_api_elapsed_bucket{handler="list",in_error="false",namespace="",le="0.25"} 0
+external_metrics_api_elapsed_bucket{handler="list",in_error="false",namespace="",le="0.5"} 0
+external_metrics_api_elapsed_bucket{handler="list",in_error="false",namespace="",le="1"} 0
+external_metrics_api_elapsed_bucket{handler="list",in_error="false",namespace="",le="2.5"} 0
+external_metrics_api_elapsed_bucket{handler="list",in_error="false",namespace="",le="5"} 0
+external_metrics_api_elapsed_bucket{handler="list",in_error="false",namespace="",le="10"} 0
+external_metrics_api_elapsed_bucket{handler="list",in_error="false",namespace="",le="+Inf"} 97
+external_metrics_api_elapsed_sum{handler="list",in_error="false",namespace=""} 4.8232774e+07
+external_metrics_api_elapsed_count{handler="list",in_error="false",namespace=""} 97
+# HELP external_metrics_api_requests Count of API Requests received
+# TYPE external_metrics_api_requests gauge
+external_metrics_api_requests{handler="get",in_error="true",namespace="datadog"} 24
+external_metrics_api_requests{handler="list",in_error="false",namespace=""} 97
 # HELP external_metrics_datadog_metrics The label valid is true if the DatadogMetric CR is valid, false otherwise
 # TYPE external_metrics_datadog_metrics gauge
 external_metrics_datadog_metrics{join_leader="true",name="dcaautogen-776576e7e4e1d57e7df8d30d63fdabe2c60edc",namespace="default",valid="false"} 1

datadog_cluster_agent/tests/test_datadog_cluster_agent.py

@@ -52,6 +52,9 @@
     'datadog.requests',
     'endpoint_checks.configs_dispatched',
     'external_metrics',
+    'external_metrics.api_elapsed.count',
+    'external_metrics.api_elapsed.sum',
+    'external_metrics.api_requests',
     'external_metrics.datadog_metrics',
     'external_metrics.delay_seconds',
     'external_metrics.processed_value',

ecs_fargate/README.md

@@ -41,6 +41,9 @@ The instructions below show you how to configure the task using the [Amazon Web
 <!-- xxx tab "Web UI" xxx -->
 ##### Web UI Task Definition
 
+<!-- partial
+{{< site-region region="us,us3,us5,eu,ap1,gov" >}}
+
 1. Log in to your [AWS Web Console][4] and navigate to the ECS section.
 2. Click on **Task Definitions** in the left menu, then click the **Create new Task Definition** button or choose an existing Fargate task definition.
 3. For new task definitions:
@@ -58,13 +61,26 @@ The instructions below show you how to configure the task using the [Amazon Web
 5. Add your other application containers to the task definition. For details on collecting integration metrics, see [Integration Setup for ECS Fargate][12].
 6. Click **Create** to create the task definition.
 
+[4]: https://aws.amazon.com/console
+[12]: http://docs.datadoghq.com/integrations/faq/integration-setup-ecs-fargate
+[41]: https://app.datadoghq.com/organization-settings/api-keys
+
+{{< /site-region >}}
+partial -->
+
 <!-- xxz tab xxx -->
 
 <!-- xxx tab "AWS CLI" xxx -->
 ##### AWS CLI Task Definition
 
 1. Download [datadog-agent-ecs-fargate.json][42]. **Note**: If you are using Internet Explorer, this may download as gzip file, which contains the JSON file mentioned below.**
+<!-- partial
+{{< site-region region="us,us3,us5,eu,ap1,gov" >}}
 2. Update the JSON with a `TASK_NAME`, your [Datadog API Key][41], and the appropriate `DD_SITE` ({{< region-param key="dd_site" code="true" >}}). **Note**: The environment variable `ECS_FARGATE` is already set to `"true"`.
+
+[41]: https://app.datadoghq.com/organization-settings/api-keys
+{{< /site-region >}}
+partial -->
 3. Add your other application containers to the task definition. For details on collecting integration metrics, see [Integration Setup for ECS Fargate][12].
 4. Optionally - Add an Agent health check.
 
@@ -92,8 +108,14 @@ aws ecs register-task-definition --cli-input-json file://<PATH_TO_FILE>/datadog-
 
 You can use [AWS CloudFormation][6] templating to configure your Fargate containers. Use the `AWS::ECS::TaskDefinition` resource within your CloudFormation template to set the Amazon ECS task and specify `FARGATE` as the required launch type for that task.
 
+<!-- partial
+{{< site-region region="us,us3,us5,eu,ap1,gov" >}}
 Update this CloudFormation template below with your [Datadog API Key][41]. As well as include the appropriate `DD_SITE` ({{< region-param key="dd_site" code="true" >}}) environment variable if necessary, as this defaults to `datadoghq.com` if you don't set it.
 
+[41]: https://app.datadoghq.com/organization-settings/api-keys
+{{< /site-region >}}
+partial -->
+
 ```yaml
 Resources:
   ECSTaskDefinition:
@@ -434,7 +456,14 @@ partial -->
   ```
 {{< /site-region >}}
 partial -->
-  **Note**: Set your `apikey` as well as the `Host` relative to your respective site `http-intake.logs.{{< region-param key="dd_site" code="true" >}}`. The full list of available parameters is described in the [Datadog Fluent Bit documentation][24].
+
+<!-- partial
+{{< site-region region="us,us3,us5,eu,ap1,gov" >}}
+**Note**: Set your `apikey` as well as the `Host` relative to your respective site `http-intake.logs.`{{< region-param key="dd_site" code="true" >}}. The full list of available parameters is described in the [Datadog Fluent Bit documentation][24].
+
+[24]: https://docs.datadoghq.com/integrations/fluentbit/#configuration-parameters
+{{< /site-region >}}
+partial -->
 
   The `dd_service`, `dd_source`, and `dd_tags` can be adjusted for your desired tags.
 
@@ -690,7 +719,11 @@ Monitor Fargate logs by using the `awslogs` log driver and a Lambda function to
 
 ### Trace collection
 
+<!-- partial
+{{< site-region region="us,us3,us5,eu,ap1,gov" >}}
 1. Follow the [instructions above](#installation) to add the Datadog Agent container to your task definition with the additional environment variable `DD_APM_ENABLED` set to `true`. Set the `DD_SITE` variable to {{< region-param key="dd_site" code="true" >}}. It defaults to `datadoghq.com` if you don't set it.
+{{< /site-region >}}
+partial -->
 
 2. [Instrument your application][32] based on your setup. With Fargate APM applications do **not** set `DD_AGENT_HOST`, the default of `localhost` works.
 

mysql/pyproject.toml

@@ -42,7 +42,7 @@ text = "BSD-3-Clause"
 [project.optional-dependencies]
 deps = [
     "cachetools==3.1.1; python_version < '3.0'",
-    "cachetools==5.2.1; python_version > '3.0'",
+    "cachetools==5.3.0; python_version > '3.0'",
     "cryptography==3.3.2; python_version < '3.0'",
     "cryptography==39.0.1; python_version > '3.0'",
     "futures==3.4.0; python_version < '3.0'",

oracle/pyproject.toml

@@ -42,10 +42,10 @@ text = "BSD-3-Clause"
 deps = [
     "jaydebeapi==1.2.3",
     "jpype1==1.4.1; python_version > '3.0'",
-    "oracledb==1.2.1; python_version >= '3.8'",
+    "oracledb==1.3.0; python_version >= '3.8'",
 ]
 client-only = [
-    "oracledb==1.2.1; python_version >= '3.8'",
+    "oracledb==1.3.0; python_version >= '3.8'",
 ]
 
 [project.urls]

postgres/pyproject.toml

@@ -42,7 +42,7 @@ text = "BSD-3-Clause"
 [project.optional-dependencies]
 deps = [
     "cachetools==3.1.1; python_version < '3.0'",
-    "cachetools==5.2.1; python_version > '3.0'",
+    "cachetools==5.3.0; python_version > '3.0'",
     "futures==3.4.0; python_version < '3.0'",
     "psycopg2-binary==2.8.6; sys_platform != 'darwin' or platform_machine != 'arm64'",
     "semver==2.13.0",

spark/pyproject.toml

@@ -10,7 +10,6 @@ build-backend = "hatchling.build"
 name = "datadog-spark"
 description = "The Spark check"
 readme = "README.md"
-license = {text = "BSD-3-Clause"}
 keywords = [
     "datadog",
     "datadog agent",
@@ -37,9 +36,12 @@ dynamic = [
     "version",
 ]
 
+[project.license]
+text = "BSD-3-Clause"
+
 [project.optional-dependencies]
 deps = [
-    "beautifulsoup4==4.11.1; python_version > '3.0'",
+    "beautifulsoup4==4.12.2; python_version > '3.0'",
     "beautifulsoup4==4.9.3; python_version < '3.0'",
 ]
 

ssh_check/pyproject.toml

@@ -41,7 +41,8 @@ text = "BSD-3-Clause"
 
 [project.optional-dependencies]
 deps = [
-    "paramiko==2.12.0",
+    "paramiko==2.12.0; python_version < '3.0'",
+    "paramiko==3.1.0; python_version > '3.0'",
 ]
 
 [project.urls]