diff --git a/.deps/metadata.json b/.deps/metadata.json index 47a3797824701..e55b164ab2c41 100644 --- a/.deps/metadata.json +++ b/.deps/metadata.json @@ -1,3 +1,3 @@ { - "sha256": "5983d33689b7f9b74eb94d7ff13b7eac3c0b2f8cd5084e8ded76c28ca5476a20" + "sha256": "4d5260e77bea1467dc82ebe382d0cbc42d3f9b62c64c36898f975393c42da56d" } diff --git a/.deps/resolved/linux-aarch64_3.12.txt b/.deps/resolved/linux-aarch64_3.12.txt index d3abef8c55543..0f6b6df2b96c2 100644 --- a/.deps/resolved/linux-aarch64_3.12.txt +++ b/.deps/resolved/linux-aarch64_3.12.txt @@ -7,7 +7,7 @@ azure-core @ https://agent-int-packages.datadoghq.com/external/azure-core/azure_ azure-identity @ https://agent-int-packages.datadoghq.com/external/azure-identity/azure_identity-1.17.1-py3-none-any.whl#sha256=db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382 bcrypt @ https://agent-int-packages.datadoghq.com/external/bcrypt/bcrypt-4.2.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=909faa1027900f2252a9ca5dfebd25fc0ef1417943824783d1c8418dd7d6df4a beautifulsoup4 @ https://agent-int-packages.datadoghq.com/external/beautifulsoup4/beautifulsoup4-4.12.3-py3-none-any.whl#sha256=b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed -binary @ https://agent-int-packages.datadoghq.com/external/binary/binary-1.0.0-py2.py3-none-any.whl#sha256=e1b61f3a5c002717d1a28e4d9d2dc8acbc9d6b12baf7b1e4ab25d743da97e323 +binary @ https://agent-int-packages.datadoghq.com/external/binary/binary-1.0.1-py3-none-any.whl#sha256=e92086be2a7204dbbdf86b55d86bd27bf4c24089db866113a90811b492241544 boto3 @ https://agent-int-packages.datadoghq.com/external/boto3/boto3-1.35.10-py3-none-any.whl#sha256=add26dd58e076dfd387013da4704716d5cff215cf14f6d4347c4b9b7fc1f0b8e botocore @ https://agent-int-packages.datadoghq.com/external/botocore/botocore-1.35.10-py3-none-any.whl#sha256=0d96d023b9b0cea99a0a428a431d011329d3a958730aee6ed6a6fec5d9bfbc03 bytecode @ https://agent-int-packages.datadoghq.com/external/bytecode/bytecode-0.16.0-py3-none-any.whl#sha256=76080b7c0eb9e7e17f961d61fd06e933aa47f3b753770a3249537439d8203a25 @@ -41,7 +41,7 @@ jellyfish @ https://agent-int-packages.datadoghq.com/external/jellyfish/jellyfis jmespath @ https://agent-int-packages.datadoghq.com/external/jmespath/jmespath-1.0.1-py3-none-any.whl#sha256=02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 jsonpatch @ https://agent-int-packages.datadoghq.com/external/jsonpatch/jsonpatch-1.33-py2.py3-none-any.whl#sha256=0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade jsonpointer @ https://agent-int-packages.datadoghq.com/external/jsonpointer/jsonpointer-3.0.0-py2.py3-none-any.whl#sha256=13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 -keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.9.0-py3-none-any.whl#sha256=c9c50ebfd012ead27bfa69b7ec093c740296dd0532e37d96a070c21cd049ece5 +keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.9.1-py3-none-any.whl#sha256=71b98835aec72a01f71c5b919c3193dac95342555e89aa35c86d3d86c4ff5f73 krb5 @ https://agent-int-packages.datadoghq.com/built/krb5/krb5-0.7.0-20241016152407-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl#sha256=37ecc257e0e35459f2438a4a8511fffae0c8056ba5a648609d9b007902806ecc kubernetes @ https://agent-int-packages.datadoghq.com/external/kubernetes/kubernetes-30.1.0-py2.py3-none-any.whl#sha256=e212e8b7579031dd2e512168b617373bc1e03888d41ac4e04039240a292d478d ldap3 @ https://agent-int-packages.datadoghq.com/external/ldap3/ldap3-2.9.1-py2.py3-none-any.whl#sha256=5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70 @@ -109,7 +109,7 @@ semver @ https://agent-int-packages.datadoghq.com/external/semver/semver-3.0.2-p service-identity @ https://agent-int-packages.datadoghq.com/external/service-identity/service_identity-24.1.0-py3-none-any.whl#sha256=a28caf8130c8a5c1c7a6f5293faaf239bbfb7751e4862436920ee6f2616f568a setuptools @ https://agent-int-packages.datadoghq.com/external/setuptools/setuptools-75.6.0-py3-none-any.whl#sha256=ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d simplejson @ https://agent-int-packages.datadoghq.com/external/simplejson/simplejson-3.19.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=d0b0efc7279d768db7c74d3d07f0b5c81280d16ae3fb14e9081dc903e8360771 -six @ https://agent-int-packages.datadoghq.com/external/six/six-1.16.0-py2.py3-none-any.whl#sha256=8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 +six @ https://agent-int-packages.datadoghq.com/external/six/six-1.17.0-py2.py3-none-any.whl#sha256=4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 snowflake-connector-python @ https://agent-int-packages.datadoghq.com/external/snowflake-connector-python/snowflake_connector_python-3.12.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=2215d8a4c5e25ea0d2183fe693c3fdf058cd6035e5c84710d532dc04ab4ffd31 sortedcontainers @ https://agent-int-packages.datadoghq.com/external/sortedcontainers/sortedcontainers-2.4.0-py2.py3-none-any.whl#sha256=a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.6-py3-none-any.whl#sha256=e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9 diff --git a/.deps/resolved/linux-x86_64_3.12.txt b/.deps/resolved/linux-x86_64_3.12.txt index e2f5751b85764..823a373300dcc 100644 --- a/.deps/resolved/linux-x86_64_3.12.txt +++ b/.deps/resolved/linux-x86_64_3.12.txt @@ -7,7 +7,7 @@ azure-core @ https://agent-int-packages.datadoghq.com/external/azure-core/azure_ azure-identity @ https://agent-int-packages.datadoghq.com/external/azure-identity/azure_identity-1.17.1-py3-none-any.whl#sha256=db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382 bcrypt @ https://agent-int-packages.datadoghq.com/external/bcrypt/bcrypt-4.2.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=cde78d385d5e93ece5479a0a87f73cd6fa26b171c786a884f955e165032b262c beautifulsoup4 @ https://agent-int-packages.datadoghq.com/external/beautifulsoup4/beautifulsoup4-4.12.3-py3-none-any.whl#sha256=b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed -binary @ https://agent-int-packages.datadoghq.com/external/binary/binary-1.0.0-py2.py3-none-any.whl#sha256=e1b61f3a5c002717d1a28e4d9d2dc8acbc9d6b12baf7b1e4ab25d743da97e323 +binary @ https://agent-int-packages.datadoghq.com/external/binary/binary-1.0.1-py3-none-any.whl#sha256=e92086be2a7204dbbdf86b55d86bd27bf4c24089db866113a90811b492241544 boto3 @ https://agent-int-packages.datadoghq.com/external/boto3/boto3-1.35.10-py3-none-any.whl#sha256=add26dd58e076dfd387013da4704716d5cff215cf14f6d4347c4b9b7fc1f0b8e botocore @ https://agent-int-packages.datadoghq.com/external/botocore/botocore-1.35.10-py3-none-any.whl#sha256=0d96d023b9b0cea99a0a428a431d011329d3a958730aee6ed6a6fec5d9bfbc03 bytecode @ https://agent-int-packages.datadoghq.com/external/bytecode/bytecode-0.16.0-py3-none-any.whl#sha256=76080b7c0eb9e7e17f961d61fd06e933aa47f3b753770a3249537439d8203a25 @@ -41,7 +41,7 @@ jellyfish @ https://agent-int-packages.datadoghq.com/external/jellyfish/jellyfis jmespath @ https://agent-int-packages.datadoghq.com/external/jmespath/jmespath-1.0.1-py3-none-any.whl#sha256=02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 jsonpatch @ https://agent-int-packages.datadoghq.com/external/jsonpatch/jsonpatch-1.33-py2.py3-none-any.whl#sha256=0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade jsonpointer @ https://agent-int-packages.datadoghq.com/external/jsonpointer/jsonpointer-3.0.0-py2.py3-none-any.whl#sha256=13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 -keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.9.0-py3-none-any.whl#sha256=c9c50ebfd012ead27bfa69b7ec093c740296dd0532e37d96a070c21cd049ece5 +keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.9.1-py3-none-any.whl#sha256=71b98835aec72a01f71c5b919c3193dac95342555e89aa35c86d3d86c4ff5f73 krb5 @ https://agent-int-packages.datadoghq.com/built/krb5/krb5-0.7.0-20241016152358-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl#sha256=eaf622a9e9e87645228c4b44a51d4096223862bfdde8f62d0b422651802843b2 kubernetes @ https://agent-int-packages.datadoghq.com/external/kubernetes/kubernetes-30.1.0-py2.py3-none-any.whl#sha256=e212e8b7579031dd2e512168b617373bc1e03888d41ac4e04039240a292d478d ldap3 @ https://agent-int-packages.datadoghq.com/external/ldap3/ldap3-2.9.1-py2.py3-none-any.whl#sha256=5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70 @@ -110,7 +110,7 @@ semver @ https://agent-int-packages.datadoghq.com/external/semver/semver-3.0.2-p service-identity @ https://agent-int-packages.datadoghq.com/external/service-identity/service_identity-24.1.0-py3-none-any.whl#sha256=a28caf8130c8a5c1c7a6f5293faaf239bbfb7751e4862436920ee6f2616f568a setuptools @ https://agent-int-packages.datadoghq.com/external/setuptools/setuptools-75.6.0-py3-none-any.whl#sha256=ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d simplejson @ https://agent-int-packages.datadoghq.com/external/simplejson/simplejson-3.19.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=7017329ca8d4dca94ad5e59f496e5fc77630aecfc39df381ffc1d37fb6b25832 -six @ https://agent-int-packages.datadoghq.com/external/six/six-1.16.0-py2.py3-none-any.whl#sha256=8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 +six @ https://agent-int-packages.datadoghq.com/external/six/six-1.17.0-py2.py3-none-any.whl#sha256=4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 snowflake-connector-python @ https://agent-int-packages.datadoghq.com/external/snowflake-connector-python/snowflake_connector_python-3.12.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f8ba9c261904c1ba7cae6035c7881224cf979da39c8b7c7cb10236fdfc57e505 sortedcontainers @ https://agent-int-packages.datadoghq.com/external/sortedcontainers/sortedcontainers-2.4.0-py2.py3-none-any.whl#sha256=a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.6-py3-none-any.whl#sha256=e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9 diff --git a/.deps/resolved/macos-x86_64_3.12.txt b/.deps/resolved/macos-x86_64_3.12.txt index 12512a7f1efce..734802d8f7240 100644 --- a/.deps/resolved/macos-x86_64_3.12.txt +++ b/.deps/resolved/macos-x86_64_3.12.txt @@ -6,7 +6,7 @@ azure-core @ https://agent-int-packages.datadoghq.com/external/azure-core/azure_ azure-identity @ https://agent-int-packages.datadoghq.com/external/azure-identity/azure_identity-1.17.1-py3-none-any.whl#sha256=db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382 bcrypt @ https://agent-int-packages.datadoghq.com/external/bcrypt/bcrypt-4.2.1-cp39-abi3-macosx_10_12_universal2.whl#sha256=8ad2f4528cbf0febe80e5a3a57d7a74e6635e41af1ea5675282a33d769fba413 beautifulsoup4 @ https://agent-int-packages.datadoghq.com/external/beautifulsoup4/beautifulsoup4-4.12.3-py3-none-any.whl#sha256=b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed -binary @ https://agent-int-packages.datadoghq.com/external/binary/binary-1.0.0-py2.py3-none-any.whl#sha256=e1b61f3a5c002717d1a28e4d9d2dc8acbc9d6b12baf7b1e4ab25d743da97e323 +binary @ https://agent-int-packages.datadoghq.com/external/binary/binary-1.0.1-py3-none-any.whl#sha256=e92086be2a7204dbbdf86b55d86bd27bf4c24089db866113a90811b492241544 boto3 @ https://agent-int-packages.datadoghq.com/external/boto3/boto3-1.35.10-py3-none-any.whl#sha256=add26dd58e076dfd387013da4704716d5cff215cf14f6d4347c4b9b7fc1f0b8e botocore @ https://agent-int-packages.datadoghq.com/external/botocore/botocore-1.35.10-py3-none-any.whl#sha256=0d96d023b9b0cea99a0a428a431d011329d3a958730aee6ed6a6fec5d9bfbc03 bytecode @ https://agent-int-packages.datadoghq.com/external/bytecode/bytecode-0.16.0-py3-none-any.whl#sha256=76080b7c0eb9e7e17f961d61fd06e933aa47f3b753770a3249537439d8203a25 @@ -18,7 +18,7 @@ charset-normalizer @ https://agent-int-packages.datadoghq.com/external/charset-n clickhouse-cityhash @ https://agent-int-packages.datadoghq.com/external/clickhouse-cityhash/clickhouse_cityhash-1.0.2.4-cp312-cp312-macosx_10_9_x86_64.whl#sha256=261fc1b0bf349de66b2d9e3d367879a561b516ca8e54e85e0c27b7c1a4f639b4 clickhouse-driver @ https://agent-int-packages.datadoghq.com/external/clickhouse-driver/clickhouse_driver-0.2.9-cp312-cp312-macosx_10_9_x86_64.whl#sha256=fcb2fd00e58650ae206a6d5dbc83117240e622471aa5124733fbf2805eb8bda0 cm-client @ https://agent-int-packages.datadoghq.com/built/cm-client/cm_client-45.0.4-20240402154932-py3-none-macosx_10_12_universal2.whl#sha256=aba3c1683ef1b2099933e030464d29b3ad1c206784ebd15d8a7147ecd6ba24e1 -confluent-kafka @ https://agent-int-packages.datadoghq.com/built/confluent-kafka/confluent_kafka-2.6.1-20241121135346-cp312-cp312-macosx_10_13_universal2.whl#sha256=dc75e64b667dcabb1f55efcc8e8da052874f26d1ed0dff747d914ee7ba21be0b +confluent-kafka @ https://agent-int-packages.datadoghq.com/built/confluent-kafka/confluent_kafka-2.6.1-20241205195042-cp312-cp312-macosx_10_13_universal2.whl#sha256=380f7e6798592b4ebbcb5697cd3b6c0aa15e2d5d48af70396461780a6b7e854e cryptography @ https://agent-int-packages.datadoghq.com/external/cryptography/cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl#sha256=ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d ddsketch @ https://agent-int-packages.datadoghq.com/external/ddsketch/ddsketch-3.0.1-py3-none-any.whl#sha256=6d047b455fe2837c43d366ff1ae6ba0c3166e15499de8688437a75cea914224e ddtrace @ https://agent-int-packages.datadoghq.com/external/ddtrace/ddtrace-2.10.6-cp312-cp312-macosx_12_0_x86_64.whl#sha256=401f77b0564c3f990b58b9f21055331ca9efcdfa06dfa6ccff13cf21f8329ba5 @@ -40,7 +40,7 @@ jellyfish @ https://agent-int-packages.datadoghq.com/external/jellyfish/jellyfis jmespath @ https://agent-int-packages.datadoghq.com/external/jmespath/jmespath-1.0.1-py3-none-any.whl#sha256=02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 jsonpatch @ https://agent-int-packages.datadoghq.com/external/jsonpatch/jsonpatch-1.33-py2.py3-none-any.whl#sha256=0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade jsonpointer @ https://agent-int-packages.datadoghq.com/external/jsonpointer/jsonpointer-3.0.0-py2.py3-none-any.whl#sha256=13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 -keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.9.0-py3-none-any.whl#sha256=c9c50ebfd012ead27bfa69b7ec093c740296dd0532e37d96a070c21cd049ece5 +keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.9.1-py3-none-any.whl#sha256=71b98835aec72a01f71c5b919c3193dac95342555e89aa35c86d3d86c4ff5f73 krb5 @ https://agent-int-packages.datadoghq.com/external/krb5/krb5-0.7.0-cp312-cp312-macosx_10_13_x86_64.whl#sha256=fa4ea45629e585787c0bcc455c7fbed7e09176031a7f9e7c87b9deaad401da36 kubernetes @ https://agent-int-packages.datadoghq.com/external/kubernetes/kubernetes-30.1.0-py2.py3-none-any.whl#sha256=e212e8b7579031dd2e512168b617373bc1e03888d41ac4e04039240a292d478d ldap3 @ https://agent-int-packages.datadoghq.com/external/ldap3/ldap3-2.9.1-py2.py3-none-any.whl#sha256=5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70 @@ -50,7 +50,7 @@ lz4 @ https://agent-int-packages.datadoghq.com/external/lz4/lz4-4.3.3-cp312-cp31 mmh3 @ https://agent-int-packages.datadoghq.com/external/mmh3/mmh3-4.1.0-cp312-cp312-macosx_10_9_x86_64.whl#sha256=d6af3e2287644b2b08b5924ed3a88c97b87b44ad08e79ca9f93d3470a54a41c5 msal @ https://agent-int-packages.datadoghq.com/external/msal/msal-1.31.1-py3-none-any.whl#sha256=29d9882de247e96db01386496d59f29035e5e841bcac892e6d7bf4390bf6bd17 msal-extensions @ https://agent-int-packages.datadoghq.com/external/msal-extensions/msal_extensions-1.2.0-py3-none-any.whl#sha256=cf5ba83a2113fa6dc011a254a72f1c223c88d7dfad74cc30617c4679a417704d -netifaces @ https://agent-int-packages.datadoghq.com/built/netifaces/netifaces-0.11.0-20241121135347-cp312-cp312-macosx_10_13_universal2.whl#sha256=7b9f6a6b674af53dcb262ab7342c1a39e82901d1be5a27b5cdf6b8d945574714 +netifaces @ https://agent-int-packages.datadoghq.com/built/netifaces/netifaces-0.11.0-20241205195042-cp312-cp312-macosx_10_13_universal2.whl#sha256=66a155ae114ae885a4a15604cd39e93e12c7dc024132de958e10ced6f375856a oauthlib @ https://agent-int-packages.datadoghq.com/external/oauthlib/oauthlib-3.2.2-py3-none-any.whl#sha256=8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca openstacksdk @ https://agent-int-packages.datadoghq.com/external/openstacksdk/openstacksdk-3.3.0-py3-none-any.whl#sha256=e6d4121b87354984caf0e3c032e2ebf4d4440374f86c81c27ec52ca5df359157 opentelemetry-api @ https://agent-int-packages.datadoghq.com/external/opentelemetry-api/opentelemetry_api-1.28.2-py3-none-any.whl#sha256=6fcec89e265beb258fe6b1acaaa3c8c705a934bd977b9f534a2b7c0d2d4275a6 @@ -76,7 +76,7 @@ pydantic @ https://agent-int-packages.datadoghq.com/external/pydantic/pydantic-2 pydantic-core @ https://agent-int-packages.datadoghq.com/external/pydantic-core/pydantic_core-2.20.1-cp312-cp312-macosx_10_12_x86_64.whl#sha256=595ba5be69b35777474fa07f80fc260ea71255656191adb22a8c53aba4479231 pyjwt @ https://agent-int-packages.datadoghq.com/external/pyjwt/PyJWT-2.9.0-py3-none-any.whl#sha256=3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850 pymongo @ https://agent-int-packages.datadoghq.com/external/pymongo/pymongo-4.8.0-cp312-cp312-macosx_10_9_x86_64.whl#sha256=e6a720a3d22b54183352dc65f08cd1547204d263e0651b213a0a2e577e838526 -pymqi @ https://agent-int-packages.datadoghq.com/built/pymqi/pymqi-1.12.10-20241121135348-cp312-cp312-macosx_10_13_universal2.whl#sha256=d55a9f66d74c2c9407dd3d7d5515dd17806bdc799812f5e122c3cb5d42f15e5b +pymqi @ https://agent-int-packages.datadoghq.com/built/pymqi/pymqi-1.12.10-20241205195043-cp312-cp312-macosx_10_13_universal2.whl#sha256=b436d180ff1d3ffa5094a610721038aa678155038597351179b19ddeb28f507d pymysql @ https://agent-int-packages.datadoghq.com/external/pymysql/PyMySQL-1.1.1-py3-none-any.whl#sha256=4de15da4c61dc132f4fb9ab763063e693d521a80fd0e87943b9a453dd4c19d6c pynacl @ https://agent-int-packages.datadoghq.com/external/pynacl/PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl#sha256=401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1 pyodbc @ https://agent-int-packages.datadoghq.com/external/pyodbc/pyodbc-5.1.0-cp312-cp312-macosx_10_9_x86_64.whl#sha256=d3d9cc4af703c4817b6e604315910b0cf5dcb68056d52b25ca072dd59c52dcbc @@ -109,7 +109,7 @@ semver @ https://agent-int-packages.datadoghq.com/external/semver/semver-3.0.2-p service-identity @ https://agent-int-packages.datadoghq.com/external/service-identity/service_identity-24.1.0-py3-none-any.whl#sha256=a28caf8130c8a5c1c7a6f5293faaf239bbfb7751e4862436920ee6f2616f568a setuptools @ https://agent-int-packages.datadoghq.com/external/setuptools/setuptools-75.6.0-py3-none-any.whl#sha256=ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d simplejson @ https://agent-int-packages.datadoghq.com/external/simplejson/simplejson-3.19.3-cp312-cp312-macosx_10_9_x86_64.whl#sha256=6ef9383c5e05f445be60f1735c1816163c874c0b1ede8bb4390aff2ced34f333 -six @ https://agent-int-packages.datadoghq.com/external/six/six-1.16.0-py2.py3-none-any.whl#sha256=8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 +six @ https://agent-int-packages.datadoghq.com/external/six/six-1.17.0-py2.py3-none-any.whl#sha256=4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 snowflake-connector-python @ https://agent-int-packages.datadoghq.com/external/snowflake-connector-python/snowflake_connector_python-3.12.3-cp312-cp312-macosx_11_0_x86_64.whl#sha256=597b0c74ec57ba693191ae2de8db9536e349ee32cab152df657473e498b6fd87 sortedcontainers @ https://agent-int-packages.datadoghq.com/external/sortedcontainers/sortedcontainers-2.4.0-py2.py3-none-any.whl#sha256=a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.6-py3-none-any.whl#sha256=e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9 @@ -120,7 +120,7 @@ tuf @ https://agent-int-packages.datadoghq.com/external/tuf/tuf-4.0.0-py3-none-a typing-extensions @ https://agent-int-packages.datadoghq.com/external/typing-extensions/typing_extensions-4.12.2-py3-none-any.whl#sha256=04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d tzlocal @ https://agent-int-packages.datadoghq.com/external/tzlocal/tzlocal-5.2-py3-none-any.whl#sha256=49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8 uhashring @ https://agent-int-packages.datadoghq.com/external/uhashring/uhashring-2.3-py3-none-any.whl#sha256=7ee8a25ca495a97effad10bd563c83b4054a6d7606d9530757049a04edab9297 -uptime @ https://agent-int-packages.datadoghq.com/built/uptime/uptime-3.0.1-20241121135348-cp312-cp312-macosx_10_13_universal2.whl#sha256=69d9ba890e3049ac5c817f73da679197e2fe2d2212000d5e290ab13ddbfec1e7 +uptime @ https://agent-int-packages.datadoghq.com/built/uptime/uptime-3.0.1-20241205195044-cp312-cp312-macosx_10_13_universal2.whl#sha256=63083f38ee611a28940cfa670df6a45df4d2d8b1960640c5480febac2f326134 urllib3 @ https://agent-int-packages.datadoghq.com/external/urllib3/urllib3-2.2.3-py3-none-any.whl#sha256=ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac vertica-python @ https://agent-int-packages.datadoghq.com/external/vertica-python/vertica_python-1.4.0-py3-none-any.whl#sha256=50fecd7687f4b0b9f6dee6e2b35c195af2a4f702ece01bd12e080b51756e000b websocket-client @ https://agent-int-packages.datadoghq.com/external/websocket-client/websocket_client-1.8.0-py3-none-any.whl#sha256=17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526 diff --git a/.deps/resolved/windows-x86_64_3.12.txt b/.deps/resolved/windows-x86_64_3.12.txt index a7eb5213c4cd8..67094404c3e2f 100644 --- a/.deps/resolved/windows-x86_64_3.12.txt +++ b/.deps/resolved/windows-x86_64_3.12.txt @@ -6,7 +6,7 @@ azure-core @ https://agent-int-packages.datadoghq.com/external/azure-core/azure_ azure-identity @ https://agent-int-packages.datadoghq.com/external/azure-identity/azure_identity-1.17.1-py3-none-any.whl#sha256=db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382 bcrypt @ https://agent-int-packages.datadoghq.com/external/bcrypt/bcrypt-4.2.1-cp39-abi3-win_amd64.whl#sha256=e84e0e6f8e40a242b11bce56c313edc2be121cec3e0ec2d76fce01f6af33c07c beautifulsoup4 @ https://agent-int-packages.datadoghq.com/external/beautifulsoup4/beautifulsoup4-4.12.3-py3-none-any.whl#sha256=b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed -binary @ https://agent-int-packages.datadoghq.com/external/binary/binary-1.0.0-py2.py3-none-any.whl#sha256=e1b61f3a5c002717d1a28e4d9d2dc8acbc9d6b12baf7b1e4ab25d743da97e323 +binary @ https://agent-int-packages.datadoghq.com/external/binary/binary-1.0.1-py3-none-any.whl#sha256=e92086be2a7204dbbdf86b55d86bd27bf4c24089db866113a90811b492241544 boto3 @ https://agent-int-packages.datadoghq.com/external/boto3/boto3-1.35.10-py3-none-any.whl#sha256=add26dd58e076dfd387013da4704716d5cff215cf14f6d4347c4b9b7fc1f0b8e botocore @ https://agent-int-packages.datadoghq.com/external/botocore/botocore-1.35.10-py3-none-any.whl#sha256=0d96d023b9b0cea99a0a428a431d011329d3a958730aee6ed6a6fec5d9bfbc03 bytecode @ https://agent-int-packages.datadoghq.com/external/bytecode/bytecode-0.16.0-py3-none-any.whl#sha256=76080b7c0eb9e7e17f961d61fd06e933aa47f3b753770a3249537439d8203a25 @@ -39,7 +39,7 @@ jellyfish @ https://agent-int-packages.datadoghq.com/external/jellyfish/jellyfis jmespath @ https://agent-int-packages.datadoghq.com/external/jmespath/jmespath-1.0.1-py3-none-any.whl#sha256=02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 jsonpatch @ https://agent-int-packages.datadoghq.com/external/jsonpatch/jsonpatch-1.33-py2.py3-none-any.whl#sha256=0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade jsonpointer @ https://agent-int-packages.datadoghq.com/external/jsonpointer/jsonpointer-3.0.0-py2.py3-none-any.whl#sha256=13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 -keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.9.0-py3-none-any.whl#sha256=c9c50ebfd012ead27bfa69b7ec093c740296dd0532e37d96a070c21cd049ece5 +keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.9.1-py3-none-any.whl#sha256=71b98835aec72a01f71c5b919c3193dac95342555e89aa35c86d3d86c4ff5f73 kubernetes @ https://agent-int-packages.datadoghq.com/external/kubernetes/kubernetes-30.1.0-py2.py3-none-any.whl#sha256=e212e8b7579031dd2e512168b617373bc1e03888d41ac4e04039240a292d478d ldap3 @ https://agent-int-packages.datadoghq.com/external/ldap3/ldap3-2.9.1-py2.py3-none-any.whl#sha256=5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70 looseversion @ https://agent-int-packages.datadoghq.com/external/looseversion/looseversion-1.3.0-py2.py3-none-any.whl#sha256=781ef477b45946fc03dd4c84ea87734b21137ecda0e1e122bcb3c8d16d2a56e0 @@ -106,7 +106,7 @@ semver @ https://agent-int-packages.datadoghq.com/external/semver/semver-3.0.2-p service-identity @ https://agent-int-packages.datadoghq.com/external/service-identity/service_identity-24.1.0-py3-none-any.whl#sha256=a28caf8130c8a5c1c7a6f5293faaf239bbfb7751e4862436920ee6f2616f568a setuptools @ https://agent-int-packages.datadoghq.com/external/setuptools/setuptools-75.6.0-py3-none-any.whl#sha256=ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d simplejson @ https://agent-int-packages.datadoghq.com/external/simplejson/simplejson-3.19.3-cp312-cp312-win_amd64.whl#sha256=1e662336db50ad665777e6548b5076329a94a0c3d4a0472971c588b3ef27de3a -six @ https://agent-int-packages.datadoghq.com/external/six/six-1.16.0-py2.py3-none-any.whl#sha256=8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 +six @ https://agent-int-packages.datadoghq.com/external/six/six-1.17.0-py2.py3-none-any.whl#sha256=4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 snowflake-connector-python @ https://agent-int-packages.datadoghq.com/external/snowflake-connector-python/snowflake_connector_python-3.12.3-cp312-cp312-win_amd64.whl#sha256=f0d0fcb948ef0812ab162ec9767622f345554043a07439c0c1a9474c86772320 sortedcontainers @ https://agent-int-packages.datadoghq.com/external/sortedcontainers/sortedcontainers-2.4.0-py2.py3-none-any.whl#sha256=a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.6-py3-none-any.whl#sha256=e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f443fae488fa8..c873cf7de3a98 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -84,7 +84,7 @@ manifest.json @DataDog/documentation @DataDog/agent-integrations /kubernetes/*.md @DataDog/container-integrations @DataDog/agent-integrations @DataDog/documentation /kubernetes/manifest.json @DataDog/container-integrations @DataDog/agent-integrations @DataDog/documentation /kubernetes_admission/ @DataDog/container-platform @DataDog/agent-integrations -/kubernetes_cluster_autoscaler/ @DataDog/container-integrations @DataDog/agent-integrations +/kubernetes_cluster_autoscaler/ @DataDog/container-integrations @DataDog/agent-integrations /kubernetes_cluster_autoscaler/*.md @DataDog/container-integrations @DataDog/agent-integrations @DataDog/documentation /kubernetes_cluster_autoscaler/manifest.json @DataDog/container-integrations @DataDog/agent-integrations @DataDog/documentation /kubernetes_state/ @DataDog/container-integrations @DataDog/agent-integrations @@ -266,6 +266,12 @@ datadog_checks_base/datadog_checks/base/checks/windows/ @DataDog/wi /freshservice/manifest.json @DataDog/saas-integrations @DataDog/documentation /freshservice/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend @DataDog/logs-core +/genesys/ @DataDog/saas-integrations +/genesys/*.md @DataDog/saas-integrations @DataDog/documentation +/genesys/manifest.json @DataDog/saas-integrations @DataDog/documentation +/genesys/metadata.csv @DataDog/saas-integrations @DataDog/documentation +/genesys/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend + /godaddy/ @DataDog/saas-integrations /godaddy/*.md @DataDog/saas-integrations @DataDog/documentation /godaddy/manifest.json @DataDog/saas-integrations @DataDog/documentation @@ -295,13 +301,18 @@ datadog_checks_base/datadog_checks/base/checks/windows/ @DataDog/wi /mimecast/manifest.json @DataDog/saas-integrations @DataDog/documentation /mimecast/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend @DataDog/logs-core +/mux/ @DataDog/saas-integrations +/mux/*.md @DataDog/saas-integrations @DataDog/documentation +/mux/manifest.json @DataDog/saas-integrations @DataDog/documentation +/mux/metadata.csv @DataDog/saas-integrations @DataDog/documentation + /palo_alto_cortex_xdr/ @DataDog/saas-integrations /palo_alto_cortex_xdr/*.md @DataDog/saas-integrations @DataDog/documentation /palo_alto_cortex_xdr/manifest.json @DataDog/saas-integrations @DataDog/documentation /palo_alto_cortex_xdr/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend @DataDog/logs-core /snowflake/ @DataDog/saas-integrations -/snowflake/*.md @DataDog/saas-integrations @DataDog/documentation +/snowflake/*.md @DataDog/saas-integrations @DataDog/documentation @DataDog/agent-integrations /snowflake/manifest.json @DataDog/saas-integrations @DataDog/documentation /sonicwall_firewall/ @DataDog/saas-integrations @@ -345,6 +356,11 @@ datadog_checks_base/datadog_checks/base/checks/windows/ @DataDog/wi /ringcentral/metadata.csv @DataDog/saas-integrations @DataDog/documentation /ringcentral/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend @DataDog/logs-core +/temporal_cloud/ @DataDog/saas-integrations +/temporal_cloud/*.md @DataDog/saas-integrations @DataDog/documentation +/temporal_cloud/manifest.json @DataDog/saas-integrations @DataDog/documentation +/temporal_cloud/metadata.csv @DataDog/saas-integrations @DataDog/documentation + /trend_micro_email_security/ @DataDog/saas-integrations /trend_micro_email_security/*.md @DataDog/saas-integrations @DataDog/documentation /trend_micro_email_security/manifest.json @DataDog/saas-integrations @DataDog/documentation @@ -365,17 +381,36 @@ datadog_checks_base/datadog_checks/base/checks/windows/ @DataDog/wi /trend_micro_vision_one_endpoint_security/manifest.json @DataDog/saas-integrations @DataDog/documentation /trend_micro_vision_one_endpoint_security/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend @DataDog/logs-core +/shopify/ @DataDog/saas-integrations +/shopify/*.md @DataDog/saas-integrations @DataDog/documentation +/shopify/manifest.json @DataDog/saas-integrations @DataDog/documentation +/shopify/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend + +/shopify/ @DataDog/saas-integrations +/shopify/*.md @DataDog/saas-integrations @DataDog/documentation +/shopify/manifest.json @DataDog/saas-integrations @DataDog/documentation +/shopify/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend + /hubspot_content_hub/ @DataDog/saas-integrations /hubspot_content_hub/*.md @DataDog/saas-integrations @DataDog/documentation /hubspot_content_hub/manifest.json @DataDog/saas-integrations @DataDog/documentation /hubspot_content_hub/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend @DataDog/logs-core - vonage/ @DataDog/saas-integrations vonage/*.md @DataDog/saas-integrations @DataDog/documentation vonage/manifest.json @DataDog/saas-integrations @DataDog/documentation vonage/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend @DataDog/logs-core +plaid/ @DataDog/saas-integrations +plaid/*.md @DataDog/saas-integrations @DataDog/documentation +plaid/manifest.json @DataDog/saas-integrations @DataDog/documentation +plaid/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend + +/streamnative/ @DataDog/saas-integrations +/streamnative/*.md @DataDog/saas-integrations @DataDog/documentation +/streamnative/manifest.json @DataDog/saas-integrations @DataDog/documentation +/streamnative/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend + # To keep Security up-to-date with changes to the signing tool. /datadog_checks_dev/datadog_checks/dev/tooling/signing.py @DataDog/agent-integrations # As well as the secure downloader. diff --git a/.github/workflows/build-ddev.yml b/.github/workflows/build-ddev.yml index 8f0e541b64b28..7c60d83ca619b 100644 --- a/.github/workflows/build-ddev.yml +++ b/.github/workflows/build-ddev.yml @@ -80,9 +80,9 @@ jobs: os: windows-2022 # macOS - target: aarch64-apple-darwin - os: macos-12 + os: macos-13 - target: x86_64-apple-darwin - os: macos-12 + os: macos-13 outputs: version: ${{ steps.version.outputs.version }} @@ -361,7 +361,7 @@ jobs: name: Build macOS installer and sign/notarize artifacts if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository needs: binaries - runs-on: macos-12 + runs-on: macos-13 env: VERSION: ${{ needs.binaries.outputs.version }} diff --git a/.github/workflows/build-deps.yml b/.github/workflows/build-deps.yml index 4d15e622e2d46..caa202a4090ad 100644 --- a/.github/workflows/build-deps.yml +++ b/.github/workflows/build-deps.yml @@ -196,7 +196,7 @@ jobs: build-macos: name: Target macOS - runs-on: macos-12 + runs-on: macos-13 env: TARGET_NAME: macos-x86_64 diff --git a/.github/workflows/config/labeler.yml b/.github/workflows/config/labeler.yml index 2ebb3d4f96817..3993f7b14ec1a 100644 --- a/.github/workflows/config/labeler.yml +++ b/.github/workflows/config/labeler.yml @@ -201,6 +201,8 @@ integration/freshservice: - freshservice/**/* integration/gearmand: - gearmand/**/* +integration/genesys: +- genesys/**/* integration/gitlab: - gitlab/**/* integration/gitlab_runner: @@ -349,6 +351,8 @@ integration/mimecast: - mimecast/**/* integration/mongo: - mongo/**/* +integration/mux: +- mux/**/* integration/mysql: - mysql/**/* integration/nagios: @@ -411,6 +415,8 @@ integration/ping_one: - ping_one/**/* integration/pivotal_pks: - pivotal_pks/**/* +integration/plaid: +- plaid/**/* integration/podman: - podman/**/* integration/postfix: @@ -447,6 +453,8 @@ integration/sap_hana: - sap_hana/**/* integration/scylla: - scylla/**/* +integration/shopify: +- shopify/**/* integration/sidekiq: - sidekiq/**/* integration/silk: @@ -501,6 +509,8 @@ integration/ssh_check: - ssh_check/**/* integration/statsd: - statsd/**/* +integration/streamnative: +- streamnative/**/* integration/strimzi: - strimzi/**/* integration/supervisord: @@ -527,6 +537,8 @@ integration/teleport: - teleport/**/* integration/temporal: - temporal/**/* +integration/temporal_cloud: +- temporal_cloud/**/* integration/tenable: - tenable/**/* integration/teradata: diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml index 98956f28222c9..fc96df3402bae 100644 --- a/.github/workflows/update-dependencies.yml +++ b/.github/workflows/update-dependencies.yml @@ -61,16 +61,10 @@ jobs: ### What does this PR do? Update the dependencies - ### Motivation - - Some of the dependencies are outdated - - ### Additional Notes - - This PR was automatically generated by the following workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + **THE CHANGELOG FILES OFTEN HAVE THE WRONG PR NUMBER. MAKE SURE TO CHECK THEM!** ### Review checklist (to be filled by reviewers) diff --git a/aerospike/changelog.d/18996.fixed b/aerospike/changelog.d/18996.fixed new file mode 100644 index 0000000000000..f46ba036b3612 --- /dev/null +++ b/aerospike/changelog.d/18996.fixed @@ -0,0 +1 @@ +Don't skip last index in each namespace diff --git a/aerospike/datadog_checks/aerospike/aerospike.py b/aerospike/datadog_checks/aerospike/aerospike.py index f52814865f090..cec0c918ffdd8 100644 --- a/aerospike/datadog_checks/aerospike/aerospike.py +++ b/aerospike/datadog_checks/aerospike/aerospike.py @@ -152,7 +152,7 @@ def check(self, _): # https://www.aerospike.com/docs/reference/info/#sindex sindex = self.get_info('sindex/{}'.format(ns)) - for idx in parse_namespace(sindex[:-1], ns, 'indexname'): + for idx in parse_namespace(sindex, ns, 'indexname'): sindex_tags = ['sindex:{}'.format(idx)] sindex_tags.extend(namespace_tags) self.collect_info('sindex/{}/{}'.format(ns, idx), SINDEX_METRIC_TYPE, tags=sindex_tags) diff --git a/aerospike/tests/common.py b/aerospike/tests/common.py index c0dd957dbabda..a42f9db0b18cd 100644 --- a/aerospike/tests/common.py +++ b/aerospike/tests/common.py @@ -104,6 +104,32 @@ ALL_METRICS = NAMESPACE_METRICS + LEGACY_SET_METRICS +INDEXES_METRICS = [ + "aerospike.sindex.delete_error", + "aerospike.sindex.delete_success", + "aerospike.sindex.entries", + "aerospike.sindex.histogram", + "aerospike.sindex.ibtr_memory_used", + "aerospike.sindex.keys", + "aerospike.sindex.load_pct", + "aerospike.sindex.loadtime", + "aerospike.sindex.nbtr_memory_used", + "aerospike.sindex.query_agg", + "aerospike.sindex.query_agg_avg_rec_count", + "aerospike.sindex.query_agg_avg_record_size", + "aerospike.sindex.query_avg_rec_count", + "aerospike.sindex.query_avg_record_size", + "aerospike.sindex.query_lookup_avg_rec_count", + "aerospike.sindex.query_lookup_avg_record_size", + "aerospike.sindex.query_lookups", + "aerospike.sindex.query_reqs", + "aerospike.sindex.si_accounted_memory", + "aerospike.sindex.stat_gc_recs", + "aerospike.sindex.stat_gc_time", + "aerospike.sindex.write_error", + "aerospike.sindex.write_success", +] + STATS_METRICS = [ 'cluster_size', 'batch_index_initiate', @@ -155,6 +181,32 @@ 'tags': ['tag:value'], } +MOCK_INDEXES_METRICS = [ + "keys=1", + "entries=1", + "ibtr_memory_used=18688", + "nbtr_memory_used=31", + "si_accounted_memory=18719", + "load_pct=100", + "loadtime=7", + "write_success=1", + "write_error=0", + "delete_success=0", + "delete_error=0", + "stat_gc_recs=0", + "stat_gc_time=0", + "query_reqs=0", + "query_avg_rec_count=0", + "query_avg_record_size=0", + "query_agg=0", + "query_agg_avg_rec_count=0", + "query_agg_avg_record_size=0", + "query_lookups=0", + "query_lookup_avg_rec_count=0", + "query_lookup_avg_record_size=0", + "histogram=false", +] + MOCK_DATACENTER_METRICS = [ 'dc_state=CLUSTER_UP', 'dc_timelag=0', diff --git a/aerospike/tests/conftest.py b/aerospike/tests/conftest.py index 01e5448a97da3..0f3b9c170191a 100644 --- a/aerospike/tests/conftest.py +++ b/aerospike/tests/conftest.py @@ -39,6 +39,8 @@ def init_db(): 'quote_cnt': 47, } client.put(key, bins) + # Create at an index + client.index_string_create('test', 'characters', 'name', 'idx_characters_name') batch_keys = [] for i in range(10): diff --git a/aerospike/tests/test_aerospike.py b/aerospike/tests/test_aerospike.py index 4914b3396a577..66646789ad770 100644 --- a/aerospike/tests/test_aerospike.py +++ b/aerospike/tests/test_aerospike.py @@ -13,6 +13,7 @@ from .common import ( EXPECTED_PROMETHEUS_METRICS, EXPECTED_PROMETHEUS_METRICS_5_6, + INDEXES_METRICS, LATENCIES_METRICS, LAZY_METRICS, LEGACY_SET_METRICS, @@ -38,7 +39,6 @@ def test_check(aggregator, instance, dd_run_check): @pytest.mark.integration def test_version_metadata(aggregator, instance, datadog_agent, dd_run_check): - check = AerospikeCheck('aerospike', {}, [instance]) check.check_id = 'test:123' @@ -129,6 +129,9 @@ def _test_check(aggregator): for metric in LEGACY_SET_METRICS: aggregator.assert_metric("aerospike.set.{}".format(metric)) + for metric in INDEXES_METRICS: + aggregator.assert_metric(metric) + aggregator.assert_all_metrics_covered() aggregator.assert_service_check('aerospike.can_connect', AerospikeCheck.OK) diff --git a/aerospike/tests/test_unit.py b/aerospike/tests/test_unit.py index e7339566dc03f..681e9b8e0b0a1 100644 --- a/aerospike/tests/test_unit.py +++ b/aerospike/tests/test_unit.py @@ -52,6 +52,36 @@ def test_xdr_metrics(aggregator): aggregator.assert_metric(metric, tags=['datacenter:test']) +def test_sindex_metrics(aggregator, dd_run_check): + check = AerospikeCheck('aerospike', {}, [common.INSTANCE]) + original_get_info = check.get_info + + def mock_get_info(command, separator=";"): + if command == "sindex/test": + return [ + "ns=test:indexname=idx_characters_name:set=characters:bin=name:type=string:indextype=default:context=null:state=RW" + ] + elif command == "sindex/test/idx_characters_name": + return common.MOCK_INDEXES_METRICS + elif command.startswith("sets/"): + return [] + return original_get_info(command, separator) + + check.get_info = mock_get_info + check._tags = [] + check._client = mock.MagicMock() + check._client.get_node_names = mock.MagicMock( + return_value={'address': common.HOST, 'port': common.PORT, 'node_name': 'test'} + ) + check.get_namespaces = mock.MagicMock(return_value=['test']) + check.collect_throughput = mock.MagicMock() + check.collect_latency = mock.MagicMock() + dd_run_check(check) + + for metric in common.INDEXES_METRICS: + aggregator.assert_metric(metric, tags=['namespace:test', 'sindex:idx_characters_name']) + + def test_multiple_xdr_metrics(aggregator): check = AerospikeCheck('aerospike', {}, [common.INSTANCE]) check.get_info = mock.MagicMock( diff --git a/agent_requirements.in b/agent_requirements.in index 17b6d28051615..b96253e8f081f 100644 --- a/agent_requirements.in +++ b/agent_requirements.in @@ -2,7 +2,7 @@ aerospike==7.1.1; sys_platform != 'win32' and sys_platform != 'darwin' aws-requests-auth==0.4.3 azure-identity==1.17.1 beautifulsoup4==4.12.3 -binary==1.0.0 +binary==1.0.1 boto3==1.35.10 botocore==1.35.10 cachetools==5.5.0 diff --git a/airflow/README.md b/airflow/README.md index 2407dfc7be1c0..94faf19dab2cc 100644 --- a/airflow/README.md +++ b/airflow/README.md @@ -429,7 +429,7 @@ Need help? Contact [Datadog support][11]. [10]: https://docs.datadoghq.com/agent/guide/agent-commands/?tab=agentv6#start-stop-and-restart-the-agent [11]: https://docs.datadoghq.com/help/ [12]: https://docs.datadoghq.com/developers/dogstatsd/?tab=kubernetes#setup -[13]: /integrations/airflow/?tab=host#connect-airflow-to-dogstatsd +[13]: https://docs.datadoghq.com/integrations/airflow/?tab=host#connect-airflow-to-dogstatsd [14]: https://docs.datadoghq.com/agent/kubernetes/integrations/?tab=kubernetes#configuration [15]: https://docs.datadoghq.com/agent/guide/agent-commands/?tab=agentv6#agent-status-and-information [16]: https://airflow.apache.org/docs/apache-airflow-providers-datadog/stable/_modules/airflow/providers/datadog/hooks/datadog.html diff --git a/amazon_msk/CHANGELOG.md b/amazon_msk/CHANGELOG.md index 262bdbdd5c747..4973f3d68464a 100644 --- a/amazon_msk/CHANGELOG.md +++ b/amazon_msk/CHANGELOG.md @@ -32,7 +32,7 @@ ***Added***: -* Update dependencies ([#18185](https://github.com/DataDog/integrations-core/pull/18185)) +* Update dependencies ([#18187](https://github.com/DataDog/integrations-core/pull/18187)) ## 4.9.0 / 2024-07-05 / Agent 7.56.0 diff --git a/argocd/CHANGELOG.md b/argocd/CHANGELOG.md index 3cc0337f4903a..ca2676e2a08e6 100644 --- a/argocd/CHANGELOG.md +++ b/argocd/CHANGELOG.md @@ -2,6 +2,12 @@ +## 3.2.0 / 2024-11-28 + +***Added***: + +* Add new Application Set metrics ([#18961](https://github.com/DataDog/integrations-core/pull/18961)) + ## 3.1.0 / 2024-10-04 / Agent 7.59.0 ***Fixed***: diff --git a/argocd/changelog.d/18961.added b/argocd/changelog.d/18961.added deleted file mode 100644 index 2709860814937..0000000000000 --- a/argocd/changelog.d/18961.added +++ /dev/null @@ -1 +0,0 @@ -Add new Application Set metrics diff --git a/argocd/datadog_checks/argocd/__about__.py b/argocd/datadog_checks/argocd/__about__.py index 00d8d3e916500..df3e4342a4a3e 100644 --- a/argocd/datadog_checks/argocd/__about__.py +++ b/argocd/datadog_checks/argocd/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2022-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '3.1.0' +__version__ = '3.2.0' diff --git a/avi_vantage/tests/test_avi_vantage.py b/avi_vantage/tests/test_avi_vantage.py index 1e9fb55aadece..04d93309c07fd 100644 --- a/avi_vantage/tests/test_avi_vantage.py +++ b/avi_vantage/tests/test_avi_vantage.py @@ -7,6 +7,7 @@ from datadog_checks.dev.utils import get_metadata_metrics +@pytest.mark.unit def test_check(mock_client, get_expected_metrics, aggregator, unit_instance, dd_run_check): check = AviVantageCheck('avi_vantage', {}, [unit_instance]) dd_run_check(check) @@ -17,6 +18,7 @@ def test_check(mock_client, get_expected_metrics, aggregator, unit_instance, dd_ aggregator.assert_metrics_using_metadata(get_metadata_metrics()) +@pytest.mark.integration def test_integration( dd_environment, get_expected_metrics, aggregator, integration_instance, dd_run_check, datadog_agent ): diff --git a/cisco_aci/CHANGELOG.md b/cisco_aci/CHANGELOG.md index 2d3253e73220a..25c1008c338ae 100644 --- a/cisco_aci/CHANGELOG.md +++ b/cisco_aci/CHANGELOG.md @@ -72,7 +72,7 @@ * [NDM] Add NDM metadata support for Cisco ACI ([#17735](https://github.com/DataDog/integrations-core/pull/17735)) * [NDM] [Cisco ACI] Add common NDM tags to metrics ([#18017](https://github.com/DataDog/integrations-core/pull/18017)) * [NDM] [Cisco ACI] Add config flag for enabling sending metadata to NDM ([#18099](https://github.com/DataDog/integrations-core/pull/18099)) -* Update dependencies ([#18185](https://github.com/DataDog/integrations-core/pull/18185)) +* Update dependencies ([#18187](https://github.com/DataDog/integrations-core/pull/18187)) ## 2.9.0 / 2024-07-05 / Agent 7.56.0 diff --git a/cisco_aci/assets/configuration/spec.yaml b/cisco_aci/assets/configuration/spec.yaml index 98b3c3d4d8502..f466b50fbe0b5 100644 --- a/cisco_aci/assets/configuration/spec.yaml +++ b/cisco_aci/assets/configuration/spec.yaml @@ -101,7 +101,7 @@ files: example: default - name: send_ndm_metadata description: | - Set to `true` to enable Network Device Monitoring metadata (for devices and interfaces) to be sent. + Set to `true` to enable Network Device Monitoring metadata (for devices, interfaces, topology) to be sent. value: type: boolean example: False diff --git a/cisco_aci/assets/dashboards/cisco_aci_dashboard.json b/cisco_aci/assets/dashboards/cisco_aci_dashboard.json index 8b145f1953a94..aac3ffdfe1bc7 100644 --- a/cisco_aci/assets/dashboards/cisco_aci_dashboard.json +++ b/cisco_aci/assets/dashboards/cisco_aci_dashboard.json @@ -26,8 +26,8 @@ "layout": { "x": 0, "y": 0, - "width": 2, - "height": 2 + "width": 6, + "height": 4 } }, { @@ -45,19 +45,136 @@ "has_padding": true }, "layout": { - "x": 2, + "x": 0, + "y": 4, + "width": 6, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 9 + } + }, + { + "id": 8073559146636168, + "definition": { + "title": "Health Overview", + "background_color": "vivid_purple", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 5679074543367742, + "definition": { + "title": "Leafs", + "title_size": "16", + "title_align": "left", + "time": {}, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:cisco_aci.fabric.node.cpu.avg{apic_role:leaf} by {device_id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "formula": "count_nonzero(query1)" + } + ] + } + ], + "autoscale": true, + "custom_links": [], + "precision": 2 + }, + "layout": { + "x": 0, "y": 0, - "width": 4, + "width": 3, "height": 2 } + }, + { + "id": 6149327972947806, + "definition": { + "title": "Spines", + "title_size": "16", + "title_align": "left", + "time": {}, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:cisco_aci.fabric.node.cpu.avg{apic_role:spine} by {device_id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "formula": "count_nonzero(query1)" + } + ] + } + ], + "autoscale": true, + "custom_links": [], + "precision": 2 + }, + "layout": { + "x": 3, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 2542134220658730, + "definition": { + "title": "Monitors overview", + "type": "manage_status", + "display_format": "countsAndList", + "color_preference": "text", + "hide_zero_counts": true, + "show_status": true, + "last_triggered_format": "relative", + "query": "cisco-aci", + "sort": "status,asc", + "count": 50, + "start": 0, + "summary_type": "monitors", + "show_priority": false, + "show_last_triggered": false + }, + "layout": { + "x": 0, + "y": 2, + "width": 6, + "height": 3 + } } ] }, "layout": { - "x": 0, + "x": 6, "y": 0, "width": 6, - "height": 3 + "height": 6 } }, { @@ -217,7 +334,7 @@ }, "layout": { "x": 6, - "y": 0, + "y": 6, "width": 6, "height": 3 } @@ -433,7 +550,7 @@ }, "layout": { "x": 0, - "y": 3, + "y": 9, "width": 12, "height": 6 } @@ -480,7 +597,7 @@ "x": 0, "y": 0, "width": 2, - "height": 1 + "height": 2 } }, { @@ -533,34 +650,40 @@ "layout": { "x": 2, "y": 0, - "width": 3, + "width": 5, "height": 2 } }, { - "id": 20, + "id": 19, "definition": { - "title": "Top 10 EPGs by IP", + "title": "Top 10 EPG Ports by traffic", "title_size": "16", "title_align": "left", "type": "toplist", "requests": [ { "style": { - "palette": "dog_classic" + "palette": "orange" }, "response_format": "scalar", "queries": [ { "data_source": "metrics", "name": "query1", - "query": "avg:cisco_aci.tenant.egress_bytes.unicast.rate{$tenant} by {ip,application,endpoint_group}", + "query": "avg:cisco_aci.fabric.port.egr_bytes.unicast{$tenant} by {node_id,port,endpoint_group}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:cisco_aci.fabric.port.ingr_bytes.unicast{$tenant} by {node_id,port,endpoint_group}", "aggregator": "avg" } ], "formulas": [ { - "formula": "query1" + "formula": "exclude_null(query1) + exclude_null(query2)" } ], "sort": { @@ -579,109 +702,103 @@ "style": {} }, "layout": { - "x": 5, + "x": 7, "y": 0, - "width": 4, - "height": 2 + "width": 5, + "height": 4 } }, { - "id": 19, + "id": 18, "definition": { - "title": "Top 10 EPG Ports by traffic", + "title": "Applications", "title_size": "16", "title_align": "left", - "type": "toplist", + "type": "query_value", "requests": [ { - "style": { - "palette": "orange" - }, "response_format": "scalar", "queries": [ { "data_source": "metrics", "name": "query1", - "query": "avg:cisco_aci.fabric.port.egr_bytes.unicast{$tenant} by {node_id,port,endpoint_group}", - "aggregator": "avg" - }, - { - "data_source": "metrics", - "name": "query2", - "query": "avg:cisco_aci.fabric.port.ingr_bytes.unicast{$tenant} by {node_id,port,endpoint_group}", - "aggregator": "avg" + "query": "avg:cisco_aci.tenant.ingress_pkts.flood.cum{$tenant} by {application}", + "aggregator": "last" } ], "formulas": [ { - "formula": "exclude_null(query1) + exclude_null(query2)" + "formula": "count_not_null(query1)" } - ], - "sort": { - "count": 10, - "order_by": [ - { - "type": "formula", - "index": 0, - "order": "desc" - } - ] - } + ] } ], + "autoscale": true, "custom_links": [], - "style": {} + "precision": 2 }, "layout": { - "x": 9, - "y": 0, - "width": 3, + "x": 0, + "y": 2, + "width": 2, "height": 2 } }, { - "id": 18, + "id": 20, "definition": { - "title": "Applications", + "title": "Top 10 EPGs by IP", "title_size": "16", "title_align": "left", - "type": "query_value", + "type": "toplist", "requests": [ { + "style": { + "palette": "dog_classic" + }, "response_format": "scalar", "queries": [ { "data_source": "metrics", "name": "query1", - "query": "avg:cisco_aci.tenant.ingress_pkts.flood.cum{$tenant} by {application}", - "aggregator": "last" + "query": "avg:cisco_aci.tenant.egress_bytes.unicast.rate{$tenant} by {ip,application,endpoint_group}", + "aggregator": "avg" } ], "formulas": [ { - "formula": "count_not_null(query1)" + "formula": "query1" } - ] + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } } ], - "autoscale": true, "custom_links": [], - "precision": 2 + "style": {} }, "layout": { - "x": 0, - "y": 1, - "width": 2, - "height": 1 + "x": 2, + "y": 2, + "width": 5, + "height": 2 } } ] }, "layout": { "x": 0, - "y": 9, + "y": 15, "width": 12, - "height": 3 + "height": 5 } }, { @@ -894,6 +1011,104 @@ "type": "group", "layout_type": "ordered", "widgets": [ + { + "id": 1547034477646386, + "definition": { + "title": "Avg CPU %", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": {}, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:cisco_aci.fabric.node.cpu.avg{apic_role:leaf} by {device_id,fabric_pod_id,node_id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 4 + } + }, + { + "id": 1671869305900852, + "definition": { + "title": "Avg Memory %", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": {}, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:cisco_aci.fabric.node.mem.avg{apic_role:leaf} by {device_id,fabric_pod_id,node_id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 4 + } + }, { "id": 2991165436190176, "definition": { @@ -979,8 +1194,8 @@ }, "layout": { "x": 0, - "y": 0, - "width": 6, + "y": 4, + "width": 12, "height": 3 } } @@ -988,9 +1203,9 @@ }, "layout": { "x": 0, - "y": 0, - "width": 6, - "height": 4 + "y": 6, + "width": 12, + "height": 8 } }, { @@ -1002,6 +1217,104 @@ "type": "group", "layout_type": "ordered", "widgets": [ + { + "id": 3145200399718776, + "definition": { + "title": "Avg CPU %", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": {}, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:cisco_aci.fabric.node.cpu.avg{apic_role:spine} by {device_id,fabric_pod_id,node_id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 4 + } + }, + { + "id": 3332653863183974, + "definition": { + "title": "Avg Memory %", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": {}, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:cisco_aci.fabric.node.mem.avg{apic_role:spine} by {device_id,fabric_pod_id,node_id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 4 + } + }, { "id": 1813163214351510, "definition": { @@ -1087,18 +1400,18 @@ }, "layout": { "x": 0, - "y": 0, - "width": 6, + "y": 4, + "width": 12, "height": 3 } } ] }, "layout": { - "x": 6, - "y": 0, - "width": 6, - "height": 4 + "x": 0, + "y": 14, + "width": 12, + "height": 8 } } ], diff --git a/cisco_aci/changelog.d/18675.added b/cisco_aci/changelog.d/18675.added new file mode 100644 index 0000000000000..72ee9491e1b34 --- /dev/null +++ b/cisco_aci/changelog.d/18675.added @@ -0,0 +1 @@ +[NDM] [Cisco ACI] Support submitting topology metadata (utilizing LLDP neighbor information) diff --git a/cisco_aci/changelog.d/19204.fixed b/cisco_aci/changelog.d/19204.fixed new file mode 100644 index 0000000000000..2ffc1d5bac760 --- /dev/null +++ b/cisco_aci/changelog.d/19204.fixed @@ -0,0 +1 @@ +[NDM] [Cisco ACI] Fix APIC device status diff --git a/cisco_aci/datadog_checks/cisco_aci/api.py b/cisco_aci/datadog_checks/cisco_aci/api.py index 4c76f3360feb7..aa6cb1c8b3977 100644 --- a/cisco_aci/datadog_checks/cisco_aci/api.py +++ b/cisco_aci/datadog_checks/cisco_aci/api.py @@ -298,6 +298,16 @@ def get_eth_list_and_stats(self, pod, node): response = self.make_request(path) return self._parse_response(response) + def get_lldp_adj_eps(self): + path = '/api/node/class/lldpAdjEp.json' + response = self.make_request(path) + return self._parse_response(response) + + def get_cdp_adj_eps(self): + path = '/api/node/class/cdpAdjEp.json' + response = self.make_request(path) + return self._parse_response(response) + def get_eqpt_capacity(self, eqpt): base_path = '/api/class/eqptcapacityEntity.json' base_query = 'query-target=self&rsp-subtree-include=stats&rsp-subtree-class=' diff --git a/cisco_aci/datadog_checks/cisco_aci/data/conf.yaml.example b/cisco_aci/datadog_checks/cisco_aci/data/conf.yaml.example index fabcec25daf24..fb2715c64aefa 100644 --- a/cisco_aci/datadog_checks/cisco_aci/data/conf.yaml.example +++ b/cisco_aci/datadog_checks/cisco_aci/data/conf.yaml.example @@ -131,7 +131,7 @@ instances: # namespace: default ## @param send_ndm_metadata - boolean - optional - default: false - ## Set to `true` to enable Network Device Monitoring metadata (for devices and interfaces) to be sent. + ## Set to `true` to enable Network Device Monitoring metadata (for devices, interfaces, topology) to be sent. # # send_ndm_metadata: false diff --git a/cisco_aci/datadog_checks/cisco_aci/fabric.py b/cisco_aci/datadog_checks/cisco_aci/fabric.py index 0d9d9fc6b40a4..8698ef5a7e09c 100644 --- a/cisco_aci/datadog_checks/cisco_aci/fabric.py +++ b/cisco_aci/datadog_checks/cisco_aci/fabric.py @@ -47,8 +47,14 @@ def collect(self): pods = self.submit_pod_health(fabric_pods) devices, interfaces = self.submit_nodes_health_and_metadata(fabric_nodes, pods) if self.ndm_enabled(): + # get topology link metadata + lldp_adj_eps = self.api.get_lldp_adj_eps() + cdp_adj_eps = self.api.get_cdp_adj_eps() + device_map = ndm.get_device_ip_mapping(devices) + links = ndm.create_topology_link_metadata(lldp_adj_eps, cdp_adj_eps, device_map, self.namespace) + collect_timestamp = int(time.time()) - batches = ndm.batch_payloads(self.namespace, devices, interfaces, collect_timestamp) + batches = ndm.batch_payloads(self.namespace, devices, interfaces, links, collect_timestamp) for batch in batches: self.event_platform_event(json.dumps(batch.model_dump(exclude_none=True)), "network-devices-metadata") diff --git a/cisco_aci/datadog_checks/cisco_aci/helpers.py b/cisco_aci/datadog_checks/cisco_aci/helpers.py index ac2cd56f40a7e..7ebe981f7fbb1 100644 --- a/cisco_aci/datadog_checks/cisco_aci/helpers.py +++ b/cisco_aci/datadog_checks/cisco_aci/helpers.py @@ -11,6 +11,7 @@ EPG_REGEX = re.compile('/epg-([^/]+)/') IP_REGEX = re.compile('/ip-([^/]+)/') NODE_REGEX = re.compile('node-([0-9]+)') +ETH_REGEX = re.compile(r'\[([^]]*)\]') def parse_capacity_tags(dn): @@ -85,6 +86,23 @@ def get_node_from_dn(dn): return _get_value_from_dn(NODE_REGEX, dn) +def get_eth_id_from_dn(dn): + """ + This parses the interface ID (eth) from a dn designator. They look like this: + topology/pod-1/node-101/sys/lldp/inst/if-[eth1/49]/adj-1 + """ + return _get_value_from_dn(ETH_REGEX, dn) + + +def get_index_from_eth_id(eth_id): + """ + This parses the interface index (eth) from an interface's ID. They look like this: + eth1/49 + """ + split = re.split('eth|/', eth_id) + return int(split[-1]) + + def _get_value_from_dn(regex, dn): if not dn: return None diff --git a/cisco_aci/datadog_checks/cisco_aci/models.py b/cisco_aci/datadog_checks/cisco_aci/models.py index 8031978213f3f..18bd6a0632b8e 100644 --- a/cisco_aci/datadog_checks/cisco_aci/models.py +++ b/cisco_aci/datadog_checks/cisco_aci/models.py @@ -8,6 +8,8 @@ from pydantic import BaseModel, ConfigDict, Field, computed_field, field_validator, model_validator +from . import helpers + """ Cisco ACI Response Models """ @@ -15,6 +17,7 @@ class NodeAttributes(BaseModel): address: Optional[str] = None + ad_st: Optional[str] = Field(default=None, alias="adSt") fabric_st: Optional[str] = Field(default=None, alias="fabricSt") role: Optional[str] = None dn: Optional[str] = None @@ -32,6 +35,22 @@ def device_type(self) -> str: return 'switch' return 'other' + @computed_field + @property + def status(self) -> int: + if self.role == 'controller': + return 1 if self.ad_st == 'on' else 2 + mapping = { + 'active': 1, + 'inactive': 2, + 'disabled': 2, + 'discovering': 2, + 'undiscovered': 2, + 'unsupported': 2, + 'unknown': 2, + } + return mapping.get(self.fabric_st, 2) + class Node(BaseModel): attributes: NodeAttributes @@ -77,6 +96,77 @@ def ethpm_phys_if(self) -> Optional[EthpmPhysIf]: return None +class LldpAdjAttributes(BaseModel): + chassis_id_t: Optional[str] = Field(default=None, alias="chassisIdT") + chassis_id_v: Optional[str] = Field(default=None, alias="chassisIdV") + dn: Optional[str] = None + mgmt_ip: Optional[str] = Field(default=None, alias="mgmtIp") + mgmt_port_mac: Optional[str] = Field(default=None, alias="mgmtPortMac") + port_desc: Optional[str] = Field(default=None, alias="portDesc") + port_id_t: Optional[str] = Field(default=None, alias="portIdT") + port_id_v: Optional[str] = Field(default=None, alias="portIdV") + sys_desc: Optional[str] = Field(default=None, alias="sysDesc") + sys_name: Optional[str] = Field(default=None, alias="sysName") + + @computed_field + @property + def ndm_remote_interface_type(self) -> str: + # map the Cisco ACI port subtype to match what NDM (writer) expects + port_subtype_mapping = { + "if-alias": "interface_alias", + "port-name": "interface_name", + "mac": "mac_address", + "nw-addr": "network_address", + "if-name": "interface_name", + "agent-ckt-id": "agent_circuit_id", + "local": "local", + } + if self.port_id_t: + return port_subtype_mapping.get(self.port_id_t, "unknown") + return "unknown" + + @computed_field + @property + def local_device_dn(self) -> str: + # example: topology/pod-1/node-101/sys/lldp/inst/if-[eth1/49]/adj-1 + return helpers.get_hostname_from_dn(self.dn) + + @computed_field + @property + def local_port_id(self) -> str: + # example: topology/pod-1/paths-201/path-ep-[eth1/1] + # use regex to extract port alias from square brackets - ex: eth1/1 + return helpers.get_eth_id_from_dn(self.dn) + + @computed_field + @property + def local_port_index(self) -> int: + return helpers.get_index_from_eth_id(self.local_port_id) + + @computed_field + @property + def remote_device_dn(self) -> str: + # example: topology/pod-1/paths-201/path-ep-[eth1/1] + # use regex to extract the pod/node - ex: pod-1-node-201 + return helpers.get_hostname_from_dn(self.sys_desc) + + @computed_field + @property + def remote_port_id(self) -> str: + # example: topology/pod-1/paths-201/path-ep-[eth1/1] + # use regex to extract port alias from square brackets - ex: eth1/1 + return helpers.get_eth_id_from_dn(self.port_desc) + + @computed_field + @property + def remote_port_index(self) -> int: + return helpers.get_index_from_eth_id(self.remote_port_id) + + +class LldpAdjEp(BaseModel): + attributes: LldpAdjAttributes + + """ NDM Models """ @@ -88,27 +178,16 @@ class DeviceMetadata(BaseModel): tags: list = Field(default_factory=list) name: Optional[str] = Field(default=None) ip_address: Optional[str] = Field(default=None) + status: Optional[int] = Field(default=None) model: Optional[str] = Field(default=None) - fabric_st: Optional[str] = Field(default=None, exclude=True) vendor: Optional[str] = Field(default=None) version: Optional[str] = Field(default=None) serial_number: Optional[str] = Field(default=None) device_type: Optional[str] = Field(default=None) integration: Optional[str] = Field(default='cisco-aci') - @computed_field - @property - def status(self) -> int: - mapping = { - 'active': 1, - 'inactive': 2, - 'disabled': 2, - 'discovering': 2, - 'undiscovered': 2, - 'unsupported': 2, - 'unknown': 2, - } - return mapping.get(self.fabric_st, 2) + # non-exported fields + pod_node_id: Optional[str] = Field(default=None, exclude=True) class DeviceMetadataList(BaseModel): @@ -136,7 +215,7 @@ class InterfaceMetadata(BaseModel): device_id: Optional[str] = Field(default=None) id_tags: list = Field(default_factory=list) raw_id: Optional[str] = Field(default=None) - raw_id_type: Optional[str] = Field(default='cisco_aci') + raw_id_type: Optional[str] = Field(default='cisco-aci') index: Optional[int] = Field(default=None) name: Optional[str] = Field(default=None) alias: Optional[str] = Field(default=None) @@ -198,10 +277,46 @@ class InterfaceMetadataList(BaseModel): interface_metadata: list = Field(default_factory=list) +class TopologyLinkDevice(BaseModel): + dd_id: Optional[str] = None + id: Optional[str] = None + id_type: Optional[str] = None + name: Optional[str] = None + description: Optional[str] = None + ip_address: Optional[str] = None + + +class TopologyLinkInterface(BaseModel): + dd_id: Optional[str] = None + id: Optional[str] = None + id_type: Optional[str] = None + description: Optional[str] = None + + +class TopologyLinkSide(BaseModel): + device: Optional[TopologyLinkDevice] = None + interface: Optional[TopologyLinkInterface] = None + + +class SourceType(StrEnum): + LLDP = "lldp" + CDP = "cdp" + OTHER = "OTHER" + + +class TopologyLinkMetadata(BaseModel): + id: Optional[str] = None + source_type: Optional[SourceType] = Field(default=None) + local: Optional[TopologyLinkSide] = Field(default=None) + remote: Optional[TopologyLinkSide] = Field(default=None) + integration: Optional[str] = Field(default='cisco-aci') + + class NetworkDevicesMetadata(BaseModel): namespace: str = None devices: Optional[list[DeviceMetadata]] = Field(default_factory=list) interfaces: Optional[list[InterfaceMetadata]] = Field(default_factory=list) + links: Optional[list[TopologyLinkMetadata]] = Field(default_factory=list) collect_timestamp: Optional[int] = None size: Optional[int] = Field(default=0, exclude=True) @@ -212,4 +327,6 @@ def append_metadata(self, metadata: DeviceMetadata | InterfaceMetadata): self.devices.append(metadata) if isinstance(metadata, InterfaceMetadata): self.interfaces.append(metadata) + if isinstance(metadata, TopologyLinkMetadata): + self.links.append(metadata) self.size += 1 diff --git a/cisco_aci/datadog_checks/cisco_aci/ndm.py b/cisco_aci/datadog_checks/cisco_aci/ndm.py index 7e9c066a72708..2336e30a0e227 100644 --- a/cisco_aci/datadog_checks/cisco_aci/ndm.py +++ b/cisco_aci/datadog_checks/cisco_aci/ndm.py @@ -5,11 +5,19 @@ from datadog_checks.cisco_aci.models import ( DeviceMetadata, InterfaceMetadata, + LldpAdjEp, NetworkDevicesMetadata, Node, PhysIf, + SourceType, + TopologyLinkDevice, + TopologyLinkInterface, + TopologyLinkMetadata, + TopologyLinkSide, ) +from . import helpers + VENDOR_CISCO = 'cisco' PAYLOAD_METADATA_BATCH_SIZE = 100 @@ -31,12 +39,13 @@ def create_node_metadata(node_attrs, tags, namespace): tags=device_tags + tags, name=hostname, ip_address=node.attributes.address, + status=node.attributes.status, model=node.attributes.model, - fabric_st=node.attributes.fabric_st, vendor=VENDOR_CISCO, version=node.attributes.version, serial_number=node.attributes.serial, device_type=node.attributes.device_type, + pod_node_id=helpers.get_hostname_from_dn(node.attributes.dn), ) return device @@ -63,6 +72,105 @@ def create_interface_metadata(phys_if, address, namespace): return interface +def create_topology_link_metadata(lldp_adj_eps, cdp_adj_eps, device_map, namespace): + """ + Create a TopologyLinkMetadata object from LLDP or CDP (only LLDP is supported as of right now) + """ + for lldp_adj_ep in lldp_adj_eps: + lldp_adj_ep = LldpAdjEp(**lldp_adj_ep.get("lldpAdjEp", {})) + lldp_attrs = lldp_adj_ep.attributes + + local_device_id = device_map.get(lldp_attrs.local_device_dn) + local_interface_id = get_interface_dd_id(local_device_id, lldp_attrs.local_port_id) + + local = TopologyLinkSide( + device=TopologyLinkDevice(dd_id=local_device_id), + interface=TopologyLinkInterface( + dd_id=local_interface_id, + id=lldp_attrs.local_port_id, + id_type='interface_name', + ), + ) + + remote_device_dd_id = get_remote_device_dd_id(device_map, lldp_attrs.remote_device_dn, lldp_attrs.mgmt_ip) + remote_device = TopologyLinkDevice( + name=lldp_attrs.sys_name, + description=lldp_attrs.sys_desc, + id=lldp_attrs.chassis_id_v, + id_type=lldp_attrs.chassis_id_t, + ip_address=lldp_attrs.mgmt_ip, + ) + if remote_device_dd_id: + remote_device.dd_id = remote_device_dd_id + remote_interface = TopologyLinkInterface( + id=lldp_attrs.port_id_v, + id_type=lldp_attrs.ndm_remote_interface_type, + description=lldp_attrs.port_desc, + ) + if remote_device_dd_id: + remote_interface.dd_id = get_interface_dd_id(remote_device_dd_id, lldp_attrs.remote_port_id) + + remote = TopologyLinkSide( + device=remote_device, + interface=remote_interface, + ) + + if remote_device_dd_id: + link_id = ( + f"{local_device_id}:{get_raw_id(lldp_attrs.local_port_id)}.{get_raw_id(lldp_attrs.remote_port_id)}" + ) + else: + link_id = f"{local_device_id}:{get_raw_id(lldp_attrs.local_port_id)}.{lldp_attrs.remote_port_index}" + + yield TopologyLinkMetadata( + id=link_id, + source_type=SourceType.LLDP, + local=local, + remote=remote, + ) + + +def get_remote_device_dd_id(device_map, remote_device_dn, mgmt_ip) -> str | None: + """ + Get the Cisco DN for a remote device, if the device is in the device map then + check that it matches the management IP of the LLDP neighbor, then return it + """ + device_id = device_map.get(remote_device_dn, "") + if device_id: + if device_id.endswith(mgmt_ip): + return device_id + return None + + +def get_interface_dd_id(device_id: str, port_id: str) -> str: + """ + Create the interface DD ID based off of the device DD ID and port ID + ex: default:10.0.200.1:cisco_aci-eth1/1 + """ + raw_id = get_raw_id(port_id) + return f"{device_id}:{raw_id}" + + +def get_raw_id(raw_id, raw_id_type="cisco-aci") -> str: + """ + Create the interface raw ID, based on the type (cisco-aci) and the interface's identifier + separated by a hyphen - ex: cisco-aci-eth1/1 + """ + return f"{raw_id_type}-{raw_id}" + + +def get_device_ip_mapping(devices): + """ + Create a mapping of node ID to device ID + ex: pod-1-node-1 -> default:10.100.0.1 + """ + devices_map = {} + for device in devices: + key = device.pod_node_id + devices_map[key] = device.id + return devices_map + + def get_device_info(device): """ Get device ID and node ID from a device object @@ -74,7 +182,7 @@ def get_device_info(device): return device.id, node_id -def batch_payloads(namespace, devices, interfaces, collect_ts): +def batch_payloads(namespace, devices, interfaces, links, collect_ts): """ Batch payloads into NetworkDevicesMetadata objects """ @@ -91,6 +199,12 @@ def batch_payloads(namespace, devices, interfaces, collect_ts): yield current_payload network_devices_metadata = new_payload + for link in links: + current_payload, new_payload = append_to_payload(link, network_devices_metadata, namespace, collect_ts) + if new_payload: + yield current_payload + network_devices_metadata = new_payload + yield network_devices_metadata diff --git a/cisco_aci/tests/common.py b/cisco_aci/tests/common.py index 79022adbb0c0e..40f1fdb2761f9 100644 --- a/cisco_aci/tests/common.py +++ b/cisco_aci/tests/common.py @@ -173,6 +173,10 @@ # 4efe80304d50330f5ed0f79252ef0a84 - Api.get_apps '_api_mo_uni_tn_DataDog_json_rsp_subtree_include_stats_no_scoped', # c8e9a0dbceac67fb1149684f7fc7772c - Api.get_tenant_stats + '_api_node_class_lldpAdjEp_json', + # f3713df3a586908a3a11f4c356153519 - Api.get_lldp_adj_eps + '_api_node_class_cdpAdjEp_json', + # 588ea77fffc6df4b37dfdfa4290cdc89 - Api.get_cdp_adj_eps '_api_node_class_topology_pod_1_node_102_l1PhysIf_json_rsp_subtree_children_rsp_subtree_include_stats_rsp_subtree_class_ethpmPhysIf_eqptEgrTotal5min_eqptIngrTotal5min_eqptEgrDropPkts5min_eqptEgrBytes5min_eqptIngrBytes5min', # fde05c4b654d2d8129c772cd5a20cbce - Api.get_eth_list_and_stats '_api_node_class_topology_pod_1_node_201_l1PhysIf_json_rsp_subtree_children_rsp_subtree_include_stats_rsp_subtree_class_ethpmPhysIf_eqptEgrTotal5min_eqptIngrTotal5min_eqptEgrDropPkts5min_eqptEgrBytes5min_eqptIngrBytes5min', diff --git a/cisco_aci/tests/fixtures/fabric/588ea77fffc6df4b37dfdfa4290cdc89.txt b/cisco_aci/tests/fixtures/fabric/588ea77fffc6df4b37dfdfa4290cdc89.txt new file mode 100644 index 0000000000000..6cc0e51dea023 --- /dev/null +++ b/cisco_aci/tests/fixtures/fabric/588ea77fffc6df4b37dfdfa4290cdc89.txt @@ -0,0 +1,4 @@ +{ + "totalCount": "0", + "imdata": [] +} \ No newline at end of file diff --git a/cisco_aci/tests/fixtures/fabric/f3713df3a586908a3a11f4c356153519.txt b/cisco_aci/tests/fixtures/fabric/f3713df3a586908a3a11f4c356153519.txt new file mode 100644 index 0000000000000..6a11461f5096e --- /dev/null +++ b/cisco_aci/tests/fixtures/fabric/f3713df3a586908a3a11f4c356153519.txt @@ -0,0 +1,61 @@ +{ + "totalCount": "1", + "imdata": [ + { + "lldpAdjEp": { + "attributes": { + "capability": "router", + "chassisIdT": "mac", + "chassisIdV": "6a:00:21:1f:55:2a", + "childAction": "", + "dn": "topology/pod-1/node-101/sys/lldp/inst/if-[eth1/49]/adj-1", + "enCap": "", + "id": "1", + "mgmtId": "0", + "mgmtIp": "10.0.200.5", + "mgmtPortMac": "unspecified", + "modTs": "2024-09-12T06:51:52.580+00:00", + "monPolDn": "uni/fabric/monfab-default", + "name": "", + "portDesc": "topology/pod-1/paths-201/pathep-[eth5/1]", + "portIdT": "mac", + "portIdV": "6a:00:21:1f:55:2a", + "portVlan": "unspecified", + "stQual": "", + "status": "", + "sysDesc": "topology/pod-1/node-201", + "sysName": "SP201", + "ttl": "120" + } + } + }, + { + "lldpAdjEp": { + "attributes": { + "capability": "router", + "chassisIdT": "mac", + "chassisIdV": "6a:00:21:1f:55:2b", + "childAction": "", + "dn": "topology/pod-1/node-102/sys/lldp/inst/if-[eth1/49]/adj-1", + "enCap": "", + "id": "1", + "mgmtId": "0", + "mgmtIp": "10.0.200.5", + "mgmtPortMac": "unspecified", + "modTs": "2024-09-12T06:51:52.580+00:00", + "monPolDn": "uni/fabric/monfab-default", + "name": "", + "portDesc": "topology/pod-1/paths-201/pathep-[eth5/2]", + "portIdT": "mac", + "portIdV": "6a:00:21:1f:55:2b", + "portVlan": "unspecified", + "stQual": "", + "status": "", + "sysDesc": "topology/pod-1/node-201", + "sysName": "SP201", + "ttl": "120" + } + } + } + ] +} \ No newline at end of file diff --git a/cisco_aci/tests/fixtures/metadata.py b/cisco_aci/tests/fixtures/metadata.py index e3b1236b34d8a..fa93687a52fec 100644 --- a/cisco_aci/tests/fixtures/metadata.py +++ b/cisco_aci/tests/fixtures/metadata.py @@ -30,7 +30,6 @@ ], 'ip_address': '10.0.200.0', 'model': 'N9K-C93180YC-FX', - 'fabric_st': 'active', 'name': 'leaf101', 'serial_number': 'FDO20440TS1', 'status': 1, @@ -62,7 +61,6 @@ ], 'ip_address': '10.0.200.1', 'model': 'N9K-C93180YC-FX', - 'fabric_st': 'active', 'name': 'leaf102', 'serial_number': 'FDO20510HCA', 'status': 1, @@ -93,10 +91,9 @@ ], 'ip_address': '10.0.200.4', 'model': 'APIC-SERVER-M1', - 'fabric_st': 'unknown', 'name': 'apic1', 'serial_number': 'FCH1928V0SL', - 'status': 2, + 'status': 1, 'vendor': 'cisco', 'version': 'A', }, @@ -125,7 +122,6 @@ ], 'ip_address': '10.0.200.5', 'model': 'N9K-C9336PQ', - 'fabric_st': 'active', 'name': 'spine201', 'serial_number': 'SAL2014N5U4', 'status': 1, @@ -272,6 +268,69 @@ }, ] +TOPOLOGY_LINK_METADATA = [ + { + 'id': 'default:10.0.200.0:cisco-aci-eth1/49.cisco-aci-eth5/1', + 'local': { + 'device': { + 'dd_id': 'default:10.0.200.0', + }, + 'interface': { + 'dd_id': 'default:10.0.200.0:cisco-aci-eth1/49', + 'id': 'eth1/49', + 'id_type': 'interface_name', + }, + }, + 'remote': { + 'device': { + 'dd_id': 'default:10.0.200.5', + 'description': 'topology/pod-1/node-201', + 'id': '6a:00:21:1f:55:2a', + 'id_type': 'mac', + 'ip_address': '10.0.200.5', + 'name': 'SP201', + }, + 'interface': { + 'dd_id': 'default:10.0.200.5:cisco-aci-eth5/1', + 'description': 'topology/pod-1/paths-201/pathep-[eth5/1]', + 'id': '6a:00:21:1f:55:2a', + 'id_type': 'mac_address', + }, + }, + 'source_type': 'lldp', + }, + { + 'id': 'default:10.0.200.1:cisco-aci-eth1/49.cisco-aci-eth5/2', + 'local': { + 'device': { + 'dd_id': 'default:10.0.200.1', + }, + 'interface': { + 'dd_id': 'default:10.0.200.1:cisco-aci-eth1/49', + 'id': 'eth1/49', + 'id_type': 'interface_name', + }, + }, + 'remote': { + 'device': { + 'dd_id': 'default:10.0.200.5', + 'description': 'topology/pod-1/node-201', + 'id': '6a:00:21:1f:55:2b', + 'id_type': 'mac', + 'ip_address': '10.0.200.5', + 'name': 'SP201', + }, + 'interface': { + 'dd_id': 'default:10.0.200.5:cisco-aci-eth5/2', + 'description': 'topology/pod-1/paths-201/pathep-[eth5/2]', + 'id': '6a:00:21:1f:55:2b', + 'id_type': 'mac_address', + }, + }, + 'source_type': 'lldp', + }, +] + EXPECTED_DEVICE_METADATA_RESULT = DeviceMetadataList(device_metadata=DEVICE_METADATA) # "2012-01-14 03:21:34" in seconds @@ -284,6 +343,7 @@ namespace='default', devices=DEVICE_METADATA, interfaces=INTERFACE_METADATA, + links=TOPOLOGY_LINK_METADATA, collect_timestamp=MOCK_TIME_EPOCH, ) ] diff --git a/cisco_sdwan/README.md b/cisco_sdwan/README.md index fa6194285587b..e383bfb9c47e1 100644 --- a/cisco_sdwan/README.md +++ b/cisco_sdwan/README.md @@ -1,4 +1,3 @@ -
The Cisco SD-WAN NDM integration is in Preview.
# Agent Check: Cisco SD-WAN diff --git a/clickhouse/CHANGELOG.md b/clickhouse/CHANGELOG.md index 7db58d203c20d..0a61ccbb4bd93 100644 --- a/clickhouse/CHANGELOG.md +++ b/clickhouse/CHANGELOG.md @@ -2,6 +2,12 @@ +## 5.1.0 / 2024-11-28 + +***Added***: + +* Add verify option when connecting to ClickHouse server. ([#19018](https://github.com/DataDog/integrations-core/pull/19018)) + ## 5.0.0 / 2024-10-04 / Agent 7.59.0 ***Removed***: diff --git a/clickhouse/changelog.d/19018.added b/clickhouse/changelog.d/19018.added deleted file mode 100644 index 26114bc6658d1..0000000000000 --- a/clickhouse/changelog.d/19018.added +++ /dev/null @@ -1 +0,0 @@ -Add verify option when connecting to ClickHouse server. \ No newline at end of file diff --git a/clickhouse/datadog_checks/clickhouse/__about__.py b/clickhouse/datadog_checks/clickhouse/__about__.py index 59ccab2a77ff8..2b46b8418a5d7 100644 --- a/clickhouse/datadog_checks/clickhouse/__about__.py +++ b/clickhouse/datadog_checks/clickhouse/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2019-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '5.0.0' +__version__ = '5.1.0' diff --git a/consul/README.md b/consul/README.md index 0f839779c5e9d..85bbc91083625 100644 --- a/consul/README.md +++ b/consul/README.md @@ -242,7 +242,7 @@ Additional helpful documentation, links, and articles: - [Key metrics for monitoring Consul][20] - [Consul monitoring tools][21] - [How to monitor Consul with Datadog][22] -- [Datadog NPM now supports Consul networking][23] +- [Datadog CNM now supports Consul networking][23] [1]: https://raw.githubusercontent.com/DataDog/integrations-core/master/consul/images/consul-dash.png [2]: https://app.datadoghq.com/account/settings/agent/latest diff --git a/couch/CHANGELOG.md b/couch/CHANGELOG.md index e874921a86cdc..7f116a589f626 100644 --- a/couch/CHANGELOG.md +++ b/couch/CHANGELOG.md @@ -2,6 +2,12 @@ +## 8.1.0 / 2024-11-28 + +***Added***: + +* Add support for Couch version 3.4.0 ([#19052](https://github.com/DataDog/integrations-core/pull/19052)) + ## 8.0.0 / 2024-10-04 / Agent 7.59.0 ***Removed***: diff --git a/couch/datadog_checks/couch/__about__.py b/couch/datadog_checks/couch/__about__.py index 90c94f7125939..1b6d66341f5d3 100644 --- a/couch/datadog_checks/couch/__about__.py +++ b/couch/datadog_checks/couch/__about__.py @@ -2,4 +2,4 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = "8.0.0" +__version__ = "8.1.0" diff --git a/couch/datadog_checks/couch/couch.py b/couch/datadog_checks/couch/couch.py index 37235cd84b2aa..b5f8267a77800 100644 --- a/couch/datadog_checks/couch/couch.py +++ b/couch/datadog_checks/couch/couch.py @@ -260,6 +260,9 @@ def _build_system_metrics(self, data, tags, prefix='couchdb.erlang'): dist_tags = list(tags) dist_tags.append("node:{0}".format(node)) self._build_system_metrics(metrics, dist_tags, "{0}.{1}".format(prefix, key)) + elif key == "distribution_events": + self.agent_check.log.debug("Skipping distribution events") + continue elif isinstance(value, dict): self._build_system_metrics(value, tags, "{0}.{1}".format(prefix, key)) else: diff --git a/couch/tests/conftest.py b/couch/tests/conftest.py index 1299b439d6d86..8e407331a251c 100644 --- a/couch/tests/conftest.py +++ b/couch/tests/conftest.py @@ -40,6 +40,15 @@ def active_tasks(): return json.loads(f.read()) +@pytest.fixture +def load_test_data(): + """ + Returns a raw response from `/_3.4_system.json` + """ + with open(os.path.join(common.HERE, 'fixtures', '_3.4_system.json')) as f: + return json.load(f) + + @pytest.fixture(scope="session") def dd_environment(): """ diff --git a/couch/tests/fixtures/_3.4_system.json b/couch/tests/fixtures/_3.4_system.json new file mode 100644 index 0000000000000..a26d70f9c4169 --- /dev/null +++ b/couch/tests/fixtures/_3.4_system.json @@ -0,0 +1,223 @@ +{ + "uptime": 87, + "memory": { + "other": 19225722, + "atom": 553593, + "atom_used": 526962, + "processes": 12616384, + "processes_used": 12607040, + "binary": 558120, + "code": 12149997, + "ets": 1847536 + }, + "run_queue": 0, + "ets_table_count": 124, + "context_switches": 83763, + "reductions": 29838712, + "garbage_collection_count": 25531, + "words_reclaimed": 27456219, + "io_input": 11354834, + "io_output": 2849837, + "os_proc_count": 0, + "stale_proc_count": 0, + "process_count": 470, + "process_limit": 262144, + "message_queues": { + "couch_file": { + "count": 34, + "min": 0, + "max": 0, + "50": 0, + "90": 0, + "99": 0 + }, + "couch_db_updater": { + "count": 22, + "min": 0, + "max": 0, + "50": 0, + "90": 0, + "99": 0 + }, + "httpc_manager": 0, + "httpc_handler_sup": 0, + "ken_sup": 0, + "ken_server": 0, + "couch_replication": 0, + "chttpd_auth_cache_lru": 0, + "couch_index_sup": 0, + "ioq_sup": 0, + "couch_index_server": 0, + "mem3_events": 0, + "jwtf_sup": 0, + "rexi_buffer_couchdb@couchdb-2.example.com": 0, + "jwtf_keystore": 0, + "rexi_buffer_couchdb@couchdb-3.example.com": 0, + "ioq": 0, + "couch_uuids": 0, + "ftp_sup": 0, + "rexi_buffer_mon": 0, + "ibrowse_sup": 0, + "couch_secondary_services": 0, + "rexi_buffer_sup": 0, + "couch_primary_services": 0, + "couch_task_status": 0, + "couch_sup": 0, + "global_changes_sup": 0, + "global_changes_server": 0, + "couch_server": 0, + "ibrowse": 0, + "config_event": 0, + "chttpd_sup": 0, + "couch_proc_manager": 0, + "release_handler": 0, + "sasl_sup": 0, + "dreyfus_sup": 0, + "standard_error_sup": 0, + "couch_event_sup2": 0, + "alarm_handler": 0, + "couch_event_server": 0, + "couch_epi_functions_gen_couch_index": 0, + "dreyfus_index_manager": 0, + "couch_epi_functions_gen_chttpd_auth": 0, + "timer_server": 0, + "couch_epi_functions_gen_couch_db": 0, + "runtime_tools_sup": 0, + "couch_epi_data_gen_flags_config": 0, + "couch_httpd_vhost": 0, + "couch_epi_functions_gen_global_changes": 0, + "couch_epi_functions_gen_chttpd_handlers": 0, + "chttpd_auth_cache": 0, + "couch_epi_functions_gen_feature_flags": 0, + "couch_stats_sup": 0, + "couch_epi_functions_gen_chttpd": 0, + "couch_plugin": 0, + "couch_stats_process_tracker": 0, + "chttpd": 0, + "kernel_safe_sup": 0, + "tftp_sup": 0, + "couch_stats_aggregator": 0, + "rexi_server_couchdb@couchdb-3.example.com": 0, + "rex": 0, + "rexi_server_couchdb@couchdb-2.example.com": 0, + "net_sup": 0, + "folsom_sup": 0, + "inet_gethost_native_sup": 0, + "kernel_sup": 0, + "ddoc_cache_sup": 0, + "global_name_server": 0, + "ddoc_cache_opener": 0, + "folsom_sample_slide_sup": 0, + "ddoc_cache_lru": 0, + "file_server_2": 0, + "standard_error": 0, + "couch_drv": 0, + "couch_peruser_sup": 0, + "tls_connection_sup": 0, + "couch_peruser": 0, + "folsom_metrics_histogram_ets": 0, + "couch_replicator_sup": 0, + "ssl_sup": 0, + "couch_replicator_scheduler_sup": 0, + "smoosh_sup": 0, + "folsom_meter_timer_server": 0, + "smoosh_server": 0, + "couch_replicator_scheduler": 0, + "rexi_buffer_couchdb@couchdb-1.example.com": 0, + "rexi_server_couchdb@couchdb-1.example.com": 0, + "mem3_sync_nodes": 0, + "couch_replicator_rate_limiter": 0, + "inet_gethost_native": 0, + "inets_sup": 0, + "setup_sup": 0, + "inet_db": 0, + "ssl_pem_cache": 0, + "mem3_sync": 0, + "ssl_manager": 0, + "mem3_sup": 0, + "ssl_listen_tracker_sup": 0, + "mem3_shards": 0, + "mem3_seeds": 0, + "httpd_sup": 0, + "couch_log_sup": 0, + "mem3_reshard_sup": 0, + "mango_sup": 0, + "couch_log_server": 0, + "couch_epi_data_gen_dreyfus_black_list": 0, + "mem3_reshard_job_sup": 0, + "erts_code_purger": 0, + "global_group": 0, + "error_logger": 0, + "couch_replicator_doc_processor": 0, + "ssl_connection_sup": 0, + "init": 0, + "mem3_reshard_dbdoc": 0, + "couch_replicator_connection": 0, + "erl_signal_server": 0, + "net_kernel": 0, + "couch_replicator_clustering": 0, + "sasl_safe_sup": 0, + "config": 0, + "mem3_reshard": 0, + "user": 0, + "couch_epi_sup": 0, + "erl_epmd": 0, + "mem3_nodes": 0, + "ssl_admin_sup": 0, + "mochiweb_clock": 0, + "dtls_udp_sup": 0, + "erl_prim_loader": 0, + "code_server": 0, + "httpc_sup": 0, + "rexi_sup": 0, + "dtls_connection_sup": 0, + "rexi_server_sup": 0, + "rexi_server_mon": 0, + "auth": 0, + "application_controller": 0, + "httpc_profile_sup": 0, + "config_sup": 0, + "rexi_server": 0 + }, + "internal_replication_jobs": 0, + "distribution": { + "couchdb@couchdb-2.example.com": { + "recv_oct": 546816, + "recv_cnt": 1865, + "recv_max": 20295, + "recv_avg": 293, + "recv_dvi": 11, + "send_oct": 326373, + "send_cnt": 2989, + "send_max": 1257, + "send_avg": 109, + "send_pend": 0 + }, + "couchdb@couchdb-3.example.com": { + "recv_oct": 495964, + "recv_cnt": 1663, + "recv_max": 20295, + "recv_avg": 298, + "recv_dvi": 20, + "send_oct": 322019, + "send_cnt": 2934, + "send_max": 2514, + "send_avg": 109, + "send_pend": 0 + } + }, + "distribution_events": { + "couchdb@couchdb-3.example.com": [ + [ + "2024-11-13T19:57:04Z", + "nodeup" + ] + ], + "couchdb@couchdb-2.example.com": [ + [ + "2024-11-13T19:57:04Z", + "nodeup" + ] + ] + } +} \ No newline at end of file diff --git a/couch/tests/test_unit.py b/couch/tests/test_unit.py index 15d86f5e71957..bfa3ba06dadae 100644 --- a/couch/tests/test_unit.py +++ b/couch/tests/test_unit.py @@ -2,11 +2,13 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) from copy import deepcopy +from unittest.mock import MagicMock import mock import pytest from datadog_checks.couch import CouchDb +from datadog_checks.couch.couch import CouchDB2 from . import common @@ -45,3 +47,24 @@ def test_config(test_case, extra_config, expected_http_kwargs): http_wargs.update(expected_http_kwargs) r.get.assert_called_with('http://{}:5984/_all_dbs/'.format(common.HOST), **http_wargs) + + +def test_new_version_system_metrics(load_test_data): + # Testing the _build_system_metrics method I'm feeding it a json that has a the updated + # keys that was added in version 3.4 that was causing the check to break. The idea here + # is that I'm going to give the method the json then assert that it's able to go through + # it thhorougly by the number of function calls and debug log calls. + + # Mock everything needed for the function to run + mock_agent_check = MagicMock() + mock_agent_check.gauge = MagicMock() + mock_agent_check.log = MagicMock() + + couchdb_check = CouchDB2(mock_agent_check) + tags = ["test:tag"] + + # The fixture file json is loaded as a fixture in the confest.py file + couchdb_check._build_system_metrics(load_test_data, tags) + + assert mock_agent_check.gauge.call_count >= 183 + mock_agent_check.log.debug.assert_any_call("Skipping distribution events") diff --git a/datadog_checks_base/CHANGELOG.md b/datadog_checks_base/CHANGELOG.md index e16e506896b63..31b58665f0adf 100644 --- a/datadog_checks_base/CHANGELOG.md +++ b/datadog_checks_base/CHANGELOG.md @@ -2,6 +2,18 @@ +## 37.2.0 / 2024-12-05 + +***Added***: + +* Bump binary package version for py3.12 ([#19190](https://github.com/DataDog/integrations-core/pull/19190)) + +## 37.1.1 / 2024-11-28 + +***Fixed***: + +* When resolving database hosts, always resolve a .local database host to itself ([#19039](https://github.com/DataDog/integrations-core/pull/19039)) + ## 37.1.0 / 2024-10-04 / Agent 7.59.0 ***Added***: @@ -67,7 +79,7 @@ * Log invalid line when failing to parse OpenMetrics response ([#17514](https://github.com/DataDog/integrations-core/pull/17514)) * Support log submission from checks ([#18019](https://github.com/DataDog/integrations-core/pull/18019)) * Allow untyped metrics that we coerce to `counter` to be collected regardless if they have `_total` or not. ([#18054](https://github.com/DataDog/integrations-core/pull/18054)) -* Update dependencies ([#18185](https://github.com/DataDog/integrations-core/pull/18185)) +* Update dependencies ([#18187](https://github.com/DataDog/integrations-core/pull/18187)) ## 36.10.0 / 2024-07-11 diff --git a/datadog_checks_base/changelog.d/18975.added b/datadog_checks_base/changelog.d/18975.added new file mode 100644 index 0000000000000..d95d103203c3e --- /dev/null +++ b/datadog_checks_base/changelog.d/18975.added @@ -0,0 +1 @@ +Show diff to closest metric match when metric test fails diff --git a/datadog_checks_base/changelog.d/19039.fixed b/datadog_checks_base/changelog.d/19039.fixed deleted file mode 100644 index 93ceec32666de..0000000000000 --- a/datadog_checks_base/changelog.d/19039.fixed +++ /dev/null @@ -1 +0,0 @@ -When resolving database hosts, always resolve a .local database host to itself diff --git a/datadog_checks_base/changelog.d/19197.fixed b/datadog_checks_base/changelog.d/19197.fixed new file mode 100644 index 0000000000000..31cd6530b5639 --- /dev/null +++ b/datadog_checks_base/changelog.d/19197.fixed @@ -0,0 +1 @@ +Fix "no snapshot data found" error when `agent check --profile-memory` diff --git a/datadog_checks_base/datadog_checks/base/__about__.py b/datadog_checks_base/datadog_checks/base/__about__.py index a1deb4b728270..d2262989bc126 100644 --- a/datadog_checks_base/datadog_checks/base/__about__.py +++ b/datadog_checks_base/datadog_checks/base/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2018-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = "37.1.0" +__version__ = "37.2.0" diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index c87b2cdb70479..9508dcc518dd4 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -1285,7 +1285,13 @@ def run(self): enter_pdb(self.check, line=self.init_config['set_breakpoint'], args=(instance,)) elif self.should_profile_memory(): - self.profile_memory(self.check, self.init_config, args=(instance,)) + # self.init_config['profile_memory'] could be `/tmp/datadog-agent-memory-profiler*` + # that is generated by Datadog Agent. + # If we use `--m-dir` for `agent check` command, a hidden flag, it should be same as a given value. + namespaces = [self.init_config['profile_memory']] + for id in self.check_id.split(":"): + namespaces.append(id) + self.profile_memory(func=self.check, namespaces=namespaces, args=(instance,)) else: self.check(instance) diff --git a/datadog_checks_base/datadog_checks/base/stubs/similar.py b/datadog_checks_base/datadog_checks/base/stubs/similar.py index 8b2e712ac3754..9d56ff3b1a89a 100644 --- a/datadog_checks_base/datadog_checks/base/stubs/similar.py +++ b/datadog_checks_base/datadog_checks/base/stubs/similar.py @@ -9,6 +9,62 @@ MAX_SIMILAR_TO_DISPLAY = 15 +def dict_diff(expected, closest): + """ + Returns an array of key/value pairs that are different between the two dicts. + """ + diff = [] + for key in closest.keys() | expected.keys(): + expected_value = expected.get(key) + closest_value = closest.get(key) + + if expected_value is not None and expected_value != closest_value: + diff.append((key, expected_value, closest_value)) + + return diff + + +def tags_list_to_dict(tags): + return {tag.split(':', 1)[0]: (tag.split(':', 1)[1] if ":" in tag else '') for tag in tags} + + +def tags_diff(expected, closest): + """ + Returns an array of key/value pairs that are different between the two lists of tags. + """ + diff = [] + expected_tags_dict = tags_list_to_dict(expected) + closest_tags_dict = tags_list_to_dict(closest) + for tag in expected_tags_dict: + if expected_tags_dict[tag] != closest_tags_dict.get(tag): + diff.append((tag, expected_tags_dict[tag], closest_tags_dict.get(tag))) + for tag in closest_tags_dict: + if tag not in expected_tags_dict: + diff.append((tag, None, closest_tags_dict[tag])) + return diff + + +def format_metric_stub_diff(expected, closest): + """ + Return formatted difference between expected and closest metric stubs + """ + diff = [] + + closest_dict = closest._asdict() + expected_dict = expected._asdict() + dict_diffs = dict_diff(expected_dict, closest_dict) + for key, expected_value, closest_value in dict_diffs: + if key == "tags": + tag_diffs = tags_diff(expected_value, closest_value) + for tag, expected_tag_value, closest_tag_value in tag_diffs: + diff.append( + f" Expected tag {tag}:{expected_tag_value}\n" + f" Found {tag}:{closest_tag_value}" + ) + else: + diff.append(f" Expected {key}: {expected_value}\n Found {closest_value}") + return diff + + def build_similar_elements_msg(expected, submitted_elements): """ Return formatted similar elements (metrics, service checks) received compared to submitted elements @@ -22,9 +78,17 @@ def build_similar_elements_msg(expected, submitted_elements): metric_stub.tags.sort() similar_metrics_to_print.append("{:.2f} {}".format(score, metric_stub)) + closest_diff = [] + if similar_metrics: + [_, closest] = similar_metrics[0] + closest_diff = format_metric_stub_diff(expected, closest) + return ( "Expected:\n" + " {}\n".format(expected) + + "Difference to closest:\n" + + "\n".join(closest_diff) + + "\n\n" + "Similar submitted:\n" + "Score Most similar\n" + "\n".join(similar_metrics_to_print) diff --git a/datadog_checks_base/pyproject.toml b/datadog_checks_base/pyproject.toml index 5aa522c223b49..8ff7f6bd6ce2c 100644 --- a/datadog_checks_base/pyproject.toml +++ b/datadog_checks_base/pyproject.toml @@ -34,7 +34,7 @@ db = [ "mmh3==4.1.0", ] deps = [ - "binary==1.0.0", + "binary==1.0.1", "cachetools==5.5.0", "cryptography==43.0.1", "ddtrace==2.10.6", diff --git a/datadog_checks_base/tests/stubs/test_aggregator_similar.py b/datadog_checks_base/tests/stubs/test_aggregator_similar.py index 390b8c60a5c84..dede6f841c5b8 100644 --- a/datadog_checks_base/tests/stubs/test_aggregator_similar.py +++ b/datadog_checks_base/tests/stubs/test_aggregator_similar.py @@ -3,6 +3,8 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import difflib +import pytest + from datadog_checks.base import AgentCheck from datadog_checks.base.stubs import similar from datadog_checks.base.stubs.aggregator import AggregatorStub @@ -24,6 +26,10 @@ def test_build_similar_elements_msg(self, aggregator): expected_msg = ''' Expected: MetricStub(name='test.similar_metric', type=None, value=None, tags=None, hostname=None, device=None, flush_first_value=None) +Difference to closest: + Expected name: test.similar_metric + Found test.most_similar_metric + Similar submitted: Score Most similar 0.88 MetricStub(name='test.most_similar_metric', type=0, value=0.0, tags=[], hostname='', device=None, flush_first_value=False) @@ -214,3 +220,33 @@ def test__build_similar_elements__histogram_buckets(self, aggregator): assert similar_histogram_bucket[3][1].name == 'histogram.bucket4' # value/monotonic/tag match assert similar_histogram_bucket[4][1].name == 'histogram.bucket5' # value/monotonic match assert similar_histogram_bucket[5][1].name == 'histogram.bucket0' # no match + + @pytest.mark.parametrize( + "expected, closest, diff", + [ + pytest.param({'a': 1, 'b': 2}, {'a': 1, 'b': 3}, [('b', 2, 3)], id="different value in closest"), + pytest.param({'a': 1}, {'b': 3}, [('a', 1, None)], id="missing key in closest"), + pytest.param({'a': 1}, {'a': 1}, [], id="no difference"), + pytest.param({'a': 1}, {}, [('a', 1, None)], id="missing key in empty closest"), + pytest.param({}, {'a': 1}, [], id="empty expected and extra key in closest"), + pytest.param({}, {}, [], id="empty"), + ], + ) + def test_dict_diff(self, expected, closest, diff): + assert similar.dict_diff(expected, closest) == diff + + @pytest.mark.parametrize( + "expected, closest, diff", + [ + pytest.param( + ['a:1', 'b:2'], ['a:1', 'c:2'], [('b', '2', None), ('c', None, '2')], id="missing tag and extra tag" + ), + pytest.param(['a:1', 'b:2'], ['a:2', 'b:2'], [('a', '1', '2')], id="different value in closest"), + pytest.param(['a:1'], ['a:1'], [], id="no difference"), + pytest.param([], [], [], id="empty lists"), + pytest.param(['a:1'], [], [("a", "1", None)], id="missing tag in empty closest"), + pytest.param([], ['a:1'], [("a", None, "1")], id="empty expected extra tag in closest"), + ], + ) + def test_tags_diff(self, expected, closest, diff): + assert similar.tags_diff(expected, closest) == diff diff --git a/datadog_checks_downloader/CHANGELOG.md b/datadog_checks_downloader/CHANGELOG.md index 84b75d6309dda..2516d2e2a3669 100644 --- a/datadog_checks_downloader/CHANGELOG.md +++ b/datadog_checks_downloader/CHANGELOG.md @@ -2,6 +2,12 @@ +## 7.0.0 / 2024-11-28 + +***Changed***: + +* v16 ceremony: bump root layout to v6. ([#19146](https://github.com/DataDog/integrations-core/pull/19146)) + ## 6.1.0 / 2024-10-31 ***Added***: diff --git a/datadog_checks_downloader/changelog.d/19146.changed b/datadog_checks_downloader/changelog.d/19146.changed deleted file mode 100644 index d23db1e8334f2..0000000000000 --- a/datadog_checks_downloader/changelog.d/19146.changed +++ /dev/null @@ -1 +0,0 @@ -v16 ceremony: bump root layout to v6. diff --git a/datadog_checks_downloader/datadog_checks/downloader/__about__.py b/datadog_checks_downloader/datadog_checks/downloader/__about__.py index 6bbcdb3fb6126..7f5e436a966ba 100644 --- a/datadog_checks_downloader/datadog_checks/downloader/__about__.py +++ b/datadog_checks_downloader/datadog_checks/downloader/__about__.py @@ -4,4 +4,4 @@ # NOTE: tie datadog-checks-downloader to v2 of our software supply chain: # https://github.com/DataDog/integrations-core/blob/6388602b6deb3b65b62cf7cda69dc20d99dede29/datadog_checks_downloader/datadog_checks/downloader/download.py#L51 -__version__ = "6.1.0" +__version__ = "7.0.0" diff --git a/datadog_checks_downloader/tests/test_downloader.py b/datadog_checks_downloader/tests/test_downloader.py index 35b9a57f2e581..1325aeaae5e70 100644 --- a/datadog_checks_downloader/tests/test_downloader.py +++ b/datadog_checks_downloader/tests/test_downloader.py @@ -53,6 +53,15 @@ "datadog-dd-cluster-agent", # excluding this since actual integration is called `datadog-cluster-agent` "datadog-kubernetes", # excluding this since `kubernetes` check is Agent v5 only "datadog-go-metro", # excluding this since `go-metro` check is Agent v5 only + "datadog-agent-metrics", # excluding this since `agent-metrics` check is Agent v5 only + "datadog-amazon-kafka", # excluding this since `amazon-kafka` wasn't an official release + "datadog-tokumx", # excluding this since `tokumx` was dropped in py3 + "datadog-ntp", # excluding this since `ntp` was Agent 5 only +] + +EXCLUDED_LOG_INTEGRATIONS = [ + # Temporary exclusion until we re-release the integration or come up with a better solution. + "datadog-zeek", # log only integration released by Florent. Will fail until we re-release it. ] # Specific integration versions released for the last time by a revoked developer but not shipped anymore. @@ -450,7 +459,7 @@ def test_downloader(): if not match: continue integration_name = match.group(1) - if integration_name in EXCLUDED_INTEGRATIONS: + if integration_name in EXCLUDED_INTEGRATIONS + EXCLUDED_LOG_INTEGRATIONS: continue if integration_name not in integrations_metadata: raise Exception( diff --git a/datadog_cluster_agent/changelog.d/19229.added b/datadog_cluster_agent/changelog.d/19229.added new file mode 100644 index 0000000000000..96fec650c7fe8 --- /dev/null +++ b/datadog_cluster_agent/changelog.d/19229.added @@ -0,0 +1 @@ +add telemetry for local load store in dca diff --git a/datadog_cluster_agent/datadog_checks/datadog_cluster_agent/check.py b/datadog_cluster_agent/datadog_checks/datadog_cluster_agent/check.py index e88ac2f4acdf6..e13153eb6a611 100644 --- a/datadog_cluster_agent/datadog_checks/datadog_cluster_agent/check.py +++ b/datadog_cluster_agent/datadog_checks/datadog_cluster_agent/check.py @@ -35,6 +35,8 @@ 'autoscaling_workload_vertical_rollout_triggered': 'autoscaling.workload.vertical_rollout_triggered', 'autoscaling_workload_vertical_scaling_received_limits': 'autoscaling.workload.vertical_scaling_received_limits', 'autoscaling_workload_vertical_scaling_received_requests': 'autoscaling.workload.vertical_scaling_received_requests', # noqa: E501 + 'autoscaling_workload_store_load_entities': 'autoscaling.workload.store_load_entities', + 'autoscaling_workload_store_job_queue_length': 'autoscaling.workload.store_job_queue_length', 'aggregator__flush': 'aggregator.flush', 'aggregator__processed': 'aggregator.processed', 'api_requests': 'api_requests', diff --git a/ddev/CHANGELOG.md b/ddev/CHANGELOG.md index 0327da035a2be..423e502adc0b3 100644 --- a/ddev/CHANGELOG.md +++ b/ddev/CHANGELOG.md @@ -2,6 +2,21 @@ +## 11.0.0 / 2024-12-09 + +***Removed***: + +* Remove manifest validation call that calls deprecated endpoint ([#19208](https://github.com/DataDog/integrations-core/pull/19208)) + +***Changed***: + +* Ddev now uses the macos-13 runner instead of macos-13 for the generated test matrix, because the macos-12 runner is being discontinued by microsoft. ([#19163](https://github.com/DataDog/integrations-core/pull/19163)) + +***Added***: + +* Add unit names for bolívar digital ([#19051](https://github.com/DataDog/integrations-core/pull/19051)) +* Bump dependencies for checking and fixing code style ([#19126](https://github.com/DataDog/integrations-core/pull/19126)) + ## 10.4.0 / 2024-11-13 ***Added***: diff --git a/ddev/changelog.d/17936.added b/ddev/changelog.d/17936.added new file mode 100644 index 0000000000000..25c19f52d2856 --- /dev/null +++ b/ddev/changelog.d/17936.added @@ -0,0 +1 @@ +Add script to convert monitor export json into the JSON we can use diff --git a/ddev/changelog.d/19051.added b/ddev/changelog.d/19051.added deleted file mode 100644 index 226133c429078..0000000000000 --- a/ddev/changelog.d/19051.added +++ /dev/null @@ -1 +0,0 @@ -Add unit names for bolívar digital diff --git a/ddev/changelog.d/19126.added b/ddev/changelog.d/19126.added deleted file mode 100644 index 778e71f577fa0..0000000000000 --- a/ddev/changelog.d/19126.added +++ /dev/null @@ -1 +0,0 @@ -Bump dependencies for checking and fixing code style diff --git a/ddev/src/ddev/cli/meta/scripts/__init__.py b/ddev/src/ddev/cli/meta/scripts/__init__.py index 2e8085a4bfc55..5605258b6499d 100644 --- a/ddev/src/ddev/cli/meta/scripts/__init__.py +++ b/ddev/src/ddev/cli/meta/scripts/__init__.py @@ -7,6 +7,7 @@ from datadog_checks.dev.tooling.commands.meta.scripts.remove_labels import remove_labels from ddev.cli.meta.scripts.generate_metrics import generate_metrics +from ddev.cli.meta.scripts.monitor import monitor from ddev.cli.meta.scripts.serve_openmetrics_payload import serve_openmetrics_payload from ddev.cli.meta.scripts.upgrade_python import upgrade_python @@ -24,3 +25,4 @@ def scripts(): scripts.add_command(remove_labels) scripts.add_command(serve_openmetrics_payload) scripts.add_command(upgrade_python) +scripts.add_command(monitor) diff --git a/ddev/src/ddev/cli/meta/scripts/monitor.py b/ddev/src/ddev/cli/meta/scripts/monitor.py new file mode 100644 index 0000000000000..43bb3576f8e67 --- /dev/null +++ b/ddev/src/ddev/cli/meta/scripts/monitor.py @@ -0,0 +1,68 @@ +import copy + +import click + +DESCRIPTION_SEED = """\ +This monitor will alert you on XXX... + +- Define the problem stated by the title. +- Answer why this is an issue worth alerting on. +- Describe the impact of the problem. + +Official guidelines: +https://docs.datadoghq.com/developers/integrations/create-an-integration-recommended-monitor/#description +""" + + +@click.group +def monitor(): + """ + Work with monitors. + """ + + +def _edit(text): + edited = click.edit(text=text, require_save=False) + return "" if edited is None else edited + + +def _drop_fields(exported): + x = copy.deepcopy(exported) + x.pop('id', None) + x['options'].pop('on_missing_data', None) + return x + + +@monitor.command +@click.argument("export_json", type=click.File()) +def create(export_json): + """ + Create monitor spec from the JSON export of the monitor in the UI. + + The exported monitor cannot be committed as-is, we have to rename, add, and drop some fields. + + After you've copied the JSON in the UI you can either save it as a file or pipe it to STDIN: + + \b + pbpaste | ddev meta monitor create - + """ + import json + from datetime import date + + exported = json.load(export_json) + today = date.today().isoformat() + wrangled = { + "version": 2, + "created_at": today, + "last_updated_at": today, + "title": _edit(text=exported["name"]).strip(), + "description": _edit(text=DESCRIPTION_SEED).strip(), + "tags": exported["tags"], + "definition": _drop_fields(exported), + } + click.echo( + json.dumps( + wrangled, + indent=2, + ) + ) diff --git a/ddev/src/ddev/cli/validate/__init__.py b/ddev/src/ddev/cli/validate/__init__.py index 95b2afaa0f721..5865a9ba66c7b 100644 --- a/ddev/src/ddev/cli/validate/__init__.py +++ b/ddev/src/ddev/cli/validate/__init__.py @@ -25,7 +25,6 @@ from ddev.cli.validate.http import http from ddev.cli.validate.labeler import labeler from ddev.cli.validate.licenses import licenses -from ddev.cli.validate.manifest import manifest from ddev.cli.validate.metadata import metadata from ddev.cli.validate.openmetrics import openmetrics from ddev.cli.validate.version import version @@ -54,7 +53,6 @@ def validate(): validate.add_command(legacy_signature) validate.add_command(license_headers) validate.add_command(licenses) -validate.add_command(manifest) validate.add_command(metadata) validate.add_command(models) validate.add_command(openmetrics) diff --git a/ddev/src/ddev/cli/validate/manifest.py b/ddev/src/ddev/cli/validate/manifest.py deleted file mode 100644 index cf8acdae216dc..0000000000000 --- a/ddev/src/ddev/cli/validate/manifest.py +++ /dev/null @@ -1,49 +0,0 @@ -# (C) Datadog, Inc. 2022-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -from __future__ import annotations - -from typing import TYPE_CHECKING - -import click - -if TYPE_CHECKING: - from ddev.cli.application import Application - - -@click.command(short_help='Validate integration manifests') -@click.argument('integrations', nargs=-1) -@click.pass_context -def manifest(ctx: click.Context, integrations: tuple[str, ...]): - """Validate integration manifests.""" - import httpx - - app: Application = ctx.obj - validation_tracker = app.create_validation_tracker('Manifests') - - dd_url = app.config.org.config.get('dd_url', '') - if not dd_url: - app.abort(f'No `dd_url` has been set for org `{app.config.org.name}`') - - validation_endpoint = f'{dd_url}/api/beta/apps/manifest/validate' - - for integration in app.repo.integrations.iter(integrations): - payload = {'data': {'type': 'app_manifest', 'attributes': integration.manifest.get('')}} - - try: - response = httpx.post(validation_endpoint, json=payload) - - if response.status_code == 400: - for error in response.json()['errors']: - validation_tracker.error((integration.display_name, 'manifest.json'), message=error) - else: - response.raise_for_status() - validation_tracker.success() - except Exception as e: - validation_tracker.error((integration.display_name, 'manifest.json'), message=str(e)) - - if validation_tracker.errors: - validation_tracker.display() - app.abort() - - validation_tracker.display() diff --git a/ddev/src/ddev/utils/scripts/ci_matrix.py b/ddev/src/ddev/utils/scripts/ci_matrix.py index 2c9da736a53c9..04895dbcfa991 100644 --- a/ddev/src/ddev/utils/scripts/ci_matrix.py +++ b/ddev/src/ddev/utils/scripts/ci_matrix.py @@ -74,8 +74,8 @@ 'linux': __plat('Linux', 'ubuntu-22.04'), # https://github.com/actions/runner-images/blob/main/images/win/Windows2022-Readme.md 'windows': __plat('Windows', 'windows-2022'), - # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md - 'macos': __plat('macOS', 'macos-12'), + # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md + 'macos': __plat('macOS', 'macos-13'), } diff --git a/ddev/tests/cli/validate/test_manifest.py b/ddev/tests/cli/validate/test_manifest.py deleted file mode 100644 index 5f5eb2084bc36..0000000000000 --- a/ddev/tests/cli/validate/test_manifest.py +++ /dev/null @@ -1,97 +0,0 @@ -# (C) Datadog, Inc. 2022-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -import json - -import pytest - -from ddev.utils.structures import EnvVars - - -@pytest.fixture(scope='module', autouse=True) -def terminal_width(): - with EnvVars({'COLUMNS': '200'}): - yield - - -def test_no_dd_url(ddev, helpers, config_file): - config_file.model.orgs['default']['dd_url'] = '' - config_file.save() - - result = ddev('validate', 'manifest', 'disk') - - assert result.exit_code == 1, result.output - assert result.output == helpers.dedent( - """ - No `dd_url` has been set for org `default` - """ - ) - - -def test_error_single_integration(ddev, repository, helpers, network_replay): - network_replay('manifest/missing_app_uuid.yaml', record_mode='none') - - check = 'mongo' - manifest_file = repository.path / check / 'manifest.json' - manifest = json.loads(manifest_file.read_text()) - del manifest['app_uuid'] - manifest_file.write_text(json.dumps(manifest)) - - result = ddev('validate', 'manifest', check) - - assert result.exit_code == 1, result.output - assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( - """ - Manifests - └── MongoDB - └── manifest.json - - API input validation failed: {'app_uuid': [u'Missing data for required field.']} - - Errors: 1 - """ - ) - - -def test_error_multiple_integrations(ddev, repository, helpers, network_replay): - network_replay('manifest/missing_app_uuid.yaml', record_mode='none') - - for check in ('mongo', 'vsphere'): - manifest_file = repository.path / check / 'manifest.json' - manifest = json.loads(manifest_file.read_text()) - del manifest['app_uuid'] - manifest_file.write_text(json.dumps(manifest)) - - result = ddev('validate', 'manifest') - - assert result.exit_code == 1, result.output - assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( - """ - Manifests - ├── MongoDB - │ └── manifest.json - │ - │ API input validation failed: {'app_uuid': [u'Missing data for required field.']} - └── vSphere - └── manifest.json - - API input validation failed: {'app_uuid': [u'Missing data for required field.']} - - Errors: 2 - """ - ) - - -def test_passing(ddev, helpers, network_replay): - network_replay('manifest/success.yaml', record_mode='none') - - result = ddev('validate', 'manifest', 'postgres') - - assert result.exit_code == 0, result.output - assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( - """ - Manifests - - Passed: 1 - """ - ) diff --git a/docs/developer/guidelines/dashboards.md b/docs/developer/guidelines/dashboards.md index 4b355ff0d9e91..eae2d680311eb 100644 --- a/docs/developer/guidelines/dashboards.md +++ b/docs/developer/guidelines/dashboards.md @@ -96,7 +96,7 @@ A dashboard that follows best practices helps users consume data quickly. Best p 1. If log collection is enabled, make a *Logs* group. Insert a timeseries widget showing a bar graph of logs by status over time. Also include a log stream of logs with the "Error" or "Critical" status. !!! tip - Consider turning groups into powerpacks if they appear repeatedly in dashboards irrespective of the integration type, so that you can insert the entire group with the correct formatting with a few clicks rather than adding the same widgets from scratch each time. + Consider turning groups into powerpacks if they appear repeatedly in dashboards irrespective of the integration type, so that you can insert the entire group with the correct formatting with a few clicks rather than adding the same widgets from scratch each time. ### Design Guidelines diff --git a/docs/developer/meta/ci/testing.md b/docs/developer/meta/ci/testing.md index 3377cdfb58406..f514b07df4f89 100644 --- a/docs/developer/meta/ci/testing.md +++ b/docs/developer/meta/ci/testing.md @@ -113,7 +113,7 @@ Configuration for targets [lives](https://github.com/DataDog/integrations-core/b | --- | --- | --- | | Linux | `linux` | [Ubuntu 22.04](https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md) | | Windows | `windows` | [Windows Server 2022](https://github.com/actions/runner-images/blob/main/images/win/Windows2022-Readme.md) | -| macOS | `macos` | [macOS 12](https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md) | +| macOS | `macos` | [macOS 13](https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md) | If an integration's `manifest.json` indicates that the only supported platform is Windows then that will be used to run tests, otherwise they will run on Linux. diff --git a/ecs_fargate/README.md b/ecs_fargate/README.md index 8a37e79439f9f..344df0a6fa375 100644 --- a/ecs_fargate/README.md +++ b/ecs_fargate/README.md @@ -18,7 +18,7 @@ The Task Metadata endpoint is only available from within the task definition its The only configuration required to enable this metrics collection is to set an environment variable `ECS_FARGATE` to `"true"` in the task definition. -**Note**: Network Performance Monitoring (NPM) is not supported for ECS Fargate. +**Note**: Cloud Network Monitoring (CNM) is not supported for ECS Fargate. ## Setup diff --git a/eks_fargate/README.md b/eks_fargate/README.md index e36fba6830831..f2640d6fc4703 100644 --- a/eks_fargate/README.md +++ b/eks_fargate/README.md @@ -7,7 +7,7 @@ Amazon EKS on AWS Fargate is a managed Kubernetes service that automates certain aspects of deployment and maintenance for any standard Kubernetes environment. Kubernetes nodes are managed by AWS Fargate and abstracted away from the user. -**Note**: Network Performance Monitoring (NPM) is not supported for EKS Fargate. +**Note**: Cloud Network Monitoring (CNM) is not supported for EKS Fargate. ## Setup diff --git a/envoy/tests/docker/api_v3/go.mod b/envoy/tests/docker/api_v3/go.mod index 031a86af1ccbe..c155a6ee937c9 100644 --- a/envoy/tests/docker/api_v3/go.mod +++ b/envoy/tests/docker/api_v3/go.mod @@ -1,9 +1,15 @@ module envoy-e2e -go 1.13 +go 1.16 require ( github.com/envoyproxy/go-control-plane v0.9.9 github.com/golang/protobuf v1.4.3 google.golang.org/grpc v1.36.0 ) + +replace github.com/envoyproxy/go-control-plane => github.com/envoyproxy/go-control-plane v0.9.9 + +replace github.com/golang/protobuf => github.com/golang/protobuf v1.4.3 + +replace google.golang.org/grpc => google.golang.org/grpc v1.36.0 diff --git a/envoy/tests/docker/api_v3/go.sum b/envoy/tests/docker/api_v3/go.sum index 15f632751d4cb..2b6350df19331 100644 --- a/envoy/tests/docker/api_v3/go.sum +++ b/envoy/tests/docker/api_v3/go.sum @@ -1,42 +1,20 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed h1:OZmjad4L3H8ncOIR8rnb5MREYqG8ixi5+WbeUsquF0c= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9 h1:vQLjymTobffN2R0F8eTqw6q7iozfRO5Z0m+/4Vw+/uA= github.com/envoyproxy/go-control-plane v0.9.9/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -52,66 +30,41 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/genesys/CHANGELOG.md b/genesys/CHANGELOG.md new file mode 100644 index 0000000000000..9fe6e7e4b6a5e --- /dev/null +++ b/genesys/CHANGELOG.md @@ -0,0 +1,7 @@ +# CHANGELOG - Genesys + +## 1.0.0 / 2024-11-13 + +***Added***: + +* Initial Release \ No newline at end of file diff --git a/genesys/README.md b/genesys/README.md new file mode 100644 index 0000000000000..63437522cef00 --- /dev/null +++ b/genesys/README.md @@ -0,0 +1,42 @@ +# Agent Check: genesys + +## Overview + +Genesys Cloud is a comprehensive cloud-based contact center platform that enables businesses to manage and optimize customer interactions across multiple channels, including voice, chat, email, social media, and messaging. It's known for its flexibility, scalability, and integration capabilities, helping businesses improve customer experience and streamline operations. + +## Setup + +### Installation + +The genesys check is included in the [Datadog Agent][2] package. +No additional installation is needed on your server. + +### Configuration + +!!! Add list of steps to set up this integration !!! + +### Validation + +!!! Add steps to validate integration is functioning as expected !!! + +## Data Collected + +### Logs + +The Genesys integration collects and forward logs to Datadog. + +### Metrics + +The Genesys integration collects and forward metrics to Datadog. + +### Events + +The Genesys integration does not include any events. + +## Troubleshooting + +Need help? Contact [Datadog support][3]. + +[1]: **LINK_TO_INTEGRATION_SITE** +[2]: https://app.datadoghq.com/account/settings/agent/latest +[3]: https://docs.datadoghq.com/help/ diff --git a/genesys/assets/service_checks.json b/genesys/assets/service_checks.json new file mode 100644 index 0000000000000..0637a088a01e8 --- /dev/null +++ b/genesys/assets/service_checks.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/genesys/manifest.json b/genesys/manifest.json new file mode 100644 index 0000000000000..7c26338ce0437 --- /dev/null +++ b/genesys/manifest.json @@ -0,0 +1,50 @@ +{ + "manifest_version": "2.0.0", + "app_uuid": "a0f0e600-1f10-4505-8fc3-64442e78b1a3", + "app_id": "genesys", + "display_on_public_website": false, + "tile": { + "overview": "README.md#Overview", + "configuration": "README.md#Setup", + "support": "README.md#Support", + "changelog": "CHANGELOG.md", + "description": "Gain insights into Conversation Analytics metrics and Audit logs", + "title": "Genesys", + "media": [], + "classifier_tags": [ + "Category::Log Collection", + "Category::Metrics", + "Category::Collaboration", + "Submitted Data Type::Logs", + "Submitted Data Type::Metrics", + "Offering::Integration" + ] + }, + "assets": { + "integration": { + "auto_install": false, + "source_type_id": 30683213, + "source_type_name": "Genesys", + "events": { + "creates_events": false + }, + "metrics": { + "prefix": "genesys.", + "check": [], + "metadata_path": "metadata.csv" + }, + "service_checks": { + "metadata_path": "assets/service_checks.json" + } + }, + "logs": { + "source": "genesys" + } + }, + "author": { + "support_email": "help@datadoghq.com", + "name": "Datadog", + "homepage": "https://www.datadoghq.com", + "sales_email": "info@datadoghq.com" + } +} \ No newline at end of file diff --git a/genesys/metadata.csv b/genesys/metadata.csv new file mode 100644 index 0000000000000..60d916455a42f --- /dev/null +++ b/genesys/metadata.csv @@ -0,0 +1 @@ +metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags \ No newline at end of file diff --git a/http_check/CHANGELOG.md b/http_check/CHANGELOG.md index dae4d86a8b0ea..6498ce6df9254 100644 --- a/http_check/CHANGELOG.md +++ b/http_check/CHANGELOG.md @@ -36,7 +36,7 @@ ***Added***: -* Update dependencies ([#18185](https://github.com/DataDog/integrations-core/pull/18185)) +* Update dependencies ([#18187](https://github.com/DataDog/integrations-core/pull/18187)) ## 9.7.0 / 2024-07-05 / Agent 7.56.0 diff --git a/hyperv/assets/dashboards/overview.json b/hyperv/assets/dashboards/overview.json index efc227a6a5699..3d2ad01e4fc0b 100644 --- a/hyperv/assets/dashboards/overview.json +++ b/hyperv/assets/dashboards/overview.json @@ -1,280 +1,366 @@ { - "title": "Hyper-V", - "description": "", - "widgets": [ - { - "id": 0, - "definition": { - "type": "query_value", - "requests": [ - { - "q": "avg:hyperv.hypervisor_logical_processor.total_run_time{$scope}", - "aggregator": "last", - "conditional_formats": [ - { - "comparator": ">", - "value": 90, - "palette": "white_on_red" - }, - { - "comparator": ">", - "value": 80, - "palette": "white_on_yellow" - }, - { - "comparator": "<", - "value": 80, - "palette": "white_on_green" - } - ] - } - ], - "custom_links": [], - "title": "Logical Processor Total Runtime %", - "title_size": "16", - "title_align": "left", - "autoscale": true, - "precision": 2 - }, - "layout": { - "x": 19, - "y": 38, - "width": 23, - "height": 11 - } - }, - { - "id": 1, - "definition": { - "type": "toplist", - "requests": [ - { - "q": "top(avg:hyperv.hypervisor_virtual_processor.total_run_time{$scope} by {instance}, 100, 'last', 'desc')", - "conditional_formats": [ - { - "comparator": ">", - "value": 90, - "palette": "white_on_red" - }, - { - "comparator": ">", - "value": 80, - "palette": "white_on_yellow" - }, - { - "comparator": "<=", - "value": 80, - "palette": "white_on_green" - } - ], - "style": { - "palette": "dog_classic" - } - } - ], - "custom_links": [], - "title": "VMs by Virtual Processor Runtime", - "title_size": "13", - "title_align": "left", - "time": { - "live_span": "1m" + "author_name": "Datadog", + "description": "", + "layout_type": "free", + "template_variables": [ + { + "available_values": [], + "default": "*", + "name": "scope" } - }, - "layout": { - "x": 1, - "y": 50, - "width": 41, - "height": 14 - } - }, - { - "id": 2, - "definition": { - "type": "timeseries", - "requests": [ - { - "q": "avg:hyperv.hypervisor_logical_processor.total_run_time{$scope}", - "display_type": "line", - "style": { - "palette": "dog_classic", - "line_type": "solid", - "line_width": "normal" + ], + "title": "Hyper-V", + "widgets": [ + { + "definition": { + "autoscale": true, + "custom_links": [], + "precision": 2, + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "palette": "white_on_red", + "value": 90 + }, + { + "comparator": ">", + "palette": "white_on_yellow", + "value": 80 + }, + { + "comparator": "<", + "palette": "white_on_green", + "value": 80 + } + ], + "queries": [ + { + "aggregator": "last", + "data_source": "metrics", + "name": "query1", + "query": "avg:hyperv.hypervisor_logical_processor.total_run_time{$scope}" + } + ], + "response_format": "scalar" + } + ], + "title": "Logical Processor Total Runtime %", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 0, + "layout": { + "height": 11, + "width": 23, + "x": 19, + "y": 38 } - } - ], - "custom_links": [], - "title": "CPU Usage by Physical Host", - "title_size": "16", - "title_align": "left", - "show_legend": true, - "legend_size": "0" - }, - "layout": { - "x": 43, - "y": 38, - "width": 61, - "height": 26 - } - }, - { - "id": 3, - "definition": { - "type": "timeseries", - "requests": [ - { - "q": "avg:hyperv.hypervisor_virtual_processor.total_run_time{$scope} by {instance}", - "display_type": "line", - "style": { - "palette": "dog_classic", - "line_type": "solid", - "line_width": "normal" + }, + { + "definition": { + "custom_links": [], + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "palette": "white_on_red", + "value": 90 + }, + { + "comparator": ">", + "palette": "white_on_yellow", + "value": 80 + }, + { + "comparator": "<=", + "palette": "white_on_green", + "value": 80 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "aggregator": "last", + "data_source": "metrics", + "name": "query1", + "query": "avg:hyperv.hypervisor_virtual_processor.total_run_time{$scope} by {instance}" + } + ], + "response_format": "scalar", + "sort": { + "count": 100, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + }, + "style": { + "palette": "dog_classic" + } + } + ], + "time": { + "live_span": "1m" + }, + "title": "VMs by Virtual Processor Runtime", + "title_align": "left", + "title_size": "13", + "type": "toplist" + }, + "id": 1, + "layout": { + "height": 14, + "width": 41, + "x": 1, + "y": 50 } - } - ], - "custom_links": [], - "yaxis": { - "min": "0", - "max": "100" }, - "title": "CPU Usage by VM", - "title_size": "16", - "title_align": "left", - "show_legend": true, - "legend_size": "0" - }, - "layout": { - "x": 43, - "y": 19, - "width": 61, - "height": 18 - } - }, - { - "id": 4, - "definition": { - "type": "query_value", - "requests": [ - { - "q": "count:hyperv.hypervisor_logical_processor.guest_run_time{$scope,!instance:_total}", - "aggregator": "last" - } - ], - "custom_links": [], - "title": "Instance Count", - "title_size": "16", - "title_align": "left", - "time": { - "live_span": "1m" + { + "definition": { + "custom_links": [], + "legend_size": "0", + "requests": [ + { + "display_type": "line", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:hyperv.hypervisor_logical_processor.total_run_time{$scope}" + } + ], + "response_format": "timeseries", + "style": { + "line_type": "solid", + "line_width": "normal", + "palette": "dog_classic" + } + } + ], + "show_legend": true, + "title": "CPU Usage by Physical Host", + "title_align": "left", + "title_size": "16", + "type": "timeseries" + }, + "id": 2, + "layout": { + "height": 26, + "width": 61, + "x": 43, + "y": 38 + } }, - "autoscale": true, - "precision": 2 - }, - "layout": { - "x": 1, - "y": 38, - "width": 17, - "height": 11 - } - }, - { - "id": 5, - "definition": { - "type": "image", - "url": "https://s25966.pcdn.co/hyper-v/wp-content/uploads/2017/12/5B5EFCA7-DF8C-4123-AF48-FA67F883AD2B.jpeg", - "sizing": "zoom" - }, - "layout": { - "x": 1, - "y": 3, - "width": 41, - "height": 15 - } - }, - { - "id": 6, - "definition": { - "type": "query_value", - "requests": [ - { - "q": "avg:hyperv.dynamic_memory_balancer.available_memory{$scope}", - "aggregator": "avg" - } - ], - "custom_links": [], - "title": "Average Available Memory (bytes)", - "title_size": "16", - "title_align": "left", - "autoscale": true, - "precision": 2 - }, - "layout": { - "x": 43, - "y": 3, - "width": 26, - "height": 15 - } - }, - { - "id": 7, - "definition": { - "type": "hostmap", - "requests": { - "fill": { - "q": "avg:hyperv.hypervisor_logical_processor.guest_run_time{*} by {host}" - } + { + "definition": { + "custom_links": [], + "legend_size": "0", + "requests": [ + { + "display_type": "line", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:hyperv.hypervisor_virtual_processor.total_run_time{$scope} by {instance}" + } + ], + "response_format": "timeseries", + "style": { + "line_type": "solid", + "line_width": "normal", + "palette": "dog_classic" + } + } + ], + "show_legend": true, + "title": "CPU Usage by VM", + "title_align": "left", + "title_size": "16", + "type": "timeseries", + "yaxis": { + "max": "100", + "min": "0" + } + }, + "id": 3, + "layout": { + "height": 18, + "width": 61, + "x": 43, + "y": 19 + } }, - "custom_links": [], - "title": "CPU Usage by Physical Host", - "title_size": "16", - "title_align": "left", - "no_metric_hosts": false, - "no_group_hosts": true, - "group": [], - "style": { - "palette": "green_to_orange", - "palette_flip": false - } - }, - "layout": { - "x": 1, - "y": 19, - "width": 41, - "height": 18 - } - }, - { - "id": 8, - "definition": { - "type": "toplist", - "requests": [ - { - "q": "top(avg:hyperv.dynamic_memory_balancer.available_memory{$scope}, 10, 'mean', 'asc')", - "style": { - "palette": "dog_classic" + { + "definition": { + "autoscale": true, + "custom_links": [], + "precision": 2, + "requests": [ + { + "queries": [ + { + "aggregator": "last", + "data_source": "metrics", + "name": "query1", + "query": "count:hyperv.hypervisor_logical_processor.guest_run_time{$scope,!instance:_total}" + } + ], + "response_format": "scalar" + } + ], + "time": { + "live_span": "1m" + }, + "title": "Instance Count", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 4, + "layout": { + "height": 11, + "width": 17, + "x": 1, + "y": 38 + } + }, + { + "definition": { + "has_background": false, + "has_border": false, + "horizontal_align": "center", + "margin": "md", + "sizing": "cover", + "type": "image", + "url": "/static/images/logos/hyper-v_large.svg", + "url_dark_theme": "/static/images/logos/hyper-v_reversed_large.svg", + "vertical_align": "center" + }, + "id": 5, + "layout": { + "height": 15, + "width": 41, + "x": 1, + "y": 3 } - } - ], - "custom_links": [], - "title": "Least Available Memory by Host", - "title_size": "16", - "title_align": "left" - }, - "layout": { - "x": 70, - "y": 3, - "width": 34, - "height": 15 - } - } - ], - "template_variables": [ - { - "name": "scope", - "default": "*", - "prefix": null - } - ], - "layout_type": "free", - "is_read_only": true, - "notify_list": [] -} + }, + { + "definition": { + "autoscale": true, + "custom_links": [], + "precision": 2, + "requests": [ + { + "queries": [ + { + "aggregator": "avg", + "data_source": "metrics", + "name": "query1", + "query": "avg:hyperv.dynamic_memory_balancer.available_memory{$scope}" + } + ], + "response_format": "scalar" + } + ], + "title": "Average Available Memory (bytes)", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 6, + "layout": { + "height": 15, + "width": 26, + "x": 43, + "y": 3 + } + }, + { + "definition": { + "custom_links": [], + "group": [], + "no_group_hosts": true, + "no_metric_hosts": false, + "requests": { + "fill": { + "q": "avg:hyperv.hypervisor_logical_processor.guest_run_time{*} by {host}" + } + }, + "style": { + "palette": "green_to_orange", + "palette_flip": false + }, + "title": "CPU Usage by Physical Host", + "title_align": "left", + "title_size": "16", + "type": "hostmap" + }, + "id": 7, + "layout": { + "height": 18, + "width": 41, + "x": 1, + "y": 19 + } + }, + { + "definition": { + "custom_links": [], + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "aggregator": "avg", + "data_source": "metrics", + "name": "query1", + "query": "avg:hyperv.dynamic_memory_balancer.available_memory{$scope}" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "asc", + "type": "formula" + } + ] + }, + "style": { + "palette": "dog_classic" + } + } + ], + "title": "Least Available Memory by Host", + "title_align": "left", + "title_size": "16", + "type": "toplist" + }, + "id": 8, + "layout": { + "height": 15, + "width": 34, + "x": 70, + "y": 3 + } + } + ] +} \ No newline at end of file diff --git a/iam_access_analyzer/README.md b/iam_access_analyzer/README.md index 9e4985f9b34f5..97355dd5464f5 100644 --- a/iam_access_analyzer/README.md +++ b/iam_access_analyzer/README.md @@ -48,6 +48,6 @@ This integration does not include any events. Need help? Contact [Datadog support][3]. -[1]: /logs/guide/forwarder/ +[1]: https://docs.datadoghq.com/logs/guide/forwarder/ [2]: https://app.datadoghq.com/logs?query=source%3Aaccess-analyzer [3]: https://docs.datadoghq.com/help diff --git a/ibm_db2/CHANGELOG.md b/ibm_db2/CHANGELOG.md index 61a94adcc26e6..0ba2aed87cfc2 100644 --- a/ibm_db2/CHANGELOG.md +++ b/ibm_db2/CHANGELOG.md @@ -2,6 +2,12 @@ +## 4.0.1 / 2024-11-28 + +***Fixed***: + +* Fixes 'unable to import module' on Windows ([#18908](https://github.com/DataDog/integrations-core/pull/18908)) + ## 4.0.0 / 2024-10-04 / Agent 7.59.0 ***Removed***: diff --git a/ibm_db2/changelog.d/18908.fixed b/ibm_db2/changelog.d/18908.fixed deleted file mode 100644 index 3dadc489fa0d5..0000000000000 --- a/ibm_db2/changelog.d/18908.fixed +++ /dev/null @@ -1 +0,0 @@ -Fixes 'unable to import module' on Windows diff --git a/ibm_db2/datadog_checks/ibm_db2/__about__.py b/ibm_db2/datadog_checks/ibm_db2/__about__.py index 78967d676d1af..59269a7bb5672 100644 --- a/ibm_db2/datadog_checks/ibm_db2/__about__.py +++ b/ibm_db2/datadog_checks/ibm_db2/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2019-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '4.0.0' +__version__ = '4.0.1' diff --git a/ibm_mq/README.md b/ibm_mq/README.md index 8bb80e851e4cd..e301efa22d713 100644 --- a/ibm_mq/README.md +++ b/ibm_mq/README.md @@ -139,6 +139,13 @@ Configure the environment variable `MQ_FILE_PATH`, to point at the data director There are many ways to set up permissions in IBM MQ. Depending on how your setup works, create a `datadog` user within MQ with read only permissions and, optionally, `+chg` permissions. `+chg` permissions are required to collect metrics for [reset queue statistics][14] (`MQCMD_RESET_Q_STATS`). If you do not wish to collect these metrics you can disable `collect_reset_queue_metrics` on the configuration. Collecting reset queue statistics performance data will also reset the performance data. +The example below sets the required permissions on the queue manager `QM1` for the `mqclient` group, the group the `datadog` user is using to execute commands. You can use wildcards to grant permissions to many queues at once. + +{{< code-block lang="shell" >}} +setmqaut -m QM1 -n SYSTEM.ADMIN.COMMAND.QUEUE -t queue -g mqclient +dsp +inq +get +put +setmqaut -m QM1 -n SYSTEM.MQEXPLORER.REPLY.MODEL -t queue -g mqclient +dsp +inq +get +put +{{< /code-block >}} + **Note**: "Queue Monitoring" must be enabled on the MQ server and set to at least "Medium". This can be done using the MQ UI or with an `mqsc` command in the server's host: ```text diff --git a/istio/README.md b/istio/README.md index f89340b6a9f84..2d657017a5b0c 100644 --- a/istio/README.md +++ b/istio/README.md @@ -5,7 +5,7 @@ Datadog monitors every aspect of your Istio environment, so you can: - Assess the health of Envoy and the Istio control plane with [logs](#log-collection). - Break down the performance of your service mesh with [request, bandwidth, and resource consumption metrics](#metrics). -- Map network communication between containers, pods, and services over the mesh with [Network Performance Monitoring][1]. +- Map network communication between containers, pods, and services over the mesh with [Cloud Network Monitoring][1]. - Drill into distributed traces for applications transacting over the mesh with [APM][2]. To learn more about monitoring your Istio environment with Datadog, [see the Monitor blog post][3]. diff --git a/kafka_consumer/CHANGELOG.md b/kafka_consumer/CHANGELOG.md index 33716717ce50e..7e993f84c168e 100644 --- a/kafka_consumer/CHANGELOG.md +++ b/kafka_consumer/CHANGELOG.md @@ -38,7 +38,7 @@ ***Added***: -* Update dependencies ([#18185](https://github.com/DataDog/integrations-core/pull/18185)) +* Update dependencies ([#18187](https://github.com/DataDog/integrations-core/pull/18187)) ## 4.5.0 / 2024-07-05 / Agent 7.56.0 diff --git a/kafka_consumer/datadog_checks/kafka_consumer/client.py b/kafka_consumer/datadog_checks/kafka_consumer/client.py index 288354acd49db..6c93f8e3887fd 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/client.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/client.py @@ -6,15 +6,13 @@ from confluent_kafka import Consumer, ConsumerGroupTopicPartitions, KafkaException, TopicPartition from confluent_kafka.admin import AdminClient -from datadog_checks.kafka_consumer.constants import KAFKA_INTERNAL_TOPICS, OFFSET_INVALID - class KafkaClient: def __init__(self, config, log) -> None: self.config = config self.log = log self._kafka_client = None - self.topic_partition_cache = {} + self._consumer = None @property def kafka_client(self): @@ -31,7 +29,7 @@ def kafka_client(self): return self._kafka_client - def __create_consumer(self, consumer_group): + def open_consumer(self, consumer_group): config = { "bootstrap.servers": self.config._kafka_connect_str, "group.id": consumer_group, @@ -41,7 +39,12 @@ def __create_consumer(self, consumer_group): } config.update(self.__get_authentication_config()) - return Consumer(config, logger=self.log) + self._consumer = Consumer(config, logger=self.log) + self.log.debug("Consumer instance %s created for group %s", self._consumer, consumer_group) + + def close_consumer(self): + self.log.debug("Closing consumer instance %s", self._consumer) + self._consumer.close() def __get_authentication_config(self): config = { @@ -79,252 +82,105 @@ def __get_authentication_config(self): return config - def get_highwater_offsets(self, consumer_offsets): - self.log.debug('Getting highwater offsets') - - cluster_id = "" - highwater_offsets = {} - topics_with_consumer_offset = set() - topic_partition_with_consumer_offset = set() - - if not self.config._monitor_all_broker_highwatermarks: - for _, topic, partition in consumer_offsets: - topics_with_consumer_offset.add(topic) - topic_partition_with_consumer_offset.add((topic, partition)) - - topic_partition_checked = set() - - for consumer_group, _topic, _partition in consumer_offsets: - self.log.debug('CONSUMER GROUP: %s', consumer_group) - if (_topic, _partition) in topic_partition_checked: - self.log.debug('Highwater offset already collected for topic %s with partition %s', _topic, _partition) - continue - - topic_partitions_for_highwater_offsets = set() - - consumer = self.__create_consumer(consumer_group) - self.log.debug("Consumer instance %s created for group %s", consumer, consumer_group) - cluster_metadata = consumer.list_topics(timeout=self.config._request_timeout) - try: - cluster_id = cluster_metadata.cluster_id - except AttributeError: - self.log.error("Failed to get cluster metadata for consumer group %s", consumer_group) - topics = cluster_metadata.topics - - for topic in topics: - if topic in KAFKA_INTERNAL_TOPICS: - self.log.debug("Skipping internal topic %s", topic) - continue - if not self.config._monitor_all_broker_highwatermarks and topic not in topics_with_consumer_offset: - self.log.debug("Skipping non-relevant topic %s", topic) - continue - - for partition in topics[topic].partitions: - if ( - self.config._monitor_all_broker_highwatermarks - or (topic, partition) in topic_partition_with_consumer_offset - ): - # Setting offset to -1 will return the latest highwater offset while calling offsets_for_times - # Reference: https://github.com/fede1024/rust-rdkafka/issues/460 - topic_partitions_for_highwater_offsets.add( - TopicPartition(topic=topic, partition=partition, offset=-1) - ) - self.log.debug('TOPIC: %s', topic) - self.log.debug('PARTITION: %s', partition) - else: - self.log.debug("Skipping non-relevant partition %s of topic %s", partition, topic) - - if len(topic_partitions_for_highwater_offsets) > 0: - self.log.debug( - 'Querying %s highwater offsets for consumer group %s', - len(topic_partitions_for_highwater_offsets), - consumer_group, - ) - for topic_partition_with_highwater_offset in consumer.offsets_for_times( - partitions=list(topic_partitions_for_highwater_offsets), - timeout=self.config._request_timeout, - ): - self.log.debug('Topic partition with highwater offset: %s', topic_partition_with_highwater_offset) - topic = topic_partition_with_highwater_offset.topic - partition = topic_partition_with_highwater_offset.partition - offset = topic_partition_with_highwater_offset.offset - highwater_offsets[(topic, partition)] = offset - self.log.debug("Adding %s %s to checked set to facilitate early exit", topic, partition) - topic_partition_checked.add((topic, partition)) - else: - self.log.debug('No new highwater offsets to query for consumer group %s', consumer_group) - - self.log.debug("Closing consumer instance %s", consumer) - consumer.close() - - self.log.debug('Got %s highwater offsets', len(highwater_offsets)) - return highwater_offsets, cluster_id + def consumer_get_cluster_id_and_list_topics(self, consumer_group): + cluster_metadata = self._consumer.list_topics(timeout=self.config._request_timeout) + try: + # TODO: remove this try-except, the attribute is always present. + cluster_id = cluster_metadata.cluster_id + except AttributeError: + self.log.error("Failed to get cluster metadata for consumer group %s", consumer_group) + return "", [] + return (cluster_id, [(name, list(metadata.partitions)) for name, metadata in cluster_metadata.topics.items()]) + + def consumer_offsets_for_times(self, partitions): + topicpartitions_for_querying = [ + # Setting offset to -1 will return the latest highwater offset while calling offsets_for_times + # Reference: https://github.com/fede1024/rust-rdkafka/issues/460 + TopicPartition(topic=topic, partition=partition, offset=-1) + for topic, partition in partitions + ] + return [ + (tp.topic, tp.partition, tp.offset) + for tp in self._consumer.offsets_for_times( + partitions=topicpartitions_for_querying, timeout=self.config._request_timeout + ) + ] def get_partitions_for_topic(self, topic): - if partitions := self.topic_partition_cache.get(topic): - return partitions - try: cluster_metadata = self.kafka_client.list_topics(topic, timeout=self.config._request_timeout) except KafkaException as e: self.log.error("Received exception when getting partitions for topic %s: %s", topic, e) - return None - else: - topic_metadata = cluster_metadata.topics[topic] - partitions = list(topic_metadata.partitions.keys()) - self.topic_partition_cache[topic] = partitions - return partitions + return [] + topic_metadata = cluster_metadata.topics[topic] + return list(topic_metadata.partitions) def request_metadata_update(self): # https://github.com/confluentinc/confluent-kafka-python/issues/594 self.kafka_client.list_topics(None, timeout=self.config._request_timeout) - def get_consumer_offsets(self): - # {(consumer_group, topic, partition): offset} - self.log.debug('Getting consumer offsets') - consumer_offsets = {} - - consumer_groups = self._get_consumer_groups() - self.log.debug('Identified %s consumer groups', len(consumer_groups)) + def list_consumer_groups(self): + groups = [] + try: + groups_res = self.kafka_client.list_consumer_groups().result() + for valid_group in groups_res.valid: + self.log.debug("Discovered consumer group: %s", valid_group.group_id) + groups.append(valid_group.group_id) + except Exception as e: + self.log.error("Failed to collect consumer groups: %s", e) + return groups + + def list_consumer_group_offsets(self, groups): + """ + For every group and (optionally) its topics and partitions retrieve consumer offsets. - futures = self._get_consumer_offset_futures(consumer_groups) - self.log.debug('%s futures to be waited on', len(futures)) + As input expects a list of tuples: (consumer_group_id, topic_partitions). + topic_partitions are either None to indicate we want all topics and partitions OR a list of (topic, partition). - for future in as_completed(futures): + Returns a list of tuples with members: + 1. group id + 2. list of tuples: (topic, partition, offset) + """ + futures = [] + for consumer_group, topic_partitions in groups: + topic_partitions = ( + topic_partitions if topic_partitions is None else [TopicPartition(t, p) for t, p in topic_partitions] + ) + futures.append( + self.kafka_client.list_consumer_group_offsets( + [ConsumerGroupTopicPartitions(group_id=consumer_group, topic_partitions=topic_partitions)] + )[consumer_group] + ) + offsets = [] + for completed in as_completed(futures): try: - response_offset_info = future.result() + response_offset_info = completed.result() except KafkaException as e: - self.log.debug("Failed to read consumer offsets for future %s: %s", future, e) - else: - consumer_group = response_offset_info.group_id - topic_partitions = response_offset_info.topic_partitions - - self.log.debug('RESULT CONSUMER GROUP: %s', consumer_group) - self.log.debug('RESULT TOPIC PARTITIONS: %s', topic_partitions) - - for topic_partition in topic_partitions: - topic = topic_partition.topic - partition = topic_partition.partition - offset = topic_partition.offset - - self.log.debug('RESULTS TOPIC: %s', topic) - self.log.debug('RESULTS PARTITION: %s', partition) - self.log.debug('RESULTS OFFSET: %s', offset) - - if topic_partition.error: - self.log.debug( - "Encountered error: %s. Occurred with topic: %s; partition: [%s]", - topic_partition.error.str(), - topic_partition.topic, - str(topic_partition.partition), - ) - continue - - if offset == OFFSET_INVALID: - continue - - if self.config._monitor_unlisted_consumer_groups or not self.config._consumer_groups_compiled_regex: - consumer_offsets[(consumer_group, topic, partition)] = offset - else: - to_match = f"{consumer_group},{topic},{partition}" - if self.config._consumer_groups_compiled_regex.match(to_match): - consumer_offsets[(consumer_group, topic, partition)] = offset - - self.log.debug('Got %s consumer offsets', len(consumer_offsets)) - return consumer_offsets - - def _get_consumer_groups(self): - # Get all consumer groups to monitor - consumer_groups = [] - if self.config._monitor_unlisted_consumer_groups or self.config._consumer_groups_compiled_regex: - consumer_groups_future = self.kafka_client.list_consumer_groups() - try: - list_consumer_groups_result = consumer_groups_future.result() - for valid_consumer_group in list_consumer_groups_result.valid: - self.log.debug("Discovered consumer group: %s", valid_consumer_group.group_id) - - consumer_groups.extend( - valid_consumer_group.group_id - for valid_consumer_group in list_consumer_groups_result.valid - if valid_consumer_group.group_id != "" - ) - except Exception as e: - self.log.error("Failed to collect consumer groups: %s", e) - return consumer_groups - else: - return self.config._consumer_groups - - def get_consumer_group_state(self, consumer_group): - consumer_group_state = "" - # Get the consumer group state if present - consumer_groups_future = self._describe_consumer_groups(consumer_group) - consumer_groups_result = consumer_groups_future[consumer_group].result() - self.log.debug( - "Consumer group: %s in state %s", - consumer_groups_result.group_id, - consumer_groups_result.state, - ) - consumer_group_result_state = str(consumer_groups_result.state) - consumer_group_state = consumer_group_result_state.split('.')[1] - - return consumer_group_state - - def _list_consumer_group_offsets(self, cg_tp): - """ - :returns: A dict of futures for each group, keyed by the group id. - The future result() method returns :class:`ConsumerGroupTopicPartitions`. - - :rtype: dict[str, future] - """ - return self.kafka_client.list_consumer_group_offsets([cg_tp]) + self.log.debug("Failed to read consumer offsets for future %s: %s", completed, e) + continue + tpo = [] + for tp in response_offset_info.topic_partitions: + if tp.error: + self.log.debug( + "Encountered error: %s. Occurred with topic: %s; partition: [%s]", + tp.error.str(), + tp.topic, + str(tp.partition), + ) + continue + tpo.append((tp.topic, tp.partition, tp.offset)) + offsets.append((response_offset_info.group_id, tpo)) + return offsets - def _describe_consumer_groups(self, consumer_group): + def describe_consumer_groups(self, consumer_group): """ :returns: A dict of futures for each group, keyed by the group_id. The future result() method returns :class:`ConsumerGroupDescription`. :rtype: dict[str, future] """ - return self.kafka_client.describe_consumer_groups([consumer_group]) + desc = self.kafka_client.describe_consumer_groups([consumer_group])[consumer_group].result() + return (desc.group_id, desc.state.value) def close_admin_client(self): self._kafka_client = None - - def _get_consumer_offset_futures(self, consumer_groups): - futures = [] - - # If either monitoring all consumer groups or regex, return all consumer group offsets (can filter later) - if self.config._monitor_unlisted_consumer_groups or self.config._consumer_groups_compiled_regex: - for consumer_group in consumer_groups: - futures.append( - self._list_consumer_group_offsets(ConsumerGroupTopicPartitions(consumer_group))[consumer_group] - ) - return futures - - for consumer_group in consumer_groups: - # If topics are specified - topics = consumer_groups.get(consumer_group) - if not topics: - futures.append( - self._list_consumer_group_offsets(ConsumerGroupTopicPartitions(consumer_group))[consumer_group] - ) - continue - - for topic in topics: - # If partitions are defined - if partitions := topics[topic]: - topic_partitions = [TopicPartition(topic, partition) for partition in partitions] - # If partitions are not defined - else: - # get all the partitions for this topic - partitions = self.get_partitions_for_topic(topic) - - topic_partitions = [TopicPartition(topic, partition) for partition in partitions] - - futures.append( - self._list_consumer_group_offsets(ConsumerGroupTopicPartitions(consumer_group, topic_partitions))[ - consumer_group - ] - ) - - return futures diff --git a/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py b/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py index ca5887039aed5..554ef4f6d3373 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py @@ -8,6 +8,7 @@ from datadog_checks.base import AgentCheck, is_affirmative from datadog_checks.kafka_consumer.client import KafkaClient from datadog_checks.kafka_consumer.config import KafkaConfig +from datadog_checks.kafka_consumer.constants import KAFKA_INTERNAL_TOPICS, OFFSET_INVALID MAX_TIMESTAMPS = 1000 @@ -22,6 +23,7 @@ def __init__(self, name, init_config, instances): self._data_streams_enabled = is_affirmative(self.instance.get('data_streams_enabled', False)) self._max_timestamps = int(self.instance.get('timestamp_history_size', MAX_TIMESTAMPS)) self.client = KafkaClient(self.config, self.log) + self.topic_partition_cache = {} self.check_initializations.insert(0, self.config.validate_config) def check(self, _): @@ -40,7 +42,7 @@ def check(self, _): try: # Fetch consumer offsets # Expected format: {(consumer_group, topic, partition): offset} - consumer_offsets = self.client.get_consumer_offsets() + consumer_offsets = self.get_consumer_offsets() except Exception: self.log.exception("There was a problem collecting consumer offsets from Kafka.") # don't raise because we might get valid broker offsets @@ -54,7 +56,7 @@ def check(self, _): if len(consumer_offsets) < self._context_limit: # Fetch highwater offsets # Expected format: ({(topic, partition): offset}, cluster_id) - highwater_offsets, cluster_id = self.client.get_highwater_offsets(consumer_offsets) + highwater_offsets, cluster_id = self.get_highwater_offsets(consumer_offsets) if self._data_streams_enabled: broker_timestamps = self._load_broker_timestamps(persistent_cache_key) self._add_broker_timestamps(broker_timestamps, highwater_offsets) @@ -95,6 +97,74 @@ def check(self, _): if self.config._close_admin_client: self.client.close_admin_client() + def get_consumer_offsets(self): + # {(consumer_group, topic, partition): offset} + self.log.debug('Getting consumer offsets') + consumer_offsets = {} + + consumer_groups = self._get_consumer_groups() + self.log.debug('Identified %s consumer groups', len(consumer_groups)) + + offsets = self._get_offsets_for_groups(consumer_groups) + self.log.debug('%s futures to be waited on', len(offsets)) + + for consumer_group, topic_partitions in offsets: + + self.log.debug('RESULT CONSUMER GROUP: %s', consumer_group) + + for topic, partition, offset in topic_partitions: + self.log.debug('RESULTS TOPIC: %s', topic) + self.log.debug('RESULTS PARTITION: %s', partition) + self.log.debug('RESULTS OFFSET: %s', offset) + + if offset == OFFSET_INVALID: + continue + + if self.config._monitor_unlisted_consumer_groups or not self.config._consumer_groups_compiled_regex: + consumer_offsets[(consumer_group, topic, partition)] = offset + else: + to_match = f"{consumer_group},{topic},{partition}" + if self.config._consumer_groups_compiled_regex.match(to_match): + consumer_offsets[(consumer_group, topic, partition)] = offset + + self.log.debug('Got %s consumer offsets', len(consumer_offsets)) + return consumer_offsets + + def _get_consumer_groups(self): + # Get all consumer groups to monitor + if self.config._monitor_unlisted_consumer_groups or self.config._consumer_groups_compiled_regex: + return [grp for grp in self.client.list_consumer_groups() if grp] + else: + return self.config._consumer_groups + + def _get_offsets_for_groups(self, consumer_groups): + groups = [] + + # If either monitoring all consumer groups or regex, return all consumer group offsets (can filter later) + if self.config._monitor_unlisted_consumer_groups or self.config._consumer_groups_compiled_regex: + for consumer_group in consumer_groups: + groups.append((consumer_group, None)) + return self.client.list_consumer_group_offsets(groups) + + for consumer_group in consumer_groups: + # If topics are specified + topics = consumer_groups.get(consumer_group) + if not topics: + groups.append((consumer_group, None)) + continue + + for topic, partitions in topics.items(): + if not partitions: + if topic in self.topic_partition_cache: + partitions = self.topic_partition_cache[topic] + else: + partitions = self.topic_partition_cache[topic] = self.client.get_partitions_for_topic(topic) + topic_partitions = [(topic, p) for p in partitions] + + groups.append((consumer_group, topic_partitions)) + + return self.client.list_consumer_group_offsets(groups) + def _load_broker_timestamps(self, persistent_cache_key): """Loads broker timestamps from persistent cache.""" broker_timestamps = defaultdict(dict) @@ -139,7 +209,7 @@ def report_consumer_offsets_and_lag( reported_contexts = 0 self.log.debug("Reporting consumer offsets and lag metrics") for (consumer_group, topic, partition), consumer_offset in consumer_offsets.items(): - consumer_group_state = self.client.get_consumer_group_state(consumer_group) + consumer_group_state = self.get_consumer_group_state(consumer_group) if reported_contexts >= contexts_limit: self.log.debug( "Reported contexts number %s greater than or equal to contexts limit of %s, returning", @@ -207,7 +277,7 @@ def report_consumer_offsets_and_lag( self.gauge('estimated_consumer_lag', lag, tags=consumer_group_tags) reported_contexts += 1 else: - if partitions is None: + if not partitions: msg = ( "Consumer group: %s has offsets for topic: %s, partition: %s, but that topic has no partitions " "in the cluster, so skipping reporting these offsets." @@ -221,6 +291,82 @@ def report_consumer_offsets_and_lag( self.client.request_metadata_update() # force metadata update on next poll() self.log.debug('%s consumer offsets reported', reported_contexts) + def get_consumer_group_state(self, consumer_group): + consumer_group_state = "" + # Get the consumer group state if present + group_id, consumer_group_state = self.client.describe_consumer_groups(consumer_group) + self.log.debug( + "Consumer group: %s in state %s", + group_id, + consumer_group_state, + ) + return consumer_group_state + + def get_highwater_offsets(self, consumer_offsets): + self.log.debug('Getting highwater offsets') + + cluster_id = "" + highwater_offsets = {} + topics_with_consumer_offset = set() + topic_partition_with_consumer_offset = set() + + if not self.config._monitor_all_broker_highwatermarks: + for _, topic, partition in consumer_offsets: + topics_with_consumer_offset.add(topic) + topic_partition_with_consumer_offset.add((topic, partition)) + + topic_partition_checked = set() + + for consumer_group, _topic, _partition in consumer_offsets: + self.log.debug('CONSUMER GROUP: %s', consumer_group) + if (_topic, _partition) in topic_partition_checked: + self.log.debug('Highwater offset already collected for topic %s with partition %s', _topic, _partition) + continue + + topic_partitions_for_highwater_offsets = set() + + self.client.open_consumer(consumer_group) + cluster_id, topics = self.client.consumer_get_cluster_id_and_list_topics(consumer_group) + + for topic, partitions in topics: + if topic in KAFKA_INTERNAL_TOPICS: + self.log.debug("Skipping internal topic %s", topic) + continue + if not self.config._monitor_all_broker_highwatermarks and topic not in topics_with_consumer_offset: + self.log.debug("Skipping non-relevant topic %s", topic) + continue + + for partition in partitions: + if ( + self.config._monitor_all_broker_highwatermarks + or (topic, partition) in topic_partition_with_consumer_offset + ): + topic_partitions_for_highwater_offsets.add((topic, partition)) + self.log.debug('TOPIC: %s', topic) + self.log.debug('PARTITION: %s', partition) + else: + self.log.debug("Skipping non-relevant partition %s of topic %s", partition, topic) + + if len(topic_partitions_for_highwater_offsets) > 0: + self.log.debug( + 'Querying %s highwater offsets for consumer group %s', + len(topic_partitions_for_highwater_offsets), + consumer_group, + ) + for topic, partition, offset in self.client.consumer_offsets_for_times( + partitions=topic_partitions_for_highwater_offsets + ): + highwater_offsets[(topic, partition)] = offset + self.log.debug("Adding %s %s to checked set to facilitate early exit", topic, partition) + topic_partition_checked.add((topic, partition)) + else: + self.log.debug('No new highwater offsets to query for consumer group %s', consumer_group) + + self.client.close_consumer() + + self.log.debug('Got %s highwater offsets', len(highwater_offsets)) + return highwater_offsets, cluster_id + def send_event(self, title, text, tags, event_type, aggregation_key, severity='info'): """Emit an event to the Datadog Event Stream.""" event_dict = { diff --git a/kafka_consumer/tests/test_unit.py b/kafka_consumer/tests/test_unit.py index 569a4e8915c9e..f38bda65e8c23 100644 --- a/kafka_consumer/tests/test_unit.py +++ b/kafka_consumer/tests/test_unit.py @@ -1,20 +1,50 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import concurrent.futures import logging from contextlib import nullcontext as does_not_raise import mock import pytest -from confluent_kafka.admin._group import ConsumerGroupListing, ListConsumerGroupsResult from datadog_checks.kafka_consumer import KafkaCheck +from datadog_checks.kafka_consumer.client import KafkaClient from datadog_checks.kafka_consumer.kafka_consumer import _get_interpolated_timestamp pytestmark = [pytest.mark.unit] +def fake_consumer_offsets_for_times(partitions): + """In our testing environment the offset is 80 for all partitions and topics.""" + + return [(t, p, 80) for t, p in partitions] + + +def seed_mock_client(): + """Set some common defaults for the mock client to kafka.""" + client = mock.create_autospec(KafkaClient) + client.list_consumer_groups.return_value = ["consumer_group1"] + client.get_partitions_for_topic.return_value = ['partition1'] + client.list_consumer_group_offsets.return_value = [("consumer_group1", [("topic1", "partition1", 2)])] + client.describe_consumer_groups.return_value = ('consumer_group', 'STABLE') + client.consumer_get_cluster_id_and_list_topics.return_value = ( + "cluster_id", + # topics + [ + # Used in unit tets + ('topic1', ["partition1"]), + ('topic2', ["partition2"]), + # Copied from integration tests + ('dc', [0, 1]), + ('unconsumed_topic', [0, 1]), + ('marvel', [0, 1]), + ('__consumer_offsets', [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + ], + ) + client.consumer_offsets_for_times = fake_consumer_offsets_for_times + return client + + @pytest.mark.parametrize( 'legacy_config, kafka_client_config, value', [ @@ -149,24 +179,15 @@ def test_oauth_config( # TODO: After these tests are finished and the revamp is complete, # the tests should be refactored to be parameters instead of separate tests -@mock.patch("datadog_checks.kafka_consumer.kafka_consumer.KafkaClient") -def test_when_consumer_lag_less_than_zero_then_emit_event( - mock_generic_client, check, kafka_instance, dd_run_check, aggregator -): +def test_when_consumer_lag_less_than_zero_then_emit_event(check, kafka_instance, dd_run_check, aggregator): # Given - # consumer_offset = {(consumer_group, topic, partition): offset} - consumer_offset = {("consumer_group1", "topic1", "partition1"): 2} - # highwater_offset = {(topic, partition): offset} - highwater_offset = {("topic1", "partition1"): 1} - mock_client = mock.MagicMock() - mock_client.get_consumer_offsets.return_value = consumer_offset - mock_client.get_highwater_offsets.return_value = (highwater_offset, "cluster_id") - mock_client.get_partitions_for_topic.return_value = ['partition1'] - mock_client.get_consumer_group_state.return_value = "STABLE" - mock_generic_client.return_value = mock_client + mock_client = seed_mock_client() + # We need the consumer offset to be higher than the highwater offset. + mock_client.list_consumer_group_offsets.return_value = [("consumer_group1", [("topic1", "partition1", 81)])] + kafka_consumer_check = check(kafka_instance) + kafka_consumer_check.client = mock_client # When - kafka_consumer_check = check(kafka_instance) dd_run_check(kafka_consumer_check) # Then @@ -216,24 +237,16 @@ def test_when_consumer_lag_less_than_zero_then_emit_event( ) -@mock.patch("datadog_checks.kafka_consumer.kafka_consumer.KafkaClient") -def test_when_partition_is_none_then_emit_warning_log( - mock_generic_client, check, kafka_instance, dd_run_check, aggregator, caplog -): +def test_when_no_partitions_then_emit_warning_log(check, kafka_instance, dd_run_check, aggregator, caplog): # Given - # consumer_offset = {(consumer_group, topic, partition): offset} - consumer_offset = {("consumer_group1", "topic1", "partition1"): 2} - # highwater_offset = {(topic, partition): offset} - highwater_offset = {("topic1", "partition1"): 1} - mock_client = mock.MagicMock() - mock_client.get_consumer_offsets.return_value = consumer_offset - mock_client.get_highwater_offsets.return_value = (highwater_offset, "cluster_id") - mock_client.get_partitions_for_topic.return_value = None - mock_generic_client.return_value = mock_client caplog.set_level(logging.WARNING) - # When + mock_client = seed_mock_client() + mock_client.get_partitions_for_topic.return_value = [] kafka_consumer_check = check(kafka_instance) + kafka_consumer_check.client = mock_client + + # When dd_run_check(kafka_consumer_check) # Then @@ -261,24 +274,18 @@ def test_when_partition_is_none_then_emit_warning_log( assert expected_warning in caplog.text -@mock.patch("datadog_checks.kafka_consumer.kafka_consumer.KafkaClient") def test_when_partition_not_in_partitions_then_emit_warning_log( - mock_generic_client, check, kafka_instance, dd_run_check, aggregator, caplog + check, kafka_instance, dd_run_check, aggregator, caplog ): # Given - # consumer_offset = {(consumer_group, topic, partition): offset} - consumer_offset = {("consumer_group1", "topic1", "partition1"): 2} - # highwater_offset = {(topic, partition): offset} - highwater_offset = {("topic1", "partition1"): 1} - mock_client = mock.MagicMock() - mock_client.get_consumer_offsets.return_value = consumer_offset - mock_client.get_highwater_offsets.return_value = (highwater_offset, "cluster_id") - mock_client.get_partitions_for_topic.return_value = ['partition2'] - mock_generic_client.return_value = mock_client caplog.set_level(logging.WARNING) - # When + mock_client = seed_mock_client() + mock_client.get_partitions_for_topic.return_value = ['partition2'] kafka_consumer_check = check(kafka_instance) + kafka_consumer_check.client = mock_client + + # When dd_run_check(kafka_consumer_check) # Then @@ -306,54 +313,44 @@ def test_when_partition_not_in_partitions_then_emit_warning_log( assert expected_warning in caplog.text -@mock.patch("datadog_checks.kafka_consumer.kafka_consumer.KafkaClient") def test_when_highwater_metric_count_hit_context_limit_then_no_more_highwater_metrics( - mock_generic_client, kafka_instance, dd_run_check, aggregator, caplog + check, kafka_instance, dd_run_check, aggregator, caplog ): # Given - # consumer_offset = {(consumer_group, topic, partition): offset} - consumer_offset = {("consumer_group1", "topic1", "partition1"): 2} - # highwater_offset = {(topic, partition): offset} - highwater_offset = {("topic1", "partition1"): 3, ("topic2", "partition2"): 3} - mock_client = mock.MagicMock() - mock_client.get_consumer_offsets.return_value = consumer_offset - mock_client.get_highwater_offsets.return_value = (highwater_offset, "cluster_id") - mock_client.get_partitions_for_topic.return_value = ['partition1'] - mock_generic_client.return_value = mock_client caplog.set_level(logging.WARNING) + mock_client = seed_mock_client() + kafka_consumer_check = check(kafka_instance, init_config={'max_partition_contexts': 2}) + kafka_consumer_check.client = mock_client + # When - kafka_consumer_check = KafkaCheck('kafka_consumer', {'max_partition_contexts': 2}, [kafka_instance]) dd_run_check(kafka_consumer_check) # Then - aggregator.assert_metric("kafka.broker_offset", count=2) - aggregator.assert_metric("kafka.consumer_offset", count=0) + aggregator.assert_metric("kafka.broker_offset", count=1) + aggregator.assert_metric("kafka.consumer_offset", count=1) aggregator.assert_metric("kafka.consumer_lag", count=0) - expected_warning = "Discovered 3 metric contexts" + expected_warning = "Discovered 2 metric contexts" assert expected_warning in caplog.text -@mock.patch("datadog_checks.kafka_consumer.kafka_consumer.KafkaClient") def test_when_consumer_metric_count_hit_context_limit_then_no_more_consumer_metrics( - mock_generic_client, kafka_instance, dd_run_check, aggregator, caplog + check, kafka_instance, dd_run_check, aggregator, caplog ): # Given - # consumer_offset = {(consumer_group, topic, partition): offset} - consumer_offset = {("consumer_group1", "topic1", "partition1"): 2, ("consumer_group1", "topic2", "partition2"): 2} - # highwater_offset = {(topic, partition): offset} - highwater_offset = {("topic1", "partition1"): 3, ("topic2", "partition2"): 3} - mock_client = mock.MagicMock() - mock_client.get_consumer_offsets.return_value = consumer_offset - mock_client.get_highwater_offsets.return_value = (highwater_offset, "cluster_id") - mock_client.get_partitions_for_topic.return_value = ['partition1'] - mock_generic_client.return_value = mock_client caplog.set_level(logging.DEBUG) + mock_client = seed_mock_client() + mock_client.list_consumer_group_offsets.return_value = [ + ("consumer_group1", [("topic1", "partition1", 2)]), + ("consumer_group1", [("topic2", "partition2", 2)]), + ] + kafka_consumer_check = check(kafka_instance, init_config={'max_partition_contexts': 3}) + kafka_consumer_check.client = mock_client + # When - kafka_consumer_check = KafkaCheck('kafka_consumer', {'max_partition_contexts': 3}, [kafka_instance]) dd_run_check(kafka_consumer_check) # Then @@ -369,19 +366,13 @@ def test_when_consumer_metric_count_hit_context_limit_then_no_more_consumer_metr def test_when_empty_string_consumer_group_then_skip(kafka_instance): - consumer_groups_result = ListConsumerGroupsResult( - valid=[ - ConsumerGroupListing(group_id="", is_simple_consumer_group=True), # Should be filtered out - ConsumerGroupListing(group_id="my_consumer", is_simple_consumer_group=True), - ] - ) - kafka_instance['monitor_unlisted_consumer_groups'] = True - future = concurrent.futures.Future() - future.set_result(consumer_groups_result) - - with mock.patch("datadog_checks.kafka_consumer.client.AdminClient.list_consumer_groups", return_value=future): + kafka_instance["monitor_unlisted_consumer_groups"] = True + with mock.patch( + "datadog_checks.kafka_consumer.kafka_consumer.KafkaClient.list_consumer_groups", + return_value=["", "my_consumer"], + ): kafka_consumer_check = KafkaCheck('kafka_consumer', {}, [kafka_instance]) - assert kafka_consumer_check.client._get_consumer_groups() == ["my_consumer"] + assert kafka_consumer_check._get_consumer_groups() == ["my_consumer"] def test_get_interpolated_timestamp(): diff --git a/kubeflow/README.md b/kubeflow/README.md index b801a22c3dedf..ac0046026ab9f 100644 --- a/kubeflow/README.md +++ b/kubeflow/README.md @@ -7,6 +7,10 @@ This check monitors [Kubeflow][1] through the Datadog Agent. ## Setup +
+This integration is currently released in Preview mode. Its availability is subject to change in the future. +
+ Follow the instructions below to install and configure this check for an Agent running on a host. For containerized environments, see the [Autodiscovery Integration Templates][3] for guidance on applying these instructions. ### Installation diff --git a/kubeflow/assets/dashboards/overview.json b/kubeflow/assets/dashboards/overview.json index b573e5f477156..333b5652c6a7c 100644 --- a/kubeflow/assets/dashboards/overview.json +++ b/kubeflow/assets/dashboards/overview.json @@ -94,7 +94,7 @@ { "definition": { "background_color": "pink", - "content": "If many widgets are empty, you are using a version of Kubeflow that does not expose certain metrics. Refer to the metadata.csv file for metrics list. \n\nReach out to support to indicate version incompatibilities.", + "content": "This integration is currently released in Preview mode. Its availability is subject to change in the future. \n\nIf many widgets are empty, you are using a version of Kubeflow that does not expose certain metrics. Refer to the metadata.csv file for metrics list. \n\nReach out to support to indicate version incompatibilities.", "font_size": "14", "has_padding": true, "show_tick": true, diff --git a/kubernetes_state_core/README.md b/kubernetes_state_core/README.md index 3550f6b26b699..ca0d16233572a 100644 --- a/kubernetes_state_core/README.md +++ b/kubernetes_state_core/README.md @@ -305,10 +305,10 @@ Need help? Contact [Datadog support][9]. [1]: https://kubernetes.io/blog/2021/04/13/kube-state-metrics-v-2-0/ [2]: #migration-from-kubernetes_state-to-kubernetes_state_core [3]: #data-collected -[4]: /agent/cluster_agent/ +[4]: https://docs.datadoghq.com/agent/cluster_agent/ [5]: https://github.com/DataDog/integrations-core/blob/master/kubernetes_state_core/metadata.csv -[6]: /getting_started/tagging/unified_service_tagging/#configuration +[6]: https://docs.datadoghq.com/getting_started/tagging/unified_service_tagging/#configuration [7]: https://github.com/DataDog/integrations-core/blob/master/kubernetes/assets/service_checks.json -[8]: /agent/guide/agent-commands/#agent-status-and-information -[9]: /help/ +[8]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information +[9]: https://docs.datadoghq.com/help/ [10]: https://www.datadoghq.com/blog/engineering/our-journey-taking-kubernetes-state-metrics-to-the-next-level/ diff --git a/kubevirt_api/README.md b/kubevirt_api/README.md index 0abed8b00a418..bf5d8dd4c7dc4 100644 --- a/kubevirt_api/README.md +++ b/kubevirt_api/README.md @@ -1,5 +1,9 @@ # Agent Check: KubeVirt API +
+This integration is in public beta and should be enabled on production workloads with caution. +
+ ## Overview This check monitors [KubeVirt API][1] through the Datadog Agent. diff --git a/kubevirt_api/manifest.json b/kubevirt_api/manifest.json index 68274dbad8c03..37a18199f5631 100644 --- a/kubevirt_api/manifest.json +++ b/kubevirt_api/manifest.json @@ -2,7 +2,7 @@ "manifest_version": "2.0.0", "app_uuid": "6b760149-4a9f-4ec7-a5bf-081fcd1d75b0", "app_id": "kubevirt-api", - "display_on_public_website": false, + "display_on_public_website": true, "tile": { "overview": "README.md#Overview", "configuration": "README.md#Setup", diff --git a/kubevirt_controller/README.md b/kubevirt_controller/README.md index e14ae467a20b4..7256e35676f67 100644 --- a/kubevirt_controller/README.md +++ b/kubevirt_controller/README.md @@ -1,5 +1,9 @@ # Agent Check: KubeVirt Controller +
+This integration is in public beta and should be enabled on production workloads with caution. +
+ ## Overview This check monitors [KubeVirt Controller][1] through the Datadog Agent. diff --git a/kubevirt_controller/manifest.json b/kubevirt_controller/manifest.json index 552896c23a325..78398a784ec82 100644 --- a/kubevirt_controller/manifest.json +++ b/kubevirt_controller/manifest.json @@ -2,7 +2,7 @@ "manifest_version": "2.0.0", "app_uuid": "f213050d-a54c-4a72-bf51-e9290a7d050c", "app_id": "kubevirt-controller", - "display_on_public_website": false, + "display_on_public_website": true, "tile": { "overview": "README.md#Overview", "configuration": "README.md#Setup", diff --git a/kubevirt_handler/CHANGELOG.md b/kubevirt_handler/CHANGELOG.md index 2644cdb8cfe3e..eb367ead79890 100644 --- a/kubevirt_handler/CHANGELOG.md +++ b/kubevirt_handler/CHANGELOG.md @@ -2,6 +2,12 @@ +## 1.0.1 / 2024-11-28 + +***Fixed***: + +* Bump base package dependency to get fixed pyyaml. ([#19156](https://github.com/DataDog/integrations-core/pull/19156)) + ## 1.0.0 / 2024-10-04 / Agent 7.59.0 ***Added***: diff --git a/kubevirt_handler/README.md b/kubevirt_handler/README.md index 33908b202c1e4..fb246f07d1bc9 100644 --- a/kubevirt_handler/README.md +++ b/kubevirt_handler/README.md @@ -1,5 +1,9 @@ # Agent Check: KubeVirt Handler +
+This integration is in public beta and should be enabled on production workloads with caution. +
+ ## Overview This check monitors [KubeVirt Handler][1] through the Datadog Agent. diff --git a/kubevirt_handler/datadog_checks/kubevirt_handler/__about__.py b/kubevirt_handler/datadog_checks/kubevirt_handler/__about__.py index acbfd1c866b84..e0db4e56d553f 100644 --- a/kubevirt_handler/datadog_checks/kubevirt_handler/__about__.py +++ b/kubevirt_handler/datadog_checks/kubevirt_handler/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '1.0.0' +__version__ = '1.0.1' diff --git a/kubevirt_handler/manifest.json b/kubevirt_handler/manifest.json index a9a0d6db1620c..d033d75c44e2b 100644 --- a/kubevirt_handler/manifest.json +++ b/kubevirt_handler/manifest.json @@ -2,7 +2,7 @@ "manifest_version": "2.0.0", "app_uuid": "751006a9-b87a-4f54-acc5-2886ec49073e", "app_id": "kubevirt-handler", - "display_on_public_website": false, + "display_on_public_website": true, "tile": { "overview": "README.md#Overview", "configuration": "README.md#Setup", diff --git a/kubevirt_handler/pyproject.toml b/kubevirt_handler/pyproject.toml index 41884d9e03d39..696173f60ec65 100644 --- a/kubevirt_handler/pyproject.toml +++ b/kubevirt_handler/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ "Topic :: System :: Monitoring", ] dependencies = [ - "datadog-checks-base>=32.6.0", + "datadog-checks-base>=36.5.0", ] dynamic = [ "version", diff --git a/mongo/CHANGELOG.md b/mongo/CHANGELOG.md index 78a233b39855b..61820b10cfe17 100644 --- a/mongo/CHANGELOG.md +++ b/mongo/CHANGELOG.md @@ -2,6 +2,20 @@ +## 8.3.0 / 2024-11-28 + +***Added***: + +* Add `metrics_collection_interval` config option to customize the collection interval for collection stats, index stats, and sharded data distribution metrics. + The default collection interval for collection stats and index stats remains unchanged at check min collection interval of 15 seconds. + The default collection interval for sharded data distribution metrics is 300 seconds. ([#19098](https://github.com/DataDog/integrations-core/pull/19098)) + +***Fixed***: + +* Fixes timezone parsing bug in slow query log, preventing incorrect timestamp conversions on non-UTC servers. ([#19057](https://github.com/DataDog/integrations-core/pull/19057)) +* Fix crash in DBM operation samples collection when a node is in recovering mode. ([#19080](https://github.com/DataDog/integrations-core/pull/19080)) +* Resolved deprecation warning for `collStats` by using `$collStats` aggregation pipeline to collect oplog size in MongoDB 6.2+. ([#19133](https://github.com/DataDog/integrations-core/pull/19133)) + ## 8.2.1 / 2024-11-06 ***Fixed***: diff --git a/mongo/changelog.d/19057.fixed b/mongo/changelog.d/19057.fixed deleted file mode 100644 index 2f93fc8d654ec..0000000000000 --- a/mongo/changelog.d/19057.fixed +++ /dev/null @@ -1 +0,0 @@ -Fixes timezone parsing bug in slow query log, preventing incorrect timestamp conversions on non-UTC servers. diff --git a/mongo/changelog.d/19080.fixed b/mongo/changelog.d/19080.fixed deleted file mode 100644 index 347af2c7cc8f3..0000000000000 --- a/mongo/changelog.d/19080.fixed +++ /dev/null @@ -1,2 +0,0 @@ -Fix crash in DBM operation samples collection when a node is in recovering mode. - diff --git a/mongo/changelog.d/19098.added b/mongo/changelog.d/19098.added deleted file mode 100644 index 827991de1eb91..0000000000000 --- a/mongo/changelog.d/19098.added +++ /dev/null @@ -1,3 +0,0 @@ -Add `metrics_collection_interval` config option to customize the collection interval for collection stats, index stats, and sharded data distribution metrics. -The default collection interval for collection stats and index stats remains unchanged at check min collection interval of 15 seconds. -The default collection interval for sharded data distribution metrics is 300 seconds. diff --git a/mongo/changelog.d/19133.fixed b/mongo/changelog.d/19133.fixed deleted file mode 100644 index 76ffde6492d4d..0000000000000 --- a/mongo/changelog.d/19133.fixed +++ /dev/null @@ -1 +0,0 @@ -Resolved deprecation warning for `collStats` by using `$collStats` aggregation pipeline to collect oplog size in MongoDB 6.2+. diff --git a/mongo/datadog_checks/mongo/__about__.py b/mongo/datadog_checks/mongo/__about__.py index 0673b43c370ef..88bbe435de59f 100644 --- a/mongo/datadog_checks/mongo/__about__.py +++ b/mongo/datadog_checks/mongo/__about__.py @@ -2,4 +2,4 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '8.2.1' +__version__ = '8.3.0' diff --git a/mongo/tests/test_integration.py b/mongo/tests/test_integration.py index f629f4ebaa396..075de05746abc 100644 --- a/mongo/tests/test_integration.py +++ b/mongo/tests/test_integration.py @@ -61,7 +61,13 @@ def _assert_mongodb_instance_event( } -@pytest.mark.parametrize("dbm", [True, False]) +@pytest.mark.parametrize( + "dbm", + [ + pytest.param(True, id="DBM enabled"), + pytest.param(False, id="DBM disabled"), + ], +) def test_integration_mongos(instance_integration_cluster, aggregator, check, dd_run_check, dbm): instance_integration_cluster['dbm'] = dbm instance_integration_cluster['operation_samples'] = {'enabled': False} diff --git a/mux/CHANGELOG.md b/mux/CHANGELOG.md new file mode 100644 index 0000000000000..0902c9b8e5fc3 --- /dev/null +++ b/mux/CHANGELOG.md @@ -0,0 +1,7 @@ +# CHANGELOG - Mux + +## 1.0.0 / 2024-09-24 + +***Added***: + +* Initial Release diff --git a/mux/README.md b/mux/README.md new file mode 100644 index 0000000000000..26d7b13e44832 --- /dev/null +++ b/mux/README.md @@ -0,0 +1,53 @@ +# MUX + +## Overview + +[Mux][1] is an all-in-one video streaming platform. It offers APIs and tools for video hosting, live streaming, etc. enabling users to easily create, manage, and optimize video content. Mux provides scalable video infrastructure to build seamless video experiences. + +Integrate Mux with Datadog to gain insights into mux video performance data. + +## Setup + +### Get config parameters from Mux + +#### Find your Access Token ID and Secret Key from Mux +1. Login to [MUX account][2]. +2. In the sidebar, click on **Settings**. +3. Click on **Access Tokens**. +4. Select **Generate new token**. +5. Choose the environment. +6. Under the **permission** section, select **Mux Data(read-only)**. +7. Enter the access token name. +8. Click on **Generate Token**. +9. Save the Access Token ID and Secret Key from the **Here's your new Access Token** tab. + +### Add your Mux credentials +- Access token ID +- Secret key + +## Data Collected + +### Logs + +The Mux integration does not include any logs. + +### Metrics + +The Mux integration collects and forwards mux metrics data to Datadog. See [metadata.csv][4] for a list of metrics provided by this integration. + +### Service Checks + +The Mux integration does not include any service checks. + +### Events + +The Mux integration does not include any events. + +## Troubleshooting + +Need help? Contact [Datadog support][3]. + +[1]: https://www.mux.com/ +[2]: https://dashboard.mux.com/ +[3]: https://docs.datadoghq.com/help/ +[4]: https://github.com/DataDog/integrations-core/blob/master/mux/metadata.csv diff --git a/mux/assets/dashboards/mux_metrics.json b/mux/assets/dashboards/mux_metrics.json new file mode 100644 index 0000000000000..0477d130ad678 --- /dev/null +++ b/mux/assets/dashboards/mux_metrics.json @@ -0,0 +1,3690 @@ +{ + "title": "Mux - Metrics", + "description": "This Dashboard provides comprehensive insights, including views, unique viewers, playing time, viewer experience score, playback success score, player startup time, smoothness score, video quality score, and rebuffer percentage, with the latest metrics showcasing consolidated statistics for video performance.", + "widgets": [ + { + "id": 8740298734186812, + "definition": { + "type": "image", + "url": "https://www.mux.com/images/mux-logo.png", + "url_dark_theme": "", + "sizing": "contain", + "margin": "sm", + "has_background": true, + "has_border": false, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 2 + } + }, + { + "id": 8407084925998778, + "definition": { + "title": "Monitors Summary", + "type": "manage_status", + "display_format": "countsAndList", + "color_preference": "background", + "hide_zero_counts": true, + "show_status": true, + "last_triggered_format": "relative", + "query": "tag:integration:mux $Video_Id", + "sort": "status,asc", + "count": 50, + "start": 0, + "summary_type": "monitors", + "show_priority": false, + "show_last_triggered": false + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 5 + } + }, + { + "id": 323399805713430, + "definition": { + "type": "note", + "content": "**[Mux](https://www.mux.com/)** is an all-in-one video streaming platform designed for developers and businesses. It offers APIs and tools for video hosting, live streaming, web player customization, and in-depth analytics, enabling users to easily create, manage, and optimize video content. \n\nThis Dashboard provides comprehensive insights, including views, unique viewers, playing time, viewer experience score, playback success score, player startup time, smoothness score, video quality score, and rebuffer percentage, with the latest metrics showcasing consolidated statistics for video performance.\n\nFor more information, see the [Mux Integration Documentation](https://docs.datadoghq.com/integrations/mux/).\n\n**Tip**:\n- Clone this dashboard to rearrange, modify and add widgets and visualizations.", + "background_color": "purple", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": true, + "tick_pos": "50%", + "tick_edge": "top", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 2, + "width": 6, + "height": 3 + } + }, + { + "id": 1220293637488098, + "definition": { + "title": "Overview", + "background_color": "purple", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 4009446443373264, + "definition": { + "title": "Views by Time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Views", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "sum:mux.metric.views{video_id:*,$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 3 + } + }, + { + "id": 3481055536288754, + "definition": { + "title": "Unique Viewers by Time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Unique Viewers", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "sum:mux.metric.unique_viewers{video_id:*,$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 3, + "width": 12, + "height": 3 + } + }, + { + "id": 6200752868207150, + "definition": { + "title": "Playing Time by Time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Playing Time", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "sum:mux.metric.playing_time{video_id:*,$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 6, + "width": 12, + "height": 3 + } + }, + { + "id": 5837183919298942, + "definition": { + "title": "Top Videos with High Views", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "sum:mux.metric.views{video_id:*,$Video_Id} by {video_id}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "dog_classic" + } + }, + "layout": { + "x": 0, + "y": 9, + "width": 12, + "height": 3 + } + }, + { + "id": 5794093284934446, + "definition": { + "title": "Views Distribution by OS", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:mux.metric.views{operating_system:*} by {operating_system}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "style": { + "palette": "datadog16" + } + } + ], + "type": "sunburst", + "legend": { + "type": "table" + } + }, + "layout": { + "x": 0, + "y": 12, + "width": 6, + "height": 4 + } + }, + { + "id": 439902379343072, + "definition": { + "title": "Views Distribution by Browser", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:mux.metric.views{browser:*} by {browser}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "style": { + "palette": "datadog16" + } + } + ], + "type": "sunburst", + "legend": { + "type": "table" + } + }, + "layout": { + "x": 6, + "y": 12, + "width": 6, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 5, + "width": 12, + "height": 17 + } + }, + { + "id": 6745713151482410, + "definition": { + "title": "Quality of Experience", + "background_color": "purple", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 6596360524579660, + "definition": { + "title": "Average Viewer Experience Score", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.viewer_experience_score{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "custom_bg", + "custom_bg_color": "#cde5f5" + } + ] + } + ], + "autoscale": true, + "precision": 0, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 8602979691001942, + "definition": { + "title": "Top Videos with High Viewer Experience Score", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.viewer_experience_score{$Video_Id} by {video_id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1 * 100" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "dog_classic" + } + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 3 + } + }, + { + "id": 2626845644049264, + "definition": { + "title": "Viewer Experience Score Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "viewer experience score", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.viewer_experience_score{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 3, + "width": 12, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 22, + "width": 12, + "height": 7 + } + }, + { + "id": 8893308504271682, + "definition": { + "title": "Playback Success", + "background_color": "purple", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 298345148190314, + "definition": { + "title": "Average Playback Success Score", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.playback_success_score{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit" + } + }, + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 0, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 4801477252304832, + "definition": { + "title": "Playback Success Score Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Playback Success Score", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.playback_success_score{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 3 + } + }, + { + "id": 4396501637723620, + "definition": { + "title": "Average Playback Failure Percentage", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.playback_failure_percentage{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_red" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 3, + "width": 3, + "height": 3 + } + }, + { + "id": 3169127014752062, + "definition": { + "title": "Playback Failure Percentage Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Playback Failure Percentage", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.playback_failure_percentage{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 3, + "width": 9, + "height": 3 + } + }, + { + "id": 2367630247389044, + "definition": { + "title": "Average Video Startup Failure Percentage", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.video_startup_failure_percentage{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_red" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 6, + "width": 3, + "height": 3 + } + }, + { + "id": 786592074502738, + "definition": { + "title": "Video Startup Failure Percentage Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Video Startup Failure Percentage", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.video_startup_failure_percentage{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 6, + "width": 9, + "height": 3 + } + }, + { + "id": 2295034455275454, + "definition": { + "title": "Average Exits Before Video Start", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.exits_before_video_start{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 9, + "width": 3, + "height": 3 + } + }, + { + "id": 2406312015360100, + "definition": { + "title": "Exits Before Video Start Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Exits Before Video Start", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.exits_before_video_start{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 9, + "width": 9, + "height": 3 + } + }, + { + "id": 5177056228052332, + "definition": { + "title": "Average Business Exception Percentage", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.playback_business_exception_percentage{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 12, + "width": 3, + "height": 3 + } + }, + { + "id": 6903804040098836, + "definition": { + "title": "Video Startup Business Exception Percentage Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Video Startup Business Exception Percentage", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.video_startup_business_exception_percentage{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 12, + "width": 9, + "height": 3 + } + }, + { + "id": 4977786892663128, + "definition": { + "title": "Average Video Startup Business Exception Percentage", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.video_startup_business_exception_percentage{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 15, + "width": 4, + "height": 3 + } + }, + { + "id": 8519844538070258, + "definition": { + "title": "Video Startup Business Exception Percentage Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Video Startup Business Exception Percentage", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.video_startup_business_exception_percentage{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 15, + "width": 8, + "height": 3 + } + }, + { + "id": 2566518125633970, + "definition": { + "title": "Top Videos with High Playback Success Score", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.playback_success_score{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1 * 100" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "dog_classic" + } + }, + "layout": { + "x": 0, + "y": 18, + "width": 6, + "height": 3 + } + }, + { + "id": 4450192488486708, + "definition": { + "title": "Top Videos with High Playback Failure Percentage", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.playback_failure_percentage{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1 * 100" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "dog_classic" + } + }, + "layout": { + "x": 6, + "y": 18, + "width": 6, + "height": 3 + } + }, + { + "id": 3013528366385814, + "definition": { + "title": "Playback Success Details", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:mux.metric.playback_success_score{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:mux.metric.playback_failure_percentage{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "avg:mux.metric.playback_business_exception_percentage{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "sort": { + "count": 1000, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "alias": "Playback Success Score", + "formula": "query1 * 100" + }, + { + "alias": "Playback Failure Percentage", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query2 * 100" + }, + { + "alias": "Playback Business Exception Percentage", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query3 * 100" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 21, + "width": 12, + "height": 4 + } + }, + { + "id": 2801208370061346, + "definition": { + "title": "Video Startup Details", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:mux.metric.video_startup_failure_percentage{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:mux.metric.video_startup_business_exception_percentage{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "avg:mux.metric.exits_before_video_start{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "sort": { + "count": 1000, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "alias": "Video Startup Failure Percentage", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query1 * 100" + }, + { + "alias": "Video Startup Business Exception Percentage", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query2 * 100" + }, + { + "cell_display_mode": "number", + "alias": "Exits Before Video Starts", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query3 * 100" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 25, + "width": 12, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 29, + "width": 12, + "height": 30 + } + }, + { + "id": 1266663704400704, + "definition": { + "title": "Startup Time", + "background_color": "purple", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 2236256623748982, + "definition": { + "title": "Average Startup Time Score", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.startup_time_score{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "custom_bg", + "custom_bg_color": "#cde5f5" + } + ] + } + ], + "autoscale": true, + "precision": 0, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 4, + "height": 3 + } + }, + { + "id": 2495307867177718, + "definition": { + "title": "Startup Time Score Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Startup Time Score", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.startup_time_score{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 0, + "width": 8, + "height": 3 + } + }, + { + "id": 474393580792922, + "definition": { + "title": "Average Page Load Time", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.page_load_time{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 3, + "width": 4, + "height": 3 + } + }, + { + "id": 7742531014991664, + "definition": { + "title": "Page Load Time Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Page Load Time", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.page_load_time{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 3, + "width": 8, + "height": 3 + } + }, + { + "id": 5326722891696668, + "definition": { + "title": "Average Video Startup Time", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.video_startup_time{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green", + "custom_bg_color": "#cde5f5" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 6, + "width": 4, + "height": 3 + } + }, + { + "id": 8707762128124154, + "definition": { + "title": "Video Startup Time Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Video Startup Time", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.video_startup_time{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 6, + "width": 8, + "height": 3 + } + }, + { + "id": 8353431131543564, + "definition": { + "title": "Average Aggregate Startup Time", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.aggregate_startup_time{$Video_Id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 9, + "width": 4, + "height": 3 + } + }, + { + "id": 6242206807126068, + "definition": { + "title": "Aggregate Startup Time Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Aggregate Startup Time", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.aggregate_startup_time{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 9, + "width": 8, + "height": 3 + } + }, + { + "id": 5162449502127588, + "definition": { + "title": "Average Player Startup Time", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.player_startup_time{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "custom_bg", + "custom_bg_color": "#cde5f5" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 12, + "width": 4, + "height": 3 + } + }, + { + "id": 1323822117530074, + "definition": { + "title": "Player Startup Time Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Player Startup Time", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.player_startup_time{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 12, + "width": 8, + "height": 3 + } + }, + { + "id": 4337513725726252, + "definition": { + "title": "Average Seek Latency", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.seek_latency{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "custom_bg", + "custom_bg_color": "#cde5f5" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 15, + "width": 4, + "height": 3 + } + }, + { + "id": 2620359520153796, + "definition": { + "title": "Seek Latency Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Seek Latency", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.seek_latency{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 15, + "width": 8, + "height": 3 + } + }, + { + "id": 8528252805378674, + "definition": { + "title": "Seek Latency Overtime by Video Id", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.seek_latency{video_id:*,$Video_Id} by {video_id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ], + "yaxis": { + "include_zero": true, + "max": "auto" + } + }, + "layout": { + "x": 0, + "y": 18, + "width": 12, + "height": 4 + } + }, + { + "id": 7582158836467120, + "definition": { + "title": "Startup Time Details", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:mux.metric.startup_time_score{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:mux.metric.video_startup_time{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "avg:mux.metric.player_startup_time{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "avg:mux.metric.page_load_time{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query5", + "query": "avg:mux.metric.aggregate_startup_time{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query6", + "query": "avg:mux.metric.seek_latency{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "sort": { + "count": 1000, + "order_by": [ + { + "type": "formula", + "index": 5, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "number", + "alias": "Video Startup Time", + "formula": "query2" + }, + { + "cell_display_mode": "number", + "alias": "Player Startup Time", + "formula": "query3" + }, + { + "cell_display_mode": "number", + "alias": "Page Load Time", + "formula": "query4" + }, + { + "cell_display_mode": "number", + "alias": "Aggregate Startup Time", + "formula": "query5" + }, + { + "cell_display_mode": "number", + "alias": "Seek Latency", + "formula": "query6" + }, + { + "alias": "Startup Time Score", + "formula": "query1 * 100" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 22, + "width": 12, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 59, + "width": 12, + "height": 27, + "is_column_break": true + } + }, + { + "id": 7334379080331064, + "definition": { + "title": "Smoothness", + "background_color": "purple", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 4608661273329986, + "definition": { + "title": "Average Smoothness Score", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.smoothness_score{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "custom_bg", + "custom_bg_color": "#cde5f5" + } + ] + } + ], + "autoscale": true, + "precision": 0, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 4, + "height": 3 + } + }, + { + "id": 4494292675722788, + "definition": { + "title": "Smoothness Score Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Smoothness Score", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.smoothness_score{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 0, + "width": 8, + "height": 3 + } + }, + { + "id": 6473814659260960, + "definition": { + "title": "Average Rebuffer Count", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.rebuffer_count{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 0, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 3, + "width": 4, + "height": 3 + } + }, + { + "id": 1932274358183966, + "definition": { + "title": "Rebuffer Count Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Rebuffer Count", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.rebuffer_count{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 3, + "width": 8, + "height": 3 + } + }, + { + "id": 4132996472752296, + "definition": { + "title": "Average Rebuffer Duration", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.rebuffer_duration{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 6, + "width": 4, + "height": 3 + } + }, + { + "id": 2813049649778638, + "definition": { + "title": "Rebuffer Duration Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Rebuffer Duration", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.rebuffer_duration{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 6, + "width": 8, + "height": 3 + } + }, + { + "id": 5931318482078592, + "definition": { + "title": "Average Rebuffer Percentage", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.rebuffer_percentage{$Video_Id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 9, + "width": 4, + "height": 3 + } + }, + { + "id": 8093326950057642, + "definition": { + "title": "Rebuffer Percentage Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Rebuffer Percentage", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.rebuffer_percentage{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 9, + "width": 8, + "height": 3 + } + }, + { + "id": 3309016372158672, + "definition": { + "title": "Average Rebuffer Frequency", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.rebuffer_frequency{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "per_unit_name": "minute" + } + }, + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 12, + "width": 4, + "height": 3 + } + }, + { + "id": 41683318688066, + "definition": { + "title": "Rebuffer Frequency Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Rebuffer Frequency", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.rebuffer_frequency{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 12, + "width": 8, + "height": 3 + } + }, + { + "id": 6656902652971822, + "definition": { + "title": "Top Videos with High Smoothness Score", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.smoothness_score{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1 * 100" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "dog_classic" + } + }, + "layout": { + "x": 0, + "y": 15, + "width": 12, + "height": 4 + } + }, + { + "id": 8881348044212578, + "definition": { + "title": "Smoothness Details", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:mux.metric.smoothness_score{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:mux.metric.rebuffer_count{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "avg:mux.metric.rebuffer_duration{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "avg:mux.metric.rebuffer_frequency{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query5", + "query": "avg:mux.metric.rebuffer_percentage{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "sort": { + "count": 1000, + "order_by": [ + { + "type": "formula", + "index": 3, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "number", + "alias": "Rebuffer Count", + "formula": "query2" + }, + { + "cell_display_mode": "number", + "alias": "Rebuffer Duration", + "formula": "query3" + }, + { + "cell_display_mode": "number", + "alias": "Rebuffer Frequency", + "number_format": { + "unit": { + "type": "canonical_unit", + "per_unit_name": "minute" + } + }, + "formula": "query4" + }, + { + "alias": "Smoothness Score", + "formula": "query1 * 100" + }, + { + "alias": "Rebuffer Percentage", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query5 * 100" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 19, + "width": 12, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 86, + "width": 12, + "height": 24 + } + }, + { + "id": 5070840369040136, + "definition": { + "title": "Video Quality", + "background_color": "purple", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 7390564078879392, + "definition": { + "title": "Average Video Quality Score", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.video_quality_score{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "custom_bg", + "custom_bg_color": "#cde5f5" + } + ] + } + ], + "autoscale": true, + "precision": 0, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 4, + "height": 3 + } + }, + { + "id": 1229883020208502, + "definition": { + "title": "Video Quality Score Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Video Quality Score", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.video_quality_score{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 0, + "width": 8, + "height": 3 + } + }, + { + "id": 2309946839235418, + "definition": { + "title": "Weighted Average Bitrate", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.weighted_average_bitrate{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "number_format": { + "unit_scale": { + "type": "canonical_unit", + "unit_name": "bit" + } + }, + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 3, + "width": 4, + "height": 3 + } + }, + { + "id": 38538954247440, + "definition": { + "title": "Weighted Average Bitrate Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Weighted Average Bitrate", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.weighted_average_bitrate{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 3, + "width": 8, + "height": 3 + } + }, + { + "id": 2741121569755672, + "definition": { + "title": "Average Upscale Percentage", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.upscale_percentage{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": false, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 6, + "width": 4, + "height": 3 + } + }, + { + "id": 4128572692652006, + "definition": { + "title": "Upscale Percentage Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Upscale Percentage", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.upscale_percentage{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 6, + "width": 8, + "height": 3 + } + }, + { + "id": 5478455287138638, + "definition": { + "title": "Average Downscale Percentage", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.downscale_percentage{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 9, + "width": 4, + "height": 3 + } + }, + { + "id": 2007567121576290, + "definition": { + "title": "Downscale Percentage Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Downscale Percentage", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.downscale_percentage{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 9, + "width": 8, + "height": 3 + } + }, + { + "id": 6862787917948638, + "definition": { + "title": "Average of Max Upscale Percentage", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.max_upscale_percentage{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "custom_bg", + "custom_bg_color": "#cde5f5" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 12, + "width": 4, + "height": 3 + } + }, + { + "id": 3869465964272090, + "definition": { + "title": "Max Upscale Percentage Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Max Upscale Percentage", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.max_upscale_percentage{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 12, + "width": 8, + "height": 3 + } + }, + { + "id": 1056987583365914, + "definition": { + "title": "Average of Max Downscale Percentage", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.max_downscale_percentage{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query1 * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "custom_bg", + "custom_bg_color": "#cde5f5" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 15, + "width": 4, + "height": 3 + } + }, + { + "id": 5172304773900428, + "definition": { + "title": "Max Downscale Percentage Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Max Downscale Percentage", + "formula": "query1 * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.max_downscale_percentage{$Video_Id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 4, + "y": 15, + "width": 8, + "height": 3 + } + }, + { + "id": 6114720235628602, + "definition": { + "title": "Average Live Stream Latency", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.live_stream_latency{$Video_Id}", + "aggregator": "avg" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "millisecond" + } + }, + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "custom_bg", + "custom_bg_color": "#cde5f5" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 18, + "width": 4, + "height": 3 + } + }, + { + "id": 8337865062113938, + "definition": { + "title": "Live Stream Latency Overtime", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "millisecond" + } + }, + "alias": "Live Stream Latency", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.live_stream_latency{$Video_Id} by {video_id}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ], + "yaxis": { + "include_zero": true, + "max": "auto" + } + }, + "layout": { + "x": 4, + "y": 18, + "width": 8, + "height": 3 + } + }, + { + "id": 6601500419927196, + "definition": { + "title": "Top Videos with High Quality Score", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:mux.metric.video_quality_score{$Video_Id} by {video_id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1 * 100" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "dog_classic" + } + }, + "layout": { + "x": 0, + "y": 21, + "width": 12, + "height": 4 + } + }, + { + "id": 3271158187474932, + "definition": { + "title": "Video Quality Details", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:mux.metric.video_quality_score{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:mux.metric.live_stream_latency{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "avg:mux.metric.weighted_average_bitrate{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "sort": { + "count": 1000, + "order_by": [ + { + "type": "formula", + "index": 2, + "order": "desc" + } + ] + }, + "formulas": [ + { + "alias": "Live Stream Latency", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "millisecond" + } + }, + "formula": "query2" + }, + { + "alias": "Weighted Average Bitrate", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "bit", + "per_unit_name": "second" + }, + "unit_scale": { + "type": "canonical_unit", + "unit_name": "bit" + } + }, + "formula": "query3" + }, + { + "alias": "Video Quality Score", + "formula": "query1 * 100" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 25, + "width": 12, + "height": 4 + } + }, + { + "id": 6980401583290302, + "definition": { + "title": "Upscale/Downscale Percentage Details", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:mux.metric.upscale_percentage{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:mux.metric.max_upscale_percentage{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "avg:mux.metric.downscale_percentage{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "avg:mux.metric.max_downscale_percentage{video_id:*,$Video_Id} by {video_id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "sort": { + "count": 1000, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "alias": "Upscale Percentage", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query1 * 100" + }, + { + "alias": "Max Upscale Percentage", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query2 * 100" + }, + { + "alias": "Downscale Percentage", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query3 * 100" + }, + { + "alias": "Max Downscale Percentage", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "query4 * 100" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 29, + "width": 12, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 110, + "width": 12, + "height": 34 + } + } + ], + "template_variables": [ + { + "name": "Video_Id", + "prefix": "video_id", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} \ No newline at end of file diff --git a/mux/assets/monitors/anomaly_detected_in_page_load_time.json b/mux/assets/monitors/anomaly_detected_in_page_load_time.json new file mode 100644 index 0000000000000..48a2c223e3e7d --- /dev/null +++ b/mux/assets/monitors/anomaly_detected_in_page_load_time.json @@ -0,0 +1,45 @@ +{ + "version": 2, + "created_at": "2024-09-24", + "last_updated_at": "2024-09-24", + "title": "Anomaly Detected in Page Load Time", + "description": "Anomalies in page load time indicate potential performance issues affecting user experience. This monitor detects deviations from normal load times to identify slowdowns. Timely alerts help ensure quick resolution, maintaining optimal site performance and user satisfaction.", + "definition": { + "id": 154116834, + "name": "[Mux] Anomaly Detected in Page Load Time", + "type": "query alert", + "query": "avg(last_2d):anomalies(max:mux.metric.page_load_time{*} by {video_id}, 'agile', 2, direction='both', interval=600, alert_window='last_4h', timezone='utc', count_default_zero='true', seasonality='daily') >= 1", + "message": "{{#is_warning}} \nThe page load time for video id: {{video_id.name}} has exceeded the threshold. \nCurrent Page Load Time: {{value}}% \nThreshold: {{warn_threshold}}% \n{{/is_warning}}\n\n{{#is_alert}} \nThe page load time for video id: {{video_id.name}} has exceeded the threshold. \nCurrent Page Load Time: {{value}}% \nThreshold: {{threshold}}% \n{{/is_alert}}\n\n\n@example@example.com", + "tags": [ + "integration:mux" + ], + "options": { + "thresholds": { + "critical": 1, + "critical_recovery": 0, + "warning": 0.5 + }, + "notify_audit": false, + "require_full_window": false, + "renotify_interval": 0, + "threshold_windows": { + "trigger_window": "last_4h", + "recovery_window": "last_1h" + }, + "on_missing_data": "default", + "include_tags": false, + "notify_by": [ + "*" + ], + "new_group_delay": 60, + "silenced": {} + }, + "priority": 3, + "restriction_policy": { + "bindings": [] + } + }, + "tags": [ + "integration:mux" + ] +} diff --git a/mux/assets/monitors/playback_failure_percentage_is_higher_than_usual.json b/mux/assets/monitors/playback_failure_percentage_is_higher_than_usual.json new file mode 100644 index 0000000000000..3316d0ed8f1ef --- /dev/null +++ b/mux/assets/monitors/playback_failure_percentage_is_higher_than_usual.json @@ -0,0 +1,35 @@ +{ + "version": 2, + "created_at": "2024-09-24", + "last_updated_at": "2024-09-24", + "title": "Playback Failure Percentage is higher than usual", + "description": "Playback failures occur when a video cannot be played successfully. This monitor tracks the percentage of these failures to identify streaming quality issues. High rates can frustrate users and reduce engagement, so timely alerts are essential for maintaining a smooth viewing experience.", + "definition": { + "id": 154118176, + "name": "[Mux] Playback Failure Percentage is higher than usual", + "type": "query alert", + "query": "max(last_4h):max:mux.metric.playback_failure_percentage{*} by {video_id} > 20", + "message": "{{#is_warning}} \nThe playback failure percentage for video id: {{video_id.name}} has exceeded the threshold. \nCurrent Playback Failure Percentage: {{value}}% \nThreshold: {{warn_threshold}}% \n{{/is_warning}}\n\n{{#is_alert}} \nThe playback failure percentage for video id: {{video_id.name}} has exceeded the threshold. \nCurrent Playback Failure Percentage: {{value}}% \nThreshold: {{threshold}}% \n{{/is_alert}}\n\n\n@example@example.com", + "tags": [ + "integration:mux" + ], + "options": { + "thresholds": { + "critical": 20, + "warning": 10 + }, + "notify_audit": false, + "on_missing_data": "default", + "include_tags": true, + "new_group_delay": 60, + "silenced": {} + }, + "priority": 2, + "restriction_policy": { + "bindings": [] + } + }, + "tags": [ + "integration:mux" + ] +} diff --git a/mux/assets/monitors/video_startup_failure_percentage_is_higher_than_usual.json b/mux/assets/monitors/video_startup_failure_percentage_is_higher_than_usual.json new file mode 100644 index 0000000000000..880bc63f322de --- /dev/null +++ b/mux/assets/monitors/video_startup_failure_percentage_is_higher_than_usual.json @@ -0,0 +1,35 @@ +{ + "version": 2, + "created_at": "2024-09-24", + "last_updated_at": "2024-09-24", + "title": "Video Startup Failure Percentage is higher than usual", + "description": "Video startup failures happen when a video fails to start for users. This monitor tracks the percentage of these failures to identify issues in delivery. High rates can lead to poor user experience and increased churn, making timely alerts essential for maintaining viewer satisfaction.", + "definition": { + "id": 154118482, + "name": "[Mux] Video Startup Failure Percentage is higher than usual", + "type": "query alert", + "query": "max(last_4h):max:mux.metric.video_startup_failure_percentage{*} by {video_id} > 20", + "message": "{{#is_warning}} \nThe Video Startup Failure Percentage for video id: {{video_id.name}} has exceeded the threshold. \nCurrent Video Startup Failure Percentage: {{value}}% \nThreshold: {{warn_threshold}}% \n{{/is_warning}}\n\n{{#is_alert}} \nThe Video Startup Failure Percentage for video id: {{video_id.name}} has exceeded the threshold. \nCurrent Video Startup Failure Percentage: {{value}}% \nThreshold: {{threshold}}% \n{{/is_alert}}\n\n\n@example@example.com", + "tags": [ + "integration:mux" + ], + "options": { + "thresholds": { + "critical": 20, + "warning": 10 + }, + "notify_audit": false, + "on_missing_data": "default", + "include_tags": true, + "new_group_delay": 60, + "silenced": {} + }, + "priority": 2, + "restriction_policy": { + "bindings": [] + } + }, + "tags": [ + "integration:mux" + ] +} diff --git a/mux/assets/mux.svg b/mux/assets/mux.svg new file mode 100644 index 0000000000000..d40e1c30ac301 --- /dev/null +++ b/mux/assets/mux.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/mux/assets/service_checks.json b/mux/assets/service_checks.json new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/mux/assets/service_checks.json @@ -0,0 +1 @@ +[] diff --git a/mux/images/mux_metrics_1.png b/mux/images/mux_metrics_1.png new file mode 100644 index 0000000000000..6fa40973b6294 Binary files /dev/null and b/mux/images/mux_metrics_1.png differ diff --git a/mux/images/mux_metrics_2.png b/mux/images/mux_metrics_2.png new file mode 100644 index 0000000000000..ced125df21148 Binary files /dev/null and b/mux/images/mux_metrics_2.png differ diff --git a/mux/images/mux_metrics_3.png b/mux/images/mux_metrics_3.png new file mode 100644 index 0000000000000..26221160c989b Binary files /dev/null and b/mux/images/mux_metrics_3.png differ diff --git a/mux/images/mux_metrics_4.png b/mux/images/mux_metrics_4.png new file mode 100644 index 0000000000000..b550dd6437ba0 Binary files /dev/null and b/mux/images/mux_metrics_4.png differ diff --git a/mux/images/mux_metrics_5.png b/mux/images/mux_metrics_5.png new file mode 100644 index 0000000000000..6f8a9376fbadb Binary files /dev/null and b/mux/images/mux_metrics_5.png differ diff --git a/mux/images/mux_overview.png b/mux/images/mux_overview.png new file mode 100644 index 0000000000000..4825148a62c2a Binary files /dev/null and b/mux/images/mux_overview.png differ diff --git a/mux/manifest.json b/mux/manifest.json new file mode 100644 index 0000000000000..5210072ad0ace --- /dev/null +++ b/mux/manifest.json @@ -0,0 +1,78 @@ +{ + "manifest_version": "2.0.0", + "app_uuid": "9c1e5a09-836f-49c8-bbb2-2dd7f86ad0de", + "app_id": "mux", + "display_on_public_website": false, + "tile": { + "overview": "README.md#Overview", + "configuration": "README.md#Setup", + "support": "README.md#Support", + "changelog": "CHANGELOG.md", + "description": "Monitor Mux video performance and metrics.", + "title": "Mux", + "media": [ + { + "caption": "Mux - Metrics", + "image_url": "images/mux_metrics_1.png", + "media_type": "image" + }, + { + "caption": "Mux - Metrics", + "image_url": "images/mux_metrics_2.png", + "media_type": "image" + }, + { + "caption": "Mux - Metrics", + "image_url": "images/mux_metrics_3.png", + "media_type": "image" + }, + { + "caption": "Mux - Metrics", + "image_url": "images/mux_metrics_4.png", + "media_type": "image" + }, + { + "caption": "Mux - Metrics", + "image_url": "images/mux_metrics_5.png", + "media_type": "image" + } + ], + "classifier_tags": [ + "Category::Metrics", + "Offering::Integration", + "Submitted Data Type::Metrics" + ] + }, + "assets": { + "integration": { + "auto_install": false, + "source_type_id": 26205496, + "source_type_name": "Mux", + "events": { + "creates_events": false + }, + "metrics": { + "prefix": "mux.", + "check": "mux.metric.views", + "metadata_path": "metadata.csv" + }, + "service_checks": { + "metadata_path": "assets/service_checks.json" + } + }, + "dashboards": { + "Mux - Metrics": "assets/dashboards/mux_metrics.json" + }, + "monitors": { + "Anomaly Detected in Page Load Time": "assets/monitors/anomaly_detected_in_page_load_time.json", + "Playback Failure Percentage is higher than usual": "assets/monitors/playback_failure_percentage_is_higher_than_usual.json", + "Video Startup Failure Percentage is higher than usual": "assets/monitors/video_startup_failure_percentage_is_higher_than_usual.json" + } + }, + "author": { + "support_email": "help@datadoghq.com", + "name": "Datadog", + "homepage": "https://www.datadoghq.com", + "sales_email": "info@datadoghq.com" + } +} diff --git a/mux/metadata.csv b/mux/metadata.csv new file mode 100644 index 0000000000000..0d4e5bc0507fa --- /dev/null +++ b/mux/metadata.csv @@ -0,0 +1,29 @@ +metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags +mux.metric.aggregate_startup_time,gauge,,millisecond,,Total time taken for the video to start across all viewers.,1,mux,aggregate_startup_time,, +mux.metric.downscale_percentage,gauge,,percent,,Percentage of time the video was downscaled to a lower resolution.,1,mux,downscale_percentage,, +mux.metric.exits_before_video_start,gauge,,percent,,Percentage of users who exited before the video started playing.,1,mux,exits_before_video_start,, +mux.metric.live_stream_latency,gauge,,millisecond,,Delay between the live stream broadcast and when viewers see it.,1,mux,live_stream_latency,, +mux.metric.max_downscale_percentage,gauge,,percent,,Highest percentage of downscaled video during playback.,1,mux,max_downscale_percentage,, +mux.metric.max_upscale_percentage,gauge,,percent,,Highest percentage of upscaled video during playback.,1,mux,max_upscale_percentage,, +mux.metric.page_load_time,gauge,,millisecond,,Time it takes for the page hosting the video to load.,1,mux,page_load_time,, +mux.metric.playback_business_exception_percentage,gauge,,percent,,Percentage of playback failures caused by business-related issues.,1,mux,playback_business_exception_percentage,, +mux.metric.playback_failure_percentage,gauge,,percent,,Percentage of playback failures due to technical issues.,1,mux,playback_failure_percentage,, +mux.metric.playback_success_score,gauge,,,,Score representing playback success score of the video.,1,mux,playback_success_score,, +mux.metric.player_startup_time,gauge,,millisecond,,Time it takes for the video player to start playback.,1,mux,player_startup_time,, +mux.metric.playing_time,gauge,,millisecond,,Total time viewers spent watching the video.,1,mux,playing_time,, +mux.metric.rebuffer_count,gauge,,,,Total number of times the video buffered during playback.,1,mux,rebuffer_count,, +mux.metric.rebuffer_duration,gauge,,millisecond,,Total duration of buffering during playback.,1,mux,rebuffer_duration,, +mux.metric.rebuffer_frequency,gauge,,,minute,Frequency of buffering events per viewer.,1,mux,rebuffer_frequency,, +mux.metric.rebuffer_percentage,gauge,,percent,,Percentage of time spent buffering during playback.,1,mux,rebuffer_percentage,, +mux.metric.seek_latency,gauge,,millisecond,,Delay experienced when seeking to different parts of the video.,1,mux,seek_latency,, +mux.metric.smoothness_score,gauge,,,,Score representing smoothness score of the video.,1,mux,smoothness_score,, +mux.metric.startup_time_score,gauge,,,,Score representing how efficiently videos are starting.,1,mux,startup_time_score,, +mux.metric.unique_viewers,gauge,,,,Number of unique viewers who watched the video.,1,mux,unique_viewers,, +mux.metric.upscale_percentage,gauge,,percent,,Percentage of time the video was upscaled to a higher resolution.,1,mux,upscale_percentage,, +mux.metric.video_quality_score,gauge,,,,Score representing overall video quality based on resolution and bitrate.,1,mux,video_quality_score,, +mux.metric.video_startup_business_exception_percentage,gauge,,percent,,Percentage of video startup issues due to business-related exceptions.,1,mux,video_startup_business_exception_percentage,, +mux.metric.video_startup_failure_percentage,gauge,,percent,,Percentage of video startup failures due to technical issues.,1,mux,video_startup_failure_percentage,, +mux.metric.video_startup_time,gauge,,millisecond,,Time it takes for the video to start after a viewer initiates playback.,1,mux,video_startup_time,, +mux.metric.viewer_experience_score,gauge,,,,Score representing the overall viewer experience based on key metrics.,1,mux,viewer_experience_score,, +mux.metric.views,gauge,,,,Total number of views for the video.,1,mux,views,, +mux.metric.weighted_average_bitrate,gauge,,bit,second,"Average bitrate of video playback, weighted by viewer experience.",1,mux,weighted_average_bitrate,, diff --git a/mysql/CHANGELOG.md b/mysql/CHANGELOG.md index 54c9711ebb1a6..8955a0d54701d 100644 --- a/mysql/CHANGELOG.md +++ b/mysql/CHANGELOG.md @@ -2,6 +2,13 @@ +## 14.3.0 / 2024-11-28 + +***Added***: + +* Added the `dbms_flavor` tag to MySQL integration metrics and events to identify the database type. This tag indicates whether the database is MySQL or MariaDB. ([#18950](https://github.com/DataDog/integrations-core/pull/18950)) +* Submit database_hostname with database instance and metrics for MySQL, Postgres, and SQLServer ([#18969](https://github.com/DataDog/integrations-core/pull/18969)) + ## 14.2.0 / 2024-11-06 ***Added***: @@ -64,7 +71,7 @@ * Adding databases (schemas) data collection to MySQL These data include information about the tables, their columns, indexes, foreign keys, and partitions. ([#17916](https://github.com/DataDog/integrations-core/pull/17916)) -* Update dependencies ([#18185](https://github.com/DataDog/integrations-core/pull/18185)) +* Update dependencies ([#18187](https://github.com/DataDog/integrations-core/pull/18187)) ***Fixed***: diff --git a/mysql/assets/configuration/spec.yaml b/mysql/assets/configuration/spec.yaml index ce50bc273edf2..4b6104ea1cf89 100644 --- a/mysql/assets/configuration/spec.yaml +++ b/mysql/assets/configuration/spec.yaml @@ -665,6 +665,8 @@ files: description: | Set to `false` to disable the collection of comments in your SQL statements. Requires `collect_metadata: true`. + Note: This option must be `true` in order to correlate Database Monitoring samples and APM traces. + See https://docs.datadoghq.com/database_monitoring/connect_dbm_and_apm value: type: boolean example: true diff --git a/mysql/changelog.d/18950.added b/mysql/changelog.d/18950.added deleted file mode 100644 index e5959091a608d..0000000000000 --- a/mysql/changelog.d/18950.added +++ /dev/null @@ -1 +0,0 @@ -Added the `dbms_flavor` tag to MySQL integration metrics and events to identify the database type. This tag indicates whether the database is MySQL or MariaDB. diff --git a/mysql/changelog.d/18969.added b/mysql/changelog.d/18969.added deleted file mode 100644 index dfc29ca0f7645..0000000000000 --- a/mysql/changelog.d/18969.added +++ /dev/null @@ -1 +0,0 @@ -Submit database_hostname with database instance and metrics for MySQL, Postgres, and SQLServer diff --git a/mysql/changelog.d/19121.added b/mysql/changelog.d/19121.added new file mode 100644 index 0000000000000..97990bdf0ca9b --- /dev/null +++ b/mysql/changelog.d/19121.added @@ -0,0 +1 @@ +Add `mysql.performance.performance_schema_digest_lost`, the number of digest instances that could not be instrumented in the `events_statements_summary_by_digest` table. diff --git a/mysql/datadog_checks/mysql/__about__.py b/mysql/datadog_checks/mysql/__about__.py index 89a066d3cc640..58eaa19f57f56 100644 --- a/mysql/datadog_checks/mysql/__about__.py +++ b/mysql/datadog_checks/mysql/__about__.py @@ -2,4 +2,4 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = "14.2.0" +__version__ = "14.3.0" diff --git a/mysql/datadog_checks/mysql/const.py b/mysql/datadog_checks/mysql/const.py index fc2cb7d1f2e48..d69dc33dae014 100644 --- a/mysql/datadog_checks/mysql/const.py +++ b/mysql/datadog_checks/mysql/const.py @@ -39,6 +39,8 @@ # Table Cache Metrics 'Open_files': ('mysql.performance.open_files', GAUGE), 'Open_tables': ('mysql.performance.open_tables', GAUGE), + # Performance schema metrics + 'Performance_schema_digest_lost': ('mysql.performance.performance_schema_digest_lost', GAUGE), # Network Metrics 'Bytes_sent': ('mysql.performance.bytes_sent', RATE), 'Bytes_received': ('mysql.performance.bytes_received', RATE), diff --git a/mysql/datadog_checks/mysql/data/conf.yaml.example b/mysql/datadog_checks/mysql/data/conf.yaml.example index 3dbf966831e81..234a9fb7312e3 100644 --- a/mysql/datadog_checks/mysql/data/conf.yaml.example +++ b/mysql/datadog_checks/mysql/data/conf.yaml.example @@ -609,6 +609,8 @@ instances: ## @param collect_comments - boolean - optional - default: true ## Set to `false` to disable the collection of comments in your SQL statements. ## Requires `collect_metadata: true`. + ## Note: This option must be `true` in order to correlate Database Monitoring samples and APM traces. + ## See https://docs.datadoghq.com/database_monitoring/connect_dbm_and_apm # # collect_comments: true diff --git a/mysql/metadata.csv b/mysql/metadata.csv index 4737e971e22c2..027b8ce425e3a 100644 --- a/mysql/metadata.csv +++ b/mysql/metadata.csv @@ -170,6 +170,7 @@ mysql.performance.max_prepared_stmt_count,gauge,,,,The maximum allowed prepared mysql.performance.open_files,gauge,,file,,The number of open files.,0,mysql,open files, mysql.performance.open_tables,gauge,,table,,The number of of tables that are open.,0,mysql,open tables, mysql.performance.opened_tables,gauge,,table,second,"The number of tables that have been opened. If `opened_tables` is big, your `table_open_cache` value is probably too small.",0,mysql,mysql performance opened_tables, +mysql.performance.performance_schema_digest_lost,gauge,,,,The number of digest instances that could not be instrumented in the events_statements_summary_by_digest table. This can be nonzero if the value of performance_schema_digests_size is too small.,0,mysql,mysql performance performance schema digest lost, mysql.performance.prepared_stmt_count,gauge,,query,second,The current number of prepared statements.,0,mysql,current prepared statements, mysql.performance.qcache.utilization,gauge,,fraction,,Fraction of the query cache memory currently being used.,0,mysql,mysql performance qcache utilization, mysql.performance.qcache_free_blocks,gauge,,block,,The number of free memory blocks in the query cache.,0,mysql,mysql performance qcache_free_blocks, diff --git a/mysql/tests/variables.py b/mysql/tests/variables.py index 246616bc1c426..825c44307f76e 100644 --- a/mysql/tests/variables.py +++ b/mysql/tests/variables.py @@ -25,6 +25,8 @@ # Table Cache Metrics 'mysql.performance.open_files', 'mysql.performance.open_tables', + # Performance schema metrics + 'mysql.performance.performance_schema_digest_lost', # Network Metrics 'mysql.performance.bytes_sent', 'mysql.performance.bytes_received', diff --git a/network_path/manifest.json b/network_path/manifest.json index ea5c107897c1f..e43f0cf937a66 100644 --- a/network_path/manifest.json +++ b/network_path/manifest.json @@ -13,6 +13,7 @@ "media": [], "classifier_tags": [ "Supported OS::Linux", + "Supported OS::Windows", "Category::Network", "Offering::Integration" ] diff --git a/nvidia_nim/CHANGELOG.md b/nvidia_nim/CHANGELOG.md index efe8873dc872e..fbd141ca35f92 100644 --- a/nvidia_nim/CHANGELOG.md +++ b/nvidia_nim/CHANGELOG.md @@ -2,3 +2,12 @@ +## 1.0.0 / 2024-11-28 + +***Added***: + +* Initial Release ([#18964](https://github.com/DataDog/integrations-core/pull/18964)) + +***Fixed***: + +* Bump base package dependency to get fixed pyyaml. ([#19156](https://github.com/DataDog/integrations-core/pull/19156)) diff --git a/nvidia_nim/assets/dashboards/nvidia_nim_overview.json b/nvidia_nim/assets/dashboards/nvidia_nim_overview.json index ee9a6dc7af428..299362b371984 100644 --- a/nvidia_nim/assets/dashboards/nvidia_nim_overview.json +++ b/nvidia_nim/assets/dashboards/nvidia_nim_overview.json @@ -205,7 +205,7 @@ "hide_zero_counts": true, "show_status": true, "last_triggered_format": "relative", - "query": "tag:(integration:vllm)", + "query": "tag:(integration:nvidia_nim)", "sort": "status,asc", "count": 50, "start": 0, @@ -265,12 +265,26 @@ "title": "Requests Waiting", "title_size": "16", "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], "time": {}, "type": "timeseries", "requests": [ { "formulas": [ { + "number_format": { + "unit": { + "type": "canonical_unit" + } + }, "formula": "query1" } ], @@ -278,7 +292,7 @@ { "name": "query1", "data_source": "metrics", - "query": "avg:nvidia_nim.num_requests.waiting{$model_name} by {model_name}" + "query": "avg:nvidia_nim.num_requests.waiting{$model_name, $host} by {model_name}" } ], "response_format": "timeseries", @@ -305,7 +319,6 @@ "title": "Requests Waiting", "title_size": "16", "title_align": "left", - "time": {}, "type": "query_value", "requests": [ { @@ -370,12 +383,26 @@ "title": "Requests Failed", "title_size": "16", "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], "time": {}, "type": "timeseries", "requests": [ { "formulas": [ { + "number_format": { + "unit": { + "type": "canonical_unit" + } + }, "formula": "query1" } ], @@ -383,7 +410,7 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:nvidia_nim.request.failure.count{$model_name} by {model_name}.as_count()" + "query": "sum:nvidia_nim.request.failure.count{$model_name, $host} by {model_name}.as_count()" } ], "response_format": "timeseries", @@ -488,7 +515,7 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:nvidia_nim.request.success.count{$model_name} by {model_name}.as_rate()" + "query": "sum:nvidia_nim.request.success.count{$model_name, $host} by {model_name}.as_rate()" } ], "response_format": "timeseries", @@ -514,7 +541,6 @@ "title": "Requests Running", "title_size": "16", "title_align": "left", - "time": {}, "type": "query_value", "requests": [ { @@ -550,9 +576,18 @@ { "id": 2448557456884510, "definition": { - "title": "K/V Cache Utilization", + "title": "GPU Cache Utilization", "title_size": "16", "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], "time": {}, "type": "timeseries", "requests": [ @@ -566,7 +601,7 @@ { "data_source": "metrics", "name": "query1", - "query": "avg:nvidia_nim.gpu_cache_usage_percent{$model_name}" + "query": "avg:nvidia_nim.gpu_cache_usage_percent{$model_name, $host} by {model_name}" } ], "response_format": "timeseries", @@ -705,12 +740,16 @@ "value", "sum" ], - "time": {}, "type": "timeseries", "requests": [ { "formulas": [ { + "number_format": { + "unit": { + "type": "canonical_unit" + } + }, "formula": "query1 / query2" } ], @@ -718,12 +757,12 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:nvidia_nim.time_to_first_token.seconds.sum{$model_name} by {model_name}.as_count()" + "query": "sum:nvidia_nim.time_to_first_token.seconds.sum{$model_name, $host} by {model_name}.as_count()" }, { "data_source": "metrics", "name": "query2", - "query": "sum:nvidia_nim.time_to_first_token.seconds.count{$model_name} by {model_name}.as_count()" + "query": "sum:nvidia_nim.time_to_first_token.seconds.count{$model_name, $host} by {model_name}.as_count()" } ], "response_format": "timeseries", @@ -759,7 +798,6 @@ "value", "sum" ], - "time": {}, "type": "timeseries", "requests": [ { @@ -775,12 +813,12 @@ ], "queries": [ { - "query": "avg:nvidia_nim.request.prompt_tokens.sum{$model_name} by {model_name}.as_count()", + "query": "avg:nvidia_nim.request.prompt_tokens.sum{$model_name, $host} by {model_name}.as_count()", "data_source": "metrics", "name": "query2" }, { - "query": "avg:nvidia_nim.request.generation_tokens.sum{$model_name} by {model_name}.as_count()", + "query": "avg:nvidia_nim.request.generation_tokens.sum{$model_name, $host} by {model_name}.as_count()", "data_source": "metrics", "name": "query1" } @@ -818,12 +856,16 @@ "value", "sum" ], - "time": {}, "type": "timeseries", "requests": [ { "formulas": [ { + "number_format": { + "unit": { + "type": "canonical_unit" + } + }, "formula": "query1 / query2" } ], @@ -831,12 +873,12 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:nvidia_nim.time_per_output_token.seconds.sum{$model_name} by {model_name}.as_count()" + "query": "sum:nvidia_nim.time_per_output_token.seconds.sum{$model_name, $host} by {model_name}.as_count()" }, { "data_source": "metrics", "name": "query2", - "query": "sum:nvidia_nim.time_per_output_token.seconds.count{$model_name} by {model_name}.as_count()" + "query": "sum:nvidia_nim.time_per_output_token.seconds.count{$model_name, $host} by {model_name}.as_count()" } ], "response_format": "timeseries", @@ -861,7 +903,7 @@ }, "layout": { "x": 0, - "y": 20, + "y": 0, "width": 12, "height": 10, "is_column_break": true @@ -924,7 +966,7 @@ { "data_source": "metrics", "name": "query1", - "query": "avg:nvidia_nim.process.resident_memory_bytes{$model_name}" + "query": "avg:nvidia_nim.process.resident_memory_bytes{$host} by {endpoint}" } ], "response_format": "timeseries", @@ -971,7 +1013,7 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:nvidia_nim.python.gc.collections.count{$model_name} by {generation}.as_count()" + "query": "sum:nvidia_nim.python.gc.collections.count{$host} by {generation,endpoint}.as_count()" } ], "response_format": "timeseries", @@ -998,7 +1040,15 @@ "title": "Uncollectable Objects", "title_size": "16", "title_align": "left", - "time": {}, + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], "type": "timeseries", "requests": [ { @@ -1011,7 +1061,7 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:nvidia_nim.python.gc.objects.uncollectable.count{$host}.as_count()" + "query": "sum:nvidia_nim.python.gc.objects.uncollectable.count{$host} by {endpoint,generation}.as_count()" } ], "response_format": "timeseries", @@ -1047,7 +1097,6 @@ "value", "sum" ], - "time": {}, "type": "timeseries", "requests": [ { @@ -1060,7 +1109,7 @@ { "data_source": "metrics", "name": "query1", - "query": "avg:nvidia_nim.process.virtual_memory_bytes{$host} by {host}" + "query": "avg:nvidia_nim.process.virtual_memory_bytes{$host} by {endpoint}" } ], "response_format": "timeseries", @@ -1107,7 +1156,7 @@ { "data_source": "metrics", "name": "query1", - "query": "sum:nvidia_nim.python.gc.objects.collected.count{$model_name} by {generation}.as_count()" + "query": "sum:nvidia_nim.python.gc.objects.collected.count{$model_name, $host} by {generation}.as_count()" } ], "response_format": "timeseries", @@ -1131,7 +1180,7 @@ }, "layout": { "x": 0, - "y": 30, + "y": 10, "width": 12, "height": 8 } diff --git a/nvidia_nim/changelog.d/18964.added b/nvidia_nim/changelog.d/18964.added deleted file mode 100644 index aa949b47b7b41..0000000000000 --- a/nvidia_nim/changelog.d/18964.added +++ /dev/null @@ -1 +0,0 @@ -Initial Release \ No newline at end of file diff --git a/nvidia_nim/datadog_checks/nvidia_nim/__about__.py b/nvidia_nim/datadog_checks/nvidia_nim/__about__.py index e9541ce83e9e5..acbfd1c866b84 100644 --- a/nvidia_nim/datadog_checks/nvidia_nim/__about__.py +++ b/nvidia_nim/datadog_checks/nvidia_nim/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '0.0.1' +__version__ = '1.0.0' diff --git a/nvidia_nim/pyproject.toml b/nvidia_nim/pyproject.toml index 9681369b06977..805bb2533fb3b 100644 --- a/nvidia_nim/pyproject.toml +++ b/nvidia_nim/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ "Topic :: System :: Monitoring", ] dependencies = [ - "datadog-checks-base>=32.6.0", + "datadog-checks-base>=33.0.0", ] dynamic = [ "version", diff --git a/openstack_controller/CHANGELOG.md b/openstack_controller/CHANGELOG.md index 5c03d3494bcde..bd68268c2a5c5 100644 --- a/openstack_controller/CHANGELOG.md +++ b/openstack_controller/CHANGELOG.md @@ -26,7 +26,7 @@ ***Added***: -* Update dependencies ([#18185](https://github.com/DataDog/integrations-core/pull/18185)) +* Update dependencies ([#18187](https://github.com/DataDog/integrations-core/pull/18187)) ## 6.8.1 / 2024-07-24 / Agent 7.56.0 diff --git a/oracle/metadata.csv b/oracle/metadata.csv index 216c5bca02281..e0aad71bbf027 100644 --- a/oracle/metadata.csv +++ b/oracle/metadata.csv @@ -1,37 +1,35 @@ -metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric -oracle.active_sessions,gauge,,,,Number of active sessions,0,oracle_database,active sessions, -oracle.asm_diskgroup.free_mb,gauge,,,,"The unused capacity of a disk group in megabytes, tagged by `state` (DBM only)",0,oracle_database,disk group free megabytes, -oracle.asm_diskgroup.total_mb,gauge,,,,"The total usable capacity of the disk group, tagged by `state` (DBM only)",0,oracle_database,disk group total megabytes, -oracle.asm_diskgroup.offline_disks,gauge,,,,"The number of disks in an ASM disk group that are offline, tagged by `state` (DBM only)",0,oracle_database,offline disks in asm disk group, -oracle.buffer_cachehit_ratio,gauge,,percent,,Ratio of buffer cache hits,1,oracle_database,buff cache hit ratio, -oracle.cache_blocks_corrupt,gauge,,block,,Corrupt cache blocks,0,oracle_database,corrupt cache blocks, -oracle.cache_blocks_lost,gauge,,block,,Lost cache blocks,0,oracle_database,lost cache blocks, -oracle.cursor_cachehit_ratio,gauge,,percent,,Ratio of cursor cache hits,1,oracle_database,cursor cache hit ratio, -oracle.database_wait_time_ratio,gauge,,percent,,Memory sorts per second,0,oracle_database,db wait-time ratio, -oracle.disk_sorts,gauge,,operation,second,Disk sorts per second,0,oracle_database,disk sorts, -oracle.enqueue_timeouts,gauge,,timeout,second,Enqueue timeouts per sec,0,oracle_database,enqueue timeouts, -oracle.gc_cr_block_received,gauge,,block,second,GC CR block received,0,oracle_database,gc cr block rcv , -oracle.library_cachehit_ratio,gauge,,percent,,Ratio of library cache hits,1,oracle_database,lib cache hit ratio, -oracle.logons,gauge,,,,Number of logon attempts,0,oracle_database,logons, -oracle.long_table_scans,gauge,,scan,second,Number of long table scans per sec,0,oracle_database,long tbl scans, -oracle.memory_sorts_ratio,gauge,,percent,,Memory sorts ratio,0,oracle_database,memory sort ratio, -oracle.physical_reads,gauge,,read,second,Physical reads per sec,0,oracle_database,phys reads, -oracle.physical_writes,gauge,,write,second,Physical writes per sec,0,oracle_database,phys reads, -oracle.process.pga_allocated_memory,gauge,,byte,,PGA memory allocated by process,0,oracle_database,pga memory allocated, -oracle.process.pga_freeable_memory,gauge,,byte,,PGA memory freeable by process,0,oracle_database,pga memory freeable, -oracle.process.pga_maximum_memory,gauge,,byte,,PGA maximum memory ever allocated by process,0,oracle_database,pga max memory allocated, -oracle.process.pga_used_memory,gauge,,byte,,PGA memory used by process,0,oracle_database,pga memory used, -oracle.rows_per_sort,gauge,,row,operation,Rows per sort,0,oracle_database,rows per sort, -oracle.service_response_time,gauge,,second,,Service response time,0,oracle_database,svc resp time, -oracle.session_count,gauge,,,,Session count,0,oracle_database,session count, -oracle.session_limit_usage,gauge,,percent,,Session limit usage,0,oracle_database,session lim usage %, -oracle.shared_pool_free,gauge,,percent,,Shared pool free memory %,0,oracle_database,shared pool free, -oracle.sorts_per_user_call,gauge,,,,Sorts per user call,0,oracle_database,sorts per user call, -oracle.tablespace.in_use,gauge,,percent,,Tablespace in-use,0,oracle_database,tablespace in-use, -oracle.tablespace.offline,gauge,,,,Tablespace offline,0,oracle_database,tablespace offline, -oracle.tablespace.size,gauge,,byte,,Tablespace size,0,oracle_database,tablespace size, -oracle.tablespace.used,gauge,,byte,,Tablespace used,0,oracle_database,tablespace used, -oracle.temp_space_used,gauge,,byte,,Temp space used,0,oracle_database,temp space used, -oracle.user_rollbacks,gauge,,operation,,number of user rollbacks,0,oracle_database,user rollbacks, - - +metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags +oracle.active_sessions,gauge,,,,Number of active sessions,0,oracle_database,active sessions,, +oracle.asm_diskgroup.free_mb,gauge,,,,"The unused capacity of a disk group in megabytes, tagged by `state` (DBM only)",0,oracle_database,disk group free megabytes,, +oracle.asm_diskgroup.offline_disks,gauge,,,,"The number of disks in an ASM disk group that are offline, tagged by `state` (DBM only)",0,oracle_database,offline disks in asm disk group,, +oracle.asm_diskgroup.total_mb,gauge,,,,"The total usable capacity of the disk group, tagged by `state` (DBM only)",0,oracle_database,disk group total megabytes,, +oracle.buffer_cachehit_ratio,gauge,,percent,,Ratio of buffer cache hits,1,oracle_database,buff cache hit ratio,, +oracle.cache_blocks_corrupt,gauge,,block,,Corrupt cache blocks,0,oracle_database,corrupt cache blocks,, +oracle.cache_blocks_lost,gauge,,block,,Lost cache blocks,0,oracle_database,lost cache blocks,, +oracle.cursor_cachehit_ratio,gauge,,percent,,Ratio of cursor cache hits,1,oracle_database,cursor cache hit ratio,, +oracle.database_wait_time_ratio,gauge,,percent,,Memory sorts per second,0,oracle_database,db wait-time ratio,, +oracle.disk_sorts,gauge,,operation,second,Disk sorts per second,0,oracle_database,disk sorts,, +oracle.enqueue_timeouts,gauge,,timeout,second,Enqueue timeouts per sec,0,oracle_database,enqueue timeouts,, +oracle.gc_cr_block_received,gauge,,block,second,GC CR block received,0,oracle_database,gc cr block rcv ,, +oracle.library_cachehit_ratio,gauge,,percent,,Ratio of library cache hits,1,oracle_database,lib cache hit ratio,, +oracle.logons,gauge,,,,Number of logon attempts,0,oracle_database,logons,, +oracle.long_table_scans,gauge,,scan,second,Number of long table scans per sec,0,oracle_database,long tbl scans,, +oracle.memory_sorts_ratio,gauge,,percent,,Memory sorts ratio,0,oracle_database,memory sort ratio,, +oracle.physical_reads,gauge,,read,second,Physical reads per sec,0,oracle_database,phys reads,, +oracle.physical_writes,gauge,,write,second,Physical writes per sec,0,oracle_database,phys reads,, +oracle.process.pga_allocated_memory,gauge,,byte,,PGA memory allocated by process,0,oracle_database,pga memory allocated,, +oracle.process.pga_freeable_memory,gauge,,byte,,PGA memory freeable by process,0,oracle_database,pga memory freeable,, +oracle.process.pga_maximum_memory,gauge,,byte,,PGA maximum memory ever allocated by process,0,oracle_database,pga max memory allocated,, +oracle.process.pga_used_memory,gauge,,byte,,PGA memory used by process,0,oracle_database,pga memory used,, +oracle.rows_per_sort,gauge,,row,operation,Rows per sort,0,oracle_database,rows per sort,, +oracle.service_response_time,gauge,,second,,Service response time,0,oracle_database,svc resp time,, +oracle.session_count,gauge,,,,Session count,0,oracle_database,session count,, +oracle.session_limit_usage,gauge,,percent,,Session limit usage,0,oracle_database,session lim usage %,, +oracle.shared_pool_free,gauge,,percent,,Shared pool free memory %,0,oracle_database,shared pool free,, +oracle.sorts_per_user_call,gauge,,,,Sorts per user call,0,oracle_database,sorts per user call,, +oracle.tablespace.in_use,gauge,,percent,,Tablespace in-use,0,oracle_database,tablespace in-use,, +oracle.tablespace.offline,gauge,,,,Tablespace offline,0,oracle_database,tablespace offline,, +oracle.tablespace.size,gauge,,byte,,Tablespace size,0,oracle_database,tablespace size,, +oracle.tablespace.used,gauge,,byte,,Tablespace used,0,oracle_database,tablespace used,, +oracle.temp_space_used,gauge,,byte,,Temp space used,0,oracle_database,temp space used,, +oracle.user_rollbacks,gauge,,operation,,number of user rollbacks,0,oracle_database,user rollbacks,, diff --git a/plaid/CHANGELOG.md b/plaid/CHANGELOG.md new file mode 100644 index 0000000000000..76f7b7ed1d71e --- /dev/null +++ b/plaid/CHANGELOG.md @@ -0,0 +1,7 @@ +# CHANGELOG - Plaid + +## 1.0.0 / 2024-10-25 + +***Added***: + +* Initial Release \ No newline at end of file diff --git a/plaid/README.md b/plaid/README.md new file mode 100644 index 0000000000000..4f508b4d0af23 --- /dev/null +++ b/plaid/README.md @@ -0,0 +1,46 @@ +# Agent Check: Plaid + +[Plaid](https://plaid.com/) specializes in financial technology by offering APIs that allow developers to integrate banking services into their applications. By connecting users' bank accounts to apps, Plaid enables features like account verification, transaction history retrieval, and balance checks. This functionality is crucial for various applications, including budgeting tools, personal finance management, and payment processing. + +## Overview + +Here are some insights that can be drawn from your Plaid dashboard: + +- **Descriptive Trends**: Assess common descriptions for categorization. +- **Failure Patterns**: Investigate failure reasons to improve reliability. +- **Network Performance**: Evaluate network effectiveness and transaction success rates. +- **Status Monitoring**: Track overall transaction statuses for operational efficiency. +- **Sweep Trends**: Analyze sweep statuses to understand fund movement dynamics. +- **Type Classification**: Categorize transactions by type for deeper financial insights. +- **Currency Insights**: Examine iso_currency_code for multi-currency transaction patterns. + +## Setup + +1. Log in to [Plaid](https://dashboard.plaid.com/signin/). +2. Client ID and Secret can be obtained through this [link](https://dashboard.plaid.com/developers/keys). + +### Configuration + +Configure the Datadog endpoint to forward Plaid logs to Datadog. +1. Navigate to Plaid. +2. Add your Plaid credentials. + +| Plaid Parameters | Description | +|----------|----------| +| Client ID | Client of the Plaid account. | +| Secret | Secret of the Plaid account | + + +## Data Collected + +The crawler will implement data collection of Plaid logs for the List of Transfer events, remove sensitive data and send it to Datadog. + + +## Troubleshooting + +Need help? Contact [Datadog support][3]. + +[1]: **LINK_TO_INTEGRATION_SITE** +[2]: https://app.datadoghq.com/account/settings/agent/latest +[3]: https://docs.datadoghq.com/help/ + diff --git a/plaid/assets/service_checks.json b/plaid/assets/service_checks.json new file mode 100644 index 0000000000000..0637a088a01e8 --- /dev/null +++ b/plaid/assets/service_checks.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/plaid/manifest.json b/plaid/manifest.json new file mode 100644 index 0000000000000..695580a4b390a --- /dev/null +++ b/plaid/manifest.json @@ -0,0 +1,47 @@ +{ + "manifest_version": "2.0.0", + "app_uuid": "104e8e43-bf5b-44f1-8ef3-76ad53a39c05", + "app_id": "plaid", + "display_on_public_website": false, + "tile": { + "overview": "README.md#Overview", + "configuration": "README.md#Setup", + "support": "README.md#Support", + "changelog": "CHANGELOG.md", + "description": "Gain insights into Plaid logs.", + "title": "Plaid", + "media": [], + "classifier_tags": [ + "Category::Log Collection", + "Offering::Integration", + "Submitted Data Type::Logs" + ] + }, + "assets": { + "integration": { + "auto_install": true, + "source_type_id": 30173459, + "source_type_name": "plaid", + "events": { + "creates_events": false + }, + "metrics": { + "prefix": "plaid.", + "check": "", + "metadata_path": "metadata.csv" + }, + "service_checks": { + "metadata_path": "assets/service_checks.json" + } + } + }, + "author": { + "support_email": "help@datadoghq.com", + "name": "Datadog", + "homepage": "https://www.datadoghq.com", + "sales_email": "info@datadoghq.com" + } +} + + + \ No newline at end of file diff --git a/plaid/metadata.csv b/plaid/metadata.csv new file mode 100644 index 0000000000000..02cde5e98381e --- /dev/null +++ b/plaid/metadata.csv @@ -0,0 +1 @@ +metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags diff --git a/postgres/CHANGELOG.md b/postgres/CHANGELOG.md index fd5c3d3d7f62b..cb8a7c57f2734 100644 --- a/postgres/CHANGELOG.md +++ b/postgres/CHANGELOG.md @@ -2,6 +2,17 @@ +## 22.3.0 / 2024-11-28 + +***Added***: + +* Submit database_hostname with database instance and metrics for MySQL, Postgres, and SQLServer ([#18969](https://github.com/DataDog/integrations-core/pull/18969)) +* Track logical replication slot catalog_xmin age ([#19083](https://github.com/DataDog/integrations-core/pull/19083)) + +***Fixed***: + +* Add alloydbadmin & alloydbmetadata to default list of databases to exclude from autodiscovery and databases to ignore to prevent failures on GCP AlloyDB for PostgreSQL. ([#19061](https://github.com/DataDog/integrations-core/pull/19061)) + ## 22.2.0 / 2024-11-06 ***Added***: @@ -88,7 +99,7 @@ * Allow filtering of schema collection in Postgres using regexes to include or exclude objects ([#18145](https://github.com/DataDog/integrations-core/pull/18145)) * Collect blk read/write time from pg_stat_database ([#18169](https://github.com/DataDog/integrations-core/pull/18169)) * Use QueryManager to collect `custom_queries` and `global_custom_queries`. `custom_queries` now supports configurable `collection_interval`. ([#18183](https://github.com/DataDog/integrations-core/pull/18183)) -* Update dependencies ([#18185](https://github.com/DataDog/integrations-core/pull/18185)) +* Update dependencies ([#18187](https://github.com/DataDog/integrations-core/pull/18187)) * Add new config option `role_arn` to AWS managed authentication to support cross account IAM auth. ([#18228](https://github.com/DataDog/integrations-core/pull/18228)) ***Fixed***: diff --git a/postgres/assets/configuration/spec.yaml b/postgres/assets/configuration/spec.yaml index deb2db6cb8f9f..016064d782779 100644 --- a/postgres/assets/configuration/spec.yaml +++ b/postgres/assets/configuration/spec.yaml @@ -888,6 +888,8 @@ files: description: | Set to `false` to disable the collection of comments in your SQL statements. Requires `collect_metadata: true`. + Note: This option must be `true` in order to correlate Database Monitoring samples and APM traces. + See https://docs.datadoghq.com/database_monitoring/connect_dbm_and_apm value: type: boolean example: true diff --git a/postgres/changelog.d/18969.added b/postgres/changelog.d/18969.added deleted file mode 100644 index dfc29ca0f7645..0000000000000 --- a/postgres/changelog.d/18969.added +++ /dev/null @@ -1 +0,0 @@ -Submit database_hostname with database instance and metrics for MySQL, Postgres, and SQLServer diff --git a/postgres/changelog.d/19061.fixed b/postgres/changelog.d/19061.fixed deleted file mode 100644 index 8fc692037374f..0000000000000 --- a/postgres/changelog.d/19061.fixed +++ /dev/null @@ -1 +0,0 @@ -Add alloydbadmin & alloydbmetadata to default list of databases to exclude from autodiscovery and databases to ignore to prevent failures on GCP AlloyDB for PostgreSQL. diff --git a/postgres/changelog.d/19083.added b/postgres/changelog.d/19083.added deleted file mode 100644 index 9a4d0e63f76db..0000000000000 --- a/postgres/changelog.d/19083.added +++ /dev/null @@ -1 +0,0 @@ -Track logical replication slot catalog_xmin age diff --git a/postgres/changelog.d/19218.added b/postgres/changelog.d/19218.added new file mode 100644 index 0000000000000..7ba3bd2933029 --- /dev/null +++ b/postgres/changelog.d/19218.added @@ -0,0 +1 @@ +Add postgresql.relation.xmin metric diff --git a/postgres/datadog_checks/postgres/__about__.py b/postgres/datadog_checks/postgres/__about__.py index 97cbee2999388..c510a97eec89f 100644 --- a/postgres/datadog_checks/postgres/__about__.py +++ b/postgres/datadog_checks/postgres/__about__.py @@ -2,4 +2,4 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = "22.2.0" +__version__ = "22.3.0" diff --git a/postgres/datadog_checks/postgres/data/conf.yaml.example b/postgres/datadog_checks/postgres/data/conf.yaml.example index e39a8b1d87fc9..d9d3c396afa0e 100644 --- a/postgres/datadog_checks/postgres/data/conf.yaml.example +++ b/postgres/datadog_checks/postgres/data/conf.yaml.example @@ -778,6 +778,8 @@ instances: ## @param collect_comments - boolean - optional - default: true ## Set to `false` to disable the collection of comments in your SQL statements. ## Requires `collect_metadata: true`. + ## Note: This option must be `true` in order to correlate Database Monitoring samples and APM traces. + ## See https://docs.datadoghq.com/database_monitoring/connect_dbm_and_apm # # collect_comments: true diff --git a/postgres/datadog_checks/postgres/relationsmanager.py b/postgres/datadog_checks/postgres/relationsmanager.py index b9dec77283709..1952534049e1c 100644 --- a/postgres/datadog_checks/postgres/relationsmanager.py +++ b/postgres/datadog_checks/postgres/relationsmanager.py @@ -187,7 +187,8 @@ pg_stat_get_vacuum_count(C.reltoastrelid), pg_stat_get_autovacuum_count(C.reltoastrelid), EXTRACT(EPOCH FROM age(CURRENT_TIMESTAMP, pg_stat_get_last_vacuum_time(C.reltoastrelid))), - EXTRACT(EPOCH FROM age(CURRENT_TIMESTAMP, pg_stat_get_last_autovacuum_time(C.reltoastrelid))) + EXTRACT(EPOCH FROM age(CURRENT_TIMESTAMP, pg_stat_get_last_autovacuum_time(C.reltoastrelid))), + C.xmin FROM pg_class C LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) LEFT JOIN pg_locks L ON C.oid = L.relation AND L.locktype = 'relation' @@ -234,6 +235,7 @@ {'name': 'toast.autovacuumed', 'type': 'monotonic_count'}, {'name': 'toast.last_vacuum_age', 'type': 'gauge'}, {'name': 'toast.last_autovacuum_age', 'type': 'gauge'}, + {'name': 'relation.xmin', 'type': 'gauge'}, ], } diff --git a/postgres/metadata.csv b/postgres/metadata.csv index 1cf6813327fe3..49183d67a61cc 100644 --- a/postgres/metadata.csv +++ b/postgres/metadata.csv @@ -116,6 +116,7 @@ postgresql.queries.time,count,,nanosecond,,"The total query execution time per q postgresql.relation.all_visible,gauge,,,,"Number of pages that are marked as all visible in the table's visibility map. This is only an estimation used by the planner and is updated by VACUUM or ANALYZE. This metric is tagged with db, schema, table, partition_of",0,postgres,relation all_visible, postgresql.relation.pages,gauge,,,,"Size of a table in pages (1 page == 8KB by default). This is only an estimation used by the planner and is updated by VACUUM or ANALYZE. This metric is tagged with db, schema, table, partition_of.",0,postgres,relation pages, postgresql.relation.tuples,gauge,,,,"Number of live rows in the table. This is only an estimation used by the planner and is updated by VACUUM or ANALYZE. If the table has never been vacuumed or analyze, -1 will be reported. This metric is tagged with db, schema, table, partition_of",0,postgres,relation tuples, +postgresql.relation.xmin,gauge,,,,"Transaction ID of the latest relation's modification in pg_class. This metric is tagged with db, schema, table",0,postgres,relation xmin, postgresql.relation_size,gauge,,byte,,"The disk space used by the specified table. TOAST data, indexes, free space map and visibility map are not included. This metric is tagged with db, schema, table.",0,postgres,relation size, postgresql.replication.backend_xmin_age,gauge,,,,The age of the standby server's xmin horizon (relative to latest stable xid) reported by hot_standby_feedback.,-1,postgres,repl backend xmin, postgresql.replication.wal_flush_lag,gauge,,second,,Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written and flushed it (but not yet applied it). This can be used to gauge the delay that synchronous_commit level on incurred while committing if this server was configured as a synchronous standby. Only available with postgresql 10 and newer.,-1,postgres,repl flush lag, diff --git a/postgres/tests/test_relations.py b/postgres/tests/test_relations.py index e2e5799d7c128..97e9c192732bb 100644 --- a/postgres/tests/test_relations.py +++ b/postgres/tests/test_relations.py @@ -197,6 +197,35 @@ def test_relations_metrics_regex(aggregator, integration_check, pg_instance): _check_metrics_for_relation_wo_index(aggregator, expected_tags[relation]) +@pytest.mark.integration +@pytest.mark.usefixtures('dd_environment') +def test_relations_xmin(aggregator, integration_check, pg_instance): + pg_instance['relations'] = ['persons'] + + conn = _get_superconn(pg_instance) + cursor = conn.cursor() + cursor.execute("SELECT xmin FROM pg_class WHERE relname='persons'") + start_xmin = float(cursor.fetchone()[0]) + + # Check that initial xmin metric match + check = integration_check(pg_instance) + check.check(pg_instance) + expected_tags = _get_expected_tags(check, pg_instance, db=pg_instance['dbname'], table='persons', schema='public') + aggregator.assert_metric('postgresql.relation.xmin', count=1, value=start_xmin, tags=expected_tags) + aggregator.reset() + + # Run multiple DDL modifying the persons relation which will increase persons' xmin in pg_class + cursor.execute("ALTER TABLE persons REPLICA IDENTITY FULL;") + cursor.execute("ALTER TABLE persons REPLICA IDENTITY DEFAULT;") + cursor.close() + conn.close() + + check.check(pg_instance) + + # xmin metric should be greater than initial xmin + assert_metric_at_least(aggregator, 'postgresql.relation.xmin', lower_bound=start_xmin + 1, tags=expected_tags) + + @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') def test_max_relations(aggregator, integration_check, pg_instance): diff --git a/redisdb/CHANGELOG.md b/redisdb/CHANGELOG.md index 5c11b27d25849..167d25edbc34f 100644 --- a/redisdb/CHANGELOG.md +++ b/redisdb/CHANGELOG.md @@ -26,7 +26,7 @@ ***Added***: -* Update dependencies ([#18185](https://github.com/DataDog/integrations-core/pull/18185)) +* Update dependencies ([#18187](https://github.com/DataDog/integrations-core/pull/18187)) ## 5.6.0 / 2024-07-05 / Agent 7.56.0 diff --git a/requirements-agent-release.txt b/requirements-agent-release.txt index 85498b3337d42..25cbc26497221 100644 --- a/requirements-agent-release.txt +++ b/requirements-agent-release.txt @@ -13,7 +13,7 @@ datadog-appgate-sdp==1.0.0; sys_platform != 'win32' datadog-arangodb==3.1.0 datadog-argo-rollouts==2.1.0 datadog-argo-workflows==2.1.0 -datadog-argocd==3.1.0 +datadog-argocd==3.2.0 datadog-aspdotnet==4.0.0; sys_platform == 'win32' datadog-avi-vantage==5.1.0 datadog-aws-neuron==2.0.1 @@ -27,21 +27,21 @@ datadog-cassandra==3.0.0 datadog-ceph==4.0.0; sys_platform != 'win32' datadog-cert-manager==5.1.0 datadog-checkpoint-quantum-firewall==1.0.0 -datadog-checks-base==37.1.0 +datadog-checks-base==37.2.0 datadog-checks-dependency-provider==3.0.0 -datadog-checks-downloader==6.1.0 +datadog-checks-downloader==7.0.0 datadog-cilium==5.0.0 datadog-cisco-aci==4.1.0 datadog-cisco-secure-firewall==1.0.0 datadog-citrix-hypervisor==5.0.0 -datadog-clickhouse==5.0.0 +datadog-clickhouse==5.1.0 datadog-cloud-foundry-api==5.0.0 datadog-cloudera==3.2.0 datadog-cockroachdb==5.0.0 datadog-confluent-platform==3.0.0 datadog-consul==4.0.0 datadog-coredns==5.0.0; sys_platform == 'linux2' -datadog-couch==8.0.0 +datadog-couch==8.1.0 datadog-couchbase==5.0.0 datadog-crio==4.0.0 datadog-datadog-cluster-agent==5.1.0 @@ -81,7 +81,7 @@ datadog-http-check==11.0.0 datadog-hudi==4.0.0 datadog-hyperv==3.0.0; sys_platform == 'win32' datadog-ibm-ace==4.0.0 -datadog-ibm-db2==4.0.0 +datadog-ibm-db2==4.0.1 datadog-ibm-i==4.0.0; sys_platform != 'win32' datadog-ibm-mq==8.0.0 datadog-ibm-was==5.0.0 @@ -107,7 +107,7 @@ datadog-kubernetes-cluster-autoscaler==2.1.0 datadog-kubernetes-state==10.0.0 datadog-kubevirt-api==1.0.0 datadog-kubevirt-controller==1.0.0 -datadog-kubevirt-handler==1.0.0 +datadog-kubevirt-handler==1.0.1 datadog-kyototycoon==4.0.0 datadog-kyverno==2.1.0 datadog-lighttpd==5.0.0 @@ -120,13 +120,14 @@ datadog-marklogic==6.0.0 datadog-mcache==6.0.0; sys_platform != 'win32' datadog-mesos-master==5.0.0; sys_platform != 'win32' datadog-mesos-slave==5.0.0; sys_platform != 'win32' -datadog-mongo==8.2.1 -datadog-mysql==14.2.0 +datadog-mongo==8.3.0 +datadog-mysql==14.3.0 datadog-nagios==3.0.0 datadog-network==5.1.0 datadog-nfsstat==3.0.0; sys_platform == 'linux2' datadog-nginx-ingress-controller==4.0.0 datadog-nginx==8.0.0 +datadog-nvidia-nim==1.0.0 datadog-nvidia-triton==2.1.0 datadog-openldap==3.0.0 datadog-openmetrics==6.0.0 @@ -141,7 +142,7 @@ datadog-pgbouncer==8.0.0; sys_platform != 'win32' datadog-php-fpm==5.0.0 datadog-ping-federate==2.0.0 datadog-postfix==3.0.0; sys_platform != 'win32' -datadog-postgres==22.2.0 +datadog-postgres==22.3.0 datadog-powerdns-recursor==4.0.0 datadog-presto==3.1.0 datadog-process==5.0.0 @@ -159,19 +160,21 @@ datadog-scylla==4.0.0 datadog-sidekiq==3.0.0 datadog-silk==4.0.0 datadog-singlestore==4.0.0 -datadog-slurm==1.0.1; sys_platform == 'linux2' -datadog-snmp==9.0.0 -datadog-snowflake==7.0.0 +datadog-slurm==1.0.3; sys_platform == 'linux2' +datadog-snmp==9.1.0 +datadog-snowflake==7.1.0 datadog-solr==2.1.0 -datadog-sonarqube==5.0.0 +datadog-sonarqube==5.1.0 +datadog-sonicwall-firewall==1.0.0 datadog-spark==6.1.0 -datadog-sqlserver==20.1.1 +datadog-sqlserver==20.2.0 datadog-squid==4.0.0 datadog-ssh-check==4.0.0 datadog-statsd==3.0.0 datadog-strimzi==3.1.0 datadog-supervisord==4.0.0 datadog-suricata==2.0.0 +datadog-symantec-endpoint-protection==1.0.0 datadog-system-core==4.0.0 datadog-system-swap==3.0.0 datadog-tcp-check==6.0.0 @@ -196,6 +199,7 @@ datadog-vertica==6.0.0 datadog-vllm==2.1.0 datadog-voltdb==5.0.0 datadog-vsphere==8.1.0 +datadog-wazuh==1.0.0 datadog-weaviate==3.1.0 datadog-weblogic==3.0.0 datadog-win32-event-log==5.0.0; sys_platform == 'win32' diff --git a/shopify/CHANGELOG.md b/shopify/CHANGELOG.md new file mode 100644 index 0000000000000..d8ad990765e59 --- /dev/null +++ b/shopify/CHANGELOG.md @@ -0,0 +1,7 @@ +# CHANGELOG - shopify + +## 1.0.0 / 2024-09-23 + +***Added***: + +* Initial Release diff --git a/shopify/README.md b/shopify/README.md new file mode 100644 index 0000000000000..dbebe4f49dc51 --- /dev/null +++ b/shopify/README.md @@ -0,0 +1,60 @@ +# Shopify + +## Overview + +[Shopify][1] is a comprehensive commerce platform designed to help individuals start, manage, and grow their businesses. It provides tools to build an online store, manage sales, market to customers, and accept payments in both digital and physical locations. + +The Shopify Integration collects Event, Product, Customer, and Order logs, sending them to Datadog for detailed analysis. + +It includes dashboards that show and analyze logs, making it easier to monitor and understand patterns. + +## Setup + +### Configuration + +#### Get Shopify credentials +1. Log in to [Shopify][2] admin account. +2. The Shopify Store name is the `xxxx` part of the Store URL (`https://admin.shopify.com/store/xxxx`). +3. Navigate to **Settings > Apps and sales channels**. +4. Select **Develop apps** and click **Allow custom app development**. +5. Click **Create a custom app**, provide the necessary details and click **Create app**. +6. Click **Configure Admin API Scopes** under the Overview tab. +7. In the **Admin API access scopes section**, select the following scopes: + - **read_orders** + - **read_products** + - **read_customers** + - **read_content** + - **read_price_rules** +8. Click **Save** to apply the changes. +9. Click **Install app**. +10. Under the **Admin API access token** section, click **Reveal token once**. + +#### Add Shopify credentials +- Shopify Store Name +- Shopify Access Token + +## Data Collected + +### Logs + +The Shopify integration collects and forwards Event, Product, Customer, and Order logs to Datadog. + +### Metrics + +The Shopify integration does not include any metrics. + +### Service Checks + +The Shopify integration does not include any service checks. + +### Events + +The Shopify integration does not include any events. + +## Troubleshooting + +Need help? Contact [Datadog support][3]. + +[1]: https://www.shopify.com/ +[2]: https://www.shopify.com/in/store-login +[3]: https://docs.datadoghq.com/help/ diff --git a/shopify/assets/dashboards/shopify_customer_overview.json b/shopify/assets/dashboards/shopify_customer_overview.json new file mode 100644 index 0000000000000..0ac696dbf349e --- /dev/null +++ b/shopify/assets/dashboards/shopify_customer_overview.json @@ -0,0 +1,1156 @@ +{ + "title": "Shopify - Customer Overview", + "description": "", + "widgets": [ + { + "id": 6457589008412640, + "definition": { + "type": "image", + "url": "https://cdn.shopify.com/shopifycloud/brochure/assets/brand-assets/shopify-logo-primary-logo-456baa801ee66a0a435671082365958316831c9960c480451dd0330bcdae304f.svg", + "url_dark_theme": "https://cdn.shopify.com/shopifycloud/brochure/assets/brand-assets/shopify-logo-inverted-primary-logo-bdc6ddd67862d9bb1f8c559e1bb50dd233112ac57b29cac2edcf17ed2e1fe6fa.svg", + "sizing": "contain", + "margin": "md", + "has_background": false, + "has_border": true, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 3 + } + }, + { + "id": 7447057642776924, + "definition": { + "type": "note", + "content": "\n**[Shopify](https://www.shopify.com/)** is a comprehensive commerce platform designed to help individuals start, manage, and grow their businesses. It provides tools to build an online store, manage sales, market to customers, and accept payments in both digital and physical locations.\n\nThe **Shopify Customer Overview** Dashboard provides an overview of customers within your store. It provides insights into customer's marketing and order details.\n\nFor more information, see the [Shopify Integration Documentation](https://docs.datadoghq.com/integrations/shopify/).\n\n### Tip:-\n- Clone this dashboard to rearrange, modify, and add widgets and visualizations.\n", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 3 + } + }, + { + "id": 8587137220738596, + "definition": { + "title": "Total Customers", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@usr.id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:customer $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "white_on_green" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 3, + "width": 4, + "height": 3 + } + }, + { + "id": 2522113371979004, + "definition": { + "title": "First Time Customers", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@usr.id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:customer @numberOfOrders:1 $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 4, + "y": 3, + "width": 4, + "height": 3 + } + }, + { + "id": 3033015122548506, + "definition": { + "title": "Returning Customers", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@usr.id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:customer @numberOfOrders:>1 $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 8, + "y": 3, + "width": 4, + "height": 3 + } + }, + { + "id": 4601366202227524, + "definition": { + "title": "Customer Returning Rate", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@usr.id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:customer @numberOfOrders:>1 $customer_name $customer_id $currency" + }, + "storage": "hot" + }, + { + "data_source": "logs", + "name": "query2", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@usr.id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:customer $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "(query1 / query2) * 100" + } + ], + "conditional_formats": [ + { + "comparator": ">=", + "value": 0, + "palette": "yellow_on_white" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 6, + "width": 4, + "height": 4 + } + }, + { + "id": 6324267437916298, + "definition": { + "title": "Sales by Customer", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@amountSpent.amount" + }, + "group_by": [ + { + "facet": "@usr.name", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@amountSpent.amount" + } + }, + { + "facet": "@amountSpent.currencyCode", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@amountSpent.amount" + } + } + ], + "search": { + "query": "source:shopify service:customer $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "text_formats": [], + "sort": { + "count": 10000, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "number", + "alias": "Order Value", + "conditional_formats": [ + { + "comparator": ">=", + "value": 0, + "palette": "black_on_light_green" + } + ], + "formula": "query1" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 4, + "y": 6, + "width": 8, + "height": 4 + } + }, + { + "id": 4312648521138476, + "definition": { + "title": "Amount Spent by First Time Customers (Avg)", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@amountSpent.amount" + }, + "group_by": [ + { + "facet": "@amountSpent.currencyCode", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@amountSpent.amount" + } + } + ], + "search": { + "query": "source:shopify service:customer @numberOfOrders:1 $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">=", + "value": 0, + "palette": "white_on_green" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 0, + "y": 10, + "width": 4, + "height": 4 + } + }, + { + "id": 8540082775305124, + "definition": { + "title": "Amount Spent by Returning Customers (Avg)", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@amountSpent.amount" + }, + "group_by": [ + { + "facet": "@amountSpent.currencyCode", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@amountSpent.amount" + } + } + ], + "search": { + "query": "source:shopify service:customer @numberOfOrders:>1 $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">=", + "value": 0, + "palette": "white_on_yellow" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 4, + "y": 10, + "width": 4, + "height": 4 + } + }, + { + "id": 6208927249201918, + "definition": { + "title": "Amount Spent by Currency (Avg)", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@amountSpent.amount" + }, + "group_by": [ + { + "facet": "@amountSpent.currencyCode", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@amountSpent.amount" + } + } + ], + "search": { + "query": "source:shopify service:customer $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 8, + "y": 10, + "width": 4, + "height": 4 + } + }, + { + "id": 2471531239284296, + "definition": { + "title": "Customers by City", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@usr.id" + }, + "group_by": [ + { + "facet": "@defaultAddress.city", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@usr.id" + } + } + ], + "search": { + "query": "source:shopify service:customer -@defaultAddress.city:\"\" $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 0, + "y": 14, + "width": 4, + "height": 4 + } + }, + { + "id": 3660022425769518, + "definition": { + "title": "Customers by Default Location", + "title_size": "16", + "title_align": "left", + "type": "geomap", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@usr.id" + }, + "group_by": [ + { + "facet": "@defaultAddress.countryCodeV2", + "limit": 250, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@usr.id" + } + } + ], + "search": { + "query": "source:shopify service:customer $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 250, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "palette": "Plasma", + "palette_flip": false + }, + "view": { + "focus": "WORLD" + } + }, + "layout": { + "x": 4, + "y": 14, + "width": 8, + "height": 4 + } + }, + { + "id": 8504166996883148, + "definition": { + "title": "Customers by Email Marketing State", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@usr.id" + }, + "group_by": [ + { + "facet": "@emailMarketingConsent.marketingState", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@usr.id" + } + } + ], + "search": { + "query": "source:shopify service:customer $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "style": { + "palette": "datadog16" + } + } + ], + "type": "sunburst", + "legend": { + "type": "table" + } + }, + "layout": { + "x": 0, + "y": 18, + "width": 6, + "height": 4 + } + }, + { + "id": 8504970604682638, + "definition": { + "title": "Customers by SMS Marketing State", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@usr.id" + }, + "group_by": [ + { + "facet": "@smsMarketingConsent.marketingState", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@usr.id" + } + } + ], + "search": { + "query": "source:shopify service:customer $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "style": { + "palette": "datadog16" + } + } + ], + "type": "sunburst", + "legend": { + "type": "table" + } + }, + "layout": { + "x": 6, + "y": 18, + "width": 6, + "height": 4 + } + }, + { + "id": 3112514483697428, + "definition": { + "title": "Customers by Last Order Value", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@lastOrder.totalPriceSet.shopMoney.amount" + }, + "group_by": [ + { + "facet": "@usr.name", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@lastOrder.totalPriceSet.shopMoney.amount" + } + }, + { + "facet": "@lastOrder.totalPriceSet.shopMoney.currencyCode", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@lastOrder.totalPriceSet.shopMoney.amount" + } + } + ], + "search": { + "query": "source:shopify service:customer $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 625, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 0, + "y": 22, + "width": 3, + "height": 4 + } + }, + { + "id": 4536649778544610, + "definition": { + "title": "Top Customers by Order Count", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "max", + "metric": "@numberOfOrders" + }, + "group_by": [ + { + "facet": "@usr.name", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "max", + "metric": "@numberOfOrders" + } + }, + { + "facet": "@usr.id", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "max", + "metric": "@numberOfOrders" + } + } + ], + "search": { + "query": "source:shopify service:customer $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 625, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "none" + }, + "palette": "datadog16" + } + }, + "layout": { + "x": 3, + "y": 22, + "width": 3, + "height": 4 + } + }, + { + "id": 2373335191656186, + "definition": { + "title": "Customers by Subscription Status", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@usr.id" + }, + "group_by": [ + { + "facet": "@productSubscriberStatus", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@usr.id" + } + } + ], + "search": { + "query": "source:shopify service:customer $customer_name $customer_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "style": { + "palette": "datadog16" + } + } + ], + "type": "sunburst", + "legend": { + "type": "table" + } + }, + "layout": { + "x": 6, + "y": 22, + "width": 6, + "height": 4 + } + }, + { + "id": 1987008830400332, + "definition": { + "title": "Customers Summary", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "event_list", + "query": { + "data_source": "logs_stream", + "query_string": "source:shopify service:customer $customer_name $customer_id $currency", + "indexes": [], + "storage": "hot" + }, + "columns": [ + { + "field": "usr.id", + "width": "auto" + }, + { + "field": "usr.name", + "width": "auto" + }, + { + "field": "numberOfOrders", + "width": "auto" + }, + { + "field": "lifetimeDuration", + "width": "auto" + }, + { + "field": "taxExempt", + "width": "auto" + }, + { + "field": "emailMarketingConsent.marketingState", + "width": "auto" + }, + { + "field": "smsMarketingConsent.marketingState", + "width": "auto" + }, + { + "field": "content", + "width": "auto" + } + ] + } + ], + "type": "list_stream" + }, + "layout": { + "x": 0, + "y": 26, + "width": 12, + "height": 4 + } + } + ], + "template_variables": [ + { + "name": "customer_id", + "prefix": "@usr.id", + "available_values": [], + "default": "*" + }, + { + "name": "customer_name", + "prefix": "@usr.name", + "available_values": [], + "default": "*" + }, + { + "name": "currency", + "prefix": "@amountSpent.currencyCode", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} \ No newline at end of file diff --git a/shopify/assets/dashboards/shopify_event_overview.json b/shopify/assets/dashboards/shopify_event_overview.json new file mode 100644 index 0000000000000..30b17a2994e9f --- /dev/null +++ b/shopify/assets/dashboards/shopify_event_overview.json @@ -0,0 +1,716 @@ +{ + "title": "Shopify - Event Overview", + "description": "", + "widgets": [ + { + "id": 2362748552655712, + "definition": { + "type": "image", + "url": "https://cdn.shopify.com/shopifycloud/brochure/assets/brand-assets/shopify-logo-primary-logo-456baa801ee66a0a435671082365958316831c9960c480451dd0330bcdae304f.svg", + "url_dark_theme": "https://cdn.shopify.com/shopifycloud/brochure/assets/brand-assets/shopify-logo-inverted-primary-logo-bdc6ddd67862d9bb1f8c559e1bb50dd233112ac57b29cac2edcf17ed2e1fe6fa.svg", + "sizing": "contain", + "margin": "md", + "has_background": false, + "has_border": true, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 3 + } + }, + { + "id": 6595375037827088, + "definition": { + "type": "note", + "content": "**[Shopify](https://www.shopify.com/)** is a comprehensive commerce platform that helps individuals to start, manage, and grow a business. It offers a set of tools to build an online store, manage sales, market to customers, and accept payments in digital and physical locations.\n\nThe **Shopify Event Overview** Dashboard provides an overview of event logs, offering insights into events, type, and action distribution.\n\nFor more information, see the [Shopify Integration Documentation](https://docs.datadoghq.com/integrations/shopify/).\n\n### Tip:-\n- Clone this dashboard to rearrange, modify, and add widgets and visualizations.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 3 + } + }, + { + "id": 4156249446142690, + "definition": { + "title": "Total Events", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:event $event_type $event_action" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "white_on_green" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 3, + "width": 3, + "height": 4 + } + }, + { + "id": 6434718208472006, + "definition": { + "title": "Events Overview", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Events", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:event $event_type $event_action" + }, + "storage": "hot" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 3, + "width": 9, + "height": 4 + } + }, + { + "id": 7313898452003926, + "definition": { + "title": "Events by Type", + "type": "treemap", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@subject_type", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:event $event_type $event_action" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "style": { + "palette": "datadog16" + } + } + ] + }, + "layout": { + "x": 0, + "y": 7, + "width": 5, + "height": 4 + } + }, + { + "id": 2516121826658084, + "definition": { + "title": "Events by Type Overview", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@subject_type", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "count", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:event $event_type $event_action" + }, + "storage": "hot" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 5, + "y": 7, + "width": 7, + "height": 4 + } + }, + { + "id": 2101689610312290, + "definition": { + "title": "Events by Action", + "requests": [ + { + "formulas": [ + { + "formula": "query1", + "limit": { + "order": "desc" + } + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@verb", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:event $event_type $event_action" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "style": { + "palette": "classic" + } + } + ], + "type": "sunburst", + "legend": { + "type": "automatic" + } + }, + "layout": { + "x": 0, + "y": 11, + "width": 5, + "height": 4 + } + }, + { + "id": 5713539676901512, + "definition": { + "title": "Events by Action Overview", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@verb", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "count", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:event $event_type $event_action" + }, + "storage": "hot" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 5, + "y": 11, + "width": 7, + "height": 4 + } + }, + { + "id": 6973490408831438, + "definition": { + "title": "Top Events", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@subject_type", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + }, + { + "facet": "@verb", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:event $event_type $event_action" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 625, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "datadog16", + "scaling": "relative" + } + }, + "layout": { + "x": 0, + "y": 15, + "width": 5, + "height": 4 + } + }, + { + "id": 5837771255186112, + "definition": { + "title": "Top Users", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@usr.name", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:event $event_type $event_action" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 5, + "y": 15, + "width": 3, + "height": 4 + } + }, + { + "id": 2751619566114168, + "definition": { + "title": "Top User Actions", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@usr.name", + "limit": 15, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + }, + { + "facet": "@subject_type", + "limit": 15, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + }, + { + "facet": "@verb", + "limit": 15, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:event $event_type $event_action" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 3375, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "number", + "alias": "Events", + "formula": "query1" + } + ] + } + ], + "has_search_bar": "never" + }, + "layout": { + "x": 8, + "y": 15, + "width": 4, + "height": 4 + } + }, + { + "id": 4587011956440926, + "definition": { + "title": "Event Summary", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "event_list", + "query": { + "data_source": "logs_stream", + "query_string": "source:shopify service:event $event_type $event_action", + "indexes": [], + "storage": "hot" + }, + "columns": [ + { + "field": "status_line", + "width": "auto" + }, + { + "field": "timestamp", + "width": "auto" + }, + { + "field": "subject_type", + "width": "auto" + }, + { + "field": "verb", + "width": "auto" + }, + { + "field": "usr.name", + "width": "auto" + }, + { + "field": "content", + "width": "auto" + } + ] + } + ], + "type": "list_stream" + }, + "layout": { + "x": 0, + "y": 19, + "width": 12, + "height": 4 + } + } + ], + "template_variables": [ + { + "name": "event_type", + "prefix": "@subject_type", + "available_values": [], + "default": "*" + }, + { + "name": "event_action", + "prefix": "@verb", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} \ No newline at end of file diff --git a/shopify/assets/dashboards/shopify_order_overview.json b/shopify/assets/dashboards/shopify_order_overview.json new file mode 100644 index 0000000000000..7173613cf3067 --- /dev/null +++ b/shopify/assets/dashboards/shopify_order_overview.json @@ -0,0 +1,1497 @@ +{ + "title": "Shopify - Order Overview", + "description": "", + "widgets": [ + { + "id": 3630919149544800, + "definition": { + "type": "image", + "url": "https://cdn.shopify.com/shopifycloud/brochure/assets/brand-assets/shopify-logo-primary-logo-456baa801ee66a0a435671082365958316831c9960c480451dd0330bcdae304f.svg", + "url_dark_theme": "https://cdn.shopify.com/shopifycloud/brochure/assets/brand-assets/shopify-logo-inverted-primary-logo-bdc6ddd67862d9bb1f8c559e1bb50dd233112ac57b29cac2edcf17ed2e1fe6fa.svg", + "sizing": "contain", + "margin": "md", + "has_background": false, + "has_border": true, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 7, + "height": 2 + } + }, + { + "id": 2768065097473154, + "definition": { + "title": "Shopify Monitor Summary", + "type": "manage_status", + "display_format": "countsAndList", + "color_preference": "background", + "hide_zero_counts": true, + "show_status": true, + "last_triggered_format": "relative", + "query": "tag:shopify", + "sort": "status,asc", + "count": 50, + "start": 0, + "summary_type": "monitors", + "show_priority": false, + "show_last_triggered": false + }, + "layout": { + "x": 7, + "y": 0, + "width": 5, + "height": 4 + } + }, + { + "id": 8602455036825926, + "definition": { + "type": "note", + "content": "\n**[Shopify](https://www.shopify.com/)** is a comprehensive commerce platform that helps individuals to start, manage, and grow a business. It offers a set of tools to build an online store, manage sales, market to customers, and accept payments in digital and physical locations.\n\nThe **Shopify Orders Overview** Dashboard provides an overview of orders within your store. It provides insights into order tracking, fulfillments, customers, and product details.\n\nFor more information, see the [Shopify Integration Documentation](https://docs.datadoghq.com/integrations/shopify/).\n\n### Tip:-\n- Clone this dashboard to rearrange, modify, and add widgets and visualizations.\n", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 2, + "width": 7, + "height": 2 + } + }, + { + "id": 4916600894722990, + "definition": { + "title": "Total Orders", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "default_zero(query1)" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "white_on_green" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 4, + "width": 3, + "height": 2 + } + }, + { + "id": 7701449148798980, + "definition": { + "title": "Confirmed Orders", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:order @confirmed:true $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "default_zero(query1)" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 3, + "y": 4, + "width": 3, + "height": 2 + } + }, + { + "id": 1812191786714824, + "definition": { + "title": "Refunded Orders", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:order @refunds.transactions.status:success $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "default_zero(query1)" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 6, + "y": 4, + "width": 3, + "height": 2 + } + }, + { + "id": 2581889973783936, + "definition": { + "title": "Marketing Accepted Orders", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:order @buyer_accepts_marketing:true $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "default_zero(query1)" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 9, + "y": 4, + "width": 3, + "height": 2 + } + }, + { + "id": 7618442638763158, + "definition": { + "title": "Completed Orders", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:order @closed_at:* $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "default_zero(query1)" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 6, + "width": 3, + "height": 2 + } + }, + { + "id": 230307842331210, + "definition": { + "title": "Cancelled Orders", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:order @cancelled_at:* $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "default_zero(query1)" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 3, + "y": 6, + "width": 3, + "height": 2 + } + }, + { + "id": 3796810043555928, + "definition": { + "title": "Tax Exempted Orders", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:order (@tax_exempt:true OR @tax_included:false) $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "default_zero(query1)" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 6, + "y": 6, + "width": 3, + "height": 2 + } + }, + { + "id": 5694819066898408, + "definition": { + "title": "Gift Card Orders", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "default_zero(query1)" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:order @line_items.gift_card:true $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 9, + "y": 6, + "width": 3, + "height": 2 + } + }, + { + "id": 4902698799419696, + "definition": { + "title": "Average Order Price by Customer", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@total_price" + }, + "group_by": [ + { + "facet": "@usr.name", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + }, + { + "facet": "@currency", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + } + ], + "search": { + "query": "source:shopify service:order -@usr.name:\" \" $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 625, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 0, + "y": 8, + "width": 3, + "height": 4 + } + }, + { + "id": 3843387744924516, + "definition": { + "title": "Order Summary (rounded price)", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@total_price" + }, + "group_by": [ + { + "facet": "@id", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + }, + { + "facet": "@currency", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + }, + { + "data_source": "logs", + "name": "query2", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@subtotal_price" + }, + "group_by": [ + { + "facet": "@id", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + }, + { + "facet": "@currency", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + }, + { + "data_source": "logs", + "name": "query3", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@total_discounts" + }, + "group_by": [ + { + "facet": "@id", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + }, + { + "facet": "@currency", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + }, + { + "data_source": "logs", + "name": "query4", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@total_line_items_price" + }, + "group_by": [ + { + "facet": "@id", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + }, + { + "facet": "@currency", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + }, + { + "data_source": "logs", + "name": "query5", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@total_tax" + }, + "group_by": [ + { + "facet": "@id", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + }, + { + "facet": "@currency", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + }, + { + "data_source": "logs", + "name": "query6", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@total_tip_received" + }, + "group_by": [ + { + "facet": "@id", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + }, + { + "facet": "@currency", + "limit": 100, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@total_price" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "text_formats": [], + "sort": { + "count": 60000, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "bar", + "alias": "Total Price", + "formula": "query1" + }, + { + "cell_display_mode": "bar", + "alias": "Sub Total Price", + "formula": "query2" + }, + { + "cell_display_mode": "bar", + "alias": "Total Discount", + "formula": "query3" + }, + { + "cell_display_mode": "bar", + "alias": "Total Line Item Price", + "formula": "query4" + }, + { + "cell_display_mode": "bar", + "alias": "Total Tax", + "formula": "query5" + }, + { + "cell_display_mode": "bar", + "alias": "Total Tip Received", + "formula": "query6" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 3, + "y": 8, + "width": 9, + "height": 4 + } + }, + { + "id": 7339194611439248, + "definition": { + "title": "Top Ordered Products", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@line_items.title", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 0, + "y": 12, + "width": 4, + "height": 4 + } + }, + { + "id": 445727493366610, + "definition": { + "title": "Top Customers by Order", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@usr.name", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:order -@usr.name:\" \" $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 4, + "y": 12, + "width": 4, + "height": 4 + } + }, + { + "id": 623764250336340, + "definition": { + "title": "Orders by Fulfillment Tracking Company", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@fulfillments.tracking_company", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 8, + "y": 12, + "width": 4, + "height": 4 + } + }, + { + "id": 4715342708650584, + "definition": { + "title": "Orders by Finance Status", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@financial_status", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "style": { + "palette": "datadog16" + } + } + ], + "type": "sunburst", + "legend": { + "type": "table" + } + }, + "layout": { + "x": 0, + "y": 16, + "width": 6, + "height": 4 + } + }, + { + "id": 1687344262907466, + "definition": { + "title": "Orders by Fulfillment Status", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@fulfillments.status", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "style": { + "palette": "datadog16" + } + } + ], + "type": "sunburst", + "legend": { + "type": "table" + } + }, + "layout": { + "x": 6, + "y": 16, + "width": 6, + "height": 4 + } + }, + { + "id": 7729756975880686, + "definition": { + "title": "Top Cities by Customer Order", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@network.client.geoip.city.name", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 0, + "y": 20, + "width": 6, + "height": 4 + } + }, + { + "id": 6552248278099246, + "definition": { + "title": "Orders by Location", + "title_size": "16", + "title_align": "left", + "type": "geomap", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@network.client.geoip.country.iso_code", + "limit": 250, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "palette": "hostmap_blues", + "palette_flip": false + }, + "view": { + "focus": "WORLD" + } + }, + "layout": { + "x": 6, + "y": 20, + "width": 6, + "height": 4 + } + }, + { + "id": 7263169748961898, + "definition": { + "title": "Device Distribution by Order Placed", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@http.useragent_details.device.category", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "style": { + "palette": "datadog16" + }, + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "type": "sunburst", + "legend": { + "type": "table" + } + }, + "layout": { + "x": 0, + "y": 24, + "width": 6, + "height": 4 + } + }, + { + "id": 2120592637920540, + "definition": { + "title": "Orders By Shipping Location", + "title_size": "16", + "title_align": "left", + "type": "geomap", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@shipping_address.country_code", + "limit": 250, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:order $order_id $customer_name $customer_city $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 250, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "palette": "hostmap_blues", + "palette_flip": false + }, + "view": { + "focus": "WORLD" + } + }, + "layout": { + "x": 6, + "y": 24, + "width": 6, + "height": 4 + } + }, + { + "id": 7126489403928608, + "definition": { + "title": "Order Details", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "event_list", + "query": { + "data_source": "logs_stream", + "query_string": "source:shopify service:order $order_id $customer_name $customer_city $currency", + "indexes": [], + "storage": "hot" + }, + "columns": [ + { + "field": "@id", + "width": "auto" + }, + { + "field": "usr.name", + "width": "auto" + }, + { + "field": "total_price", + "width": "auto" + }, + { + "field": "financial_status", + "width": "auto" + }, + { + "field": "currency", + "width": "auto" + }, + { + "field": "content", + "width": "auto" + } + ] + } + ], + "type": "list_stream" + }, + "layout": { + "x": 0, + "y": 28, + "width": 12, + "height": 4 + } + } + ], + "template_variables": [ + { + "name": "order_id", + "prefix": "@id", + "available_values": [], + "default": "*" + }, + { + "name": "customer_name", + "prefix": "@usr.name", + "available_values": [], + "default": "*" + }, + { + "name": "customer_city", + "prefix": "@network.client.geoip.city.name", + "available_values": [], + "default": "*" + }, + { + "name": "currency", + "prefix": "@currency", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} \ No newline at end of file diff --git a/shopify/assets/dashboards/shopify_product_overview.json b/shopify/assets/dashboards/shopify_product_overview.json new file mode 100644 index 0000000000000..6d10b3c2a8954 --- /dev/null +++ b/shopify/assets/dashboards/shopify_product_overview.json @@ -0,0 +1,621 @@ +{ + "title": "Shopify - Product Overview", + "description": "", + "widgets": [ + { + "id": 8506652285430826, + "definition": { + "type": "image", + "url": "https://cdn.shopify.com/shopifycloud/brochure/assets/brand-assets/shopify-logo-primary-logo-456baa801ee66a0a435671082365958316831c9960c480451dd0330bcdae304f.svg", + "url_dark_theme": "https://cdn.shopify.com/shopifycloud/brochure/assets/brand-assets/shopify-logo-inverted-primary-logo-bdc6ddd67862d9bb1f8c559e1bb50dd233112ac57b29cac2edcf17ed2e1fe6fa.svg", + "sizing": "contain", + "margin": "md", + "has_background": false, + "has_border": true, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 7, + "height": 2 + } + }, + { + "id": 8770406996449935, + "definition": { + "title": "Shopify Monitor Summary", + "type": "manage_status", + "display_format": "countsAndList", + "color_preference": "background", + "hide_zero_counts": true, + "show_status": true, + "last_triggered_format": "relative", + "query": "tag:shopify", + "sort": "status,asc", + "count": 50, + "start": 0, + "summary_type": "monitors", + "show_priority": false, + "show_last_triggered": false + }, + "layout": { + "x": 7, + "y": 0, + "width": 5, + "height": 4 + } + }, + { + "id": 1767411351951850, + "definition": { + "type": "note", + "content": "**[Shopify](https://www.shopify.com/)** is a comprehensive commerce platform that helps individuals to start, manage, and grow a business. It offers a set of tools to build an online store, manage sales, market to customers, and accept payments in digital and physical locations.\n\nThe **Shopify Product Overview** Dashboard provides an overview of products within your store. It provides insights into product types, status distribution, inventory, and variant details.\n\nFor more information, see the [Shopify Integration Documentation](https://docs.datadoghq.com/integrations/shopify/).\n\n### Tip:-\n- Clone this dashboard to rearrange, modify, and add widgets and visualizations.\n", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 2, + "width": 7, + "height": 2 + } + }, + { + "id": 2446181804882916, + "definition": { + "title": "Total Products", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:product $product_name $product_type $product_id $currency" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "white_on_green" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 4, + "width": 3, + "height": 3 + } + }, + { + "id": 6613518820826564, + "definition": { + "title": "Top Product Count by Type", + "type": "treemap", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@productType", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:product -@productType:\"\" $product_name $product_type $product_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "style": { + "palette": "datadog16" + } + } + ] + }, + "layout": { + "x": 3, + "y": 4, + "width": 9, + "height": 3 + } + }, + { + "id": 4043889896241554, + "definition": { + "title": "Products with Out of Stock Variants", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:product @hasOutOfStockVariants:true $product_name $product_type $product_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 7, + "width": 4, + "height": 2 + } + }, + { + "id": 1141858048342154, + "definition": { + "title": "Products with Out of Stock Inventory", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:product @tracksInventory:true @totalInventory:<=0 $product_type $product_name $product_id $currency" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 4, + "y": 7, + "width": 4, + "height": 2 + } + }, + { + "id": 5213489288870882, + "definition": { + "title": "Products with Inventory Tracking", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:product @tracksInventory:true $product_type $product_name $product_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 8, + "y": 7, + "width": 4, + "height": 2 + } + }, + { + "id": 8353371234267758, + "definition": { + "title": "Products by Status", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [ + { + "facet": "@product_status", + "limit": 25, + "sort": { + "order": "desc", + "aggregation": "cardinality", + "metric": "@id" + } + } + ], + "search": { + "query": "source:shopify service:product $product_type $product_name $product_id $currency" + }, + "storage": "hot" + } + ], + "style": { + "palette": "datadog16" + }, + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "type": "sunburst", + "legend": { + "type": "inline" + } + }, + "layout": { + "x": 0, + "y": 9, + "width": 4, + "height": 4 + } + }, + { + "id": 3849686844214096, + "definition": { + "title": "Product Variant Price (Avg)", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@priceRangeV2.maxVariantPrice.amount" + }, + "group_by": [ + { + "facet": "@legacyResourceId", + "limit": 15, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@priceRangeV2.maxVariantPrice.amount" + } + }, + { + "facet": "@title", + "limit": 15, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@priceRangeV2.maxVariantPrice.amount" + } + }, + { + "facet": "@priceRangeV2.maxVariantPrice.currencyCode", + "limit": 15, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@priceRangeV2.maxVariantPrice.amount" + } + } + ], + "search": { + "query": "source:shopify service:product $product_type $product_name $product_id $currency" + }, + "storage": "hot" + }, + { + "data_source": "logs", + "name": "query2", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "avg", + "metric": "@priceRangeV2.minVariantPrice.amount" + }, + "group_by": [ + { + "facet": "@legacyResourceId", + "limit": 15, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@priceRangeV2.maxVariantPrice.amount" + } + }, + { + "facet": "@title", + "limit": 15, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@priceRangeV2.maxVariantPrice.amount" + } + }, + { + "facet": "@priceRangeV2.maxVariantPrice.currencyCode", + "limit": 15, + "sort": { + "order": "desc", + "aggregation": "avg", + "metric": "@priceRangeV2.maxVariantPrice.amount" + } + } + ], + "search": { + "query": "source:shopify service:product $product_type $product_name $product_id $currency" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 6750, + "order_by": [ + { + "type": "formula", + "index": 2, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "number", + "alias": "Max Variant Price", + "formula": "query1" + }, + { + "cell_display_mode": "number", + "alias": "Min Variant Price", + "formula": "query2" + }, + { + "alias": "Average Variant Price", + "cell_display_mode": "bar", + "formula": "(query1 + query2) / 2" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 4, + "y": 9, + "width": 8, + "height": 4 + } + }, + { + "id": 6140658451404220, + "definition": { + "title": "Product Summary", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "event_list", + "query": { + "data_source": "logs_stream", + "query_string": "source:shopify service:product $product_name $product_type $product_id $currency", + "indexes": [], + "storage": "hot" + }, + "columns": [ + { + "field": "legacyResourceId", + "width": "auto" + }, + { + "field": "title", + "width": "auto" + }, + { + "field": "productType", + "width": "auto" + }, + { + "field": "product_status", + "width": "auto" + }, + { + "field": "tracksInventory", + "width": "auto" + }, + { + "field": "totalInventory", + "width": "auto" + }, + { + "field": "priceRangeV2.maxVariantPrice.amount", + "width": "auto" + }, + { + "field": "priceRangeV2.minVariantPrice.amount", + "width": "auto" + }, + { + "field": "priceRangeV2.maxVariantPrice.currencyCode", + "width": "auto" + }, + { + "field": "vendor", + "width": "auto" + }, + { + "field": "isGiftCard", + "width": "auto" + } + ] + } + ], + "type": "list_stream" + }, + "layout": { + "x": 0, + "y": 13, + "width": 12, + "height": 5 + } + } + ], + "template_variables": [ + { + "name": "product_id", + "prefix": "@id", + "available_values": [], + "default": "*" + }, + { + "name": "product_name", + "prefix": "@title", + "available_values": [], + "default": "*" + }, + { + "name": "product_type", + "prefix": "@productType", + "available_values": [], + "default": "*" + }, + { + "name": "currency", + "prefix": "@priceRangeV2.maxVariantPrice.currencyCode", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} \ No newline at end of file diff --git a/shopify/assets/logs/shopify.yaml b/shopify/assets/logs/shopify.yaml new file mode 100644 index 0000000000000..c099b9da32853 --- /dev/null +++ b/shopify/assets/logs/shopify.yaml @@ -0,0 +1,246 @@ +id: shopify +metric_id: shopify +backend_only: false +facets: + - groups: + - Web Access + name: User-Agent + path: http.useragent + source: log + - groups: + - Web Access + name: Browser + path: http.useragent_details.browser.family + source: log + - groups: + - Web Access + name: Device + path: http.useragent_details.device.family + source: log + - groups: + - Web Access + name: OS + path: http.useragent_details.os.family + source: log + - groups: + - Geoip + name: City Name + path: network.client.geoip.city.name + source: log + - groups: + - Geoip + name: Continent Code + path: network.client.geoip.continent.code + source: log + - groups: + - Geoip + name: Continent Name + path: network.client.geoip.continent.name + source: log + - groups: + - Geoip + name: Country ISO Code + path: network.client.geoip.country.iso_code + source: log + - groups: + - Geoip + name: Country Name + path: network.client.geoip.country.name + source: log + - groups: + - Geoip + name: Subdivision ISO Code + path: network.client.geoip.subdivision.iso_code + source: log + - groups: + - Geoip + name: Subdivision Name + path: network.client.geoip.subdivision.name + source: log + - groups: + - Web Access + name: Client IP + path: network.client.ip + source: log + - groups: + - User + name: User Email + path: usr.email + source: log + - groups: + - User + name: User ID + path: usr.id + source: log + - groups: + - User + name: User Name + path: usr.name + source: log +pipeline: + type: pipeline + name: Shopify + enabled: true + filter: + query: "source:shopify" + processors: + - type: pipeline + name: Event + enabled: true + filter: + query: "service:event" + processors: + - type: date-remapper + name: Define `created_at` as the official date of the log + enabled: true + sources: + - created_at + - type: attribute-remapper + name: Map `author` to `usr.name` + enabled: true + sources: + - author + sourceType: attribute + target: usr.name + targetType: attribute + preserveSource: false + overrideOnConflict: false + - type: pipeline + name: Order + enabled: true + filter: + query: "service:order" + processors: + - type: date-remapper + name: Define `updated_at` as the official date of the log + enabled: true + sources: + - updated_at + - type: attribute-remapper + name: Map `client_details.browser_ip` to `network.client.ip` + enabled: true + sources: + - client_details.browser_ip + sourceType: attribute + target: network.client.ip + targetType: attribute + preserveSource: false + overrideOnConflict: false + - type: attribute-remapper + name: Map `client_details.user_agent` to `http.useragent` + enabled: true + sources: + - client_details.user_agent + sourceType: attribute + target: http.useragent + targetType: attribute + preserveSource: false + overrideOnConflict: false + - type: attribute-remapper + name: Map `customer.id` to `usr.id` + enabled: true + sources: + - customer.id + sourceType: attribute + target: usr.id + targetType: attribute + preserveSource: false + overrideOnConflict: false + - type: attribute-remapper + name: Map `customer.email` to `usr.email` + enabled: true + sources: + - customer.email + sourceType: attribute + target: usr.email + targetType: attribute + preserveSource: false + overrideOnConflict: false + - type: string-builder-processor + name: Define `usr.name` equal to %{customer.first_name} %{customer.last_name} + enabled: true + template: "%{customer.first_name} %{customer.last_name}" + target: usr.name + replaceMissing: false + - type: geo-ip-parser + name: GeoIP Parser for `network.client.ip` + enabled: true + sources: + - network.client.ip + target: network.client.geoip + ip_processing_behavior: do-nothing + - type: user-agent-parser + name: Extract details from `http.useragent` + enabled: true + sources: + - http.useragent + target: http.useragent_details + encoded: false + combineVersionDetails: false + - type: pipeline + name: Customer + enabled: true + filter: + query: "service:customer" + processors: + - type: date-remapper + name: Define `updatedAt` as the official date of the log + enabled: true + sources: + - updatedAt + - type: attribute-remapper + name: Map `legacyResourceId` to `usr.id` + enabled: true + sources: + - legacyResourceId + sourceType: attribute + target: usr.id + targetType: attribute + preserveSource: false + overrideOnConflict: false + - type: attribute-remapper + name: Map `displayName` to `usr.name` + enabled: true + sources: + - displayName + sourceType: attribute + target: usr.name + targetType: attribute + preserveSource: false + overrideOnConflict: false + - type: pipeline + name: Product + enabled: true + filter: + query: "service:product" + processors: + - type: date-remapper + name: Define `updatedAt` as the official date of the log + enabled: true + sources: + - updatedAt + - type: attribute-remapper + name: Map `status` to `product_status` + enabled: true + sources: + - status + sourceType: attribute + target: product_status + targetType: attribute + preserveSource: false + overrideOnConflict: false + - name: Lookup for `product_status` to `status` + enabled: true + source: product_status + target: status + lookupTable: |- + ACTIVE,info + ARCHIVED,info + DRAFT,info + defaultLookup: info + type: lookup-processor + - type: status-remapper + name: Define `status` as the official status of the log + enabled: true + sources: + - status diff --git a/shopify/assets/logs/shopify_tests.yaml b/shopify/assets/logs/shopify_tests.yaml new file mode 100644 index 0000000000000..3245c3565d2b7 --- /dev/null +++ b/shopify/assets/logs/shopify_tests.yaml @@ -0,0 +1,1565 @@ +id: shopify +tests: + - + sample: |- + { + "subject_id" : 1234, + "path" : "/admin/pages", + "subject_type" : "Blog", + "author" : "Shopify", + "verb" : "create", + "created_at" : "2024-08-27T15:36:38+05:30", + "description" : "Blog was created: News.", + "arguments" : [ "News" ], + "id" : 222397999050809, + "message" : "Blog was created: News." + } + service: "event" + result: + custom: + arguments: + - "News" + created_at: "2024-08-27T15:36:38+05:30" + description: "Blog was created: News." + id: 222397999050809 + path: "/admin/pages" + subject_id: 1234 + subject_type: "Blog" + usr: + name: "Shopify" + verb: "create" + message: "Blog was created: News." + service: "event" + tags: + - "source:LOGS_SOURCE" + timestamp: 1724753198000 + - + sample: |- + { + "confirmation_number" : "L5MIX0PUW", + "fulfillment_status" : "fulfilled", + "total_outstanding" : "1296.82", + "order_number" : 1002, + "created_at" : "2024-08-28T14:48:43+05:30", + "taxes_included" : false, + "line_items" : [ { + "variant_title" : "Black", + "fulfillment_status" : "fulfilled", + "total_discount" : "0.00", + "gift_card" : false, + "requires_shipping" : true, + "total_discount_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "INR" + } + }, + "title" : "Shoes", + "product_exists" : true, + "variant_id" : 43744593936441, + "tax_lines" : [ { + "channel_liable" : false, + "rate" : 0.18, + "price" : "233.82", + "price_set" : { + "shop_money" : { + "amount" : "233.82", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "233.82", + "currency_code" : "INR" + } + }, + "title" : "IGST" + } ], + "price" : "1299.00", + "vendor" : "Dummy Vendor", + "product_id" : 7688312913977, + "id" : 12778730618937, + "grams" : 500, + "sku" : "SHOES-1", + "fulfillable_quantity" : 0, + "quantity" : 1, + "fulfillment_service" : "manual", + "taxable" : true, + "variant_inventory_management" : "shopify", + "current_quantity" : 1, + "admin_graphql_api_id" : "gid://shopify/LineItem/12778730618937", + "name" : "Shoes - Black", + "price_set" : { + "shop_money" : { + "amount" : "1299.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "1299.00", + "currency_code" : "INR" + } + } + }, { + "variant_title" : "L", + "fulfillment_status" : "fulfilled", + "total_discount" : "0.00", + "gift_card" : false, + "requires_shipping" : true, + "total_discount_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "INR" + } + }, + "title" : "Cotton tshirt", + "product_exists" : true, + "variant_id" : 43741030744121, + "tax_lines" : [ { + "channel_liable" : false, + "rate" : 0.18, + "price" : "53.82", + "price_set" : { + "shop_money" : { + "amount" : "53.82", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "53.82", + "currency_code" : "INR" + } + }, + "title" : "IGST" + } ], + "price" : "299.00", + "vendor" : "My Store", + "product_id" : 7687579762745, + "id" : 12801614413881, + "grams" : 100, + "sku" : "", + "fulfillable_quantity" : 0, + "quantity" : 1, + "fulfillment_service" : "manual", + "taxable" : true, + "variant_inventory_management" : "shopify", + "current_quantity" : 1, + "admin_graphql_api_id" : "gid://shopify/LineItem/12801614413881", + "name" : "Cotton tshirt - L", + "price_set" : { + "shop_money" : { + "amount" : "299.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "299.00", + "currency_code" : "INR" + } + } + }, { + "fulfillable_quantity" : 0, + "fulfillment_status" : "fulfilled", + "quantity" : 1, + "total_discount" : "0.00", + "fulfillment_service" : "manual", + "gift_card" : false, + "taxable" : true, + "requires_shipping" : true, + "total_discount_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "INR" + } + }, + "title" : "Desk Lamp", + "current_quantity" : 1, + "product_exists" : true, + "variant_id" : 43744917815353, + "tax_lines" : [ { + "channel_liable" : false, + "rate" : 0.18, + "price" : "144.00", + "price_set" : { + "shop_money" : { + "amount" : "144.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "144.00", + "currency_code" : "INR" + } + }, + "title" : "IGST" + } ], + "price" : "800.00", + "vendor" : "Dummy Vendor", + "admin_graphql_api_id" : "gid://shopify/LineItem/12801628569657", + "product_id" : 7688576860217, + "name" : "Desk Lamp", + "id" : 12801628569657, + "grams" : 0, + "price_set" : { + "shop_money" : { + "amount" : "800.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "800.00", + "currency_code" : "INR" + } + }, + "sku" : "DL789" + } ], + "buyer_accepts_marketing" : false, + "presentment_currency" : "INR", + "confirmed" : true, + "total_weight" : 600, + "total_discounts" : "0.00", + "fulfillments" : [ { + "updated_at" : "2024-09-20T10:13:23+05:30", + "service" : "manual", + "admin_graphql_api_id" : "gid://shopify/Fulfillment/4542375952441", + "name" : "#1002.1", + "created_at" : "2024-09-20T10:13:23+05:30", + "id" : 4542375952441, + "line_items" : [ { + "variant_title" : "Black", + "fulfillment_status" : "fulfilled", + "total_discount" : "0.00", + "gift_card" : false, + "requires_shipping" : true, + "total_discount_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "INR" + } + }, + "title" : "Shoes", + "product_exists" : true, + "variant_id" : 43744593936441, + "tax_lines" : [ { + "channel_liable" : false, + "rate" : 0.18, + "price" : "233.82", + "price_set" : { + "shop_money" : { + "amount" : "233.82", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "233.82", + "currency_code" : "INR" + } + }, + "title" : "IGST" + } ], + "price" : "1299.00", + "vendor" : "Dummy Vendor", + "product_id" : 7688312913977, + "id" : 12778730618937, + "grams" : 500, + "sku" : "SHOES-1", + "fulfillable_quantity" : 0, + "quantity" : 1, + "fulfillment_service" : "manual", + "taxable" : true, + "variant_inventory_management" : "shopify", + "current_quantity" : 1, + "admin_graphql_api_id" : "gid://shopify/LineItem/12778730618937", + "name" : "Shoes - Black", + "price_set" : { + "shop_money" : { + "amount" : "1299.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "1299.00", + "currency_code" : "INR" + } + } + }, { + "variant_title" : "L", + "fulfillment_status" : "fulfilled", + "total_discount" : "0.00", + "gift_card" : false, + "requires_shipping" : true, + "total_discount_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "INR" + } + }, + "title" : "Cotton tshirt", + "product_exists" : true, + "variant_id" : 43741030744121, + "tax_lines" : [ { + "channel_liable" : false, + "rate" : 0.18, + "price" : "53.82", + "price_set" : { + "shop_money" : { + "amount" : "53.82", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "53.82", + "currency_code" : "INR" + } + }, + "title" : "IGST" + } ], + "price" : "299.00", + "vendor" : "My Store", + "product_id" : 7687579762745, + "id" : 12801614413881, + "grams" : 100, + "sku" : "", + "fulfillable_quantity" : 0, + "quantity" : 1, + "fulfillment_service" : "manual", + "taxable" : true, + "variant_inventory_management" : "shopify", + "current_quantity" : 1, + "admin_graphql_api_id" : "gid://shopify/LineItem/12801614413881", + "name" : "Cotton tshirt - L", + "price_set" : { + "shop_money" : { + "amount" : "299.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "299.00", + "currency_code" : "INR" + } + } + }, { + "fulfillable_quantity" : 0, + "fulfillment_status" : "fulfilled", + "quantity" : 1, + "total_discount" : "0.00", + "fulfillment_service" : "manual", + "gift_card" : false, + "taxable" : true, + "requires_shipping" : true, + "total_discount_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "INR" + } + }, + "title" : "Desk Lamp", + "current_quantity" : 1, + "product_exists" : true, + "variant_id" : 43744917815353, + "tax_lines" : [ { + "channel_liable" : false, + "rate" : 0.18, + "price" : "144.00", + "price_set" : { + "shop_money" : { + "amount" : "144.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "144.00", + "currency_code" : "INR" + } + }, + "title" : "IGST" + } ], + "price" : "800.00", + "vendor" : "Dummy Vendor", + "admin_graphql_api_id" : "gid://shopify/LineItem/12801628569657", + "product_id" : 7688576860217, + "name" : "Desk Lamp", + "id" : 12801628569657, + "grams" : 0, + "price_set" : { + "shop_money" : { + "amount" : "800.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "800.00", + "currency_code" : "INR" + } + }, + "sku" : "DL789" + } ], + "order_id" : 5030430111111, + "location_id" : 75274649657, + "status" : "success" + } ], + "client_details" : { + "browser_ip" : "10.0.0.0", + "user_agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + }, + "updated_at" : "2024-09-20T10:13:23+05:30", + "customer_locale" : "en", + "processed_at" : "2024-08-28T14:48:43+05:30", + "currency" : "INR", + "id" : 5030430111111, + "subtotal_price" : "2398.00", + "total_price" : "2829.64", + "total_line_items_price" : "2398.00", + "total_tax" : "431.64", + "total_tip_received" : "0.00", + "financial_status" : "partially_paid", + "name" : "#1002", + "customer" : { + "note" : "", + "tax_exempt" : false, + "email_marketing_consent" : { + "state" : "not_subscribed", + "opt_in_level" : "single_opt_in" + }, + "created_at" : "2024-08-28T15:50:36+05:30", + "last_name" : "Doe", + "verified_email" : true, + "tags" : "", + "default_address" : { + "zip" : "", + "country" : "India", + "address2" : "", + "city" : "Ahmedabad", + "address1" : "", + "last_name" : "Doe", + "province_code" : "GJ", + "country_code" : "IN", + "default" : true, + "province" : "Gujarat", + "name" : "John Doe", + "country_name" : "India", + "company" : "", + "id" : 8518800900153, + "customer_id" : 7597194181111, + "first_name" : "John" + }, + "updated_at" : "2024-09-23T12:22:38+05:30", + "admin_graphql_api_id" : "gid://shopify/Customer/7597194181111", + "currency" : "INR", + "id" : 7597194181111, + "state" : "disabled", + "first_name" : "John", + "email" : "example@gmail.com" + } + } + service: "order" + result: + custom: + buyer_accepts_marketing: false + confirmation_number: "L5MIX0PUW" + confirmed: true + created_at: "2024-08-28T14:48:43+05:30" + currency: "INR" + customer: + admin_graphql_api_id: "gid://shopify/Customer/7597194181111" + created_at: "2024-08-28T15:50:36+05:30" + currency: "INR" + default_address: + address1: "" + address2: "" + city: "Ahmedabad" + company: "" + country: "India" + country_code: "IN" + country_name: "India" + customer_id: 7597194181111 + default: true + first_name: "John" + id: 8518800900153 + last_name: "Doe" + name: "John Doe" + province: "Gujarat" + province_code: "GJ" + zip: "" + email_marketing_consent: + opt_in_level: "single_opt_in" + state: "not_subscribed" + first_name: "John" + last_name: "Doe" + note: "" + state: "disabled" + tags: "" + tax_exempt: false + updated_at: "2024-09-23T12:22:38+05:30" + verified_email: true + customer_locale: "en" + financial_status: "partially_paid" + fulfillment_status: "fulfilled" + fulfillments: + - + updated_at: "2024-09-20T10:13:23+05:30" + service: "manual" + admin_graphql_api_id: "gid://shopify/Fulfillment/4542375952441" + name: "#1002.1" + created_at: "2024-09-20T10:13:23+05:30" + id: 4542375952441 + line_items: + - + variant_title: "Black" + fulfillment_status: "fulfilled" + total_discount: "0.00" + gift_card: false + requires_shipping: true + total_discount_set: + shop_money: + amount: "0.00" + currency_code: "INR" + presentment_money: + amount: "0.00" + currency_code: "INR" + title: "Shoes" + product_exists: true + variant_id: 43744593936441 + tax_lines: + - + channel_liable: false + rate: 0.18 + price: "233.82" + price_set: + shop_money: + amount: "233.82" + currency_code: "INR" + presentment_money: + amount: "233.82" + currency_code: "INR" + title: "IGST" + price: "1299.00" + vendor: "Dummy Vendor" + product_id: 7688312913977 + id: 12778730618937 + grams: 500 + sku: "SHOES-1" + fulfillable_quantity: 0 + quantity: 1 + fulfillment_service: "manual" + taxable: true + variant_inventory_management: "shopify" + current_quantity: 1 + admin_graphql_api_id: "gid://shopify/LineItem/12778730618937" + name: "Shoes - Black" + price_set: + shop_money: + amount: "1299.00" + currency_code: "INR" + presentment_money: + amount: "1299.00" + currency_code: "INR" + - + variant_title: "L" + fulfillment_status: "fulfilled" + total_discount: "0.00" + gift_card: false + requires_shipping: true + total_discount_set: + shop_money: + amount: "0.00" + currency_code: "INR" + presentment_money: + amount: "0.00" + currency_code: "INR" + title: "Cotton tshirt" + product_exists: true + variant_id: 43741030744121 + tax_lines: + - + channel_liable: false + rate: 0.18 + price: "53.82" + price_set: + shop_money: + amount: "53.82" + currency_code: "INR" + presentment_money: + amount: "53.82" + currency_code: "INR" + title: "IGST" + price: "299.00" + vendor: "My Store" + product_id: 7687579762745 + id: 12801614413881 + grams: 100 + sku: "" + fulfillable_quantity: 0 + quantity: 1 + fulfillment_service: "manual" + taxable: true + variant_inventory_management: "shopify" + current_quantity: 1 + admin_graphql_api_id: "gid://shopify/LineItem/12801614413881" + name: "Cotton tshirt - L" + price_set: + shop_money: + amount: "299.00" + currency_code: "INR" + presentment_money: + amount: "299.00" + currency_code: "INR" + - + fulfillable_quantity: 0 + fulfillment_status: "fulfilled" + quantity: 1 + total_discount: "0.00" + fulfillment_service: "manual" + gift_card: false + taxable: true + requires_shipping: true + total_discount_set: + shop_money: + amount: "0.00" + currency_code: "INR" + presentment_money: + amount: "0.00" + currency_code: "INR" + title: "Desk Lamp" + current_quantity: 1 + product_exists: true + variant_id: 43744917815353 + tax_lines: + - + channel_liable: false + rate: 0.18 + price: "144.00" + price_set: + shop_money: + amount: "144.00" + currency_code: "INR" + presentment_money: + amount: "144.00" + currency_code: "INR" + title: "IGST" + price: "800.00" + vendor: "Dummy Vendor" + admin_graphql_api_id: "gid://shopify/LineItem/12801628569657" + product_id: 7688576860217 + name: "Desk Lamp" + id: 12801628569657 + grams: 0 + price_set: + shop_money: + amount: "800.00" + currency_code: "INR" + presentment_money: + amount: "800.00" + currency_code: "INR" + sku: "DL789" + order_id: 5030430111111 + location_id: 75274649657 + status: "success" + http: + useragent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + useragent_details: + browser: + family: "Chrome" + major: "128" + minor: "0" + patch: "0" + patch_minor: "0" + device: + category: "Desktop" + family: "Other" + os: + family: "Windows" + major: "10" + id: 5030430111111 + line_items: + - + variant_title: "Black" + fulfillment_status: "fulfilled" + total_discount: "0.00" + gift_card: false + requires_shipping: true + total_discount_set: + shop_money: + amount: "0.00" + currency_code: "INR" + presentment_money: + amount: "0.00" + currency_code: "INR" + title: "Shoes" + product_exists: true + variant_id: 43744593936441 + tax_lines: + - + channel_liable: false + rate: 0.18 + price: "233.82" + price_set: + shop_money: + amount: "233.82" + currency_code: "INR" + presentment_money: + amount: "233.82" + currency_code: "INR" + title: "IGST" + price: "1299.00" + vendor: "Dummy Vendor" + product_id: 7688312913977 + id: 12778730618937 + grams: 500 + sku: "SHOES-1" + fulfillable_quantity: 0 + quantity: 1 + fulfillment_service: "manual" + taxable: true + variant_inventory_management: "shopify" + current_quantity: 1 + admin_graphql_api_id: "gid://shopify/LineItem/12778730618937" + name: "Shoes - Black" + price_set: + shop_money: + amount: "1299.00" + currency_code: "INR" + presentment_money: + amount: "1299.00" + currency_code: "INR" + - + variant_title: "L" + fulfillment_status: "fulfilled" + total_discount: "0.00" + gift_card: false + requires_shipping: true + total_discount_set: + shop_money: + amount: "0.00" + currency_code: "INR" + presentment_money: + amount: "0.00" + currency_code: "INR" + title: "Cotton tshirt" + product_exists: true + variant_id: 43741030744121 + tax_lines: + - + channel_liable: false + rate: 0.18 + price: "53.82" + price_set: + shop_money: + amount: "53.82" + currency_code: "INR" + presentment_money: + amount: "53.82" + currency_code: "INR" + title: "IGST" + price: "299.00" + vendor: "My Store" + product_id: 7687579762745 + id: 12801614413881 + grams: 100 + sku: "" + fulfillable_quantity: 0 + quantity: 1 + fulfillment_service: "manual" + taxable: true + variant_inventory_management: "shopify" + current_quantity: 1 + admin_graphql_api_id: "gid://shopify/LineItem/12801614413881" + name: "Cotton tshirt - L" + price_set: + shop_money: + amount: "299.00" + currency_code: "INR" + presentment_money: + amount: "299.00" + currency_code: "INR" + - + fulfillable_quantity: 0 + fulfillment_status: "fulfilled" + quantity: 1 + total_discount: "0.00" + fulfillment_service: "manual" + gift_card: false + taxable: true + requires_shipping: true + total_discount_set: + shop_money: + amount: "0.00" + currency_code: "INR" + presentment_money: + amount: "0.00" + currency_code: "INR" + title: "Desk Lamp" + current_quantity: 1 + product_exists: true + variant_id: 43744917815353 + tax_lines: + - + channel_liable: false + rate: 0.18 + price: "144.00" + price_set: + shop_money: + amount: "144.00" + currency_code: "INR" + presentment_money: + amount: "144.00" + currency_code: "INR" + title: "IGST" + price: "800.00" + vendor: "Dummy Vendor" + admin_graphql_api_id: "gid://shopify/LineItem/12801628569657" + product_id: 7688576860217 + name: "Desk Lamp" + id: 12801628569657 + grams: 0 + price_set: + shop_money: + amount: "800.00" + currency_code: "INR" + presentment_money: + amount: "800.00" + currency_code: "INR" + sku: "DL789" + name: "#1002" + network: + client: + geoip: {} + ip: "10.0.0.0" + order_number: 1002 + presentment_currency: "INR" + processed_at: "2024-08-28T14:48:43+05:30" + subtotal_price: "2398.00" + taxes_included: false + total_discounts: "0.00" + total_line_items_price: "2398.00" + total_outstanding: "1296.82" + total_price: "2829.64" + total_tax: "431.64" + total_tip_received: "0.00" + total_weight: 600 + updated_at: "2024-09-20T10:13:23+05:30" + usr: + email: "example@gmail.com" + id: 7597194181111 + name: "John Doe" + message: |- + { + "confirmation_number" : "L5MIX0PUW", + "fulfillment_status" : "fulfilled", + "total_outstanding" : "1296.82", + "order_number" : 1002, + "created_at" : "2024-08-28T14:48:43+05:30", + "taxes_included" : false, + "line_items" : [ { + "variant_title" : "Black", + "fulfillment_status" : "fulfilled", + "total_discount" : "0.00", + "gift_card" : false, + "requires_shipping" : true, + "total_discount_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "INR" + } + }, + "title" : "Shoes", + "product_exists" : true, + "variant_id" : 43744593936441, + "tax_lines" : [ { + "channel_liable" : false, + "rate" : 0.18, + "price" : "233.82", + "price_set" : { + "shop_money" : { + "amount" : "233.82", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "233.82", + "currency_code" : "INR" + } + }, + "title" : "IGST" + } ], + "price" : "1299.00", + "vendor" : "Dummy Vendor", + "product_id" : 7688312913977, + "id" : 12778730618937, + "grams" : 500, + "sku" : "SHOES-1", + "fulfillable_quantity" : 0, + "quantity" : 1, + "fulfillment_service" : "manual", + "taxable" : true, + "variant_inventory_management" : "shopify", + "current_quantity" : 1, + "admin_graphql_api_id" : "gid://shopify/LineItem/12778730618937", + "name" : "Shoes - Black", + "price_set" : { + "shop_money" : { + "amount" : "1299.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "1299.00", + "currency_code" : "INR" + } + } + }, { + "variant_title" : "L", + "fulfillment_status" : "fulfilled", + "total_discount" : "0.00", + "gift_card" : false, + "requires_shipping" : true, + "total_discount_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "INR" + } + }, + "title" : "Cotton tshirt", + "product_exists" : true, + "variant_id" : 43741030744121, + "tax_lines" : [ { + "channel_liable" : false, + "rate" : 0.18, + "price" : "53.82", + "price_set" : { + "shop_money" : { + "amount" : "53.82", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "53.82", + "currency_code" : "INR" + } + }, + "title" : "IGST" + } ], + "price" : "299.00", + "vendor" : "My Store", + "product_id" : 7687579762745, + "id" : 12801614413881, + "grams" : 100, + "sku" : "", + "fulfillable_quantity" : 0, + "quantity" : 1, + "fulfillment_service" : "manual", + "taxable" : true, + "variant_inventory_management" : "shopify", + "current_quantity" : 1, + "admin_graphql_api_id" : "gid://shopify/LineItem/12801614413881", + "name" : "Cotton tshirt - L", + "price_set" : { + "shop_money" : { + "amount" : "299.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "299.00", + "currency_code" : "INR" + } + } + }, { + "fulfillable_quantity" : 0, + "fulfillment_status" : "fulfilled", + "quantity" : 1, + "total_discount" : "0.00", + "fulfillment_service" : "manual", + "gift_card" : false, + "taxable" : true, + "requires_shipping" : true, + "total_discount_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "INR" + } + }, + "title" : "Desk Lamp", + "current_quantity" : 1, + "product_exists" : true, + "variant_id" : 43744917815353, + "tax_lines" : [ { + "channel_liable" : false, + "rate" : 0.18, + "price" : "144.00", + "price_set" : { + "shop_money" : { + "amount" : "144.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "144.00", + "currency_code" : "INR" + } + }, + "title" : "IGST" + } ], + "price" : "800.00", + "vendor" : "Dummy Vendor", + "admin_graphql_api_id" : "gid://shopify/LineItem/12801628569657", + "product_id" : 7688576860217, + "name" : "Desk Lamp", + "id" : 12801628569657, + "grams" : 0, + "price_set" : { + "shop_money" : { + "amount" : "800.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "800.00", + "currency_code" : "INR" + } + }, + "sku" : "DL789" + } ], + "buyer_accepts_marketing" : false, + "presentment_currency" : "INR", + "confirmed" : true, + "total_weight" : 600, + "total_discounts" : "0.00", + "fulfillments" : [ { + "updated_at" : "2024-09-20T10:13:23+05:30", + "service" : "manual", + "admin_graphql_api_id" : "gid://shopify/Fulfillment/4542375952441", + "name" : "#1002.1", + "created_at" : "2024-09-20T10:13:23+05:30", + "id" : 4542375952441, + "line_items" : [ { + "variant_title" : "Black", + "fulfillment_status" : "fulfilled", + "total_discount" : "0.00", + "gift_card" : false, + "requires_shipping" : true, + "total_discount_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "INR" + } + }, + "title" : "Shoes", + "product_exists" : true, + "variant_id" : 43744593936441, + "tax_lines" : [ { + "channel_liable" : false, + "rate" : 0.18, + "price" : "233.82", + "price_set" : { + "shop_money" : { + "amount" : "233.82", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "233.82", + "currency_code" : "INR" + } + }, + "title" : "IGST" + } ], + "price" : "1299.00", + "vendor" : "Dummy Vendor", + "product_id" : 7688312913977, + "id" : 12778730618937, + "grams" : 500, + "sku" : "SHOES-1", + "fulfillable_quantity" : 0, + "quantity" : 1, + "fulfillment_service" : "manual", + "taxable" : true, + "variant_inventory_management" : "shopify", + "current_quantity" : 1, + "admin_graphql_api_id" : "gid://shopify/LineItem/12778730618937", + "name" : "Shoes - Black", + "price_set" : { + "shop_money" : { + "amount" : "1299.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "1299.00", + "currency_code" : "INR" + } + } + }, { + "variant_title" : "L", + "fulfillment_status" : "fulfilled", + "total_discount" : "0.00", + "gift_card" : false, + "requires_shipping" : true, + "total_discount_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "INR" + } + }, + "title" : "Cotton tshirt", + "product_exists" : true, + "variant_id" : 43741030744121, + "tax_lines" : [ { + "channel_liable" : false, + "rate" : 0.18, + "price" : "53.82", + "price_set" : { + "shop_money" : { + "amount" : "53.82", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "53.82", + "currency_code" : "INR" + } + }, + "title" : "IGST" + } ], + "price" : "299.00", + "vendor" : "My Store", + "product_id" : 7687579762745, + "id" : 12801614413881, + "grams" : 100, + "sku" : "", + "fulfillable_quantity" : 0, + "quantity" : 1, + "fulfillment_service" : "manual", + "taxable" : true, + "variant_inventory_management" : "shopify", + "current_quantity" : 1, + "admin_graphql_api_id" : "gid://shopify/LineItem/12801614413881", + "name" : "Cotton tshirt - L", + "price_set" : { + "shop_money" : { + "amount" : "299.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "299.00", + "currency_code" : "INR" + } + } + }, { + "fulfillable_quantity" : 0, + "fulfillment_status" : "fulfilled", + "quantity" : 1, + "total_discount" : "0.00", + "fulfillment_service" : "manual", + "gift_card" : false, + "taxable" : true, + "requires_shipping" : true, + "total_discount_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "INR" + } + }, + "title" : "Desk Lamp", + "current_quantity" : 1, + "product_exists" : true, + "variant_id" : 43744917815353, + "tax_lines" : [ { + "channel_liable" : false, + "rate" : 0.18, + "price" : "144.00", + "price_set" : { + "shop_money" : { + "amount" : "144.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "144.00", + "currency_code" : "INR" + } + }, + "title" : "IGST" + } ], + "price" : "800.00", + "vendor" : "Dummy Vendor", + "admin_graphql_api_id" : "gid://shopify/LineItem/12801628569657", + "product_id" : 7688576860217, + "name" : "Desk Lamp", + "id" : 12801628569657, + "grams" : 0, + "price_set" : { + "shop_money" : { + "amount" : "800.00", + "currency_code" : "INR" + }, + "presentment_money" : { + "amount" : "800.00", + "currency_code" : "INR" + } + }, + "sku" : "DL789" + } ], + "order_id" : 5030430111111, + "location_id" : 75274649657, + "status" : "success" + } ], + "client_details" : { + "browser_ip" : "10.0.0.0", + "user_agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + }, + "updated_at" : "2024-09-20T10:13:23+05:30", + "customer_locale" : "en", + "processed_at" : "2024-08-28T14:48:43+05:30", + "currency" : "INR", + "id" : 5030430111111, + "subtotal_price" : "2398.00", + "total_price" : "2829.64", + "total_line_items_price" : "2398.00", + "total_tax" : "431.64", + "total_tip_received" : "0.00", + "financial_status" : "partially_paid", + "name" : "#1002", + "customer" : { + "note" : "", + "tax_exempt" : false, + "email_marketing_consent" : { + "state" : "not_subscribed", + "opt_in_level" : "single_opt_in" + }, + "created_at" : "2024-08-28T15:50:36+05:30", + "last_name" : "Doe", + "verified_email" : true, + "tags" : "", + "default_address" : { + "zip" : "", + "country" : "India", + "address2" : "", + "city" : "Ahmedabad", + "address1" : "", + "last_name" : "Doe", + "province_code" : "GJ", + "country_code" : "IN", + "default" : true, + "province" : "Gujarat", + "name" : "John Doe", + "country_name" : "India", + "company" : "", + "id" : 8518800900153, + "customer_id" : 7597194181111, + "first_name" : "John" + }, + "updated_at" : "2024-09-23T12:22:38+05:30", + "admin_graphql_api_id" : "gid://shopify/Customer/7597194181111", + "currency" : "INR", + "id" : 7597194181111, + "state" : "disabled", + "first_name" : "John", + "email" : "example@gmail.com" + } + } + service: "order" + tags: + - "source:LOGS_SOURCE" + timestamp: 1726807403000 + - + sample: |- + { + "note" : "Note", + "taxExempt" : false, + "productSubscriberStatus" : "NEVER_SUBSCRIBED", + "lastOrder" : { + "totalPriceSet" : { + "shopMoney" : { + "amount" : "3065.64", + "currencyCode" : "INR" + } + } + }, + "displayName" : "John Doe", + "smsMarketingConsent" : { + "marketingOptInLevel" : "SINGLE_OPT_IN", + "consentCollectedFrom" : "SHOPIFY", + "marketingState" : "NOT_SUBSCRIBED" + }, + "validEmailAddress" : true, + "numberOfOrders" : "1", + "emailMarketingConsent" : { + "marketingOptInLevel" : "SINGLE_OPT_IN", + "marketingState" : "SUBSCRIBED", + "consentUpdatedAt" : "2024-09-03T09:31:52Z" + }, + "createdAt" : "2024-09-03T09:31:52Z", + "legacyResourceId" : "123", + "canDelete" : false, + "lifetimeDuration" : "22 days", + "id" : "gid://shopify/Customer/123", + "state" : "DISABLED", + "dataSaleOptOut" : false, + "amountSpent" : { + "amount" : "3065.64", + "currencyCode" : "INR" + }, + "updatedAt" : "2024-09-18T12:23:20Z", + "defaultAddress" : { + "country" : "India", + "city" : "Ahmedabad", + "countryCodeV2" : "IN" + }, + "verifiedEmail" : true + } + service: "customer" + result: + custom: + amountSpent: + amount: "3065.64" + currencyCode: "INR" + canDelete: false + createdAt: "2024-09-03T09:31:52Z" + dataSaleOptOut: false + defaultAddress: + city: "Ahmedabad" + country: "India" + countryCodeV2: "IN" + emailMarketingConsent: + consentUpdatedAt: "2024-09-03T09:31:52Z" + marketingOptInLevel: "SINGLE_OPT_IN" + marketingState: "SUBSCRIBED" + id: "gid://shopify/Customer/123" + lastOrder: + totalPriceSet: + shopMoney: + amount: "3065.64" + currencyCode: "INR" + lifetimeDuration: "22 days" + note: "Note" + numberOfOrders: "1" + productSubscriberStatus: "NEVER_SUBSCRIBED" + smsMarketingConsent: + consentCollectedFrom: "SHOPIFY" + marketingOptInLevel: "SINGLE_OPT_IN" + marketingState: "NOT_SUBSCRIBED" + state: "DISABLED" + taxExempt: false + updatedAt: "2024-09-18T12:23:20Z" + usr: + id: "123" + name: "John Doe" + validEmailAddress: true + verifiedEmail: true + message: |- + { + "note" : "Note", + "taxExempt" : false, + "productSubscriberStatus" : "NEVER_SUBSCRIBED", + "lastOrder" : { + "totalPriceSet" : { + "shopMoney" : { + "amount" : "3065.64", + "currencyCode" : "INR" + } + } + }, + "displayName" : "John Doe", + "smsMarketingConsent" : { + "marketingOptInLevel" : "SINGLE_OPT_IN", + "consentCollectedFrom" : "SHOPIFY", + "marketingState" : "NOT_SUBSCRIBED" + }, + "validEmailAddress" : true, + "numberOfOrders" : "1", + "emailMarketingConsent" : { + "marketingOptInLevel" : "SINGLE_OPT_IN", + "marketingState" : "SUBSCRIBED", + "consentUpdatedAt" : "2024-09-03T09:31:52Z" + }, + "createdAt" : "2024-09-03T09:31:52Z", + "legacyResourceId" : "123", + "canDelete" : false, + "lifetimeDuration" : "22 days", + "id" : "gid://shopify/Customer/123", + "state" : "DISABLED", + "dataSaleOptOut" : false, + "amountSpent" : { + "amount" : "3065.64", + "currencyCode" : "INR" + }, + "updatedAt" : "2024-09-18T12:23:20Z", + "defaultAddress" : { + "country" : "India", + "city" : "Ahmedabad", + "countryCodeV2" : "IN" + }, + "verifiedEmail" : true + } + service: "customer" + tags: + - "source:LOGS_SOURCE" + timestamp: 1726662200000 + - + sample: |- + { + "sellingPlanGroupsCount" : { + "count" : 0 + }, + "isGiftCard" : false, + "publishedAt" : "2024-09-25T05:37:35Z", + "description" : "zx xvb", + "handle" : "black-pama-shoes", + "totalInventory" : 0, + "tracksInventory" : true, + "title" : "Black Pama Shoes", + "requiresSellingPlan" : false, + "hasOutOfStockVariants" : true, + "variantsCount" : { + "count" : 1 + }, + "priceRangeV2" : { + "maxVariantPrice" : { + "amount" : "0.0", + "currencyCode" : "INR" + }, + "minVariantPrice" : { + "amount" : "0.0", + "currencyCode" : "INR" + } + }, + "createdAt" : "2024-09-25T05:37:35Z", + "hasOnlyDefaultVariant" : true, + "legacyResourceId" : "7715954491449", + "vendor" : "Dummy Vendor", + "mediaCount" : { + "count" : 0 + }, + "onlineStoreUrl" : "https://8856f0-0b.myshopify.com/products/black-pama-shoes", + "id" : "gid://shopify/Product/7715954491449", + "category" : { + "isArchived" : false, + "name" : "Apparel & Accessories", + "fullName" : "Apparel & Accessories", + "id" : "gid://shopify/TaxonomyCategory/aa" + }, + "hasVariantsThatRequiresComponents" : false, + "productType" : "", + "status" : "ACTIVE", + "updatedAt" : "2024-09-25T10:56:37Z" + } + service: "product" + result: + custom: + category: + fullName: "Apparel & Accessories" + id: "gid://shopify/TaxonomyCategory/aa" + isArchived: false + name: "Apparel & Accessories" + createdAt: "2024-09-25T05:37:35Z" + description: "zx xvb" + handle: "black-pama-shoes" + hasOnlyDefaultVariant: true + hasOutOfStockVariants: true + hasVariantsThatRequiresComponents: false + id: "gid://shopify/Product/7715954491449" + isGiftCard: false + legacyResourceId: "7715954491449" + mediaCount: + count: 0 + onlineStoreUrl: "https://8856f0-0b.myshopify.com/products/black-pama-shoes" + priceRangeV2: + maxVariantPrice: + amount: "0.0" + currencyCode: "INR" + minVariantPrice: + amount: "0.0" + currencyCode: "INR" + productType: "" + product_status: "ACTIVE" + publishedAt: "2024-09-25T05:37:35Z" + requiresSellingPlan: false + sellingPlanGroupsCount: + count: 0 + status: "info" + title: "Black Pama Shoes" + totalInventory: 0 + tracksInventory: true + updatedAt: "2024-09-25T10:56:37Z" + variantsCount: + count: 1 + vendor: "Dummy Vendor" + message: |- + { + "sellingPlanGroupsCount" : { + "count" : 0 + }, + "isGiftCard" : false, + "publishedAt" : "2024-09-25T05:37:35Z", + "description" : "zx xvb", + "handle" : "black-pama-shoes", + "totalInventory" : 0, + "tracksInventory" : true, + "title" : "Black Pama Shoes", + "requiresSellingPlan" : false, + "hasOutOfStockVariants" : true, + "variantsCount" : { + "count" : 1 + }, + "priceRangeV2" : { + "maxVariantPrice" : { + "amount" : "0.0", + "currencyCode" : "INR" + }, + "minVariantPrice" : { + "amount" : "0.0", + "currencyCode" : "INR" + } + }, + "createdAt" : "2024-09-25T05:37:35Z", + "hasOnlyDefaultVariant" : true, + "legacyResourceId" : "7715954491449", + "vendor" : "Dummy Vendor", + "mediaCount" : { + "count" : 0 + }, + "onlineStoreUrl" : "https://8856f0-0b.myshopify.com/products/black-pama-shoes", + "id" : "gid://shopify/Product/7715954491449", + "category" : { + "isArchived" : false, + "name" : "Apparel & Accessories", + "fullName" : "Apparel & Accessories", + "id" : "gid://shopify/TaxonomyCategory/aa" + }, + "hasVariantsThatRequiresComponents" : false, + "productType" : "", + "status" : "ACTIVE", + "updatedAt" : "2024-09-25T10:56:37Z" + } + service: "product" + status: "info" + tags: + - "source:LOGS_SOURCE" + timestamp: 1727261797000 diff --git a/shopify/assets/monitors/order_cancellation_rate.json b/shopify/assets/monitors/order_cancellation_rate.json new file mode 100644 index 0000000000000..bbce19a316d3a --- /dev/null +++ b/shopify/assets/monitors/order_cancellation_rate.json @@ -0,0 +1,74 @@ +{ + "version": 2, + "created_at": "2024-10-01", + "last_updated_at": "2024-10-01", + "title": "Order Cancellation Rate is High", + "description": "The order cancellation rate is the percentage of orders that are canceled, compared to the total number of orders placed within a time interval. This monitor tracks the order cancellations to help identify issues, improve the shopping experience, and reduce lost sales.", + "definition": { + "id": 155142683, + "name": "Order Cancellation Rate is High", + "type": "log alert", + "query": "formula(\"query * 100 / query1\").last(\"4h\") >= 5", + "message": "{{#is_warning}}\nThe order cancellation rate has exceeded the defined threshold. \nCurrent Cancellation Rate: {{value}}% \nThreshold: {{warn_threshold}}% \n{{/is_warning}}\n\n{{#is_alert}}\nThe order cancellation rate has exceeded the defined threshold. \nCurrent Cancellation Rate: {{value}}% \nThreshold: {{threshold}}% \n{{/is_alert}}\n\n@abc@example.com", + "tags": [ + "shopify" + ], + "options": { + "thresholds": { + "critical": 5, + "warning": 2 + }, + "enable_logs_sample": false, + "notify_audit": false, + "on_missing_data": "resolve", + "include_tags": false, + "variables": [ + { + "data_source": "logs", + "name": "query", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:order @cancelled_at:*" + }, + "storage": "hot" + }, + { + "data_source": "logs", + "name": "query1", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "cardinality", + "metric": "@id" + }, + "group_by": [], + "search": { + "query": "source:shopify service:order" + }, + "storage": "hot" + } + ], + "renotify_interval": 0, + "escalation_message": "", + "notification_preset_name": "hide_query", + "new_host_delay": 300, + "groupby_simple_monitor": false, + "silenced": {} + }, + "priority": null, + "restriction_policy": { + "bindings": [] + } + }, + "tags": [ + "integration:shopify" + ] +} \ No newline at end of file diff --git a/shopify/assets/monitors/product_inventory_out_of_stock.json b/shopify/assets/monitors/product_inventory_out_of_stock.json new file mode 100644 index 0000000000000..40aca03d25750 --- /dev/null +++ b/shopify/assets/monitors/product_inventory_out_of_stock.json @@ -0,0 +1,39 @@ +{ + "version": 2, + "created_at": "2024-10-01", + "last_updated_at": "2024-10-01", + "title": "Product Inventory is Out of Stock", + "description": "Out-of-stock inventory refers to products that are sold out and unavailable for purchase. This monitor tracks product inventory to help manage stock levels, prevent overselling, and ensure timely restocking.", + "definition": { + "id": 155142623, + "name": "Product Inventory is Out of Stock", + "type": "log alert", + "query": "logs(\"source:shopify service:product @tracksInventory:true @totalInventory:<=0\").index(\"*\").rollup(\"count\").by(\"@legacyResourceId,@title\").last(\"4h\") > 0", + "message": "{{#is_alert}}\nProduct {{@title.name}} (ID: {{@legacyResourceId.name}}) is currently out of stock. Please restock to ensure it's availability.\n{{/is_alert}}\n\n@abc@example.com", + "tags": [ + "shopify" + ], + "options": { + "thresholds": { + "critical": 0 + }, + "enable_logs_sample": false, + "notify_audit": false, + "on_missing_data": "resolve", + "include_tags": false, + "new_group_delay": 60, + "renotify_interval": 0, + "escalation_message": "", + "notification_preset_name": "hide_query", + "groupby_simple_monitor": false, + "silenced": {} + }, + "priority": null, + "restriction_policy": { + "bindings": [] + } + }, + "tags": [ + "integration:shopify" + ] +} \ No newline at end of file diff --git a/shopify/assets/service_checks.json b/shopify/assets/service_checks.json new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/shopify/assets/service_checks.json @@ -0,0 +1 @@ +[] diff --git a/shopify/assets/shopify_dark_theme.svg b/shopify/assets/shopify_dark_theme.svg new file mode 100644 index 0000000000000..abc7797f9c2cb --- /dev/null +++ b/shopify/assets/shopify_dark_theme.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/shopify/assets/shopify_white_them.svg b/shopify/assets/shopify_white_them.svg new file mode 100644 index 0000000000000..55cde5746ce75 --- /dev/null +++ b/shopify/assets/shopify_white_them.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/shopify/images/shopify_customer_overview.png b/shopify/images/shopify_customer_overview.png new file mode 100644 index 0000000000000..26d117353e64b Binary files /dev/null and b/shopify/images/shopify_customer_overview.png differ diff --git a/shopify/images/shopify_event_overview.png b/shopify/images/shopify_event_overview.png new file mode 100644 index 0000000000000..bc597771a9141 Binary files /dev/null and b/shopify/images/shopify_event_overview.png differ diff --git a/shopify/images/shopify_order_overview.png b/shopify/images/shopify_order_overview.png new file mode 100644 index 0000000000000..7edf836c115f9 Binary files /dev/null and b/shopify/images/shopify_order_overview.png differ diff --git a/shopify/images/shopify_product_overview.png b/shopify/images/shopify_product_overview.png new file mode 100644 index 0000000000000..9cdd3ba4e56f1 Binary files /dev/null and b/shopify/images/shopify_product_overview.png differ diff --git a/shopify/manifest.json b/shopify/manifest.json new file mode 100644 index 0000000000000..9043670e922c9 --- /dev/null +++ b/shopify/manifest.json @@ -0,0 +1,73 @@ +{ + "manifest_version": "2.0.0", + "app_uuid": "81c0f478-e722-454a-83d3-5e3f45e11ca8", + "app_id": "shopify", + "display_on_public_website": false, + "tile": { + "overview": "README.md#Overview", + "configuration": "README.md#Setup", + "support": "README.md#Support", + "changelog": "CHANGELOG.md", + "description": "Gain insights into Shopify Event, Product, Customer and Order logs.", + "title": "Shopify", + "media": [ + { + "caption": "Shopify - Event Overview", + "image_url": "images/shopify_event_overview.png", + "media_type": "image" + }, + { + "caption": "Shopify - Product Overview", + "image_url": "images/shopify_product_overview.png", + "media_type": "image" + }, + { + "caption": "Shopify - Customer Overview", + "image_url": "images/shopify_customer_overview.png", + "media_type": "image" + }, + { + "caption": "Shopify - Order Overview", + "image_url": "images/shopify_order_overview.png", + "media_type": "image" + } + ], + "classifier_tags": [ + "Category::Log Collection", + "Submitted Data Type::Logs", + "Offering::Integration" + ] + }, + "assets": { + "integration": { + "auto_install": false, + "source_type_id": 622, + "source_type_name": "Shopify", + "events": { + "creates_events": false + }, + "service_checks": { + "metadata_path": "assets/service_checks.json" + } + }, + "dashboards": { + "Shopify - Event Overview" : "assets/dashboards/shopify_event_overview.json", + "Shopify - Product Overview" : "assets/dashboards/shopify_product_overview.json", + "Shopify - Customer Overview" : "assets/dashboards/shopify_customer_overview.json", + "Shopify - Order Overview" : "assets/dashboards/shopify_order_overview.json" + }, + "monitors" : { + "Product Inventory is Out of Stock" : "assets/monitors/product_inventory_out_of_stock.json", + "Order Cancellation Rate is High" : "assets/monitors/order_cancellation_rate.json" + }, + "logs": { + "source": "shopify" + } + }, + "author": { + "support_email": "help@datadoghq.com", + "name": "Datadog", + "homepage": "https://www.datadoghq.com", + "sales_email": "info@datadoghq.com" + } +} diff --git a/slurm/CHANGELOG.md b/slurm/CHANGELOG.md index 2df8076b53349..93fbb1fcc2497 100644 --- a/slurm/CHANGELOG.md +++ b/slurm/CHANGELOG.md @@ -2,6 +2,18 @@ +## 1.0.3 / 2024-12-06 + +***Fixed***: + +* Add all user query param to the different queries ([#19182](https://github.com/DataDog/integrations-core/pull/19182)) + +## 1.0.2 / 2024-11-28 + +***Fixed***: + +* Bump base package dependency to get fixed pyyaml. ([#19156](https://github.com/DataDog/integrations-core/pull/19156)) + ## 1.0.1 / 2024-11-25 ***Fixed***: diff --git a/slurm/datadog_checks/slurm/__about__.py b/slurm/datadog_checks/slurm/__about__.py index e0db4e56d553f..bfba3d18c9577 100644 --- a/slurm/datadog_checks/slurm/__about__.py +++ b/slurm/datadog_checks/slurm/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '1.0.1' +__version__ = '1.0.3' diff --git a/slurm/datadog_checks/slurm/constants.py b/slurm/datadog_checks/slurm/constants.py index ce66074dfd5bf..f16453e7e9ae1 100644 --- a/slurm/datadog_checks/slurm/constants.py +++ b/slurm/datadog_checks/slurm/constants.py @@ -2,16 +2,16 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) SINFO_PARTITION_PARAMS = [ - "-hO", + "-ahO", "Partition:|,NodeList:|,CPUs:|,Available:|,Memory:|,Cluster:|,NodeAIOT:|,StateLong:|,Nodes:", ] -SINFO_NODE_PARAMS = ["-hNO", "PartitionName:|,Available:|,NodeList:|,NodeAIOT:|,Memory:|,Cluster:"] +SINFO_NODE_PARAMS = ["-haNO", "PartitionName:|,Available:|,NodeList:|,NodeAIOT:|,Memory:|,Cluster:"] SINFO_ADDITIONAL_NODE_PARAMS = "|,CPUsLoad:|,FreeMem:|,Disk:|,StateLong:|,Reason:|,features_act:|,Threads:" GPU_PARAMS = "|,Gres:|,GresUsed:" -SQUEUE_PARAMS = ["-ho", "%A|%u|%j|%T|%N|%C|%R|%m"] -SSHARE_PARAMS = ["-lnPU"] +SQUEUE_PARAMS = ["-aho", "%A|%u|%j|%T|%N|%C|%R|%m"] +SSHARE_PARAMS = ["-alnPU"] SACCT_PARAMS = [ - "-npo", + "-anpo", "JobID,JobName%40,Partition,Account,AllocCPUs,AllocTRES%40,Elapsed,CPUTimeRAW,MaxRSS,MaxVMSize,AveCPU,AveRSS,State,ExitCode,Start,End,NodeList", "--units=M", ] diff --git a/slurm/pyproject.toml b/slurm/pyproject.toml index e1dc5880863bf..d24d3459029ae 100644 --- a/slurm/pyproject.toml +++ b/slurm/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ "Topic :: System :: Monitoring", ] dependencies = [ - "datadog-checks-base>=32.6.0", + "datadog-checks-base>=33.0.0", ] dynamic = [ "version", diff --git a/snmp/CHANGELOG.md b/snmp/CHANGELOG.md index 34638ae4e8f8f..9c4c2edd8dd7d 100644 --- a/snmp/CHANGELOG.md +++ b/snmp/CHANGELOG.md @@ -2,6 +2,12 @@ +## 9.1.0 / 2024-11-28 + +***Added***: + +* [NDMII-3147] update Cisco IP SLA metric tags and description. ([#19079](https://github.com/DataDog/integrations-core/pull/19079)) + ## 9.0.0 / 2024-10-04 / Agent 7.59.0 ***Removed***: @@ -37,7 +43,7 @@ ***Added***: -* Update dependencies ([#18185](https://github.com/DataDog/integrations-core/pull/18185)) +* Update dependencies ([#18187](https://github.com/DataDog/integrations-core/pull/18187)) ## 7.3.1 / 2024-06-05 / Agent 7.55.0 diff --git a/snmp/assets/dashboards/interface_performance.json b/snmp/assets/dashboards/interface_performance.json index e7228088ae2b8..138f85b5a2d5a 100644 --- a/snmp/assets/dashboards/interface_performance.json +++ b/snmp/assets/dashboards/interface_performance.json @@ -587,9 +587,9 @@ "cell_display_mode": ["bar"], "conditional_formats": [ { - "comparator": ">", - "palette": "white_on_yellow", - "value": 2 + "comparator": "<=", + "palette": "white_on_green", + "value": 1 }, { "comparator": "<=", @@ -598,8 +598,8 @@ }, { "comparator": "<=", - "palette": "white_on_green", - "value": 1 + "palette": "white_on_yellow", + "value": 3 } ], "q": "avg:snmp.ifAdminStatus{$snmp_host,$interface,$snmp_device} by {snmp_device,snmp_host,interface}", diff --git a/snmp/assets/service_checks.json b/snmp/assets/service_checks.json index 7611d3f115ed1..9a6b400cfbced 100644 --- a/snmp/assets/service_checks.json +++ b/snmp/assets/service_checks.json @@ -13,6 +13,6 @@ "snmp_device" ], "name": "Can check", - "description": "Returns `CRITICAL` if the Agent check is unable to collect metrics from SNMP, and `WARNING` if it partially works but metrics configuration is incorrect. Returns `OK` otherwise." + "description": "Returns `CRITICAL` if the Agent check is unable to collect SNMP metrics from the SNMP Agent, and `WARNING` if it partially works but metrics configuration is incorrect. Returns `OK` otherwise." } ] diff --git a/snmp/changelog.d/19079.added b/snmp/changelog.d/19079.added deleted file mode 100644 index 74814808c1f37..0000000000000 --- a/snmp/changelog.d/19079.added +++ /dev/null @@ -1 +0,0 @@ -[NDMII-3147] update Cisco IP SLA metric tags and description. diff --git a/snmp/datadog_checks/snmp/__about__.py b/snmp/datadog_checks/snmp/__about__.py index d028d71da95d0..29b2032fbf67a 100644 --- a/snmp/datadog_checks/snmp/__about__.py +++ b/snmp/datadog_checks/snmp/__about__.py @@ -2,4 +2,4 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '9.0.0' +__version__ = '9.1.0' diff --git a/snmp/metadata.csv b/snmp/metadata.csv index d8d924476d837..1931558493c59 100644 --- a/snmp/metadata.csv +++ b/snmp/metadata.csv @@ -988,35 +988,35 @@ snmp.ibm.imm.systemHealthSummary,gauge,,,,"[IMM-MIB] Table of System Health summ snmp.ibm.imm.systemMemoryVpd,gauge,,,,"[IMM-MIB] Table of the system Memory VPD information. View as a table and not as individual entries for consistent results. (Make 'sum by {X}' queries to count elements with the tag X.)",0,snmp,, snmp.ibm.imm.tempReading,gauge,,,,"[IMM-MIB] The measured temperature.",0,snmp,, snmp.ibm.imm.voltReading,gauge,,,,"[IMM-MIB] The measured voltage.",0,snmp,, -snmp.ifAdminStatus,gauge,,,,[Generic router] [F5 BIG-IP] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The desired state of the interface.,0,snmp,, +snmp.ifAdminStatus,gauge,,,,[Generic router] [F5 BIG-IP] The desired state of the interface.,0,snmp,, snmp.ifBandwidthInUsage.rate,gauge,,percent,,"[Generic router] The percent rate of used received bandwidth.",0,snmp,, snmp.ifBandwidthOutUsage.rate,gauge,,percent,,"[Generic router] The percent rate of used sent bandwidth.",0,snmp,, -snmp.ifHCInBroadcastPkts,count,,packet,,[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The number of packets delivered by this sub-layer to a higher (sub-)layer that were addressed to a broadcast address at this sub-layer.,0,snmp,, -snmp.ifHCInMulticastPkts,count,,packet,,[Generic router] [F5 BIG-IP] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The number of packets delivered by this sub-layer to a higher (sub-)layer which were addressed to a multicast address at this sub-layer.,0,snmp,, -snmp.ifHCInOctets,count,,byte,,[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The total number of octets received on the interface including framing characters.,0,snmp,, -snmp.ifHCInOctets.rate,gauge,,byte,second,[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The total number of octets received on the interface including framing characters.,0,snmp,, -snmp.ifHCInUcastPkts,count,,packet,,[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The number of packets delivered by this sub-layer to a higher (sub-)layer that were not addressed to a multicast or broadcast address at this sub-layer.,0,snmp,, -snmp.ifHCOutBroadcastPkts,count,,packet,,"[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The total number of packets that higher-level protocols requested be transmitted that were addressed to a broadcast address at this sub-layer, including those that were discarded or not sent.",0,snmp,, -snmp.ifHCOutMulticastPkts,count,,packet,,[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The total number of packets that higher-level protocols requested be transmitted that were addressed to a multicast address at this sub-layer including those that were discarded or not sent.,0,snmp,, -snmp.ifHCOutOctets,count,,byte,,[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The total number of octets transmitted out of the interface including framing characters.,0,snmp,, -snmp.ifHCOutOctets.rate,gauge,,byte,second,[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The total number of octets transmitted out of the interface including framing characters.,0,snmp,, -snmp.ifHCOutUcastPkts,count,,packet,,[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The total number of packets higher-level protocols requested be transmitted that were not addressed to a multicast or broadcast address at this sub-layer including those that were discarded or not sent.,0,snmp,, +snmp.ifHCInBroadcastPkts,count,,packet,,[Generic router] The number of packets delivered by this sub-layer to a higher (sub-)layer that were addressed to a broadcast address at this sub-layer.,0,snmp,, +snmp.ifHCInMulticastPkts,count,,packet,,[Generic router] [F5 BIG-IP] The number of packets delivered by this sub-layer to a higher (sub-)layer which were addressed to a multicast address at this sub-layer.,0,snmp,, +snmp.ifHCInOctets,count,,byte,,[Generic router] The total number of octets received on the interface including framing characters.,0,snmp,, +snmp.ifHCInOctets.rate,gauge,,byte_in_bits_family,second,[Generic router] The inbound data rate on the interface including framing characters.,0,snmp,, +snmp.ifHCInUcastPkts,count,,packet,,[Generic router] The number of packets delivered by this sub-layer to a higher (sub-)layer that were not addressed to a multicast or broadcast address at this sub-layer.,0,snmp,, +snmp.ifHCOutBroadcastPkts,count,,packet,,"[Generic device] The total number of packets that higher-level protocols requested be transmitted that were addressed to a broadcast address at this sub-layer, including those that were discarded or not sent.",0,snmp,, +snmp.ifHCOutMulticastPkts,count,,packet,,[Generic device] The total number of packets that higher-level protocols requested be transmitted that were addressed to a multicast address at this sub-layer including those that were discarded or not sent.,0,snmp,, +snmp.ifHCOutOctets,count,,byte,,[Generic device] The total number of octets transmitted out of the interface including framing characters.,0,snmp,, +snmp.ifHCOutOctets.rate,gauge,,byte_in_bits_family,second,[Generic device] The outbound data rate on the interface including framing characters.,0,snmp,, +snmp.ifHCOutUcastPkts,count,,packet,,[Generic device] The total number of packets higher-level protocols requested be transmitted that were not addressed to a multicast or broadcast address at this sub-layer including those that were discarded or not sent.,0,snmp,, snmp.ifHighInOctets,count,,byte,,"[NetApp] The total number of bytes received on the interface, including framing characters.",0,snmp,, -snmp.ifHighInOctets.rate,gauge,,byte,second,"[NetApp] The number bytes per second received on the interface, including framing characters.",0,snmp,, -snmp.ifHighSpeed,gauge,,,,"[Generic router] An estimate of the interface's current bandwidth in units of 1,000,000 bits per second, or the nominal bandwidth.",0,snmp,, -snmp.ifInDiscards,count,,packet,,[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The number of inbound packets chosen to be discarded even though no errors had been detected to prevent them being deliverable to a higher-layer protocol.,0,snmp,, -snmp.ifInDiscards.rate,gauge,,packet,second,[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The number of inbound packets chosen to be discarded even though no errors had been detected to prevent them being deliverable to a higher-layer protocol.,0,snmp,, -snmp.ifInErrors,count,,packet,,[Generic router] [Cisco c3850] [Cisco Nexus][Cisco ASA 5525] [F5 BIG-IP] The number of inbound packets that contained errors preventing them from being deliverable to a higher-layer protocol.,0,snmp,, -snmp.ifInErrors.rate,gauge,,packet,second,[Generic router] [Cisco c3850] [Cisco Nexus][Cisco ASA 5525] [F5 BIG-IP] The number of inbound packets that contained errors preventing them from being deliverable to a higher-layer protocol.,0,snmp,, -snmp.ifInSpeed,gauge,,,,"[Generic router] An estimate of the interface's current inbound bandwidth in bits per second.",0,snmp,, +snmp.ifHighInOctets.rate,gauge,,byte_in_bits_family,second,"[Generic device] The inbound data rate on the interface, including framing characters.",0,snmp,, +snmp.ifHighSpeed,gauge,,megabit,,"[Generic device] An estimate of the interface's current bandwidth in units of 1,000,000 bits per second, or the nominal bandwidth.",0,snmp,, +snmp.ifInDiscards,count,,packet,,[Generic device] The number of inbound packets chosen to be discarded even though no errors had been detected to prevent them being deliverable to a higher-layer protocol.,0,snmp,, +snmp.ifInDiscards.rate,gauge,,packet,second,[Generic device] The number of inbound packets chosen to be discarded even though no errors had been detected to prevent them being deliverable to a higher-layer protocol.,0,snmp,, +snmp.ifInErrors,count,,packet,,[Generic router] The number of inbound packets that contained errors preventing them from being deliverable to a higher-layer protocol.,0,snmp,, +snmp.ifInErrors.rate,gauge,,packet,second,[Generic router] The number of inbound packets that contained errors preventing them from being deliverable to a higher-layer protocol.,0,snmp,, +snmp.ifInSpeed,gauge,,bit_in_bits_family,,"[Generic router] An estimate of the interface's current inbound bandwidth in bits per second.",0,snmp,, snmp.ifNumber,gauge,,,,[Generic router] The number of network interfaces (regardless of their current state) present on this system.,0,snmp,, -snmp.ifOperStatus,gauge,,,,[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The current operational state of the interface.,0,snmp,, -snmp.ifOutDiscards,count,,packet,,[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The number of outbound packets chosen to be discarded even though no errors had been detected to prevent them being transmitted.,0,snmp,, -snmp.ifOutDiscards.rate,gauge,,packet,second,[Generic router] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The number of outbound packets chosen to be discarded even though no errors had been detected to prevent them being transmitted.,0,snmp,, -snmp.ifOutErrors,count,,packet,,[Generic router] [F5 BIG-IP] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The number of outbound packets that could not be transmitted because of errors.,0,snmp,, -snmp.ifOutErrors.rate,gauge,,packet,second,[Generic router] [F5 BIG-IP] [Cisco c3850] [Cisco Nexus] [Cisco ASA 5525] The number of outbound packets that could not be transmitted because of errors.,0,snmp,, -snmp.ifOutSpeed,gauge,,,,"[Generic router] An estimate of the interface's current outbound bandwidth in bits per second.",0,snmp,, -snmp.ifSpeed,gauge,,,,"[Generic router] An estimate of the interface's current bandwidth in bits per second, or the nominal bandwidth.",0,snmp,, +snmp.ifOperStatus,gauge,,,,[Generic router] The current operational state of the interface.,0,snmp,, +snmp.ifOutDiscards,count,,packet,,[Generic router] The number of outbound packets chosen to be discarded even though no errors had been detected to prevent them being transmitted.,0,snmp,, +snmp.ifOutDiscards.rate,gauge,,packet,second,[Generic router] The number of outbound packets chosen to be discarded even though no errors had been detected to prevent them being transmitted.,0,snmp,, +snmp.ifOutErrors,count,,packet,,[Generic router] The number of outbound packets that could not be transmitted because of errors.,0,snmp,, +snmp.ifOutErrors.rate,gauge,,packet,second,[Generic router] The number of outbound packets that could not be transmitted because of errors.,0,snmp,, +snmp.ifOutSpeed,gauge,,bit_in_bits_family,,"[Generic router] An estimate of the interface's current outbound bandwidth in bits per second.",0,snmp,, +snmp.ifSpeed,gauge,,bit_in_bits_family,,"[Generic router] An estimate of the interface's current bandwidth in bits per second, or the nominal bandwidth.",0,snmp,, snmp.ifsTotalBytes,gauge,,byte,,[Isilon] The total cluster capacity of the /ifs filesystem in bytes.,0,snmp,, snmp.ifsUsedBytes,gauge,,byte,,[Isilon] The number of bytes used in the /ifs filesystem.,0,snmp,, snmp.interface.status,gauge,,,,"For each interface of each monitored network device, this metric reports always 1 with the admin_status and oper_status as tags, as long as a 'combined' status that can be used for monitors.",0,snmp,, diff --git a/snowflake/CHANGELOG.md b/snowflake/CHANGELOG.md index b909256429882..def3fc861a5a8 100644 --- a/snowflake/CHANGELOG.md +++ b/snowflake/CHANGELOG.md @@ -2,6 +2,12 @@ +## 7.1.0 / 2024-11-28 + +***Added***: + +* Upgrade `snowflake-connector-python` to 3.12.3 ([#19010](https://github.com/DataDog/integrations-core/pull/19010)) + ## 7.0.0 / 2024-10-04 / Agent 7.59.0 ***Removed***: @@ -32,7 +38,7 @@ ***Added***: -* Update dependencies ([#18185](https://github.com/DataDog/integrations-core/pull/18185)) +* Update dependencies ([#18187](https://github.com/DataDog/integrations-core/pull/18187)) ## 5.7.0 / 2024-07-05 / Agent 7.56.0 diff --git a/snowflake/changelog.d/19010.added b/snowflake/changelog.d/19010.added deleted file mode 100644 index dc4809c026413..0000000000000 --- a/snowflake/changelog.d/19010.added +++ /dev/null @@ -1 +0,0 @@ -Upgrade `snowflake-connector-python` to 3.12.3 \ No newline at end of file diff --git a/snowflake/datadog_checks/snowflake/__about__.py b/snowflake/datadog_checks/snowflake/__about__.py index 7d1ab144cf5b2..1440196a477c0 100644 --- a/snowflake/datadog_checks/snowflake/__about__.py +++ b/snowflake/datadog_checks/snowflake/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2020-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '7.0.0' +__version__ = '7.1.0' diff --git a/sonarqube/CHANGELOG.md b/sonarqube/CHANGELOG.md index ed7433e548344..6ac541b49131e 100644 --- a/sonarqube/CHANGELOG.md +++ b/sonarqube/CHANGELOG.md @@ -2,6 +2,12 @@ +## 5.1.0 / 2024-11-28 + +***Added***: + +* Add `MAX_PAGES` to Sonarqube API request ([#19149](https://github.com/DataDog/integrations-core/pull/19149)) + ## 5.0.0 / 2024-10-04 / Agent 7.59.0 ***Removed***: diff --git a/sonarqube/changelog.d/19149.added b/sonarqube/changelog.d/19149.added deleted file mode 100644 index 74d0f05524b38..0000000000000 --- a/sonarqube/changelog.d/19149.added +++ /dev/null @@ -1 +0,0 @@ -Add `MAX_PAGES` to Sonarqube API request \ No newline at end of file diff --git a/sonarqube/datadog_checks/sonarqube/__about__.py b/sonarqube/datadog_checks/sonarqube/__about__.py index 1fda5ed1ca206..591962e51d2ac 100644 --- a/sonarqube/datadog_checks/sonarqube/__about__.py +++ b/sonarqube/datadog_checks/sonarqube/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2020-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '5.0.0' +__version__ = '5.1.0' diff --git a/sonicwall_firewall/CHANGELOG.md b/sonicwall_firewall/CHANGELOG.md index 87e70fa519fde..2720565e415a1 100644 --- a/sonicwall_firewall/CHANGELOG.md +++ b/sonicwall_firewall/CHANGELOG.md @@ -2,3 +2,8 @@ +## 1.0.0 / 2024-11-28 + +***Added***: + +* Initial Release ([#18667](https://github.com/DataDog/integrations-core/pull/18667)) diff --git a/sonicwall_firewall/changelog.d/18667.added b/sonicwall_firewall/changelog.d/18667.added deleted file mode 100644 index aa949b47b7b41..0000000000000 --- a/sonicwall_firewall/changelog.d/18667.added +++ /dev/null @@ -1 +0,0 @@ -Initial Release \ No newline at end of file diff --git a/sonicwall_firewall/datadog_checks/sonicwall_firewall/__about__.py b/sonicwall_firewall/datadog_checks/sonicwall_firewall/__about__.py index e9541ce83e9e5..acbfd1c866b84 100644 --- a/sonicwall_firewall/datadog_checks/sonicwall_firewall/__about__.py +++ b/sonicwall_firewall/datadog_checks/sonicwall_firewall/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '0.0.1' +__version__ = '1.0.0' diff --git a/sqlserver/CHANGELOG.md b/sqlserver/CHANGELOG.md index b9771578f65eb..58032c651ecc2 100644 --- a/sqlserver/CHANGELOG.md +++ b/sqlserver/CHANGELOG.md @@ -2,6 +2,28 @@ +## 20.2.0 / 2024-11-28 + +***Added***: + +* Submit database_hostname with database instance and metrics for MySQL, Postgres, and SQLServer ([#18969](https://github.com/DataDog/integrations-core/pull/18969)) +* Add lookback_window config parameter to query_metrics. + + The current lookback window defaults to 2 times the collection interval, and is not able to be overridden. + This means that infrequently-run queries are unlikely to have metrics captured for them. One common + use case that falls into this bucket is ETL queries which can run hourly or even daily. These have + a very small chance of having metrics captured for them. In that case, we will support setting a lookback + window that will include such queries. ([#18979](https://github.com/DataDog/integrations-core/pull/18979)) +* Add config option `azure.aggregate_sql_databases` to report multiple azure sql databases as one database host. This is an opted in feature and is disabled by default. ([#19032](https://github.com/DataDog/integrations-core/pull/19032)) + +***Fixed***: + +* Fix missing appended SQL comments. ([#18958](https://github.com/DataDog/integrations-core/pull/18958)) +* Fix `azure_sql_server_database` resource tag to use Azure SQL Database `{fully_qualified_doman_name}/{database_name}`. ([#19014](https://github.com/DataDog/integrations-core/pull/19014)) +* Update SQLServer agent jobs metrics to be DBM only. ([#19033](https://github.com/DataDog/integrations-core/pull/19033)) +* Fix duplicate deadlock events ([#19139](https://github.com/DataDog/integrations-core/pull/19139)) +* Fix poor query signature correlation for deadlocks. ([#19142](https://github.com/DataDog/integrations-core/pull/19142)) + ## 20.1.1 / 2024-11-25 ***Fixed***: diff --git a/sqlserver/assets/configuration/spec.yaml b/sqlserver/assets/configuration/spec.yaml index 4fde7367ce475..9497f4ef208be 100644 --- a/sqlserver/assets/configuration/spec.yaml +++ b/sqlserver/assets/configuration/spec.yaml @@ -129,12 +129,230 @@ files: type: boolean example: false display_default: true - - name: include_ao_metrics + - name: database_metrics description: | - Include AlwaysOn availability group metrics. - value: - type: boolean - example: false + Configure the collection of database metrics + options: + - name: ao_metrics + description: | + Configure collection of AlwaysOn availability group metrics. + + When the `ao_metrics.enabled` is True, use `ao_metrics.availability_group` to specify the + resource group id of a specific availability group that you would like to monitor. + If no availability group is specified, then we will collect AlwaysOn metrics for all + availability groups on the current replica. + + Primary replicas may emit metrics for remote secondary replicas + in the same availability group. If `ao_metrics.only_emit_local` is set to true, + the primary replica will only emit information local to itself. + + If `ao_metrics.ao_database` is set, AlwaysOn metrics are only emitted for the selected `ao_database`. + value: + type: object + properties: + - name: enabled + type: boolean + example: false + - name: availability_group + type: string + - name: only_emit_local + type: boolean + example: false + - name: ao_database + type: string + - name: db_backup_metrics + description: | + Configure collection of database backup metrics. + Use `db_backup_metrics.collection_interval` to set the interval (in seconds) for the collection of + database backup metrics. Defaults to 300 seconds (5 minutes). If you intend on updating this value, + it is strongly recommended to use a consistent value throughout all SQL Server agent deployments. + hidden: true + value: + type: object + properties: + - name: enabled + type: boolean + example: true + - name: collection_interval + type: integer + example: 300 + display_default: 300 + - name: db_files_metrics + description: | + Configure collection of database files metrics. + hidden: true + value: + type: object + properties: + - name: enabled + type: boolean + example: true + - name: db_stats_metrics + description: | + Configure collection of database stats metrics + hidden: true + value: + type: object + properties: + - name: enabled + type: boolean + example: true + - name: db_fragmentation_metrics + description: | + Configure collection of database fragmentation metrics. + Note these queries can be resource intensive on large datasets. Recommend to limit these via + autodiscovery or specific database instances. + + Use `db_fragmentation_metrics.enabled_tempdb` to enable collection of database index fragmentation statistics + in tempdb database from the `sys.dm_db_index_physical_stats` DMF. + By default, we do not collect index fragmentation statistics in the tempdb database, as those queries + might cause blocking. This configuration parameter allows enabling the collection of this metric. + This parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false. + + Use `db_fragmentation_metrics.collection_interval` to set the interval (in seconds) for the collection of + database fragmentation metrics from the `sys.dm_db_index_physical_stats` DMF. + Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + to use a consistent value throughout all SQL Server agent deployments. + value: + type: object + properties: + - name: enabled + type: boolean + example: false + - name: enabled_tempdb + type: boolean + example: false + - name: collection_interval + type: integer + example: 300 + display_default: 300 + - name: fci_metrics + description: | + Configure collection of failover Cluster Instance metrics. Note that these metrics + requires a SQLServer set up with Failover Clustering enabled. + value: + type: object + properties: + - name: enabled + type: boolean + example: false + - name: file_stats_metrics + description: | + Configure collection of file stats metrics. + hidden: true + value: + type: object + properties: + - name: enabled + type: boolean + example: true + - name: index_usage_metrics + description: | + Configure collection of user table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. + Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable + `database_autodiscovery` or set `database`. + + Use `index_usage_metrics.enabled_tempdb` to enable collection of user table index usage statistics in tempdb + database from the `sys.dm_db_index_usage_stats` DMV. + By default, we do not collect index usage statistics in the tempdb database, as those queries + might cause blocking. This configuration parameter allows enabling the collection of this metric. + This parameter is ignored if 'index_usage_metrics.enabled' is set to false. + + Use `index_usage_metrics.collection_interval` to set the interval (in seconds) for the collection of index + usage statistics from the `sys.dm_db_index_usage_stats` DMV. + Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + to use a consistent value throughout all SQL Server agent deployments. + value: + type: object + properties: + - name: enabled + type: boolean + example: false + - name: enabled_tempdb + type: boolean + example: false + - name: collection_interval + type: integer + example: 300 + display_default: 300 + - name: instance_metrics + description: | + Configure collection of server-level instance metrics. When setting up multiple instances for + different databases on the same host these metrics will be duplicated unless this option is turned off. + value: + type: object + properties: + - name: enabled + type: boolean + example: true + - name: master_files_metrics + description: | + Configure collection of database file size and state from `sys.master_files` + value: + type: object + properties: + - name: enabled + type: boolean + example: false + - name: primary_log_shipping_metrics + description: | + Configure collection of metrics for a log shipping setup. Required to run against the + primary instance in a transaction log shipping configuration. Note that + the Datadog user needs to be present in msdb and must be added to the db_datareader role. + value: + type: object + properties: + - name: enabled + type: boolean + example: false + - name: secondary_log_shipping_metrics + description: | + Configure collection of metrics for a log shipping setup. Required to run against the + secondary instance in a transaction log shipping configuration. Note that + the Datadog user needs to be present in msdb and must be added to the db_datareader role. + value: + type: object + properties: + - name: enabled + type: boolean + example: false + - name: server_state_metrics + description: | + Configure collection of server state metrics + hidden: true + value: + type: object + properties: + - name: enabled + type: boolean + example: true + - name: task_scheduler_metrics + description: | + Configure collection of additional Task and Scheduler metrics. + value: + type: object + properties: + - name: enabled + type: boolean + example: false + - name: tempdb_file_space_usage_metrics + description: | + Configure collection of tempdb file space usage metrics for how space is used in tempdb data files. + value: + type: object + properties: + - name: enabled + type: boolean + example: false + - name: xe_metrics + description: | + Configure collection of extended events (XE) metrics. + value: + type: object + properties: + - name: enabled + type: boolean + example: false - name: agent_jobs description: Configure collection of agent jobs events and metrics options: @@ -157,116 +375,6 @@ files: value: type: integer example: 10000 - - name: availability_group - description: | - When `include_ao_metrics` is enabled, you can provide the resource - group id of a specific availability group that you would like to monitor. - If no availability group is specified, then we will collect AlwaysOn metrics - for all availability groups on the current replica. - value: - type: string - - name: only_emit_local - description: | - Primary replicas may emit metrics for remote secondary replicas - in the same availability group. If this option is set to true, - the primary replica will only emit information local to itself. - value: - type: boolean - example: false - - name: ao_database - description: | - AlwaysOn metrics are only emitted for the selected `ao_database` if not empty. - value: - type: string - - name: include_master_files_metrics - description: | - Include database file size and state from `sys.master_files`. - value: - type: boolean - example: false - - name: include_fci_metrics - description: | - Include Failover Cluster Instance metrics. Note that these metrics - requires a SQLServer set up with Failover Clustering enabled. - value: - type: boolean - example: false - - name: include_primary_log_shipping_metrics - description: | - Include log_shipping_primary metrics for a log shipping setup. Required to run - against the primary instance in a transaction log shipping configuration. Note that - the Datadog user needs to be present in msdb and must be added to the db_datareader role. - value: - type: boolean - example: false - - name: include_secondary_log_shipping_metrics - description: | - Include log_shipping_secondary metrics for a log shipping setup. Required to run - against a secondary instance in a transaction log shipping configuration. Note that - the Datadog user needs to be present in msdb and must be added to the db_datareader role. - value: - type: boolean - example: false - - name: include_instance_metrics - description: | - Include server-level instance metrics. When setting up multiple instances for - different databases on the same host these metrics will be duplicated unless this option is turned off. - value: - type: boolean - example: true - - name: include_task_scheduler_metrics - description: Include additional Task and Scheduler metrics. - value: - type: boolean - example: false - - name: include_db_fragmentation_metrics - description: | - Include database fragmentation metrics. Note these queries can be resource intensive on large datasets. - Recommend to limit these via autodiscovery or specific database instances. - value: - type: boolean - example: false - - name: include_db_fragmentation_metrics_tempdb - description: | - Configure the collection of database index fragmentation statistics in tempdb database from the - `sys.dm_db_index_physical_stats` DMF. - - By default, we do not collect index fragmentation statistics in the tempdb database, as those queries - might cause blocking. This configuration parameter allows enabling the collection of this metric. - This parameter is ignored if 'include_db_fragmentation_metrics' is set to false. - value: - type: boolean - example: false - - name: include_index_usage_metrics - description: | - Configure the collection of user table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. - - Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable - `database_autodiscovery` or set `database`. - value: - type: boolean - example: true - - name: include_index_usage_metrics_tempdb - description: | - Configure the collection of user table index usage statistics in tempdb database from the - `sys.dm_db_index_usage_stats` DMV. - - By default, we do not collect index usage statistics in the tempdb database, as those queries - might cause blocking. This configuration parameter allows enabling the collection of this metric. - This parameter is ignored if 'include_index_usage_metrics' is set to false. - value: - type: boolean - example: false - - name: index_usage_metrics_interval - description: | - Configure the interval (in seconds) for the collection of index usage statistics from the - `sys.dm_db_index_usage_stats` DMV. - Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - to use a consistent value throughout all SQL Server agent deployments. - value: - type: integer - example: 300 - display_default: 300 - name: db_fragmentation_object_names description: | Fragmentation metrics normally emit metrics for all objects within a database. @@ -276,19 +384,6 @@ files: type: array items: type: string - - name: include_tempdb_file_space_usage_metrics - description: | - Include tempdb file space usage metrics for how space is used in tempdb data files. - value: - type: boolean - example: true - - name: include_xe_metrics - description: | - Include extended events (XE) metrics. The collection of XE metrics is automatically enabled - when `deadlocks_collection` is enabled. - value: - type: boolean - example: true - name: adoprovider description: | Choose the ADO provider. Note that the (default) provider @@ -649,6 +744,8 @@ files: description: | Set to `false` to disable the collection of comments in your SQL statements. Requires `collect_metadata: true`. + Note: This option must be `true` in order to correlate Database Monitoring samples and APM traces. + See https://docs.datadoghq.com/database_monitoring/connect_dbm_and_apm value: type: boolean example: true @@ -764,7 +861,7 @@ files: value: type: number example: 1800 - display_default: false + display_default: 300 - name: schemas_collection description: | Available for Agent 7.56 and newer. diff --git a/sqlserver/changelog.d/18958.fixed b/sqlserver/changelog.d/18958.fixed deleted file mode 100644 index 0ba896b9a6e49..0000000000000 --- a/sqlserver/changelog.d/18958.fixed +++ /dev/null @@ -1 +0,0 @@ -Fix missing appended SQL comments. diff --git a/sqlserver/changelog.d/18969.added b/sqlserver/changelog.d/18969.added deleted file mode 100644 index dfc29ca0f7645..0000000000000 --- a/sqlserver/changelog.d/18969.added +++ /dev/null @@ -1 +0,0 @@ -Submit database_hostname with database instance and metrics for MySQL, Postgres, and SQLServer diff --git a/sqlserver/changelog.d/18979.added b/sqlserver/changelog.d/18979.added deleted file mode 100644 index 5edd40d1e74fb..0000000000000 --- a/sqlserver/changelog.d/18979.added +++ /dev/null @@ -1,7 +0,0 @@ -Add lookback_window config parameter to query_metrics. - -The current lookback window defaults to 2 times the collection interval, and is not able to be overridden. -This means that infrequently-run queries are unlikely to have metrics captured for them. One common -use case that falls into this bucket is ETL queries which can run hourly or even daily. These have -a very small chance of having metrics captured for them. In that case, we will support setting a lookback -window that will include such queries. diff --git a/sqlserver/changelog.d/19014.fixed b/sqlserver/changelog.d/19014.fixed deleted file mode 100644 index 1a3ee8c5f1408..0000000000000 --- a/sqlserver/changelog.d/19014.fixed +++ /dev/null @@ -1 +0,0 @@ -Fix `azure_sql_server_database` resource tag to use Azure SQL Database `{fully_qualified_doman_name}/{database_name}`. diff --git a/sqlserver/changelog.d/19032.added b/sqlserver/changelog.d/19032.added deleted file mode 100644 index 4a64ae6d50354..0000000000000 --- a/sqlserver/changelog.d/19032.added +++ /dev/null @@ -1 +0,0 @@ -Add config option `azure.aggregate_sql_databases` to report multiple azure sql databases as one database host. This is an opted in feature and is disabled by default. diff --git a/sqlserver/changelog.d/19033.fixed b/sqlserver/changelog.d/19033.fixed deleted file mode 100644 index 14d0c87a667da..0000000000000 --- a/sqlserver/changelog.d/19033.fixed +++ /dev/null @@ -1 +0,0 @@ -Update SQLServer agent jobs metrics to be DBM only. diff --git a/sqlserver/changelog.d/19111.added b/sqlserver/changelog.d/19111.added new file mode 100644 index 0000000000000..c3ae6706f8277 --- /dev/null +++ b/sqlserver/changelog.d/19111.added @@ -0,0 +1 @@ +Update configuration structure and allow configuration of all database metrics \ No newline at end of file diff --git a/sqlserver/changelog.d/19139.fixed b/sqlserver/changelog.d/19139.fixed deleted file mode 100644 index cec4e36064310..0000000000000 --- a/sqlserver/changelog.d/19139.fixed +++ /dev/null @@ -1 +0,0 @@ -Fix duplicate deadlock events diff --git a/sqlserver/changelog.d/19142.fixed b/sqlserver/changelog.d/19142.fixed deleted file mode 100644 index e04ecf292d982..0000000000000 --- a/sqlserver/changelog.d/19142.fixed +++ /dev/null @@ -1 +0,0 @@ -Fix poor query signature correlation for deadlocks. diff --git a/sqlserver/changelog.d/19189.changed b/sqlserver/changelog.d/19189.changed new file mode 100644 index 0000000000000..c79730c4307db --- /dev/null +++ b/sqlserver/changelog.d/19189.changed @@ -0,0 +1 @@ +Fall back to ``system_health/event_file`` when querying deadlocks if `datadog` XE session wasn't created. diff --git a/sqlserver/datadog_checks/sqlserver/__about__.py b/sqlserver/datadog_checks/sqlserver/__about__.py index 88c205094f084..cc600abdac80d 100644 --- a/sqlserver/datadog_checks/sqlserver/__about__.py +++ b/sqlserver/datadog_checks/sqlserver/__about__.py @@ -2,4 +2,4 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '20.1.1' +__version__ = '20.2.0' diff --git a/sqlserver/datadog_checks/sqlserver/config.py b/sqlserver/datadog_checks/sqlserver/config.py index 92508a44e5721..bcbbb9be07b66 100644 --- a/sqlserver/datadog_checks/sqlserver/config.py +++ b/sqlserver/datadog_checks/sqlserver/config.py @@ -10,6 +10,7 @@ from datadog_checks.base.utils.db.utils import get_agent_host_tags from datadog_checks.sqlserver.const import ( DEFAULT_AUTODISCOVERY_INTERVAL, + DEFAULT_LONG_METRICS_COLLECTION_INTERVAL, PROC_CHAR_LIMIT, ) @@ -28,17 +29,14 @@ def __init__(self, init_config, instance, log): self.autodiscovery_db_service_check: bool = is_affirmative(instance.get('autodiscovery_db_service_check', True)) self.min_collection_interval: int = instance.get('min_collection_interval', 15) self.autodiscovery_interval: int = instance.get('autodiscovery_interval', DEFAULT_AUTODISCOVERY_INTERVAL) + self.database_instance_collection_interval: int = instance.get( + 'database_instance_collection_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL + ) self._include_patterns = self._compile_valid_patterns(self.autodiscovery_include) self._exclude_patterns = self._compile_valid_patterns(self.autodiscovery_exclude) self.proc: str = instance.get('stored_procedure') self.custom_metrics: list[dict] = init_config.get('custom_metrics', []) or [] - self.include_index_usage_metrics_tempdb: bool = is_affirmative( - instance.get('include_index_usage_metrics_tempdb', False) - ) - self.include_db_fragmentation_metrics_tempdb: bool = is_affirmative( - instance.get('include_db_fragmentation_metrics_tempdb', False) - ) self.ignore_missing_database = is_affirmative(instance.get("ignore_missing_database", False)) if self.ignore_missing_database: self.log.warning( @@ -48,6 +46,7 @@ def __init__(self, init_config, instance, log): # DBM self.dbm_enabled: bool = is_affirmative(instance.get('dbm', False)) + self.database_metrics_config: dict = self._build_database_metrics_configs(instance) self.statement_metrics_config: dict = instance.get('query_metrics', {}) or {} self.agent_jobs_config: dict = instance.get('agent_jobs', {}) or {} self.procedure_metrics_config: dict = instance.get('procedure_metrics', {}) or {} @@ -110,10 +109,10 @@ def __init__(self, init_config, instance, log): ) self.log_unobfuscated_queries: bool = is_affirmative(instance.get('log_unobfuscated_queries', False)) self.log_unobfuscated_plans: bool = is_affirmative(instance.get('log_unobfuscated_plans', False)) - self.database_instance_collection_interval: int = instance.get('database_instance_collection_interval', 300) self.stored_procedure_characters_limit: int = instance.get('stored_procedure_characters_limit', PROC_CHAR_LIMIT) self.connection_host: str = instance['host'] self.service = instance.get('service') or init_config.get('service') or '' + self.db_fragmentation_object_names = instance.get('db_fragmentation_object_names', []) or [] def _compile_valid_patterns(self, patterns: list[str]) -> re.Pattern: valid_patterns = [] @@ -170,3 +169,83 @@ def _should_propagate_agent_tags(instance, init_config) -> bool: return init_config_propagate_agent_tags # if neither the instance nor the init_config has set the value, return False return False + + def _build_database_metrics_configs(self, instance): + # Set defaults for database metrics + configurable_metrics = { + "ao_metrics": {'enabled': False, 'availability_group': None, 'ao_database': None, 'only_emit_local': False}, + "db_backup_metrics": {'enabled': True, 'collection_interval': DEFAULT_LONG_METRICS_COLLECTION_INTERVAL}, + "db_files_metrics": {'enabled': True}, + "db_stats_metrics": {'enabled': True}, + "db_fragmentation_metrics": { + 'enabled': False, + 'enabled_tempdb': False, + 'collection_interval': DEFAULT_LONG_METRICS_COLLECTION_INTERVAL, + }, + "fci_metrics": {'enabled': False}, + "file_stats_metrics": {'enabled': True}, + "index_usage_metrics": { + 'enabled': True, + 'collection_interval': DEFAULT_LONG_METRICS_COLLECTION_INTERVAL, + 'enabled_tempdb': False, + }, + "instance_metrics": {'enabled': True}, + "master_files_metrics": {'enabled': False}, + "primary_log_shipping_metrics": {'enabled': False}, + "secondary_log_shipping_metrics": {'enabled': False}, + "server_state_metrics": {'enabled': True}, + "task_scheduler_metrics": {'enabled': False}, + "tempdb_file_space_usage_metrics": {'enabled': True}, + "xe_metrics": {'enabled': False}, + } + # Check if the instance has any configuration for the metrics in legacy structure + legacy_configuration_metrics = { + "include_ao_metrics": "ao_metrics", + "include_master_files_metrics": "master_files_metrics", + "include_fci_metrics": "fci_metrics", + "include_primary_log_shipping_metrics": "primary_log_shipping_metrics", + "include_secondary_log_shipping_metrics": "secondary_log_shipping_metrics", + "include_instance_metrics": "instance_metrics", + "include_task_scheduler_metrics": "task_scheduler_metrics", + "include_db_fragmentation_metrics": "db_fragmentation_metrics", + "include_index_usage_metrics": "index_usage_metrics", + "include_tempdb_file_space_usage_metrics": "tempdb_file_space_usage_metrics", + "include_xe_metrics": "xe_metrics", + } + for metric, config_key in legacy_configuration_metrics.items(): + if instance.get(metric) is not None: + configurable_metrics[config_key]['enabled'] = instance[metric] + # Manual look ups for legacy configuration structure + configurable_metrics['ao_metrics']['availability_group'] = instance.get( + 'availability_group', configurable_metrics['ao_metrics']['availability_group'] + ) + configurable_metrics['ao_metrics']['ao_database'] = instance.get( + 'ao_database', configurable_metrics['ao_metrics']['ao_database'] + ) + configurable_metrics['ao_metrics']['only_emit_local'] = instance.get( + 'only_emit_local', configurable_metrics['ao_metrics']['only_emit_local'] + ) + configurable_metrics['db_backup_metrics']['collection_interval'] = instance.get( + 'database_backup_metrics_interval', configurable_metrics['db_backup_metrics']['collection_interval'] + ) + configurable_metrics['db_fragmentation_metrics']['enabled_tempdb'] = instance.get( + 'include_db_fragmentation_metrics_tempdb', + configurable_metrics['db_fragmentation_metrics']['enabled_tempdb'], + ) + configurable_metrics['db_fragmentation_metrics']['collection_interval'] = instance.get( + 'db_fragmentation_metrics_interval', configurable_metrics['db_fragmentation_metrics']['collection_interval'] + ) + configurable_metrics['index_usage_metrics']['enabled_tempdb'] = instance.get( + 'include_index_usage_metrics_tempdb', configurable_metrics['index_usage_metrics']['enabled_tempdb'] + ) + configurable_metrics['index_usage_metrics']['collection_interval'] = instance.get( + 'index_usage_stats_interval', configurable_metrics['index_usage_metrics']['collection_interval'] + ) + # Check if the instance has any configuration for the metrics + database_metrics = instance.get('database_metrics', {}) + for metric, config in configurable_metrics.items(): + metric_config = database_metrics.get(metric, {}) + for key, value in metric_config.items(): + if value is not None: + config[key] = value + return configurable_metrics diff --git a/sqlserver/datadog_checks/sqlserver/config_models/defaults.py b/sqlserver/datadog_checks/sqlserver/config_models/defaults.py index 8dfc29b5ddfa7..1c6fcda461355 100644 --- a/sqlserver/datadog_checks/sqlserver/config_models/defaults.py +++ b/sqlserver/datadog_checks/sqlserver/config_models/defaults.py @@ -41,7 +41,7 @@ def instance_database_autodiscovery_interval(): def instance_database_instance_collection_interval(): - return False + return 300 def instance_dbm(): @@ -64,62 +64,6 @@ def instance_ignore_missing_database(): return False -def instance_include_ao_metrics(): - return False - - -def instance_include_db_fragmentation_metrics(): - return False - - -def instance_include_db_fragmentation_metrics_tempdb(): - return False - - -def instance_include_fci_metrics(): - return False - - -def instance_include_index_usage_metrics(): - return True - - -def instance_include_index_usage_metrics_tempdb(): - return False - - -def instance_include_instance_metrics(): - return True - - -def instance_include_master_files_metrics(): - return False - - -def instance_include_primary_log_shipping_metrics(): - return False - - -def instance_include_secondary_log_shipping_metrics(): - return False - - -def instance_include_task_scheduler_metrics(): - return False - - -def instance_include_tempdb_file_space_usage_metrics(): - return True - - -def instance_include_xe_metrics(): - return True - - -def instance_index_usage_metrics_interval(): - return 300 - - def instance_log_unobfuscated_plans(): return False @@ -136,10 +80,6 @@ def instance_only_custom_queries(): return False -def instance_only_emit_local(): - return False - - def instance_proc_only_if_database(): return 'master' diff --git a/sqlserver/datadog_checks/sqlserver/config_models/instance.py b/sqlserver/datadog_checks/sqlserver/config_models/instance.py index ad93d2bdb6f48..1f83cbd0ccc1d 100644 --- a/sqlserver/datadog_checks/sqlserver/config_models/instance.py +++ b/sqlserver/datadog_checks/sqlserver/config_models/instance.py @@ -12,7 +12,7 @@ from types import MappingProxyType from typing import Any, Optional -from pydantic import BaseModel, ConfigDict, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from datadog_checks.base.utils.functions import identity from datadog_checks.base.utils.models import validation @@ -69,6 +69,165 @@ class CustomQuery(BaseModel): tags: Optional[tuple[str, ...]] = None +class AoMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + ao_database: Optional[str] = None + availability_group: Optional[str] = None + enabled: Optional[bool] = Field(None, examples=[False]) + only_emit_local: Optional[bool] = Field(None, examples=[False]) + + +class DbBackupMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[int] = Field(None, examples=[300]) + enabled: Optional[bool] = Field(None, examples=[True]) + + +class DbFilesMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[True]) + + +class DbFragmentationMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[int] = Field(None, examples=[300]) + enabled: Optional[bool] = Field(None, examples=[False]) + enabled_tempdb: Optional[bool] = Field(None, examples=[False]) + + +class DbStatsMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[True]) + + +class FciMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class FileStatsMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[True]) + + +class IndexUsageMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[int] = Field(None, examples=[300]) + enabled: Optional[bool] = Field(None, examples=[False]) + enabled_tempdb: Optional[bool] = Field(None, examples=[False]) + + +class InstanceMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[True]) + + +class MasterFilesMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class PrimaryLogShippingMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class SecondaryLogShippingMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class ServerStateMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[True]) + + +class TaskSchedulerMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class TempdbFileSpaceUsageMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class XeMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class DatabaseMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + ao_metrics: Optional[AoMetrics] = None + db_backup_metrics: Optional[DbBackupMetrics] = None + db_files_metrics: Optional[DbFilesMetrics] = None + db_fragmentation_metrics: Optional[DbFragmentationMetrics] = None + db_stats_metrics: Optional[DbStatsMetrics] = None + fci_metrics: Optional[FciMetrics] = None + file_stats_metrics: Optional[FileStatsMetrics] = None + index_usage_metrics: Optional[IndexUsageMetrics] = None + instance_metrics: Optional[InstanceMetrics] = None + master_files_metrics: Optional[MasterFilesMetrics] = None + primary_log_shipping_metrics: Optional[PrimaryLogShippingMetrics] = None + secondary_log_shipping_metrics: Optional[SecondaryLogShippingMetrics] = None + server_state_metrics: Optional[ServerStateMetrics] = None + task_scheduler_metrics: Optional[TaskSchedulerMetrics] = None + tempdb_file_space_usage_metrics: Optional[TempdbFileSpaceUsageMetrics] = None + xe_metrics: Optional[XeMetrics] = None + + class DeadlocksCollection(BaseModel): model_config = ConfigDict( arbitrary_types_allowed=True, @@ -179,11 +338,9 @@ class InstanceConfig(BaseModel): ) adoprovider: Optional[str] = None agent_jobs: Optional[AgentJobs] = None - ao_database: Optional[str] = None autodiscovery_db_service_check: Optional[bool] = None autodiscovery_exclude: Optional[tuple[str, ...]] = None autodiscovery_include: Optional[tuple[str, ...]] = None - availability_group: Optional[str] = None aws: Optional[Aws] = None azure: Optional[Azure] = None collect_settings: Optional[CollectSettings] = None @@ -195,6 +352,7 @@ class InstanceConfig(BaseModel): database_autodiscovery: Optional[bool] = None database_autodiscovery_interval: Optional[int] = None database_instance_collection_interval: Optional[float] = None + database_metrics: Optional[DatabaseMetrics] = None db_fragmentation_object_names: Optional[tuple[str, ...]] = None dbm: Optional[bool] = None deadlocks_collection: Optional[DeadlocksCollection] = None @@ -205,20 +363,6 @@ class InstanceConfig(BaseModel): gcp: Optional[Gcp] = None host: str ignore_missing_database: Optional[bool] = None - include_ao_metrics: Optional[bool] = None - include_db_fragmentation_metrics: Optional[bool] = None - include_db_fragmentation_metrics_tempdb: Optional[bool] = None - include_fci_metrics: Optional[bool] = None - include_index_usage_metrics: Optional[bool] = None - include_index_usage_metrics_tempdb: Optional[bool] = None - include_instance_metrics: Optional[bool] = None - include_master_files_metrics: Optional[bool] = None - include_primary_log_shipping_metrics: Optional[bool] = None - include_secondary_log_shipping_metrics: Optional[bool] = None - include_task_scheduler_metrics: Optional[bool] = None - include_tempdb_file_space_usage_metrics: Optional[bool] = None - include_xe_metrics: Optional[bool] = None - index_usage_metrics_interval: Optional[int] = None log_unobfuscated_plans: Optional[bool] = None log_unobfuscated_queries: Optional[bool] = None managed_identity: Optional[ManagedIdentity] = None @@ -226,7 +370,6 @@ class InstanceConfig(BaseModel): min_collection_interval: Optional[float] = None obfuscator_options: Optional[ObfuscatorOptions] = None only_custom_queries: Optional[bool] = None - only_emit_local: Optional[bool] = None password: Optional[str] = None proc_only_if: Optional[str] = None proc_only_if_database: Optional[str] = None diff --git a/sqlserver/datadog_checks/sqlserver/const.py b/sqlserver/datadog_checks/sqlserver/const.py index 7fc1cf86a659e..5d29c54041040 100644 --- a/sqlserver/datadog_checks/sqlserver/const.py +++ b/sqlserver/datadog_checks/sqlserver/const.py @@ -271,3 +271,5 @@ PROC_CHAR_LIMIT = 500 DEFAULT_SCHEMAS_COLLECTION_INTERVAL = 600 + +DEFAULT_LONG_METRICS_COLLECTION_INTERVAL = 300 diff --git a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example index 5418c1b76b220..7ea10e54d507d 100644 --- a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example +++ b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example @@ -127,10 +127,107 @@ instances: # autodiscovery_db_service_check: false - ## @param include_ao_metrics - boolean - optional - default: false - ## Include AlwaysOn availability group metrics. + ## Configure the collection of database metrics # - # include_ao_metrics: false + # database_metrics: + + ## @param ao_metrics - mapping - optional + ## Configure collection of AlwaysOn availability group metrics. + ## + ## When the `ao_metrics.enabled` is True, use `ao_metrics.availability_group` to specify the + ## resource group id of a specific availability group that you would like to monitor. + ## If no availability group is specified, then we will collect AlwaysOn metrics for all + ## availability groups on the current replica. + ## + ## Primary replicas may emit metrics for remote secondary replicas + ## in the same availability group. If `ao_metrics.only_emit_local` is set to true, + ## the primary replica will only emit information local to itself. + ## + ## If `ao_metrics.ao_database` is set, AlwaysOn metrics are only emitted for the selected `ao_database`. + # + # ao_metrics: {} + + ## @param db_fragmentation_metrics - mapping - optional + ## Configure collection of database fragmentation metrics. + ## Note these queries can be resource intensive on large datasets. Recommend to limit these via + ## autodiscovery or specific database instances. + ## + ## Use `db_fragmentation_metrics.enabled_tempdb` to enable collection of database index fragmentation statistics + ## in tempdb database from the `sys.dm_db_index_physical_stats` DMF. + ## By default, we do not collect index fragmentation statistics in the tempdb database, as those queries + ## might cause blocking. This configuration parameter allows enabling the collection of this metric. + ## This parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false. + ## + ## Use `db_fragmentation_metrics.collection_interval` to set the interval (in seconds) for the collection of + ## database fragmentation metrics from the `sys.dm_db_index_physical_stats` DMF. + ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + ## to use a consistent value throughout all SQL Server agent deployments. + # + # db_fragmentation_metrics: {} + + ## @param fci_metrics - mapping - optional + ## Configure collection of failover Cluster Instance metrics. Note that these metrics + ## requires a SQLServer set up with Failover Clustering enabled. + # + # fci_metrics: {} + + ## @param index_usage_metrics - mapping - optional + ## Configure collection of user table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. + ## Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable + ## `database_autodiscovery` or set `database`. + ## + ## Use `index_usage_metrics.enabled_tempdb` to enable collection of user table index usage statistics in tempdb + ## database from the `sys.dm_db_index_usage_stats` DMV. + ## By default, we do not collect index usage statistics in the tempdb database, as those queries + ## might cause blocking. This configuration parameter allows enabling the collection of this metric. + ## This parameter is ignored if 'index_usage_metrics.enabled' is set to false. + ## + ## Use `index_usage_metrics.collection_interval` to set the interval (in seconds) for the collection of index + ## usage statistics from the `sys.dm_db_index_usage_stats` DMV. + ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + ## to use a consistent value throughout all SQL Server agent deployments. + # + # index_usage_metrics: {} + + ## @param instance_metrics - mapping - optional + ## Configure collection of server-level instance metrics. When setting up multiple instances for + ## different databases on the same host these metrics will be duplicated unless this option is turned off. + # + # instance_metrics: {} + + ## @param master_files_metrics - mapping - optional + ## Configure collection of database file size and state from `sys.master_files` + # + # master_files_metrics: {} + + ## @param primary_log_shipping_metrics - mapping - optional + ## Configure collection of metrics for a log shipping setup. Required to run against the + ## primary instance in a transaction log shipping configuration. Note that + ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. + # + # primary_log_shipping_metrics: {} + + ## @param secondary_log_shipping_metrics - mapping - optional + ## Configure collection of metrics for a log shipping setup. Required to run against the + ## secondary instance in a transaction log shipping configuration. Note that + ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. + # + # secondary_log_shipping_metrics: {} + + ## @param task_scheduler_metrics - mapping - optional + ## Configure collection of additional Task and Scheduler metrics. + # + # task_scheduler_metrics: {} + + ## @param tempdb_file_space_usage_metrics - mapping - optional + ## Configure collection of tempdb file space usage metrics for how space is used in tempdb data files. + # + # tempdb_file_space_usage_metrics: {} + + ## @param xe_metrics - mapping - optional + ## Configure collection of extended events (XE) metrics. + # + # xe_metrics: {} ## Configure collection of agent jobs events and metrics # @@ -152,104 +249,6 @@ instances: # # history_row_limit: 10000 - ## @param availability_group - string - optional - ## When `include_ao_metrics` is enabled, you can provide the resource - ## group id of a specific availability group that you would like to monitor. - ## If no availability group is specified, then we will collect AlwaysOn metrics - ## for all availability groups on the current replica. - # - # availability_group: - - ## @param only_emit_local - boolean - optional - default: false - ## Primary replicas may emit metrics for remote secondary replicas - ## in the same availability group. If this option is set to true, - ## the primary replica will only emit information local to itself. - # - # only_emit_local: false - - ## @param ao_database - string - optional - ## AlwaysOn metrics are only emitted for the selected `ao_database` if not empty. - # - # ao_database: - - ## @param include_master_files_metrics - boolean - optional - default: false - ## Include database file size and state from `sys.master_files`. - # - # include_master_files_metrics: false - - ## @param include_fci_metrics - boolean - optional - default: false - ## Include Failover Cluster Instance metrics. Note that these metrics - ## requires a SQLServer set up with Failover Clustering enabled. - # - # include_fci_metrics: false - - ## @param include_primary_log_shipping_metrics - boolean - optional - default: false - ## Include log_shipping_primary metrics for a log shipping setup. Required to run - ## against the primary instance in a transaction log shipping configuration. Note that - ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. - # - # include_primary_log_shipping_metrics: false - - ## @param include_secondary_log_shipping_metrics - boolean - optional - default: false - ## Include log_shipping_secondary metrics for a log shipping setup. Required to run - ## against a secondary instance in a transaction log shipping configuration. Note that - ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. - # - # include_secondary_log_shipping_metrics: false - - ## @param include_instance_metrics - boolean - optional - default: true - ## Include server-level instance metrics. When setting up multiple instances for - ## different databases on the same host these metrics will be duplicated unless this option is turned off. - # - # include_instance_metrics: true - - ## @param include_task_scheduler_metrics - boolean - optional - default: false - ## Include additional Task and Scheduler metrics. - # - # include_task_scheduler_metrics: false - - ## @param include_db_fragmentation_metrics - boolean - optional - default: false - ## Include database fragmentation metrics. Note these queries can be resource intensive on large datasets. - ## Recommend to limit these via autodiscovery or specific database instances. - # - # include_db_fragmentation_metrics: false - - ## @param include_db_fragmentation_metrics_tempdb - boolean - optional - default: false - ## Configure the collection of database index fragmentation statistics in tempdb database from the - ## `sys.dm_db_index_physical_stats` DMF. - ## - ## By default, we do not collect index fragmentation statistics in the tempdb database, as those queries - ## might cause blocking. This configuration parameter allows enabling the collection of this metric. - ## This parameter is ignored if 'include_db_fragmentation_metrics' is set to false. - # - # include_db_fragmentation_metrics_tempdb: false - - ## @param include_index_usage_metrics - boolean - optional - default: true - ## Configure the collection of user table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. - ## - ## Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable - ## `database_autodiscovery` or set `database`. - # - # include_index_usage_metrics: true - - ## @param include_index_usage_metrics_tempdb - boolean - optional - default: false - ## Configure the collection of user table index usage statistics in tempdb database from the - ## `sys.dm_db_index_usage_stats` DMV. - ## - ## By default, we do not collect index usage statistics in the tempdb database, as those queries - ## might cause blocking. This configuration parameter allows enabling the collection of this metric. - ## This parameter is ignored if 'include_index_usage_metrics' is set to false. - # - # include_index_usage_metrics_tempdb: false - - ## @param index_usage_metrics_interval - integer - optional - default: 300 - ## Configure the interval (in seconds) for the collection of index usage statistics from the - ## `sys.dm_db_index_usage_stats` DMV. - ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - ## to use a consistent value throughout all SQL Server agent deployments. - # - # index_usage_metrics_interval: 300 - ## @param db_fragmentation_object_names - list of strings - optional ## Fragmentation metrics normally emit metrics for all objects within a database. ## This option allows you to specify database object names to query for fragmentation metrics. @@ -257,17 +256,6 @@ instances: # # db_fragmentation_object_names: [] - ## @param include_tempdb_file_space_usage_metrics - boolean - optional - default: true - ## Include tempdb file space usage metrics for how space is used in tempdb data files. - # - # include_tempdb_file_space_usage_metrics: true - - ## @param include_xe_metrics - boolean - optional - default: true - ## Include extended events (XE) metrics. The collection of XE metrics is automatically enabled - ## when `deadlocks_collection` is enabled. - # - # include_xe_metrics: true - ## @param adoprovider - string - optional - default: SQLOLEDB ## Choose the ADO provider. Note that the (default) provider ## SQLOLEDB is being deprecated. To use the newer MSOLEDBSQL @@ -556,6 +544,8 @@ instances: ## @param collect_comments - boolean - optional - default: true ## Set to `false` to disable the collection of comments in your SQL statements. ## Requires `collect_metadata: true`. + ## Note: This option must be `true` in order to correlate Database Monitoring samples and APM traces. + ## See https://docs.datadoghq.com/database_monitoring/connect_dbm_and_apm # # collect_comments: true diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py index 31b3238ba8752..8e83034bda518 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py @@ -4,7 +4,6 @@ from typing import List -from datadog_checks.base.config import is_affirmative from datadog_checks.sqlserver.utils import is_azure_database from .base import SqlserverDatabaseMetricsBase @@ -61,7 +60,7 @@ class SqlserverAoMetrics(SqlserverDatabaseMetricsBase): @property def include_ao_metrics(self) -> bool: - return is_affirmative(self.instance_config.get('include_ao_metrics', False)) + return self.config.database_metrics_config["ao_metrics"]["enabled"] @property def enabled(self) -> bool: diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py index c3c7937f65c2b..ed3558c3d3a1c 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py @@ -2,8 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase AVAILABILITY_GROUPS_METRICS_QUERY = { @@ -38,11 +36,11 @@ class SqlserverAvailabilityGroupsMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-hadr-availability-group-states-transact-sql?view=sql-server-ver15 @property def include_ao_metrics(self) -> bool: - return is_affirmative(self.instance_config.get('include_ao_metrics', False)) + return self.config.database_metrics_config["ao_metrics"]["enabled"] @property def availability_group(self): - return self.instance_config.get('availability_group') + return self.config.database_metrics_config["ao_metrics"]["availability_group"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py index 95b0c9041108c..69bccb89af457 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py @@ -2,8 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase AVAILABILITY_REPLICAS_METRICS_QUERY = { @@ -49,19 +47,19 @@ class SqlserverAvailabilityReplicasMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-availability-replicas-transact-sql?view=sql-server-ver15 @property def include_ao_metrics(self) -> bool: - return is_affirmative(self.instance_config.get('include_ao_metrics', False)) + return self.config.database_metrics_config["ao_metrics"]["enabled"] @property def availability_group(self): - return self.instance_config.get('availability_group') + return self.config.database_metrics_config["ao_metrics"]["availability_group"] @property def only_emit_local(self): - return is_affirmative(self.instance_config.get('only_emit_local', False)) + return self.config.database_metrics_config["ao_metrics"]["only_emit_local"] @property def ao_database(self): - return self.instance_config.get('ao_database') + return self.config.database_metrics_config["ao_metrics"]["ao_database"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/base.py b/sqlserver/datadog_checks/sqlserver/database_metrics/base.py index bc61f01c71491..013c68d9703e7 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/base.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/base.py @@ -15,7 +15,6 @@ class SqlserverDatabaseMetricsBase: def __init__( self, config, - instance_config, new_query_executor, server_static_info, execute_query_handler, @@ -23,7 +22,6 @@ def __init__( databases=None, ): self.config: SQLServerConfig = config - self.instance_config: dict = instance_config # TODO: Remove instance_config and use self.config self.server_static_info: dict = server_static_info self.new_query_executor: Callable[ [List[dict], Callable, Optional[List[str]], Optional[bool]], QueryExecutor diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_agent_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_agent_metrics.py index 6d99655392eaf..0c2491b0b056f 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_agent_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_agent_metrics.py @@ -110,7 +110,7 @@ class SqlserverAgentMetrics(SqlserverDatabaseMetricsBase): def include_agent_metrics(self) -> bool: if not self.config.dbm_enabled: return False - agent_jobs_config = self.instance_config.get('agent_jobs', {}) + agent_jobs_config = self.config.agent_jobs_config if agent_jobs_config: return is_affirmative(agent_jobs_config.get('enabled', False)) return False @@ -128,7 +128,7 @@ def collection_interval(self) -> int: Returns the interval in seconds at which to collect index usage metrics. Note: The index usage metrics query can be expensive, so it is recommended to set a higher interval. ''' - agent_jobs_config = self.instance_config.get('agent_jobs', {}) + agent_jobs_config = self.config.agent_jobs_config if agent_jobs_config: return int(agent_jobs_config.get('collection_interval', 15)) return 15 # 15 seconds diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py index e82cd65963204..21be687d112b1 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py @@ -30,26 +30,23 @@ class SqlserverDatabaseBackupMetrics(SqlserverDatabaseMetricsBase): # Contains a row for each backup set. A backup set # contains the backup from a single, successful backup operation. # https://docs.microsoft.com/en-us/sql/relational-databases/system-tables/backupset-transact-sql?view=sql-server-ver15 + @property + def include_database_backup_metrics(self) -> bool: + return self.config.database_metrics_config["db_backup_metrics"]["enabled"] + @property def enabled(self): - if is_azure_sql_database(self.engine_edition): + if not self.include_database_backup_metrics or is_azure_sql_database(self.engine_edition): return False return True - @property - def _default_collection_interval(self) -> int: - ''' - Returns the default interval in seconds at which to collect database backup metrics. - ''' - return 5 * 60 # 5 minutes - @property def collection_interval(self) -> int: ''' Returns the interval in seconds at which to collect database backup metrics. Note: The database backup metrics query can be expensive, so it is recommended to set a higher interval. ''' - return int(self.instance_config.get('database_backup_metrics_interval', self._default_collection_interval)) + return self.config.database_metrics_config["db_backup_metrics"]["collection_interval"] @property def queries(self): @@ -63,6 +60,7 @@ def __repr__(self) -> str: return ( f"{self.__class__.__name__}(" f"enabled={self.enabled}, " + f"include_database_backup_metrics={self.include_database_backup_metrics}), " f"engine_edition={self.engine_edition}, " f"collection_interval={self.collection_interval})" ) diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_files_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_files_metrics.py index 0786ebd5e999c..5dd458bfd6005 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_files_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_files_metrics.py @@ -46,8 +46,14 @@ class SqlserverDatabaseFilesMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-database-files-transact-sql + @property + def include_database_files_metrics(self) -> bool: + return self.config.database_metrics_config["db_files_metrics"]["enabled"] + @property def enabled(self): + if not self.include_database_files_metrics: + return False return True @property @@ -55,7 +61,11 @@ def queries(self): return [DATABASE_FILES_METRICS_QUERY] def __repr__(self) -> str: - return f"{self.__class__.__name__}(" f"enabled={self.enabled})" + return ( + f"{self.__class__.__name__}(" + f"enabled={self.enabled}, " + f"include_database_files_metrics={self.include_database_files_metrics})" + ) def _build_query_executors(self): executors = [] diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py index e1233e353cf83..cbc5503855c80 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py @@ -2,8 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase DATABASE_REPLICATION_STATS_METRICS_QUERY = { @@ -34,15 +32,15 @@ class SqlserverDatabaseReplicationStatsMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-hadr-database-replica-states-transact-sql?view=sql-server-ver15 @property def include_ao_metrics(self) -> bool: - return is_affirmative(self.instance_config.get('include_ao_metrics', False)) + return self.config.database_metrics_config["ao_metrics"]["enabled"] @property def availability_group(self): - return self.instance_config.get('availability_group') + return self.config.database_metrics_config["ao_metrics"]["availability_group"] @property def only_emit_local(self): - return is_affirmative(self.instance_config.get('only_emit_local', False)) + return self.config.database_metrics_config["ao_metrics"]["only_emit_local"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_stats_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_stats_metrics.py index d8024f4deb929..3c34a16004f81 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_stats_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_stats_metrics.py @@ -33,8 +33,14 @@ class SqlserverDatabaseStatsMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-databases-transact-sql?view=sql-server-ver15 + @property + def include_database_stats_metrics(self) -> bool: + return self.config.database_metrics_config["db_stats_metrics"]["enabled"] + @property def enabled(self): + if not self.include_database_stats_metrics: + return False return True @property @@ -42,4 +48,8 @@ def queries(self): return [DATABASE_STATS_METRICS_QUERY] def __repr__(self) -> str: - return f"{self.__class__.__name__}(" f"enabled={self.enabled}" + return ( + f"{self.__class__.__name__}(" + f"enabled={self.enabled}, " + f"include_database_stats_metrics={self.include_database_stats_metrics})" + ) diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py index 2b05f9378d37c..1411449ac5e21 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py @@ -5,7 +5,6 @@ import copy import functools -from datadog_checks.base.config import is_affirmative from datadog_checks.base.errors import ConfigurationError from .base import SqlserverDatabaseMetricsBase @@ -52,15 +51,15 @@ class SqlserverDBFragmentationMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-index-physical-stats-transact-sql?view=sql-server-ver15 @property def include_db_fragmentation_metrics(self): - return is_affirmative(self.instance_config.get('include_db_fragmentation_metrics', False)) + return self.config.database_metrics_config["db_fragmentation_metrics"]["enabled"] @property def include_db_fragmentation_metrics_tempdb(self): - return is_affirmative(self.instance_config.get('include_db_fragmentation_metrics_tempdb', False)) + return self.config.database_metrics_config["db_fragmentation_metrics"]["enabled_tempdb"] @property def db_fragmentation_object_names(self): - return self.instance_config.get('db_fragmentation_object_names', []) or [] + return self.config.db_fragmentation_object_names @property def enabled(self): @@ -68,20 +67,13 @@ def enabled(self): return False return True - @property - def _default_collection_interval(self) -> int: - ''' - Returns the default interval in seconds at which to collect database index fragmentation metrics. - ''' - return 5 * 60 # 5 minutes - @property def collection_interval(self) -> int: ''' Returns the interval in seconds at which to collect database index fragmentation metrics. Note: The index fragmentation metrics query can be expensive, so it is recommended to set a higher interval. ''' - return int(self.instance_config.get('db_fragmentation_metrics_interval', self._default_collection_interval)) + return self.config.database_metrics_config["db_fragmentation_metrics"]["collection_interval"] @property def databases(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py index 81bbf485509d3..5bd821d07b8de 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py @@ -2,7 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative from datadog_checks.sqlserver.const import ( ENGINE_EDITION_AZURE_MANAGED_INSTANCE, ) @@ -37,8 +36,8 @@ class SqlserverFciMetrics(SqlserverDatabaseMetricsBase): @property - def include_fci_metrics(self): - return is_affirmative(self.instance_config.get('include_fci_metrics', False)) + def include_fci_metrics(self) -> bool: + return self.config.database_metrics_config["fci_metrics"]["enabled"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/file_stats_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/file_stats_metrics.py index 79e0877769d7a..85b3a0d25c26d 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/file_stats_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/file_stats_metrics.py @@ -9,8 +9,14 @@ class SqlserverFileStatsMetrics(SqlserverDatabaseMetricsBase): + @property + def include_file_stats_metrics(self) -> bool: + return self.config.database_metrics_config["file_stats_metrics"]["enabled"] + @property def enabled(self): + if not self.include_file_stats_metrics: + return False if not self.major_version and not is_azure_database(self.engine_edition): return False return True diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py index 34c9b24e8d668..d91a935c19133 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py @@ -4,7 +4,6 @@ import functools -from datadog_checks.base.config import is_affirmative from datadog_checks.base.errors import ConfigurationError from .base import SqlserverDatabaseMetricsBase @@ -45,18 +44,11 @@ class SqlserverIndexUsageMetrics(SqlserverDatabaseMetricsBase): @property def include_index_usage_metrics(self) -> bool: - return is_affirmative(self.instance_config.get('include_index_usage_metrics', True)) + return self.config.database_metrics_config["index_usage_metrics"]["enabled"] @property def include_index_usage_metrics_tempdb(self) -> bool: - return is_affirmative(self.instance_config.get('include_index_usage_metrics_tempdb', False)) - - @property - def _default_collection_interval(self) -> int: - ''' - Returns the default interval in seconds at which to collect index usage metrics. - ''' - return 5 * 60 # 5 minutes + return self.config.database_metrics_config["index_usage_metrics"]["enabled_tempdb"] @property def collection_interval(self) -> int: @@ -64,7 +56,7 @@ def collection_interval(self) -> int: Returns the interval in seconds at which to collect index usage metrics. Note: The index usage metrics query can be expensive, so it is recommended to set a higher interval. ''' - return int(self.instance_config.get('index_usage_stats_interval', self._default_collection_interval)) + return self.config.database_metrics_config["index_usage_metrics"]["collection_interval"] @property def databases(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py index a3ebf5edd89ef..2dd429807c0dc 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py @@ -2,8 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase MASTER_FILES_METRICS_QUERY = { @@ -48,7 +46,7 @@ class SqlserverMasterFilesMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-master-files-transact-sql @property def include_master_files_metrics(self): - return is_affirmative(self.instance_config.get('include_master_files_metrics', False)) + return self.config.database_metrics_config["master_files_metrics"]["enabled"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py index c1dcfd8092652..0363040d455f2 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py @@ -2,8 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase OS_SCHEDULERS_METRICS_QUERY = { @@ -34,7 +32,7 @@ class SqlserverOsSchedulersMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-schedulers-transact-sql @property def include_task_scheduler_metrics(self): - return is_affirmative(self.instance_config.get('include_task_scheduler_metrics', False)) + return self.config.database_metrics_config["task_scheduler_metrics"]["enabled"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py index 036ef35e36bd8..2fa631c28e7d7 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py @@ -2,8 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase OS_TASKS_METRICS_QUERY = { @@ -30,7 +28,7 @@ class SqlserverOsTasksMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-tasks-transact-sql @property def include_task_scheduler_metrics(self): - return is_affirmative(self.instance_config.get('include_task_scheduler_metrics', False)) + return self.config.database_metrics_config["task_scheduler_metrics"]["enabled"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py index 66724c1bacb09..709c8e48f2146 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py @@ -2,9 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) - -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase QUERY_LOG_SHIPPING_PRIMARY = { @@ -29,8 +26,8 @@ class SqlserverPrimaryLogShippingMetrics(SqlserverDatabaseMetricsBase): @property - def include_primary_log_shipping_metrics(self): - return is_affirmative(self.instance_config.get('include_primary_log_shipping_metrics', False)) + def include_primary_log_shipping_metrics(self) -> bool: + return self.config.database_metrics_config["primary_log_shipping_metrics"]["enabled"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py index daa4476058b2d..4fe6fb3db106d 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py @@ -2,9 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) - -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase QUERY_LOG_SHIPPING_SECONDARY = { @@ -37,8 +34,8 @@ class SqlserverSecondaryLogShippingMetrics(SqlserverDatabaseMetricsBase): @property - def include_secondary_log_shipping_metrics(self): - return is_affirmative(self.instance_config.get('include_secondary_log_shipping_metrics', False)) + def include_secondary_log_shipping_metrics(self) -> bool: + return self.config.database_metrics_config['secondary_log_shipping_metrics']['enabled'] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/server_state_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/server_state_metrics.py index 536b67f0be4ae..7d437637e3056 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/server_state_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/server_state_metrics.py @@ -30,10 +30,17 @@ class SqlserverServerStateMetrics(SqlserverDatabaseMetricsBase): + @property + def include_server_state_metrics(self) -> bool: + return self.config.database_metrics_config['server_state_metrics']['enabled'] + @property def enabled(self): # Server state queries require VIEW SERVER STATE permissions, which some managed database # versions do not support. + + if not self.include_server_state_metrics: + return False if self.engine_edition in [ENGINE_EDITION_SQL_DATABASE]: return False return True @@ -46,6 +53,7 @@ def __repr__(self) -> str: return ( f"{self.__class__.__name__}(" f"enabled={self.enabled}, " + f"include_server_state_metrics={self.include_server_state_metrics}, " f"major_version={self.major_version}, " f"engine_edition={self.engine_edition})" ) diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py index 6d926c0c2f74f..5e18c1838679c 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py @@ -4,7 +4,6 @@ import functools -from datadog_checks.base.config import is_affirmative from datadog_checks.sqlserver.utils import is_azure_sql_database from .base import SqlserverDatabaseMetricsBase @@ -32,8 +31,8 @@ class SqlserverTempDBFileSpaceUsageMetrics(SqlserverDatabaseMetricsBase): @property - def include_tempdb_file_space_usage_metrics(self): - return is_affirmative(self.instance_config.get('include_tempdb_file_space_usage_metrics', True)) + def include_tempdb_file_space_usage_metrics(self) -> bool: + return self.config.database_metrics_config['tempdb_file_space_usage_metrics']['enabled'] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py index 5f5d57bbe938e..db99837c38b3f 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py @@ -6,6 +6,9 @@ from .base import SqlserverDatabaseMetricsBase +XE_RING_BUFFER = "ring_buffer" +XE_EVENT_FILE = "event_file" + XE_SESSION_STATUS_QUERY = { "name": "sys.dm_xe_sessions", "query": """SELECT @@ -44,8 +47,8 @@ class SQLServerXESessionMetrics(SqlserverDatabaseMetricsBase): @property def enabled(self): - self.deadlocks_config: dict = self.instance_config.get('deadlocks_collection', {}) or {} - return is_affirmative(self.instance_config.get("include_xe_metrics", False)) or is_affirmative( + self.deadlocks_config: dict = self.config.deadlocks_config + return self.config.database_metrics_config["xe_metrics"]["enabled"] or is_affirmative( self.deadlocks_config.get('enabled', False) ) diff --git a/sqlserver/datadog_checks/sqlserver/deadlocks.py b/sqlserver/datadog_checks/sqlserver/deadlocks.py index 6c6d6d0cca8c3..06debaa260d4b 100644 --- a/sqlserver/datadog_checks/sqlserver/deadlocks.py +++ b/sqlserver/datadog_checks/sqlserver/deadlocks.py @@ -11,6 +11,7 @@ from datadog_checks.base.utils.tracking import tracked_method from datadog_checks.sqlserver.config import SQLServerConfig from datadog_checks.sqlserver.const import STATIC_INFO_ENGINE_EDITION, STATIC_INFO_VERSION +from datadog_checks.sqlserver.database_metrics.xe_session_metrics import XE_EVENT_FILE, XE_RING_BUFFER from datadog_checks.sqlserver.queries import ( DEADLOCK_TIMESTAMP_ALIAS, DEADLOCK_XML_ALIAS, @@ -53,6 +54,7 @@ def __init__(self, check, config: SQLServerConfig): self.collection_interval = config.deadlocks_config.get("collection_interval", DEFAULT_COLLECTION_INTERVAL) self._force_convert_xml_to_str = False self._xe_session_name = None + self._xe_session_target = None super(Deadlocks, self).__init__( check, run_sync=True, @@ -134,14 +136,24 @@ def _set_xe_session_name(self): if not rows: raise NoXESessionError(NO_XE_SESSION_ERROR) xe_system_found = False + xe_system_xe_file_found = False for row in rows: - if (session := row[0]) in (XE_SESSION_DATADOG): + (session, target) = row + if session in (XE_SESSION_DATADOG): self._xe_session_name = session + self._xe_session_target = target return if session == XE_SESSION_SYSTEM: xe_system_found = True + if target == XE_EVENT_FILE: + xe_system_xe_file_found = True + if xe_system_found: self._xe_session_name = XE_SESSION_SYSTEM + if xe_system_xe_file_found: + self._xe_session_target = XE_EVENT_FILE + else: + self._xe_session_target = XE_RING_BUFFER return raise NoXESessionError(NO_XE_SESSION_ERROR) @@ -152,7 +164,9 @@ def _query_deadlocks(self): except NoXESessionError as e: self._log.error(str(e)) return - self._log.info(f'Using XE session {self._xe_session_name} to collect deadlocks') + self._log.info( + f'Using XE session [{self._xe_session_name}], target [{self._xe_session_target}] to collect deadlocks' + ) with self._check.connection.open_managed_default_connection(key_prefix=self._conn_key_prefix): with self._check.connection.get_managed_cursor(key_prefix=self._conn_key_prefix) as cursor: @@ -160,7 +174,9 @@ def _query_deadlocks(self): if self._force_convert_xml_to_str or self._get_connector() == "adodbapi": convert_xml_to_str = True query = get_deadlocks_query( - convert_xml_to_str=convert_xml_to_str, xe_session_name=self._xe_session_name + convert_xml_to_str=convert_xml_to_str, + xe_session_name=self._xe_session_name, + xe_target_name=self._xe_session_target, ) lookback = self._get_lookback_seconds() self._log.debug( diff --git a/sqlserver/datadog_checks/sqlserver/queries.py b/sqlserver/datadog_checks/sqlserver/queries.py index 2b6d6adf6b883..45d3f2ae0c919 100644 --- a/sqlserver/datadog_checks/sqlserver/queries.py +++ b/sqlserver/datadog_checks/sqlserver/queries.py @@ -2,6 +2,7 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from datadog_checks.sqlserver.database_metrics.xe_session_metrics import XE_RING_BUFFER DB_QUERY = """ SELECT @@ -135,22 +136,21 @@ XE_SESSION_SYSTEM = "system_health" XE_SESSIONS_QUERY = f""" SELECT - s.name AS session_name + s.name AS session_name, t.target_name AS target_name FROM sys.dm_xe_sessions s JOIN sys.dm_xe_session_targets t ON s.address = t.event_session_address WHERE - t.target_name = 'ring_buffer' - AND s.name IN ('{XE_SESSION_DATADOG}', '{XE_SESSION_SYSTEM}'); + s.name IN ('{XE_SESSION_DATADOG}', '{XE_SESSION_SYSTEM}'); """ DEADLOCK_TIMESTAMP_ALIAS = "timestamp" DEADLOCK_XML_ALIAS = "event_xml" -def get_deadlocks_query(convert_xml_to_str=False, xe_session_name="datadog"): +def get_deadlocks_query(convert_xml_to_str=False, xe_session_name=XE_SESSION_DATADOG, xe_target_name=XE_RING_BUFFER): """ Construct the query to fetch deadlocks from the system_health extended event session :param convert_xml_to_str: Whether to convert the XML to a string. This option is for MSOLEDB drivers @@ -161,16 +161,27 @@ def get_deadlocks_query(convert_xml_to_str=False, xe_session_name="datadog"): if convert_xml_to_str: xml_expression = "CAST(xdr.query('.') AS NVARCHAR(MAX))" - return f""" - SELECT TOP(?) xdr.value('@timestamp', 'datetime') AS [{DEADLOCK_TIMESTAMP_ALIAS}], - {xml_expression} AS [{DEADLOCK_XML_ALIAS}] + if xe_target_name == XE_RING_BUFFER: + return f"""SELECT TOP(?) xdr.value('@timestamp', 'datetime') AS [{DEADLOCK_TIMESTAMP_ALIAS}], + {xml_expression} AS [{DEADLOCK_XML_ALIAS}] FROM (SELECT CAST([target_data] AS XML) AS Target_Data FROM sys.dm_xe_session_targets AS xt INNER JOIN sys.dm_xe_sessions AS xs ON xs.address = xt.event_session_address WHERE xs.name = N'{xe_session_name}' - AND xt.target_name = N'ring_buffer' + AND xt.target_name = N'{XE_RING_BUFFER}' ) AS XML_Data CROSS APPLY Target_Data.nodes('RingBufferTarget/event[@name="xml_deadlock_report"]') AS XEventData(xdr) WHERE xdr.value('@timestamp', 'datetime') >= DATEADD(SECOND, ?, TODATETIMEOFFSET(GETDATE(), DATEPART(TZOFFSET, SYSDATETIMEOFFSET())) AT TIME ZONE 'UTC') ;""" + + return f"""SELECT TOP(?) +event_data AS [{DEADLOCK_XML_ALIAS}], +CONVERT(xml, event_data).value('(event[@name="xml_deadlock_report"]/@timestamp)[1]','datetime') + AS [{DEADLOCK_TIMESTAMP_ALIAS}] +FROM +sys.fn_xe_file_target_read_file +('system_health*.xel', null, null, null) +WHERE object_name like 'xml_deadlock_report' + and CONVERT(xml, event_data).value('(event[@name="xml_deadlock_report"]/@timestamp)[1]','datetime') + >= DATEADD(SECOND, ?, TODATETIMEOFFSET(GETDATE(), DATEPART(TZOFFSET, SYSDATETIMEOFFSET())) AT TIME ZONE 'UTC');""" diff --git a/sqlserver/datadog_checks/sqlserver/sqlserver.py b/sqlserver/datadog_checks/sqlserver/sqlserver.py index 6016f5fa6f95f..67189e3145c4d 100644 --- a/sqlserver/datadog_checks/sqlserver/sqlserver.py +++ b/sqlserver/datadog_checks/sqlserver/sqlserver.py @@ -757,7 +757,6 @@ def check(self, _): def _new_database_metric_executor(self, database_metric_class, db_names=None): return database_metric_class( config=self._config, - instance_config=self.instance, new_query_executor=self._new_query_executor, server_static_info=self.static_info_cache, execute_query_handler=self.execute_query_raw, diff --git a/sqlserver/tests/conftest.py b/sqlserver/tests/conftest.py index b0985ea6a8414..bfe6dd7fdf71f 100644 --- a/sqlserver/tests/conftest.py +++ b/sqlserver/tests/conftest.py @@ -99,11 +99,23 @@ def instance_minimal_defaults(): def instance_docker(instance_docker_defaults): instance_docker_defaults.update( { - 'include_task_scheduler_metrics': True, - 'include_db_fragmentation_metrics': True, - 'include_fci_metrics': True, - 'include_ao_metrics': False, - 'include_master_files_metrics': True, + 'database_metrics': { + 'ao_metrics': { + 'enabled': False, + }, + 'task_scheduler_metrics': { + 'enabled': True, + }, + 'db_fragmentation_metrics': { + 'enabled': True, + }, + 'fci_metrics': { + 'enabled': True, + }, + 'master_files_metrics': { + 'enabled': True, + }, + }, 'disable_generic_tags': True, } ) @@ -233,21 +245,21 @@ def instance_e2e(instance_docker): @pytest.fixture def instance_ao_docker_primary(instance_docker): - instance_docker['include_ao_metrics'] = True + instance_docker['database_metrics']['ao_metrics']['enabled'] = True return instance_docker @pytest.fixture def instance_ao_docker_primary_local_only(instance_ao_docker_primary): instance = deepcopy(instance_ao_docker_primary) - instance['only_emit_local'] = True + instance['database_metrics']['ao_metrics']['only_emit_local'] = True return instance @pytest.fixture def instance_ao_docker_primary_non_existing_ag(instance_ao_docker_primary): instance = deepcopy(instance_ao_docker_primary) - instance['availability_group'] = 'AG2' + instance['database_metrics']['ao_metrics']['availability_group'] = 'AG2' return instance diff --git a/sqlserver/tests/test_activity.py b/sqlserver/tests/test_activity.py index 7f4f84f027abb..7bf91d8fd7159 100644 --- a/sqlserver/tests/test_activity.py +++ b/sqlserver/tests/test_activity.py @@ -57,6 +57,7 @@ def dbm_instance(instance_docker): return copy(instance_docker) +@pytest.mark.flaky @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') @pytest.mark.parametrize("use_autocommit", [True, False]) diff --git a/sqlserver/tests/test_database_metrics.py b/sqlserver/tests/test_database_metrics.py index 0330ebbe36a1e..7203196159b4b 100644 --- a/sqlserver/tests/test_database_metrics.py +++ b/sqlserver/tests/test_database_metrics.py @@ -52,13 +52,18 @@ @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') +@pytest.mark.parametrize('include_file_stats_metrics', [True, False]) def test_sqlserver_file_stats_metrics( aggregator, dd_run_check, init_config, instance_docker_metrics, + include_file_stats_metrics, ): instance_docker_metrics['database_autodiscovery'] = True + instance_docker_metrics['database_metrics'] = { + 'file_stats_metrics': {'enabled': include_file_stats_metrics}, + } mocked_results = [ ('master', 'ONLINE', 'master', '/xx/master.mdf', 89, 0, 0, 73, 16, 3153920, 933888, 59, 98, 4194304), @@ -78,7 +83,6 @@ def execute_query_handler_mocked(query, db=None): file_stats_metrics = SqlserverFileStatsMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -88,18 +92,21 @@ def execute_query_handler_mocked(query, db=None): dd_run_check(sqlserver_check) - tags = sqlserver_check._config.tags - for result in mocked_results: - db, state, logical_name, file_location, *metric_values = result - metrics = zip(file_stats_metrics.metric_names()[0], metric_values) - expected_tags = [ - f'db:{db}', - f'state:{state}', - f'logical_name:{logical_name}', - f'file_location:{file_location}', - ] + tags - for metric_name, metric_value in metrics: - aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) + if not include_file_stats_metrics: + assert file_stats_metrics.enabled is False + else: + tags = sqlserver_check._config.tags + for result in mocked_results: + db, state, logical_name, file_location, *metric_values = result + metrics = zip(file_stats_metrics.metric_names()[0], metric_values) + expected_tags = [ + f'db:{db}', + f'state:{state}', + f'logical_name:{logical_name}', + f'file_location:{file_location}', + ] + tags + for metric_name, metric_value in metrics: + aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) @pytest.mark.integration @@ -113,7 +120,9 @@ def test_sqlserver_ao_metrics( include_ao_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_ao_metrics'] = include_ao_metrics + instance_docker_metrics['database_metrics'] = { + 'ao_metrics': {'enabled': include_ao_metrics}, + } # Mocked results mocked_ao_availability_groups = [ @@ -155,7 +164,6 @@ def test_sqlserver_ao_metrics( ao_metrics = SqlserverAoMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -254,9 +262,11 @@ def test_sqlserver_availability_groups_metrics( mocked_results, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_ao_metrics'] = include_ao_metrics + instance_docker_metrics['database_metrics'] = { + 'ao_metrics': {'enabled': include_ao_metrics}, + } if availability_group: - instance_docker_metrics['availability_group'] = availability_group + instance_docker_metrics['database_metrics']['ao_metrics']['availability_group'] = availability_group sqlserver_check = SQLServer(CHECK_NAME, init_config, [instance_docker_metrics]) @@ -265,7 +275,6 @@ def execute_query_handler_mocked(query, db=None): availability_groups_metrics = SqlserverAvailabilityGroupsMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -342,11 +351,13 @@ def test_sqlserver_database_replication_stats_metrics( mocked_results, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_ao_metrics'] = include_ao_metrics + instance_docker_metrics['database_metrics'] = { + 'ao_metrics': {'enabled': include_ao_metrics}, + } if availability_group: - instance_docker_metrics['availability_group'] = availability_group + instance_docker_metrics['database_metrics']['ao_metrics']['availability_group'] = availability_group if only_emit_local: - instance_docker_metrics['only_emit_local'] = only_emit_local + instance_docker_metrics['database_metrics']['ao_metrics']['only_emit_local'] = only_emit_local sqlserver_check = SQLServer(CHECK_NAME, init_config, [instance_docker_metrics]) @@ -355,7 +366,6 @@ def execute_query_handler_mocked(query, db=None): database_replication_stats_metrics = SqlserverDatabaseReplicationStatsMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -475,13 +485,15 @@ def test_sqlserver_availability_replicas_metrics( mocked_results, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_ao_metrics'] = include_ao_metrics + instance_docker_metrics['database_metrics'] = { + 'ao_metrics': {'enabled': include_ao_metrics}, + } if availability_group: - instance_docker_metrics['availability_group'] = availability_group + instance_docker_metrics['database_metrics']['ao_metrics']['availability_group'] = availability_group if only_emit_local: - instance_docker_metrics['only_emit_local'] = only_emit_local + instance_docker_metrics['database_metrics']['ao_metrics']['only_emit_local'] = only_emit_local if ao_database: - instance_docker_metrics['ao_database'] = ao_database + instance_docker_metrics['database_metrics']['ao_metrics']['ao_database'] = ao_database sqlserver_check = SQLServer(CHECK_NAME, init_config, [instance_docker_metrics]) @@ -490,7 +502,6 @@ def execute_query_handler_mocked(query, db=None): availability_replicas_metrics = SqlserverAvailabilityReplicasMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -548,7 +559,9 @@ def test_sqlserver_fci_metrics( include_fci_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_fci_metrics'] = include_fci_metrics + instance_docker_metrics['database_metrics'] = { + 'fci_metrics': {'enabled': include_fci_metrics}, + } mocked_results = [ ('node1', 'up', 'cluster1', 0, 1), @@ -561,7 +574,6 @@ def execute_query_handler_mocked(query, db=None): fci_metrics = SqlserverFciMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -598,8 +610,9 @@ def test_sqlserver_primary_log_shipping_metrics( include_primary_log_shipping_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_primary_log_shipping_metrics'] = include_primary_log_shipping_metrics - + instance_docker_metrics['database_metrics'] = { + 'primary_log_shipping_metrics': {'enabled': include_primary_log_shipping_metrics}, + } mocked_results = [('97E29D89-2FA0-44FF-9EF7-65DA75FE0E3E', 'EC2AMAZ-Q0NCNV5', 'MyDummyDB', 500, 3600)] sqlserver_check = SQLServer(CHECK_NAME, init_config, [instance_docker_metrics]) @@ -609,7 +622,6 @@ def execute_query_handler_mocked(query, db=None): primary_log_shipping_metrics = SqlserverPrimaryLogShippingMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -646,8 +658,9 @@ def test_sqlserver_secondary_log_shipping_metrics( include_secondary_log_shipping_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_secondary_log_shipping_metrics'] = include_secondary_log_shipping_metrics - + instance_docker_metrics['database_metrics'] = { + 'secondary_log_shipping_metrics': {'enabled': include_secondary_log_shipping_metrics}, + } mocked_results = [ ( r'EC2AMAZ-Q0NCNV5\MYSECONDARY', @@ -669,7 +682,6 @@ def execute_query_handler_mocked(query, db=None): primary_log_shipping_metrics = SqlserverSecondaryLogShippingMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -699,13 +711,14 @@ def execute_query_handler_mocked(query, db=None): @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') +@pytest.mark.parametrize('include_server_state_metrics', [True, False]) def test_sqlserver_server_state_metrics( - aggregator, - dd_run_check, - init_config, - instance_docker_metrics, + aggregator, dd_run_check, init_config, instance_docker_metrics, include_server_state_metrics ): instance_docker_metrics['database_autodiscovery'] = True + instance_docker_metrics['database_metrics'] = { + 'server_state_metrics': {'enabled': include_server_state_metrics}, + } mocked_results = [(1000, 4, 8589934592, 17179869184, 4294967296, 8589934592)] @@ -716,7 +729,6 @@ def execute_query_handler_mocked(query, db=None): server_state_metrics = SqlserverServerStateMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -726,11 +738,14 @@ def execute_query_handler_mocked(query, db=None): dd_run_check(sqlserver_check) - tags = sqlserver_check._config.tags - for result in mocked_results: - metrics = zip(server_state_metrics.metric_names()[0], result) - for metric_name, metric_value in metrics: - aggregator.assert_metric(metric_name, value=metric_value, tags=tags) + if not include_server_state_metrics: + assert server_state_metrics.enabled is False + else: + tags = sqlserver_check._config.tags + for result in mocked_results: + metrics = zip(server_state_metrics.metric_names()[0], result) + for metric_name, metric_value in metrics: + aggregator.assert_metric(metric_name, value=metric_value, tags=tags) @pytest.mark.integration @@ -744,8 +759,9 @@ def test_sqlserver_tempdb_file_space_usage_metrics( include_tempdb_file_space_usage_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_tempdb_file_space_usage_metrics'] = include_tempdb_file_space_usage_metrics - + instance_docker_metrics['database_metrics'] = { + 'tempdb_file_space_usage_metrics': {'enabled': include_tempdb_file_space_usage_metrics} + } mocked_results = [ [(2, Decimal('5.375000'), Decimal('0.000000'), Decimal('0.000000'), Decimal('1.312500'), Decimal('1.312500'))] ] @@ -757,7 +773,6 @@ def execute_query_handler_mocked(query, db=None): tempdb_file_space_usage_metrics = SqlserverTempDBFileSpaceUsageMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -798,10 +813,16 @@ def test_sqlserver_index_usage_metrics( index_usage_stats_interval, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_index_usage_metrics'] = include_index_usage_metrics - instance_docker_metrics['include_index_usage_metrics_tempdb'] = include_index_usage_metrics_tempdb + instance_docker_metrics['database_metrics'] = { + 'index_usage_metrics': { + 'enabled': include_index_usage_metrics, + 'enabled_tempdb': include_index_usage_metrics_tempdb, + }, + } if index_usage_stats_interval: - instance_docker_metrics['index_usage_stats_interval'] = index_usage_stats_interval + instance_docker_metrics['database_metrics']['index_usage_metrics'][ + 'collection_interval' + ] = index_usage_stats_interval mocked_results_non_tempdb = [ [ @@ -830,14 +851,13 @@ def test_sqlserver_index_usage_metrics( index_usage_metrics = SqlserverIndexUsageMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, databases=AUTODISCOVERY_DBS + ['tempdb'], ) - expected_collection_interval = index_usage_stats_interval or index_usage_metrics._default_collection_interval + expected_collection_interval = index_usage_stats_interval or index_usage_metrics.collection_interval assert index_usage_metrics.queries[0]['collection_interval'] == expected_collection_interval sqlserver_check._database_metrics = [index_usage_metrics] @@ -888,11 +908,17 @@ def test_sqlserver_db_fragmentation_metrics( db_fragmentation_metrics_interval, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_db_fragmentation_metrics'] = include_db_fragmentation_metrics - instance_docker_metrics['include_db_fragmentation_metrics_tempdb'] = include_db_fragmentation_metrics_tempdb + instance_docker_metrics['database_metrics'] = { + 'db_fragmentation_metrics': { + 'enabled': include_db_fragmentation_metrics, + 'enabled_tempdb': include_db_fragmentation_metrics_tempdb, + }, + } if db_fragmentation_metrics_interval: - instance_docker_metrics['db_fragmentation_metrics_interval'] = db_fragmentation_metrics_interval - + instance_docker_metrics['database_metrics']['db_fragmentation_metrics'][ + 'collection_interval' + ] = db_fragmentation_metrics_interval + print(instance_docker_metrics) mocked_results = [ [ ('master', 'spt_fallback_db', 0, None, 0, 0.0, 0, 0.0), @@ -934,7 +960,6 @@ def test_sqlserver_db_fragmentation_metrics( db_fragmentation_metrics = SqlserverDBFragmentationMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -944,9 +969,7 @@ def test_sqlserver_db_fragmentation_metrics( if db_fragmentation_object_names: assert db_fragmentation_metrics.db_fragmentation_object_names == db_fragmentation_object_names - expected_collection_interval = ( - db_fragmentation_metrics_interval or db_fragmentation_metrics._default_collection_interval - ) + expected_collection_interval = db_fragmentation_metrics_interval or db_fragmentation_metrics.collection_interval assert db_fragmentation_metrics.queries[0]['collection_interval'] == expected_collection_interval sqlserver_check._database_metrics = [db_fragmentation_metrics] @@ -995,7 +1018,9 @@ def test_sqlserver_os_schedulers_metrics( include_task_scheduler_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_task_scheduler_metrics'] = include_task_scheduler_metrics + instance_docker_metrics['database_metrics'] = { + 'task_scheduler_metrics': {'enabled': include_task_scheduler_metrics}, + } mocked_results = [ (0, 0, 4, 6, 4, 0, 0), @@ -1024,7 +1049,6 @@ def execute_query_handler_mocked(query, db=None): os_schedulers_metrics = SqlserverOsSchedulersMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -1060,8 +1084,9 @@ def test_sqlserver_os_tasks_metrics( include_task_scheduler_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_task_scheduler_metrics'] = include_task_scheduler_metrics - + instance_docker_metrics['database_metrics'] = { + 'task_scheduler_metrics': {'enabled': include_task_scheduler_metrics}, + } mocked_results = [ (0, 40, 0, 0, 0), (9, 46, 0, 0, 0), @@ -1089,7 +1114,6 @@ def execute_query_handler_mocked(query, db=None): os_tasks_metrics = SqlserverOsTasksMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -1124,8 +1148,9 @@ def test_sqlserver_master_files_metrics( include_master_files_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_master_files_metrics'] = include_master_files_metrics - + instance_docker_metrics['database_metrics'] = { + 'master_files_metrics': {'enabled': include_master_files_metrics}, + } mocked_results = [ ('master', 'master', 1, 'data', '/var/opt/mssql/data/master.mdf', 'ONLINE', 4096, 0), ('master', 'master', 2, 'transaction_log', '/var/opt/mssql/data/mastlog.ldf', 'ONLINE', 512, 0), @@ -1155,7 +1180,6 @@ def execute_query_handler_mocked(query, db=None): master_files_metrics = SqlserverMasterFilesMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -1187,13 +1211,18 @@ def execute_query_handler_mocked(query, db=None): @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') +@pytest.mark.parametrize('include_database_files_metrics', [True, False]) def test_sqlserver_database_files_metrics( aggregator, dd_run_check, init_config, instance_docker_metrics, + include_database_files_metrics, ): instance_docker_metrics['database_autodiscovery'] = True + instance_docker_metrics['database_metrics'] = { + 'db_files_metrics': {'enabled': include_database_files_metrics}, + } mocked_results = [ [ @@ -1226,7 +1255,6 @@ def test_sqlserver_database_files_metrics( database_files_metrics = SqlserverDatabaseFilesMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -1237,35 +1265,43 @@ def test_sqlserver_database_files_metrics( dd_run_check(sqlserver_check) - tags = sqlserver_check._config.tags - for db, result in zip(AUTODISCOVERY_DBS, mocked_results): - for row in result: - file_id, file_type, file_location, file_name, database_files_state_desc, size, space_used, state = row - size *= 8 # size is in pages, 1 page = 8 KB - space_used *= 8 # space_used is in pages, 1 page = 8 KB - metrics = zip(database_files_metrics.metric_names()[0], [state, size, space_used]) - expected_tags = [ - f'db:{db}', - f'database:{db}', - f'file_id:{file_id}', - f'file_type:{file_type}', - f'file_location:{file_location}', - f'file_name:{file_name}', - f'database_files_state_desc:{database_files_state_desc}', - ] + tags - for metric_name, metric_value in metrics: - aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) + if not include_database_files_metrics: + assert database_files_metrics.enabled is False + else: + tags = sqlserver_check._config.tags + for db, result in zip(AUTODISCOVERY_DBS, mocked_results): + for row in result: + file_id, file_type, file_location, file_name, database_files_state_desc, size, space_used, state = row + size *= 8 # size is in pages, 1 page = 8 KB + space_used *= 8 # space_used is in pages, 1 page = 8 KB + metrics = zip(database_files_metrics.metric_names()[0], [state, size, space_used]) + expected_tags = [ + f'db:{db}', + f'database:{db}', + f'file_id:{file_id}', + f'file_type:{file_type}', + f'file_location:{file_location}', + f'file_name:{file_name}', + f'database_files_state_desc:{database_files_state_desc}', + ] + tags + for metric_name, metric_value in metrics: + aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') +@pytest.mark.parametrize('include_database_files_metrics', [True, False]) def test_sqlserver_database_stats_metrics( aggregator, dd_run_check, init_config, instance_docker_metrics, + include_database_files_metrics, ): instance_docker_metrics['database_autodiscovery'] = True + instance_docker_metrics['database_metrics'] = { + 'db_stats_metrics': {'enabled': include_database_files_metrics}, + } mocked_results = [ ('master', 'master', 'ONLINE', 'SIMPLE', 0, False, False, False), @@ -1282,7 +1318,6 @@ def execute_query_handler_mocked(query, db=None): database_stats_metrics = SqlserverDatabaseStatsMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -1292,33 +1327,43 @@ def execute_query_handler_mocked(query, db=None): dd_run_check(sqlserver_check) - tags = sqlserver_check._config.tags - for result in mocked_results: - db, database, database_state_desc, database_recovery_model_desc, *metric_values = result - metrics = zip(database_stats_metrics.metric_names()[0], metric_values) - expected_tags = [ - f'db:{db}', - f'database:{database}', - f'database_state_desc:{database_state_desc}', - f'database_recovery_model_desc:{database_recovery_model_desc}', - ] + tags - for metric_name, metric_value in metrics: - aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) + if not include_database_files_metrics: + assert database_stats_metrics.enabled is False + else: + tags = sqlserver_check._config.tags + for result in mocked_results: + db, database, database_state_desc, database_recovery_model_desc, *metric_values = result + metrics = zip(database_stats_metrics.metric_names()[0], metric_values) + expected_tags = [ + f'db:{db}', + f'database:{database}', + f'database_state_desc:{database_state_desc}', + f'database_recovery_model_desc:{database_recovery_model_desc}', + ] + tags + for metric_name, metric_value in metrics: + aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') @pytest.mark.parametrize('database_backup_metrics_interval', [None, 600]) +@pytest.mark.parametrize('include_database_backup_metrics', [True, False]) def test_sqlserver_database_backup_metrics( aggregator, dd_run_check, init_config, instance_docker_metrics, database_backup_metrics_interval, + include_database_backup_metrics, ): instance_docker_metrics['database_autodiscovery'] = True + instance_docker_metrics['database_metrics'] = { + 'db_backup_metrics': {'enabled': include_database_backup_metrics}, + } if database_backup_metrics_interval: - instance_docker_metrics['database_backup_metrics_interval'] = database_backup_metrics_interval + instance_docker_metrics['database_metrics']['db_backup_metrics'][ + 'collection_interval' + ] = database_backup_metrics_interval mocked_results = [ ('master', 'master', 0), @@ -1335,36 +1380,36 @@ def execute_query_handler_mocked(query, db=None): database_backup_metrics = SqlserverDatabaseBackupMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, ) - expected_collection_interval = ( - database_backup_metrics_interval or database_backup_metrics._default_collection_interval - ) + expected_collection_interval = database_backup_metrics_interval or database_backup_metrics.collection_interval assert database_backup_metrics.queries[0]['collection_interval'] == expected_collection_interval sqlserver_check._database_metrics = [database_backup_metrics] dd_run_check(sqlserver_check) - tags = sqlserver_check._config.tags - for result in mocked_results: - db, database, *metric_values = result - metrics = zip(database_backup_metrics.metric_names()[0], metric_values) - expected_tags = [ - f'db:{db}', - f'database:{database}', - ] + tags - for metric_name, metric_value in metrics: - aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) - - # database_backup_metrics should not be collected because the collection interval is not reached - aggregator.reset() - dd_run_check(sqlserver_check) - for metric_name in database_backup_metrics.metric_names()[0]: - aggregator.assert_metric(metric_name, count=0) + if not include_database_backup_metrics: + assert database_backup_metrics.enabled is False + else: + tags = sqlserver_check._config.tags + for result in mocked_results: + db, database, *metric_values = result + metrics = zip(database_backup_metrics.metric_names()[0], metric_values) + expected_tags = [ + f'db:{db}', + f'database:{database}', + ] + tags + for metric_name, metric_value in metrics: + aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) + + # database_backup_metrics should not be collected because the collection interval is not reached + aggregator.reset() + dd_run_check(sqlserver_check) + for metric_name in database_backup_metrics.metric_names()[0]: + aggregator.assert_metric(metric_name, count=0) @pytest.mark.integration @@ -1391,3 +1436,46 @@ def test_sqlserver_xe_session_metrics( expected_tags = sqlserver_check._config.tags expected_tags.append('session_name:datadog') aggregator.assert_metric("sqlserver.xe.session_status", value=1, tags=expected_tags) + + +@pytest.mark.integration +@pytest.mark.usefixtures('dd_environment') +def test_sqlserver_database_metrics_defaults( + aggregator, + dd_run_check, + init_config, + instance_docker_metrics, +): + include_defaults = { + SqlserverAoMetrics: False, + SqlserverAvailabilityGroupsMetrics: False, + SqlserverAvailabilityReplicasMetrics: False, + SqlserverDatabaseBackupMetrics: True, + SqlserverDatabaseFilesMetrics: True, + SqlserverDatabaseReplicationStatsMetrics: False, + SqlserverDatabaseStatsMetrics: True, + SqlserverDBFragmentationMetrics: False, + SqlserverFciMetrics: False, + SqlserverFileStatsMetrics: True, + SqlserverIndexUsageMetrics: True, + SqlserverMasterFilesMetrics: False, + SqlserverOsSchedulersMetrics: False, + SqlserverOsTasksMetrics: False, + SqlserverPrimaryLogShippingMetrics: False, + SqlserverSecondaryLogShippingMetrics: False, + SqlserverServerStateMetrics: True, + SqlserverTempDBFileSpaceUsageMetrics: True, + } + instance_docker_metrics['database_autodiscovery'] = True + + sqlserver_check = SQLServer(CHECK_NAME, init_config, [instance_docker_metrics]) + + for metric, enabled in include_defaults.items(): + database_metrics = metric( + config=sqlserver_check._config, + new_query_executor=sqlserver_check._new_query_executor, + server_static_info=STATIC_SERVER_INFO, + execute_query_handler=None, + databases=AUTODISCOVERY_DBS, + ) + assert database_metrics.enabled == enabled diff --git a/sqlserver/tests/test_deadlocks.py b/sqlserver/tests/test_deadlocks.py index ddbcb8fc317e2..1b1bb5b666311 100644 --- a/sqlserver/tests/test_deadlocks.py +++ b/sqlserver/tests/test_deadlocks.py @@ -16,13 +16,18 @@ from mock import patch from datadog_checks.sqlserver import SQLServer +from datadog_checks.sqlserver.database_metrics.xe_session_metrics import XE_EVENT_FILE, XE_RING_BUFFER from datadog_checks.sqlserver.deadlocks import ( PAYLOAD_QUERY_SIGNATURE, PAYLOAD_TIMESTAMP, - XE_SESSION_DATADOG, Deadlocks, ) -from datadog_checks.sqlserver.queries import DEADLOCK_TIMESTAMP_ALIAS, DEADLOCK_XML_ALIAS +from datadog_checks.sqlserver.queries import ( + DEADLOCK_TIMESTAMP_ALIAS, + DEADLOCK_XML_ALIAS, + XE_SESSION_DATADOG, + XE_SESSION_SYSTEM, +) from .common import CHECK_NAME @@ -137,9 +142,18 @@ def _create_deadlock(dd_environment, dbm_instance): @pytest.mark.usefixtures('dd_environment') @pytest.mark.usefixtures('_create_deadlock') @pytest.mark.parametrize("convert_xml_to_str", [False, True]) -def test_deadlocks(aggregator, dd_run_check, dbm_instance, convert_xml_to_str): +@pytest.mark.parametrize( + "xe_session_name, xe_session_target", + [ + [XE_SESSION_DATADOG, XE_RING_BUFFER], + [XE_SESSION_SYSTEM, XE_EVENT_FILE], + ], +) +def test_deadlocks(aggregator, dd_run_check, dbm_instance, convert_xml_to_str, xe_session_name, xe_session_target): check = SQLServer(CHECK_NAME, {}, [dbm_instance]) check.deadlocks._force_convert_xml_to_str = convert_xml_to_str + check.deadlocks._xe_session_name = xe_session_name + check.deadlocks._xe_session_target = xe_session_target dbm_instance['dbm_enabled'] = True deadlock_payloads = _run_check_and_get_deadlock_payloads(dd_run_check, check, aggregator) diff --git a/streamnative/CHANGELOG.md b/streamnative/CHANGELOG.md new file mode 100644 index 0000000000000..acabc5348fcaf --- /dev/null +++ b/streamnative/CHANGELOG.md @@ -0,0 +1,7 @@ +# CHANGELOG - StreamNative + +## 1.0.0 / 2024-10-01 + +***Added***: + +* Initial Release diff --git a/streamnative/README.md b/streamnative/README.md new file mode 100644 index 0000000000000..ab65f9b40ca2b --- /dev/null +++ b/streamnative/README.md @@ -0,0 +1,72 @@ +# StreamNative + +## Overview + +[StreamNative][1] provides an enterprise-grade messaging and event streaming platform built on Apache Pulsar. It offers scalable, real-time data streaming solutions with features like multi-tenancy, geo-replication, and seamless integration with cloud services. + +The StreamNative integration collects the following types of [metrics][2]: + +1. Health +2. Pulsar Resource +3. Source Connector +4. Sink Connector +5. Kafka Connect + +## Setup + +### Configuration + +#### Get StreamNative credentials + +Log into the [StreamNative Cloud Console Account][3]. +##### Get the `Organization ID` and `Instance Name`: + +1. Click the profile icon and select **Organizations**. +2. Choose the **Organization** for which data needs to be collected. +3. From the **Select an Instance** dropdown, note the **Instance Name**. + +##### Get the `Client ID` and `Client Secret`: + +1. Click the profile icon and navigate to the **Accounts & Accesses** tab. +2. Find the Service Account with **Admin** permissions set to **Enabled**. + - If no Service Account exists, select **New -> Service Account** to create one, and make sure to enable the **Super Admin** option. +3. On the right side of the chosen Service Account, click the `...` button. +4. Select **Download OAuth2 Key** to obtain the **Client ID** and **Client Secret**. + + +#### Add StreamNative credentials + +- Organization ID +- Instance Name +- Client ID +- Client Secret + + +## Data Collected + +### Logs + +The StreamNative integration does not include any logs. + +### Metrics + +The StreamNative integration collects and forwards the following metrics to Datadog. + +{{< get-metrics-from-git "streamnative" >}} + +### Service Checks + +The StreamNative integration does not include any service checks. + +### Events + +The StreamNative integration does not include any events. + +## Troubleshooting + +Need help? Contact [Datadog support][4]. + +[1]: https://streamnative.io/ +[2]: https://docs.streamnative.io/docs/cloud-metrics-api#metrics-endpoint +[3]: https://console.streamnative.cloud/ +[4]: https://docs.datadoghq.com/help/ \ No newline at end of file diff --git a/streamnative/assets/dashboards/streamnative_health.json b/streamnative/assets/dashboards/streamnative_health.json new file mode 100644 index 0000000000000..9859e94c4d3b0 --- /dev/null +++ b/streamnative/assets/dashboards/streamnative_health.json @@ -0,0 +1,604 @@ +{ + "title": "StreamNative - Health", + "description": "", + "widgets": [ + { + "id": 5694301138076536, + "definition": { + "type": "image", + "url": "https://cdn.prod.website-files.com/638a1dc72083d166ed6e3d76/63926c17a52727a15e13decd_Logo-streamnative-150px.svg", + "url_dark_theme": "https://cdn.prod.website-files.com/638a1dc72083d166ed6e3d76/66503b265696d89c26062701_Group%2021.svg", + "sizing": "contain", + "margin": "md", + "has_background": false, + "has_border": false, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 7, + "height": 3 + } + }, + { + "id": 4734527193949666, + "definition": { + "title": "Monitor Summary", + "type": "manage_status", + "display_format": "countsAndList", + "color_preference": "background", + "hide_zero_counts": true, + "show_status": true, + "last_triggered_format": "relative", + "query": "tag:streamnative", + "sort": "status,asc", + "count": 50, + "start": 0, + "summary_type": "monitors", + "show_priority": false, + "show_last_triggered": false + }, + "layout": { + "x": 7, + "y": 0, + "width": 5, + "height": 5 + } + }, + { + "id": 6669352432630000, + "definition": { + "type": "note", + "content": "**[StreamNative](https://www.streamnative.com/)** provides an enterprise-grade messaging platform built on Apache Pulsar, offering scalable real-time data streaming with multi-tenancy, geo-replication, and cloud integration.\n\nThis Health dashboard tracks messaging latency (end-to-end, publish, cross-cluster) and service availability. \n\nFor more information, see the [StreamNative Integration Documentation](https://docs.datadoghq.com/integrations/streamnative/).", + "background_color": "orange", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "top", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 3, + "width": 4, + "height": 2 + } + }, + { + "id": 5437379294232914, + "definition": { + "type": "note", + "content": "**Note**:\n- The dashboard widgets display the latest metric values captured based on health statistics.\n\n**Tip**:\n- Clone this dashboard to rearrange, modify, and add widgets and visualizations.", + "background_color": "yellow", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "top", + "has_padding": true + }, + "layout": { + "x": 4, + "y": 3, + "width": 3, + "height": 2 + } + }, + { + "id": 1229081970979708, + "definition": { + "title": "End to End Latency over time (Median)", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "End To End Latency", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:streamnative.health.pulsar_detector_e2e_latency_ms{$Organization,$Instance,$Namespace,$Cluster,quantile:0.5} by {cloud_streamnative_io_cluster}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 5, + "width": 12, + "height": 3 + } + }, + { + "id": 4014832880981910, + "definition": { + "title": "Publish Latency over time (Median)", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Publish Latency", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.health.pulsar_detector_publish_latency_ms{$Organization,$Cluster,$Instance,$Namespace,quantile:0.5} by {cloud_streamnative_io_cluster}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 8, + "width": 12, + "height": 3 + } + }, + { + "id": 762054180789370, + "definition": { + "title": "Average End To End Latency by Quantile", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "End To End latency", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.health.pulsar_detector_e2e_latency_ms{$Organization,$Instance,$Namespace,$Cluster} by {quantile}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 11, + "width": 12, + "height": 4 + } + }, + { + "id": 8630166971873926, + "definition": { + "title": "Average Publish Latency by Quantile", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Publish Latency", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.health.pulsar_detector_publish_latency_ms{$Organization,$Instance,$Namespace,$Cluster} by {quantile}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 4, + "is_column_break": true + } + }, + { + "id": 1445816428377812, + "definition": { + "title": "Service Status", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.health.pulsar_detector_pulsar_sla_webservice_up{$Organization,$Instance,$Namespace,$Cluster} by {cloud_streamnative_io_organization_name,cloud_streamnative_io_pulsar_instance,cloud_streamnative_io_cluster}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:streamnative.health.pulsar_detector_pulsar_sla_messaging_up{$Organization,$Instance,$Namespace,$Cluster} by {cloud_streamnative_io_organization_name,cloud_streamnative_io_pulsar_instance,cloud_streamnative_io_cluster}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "conditional_formats": [ + { + "comparator": "=", + "value": 1, + "palette": "white_on_green" + }, + { + "comparator": "<", + "value": 1, + "palette": "black_on_light_red" + } + ], + "cell_display_mode": "number", + "alias": "Webservice Status", + "formula": "query1" + }, + { + "conditional_formats": [ + { + "comparator": "=", + "value": 1, + "palette": "white_on_green" + }, + { + "comparator": "<", + "value": 1, + "palette": "black_on_light_red" + } + ], + "cell_display_mode": "number", + "alias": "Messaging Service Status", + "formula": "query2" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 4, + "width": 12, + "height": 3 + } + }, + { + "id": 5425815181046796, + "definition": { + "title": "Service Status over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Webservice", + "style": { + "palette": "dd20", + "palette_index": 2 + }, + "formula": "count_nonzero(query1)" + }, + { + "alias": "Messaging Service", + "style": { + "palette": "classic", + "palette_index": 1 + }, + "formula": "count_nonzero(query2)" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.health.pulsar_detector_pulsar_sla_webservice_up{$Organization,$Instance,$Namespace,$Cluster}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:streamnative.health.pulsar_detector_pulsar_sla_messaging_up{$Organization,$Instance,$Namespace,$Cluster}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 7, + "width": 12, + "height": 4 + } + }, + { + "id": 1936496724376416, + "definition": { + "title": "Overall Summary", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.health.pulsar_detector_e2e_latency_ms{$Organization,$Cluster,$Instance,$Namespace,quantile:0.5} by {cloud_streamnative_io_organization_name,cloud_streamnative_io_pulsar_instance}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:streamnative.health.pulsar_detector_publish_latency_ms{$Organization,$Cluster,$Instance,$Namespace,quantile:0.5} by {cloud_streamnative_io_organization_name,cloud_streamnative_io_pulsar_instance}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "avg:streamnative.health.pulsar_detector_pulsar_sla_webservice_up{$Organization,$Cluster,$Instance,$Namespace} by {cloud_streamnative_io_organization_name,cloud_streamnative_io_pulsar_instance}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "avg:streamnative.health.pulsar_detector_pulsar_sla_messaging_up{$Organization,$Cluster,$Instance,$Namespace} by {cloud_streamnative_io_organization_name,cloud_streamnative_io_pulsar_instance}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "bar", + "alias": "End to End Latency (Median)", + "formula": "query1" + }, + { + "cell_display_mode": "bar", + "alias": "Publish Latency (Median)", + "formula": "query2" + }, + { + "cell_display_mode": "bar", + "alias": "Webservice Status", + "formula": "query3", + "conditional_formats": [ + { + "comparator": "=", + "value": 1, + "palette": "black_on_light_green" + }, + { + "comparator": "<", + "value": 1, + "palette": "black_on_light_red" + } + ] + }, + { + "cell_display_mode": "bar", + "alias": "Messaging Status", + "formula": "query4", + "conditional_formats": [ + { + "comparator": "=", + "value": 1, + "palette": "black_on_light_green" + }, + { + "comparator": "<", + "value": 1, + "palette": "black_on_light_red" + } + ] + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 11, + "width": 12, + "height": 4 + } + } + ], + "template_variables": [ + { + "name": "Organization", + "prefix": "cloud_streamnative_io_organization_name", + "available_values": [], + "default": "*" + }, + { + "name": "Instance", + "prefix": "cloud_streamnative_io_pulsar_instance", + "available_values": [], + "default": "*" + }, + { + "name": "Cluster", + "prefix": "cloud_streamnative_io_pulsar_cluster", + "available_values": [], + "default": "*" + }, + { + "name": "Namespace", + "prefix": "kubernetes_namespace", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} \ No newline at end of file diff --git a/streamnative/assets/dashboards/streamnative_kafka_connect.json b/streamnative/assets/dashboards/streamnative_kafka_connect.json new file mode 100644 index 0000000000000..0927c21ccf923 --- /dev/null +++ b/streamnative/assets/dashboards/streamnative_kafka_connect.json @@ -0,0 +1,2106 @@ +{ + "title": "StreamNative - Kafka Connect", + "description": "", + "widgets": [ + { + "id": 6881134961056018, + "definition": { + "type": "image", + "url": "https://cdn.prod.website-files.com/638a1dc72083d166ed6e3d76/63926c17a52727a15e13decd_Logo-streamnative-150px.svg", + "url_dark_theme": "https://cdn.prod.website-files.com/638a1dc72083d166ed6e3d76/66503b265696d89c26062701_Group%2021.svg", + "sizing": "contain", + "margin": "md", + "has_background": false, + "has_border": false, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 3 + } + }, + { + "id": 2417374102724764, + "definition": { + "type": "note", + "content": "**[StreamNative](https://www.streamnative.com/)** provides an enterprise-grade messaging platform built on Apache Pulsar, offering scalable real-time data streaming with multi-tenancy, geo-replication, and cloud integration.\n\nThis Kafka Connect dashboard track records polled/written to Kafka, task offset completions, time for reading/sending records, and JVM resource management for system insights.\n\nFor more information, see the [StreamNative Integration Documentation](https://docs.datadoghq.com/integrations/streamnative/).", + "background_color": "orange", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "top", + "has_padding": true + }, + "layout": { + "x": 6, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 7337277095932392, + "definition": { + "type": "note", + "content": "**Note**:\n- The dashboard widgets display the latest metric values captured on Kafka connect statistics.\n\n**Tip**:\n- Clone this dashboard to rearrange, modify, and add widgets and visualizations.", + "background_color": "yellow", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "top", + "has_padding": true + }, + "layout": { + "x": 9, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 8576227419060516, + "definition": { + "title": "Connector Statistics", + "background_color": "green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 2900243942491976, + "definition": { + "title": "Total Source Task Records Polled by Connector", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_source_task_source_record_poll{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {connector}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 4 + } + }, + { + "id": 7020351017682740, + "definition": { + "title": "Time Taken To Fetch Batch Records over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "millisecond" + } + }, + "alias": "Avg Time", + "formula": "query1" + }, + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "millisecond" + } + }, + "alias": "Max Time", + "formula": "query2" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.kafka_connect.kafka_connect_source_task_poll_batch_avg_time_ms{$Namespace,$Tenant,$Cluster,$Instance,$Connector}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:streamnative.kafka_connect.kafka_connect_source_task_poll_batch_max_time_ms{$Namespace,$Tenant,$Cluster,$Instance,$Connector}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 4, + "width": 12, + "height": 4 + } + }, + { + "id": 6859350096416348, + "definition": { + "title": "Task Count over time", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Task Count", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_total_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 8, + "width": 12, + "height": 3 + } + }, + { + "id": 5481643224766836, + "definition": { + "title": "Task Batch Average Size over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Avg Batch Size", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.kafka_connect.kafka_connect_connector_task_batch_size_avg{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {connector}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 11, + "width": 6, + "height": 4 + } + }, + { + "id": 4078946700674062, + "definition": { + "title": "Task Offset Commit Average Time Overview", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "millisecond" + } + }, + "alias": "Avg Time", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.kafka_connect.kafka_connect_connector_task_offset_commit_avg_time_ms{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {connector}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 6, + "y": 11, + "width": 6, + "height": 4 + } + }, + { + "id": 8129683079934458, + "definition": { + "title": "Task Offset Commit Attempt over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Success Rate", + "style": { + "palette": "green", + "palette_index": 6 + }, + "formula": "query1" + }, + { + "alias": "Failure Rate", + "style": { + "palette": "warm", + "palette_index": 6 + }, + "formula": "query2" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.kafka_connect.kafka_connect_connector_task_offset_commit_success_percentage{$Instance,$Cluster,$Tenant,$Namespace,$Connector}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:streamnative.kafka_connect.kafka_connect_connector_task_offset_commit_failure_percentage{$Instance,$Cluster,$Tenant,$Namespace,$Connector}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 15, + "width": 12, + "height": 3 + } + }, + { + "id": 6055621104791106, + "definition": { + "title": "Task Ratio over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Running Ratio", + "formula": "query1" + }, + { + "alias": "Pause Ratio", + "formula": "query2" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.kafka_connect.kafka_connect_connector_task_running_ratio{$Namespace,$Tenant,$Cluster,$Instance,$Connector}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:streamnative.kafka_connect.kafka_connect_connector_task_pause_ratio{$Namespace,$Tenant,$Cluster,$Instance,$Connector}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 18, + "width": 12, + "height": 3 + } + }, + { + "id": 1733782152939984, + "definition": { + "title": "Record Poll Rate", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.kafka_connect.kafka_connect_source_task_source_record_poll_rate{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 21, + "width": 3, + "height": 3 + } + }, + { + "id": 6912818600913508, + "definition": { + "title": "Records Polled over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Count", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_source_task_source_record_poll{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {connector}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 21, + "width": 9, + "height": 3 + } + }, + { + "id": 8975879939332920, + "definition": { + "title": "Record Write Rate", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.kafka_connect.kafka_connect_source_task_source_record_write_rate{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 24, + "width": 3, + "height": 3 + } + }, + { + "id": 1751252889816062, + "definition": { + "title": "Records Written over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_source_task_source_record_write{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {connector}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 24, + "width": 9, + "height": 3 + } + }, + { + "id": 6347246414024880, + "definition": { + "title": "Active Source Record Counts", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_source_task_source_record_active_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 27, + "width": 3, + "height": 3 + } + }, + { + "id": 917484482480764, + "definition": { + "title": "Active Source Average Record Count over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Avg Count", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.kafka_connect.kafka_connect_source_task_source_record_active_count_avg{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {connector}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 27, + "width": 9, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 3, + "width": 12, + "height": 31 + } + }, + { + "id": 8141984939677466, + "definition": { + "title": "Task Statistics", + "background_color": "yellow", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 4937187724533902, + "definition": { + "title": "Total Task count", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_total_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">=", + "value": 0, + "palette": "black_on_light_green", + "custom_bg_color": "#d3dbfd" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 2 + } + }, + { + "id": 1732765197010766, + "definition": { + "title": "Running Tasks", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_running_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "white_on_green", + "custom_bg_color": "#94bee6" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 2, + "width": 4, + "height": 2 + } + }, + { + "id": 5731082836012438, + "definition": { + "title": "Restarted Tasks", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_restarting_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow", + "custom_bg_color": "#affdc7" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 4, + "y": 2, + "width": 4, + "height": 2 + } + }, + { + "id": 7579566112672550, + "definition": { + "title": "Paused Tasks", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_paused_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow", + "custom_bg_color": "#dbdbdb" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 8, + "y": 2, + "width": 4, + "height": 2 + } + }, + { + "id": 1674575362095614, + "definition": { + "title": "Unassigned Tasks", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_unassigned_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow", + "custom_bg_color": "#f6fdbf" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green", + "custom_bg_color": "#adc7db" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 4, + "width": 4, + "height": 2 + } + }, + { + "id": 896751850091798, + "definition": { + "title": "Failed Tasks", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_failed_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_red" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 4, + "y": 4, + "width": 4, + "height": 2 + } + }, + { + "id": 4136437594096920, + "definition": { + "title": "Destroyed Tasks", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_destroyed_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_red" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 8, + "y": 4, + "width": 4, + "height": 2 + } + }, + { + "id": 952345246794038, + "definition": { + "title": "Overall Task Status", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_running_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {connector,cloud_streamnative_io_kafka_connector_type}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_paused_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {connector,cloud_streamnative_io_kafka_connector_type}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_failed_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {connector,cloud_streamnative_io_kafka_connector_type}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_restarting_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {connector,cloud_streamnative_io_kafka_connector_type}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query5", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_unassigned_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {connector,cloud_streamnative_io_kafka_connector_type}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query6", + "query": "sum:streamnative.kafka_connect.kafka_connect_worker_connector_destroyed_task_count{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {connector,cloud_streamnative_io_kafka_connector_type}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_red" + } + ], + "cell_display_mode": "number", + "alias": "Running Tasks", + "formula": "query1" + }, + { + "conditional_formats": [ + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + }, + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ], + "cell_display_mode": "number", + "alias": "Paused Tasks", + "formula": "query2" + }, + { + "conditional_formats": [ + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + }, + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_red" + } + ], + "cell_display_mode": "number", + "alias": "Failed Tasks", + "formula": "query3" + }, + { + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ], + "cell_display_mode": "number", + "alias": "Restarted Tasks", + "formula": "query4" + }, + { + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "custom_bg", + "custom_bg_color": "#aaa7a7" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ], + "cell_display_mode": "number", + "alias": "Unassigned Tasks", + "formula": "query5" + }, + { + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_red", + "custom_bg_color": "#aaa7a7" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ], + "cell_display_mode": "number", + "alias": "Destroyed Tasks", + "formula": "query6" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 6, + "width": 12, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 34, + "width": 12, + "height": 10, + "is_column_break": true + } + }, + { + "id": 1459531063639174, + "definition": { + "title": "Error Statistics", + "background_color": "orange", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 8244539431747950, + "definition": { + "title": "Total Logged Errors", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_task_error_total_errors_logged{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_red" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 1174588026581614, + "definition": { + "title": "Last Error Occurred", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.kafka_connect.kafka_connect_task_error_last_error_timestamp{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 3, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 3375082908184278, + "definition": { + "title": "Total Erroneous Records over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Total Count", + "formula": "query4" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query4", + "query": "sum:streamnative.kafka_connect.kafka_connect_task_error_total_record_errors{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {cloud_streamnative_io_pulsar_component}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 4 + } + }, + { + "id": 901155958667404, + "definition": { + "title": "Operations Retries Attempted", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_task_error_total_retries{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 2, + "width": 6, + "height": 2 + } + }, + { + "id": 8397701140048364, + "definition": { + "title": "Records by Status over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Skipped Records", + "formula": "query4" + }, + { + "alias": "Failed Records", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query4", + "query": "sum:streamnative.kafka_connect.kafka_connect_task_error_total_records_skipped{$Namespace,$Tenant,$Cluster,$Instance,$Connector}" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_task_error_total_record_failures{$Namespace,$Tenant,$Cluster,$Instance,$Connector}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 4, + "width": 12, + "height": 4 + } + }, + { + "id": 2134993726896258, + "definition": { + "title": "Dead Letter Queue Writes over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Failed Writes", + "formula": "query1" + }, + { + "alias": "Requested Writes", + "formula": "query2" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.kafka_connect_task_error_deadletterqueue_produce_failures{$Namespace,$Tenant,$Cluster,$Instance,$Connector}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "sum:streamnative.kafka_connect.kafka_connect_task_error_deadletterqueue_produce_requests{$Namespace,$Tenant,$Cluster,$Instance,$Connector}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 8, + "width": 12, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 44, + "width": 12, + "height": 13 + } + }, + { + "id": 6566116028475434, + "definition": { + "title": "JVM Statistics", + "background_color": "purple", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 860199545188632, + "definition": { + "title": "Max Memory Bytes", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.jvm_memory_max_bytes{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte" + } + }, + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 4337310144041626, + "definition": { + "title": "Memory Space over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte" + } + }, + "alias": "Initial Memory Bytes", + "formula": "query1" + }, + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte" + } + }, + "alias": "Used Memory Bytes", + "formula": "query2" + }, + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte" + } + }, + "alias": "Committed Memory Bytes", + "formula": "query3" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.jvm_memory_init_bytes{$Namespace,$Tenant,$Cluster,$Instance,$Connector}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "sum:streamnative.kafka_connect.jvm_memory_used_bytes{$Namespace,$Tenant,$Cluster,$Instance,$Connector}" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:streamnative.kafka_connect.jvm_memory_committed_bytes{$Namespace,$Tenant,$Cluster,$Instance,$Connector}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 3 + } + }, + { + "id": 7417774753206652, + "definition": { + "title": "Committed Memory Bytes", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.jvm_memory_committed_bytes{$Namespace,$Tenant,$Cluster,$Instance,$Connector}", + "aggregator": "last" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte" + } + }, + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 3, + "width": 3, + "height": 3 + } + }, + { + "id": 7133220908988536, + "definition": { + "title": "Time Spent In JVM Garbage Collector", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "second" + } + }, + "alias": "GC Time", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.kafka_connect.jvm_gc_collection_seconds_sum{$Namespace,$Tenant,$Cluster,$Instance,$Connector} by {cloud_streamnative_io_kafka_connector_type}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 3, + "width": 9, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 57, + "width": 12, + "height": 7 + } + } + ], + "template_variables": [ + { + "name": "Instance", + "prefix": "cloud_streamnative_io_pulsar_instance", + "available_values": [], + "default": "*" + }, + { + "name": "Cluster", + "prefix": "cloud_streamnative_io_pulsar_cluster", + "available_values": [], + "default": "*" + }, + { + "name": "Tenant", + "prefix": "cloud_streamnative_io_pulsar_tenant", + "available_values": [], + "default": "*" + }, + { + "name": "Namespace", + "prefix": "kubernetes_namespace", + "available_values": [], + "default": "*" + }, + { + "name": "Connector", + "prefix": "cloud_streamnative_io_pulsar_component", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} \ No newline at end of file diff --git a/streamnative/assets/dashboards/streamnative_pulsar_resource.json b/streamnative/assets/dashboards/streamnative_pulsar_resource.json new file mode 100644 index 0000000000000..4ada84aa4cf0a --- /dev/null +++ b/streamnative/assets/dashboards/streamnative_pulsar_resource.json @@ -0,0 +1,1144 @@ +{ + "title": "StreamNative - Pulsar Resource", + "description": "", + "widgets": [ + { + "id": 1435610181574376, + "definition": { + "type": "image", + "url": "https://cdn.prod.website-files.com/638a1dc72083d166ed6e3d76/63926c17a52727a15e13decd_Logo-streamnative-150px.svg", + "url_dark_theme": "https://cdn.prod.website-files.com/638a1dc72083d166ed6e3d76/66503b265696d89c26062701_Group%2021.svg", + "sizing": "contain", + "margin": "md", + "has_background": false, + "has_border": false, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 7, + "height": 3 + } + }, + { + "id": 6048556896224372, + "definition": { + "title": "Monitor Summary", + "type": "manage_status", + "display_format": "countsAndList", + "color_preference": "background", + "hide_zero_counts": true, + "show_status": true, + "last_triggered_format": "relative", + "query": "tag:streamnative", + "sort": "status,asc", + "count": 50, + "start": 0, + "summary_type": "monitors", + "show_priority": false, + "show_last_triggered": false + }, + "layout": { + "x": 7, + "y": 0, + "width": 5, + "height": 5 + } + }, + { + "id": 8952756529161660, + "definition": { + "type": "note", + "content": "**[StreamNative](https://www.streamnative.com/)** provides an enterprise-grade messaging platform built on Apache Pulsar, offering scalable real-time data streaming with multi-tenancy, geo-replication, and cloud integration.\n\nThis Pulsar Resource dashboard tracks topics, subscriptions, message rates, and throughput, along with storage metrics like size, backlog, and offloaded data. Latency and entry size distributions are also monitored for efficient data flow and storage management.\n\nFor more information, see the [StreamNative Integration Documentation](https://docs.datadoghq.com/integrations/streamnative/).", + "background_color": "orange", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "top", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 3, + "width": 4, + "height": 2 + } + }, + { + "id": 393756623944156, + "definition": { + "type": "note", + "content": "**Note**:\n- The dashboard widgets display the latest metric values captured on Pulsar resource statistics.\n\n**Tip**:\n- Clone this dashboard to rearrange, modify, and add widgets and visualizations.", + "background_color": "yellow", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "top", + "has_padding": true + }, + "layout": { + "x": 4, + "y": 3, + "width": 3, + "height": 2 + } + }, + { + "id": 3401525582219446, + "definition": { + "title": "Topic Statistics", + "background_color": "green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 7407785548312006, + "definition": { + "title": "Topics (Partitions)", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.pulsar_resource.pulsar_topics_count{$Namespace,$Instance,$Organization,$Cluster}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">=", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 449304534867526, + "definition": { + "title": "Producers", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.pulsar_resource.pulsar_producers_count{$Instance,$Namespace,$Organization,$Cluster,$Topic}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">=", + "value": 0, + "palette": "black_on_light_yellow", + "custom_bg_color": "#e1d7a3" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 3, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 4049965368212942, + "definition": { + "title": "Consumers", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.pulsar_resource.pulsar_consumers_count{$Instance,$Namespace,$Organization,$Cluster,$Topic}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">=", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 6, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 8787839035935990, + "definition": { + "title": "Subscriptions", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.pulsar_resource.pulsar_subscriptions_count{$Namespace,$Instance,$Organization,$Cluster,$Topic}", + "aggregator": "last" + } + ], + "conditional_formats": [ + { + "comparator": ">=", + "value": 0, + "palette": "black_on_light_green", + "custom_bg_color": "#d2ecc1" + } + ], + "formulas": [ + { + "formula": "query1" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 9, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 6918508884786830, + "definition": { + "title": "Topic Statistics", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "query2", + "query": "sum:streamnative.pulsar_resource.pulsar_consumers_count{$Namespace,$Instance,$Organization,$Cluster,$Topic} by {cluster,topic,namespace}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:streamnative.pulsar_resource.pulsar_producers_count{$Namespace,$Instance,$Organization,$Cluster,$Topic} by {cluster,topic,namespace}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "number", + "alias": "Consumers Count", + "formula": "query2" + }, + { + "cell_display_mode": "number", + "alias": "Producers Count", + "formula": "query3" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 3, + "width": 12, + "height": 5 + } + }, + { + "id": 1225105301796874, + "definition": { + "title": "Backlog Size", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.pulsar_resource.pulsar_storage_backlog_size{$Namespace,$Instance,$Organization,$Cluster,$Topic}", + "aggregator": "last" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_red" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green", + "custom_bg_color": "#8d8ebf" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte" + } + }, + "formula": "query1" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 8, + "width": 3, + "height": 3 + } + }, + { + "id": 6387866884258252, + "definition": { + "title": "Backlog Size by Topic", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.pulsar_resource.pulsar_storage_backlog_size{$Namespace,$Instance,$Organization,$Cluster,$Topic} by {topic}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 25, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "inline" + }, + "palette": "datadog16", + "scaling": "absolute" + } + }, + "layout": { + "x": 3, + "y": 8, + "width": 9, + "height": 3 + } + }, + { + "id": 2251274279995436, + "definition": { + "title": "Backlog Size over time", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.pulsar_resource.pulsar_storage_backlog_size{$Namespace,$Instance,$Organization,$Cluster,$Topic}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "datadog16", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "area" + } + ] + }, + "layout": { + "x": 0, + "y": 11, + "width": 12, + "height": 3 + } + }, + { + "id": 2143008222517982, + "definition": { + "type": "note", + "content": "The total message batches (entries) delayed for dispatching.", + "background_color": "yellow", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": true, + "tick_pos": "50%", + "tick_edge": "right", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 14, + "width": 2, + "height": 2 + } + }, + { + "id": 8915007273971110, + "definition": { + "title": "Total Subscription Delay", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.pulsar_resource.pulsar_subscription_delayed{$Namespace,$Instance,$Organization,$Cluster,$Topic}", + "aggregator": "last" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_red" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 2, + "y": 14, + "width": 4, + "height": 2 + } + }, + { + "id": 5855029633521978, + "definition": { + "title": "Total Offloaded Size", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.pulsar_resource.pulsar_storage_offloaded_size{$Namespace,$Instance,$Organization,$Cluster,$Topic}", + "aggregator": "last" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_yellow", + "custom_bg_color": "#c4b0e8" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte" + } + }, + "formula": "query1" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 6, + "y": 14, + "width": 4, + "height": 2 + } + }, + { + "id": 6171745012404356, + "definition": { + "type": "note", + "content": "The total amount of the data in this topic offloaded to the tiered storage (bytes).", + "background_color": "yellow", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": true, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true + }, + "layout": { + "x": 10, + "y": 14, + "width": 2, + "height": 2 + } + }, + { + "id": 2700175861535110, + "definition": { + "title": "Storage ", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.pulsar_resource.pulsar_storage_size{$Namespace,$Instance,$Organization,$Cluster,$Topic}", + "aggregator": "last" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte" + } + }, + "formula": "query1" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 16, + "width": 3, + "height": 3 + } + }, + { + "id": 713922844627902, + "definition": { + "title": "Storage Size over time", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.pulsar_resource.pulsar_storage_size{$Namespace,$Instance,$Organization,$Cluster,$Topic}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "area" + } + ] + }, + "layout": { + "x": 3, + "y": 16, + "width": 9, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 5, + "width": 12, + "height": 20 + } + }, + { + "id": 627038884644444, + "definition": { + "title": "Performance Statistics", + "background_color": "yellow", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 5643112798519754, + "definition": { + "title": "Rate Flow over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Rate In", + "formula": "query1" + }, + { + "alias": "Rate Out", + "formula": "query2" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.pulsar_resource.pulsar_rate_in{$Namespace,$Instance,$Organization,$Cluster,$Topic}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:streamnative.pulsar_resource.pulsar_rate_out{$Namespace,$Instance,$Organization,$Cluster,$Topic}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 4 + } + }, + { + "id": 7237859763233540, + "definition": { + "title": "Throughput over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Throughput In", + "formula": "query1" + }, + { + "alias": "Throughput Out", + "formula": "query2" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.pulsar_resource.pulsar_throughput_in{$Namespace,$Instance,$Organization,$Cluster,$Topic}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:streamnative.pulsar_resource.pulsar_throughput_out{$Namespace,$Instance,$Organization,$Cluster,$Topic}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 4, + "width": 6, + "height": 4 + } + }, + { + "id": 7636082687940984, + "definition": { + "title": "Storage I/O over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Read Rate", + "formula": "query1" + }, + { + "alias": "Write Rate", + "formula": "query2" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.pulsar_resource.pulsar_storage_read_rate{$Namespace,$Instance,$Organization,$Cluster,$Topic}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:streamnative.pulsar_resource.pulsar_storage_write_rate{$Namespace,$Instance,$Organization,$Cluster,$Topic}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 6, + "y": 4, + "width": 6, + "height": 4 + } + }, + { + "id": 1748232157544970, + "definition": { + "title": "Overall Summary", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "avg:streamnative.pulsar_resource.pulsar_storage_read_rate{$Namespace,$Instance,$Organization,$Cluster,$Topic} by {cluster,namespace,topic}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query5", + "query": "avg:streamnative.pulsar_resource.pulsar_storage_write_rate{$Namespace,$Instance,$Organization,$Cluster,$Topic} by {cluster,namespace,topic}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:streamnative.pulsar_resource.pulsar_throughput_in{$Namespace,$Instance,$Organization,$Cluster,$Topic} by {cluster,namespace,topic}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "avg:streamnative.pulsar_resource.pulsar_throughput_out{$Namespace,$Instance,$Organization,$Cluster,$Topic} by {cluster,namespace,topic}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "avg:streamnative.pulsar_resource.pulsar_rate_in{$Namespace,$Instance,$Organization,$Cluster,$Topic} by {cluster,namespace,topic}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "query6", + "query": "avg:streamnative.pulsar_resource.pulsar_rate_out{$Namespace,$Instance,$Organization,$Cluster,$Topic} by {cluster,namespace,topic}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "number", + "alias": "Storage Read Rate", + "formula": "query1" + }, + { + "cell_display_mode": "number", + "alias": "Storage Write Rate", + "formula": "query5" + }, + { + "cell_display_mode": "number", + "alias": "Throughput In", + "formula": "query2" + }, + { + "cell_display_mode": "number", + "alias": "Throughput Out", + "formula": "query3" + }, + { + "cell_display_mode": "number", + "alias": "Rate In", + "formula": "query4" + }, + { + "cell_display_mode": "number", + "alias": "Rate Out", + "formula": "query6" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 8, + "width": 12, + "height": 5 + } + } + ] + }, + "layout": { + "x": 0, + "y": 25, + "width": 12, + "height": 14, + "is_column_break": true + } + } + ], + "template_variables": [ + { + "name": "Organization", + "prefix": "cloud_streamnative_io_organization_name", + "available_values": [], + "default": "*" + }, + { + "name": "Instance", + "prefix": "cloud_streamnative_io_pulsar_instance", + "available_values": [], + "default": "*" + }, + { + "name": "Cluster", + "prefix": "cluster", + "available_values": [], + "default": "*" + }, + { + "name": "Namespace", + "prefix": "namespace", + "available_values": [], + "default": "*" + }, + { + "name": "Topic", + "prefix": "topic", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} \ No newline at end of file diff --git a/streamnative/assets/dashboards/streamnative_sink_connector.json b/streamnative/assets/dashboards/streamnative_sink_connector.json new file mode 100644 index 0000000000000..3ffc856038fe1 --- /dev/null +++ b/streamnative/assets/dashboards/streamnative_sink_connector.json @@ -0,0 +1,774 @@ +{ + "title": "StreamNative - Sink Connector", + "description": "", + "widgets": [ + { + "id": 4904783609376248, + "definition": { + "type": "image", + "url": "https://cdn.prod.website-files.com/638a1dc72083d166ed6e3d76/63926c17a52727a15e13decd_Logo-streamnative-150px.svg", + "url_dark_theme": "https://cdn.prod.website-files.com/638a1dc72083d166ed6e3d76/66503b265696d89c26062701_Group%2021.svg", + "sizing": "contain", + "margin": "md", + "has_background": false, + "has_border": false, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 3 + } + }, + { + "id": 8995860272844608, + "definition": { + "type": "note", + "content": "**[StreamNative](https://www.streamnative.com/)** provides an enterprise-grade messaging platform built on Apache Pulsar, offering scalable real-time data streaming with multi-tenancy, geo-replication, and cloud integration.\n\nThis Sink Connector dashboard provides insights into sink operations, data flow, exceptions, and JVM resource management for efficient Pulsar performance.\n\nFor more information, see the [StreamNative Integration Documentation](https://docs.datadoghq.com/integrations/streamnative/).", + "background_color": "orange", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "top", + "has_padding": true + }, + "layout": { + "x": 6, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 5920141165566776, + "definition": { + "type": "note", + "content": "**Note**:\n- The dashboard widgets display the latest metric values captured on sink connector statistics.\n\n**Tip**:\n- Clone this dashboard to rearrange, modify, and add widgets and visualizations.", + "background_color": "yellow", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "top", + "has_padding": true + }, + "layout": { + "x": 9, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 8920403223435398, + "definition": { + "title": "Messages Written", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ], + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "sum:streamnative.sink_connector.pulsar_sink_written_total{$Sink_Name,$Organization,$Instance}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green", + "custom_bg_color": "#f0d999" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 3, + "width": 3, + "height": 3 + } + }, + { + "id": 1093006653689844, + "definition": { + "title": "Messages Written over time", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Pulsar Sink Written", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.sink_connector.pulsar_sink_written_total{$Sink_Name,$Organization,$Instance}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "area" + } + ] + }, + "layout": { + "x": 3, + "y": 3, + "width": 9, + "height": 3 + } + }, + { + "id": 7549091675650990, + "definition": { + "title": "Messages Received", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.sink_connector.pulsar_sink_received_total{$Sink_Name,$Organization,$Instance}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow", + "custom_bg_color": "#c7dcda" + } + ] + } + ], + "autoscale": true, + "text_align": "center", + "precision": 0 + }, + "layout": { + "x": 0, + "y": 6, + "width": 3, + "height": 3 + } + }, + { + "id": 7401087282347594, + "definition": { + "title": "Messages Received over time", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Pulsar Sink Received", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.sink_connector.pulsar_sink_received_total{$Sink_Name,$Organization,$Instance}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "area" + } + ] + }, + "layout": { + "x": 3, + "y": 6, + "width": 9, + "height": 3 + } + }, + { + "id": 3549300876223298, + "definition": { + "title": "User and System CPU Time", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "second" + } + }, + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.sink_connector.process_cpu_seconds_total{$Sink_Name,$Organization,$Instance}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 1, + "palette": "yellow_on_white", + "custom_bg_color": "#e5c7f0" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 9, + "width": 6, + "height": 2 + } + }, + { + "id": 1900009483847726, + "definition": { + "title": "Last Invocation Time", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.sink_connector.pulsar_sink_last_invocation{$Sink_Name,$Organization,$Instance}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "yellow_on_white", + "custom_bg_color": "#c9e69e" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 6, + "y": 9, + "width": 6, + "height": 2 + } + }, + { + "id": 7809318879280602, + "definition": { + "title": "Sink Exceptions", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.sink_connector.pulsar_sink_sink_exceptions_total{$Sink_Name,$Organization,$Instance}", + "aggregator": "last" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_red" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 2, + "is_column_break": true + } + }, + { + "id": 6707352404639228, + "definition": { + "title": "System Exceptions", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.sink_connector.pulsar_sink_system_exceptions_total{$Sink_Name,$Organization,$Instance}", + "aggregator": "last" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_red", + "custom_bg_color": "#eedef7" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 2 + } + }, + { + "id": 8445989619646756, + "definition": { + "title": "JVM Statistics", + "background_color": "purple", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 8970146552931028, + "definition": { + "title": "Max Memory Bytes", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.sink_connector.jvm_memory_bytes_max{$Organization,$Instance,$Sink_Name}", + "aggregator": "last" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte" + } + }, + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 4546078437502932, + "definition": { + "title": "Memory Space over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Committed Memory Bytes Bytes", + "formula": "query1" + }, + { + "alias": "Max Memory Bytes", + "formula": "query2" + }, + { + "alias": "Initial Memory Bytes ", + "formula": "query3" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.sink_connector.jvm_memory_bytes_committed{$Sink_Name,$Organization,$Instance}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "sum:streamnative.sink_connector.jvm_memory_bytes_max{$Sink_Name,$Organization,$Instance}" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:streamnative.sink_connector.jvm_memory_bytes_init{$Sink_Name,$Organization,$Instance}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 3 + } + }, + { + "id": 983596439967046, + "definition": { + "title": "Committed Memory Bytes", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.sink_connector.jvm_memory_bytes_committed{$Organization,$Instance,$Sink_Name}", + "aggregator": "last" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte" + } + }, + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 3, + "width": 3, + "height": 3 + } + }, + { + "id": 2285216008395834, + "definition": { + "title": "Time Spent In JVM Garbage Collector", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "GC Time", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.sink_connector.jvm_gc_collection_seconds_sum{$Sink_Name,$Organization,$Instance}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 3, + "width": 9, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 13, + "width": 12, + "height": 7 + } + } + ], + "template_variables": [ + { + "name": "Organization", + "prefix": "kubernetes_namespace", + "available_values": [], + "default": "*" + }, + { + "name": "Instance", + "prefix": "cloud_streamnative_io_pulsar_instance", + "available_values": [], + "default": "*" + }, + { + "name": "Sink_Name", + "prefix": "pulsar_component", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} \ No newline at end of file diff --git a/streamnative/assets/dashboards/streamnative_source_connector.json b/streamnative/assets/dashboards/streamnative_source_connector.json new file mode 100644 index 0000000000000..fb36fd9efadaf --- /dev/null +++ b/streamnative/assets/dashboards/streamnative_source_connector.json @@ -0,0 +1,777 @@ +{ + "title": "StreamNative - Source Connector", + "description": "", + "widgets": [ + { + "id": 2511154829172922, + "definition": { + "type": "image", + "url": "https://cdn.prod.website-files.com/638a1dc72083d166ed6e3d76/63926c17a52727a15e13decd_Logo-streamnative-150px.svg", + "url_dark_theme": "https://cdn.prod.website-files.com/638a1dc72083d166ed6e3d76/66503b265696d89c26062701_Group%2021.svg", + "sizing": "contain", + "margin": "md", + "has_background": false, + "has_border": false, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 3 + } + }, + { + "id": 7639788193841362, + "definition": { + "type": "note", + "content": "**[StreamNative](https://www.streamnative.com/)** provides an enterprise-grade messaging platform built on Apache Pulsar, offering scalable real-time data streaming with multi-tenancy, geo-replication, and cloud integration.\n\nThis Source Connector dashboard provides insights into source operations, data flow, exceptions, and JVM resource management for efficient Pulsar performance.\n\nFor more information, see the [StreamNative Integration Documentation](https://docs.datadoghq.com/integrations/streamnative/).", + "background_color": "orange", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "top", + "has_padding": true + }, + "layout": { + "x": 6, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 863470997872514, + "definition": { + "type": "note", + "content": "**Note**:\n- The dashboard widgets display the latest metric values captured on source connector statistics.\n\n**Tip**:\n- Clone this dashboard to rearrange, modify, and add widgets and visualizations.", + "background_color": "yellow", + "font_size": "14", + "text_align": "left", + "vertical_align": "top", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "top", + "has_padding": true + }, + "layout": { + "x": 9, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 510982300759208, + "definition": { + "title": "Messages Written", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.source_connector.pulsar_source_written_total{$Source_Name,$Organization,$Instance}", + "aggregator": "last" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green", + "custom_bg_color": "#5db0b1" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 3, + "width": 3, + "height": 3 + } + }, + { + "id": 902261882355924, + "definition": { + "title": "Messages Written over time", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Pulsar Source Written", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.source_connector.pulsar_source_written_total{$Source_Name,$Organization,$Instance}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "area" + } + ] + }, + "layout": { + "x": 3, + "y": 3, + "width": 9, + "height": 3 + } + }, + { + "id": 2464648447453060, + "definition": { + "title": "Messages Received", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.source_connector.pulsar_source_received_total{$Source_Name,$Organization,$Instance}", + "aggregator": "last" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow", + "custom_bg_color": "#ea80b3" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 6, + "width": 3, + "height": 3 + } + }, + { + "id": 5636435429995700, + "definition": { + "title": "Messages Received over time", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Pulsar Source Received", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.source_connector.pulsar_source_received_total{$Source_Name,$Organization,$Instance}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "area" + } + ] + }, + "layout": { + "x": 3, + "y": 6, + "width": 9, + "height": 3 + } + }, + { + "id": 3463292503235014, + "definition": { + "title": "User and System CPU Time", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "second" + } + }, + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.source_connector.process_cpu_seconds_total{$Source_Name,$Organization,$Instance}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 1, + "palette": "yellow_on_white" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 9, + "width": 6, + "height": 2 + } + }, + { + "id": 2681497336240052, + "definition": { + "title": "Last Invocation Time", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.source_connector.pulsar_source_last_invocation{$Source_Name,$Organization,$Instance}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "yellow_on_white" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 6, + "y": 9, + "width": 6, + "height": 2 + } + }, + { + "id": 1092066668982220, + "definition": { + "title": "Source Exceptions", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.source_connector.pulsar_source_source_exceptions_total{$Source_Name,$Organization,$Instance}", + "aggregator": "last" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_red" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 2, + "is_column_break": true + } + }, + { + "id": 2954779705730000, + "definition": { + "title": "System Exceptions", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.source_connector.pulsar_source_system_exceptions_total{$Source_Name,$Organization,$Instance}", + "aggregator": "last" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_red", + "custom_bg_color": "#eedef7" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit" + } + } + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 2 + } + }, + { + "id": 7961217489134972, + "definition": { + "title": "JVM Statistics", + "background_color": "purple", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 4247332022096914, + "definition": { + "title": "Max Memory Bytes", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.source_connector.jvm_memory_bytes_max{$Organization,$Instance,$Source_Name}", + "aggregator": "last" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte" + } + }, + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow", + "custom_bg_color": "#a2d9dd" + }, + { + "comparator": "=", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 8539224685744776, + "definition": { + "title": "Memory Space over time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Committed Memory Bytes", + "formula": "query1" + }, + { + "alias": "Max Memory Bytes", + "formula": "query2" + }, + { + "alias": "Initial Memory Bytes", + "formula": "query3" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.source_connector.jvm_memory_bytes_committed{$Source_Name,$Organization,$Instance}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "sum:streamnative.source_connector.jvm_memory_bytes_max{$Source_Name,$Organization,$Instance}" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:streamnative.source_connector.jvm_memory_bytes_init{$Source_Name,$Organization,$Instance}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 3 + } + }, + { + "id": 7759610259055016, + "definition": { + "title": "Committed Memory Bytes", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "hour", + "value": 4 + }, + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.source_connector.jvm_memory_bytes_committed{$Organization,$Instance,$Source_Name}", + "aggregator": "last" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte" + } + }, + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 3, + "width": 3, + "height": 3 + } + }, + { + "id": 8264813475686008, + "definition": { + "title": "Time Spent In JVM Garbage Collector", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "type": "live", + "unit": "month", + "value": 1 + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "GC Time", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:streamnative.source_connector.jvm_gc_collection_seconds_sum{$Source_Name,$Organization,$Instance}" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 3, + "y": 3, + "width": 9, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 13, + "width": 12, + "height": 7 + } + } + ], + "template_variables": [ + { + "name": "Organization", + "prefix": "kubernetes_namespace", + "available_values": [], + "default": "*" + }, + { + "name": "Instance", + "prefix": "cloud_streamnative_io_pulsar_instance", + "available_values": [], + "default": "*" + }, + { + "name": "Source_Name", + "prefix": "pulsar_component", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} \ No newline at end of file diff --git a/streamnative/assets/monitors/backlog_size_exceeding_threshold.json b/streamnative/assets/monitors/backlog_size_exceeding_threshold.json new file mode 100644 index 0000000000000..9b1db8b4f58aa --- /dev/null +++ b/streamnative/assets/monitors/backlog_size_exceeding_threshold.json @@ -0,0 +1,34 @@ +{ + "version": 2, + "created_at": "2024-11-07", + "last_updated_at": "2024-11-07", + "title": "Backlog size exceeding threshold", + "description": "Backlog size has exceeded the threshold for the specified organization, instance, and cluster.", + "definition": { + "id": 155337177, + "name": "Backlog size exceeding threshold", + "type": "query alert", + "query": "max(last_5m):avg:streamnative.pulsar_resource.pulsar_storage_backlog_size{*} by {cloud_streamnative_io_organization_name,cloud_streamnative_io_pulsar_instance,cloud_streamnative_io_pulsar_cluster} >= 10737418240", + "message": "{{#is_alert}} \nBacklog size has exceeded threshold for Organization: {{cloud_streamnative_io_organization_name.name}}, Instance: {{cloud_streamnative_io_pulsar_instance.name}}, Cluster: {{cloud_streamnative_io_cluster.name}}\nCurrent backlog size: {{eval \"round(value/1073741824, 3)\"}} GiB\nThreshold backlog size: {{eval \"round(threshold/1073741824, 3)\"}} GiB\n{{/is_alert}}\n@email@example.com", + "tags": [ + "streamnative" + ], + "options": { + "thresholds": { + "critical": 10737418240 + }, + "notify_audit": false, + "on_missing_data": "show_no_data", + "include_tags": true, + "new_group_delay": 60, + "silenced": {} + }, + "priority": null, + "restriction_policy": { + "bindings": [] + } + }, + "tags": [ + "integration:streamnative" + ] +} \ No newline at end of file diff --git a/streamnative/assets/monitors/messaging_service_down.json b/streamnative/assets/monitors/messaging_service_down.json new file mode 100644 index 0000000000000..bf378e47b8705 --- /dev/null +++ b/streamnative/assets/monitors/messaging_service_down.json @@ -0,0 +1,34 @@ +{ + "version": 2, + "created_at": "2024-11-07", + "last_updated_at": "2024-11-07", + "title": "Messaging service is down", + "description": "Messaging service is down for the specified organization, instance, and cluster.", + "definition": { + "id": 155399567, + "name": "Messaging service is down", + "type": "query alert", + "query": "avg(last_5m):avg:streamnative.health.pulsar_detector_pulsar_sla_messaging_up{*} by {cloud_streamnative_io_organization_name,cloud_streamnative_io_pulsar_instance,cloud_streamnative_io_pulsar_cluster} < 1", + "message": "{{#is_alert}}\nMessaging Service is down for Organization: {{cloud_streamnative_io_organization_name.name}}, Instance: {{cloud_streamnative_io_pulsar_instance.name}}, Cluster: {{cloud_streamnative_io_pulsar_cluster.name}}\n{{/is_alert}}\n@email@example.com", + "tags": [ + "streamnative" + ], + "options": { + "thresholds": { + "critical": 1 + }, + "notify_audit": false, + "on_missing_data": "show_no_data", + "include_tags": false, + "new_group_delay": 0, + "silenced": {} + }, + "priority": null, + "restriction_policy": { + "bindings": [] + } + }, + "tags": [ + "integration:streamnative" + ] +} \ No newline at end of file diff --git a/streamnative/assets/monitors/webservice_down.json b/streamnative/assets/monitors/webservice_down.json new file mode 100644 index 0000000000000..44e8c813a4e52 --- /dev/null +++ b/streamnative/assets/monitors/webservice_down.json @@ -0,0 +1,34 @@ +{ + "version": 2, + "created_at": "2024-11-07", + "last_updated_at": "2024-11-07", + "title": "Webservice is down", + "description": "Webservice is down for the specified organization, instance, and cluster.", + "definition": { + "id": 155033525, + "name": "Webservice is down", + "type": "query alert", + "query": "avg(last_5m):avg:streamnative.health.pulsar_detector_pulsar_sla_webservice_up{*} by {cloud_streamnative_io_organization_name,cloud_streamnative_io_pulsar_instance,cloud_streamnative_io_pulsar_cluster} < 1", + "message": "{{#is_alert}}\nWebservice is down for Organization: {{cloud_streamnative_io_organization_name.name}}, Instance: {{cloud_streamnative_io_pulsar_instance.name}}, Cluster: {{cloud_streamnative_io_pulsar_cluster.name}}\n{{/is_alert}}\n@email@example.com", + "tags": [ + "streamnative" + ], + "options": { + "thresholds": { + "critical": 1 + }, + "notify_audit": false, + "on_missing_data": "show_no_data", + "include_tags": false, + "new_group_delay": 0, + "silenced": {} + }, + "priority": null, + "restriction_policy": { + "bindings": [] + } + }, + "tags": [ + "integration:streamnative" + ] +} \ No newline at end of file diff --git a/streamnative/assets/service_checks.json b/streamnative/assets/service_checks.json new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/streamnative/assets/service_checks.json @@ -0,0 +1 @@ +[] diff --git a/streamnative/assets/streamnative.svg b/streamnative/assets/streamnative.svg new file mode 100644 index 0000000000000..2d54b8e55e549 --- /dev/null +++ b/streamnative/assets/streamnative.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/streamnative/images/streamnative_health.png b/streamnative/images/streamnative_health.png new file mode 100644 index 0000000000000..77d6251b969c7 Binary files /dev/null and b/streamnative/images/streamnative_health.png differ diff --git a/streamnative/images/streamnative_kafka_connect.png b/streamnative/images/streamnative_kafka_connect.png new file mode 100644 index 0000000000000..6b71830b70a93 Binary files /dev/null and b/streamnative/images/streamnative_kafka_connect.png differ diff --git a/streamnative/images/streamnative_pulsar_resource.png b/streamnative/images/streamnative_pulsar_resource.png new file mode 100644 index 0000000000000..9233bd8bd7a16 Binary files /dev/null and b/streamnative/images/streamnative_pulsar_resource.png differ diff --git a/streamnative/images/streamnative_sink_connector.png b/streamnative/images/streamnative_sink_connector.png new file mode 100644 index 0000000000000..99ba1c37b083a Binary files /dev/null and b/streamnative/images/streamnative_sink_connector.png differ diff --git a/streamnative/images/streamnative_source_connector.png b/streamnative/images/streamnative_source_connector.png new file mode 100644 index 0000000000000..a811dfed76b77 Binary files /dev/null and b/streamnative/images/streamnative_source_connector.png differ diff --git a/streamnative/manifest.json b/streamnative/manifest.json new file mode 100644 index 0000000000000..4d30758819de1 --- /dev/null +++ b/streamnative/manifest.json @@ -0,0 +1,81 @@ +{ + "manifest_version": "2.0.0", + "app_uuid": "e92fa53b-f620-4167-bdaa-31ac3bc6be35", + "app_id": "streamnative", + "display_on_public_website": false, + "tile": { + "overview": "README.md#Overview", + "configuration": "README.md#Setup", + "support": "README.md#Support", + "changelog": "CHANGELOG.md", + "description": "Gain insights into StreamNative metrics data.", + "title": "StreamNative", + "media": [ + { + "caption": "StreamNative - Health", + "image_url": "images/streamnative_health.png", + "media_type": "image" + }, + { + "caption": "StreamNative - Kafka Connect", + "image_url": "images/streamnative_kafka_connect.png", + "media_type": "image" + }, + { + "caption": "StreamNative - Pulsar Resource", + "image_url": "images/streamnative_pulsar_resource.png", + "media_type": "image" + }, + { + "caption": "StreamNative - Sink Connector", + "image_url": "images/streamnative_sink_connector.png", + "media_type": "image" + }, + { + "caption": "StreamNative - Source Connector", + "image_url": "images/streamnative_source_connector.png", + "media_type": "image" + }], + "classifier_tags": [ + "Category::Metrics", + "Submitted Data Type::Metrics", + "Offering::Integration" + ] + }, + "assets": { + "integration": { + "auto_install": false, + "source_type_id": 27153739, + "source_type_name": "StreamNative", + "events": { + "creates_events": false + }, + "metrics": { + "prefix": "streamnative.", + "check": ["streamnative.pulsar_resource.pulsar_consumers_count"], + "metadata_path": "metadata.csv" + }, + "service_checks": { + "metadata_path": "assets/service_checks.json" + } + }, + "dashboards": { + "StreamNative - Health" : "assets/dashboards/streamnative_health.json", + "StreamNative - Kafka Connect" : "assets/dashboards/streamnative_kafka_connect.json", + "StreamNative - Pulsar Resource" : "assets/dashboards/streamnative_pulsar_resource.json", + "StreamNative - Sink Connector" : "assets/dashboards/streamnative_sink_connector.json", + "StreamNative - Source Connector" : "assets/dashboards/streamnative_source_connector.json" + }, + "monitors": { + "Backlog size exceeding threshold" : "assets/monitors/backlog_size_exceeding_threshold.json", + "Messaging service is down" : "assets/monitors/messaging_service_down.json", + "Webservice is down" : "assets/monitors/webservice_down.json" + } + }, + "author": { + "support_email": "help@datadoghq.com", + "name": "Datadog", + "homepage": "https://www.datadoghq.com", + "sales_email": "info@datadoghq.com" + } +} diff --git a/streamnative/metadata.csv b/streamnative/metadata.csv new file mode 100644 index 0000000000000..1a62208633a75 --- /dev/null +++ b/streamnative/metadata.csv @@ -0,0 +1,126 @@ +metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags +streamnative.health.pulsar_detector_e2e_latency_ms,gauge,,millisecond,,The latency distribution from message sending to message consumption,0,streamnative,Health Pulsar Detector E2E Latency,, +streamnative.health.pulsar_detector_geo_latency_ms,gauge,,millisecond,,The latency distribution Latency distribution from message sending to message consumption across clusters,0,streamnative,Health Pulsar Detector Geo Latency,, +streamnative.health.pulsar_detector_publish_latency_ms,gauge,,millisecond,,The latency distribution of message sending,0,streamnative,Health Pulsar Detector Publish Latency,, +streamnative.health.pulsar_detector_pulsar_sla_messaging_up,gauge,,,,The gauge for indicating the messaging service up or down,0,streamnative,Health Pulsar Detector Pulsar Sla Messaging Up,, +streamnative.health.pulsar_detector_pulsar_sla_webservice_up,gauge,,,,The gauge for indicating the webservice up or down,0,streamnative,Health Pulsar Detector Pulsar Sla Webservice Up,, +streamnative.kafka_connect.jvm_gc_collection_seconds_sum,gauge,,second,,Time spent in a given JVM garbage collector in seconds.,0,streamnative,Kafka Connect Jvm GC Collection Seconds Sum,, +streamnative.kafka_connect.jvm_memory_committed_bytes,gauge,,byte,,Committed bytes of a given JVM memory area,0,streamnative,Kafka Connect Jvm Memory Committed Bytes,, +streamnative.kafka_connect.jvm_memory_init_bytes,gauge,,byte,,Initial bytes of a given JVM memory area,0,streamnative,Kafka Connect Jvm Memory Init Bytes,, +streamnative.kafka_connect.jvm_memory_max_bytes,gauge,,byte,,Max bytes of a given JVM memory area,0,streamnative,Kafka Connect Jvm Memory Max Bytes,, +streamnative.kafka_connect.jvm_memory_used_bytes,gauge,,byte,,Used bytes of a given JVM memory area,0,streamnative,Kafka Connect Jvm Memory Used Bytes,, +streamnative.kafka_connect.kafka_connect_connector_task_batch_size_avg,gauge,,,,The average size of the batches processed by the connector,0,streamnative,Kafka Connect Connector Task Batch Size Avg,, +streamnative.kafka_connect.kafka_connect_connector_task_batch_size_max,gauge,,,,The maximum size of the batches processed by the connector,0,streamnative,Kafka Connect Connector Task Batch Size Max,, +streamnative.kafka_connect.kafka_connect_connector_task_offset_commit_avg_time_ms,gauge,,millisecond,,The average time in milliseconds taken by this task to commit offsets,0,streamnative,Kafka Connect Connector Task Offset Commit Avg Time,, +streamnative.kafka_connect.kafka_connect_connector_task_offset_commit_failure_percentage,gauge,,fraction,,The average percentage of this task's offset commit attempts that failed,0,streamnative,Kafka Connect Connector Task Offset Commit Failure Percentage,, +streamnative.kafka_connect.kafka_connect_connector_task_offset_commit_max_time_ms,gauge,,millisecond,,The maximum time in milliseconds taken by this task to commit offsets,0,streamnative,Kafka Connect Connector Task Offset Commit Max Time,, +streamnative.kafka_connect.kafka_connect_connector_task_offset_commit_success_percentage,gauge,,fraction,,The average percentage of this task's offset commit attempts that succeeded,0,streamnative,Kafka Connect Connector Task Offset Commit Success Percentage,, +streamnative.kafka_connect.kafka_connect_connector_task_pause_ratio,gauge,,fraction,,The fraction of time this task has spent in the pause state,0,streamnative,Kafka Connect Connector Task Pause Ratio,, +streamnative.kafka_connect.kafka_connect_connector_task_running_ratio,gauge,,fraction,,The fraction of time this task has spent in the running state,0,streamnative,Kafka Connect Connector Task Running Ratio,, +streamnative.kafka_connect.kafka_connect_sink_task_offset_commit_completion,gauge,,offset,,The total number of offset commit completions that were completed successfully,0,streamnative,Kafka Connect Sink Task Offset Commit Completion,, +streamnative.kafka_connect.kafka_connect_sink_task_offset_commit_completion_rate,gauge,,offset,second,The average per-second number of offset commit completions that were completed successfully,0,streamnative,Kafka Connect Sink Task Offset Commit Completion Rate,, +streamnative.kafka_connect.kafka_connect_sink_task_offset_commit_seq_no,gauge,,,,The current sequence number for offset commits,0,streamnative,Kafka Connect Sink Task Offset Commit Seq No,, +streamnative.kafka_connect.kafka_connect_sink_task_offset_commit_skip,gauge,,offset,,The total number of offset commit completions that were received too late and skipped/ignored,0,streamnative,Kafka Connect Sink Task Offset Commit Skip,, +streamnative.kafka_connect.kafka_connect_sink_task_offset_commit_skip_rate,gauge,,offset,second,The average per-second number of offset commit completions that were received too late and skipped/ignored,0,streamnative,Kafka Connect Sink Task Offset Commit Skip Rate,, +streamnative.kafka_connect.kafka_connect_sink_task_partition_count,gauge,,,,The number of topic partitions assigned to this task belonging to the named sink connector in this worker,0,streamnative,Kafka Connect Sink Task Partition Count,, +streamnative.kafka_connect.kafka_connect_sink_task_put_batch_avg_time_ms,gauge,,millisecond,,The average time taken by this task to put a batch of sinks records,0,streamnative,Kafka Connect Sink Task Put Batch Avg Time,, +streamnative.kafka_connect.kafka_connect_sink_task_put_batch_max_time_ms,gauge,,millisecond,,The maximum time taken by this task to put a batch of sinks records,0,streamnative,Kafka Connect Sink Task Put Batch Max Time,, +streamnative.kafka_connect.kafka_connect_sink_task_sink_record_active_count,gauge,,record,,The number of records that have been read from Kafka but not yet completely committed/flushed/acknowledged by the sink task,0,streamnative,Kafka Connect Sink Task Sink Record Active Count,, +streamnative.kafka_connect.kafka_connect_sink_task_sink_record_active_count_avg,gauge,,record,,The average number of records that have been read from Kafka but not yet completely committed/flushed/acknowledged by the sink task,0,streamnative,Kafka Connect Sink Task Sink Record Active Count Avg,, +streamnative.kafka_connect.kafka_connect_sink_task_sink_record_active_count_max,gauge,,record,,The maximum number of records that have been read from Kafka but not yet completely committed/flushed/acknowledged by the sink task,0,streamnative,Kafka Connect Sink Task Sink Record Active Count Max,, +streamnative.kafka_connect.kafka_connect_sink_task_sink_record_read,gauge,,record,,"The total number of records read from Kafka by this task belonging to the named sink connector in this worker, since the task was last restarted",0,streamnative,Kafka Connect Sink Task Sink Record Read,, +streamnative.kafka_connect.kafka_connect_sink_task_sink_record_read_rate,gauge,,record,second,The average per-second number of records read from Kafka for this task belonging to the named sink connector in this worker. This is before transformations are applied,0,streamnative,Kafka Connect Sink Task Sink Record Read Rate,, +streamnative.kafka_connect.kafka_connect_sink_task_sink_record_send,gauge,,record,,"The total number of records output from the transformations and sent/put to this task belonging to the named sink connector in this worker, since the task was last restarted",0,streamnative,Kafka Connect Sink Task Sink Record Send,, +streamnative.kafka_connect.kafka_connect_sink_task_sink_record_send_rate,gauge,,record,second,The average per-second number of records output from the transformations and sent/put to this task belonging to the named sink connector in this worker,0,streamnative,Kafka Connect Sink Task Sink Record Send Rate,, +streamnative.kafka_connect.kafka_connect_source_task_poll_batch_avg_time_ms,gauge,,millisecond,,The average time in milliseconds taken by this task to poll for a batch of source records,0,streamnative,Kafka Connect Source Task Poll Batch Avg Time,, +streamnative.kafka_connect.kafka_connect_source_task_poll_batch_max_time_ms,gauge,,millisecond,,The maximum time in milliseconds taken by this task to poll for a batch of source records,0,streamnative,Kafka Connect Source Task Poll Batch Max Time,, +streamnative.kafka_connect.kafka_connect_source_task_source_record_active_count,gauge,,record,,The number of records that have been produced by this task but not yet completely written to Kafka,0,streamnative,Kafka Connect Source Task Source Record Active Count,, +streamnative.kafka_connect.kafka_connect_source_task_source_record_active_count_avg,gauge,,record,,The average number of records that have been produced by this task but not yet completely written to Kafka,0,streamnative,Kafka Connect Source Task Source Record Active Count Avg,, +streamnative.kafka_connect.kafka_connect_source_task_source_record_active_count_max,gauge,,record,,The maximum number of records that have been produced by this task but not yet completely written to Kafka,0,streamnative,Kafka Connect Source Task Source Record Active Count Max,, +streamnative.kafka_connect.kafka_connect_source_task_source_record_poll,gauge,,record,,The total number of records produced/polled (before transformation) by this task belonging to the named source connector in this worker,0,streamnative,Kafka Connect Source Task Source Record Poll,, +streamnative.kafka_connect.kafka_connect_source_task_source_record_poll_rate,gauge,,record,second,The average per-second number of records produced/polled (before transformation) by this task belonging to the named source connector in this worker,0,streamnative,Kafka Connect Source Task Source Record Poll Rate,, +streamnative.kafka_connect.kafka_connect_source_task_source_record_write,gauge,,record,,"The number of records output from the transformations and written to Kafka for this task belonging to the named source connector in this worker, since the task was last restarted",0,streamnative,Kafka Connect Source Task Source Record Write,, +streamnative.kafka_connect.kafka_connect_source_task_source_record_write_rate,gauge,,record,second,The average per-second number of records output from the transformations and written to Kafka for this task belonging to the named source connector in this worker,0,streamnative,Kafka Connect Source Task Source Record Write Rate,, +streamnative.kafka_connect.kafka_connect_task_error_deadletterqueue_produce_failures,gauge,,,,The number of failed writes to the dead letter queue,0,streamnative,Kafka Connect Task Error Deadletterqueue Produce Failures,, +streamnative.kafka_connect.kafka_connect_task_error_deadletterqueue_produce_requests,gauge,,,,The number of attempted writes to the dead letter queue,0,streamnative,Kafka Connect Task Error Deadletterqueue Produce Requests,, +streamnative.kafka_connect.kafka_connect_task_error_last_error_timestamp,gauge,,millisecond,,The epoch timestamp when this task last encountered an error,0,streamnative,Kafka Connect Task Error Last Error Timestamp,, +streamnative.kafka_connect.kafka_connect_task_error_total_errors_logged,gauge,,error,,The total number of errors that were logged,0,streamnative,Kafka Connect Task Error Total Errors Logged,, +streamnative.kafka_connect.kafka_connect_task_error_total_record_errors,gauge,,record,,The total number of record processing errors in this task,0,streamnative,Kafka Connect Task Error Total Record Errors,, +streamnative.kafka_connect.kafka_connect_task_error_total_record_failures,gauge,,record,,The total number of record processing failures in this task,0,streamnative,Kafka Connect Task Error Total Record Failures,, +streamnative.kafka_connect.kafka_connect_task_error_total_records_skipped,gauge,,record,,The total number of records skipped due to errors,0,streamnative,Kafka Connect Task Error Total Records Skipped,, +streamnative.kafka_connect.kafka_connect_task_error_total_retries,gauge,,operation,,The total number of operations retried,0,streamnative,Kafka Connect Task Error Total Retries,, +streamnative.kafka_connect.kafka_connect_worker_connector_destroyed_task_count,gauge,,task,,The number of destroyed tasks of the connector on the worker,0,streamnative,Kafka Connect Worker Connector Destroyed Task Count,, +streamnative.kafka_connect.kafka_connect_worker_connector_failed_task_count,gauge,,task,,The number of failed tasks of the connector on the worker,0,streamnative,Kafka Connect Worker Connector Failed Task Count,, +streamnative.kafka_connect.kafka_connect_worker_connector_paused_task_count,gauge,,task,,The number of paused tasks of the connector on the worker,0,streamnative,Kafka Connect Worker Connector Paused Task Count,, +streamnative.kafka_connect.kafka_connect_worker_connector_restarting_task_count,gauge,,task,,The number of restarting tasks of the connector on the worker,0,streamnative,Kafka Connect Worker Connector Restarting Task Count,, +streamnative.kafka_connect.kafka_connect_worker_connector_running_task_count,gauge,,task,,The number of running tasks of the connector on the worker,0,streamnative,Kafka Connect Worker Connector Running Task Count,, +streamnative.kafka_connect.kafka_connect_worker_connector_total_task_count,gauge,,task,,The number of tasks of the connector on the worker,0,streamnative,Kafka Connect Worker Connector Total Task Count,, +streamnative.kafka_connect.kafka_connect_worker_connector_unassigned_task_count,gauge,,task,,The number of unassigned tasks of the connector on the worker,0,streamnative,Kafka Connect Worker Connector Unassigned Task Count,, +streamnative.kafka_connect.process_cpu_seconds_total,gauge,,second,,Total user and system CPU time spent in seconds,0,streamnative,Kafka Connect Process Cpu Seconds Total,, +streamnative.pulsar_resource.pulsar_consumers_count,gauge,,,,The number of active consumers of the topic connected to this broker.,0,streamnative,Pulsar Resource Pulsar Consumers Count,, +streamnative.pulsar_resource.pulsar_entry_size_le_100_kb,gauge,,fraction,,The entry rate for a topic with entry size smaller than or equal to 100 kilobytes,0,streamnative,Pulsar Resource Pulsar Entry Size Le_100 KB,, +streamnative.pulsar_resource.pulsar_entry_size_le_128,gauge,,fraction,,The entry rate for a topic with entry size smaller than or equal to 128 bytes,0,streamnative,Pulsar Resource Pulsar Entry Size Le_128,, +streamnative.pulsar_resource.pulsar_entry_size_le_16_kb,gauge,,fraction,,The entry rate for a topic with entry size smaller than or equal to 16 kilobytes,0,streamnative,Pulsar Resource Pulsar Entry Size Le_16 KB,, +streamnative.pulsar_resource.pulsar_entry_size_le_1_kb,gauge,,fraction,,The entry rate for a topic with entry size smaller than or equal to 1 kilobyte,0,streamnative,Pulsar Resource Pulsar Entry Size Le_1 KB,, +streamnative.pulsar_resource.pulsar_entry_size_le_1_mb,gauge,,fraction,,The entry rate for a topic with entry size smaller than or equal to 1 megabyte,0,streamnative,Pulsar Resource Pulsar Entry Size Le_1 MB,, +streamnative.pulsar_resource.pulsar_entry_size_le_2_kb,gauge,,fraction,,The entry rate for a topic with entry size smaller than or equal to 2 kilobytes,0,streamnative,Pulsar Resource Pulsar Entry Size Le_2 KB,, +streamnative.pulsar_resource.pulsar_entry_size_le_4_kb,gauge,,fraction,,The entry rate for a topic with entry size smaller than or equal to 4 kilobytes,0,streamnative,Pulsar Resource Pulsar Entry Size Le_4 KB,, +streamnative.pulsar_resource.pulsar_entry_size_le_512,gauge,,fraction,,The entry rate for a topic with entry size smaller than or equal to 512 bytes,0,streamnative,Pulsar Resource Pulsar Entry Size Le_512,, +streamnative.pulsar_resource.pulsar_entry_size_le_overflow,gauge,,fraction,,The entry rate for a topic with entry size greater than 1 megabyte,0,streamnative,Pulsar Resource Pulsar Entry Size Le_Overflow,, +streamnative.pulsar_resource.pulsar_producers_count,gauge,,,,The number of active producers of the topic connected to this broker,0,streamnative,Pulsar Resource Pulsar Producers Count,, +streamnative.pulsar_resource.pulsar_rate_in,gauge,,message,second,The total message rate of the namespace coming into this broker,0,streamnative,Pulsar Resource Pulsar Rate In,, +streamnative.pulsar_resource.pulsar_rate_out,gauge,,message,second,The total message rate of the namespace going out from this broker,0,streamnative,Pulsar Resource Pulsar Rate Out,, +streamnative.pulsar_resource.pulsar_storage_backlog_size,gauge,,byte,,The total backlog size of the topics of this topic owned by this broker,0,streamnative,Pulsar Resource Pulsar Storage Backlog Size,, +streamnative.pulsar_resource.pulsar_storage_offloaded_size,gauge,,byte,,The total amount of the data in this topic offloaded to the tiered storage,0,streamnative,Pulsar Resource Pulsar Storage Offloaded Size,, +streamnative.pulsar_resource.pulsar_storage_read_rate,gauge,,,,The total message batches (entries) read from the storage for this topic,0,streamnative,Pulsar Resource Pulsar Storage Read Rate,, +streamnative.pulsar_resource.pulsar_storage_size,gauge,,byte,,The total storage size of the topics in this topic owned by this broker,0,streamnative,Pulsar Resource Pulsar Storage Size,, +streamnative.pulsar_resource.pulsar_storage_write_latency_le_0_5,gauge,,fraction,,The entry rate for a topic where the storage write latency is less than or equal to 0.5 milliseconds,0,streamnative,Pulsar Resource Pulsar Storage Write Latency Le_0_5,, +streamnative.pulsar_resource.pulsar_storage_write_latency_le_1,gauge,,fraction,,The entry rate for a topic where the storage write latency is less than or equal to 1 millisecond,0,streamnative,Pulsar Resource Pulsar Storage Write Latency Le_1,, +streamnative.pulsar_resource.pulsar_storage_write_latency_le_10,gauge,,fraction,,The entry rate for a topic where the storage write latency is less than or equal to 10 milliseconds,0,streamnative,Pulsar Resource Pulsar Storage Write Latency Le_10,, +streamnative.pulsar_resource.pulsar_storage_write_latency_le_100,gauge,,fraction,,The entry rate for a topic where the storage write latency is less than or equal to 100 milliseconds,0,streamnative,Pulsar Resource Pulsar Storage Write Latency Le_100,, +streamnative.pulsar_resource.pulsar_storage_write_latency_le_1000,gauge,,fraction,,The entry rate for a topic where the storage write latency is less than or equal to 1000 milliseconds,0,streamnative,Pulsar Resource Pulsar Storage Write Latency Le_1000,, +streamnative.pulsar_resource.pulsar_storage_write_latency_le_20,gauge,,fraction,,The entry rate for a topic where the storage write latency is less than or equal to 20 milliseconds,0,streamnative,Pulsar Resource Pulsar Storage Write Latency Le_20,, +streamnative.pulsar_resource.pulsar_storage_write_latency_le_200,gauge,,fraction,,The entry rate for a topic where the storage write latency is less than or equal to 200 milliseconds,0,streamnative,Pulsar Resource Pulsar Storage Write Latency Le_200,, +streamnative.pulsar_resource.pulsar_storage_write_latency_le_5,gauge,,fraction,,The entry rate for a topic where the storage write latency is less than or equal to 5 milliseconds,0,streamnative,Pulsar Resource Pulsar Storage Write Latency Le_5,, +streamnative.pulsar_resource.pulsar_storage_write_latency_le_50,gauge,,fraction,,The entry rate for a topic where the storage write latency is less than or equal to 50 milliseconds,0,streamnative,Pulsar Resource Pulsar Storage Write Latency Le_50,, +streamnative.pulsar_resource.pulsar_storage_write_latency_le_overflow,gauge,,fraction,,The entry rate for a topic where the storage write latency is greater than 1 second,0,streamnative,Pulsar Resource Pulsar Storage Write Latency Le_Overflow,, +streamnative.pulsar_resource.pulsar_storage_write_rate,gauge,,,,The total message batches (entries) written to the storage for this topic,0,streamnative,Pulsar Resource Pulsar Storage Write Rate,, +streamnative.pulsar_resource.pulsar_subscription_delayed,gauge,,,,The total message batches (entries) are delayed for dispatching,0,streamnative,Pulsar Resource Pulsar Subscription Delayed,, +streamnative.pulsar_resource.pulsar_subscriptions_count,gauge,,,,The number of Pulsar subscriptions of the topic served by this broker,0,streamnative,Pulsar Resource Pulsar Subscriptions Count,, +streamnative.pulsar_resource.pulsar_throughput_in,gauge,,byte,second,The total throughput of the topic coming into this broker,0,streamnative,Pulsar Resource Pulsar Throughput In,, +streamnative.pulsar_resource.pulsar_throughput_out,gauge,,byte,second,The total throughput of the topic going out from this broker,0,streamnative,Pulsar Resource Pulsar Throughput Out,, +streamnative.pulsar_resource.pulsar_topics_count,gauge,,,,The number of Pulsar topics of the namespace owned by this broker,0,streamnative,Pulsar Resource Pulsar Topics Count,, +streamnative.sink_connector.jvm_gc_collection_seconds_sum,gauge,,second,,Time spent in a given JVM garbage collector in seconds,0,streamnative,Sink Connector Jvm GC Collection Seconds Sum,, +streamnative.sink_connector.jvm_memory_bytes_committed,gauge,,byte,,Committed bytes of a given JVM memory area,0,streamnative,Sink Connector Jvm Memory Bytes Committed,, +streamnative.sink_connector.jvm_memory_bytes_init,gauge,,byte,,Initial bytes of a given JVM memory area,0,streamnative,Sink Connector Jvm Memory Bytes Init,, +streamnative.sink_connector.jvm_memory_bytes_max,gauge,,byte,,Max bytes of a given JVM memory area,0,streamnative,Sink Connector Jvm Memory Bytes Max,, +streamnative.sink_connector.jvm_memory_direct_bytes_used,gauge,,byte,,Used bytes of a given JVM memory area,0,streamnative,Sink Connector Jvm Memory Bytes Used,, +streamnative.sink_connector.process_cpu_seconds_total,gauge,,second,,Total user and system CPU time spent in seconds,0,streamnative,Sink Connector Process Cpu Seconds Total,, +streamnative.sink_connector.pulsar_sink_last_invocation,gauge,,millisecond,,The timestamp of the last invocation of the sink,0,streamnative,Sink Connector Pulsar Sink Last Invocation,, +streamnative.sink_connector.pulsar_sink_received_1min_total,gauge,,record,,The total number of records received from sink in the last 1 minute,0,streamnative,Sink Connector Pulsar Sink Received 1min Total,, +streamnative.sink_connector.pulsar_sink_received_total,gauge,,record,,The total number of records received from sink,0,streamnative,Sink Connector Pulsar Sink Received Total,, +streamnative.sink_connector.pulsar_sink_sink_exception,gauge,,,,The exception from a sink,0,streamnative,Sink Connector Pulsar Sink Sink Exception,, +streamnative.sink_connector.pulsar_sink_sink_exceptions_1min_total,gauge,,exception,,The total number of sink exceptions in the last 1 minute,0,streamnative,Sink Connector Pulsar Sink Sink Exceptions 1min Total,, +streamnative.sink_connector.pulsar_sink_sink_exceptions_total,gauge,,exception,,The total number of sink exceptions,0,streamnative,Sink Connector Pulsar Sink Sink Exceptions Total,, +streamnative.sink_connector.pulsar_sink_system_exception,gauge,,,,The exception from system code,0,streamnative,Sink Connector Pulsar Sink System Exception,, +streamnative.sink_connector.pulsar_sink_system_exceptions_1min_total,gauge,,exception,,The total number of system exceptions in the last 1 minute,0,streamnative,Sink Connector Pulsar Sink System Exceptions 1min Total,, +streamnative.sink_connector.pulsar_sink_system_exceptions_total,gauge,,exception,,The total number of system exceptions,0,streamnative,Sink Connector Pulsar Sink System Exceptions Total,, +streamnative.sink_connector.pulsar_sink_written_1min_total,gauge,,record,,The total number of records written to a Pulsar topic in the last 1 minute,0,streamnative,Sink Connector Pulsar Sink Written_1min Total,, +streamnative.sink_connector.pulsar_sink_written_total,gauge,,record,,The total number of records written to a Pulsar topic,0,streamnative,Sink Connector Pulsar Sink Written Total,, +streamnative.source_connector.jvm_gc_collection_seconds_sum,gauge,,second,,Time spent in a given JVM garbage collector in seconds,0,streamnative,Source Connector Jvm GC Collection Seconds Sum,, +streamnative.source_connector.jvm_memory_bytes_committed,gauge,,byte,,Committed bytes of a given JVM memory area,0,streamnative,Source Connector Jvm Memory Bytes Committed,, +streamnative.source_connector.jvm_memory_bytes_init,gauge,,byte,,Initial bytes of a given JVM memory area,0,streamnative,Source Connector Jvm Memory Bytes Init,, +streamnative.source_connector.jvm_memory_bytes_max,gauge,,byte,,Max bytes of a given JVM memory area,0,streamnative,Source Connector Jvm Memory Bytes Max,, +streamnative.source_connector.jvm_memory_direct_bytes_used,gauge,,byte,,Used bytes of a given JVM memory area,0,streamnative,Source Connector Jvm Memory Direct Bytes Used,, +streamnative.source_connector.process_cpu_seconds_total,gauge,,second,,Total user and system CPU time spent in seconds,0,streamnative,Source Connector Process Cpu Seconds Total,, +streamnative.source_connector.pulsar_source_last_invocation,gauge,,millisecond,,The timestamp of the last invocation of the source,0,streamnative,Source Connector Pulsar Source Last Invocation,, +streamnative.source_connector.pulsar_source_received_1min_total,gauge,,record,,The total number of records received from source in the last 1 minute,0,streamnative,Source Connector Pulsar Source Received 1min Total,, +streamnative.source_connector.pulsar_source_received_total,gauge,,record,,The total number of records received from source,0,streamnative,Source Connector Pulsar Source Received Total,, +streamnative.source_connector.pulsar_source_source_exception,gauge,,,,The exception from a source,0,streamnative,Source Connector Pulsar Source Source Exception,, +streamnative.source_connector.pulsar_source_source_exceptions_1min_total,gauge,,exception,,The total number of source exceptions in the last 1 minute,0,streamnative,Source Connector Pulsar Source Source Exceptions 1min Total,, +streamnative.source_connector.pulsar_source_source_exceptions_total,gauge,,exception,,The total number of source exceptions,0,streamnative,Source Connector Pulsar Source Source Exceptions Total,, +streamnative.source_connector.pulsar_source_system_exception,gauge,,,,The exception from system code,0,streamnative,Source Connector Pulsar Source System Exception,, +streamnative.source_connector.pulsar_source_system_exceptions_1min_total,gauge,,exception,,The total number of system exceptions in the last 1 minute,0,streamnative,Source Connector Pulsar Source System Exceptions 1min Total,, +streamnative.source_connector.pulsar_source_system_exceptions_total,gauge,,exception,,The total number of system exceptions,0,streamnative,Source Connector Pulsar Source System Exceptions Total,, +streamnative.source_connector.pulsar_source_written_1min_total,gauge,,record,,The total number of records written to a Pulsar topic in the last 1 minute,0,streamnative,Source Connector Pulsar Source Written 1min Total,, +streamnative.source_connector.pulsar_source_written_total,gauge,,record,,The total number of records written to a Pulsar topic,0,streamnative,Source Connector Pulsar Source Written Total,, \ No newline at end of file diff --git a/symantec_endpoint_protection/CHANGELOG.md b/symantec_endpoint_protection/CHANGELOG.md index 52576ce2b70ae..058b2807cc86f 100644 --- a/symantec_endpoint_protection/CHANGELOG.md +++ b/symantec_endpoint_protection/CHANGELOG.md @@ -1,3 +1,9 @@ # CHANGELOG - symantec-endpoint-protection + +## 1.0.0 / 2024-11-28 + +***Added***: + +* Initial Release ([#18714](https://github.com/DataDog/integrations-core/pull/18714)) diff --git a/symantec_endpoint_protection/changelog.d/18714.added b/symantec_endpoint_protection/changelog.d/18714.added deleted file mode 100644 index aa949b47b7b41..0000000000000 --- a/symantec_endpoint_protection/changelog.d/18714.added +++ /dev/null @@ -1 +0,0 @@ -Initial Release \ No newline at end of file diff --git a/symantec_endpoint_protection/datadog_checks/symantec_endpoint_protection/__about__.py b/symantec_endpoint_protection/datadog_checks/symantec_endpoint_protection/__about__.py index e9541ce83e9e5..acbfd1c866b84 100644 --- a/symantec_endpoint_protection/datadog_checks/symantec_endpoint_protection/__about__.py +++ b/symantec_endpoint_protection/datadog_checks/symantec_endpoint_protection/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '0.0.1' +__version__ = '1.0.0' diff --git a/temporal_cloud/CHANGELOG.md b/temporal_cloud/CHANGELOG.md new file mode 100644 index 0000000000000..a64023ed3d344 --- /dev/null +++ b/temporal_cloud/CHANGELOG.md @@ -0,0 +1,7 @@ +# CHANGELOG - Temporal_Cloud + +## 1.0.0 / 2024-11-26 + +***Added***: + +* Initial Release diff --git a/temporal_cloud/README.md b/temporal_cloud/README.md new file mode 100644 index 0000000000000..c780d1f397562 --- /dev/null +++ b/temporal_cloud/README.md @@ -0,0 +1,41 @@ +## Overview + +This check monitors [Temporal_Cloud][1]. + +## Setup + +### Installation + +The Temporal_Cloud check is included in the [Datadog Agent][2] package. +No additional installation is needed on your server. + +### Configuration + +!!! Add list of steps to set up this integration !!! + +### Validation + +!!! Add steps to validate integration is functioning as expected !!! + +## Data Collected + +### Metrics + +Temporal_Cloud does not include any metrics. + +### Service Checks + +Temporal_Cloud does not include any service checks. + +### Events + +Temporal_Cloud does not include any events. + +## Troubleshooting + +Need help? Contact [Datadog support][3]. + +[1]: **LINK_TO_INTEGRATION_SITE** +[2]: https://app.datadoghq.com/account/settings/agent/latest +[3]: https://docs.datadoghq.com/help/ + diff --git a/temporal_cloud/assets/service_checks.json b/temporal_cloud/assets/service_checks.json new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/temporal_cloud/assets/service_checks.json @@ -0,0 +1 @@ +[] diff --git a/temporal_cloud/manifest.json b/temporal_cloud/manifest.json new file mode 100644 index 0000000000000..6a777edc5ab90 --- /dev/null +++ b/temporal_cloud/manifest.json @@ -0,0 +1,44 @@ +{ + "manifest_version": "2.0.0", + "app_uuid": "4fc358f8-ab2d-43ae-86e5-129ef4e4e6a1", + "app_id": "temporal-cloud", + "display_on_public_website": false, + "tile": { + "overview": "README.md#Overview", + "configuration": "README.md#Setup", + "support": "README.md#Support", + "changelog": "CHANGELOG.md", + "description": "", + "title": "Temporal Cloud", + "media": [], + "classifier_tags": [ + "Category::Metrics", + "Offering::Integration", + "Submitted Data Type::Metrics" + ] + }, + "assets": { + "integration": { + "auto_install": false, + "source_type_id": 32597071, + "source_type_name": "Temporal Cloud", + "events": { + "creates_events": false + }, + "metrics": { + "prefix": "temporal_cloud.", + "check": [], + "metadata_path": "metadata.csv" + }, + "service_checks": { + "metadata_path": "assets/service_checks.json" + } + } + }, + "author": { + "support_email": "help@datadoghq.com", + "name": "Datadog", + "homepage": "https://www.datadoghq.com", + "sales_email": "info@datadoghq.com" + } +} diff --git a/temporal_cloud/metadata.csv b/temporal_cloud/metadata.csv new file mode 100644 index 0000000000000..02cde5e98381e --- /dev/null +++ b/temporal_cloud/metadata.csv @@ -0,0 +1 @@ +metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags diff --git a/tls/CHANGELOG.md b/tls/CHANGELOG.md index 4f6d290038601..b2068b73f2f7f 100644 --- a/tls/CHANGELOG.md +++ b/tls/CHANGELOG.md @@ -36,7 +36,7 @@ ***Added***: -* Update dependencies ([#18185](https://github.com/DataDog/integrations-core/pull/18185)) +* Update dependencies ([#18187](https://github.com/DataDog/integrations-core/pull/18187)) ## 2.18.0 / 2024-07-05 / Agent 7.56.0 diff --git a/vault/tests/test_integration.py b/vault/tests/test_integration.py index 77e2802e46dd3..3097f19d7ffa2 100644 --- a/vault/tests/test_integration.py +++ b/vault/tests/test_integration.py @@ -10,12 +10,24 @@ @auth_required @pytest.mark.usefixtures('dd_environment') -@pytest.mark.flaky @pytest.mark.integration -@pytest.mark.parametrize('use_openmetrics', [False, True], indirect=True) -@pytest.mark.parametrize('use_auth_file', [False, True]) -def test_integration(aggregator, dd_run_check, check, instance, global_tags, use_openmetrics, use_auth_file): - instance = dict(instance(use_auth_file)) +@pytest.mark.parametrize('use_openmetrics', [True, False], indirect=True, ids=['legacy', 'openmetrics']) +def test_integration(aggregator, dd_run_check, check, instance, global_tags, use_openmetrics): + instance = dict(instance(False)) + instance['use_openmetrics'] = use_openmetrics + + check = check(instance) + dd_run_check(check) + + assert_collection(aggregator, global_tags, use_openmetrics) + + +@auth_required +@pytest.mark.usefixtures('dd_environment') +@pytest.mark.integration +@pytest.mark.parametrize('use_openmetrics', [True, False], indirect=True, ids=['legacy', 'openmetrics']) +def test_integration_auth_file(aggregator, dd_run_check, check, instance, global_tags, use_openmetrics): + instance = dict(instance(True)) instance['use_openmetrics'] = use_openmetrics check = check(instance) @@ -26,9 +38,8 @@ def test_integration(aggregator, dd_run_check, check, instance, global_tags, use @noauth_required @pytest.mark.usefixtures('dd_environment') -@pytest.mark.flaky @pytest.mark.integration -@pytest.mark.parametrize('use_openmetrics', [False, True], indirect=True) +@pytest.mark.parametrize('use_openmetrics', [False], indirect=True, ids=['legacy']) def test_integration_noauth(aggregator, dd_run_check, check, no_token_instance, global_tags, use_openmetrics): instance = dict(no_token_instance) instance['use_openmetrics'] = use_openmetrics diff --git a/vertica/CHANGELOG.md b/vertica/CHANGELOG.md index 96f8b93f65ddc..380420c558321 100644 --- a/vertica/CHANGELOG.md +++ b/vertica/CHANGELOG.md @@ -26,7 +26,7 @@ ***Added***: -* Update dependencies ([#18185](https://github.com/DataDog/integrations-core/pull/18185)) +* Update dependencies ([#18187](https://github.com/DataDog/integrations-core/pull/18187)) ## 4.5.0 / 2024-03-22 / Agent 7.53.0 diff --git a/vllm/README.md b/vllm/README.md index 420aa6cfcf32e..b69854afb0e9a 100644 --- a/vllm/README.md +++ b/vllm/README.md @@ -48,6 +48,10 @@ In either case, make sure that the `source` value for your logs is `vllm`. This Need help? Contact [Datadog support][9]. +## Further Reading +Additional helpful documentation, links, and articles: +- [Optimize LLM application performance with Datadog's vLLM integration][13] + [1]: https://docs.vllm.ai/en/stable/ [2]: https://app.datadoghq.com/account/settings/agent/latest @@ -60,3 +64,4 @@ Need help? Contact [Datadog support][9]. [10]: https://docs.datadoghq.com/containers/docker/log/?tab=containerinstallation#installation [11]: https://docs.datadoghq.com/containers/docker/log/?tab=hostagent#installation [12]: https://docs.datadoghq.com/containers/docker/log/?tab=dockerfile#log-integrations +[13]: https://www.datadoghq.com/blog/vllm-integration/ \ No newline at end of file diff --git a/vllm/assets/dashboards/overview.json b/vllm/assets/dashboards/overview.json index dac7d8cf0f7b5..5df9e28ee5115 100644 --- a/vllm/assets/dashboards/overview.json +++ b/vllm/assets/dashboards/overview.json @@ -448,7 +448,7 @@ "aggregator": "avg", "data_source": "metrics", "name": "query1", - "query": "avg:vllm.avg.generation_throughput.toks_per_s{$model_name}" + "query": "sum:vllm.avg.generation_throughput.toks_per_s{$model_name}" } ], "response_format": "scalar" diff --git a/vllm/manifest.json b/vllm/manifest.json index c81c4669a8f8b..b5d9c61852e7a 100644 --- a/vllm/manifest.json +++ b/vllm/manifest.json @@ -19,6 +19,12 @@ "Category::AI/ML", "Submitted Data Type::Metrics", "Offering::Integration" + ], + "resources": [ + { + "resource_type": "blog", + "url": "https://www.datadoghq.com/blog/vllm-integration/" + } ] }, "assets": { diff --git a/vsphere/README.md b/vsphere/README.md index 2a341391352aa..7296b1cabbcc5 100644 --- a/vsphere/README.md +++ b/vsphere/README.md @@ -97,7 +97,11 @@ See [service_checks.json][12] for a list of service checks provided by this inte You can limit the number of VMs pulled in with the VMWare integration using the `vsphere.d/conf.yaml` file. See the `resource_filters` parameter section in the [sample vsphere.d/conf.yaml][4]. -### Monitoring vSphere Tanzu Kubernetes Grid (TKG) +## Billing + +- [vSphere Integration Billing][17] + +## Monitoring vSphere Tanzu Kubernetes Grid (TKG) The Datadog vSphere integration collects metrics and events from your [TKG][13] VMs and control plane VMs automatically. To collect more granular information about your TKG cluster, including container-, pod-, and node-level metrics, you can install the [Datadog Agent][14] on your cluster. See the [distribution documentation][15] for example configuration files specific to TKG. @@ -121,3 +125,4 @@ The Datadog vSphere integration collects metrics and events from your [TKG][13] [14]: https://docs.datadoghq.com/containers/kubernetes/installation/?tab=operator [15]: https://docs.datadoghq.com/containers/kubernetes/distributions/?tab=operator#TKG [16]: https://www.datadoghq.com/blog/unified-vsphere-app-monitoring-datadog/#auto-discovery-across-vm-and-app-layers +[17]: https://docs.datadoghq.com/account_management/billing/vsphere diff --git a/wazuh/CHANGELOG.md b/wazuh/CHANGELOG.md index 340981538e0aa..577348c192531 100644 --- a/wazuh/CHANGELOG.md +++ b/wazuh/CHANGELOG.md @@ -2,3 +2,8 @@ +## 1.0.0 / 2024-11-28 + +***Added***: + +* Initial Release ([#18646](https://github.com/DataDog/integrations-core/pull/18646)) diff --git a/wazuh/changelog.d/18646.added b/wazuh/changelog.d/18646.added deleted file mode 100644 index aa949b47b7b41..0000000000000 --- a/wazuh/changelog.d/18646.added +++ /dev/null @@ -1 +0,0 @@ -Initial Release \ No newline at end of file diff --git a/wazuh/datadog_checks/wazuh/__about__.py b/wazuh/datadog_checks/wazuh/__about__.py index e9541ce83e9e5..acbfd1c866b84 100644 --- a/wazuh/datadog_checks/wazuh/__about__.py +++ b/wazuh/datadog_checks/wazuh/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '0.0.1' +__version__ = '1.0.0'