From 3ae6bd762a6dd885c17ae9d0f07a6a00d0ea854b Mon Sep 17 00:00:00 2001 From: Mike Garabedian Date: Wed, 24 Feb 2021 13:36:28 -0500 Subject: [PATCH] Add support for ActiveMQ Artemis (#8527) * Initial Artemis JMX integration * Add artemis metrics and e2e env * Add test queues for artemis env * Remove new integration files * Update metadata * Update e2e tests * Fix style * Update README * Add address metrics * Update tests * Update activemq/README.md Co-authored-by: Christine Chen * Update metric types * Fix metadata * Update metric types and add rate test Co-authored-by: Christine Chen --- activemq/README.md | 4 +- .../datadog_checks/activemq/data/metrics.yaml | 84 +++++++++++++++++++ activemq/metadata.csv | 23 +++++ activemq/tests/common.py | 40 ++++++++- .../{docker-compose.yaml => activemq.yaml} | 1 + activemq/tests/compose/artemis.yaml | 23 +++++ activemq/tests/conftest.py | 62 +++++++++++--- activemq/tests/test_check.py | 18 +++- activemq/tox.ini | 7 +- 9 files changed, 247 insertions(+), 15 deletions(-) rename activemq/tests/compose/{docker-compose.yaml => activemq.yaml} (99%) create mode 100644 activemq/tests/compose/artemis.yaml diff --git a/activemq/README.md b/activemq/README.md index 5ac074a3af0b0..47d3c0f653061 100644 --- a/activemq/README.md +++ b/activemq/README.md @@ -4,6 +4,8 @@ The ActiveMQ check collects metrics for brokers and queues, producers and consumers, and more. +**Note:** This check also supports ActiveMQ Artemis (future ActiveMQ version `6`) and reports metrics under the `activemq.artemis` namespace. See [metrics][#metrics] for more details. + **Note**: If you are running a ActiveMQ version older than 5.8.0, see the [Agent 5.10.x released sample files][1]. ## Setup @@ -103,7 +105,7 @@ Collecting logs is disabled by default in the Datadog Agent. To enable it, see [ ### Metrics -See [metadata.csv][9] for a list of metrics provided by this integration. +See [metadata.csv][9] for a list of metrics provided by this integration. Metrics associated with ActiveMQ Artemis flavor have `artemis` in their metric name, all others are reported for ActiveMQ "classic". ### Events diff --git a/activemq/datadog_checks/activemq/data/metrics.yaml b/activemq/datadog_checks/activemq/data/metrics.yaml index 41d6593299ec0..3f66c665ca667 100644 --- a/activemq/datadog_checks/activemq/data/metrics.yaml +++ b/activemq/datadog_checks/activemq/data/metrics.yaml @@ -54,3 +54,87 @@ jmx_metrics: MemoryPercentUsage: alias: activemq.broker.memory_pct metric_type: gauge + + - include: + domain: org.apache.activemq.artemis + attribute: + AddressMemoryUsage: + alias: activemq.artemis.address_memory_usage + metric_type: gauge + AddressMemoryUsagePercentage: + alias: activemq.artemis.address_memory_usage_pct + metric_type: gauge + MaxDiskUsage: + alias: activemq.artemis.max_disk_usage + metric_type: gauge + DiskStoreUsage: + alias: activemq.artemis.disk_store_usage_pct + metric_type: gauge + ConnectionCount: + alias: activemq.artemis.connection_count + metric_type: counter + TotalConnectionCount: + alias: activemq.artemis.total_connection_count + metric_type: monotonic_count + TotalMessageCount: + alias: activemq.artemis.total_message_count + metric_type: monotonic_count + TotalMessagesAdded: + alias: activemq.artemis.total_messages_added + metric_type: monotonic_count + TotalMessagesAcknowledged: + alias: activemq.artemis.total_messages_acknowledged + metric_type: monotonic_count + TotalConsumerCount: + alias: activemq.artemis.total_consumer_count + metric_type: monotonic_count + + - include: + domain: org.apache.activemq.artemis + component: addresses + attribute: + AddressSize: + alias: activemq.artemis.address.size + metric_type: gauge + NumberOfPages: + alias: activemq.artemis.address.pages_count + metric_type: counter + NumberOfMessages: + alias: activemq.artemis.address.number_of_messages + metric_type: monotonic_count + NumberOfBytesPerPage: + alias: activemq.artemis.address.bytes_per_page + metric_type: gauge + RoutedMessageCount: + alias: activemq.artemis.address.routed_messages + metric_type: monotonic_count + UnRoutedMessageCount: + alias: activemq.artemis.address.unrouted_messages + metric_type: monotonic_count + + - include: + domain: org.apache.activemq.artemis + subcomponent: queues + attribute: + MessageCount: + alias: activemq.artemis.queue.message_count + metric_type: monotonic_count + ConsumerCount: + alias: activemq.artemis.queue.consumer_count + metric_type: gauge + MaxConsumers: + alias: activemq.artemis.queue.max_consumers + metric_type: gauge + MessagesAdded: + alias: activemq.artemis.queue.messages_added + metric_type: monotonic_count + MessagesExpired: + # implied as monotonic count, reports as gauge + alias: activemq.artemis.queue.messages_expired + metric_type: gauge + MessagesAcknowledged: + alias: activemq.artemis.queue.messages_acknowledged + metric_type: monotonic_count + MessagesKilled: + alias: activemq.artemis.queue.messages_killed + metric_type: monotonic_count diff --git a/activemq/metadata.csv b/activemq/metadata.csv index 226e5378feedc..fd278a95c8a88 100644 --- a/activemq/metadata.csv +++ b/activemq/metadata.csv @@ -14,3 +14,26 @@ activemq.queue.in_flight_count,gauge,10,message,,The amount of messages that hav activemq.broker.store_pct,gauge,10,percent,,The percentage of store in use.,0,activemq,str pct activemq.broker.temp_pct,gauge,10,percent,,The percentage of temporary in use.,0,activemq,temp pct activemq.broker.memory_pct,gauge,10,percent,,The percentage of memory in use.,0,activemq,mem pct +activemq.artemis.address_memory_usage,gauge,,,,(Artemis only) Memory used by all the addresses on broker for in-memory messages,0,activemq,art mem usage +activemq.artemis.address_memory_usage_pct,gauge,,,,(Artemis only) Memory used by all the addresses on broker as a percentage of the global-max-size,0,activemq,art mem pct +activemq.artemis.max_disk_usage,gauge,,,,(Artemis only) Maximum limit for disk use in percentage,0,activemq,art max disk +activemq.artemis.disk_store_usage_pct,gauge,,,,(Artemis only) Percentage of total disk store used,0,activemq,art disk pct +activemq.artemis.connection_count,gauge,,,,(Artemis only) Number of clients connected to this server,0,activemq,art conn count +activemq.artemis.total_connection_count,rate,,,,(Artemis only) Number of clients which have connected to this server since it was started,0,activemq,art total conns +activemq.artemis.total_message_count,rate,,,,(Artemis only) Number of messages in all queues on the server,0,activemq,art total messages +activemq.artemis.total_messages_added,rate,,,,(Artemis only) Number of messages sent to this server since it was started,0,activemq,art total mess added +activemq.artemis.total_messages_acknowledged,rate,,,,(Artemis only) Number of messages acknowledged from all the queues on this server since it was started,0,activemq,art total mess acked +activemq.artemis.total_consumer_count,rate,,,,(Artemis only) Number of consumers consuming messages from all the queues on this server,0,activemq,art total consumers +activemq.artemis.address.size,gauge,,,,(Artemis only) Number of estimated bytes being used by all the queue(s) bound to this address; used to control paging and blocking,0,activemq,add size +activemq.artemis.address.pages_count,gauge,,,,(Artemis only) Number of pages used by this address,0,activemq,add pages +activemq.artemis.address.number_of_messages,rate,,,,"(Artemis only) The sum of messages on queue(s), including messages in delivery",0,activemq,add num mess +activemq.artemis.address.bytes_per_page,gauge,,,,(Artemis only) Number of bytes used by each page for this address,0,activemq,add bytes page +activemq.artemis.address.routed_messages,rate,,,,(Artemis only) Number of messages routed to one or more bindings,0,activemq,add routed +activemq.artemis.address.unrouted_messages,rate,,,,(Artemis only) Number of messages not routed to any bindings,0,activemq,add unrouted +activemq.artemis.queue.message_count,rate,,,,"(Artemis only) Number of messages currently in this queue (includes scheduled, paged, and in-delivery messages)",0,activemq,art q mess count +activemq.artemis.queue.consumer_count,gauge,,,,(Artemis only) Number of consumers consuming messages from this queue,0,activemq,art q consumers +activemq.artemis.queue.max_consumers,gauge,,,,(Artemis only) Maximum number of consumers allowed on this queue at any one time,0,activemq,art q max consumers +activemq.artemis.queue.messages_added,rate,,,,(Artemis only) Number of messages added to this queue since it was created,0,activemq,art q mess added +activemq.artemis.queue.messages_expired,gauge,,,,(Artemis only) Number of messages expired from this queue since it was created,0,activemq,art q mess exp +activemq.artemis.queue.messages_acknowledged,rate,,,,(Artemis only) Number of messages acknowledged from this queue since it was created,0,activemq,art q mess acked +activemq.artemis.queue.messages_killed,rate,,,,(Artemis only) Number of messages removed from this queue since it was created due to exceeding the max delivery attempts,0,activemq,art q mess killed diff --git a/activemq/tests/common.py b/activemq/tests/common.py index 4764b8b1893ef..5e934ab91cb71 100644 --- a/activemq/tests/common.py +++ b/activemq/tests/common.py @@ -1,10 +1,20 @@ # (C) Datadog, Inc. 2019-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import os + +import pytest + from datadog_checks.dev import get_docker_hostname, get_here CHECK_NAME = 'activemq' +COMPOSE_FILE = os.getenv('COMPOSE_FILE') +IS_ARTEMIS = COMPOSE_FILE == 'artemis.yaml' + +artemis = pytest.mark.skipif(not IS_ARTEMIS, reason='Test only valid for ActiveMQ Artemis versions') +not_artemis = pytest.mark.skipif(IS_ARTEMIS, reason='Test only valid for non-Artemis versions') + HERE = get_here() HOST = get_docker_hostname() @@ -16,7 +26,9 @@ TEST_AUTH = ('admin', 'admin') TEST_PORT = 8161 -BASE_URL = 'http://{}:{}/api/message'.format(HOST, TEST_PORT) +BASE_URL = 'http://{}:{}'.format(HOST, TEST_PORT) +ACTIVEMQ_URL = '{}/api/message'.format(BASE_URL) +ARTEMIS_URL = '{}/console/jolokia'.format(BASE_URL) # not all metrics will be available in our E2E environment, specifically: # "activemq.queue.dequeue_count", @@ -37,3 +49,29 @@ "activemq.broker.temp_pct", "activemq.broker.memory_pct", ] + +ARTEMIS_E2E_METRICS = [ + "activemq.artemis.address.bytes_per_page", + "activemq.artemis.address.number_of_messages", + "activemq.artemis.address.pages_count", + "activemq.artemis.address.routed_messages", + "activemq.artemis.address.size", + "activemq.artemis.address.unrouted_messages", + "activemq.artemis.address_memory_usage", + "activemq.artemis.address_memory_usage_pct", + "activemq.artemis.connection_count", + "activemq.artemis.disk_store_usage_pct", + "activemq.artemis.max_disk_usage", + "activemq.artemis.queue.consumer_count", + "activemq.artemis.queue.max_consumers", + "activemq.artemis.queue.message_count", + "activemq.artemis.queue.messages_acknowledged", + "activemq.artemis.queue.messages_added", + "activemq.artemis.queue.messages_expired", + "activemq.artemis.queue.messages_killed", + "activemq.artemis.total_connection_count", + "activemq.artemis.total_consumer_count", + "activemq.artemis.total_message_count", + "activemq.artemis.total_messages_acknowledged", + "activemq.artemis.total_messages_added", +] diff --git a/activemq/tests/compose/docker-compose.yaml b/activemq/tests/compose/activemq.yaml similarity index 99% rename from activemq/tests/compose/docker-compose.yaml rename to activemq/tests/compose/activemq.yaml index 8f4a47d90c972..8df30e986d829 100644 --- a/activemq/tests/compose/docker-compose.yaml +++ b/activemq/tests/compose/activemq.yaml @@ -14,3 +14,4 @@ services: ports: - "8161:8161" - "${JMX_PORT}:${JMX_PORT}" + diff --git a/activemq/tests/compose/artemis.yaml b/activemq/tests/compose/artemis.yaml new file mode 100644 index 0000000000000..e2d29beea16d1 --- /dev/null +++ b/activemq/tests/compose/artemis.yaml @@ -0,0 +1,23 @@ +version: "3" + +services: + namenode: + image: vromero/activemq-artemis:${ARTEMIS_VERSION} + container_name: dd-test-activemq-server + environment: + JAVA_OPTS: >- + -Dcom.sun.management.jmxremote.port=${JMX_PORT} + -Dcom.sun.management.jmxremote.rmi.port=${JMX_PORT} + -Dcom.sun.management.jmxremote.ssl=false + -Dcom.sun.management.jmxremote.authenticate=false + -Djava.rmi.server.hostname=localhost + ARTEMIS_USERNAME: admin + ARTEMIS_PASSWORD: admin + DISABLE_SECURITY: "true" + ENABLE_JMX: "true" + JMX_PORT: ${JMX_PORT} + JMX_RMI_PORT: ${JMX_PORT} + ports: + - "8161:8161" + - "${JMX_PORT}:${JMX_PORT}" + - "61616:61616" diff --git a/activemq/tests/conftest.py b/activemq/tests/conftest.py index d3b6f543fe0b3..ab1815eae2585 100644 --- a/activemq/tests/conftest.py +++ b/activemq/tests/conftest.py @@ -12,29 +12,71 @@ from datadog_checks.dev.conditions import WaitForPortListening from datadog_checks.dev.utils import load_jmx_config -from .common import BASE_URL, HERE, HOST, JMX_PORT, TEST_AUTH, TEST_MESSAGE, TEST_PORT, TEST_QUEUES, TEST_TOPICS +from .common import ( + ACTIVEMQ_URL, + ARTEMIS_URL, + BASE_URL, + COMPOSE_FILE, + HERE, + HOST, + IS_ARTEMIS, + JMX_PORT, + TEST_AUTH, + TEST_MESSAGE, + TEST_PORT, + TEST_QUEUES, + TEST_TOPICS, +) def populate_server(): """Add some queues and topics to ensure more metrics are available.""" time.sleep(3) - for queue in TEST_QUEUES: - url = '{}/{}?type=queue'.format(BASE_URL, queue) - requests.post(url, data=TEST_MESSAGE, auth=TEST_AUTH) + if IS_ARTEMIS: + s = requests.Session() + s.auth = TEST_AUTH + s.headers = {'accept': 'application/json', 'origin': BASE_URL} + data = s.get(ARTEMIS_URL + '/list') + channels = data.json()['value']['org.apache.activemq.artemis'] + broker = [k for k in channels.keys() if k.startswith('broker') and ',' not in k][0] + bean = 'org.apache.activemq.artemis:{}'.format(broker) - for topic in TEST_TOPICS: - url = '{}/{}?type=topic'.format(BASE_URL, topic) - requests.post(url, data=TEST_MESSAGE, auth=TEST_AUTH) + for queue in TEST_QUEUES: + body = { + "type": "exec", + "mbean": bean, + "operation": "createQueue(" + "java.lang.String," + "java.lang.String," + "java.lang.String," + "java.lang.String," + "boolean,int,boolean,boolean)", + "arguments": ["activemq.notifications", "ANYCAST", queue, None, True, -1, False, True], + } + s.post(ARTEMIS_URL + '/exec', json=body) + + else: + for queue in TEST_QUEUES: + url = '{}/{}?type=queue'.format(ACTIVEMQ_URL, queue) + requests.post(url, data=TEST_MESSAGE, auth=TEST_AUTH) + + for topic in TEST_TOPICS: + url = '{}/{}?type=topic'.format(ACTIVEMQ_URL, topic) + requests.post(url, data=TEST_MESSAGE, auth=TEST_AUTH) @pytest.fixture(scope="session") def dd_environment(): envs = {'JMX_PORT': str(JMX_PORT)} - compose_file = os.path.join(HERE, 'compose', 'docker-compose.yaml') + + log_pattern = 'ActiveMQ Jolokia REST API available' + if IS_ARTEMIS: + log_pattern = 'HTTP Server started at http://0.0.0.0:8161' + with docker_run( - compose_file, - log_patterns=['ActiveMQ Jolokia REST API available'], + os.path.join(HERE, 'compose', COMPOSE_FILE), + log_patterns=[log_pattern], conditions=[WaitForPortListening(HOST, TEST_PORT), populate_server], env_vars=envs, ): diff --git a/activemq/tests/test_check.py b/activemq/tests/test_check.py index 2664080b66c8e..2960ca7089c81 100644 --- a/activemq/tests/test_check.py +++ b/activemq/tests/test_check.py @@ -7,11 +7,12 @@ from datadog_checks.dev.jmx import JVM_E2E_METRICS from datadog_checks.dev.utils import get_metadata_metrics -from .common import ACTIVEMQ_E2E_METRICS +from .common import ACTIVEMQ_E2E_METRICS, ARTEMIS_E2E_METRICS, artemis, not_artemis +@not_artemis @pytest.mark.e2e -def test(dd_agent_check): +def test_activemq_metrics(dd_agent_check): instance = {} aggregator = dd_agent_check(instance) @@ -20,3 +21,16 @@ def test(dd_agent_check): aggregator.assert_all_metrics_covered() aggregator.assert_metrics_using_metadata(get_metadata_metrics(), exclude=JVM_E2E_METRICS) + + +@artemis +@pytest.mark.e2e +def test_artemis_metrics(dd_agent_check): + instance = {} + aggregator = dd_agent_check(instance, rate=True) + + for metric in ARTEMIS_E2E_METRICS + JVM_E2E_METRICS: + aggregator.assert_metric(metric) + + aggregator.assert_all_metrics_covered() + aggregator.assert_metrics_using_metadata(get_metadata_metrics(), exclude=JVM_E2E_METRICS) diff --git a/activemq/tox.ini b/activemq/tox.ini index 399fa495ac460..c3ba41f92aa10 100644 --- a/activemq/tox.ini +++ b/activemq/tox.ini @@ -4,6 +4,7 @@ skip_missing_interpreters = true basepython = py38 envlist = py38 + py38-artemis [testenv] ensure_default_envdir = true @@ -20,6 +21,10 @@ deps = passenv = DOCKER* COMPOSE* -setenv = ACTIVEMQ_VERSION=5.15.9 commands = pytest -v {posargs} +setenv = + ACTIVEMQ_VERSION=5.15.9 + COMPOSE_FILE=activemq.yaml + artemis: COMPOSE_FILE=artemis.yaml + artemis: ARTEMIS_VERSION=2.15.0