Add support for ActiveMQ Artemis (#8527)

* Initial Artemis JMX integration

* Add artemis metrics and e2e env

* Add test queues for artemis env

* Remove new integration files

* Update metadata

* Update e2e tests

* Fix style

* Update README

* Add address metrics

* Update tests

* Update activemq/README.md

Co-authored-by: Christine Chen <ChristineTChen@users.noreply.github.com>

* Update metric types

* Fix metadata

* Update metric types and add rate test

Co-authored-by: Christine Chen <ChristineTChen@users.noreply.github.com>

activemq/README.md

@@ -4,6 +4,8 @@
 
 The ActiveMQ check collects metrics for brokers and queues, producers and consumers, and more.
 
+**Note:** This check also supports ActiveMQ Artemis (future ActiveMQ version `6`) and reports metrics under the `activemq.artemis` namespace. See [metrics][#metrics] for more details.
+
 **Note**: If you are running a ActiveMQ version older than 5.8.0, see the [Agent 5.10.x released sample files][1].
 
 ## Setup
@@ -103,7 +105,7 @@ Collecting logs is disabled by default in the Datadog Agent. To enable it, see [
 
 ### Metrics
 
-See [metadata.csv][9] for a list of metrics provided by this integration.
+See [metadata.csv][9] for a list of metrics provided by this integration.  Metrics associated with ActiveMQ Artemis flavor have `artemis` in their metric name, all others are reported for ActiveMQ "classic".
 
 ### Events
 

activemq/datadog_checks/activemq/data/metrics.yaml

@@ -54,3 +54,87 @@ jmx_metrics:
         MemoryPercentUsage:
           alias: activemq.broker.memory_pct
           metric_type: gauge
+
+  - include:
+      domain: org.apache.activemq.artemis
+      attribute:
+        AddressMemoryUsage:
+          alias: activemq.artemis.address_memory_usage
+          metric_type: gauge
+        AddressMemoryUsagePercentage:
+          alias: activemq.artemis.address_memory_usage_pct
+          metric_type: gauge
+        MaxDiskUsage:
+          alias: activemq.artemis.max_disk_usage
+          metric_type: gauge
+        DiskStoreUsage:
+          alias: activemq.artemis.disk_store_usage_pct
+          metric_type: gauge
+        ConnectionCount:
+          alias: activemq.artemis.connection_count
+          metric_type: counter
+        TotalConnectionCount:
+          alias: activemq.artemis.total_connection_count
+          metric_type: monotonic_count
+        TotalMessageCount:
+          alias: activemq.artemis.total_message_count
+          metric_type: monotonic_count
+        TotalMessagesAdded:
+          alias: activemq.artemis.total_messages_added
+          metric_type: monotonic_count
+        TotalMessagesAcknowledged:
+          alias: activemq.artemis.total_messages_acknowledged
+          metric_type: monotonic_count
+        TotalConsumerCount:
+          alias: activemq.artemis.total_consumer_count
+          metric_type: monotonic_count
+
+  - include:
+      domain: org.apache.activemq.artemis
+      component: addresses
+      attribute:
+        AddressSize:
+          alias: activemq.artemis.address.size
+          metric_type: gauge
+        NumberOfPages:
+          alias: activemq.artemis.address.pages_count
+          metric_type: counter
+        NumberOfMessages:
+          alias: activemq.artemis.address.number_of_messages
+          metric_type: monotonic_count
+        NumberOfBytesPerPage:
+          alias: activemq.artemis.address.bytes_per_page
+          metric_type: gauge
+        RoutedMessageCount:
+          alias: activemq.artemis.address.routed_messages
+          metric_type: monotonic_count
+        UnRoutedMessageCount:
+          alias: activemq.artemis.address.unrouted_messages
+          metric_type: monotonic_count
+
+  - include:
+      domain: org.apache.activemq.artemis
+      subcomponent: queues
+      attribute:
+        MessageCount:
+          alias: activemq.artemis.queue.message_count
+          metric_type: monotonic_count
+        ConsumerCount:
+          alias: activemq.artemis.queue.consumer_count
+          metric_type: gauge
+        MaxConsumers:
+          alias: activemq.artemis.queue.max_consumers
+          metric_type: gauge
+        MessagesAdded:
+          alias: activemq.artemis.queue.messages_added
+          metric_type: monotonic_count
+        MessagesExpired:
+          # implied as monotonic count, reports as gauge
+          alias: activemq.artemis.queue.messages_expired
+          metric_type: gauge
+        MessagesAcknowledged:
+          alias: activemq.artemis.queue.messages_acknowledged
+          metric_type: monotonic_count
+        MessagesKilled:
+          alias: activemq.artemis.queue.messages_killed
+          metric_type: monotonic_count

activemq/metadata.csv

@@ -14,3 +14,26 @@ activemq.queue.in_flight_count,gauge,10,message,,The amount of messages that hav
 activemq.broker.store_pct,gauge,10,percent,,The percentage of store in use.,0,activemq,str pct
 activemq.broker.temp_pct,gauge,10,percent,,The percentage of temporary in use.,0,activemq,temp pct
 activemq.broker.memory_pct,gauge,10,percent,,The percentage of memory in use.,0,activemq,mem pct
+activemq.artemis.address_memory_usage,gauge,,,,(Artemis only) Memory used by all the addresses on broker for in-memory messages,0,activemq,art mem usage
+activemq.artemis.address_memory_usage_pct,gauge,,,,(Artemis only) Memory used by all the addresses on broker as a percentage of the global-max-size,0,activemq,art mem pct
+activemq.artemis.max_disk_usage,gauge,,,,(Artemis only) Maximum limit for disk use in percentage,0,activemq,art max disk
+activemq.artemis.disk_store_usage_pct,gauge,,,,(Artemis only) Percentage of total disk store used,0,activemq,art disk pct
+activemq.artemis.connection_count,gauge,,,,(Artemis only) Number of clients connected to this server,0,activemq,art conn count
+activemq.artemis.total_connection_count,rate,,,,(Artemis only) Number of clients which have connected to this server since it was started,0,activemq,art total conns
+activemq.artemis.total_message_count,rate,,,,(Artemis only) Number of messages in all queues on the server,0,activemq,art total messages
+activemq.artemis.total_messages_added,rate,,,,(Artemis only) Number of messages sent to this server since it was started,0,activemq,art total mess added
+activemq.artemis.total_messages_acknowledged,rate,,,,(Artemis only) Number of messages acknowledged from all the queues on this server since it was started,0,activemq,art total mess acked
+activemq.artemis.total_consumer_count,rate,,,,(Artemis only) Number of consumers consuming messages from all the queues on this server,0,activemq,art total consumers
+activemq.artemis.address.size,gauge,,,,(Artemis only) Number of estimated bytes being used by all the queue(s) bound to this address; used to control paging and blocking,0,activemq,add size
+activemq.artemis.address.pages_count,gauge,,,,(Artemis only) Number of pages used by this address,0,activemq,add pages
+activemq.artemis.address.number_of_messages,rate,,,,"(Artemis only) The sum of messages on queue(s), including messages in delivery",0,activemq,add num mess
+activemq.artemis.address.bytes_per_page,gauge,,,,(Artemis only) Number of bytes used by each page for this address,0,activemq,add bytes page
+activemq.artemis.address.routed_messages,rate,,,,(Artemis only) Number of messages routed to one or more bindings,0,activemq,add routed
+activemq.artemis.address.unrouted_messages,rate,,,,(Artemis only) Number of messages not routed to any bindings,0,activemq,add unrouted
+activemq.artemis.queue.message_count,rate,,,,"(Artemis only) Number of messages currently in this queue (includes scheduled, paged, and in-delivery messages)",0,activemq,art q mess count
+activemq.artemis.queue.consumer_count,gauge,,,,(Artemis only) Number of consumers consuming messages from this queue,0,activemq,art q consumers
+activemq.artemis.queue.max_consumers,gauge,,,,(Artemis only) Maximum number of consumers allowed on this queue at any one time,0,activemq,art q max consumers
+activemq.artemis.queue.messages_added,rate,,,,(Artemis only) Number of messages added to this queue since it was created,0,activemq,art q mess added
+activemq.artemis.queue.messages_expired,gauge,,,,(Artemis only) Number of messages expired from this queue since it was created,0,activemq,art q mess exp
+activemq.artemis.queue.messages_acknowledged,rate,,,,(Artemis only) Number of messages acknowledged from this queue since it was created,0,activemq,art q mess acked
+activemq.artemis.queue.messages_killed,rate,,,,(Artemis only) Number of messages removed from this queue since it was created due to exceeding the max delivery attempts,0,activemq,art q mess killed

activemq/tests/common.py

@@ -1,10 +1,20 @@
 # (C) Datadog, Inc. 2019-present
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
+import os
+
+import pytest
+
 from datadog_checks.dev import get_docker_hostname, get_here
 
 CHECK_NAME = 'activemq'
 
+COMPOSE_FILE = os.getenv('COMPOSE_FILE')
+IS_ARTEMIS = COMPOSE_FILE == 'artemis.yaml'
+
+artemis = pytest.mark.skipif(not IS_ARTEMIS, reason='Test only valid for ActiveMQ Artemis versions')
+not_artemis = pytest.mark.skipif(IS_ARTEMIS, reason='Test only valid for non-Artemis versions')
+
 HERE = get_here()
 HOST = get_docker_hostname()
 
@@ -16,7 +26,9 @@
 TEST_AUTH = ('admin', 'admin')
 
 TEST_PORT = 8161
-BASE_URL = 'http://{}:{}/api/message'.format(HOST, TEST_PORT)
+BASE_URL = 'http://{}:{}'.format(HOST, TEST_PORT)
+ACTIVEMQ_URL = '{}/api/message'.format(BASE_URL)
+ARTEMIS_URL = '{}/console/jolokia'.format(BASE_URL)
 
 # not all metrics will be available in our E2E environment, specifically:
 # "activemq.queue.dequeue_count",
@@ -37,3 +49,29 @@
     "activemq.broker.temp_pct",
     "activemq.broker.memory_pct",
 ]
+
+ARTEMIS_E2E_METRICS = [
+    "activemq.artemis.address.bytes_per_page",
+    "activemq.artemis.address.number_of_messages",
+    "activemq.artemis.address.pages_count",
+    "activemq.artemis.address.routed_messages",
+    "activemq.artemis.address.size",
+    "activemq.artemis.address.unrouted_messages",
+    "activemq.artemis.address_memory_usage",
+    "activemq.artemis.address_memory_usage_pct",
+    "activemq.artemis.connection_count",
+    "activemq.artemis.disk_store_usage_pct",
+    "activemq.artemis.max_disk_usage",
+    "activemq.artemis.queue.consumer_count",
+    "activemq.artemis.queue.max_consumers",
+    "activemq.artemis.queue.message_count",
+    "activemq.artemis.queue.messages_acknowledged",
+    "activemq.artemis.queue.messages_added",
+    "activemq.artemis.queue.messages_expired",
+    "activemq.artemis.queue.messages_killed",
+    "activemq.artemis.total_connection_count",
+    "activemq.artemis.total_consumer_count",
+    "activemq.artemis.total_message_count",
+    "activemq.artemis.total_messages_acknowledged",
+    "activemq.artemis.total_messages_added",
+]

activemq/tests/compose/docker-compose.yaml → activemq/tests/compose/activemq.yaml

@@ -14,3 +14,4 @@ services:
     ports:
       - "8161:8161"
       - "${JMX_PORT}:${JMX_PORT}"
+

activemq/tests/compose/artemis.yaml

@@ -0,0 +1,23 @@
+version: "3"
+
+services:
+  namenode:
+    image: vromero/activemq-artemis:${ARTEMIS_VERSION}
+    container_name: dd-test-activemq-server
+    environment:
+      JAVA_OPTS: >-
+        -Dcom.sun.management.jmxremote.port=${JMX_PORT}
+        -Dcom.sun.management.jmxremote.rmi.port=${JMX_PORT}
+        -Dcom.sun.management.jmxremote.ssl=false
+        -Dcom.sun.management.jmxremote.authenticate=false
+        -Djava.rmi.server.hostname=localhost
+      ARTEMIS_USERNAME: admin
+      ARTEMIS_PASSWORD: admin
+      DISABLE_SECURITY: "true"
+      ENABLE_JMX: "true"
+      JMX_PORT: ${JMX_PORT}
+      JMX_RMI_PORT: ${JMX_PORT}
+    ports:
+      - "8161:8161"
+      - "${JMX_PORT}:${JMX_PORT}"
+      - "61616:61616"

activemq/tests/conftest.py

@@ -12,29 +12,71 @@
 from datadog_checks.dev.conditions import WaitForPortListening
 from datadog_checks.dev.utils import load_jmx_config
 
-from .common import BASE_URL, HERE, HOST, JMX_PORT, TEST_AUTH, TEST_MESSAGE, TEST_PORT, TEST_QUEUES, TEST_TOPICS
+from .common import (
+    ACTIVEMQ_URL,
+    ARTEMIS_URL,
+    BASE_URL,
+    COMPOSE_FILE,
+    HERE,
+    HOST,
+    IS_ARTEMIS,
+    JMX_PORT,
+    TEST_AUTH,
+    TEST_MESSAGE,
+    TEST_PORT,
+    TEST_QUEUES,
+    TEST_TOPICS,
+)
 
 
 def populate_server():
     """Add some queues and topics to ensure more metrics are available."""
     time.sleep(3)
 
-    for queue in TEST_QUEUES:
-        url = '{}/{}?type=queue'.format(BASE_URL, queue)
-        requests.post(url, data=TEST_MESSAGE, auth=TEST_AUTH)
+    if IS_ARTEMIS:
+        s = requests.Session()
+        s.auth = TEST_AUTH
+        s.headers = {'accept': 'application/json', 'origin': BASE_URL}
+        data = s.get(ARTEMIS_URL + '/list')
+        channels = data.json()['value']['org.apache.activemq.artemis']
+        broker = [k for k in channels.keys() if k.startswith('broker') and ',' not in k][0]
+        bean = 'org.apache.activemq.artemis:{}'.format(broker)
 
-    for topic in TEST_TOPICS:
-        url = '{}/{}?type=topic'.format(BASE_URL, topic)
-        requests.post(url, data=TEST_MESSAGE, auth=TEST_AUTH)
+        for queue in TEST_QUEUES:
+            body = {
+                "type": "exec",
+                "mbean": bean,
+                "operation": "createQueue("
+                "java.lang.String,"
+                "java.lang.String,"
+                "java.lang.String,"
+                "java.lang.String,"
+                "boolean,int,boolean,boolean)",
+                "arguments": ["activemq.notifications", "ANYCAST", queue, None, True, -1, False, True],
+            }
+            s.post(ARTEMIS_URL + '/exec', json=body)
+
+    else:
+        for queue in TEST_QUEUES:
+            url = '{}/{}?type=queue'.format(ACTIVEMQ_URL, queue)
+            requests.post(url, data=TEST_MESSAGE, auth=TEST_AUTH)
+
+        for topic in TEST_TOPICS:
+            url = '{}/{}?type=topic'.format(ACTIVEMQ_URL, topic)
+            requests.post(url, data=TEST_MESSAGE, auth=TEST_AUTH)
 
 
 @pytest.fixture(scope="session")
 def dd_environment():
     envs = {'JMX_PORT': str(JMX_PORT)}
-    compose_file = os.path.join(HERE, 'compose', 'docker-compose.yaml')
+
+    log_pattern = 'ActiveMQ Jolokia REST API available'
+    if IS_ARTEMIS:
+        log_pattern = 'HTTP Server started at http://0.0.0.0:8161'
+
     with docker_run(
-        compose_file,
-        log_patterns=['ActiveMQ Jolokia REST API available'],
+        os.path.join(HERE, 'compose', COMPOSE_FILE),
+        log_patterns=[log_pattern],
         conditions=[WaitForPortListening(HOST, TEST_PORT), populate_server],
         env_vars=envs,
     ):

activemq/tests/test_check.py

@@ -7,11 +7,12 @@
 from datadog_checks.dev.jmx import JVM_E2E_METRICS
 from datadog_checks.dev.utils import get_metadata_metrics
 
-from .common import ACTIVEMQ_E2E_METRICS
+from .common import ACTIVEMQ_E2E_METRICS, ARTEMIS_E2E_METRICS, artemis, not_artemis
 
 
+@not_artemis
 @pytest.mark.e2e
-def test(dd_agent_check):
+def test_activemq_metrics(dd_agent_check):
     instance = {}
     aggregator = dd_agent_check(instance)
 
@@ -20,3 +21,16 @@ def test(dd_agent_check):
 
     aggregator.assert_all_metrics_covered()
     aggregator.assert_metrics_using_metadata(get_metadata_metrics(), exclude=JVM_E2E_METRICS)
+
+
+@artemis
+@pytest.mark.e2e
+def test_artemis_metrics(dd_agent_check):
+    instance = {}
+    aggregator = dd_agent_check(instance, rate=True)
+
+    for metric in ARTEMIS_E2E_METRICS + JVM_E2E_METRICS:
+        aggregator.assert_metric(metric)
+
+    aggregator.assert_all_metrics_covered()
+    aggregator.assert_metrics_using_metadata(get_metadata_metrics(), exclude=JVM_E2E_METRICS)

activemq/tox.ini

@@ -4,6 +4,7 @@ skip_missing_interpreters = true
 basepython = py38
 envlist =
     py38
+    py38-artemis
 
 [testenv]
 ensure_default_envdir = true
@@ -20,6 +21,10 @@ deps =
 passenv =
     DOCKER*
     COMPOSE*
-setenv = ACTIVEMQ_VERSION=5.15.9
 commands =
     pytest -v {posargs}
+setenv =
+    ACTIVEMQ_VERSION=5.15.9
+    COMPOSE_FILE=activemq.yaml
+    artemis: COMPOSE_FILE=artemis.yaml
+    artemis: ARTEMIS_VERSION=2.15.0