Add RethinkDB integration

.azure-pipelines/templates/test-all-checks.yml

@@ -318,6 +318,9 @@ jobs:
     - checkName: redisdb
       displayName: Redis
       os: linux
+    - checkName: rethinkdb
+      displayName: RethinkDB
+      os: linux
     - checkName: riak
       displayName: Riak
       os: linux

datadog_checks_base/datadog_checks/base/data/agent_requirements.in

@@ -61,6 +61,7 @@ redis==3.3.11
 requests==2.22.0
 requests-kerberos==0.12.0
 requests_ntlm==1.1.0
+rethinkdb==2.4.4
 scandir==1.8
 securesystemslib[crypto,pynacl]==0.14.2
 selectors34==1.2.0; sys_platform == 'win32' and python_version < '3.4'

datadog_checks_dev/datadog_checks/dev/docker.py

@@ -3,6 +3,7 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import os
 from contextlib import contextmanager
+from typing import Iterator
 
 from six import string_types
 from six.moves.urllib.parse import urlparse
@@ -246,3 +247,11 @@ def _read_example_logs_config(check_root):
                 return option['example']
 
     raise ValueError('No logs example found')
+
+
+@contextmanager
+def temporarily_stop_service(service, compose_file, check=True):
+    # type: (str, str, bool) -> Iterator[None]
+    run_command(['docker-compose', '-f', compose_file, 'stop', service], capture=False, check=check)
+    yield
+    run_command(['docker-compose', '-f', compose_file, 'start', service], capture=False, check=check)

rethinkdb/CHANGELOG.md

@@ -0,0 +1,2 @@
+# CHANGELOG - RethinkDB
+

rethinkdb/MANIFEST.in

@@ -0,0 +1,10 @@
+graft datadog_checks
+graft tests
+
+include MANIFEST.in
+include README.md
+include requirements.in
+include requirements-dev.txt
+include manifest.json
+
+global-exclude *.py[cod] __pycache__

rethinkdb/README.md

@@ -0,0 +1,82 @@
+# Agent Check: RethinkDB
+
+## Overview
+
+[RethinkDB][1] is a distributed documented-oriented NoSQL database, with first class support for realtime change feeds.
+
+This check monitors a RethinkDB cluster through the Datadog Agent and collects metrics about performance, data availability, cluster configuration, and more.
+
+**Note**: this integration is compatible with RethinkDB **version 2.3.6 and above**.
+
+## Setup
+
+Follow the instructions below to install and configure this check for an Agent running on a host. For containerized environments, see the [Autodiscovery Integration Templates][2] for guidance on applying these instructions.
+
+### Installation
+
+The RethinkDB check is included in the [Datadog Agent][3] package. No additional installation is needed on your server.
+
+### Configuration
+
+1. If using RethinkDB 2.4+, add a `datadog-agent` user with read-only permissions on the `rethinkdb` database. You can use the following ReQL commands, and refer to [Permissions and user accounts][4] for details:
+
+    ```python
+    r.db('rethinkdb').table('users').insert({'id': 'datadog-agent', 'password': '<PASSWORD>'})
+    r.db('rethinkdb').grant('datadog-agent', {'read': True})
+    ```
+
+    **Note**: on RethinkDB 2.3.x, granting permissions on the `rethinkdb` database is not supported. Skip this step and use your [admin account][5] below instead.
+
+2. Edit the `rethinkdb.d/conf.yaml` file in the `conf.d/` folder at the root of your [Agent's configuration directory][6]. See the [sample rethinkdb.d/conf.yaml][7] for all available configuration options.
+
+    ```yaml
+    init_config:
+
+    instances:
+      - host: localhost
+        port: 28015
+        user: "<USER>"
+        password: "<PASSWORD>"
+    ```
+
+3. [Restart the Agent][8].
+
+**Note**: this integration collects metrics from all servers in the cluster, so you only need a single Agent.
+
+### Validation
+
+[Run the Agent's status subcommand][9] and look for `rethinkdb` under the Checks section.
+
+## Data Collected
+
+### Metrics
+
+See [metadata.csv][10] for a list of metrics provided by this check.
+
+### Service Checks
+
+- `rethinkdb.can_connect`: Returns `CRITICAL` if the Agent cannot reach the configured RethinkDB server, `OK` otherwise.
+- `rethinkdb.table_status.status.ready_for_outdated_reads`: Returns `OK` if all shards of a table are ready to accept outdated read queries, `WARNING` otherwise.
+- `rethinkdb.table_status.status.ready_for_reads`: Returns `OK` if all shards of a table are ready to accept read queries, `WARNING` otherwise.
+- `rethinkdb.table_status.status.ready_for_writes`: Returns `OK` if all shards of a table are ready to accept write queries, `WARNING` otherwise.
+- `rethinkdb.table_status.status.all_replicas_ready`: Returns `OK` if all replicas are ready for reads and writes, `WARNING` otherwise (e.g. if backfills are in progress).
+
+### Events
+
+RethinkDB does not include any events.
+
+## Troubleshooting
+
+Need help? Contact [Datadog support][11].
+
+[1]: https://rethinkdb.com/
+[2]: https://docs.datadoghq.com/agent/autodiscovery/integrations
+[3]: https://docs.datadoghq.com/agent
+[4]: https://rethinkdb.com/docs/permissions-and-accounts/
+[5]: https://rethinkdb.com/docs/security/#the-admin-account
+[6]: https://docs.datadoghq.com/agent/guide/agent-configuration-files/#agent-configuration-directory
+[7]: https://github.com/DataDog/integrations-core/blob/master/rethinkdb/datadog_checks/rethinkdb/data/conf.yaml.example
+[8]: https://docs.datadoghq.com/agent/guide/agent-commands/#start-stop-and-restart-the-agent
+[9]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information
+[10]: https://github.com/DataDog/integrations-core/blob/master/rethinkdb/metadata.csv
+[11]: https://docs.datadoghq.com/help

rethinkdb/assets/configuration/spec.yaml

@@ -0,0 +1,44 @@
+name: RethinkDB
+
+files:
+  - name: rethinkdb.yaml
+
+    options:
+      - template: init_config
+        options:
+          - template: init_config/default
+
+      - template: instances
+        options:
+          - name: host
+            required: true
+            description: Host of the RethinkDB server.
+            value:
+              example: localhost
+              type: string
+
+          - name: port
+            required: true
+            description: Driver port of the RethinkDB server.
+            value:
+              example: 28015
+              type: integer
+
+          - name: username
+            description: The user account to connect as.
+            value:
+              type: string
+
+          - name: password
+            description: The password for the user account to connect as.
+            value:
+              type: string
+
+          - name: tls_ca_cert
+            description: |
+              Path to a TLS client certificate to use when connecting to the RethinkDB server.
+              See also: https://rethinkdb.com/docs/security/#using-tls
+            value:
+              type: string
+
+          - template: instances/default

rethinkdb/assets/service_checks.json

@@ -0,0 +1,77 @@
+[
+    {
+        "agent_version": "6.19.0",
+        "integration": "RethinkDB",
+        "groups": [
+            "host",
+            "port"
+        ],
+        "check": "rethinkdb.can_connect",
+        "statuses": [
+            "ok",
+            "critical"
+        ],
+        "name": "Can Connect",
+        "description": "Returns `CRITICAL` if the Agent is unable to reach the configured RethinkDB server, `OK` otherwise."
+    },
+    {
+        "agent_version": "6.19.0",
+        "integration": "RethinkDB",
+        "groups": [
+            "database",
+            "table"
+        ],
+        "check": "rethinkdb.table_status.status.ready_for_outdated_reads",
+        "statuses": [
+            "ok",
+            "warning"
+        ],
+        "name": "Table Ready For Outdated Reads",
+        "description": "Returns `OK` if all shards of a table are ready to accept outdated read queries, `WARNING` otherwise."
+    },
+    {
+        "agent_version": "6.19.0",
+        "integration": "RethinkDB",
+        "groups": [
+            "database",
+            "table"
+        ],
+        "check": "rethinkdb.table_status.status.ready_for_reads",
+        "statuses": [
+            "ok",
+            "warning"
+        ],
+        "name": "Table Ready For Reads",
+        "description": "Returns `OK` if all shards of a table are ready to accept read queries, `WARNING` otherwise."
+    },
+    {
+        "agent_version": "6.19.0",
+        "integration": "RethinkDB",
+        "groups": [
+            "database",
+            "table"
+        ],
+        "check": "rethinkdb.table_status.status.ready_for_writes",
+        "statuses": [
+            "ok",
+            "warning"
+        ],
+        "name": "Table Ready For Writes",
+        "description": "Returns `OK` if all shards of a table are ready to accept write queries, `WARNING` otherwise."
+    },
+    {
+        "agent_version": "6.19.0",
+        "integration": "RethinkDB",
+        "groups": [
+            "database",
+            "table"
+        ],
+        "check": "rethinkdb.table_status.status.all_replicas_ready",
+        "statuses": [
+            "ok",
+            "warning"
+        ],
+        "name": "Table All Replicas Ready",
+        "description": "Returns `OK` if all replicas are ready for reads and writes, `WARNING` otherwise (e.g. if backfills are in progress)."
+    }
+]

rethinkdb/datadog_checks/__init__.py

@@ -0,0 +1,4 @@
+# (C) Datadog, Inc. 2020-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore

rethinkdb/datadog_checks/rethinkdb/__about__.py

@@ -0,0 +1,4 @@
+# (C) Datadog, Inc. 2020-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+__version__ = '0.0.1'

rethinkdb/datadog_checks/rethinkdb/__init__.py

@@ -0,0 +1,7 @@
+# (C) Datadog, Inc. 2020-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from .__about__ import __version__
+from .check import RethinkDBCheck
+
+__all__ = ['__version__', 'RethinkDBCheck']

rethinkdb/datadog_checks/rethinkdb/check.py

@@ -0,0 +1,115 @@
+# (C) Datadog, Inc. 2020-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from contextlib import contextmanager
+from typing import Any, Callable, Iterator, Sequence, cast
+
+import rethinkdb
+
+from datadog_checks.base import AgentCheck
+
+from . import operations, queries
+from .config import Config
+from .document_db import DocumentQuery
+from .document_db.types import Metric
+from .types import Instance
+from .version import parse_version
+
+
+class RethinkDBCheck(AgentCheck):
+    """
+    Collect metrics from a RethinkDB cluster.
+    """
+
+    SERVICE_CHECK_CONNECT = 'rethinkdb.can_connect'
+
+    def __init__(self, *args, **kwargs):
+        # type: (*Any, **Any) -> None
+        super(RethinkDBCheck, self).__init__(*args, **kwargs)
+
+        self.config = Config(cast(Instance, self.instance))
+
+        if self.config.password:
+            self.register_secret(self.config.password)
+
+        self.queries = (
+            queries.config_summary,
+            queries.cluster_statistics,
+            queries.server_statistics,
+            queries.table_statistics,
+            queries.replica_statistics,
+            queries.table_statuses,
+            queries.server_statuses,
+            queries.system_jobs,
+            queries.current_issues_summary,
+        )  # type: Sequence[DocumentQuery]
+
+    @contextmanager
+    def connect_submitting_service_checks(self):
+        # type: () -> Iterator[rethinkdb.net.Connection]
+        config = self.config
+        tags = config.service_check_tags
+
+        try:
+            with rethinkdb.r.connect(
+                host=config.host,
+                port=config.port,
+                user=config.user,
+                password=config.password,
+                ssl={'ca_certs': config.tls_ca_cert} if config.tls_ca_cert is not None else {},
+            ) as conn:
+                yield conn
+        except rethinkdb.errors.ReqlDriverError as exc:
+            message = 'Could not connect to RethinkDB server: {!r}'.format(exc)
+            self.log.error(message)
+            self.service_check(self.SERVICE_CHECK_CONNECT, self.CRITICAL, tags=tags, message=message)
+            raise
+        except Exception as exc:
+            message = 'Unexpected error while executing RethinkDB check: {!r}'.format(exc)
+            self.log.error(message)
+            self.service_check(self.SERVICE_CHECK_CONNECT, self.CRITICAL, tags=tags, message=message)
+            raise
+        else:
+            self.service_check(self.SERVICE_CHECK_CONNECT, self.OK, tags=tags)
+
+    def collect_metrics(self, conn):
+        # type: (rethinkdb.net.Connection) -> Iterator[Metric]
+        """
+        Collect metrics from the RethinkDB cluster we are connected to.
+        """
+        for query in self.queries:
+            for metric in query.run(logger=self.log, conn=conn, config=self.config):
+                yield metric
+
+    def submit_metric(self, metric):
+        # type: (Metric) -> None
+        submit = getattr(self, metric['type'])  # type: Callable
+        submit(metric['name'], metric['value'], tags=self.config.tags + metric['tags'])
+
+    def submit_version_metadata(self, conn):
+        # type: (rethinkdb.net.Connection) -> None
+        try:
+            raw_version = operations.get_connected_server_raw_version(conn)
+        except Exception as exc:
+            self.log.error('Error collecting version metadata: %s', exc)
+            return
+
+        if raw_version is None:
+            return
+
+        try:
+            version = parse_version(raw_version)
+        except ValueError as exc:
+            self.log.error('Failed to parse version: %s', exc)
+            return
+
+        self.set_metadata('version', version)
+
+    def check(self, instance):
+        # type: (Any) -> None
+        with self.connect_submitting_service_checks() as conn:
+            for metric in self.collect_metrics(conn):
+                self.submit_metric(metric)
+
+            if self.is_metadata_collection_enabled():
+                self.submit_version_metadata(conn)