Add RethinkDB integration

.azure-pipelines/templates/test-all-checks.yml

@@ -318,6 +318,9 @@ jobs:
     - checkName: redisdb
       displayName: Redis
       os: linux
+    - checkName: rethinkdb
+      displayName: RethinkDB
+      os: linux
     - checkName: riak
       displayName: Riak
       os: linux

datadog_checks_base/datadog_checks/base/data/agent_requirements.in

@@ -60,6 +60,7 @@ redis==3.3.11
 requests==2.22.0
 requests-kerberos==0.12.0
 requests_ntlm==1.1.0
+rethinkdb==2.4.4
 scandir==1.8
 securesystemslib[crypto,pynacl]==0.14.2
 selectors34==1.2.0; sys_platform == 'win32' and python_version < '3.4'

datadog_checks_dev/datadog_checks/dev/docker.py

@@ -3,6 +3,7 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import os
 from contextlib import contextmanager
+from typing import Iterator
 
 from six import string_types
 from six.moves.urllib.parse import urlparse
@@ -246,3 +247,15 @@ def _read_example_logs_config(check_root):
                 return option['example']
 
     raise ValueError('No logs example found')
+
+
+@contextmanager
+def temporarily_stop_service(service, compose_file, check=True):
+    # type: (str, str, bool) -> Iterator[None]
+    run_command(['docker-compose', '-f', compose_file, 'stop', service], capture=False, check=check)
+    try:
+        yield
+    except Exception:
+        raise
+    else:
+        run_command(['docker-compose', '-f', compose_file, 'start', service], capture=False, check=check)

rethinkdb/CHANGELOG.md

@@ -0,0 +1,2 @@
+# CHANGELOG - RethinkDB
+

rethinkdb/MANIFEST.in

@@ -0,0 +1,10 @@
+graft datadog_checks
+graft tests
+
+include MANIFEST.in
+include README.md
+include requirements.in
+include requirements-dev.txt
+include manifest.json
+
+global-exclude *.py[cod] __pycache__

rethinkdb/README.md

@@ -0,0 +1,77 @@
+# Agent Check: RethinkDB
+
+## Overview
+
+[RethinkDB][1] is a distributed documented-oriented NoSQL database, with first class support for realtime change feeds.
+
+This check monitors a RethinkDB cluster through the Datadog Agent and collects metrics about performance, data availability, cluster configuration, and more.
+
+## Setup
+
+Follow the instructions below to install and configure this check for an Agent running on a host. For containerized environments, see the [Autodiscovery Integration Templates][2] for guidance on applying these instructions.
+
+### Installation
+
+The RethinkDB check is included in the [Datadog Agent][3] package. No additional installation is needed on your server.
+
+### Configuration
+
+1. Add a `datadog-agent` user with read-only permissions on the `rethinkdb` database. Use the following ReQL commands, referring to [Permissions and user accounts][4] for details:
+
+    ```python
+    r.db('rethinkdb').table('users').insert({'id': 'datadog-agent', 'password': '<PASSWORD>'})
+    r.db('rethinkdb').grant('datadog-agent', {'read': True})
+    ```
+
+2. Edit the `rethinkdb.d/conf.yaml` file in the `conf.d/` folder at the root of your [Agent's configuration directory][5]. See the [sample rethinkdb.d/conf.yaml][6] for all available configuration options.
+
+    ```yaml
+    init_config:
+
+    instances:
+      - host: localhost
+        port: 28015
+        user: datadog-agent
+        password: "<PASSWORD>"
+    ```
+
+3. [Restart the Agent][7].
+
+**Note**: this integration collects metrics from all servers in the cluster, so you only need a single Agent.
+
+### Validation
+
+[Run the Agent's status subcommand][8] and look for `rethinkdb` under the Checks section.
+
+## Data Collected
+
+### Metrics
+
+See [metadata.csv][9] for a list of metrics provided by this check.
+
+### Service Checks
+
+- `rethinkdb.can_connect`: Returns `CRITICAL` if the Agent cannot reach the configured RethinkDB server, `OK` otherwise.
+- `rethinkdb.table_status.ready_for_outdated_reads`: Returns `OK` if all shards of a table are ready to accept outdated read queries, `WARNING` otherwise.
+- `rethinkdb.table_status.ready_for_reads`: Returns `OK` if all shards of a table are ready to accept read queries, `WARNING` otherwise.
+- `rethinkdb.table_status.ready_for_writes`: Returns `OK` if all shards of a table are ready to accept write queries, `WARNING` otherwise.
+- `rethinkdb.table_status.all_replicas_ready`: Returns `OK` if all replicas are ready for reads and writes, `WARNING` otherwise (e.g. if backfills are in progress).
+
+### Events
+
+RethinkDB does not include any events.
+
+## Troubleshooting
+
+Need help? Contact [Datadog support][10].
+
+[1]: https://rethinkdb.com/
+[2]: https://docs.datadoghq.com/agent/autodiscovery/integrations
+[3]: https://docs.datadoghq.com/agent
+[4]: https://rethinkdb.com/docs/permissions-and-accounts/
+[5]: https://docs.datadoghq.com/agent/guide/agent-configuration-files/#agent-configuration-directory
+[6]: https://github.com/DataDog/integrations-core/blob/master/rethinkdb/datadog_checks/rethinkdb/data/conf.yaml.example
+[7]: https://docs.datadoghq.com/agent/guide/agent-commands/#start-stop-and-restart-the-agent
+[8]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information
+[9]: https://github.com/DataDog/integrations-core/blob/master/rethinkdb/metadata.csv
+[10]: https://docs.datadoghq.com/help

rethinkdb/assets/configuration/spec.yaml

@@ -0,0 +1,47 @@
+name: RethinkDB
+
+files:
+  - name: rethinkdb.yaml
+
+    options:
+      - template: init_config
+        options:
+          - template: init_config/default
+
+      - template: instances
+        options:
+          - name: host
+            required: false
+            description: Host of the RethinkDB server.
+            value:
+              example: localhost
+              type: string
+
+          - name: port
+            required: false
+            description: Driver port of the RethinkDB server.
+            value:
+              example: 28015
+              type: integer
+
+          - name: username
+            required: false
+            description: The user account to connect as.
+            value:
+              type: string
+
+          - name: password
+            required: false
+            description: The password for the user account to connect as.
+            value:
+              type: string
+
+          - name: tls_ca_cert
+            required: false
+            description: |
+              Path to a TLS client certificate to use when connecting to the RethinkDB server.
+              See also: https://rethinkdb.com/docs/security/#using-tls
+            value:
+              type: string
+
+          - template: instances/default

rethinkdb/assets/service_checks.json

@@ -0,0 +1,77 @@
+[
+    {
+        "agent_version": "6.19.0",
+        "integration": "RethinkDB",
+        "groups": [
+            "host",
+            "port"
+        ],
+        "check": "rethinkdb.can_connect",
+        "statuses": [
+            "ok",
+            "critical"
+        ],
+        "name": "Can Connect",
+        "description": "Returns `CRITICAL` if the Agent is unable to reach the configured RethinkDB server, `OK` otherwise."
+    },
+    {
+        "agent_version": "6.19.0",
+        "integration": "RethinkDB",
+        "groups": [
+            "database",
+            "table"
+        ],
+        "check": "rethinkdb.table_status.ready_for_outdated_reads",
+        "statuses": [
+            "ok",
+            "warning"
+        ],
+        "name": "Table Ready For Outdated Reads",
+        "description": "Returns `OK` if all shards of a table are ready to accept outdated read queries, `WARNING` otherwise."
+    },
+    {
+        "agent_version": "6.19.0",
+        "integration": "RethinkDB",
+        "groups": [
+            "database",
+            "table"
+        ],
+        "check": "rethinkdb.table_status.ready_for_reads",
+        "statuses": [
+            "ok",
+            "warning"
+        ],
+        "name": "Table Ready For Reads",
+        "description": "Returns `OK` if all shards of a table are ready to accept read queries, `WARNING` otherwise."
+    },
+    {
+        "agent_version": "6.19.0",
+        "integration": "RethinkDB",
+        "groups": [
+            "database",
+            "table"
+        ],
+        "check": "rethinkdb.table_status.ready_for_writes",
+        "statuses": [
+            "ok",
+            "warning"
+        ],
+        "name": "Table Ready For Writes",
+        "description": "Returns `OK` if all shards of a table are ready to accept write queries, `WARNING` otherwise."
+    },
+    {
+        "agent_version": "6.19.0",
+        "integration": "RethinkDB",
+        "groups": [
+            "database",
+            "table"
+        ],
+        "check": "rethinkdb.table_status.all_replicas_ready",
+        "statuses": [
+            "ok",
+            "warning"
+        ],
+        "name": "Table All Replicas Ready",
+        "description": "Returns `OK` if all replicas are ready for reads and writes, `WARNING` otherwise (e.g. if backfills are in progress)."
+    }
+]

rethinkdb/datadog_checks/__init__.py

@@ -0,0 +1,4 @@
+# (C) Datadog, Inc. 2020-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore

rethinkdb/datadog_checks/rethinkdb/__about__.py

@@ -0,0 +1,4 @@
+# (C) Datadog, Inc. 2020-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+__version__ = '0.0.1'

rethinkdb/datadog_checks/rethinkdb/__init__.py

@@ -0,0 +1,7 @@
+# (C) Datadog, Inc. 2020-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from .__about__ import __version__
+from .check import RethinkDBCheck
+
+__all__ = ['__version__', 'RethinkDBCheck']

rethinkdb/datadog_checks/rethinkdb/backends.py

@@ -0,0 +1,85 @@
+# (C) Datadog, Inc. 2020-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from typing import Callable, Iterator, Sequence
+
+import rethinkdb
+
+from .config import Config
+from .connections import Connection
+from .exceptions import CouldNotConnect
+from .metrics.config import collect_config_totals
+from .metrics.current_issues import collect_current_issues
+from .metrics.statistics import (
+    collect_cluster_statistics,
+    collect_replica_statistics,
+    collect_server_statistics,
+    collect_table_statistics,
+)
+from .metrics.statuses import collect_server_status, collect_table_status
+from .metrics.system_jobs import collect_system_jobs
+from .queries import QueryEngine
+from .types import Metric
+from .version import parse_version
+
+
+class Backend(object):
+    """
+    An interface for high-level operations performed during a RethinkDB check.
+
+    Abstracts away any interfaces specific to the `rethinkdb` client library, while providing a default
+    implementation that uses that library.
+    """
+
+    def __init__(self):
+        # type: () -> None
+        # NOTE: the name 'r' may look off-putting at first, but it was chosen for consistency with the officially
+        # advertised ReQL usage. For example, see: https://rethinkdb.com/docs/guide/python/
+        self._r = rethinkdb.r
+        self._query_engine = QueryEngine(r=self._r)
+        self._collect_funcs = (
+            collect_config_totals,
+            collect_cluster_statistics,
+            collect_server_statistics,
+            collect_table_statistics,
+            collect_replica_statistics,
+            collect_server_status,
+            collect_table_status,
+            collect_system_jobs,
+            collect_current_issues,
+        )  # type: Sequence[Callable[[QueryEngine, Connection], Iterator[Metric]]]
+
+    def connect(self, config):
+        # type: (Config) -> Connection
+        """
+        Establish a connection with the configured RethinkDB server.
+        """
+        try:
+            conn = self._r.connect(
+                host=config.host,
+                port=config.port,
+                user=config.user,
+                password=config.password,
+                ssl={'ca_certs': config.tls_ca_cert} if config.tls_ca_cert is not None else None,
+            )
+        except rethinkdb.errors.ReqlDriverError as exc:
+            raise CouldNotConnect(exc)
+
+        return Connection(conn)
+
+    def collect_metrics(self, conn):
+        # type: (Connection) -> Iterator[Metric]
+        """
+        Collect metrics from the RethinkDB cluster we are connected to.
+        """
+        for collect in self._collect_funcs:
+            for metric in collect(self._query_engine, conn):
+                yield metric
+
+    def collect_connected_server_version(self, conn):
+        # type: (Connection) -> str
+        """
+        Return the version of RethinkDB run by the server at the other end of the connection, in SemVer format.
+        """
+        version_string = self._query_engine.query_connected_server_version_string(conn)
+        return parse_version(version_string)