Merge pull request #876 from DataDog/nginx-plus

update nginx check to support nginx plus status format

checks.d/nginx.py

@@ -1,7 +1,7 @@
 import re
 import urllib2
 
-from util import headers
+from util import headers, json
 from checks import AgentCheck
 from checks.utils import add_basic_auth
 
@@ -25,41 +25,125 @@ def check(self, instance):
             raise Exception('NginX instance missing "nginx_status_url" value.')
         tags = instance.get('tags', [])
 
-        response = self._get_data(instance)
-        self._get_metrics(response, tags)
+        response, content_type = self._get_data(instance)
+        if content_type == 'application/json':
+            metrics = self.parse_json(response, tags)
+        else:
+            metrics = self.parse_text(response, tags)
+
+        funcs = {
+            'gauge': self.gauge,
+            'rate': self.rate
+        }
+        for row in metrics:
+            try:
+                name, value, tags, metric_type = row
+                func = funcs[metric_type]
+                func(name, value, tags)
+            except Exception:
+                self.log.error(u'Could not submit metric: %s' % repr(row))
 
     def _get_data(self, instance):
         url = instance.get('nginx_status_url')
         req = urllib2.Request(url, None, headers(self.agentConfig))
         if 'user' in instance and 'password' in instance:
             add_basic_auth(req, instance['user'], instance['password'])
-        request = urllib2.urlopen(req)
-        return request.read()
-
+        response = urllib2.urlopen(req)
+        body = response.read()
+        resp_headers = response.info()
+        return body, resp_headers.get('Content-Type', 'text/plain')
 
-    def _get_metrics(self, response, tags):
+    @classmethod
+    def parse_text(cls, raw, tags):
         # Thanks to http://hostingfu.com/files/nginx/nginxstats.py for this code
         # Connections
-        parsed = re.search(r'Active connections:\s+(\d+)', response)
+        output = []
+        parsed = re.search(r'Active connections:\s+(\d+)', raw)
         if parsed:
             connections = int(parsed.group(1))
-            self.gauge("nginx.net.connections", connections, tags=tags)
+            output.append(('nginx.net.connections', connections, tags, 'gauge'))
 
         # Requests per second
-        parsed = re.search(r'\s*(\d+)\s+(\d+)\s+(\d+)', response)
+        parsed = re.search(r'\s*(\d+)\s+(\d+)\s+(\d+)', raw)
         if parsed:
             conn = int(parsed.group(1))
             requests = int(parsed.group(3))
-            self.rate("nginx.net.conn_opened_per_s", conn, tags=tags)
-            self.rate("nginx.net.request_per_s", requests, tags=tags)
+            output.extend([('nginx.net.conn_opened_per_s', conn, tags, 'rate'),
+                           ('nginx.net.request_per_s', requests, tags, 'rate')])
 
         # Connection states, reading, writing or waiting for clients
-        parsed = re.search(r'Reading: (\d+)\s+Writing: (\d+)\s+Waiting: (\d+)', response)
+        parsed = re.search(r'Reading: (\d+)\s+Writing: (\d+)\s+Waiting: (\d+)', raw)
         if parsed:
-            reading, writing, waiting = map(int, parsed.groups())
-            self.gauge("nginx.net.reading", reading, tags=tags)
-            self.gauge("nginx.net.writing", writing, tags=tags)
-            self.gauge("nginx.net.waiting", waiting, tags=tags)
+            reading, writing, waiting = parsed.groups()
+            output.extend([
+                ("nginx.net.reading", int(reading), tags, 'gauge'),
+                ("nginx.net.writing", int(writing), tags, 'gauge'),
+                ("nginx.net.waiting", int(waiting), tags, 'gauge'),
+            ])
+        return output
+
+    @classmethod
+    def parse_json(cls, raw, tags=None):
+        if tags is None:
+            tags = []
+        parsed = json.loads(raw)
+        metric_base = 'nginx'
+        output = []
+        all_keys = parsed.keys()
+
+        tagged_keys = [('caches', 'cache'), ('server_zones', 'server_zone'),
+                       ('upstreams', 'upstream')]
+
+        # Process the special keys that should turn into tags instead of
+        # getting concatenated to the metric name
+        for key, tag_name in tagged_keys:
+            metric_name = '%s.%s' % (metric_base, tag_name)
+            for tag_val, data in parsed.get(key, {}).iteritems():
+                tag = '%s:%s' % (tag_name, tag_val)
+                output.extend(cls._flatten_json(metric_name, data, tags + [tag]))
+
+        # Process the rest of the keys
+        rest = set(all_keys) - set([k for k, _ in tagged_keys])
+        for key in rest:
+            metric_name = '%s.%s' % (metric_base, key)
+            output.extend(cls._flatten_json(metric_name, parsed[key], tags))
+
+        return output
+
+    @classmethod
+    def _flatten_json(cls, metric_base, val, tags):
+        ''' Recursively flattens the nginx json object. Returns the following:
+            [(metric_name, value, tags)]
+        '''
+        output = []
+        if isinstance(val, dict):
+            # Pull out the server as a tag instead of trying to read as a metric
+            if 'server' in val and val['server']:
+                server = 'server:%s' % val.pop('server')
+                if tags is None:
+                    tags = [server]
+                else:
+                    tags = tags + [server]
+            for key, val2 in val.iteritems():
+                metric_name = '%s.%s' % (metric_base, key)
+                output.extend(cls._flatten_json(metric_name, val2, tags))
+
+        elif isinstance(val, list):
+            for val2 in val:
+                output.extend(cls._flatten_json(metric_base, val2, tags))
+
+        elif isinstance(val, bool):
+            # Turn bools into 0/1 values
+            if val:
+                val = 1
+            else:
+                val = 0
+            output.append((metric_base, val, tags, 'gauge'))
+
+        elif isinstance(val, (int, float)):
+            output.append((metric_base, val, tags, 'gauge'))
+
+        return output
 
     @staticmethod
     def parse_agent_config(agentConfig):

tests/common.py

@@ -75,3 +75,6 @@ def get_check(name, config_str):
 
     return check_class.from_yaml(yaml_text=config_str, check_name=name,
         agentConfig=agentConfig)
+
+def read_test_data(filename):
+    return open(os.path.join(os.path.dirname(__file__), 'data', filename)).read()