[yarn] refactor application_tags

Change config, hopefully with more documentation.

checks.d/yarn.py

@@ -148,7 +148,14 @@ def check(self, instance):
 
         # Get properties from conf file
         rm_address = instance.get('resourcemanager_uri', DEFAULT_RM_URI)
-        app_tags = instance.get('application_tags', [])
+        app_tags = instance.get('application_tags', {})
+
+        if type(app_tags) is not dict:
+            self.log.error('application_tags is incorrect: %s is not a dictionary', app_tags)
+            app_tags = {}
+
+        # Collected by default
+        app_tags['app_name'] = 'name'
 
         # Get additional tags from the conf file
         tags = instance.get('tags', [])
@@ -187,50 +194,46 @@ def _yarn_app_metrics(self, rm_address, app_tags, addl_tags):
         '''
         Get metrics for running applications
         '''
-        metrics_json = self._rest_request_to_json(rm_address,
+        metrics_json = self._rest_request_to_json(
+            rm_address,
             YARN_APPS_PATH,
-            states=YARN_APPLICATION_STATES)
+            states=YARN_APPLICATION_STATES
+        )
 
-        if metrics_json:
-            if metrics_json['apps'] is not None:
-                if metrics_json['apps']['app'] is not None:
+        if (metrics_json and metrics_json['apps'] is not None and
+            metrics_json['apps']['app'] is not None):
 
-                    for app_json in metrics_json['apps']['app']:
+            for app_json in metrics_json['apps']['app']:
 
-                        tags = []
+                tags = []
+                for dd_tag, yarn_key in app_tags.iteritems():
+                    try:
+                        tags.append("{tag}:{value}".format(
+                            tag=dd_tag, value=app_json[yarn_key]
+                        ))
+                    except KeyError:
+                        self.log.error("Invalid value %s for application_tag", yarn_key)
 
-                        for tag in app_tags:
-                            try:
-                                key, value = tag.split(':')
-                                tags.append("{tag}:{value}".format(
-                                    tag=value, value=str(app_json[key])
-                                ))
-                            except ValueError:
-                                self.log.error("Invalid value %s for application_tag", tag)
-                            except KeyError:
-                                self.log.warning("Application tag %s not found, so it will be ignored", key)
+                tags.extend(addl_tags)
 
-                        tags.extend(addl_tags)
-
-                        self._set_yarn_metrics_from_json(tags, app_json, YARN_APP_METRICS)
+                self._set_yarn_metrics_from_json(tags, app_json, YARN_APP_METRICS)
 
     def _yarn_node_metrics(self, rm_address, addl_tags):
         '''
         Get metrics related to YARN nodes
         '''
         metrics_json = self._rest_request_to_json(rm_address, YARN_NODES_PATH)
 
-        if metrics_json:
-            if metrics_json['nodes'] is not None:
-                if metrics_json['nodes']['node'] is not None:
+        if (metrics_json and metrics_json['nodes'] is not None and
+            metrics_json['nodes']['node'] is not None):
 
-                    for node_json in metrics_json['nodes']['node']:
-                        node_id = node_json['id']
+            for node_json in metrics_json['nodes']['node']:
+                node_id = node_json['id']
 
-                        tags = ['node_id:%s' % str(node_id)]
-                        tags.extend(addl_tags)
+                tags = ['node_id:%s' % str(node_id)]
+                tags.extend(addl_tags)
 
-                        self._set_yarn_metrics_from_json(tags, node_json, YARN_NODE_METRICS)
+                self._set_yarn_metrics_from_json(tags, node_json, YARN_NODE_METRICS)
 
     def _set_yarn_metrics_from_json(self, tags, metrics_json, yarn_metrics):
         '''
@@ -254,7 +257,7 @@ def _set_metric(self, metric_name, metric_type, value, tags=None, device_name=No
         elif metric_type == INCREMENT:
             self.increment(metric_name, value, tags=tags, device_name=device_name)
         else:
-            self.log.error('Metric type "%s" unknown' % (metric_type))
+            self.log.error('Metric type "%s" unknown', metric_type)
 
     def _rest_request_to_json(self, address, object_path, *args, **kwargs):
         '''

conf.d/yarn.yaml.example

@@ -3,7 +3,7 @@ init_config:
 instances:
   # The YARN check retrieves metrics from YARNS's ResourceManager. This
   # check must be run from the Master Node and the ResourceManager URI must
-  # be specified below. The ResourceManager URI is composed of the 
+  # be specified below. The ResourceManager URI is composed of the
   # ResourceManager's hostname and port.
   #
   # The ResourceManager hostname can be found in the yarn-site.xml conf file
@@ -17,16 +17,19 @@ instances:
     # A Required friendly name for the cluster.
     # cluster_name: MyYarnCluster
 
-    # Values from running applications response that will be applied as datadog tags, set as app_key:datadog_tag
-    # You can find the whole list keys on yarn api reference - application response body:
-    # https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html#Cluster_Applications_API
-    # Example:
-    #   - queue:app_queue
-    #   - user:app_user
-     application_tags:
-       - name:app_name
-
     # Optional tags to be applied to every emitted metric.
     # tags:
     #   - key:value
-    #   - instance:production
+    #   - instance:production
+
+    # Optional tags retrieved from the application data to be applied to the
+    # application metrics.
+    # application_tags:
+    #   # tag_prefix: yarn_key
+    #   - app_queue: queue
+    # This will add a tag 'app_queue:name_of_the_queue' to the app metrics,
+    # app_queue being the tag_prefix and queue the actual YARN key. Whole list
+    # of keys on yarn api reference:
+    # https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html#Cluster_Applications_API
+    # By default, the application name is collected with the prefix app_name.
+

tests/checks/mock/test_yarn.py

@@ -57,10 +57,9 @@ class YARNCheck(AgentCheckTest):
         'tags': [
             'opt_key:opt_value'
         ],
-        'application_tags': [
-            'id:app_id',
-            'name:app_name'
-        ]
+        'application_tags': {
+            'app_id': 'id'
+        }
     }
 
     YARN_CLUSTER_METRICS_VALUES = {