From 325f1e6d464a694cb5b911a9d23dc7f85d901957 Mon Sep 17 00:00:00 2001 From: Quentin Madec Date: Tue, 22 Nov 2016 18:00:39 -0500 Subject: [PATCH] [yarn] refactor application_tags Change config, hopefully with more documentation. --- checks.d/yarn.py | 63 ++++++++++++++++++---------------- conf.d/yarn.yaml.example | 25 ++++++++------ tests/checks/mock/test_yarn.py | 7 ++-- 3 files changed, 50 insertions(+), 45 deletions(-) diff --git a/checks.d/yarn.py b/checks.d/yarn.py index 5dde9c954d..21fca85fc5 100644 --- a/checks.d/yarn.py +++ b/checks.d/yarn.py @@ -148,7 +148,14 @@ def check(self, instance): # Get properties from conf file rm_address = instance.get('resourcemanager_uri', DEFAULT_RM_URI) - app_tags = instance.get('application_tags', []) + app_tags = instance.get('application_tags', {}) + + if type(app_tags) is not dict: + self.log.error('application_tags is incorrect: %s is not a dictionary', app_tags) + app_tags = {} + + # Collected by default + app_tags['app_name'] = 'name' # Get additional tags from the conf file tags = instance.get('tags', []) @@ -187,32 +194,29 @@ def _yarn_app_metrics(self, rm_address, app_tags, addl_tags): ''' Get metrics for running applications ''' - metrics_json = self._rest_request_to_json(rm_address, + metrics_json = self._rest_request_to_json( + rm_address, YARN_APPS_PATH, - states=YARN_APPLICATION_STATES) + states=YARN_APPLICATION_STATES + ) - if metrics_json: - if metrics_json['apps'] is not None: - if metrics_json['apps']['app'] is not None: + if (metrics_json and metrics_json['apps'] is not None and + metrics_json['apps']['app'] is not None): - for app_json in metrics_json['apps']['app']: + for app_json in metrics_json['apps']['app']: - tags = [] + tags = [] + for dd_tag, yarn_key in app_tags.iteritems(): + try: + tags.append("{tag}:{value}".format( + tag=dd_tag, value=app_json[yarn_key] + )) + except KeyError: + self.log.error("Invalid value %s for application_tag", yarn_key) - for tag in app_tags: - try: - key, value = tag.split(':') - tags.append("{tag}:{value}".format( - tag=value, value=str(app_json[key]) - )) - except ValueError: - self.log.error("Invalid value %s for application_tag", tag) - except KeyError: - self.log.warning("Application tag %s not found, so it will be ignored", key) + tags.extend(addl_tags) - tags.extend(addl_tags) - - self._set_yarn_metrics_from_json(tags, app_json, YARN_APP_METRICS) + self._set_yarn_metrics_from_json(tags, app_json, YARN_APP_METRICS) def _yarn_node_metrics(self, rm_address, addl_tags): ''' @@ -220,17 +224,16 @@ def _yarn_node_metrics(self, rm_address, addl_tags): ''' metrics_json = self._rest_request_to_json(rm_address, YARN_NODES_PATH) - if metrics_json: - if metrics_json['nodes'] is not None: - if metrics_json['nodes']['node'] is not None: + if (metrics_json and metrics_json['nodes'] is not None and + metrics_json['nodes']['node'] is not None): - for node_json in metrics_json['nodes']['node']: - node_id = node_json['id'] + for node_json in metrics_json['nodes']['node']: + node_id = node_json['id'] - tags = ['node_id:%s' % str(node_id)] - tags.extend(addl_tags) + tags = ['node_id:%s' % str(node_id)] + tags.extend(addl_tags) - self._set_yarn_metrics_from_json(tags, node_json, YARN_NODE_METRICS) + self._set_yarn_metrics_from_json(tags, node_json, YARN_NODE_METRICS) def _set_yarn_metrics_from_json(self, tags, metrics_json, yarn_metrics): ''' @@ -254,7 +257,7 @@ def _set_metric(self, metric_name, metric_type, value, tags=None, device_name=No elif metric_type == INCREMENT: self.increment(metric_name, value, tags=tags, device_name=device_name) else: - self.log.error('Metric type "%s" unknown' % (metric_type)) + self.log.error('Metric type "%s" unknown', metric_type) def _rest_request_to_json(self, address, object_path, *args, **kwargs): ''' diff --git a/conf.d/yarn.yaml.example b/conf.d/yarn.yaml.example index 5d3df17857..3027cb8273 100644 --- a/conf.d/yarn.yaml.example +++ b/conf.d/yarn.yaml.example @@ -3,7 +3,7 @@ init_config: instances: # The YARN check retrieves metrics from YARNS's ResourceManager. This # check must be run from the Master Node and the ResourceManager URI must - # be specified below. The ResourceManager URI is composed of the + # be specified below. The ResourceManager URI is composed of the # ResourceManager's hostname and port. # # The ResourceManager hostname can be found in the yarn-site.xml conf file @@ -17,16 +17,19 @@ instances: # A Required friendly name for the cluster. # cluster_name: MyYarnCluster - # Values from running applications response that will be applied as datadog tags, set as app_key:datadog_tag - # You can find the whole list keys on yarn api reference - application response body: - # https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html#Cluster_Applications_API - # Example: - # - queue:app_queue - # - user:app_user - application_tags: - - name:app_name - # Optional tags to be applied to every emitted metric. # tags: # - key:value - # - instance:production \ No newline at end of file + # - instance:production + + # Optional tags retrieved from the application data to be applied to the + # application metrics. + # application_tags: + # # tag_prefix: yarn_key + # - app_queue: queue + # This will add a tag 'app_queue:name_of_the_queue' to the app metrics, + # app_queue being the tag_prefix and queue the actual YARN key. Whole list + # of keys on yarn api reference: + # https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html#Cluster_Applications_API + # By default, the application name is collected with the prefix app_name. + diff --git a/tests/checks/mock/test_yarn.py b/tests/checks/mock/test_yarn.py index 4786e0109b..02d85b1a21 100644 --- a/tests/checks/mock/test_yarn.py +++ b/tests/checks/mock/test_yarn.py @@ -57,10 +57,9 @@ class YARNCheck(AgentCheckTest): 'tags': [ 'opt_key:opt_value' ], - 'application_tags': [ - 'id:app_id', - 'name:app_name' - ] + 'application_tags': { + 'app_id': 'id' + } } YARN_CLUSTER_METRICS_VALUES = {