From 3b7460940cb4892d72d2dbf683593e5f526f0641 Mon Sep 17 00:00:00 2001 From: Alexis Le-Quoc Date: Fri, 13 Dec 2013 15:52:33 -0500 Subject: [PATCH 01/10] In progress --- .gitignore | 3 +- checks.d/postgres.py | 167 ++++++++++++++++++++++++++++++------------- 2 files changed, 121 insertions(+), 49 deletions(-) diff --git a/.gitignore b/.gitignore index 255a0d6f0d..93cd309bd4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ stats.dat conf.d/*.yaml !conf.d/network.yaml packaging/build/ -packaging/root/ \ No newline at end of file +packaging/root/ +/.vagrant/machines/default/virtualbox/id diff --git a/checks.d/postgres.py b/checks.d/postgres.py index 09b36795ea..8c41e46755 100644 --- a/checks.d/postgres.py +++ b/checks.d/postgres.py @@ -3,21 +3,36 @@ GAUGE = 'gauge' RATE = 'rate' -METRICS = { - 'numbackends' : ('connections', GAUGE), - 'xact_commit' : ('commits', RATE), - 'xact_rollback' : ('rollbacks', RATE), - 'blks_read' : ('disk_read', RATE), - 'blks_hit' : ('buffer_hit', RATE), - 'tup_returned' : ('rows_returned', RATE), - 'tup_fetched' : ('rows_fetched', RATE), - 'tup_inserted' : ('rows_inserted', RATE), - 'tup_updated' : ('rows_updated', RATE), - 'tup_deleted' : ('rows_deleted', RATE), - +# Comment here + +# turning columns into tags +DB_METRICS = { + 'descriptors': [ + ('datname', 'db') + ], + 'metrics': { + 'numbackends' : ('connections', GAUGE), + 'xact_commit' : ('commits', RATE), + 'xact_rollback' : ('rollbacks', RATE), + 'blks_read' : ('disk_read', RATE), + 'blks_hit' : ('buffer_hit', RATE), + 'tup_returned' : ('rows_returned', RATE), + 'tup_fetched' : ('rows_fetched', RATE), + 'tup_inserted' : ('rows_inserted', RATE), + 'tup_updated' : ('rows_updated', RATE), + 'tup_deleted' : ('rows_deleted', RATE), + }, + 'query': """ +SELECT datname, + %s + FROM pg_stat_database + WHERE datname not ilike 'template%%' + AND datname not ilike 'postgres' +""", + 'relation': False, } -NEWER_92_METRICS = { +NEWER_92_DB_METRICS = { 'blk_read_time' : ('disk_read_time', GAUGE), 'blk_write_time' : ('disk_write_time', GAUGE), 'deadlocks' : ('deadlocks', GAUGE), @@ -25,6 +40,50 @@ 'temp_files' : ('temp_files', RATE), } +REL_METRICS = { + 'descriptors': [ + ('relname', 'table') + ], + 'metrics': { + 'seq_scan' : ('seq_scans', RATE), + 'seq_tup_read' : ('seq_rows_read', RATE), + 'idx_scan' : ('index_scans', RATE), + 'idx_tup_fetch' : ('index_rows_fetched', RATE), + 'n_tup_ins' : ('rows_inserted', RATE), + 'n_tup_upd' : ('rows_updated', RATE), + 'n_tup_del' : ('rows_deleted', RATE), + 'n_tup_hot_upd' : ('rows_hot_updated', RATE), + 'n_live_tup' : ('live_rows', GAUGE), + 'n_dead_tup' : ('dead_rows', GAUGE), + }, + 'query': """ +SELECT relname, + %s + FROM pg_stat_user_tables + WHERE relname = %s""", + 'relation': True, +} + +IDX_METRICS = { + 'descriptors': [ + ('relname', 'table'), + ('indexrelname', 'index') + ], + 'metrics': { + 'idx_scan' : ('index_scans', RATE), + 'idx_tup_read' : ('index_rows_read', RATE), + 'idx_tup_fetch' : ('index_rows_fetched', RATE), + }, + 'query': """ +SELECT relname, + indexrelname, + %s + FROM pg_stat_user_indexes + WHERE relname = %s""", + 'relation': True, +} + + class PostgreSql(AgentCheck): def __init__(self, name, init_config, agentConfig): AgentCheck.__init__(self, name, init_config, agentConfig) @@ -62,48 +121,59 @@ def _is_9_2_or_above(self, key, db): return False - - def _collect_stats(self, key, db, instance_tags): - - metrics_to_collect = METRICS + def _collect_stats(self, key, db, instance_tags, relations): + """Query pg_stat_* for various metrics + If relations is not an empty list, gather per-relation metrics + on top of that. + """ + def get_dd_metric_name(pg_name, mapping): + "Turn a pg metric name into a dd metric name" + return "postgresql.%s" % mapping.get('metrics', {}).get(pg_name, pg_name) + + # Extended 9.2+ metrics if self._is_9_2_or_above(key, db): - metrics_to_collect.update(NEWER_92_METRICS) - - - metrics_keys = metrics_to_collect.keys() - fields = ",".join(metrics_keys) - query = """SELECT datname, - %s - FROM pg_stat_database - WHERE datname not ilike 'template%%' - AND datname not ilike 'postgres' - ;""" % fields + DB_METRICS['metrics'].update(NEWER_92_METRICS) cursor = db.cursor() - cursor.execute(query) - result = cursor.fetchone() - while result is not None: - dbname = result[0] - try: - tags = ['db:%s' % dbname] + instance_tags - except Exception: - # if tags is none or is not of the right type - tags = ['db:%s' % dbname] - - for i, value in enumerate(result): - if i == 0: - # This is the dbname - continue - - metric_name = "postgresql.%s" % metrics_to_collect[metrics_keys[i-1]][0] + try: + for scope in (DB_METRICS, REL_METRICS, IDX_METRICS): + # build query + fields = ",".join(scope['metrics'].keys()) + query = scope['query'] % fields + + # execute query + cursor.execute(query) + results = cursor.fetchall() + + # parse results + # A row should look like this + # (descriptor, descriptor, ..., value, value, value, value, ...) + # with descriptor a table, relation or index name + + + for row in results: + # [(metric-map, value), (metric-map, value), ...] + # shift the results since the first columns will be the "descriptors" + desc = scope['descriptors'] + x = zip(scope['metrics'], row[len(desc):]) + + # compute tags + if instance_tags is None: + instance_tags = [] + # turn descriptors into tags + tags = instance_tags.extend(["%s:%s" % (d[0][1], d for d in zip(desc, row[:len(desc)])]) + + # [(metric, value), (metric, value), ...] + metric_name = lambda name: "postgresql.%s" % metrics_to_collect[metrics_keys[i-1]][0] metric_type = metrics_to_collect[metrics_keys[i-1]][1] if metric_type == GAUGE: self.gauge(metric_name, value, tags=tags) elif metric_type == RATE: self.rate(metric_name, value, tags=tags) - - result = cursor.fetchone() - del cursor + + result = cursor.fetchone() + finally: + del cursor def get_connection(self, key, host, port, user, password, dbname): @@ -134,6 +204,7 @@ def check(self, instance): password = instance.get('password', '') tags = instance.get('tags', []) dbname = instance.get('database', 'postgres') + relations = instance.get('relations', []) # Clean up tags in case there was a None entry in the instance # e.g. if the yaml contains tags: but no actual tags if tags is None: @@ -147,7 +218,7 @@ def check(self, instance): self.log.debug("Running check against version %s" % version) # Collect metrics - self._collect_stats(key, db, tags) + self._collect_stats(key, db, tags, relations) @staticmethod def parse_agent_config(agentConfig): From 556d114700097f35fdb4680c9bcebef09ce982d1 Mon Sep 17 00:00:00 2001 From: Alexis Le-Quoc Date: Fri, 13 Dec 2013 22:50:04 -0500 Subject: [PATCH 02/10] MANIFEST has jmxfetch --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 2a20c56cb5..9364a95566 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include datadog.conf.example -include checks/libs/jmxterm-1.0-alpha-4-uber.jar +include checks/libs/jmxfetch-0.1.2-jar-with-dependencies.jar include pup/pup.html include pup/static/* From d11e9f780368fde75a6774dbdb92c5d13f119813 Mon Sep 17 00:00:00 2001 From: Alexis Le-Quoc Date: Sat, 14 Dec 2013 22:01:52 -0500 Subject: [PATCH 03/10] Only works for db-wide metrics --- checks.d/postgres.py | 231 ++++++++++++++++++++++--------------------- 1 file changed, 119 insertions(+), 112 deletions(-) diff --git a/checks.d/postgres.py b/checks.d/postgres.py index 8c41e46755..73930669ad 100644 --- a/checks.d/postgres.py +++ b/checks.d/postgres.py @@ -1,90 +1,89 @@ -from checks import AgentCheck - -GAUGE = 'gauge' -RATE = 'rate' - -# Comment here - -# turning columns into tags -DB_METRICS = { - 'descriptors': [ - ('datname', 'db') - ], - 'metrics': { - 'numbackends' : ('connections', GAUGE), - 'xact_commit' : ('commits', RATE), - 'xact_rollback' : ('rollbacks', RATE), - 'blks_read' : ('disk_read', RATE), - 'blks_hit' : ('buffer_hit', RATE), - 'tup_returned' : ('rows_returned', RATE), - 'tup_fetched' : ('rows_fetched', RATE), - 'tup_inserted' : ('rows_inserted', RATE), - 'tup_updated' : ('rows_updated', RATE), - 'tup_deleted' : ('rows_deleted', RATE), - }, - 'query': """ +from checks import AgentCheck, CheckException + +class PostgreSql(AgentCheck): + """ + """ + + RATE = AgentCheck.rate + GAUGE = AgentCheck.gauge + + # turning columns into tags + DB_METRICS = { + 'descriptors': [ + ('datname', 'db') + ], + 'metrics': { + 'numbackends' : ('postgresql.connections', GAUGE), + 'xact_commit' : ('postgresql.commits', RATE), + 'xact_rollback' : ('postgresql.rollbacks', RATE), + 'blks_read' : ('postgresql.disk_read', RATE), + 'blks_hit' : ('postgresql.buffer_hit', RATE), + 'tup_returned' : ('postgresql.rows_returned', RATE), + 'tup_fetched' : ('postgresql.rows_fetched', RATE), + 'tup_inserted' : ('postgresql.rows_inserted', RATE), + 'tup_updated' : ('postgresql.rows_updated', RATE), + 'tup_deleted' : ('postgresql.rows_deleted', RATE), + }, + 'query': """ SELECT datname, %s FROM pg_stat_database WHERE datname not ilike 'template%%' AND datname not ilike 'postgres' """, - 'relation': False, -} - -NEWER_92_DB_METRICS = { - 'blk_read_time' : ('disk_read_time', GAUGE), - 'blk_write_time' : ('disk_write_time', GAUGE), - 'deadlocks' : ('deadlocks', GAUGE), - 'temp_bytes' : ('temp_bytes', RATE), - 'temp_files' : ('temp_files', RATE), -} - -REL_METRICS = { - 'descriptors': [ - ('relname', 'table') - ], - 'metrics': { - 'seq_scan' : ('seq_scans', RATE), - 'seq_tup_read' : ('seq_rows_read', RATE), - 'idx_scan' : ('index_scans', RATE), - 'idx_tup_fetch' : ('index_rows_fetched', RATE), - 'n_tup_ins' : ('rows_inserted', RATE), - 'n_tup_upd' : ('rows_updated', RATE), - 'n_tup_del' : ('rows_deleted', RATE), - 'n_tup_hot_upd' : ('rows_hot_updated', RATE), - 'n_live_tup' : ('live_rows', GAUGE), - 'n_dead_tup' : ('dead_rows', GAUGE), - }, - 'query': """ + 'relation': False, + } + + NEWER_92_METRICS = { + 'deadlocks' : ('postgresql.deadlocks', GAUGE), + 'temp_bytes' : ('postgresql.temp_bytes', RATE), + 'temp_files' : ('postgresql.temp_files', RATE), + } + + REL_METRICS = { + 'descriptors': [ + ('relname', 'table') + ], + 'metrics': { + 'seq_scan' : ('postgresql.seq_scans', RATE), + 'seq_tup_read' : ('postgresql.seq_rows_read', RATE), + 'idx_scan' : ('postgresql.index_scans', RATE), + 'idx_tup_fetch' : ('postgresql.index_rows_fetched', RATE), + 'n_tup_ins' : ('postgresql.rows_inserted', RATE), + 'n_tup_upd' : ('postgresql.rows_updated', RATE), + 'n_tup_del' : ('postgresql.rows_deleted', RATE), + 'n_tup_hot_upd' : ('postgresql.rows_hot_updated', RATE), + 'n_live_tup' : ('postgresql.live_rows', GAUGE), + 'n_dead_tup' : ('postgresql.dead_rows', GAUGE), + }, + 'query': """ SELECT relname, %s FROM pg_stat_user_tables WHERE relname = %s""", - 'relation': True, -} - -IDX_METRICS = { - 'descriptors': [ - ('relname', 'table'), - ('indexrelname', 'index') - ], - 'metrics': { - 'idx_scan' : ('index_scans', RATE), - 'idx_tup_read' : ('index_rows_read', RATE), - 'idx_tup_fetch' : ('index_rows_fetched', RATE), - }, - 'query': """ + 'relation': True, + } + + IDX_METRICS = { + 'descriptors': [ + ('relname', 'table'), + ('indexrelname', 'index') + ], + 'metrics': { + 'idx_scan' : ('postgresql.index_scans', RATE), + 'idx_tup_read' : ('postgresql.index_rows_read', RATE), + 'idx_tup_fetch' : ('postgresql.index_rows_fetched', RATE), + }, + 'query': """ SELECT relname, indexrelname, %s FROM pg_stat_user_indexes WHERE relname = %s""", - 'relation': True, -} + 'relation': True, + } -class PostgreSql(AgentCheck): def __init__(self, name, init_config, agentConfig): AgentCheck.__init__(self, name, init_config, agentConfig) self.dbs = {} @@ -126,20 +125,22 @@ def _collect_stats(self, key, db, instance_tags, relations): If relations is not an empty list, gather per-relation metrics on top of that. """ - def get_dd_metric_name(pg_name, mapping): - "Turn a pg metric name into a dd metric name" - return "postgresql.%s" % mapping.get('metrics', {}).get(pg_name, pg_name) - + + # Clean up initial args + if instance_tags is None: + instance_tags = [] + # Extended 9.2+ metrics if self._is_9_2_or_above(key, db): - DB_METRICS['metrics'].update(NEWER_92_METRICS) - + self.DB_METRICS['metrics'].update(self.NEWER_92_METRICS) + cursor = db.cursor() try: - for scope in (DB_METRICS, REL_METRICS, IDX_METRICS): + for scope in (self.DB_METRICS, self.REL_METRICS, self.IDX_METRICS): # build query - fields = ",".join(scope['metrics'].keys()) - query = scope['query'] % fields + cols = scope['metrics'].keys() # list of metrics to query, in some order + # we must remember that order to parse results + query = scope['query'] % (", ".join(cols)) # assembled query # execute query cursor.execute(query) @@ -148,53 +149,55 @@ def get_dd_metric_name(pg_name, mapping): # parse results # A row should look like this # (descriptor, descriptor, ..., value, value, value, value, ...) - # with descriptor a table, relation or index name - + # with descriptor a PG relation or index name, which we use to create the tags for row in results: - # [(metric-map, value), (metric-map, value), ...] - # shift the results since the first columns will be the "descriptors" desc = scope['descriptors'] - x = zip(scope['metrics'], row[len(desc):]) - - # compute tags - if instance_tags is None: - instance_tags = [] # turn descriptors into tags - tags = instance_tags.extend(["%s:%s" % (d[0][1], d for d in zip(desc, row[:len(desc)])]) - - # [(metric, value), (metric, value), ...] - metric_name = lambda name: "postgresql.%s" % metrics_to_collect[metrics_keys[i-1]][0] - metric_type = metrics_to_collect[metrics_keys[i-1]][1] - if metric_type == GAUGE: - self.gauge(metric_name, value, tags=tags) - elif metric_type == RATE: - self.rate(metric_name, value, tags=tags) - - result = cursor.fetchone() + tags = instance_tags.extend(["%s:%s" % (d[0][1], d) for d in zip(desc, row[:len(desc)])]) + print tags + + # [(metric-map, value), (metric-map, value), ...] + # metric-map is: (dd_name, "rate"|"gauge") + # shift the results since the first columns will be the "descriptors" + values = zip([scope['metrics'][c] for c in cols], row[len(desc):]) + print values + + # To submit simply call the function for each value v + # v[0] == (metric_name, submit_function) + # v[1] == the actual value + # FIXME namedtuple probably better here + [v[0][1](self, v[0][0], v[1], tags=tags) for v in values] finally: del cursor def get_connection(self, key, host, port, user, password, dbname): - + "Get and memoize connections to instances" if key in self.dbs: return self.dbs[key] - elif host != '' and user != '': + elif host != "" and user != "": try: import psycopg2 as pg + if host == 'localhost' and password == '': + # Use ident method + return pg.connect("user=%s dbname=%s" % (user, dbname)) + elif port != '': + return pg.connect(host=host, port=port, user=user, + password=password, database=dbname) + else: + return pg.connect(host=host, user=user, password=password, + database=dbname) except ImportError: - raise ImportError("psycopg2 library can not be imported. Please check the installation instruction on the Datadog Website") - - if host == 'localhost' and password == '': - # Use ident method - return pg.connect("user=%s dbname=%s" % (user, dbname)) - elif port != '': - return pg.connect(host=host, port=port, user=user, - password=password, database=dbname) + raise ImportError("psycopg2 library can not be imported. Please check the installation instruction on the Datadog Website.") + + else: + if host is None or host == "": + raise CheckException("Please specify a Postgres host to connect to.") + elif user is None or user == "": + raise CheckException("Please specify a user to connect to Postgres as.") else: - return pg.connect(host=host, user=user, password=password, - database=dbname) + raise CheckException("Cannot connect to Postgres.") def check(self, instance): @@ -216,7 +219,7 @@ def check(self, instance): # Check version version = self._get_version(key, db) self.log.debug("Running check against version %s" % version) - + # Collect metrics self._collect_stats(key, db, tags, relations) @@ -238,3 +241,7 @@ def parse_agent_config(agentConfig): } return False + +if __name__ == '__main__': + p = PostgreSql("", {}, {}) + p.check({"host": "localhost", "port": 5432, "username": "alq", "password": "", "tags": ["code"]}) From 8904c361e53bff49d7c98ca9934225c110b2732a Mon Sep 17 00:00:00 2001 From: Alexis Le-Quoc Date: Sat, 14 Dec 2013 22:11:08 -0500 Subject: [PATCH 04/10] Tags are correct now --- checks.d/postgres.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/checks.d/postgres.py b/checks.d/postgres.py index 73930669ad..8fa63e8926 100644 --- a/checks.d/postgres.py +++ b/checks.d/postgres.py @@ -133,10 +133,17 @@ def _collect_stats(self, key, db, instance_tags, relations): # Extended 9.2+ metrics if self._is_9_2_or_above(key, db): self.DB_METRICS['metrics'].update(self.NEWER_92_METRICS) + + # Do we need relation-specific metrics? + if relations is None or relations == []: + metric_scope = (self.DB_METRICS, ) + else: + metric_scope = (self.DB_METRICS, self.REL_METRICS, self.IDX_METRICS) cursor = db.cursor() + try: - for scope in (self.DB_METRICS, self.REL_METRICS, self.IDX_METRICS): + for scope in metric_scope: # build query cols = scope['metrics'].keys() # list of metrics to query, in some order # we must remember that order to parse results @@ -152,10 +159,10 @@ def _collect_stats(self, key, db, instance_tags, relations): # with descriptor a PG relation or index name, which we use to create the tags for row in results: - desc = scope['descriptors'] # turn descriptors into tags - tags = instance_tags.extend(["%s:%s" % (d[0][1], d) for d in zip(desc, row[:len(desc)])]) - print tags + desc = scope['descriptors'] + # descriptors are: (pg_name, dd_tag_name): value + instance_tags.extend(["%s:%s" % (d[0][1], d[1]) for d in zip(desc, row[:len(desc)])]) # [(metric-map, value), (metric-map, value), ...] # metric-map is: (dd_name, "rate"|"gauge") @@ -167,7 +174,7 @@ def _collect_stats(self, key, db, instance_tags, relations): # v[0] == (metric_name, submit_function) # v[1] == the actual value # FIXME namedtuple probably better here - [v[0][1](self, v[0][0], v[1], tags=tags) for v in values] + [v[0][1](self, v[0][0], v[1], tags=instance_tags) for v in values] finally: del cursor @@ -244,4 +251,9 @@ def parse_agent_config(agentConfig): if __name__ == '__main__': p = PostgreSql("", {}, {}) - p.check({"host": "localhost", "port": 5432, "username": "alq", "password": "", "tags": ["code"]}) + p.check({"host": "localhost", + "port": 5432, + "username": "alq", + "password": "", + "tags": ["code"], + "relations": ["hourly_usage"]}) From 1849d92df6cfbe6af542b2313fd8490e22c0b89f Mon Sep 17 00:00:00 2001 From: Alexis Le-Quoc Date: Sat, 14 Dec 2013 23:52:06 -0500 Subject: [PATCH 05/10] Per-relation and per-index metric work --- checks.d/postgres.py | 125 +++++++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 57 deletions(-) diff --git a/checks.d/postgres.py b/checks.d/postgres.py index 8fa63e8926..faa2b93be4 100644 --- a/checks.d/postgres.py +++ b/checks.d/postgres.py @@ -1,7 +1,7 @@ from checks import AgentCheck, CheckException class PostgreSql(AgentCheck): - """ + """Collects per-database, and optionally per-relation metrics """ RATE = AgentCheck.rate @@ -60,7 +60,7 @@ class PostgreSql(AgentCheck): SELECT relname, %s FROM pg_stat_user_tables - WHERE relname = %s""", + WHERE relname = ANY(%s)""", 'relation': True, } @@ -79,7 +79,7 @@ class PostgreSql(AgentCheck): indexrelname, %s FROM pg_stat_user_indexes - WHERE relname = %s""", + WHERE relname = ANY(%s)""", 'relation': True, } @@ -89,16 +89,6 @@ def __init__(self, name, init_config, agentConfig): self.dbs = {} self.versions = {} - def get_library_versions(self): - try: - import psycopg2 - version = psycopg2.__version__ - except ImportError: - version = "Not Found" - except AttributeError: - version = "Unknown" - - return {"psycopg2": version} def _get_version(self, key, db): if key not in self.versions: @@ -116,7 +106,7 @@ def _get_version(self, key, db): def _is_9_2_or_above(self, key, db): version = self._get_version(key, db) if type(version) == list: - return version >= [9,2,0] + return version >= [9, 2, 0] return False @@ -126,57 +116,66 @@ def _collect_stats(self, key, db, instance_tags, relations): on top of that. """ - # Clean up initial args - if instance_tags is None: - instance_tags = [] - # Extended 9.2+ metrics if self._is_9_2_or_above(key, db): self.DB_METRICS['metrics'].update(self.NEWER_92_METRICS) # Do we need relation-specific metrics? if relations is None or relations == []: - metric_scope = (self.DB_METRICS, ) + metric_scope = (self.DB_METRICS,) else: metric_scope = (self.DB_METRICS, self.REL_METRICS, self.IDX_METRICS) - cursor = db.cursor() - - try: - for scope in metric_scope: - # build query - cols = scope['metrics'].keys() # list of metrics to query, in some order - # we must remember that order to parse results - query = scope['query'] % (", ".join(cols)) # assembled query - - # execute query + for scope in metric_scope: + # build query + cols = scope['metrics'].keys() # list of metrics to query, in some order + # we must remember that order to parse results + cursor = db.cursor() + + # if this is a relation-specific query, we need to list all relations last + if scope['relation'] and len(relations) > 0: + query = scope['query'] % (", ".join(cols), "%s") # Keep the last %s intact + cursor.execute(query, (relations, )) + else: + query = scope['query'] % (", ".join(cols)) cursor.execute(query) - results = cursor.fetchall() - # parse results - # A row should look like this - # (descriptor, descriptor, ..., value, value, value, value, ...) - # with descriptor a PG relation or index name, which we use to create the tags + results = cursor.fetchall() + cursor.close() - for row in results: - # turn descriptors into tags - desc = scope['descriptors'] - # descriptors are: (pg_name, dd_tag_name): value - instance_tags.extend(["%s:%s" % (d[0][1], d[1]) for d in zip(desc, row[:len(desc)])]) + # parse & submit results + # A row should look like this + # (descriptor, descriptor, ..., value, value, value, value, ...) + # with descriptor a PG relation or index name, which we use to create the tags + for row in results: + # turn descriptors into tags + desc = scope['descriptors'] + # Check that all columns will be processed + assert len(row) == len(cols) + len(desc) + + # Build tags + # descriptors are: (pg_name, dd_tag_name): value + # Special-case the "db" tag, which overrides the one that is passed as instance_tag + # The reason is that pg_stat_database returns all databases regardless of the + # connection. + if not scope['relation']: + tags = [t for t in instance_tags if not t.startswith("db:")] + else: + tags = [t for t in instance_tags] - # [(metric-map, value), (metric-map, value), ...] - # metric-map is: (dd_name, "rate"|"gauge") - # shift the results since the first columns will be the "descriptors" - values = zip([scope['metrics'][c] for c in cols], row[len(desc):]) - print values - - # To submit simply call the function for each value v - # v[0] == (metric_name, submit_function) - # v[1] == the actual value - # FIXME namedtuple probably better here - [v[0][1](self, v[0][0], v[1], tags=instance_tags) for v in values] - finally: - del cursor + tags += ["%s:%s" % (d[0][1], d[1]) for d in zip(desc, row[:len(desc)])] + + # [(metric-map, value), (metric-map, value), ...] + # metric-map is: (dd_name, "rate"|"gauge") + # shift the results since the first columns will be the "descriptors" + values = zip([scope['metrics'][c] for c in cols], row[len(desc):]) + + # To submit simply call the function for each value v + # v[0] == (metric_name, submit_function) + # v[1] == the actual value + # tags are + [v[0][1](self, v[0][0], v[1], tags=tags) for v in values] + def get_connection(self, key, host, port, user, password, dbname): "Get and memoize connections to instances" @@ -215,13 +214,17 @@ def check(self, instance): tags = instance.get('tags', []) dbname = instance.get('database', 'postgres') relations = instance.get('relations', []) + + key = '%s:%s' % (host, port) + db = self.get_connection(key, host, port, user, password, dbname) + # Clean up tags in case there was a None entry in the instance # e.g. if the yaml contains tags: but no actual tags if tags is None: tags = [] - key = '%s:%s' % (host, port) - - db = self.get_connection(key, host, port, user, password, dbname) + + # preset tags to the database name + tags.extend(["db:%s" % dbname]) # Check version version = self._get_version(key, db) @@ -250,10 +253,18 @@ def parse_agent_config(agentConfig): return False if __name__ == '__main__': + # Assumes that you have a pg db that bears the same name + # as your unix username. Furthermore assumes that you can connect + # using ident. + # if you want to collect per-table stats, simply pass them as arguments p = PostgreSql("", {}, {}) + # get current username + import getpass, sys + usr = getpass.getuser() p.check({"host": "localhost", "port": 5432, - "username": "alq", + "username": usr, "password": "", "tags": ["code"], - "relations": ["hourly_usage"]}) + "database": usr, + "relations": sys.argv[1:]}) From ba8b57d4a29514aae8e1e42a8292976b77ad36fe Mon Sep 17 00:00:00 2001 From: Alexis Le-Quoc Date: Sat, 14 Dec 2013 23:59:17 -0500 Subject: [PATCH 06/10] Update configuration --- conf.d/postgres.yaml.example | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/conf.d/postgres.yaml.example b/conf.d/postgres.yaml.example index 0c6e22d4a8..a07f8fdb59 100644 --- a/conf.d/postgres.yaml.example +++ b/conf.d/postgres.yaml.example @@ -8,4 +8,15 @@ instances: # dbname: db_name # tags: # - optional_tag1 -# - optional_tag2 \ No newline at end of file +# - optional_tag2 + +# Custom-metrics section + +# You can now track per-relation (table) metrics +# You need to specify the list. Each relation +# generates a lot of metrics (10 + 10 per index) +# so you want to only use the ones you really care about + +# relations: +# - my_table +# - my_other_table From 3ba5eef053653fb76a8ee90fc14144c389be2d9a Mon Sep 17 00:00:00 2001 From: Alexis Le-Quoc Date: Sun, 15 Dec 2013 00:02:51 -0500 Subject: [PATCH 07/10] Reinstate get_library_versions --- checks.d/postgres.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/checks.d/postgres.py b/checks.d/postgres.py index cb99a4de56..d713a9b17a 100644 --- a/checks.d/postgres.py +++ b/checks.d/postgres.py @@ -89,6 +89,15 @@ def __init__(self, name, init_config, agentConfig): self.dbs = {} self.versions = {} + def get_library_versions(self): + try: + import psycopg2 + version = psycopg2.__version__ + except ImportError: + version = "Not Found" + except AttributeError: + version = "Unknown" + return {"psycopg2": version} def _get_version(self, key, db): if key not in self.versions: From f731cc48a5aab76426b1d2a0188082075dc1ce20 Mon Sep 17 00:00:00 2001 From: Alexis Le-Quoc Date: Sun, 15 Dec 2013 18:37:00 -0500 Subject: [PATCH 08/10] Ignore vagrant files --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 93cd309bd4..5d420bb7d2 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,4 @@ conf.d/*.yaml !conf.d/network.yaml packaging/build/ packaging/root/ -/.vagrant/machines/default/virtualbox/id +.vagrant/* From 9e5e02fcc2953b73e7a74fdf7c762a014827e9ff Mon Sep 17 00:00:00 2001 From: Alexis Le-Quoc Date: Fri, 10 Jan 2014 17:22:39 -0500 Subject: [PATCH 09/10] check can be tested on the command line No need for a __main__ section. --- MANIFEST.in | 1 + checks.d/postgres.py | 17 ----------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 9364a95566..2dc886532c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include datadog.conf.example +include checks/libs/jmxterm-1.0-alpha-4-uber.jar include checks/libs/jmxfetch-0.1.2-jar-with-dependencies.jar include pup/pup.html include pup/static/* diff --git a/checks.d/postgres.py b/checks.d/postgres.py index d713a9b17a..57bc572b37 100644 --- a/checks.d/postgres.py +++ b/checks.d/postgres.py @@ -272,20 +272,3 @@ def parse_agent_config(agentConfig): } return False - -if __name__ == '__main__': - # Assumes that you have a pg db that bears the same name - # as your unix username. Furthermore assumes that you can connect - # using ident. - # if you want to collect per-table stats, simply pass them as arguments - p = PostgreSql("", {}, {}) - # get current username - import getpass, sys - usr = getpass.getuser() - p.check({"host": "localhost", - "port": 5432, - "username": usr, - "password": "", - "tags": ["code"], - "database": usr, - "relations": sys.argv[1:]}) From 2985f7e5f13aec367af50c44ee0f14874a5776f5 Mon Sep 17 00:00:00 2001 From: Alexis Le-Quoc Date: Fri, 10 Jan 2014 17:24:36 -0500 Subject: [PATCH 10/10] Cosmetic change to test --- checks.d/postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checks.d/postgres.py b/checks.d/postgres.py index 57bc572b37..99d63445d8 100644 --- a/checks.d/postgres.py +++ b/checks.d/postgres.py @@ -130,7 +130,7 @@ def _collect_stats(self, key, db, instance_tags, relations): self.DB_METRICS['metrics'].update(self.NEWER_92_METRICS) # Do we need relation-specific metrics? - if relations is None or relations == []: + if not relations: metric_scope = (self.DB_METRICS,) else: metric_scope = (self.DB_METRICS, self.REL_METRICS, self.IDX_METRICS)