Add gsum/gcount to GaugeHistogram.

Allow gsum, gcount, and created to be sanely returned in Prometheus
format.

Extend openmetrics parser unittests to cover Info and StateSet.

Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>

prometheus_client/core.py

@@ -122,6 +122,7 @@ def _get_names(self, collector):
             'counter': ['_total', '_created'],
             'summary': ['', '_sum', '_count', '_created'],
             'histogram': ['_bucket', '_sum', '_count', '_created'],
+            'gaugehistogram': ['_bucket', '_gsum', '_gcount'],
             'info': ['_info'],
         }
         for metric in desc_func():
@@ -391,29 +392,33 @@ class GaugeHistogramMetricFamily(Metric):
 
     For use by custom collectors.
     '''
-    def __init__(self, name, documentation, buckets=None, labels=None, unit=''):
+    def __init__(self, name, documentation, buckets=None, gsum_value=None, labels=None, unit=''):
         Metric.__init__(self, name, documentation, 'gaugehistogram', unit)
         if labels is not None and buckets is not None:
             raise ValueError('Can only specify at most one of buckets and labels.')
         if labels is None:
             labels = []
         self._labelnames = tuple(labels)
         if buckets is not None:
-            self.add_metric([], buckets)
+            self.add_metric([], buckets, gsum_value)
 
-    def add_metric(self, labels, buckets, timestamp=None):
+    def add_metric(self, labels, buckets, gsum_value, timestamp=None):
         '''Add a metric to the metric family.
 
         Args:
           labels: A list of label values
           buckets: A list of pairs of bucket names and values.
               The buckets must be sorted, and +Inf present.
+          gsum_value: The sum value of the metric.
         '''
         for bucket, value in buckets:
             self.samples.append(Sample(
                 self.name + '_bucket',
                 dict(list(zip(self._labelnames, labels)) + [('le', bucket)]),
                 value, timestamp))
+        # +Inf is last and provides the count value.
+        self.samples.append(Sample(self.name + '_gcount', dict(zip(self._labelnames, labels)), buckets[-1][1], timestamp))
+        self.samples.append(Sample(self.name + '_gsum', dict(zip(self._labelnames, labels)), gsum_value, timestamp))
 
 
 class InfoMetricFamily(Metric):
@@ -465,7 +470,7 @@ def add_metric(self, labels, value, timestamp=None):
           value: A dict of string state names to booleans
         '''
         labels = tuple(labels)
-        for state, enabled in value.items():
+        for state, enabled in sorted(value.items()):
             v = (1 if enabled else 0)
             self.samples.append(Sample(self.name,
                 dict(zip(self._labelnames + (self.name,), labels + (state,))), v, timestamp))

prometheus_client/exposition.py

@@ -67,6 +67,22 @@ def start_wsgi_server(port, addr='', registry=core.REGISTRY):
 
 def generate_latest(registry=core.REGISTRY):
     '''Returns the metrics from the registry in latest text format as a string.'''
+
+    def sample_line(s):
+        if s.labels:
+            labelstr = '{{{0}}}'.format(','.join(
+                ['{0}="{1}"'.format(
+                 k, v.replace('\\', r'\\').replace('\n', r'\n').replace('"', r'\"'))
+                 for k, v in sorted(s.labels.items())]))
+        else:
+            labelstr = ''
+        timestamp = ''
+        if s.timestamp is not None:
+            # Convert to milliseconds.
+            timestamp = ' {0:d}'.format(int(float(s.timestamp) * 1000))
+        return '{0}{1} {2}{3}\n'.format(
+            s.name, labelstr, core._floatToGoString(s.value), timestamp)
+
     output = []
     for metric in registry.collect():
         mname = metric.name
@@ -86,25 +102,22 @@ def generate_latest(registry=core.REGISTRY):
         elif mtype == 'unknown':
             mtype = 'untyped'
 
-        output.append('# HELP {0} {1}'.format(
+        output.append('# HELP {0} {1}\n'.format(
             mname, metric.documentation.replace('\\', r'\\').replace('\n', r'\n')))
-        output.append('\n# TYPE {0} {1}\n'.format(mname, mtype))
+        output.append('# TYPE {0} {1}\n'.format(mname, mtype))
+
+        om_samples = {}
         for s in metric.samples:
-            if s.name == metric.name + '_created':
-                continue  # Ignore OpenMetrics specific sample. TODO: Make these into a gauge.
-            if s.labels:
-                labelstr = '{{{0}}}'.format(','.join(
-                    ['{0}="{1}"'.format(
-                     k, v.replace('\\', r'\\').replace('\n', r'\n').replace('"', r'\"'))
-                     for k, v in sorted(s.labels.items())]))
+            for suffix in ['_created', '_gsum', '_gcount']:
+                if s.name == metric.name + suffix:
+                    # OpenMetrics specific sample, put in a gauge at the end.
+                    om_samples.setdefault(suffix, []).append(sample_line(s))
+                    break
             else:
-                labelstr = ''
-            timestamp = ''
-            if s.timestamp is not None:
-                # Convert to milliseconds.
-                timestamp = ' {0:d}'.format(int(float(s.timestamp) * 1000))
-            output.append('{0}{1} {2}{3}\n'.format(
-                s.name, labelstr, core._floatToGoString(s.value), timestamp))
+                output.append(sample_line(s))
+        for suffix, lines in sorted(om_samples.items()):
+            output.append('# TYPE {0}{1} gauge\n'.format(metric.name, suffix))
+            output.extend(lines)
     return ''.join(output).encode('utf-8')
 
 

prometheus_client/openmetrics/exposition.py

@@ -26,7 +26,7 @@ def generate_latest(registry):
             else:
                 labelstr = ''
             if s.exemplar:
-                if metric.type != 'histogram' or not s.name.endswith('_bucket'):
+                if metric.type not in ('histogram', 'gaugehistogram') or not s.name.endswith('_bucket'):
                     raise ValueError("Metric {0} has exemplars, but is not a histogram bucket".format(metric.name))
                 labels = '{{{0}}}'.format(','.join(
                     ['{0}="{1}"'.format(
@@ -42,7 +42,6 @@ def generate_latest(registry):
                 exemplarstr = ''
             timestamp = ''
             if s.timestamp is not None:
-                # Convert to milliseconds.
                 timestamp = ' {0}'.format(s.timestamp)
             output.append('{0}{1} {2}{3}{4}\n'.format(s.name, labelstr,
                 core._floatToGoString(s.value), timestamp, exemplarstr))

prometheus_client/openmetrics/parser.py

@@ -302,7 +302,8 @@ def build_metric(name, documentation, typ, unit, samples):
                     'counter': ['_total', '_created'],
                     'summary': ['_count', '_sum', '', '_created'],
                     'histogram': ['_count', '_sum', '_bucket', 'created'],
-                    'gaugehistogram': ['_bucket'],
+                    'gaugehistogram': ['_gcount', '_gsum', '_bucket'],
+                    'info': ['_info'],
                 }.get(typ, [''])
                 allowed_names = [name + n for n in allowed_names]
             elif parts[1] == 'UNIT':

tests/openmetrics/test_exposition.py

@@ -134,11 +134,13 @@ def collect(self):
             generate_latest(self.registry)
 
     def test_gaugehistogram(self):
-        self.custom_collector(GaugeHistogramMetricFamily('gh', 'help', buckets=[('1.0', 4), ('+Inf', (5))]))
+        self.custom_collector(GaugeHistogramMetricFamily('gh', 'help', buckets=[('1.0', 4), ('+Inf', (5))], gsum_value=7))
         self.assertEqual(b'''# HELP gh help
 # TYPE gh gaugehistogram
 gh_bucket{le="1.0"} 4.0
 gh_bucket{le="+Inf"} 5.0
+gh_gcount 5.0
+gh_gsum 7.0
 # EOF
 ''', generate_latest(self.registry))
 

tests/openmetrics/test_parser.py

@@ -13,10 +13,13 @@
     CollectorRegistry,
     CounterMetricFamily,
     Exemplar,
+    GaugeHistogramMetricFamily,
     GaugeMetricFamily,
     HistogramMetricFamily,
+    InfoMetricFamily,
     Metric,
     Sample,
+    StateSetMetricFamily,
     SummaryMetricFamily,
     Timestamp,
 )
@@ -120,6 +123,48 @@ def test_histogram_exemplars(self):
         hfm.add_sample("a_bucket", {"le": "+Inf"}, 3.0, None, Exemplar({"a": "d"}, 4, Timestamp(123, 0)))
         self.assertEqual([hfm], list(families))
 
+    def test_simple_gaugehistogram(self):
+        families = text_string_to_metric_families("""# TYPE a gaugehistogram
+# HELP a help
+a_bucket{le="1"} 0
+a_bucket{le="+Inf"} 3
+a_gcount 3
+a_gsum 2
+# EOF
+""")
+        self.assertEqual([GaugeHistogramMetricFamily("a", "help", gsum_value=2, buckets=[("1", 0.0), ("+Inf", 3.0)])], list(families))
+
+    def test_histogram_exemplars(self):
+        families = text_string_to_metric_families("""# TYPE a gaugehistogram
+# HELP a help
+a_bucket{le="1"} 0 # {a="b"} 0.5
+a_bucket{le="2"} 2 123 # {a="c"} 0.5
+a_bucket{le="+Inf"} 3 # {a="d"} 4 123
+# EOF
+""")
+        hfm = GaugeHistogramMetricFamily("a", "help")
+        hfm.add_sample("a_bucket", {"le": "1"}, 0.0, None, Exemplar({"a": "b"}, 0.5))
+        hfm.add_sample("a_bucket", {"le": "2"}, 2.0, Timestamp(123, 0), Exemplar({"a": "c"}, 0.5)), 
+        hfm.add_sample("a_bucket", {"le": "+Inf"}, 3.0, None, Exemplar({"a": "d"}, 4, Timestamp(123, 0)))
+        self.assertEqual([hfm], list(families))
+
+    def test_simple_info(self):
+        families = text_string_to_metric_families("""# TYPE a info
+# HELP a help
+a_info{foo="bar"} 1
+# EOF
+""")
+        self.assertEqual([InfoMetricFamily("a", "help", {'foo': 'bar'})], list(families))
+
+    def test_simple_stateset(self):
+        families = text_string_to_metric_families("""# TYPE a stateset
+# HELP a help
+a{a="bar"} 0
+a{a="foo"} 1
+# EOF
+""")
+        self.assertEqual([StateSetMetricFamily("a", "help", {'foo': True, 'bar': False})], list(families))
+
     def test_no_metadata(self):
         families = text_string_to_metric_families("""a 1
 # EOF

tests/test_core.py

@@ -569,10 +569,12 @@ def test_gaugehistogram(self):
 
     def test_gaugehistogram_labels(self):
         cmf = GaugeHistogramMetricFamily('h', 'help', labels=['a'])
-        cmf.add_metric(['b'], buckets=[('0', 1), ('+Inf', 2)])
+        cmf.add_metric(['b'], buckets=[('0', 1), ('+Inf', 2)], gsum_value=3)
         self.custom_collector(cmf)
         self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'a': 'b', 'le': '0'}))
         self.assertEqual(2, self.registry.get_sample_value('h_bucket', {'a': 'b', 'le': '+Inf'}))
+        self.assertEqual(2, self.registry.get_sample_value('h_gcount', {'a': 'b'}))
+        self.assertEqual(3, self.registry.get_sample_value('h_gsum', {'a': 'b'}))
 
     def test_info(self):
         self.custom_collector(InfoMetricFamily('i', 'help', value={'a': 'b'}))

tests/test_exposition.py

@@ -2,6 +2,7 @@
 
 import sys
 import threading
+import time
 
 if sys.version_info < (2, 7):
     # We need the skip decorators from unittest2 on Python 2.6.
@@ -29,6 +30,13 @@ class TestGenerateText(unittest.TestCase):
     def setUp(self):
         self.registry = CollectorRegistry()
 
+        # Mock time so _created values are fixed.
+        self.old_time = time.time
+        time.time = lambda: 123.456
+
+    def tearDown(self):
+        time.time = self.old_time
+
     def custom_collector(self, metric_family):
         class CustomCollector(object):
             def collect(self):
@@ -38,12 +46,23 @@ def collect(self):
     def test_counter(self):
         c = Counter('cc', 'A counter', registry=self.registry)
         c.inc()
-        self.assertEqual(b'# HELP cc_total A counter\n# TYPE cc_total counter\ncc_total 1.0\n', generate_latest(self.registry))
+        self.assertEqual(b'''# HELP cc_total A counter
+# TYPE cc_total counter
+cc_total 1.0
+# TYPE cc_created gauge
+cc_created 123.456
+''', generate_latest(self.registry))
 
     def test_counter_total(self):
         c = Counter('cc_total', 'A counter', registry=self.registry)
         c.inc()
-        self.assertEqual(b'# HELP cc_total A counter\n# TYPE cc_total counter\ncc_total 1.0\n', generate_latest(self.registry))
+        self.assertEqual(b'''# HELP cc_total A counter
+# TYPE cc_total counter
+cc_total 1.0
+# TYPE cc_created gauge
+cc_created 123.456
+''', generate_latest(self.registry))
+
     def test_gauge(self):
         g = Gauge('gg', 'A gauge', registry=self.registry)
         g.set(17)
@@ -52,7 +71,13 @@ def test_gauge(self):
     def test_summary(self):
         s = Summary('ss', 'A summary', ['a', 'b'], registry=self.registry)
         s.labels('c', 'd').observe(17)
-        self.assertEqual(b'# HELP ss A summary\n# TYPE ss summary\nss_count{a="c",b="d"} 1.0\nss_sum{a="c",b="d"} 17.0\n', generate_latest(self.registry))
+        self.assertEqual(b'''# HELP ss A summary
+# TYPE ss summary
+ss_count{a="c",b="d"} 1.0
+ss_sum{a="c",b="d"} 17.0
+# TYPE ss_created gauge
+ss_created{a="c",b="d"} 123.456
+''', generate_latest(self.registry))
 
     @unittest.skipIf(sys.version_info < (2, 7), "Test requires Python 2.7+.")
     def test_histogram(self):
@@ -77,11 +102,21 @@ def test_histogram(self):
 hh_bucket{le="+Inf"} 1.0
 hh_count 1.0
 hh_sum 0.05
+# TYPE hh_created gauge
+hh_created 123.456
 ''', generate_latest(self.registry))
 
     def test_gaugehistogram(self):
-        self.custom_collector(GaugeHistogramMetricFamily('gh', 'help', buckets=[('1.0', 4), ('+Inf', (5))]))
-        self.assertEqual(b'''# HELP gh help\n# TYPE gh histogram\ngh_bucket{le="1.0"} 4.0\ngh_bucket{le="+Inf"} 5.0\n''', generate_latest(self.registry))
+        self.custom_collector(GaugeHistogramMetricFamily('gh', 'help', buckets=[('1.0', 4), ('+Inf', 5)], gsum_value=7))
+        self.assertEqual(b'''# HELP gh help
+# TYPE gh histogram
+gh_bucket{le="1.0"} 4.0
+gh_bucket{le="+Inf"} 5.0
+# TYPE gh_gcount gauge
+gh_gcount 5.0
+# TYPE gh_gsum gauge
+gh_gsum 7.0
+''', generate_latest(self.registry))
 
     def test_info(self):
         i = Info('ii', 'A info', ['a', 'b'], registry=self.registry)
@@ -94,14 +129,14 @@ def test_enum(self):
         self.assertEqual(b'# HELP ee An enum\n# TYPE ee gauge\nee{a="c",b="d",ee="foo"} 0.0\nee{a="c",b="d",ee="bar"} 1.0\n', generate_latest(self.registry))
 
     def test_unicode(self):
-        c = Counter('cc', '\u4500', ['l'], registry=self.registry)
+        c = Gauge('cc', '\u4500', ['l'], registry=self.registry)
         c.labels('\u4500').inc()
-        self.assertEqual(b'# HELP cc_total \xe4\x94\x80\n# TYPE cc_total counter\ncc_total{l="\xe4\x94\x80"} 1.0\n', generate_latest(self.registry))
+        self.assertEqual(b'# HELP cc \xe4\x94\x80\n# TYPE cc gauge\ncc{l="\xe4\x94\x80"} 1.0\n', generate_latest(self.registry))
 
     def test_escaping(self):
-        c = Counter('cc', 'A\ncount\\er', ['a'], registry=self.registry)
-        c.labels('\\x\n"').inc(1)
-        self.assertEqual(b'# HELP cc_total A\\ncount\\\\er\n# TYPE cc_total counter\ncc_total{a="\\\\x\\n\\""} 1.0\n', generate_latest(self.registry))
+        g = Gauge('cc', 'A\ngaug\\e', ['a'], registry=self.registry)
+        g.labels('\\x\n"').inc(1)
+        self.assertEqual(b'# HELP cc A\\ngaug\\\\e\n# TYPE cc gauge\ncc{a="\\\\x\\n\\""} 1.0\n', generate_latest(self.registry))
 
     def test_nonnumber(self):