Change compat metrics to counters (#3686)

This commit changes the metrics in the compat package from gauges
to counters. The reason for this is that in some cases the gauge
should behave like a gauge (i.e. loading configurations) but in
other cases should behave like a counter (i.e. HTTP requests).

Second, because the compat package is a global package
(due to how config.Load works), in tenanted systems like Cortex
and Mimir it was non-trivial to reset the gauges per tenant
each time their configuration was reloaded.

Instead, it's easier to compute the rate of increase as 0 instead
of check that the gauge is 0 to know if UTF-8 strict mode can be
enabled.

Signed-off-by: George Robinson <george.robinson@grafana.com>

matchers/compat/metrics.go

@@ -31,28 +31,28 @@ var DefaultOrigins = []string{
 var RegisteredMetrics = NewMetrics(prometheus.DefaultRegisterer)
 
 type Metrics struct {
-	Total             *prometheus.GaugeVec
-	DisagreeTotal     *prometheus.GaugeVec
-	IncompatibleTotal *prometheus.GaugeVec
-	InvalidTotal      *prometheus.GaugeVec
+	Total             *prometheus.CounterVec
+	DisagreeTotal     *prometheus.CounterVec
+	IncompatibleTotal *prometheus.CounterVec
+	InvalidTotal      *prometheus.CounterVec
 }
 
 func NewMetrics(r prometheus.Registerer) *Metrics {
 	m := &Metrics{
-		Total: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
-			Name: "alertmanager_matchers_parse",
+		Total: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
+			Name: "alertmanager_matchers_parse_total",
 			Help: "Total number of matcher inputs parsed, including invalid inputs.",
 		}, []string{"origin"}),
-		DisagreeTotal: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
-			Name: "alertmanager_matchers_disagree",
+		DisagreeTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
+			Name: "alertmanager_matchers_disagree_total",
 			Help: "Total number of matcher inputs which produce different parsings (disagreement).",
 		}, []string{"origin"}),
-		IncompatibleTotal: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
-			Name: "alertmanager_matchers_incompatible",
+		IncompatibleTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
+			Name: "alertmanager_matchers_incompatible_total",
 			Help: "Total number of matcher inputs that are incompatible with the UTF-8 parser.",
 		}, []string{"origin"}),
-		InvalidTotal: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
-			Name: "alertmanager_matchers_invalid",
+		InvalidTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
+			Name: "alertmanager_matchers_invalid_total",
 			Help: "Total number of matcher inputs that could not be parsed.",
 		}, []string{"origin"}),
 	}

matchers/compat/parse_test.go

@@ -236,7 +236,7 @@ func TestIsValidUTF8LabelName(t *testing.T) {
 	}
 }
 
-func requireMetric(t *testing.T, expected float64, m *prometheus.GaugeVec) {
+func requireMetric(t *testing.T, expected float64, m *prometheus.CounterVec) {
 	if expected == 0 {
 		require.Equal(t, 0, testutil.CollectAndCount(m))
 	} else {