Prometheus metrics & mixin

Makefile

@@ -135,6 +135,8 @@ controller.yaml: controller.jsonnet controller-norbac.jsonnet
 
 controller-norbac.yaml: controller-norbac.jsonnet
 
+controller-podmonitor.yaml: controller.jsonnet controller.image controller-norbac.jsonnet
+
 test:
 	$(GO) test $(GO_FLAGS) $(GO_PACKAGES)
 

cmd/controller/controller.go

@@ -198,9 +198,11 @@ func (c *Controller) processNextItem() bool {
 }
 
 func (c *Controller) unseal(key string) error {
+	unsealRequestsTotal.Inc()
 	obj, exists, err := c.informer.GetIndexer().GetByKey(key)
 	if err != nil {
 		log.Printf("Error fetching object with key %s from store: %v", key, err)
+		unsealErrorsTotal.WithLabelValues("fetch").Inc()
 		return err
 	}
 
@@ -229,6 +231,7 @@ func (c *Controller) unseal(key string) error {
 	newSecret, err := c.attemptUnseal(ssecret)
 	if err != nil {
 		c.recorder.Eventf(ssecret, corev1.EventTypeWarning, ErrUnsealFailed, "Failed to unseal: %v", err)
+		unsealErrorsTotal.WithLabelValues("unseal").Inc()
 		return err
 	}
 
@@ -238,12 +241,14 @@ func (c *Controller) unseal(key string) error {
 	}
 	if err != nil {
 		c.recorder.Event(ssecret, corev1.EventTypeWarning, ErrUpdateFailed, err.Error())
+		unsealErrorsTotal.WithLabelValues("update").Inc()
 		return err
 	}
 
 	if !metav1.IsControlledBy(secret, ssecret) && !isAnnotatedToBeManaged(secret) {
 		msg := fmt.Sprintf("Resource %q already exists and is not managed by SealedSecret", secret.Name)
 		c.recorder.Event(ssecret, corev1.EventTypeWarning, ErrUpdateFailed, msg)
+		unsealErrorsTotal.WithLabelValues("unmanaged").Inc()
 		return fmt.Errorf("failed update: %s", msg)
 	}
 
@@ -260,6 +265,7 @@ func (c *Controller) unseal(key string) error {
 		secret, err = c.sclient.Secrets(ssecret.GetObjectMeta().GetNamespace()).Update(secret)
 		if err != nil {
 			c.recorder.Event(ssecret, corev1.EventTypeWarning, ErrUpdateFailed, err.Error())
+			unsealErrorsTotal.WithLabelValues("update").Inc()
 			return err
 		}
 	}
@@ -268,6 +274,7 @@ func (c *Controller) unseal(key string) error {
 	if err != nil {
 		// Non-fatal.  Log and continue.
 		log.Printf("Error updating SealedSecret %s status: %v", key, err)
+		unsealErrorsTotal.WithLabelValues("status").Inc()
 	}
 
 	c.recorder.Event(ssecret, corev1.EventTypeNormal, SuccessUnsealed, "SealedSecret unsealed successfully")

cmd/controller/metrics.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Define Prometheus Exporter namespace (prefix) for all metric names
+const metricNamespace string = "sealed_secrets_controller"
+
+// Define Prometheus metrics to expose
+var (
+	buildInfo = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace:   metricNamespace,
+			Name:        "build_info",
+			Help:        "Build information.",
+			ConstLabels: prometheus.Labels{"revision": VERSION},
+		},
+	)
+	// TODO: rename metric, change increment logic, or accept behaviour
+	// when a SealedSecret is deleted the unseal() function is called which is
+	// not technically an 'unseal request'.
+	unsealRequestsTotal = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricNamespace,
+			Name:      "unseal_requests_total",
+			Help:      "Total number of sealed secret unseal requests",
+		},
+	)
+	unsealErrorsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricNamespace,
+			Name:      "unseal_errors_total",
+			Help:      "Total number of sealed secret unseal errors by reason",
+		},
+		[]string{"reason"},
+	)
+)
+
+func init() {
+	// Register metrics with Prometheus
+	prometheus.MustRegister(buildInfo)
+	prometheus.MustRegister(prometheus.NewBuildInfoCollector())
+	prometheus.MustRegister(unsealRequestsTotal)
+	prometheus.MustRegister(unsealErrorsTotal)
+
+	// Initialise known label values
+	for _, val := range []string{"fetch", "status", "unmanaged", "unseal", "update"} {
+		unsealErrorsTotal.WithLabelValues(val)
+	}
+
+}

cmd/controller/server.go

@@ -9,6 +9,8 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
 	flag "github.com/spf13/pflag"
 	"github.com/throttled/throttled"
 	"github.com/throttled/throttled/store/memstore"
@@ -40,6 +42,8 @@ func httpserver(cp certProvider, sc secretChecker, sr secretRotator) *http.Serve
 		io.WriteString(w, "ok\n")
 	})
 
+	mux.Handle("/metrics", promhttp.Handler())
+
 	mux.Handle("/v1/verify", httpRateLimiter.RateLimit(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		content, err := ioutil.ReadAll(r.Body)
 

controller-podmonitor.jsonnet

@@ -0,0 +1,37 @@
+// Prometheus Service Monitor manifest
+
+local controller = import 'controller.jsonnet';
+
+controller {
+  serviceMonitor: {
+    apiVersion: 'monitoring.coreos.com/v1',
+    kind: 'PodMonitor',
+    metadata: {
+      name: 'sealed-secrets-controller',
+      namespace: $.namespace,
+      labels: {
+        name: 'sealed-secrets-controller',
+      },
+    },
+    spec: {
+      jobLabel: 'name',
+      selector: {
+        matchLabels: {
+          name: 'sealed-secrets-controller',
+        },
+      },
+      namespaceSelector: {
+        matchNames: [
+          $.namespace,
+        ],
+      },
+      podMetricsEndpoints: [
+        {
+          port: 'http',
+          interval: '30s',
+        },
+      ],
+      sampleLimit: 1000,
+    },
+  },
+}

go.mod

@@ -11,12 +11,13 @@ require (
 	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible // indirect
 	github.com/google/go-cmp v0.3.0
-	github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf
+	github.com/google/gofuzz v1.0.0
 	github.com/googleapis/gnostic v0.0.0-20171211024024-933c109c13ce // indirect
 	github.com/mattn/go-isatty v0.0.10
 	github.com/mkmik/multierror v0.3.0
 	github.com/onsi/ginkgo v1.6.0
 	github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3
+	github.com/prometheus/client_golang v1.2.1
 	github.com/spf13/pflag v1.0.3
 	github.com/throttled/throttled v2.2.2+incompatible
 	golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8