Prometheus metrics & mixin

Makefile

@@ -135,6 +135,8 @@ controller.yaml: controller.jsonnet controller-norbac.jsonnet
 
 controller-norbac.yaml: controller-norbac.jsonnet
 
+controller-podmonitor.yaml: controller.jsonnet controller-norbac.jsonnet
+
 test:
 	$(GO) test $(GO_FLAGS) $(GO_PACKAGES)
 

cmd/controller/controller.go

@@ -198,9 +198,11 @@ func (c *Controller) processNextItem() bool {
 }
 
 func (c *Controller) unseal(key string) error {
+	unsealRequestsTotal.Inc()
 	obj, exists, err := c.informer.GetIndexer().GetByKey(key)
 	if err != nil {
 		log.Printf("Error fetching object with key %s from store: %v", key, err)
+		unsealErrorsTotal.WithLabelValues("fetch").Inc()
 		return err
 	}
 
@@ -229,6 +231,7 @@ func (c *Controller) unseal(key string) error {
 	newSecret, err := c.attemptUnseal(ssecret)
 	if err != nil {
 		c.recorder.Eventf(ssecret, corev1.EventTypeWarning, ErrUnsealFailed, "Failed to unseal: %v", err)
+		unsealErrorsTotal.WithLabelValues("unseal").Inc()
 		return err
 	}
 
@@ -238,12 +241,14 @@ func (c *Controller) unseal(key string) error {
 	}
 	if err != nil {
 		c.recorder.Event(ssecret, corev1.EventTypeWarning, ErrUpdateFailed, err.Error())
+		unsealErrorsTotal.WithLabelValues("update").Inc()
 		return err
 	}
 
 	if !metav1.IsControlledBy(secret, ssecret) && !isAnnotatedToBeManaged(secret) {
 		msg := fmt.Sprintf("Resource %q already exists and is not managed by SealedSecret", secret.Name)
 		c.recorder.Event(ssecret, corev1.EventTypeWarning, ErrUpdateFailed, msg)
+		unsealErrorsTotal.WithLabelValues("unmanaged").Inc()
 		return fmt.Errorf("failed update: %s", msg)
 	}
 
@@ -260,6 +265,7 @@ func (c *Controller) unseal(key string) error {
 		secret, err = c.sclient.Secrets(ssecret.GetObjectMeta().GetNamespace()).Update(secret)
 		if err != nil {
 			c.recorder.Event(ssecret, corev1.EventTypeWarning, ErrUpdateFailed, err.Error())
+			unsealErrorsTotal.WithLabelValues("update").Inc()
 			return err
 		}
 	}
@@ -268,6 +274,7 @@ func (c *Controller) unseal(key string) error {
 	if err != nil {
 		// Non-fatal.  Log and continue.
 		log.Printf("Error updating SealedSecret %s status: %v", key, err)
+		unsealErrorsTotal.WithLabelValues("status").Inc()
 	}
 
 	c.recorder.Event(ssecret, corev1.EventTypeNormal, SuccessUnsealed, "SealedSecret unsealed successfully")

cmd/controller/metrics.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Define Prometheus Exporter namespace (prefix) for all metric names
+const metricNamespace string = "sealed_secrets_controller"
+
+// Define Prometheus metrics to expose
+var (
+	buildInfo = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace:   metricNamespace,
+			Name:        "build_info",
+			Help:        "Build information.",
+			ConstLabels: prometheus.Labels{"revision": VERSION},
+		},
+	)
+	// TODO: rename metric, change increment logic, or accept behaviour
+	// when a SealedSecret is deleted the unseal() function is called which is
+	// not technically an 'unseal request'.
+	unsealRequestsTotal = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricNamespace,
+			Name:      "unseal_requests_total",
+			Help:      "Total number of sealed secret unseal requests",
+		},
+	)
+	unsealErrorsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricNamespace,
+			Name:      "unseal_errors_total",
+			Help:      "Total number of sealed secret unseal errors by reason",
+		},
+		[]string{"reason"},
+	)
+)
+
+func init() {
+	// Register metrics with Prometheus
+	prometheus.MustRegister(buildInfo)
+	prometheus.MustRegister(prometheus.NewBuildInfoCollector())
+	prometheus.MustRegister(unsealRequestsTotal)
+	prometheus.MustRegister(unsealErrorsTotal)
+
+	// Initialise known label values
+	for _, val := range []string{"fetch", "status", "unmanaged", "unseal", "update"} {
+		unsealErrorsTotal.WithLabelValues(val)
+	}
+
+}

cmd/controller/server.go

@@ -9,6 +9,8 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
 	flag "github.com/spf13/pflag"
 	"github.com/throttled/throttled"
 	"github.com/throttled/throttled/store/memstore"
@@ -40,6 +42,8 @@ func httpserver(cp certProvider, sc secretChecker, sr secretRotator) *http.Serve
 		io.WriteString(w, "ok\n")
 	})
 
+	mux.Handle("/metrics", promhttp.Handler())
+
 	mux.Handle("/v1/verify", httpRateLimiter.RateLimit(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		content, err := ioutil.ReadAll(r.Body)
 

contrib/prometheus-mixin/.gitignore

@@ -0,0 +1 @@
+manifests/

contrib/prometheus-mixin/Makefile

@@ -0,0 +1,56 @@
+# Prometheus Mixin Makefile
+# Heavily copied from upstream project kubenetes-mixin
+
+PROMETHEUS_IMAGE := prom/prometheus:v2.16.0
+
+JSONNET_FMT := jsonnetfmt
+
+all: fmt prometheus_alerts.yaml prometheus_rules.yaml dashboards_out lint test ## Generate files, lint and test
+
+fmt: ## Format Jsonnet
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		xargs -n 1 -- $(JSONNET_FMT) -i
+
+prometheus_alerts.yaml: mixin.libsonnet lib/alerts.jsonnet alerts/*.libsonnet ## Generate Alerts YAML
+	@mkdir -p manifests
+	jsonnet -S lib/alerts.jsonnet > manifests/$@
+
+prometheus_rules.yaml: mixin.libsonnet lib/rules.jsonnet rules/*.libsonnet ## Generate Rules YAML
+	@mkdir -p manifests
+	jsonnet -S lib/rules.jsonnet > manifests/$@
+
+dashboards_out: mixin.libsonnet lib/dashboards.jsonnet dashboards/*.libsonnet ## Generate Dashboards JSON
+	jsonnet -J vendor -m manifests lib/dashboards.jsonnet
+
+lint: prometheus_alerts.yaml prometheus_rules.yaml ## Lint and check YAML
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		while read f; do \
+			$(JSONNET_FMT) "$$f" | diff -u "$$f" -; \
+		done
+	docker run \
+		-v $(PWD)/manifests:/tmp \
+		--entrypoint '/bin/promtool' \
+		$(PROMETHEUS_IMAGE) \
+		check rules /tmp/prometheus_rules.yaml; \
+	docker run \
+		-v $(PWD)/manifests:/tmp \
+		--entrypoint '/bin/promtool' \
+		$(PROMETHEUS_IMAGE) \
+		check rules /tmp/prometheus_alerts.yaml
+
+clean: ## Clean up generated files
+	rm -rf manifests/
+
+# TODO: Find out why official prom images segfaults during `test rules` if not root
+test: prometheus_alerts.yaml prometheus_rules.yaml ## Test generated files
+	docker run \
+		-v $(PWD):/tmp \
+		--user root \
+		--entrypoint '/bin/promtool' \
+		$(PROMETHEUS_IMAGE) \
+		test rules /tmp/tests.yaml
+
+.PHONY: help
+help:
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+

contrib/prometheus-mixin/README.md

@@ -0,0 +1,29 @@
+# Prometheus Mixin
+
+A Prometheus mixin includes dashboards, recording rules and alerts provided to monitor
+the application. For more details see 
+[monitoring-mixins](https://github.com/monitoring-mixins/docs).
+
+## Grafana dashboard
+
+The [dashboard](./dashboards/sealed-secrets-controller.json) can be imported
+standalone into Grafana. You may need to edit the datasource if you have
+configured your Prometheus datasource with a different name.
+
+## Using the mixin as jsonnet
+
+See the [kube-prometheus](https://github.com/coreos/kube-prometheus#kube-prometheus)
+project documentation for instructions on importing mixins.
+
+## Generating YAML files and validating changes
+
+Install the `jsonnet` dependencies:
+```
+$ go get github.com/google/go-jsonnet/cmd/jsonnet
+$ go get github.com/google/go-jsonnet/cmd/jsonnetfmt
+```
+
+Generate yaml and run tests:
+```
+$ make
+```

contrib/prometheus-mixin/alerts/alerts.libsonnet

@@ -0,0 +1,3 @@
+// Sealed Secrets Alertmanager Alerts
+
+(import 'sealed-secrets-alerts.libsonnet')

contrib/prometheus-mixin/alerts/sealed-secrets-alerts.libsonnet

@@ -0,0 +1,33 @@
+{
+  prometheusAlerts+:: {
+    groups+: [{
+      name: 'sealed-secrets',
+      rules: [
+        // SealedSecretsErrorRateHigh:
+        // Method: Alert on occurence of errors by looking for a non-zero rate of errors over past 5 minutes
+        // Pros:
+        // - An app deploy is likely broken if a secret can't be updated by Controller.
+        // Caveats:
+        // - Probably better to leave app deploy breakages to the app or CD systems monitoring.
+        // - Potentially noisy. Controller attempts to unseal 5 times, so if it exceeds on the 4th attempt then all is fine but this alert will trigger.
+        // - Usage of an invalid cert.pem with kubeseal will trigger this alert, it would be better to distinguish alerts due to controller or user
+        // - 'for' clause not used because we are unlikely to have a sustained rate of errors unless there is a LOT of secret churn in cluster.
+        // Rob Ewaschuk - My Philosophy on Alerting: https://docs.google.com/document/d/199PqyG3UsyXlwieHaqbGiWVa8eMWi8zzAn0YfcApr8Q/edit
+        {
+          alert: 'SealedSecretsUnsealErrorRateHigh',
+          expr: |||
+            sum(rate(sealed_secrets_controller_unseal_errors_total{}[5m])) > 0
+          ||| % $._config,
+          // 'for': '5m', // Not used, see caveats above.
+          labels: {
+            severity: 'warning',
+          },
+          annotations: {
+            message: 'High rate of errors unsealing Sealed Secrets',
+            runbook: 'https://github.com/bitnami-labs/sealed-secrets',
+          },
+        },
+      ],
+    }],
+  },
+}

contrib/prometheus-mixin/config.libsonnet

@@ -0,0 +1,4 @@
+// Sealed Secrets Prometheus Mixin Config
+{
+  _config+:: {},
+}

contrib/prometheus-mixin/dashboards/dashboards.libsonnet

@@ -0,0 +1,7 @@
+// Sealed Secrets Grafana Dashboards
+
+{
+  grafanaDashboards+:: {
+    'sealed-secrets-controller.json': (import 'sealed-secrets-controller.json'),
+  },
+}