Add prometheus metrics

Gopkg.toml

@@ -24,7 +24,7 @@
 
 [[constraint]]
   name = "github.com/prometheus/client_golang"
-  version = "0.8.0"
+  revision = "3653aff4d509dd87a6ba41b82d57c0c662733e14"
 
 [[override]]
   name = "github.com/golang/protobuf"

chaoskube/chaoskube.go

@@ -7,15 +7,14 @@ import (
 	"math/rand"
 	"time"
 
+	"github.com/linki/chaoskube/metrics"
+	"github.com/linki/chaoskube/util"
 	log "github.com/sirupsen/logrus"
-
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/client-go/kubernetes"
-
-	"github.com/linki/chaoskube/util"
 )
 
 // Chaoskube represents an instance of chaoskube
@@ -87,8 +86,13 @@ func New(client kubernetes.Interface, labels, annotations, namespaces labels.Sel
 // described by channel next. It returns when the given context is canceled.
 func (c *Chaoskube) Run(ctx context.Context, next <-chan time.Time) {
 	for {
+		metrics.RunCounter.Inc()
+
 		if err := c.TerminateVictim(); err != nil {
+			metrics.ErrorCounter.Inc()
 			c.Logger.WithField("err", err).Error("failed to terminate victim")
+		} else {
+			metrics.PodsDeletedCounter.Inc()
 		}
 
 		c.Logger.Debug("sleeping...")
@@ -191,7 +195,11 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error {
 		return nil
 	}
 
-	return c.Client.CoreV1().Pods(victim.Namespace).Delete(victim.Name, nil)
+	start := time.Now()
+	e := c.Client.CoreV1().Pods(victim.Namespace).Delete(victim.Name, nil)
+	metrics.TerminationHistogram.Observe(time.Since(start).Seconds())
+
+	return e
 }
 
 // filterByNamespaces filters a list of pods by a given namespace selector.

metrics/metrics.go

@@ -0,0 +1,29 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	// PodsDeletedCounter is the pods deleted counter
+	PodsDeletedCounter = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "pods_deleted",
+		Help: "The total number of pods deleted",
+	})
+	// RunCounter is the run function executions counter
+	RunCounter = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "run_counts",
+		Help: "The total number of pod termination logic runs",
+	})
+	// ErrorCounter is the run function executions counter
+	ErrorCounter = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "termination_errors",
+		Help: "The total number of errors on terminate victim operation",
+	})
+	// errorCounter is the run function executions counter
+	TerminationHistogram = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name: "termination_time_seconds",
+		Help: "The time took single pod termination to finish",
+	})
+)