From 8d8d80acf41c90816f6e64cc28d1fa2141b73e9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20GLON?= <sebastien.glon@ubble.ai>
Date: Tue, 26 Nov 2024 12:23:48 +0100
Subject: [PATCH] feat(job): add duration histogram

---
 pkg/controller/collectors.go | 13 +++++++++++++
 pkg/controller/jobs.go       |  6 ++++++
 pkg/controller/jobs_test.go  |  8 ++++++++
 pkg/controller/metrics.go    |  1 +
 pkg/schemas/metric.go        |  3 +++
 5 files changed, 31 insertions(+)

diff --git a/pkg/controller/collectors.go b/pkg/controller/collectors.go
index a9265cec..1a80aa27 100644
--- a/pkg/controller/collectors.go
+++ b/pkg/controller/collectors.go
@@ -11,6 +11,7 @@ var (
 	testSuiteLabels              = []string{"test_suite_name"}
 	testCaseLabels               = []string{"test_case_name", "test_case_classname"}
 	statusesList                 = [...]string{"created", "waiting_for_resource", "preparing", "pending", "running", "success", "failed", "canceled", "skipped", "manual", "scheduled", "error"}
+	latencyHistogramBuckets      = []float64{.1, .25, .5, 1, 2.5, 5, 10, 15, 20, 30, 40, 50, 60, 90, 150, 210, 270, 330, 390, 450, 500, 600, 1200, 1800, 2700, 3600}
 )
 
 // NewInternalCollectorCurrentlyQueuedTasksCount returns a new collector for the gcpe_currently_queued_tasks_count metric.
@@ -266,6 +267,18 @@ func NewCollectorJobDurationSeconds() prometheus.Collector {
 	)
 }
 
+// NewCollectorJobDurationHistogram returns a new collector for the gitlab_ci_pipeline_job_duration_seconds histogram metrics.
+func NewCollectorJobDurationHistogram() prometheus.Collector {
+	return prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "duration_seconds",
+			Help:    "Histogram of duration (seconds) of finished gitlab jobs",
+			Buckets: latencyHistogramBuckets,
+		},
+		append(defaultLabels, jobLabels...),
+	)
+}
+
 // NewCollectorJobID returns a new collector for the gitlab_ci_pipeline_job_id metric.
 func NewCollectorJobID() prometheus.Collector {
 	return prometheus.NewGaugeVec(
diff --git a/pkg/controller/jobs.go b/pkg/controller/jobs.go
index 4722f9c9..9eb63730 100644
--- a/pkg/controller/jobs.go
+++ b/pkg/controller/jobs.go
@@ -128,6 +128,12 @@ func (c *Controller) ProcessJobMetrics(ctx context.Context, ref schemas.Ref, job
 		Value:  job.DurationSeconds,
 	})
 
+	storeSetMetric(ctx, c.Store, schemas.Metric{
+		Kind:   schemas.MetricKindJobDurationHistogram,
+		Labels: labels,
+		Value:  job.DurationSeconds,
+	})
+
 	storeSetMetric(ctx, c.Store, schemas.Metric{
 		Kind:   schemas.MetricKindJobQueuedDurationSeconds,
 		Labels: labels,
diff --git a/pkg/controller/jobs_test.go b/pkg/controller/jobs_test.go
index 545a8eca..1b777a00 100644
--- a/pkg/controller/jobs_test.go
+++ b/pkg/controller/jobs_test.go
@@ -149,6 +149,14 @@ func TestProcessJobMetrics(t *testing.T) {
 	}
 	assert.Equal(t, lastRunJobDuration, metrics[lastRunJobDuration.Key()])
 
+	jobDurationHistogram := schemas.Metric{
+		Kind:   schemas.MetricKindJobDurationHistogram,
+		Labels: labels,
+		Value:  newJob.DurationSeconds,
+	}
+
+	assert.Equal(t, jobDurationHistogram, metrics[jobDurationHistogram.Key()])
+
 	jobRunCount := schemas.Metric{
 		Kind:   schemas.MetricKindJobRunCount,
 		Labels: labels,
diff --git a/pkg/controller/metrics.go b/pkg/controller/metrics.go
index a03aea8c..406a86d8 100644
--- a/pkg/controller/metrics.go
+++ b/pkg/controller/metrics.go
@@ -53,6 +53,7 @@ func NewRegistry(ctx context.Context) *Registry {
 			schemas.MetricKindID:                                   NewCollectorID(),
 			schemas.MetricKindJobArtifactSizeBytes:                 NewCollectorJobArtifactSizeBytes(),
 			schemas.MetricKindJobDurationSeconds:                   NewCollectorJobDurationSeconds(),
+			schemas.MetricKindJobDurationHistogram:                 NewCollectorJobDurationHistogram(),
 			schemas.MetricKindJobID:                                NewCollectorJobID(),
 			schemas.MetricKindJobQueuedDurationSeconds:             NewCollectorJobQueuedDurationSeconds(),
 			schemas.MetricKindJobRunCount:                          NewCollectorJobRunCount(),
diff --git a/pkg/schemas/metric.go b/pkg/schemas/metric.go
index bbf19752..16326f1c 100644
--- a/pkg/schemas/metric.go
+++ b/pkg/schemas/metric.go
@@ -48,6 +48,9 @@ const (
 	// MetricKindJobDurationSeconds ..
 	MetricKindJobDurationSeconds
 
+	// MetricKindJobDurationHistogram ..
+	MetricKindJobDurationHistogram
+
 	// MetricKindJobID ..
 	MetricKindJobID