Add new metric to show number of running pipelineruns

added new gauge metric to show number of running pipelineruns.
added tests and docs accordingly.

https://issues.redhat.com/browse/SRVKP-6375

Signed-off-by: Zaki Shaikh <zashaikh@redhat.com>

docs/content/docs/install/metrics.md

@@ -14,3 +14,4 @@ You can configure these exporters by referring to the [observability configurati
 |------------------------------------------------------|---------|--------------------------------------------------------------------|
 | `pipelines_as_code_pipelinerun_count`                | Counter | Number of pipelineruns created by pipelines-as-code                |
 | `pipelines_as_code_pipelinerun_duration_seconds_sum` | Counter | Number of seconds all pipelineruns have taken in pipelines-as-code |
+| `pipelines_as_code_running_pipelineruns_count`       | Gauge   | Number of running pipelineruns in pipelines-as-code                |

pkg/metrics/metrics.go

@@ -19,6 +19,10 @@ var prDurationCount = stats.Float64("pipelines_as_code_pipelinerun_duration_seco
 	"number of seconds all pipelineruns completed in by pipelines as code",
 	stats.UnitDimensionless)
 
+var runningPRCount = stats.Float64("pipelines_as_code_running_pipelineruns_count",
+	"number of running pipeline runs by pipelines as code",
+	stats.UnitDimensionless)
+
 // Recorder holds keys for metrics.
 type Recorder struct {
 	initialized     bool
@@ -90,6 +94,12 @@ func NewRecorder() (*Recorder, error) {
 			Aggregation: view.Sum(),
 			TagKeys:     []tag.Key{r.namespace, r.repository, r.status, r.reason},
 		},
+		&view.View{
+			Description: runningPRCount.Description(),
+			Measure:     runningPRCount,
+			Aggregation: view.LastValue(),
+			TagKeys:     []tag.Key{r.provider, r.eventType, r.namespace, r.repository},
+		},
 	)
 	if err != nil {
 		r.initialized = false
@@ -103,7 +113,7 @@ func NewRecorder() (*Recorder, error) {
 func (r *Recorder) Count(provider, event, namespace, repository string) error {
 	if !r.initialized {
 		return fmt.Errorf(
-			"ignoring the metrics recording for pipeline runs,  failed to initialize the metrics recorder")
+			"ignoring the metrics recording for pipelineruns, failed to initialize the metrics recorder")
 	}
 
 	ctx, err := tag.New(
@@ -142,3 +152,25 @@ func (r *Recorder) CountPRDuration(namespace, repository, status, reason string,
 	metrics.Record(ctx, prDurationCount.M(duration.Seconds()))
 	return nil
 }
+
+// CountRunningPRs emits the number of running pipelineruns for a repository and namespace.
+func (r *Recorder) CountRunningPRs(provider, event, namespace, repository string, runningPRs float64) error {
+	if !r.initialized {
+		return fmt.Errorf(
+			"ignoring the metrics recording for pipelineruns, failed to initialize the metrics recorder")
+	}
+
+	ctx, err := tag.New(
+		context.Background(),
+		tag.Insert(r.provider, provider),
+		tag.Insert(r.eventType, event),
+		tag.Insert(r.namespace, namespace),
+		tag.Insert(r.repository, repository),
+	)
+	if err != nil {
+		return err
+	}
+
+	metrics.Record(ctx, runningPRCount.M(runningPRs))
+	return nil
+}

pkg/reconciler/emit_metrics_test.go

@@ -86,7 +86,7 @@ func TestCountPipelineRun(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			metricstest.Unregister("pipelines_as_code_pipelinerun_count")
+			unregisterMetrics()
 			m, err := metrics.NewRecorder()
 			assert.NilError(t, err)
 			r := &Reconciler{
@@ -224,7 +224,7 @@ func TestCalculatePipelineRunDuration(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			metricstest.Unregister("pipelines_as_code_pipelinerun_duration_seconds_sum")
+			unregisterMetrics()
 			m, err := metrics.NewRecorder()
 			assert.NilError(t, err)
 			r := &Reconciler{
@@ -262,3 +262,55 @@ func TestCalculatePipelineRunDuration(t *testing.T) {
 		})
 	}
 }
+
+func TestCountRunningPRs(t *testing.T) {
+	annotations := map[string]string{
+		keys.GitProvider: "github",
+		keys.EventType:   "pull_request",
+		keys.Repository:  "pac-repo",
+	}
+	var prl []*tektonv1.PipelineRun
+	pr := &tektonv1.PipelineRun{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:   "pac-ns",
+			Annotations: annotations,
+		},
+		Status: tektonv1.PipelineRunStatus{
+			Status: duckv1.Status{Conditions: []apis.Condition{
+				{
+					Type:   apis.ConditionReady,
+					Status: corev1.ConditionTrue,
+					Reason: tektonv1.PipelineRunReasonRunning.String(),
+				},
+			}},
+		},
+	}
+
+	numberOfRunningPRs := 10
+	for i := 0; i < numberOfRunningPRs; i++ {
+		prl = append(prl, pr)
+	}
+
+	unregisterMetrics()
+	m, err := metrics.NewRecorder()
+	assert.NilError(t, err)
+	r := &Reconciler{
+		metrics: m,
+	}
+
+	err = r.emitRunningPipelineRunsMetric(prl)
+	assert.NilError(t, err)
+	tags := map[string]string{
+		"namespace":  "pac-ns",
+		"repository": "pac-repo",
+		"event-type": "pull_request",
+		"provider":   "github",
+	}
+	metricstest.CheckLastValueData(t, "pipelines_as_code_running_pipelineruns_count", tags, float64(numberOfRunningPRs))
+}
+
+func unregisterMetrics() {
+	metricstest.Unregister("pipelines_as_code_pipelinerun_count",
+		"pipelines_as_code_pipelinerun_duration_seconds_sum",
+		"pipelines_as_code_running_pipelineruns_count")
+}

pkg/reconciler/reconciler.go

@@ -11,6 +11,7 @@ import (
 	tektonv1lister "github.com/tektoncd/pipeline/pkg/client/listers/pipeline/v1"
 	"go.uber.org/zap"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"knative.dev/pkg/logging"
 	pkgreconciler "knative.dev/pkg/reconciler"
 	"knative.dev/pkg/system"
@@ -52,6 +53,15 @@ var (
 
 // ReconcileKind is the main entry point for reconciling PipelineRun resources.
 func (r *Reconciler) ReconcileKind(ctx context.Context, pr *tektonv1.PipelineRun) pkgreconciler.Event {
+	prl, err := r.pipelineRunLister.List(labels.Everything())
+	if err != nil {
+		return err
+	}
+
+	if err := r.emitRunningPipelineRunsMetric(prl); err != nil {
+		return err
+	}
+
 	ctx = info.StoreNS(ctx, system.Namespace())
 	logger := logging.FromContext(ctx).With("namespace", pr.GetNamespace())
 	// if pipelineRun is in completed or failed state then return
@@ -318,3 +328,18 @@ func (r *Reconciler) updatePipelineRunState(ctx context.Context, logger *zap.Sug
 	}
 	return patchedPR, nil
 }
+
+func (r *Reconciler) emitRunningPipelineRunsMetric(prl []*tektonv1.PipelineRun) error {
+	gitProvider := prl[0].GetAnnotations()[keys.GitProvider]
+	eventType := prl[0].GetAnnotations()[keys.EventType]
+	repository := prl[0].GetAnnotations()[keys.Repository]
+
+	runningPRs := 0
+	for _, pr := range prl {
+		if !pr.IsDone() {
+			runningPRs++
+		}
+	}
+
+	return r.metrics.CountRunningPRs(gitProvider, eventType, prl[0].GetNamespace(), repository, float64(runningPRs))
+}

pkg/reconciler/reconciler_test.go

@@ -180,6 +180,10 @@ func TestReconciler_ReconcileKind(t *testing.T) {
 			}
 			stdata, informers := testclient.SeedTestData(t, ctx, testData)
 
+			// needs to unregister because this test call NewRecorder func
+			// which registers metrics and causes `already registered metric` error
+			// in other metric tests.
+			unregisterMetrics()
 			metrics, err := metrics.NewRecorder()
 			assert.NilError(t, err)