Add Prometheus metrics to fleet-controller

Refers to rancher#2172, SURE-4340

Expose Prometheus metrics of the fleet-controller for the following
controllers:

- GitRepo
- Bundle
- BundleDeployment
- Cluster
- ClusterGroup

.github/workflows/e2e-ci.yml

@@ -106,21 +106,24 @@ jobs:
           # 1. Run test cases not needing infra
           ginkgo --label-filter='!infra-setup' e2e/single-cluster e2e/keep-resources
 
-          # 2. Run tests requiring only the git server
+          # 2. Run tests for metrics
+          ginkgo e2e/metrics
+
+          # 3. Run tests requiring only the git server
           e2e/testenv/infra/infra setup --git-server=true
           ginkgo --label-filter='infra-setup && !helm-registry && !oci-registry' e2e/single-cluster/
 
-          # 3. Run tests requiring a Helm registry
+          # 4. Run tests requiring a Helm registry
           e2e/testenv/infra/infra setup --helm-registry=true
           ginkgo --label-filter='helm-registry' e2e/single-cluster
 
           e2e/testenv/infra/infra teardown --helm-registry=true
 
-          # 4. Run tests requiring an OCI registry
+          # 5. Run tests requiring an OCI registry
           e2e/testenv/infra/infra setup --oci-registry=true
           ginkgo --label-filter='oci-registry' e2e/single-cluster
 
-          # 5. Tear down all infra
+          # 6. Tear down all infra
           e2e/testenv/infra/infra teardown
 
       -

.gitignore

@@ -15,3 +15,5 @@ e2e/testenv/infra/infra
 ^fleet$
 FleetCI-RootCA
 .envrc
+env.multi-cluster
+env.single-cluster

charts/fleet/templates/deployment.yaml

@@ -58,11 +58,19 @@ spec:
         image: '{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}'
         name: fleet-controller
         imagePullPolicy: "{{ .Values.image.imagePullPolicy }}"
+        {{- if .Values.metrics.enabled }}
+        ports:
+        - containerPort: 8080
+          name: metrics
+        {{- end }}
         command:
         - fleetcontroller
         {{- if not .Values.gitops.enabled }}
         - --disable-gitops
         {{- end }}
+        {{- if not .Values.metrics.enabled }}
+        - --disable-metrics
+        {{- end }}
         {{- if .Values.debug }}
         - --debug
         - --debug-level

charts/fleet/templates/service.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.metrics.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: monitoring-fleet-controller
+  labels:
+    app: fleet-controller
+spec:
+  type: ClusterIP
+  ports:
+  - port: 8080
+    targetPort: 8080
+    protocol: TCP
+    name: metrics
+  selector:
+    app: fleet-controller
+{{- end }}

charts/fleet/values.yaml

@@ -66,6 +66,9 @@ priorityClassName: ""
 gitops:
   enabled: true
 
+metrics:
+  enabled: true
+
 debug: false
 debugLevel: 0
 propagateDebugSettingsToAgents: true

e2e/assets/clustergroup-template.yaml

@@ -0,0 +1,11 @@
+apiVersion: fleet.cattle.io/v1alpha1
+kind: ClusterGroup
+metadata:
+  name: {{ .Name }}
+  namespace: {{ .Namespace }}
+spec:
+  selector:
+    matchLabels:
+      {{- range $key, $value := .MatchLabels}}
+      {{$key}}: {{$value}}
+      {{- end}}

e2e/assets/metrics/fleetcontroller_service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Name }}
+spec:
+  selector:
+    app: fleet-controller
+  ports:
+    - protocol: TCP
+      port: {{ .Port }}
+      targetPort: metrics
+  type: LoadBalancer

e2e/metrics/bundle_test.go

@@ -0,0 +1,156 @@
+package metrics_test
+
+import (
+	"fmt"
+	"math/rand"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/rancher/fleet/e2e/metrics"
+	"github.com/rancher/fleet/e2e/testenv"
+	"github.com/rancher/fleet/e2e/testenv/kubectl"
+)
+
+var _ = Describe("Bundle Metrics", Label("bundle"), func() {
+	const (
+		objName = "metrics"
+		branch  = "master"
+	)
+
+	var (
+		// kw is the kubectl command for namespace the workload is deployed to
+		kw        kubectl.Command
+		namespace string
+	)
+
+	BeforeEach(func() {
+		k = env.Kubectl.Namespace(env.Namespace)
+		namespace = testenv.NewNamespaceName(
+			objName,
+			rand.New(rand.NewSource(time.Now().UnixNano())),
+		)
+		kw = k.Namespace(namespace)
+
+		out, err := k.Create("ns", namespace)
+		Expect(err).ToNot(HaveOccurred(), out)
+
+		err = testenv.CreateGitRepo(
+			kw,
+			namespace,
+			objName,
+			branch,
+			"simple-manifest",
+		)
+		Expect(err).ToNot(HaveOccurred())
+
+		DeferCleanup(func() {
+			out, err = k.Delete("ns", namespace)
+			Expect(err).ToNot(HaveOccurred(), out)
+		})
+	})
+
+	When("testing Bundle metrics", func() {
+		bundleMetricNames := []string{
+			"fleet_bundle_desired_ready",
+			"fleet_bundle_err_applied",
+			"fleet_bundle_modified",
+			"fleet_bundle_not_ready",
+			"fleet_bundle_out_of_sync",
+			"fleet_bundle_pending",
+			"fleet_bundle_ready",
+			"fleet_bundle_wait_applied",
+		}
+
+		It("should have exactly one metric for the bundle", func() {
+			et := metrics.NewExporterTest(metricsURL)
+			Eventually(func() error {
+				for _, metricName := range bundleMetricNames {
+					metric, err := et.FindOneMetric(
+						metricName,
+						map[string]string{
+							"name":      objName + "-simple-manifest",
+							"namespace": namespace,
+						},
+					)
+					if err != nil {
+						return err
+					}
+					Expect(metric.Gauge.GetValue()).To(Equal(float64(0)))
+				}
+				return nil
+			}).ShouldNot(HaveOccurred())
+		})
+
+		Context("when the GitRepo (and therefore Bundle) is changed", Label("bundle-altered"), func() {
+			It("it should not duplicate metrics if Bundle is updated", Label("bundle-update"), func() {
+				et := metrics.NewExporterTest(metricsURL)
+				out, err := kw.Patch(
+					"gitrepo", objName,
+					"--type=json",
+					"-p", `[{"op": "replace", "path": "/spec/paths", "value": ["simple-chart"]}]`,
+				)
+				Expect(err).ToNot(HaveOccurred(), out)
+				Expect(out).To(ContainSubstring("gitrepo.fleet.cattle.io/metrics patched"))
+
+				// Wait for it to be changed and fetched.
+				Eventually(func() (string, error) {
+					return kw.Get("gitrepo", objName, "-o", "jsonpath={.status.commit}")
+				}).ShouldNot(BeEmpty())
+
+				var metric *metrics.Metric
+				// Expect still no metrics to be duplicated.
+				Eventually(func() error {
+					for _, metricName := range bundleMetricNames {
+						metric, err = et.FindOneMetric(
+							metricName,
+							map[string]string{
+								"name":      objName + "-simple-chart",
+								"namespace": namespace,
+							},
+						)
+						if err != nil {
+							return err
+						}
+						if metric.LabelValue("paths") == "simple-manifest" {
+							return fmt.Errorf("path for metric %s unchanged", metricName)
+						}
+					}
+					return nil
+				}).ShouldNot(HaveOccurred())
+			})
+
+			It("should not keep metrics if Bundle is deleted", Label("bundle-delete"), func() {
+				et := metrics.NewExporterTest(metricsURL)
+
+				objName := objName + "-simple-manifest"
+
+				var (
+					out string
+					err error
+				)
+				Eventually(func() error {
+					out, err = kw.Delete("bundle", objName)
+					return err
+				}).ShouldNot(HaveOccurred(), out)
+
+				Eventually(func() error {
+					for _, metricName := range bundleMetricNames {
+						_, err := et.FindOneMetric(
+							metricName,
+							map[string]string{
+								"name":      objName,
+								"namespace": namespace,
+							},
+						)
+						if err == nil {
+							return fmt.Errorf("metric %s found but not expected", metricName)
+						}
+					}
+					return nil
+				}).ShouldNot(HaveOccurred())
+			})
+		})
+	})
+})