enable clusterlifecycle-metric on AKS (#984)

Signed-off-by: Zhiwei Yin <zyin@redhat.com>
stolostron · Nov 14, 2024 · 02fae54 · 02fae54
1 parent 9ed7592
commit 02fae54
Show file tree

Hide file tree

Showing 27 changed files with 97 additions and 46 deletions.
diff --git a/controllers/toggle_components.go b/controllers/toggle_components.go
@@ -779,19 +779,21 @@ func (r *MultiClusterEngineReconciler) ensureNoImageBasedInstallOperator(ctx con
 func (r *MultiClusterEngineReconciler) ensureClusterLifecycle(ctx context.Context,
 	mce *backplanev1.MultiClusterEngine) (ctrl.Result, error) {
 
-	namespacedName := types.NamespacedName{Name: "cluster-curator-controller", Namespace: mce.Spec.TargetNamespace}
-	r.StatusManager.RemoveComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
-	r.StatusManager.AddComponent(toggle.EnabledStatus(namespacedName))
-	namespacedName = types.NamespacedName{Name: "clusterclaims-controller", Namespace: mce.Spec.TargetNamespace}
-	r.StatusManager.RemoveComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
-	r.StatusManager.AddComponent(toggle.EnabledStatus(namespacedName))
-	namespacedName = types.NamespacedName{Name: "provider-credential-controller", Namespace: mce.Spec.TargetNamespace}
-	r.StatusManager.RemoveComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
-	r.StatusManager.AddComponent(toggle.EnabledStatus(namespacedName))
-	namespacedName = types.NamespacedName{Name: "clusterlifecycle-state-metrics-v2", Namespace: mce.Spec.TargetNamespace}
-	r.StatusManager.RemoveComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
-	r.StatusManager.AddComponent(toggle.EnabledStatus(namespacedName))
-	namespacedName = types.NamespacedName{Name: "cluster-image-set-controller", Namespace: mce.Spec.TargetNamespace}
+	if utils.DeployOnOCP() {
+		namespacedName := types.NamespacedName{Name: "cluster-curator-controller", Namespace: mce.Spec.TargetNamespace}
+		r.StatusManager.RemoveComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
+		r.StatusManager.AddComponent(toggle.EnabledStatus(namespacedName))
+		namespacedName = types.NamespacedName{Name: "clusterclaims-controller", Namespace: mce.Spec.TargetNamespace}
+		r.StatusManager.RemoveComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
+		r.StatusManager.AddComponent(toggle.EnabledStatus(namespacedName))
+		namespacedName = types.NamespacedName{Name: "provider-credential-controller", Namespace: mce.Spec.TargetNamespace}
+		r.StatusManager.RemoveComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
+		r.StatusManager.AddComponent(toggle.EnabledStatus(namespacedName))
+		namespacedName = types.NamespacedName{Name: "cluster-image-set-controller", Namespace: mce.Spec.TargetNamespace}
+		r.StatusManager.RemoveComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
+		r.StatusManager.AddComponent(toggle.EnabledStatus(namespacedName))
+	}
+	namespacedName := types.NamespacedName{Name: "clusterlifecycle-state-metrics-v2", Namespace: mce.Spec.TargetNamespace}
 	r.StatusManager.RemoveComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
 	r.StatusManager.AddComponent(toggle.EnabledStatus(namespacedName))
 
@@ -849,18 +851,20 @@ func (r *MultiClusterEngineReconciler) ensureNoClusterLifecycle(ctx context.Cont
 		return ctrl.Result{RequeueAfter: requeuePeriod}, nil
 	}
 
-	namespacedName := types.NamespacedName{Name: "cluster-curator-controller", Namespace: mce.Spec.TargetNamespace}
-	r.StatusManager.RemoveComponent(toggle.EnabledStatus(namespacedName))
-	r.StatusManager.AddComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
-	namespacedName = types.NamespacedName{Name: "clusterclaims-controller", Namespace: mce.Spec.TargetNamespace}
-	r.StatusManager.RemoveComponent(toggle.EnabledStatus(namespacedName))
-	r.StatusManager.AddComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
-	namespacedName = types.NamespacedName{Name: "provider-credential-controller", Namespace: mce.Spec.TargetNamespace}
-	r.StatusManager.RemoveComponent(toggle.EnabledStatus(namespacedName))
-	r.StatusManager.AddComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
-	namespacedName = types.NamespacedName{Name: "cluster-image-set-controller", Namespace: mce.Spec.TargetNamespace}
-	r.StatusManager.RemoveComponent(toggle.EnabledStatus(namespacedName))
-	r.StatusManager.AddComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
+	if utils.DeployOnOCP() {
+		namespacedName := types.NamespacedName{Name: "cluster-curator-controller", Namespace: mce.Spec.TargetNamespace}
+		r.StatusManager.RemoveComponent(toggle.EnabledStatus(namespacedName))
+		r.StatusManager.AddComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
+		namespacedName = types.NamespacedName{Name: "clusterclaims-controller", Namespace: mce.Spec.TargetNamespace}
+		r.StatusManager.RemoveComponent(toggle.EnabledStatus(namespacedName))
+		r.StatusManager.AddComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
+		namespacedName = types.NamespacedName{Name: "provider-credential-controller", Namespace: mce.Spec.TargetNamespace}
+		r.StatusManager.RemoveComponent(toggle.EnabledStatus(namespacedName))
+		r.StatusManager.AddComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
+		namespacedName = types.NamespacedName{Name: "cluster-image-set-controller", Namespace: mce.Spec.TargetNamespace}
+		r.StatusManager.RemoveComponent(toggle.EnabledStatus(namespacedName))
+		r.StatusManager.AddComponent(toggle.DisabledStatus(namespacedName, []*unstructured.Unstructured{}))
+	}
 
 	// Deletes all templates
 	for _, template := range templates {

diff --git a/main.go b/main.go
@@ -270,6 +270,10 @@ func main() {
 					Name:      "ocm-webhook",
 					HostNames: []string{fmt.Sprintf("ocm-webhook.%s.svc", operatorNamespace)},
 				},
+				{
+					Name:      "clusterlifecycle-state-metrics-certs",
+					HostNames: []string{fmt.Sprintf("clusterlifecycle-state-metrics-v2.%s.svc", operatorNamespace)},
+				},
 			}).Start(ctx)
 
 		if err = (&mcewebhook.Reconciler{

diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/cluster-curator-clusterrole.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/cluster-curator-clusterrole.yaml
@@ -1,5 +1,6 @@
 # Copyright Contributors to the Open Cluster Management project
 ---
+{{- if .Values.global.deployOnOCP }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -112,3 +113,4 @@ rules:
   - delete
   - update
   - list
+{{- end }}
diff --git a/...ates/charts/toggle/cluster-lifecycle/templates/cluster-curator-controller-deployment.yaml b/...ates/charts/toggle/cluster-lifecycle/templates/cluster-curator-controller-deployment.yaml
@@ -1,6 +1,6 @@
 
 # Copyright Contributors to the Open Cluster Management project.
-
+{{- if .Values.global.deployOnOCP }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -111,4 +111,5 @@ spec:
         {{ if .Effect }} effect: {{ .Effect }} {{- end }}
         {{ if .TolerationSeconds }} tolerationSeconds: {{ .TolerationSeconds }} {{- end }}
         {{- end }}
-{{- end }}
+{{- end }}
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/cluster-curator-rolebinding.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/cluster-curator-rolebinding.yaml
@@ -1,6 +1,7 @@
 
 # Copyright Contributors to the Open Cluster Management project
 ---
+{{- if .Values.global.deployOnOCP }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -12,4 +13,5 @@ subjects:
 roleRef:
   kind: ClusterRole
   name: {{ .Values.org }}.cluster-lifecycle.cluster-curator
-  apiGroup: rbac.authorization.k8s.io
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/cluster-curator-service_account.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/cluster-curator-service_account.yaml
@@ -1,6 +1,8 @@
 # Copyright Contributors to the Open Cluster Management project
 ---
+{{- if .Values.global.deployOnOCP }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: cluster-curator
+  name: cluster-curator
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/cluster-image-set-clusterrole.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/cluster-image-set-clusterrole.yaml
@@ -1,5 +1,6 @@
 # Copyright Contributors to the Open Cluster Management project
 ---
+{{- if .Values.global.deployOnOCP }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -59,3 +60,4 @@ rules:
   - get
   - create
   - update
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/cluster-image-set-deployment.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/cluster-image-set-deployment.yaml
@@ -1,6 +1,6 @@
 
 # Copyright Contributors to the Open Cluster Management project.
-
+{{- if .Values.global.deployOnOCP }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -123,3 +123,4 @@ spec:
         {{ if .TolerationSeconds }} tolerationSeconds: {{ .TolerationSeconds }} {{- end }}
         {{- end }}
 {{- end }}
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/cluster-image-set-rolebinding.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/cluster-image-set-rolebinding.yaml
@@ -1,6 +1,7 @@
 
 # Copyright Contributors to the Open Cluster Management project
 ---
+{{- if .Values.global.deployOnOCP }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -13,3 +14,4 @@ roleRef:
   kind: ClusterRole
   name: {{ .Values.org }}.cluster-lifecycle.cluster-image-set
   apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/...emplates/charts/toggle/cluster-lifecycle/templates/cluster-image-set-service_account.yaml b/...emplates/charts/toggle/cluster-lifecycle/templates/cluster-image-set-service_account.yaml
@@ -1,6 +1,8 @@
 # Copyright Contributors to the Open Cluster Management project
 ---
+{{- if .Values.global.deployOnOCP }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: cluster-image-set
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/clusterclaims-clusterrole.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/clusterclaims-clusterrole.yaml
@@ -1,5 +1,6 @@
 # Copyright Contributors to the Open Cluster Management project
 ---
+{{- if .Values.global.deployOnOCP }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -89,4 +90,5 @@ rules:
   resources:
   - events
   verbs:
-  - create
+  - create
+{{- end }}
diff --git a/...plates/charts/toggle/cluster-lifecycle/templates/clusterclaims-controller-deployment.yaml b/...plates/charts/toggle/cluster-lifecycle/templates/clusterclaims-controller-deployment.yaml
@@ -1,5 +1,5 @@
 # Copyright Contributors to the Open Cluster Management project.
-
+{{- if .Values.global.deployOnOCP }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:  
@@ -137,4 +137,5 @@ spec:
         {{ if .Effect }} effect: {{ .Effect }} {{- end }}
         {{ if .TolerationSeconds }} tolerationSeconds: {{ .TolerationSeconds }} {{- end }}
         {{- end }}
-{{- end }}
+{{- end }}
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/clusterclaims-rolebinding.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/clusterclaims-rolebinding.yaml
@@ -1,6 +1,7 @@
 
 # Copyright Contributors to the Open Cluster Management project
 ---
+{{- if .Values.global.deployOnOCP }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -12,4 +13,5 @@ subjects:
 roleRef:
   kind: ClusterRole
   name: {{ .Values.org }}.cluster-lifecycle.clusterclaims
-  apiGroup: rbac.authorization.k8s.io
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/clusterclaims-service_account.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/clusterclaims-service_account.yaml
@@ -1,7 +1,9 @@
 
 # Copyright Contributors to the Open Cluster Management project
 ---
+{{- if .Values.global.deployOnOCP }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: clusterclaims
+  name: clusterclaims
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/clusterrole-clustermanageradmin.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/clusterrole-clustermanageradmin.yaml
@@ -1,5 +1,5 @@
 # Copyright Contributors to the Open Cluster Management project
-
+{{- if .Values.global.deployOnOCP }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -11,3 +11,4 @@ aggregationRule:
     - matchLabels:
         rbac.authorization.k8s.io/aggregate-to-ocm-cluster-manager-admin: 'true'
 rules: []
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/metrics-clusterrole.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/metrics-clusterrole.yaml
@@ -15,6 +15,12 @@ rules:
   - update
   - get
   - delete
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
 # Allow hub to monitor and update status of csr
 - apiGroups: ["hive.openshift.io"]
   resources: ["clusterdeployments"]

diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/metrics-clusterrolebinding-prom.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/metrics-clusterrolebinding-prom.yaml
@@ -1,5 +1,5 @@
 # Copyright Contributors to the Open Cluster Management project
-
+{{- if .Values.global.deployOnOCP }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -11,4 +11,5 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: prometheus-k8s
-    namespace: openshift-monitoring
+    namespace: openshift-monitoring
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/metrics-deployment.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/metrics-deployment.yaml
@@ -101,10 +101,12 @@ spec:
             name: certs
             readOnly: true
       securityContext:
+        {{- if .Values.global.deployOnOCP }}
         {{- if semverCompare ">=4.11.0" .Values.hubconfig.ocpVersion }}
         seccompProfile:
           type: RuntimeDefault
         {{- end }}
+        {{- end }}
       volumes:
       - name: certs
         secret:

diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/metrics-prometheusrule.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/metrics-prometheusrule.yaml
@@ -1,5 +1,5 @@
 # Copyright Contributors to the Open Cluster Management project
-
+{{- if .Values.global.deployOnOCP }}
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
@@ -10,4 +10,5 @@ spec:
     - name: acm_managed_cluster_worker_cores.rules
       rules:
         - expr: max by (hub_cluster_id, managed_cluster_id) (acm_managed_cluster_worker_cores)
-          record: 'acm_managed_cluster_worker_cores:max'
+          record: 'acm_managed_cluster_worker_cores:max'
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/metrics-servicemonitor.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/metrics-servicemonitor.yaml
@@ -1,5 +1,5 @@
 # Copyright Contributors to the Open Cluster Management project
-
+{{- if .Values.global.deployOnOCP }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -20,4 +20,5 @@ spec:
       clc-app: clusterlifecycle-state-metrics-v2
   namespaceSelector:
     matchNames:
-      - {{ .Values.global.namespace }}
+      - {{ .Values.global.namespace }}
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/templates/provider-credential-clusterrole.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/templates/provider-credential-clusterrole.yaml
@@ -1,5 +1,6 @@
 # Copyright Contributors to the Open Cluster Management project
 ---
+{{- if .Values.global.deployOnOCP }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -33,4 +34,5 @@ rules:
   - events
   verbs:
   - create
-  - patch
+  - patch
+{{- end }}
diff --git a/...tes/charts/toggle/cluster-lifecycle/templates/provider-credential-clusterrolebinding.yaml b/...tes/charts/toggle/cluster-lifecycle/templates/provider-credential-clusterrolebinding.yaml
@@ -1,5 +1,6 @@
 # Copyright Contributors to the Open Cluster Management project
 ---
+{{- if .Values.global.deployOnOCP }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -11,4 +12,5 @@ subjects:
 roleRef:
   kind: ClusterRole
   name: {{ .Values.org }}.cluster-lifecycle.provider-credential
-  apiGroup: rbac.authorization.k8s.io
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/.../charts/toggle/cluster-lifecycle/templates/provider-credential-controller-deployment.yaml b/.../charts/toggle/cluster-lifecycle/templates/provider-credential-controller-deployment.yaml
@@ -1,4 +1,5 @@
 # Copyright Contributors to the Open Cluster Management project
+{{- if .Values.global.deployOnOCP }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -132,4 +133,4 @@ spec:
         {{ if .TolerationSeconds }} tolerationSeconds: {{ .TolerationSeconds }} {{- end }}
         {{- end }}
 {{- end }}
-      
+{{- end }}
diff --git a/...plates/charts/toggle/cluster-lifecycle/templates/provider-credential-service_account.yaml b/...plates/charts/toggle/cluster-lifecycle/templates/provider-credential-service_account.yaml
@@ -1,6 +1,8 @@
 # Copyright Contributors to the Open Cluster Management project
 ---
+{{- if .Values.global.deployOnOCP }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: provider-credential
+  name: provider-credential
+{{- end }}
diff --git a/pkg/templates/charts/toggle/cluster-lifecycle/values.yaml b/pkg/templates/charts/toggle/cluster-lifecycle/values.yaml
@@ -9,6 +9,7 @@ global:
   pullSecret: ""
   namespace: default
   hubSize: Small
+  deployOnOCP: ""
 hubconfig:
   nodeSelector: {}
   proxyConfigs: {}

diff --git a/pkg/templates/rbac_gen.go b/pkg/templates/rbac_gen.go
diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
@@ -69,6 +69,7 @@ var nonOCPComponents = []string{
 	backplanev1.HyperShift,
 	backplanev1.HypershiftLocalHosting,
 	backplanev1.LocalCluster,
+	backplanev1.ClusterLifecycle,
 }
 
 var GlobalDeployOnOCP = true