Merge pull request opendatahub-io#124 from zdtsw-forking/fix_revert_m…

…onitoring_rules fix(monitoring): when DSC is removed entry in rule_files should be cleanup without panic
VaishnaviHire · Nov 20, 2023 · c8a6d91 · c8a6d91
2 parents dafe183 + 786f815
commit c8a6d91
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 11 deletions.
diff --git a/config/monitoring/prometheus/apps/prometheus-configs.yaml b/config/monitoring/prometheus/apps/prometheus-configs.yaml
@@ -7,8 +7,8 @@ metadata:
 data:
   prometheus.yml: |
     rule_files:
-      - 'operator-recording.rules'
-      - 'deadmanssnitch-alerting.rules'
+      - operator-recording.rules
+      - deadmanssnitch-alerting.rules
 
     global:
       scrape_interval:     10s
@@ -333,14 +333,14 @@ data:
       - name: DeadManSnitch
         interval: 1m
         rules:
-          - alert: DeadManSnitch
-            expr: vector(1)
-            labels:
-              severity: critical
-              namespace: redhat-ods-monitoring
-            annotations:
-              description: This is a DeadManSnitch to ensure RHODS monitoring and alerting pipeline is online.
-              summary: Alerting DeadManSnitch
+        - alert: DeadManSnitch
+          expr: vector(1)
+          labels:
+            severity: critical
+            namespace: redhat-ods-monitoring
+          annotations:
+            description: This is a DeadManSnitch to ensure RHODS monitoring and alerting pipeline is online.
+            summary: Alerting DeadManSnitch
 
   codeflare-recording.rules: |
     groups:

diff --git a/controllers/dscinitialization/monitoring.go b/controllers/dscinitialization/monitoring.go
@@ -50,9 +50,18 @@ func (r *DSCInitializationReconciler) configureManagedMonitoring(ctx context.Con
 		}
 	}
 	if initial == "revertbackup" {
+		// TODO: implement with a better solution
+		// to have - before component name is to filter out the real rules file line
+		// e.g line of "workbenches-recording.rules: |"
 		err := common.MatchLineInFile(filepath.Join(prometheusConfigPath, "prometheus-configs.yaml"),
 			map[string]string{
-				"*.rules: ": "",
+				"(.*)-(.*)workbenches(.*).rules":                     "",
+				"(.*)-(.*)rhods-dashboard(.*).rules":                 "",
+				"(.*)-(.*)codeflare(.*).rules":                       "",
+				"(.*)-(.*)data-science-pipelines-operator(.*).rules": "",
+				"(.*)-(.*)model-mesh(.*).rules":                      "",
+				"(.*)-(.*)odh-model-controller(.*).rules":            "",
+				"(.*)-(.*)ray(.*).rules":                             "",
 			})
 		if err != nil {
 			r.Log.Error(err, "error to remove previous enabled component rules")