Implement regex exclusions for alerts

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>
fluxcd · Feb 8, 2021 · 273fe7d · 273fe7d
1 parent ac36348
commit 273fe7d
Show file tree

Hide file tree

Showing 6 changed files with 91 additions and 10 deletions.
diff --git a/api/v1beta1/alert_types.go b/api/v1beta1/alert_types.go
@@ -27,7 +27,7 @@ const (
 
 // AlertSpec defines an alerting rule for events involving a list of objects
 type AlertSpec struct {
-	// Send events using this provider
+	// Send events using this provider.
 	// +required
 	ProviderRef meta.LocalObjectReference `json:"providerRef"`
 
@@ -38,10 +38,14 @@ type AlertSpec struct {
 	// +optional
 	EventSeverity string `json:"eventSeverity,omitempty"`
 
-	// Filter events based on the involved objects
+	// Filter events based on the involved objects.
 	// +required
 	EventSources []CrossNamespaceObjectReference `json:"eventSources"`
 
+	// A list of Golang regular expressions to be used for excluding messages.
+	// +optional
+	ExclusionList []string `json:"exclusionList,omitempty"`
+
 	// Short description of the impact and affected cluster.
 	// +optional
 	Summary string `json:"summary,omitempty"`

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/notification.toolkit.fluxcd.io_alerts.yaml b/config/crd/bases/notification.toolkit.fluxcd.io_alerts.yaml
@@ -56,7 +56,7 @@ spec:
                 - error
                 type: string
               eventSources:
-                description: Filter events based on the involved objects
+                description: Filter events based on the involved objects.
                 items:
                   description: CrossNamespaceObjectReference contains enough information
                     to let you locate the typed referenced object at cluster level
@@ -91,8 +91,14 @@ spec:
                   - name
                   type: object
                 type: array
+              exclusionList:
+                description: A list of Golang regular expressions to be used for excluding
+                  messages.
+                items:
+                  type: string
+                type: array
               providerRef:
-                description: Send events using this provider
+                description: Send events using this provider.
                 properties:
                   name:
                     description: Name of the referent

diff --git a/docs/api/notification.md b/docs/api/notification.md
@@ -80,7 +80,7 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference
 </em>
 </td>
 <td>
-<p>Send events using this provider</p>
+<p>Send events using this provider.</p>
 </td>
 </tr>
 <tr>
@@ -106,7 +106,19 @@ If set to &lsquo;info&rsquo; no events will be filtered.</p>
 </em>
 </td>
 <td>
-<p>Filter events based on the involved objects</p>
+<p>Filter events based on the involved objects.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>exclusionList</code><br>
+<em>
+[]string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>A list of Golang regular expressions to be used for excluding messages.</p>
 </td>
 </tr>
 <tr>
@@ -465,7 +477,7 @@ github.com/fluxcd/pkg/apis/meta.LocalObjectReference
 </em>
 </td>
 <td>
-<p>Send events using this provider</p>
+<p>Send events using this provider.</p>
 </td>
 </tr>
 <tr>
@@ -491,7 +503,19 @@ If set to &lsquo;info&rsquo; no events will be filtered.</p>
 </em>
 </td>
 <td>
-<p>Filter events based on the involved objects</p>
+<p>Filter events based on the involved objects.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>exclusionList</code><br>
+<em>
+[]string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>A list of Golang regular expressions to be used for excluding messages.</p>
 </td>
 </tr>
 <tr>

diff --git a/docs/spec/v1beta1/alert.md b/docs/spec/v1beta1/alert.md
@@ -8,7 +8,7 @@ Spec:
 
 ```go
 type AlertSpec struct {
-	// Send events using this provider
+	// Send events using this provider.
 	// +required
 	ProviderRef meta.LocalObjectReference `json:"providerRef"`
 
@@ -17,10 +17,14 @@ type AlertSpec struct {
 	// +optional
 	EventSeverity string `json:"eventSeverity,omitempty"`
 
-	// Filter events based on the involved objects
+	// Filter events based on the involved objects.
 	// +required
 	EventSources []CrossNamespaceObjectReference `json:"eventSources"`
 
+	// A list of Golang regular expressions to be used for excluding messages.
+	// +optional
+	ExclusionList []string `json:"exclusionList,omitempty"`
+
 	// Short description of the impact and affected cluster.
 	// +optional
 	Summary string `json:"summary,omitempty"`
@@ -115,3 +119,29 @@ spec:
     - kind: HelmRelease
       name: nginx-ingress
 ```
+
+Skip alerting if the message matches a [Go regex](https://golang.org/pkg/regexp/syntax)
+from the exclusion list:
+
+```yaml
+apiVersion: notification.toolkit.fluxcd.io/v1beta1
+kind: Alert
+metadata:
+  name: flux-system
+  namespace: flux-system
+spec:
+  providerRef: 
+    name: on-call-slack
+  eventSeverity: error
+  eventSources:
+    - kind: GitRepository
+      name: flux-system
+  exclusionList:
+    - "waiting.*socket"
+```
+
+The above definition will not send alerts for transient Git clone errors like:
+
+```
+unable to clone 'ssh://git@ssh.dev.azure.com/v3/...', error: SSH could not read data: Error waiting on socket
+```
diff --git a/internal/server/event_handlers.go b/internal/server/event_handlers.go
@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"net/http"
+	"regexp"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
@@ -72,6 +73,17 @@ func (s *EventServer) handleEvent() func(w http.ResponseWriter, r *http.Request)
 				continue
 			}
 
+			// skip alert if the message matches a regex from the exclusion list
+			if len(alert.Spec.ExclusionList) > 0 {
+				for _, exp := range alert.Spec.ExclusionList {
+					if r, err := regexp.Compile(exp); err == nil {
+						if r.Match([]byte(event.Message)) {
+							continue
+						}
+					}
+				}
+			}
+
 			// filter alerts by object and severity
 			for _, source := range alert.Spec.EventSources {
 				if source.Namespace == "" {