Prevent alerts from inhibiting themselves

inhibit/inhibit.go

@@ -139,7 +139,8 @@ func (ih *Inhibitor) Mutes(lset model.LabelSet) bool {
 	fp := lset.Fingerprint()
 
 	for _, r := range ih.rules {
-		if inhibitedByFP, eq := r.hasEqual(lset); r.TargetMatchers.Match(lset) && eq {
+		// Only inhibit if target matchers match but source matchers don't.
+		if inhibitedByFP, eq := r.hasEqual(lset); !r.SourceMatchers.Match(lset) && r.TargetMatchers.Match(lset) && eq {
 			ih.marker.SetInhibited(fp, fmt.Sprintf("%d", inhibitedByFP))
 			return true
 		}

inhibit/inhibit_test.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
+	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/types"
 	"github.com/prometheus/common/model"
 )
@@ -66,7 +67,7 @@ func TestInhibitRuleHasEqual(t *testing.T) {
 			result: false,
 		},
 		{
-			// Matching but already resolved.
+			// Matching and unresolved.
 			initial: map[model.Fingerprint]*types.Alert{
 				1: &types.Alert{
 					Alert: model.Alert{
@@ -79,13 +80,13 @@ func TestInhibitRuleHasEqual(t *testing.T) {
 					Alert: model.Alert{
 						Labels:   model.LabelSet{"a": "b", "c": "f"},
 						StartsAt: now.Add(-time.Minute),
-						EndsAt:   now.Add(-time.Second),
+						EndsAt:   now.Add(time.Hour),
 					},
 				},
 			},
 			equal:  model.LabelNames{"a"},
 			input:  model.LabelSet{"a": "b"},
-			result: false,
+			result: true,
 		},
 		{
 			// Equal label does not match.
@@ -133,6 +134,65 @@ func TestInhibitRuleHasEqual(t *testing.T) {
 	}
 }
 
+func TestInhibitRuleMatches(t *testing.T) {
+	// Simple inhibut rule
+	cr := config.InhibitRule{
+		SourceMatch: map[string]string{"s": "1"},
+		TargetMatch: map[string]string{"t": "1"},
+		Equal:       model.LabelNames{"e"},
+	}
+	m := types.NewMarker()
+	ih := NewInhibitor(nil, []*config.InhibitRule{&cr}, m, nil)
+	ir := ih.rules[0]
+	now := time.Now()
+	// Active alert that matches the source filter
+	sourceAlert := types.Alert{
+		Alert: model.Alert{
+			Labels:   model.LabelSet{"s": "1", "e": "1"},
+			StartsAt: now.Add(-time.Minute),
+			EndsAt:   now.Add(time.Hour),
+		},
+	}
+	ir.scache = map[model.Fingerprint]*types.Alert{1: &sourceAlert}
+
+	cases := []struct {
+		target   model.LabelSet
+		expected bool
+	}{
+		{
+			// Matches target filter, inhibited
+			target:   model.LabelSet{"t": "1", "e": "1"},
+			expected: true,
+		},
+		{
+			// Matches target filter (plus noise), inhibited
+			target:   model.LabelSet{"t": "1", "t2": "1", "e": "1"},
+			expected: true,
+		},
+		{
+			// Doesn't match target filter, not inhibited
+			target:   model.LabelSet{"t": "0", "e": "1"},
+			expected: false,
+		},
+		{
+			// Matches both source and target filters, not inhibited
+			target:   model.LabelSet{"s": "1", "t": "1", "e": "1"},
+			expected: false,
+		},
+		{
+			// Matches target filter, equal label doesn't match, not inhibited
+			target:   model.LabelSet{"t": "1", "e": "0"},
+			expected: false,
+		},
+	}
+
+	for _, c := range cases {
+		if actual := ih.Mutes(c.target); actual != c.expected {
+			t.Errorf("Expected (*Inhibitor).Mutes(%v) to return %t but got %t", c.target, c.expected, actual)
+		}
+	}
+}
+
 func TestInhibitRuleGC(t *testing.T) {
 	// TODO(fabxc): add now() injection function to Resolved() to remove
 	// dependency on machine time in this test.