Merge pull request #2393 from benridley/dev_time_interval

Add time-based muting to routing tree

cmd/alertmanager/main.go

@@ -60,6 +60,7 @@ import (
 	"github.com/prometheus/alertmanager/provider/mem"
 	"github.com/prometheus/alertmanager/silence"
 	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/timeinterval"
 	"github.com/prometheus/alertmanager/types"
 	"github.com/prometheus/alertmanager/ui"
 )
@@ -413,6 +414,12 @@ func run() int {
 			integrationsNum += len(integrations)
 		}
 
+		// Build the map of time interval names to mute time definitions.
+		muteTimes := make(map[string][]timeinterval.TimeInterval, len(conf.MuteTimeIntervals))
+		for _, ti := range conf.MuteTimeIntervals {
+			muteTimes[ti.Name] = ti.TimeIntervals
+		}
+
 		inhibitor.Stop()
 		disp.Stop()
 
@@ -423,6 +430,7 @@ func run() int {
 			waitFunc,
 			inhibitor,
 			silencer,
+			muteTimes,
 			notificationLog,
 			peer,
 		)

config/config.go

@@ -31,6 +31,7 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/prometheus/alertmanager/pkg/labels"
+	"github.com/prometheus/alertmanager/timeinterval"
 )
 
 const secretToken = "<secret>"
@@ -219,13 +220,32 @@ func resolveFilepaths(baseDir string, cfg *Config) {
 	}
 }
 
+// MuteTimeInterval represents a named set of time intervals for which a route should be muted.
+type MuteTimeInterval struct {
+	Name          string                      `yaml:"name"`
+	TimeIntervals []timeinterval.TimeInterval `yaml:"time_intervals"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface for MuteTimeInterval.
+func (mt *MuteTimeInterval) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type plain MuteTimeInterval
+	if err := unmarshal((*plain)(mt)); err != nil {
+		return err
+	}
+	if mt.Name == "" {
+		return fmt.Errorf("missing name in mute time interval")
+	}
+	return nil
+}
+
 // Config is the top-level configuration for Alertmanager's config files.
 type Config struct {
-	Global       *GlobalConfig  `yaml:"global,omitempty" json:"global,omitempty"`
-	Route        *Route         `yaml:"route,omitempty" json:"route,omitempty"`
-	InhibitRules []*InhibitRule `yaml:"inhibit_rules,omitempty" json:"inhibit_rules,omitempty"`
-	Receivers    []*Receiver    `yaml:"receivers,omitempty" json:"receivers,omitempty"`
-	Templates    []string       `yaml:"templates" json:"templates"`
+	Global            *GlobalConfig      `yaml:"global,omitempty" json:"global,omitempty"`
+	Route             *Route             `yaml:"route,omitempty" json:"route,omitempty"`
+	InhibitRules      []*InhibitRule     `yaml:"inhibit_rules,omitempty" json:"inhibit_rules,omitempty"`
+	Receivers         []*Receiver        `yaml:"receivers,omitempty" json:"receivers,omitempty"`
+	Templates         []string           `yaml:"templates" json:"templates"`
+	MuteTimeIntervals []MuteTimeInterval `yaml:"mute_time_intervals,omitempty" json:"mute_time_intervals,omitempty"`
 
 	// original is the input from which the config was parsed.
 	original string
@@ -411,9 +431,23 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if len(c.Route.Match) > 0 || len(c.Route.MatchRE) > 0 {
 		return fmt.Errorf("root route must not have any matchers")
 	}
+	if len(c.Route.MuteTimeIntervals) > 0 {
+		return fmt.Errorf("root route must not have any mute time intervals")
+	}
 
 	// Validate that all receivers used in the routing tree are defined.
-	return checkReceiver(c.Route, names)
+	if err := checkReceiver(c.Route, names); err != nil {
+		return err
+	}
+
+	tiNames := make(map[string]struct{})
+	for _, mt := range c.MuteTimeIntervals {
+		if _, ok := tiNames[mt.Name]; ok {
+			return fmt.Errorf("mute time interval %q is not unique", mt.Name)
+		}
+		tiNames[mt.Name] = struct{}{}
+	}
+	return checkTimeInterval(c.Route, tiNames)
 }
 
 // checkReceiver returns an error if a node in the routing tree
@@ -433,6 +467,23 @@ func checkReceiver(r *Route, receivers map[string]struct{}) error {
 	return nil
 }
 
+func checkTimeInterval(r *Route, timeIntervals map[string]struct{}) error {
+	for _, sr := range r.Routes {
+		if err := checkTimeInterval(sr, timeIntervals); err != nil {
+			return err
+		}
+	}
+	if len(r.MuteTimeIntervals) == 0 {
+		return nil
+	}
+	for _, mt := range r.MuteTimeIntervals {
+		if _, ok := timeIntervals[mt]; !ok {
+			return fmt.Errorf("undefined time interval %q used in route", mt)
+		}
+	}
+	return nil
+}
+
 // DefaultGlobalConfig returns GlobalConfig with default values.
 func DefaultGlobalConfig() GlobalConfig {
 	return GlobalConfig{
@@ -582,10 +633,11 @@ type Route struct {
 	// Deprecated. Remove before v1.0 release.
 	Match map[string]string `yaml:"match,omitempty" json:"match,omitempty"`
 	// Deprecated. Remove before v1.0 release.
-	MatchRE  MatchRegexps `yaml:"match_re,omitempty" json:"match_re,omitempty"`
-	Matchers Matchers     `yaml:"matchers,omitempty" json:"matchers,omitempty"`
-	Continue bool         `yaml:"continue" json:"continue,omitempty"`
-	Routes   []*Route     `yaml:"routes,omitempty" json:"routes,omitempty"`
+	MatchRE           MatchRegexps `yaml:"match_re,omitempty" json:"match_re,omitempty"`
+	Matchers          Matchers     `yaml:"matchers,omitempty" json:"matchers,omitempty"`
+	MuteTimeIntervals []string     `yaml:"mute_time_intervals,omitempty" json:"mute_time_intervals,omitempty"`
+	Continue          bool         `yaml:"continue" json:"continue,omitempty"`
+	Routes            []*Route     `yaml:"routes,omitempty" json:"routes,omitempty"`
 
 	GroupWait      *model.Duration `yaml:"group_wait,omitempty" json:"group_wait,omitempty"`
 	GroupInterval  *model.Duration `yaml:"group_interval,omitempty" json:"group_interval,omitempty"`

config/config_test.go

@@ -151,6 +151,103 @@ receivers:
 
 }
 
+func TestMuteTimeExists(t *testing.T) {
+	in := `
+route:
+    receiver: team-Y
+    routes:
+    -  match:
+        severity: critical
+       mute_time_intervals:
+       - business_hours
+
+receivers:
+- name: 'team-Y'
+`
+	_, err := Load(in)
+
+	expected := "undefined time interval \"business_hours\" used in route"
+
+	if err == nil {
+		t.Fatalf("no error returned, expected:\n%q", expected)
+	}
+	if err.Error() != expected {
+		t.Errorf("\nexpected:\n%q\ngot:\n%q", expected, err.Error())
+	}
+
+}
+
+func TestMuteTimeHasName(t *testing.T) {
+	in := `
+mute_time_intervals:
+- name: 
+  time_intervals:
+  - times:
+     - start_time: '09:00'
+       end_time: '17:00'
+
+receivers:
+- name: 'team-X-mails'
+
+route:
+  receiver: 'team-X-mails'
+  routes:
+  -  match:
+      severity: critical
+     mute_time_intervals:
+     - business_hours
+`
+	_, err := Load(in)
+
+	expected := "missing name in mute time interval"
+
+	if err == nil {
+		t.Fatalf("no error returned, expected:\n%q", expected)
+	}
+	if err.Error() != expected {
+		t.Errorf("\nexpected:\n%q\ngot:\n%q", expected, err.Error())
+	}
+
+}
+
+func TestMuteTimeNoDuplicates(t *testing.T) {
+	in := `
+mute_time_intervals:
+- name: duplicate
+  time_intervals:
+  - times:
+     - start_time: '09:00'
+       end_time: '17:00'
+- name: duplicate
+  time_intervals:
+  - times:
+     - start_time: '10:00'
+       end_time: '14:00'
+
+receivers:
+- name: 'team-X-mails'
+
+route:
+  receiver: 'team-X-mails'
+  routes:
+  -  match:
+      severity: critical
+     mute_time_intervals:
+     - business_hours
+`
+	_, err := Load(in)
+
+	expected := "mute time interval \"duplicate\" is not unique"
+
+	if err == nil {
+		t.Fatalf("no error returned, expected:\n%q", expected)
+	}
+	if err.Error() != expected {
+		t.Errorf("\nexpected:\n%q\ngot:\n%q", expected, err.Error())
+	}
+
+}
+
 func TestGroupByHasNoDuplicatedLabels(t *testing.T) {
 	in := `
 route:
@@ -231,6 +328,36 @@ receivers:
 
 }
 
+func TestRootRouteNoMuteTimes(t *testing.T) {
+	in := `
+mute_time_intervals:
+- name: my_mute_time
+  time_intervals:
+  - times:
+     - start_time: '09:00'
+       end_time: '17:00'
+
+receivers:
+- name: 'team-X-mails'
+
+route:
+  receiver: 'team-X-mails'
+  mute_time_intervals:
+  - my_mute_time
+`
+	_, err := Load(in)
+
+	expected := "root route must not have any mute time intervals"
+
+	if err == nil {
+		t.Fatalf("no error returned, expected:\n%q", expected)
+	}
+	if err.Error() != expected {
+		t.Errorf("\nexpected:\n%q\ngot:\n%q", expected, err.Error())
+	}
+
+}
+
 func TestRootRouteHasNoMatcher(t *testing.T) {
 	in := `
 route:

dispatch/dispatch.go

@@ -404,6 +404,7 @@ func (ag *aggrGroup) run(nf notifyFunc) {
 			ctx = notify.WithGroupLabels(ctx, ag.labels)
 			ctx = notify.WithReceiverName(ctx, ag.opts.Receiver)
 			ctx = notify.WithRepeatInterval(ctx, ag.opts.RepeatInterval)
+			ctx = notify.WithMuteTimeIntervals(ctx, ag.opts.MuteTimeIntervals)
 
 			// Wait the configured interval before calling flush again.
 			ag.mtx.Lock()

dispatch/route.go

@@ -29,11 +29,12 @@ import (
 // DefaultRouteOpts are the defaulting routing options which apply
 // to the root route of a routing tree.
 var DefaultRouteOpts = RouteOpts{
-	GroupWait:      30 * time.Second,
-	GroupInterval:  5 * time.Minute,
-	RepeatInterval: 4 * time.Hour,
-	GroupBy:        map[model.LabelName]struct{}{},
-	GroupByAll:     false,
+	GroupWait:         30 * time.Second,
+	GroupInterval:     5 * time.Minute,
+	RepeatInterval:    4 * time.Hour,
+	GroupBy:           map[model.LabelName]struct{}{},
+	GroupByAll:        false,
+	MuteTimeIntervals: []string{},
 }
 
 // A Route is a node that contains definitions of how to handle alerts.
@@ -65,6 +66,7 @@ func NewRoute(cr *config.Route, parent *Route) *Route {
 	if cr.Receiver != "" {
 		opts.Receiver = cr.Receiver
 	}
+
 	if cr.GroupBy != nil {
 		opts.GroupBy = map[model.LabelName]struct{}{}
 		for _, ln := range cr.GroupBy {
@@ -115,6 +117,8 @@ func NewRoute(cr *config.Route, parent *Route) *Route {
 
 	sort.Sort(matchers)
 
+	opts.MuteTimeIntervals = cr.MuteTimeIntervals
+
 	route := &Route{
 		parent:    parent,
 		RouteOpts: opts,
@@ -203,6 +207,9 @@ type RouteOpts struct {
 	GroupWait      time.Duration
 	GroupInterval  time.Duration
 	RepeatInterval time.Duration
+
+	// A list of time intervals for which the route is muted.
+	MuteTimeIntervals []string
 }
 
 func (ro *RouteOpts) String() string {