Skip to content

Commit

Permalink
Initial Prometheus alertmanager webhook
Browse files Browse the repository at this point in the history
  • Loading branch information
russss committed Nov 27, 2019
1 parent d15bdb7 commit 573057a
Show file tree
Hide file tree
Showing 8 changed files with 136 additions and 0 deletions.
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,13 @@ GitHub can be configured to deliver webhooks to irccat on an organisation level
with the `default_channel` setting, significantly reduces configuration effort compared to
GitHub's old integrations system.

### Prometheus Alertmanager Webhook
```json
"prometheus": "#channel"
```

Receives [Prometheus Alertmanager webhooks](https://prometheus.io/docs/alerting/configuration/#webhook_config) at `/prometheus`. They will be sent to the channel defined in `http.listeners.prometheus`. Note that this endpoint is unauthenticated.

## IRC → Shell
You can use irccat to execute commands from IRC:

Expand Down
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ require (
github.com/deckarep/golang-set v1.7.1
github.com/fsnotify/fsnotify v1.4.7
github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.5.0
github.com/stretchr/testify v1.3.0 // indirect
github.com/thoj/go-ircevent v0.0.0-20190807115034-8e7ce4b5a1eb
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e // indirect
gopkg.in/go-playground/webhooks.v5 v5.13.0
gopkg.in/yaml.v2 v2.2.7 // indirect
)
13 changes: 13 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ=
Expand Down Expand Up @@ -48,6 +49,7 @@ github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVE
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
Expand Down Expand Up @@ -77,6 +79,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
Expand All @@ -88,11 +91,16 @@ github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.5.0 h1:GpsTwfsQ27oS/Aha/6d1oD7tpKIqWnOA6tgOX9HHkt4=
github.com/spf13/viper v1.5.0/go.mod h1:AkYRkVJF8TkSG/xet6PzXX+l39KhhXa2pdqVSxnTcn4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/thoj/go-ircevent v0.0.0-20190807115034-8e7ce4b5a1eb h1:EavwSqheIJl3nb91HhkL73DwnT2Fk8W3yM7T7TuLZvA=
Expand All @@ -106,6 +114,7 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
Expand All @@ -124,6 +133,8 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
Expand All @@ -148,4 +159,6 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
5 changes: 5 additions & 0 deletions httplistener/httplistener.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,11 @@ func New(irc *irc.Connection) (*HTTPListener, error) {
mux.HandleFunc("/github", hl.githubHandler)
}

if viper.IsSet("http.listeners.prometheus") {
log.Infof("Listening for Prometheus Alertmanager webhooks at /prometheus")
mux.HandleFunc("/prometheus", hl.prometheusHandler)
}

hl.http.Handler = mux
if viper.GetBool("http.tls") {
go hl.http.ListenAndServeTLS(viper.GetString("http.tls_cert"), viper.GetString("http.tls_key"))
Expand Down
53 changes: 53 additions & 0 deletions httplistener/prometheus.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package httplistener

import (
"bytes"
"encoding/json"
"github.com/spf13/viper"
"net/http"
)

type HookMessage struct {
Version string `json:"version"`
GroupKey string `json:"groupKey"`
Status string `json:"status"`
Receiver string `json:"receiver"`
GroupLabels map[string]string `json:"groupLabels"`
CommonLabels map[string]string `json:"commonLabels"`
CommonAnnotations map[string]string `json:"commonAnnotations"`
ExternalURL string `json:"externalURL"`
Alerts []Alert `json:"alerts"`
}

type Alert struct {
Status string `json:"status"`
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
StartsAt string `json:"startsAt,omitempty"`
EndsAt string `json:"endsAt,omitempty"`
GeneratorURL string `json:"generatorURL"`
}

func (hl *HTTPListener) prometheusHandler(w http.ResponseWriter, request *http.Request) {
if request.Method != "POST" {
http.NotFound(w, request)
return
}

var message HookMessage
buf := new(bytes.Buffer)
buf.ReadFrom(request.Body)
json.Unmarshal(buf.Bytes(), &message)
log.Infof("%s [%s] Prometheus alert", request.RemoteAddr, viper.GetString("http.listeners.prometheus"))

msgs, err := hl.renderTemplate("prometheus.alert", message)
if err != nil {
log.Errorf("Unable to render template: %s", err)
return
}

channel := viper.GetString("http.listeners.prometheus")
for _, msg := range msgs {
hl.irc.Privmsg(channel, msg)
}
}
2 changes: 2 additions & 0 deletions httplistener/templates.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ var defaultTemplates = map[string]string{
"github.issue": "[{{b .Repository.Name}}] {{g .Sender.Login}} {{.Action}} issue #{{.Issue.Number}}: {{.Issue.Title}} {{.Issue.HTMLURL}}",
"github.issuecomment": "[{{b .Repository.Name}}] {{g .Comment.User.Login}} commented on issue #{{.Issue.Number}}: {{trunc .Comment.Body 150}} {{.Comment.HTMLURL}}",
"github.pullrequest": "[{{b .Repository.Name}}] {{g .Sender.Login}} {{if .PullRequest.Merged}}merged{{else}}{{.Action}}{{end}} pull request #{{.PullRequest.Number}} (\x0303{{.PullRequest.Base.Ref}}…{{.PullRequest.Head.Ref}}\x0f): {{.PullRequest.Title}} {{.PullRequest.HTMLURL}}",
"prometheus.alert": `{{range .Alerts}}[{{b "Prometheus"}}] {{if (eq .Status "firing")}}{{b "alerting"}}{{else}}{{h "resolved"}}{{end}}: {{.Annotations.summary}}
{{end}}`,
}

func refName(ref string) string {
Expand Down
26 changes: 26 additions & 0 deletions webhook_test_data/prometheus/alert.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
{
"version": "4",
"groupKey": "group",
"status": "firing",
"receiver": "irccat",
"groupLabels": {},
"commonLabels": {},
"commonAnnotations": {},
"externalURL": "http://alertmanager:9093",
"alerts": [
{
"status": "firing",
"labels": {
"alertname": "Test Alert",
"instance": "test.example.com",
"job": "jobname"
},
"annotations": {
"summary": "Alert summary"
},
"startsAt": "2019-11-10T23:20:50.52Z",
"endsAt": "2019-11-10T23:20:50.52Z",
"generatorURL": "http://prometheus:9090/graph?"
}
]
}
26 changes: 26 additions & 0 deletions webhook_test_data/prometheus/resolved.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
{
"version": "4",
"groupKey": "group",
"status": "firing",
"receiver": "irccat",
"groupLabels": {},
"commonLabels": {},
"commonAnnotations": {},
"externalURL": "http://alertmanager:9093",
"alerts": [
{
"status": "firing",
"labels": {
"alertname": "Test Alert",
"instance": "test.example.com",
"job": "jobname"
},
"annotations": {
"summary": "Alert summary"
},
"startsAt": "2019-11-10T23:20:50.52Z",
"endsAt": "2019-11-10T23:20:50.52Z",
"generatorURL": "http://prometheus:9090/graph?"
}
]
}

0 comments on commit 573057a

Please sign in to comment.