From 6ebeb130177ad1056730d737c44ec1040b6cdcb0 Mon Sep 17 00:00:00 2001 From: Ross Guarino Date: Mon, 20 Aug 2018 08:34:07 -0700 Subject: [PATCH] Expose workqueue stats This CL adds an http endpoint with one handler "/metrics" that serves Prometheus metrics exported by MetaController. The metrics are collected using OpenCensus and exported using the Prometheus HTTP exposition format. This CL also configures the Kubernetes workqueue package to collect its statistics and expose them via the Prometheus http endpoint. We collect and expose all of the workqueue metrics (Depth, Adds, Latency, WorkDuration, Retries) tagged with the name of the queue. --- Gopkg.lock | 231 +++++++++++++++++++++++++++++++++++++++++++++++++---- Gopkg.toml | 5 ++ main.go | 32 ++++++++ 3 files changed, 251 insertions(+), 17 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 3483466..280bbc6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,204 +2,343 @@ [[projects]] + branch = "master" + digest = "1:3dcf9df58e425f7ace052d832e683930533be49fcde3b6d66cebb2ed5a97bb3f" + name = "github.com/0xRLG/ocworkqueue" + packages = ["."] + pruneopts = "UT" + revision = "40aa506a5a3c9f494b65cf929bb34b6ab1d02617" + +[[projects]] + digest = "1:d1665c44bd5db19aaee18d1b6233c99b0b9a986e8bccb24ef54747547a48027f" name = "github.com/PuerkitoBio/purell" packages = ["."] + pruneopts = "UT" revision = "0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:c739832d67eb1e9cc478a19cc1a1ccd78df0397bf8a32978b759152e205f644b" name = "github.com/PuerkitoBio/urlesc" packages = ["."] + pruneopts = "UT" revision = "de5bf2ad457846296e2031421a34e2568e304e35" [[projects]] + branch = "master" + digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" + name = "github.com/beorn7/perks" + packages = ["quantile"] + pruneopts = "UT" + revision = "3a771d992973f24aa725d07868b467d1ddfceafb" + +[[projects]] + digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" name = "github.com/davecgh/go-spew" packages = ["spew"] + pruneopts = "UT" revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" [[projects]] + digest = "1:899234af23e5793c34e06fd397f86ba33af5307b959b6a7afd19b63db065a9d7" name = "github.com/emicklei/go-restful" packages = [ ".", - "log" + "log", ] + pruneopts = "UT" revision = "3eb9738c1697594ea6e71a7156a9bb32ed216cf0" version = "v2.8.0" [[projects]] + digest = "1:2cd7915ab26ede7d95b8749e6b1f933f1c6d5398030684e6505940a10f31cfda" name = "github.com/ghodss/yaml" packages = ["."] + pruneopts = "UT" revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" version = "v1.0.0" [[projects]] branch = "master" + digest = "1:2997679181d901ac8aaf4330d11138ecf3974c6d3334995ff36f20cbd597daf8" name = "github.com/go-openapi/jsonpointer" packages = ["."] + pruneopts = "UT" revision = "3a0015ad55fa9873f41605d3e8f28cd279c32ab2" [[projects]] branch = "master" + digest = "1:1ae3f233d75a731b164ca9feafd8ed646cbedf1784095876ed6988ce8aa88b1f" name = "github.com/go-openapi/jsonreference" packages = ["."] + pruneopts = "UT" revision = "3fb327e6747da3043567ee86abd02bb6376b6be2" [[projects]] branch = "master" + digest = "1:b37bb08b92684db03722b62592cd23df06341a9642718cb0d1b107d57b48b334" name = "github.com/go-openapi/spec" packages = ["."] + pruneopts = "UT" revision = "90cb6221326a6e7a2ae375729b6f724c7ae66899" [[projects]] branch = "master" + digest = "1:731022b436cdb9b4b2a53be2ead693467a1474b8b873d4f90cb424fffdc3d0ff" name = "github.com/go-openapi/swag" packages = ["."] + pruneopts = "UT" revision = "2b0bd4f193d011c203529df626a65d63cb8a79e8" [[projects]] + digest = "1:a7534feda0f15b5fd691e59e4fb6b7547e27df4b415a62e02c7cb71b3439c1b1" name = "github.com/gogo/protobuf" packages = [ "proto", - "sortkeys" + "sortkeys", ] + pruneopts = "UT" revision = "1adfc126b41513cc696b209667c8656ea7aac67c" version = "v1.0.0" [[projects]] branch = "master" + digest = "1:1ba1d79f2810270045c328ae5d674321db34e3aae468eb4233883b473c5c0467" name = "github.com/golang/glog" packages = ["."] + pruneopts = "UT" revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" [[projects]] + digest = "1:8caffcd8995b0eae7d2c18b2baefabef331bb05b1e16ed2b26b732c3349e6989" name = "github.com/golang/protobuf" packages = [ "proto", "ptypes", "ptypes/any", "ptypes/duration", - "ptypes/timestamp" + "ptypes/timestamp", ] + pruneopts = "UT" revision = "925541529c1fa6821df4e44ce2723319eb2be768" version = "v1.0.0" [[projects]] branch = "master" + digest = "1:9887333bbef17574b1db5f9893ea137ac44107235d624408a3ac9e0b98fbb2cb" name = "github.com/google/btree" packages = ["."] + pruneopts = "UT" revision = "e89373fe6b4a7413d7acd6da1725b83ef713e6e4" [[projects]] branch = "master" + digest = "1:9e21a98615890056a2b052e94035c87b21f8fea93d01aeeebae605fcc398b3e4" name = "github.com/google/go-jsonnet" packages = [ ".", "ast", - "parser" + "parser", ] + pruneopts = "UT" revision = "b0459e4867cd151f426eebf22bafe08c7eb608bf" [[projects]] branch = "master" + digest = "1:3ee90c0d94da31b442dde97c99635aaafec68d0b8a3c12ee2075c6bdabeec6bb" name = "github.com/google/gofuzz" packages = ["."] + pruneopts = "UT" revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1" [[projects]] + digest = "1:5e031a35b76ee001fa9ca9d598298054a123d080e00d13a8dafcfc5e3ecd5b58" name = "github.com/googleapis/gnostic" packages = [ "OpenAPIv2", "compiler", - "extensions" + "extensions", ] + pruneopts = "UT" revision = "ee43cbb60db7bd22502942cccbc39059117352ab" version = "v0.1.0" [[projects]] branch = "master" + digest = "1:86c1210529e69d69860f2bb3ee9ccce0b595aa3f9165e7dd1388e5c612915888" name = "github.com/gregjones/httpcache" packages = [ ".", - "diskcache" + "diskcache", ] + pruneopts = "UT" revision = "9cad4c3443a7200dd6400aef47183728de563a38" [[projects]] branch = "master" + digest = "1:cf296baa185baae04a9a7004efee8511d08e2f5f51d4cbe5375da89722d681db" name = "github.com/hashicorp/golang-lru" packages = [ ".", - "simplelru" + "simplelru", ] + pruneopts = "UT" revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3" [[projects]] + digest = "1:eaefc85d32c03e5f0c2b88ea2f79fce3d993e2c78316d21319575dd4ea9153ca" name = "github.com/json-iterator/go" packages = ["."] + pruneopts = "UT" revision = "ab8a2e0c74be9d3be70b3184d9acc634935ded82" version = "1.1.4" [[projects]] + digest = "1:190ff84d9b2ed6589088f178cba8edb4b8ecb334df4572421fb016be1ac20463" name = "github.com/juju/ratelimit" packages = ["."] + pruneopts = "UT" revision = "59fac5042749a5afb9af70e813da1dd5474f0167" version = "1.0.1" [[projects]] branch = "master" + digest = "1:ada518b8c338e10e0afa443d84671476d3bd1d926e13713938088e8ddbee1a3e" name = "github.com/mailru/easyjson" packages = [ "buffer", "jlexer", - "jwriter" + "jwriter", ] + pruneopts = "UT" revision = "3fdea8d05856a0c8df22ed4bc71b3219245e4485" [[projects]] + digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc" + name = "github.com/matttproud/golang_protobuf_extensions" + packages = ["pbutil"] + pruneopts = "UT" + revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + version = "v1.0.1" + +[[projects]] + digest = "1:33422d238f147d247752996a26574ac48dcf472976eda7f5134015f06bf16563" name = "github.com/modern-go/concurrent" packages = ["."] + pruneopts = "UT" revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94" version = "1.0.3" [[projects]] + digest = "1:e32bdbdb7c377a07a9a46378290059822efdce5c8d96fe71940d87cb4f918855" name = "github.com/modern-go/reflect2" packages = ["."] + pruneopts = "UT" revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" version = "1.0.1" [[projects]] branch = "master" + digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2" name = "github.com/petar/GoLLRB" packages = ["llrb"] + pruneopts = "UT" revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" [[projects]] + digest = "1:0e7775ebbcf00d8dd28ac663614af924411c868dca3d5aa762af0fae3808d852" name = "github.com/peterbourgon/diskv" packages = ["."] + pruneopts = "UT" revision = "5f041e8faa004a95c88a202771f4cc3e991971e6" version = "v2.0.1" [[projects]] + digest = "1:d14a5f4bfecf017cb780bdde1b6483e5deb87e12c332544d2c430eda58734bcb" + name = "github.com/prometheus/client_golang" + packages = [ + "prometheus", + "prometheus/promhttp", + ] + pruneopts = "UT" + revision = "c5b7fccd204277076155f10851dad72b76a49317" + version = "v0.8.0" + +[[projects]] + digest = "1:32d10bdfa8f09ecf13598324dba86ab891f11db3c538b6a34d1c3b5b99d7c36b" + name = "github.com/prometheus/client_model" + packages = ["go"] + pruneopts = "UT" + revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" + +[[projects]] + branch = "master" + digest = "1:63b68062b8968092eb86bedc4e68894bd096ea6b24920faca8b9dcf451f54bb5" + name = "github.com/prometheus/common" + packages = [ + "expfmt", + "internal/bitbucket.org/ww/goautoneg", + "model", + ] + pruneopts = "UT" + revision = "c7de2306084e37d54b8be01f3541a8464345e9a5" + +[[projects]] + branch = "master" + digest = "1:8c49953a1414305f2ff5465147ee576dd705487c35b15918fcd4efdc0cb7a290" + name = "github.com/prometheus/procfs" + packages = [ + ".", + "internal/util", + "nfs", + "xfs", + ] + pruneopts = "UT" + revision = "05ee40e3a273f7245e8777337fc7b46e533a9a92" + +[[projects]] + digest = "1:1b21a2b4058a779f290c7341cd93267492e0ecea6c8b54f64a4a5fd7ff131034" name = "github.com/spf13/pflag" packages = ["."] + pruneopts = "UT" revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" version = "v1.0.0" +[[projects]] + digest = "1:5845e9795d688bc745679ac1062a45865934c437d71825391c99ced0a90112da" + name = "go.opencensus.io" + packages = [ + ".", + "exporter/prometheus", + "internal", + "internal/tagencoding", + "stats", + "stats/internal", + "stats/view", + "tag", + ] + pruneopts = "UT" + revision = "7b558058b7cc960667590e5413ef55157b06652e" + version = "v0.15.0" + [[projects]] branch = "master" + digest = "1:288df6b6054389a21896d57f73bc9584110d929f84aa67eaddf8c77b1e9a8d65" name = "golang.org/x/net" packages = [ "context", "http2", "http2/hpack", "idna", - "lex/httplex" + "lex/httplex", ] + pruneopts = "UT" revision = "e0c57d8f86c17f0724497efcb3bc617e82834821" [[projects]] + digest = "1:0c56024909189aee3364b7f21a95a27459f718aa7c199a5c111c36cfffd9eaef" name = "golang.org/x/text" packages = [ "collate", @@ -216,35 +355,43 @@ "unicode/cldr", "unicode/norm", "unicode/rangetable", - "width" + "width", ] + pruneopts = "UT" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" [[projects]] branch = "master" + digest = "1:8e85b51231292d9427635ba825850a3972ab2a43894e025dc6560a48eba186fd" name = "golang.org/x/tools" packages = [ "go/ast/astutil", "imports", - "internal/fastwalk" + "internal/fastwalk", ] + pruneopts = "UT" revision = "d600f31f815368e57b551e524b0169bd175f7edb" [[projects]] + digest = "1:ef72505cf098abdd34efeea032103377bec06abb61d8a06f002d5d296a4b1185" name = "gopkg.in/inf.v0" packages = ["."] + pruneopts = "UT" revision = "3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4" version = "v0.9.0" [[projects]] + digest = "1:2a81c6e126d36ad027328cffaa4888fc3be40f09dc48028d1f93705b718130b9" name = "gopkg.in/yaml.v2" packages = ["."] + pruneopts = "UT" revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5" version = "v2.1.1" [[projects]] branch = "master" + digest = "1:da85b03f43f9b8def4df7b1740755f85b149af652efecf1f569f73bcc3af9854" name = "k8s.io/api" packages = [ "admissionregistration/v1alpha1", @@ -274,11 +421,13 @@ "settings/v1alpha1", "storage/v1", "storage/v1alpha1", - "storage/v1beta1" + "storage/v1beta1", ] + pruneopts = "UT" revision = "5584376ceeffeb13a2e98b5e9f0e9dab37de4bab" [[projects]] + digest = "1:f4a207f5840646fda14928f2508187fbc637e4ca7eb04f7e133e631f85a2f408" name = "k8s.io/apimachinery" packages = [ "pkg/api/equality", @@ -326,12 +475,14 @@ "pkg/util/yaml", "pkg/version", "pkg/watch", - "third_party/forked/golang/reflect" + "third_party/forked/golang/reflect", ] + pruneopts = "UT" revision = "fb40df2b502912cbe3a93aa61c2b2487f39cb42f" version = "kubernetes-1.9.9" [[projects]] + digest = "1:35eee68628a3507eff6f4e71a284d9bffc2d35cf29cd8edc8b984c9e9d3d1e01" name = "k8s.io/client-go" packages = [ "discovery", @@ -350,12 +501,14 @@ "util/flowcontrol", "util/integer", "util/retry", - "util/workqueue" + "util/workqueue", ] + pruneopts = "UT" revision = "78700dec6369ba22221b72770783300f143df150" version = "v6.0.0" [[projects]] + digest = "1:32d34f7ec0e85817a23e6aaed502f37e82b1e12ede315759bbedc77fecf93aaa" name = "k8s.io/code-generator" packages = [ "cmd/client-gen", @@ -370,13 +523,15 @@ "cmd/informer-gen", "cmd/informer-gen/generators", "cmd/lister-gen", - "cmd/lister-gen/generators" + "cmd/lister-gen/generators", ] + pruneopts = "UT" revision = "0ab89e584187c20cc7c1a3f30db69f3b4ab64196" version = "kubernetes-1.9.9" [[projects]] branch = "master" + digest = "1:62b0d3121a45ef78635a23e46a16820f1a000e1d9edf2a898e1c056e41fec5ce" name = "k8s.io/gengo" packages = [ "args", @@ -385,25 +540,67 @@ "generator", "namer", "parser", - "types" + "types", ] + pruneopts = "UT" revision = "fdcf9f9480fdd5bf2b3c3df9bf4ecd22b25b87e2" [[projects]] branch = "master" + digest = "1:f60e3ccfe35ad7632217492c6709871d6c84b2ba77219f054f8d0ef7d1b996e2" name = "k8s.io/kube-openapi" packages = ["pkg/common"] + pruneopts = "UT" revision = "d83b052f768a50a309c692a9c271da3f3276ff88" [[projects]] + digest = "1:e7efc01ed4d7b67406bed902b048638045bcbbbb0cc3a9dda5adfd461f69a52f" name = "k8s.io/kubernetes" packages = ["pkg/util/pointer"] + pruneopts = "UT" revision = "bee2d1505c4fe820744d26d41ecd3fdd4a3d6546" version = "v1.9.4" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "b61d80b6a391ef6b4edad0fdd2dc3a9f9186a1c790353533c85d18b0aef09cd8" + input-imports = [ + "github.com/0xRLG/ocworkqueue", + "github.com/golang/glog", + "github.com/google/go-jsonnet", + "github.com/google/go-jsonnet/ast", + "go.opencensus.io/exporter/prometheus", + "go.opencensus.io/stats/view", + "k8s.io/apimachinery/pkg/api/equality", + "k8s.io/apimachinery/pkg/api/errors", + "k8s.io/apimachinery/pkg/api/testing/fuzzer", + "k8s.io/apimachinery/pkg/api/testing/roundtrip", + "k8s.io/apimachinery/pkg/apis/meta/fuzzer", + "k8s.io/apimachinery/pkg/apis/meta/v1", + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured", + "k8s.io/apimachinery/pkg/labels", + "k8s.io/apimachinery/pkg/runtime", + "k8s.io/apimachinery/pkg/runtime/schema", + "k8s.io/apimachinery/pkg/runtime/serializer", + "k8s.io/apimachinery/pkg/types", + "k8s.io/apimachinery/pkg/util/diff", + "k8s.io/apimachinery/pkg/util/errors", + "k8s.io/apimachinery/pkg/util/json", + "k8s.io/apimachinery/pkg/util/runtime", + "k8s.io/apimachinery/pkg/util/wait", + "k8s.io/apimachinery/pkg/watch", + "k8s.io/client-go/discovery", + "k8s.io/client-go/dynamic", + "k8s.io/client-go/rest", + "k8s.io/client-go/tools/cache", + "k8s.io/client-go/util/flowcontrol", + "k8s.io/client-go/util/retry", + "k8s.io/client-go/util/workqueue", + "k8s.io/code-generator/cmd/client-gen", + "k8s.io/code-generator/cmd/deepcopy-gen", + "k8s.io/code-generator/cmd/informer-gen", + "k8s.io/code-generator/cmd/lister-gen", + "k8s.io/kubernetes/pkg/util/pointer", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index a70362e..d4e8577 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -44,6 +44,11 @@ required = [ name = "github.com/json-iterator/go" version = "1.1.4" # as used by apimachinery@kubernetes-1.9.9 +[[override]] + name = "github.com/prometheus/client_model" + revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" +# last version compatable with golang/protobuf@v1.0.0 (the version used by apimachinery@kubernetes-1.9.9) + [prune] go-tests = true unused-packages = true diff --git a/main.go b/main.go index aabba57..c75e958 100644 --- a/main.go +++ b/main.go @@ -17,17 +17,23 @@ limitations under the License. package main import ( + "context" "flag" + "net/http" "os" "os/signal" "sync" "syscall" "time" + "github.com/0xRLG/ocworkqueue" "github.com/golang/glog" + "go.opencensus.io/exporter/prometheus" + "go.opencensus.io/stats/view" "k8s.io/client-go/discovery" "k8s.io/client-go/rest" + "k8s.io/client-go/util/workqueue" "k8s.io/metacontroller/apis/metacontroller/v1alpha1" mcclientset "k8s.io/metacontroller/client/generated/clientset/internalclientset" @@ -42,6 +48,7 @@ import ( var ( discoveryInterval = flag.Duration("discovery-interval", 30*time.Second, "How often to refresh discovery cache to pick up newly-installed resources") informerRelist = flag.Duration("cache-flush-interval", 30*time.Minute, "How often to flush local caches and relist objects from the API server") + debugAddr = flag.String("debug-addr", "localhost:9999", "The address to bind the debug http endpoints") ) type controller interface { @@ -54,6 +61,7 @@ func main() { glog.Infof("Discovery cache flush interval: %v", *discoveryInterval) glog.Infof("API server object cache flush interval: %v", *informerRelist) + glog.Infof("Debug http server address: %v", *debugAddr) config, err := rest.InClusterConfig() if err != nil { @@ -78,6 +86,9 @@ func main() { // Create dynamic informer factory (for sharing dynamic informers). dynInformers := dynamicinformer.NewSharedInformerFactory(dynClient, *informerRelist) + workqueue.SetProvider(ocworkqueue.MetricsProvider()) + view.Register(ocworkqueue.DefaultViews...) + // Start metacontrollers (controllers that spawn controllers). // Each one requests the informers it needs from the factory. controllers := []controller{ @@ -94,6 +105,22 @@ func main() { c.Start() } + exporter, err := prometheus.NewExporter(prometheus.Options{}) + if err != nil { + glog.Fatalf("can't create prometheus exporter: %v", err) + } + view.RegisterExporter(exporter) + + mux := http.NewServeMux() + mux.Handle("/metrics", exporter) + srv := &http.Server{ + Addr: *debugAddr, + Handler: mux, + } + go func() { + glog.Fatalf("cannot serve debug endpoint: %v", srv.ListenAndServe()) + }() + // On SIGTERM, stop all controllers gracefully. sigchan := make(chan os.Signal, 2) signal.Notify(sigchan, os.Interrupt, syscall.SIGTERM) @@ -108,5 +135,10 @@ func main() { c.Stop() }(c) } + wg.Add(1) + go func() { + defer wg.Done() + srv.Shutdown(context.Background()) + }() wg.Wait() }