From ac0a0d49424f1f19b5044ea84a245e3139b5adb3 Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Fri, 20 Dec 2019 12:27:05 -0600 Subject: [PATCH] indexer: add Accept-Encoding aware middleware This commit adds a middleware that examines the incoming "accept-encoding" header and attempt to compress the payload accordingly, and sets the `clair` binary to use it automatically. --- cmd/clair/httpcompress.go | 151 +++++++++++++++++++++++++++++++++++++ cmd/clair/httptransport.go | 11 +-- go.mod | 2 + go.sum | 4 + 4 files changed, 163 insertions(+), 5 deletions(-) create mode 100644 cmd/clair/httpcompress.go diff --git a/cmd/clair/httpcompress.go b/cmd/clair/httpcompress.go new file mode 100644 index 0000000000..72d763e7b5 --- /dev/null +++ b/cmd/clair/httpcompress.go @@ -0,0 +1,151 @@ +package main + +import ( + "fmt" + "io" + "net/http" + "sync" + + "github.com/klauspost/compress/flate" + "github.com/klauspost/compress/gzip" + "github.com/klauspost/compress/snappy" + "github.com/markusthoemmes/goautoneg" +) + +// Compress wraps the provided http.Handler and provides transparent body +// compression based on a Request's "Accept-Encoding" header. +func Compress(next http.Handler) http.Handler { + h := compressHandler{ + next: next, + } + h.snappy.New = func() interface{} { + return snappy.NewBufferedWriter(nil) + } + h.gzip.New = func() interface{} { + w, _ := gzip.NewWriterLevel(nil, gzip.BestSpeed) + return w + } + h.flate.New = func() interface{} { + w, _ := flate.NewWriter(nil, flate.BestSpeed) + return w + } + + return &h +} + +var _ http.Handler = (*compressHandler)(nil) + +// CompressHandler performs transparent HTTP body compression. +type compressHandler struct { + snappy, gzip, flate sync.Pool + next http.Handler +} + +// Header is an interface that has the http.ResponseWriter's Header-related +// methods. +type header interface { + Header() http.Header + WriteHeader(int) +} + +// ServeHTTP implements http.Handler. +func (c *compressHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var ( + flusher http.Flusher + pusher http.Pusher + cw io.WriteCloser + ) + flusher, _ = w.(http.Flusher) + pusher, _ = w.(http.Pusher) + + // Find the first accept-encoding we support. + for _, a := range goautoneg.ParseAccept(r.Header.Get("accept-encoding")) { + switch a.Type { + case "gzip": + w.Header().Set("content-encoding", "gzip") + gz := c.gzip.Get().(*gzip.Writer) + defer c.gzip.Put(gz) + cw = gz + case "deflate": + w.Header().Set("content-encoding", "deflate") + z := c.flate.Get().(*flate.Writer) + defer c.flate.Put(z) + cw = z + case "snappy": // Nonstandard + w.Header().Set("content-encoding", "snappy") + s := c.snappy.Get().(*snappy.Writer) + defer c.snappy.Put(s) + cw = s + case "identity": + w.Header().Set("content-encoding", "identity") + case "*": + default: + continue + } + break + } + // Do some setup so we can see the error, albeit as a trailer. + if cw != nil { + const errHeader = `clair-error` + w.Header().Add("trailer", errHeader) + defer func() { + if err := cw.Close(); err != nil { + w.Header().Add(errHeader, err.Error()) + } + }() + } + + // Nw is the http.ResponseWriter for our next http.Handler. + var nw http.ResponseWriter + // This is a giant truth table to make anonymous types that satisfy as many + // optional interfaces as possible. + // + // We care about 3 interfaces, so there are 2^3 == 8 combinations + switch { + case flusher == nil && pusher == nil && cw == nil: + nw = w + case flusher == nil && pusher == nil && cw != nil: + nw = struct { + header + io.Writer + }{w, cw} + case flusher == nil && pusher != nil && cw == nil: + nw = struct { + http.ResponseWriter + http.Pusher + }{w, pusher} + case flusher == nil && pusher != nil && cw != nil: + nw = struct { + header + io.Writer + http.Pusher + }{w, cw, pusher} + case flusher != nil && pusher == nil && cw == nil: + nw = struct { + http.ResponseWriter + http.Flusher + }{w, flusher} + case flusher != nil && pusher == nil && cw != nil: + nw = struct { + header + io.Writer + http.Flusher + }{w, cw, flusher} + case flusher != nil && pusher != nil && cw == nil: + nw = struct { + http.ResponseWriter + http.Flusher + http.Pusher + }{w, flusher, pusher} + case flusher != nil && pusher != nil && cw != nil: + nw = struct { + header + io.Writer + http.Flusher + http.Pusher + }{w, cw, flusher, pusher} + default: + panic(fmt.Sprintf("unexpect type combination: %T/%T/%T", flusher, pusher, cw)) + } + c.next.ServeHTTP(nw, r) +} diff --git a/cmd/clair/httptransport.go b/cmd/clair/httptransport.go index 7f50acbf38..8a28e3b38c 100644 --- a/cmd/clair/httptransport.go +++ b/cmd/clair/httptransport.go @@ -6,11 +6,12 @@ import ( "net/http" "time" + "github.com/quay/claircore/libindex" + "github.com/quay/claircore/libvuln" + "github.com/quay/clair/v4/config" "github.com/quay/clair/v4/indexer" "github.com/quay/clair/v4/matcher" - "github.com/quay/claircore/libindex" - "github.com/quay/claircore/libvuln" ) const ( @@ -60,7 +61,7 @@ func devMode(ctx context.Context, conf config.Config) (*http.Server, error) { matcher.Register(mux) return &http.Server{ Addr: conf.HTTPListenAddr, - Handler: mux, + Handler: Compress(mux), }, nil } @@ -81,7 +82,7 @@ func indexerMode(ctx context.Context, conf config.Config) (*http.Server, error) } return &http.Server{ Addr: conf.Indexer.HTTPListenAddr, - Handler: indexer, + Handler: Compress(indexer), }, nil } @@ -105,6 +106,6 @@ func matcherMode(ctx context.Context, conf config.Config) (*http.Server, error) } return &http.Server{ Addr: conf.Matcher.HTTPListenAddr, - Handler: matcher, + Handler: Compress(matcher), }, nil } diff --git a/go.mod b/go.mod index 7a3b404229..f3a0de99fc 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/quay/clair/v4 go 1.13 require ( + github.com/klauspost/compress v1.9.4 + github.com/markusthoemmes/goautoneg v0.0.0-20190713162725-c6008fefa5b1 github.com/mattn/go-sqlite3 v1.11.0 // indirect github.com/quay/claircore v0.0.10-0.20191211195844-8a5a18affde1 github.com/rs/zerolog v1.16.0 diff --git a/go.sum b/go.sum index 6b945527bb..0d1a483623 100644 --- a/go.sum +++ b/go.sum @@ -188,6 +188,8 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.4 h1:xhvAeUPQ2drNUhKtrGdTGNvV9nNafHMUkRyLkzxJoB4= +github.com/klauspost/compress v1.9.4/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/knqyf263/go-cpe v0.0.0-20180327054844-659663f6eca2 h1:9CYbtr3i56D/rD6u6jJ/Aocsic9G+MupyVu7gb+QHF4= github.com/knqyf263/go-cpe v0.0.0-20180327054844-659663f6eca2/go.mod h1:XM58Cg7dN+g0J9UPVmKjiXWlGi55lx+9IMs0IMoFWQo= github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d h1:X4cedH4Kn3JPupAwwWuo4AzYp16P0OyLO9d7OnMZc/c= @@ -214,6 +216,8 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/markusthoemmes/goautoneg v0.0.0-20190713162725-c6008fefa5b1 h1:Qhv4Ni88zV+8TY65yr2ak8xU4sblgs6aRT9RuGM5SNU= +github.com/markusthoemmes/goautoneg v0.0.0-20190713162725-c6008fefa5b1/go.mod h1:qFhy2RoC9EWZC7fgczcBbUpzGNFfIm5//VO/gde0AbI= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=