diff --git a/CHANGELOG.md b/CHANGELOG.md index 0654331d82e..c02e3f407cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ - Remove `config.Pipeline.InputDataType` (#4343) - otlpexporter: Do not retry on PermissionDenied and Unauthenticated (#4349) +## 💡 Enhancements 💡 +- Supports more compression methods(`snappy` and `zstd`) for configgrpc, in addition to current `gzip` (#4088) + ## v0.38.0 Beta ## 🛑 Breaking changes 🛑 diff --git a/config/configgrpc/README.md b/config/configgrpc/README.md index 7b82bb5f62e..3a343eb72fe 100644 --- a/config/configgrpc/README.md +++ b/config/configgrpc/README.md @@ -16,7 +16,7 @@ configuration. For more information, see [configtls README](../configtls/README.md). - [`balancer_name`](https://github.com/grpc/grpc-go/blob/master/examples/features/load_balancing/README.md) -- `compression` (default = gzip): Compression type to use (only gzip is supported today) +- `compression` Compression type to use among `gzip`, `snappy` and `zstd` - `endpoint`: Valid value syntax available [here](https://github.com/grpc/grpc/blob/master/doc/naming.md) - [`tls`](../configtls/README.md) - `headers`: name/value pairs added to the request diff --git a/config/configgrpc/configgrpc.go b/config/configgrpc/configgrpc.go index 9f58844452c..5a2bf891aaf 100644 --- a/config/configgrpc/configgrpc.go +++ b/config/configgrpc/configgrpc.go @@ -21,6 +21,8 @@ import ( "strings" "time" + "github.com/mostynb/go-grpc-compression/snappy" + "github.com/mostynb/go-grpc-compression/zstd" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel" "google.golang.org/grpc" @@ -39,12 +41,16 @@ import ( const ( CompressionUnsupported = "" CompressionGzip = "gzip" + CompressionSnappy = "snappy" + CompressionZstd = "zstd" ) var ( // Map of opentelemetry compression types to grpc registered compression types. gRPCCompressionKeyMap = map[string]string{ - CompressionGzip: gzip.Name, + CompressionGzip: gzip.Name, + CompressionSnappy: snappy.Name, + CompressionZstd: zstd.Name, } ) @@ -67,8 +73,7 @@ type GRPCClientSettings struct { // https://github.com/grpc/grpc/blob/master/doc/naming.md. Endpoint string `mapstructure:"endpoint"` - // The compression key for supported compression types within - // collector. Currently the only supported mode is `gzip`. + // The compression key for supported compression types within collector. Compression string `mapstructure:"compression"` // TLSSetting struct exposes TLS client configuration. diff --git a/config/configgrpc/configgrpc_test.go b/config/configgrpc/configgrpc_test.go index 1c1c94b200c..18e81155719 100644 --- a/config/configgrpc/configgrpc_test.go +++ b/config/configgrpc/configgrpc_test.go @@ -330,6 +330,22 @@ func TestGetGRPCCompressionKey(t *testing.T) { t.Error("Capitalization of CompressionGzip should not matter") } + if GetGRPCCompressionKey("snappy") != CompressionSnappy { + t.Error("snappy is marked as supported but returned unsupported") + } + + if GetGRPCCompressionKey("Snappy") != CompressionSnappy { + t.Error("Capitalization of CompressionSnappy should not matter") + } + + if GetGRPCCompressionKey("zstd") != CompressionZstd { + t.Error("zstd is marked as supported but returned unsupported") + } + + if GetGRPCCompressionKey("Zstd") != CompressionZstd { + t.Error("Capitalization of CompressionZstd should not matter") + } + if GetGRPCCompressionKey("badType") != CompressionUnsupported { t.Error("badType is not supported but was returned as supported") } diff --git a/exporter/otlpexporter/cfg-schema.yaml b/exporter/otlpexporter/cfg-schema.yaml index abecc7d95cb..bfabb9207b8 100644 --- a/exporter/otlpexporter/cfg-schema.yaml +++ b/exporter/otlpexporter/cfg-schema.yaml @@ -64,7 +64,7 @@ fields: kind: string doc: | The compression key for supported compression types within - collector. Currently the only supported mode is `gzip`. + collector. Supports `gzip`, `snappy` and `zstd`. - name: ca_file kind: string doc: | diff --git a/exporter/otlpexporter/factory_test.go b/exporter/otlpexporter/factory_test.go index e791114d6ad..4fcf05f0ccc 100644 --- a/exporter/otlpexporter/factory_test.go +++ b/exporter/otlpexporter/factory_test.go @@ -99,7 +99,7 @@ func TestCreateTracesExporter(t *testing.T) { }, }, { - name: "Compression", + name: "GzipCompression", config: Config{ ExporterSettings: config.NewExporterSettings(config.NewComponentID(typeStr)), GRPCClientSettings: configgrpc.GRPCClientSettings{ @@ -108,6 +108,24 @@ func TestCreateTracesExporter(t *testing.T) { }, }, }, + { + name: "SnappyCompression", + config: Config{ + GRPCClientSettings: configgrpc.GRPCClientSettings{ + Endpoint: endpoint, + Compression: configgrpc.CompressionSnappy, + }, + }, + }, + { + name: "ZstdCompression", + config: Config{ + GRPCClientSettings: configgrpc.GRPCClientSettings{ + Endpoint: endpoint, + Compression: configgrpc.CompressionZstd, + }, + }, + }, { name: "Headers", config: Config{ diff --git a/go.mod b/go.mod index 5e1363e94d9..71d3a87d6f2 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/knadh/koanf v1.3.0 github.com/magiconair/properties v1.8.5 github.com/mitchellh/mapstructure v1.4.2 + github.com/mostynb/go-grpc-compression v1.1.14 github.com/prometheus/common v0.32.1 github.com/rs/cors v1.8.0 github.com/shirou/gopsutil/v3 v3.21.9 @@ -34,7 +35,7 @@ require ( go.uber.org/zap v1.19.1 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71 google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08 - google.golang.org/grpc v1.41.0 + google.golang.org/grpc v1.42.0 google.golang.org/protobuf v1.27.1 gopkg.in/yaml.v2 v2.4.0 ) @@ -51,8 +52,9 @@ require ( github.com/go-ole/go-ole v1.2.5 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/kr/pretty v0.3.0 // indirect + github.com/klauspost/compress v1.13.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect diff --git a/go.sum b/go.sum index eb23bea7257..45fa65f3c67 100644 --- a/go.sum +++ b/go.sum @@ -83,7 +83,10 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -103,6 +106,7 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -164,6 +168,8 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -257,6 +263,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/knadh/koanf v1.3.0 h1:nNmG4HGbpJUv7eUV1skDvHzzFS+35Q3b+OsYvoXyt2E= github.com/knadh/koanf v1.3.0/go.mod h1:HZ7HMLIGbrWJUfgtEzfHvzR/rX+eIqQlBNPRr4Vt42s= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -303,6 +311,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mostynb/go-grpc-compression v1.1.14 h1:dxtryvqLCzScC6mIpw5oWZqyBgucdMmnVWRNFkVX9DY= +github.com/mostynb/go-grpc-compression v1.1.14/go.mod h1:z6FFnn1nEs6zxqgicvXTqb9IFVmb+R9zdr2BvaYH8tU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= @@ -312,6 +322,7 @@ github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAv github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -777,8 +788,9 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=