Add APM tracing for client-go requests to the Kubernetes API (#5651)

This adds a custom APM round tripper for requests to the Kubernetes API. This can potentially be contributed to elastic/apm-agent-go as a new module. But I think it would be good to try it first in the context of the eck-operator. I structured the code in such a way to allow that if we decide it is worth contributing.
Why a custom round tripper? To get a meaningful span summary that is specific to k8s. The current implementation is not perfect in this regard especially sub-resource requests are represented somewhat arbitrarily by subsections of the request path. The other thing implement here is to track requests done in the background by the client-go library to maintain the local cache. As far as I know there is no way to inject a context into the library that would enable tracing. This PR implements instead a default transaction mechanism to track those requests.

cmd/manager/main.go

@@ -19,7 +19,7 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"go.elastic.co/apm"
+	"go.elastic.co/apm/v2"
 	"go.uber.org/automaxprocs/maxprocs"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -65,6 +65,7 @@ import (
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler"
 	controllerscheme "github.com/elastic/cloud-on-k8s/pkg/controller/common/scheme"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/tracing"
+	"github.com/elastic/cloud-on-k8s/pkg/controller/common/tracing/apmclientgo"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/version"
 	commonwebhook "github.com/elastic/cloud-on-k8s/pkg/controller/common/webhook"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch"
@@ -457,9 +458,18 @@ func startOperator(ctx context.Context) error {
 		return err
 	}
 
+	// set up APM  tracing if configured
+	var tracer *apm.Tracer
+	if viper.GetBool(operator.EnableTracingFlag) {
+		tracer = tracing.NewTracer("elastic-operator")
+		// set up APM tracing for client-go
+		cfg.Wrap(tracing.ClientGoTransportWrapper(
+			apmclientgo.WithDefaultTransaction(tracing.ClientGoCacheTx(tracer)),
+		))
+	}
+
 	// set the timeout for API client
 	cfg.Timeout = viper.GetDuration(operator.KubeClientTimeout)
-
 	// set the timeout for Elasticsearch requests
 	esclient.DefaultESClientTimeout = viper.GetDuration(operator.ElasticsearchClientTimeout)
 
@@ -556,10 +566,6 @@ func startOperator(ctx context.Context) error {
 	}
 
 	log.Info("Setting up controllers")
-	var tracer *apm.Tracer
-	if viper.GetBool(operator.EnableTracingFlag) {
-		tracer = tracing.NewTracer("elastic-operator")
-	}
 
 	exposedNodeLabels, err := esvalidation.NewExposedNodeLabels(viper.GetStringSlice(operator.ExposedNodeLabels))
 	if err != nil {

docs/reference/dependencies.asciidoc

@@ -44,8 +44,9 @@ This page lists the third-party dependencies used to build {n}.
 | link:https://github.com/spf13/viper[$$github.com/spf13/viper$$] | v1.11.0 | MIT
 | link:https://github.com/stretchr/testify[$$github.com/stretchr/testify$$] | v1.7.1 | MIT
 | link:https://github.com/tsenart/vegeta[$$github.com/tsenart/vegeta$$] | v12.7.0+incompatible | MIT
-| link:https://go.elastic.co/apm[$$go.elastic.co/apm$$] | v1.15.0 | Apache-2.0
-| link:https://go.elastic.co/apm/module/apmelasticsearch[$$go.elastic.co/apm/module/apmelasticsearch$$] | v1.15.0 | Apache-2.0
+| link:https://go.elastic.co/apm/module/apmelasticsearch/v2[$$go.elastic.co/apm/module/apmelasticsearch/v2$$] | v2.0.0 | Apache-2.0
+| link:https://go.elastic.co/apm/module/apmhttp/v2[$$go.elastic.co/apm/module/apmhttp/v2$$] | v2.0.0 | Apache-2.0
+| link:https://go.elastic.co/apm/v2[$$go.elastic.co/apm/v2$$] | v2.0.0 | Apache-2.0
 | link:https://go.uber.org/automaxprocs[$$go.uber.org/automaxprocs$$] | v1.5.1 | MIT
 | link:https://go.uber.org/zap[$$go.uber.org/zap$$] | v1.21.0 | MIT
 | link:https://golang.org/x/crypto[$$golang.org/x/crypto$$] | v0.0.0-20220411220226-7b82a4e95df4 | BSD-3-Clause
@@ -89,9 +90,9 @@ This page lists the third-party dependencies used to build {n}.
 | link:https://github.com/docker/distribution[$$github.com/docker/distribution$$] | v2.8.1+incompatible | Apache-2.0
 | link:https://github.com/docker/docker[$$github.com/docker/docker$$] | v20.10.16+incompatible | Apache-2.0
 | link:https://github.com/docker/docker-credential-helpers[$$github.com/docker/docker-credential-helpers$$] | v0.6.4 | MIT
-| link:https://github.com/elastic/go-licenser[$$github.com/elastic/go-licenser$$] | v0.3.1 | Apache-2.0
-| link:https://github.com/elastic/go-sysinfo[$$github.com/elastic/go-sysinfo$$] | v1.1.1 | Apache-2.0
-| link:https://github.com/elastic/go-windows[$$github.com/elastic/go-windows$$] | v1.0.0 | Apache-2.0
+| link:https://github.com/elastic/go-licenser[$$github.com/elastic/go-licenser$$] | v0.4.0 | Apache-2.0
+| link:https://github.com/elastic/go-sysinfo[$$github.com/elastic/go-sysinfo$$] | v1.7.1 | Apache-2.0
+| link:https://github.com/elastic/go-windows[$$github.com/elastic/go-windows$$] | v1.0.1 | Apache-2.0
 | link:https://github.com/elazarl/goproxy[$$github.com/elazarl/goproxy$$] | v0.0.0-20190711103511-473e67f1d7d2 | BSD-3-Clause
 | link:https://github.com/emicklei/go-restful[$$github.com/emicklei/go-restful$$] | v2.9.5+incompatible | MIT
 | link:https://github.com/evanphx/json-patch[$$github.com/evanphx/json-patch$$] | v4.12.0+incompatible | BSD-3-Clause
@@ -174,13 +175,11 @@ This page lists the third-party dependencies used to build {n}.
 | link:https://github.com/stretchr/objx[$$github.com/stretchr/objx$$] | v0.1.1 | MIT
 | link:https://github.com/subosito/gotenv[$$github.com/subosito/gotenv$$] | v1.2.0 | MIT
 | link:https://github.com/vbatts/tar-split[$$github.com/vbatts/tar-split$$] | v0.11.2 | BSD-3-Clause
-| link:https://go.elastic.co/apm/module/apmhttp[$$go.elastic.co/apm/module/apmhttp$$] | v1.15.0 | Apache-2.0
 | link:https://go.elastic.co/fastjson[$$go.elastic.co/fastjson$$] | v1.1.0 | MIT
 | link:https://go.uber.org/atomic[$$go.uber.org/atomic$$] | v1.9.0 | MIT
 | link:https://go.uber.org/goleak[$$go.uber.org/goleak$$] | v1.1.12 | MIT
 | link:https://go.uber.org/multierr[$$go.uber.org/multierr$$] | v1.6.0 | MIT
 | link:https://golang.org/x/exp[$$golang.org/x/exp$$] | v0.0.0-20200224162631-6cc2880d07d6 | BSD-3-Clause
-| link:https://golang.org/x/lint[$$golang.org/x/lint$$] | v0.0.0-20210508222113-6edffad5e616 | BSD-3-Clause
 | link:https://golang.org/x/mod[$$golang.org/x/mod$$] | v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 | BSD-3-Clause
 | link:https://golang.org/x/net[$$golang.org/x/net$$] | v0.0.0-20220516155154-20f960328961 | BSD-3-Clause
 | link:https://golang.org/x/oauth2[$$golang.org/x/oauth2$$] | v0.0.0-20220411215720-9780585627b5 | BSD-3-Clause
@@ -202,7 +201,7 @@ This page lists the third-party dependencies used to build {n}.
 | link:https://gopkg.in/square/go-jose.v2[$$gopkg.in/square/go-jose.v2$$] | v2.5.1 | Apache-2.0
 | link:https://gopkg.in/tomb.v1[$$gopkg.in/tomb.v1$$] | v1.0.0-20141024135613-dd632973f1e7 | BSD-3-Clause
 | link:https://gotest.tools/v3[$$gotest.tools/v3$$] | v3.0.3 | Apache-2.0
-| link:https://gitlab.howett.net/go/plist[$$howett.net/plist$$] | v0.0.0-20181124034731-591f970eefbb | BSD-2-Clause
+| link:https://gitlab.howett.net/go/plist[$$howett.net/plist$$] | v1.0.0 | BSD-2-Clause
 | link:https://github.com/kubernetes/apiextensions-apiserver[$$k8s.io/apiextensions-apiserver$$] | v0.24.0 | Apache-2.0
 | link:https://github.com/kubernetes/component-base[$$k8s.io/component-base$$] | v0.24.0 | Apache-2.0
 | link:https://github.com/kubernetes/kube-openapi[$$k8s.io/kube-openapi$$] | v0.0.0-20220328201542-3ee0da9b0b42 | Apache-2.0

go.mod

@@ -31,8 +31,9 @@ require (
 	github.com/spf13/viper v1.11.0
 	github.com/stretchr/testify v1.7.1
 	github.com/tsenart/vegeta v12.7.0+incompatible
-	go.elastic.co/apm v1.15.0
-	go.elastic.co/apm/module/apmelasticsearch v1.15.0
+	go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0
+	go.elastic.co/apm/module/apmhttp/v2 v2.0.0
+	go.elastic.co/apm/v2 v2.0.0
 	go.uber.org/automaxprocs v1.5.1
 	go.uber.org/zap v1.21.0
 	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4
@@ -66,9 +67,9 @@ require (
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/docker v20.10.16+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.6.4 // indirect
-	github.com/elastic/go-licenser v0.3.1 // indirect
-	github.com/elastic/go-sysinfo v1.1.1 // indirect
-	github.com/elastic/go-windows v1.0.0 // indirect
+	github.com/elastic/go-licenser v0.4.0 // indirect
+	github.com/elastic/go-sysinfo v1.7.1 // indirect
+	github.com/elastic/go-windows v1.0.1 // indirect
 	github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 // indirect
 	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
@@ -140,11 +141,9 @@ require (
 	github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/vbatts/tar-split v0.11.2 // indirect
-	go.elastic.co/apm/module/apmhttp v1.15.0 // indirect
 	go.elastic.co/fastjson v1.1.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
 	golang.org/x/net v0.0.0-20220516155154-20f960328961 // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
@@ -162,7 +161,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
-	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
+	howett.net/plist v1.0.0 // indirect
 	k8s.io/apiextensions-apiserver v0.24.0 // indirect
 	k8s.io/component-base v0.24.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect