TEP-0124 implement opentelemetry and jaeger tracing

Adds opentelemetry instrumentation code to pipelinerun and taskrun
reconcilers. Also made required changes in the main.go to include jaeger
as tracing backend

cmd/controller/main.go

@@ -17,10 +17,12 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"flag"
 	"log"
 	"net/http"
 	"os"
+	"time"
 
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
@@ -36,11 +38,23 @@ import (
 	"knative.dev/pkg/injection"
 	"knative.dev/pkg/injection/sharedmain"
 	"knative.dev/pkg/signals"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/jaeger"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	tracesdk "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
+	"go.opentelemetry.io/otel/trace"
 )
 
 const (
 	// ControllerLogKey is the name of the logger for the controller cmd
 	ControllerLogKey = "tekton-pipelines-controller"
+	// TraceProvider name for pipeline reconciler
+	TracerProviderPipelineRun = "pipeline-reconciler"
+	// TraceProvider name for taskrun reconciler
+	TracerProviderTaskRun = "taskrun-reconciler"
 )
 
 func main() {
@@ -102,17 +116,80 @@ func main() {
 		log.Fatal(http.ListenAndServe(":"+port, mux))
 	}()
 
+	// initialize opentelemetry
+	tpPipelineRun, err := tracerProvider(TracerProviderPipelineRun)
+	if err != nil {
+		log.Printf("failed to initialize tracerProvider for pipelinerun, falling back to no-op provider, %s", err.Error())
+		tpPipelineRun = trace.NewNoopTracerProvider()
+	}
+	tpTaskrun, err := tracerProvider(TracerProviderTaskRun)
+	if err != nil {
+		log.Printf("failed to initialize tracerProvider for taskrun, falling back to no-op provider, %s", err.Error())
+		tpTaskrun = trace.NewNoopTracerProvider()
+	}
+	otel.SetTextMapPropagator(propagation.TraceContext{})
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
 	ctx = filteredinformerfactory.WithSelectors(ctx, v1beta1.ManagedByLabelKey)
 	sharedmain.MainWithConfig(ctx, ControllerLogKey, cfg,
-		taskrun.NewController(opts, clock.RealClock{}),
-		pipelinerun.NewController(opts, clock.RealClock{}),
+		taskrun.NewController(opts, clock.RealClock{}, tpTaskrun),
+		pipelinerun.NewController(opts, clock.RealClock{}, tpPipelineRun),
 		run.NewController(),
 		resolutionrequest.NewController(clock.RealClock{}),
 		// TODO(jerop, abayer) uncomment after we align on retries in customruns
 		// customrun.NewController(),
 	)
+
+	// Cleanly shutdown and flush telemetry when the application exits.
+	defer func(ctx context.Context) {
+		// Do not make the application hang when it is shutdown.
+		ctx, cancel = context.WithTimeout(ctx, time.Second*5)
+		defer cancel()
+
+		// shutdown is only needed when tracerProvider is inialized with jaeger
+		// not needed when tracerProvider is NewNoopTracerProvider
+		if tp, ok := tpPipelineRun.(*tracesdk.TracerProvider); ok {
+			tp.Shutdown(ctx)
+		}
+		if tp, ok := tpTaskrun.(*tracesdk.TracerProvider); ok {
+			tp.Shutdown(ctx)
+		}
+	}(ctx)
 }
 
 func handler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 }
+
+// tracerProvider returns an OpenTelemetry TracerProvider configured to use
+// the Jaeger exporter that will send spans to the provided url. The returned
+// TracerProvider will also use a Resource configured with all the information
+// about the application.
+func tracerProvider(service string) (trace.TracerProvider, error) {
+	// Create the Jaeger exporter
+	// The following env variables are used by the sdk for creating the exporter
+	// - OTEL_EXPORTER_JAEGER_ENDPOINT is the HTTP endpoint for sending spans directly to a collector.
+	// - OTEL_EXPORTER_JAEGER_USER is the username to be sent as authentication to the collector endpoint.
+	// - OTEL_EXPORTER_JAEGER_PASSWORD is the password to be sent as authentication to the collector endpoint.
+
+	if _, e := os.LookupEnv("OTEL_EXPORTER_JAEGER_ENDPOINT"); !e {
+		// jaeger endpoint is not defined, disable tracing and return no-op tracerProvider
+		return trace.NewNoopTracerProvider(), nil
+	}
+
+	exp, err := jaeger.New(jaeger.WithCollectorEndpoint())
+	if err != nil {
+		return nil, err
+	}
+	// Initialize tracerProvider with the jaeger exporter
+	tp := tracesdk.NewTracerProvider(
+		tracesdk.WithBatcher(exp),
+		// Record information about the service in a Resource.
+		tracesdk.WithResource(resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceNameKey.String(service),
+		)),
+	)
+	return tp, nil
+}

config/controller.yaml

@@ -124,6 +124,13 @@ spec:
           value: /etc/ssl/certs
         - name: METRICS_DOMAIN
           value: tekton.dev/pipeline
+       # The following variables can be uncommented with correct values to enable Jaeger tracing
+       #- name: OTEL_EXPORTER_JAEGER_ENDPOINT
+       #  value: http://jaeger-collector.jaeger:14268/api/traces
+       #- name: OTEL_EXPORTER_JAEGER_USER
+       #  value: username
+       #- name: OTEL_EXPORTER_JAEGER_PASSWORD
+       #  value: password
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:

docs/developers/README.md

@@ -11,6 +11,7 @@ channel for training and tutorials on Tekton!
 - Developing on Tekton:
   - [Local Setup](./local-setup.md): Getting your local environment set up to develop on Tekton.
   - [Testing](../../test/README.md): Running Tekton tests.
+  - [Tracing](./tracing.md): Enabling Jaeger tracing
 - How Tekton is run on Kubernetes:
   - [Controller Logic](./controller-logic.md): How Tekton extends Kubernetes using Knative.
   - [TaskRun Logic](./taskruns.md): How TaskRuns are run in pods.

docs/developers/tracing.md

@@ -0,0 +1,35 @@
+# Tracing setup
+
+This sections shows how to enable tracing for tekton reconcilers and
+capture traces in Jaeger
+
+## Prerequisites
+
+Jaeger should be installed and accessible from the cluster. The easiest
+way to set it up is using helm as below
+
+the following command installs Jaeger in `jaeger` namespace
+
+```
+helm repo add jaegertracing https://jaegertracing.github.io/helm-charts
+helm upgrade -i jaeger jaegertracing/jaeger -n jaeger --create-namespace
+```
+
+Use port-forwarding to open the jaeger query UI or adjust the service
+type to Loadbalancer for accessing the service directly
+
+```
+kubectl port-forward svc/jaeger-query -n jaeger 8080:80
+```
+
+Check the official [Jaeger docs](https://www.jaegertracing.io/docs/) on how to work with Jaeger
+
+## Enabling tracing
+
+Tekton pipelines controller expects the following environment variables to be able to connect to jaeger:
+
+* `OTEL_EXPORTER_JAEGER_ENDPOINT` is the HTTP endpoint for sending spans directly to a collector.
+* `OTEL_EXPORTER_JAEGER_USER` is the username to be sent as authentication to the collector endpoint.
+* `OTEL_EXPORTER_JAEGER_PASSWORD` is the password to be sent as authentication to the collector endpoint.
+
+`OTEL_EXPORTER_JAEGER_ENDPOINT` is the only manadatory variable to enable tracing. You can find these variables in the controller manifest as well.

go.mod

@@ -45,6 +45,10 @@ require (
 	github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20221030203717-1711cefd7eec
 	github.com/letsencrypt/boulder v0.0.0-20220929215747-76583552c2be
 	github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399
+	go.opentelemetry.io/otel v1.11.1
+	go.opentelemetry.io/otel/exporters/jaeger v1.11.1
+	go.opentelemetry.io/otel/sdk v1.11.1
+	go.opentelemetry.io/otel/trace v1.11.1
 	k8s.io/utils v0.0.0-20221012122500-cfd413dd9e85
 )
 
@@ -64,6 +68,7 @@ require (
 require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/google/gnostic v0.6.9 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect

pkg/reconciler/pipelinerun/controller.go

@@ -34,6 +34,7 @@ import (
 	cloudeventclient "github.com/tektoncd/pipeline/pkg/reconciler/events/cloudevent"
 	"github.com/tektoncd/pipeline/pkg/reconciler/volumeclaim"
 	resolution "github.com/tektoncd/pipeline/pkg/resolution/resource"
+	"go.opentelemetry.io/otel/trace"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/utils/clock"
 	kubeclient "knative.dev/pkg/client/injection/kube/client"
@@ -43,7 +44,7 @@ import (
 )
 
 // NewController instantiates a new controller.Impl from knative.dev/pkg/controller
-func NewController(opts *pipeline.Options, clock clock.PassiveClock) func(context.Context, configmap.Watcher) *controller.Impl {
+func NewController(opts *pipeline.Options, clock clock.PassiveClock, tracerProvider trace.TracerProvider) func(context.Context, configmap.Watcher) *controller.Impl {
 	return func(ctx context.Context, cmw configmap.Watcher) *controller.Impl {
 		logger := logging.FromContext(ctx)
 		kubeclientset := kubeclient.Get(ctx)
@@ -69,6 +70,7 @@ func NewController(opts *pipeline.Options, clock clock.PassiveClock) func(contex
 			metrics:             pipelinerunmetrics.Get(ctx),
 			pvcHandler:          volumeclaim.NewPVCHandler(kubeclientset, logger),
 			resolutionRequester: resolution.NewCRDRequester(resolutionclient.Get(ctx), resolutionInformer.Lister()),
+			tracerProvider:      tracerProvider,
 		}
 		impl := pipelinerunreconciler.NewImpl(ctx, c, func(impl *controller.Impl) controller.Options {
 			return controller.Options{