Remove the "operator" init container

- Remove the "operator" init container. Now there is a proper operator
  deployment, so we don't need to create CRDs in the init container.
- Update the Tetragon daemonset ClusterRole accordingly.
- Modify the Tetragon daemonset initialization logic to wait for all the
  required CRDs to show up before proceeding.

Signed-off-by: Michi Mutsuzaki <michi@isovalent.com>

cmd/tetragon/flags.go

@@ -80,6 +80,8 @@ const (
 	keyEnableMsgHandlingLatency = "enable-msg-handling-latency"
 
 	keyKmods = "kmods"
+
+	keyEnablePodInfo = "enable-pod-info"
 )
 
 func readAndSetFlags() {
@@ -144,6 +146,8 @@ func readAndSetFlags() {
 
 	option.Config.KMods = viper.GetStringSlice(keyKmods)
 
+	option.Config.EnablePodInfo = viper.GetBool(keyEnablePodInfo)
+
 	if viper.IsSet(keyTracingPolicy) {
 		option.Config.TracingPolicy = viper.GetString(keyTracingPolicy)
 	}

cmd/tetragon/main.go

@@ -53,13 +53,19 @@ import (
 	_ "github.com/cilium/tetragon/pkg/sensors"
 
 	"github.com/cilium/lumberjack/v2"
+	"github.com/cilium/tetragon/pkg/k8s/apis/cilium.io/v1alpha1"
 	gops "github.com/google/gops/agent"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"golang.org/x/exp/maps"
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/types/known/durationpb"
+	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	apiextensionsinformer "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
 )
 
 var (
@@ -310,16 +316,64 @@ func tetragonExecute() error {
 	// Probe runtime configuration and do not fail on errors
 	obs.UpdateRuntimeConf(option.Config.MapDir)
 
-	watcher, err := getWatcher()
-	if err != nil {
-		return err
-	}
-	_, err = cilium.InitCiliumState(ctx, option.Config.EnableCilium)
+	var k8sWatcher watcher.K8sResourceWatcher
+	if option.Config.EnableK8s {
+		log.Info("Enabling Kubernetes API")
+		crds := map[string]struct{}{
+			v1alpha1.TPName:           {},
+			v1alpha1.TPNamespacedName: {},
+		}
+		if option.Config.EnablePodInfo {
+			crds[v1alpha1.PIName] = struct{}{}
+		}
+		config, err := k8sconf.K8sConfig()
+		if err != nil {
+			return err
+		}
+		log.WithField("crds", maps.Keys(crds)).Info("Waiting for required CRDs")
+		var wg sync.WaitGroup
+		wg.Add(1)
+		k8sClient := kubernetes.NewForConfigOrDie(config)
+		crdClient := apiextensionsclientset.NewForConfigOrDie(config)
+		crdInformer := apiextensionsinformer.NewCustomResourceDefinitionInformer(crdClient, 0*time.Second, nil)
+		_, err = crdInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				crdObject, ok := obj.(*v1.CustomResourceDefinition)
+				if !ok {
+					log.WithField("obj", obj).Warn("Received an invalid object")
+					return
+				}
+				if _, ok := crds[crdObject.Name]; ok {
+					log.WithField("crd", crdObject.Name).Info("Found CRD")
+					delete(crds, crdObject.Name)
+				}
+				if len(crds) == 0 {
+					log.Info("Found all the required CRDs")
+					wg.Done()
+				}
+			},
+		})
+		if err != nil {
+			log.WithError(err).Error("failed to add event handler")
+			return err
+		}
+		stop := make(chan struct{})
+		go func() {
+			crdInformer.Run(stop)
+		}()
+		wg.Wait()
+		close(stop)
+		k8sWatcher = watcher.NewK8sWatcher(k8sClient, 60*time.Second)
+	} else {
+		log.Info("Disabling Kubernetes API")
+		k8sWatcher = watcher.NewFakeK8sWatcher(nil)
+	}
+	_, err := cilium.InitCiliumState(ctx, option.Config.EnableCilium)
 	if err != nil {
 		return err
 	}
 
-	if err := process.InitCache(watcher, option.Config.ProcessCacheSize); err != nil {
+	if err := process.InitCache(k8sWatcher, option.Config.ProcessCacheSize); err != nil {
 		return err
 	}
 
@@ -338,7 +392,7 @@ func tetragonExecute() error {
 	ctx, cancel2 := context.WithCancel(ctx)
 	defer cancel2()
 
-	hookRunner := rthooks.GlobalRunner().WithWatcher(watcher)
+	hookRunner := rthooks.GlobalRunner().WithWatcher(k8sWatcher)
 
 	pm, err := tetragonGrpc.NewProcessManager(
 		ctx,
@@ -631,21 +685,6 @@ func Serve(ctx context.Context, listenAddr string, srv *server.Server) error {
 	return nil
 }
 
-func getWatcher() (watcher.K8sResourceWatcher, error) {
-	if option.Config.EnableK8s {
-		log.Info("Enabling Kubernetes API")
-		config, err := k8sconf.K8sConfig()
-		if err != nil {
-			return nil, err
-		}
-		k8sClient := kubernetes.NewForConfigOrDie(config)
-		return watcher.NewK8sWatcher(k8sClient, 60*time.Second), nil
-
-	}
-	log.Info("Disabling Kubernetes API")
-	return watcher.NewFakeK8sWatcher(nil), nil
-}
-
 func startGopsServer() error {
 	// Empty means no gops
 	if option.Config.GopsAddr == "" {
@@ -773,6 +812,8 @@ func execute() error {
 
 	flags.Int(keyRBQueueSize, 65535, "Set size of channel between ring buffer and sensor go routines (default 65k)")
 
+	flags.Bool(keyEnablePodInfo, false, "Enable PodInfo custom resource")
+
 	viper.BindPFlags(flags)
 	return rootCmd.Execute()
 }

go.mod

@@ -38,6 +38,7 @@ require (
 	github.com/vishvananda/netlink v1.2.1-beta.2.0.20230807190133-6afddb37c1f0
 	go.uber.org/atomic v1.11.0
 	go.uber.org/multierr v1.11.0
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 	golang.org/x/sync v0.4.0
 	golang.org/x/sys v0.13.0
 	golang.org/x/time v0.3.0
@@ -167,7 +168,6 @@ require (
 	go.opentelemetry.io/otel/trace v1.17.0 // indirect
 	go.uber.org/dig v1.17.0 // indirect
 	go.uber.org/zap v1.25.0 // indirect
-	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/oauth2 v0.12.0 // indirect

install/kubernetes/templates/_container_tetragon.tpl

@@ -80,14 +80,3 @@
 {{- end -}}
 {{- end -}}
 
-{{- define "container.tetragon.init-operator" -}}
-- name: {{ include "container.tetragon.name" . }}-operator
-  image: "{{ if .Values.tetragonOperator.image.override }}{{ .Values.tetragonOperator.image.override }}{{ else }}{{ .Values.tetragonOperator.image.repository }}{{ .Values.tetragonOperator.image.suffix }}:{{ .Values.tetragonOperator.image.tag }}{{ end }}"
-  imagePullPolicy: {{ .Values.imagePullPolicy }}
-  args:
-    - --config-dir=/etc/tetragon/operator.conf.d/
-  volumeMounts:
-    - mountPath: /etc/tetragon/operator.conf.d/
-      name: tetragon-operator-config
-      readOnly: true
-{{- end -}}

install/kubernetes/templates/clusterrole.yaml

@@ -31,23 +31,7 @@ rules:
     resources:
       - customresourcedefinitions
     verbs:
-      - create
-  - apiGroups:
-      - apiextensions.k8s.io
-    resources:
-      - customresourcedefinitions
-    verbs:
-      - create
-  - apiGroups:
-      - apiextensions.k8s.io
-    resources:
-      - customresourcedefinitions
-    resourceNames:
-      - tracingpolicies.cilium.io
-      - tracingpoliciesnamespaced.cilium.io
-      - podinfo.cilium.io
-    verbs:
-      - update
       - get
       - list
+      - watch
 {{- end }}

install/kubernetes/templates/daemonset.yaml

@@ -44,10 +44,6 @@ spec:
       securityContext:
         {{- toYaml . | nindent 6 }}
       {{- end }}
-{{- if .Values.tetragon.enabled }}
-      initContainers:
-      {{- include "container.tetragon.init-operator" . | nindent 6 -}}
-{{- end }}
       containers:
 {{- if eq .Values.export.mode "stdout" }}
       {{- include "container.export.stdout" . | nindent 6 -}}
@@ -96,9 +92,6 @@ spec:
         name: metadata-files
 {{- end }}
 {{- end }}
-      - name: tetragon-operator-config
-        configMap:
-          name: {{ .Release.Name }}-operator-config
       {{- with .Values.extraVolumes }}
         {{- toYaml . | nindent 6 }}
       {{- end }}

install/kubernetes/templates/tetragon_configmap.yaml

@@ -54,3 +54,4 @@ data:
 {{- if .Values.tetragon.enableMsgHandlingLatency }}
   enable-msg-handling-latency: "true"
 {{- end }}
+  enable-pod-info: {{ .Values.tetragonOperator.podInfo.enabled | quote }}

pkg/option/config.go

@@ -81,6 +81,8 @@ type config struct {
 	EnableMsgHandlingLatency bool
 
 	KMods []string
+
+	EnablePodInfo bool
 }
 
 var (