feat: improve Operator performance by using caching to reduce api call

 and network impact
(work based on opendatahub-io#1189)
- secret: default application namespace + other default ones + istio cert
- configmap: all
- namespace: all
- ingressctrler: "default" one
- deployment: default application namespaces + default namespaces

Signed-off-by: Wen Zhou <wenzhou@redhat.com>

bundle/manifests/opendatahub-operator-webhook-service_v1_service.yaml

@@ -3,14 +3,11 @@ kind: Service
 metadata:
   annotations:
     service.beta.openshift.io/inject-cabundle: "true"
-    service.beta.openshift.io/serving-cert-secret-name: opendatahub-operator-controller-webhook-cert
+    service.beta.openshift.io/serving-cert-secret-name: opendatahub-operator-webhook-cert
   creationTimestamp: null
   labels:
-    app.kubernetes.io/component: webhook
     app.kubernetes.io/created-by: opendatahub-operator
-    app.kubernetes.io/instance: webhook-service
     app.kubernetes.io/managed-by: kustomize
-    app.kubernetes.io/name: service
     app.kubernetes.io/part-of: opendatahub-operator
   name: opendatahub-operator-webhook-service
 spec:

bundle/manifests/opendatahub-operator.clusterserviceversion.yaml

@@ -1176,7 +1176,7 @@ spec:
               - name: cert
                 secret:
                   defaultMode: 420
-                  secretName: opendatahub-operator-controller-webhook-cert
+                  secretName: opendatahub-operator-controller-manager-service-cert
     strategy: deployment
   installModes:
   - supported: false

config/default/manager_webhook_patch.yaml

@@ -20,4 +20,4 @@ spec:
       - name: cert
         secret:
           defaultMode: 420
-          secretName: opendatahub-operator-controller-webhook-cert
+          secretName: opendatahub-operator-webhook-service

config/webhook/kustomization.yaml

@@ -6,4 +6,4 @@ commonAnnotations:
   service.beta.openshift.io/inject-cabundle: "true"
 
 configurations:
-- kustomizeconfig.yaml
+- kustomizeconfig.yaml

config/webhook/kustomizeconfig.yaml

@@ -4,18 +4,11 @@ nameReference:
 - kind: Service
   version: v1
   fieldSpecs:
-  - kind: MutatingWebhookConfiguration
-    group: admissionregistration.k8s.io
-    path: webhooks/clientConfig/service/name
   - kind: ValidatingWebhookConfiguration
     group: admissionregistration.k8s.io
     path: webhooks/clientConfig/service/name
 
 namespace:
-- kind: MutatingWebhookConfiguration
-  group: admissionregistration.k8s.io
-  path: webhooks/clientConfig/service/namespace
-  create: true
 - kind: ValidatingWebhookConfiguration
   group: admissionregistration.k8s.io
   path: webhooks/clientConfig/service/namespace

config/webhook/service.yaml

@@ -1,18 +1,13 @@
-
 apiVersion: v1
 kind: Service
 metadata:
   labels:
-    app.kubernetes.io/name: service
-    app.kubernetes.io/instance: webhook-service
-    app.kubernetes.io/component: webhook
     app.kubernetes.io/created-by: opendatahub-operator
     app.kubernetes.io/part-of: opendatahub-operator
     app.kubernetes.io/managed-by: kustomize
   name: webhook-service
-  namespace: system
   annotations:
-    service.beta.openshift.io/serving-cert-secret-name: opendatahub-operator-controller-webhook-cert
+    service.beta.openshift.io/serving-cert-secret-name: opendatahub-operator-webhook-cert
 spec:
   ports:
     - port: 443

controllers/webhook/webhook_suite_test.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -127,6 +128,7 @@ var _ = BeforeSuite(func() {
 			Host:    webhookInstallOptions.LocalServingHost,
 			CertDir: webhookInstallOptions.LocalServingCertDir,
 		}),
+		Cache: cache.Options{Scheme: scheme},
 	})
 	Expect(err).NotTo(HaveOccurred())
 

main.go

@@ -38,12 +38,14 @@ import (
 	networkingv1 "k8s.io/api/networking/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -98,7 +100,7 @@ func init() { //nolint:gochecknoinits
 	utilruntime.Must(operatorv1.Install(scheme))
 }
 
-func main() { //nolint:funlen
+func main() { //nolint:funlen,maintidx
 	var metricsAddr string
 	var enableLeaderElection bool
 	var probeAddr string
@@ -126,14 +128,58 @@ func main() { //nolint:funlen
 	// root context
 	ctx := ctrl.SetupSignalHandler()
 
+	cacheOptions := cache.Options{
+		// opendatahub.io/generated-namespace: 'true'
+		Scheme: scheme,
+		ByObject: map[client.Object]cache.ByObject{
+			// all CRD: mainly for pipeline v1 teckon and v2 argo and dashboard's own CRD
+			&apiextensionsv1.CustomResourceDefinition{}: {},
+			// Cannot find a label on various screts, so we need to watch all secrets
+			// this include, monitoring, dashboard, trustcabundle default cert etc for these NS
+			&corev1.Secret{}: {
+				Namespaces: map[string]cache.Config{
+					"redhat-ods-monitoring":   {},
+					"redhat-ods-applications": {},
+					"opendatahub":             {},
+					"istio-system":            {},
+					"openshift-ingress":       {},
+				},
+			},
+			// it is hard to find a label can be used for both trustCAbundle configmap and inferenceservice-config
+			&corev1.ConfigMap{}: {},
+			// TODO: we can limit scope of namespace if we find a way to only get list of DSproject
+			// also need for monitoring, trustcabundle
+			&corev1.Namespace{}: {},
+			// For catsrc (avoid frequently check cluster type)
+			&ofapiv1alpha1.CatalogSource{}: {
+				Field: fields.Set{"metadata.name": "addon-managed-odh-catalog"}.AsSelector(),
+			},
+			// For domain to get OpenshiftIngress and default cert
+			&operatorv1.IngressController{}: {
+				Field: fields.Set{"metadata.name": "default"}.AsSelector(),
+			},
+			// for prometheus and black-box deployment and ones we owns
+			&appsv1.Deployment{}: {
+				Namespaces: map[string]cache.Config{
+					"redhat-ods-monitoring":   {},
+					"redhat-ods-applications": {},
+					"odh-model-registries":    {},
+					"rhods-notebooks":         {},
+					"opendatahub":             {},
+				},
+			},
+		},
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ // single pod does not need to have LeaderElection
 		Scheme:  scheme,
 		Metrics: ctrlmetrics.Options{BindAddress: metricsAddr},
 		WebhookServer: ctrlwebhook.NewServer(ctrlwebhook.Options{
 			Port: 9443,
-			// TLSOpts: , // TODO: do we need tls for webhook
+			// TLSOpts: , // TODO: it was not set in the old code
 		}),
 		HealthProbeBindAddress: probeAddr,
+		Cache:                  cacheOptions,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "07ed84f7.opendatahub.io",
 		// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily