CP/DP split: update/delete user secrets

Problem: When a user updates or deletes their docker registry or NGINX Plus secrets, those changes need to be propagated to all duplicate secrets that we've provisioned for the Gateway resources.

Solution: If updated, update the provisioned secret. If deleted, delete the provisioned secret.

internal/framework/controller/predicate/annotation.go

@@ -19,24 +19,24 @@ type AnnotationPredicate struct {
 }
 
 // Create filters CreateEvents based on the Annotation.
-func (cp AnnotationPredicate) Create(e event.CreateEvent) bool {
+func (ap AnnotationPredicate) Create(e event.CreateEvent) bool {
 	if e.Object == nil {
 		return false
 	}
 
-	_, ok := e.Object.GetAnnotations()[cp.Annotation]
+	_, ok := e.Object.GetAnnotations()[ap.Annotation]
 	return ok
 }
 
 // Update filters UpdateEvents based on the Annotation.
-func (cp AnnotationPredicate) Update(e event.UpdateEvent) bool {
+func (ap AnnotationPredicate) Update(e event.UpdateEvent) bool {
 	if e.ObjectOld == nil || e.ObjectNew == nil {
 		// this case should not happen
 		return false
 	}
 
-	oldAnnotationVal := e.ObjectOld.GetAnnotations()[cp.Annotation]
-	newAnnotationVal := e.ObjectNew.GetAnnotations()[cp.Annotation]
+	oldAnnotationVal := e.ObjectOld.GetAnnotations()[ap.Annotation]
+	newAnnotationVal := e.ObjectNew.GetAnnotations()[ap.Annotation]
 
 	return oldAnnotationVal != newAnnotationVal
 }
@@ -52,7 +52,7 @@ type RestartDeploymentAnnotationPredicate struct {
 }
 
 // Update filters UpdateEvents based on if the annotation is present or changed.
-func (cp RestartDeploymentAnnotationPredicate) Update(e event.UpdateEvent) bool {
+func (RestartDeploymentAnnotationPredicate) Update(e event.UpdateEvent) bool {
 	if e.ObjectOld == nil || e.ObjectNew == nil {
 		// this case should not happen
 		return false

internal/framework/controller/predicate/secret.go

@@ -0,0 +1,77 @@
+package predicate
+
+import (
+	"slices"
+
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+// SecretNamePredicate implements a predicate function that returns true if the Secret matches the expected
+// namespace and one of the expected names.
+type SecretNamePredicate struct {
+	predicate.Funcs
+	Namespace   string
+	SecretNames []string
+}
+
+// Create filters CreateEvents based on the Secret name.
+func (sp SecretNamePredicate) Create(e event.CreateEvent) bool {
+	if e.Object == nil {
+		return false
+	}
+
+	if secret, ok := e.Object.(*corev1.Secret); ok {
+		return secretMatches(secret, sp.Namespace, sp.SecretNames)
+	}
+
+	return false
+}
+
+// Update filters UpdateEvents based on the Secret name.
+func (sp SecretNamePredicate) Update(e event.UpdateEvent) bool {
+	if e.ObjectNew == nil {
+		return false
+	}
+
+	if secret, ok := e.ObjectNew.(*corev1.Secret); ok {
+		return secretMatches(secret, sp.Namespace, sp.SecretNames)
+	}
+
+	return false
+}
+
+// Delete filters DeleteEvents based on the Secret name.
+func (sp SecretNamePredicate) Delete(e event.DeleteEvent) bool {
+	if e.Object == nil {
+		return false
+	}
+
+	if secret, ok := e.Object.(*corev1.Secret); ok {
+		return secretMatches(secret, sp.Namespace, sp.SecretNames)
+	}
+
+	return false
+}
+
+// Generic filters GenericEvents based on the Secret name.
+func (sp SecretNamePredicate) Generic(e event.GenericEvent) bool {
+	if e.Object == nil {
+		return false
+	}
+
+	if secret, ok := e.Object.(*corev1.Secret); ok {
+		return secretMatches(secret, sp.Namespace, sp.SecretNames)
+	}
+
+	return false
+}
+
+func secretMatches(secret *corev1.Secret, namespace string, names []string) bool {
+	if secret.GetNamespace() != namespace {
+		return false
+	}
+
+	return slices.Contains(names, secret.GetName())
+}

internal/framework/controller/predicate/secret_test.go

@@ -0,0 +1,194 @@
+package predicate
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+func TestSecretNamePredicate(t *testing.T) {
+	t.Parallel()
+
+	pred := SecretNamePredicate{
+		Namespace:   "test-namespace",
+		SecretNames: []string{"secret1", "secret2"},
+	}
+
+	tests := []struct {
+		createEvent  *event.CreateEvent
+		updateEvent  *event.UpdateEvent
+		deleteEvent  *event.DeleteEvent
+		genericEvent *event.GenericEvent
+		name         string
+		expUpdate    bool
+	}{
+		{
+			name: "Create event with matching secret",
+			createEvent: &event.CreateEvent{
+				Object: &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "secret1",
+						Namespace: "test-namespace",
+					},
+				},
+			},
+			expUpdate: true,
+		},
+		{
+			name: "Create event with non-matching secret",
+			createEvent: &event.CreateEvent{
+				Object: &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "secret3",
+						Namespace: "test-namespace",
+					},
+				},
+			},
+			expUpdate: false,
+		},
+		{
+			name: "Create event with non-matching namespace",
+			createEvent: &event.CreateEvent{
+				Object: &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "secret1",
+						Namespace: "other-namespace",
+					},
+				},
+			},
+			expUpdate: false,
+		},
+		{
+			name: "Update event with matching secret",
+			updateEvent: &event.UpdateEvent{
+				ObjectNew: &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "secret2",
+						Namespace: "test-namespace",
+					},
+				},
+			},
+			expUpdate: true,
+		},
+		{
+			name: "Update event with non-matching secret",
+			updateEvent: &event.UpdateEvent{
+				ObjectNew: &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "secret3",
+						Namespace: "test-namespace",
+					},
+				},
+			},
+			expUpdate: false,
+		},
+		{
+			name: "Update event with non-matching namespace",
+			updateEvent: &event.UpdateEvent{
+				ObjectNew: &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "secret1",
+						Namespace: "other-namespace",
+					},
+				},
+			},
+			expUpdate: false,
+		},
+		{
+			name: "Delete event with matching secret",
+			deleteEvent: &event.DeleteEvent{
+				Object: &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "secret1",
+						Namespace: "test-namespace",
+					},
+				},
+			},
+			expUpdate: true,
+		},
+		{
+			name: "Delete event with non-matching secret",
+			deleteEvent: &event.DeleteEvent{
+				Object: &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "secret3",
+						Namespace: "test-namespace",
+					},
+				},
+			},
+			expUpdate: false,
+		},
+		{
+			name: "Delete event with non-matching namespace",
+			deleteEvent: &event.DeleteEvent{
+				Object: &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "secret1",
+						Namespace: "other-namespace",
+					},
+				},
+			},
+			expUpdate: false,
+		},
+		{
+			name: "Generic event with non-matching secret",
+			genericEvent: &event.GenericEvent{
+				Object: &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "secret3",
+						Namespace: "test-namespace",
+					},
+				},
+			},
+			expUpdate: false,
+		},
+		{
+			name: "Generic event with non-matching secret",
+			genericEvent: &event.GenericEvent{
+				Object: &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "secret3",
+						Namespace: "test-namespace",
+					},
+				},
+			},
+			expUpdate: false,
+		},
+		{
+			name: "Generic event with non-matching namespace",
+			genericEvent: &event.GenericEvent{
+				Object: &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "secret1",
+						Namespace: "other-namespace",
+					},
+				},
+			},
+			expUpdate: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			g := NewWithT(t)
+
+			var result bool
+			switch {
+			case test.createEvent != nil:
+				result = pred.Create(*test.createEvent)
+			case test.updateEvent != nil:
+				result = pred.Update(*test.updateEvent)
+			case test.deleteEvent != nil:
+				result = pred.Delete(*test.deleteEvent)
+			default:
+				result = pred.Generic(*test.genericEvent)
+			}
+
+			g.Expect(test.expUpdate).To(Equal(result))
+		})
+	}
+}

internal/framework/controller/resource.go

@@ -1,9 +1,22 @@
 package controller
 
-import "fmt"
+import (
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
 
 // CreateNginxResourceName creates the base resource name for all nginx resources
 // created by the control plane.
 func CreateNginxResourceName(prefix, suffix string) string {
 	return fmt.Sprintf("%s-%s", prefix, suffix)
 }
+
+// ObjectMetaToNamespacedName converts ObjectMeta to NamespacedName.
+func ObjectMetaToNamespacedName(meta metav1.ObjectMeta) types.NamespacedName {
+	return types.NamespacedName{
+		Namespace: meta.Namespace,
+		Name:      meta.Name,
+	}
+}

internal/mode/static/provisioner/eventloop.go

@@ -18,6 +18,7 @@ import (
 	"github.com/nginx/nginx-gateway-fabric/internal/framework/controller/predicate"
 	"github.com/nginx/nginx-gateway-fabric/internal/framework/events"
 	ngftypes "github.com/nginx/nginx-gateway-fabric/internal/framework/types"
+	"github.com/nginx/nginx-gateway-fabric/internal/mode/static/config"
 )
 
 func newEventLoop(
@@ -26,11 +27,30 @@ func newEventLoop(
 	handler *eventHandler,
 	logger logr.Logger,
 	selector metav1.LabelSelector,
+	ngfNamespace string,
+	dockerSecrets []string,
+	usageConfig *config.UsageReportConfig,
 ) (*events.EventLoop, error) {
 	nginxResourceLabelPredicate := predicate.NginxLabelPredicate(selector)
 
+	secretsToWatch := make([]string, 0, len(dockerSecrets)+3)
+	secretsToWatch = append(secretsToWatch, dockerSecrets...)
+
+	if usageConfig != nil {
+		if usageConfig.SecretName != "" {
+			secretsToWatch = append(secretsToWatch, usageConfig.SecretName)
+		}
+		if usageConfig.CASecretName != "" {
+			secretsToWatch = append(secretsToWatch, usageConfig.CASecretName)
+		}
+		if usageConfig.ClientSSLSecretName != "" {
+			secretsToWatch = append(secretsToWatch, usageConfig.ClientSSLSecretName)
+		}
+	}
+
 	controllerRegCfgs := []struct {
 		objectType ngftypes.ObjectType
+		name       string
 		options    []controller.Option
 	}{
 		{
@@ -85,15 +105,18 @@ func newEventLoop(
 			options: []controller.Option{
 				controller.WithK8sPredicate(
 					k8spredicate.And(
-						k8spredicate.GenerationChangedPredicate{},
-						nginxResourceLabelPredicate,
+						k8spredicate.ResourceVersionChangedPredicate{},
+						k8spredicate.Or(
+							nginxResourceLabelPredicate,
+							predicate.SecretNamePredicate{Namespace: ngfNamespace, SecretNames: secretsToWatch},
+						),
 					),
 				),
 			},
 		},
 	}
 
-	eventCh := make(chan interface{})
+	eventCh := make(chan any)
 	for _, regCfg := range controllerRegCfgs {
 		gvk, err := apiutil.GVKForObject(regCfg.objectType, mgr.GetScheme())
 		if err != nil {