feat(diag): expose fallback configuration diagnostics (#6159)

Expose fallback configuration generation diagnostics data via the `GET /debug/config/fallback` endpoint.

internal/dataplane/fallback/fallback.go

@@ -31,52 +31,52 @@ func NewGenerator(cacheGraphProvider CacheGraphProvider, logger logr.Logger) *Ge
 func (g *Generator) GenerateExcludingBrokenObjects(
 	cache store.CacheStores,
 	brokenObjects []ObjectHash,
-) (store.CacheStores, error) {
+) (store.CacheStores, GeneratedCacheMetadata, error) {
+	metadataCollector := NewGenerateCacheMetadataCollector(brokenObjects...)
+
 	graph, err := g.cacheGraphProvider.CacheToGraph(cache)
 	if err != nil {
-		return store.CacheStores{}, fmt.Errorf("failed to build cache graph: %w", err)
+		return store.CacheStores{}, GeneratedCacheMetadata{}, fmt.Errorf("failed to build cache graph: %w", err)
 	}
 
 	fallbackCache, err := cache.TakeSnapshot()
 	if err != nil {
-		return store.CacheStores{}, fmt.Errorf("failed to take cache snapshot: %w", err)
+		return store.CacheStores{}, GeneratedCacheMetadata{}, fmt.Errorf("failed to take cache snapshot: %w", err)
 	}
 
 	for _, brokenObject := range brokenObjects {
 		subgraphObjects, err := graph.SubgraphObjects(brokenObject)
 		if err != nil {
-			return store.CacheStores{}, fmt.Errorf("failed to find dependants for %s: %w", brokenObject, err)
+			return store.CacheStores{}, GeneratedCacheMetadata{}, fmt.Errorf("failed to find dependants for %s: %w", brokenObject, err)
 		}
 		for _, obj := range subgraphObjects {
 			if err := fallbackCache.Delete(obj); err != nil {
-				return store.CacheStores{}, fmt.Errorf("failed to delete %s from the cache: %w", GetObjectHash(obj), err)
+				return store.CacheStores{}, GeneratedCacheMetadata{}, fmt.Errorf("failed to delete %s from the cache: %w", GetObjectHash(obj), err)
 			}
-			g.logger.V(util.DebugLevel).Info("Excluded object from fallback cache",
-				"object_kind", obj.GetObjectKind(),
-				"object_name", obj.GetName(),
-				"object_namespace", obj.GetNamespace(),
-			)
+			metadataCollector.CollectExcluded(obj, brokenObject)
 		}
 	}
 
-	return fallbackCache, nil
+	return fallbackCache, metadataCollector.Metadata(), nil
 }
 
 func (g *Generator) GenerateBackfillingBrokenObjects(
 	currentCache store.CacheStores,
 	lastValidCacheSnapshot *store.CacheStores,
 	brokenObjects []ObjectHash,
-) (store.CacheStores, error) {
+) (store.CacheStores, GeneratedCacheMetadata, error) {
+	metadataCollector := NewGenerateCacheMetadataCollector(brokenObjects...)
+
 	// Build a graph from the current cache.
 	currentGraph, err := g.cacheGraphProvider.CacheToGraph(currentCache)
 	if err != nil {
-		return store.CacheStores{}, fmt.Errorf("failed to build current cache graph: %w", err)
+		return store.CacheStores{}, GeneratedCacheMetadata{}, fmt.Errorf("failed to build current cache graph: %w", err)
 	}
 
 	// Take a snapshot of the current cache to use as a fallback.
 	fallbackCache, err := currentCache.TakeSnapshot()
 	if err != nil {
-		return store.CacheStores{}, fmt.Errorf("failed to take current cache snapshot: %w", err)
+		return store.CacheStores{}, GeneratedCacheMetadata{}, fmt.Errorf("failed to take current cache snapshot: %w", err)
 	}
 
 	// Exclude the affected objects from the fallback cache. Also, collect all the affected objects as they will be
@@ -85,49 +85,41 @@ func (g *Generator) GenerateBackfillingBrokenObjects(
 	for _, brokenObject := range brokenObjects {
 		subgraphObjects, err := currentGraph.SubgraphObjects(brokenObject)
 		if err != nil {
-			return store.CacheStores{}, fmt.Errorf("failed to find dependants for %s: %w", brokenObject, err)
+			return store.CacheStores{}, GeneratedCacheMetadata{}, fmt.Errorf("failed to find dependants for %s: %w", brokenObject, err)
 		}
 		for _, obj := range subgraphObjects {
 			if err := fallbackCache.Delete(obj); err != nil {
-				return store.CacheStores{}, fmt.Errorf("failed to delete %s from the fallback cache: %w", GetObjectHash(obj), err)
+				return store.CacheStores{}, GeneratedCacheMetadata{}, fmt.Errorf("failed to delete %s from the fallback cache: %w", GetObjectHash(obj), err)
 			}
-			g.logger.V(util.DebugLevel).Info("Excluded object from fallback cache",
-				"object_kind", obj.GetObjectKind(),
-				"object_name", obj.GetName(),
-				"object_namespace", obj.GetNamespace(),
-			)
 			affectedObjects = append(affectedObjects, GetObjectHash(obj))
+			metadataCollector.CollectExcluded(obj, brokenObject)
 		}
 	}
 
 	if lastValidCacheSnapshot == nil {
 		g.logger.V(util.DebugLevel).Info("No previous valid cache snapshot found, skipping backfilling")
-		return fallbackCache, nil
+		return fallbackCache, metadataCollector.Metadata(), nil
 	}
 
 	// Build a graph from the last valid cache snapshot.
 	lastValidGraph, err := g.cacheGraphProvider.CacheToGraph(*lastValidCacheSnapshot)
 	if err != nil {
-		return store.CacheStores{}, fmt.Errorf("failed to build cache graph: %w", err)
+		return store.CacheStores{}, GeneratedCacheMetadata{}, fmt.Errorf("failed to build cache graph: %w", err)
 	}
 
 	// Backfill the affected objects from the last valid cache snapshot.
 	for _, affectedObject := range affectedObjects {
 		objectsToBackfill, err := lastValidGraph.SubgraphObjects(affectedObject)
 		if err != nil {
-			return store.CacheStores{}, fmt.Errorf("failed to find dependants for %s: %w", affectedObject, err)
+			return store.CacheStores{}, GeneratedCacheMetadata{}, fmt.Errorf("failed to find dependants for %s: %w", affectedObject, err)
 		}
 
 		for _, obj := range objectsToBackfill {
 			if err := fallbackCache.Add(obj); err != nil {
-				return store.CacheStores{}, fmt.Errorf("failed to add %s to the cache: %w", GetObjectHash(obj), err)
+				return store.CacheStores{}, GeneratedCacheMetadata{}, fmt.Errorf("failed to add %s to the cache: %w", GetObjectHash(obj), err)
 			}
-			g.logger.V(util.DebugLevel).Info("Backfilled object to fallback cache from previous valid cache snapshot",
-				"object_kind", obj.GetObjectKind(),
-				"object_name", obj.GetName(),
-				"object_namespace", obj.GetNamespace(),
-			)
+			metadataCollector.CollectBackfilled(obj, affectedObject)
 		}
 	}
-	return fallbackCache, nil
+	return fallbackCache, metadataCollector.Metadata(), nil
 }

internal/dataplane/fallback/fallback_meta.go

@@ -0,0 +1,75 @@
+package fallback
+
+import (
+	"github.com/samber/lo"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// GeneratedCacheMetadata contains metadata generated during the fallback process.
+type GeneratedCacheMetadata struct {
+	// BrokenObjects are objects that were reported as broken by the Kong Admin API.
+	BrokenObjects []ObjectHash
+	// ExcludedObjects are objects that were excluded from the fallback configuration as they were broken or either of their
+	// dependencies was broken.
+	ExcludedObjects []AffectedCacheObjectMetadata
+	// BackfilledObjects are objects that were backfilled from the last valid cache state as they were broken or either of
+	// their dependencies was broken.
+	BackfilledObjects []AffectedCacheObjectMetadata
+}
+
+// GeneratedCacheMetadataCollector is a collector for cache metadata generated during the fallback process.
+// It's primarily used to deduplicate the metadata and make it easier to work with.
+type GeneratedCacheMetadataCollector struct {
+	brokenObjects     []ObjectHash
+	excludedObjects   map[ObjectHash]AffectedCacheObjectMetadata
+	backfilledObjects map[ObjectHash]AffectedCacheObjectMetadata
+}
+
+// AffectedCacheObjectMetadata contains an object and a list of objects that caused it to be excluded or backfilled
+// during the fallback process.
+type AffectedCacheObjectMetadata struct {
+	Object         client.Object
+	CausingObjects []ObjectHash
+}
+
+// NewGenerateCacheMetadataCollector creates a new GeneratedCacheMetadataCollector instance.
+func NewGenerateCacheMetadataCollector(brokenObjects ...ObjectHash) *GeneratedCacheMetadataCollector {
+	return &GeneratedCacheMetadataCollector{
+		brokenObjects:     brokenObjects,
+		excludedObjects:   make(map[ObjectHash]AffectedCacheObjectMetadata),
+		backfilledObjects: make(map[ObjectHash]AffectedCacheObjectMetadata),
+	}
+}
+
+// CollectExcluded collects an excluded object (an object that was excluded from the fallback configuration as it was
+// broken or one of its dependencies was broken).
+func (m *GeneratedCacheMetadataCollector) CollectExcluded(excluded client.Object, causing ObjectHash) {
+	objHash := GetObjectHash(excluded)
+	if existingEntry, ok := m.excludedObjects[objHash]; ok {
+		existingEntry.CausingObjects = append(existingEntry.CausingObjects, causing)
+		m.excludedObjects[objHash] = existingEntry
+	} else {
+		m.excludedObjects[objHash] = AffectedCacheObjectMetadata{Object: excluded, CausingObjects: []ObjectHash{causing}}
+	}
+}
+
+// CollectBackfilled collects a backfilled object (an object that was backfilled from the last valid cache state as that or
+// one of its dependencies was broken).
+func (m *GeneratedCacheMetadataCollector) CollectBackfilled(backfilled client.Object, causing ObjectHash) {
+	objHash := GetObjectHash(backfilled)
+	if existingEntry, ok := m.backfilledObjects[objHash]; ok {
+		existingEntry.CausingObjects = append(existingEntry.CausingObjects, causing)
+		m.backfilledObjects[objHash] = existingEntry
+	} else {
+		m.backfilledObjects[objHash] = AffectedCacheObjectMetadata{Object: backfilled, CausingObjects: []ObjectHash{causing}}
+	}
+}
+
+// Metadata generates the final cache metadata from the collected data.
+func (m *GeneratedCacheMetadataCollector) Metadata() GeneratedCacheMetadata {
+	return GeneratedCacheMetadata{
+		BrokenObjects:     m.brokenObjects,
+		ExcludedObjects:   lo.Values(m.excludedObjects),
+		BackfilledObjects: lo.Values(m.backfilledObjects),
+	}
+}

internal/dataplane/fallback/fallback_meta_test.go

@@ -0,0 +1,94 @@
+package fallback_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/kong/kubernetes-ingress-controller/v3/internal/dataplane/fallback"
+)
+
+func TestGeneratedCacheMetadataCollector(t *testing.T) {
+	excluded1 := testService(t, "excluded-1")
+	excluded2 := testService(t, "excluded-2")
+	backfilled1 := testService(t, "backfilled-1")
+	backfilled2 := testService(t, "backfilled-2")
+	causing1 := testService(t, "causing-1")
+	causing2 := testService(t, "causing-2")
+
+	t.Run("only excluded", func(t *testing.T) {
+		c := fallback.NewGenerateCacheMetadataCollector(
+			fallback.GetObjectHash(causing1),
+			fallback.GetObjectHash(causing2),
+		)
+		c.CollectExcluded(excluded1, fallback.GetObjectHash(causing1))
+		c.CollectExcluded(excluded2, fallback.GetObjectHash(causing1))
+		c.CollectExcluded(excluded2, fallback.GetObjectHash(causing2)) // Duplicate with another causing object.
+
+		meta := c.Metadata()
+		require.ElementsMatch(t, []fallback.ObjectHash{
+			fallback.GetObjectHash(causing1),
+			fallback.GetObjectHash(causing2),
+		},
+			meta.BrokenObjects,
+		)
+		require.ElementsMatch(t, []fallback.AffectedCacheObjectMetadata{
+			{
+				Object:         excluded1,
+				CausingObjects: []fallback.ObjectHash{fallback.GetObjectHash(causing1)},
+			},
+			{
+				Object:         excluded2,
+				CausingObjects: []fallback.ObjectHash{fallback.GetObjectHash(causing1), fallback.GetObjectHash(causing2)},
+			},
+		},
+			meta.ExcludedObjects,
+		)
+	})
+
+	t.Run("excluded and backfilled", func(t *testing.T) {
+		c := fallback.NewGenerateCacheMetadataCollector(
+			fallback.GetObjectHash(causing1),
+			fallback.GetObjectHash(causing2),
+		)
+		c.CollectExcluded(excluded1, fallback.GetObjectHash(causing1))
+		c.CollectExcluded(excluded2, fallback.GetObjectHash(causing1))
+		c.CollectExcluded(excluded2, fallback.GetObjectHash(causing2)) // Duplicate with another causing object.
+
+		c.CollectBackfilled(backfilled1, fallback.GetObjectHash(causing1))
+		c.CollectBackfilled(backfilled2, fallback.GetObjectHash(causing1))
+		c.CollectBackfilled(backfilled2, fallback.GetObjectHash(causing2)) // Duplicate with another causing object.
+
+		meta := c.Metadata()
+		require.ElementsMatch(t, []fallback.ObjectHash{
+			fallback.GetObjectHash(causing1),
+			fallback.GetObjectHash(causing2),
+		},
+			meta.BrokenObjects,
+		)
+		require.ElementsMatch(t, []fallback.AffectedCacheObjectMetadata{
+			{
+				Object:         excluded1,
+				CausingObjects: []fallback.ObjectHash{fallback.GetObjectHash(causing1)},
+			},
+			{
+				Object:         excluded2,
+				CausingObjects: []fallback.ObjectHash{fallback.GetObjectHash(causing1), fallback.GetObjectHash(causing2)},
+			},
+		},
+			meta.ExcludedObjects,
+		)
+		require.ElementsMatch(t, []fallback.AffectedCacheObjectMetadata{
+			{
+				Object:         backfilled1,
+				CausingObjects: []fallback.ObjectHash{fallback.GetObjectHash(causing1)},
+			},
+			{
+				Object:         backfilled2,
+				CausingObjects: []fallback.ObjectHash{fallback.GetObjectHash(causing1), fallback.GetObjectHash(causing2)},
+			},
+		},
+			meta.BackfilledObjects,
+		)
+	})
+}