Implement a Catalog Controllers Lifecycle Integration Test (#17435)

* Implement a Catalog Controllers Lifecycle Integration Test

* Prevent triggering the race detector.

This allows defining some variables for protobuf constants and using those in comparisons. Without that, something internal in the fmt package ended up looking at the protobuf message size cache and triggering the race detector.

internal/catalog/catalogtest/run_test.go

@@ -37,3 +37,8 @@ func TestControllers_Integration(t *testing.T) {
 	client := runInMemResourceServiceAndControllers(t, catalog.DefaultControllerDependencies())
 	RunCatalogV1Alpha1IntegrationTest(t, client)
 }
+
+func TestControllers_Lifecycle(t *testing.T) {
+	client := runInMemResourceServiceAndControllers(t, catalog.DefaultControllerDependencies())
+	RunCatalogV1Alpha1LifecycleIntegrationTest(t, client)
+}

internal/catalog/catalogtest/test_integration_v1alpha1.go

@@ -698,6 +698,7 @@ func expectedGRPCApiServiceEndpoints(t *testing.T, c *rtest.Client) *pbcatalog.S
 }
 
 func verifyServiceEndpoints(t *testing.T, c *rtest.Client, id *pbresource.ID, expected *pbcatalog.ServiceEndpoints) {
+	t.Helper()
 	c.WaitForResourceState(t, id, func(t rtest.T, res *pbresource.Resource) {
 		var actual pbcatalog.ServiceEndpoints
 		err := res.Data.UnmarshalTo(&actual)

internal/catalog/exports.go

@@ -5,6 +5,9 @@ package catalog
 
 import (
 	"github.com/hashicorp/consul/internal/catalog/internal/controllers"
+	"github.com/hashicorp/consul/internal/catalog/internal/controllers/endpoints"
+	"github.com/hashicorp/consul/internal/catalog/internal/controllers/nodehealth"
+	"github.com/hashicorp/consul/internal/catalog/internal/controllers/workloadhealth"
 	"github.com/hashicorp/consul/internal/catalog/internal/mappers/nodemapper"
 	"github.com/hashicorp/consul/internal/catalog/internal/mappers/selectiontracker"
 	"github.com/hashicorp/consul/internal/catalog/internal/types"
@@ -40,6 +43,21 @@ var (
 	HealthStatusV1Alpha1Type     = types.HealthStatusV1Alpha1Type
 	HealthChecksV1Alpha1Type     = types.HealthChecksV1Alpha1Type
 	DNSPolicyV1Alpha1Type        = types.DNSPolicyV1Alpha1Type
+
+	// Controller Statuses
+	NodeHealthStatusKey              = nodehealth.StatusKey
+	NodeHealthStatusConditionHealthy = nodehealth.StatusConditionHealthy
+	NodeHealthConditions             = nodehealth.Conditions
+
+	WorkloadHealthStatusKey              = workloadhealth.StatusKey
+	WorkloadHealthStatusConditionHealthy = workloadhealth.StatusConditionHealthy
+	WorkloadHealthConditions             = workloadhealth.WorkloadConditions
+	WorkloadAndNodeHealthConditions      = workloadhealth.NodeAndWorkloadConditions
+
+	EndpointsStatusKey                       = endpoints.StatusKey
+	EndpointsStatusConditionEndpointsManaged = endpoints.StatusConditionEndpointsManaged
+	EndpointsStatusConditionManaged          = endpoints.ConditionManaged
+	EndpointsStatusConditionUnmanaged        = endpoints.ConditionUnmanaged
 )
 
 // RegisterTypes adds all resource types within the "catalog" API group

internal/resource/resourcetest/builder.go

@@ -3,9 +3,12 @@ package resourcetest
 import (
 	"context"
 
+	"github.com/hashicorp/consul/internal/resource"
 	"github.com/hashicorp/consul/proto-public/pbresource"
 	"github.com/oklog/ulid/v2"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/reflect/protoreflect"
 	"google.golang.org/protobuf/types/known/anypb"
@@ -37,6 +40,14 @@ func Resource(rtype *pbresource.Type, name string) *resourceBuilder {
 	}
 }
 
+func ResourceID(id *pbresource.ID) *resourceBuilder {
+	return &resourceBuilder{
+		resource: &pbresource.Resource{
+			Id: id,
+		},
+	}
+}
+
 func (b *resourceBuilder) WithData(t T, data protoreflect.ProtoMessage) *resourceBuilder {
 	t.Helper()
 
@@ -123,7 +134,11 @@ func (b *resourceBuilder) Write(t T, client pbresource.ResourceServiceClient) *p
 			_, err := client.Delete(context.Background(), &pbresource.DeleteRequest{
 				Id: rsp.Resource.Id,
 			})
-			require.NoError(t, err)
+
+			// ignore not found errors
+			if err != nil && status.Code(err) != codes.NotFound {
+				t.Fatalf("Failed to delete resource %s of type %s: %v", rsp.Resource.Id.Name, resource.ToGVK(rsp.Resource.Id.Type), err)
+			}
 		})
 	}
 

internal/resource/resourcetest/client.go

@@ -35,6 +35,7 @@ func (client *Client) SetRetryerConfig(timeout time.Duration, wait time.Duration
 }
 
 func (client *Client) retry(t T, fn func(r *retry.R)) {
+	t.Helper()
 	retryer := &retry.Timer{Timeout: client.timeout, Wait: client.wait}
 	retry.RunWith(retryer, t, fn)
 }
@@ -181,7 +182,7 @@ func (client *Client) WaitForStatusCondition(t T, id *pbresource.ID, statusKey s
 
 	var res *pbresource.Resource
 	client.retry(t, func(r *retry.R) {
-		res = client.RequireStatusConditionForCurrentGen(t, id, statusKey, condition)
+		res = client.RequireStatusConditionForCurrentGen(r, id, statusKey, condition)
 	})
 
 	return res
@@ -209,10 +210,27 @@ func (client *Client) WaitForResourceState(t T, id *pbresource.ID, verify func(T
 	return res
 }
 
+func (client *Client) WaitForDeletion(t T, id *pbresource.ID) {
+	t.Helper()
+
+	client.retry(t, func(r *retry.R) {
+		client.RequireResourceNotFound(r, id)
+	})
+}
+
 // ResolveResourceID will read the specified resource and returns its full ID.
 // This is mainly useful to get the ID with the Uid filled out.
 func (client *Client) ResolveResourceID(t T, id *pbresource.ID) *pbresource.ID {
 	t.Helper()
 
 	return client.RequireResourceExists(t, id).Id
 }
+
+func (client *Client) MustDelete(t T, id *pbresource.ID) {
+	_, err := client.Delete(context.Background(), &pbresource.DeleteRequest{Id: id})
+	if status.Code(err) == codes.NotFound {
+		return
+	}
+
+	require.NoError(t, err)
+}

proto/private/prototest/testing.go

@@ -100,5 +100,5 @@ func AssertContainsElement[V any](t TestingT, list []V, element V, opts ...cmp.O
 		}
 	}
 
-	t.Fatalf("assertion failed: list does not contain element\n--- list\n%#v\n--- element: %#v", list, element)
+	t.Fatalf("assertion failed: list does not contain element\n--- list\n%+v\n--- element: %+v", list, element)
 }