feat(gke): allow synchronous cluster teardown

CHANGELOG.md

@@ -5,7 +5,8 @@
 - GKE cluster builder allows creating a subnet for the cluster instead of using 
   a default one.
   [#490](https://github.com/Kong/kubernetes-testing-framework/pull/490)
-
+- GKE cluster is able to wait for its cleanup synchronously. 
+  [#491](https://github.com/Kong/kubernetes-testing-framework/pull/491)
 - MetalLB addon will use an extended timeout when fetching manifests from GH which 
   should improve its stability. 
   [#492](https://github.com/Kong/kubernetes-testing-framework/pull/492)

pkg/clusters/types/gke/builder.go

@@ -28,6 +28,7 @@ type Builder struct {
 	Name              string
 	project, location string
 	jsonCreds         []byte
+	waitForTeardown   bool
 
 	createSubnet   bool
 	addons         clusters.Addons
@@ -68,6 +69,15 @@ func (b *Builder) WithClusterMinorVersion(major, minor uint64) *Builder {
 	return b
 }
 
+// WithWaitForTeardown sets a flag telling whether the cluster should wait for
+// a cleanup operation synchronously.
+//
+// Default: `false`.
+func (b *Builder) WithWaitForTeardown(wait bool) *Builder {
+	b.waitForTeardown = wait
+	return b
+}
+
 // WithCreateSubnet sets a flag telling whether the builder should create a subnet
 // for the cluster. If set to `true`, it will create a subnetwork in a default VPC
 // with a uniquely generated name. The subnetwork will be removed once the cluster
@@ -176,14 +186,15 @@ func (b *Builder) Build(ctx context.Context) (clusters.Cluster, error) {
 	}
 
 	cluster := &Cluster{
-		name:      b.Name,
-		project:   b.project,
-		location:  b.location,
-		jsonCreds: b.jsonCreds,
-		client:    k8s,
-		cfg:       restCFG,
-		addons:    make(clusters.Addons),
-		l:         &sync.RWMutex{},
+		name:            b.Name,
+		project:         b.project,
+		location:        b.location,
+		jsonCreds:       b.jsonCreds,
+		waitForTeardown: b.waitForTeardown,
+		client:          k8s,
+		cfg:             restCFG,
+		addons:          make(clusters.Addons),
+		l:               &sync.RWMutex{},
 	}
 
 	if err := utils.ClusterInitHooks(ctx, cluster); err != nil {

pkg/clusters/types/gke/cluster.go

@@ -6,8 +6,10 @@ import (
 	"os"
 	"strings"
 	"sync"
+	"time"
 
 	container "cloud.google.com/go/container/apiv1"
+	"cloud.google.com/go/container/apiv1/containerpb"
 	"github.com/blang/semver/v4"
 	"google.golang.org/api/option"
 	"k8s.io/client-go/kubernetes"
@@ -22,14 +24,15 @@ import (
 
 // Cluster is a clusters.Cluster implementation backed by Google Kubernetes Engine (GKE)
 type Cluster struct {
-	name      string
-	project   string
-	location  string
-	jsonCreds []byte
-	client    *kubernetes.Clientset
-	cfg       *rest.Config
-	addons    clusters.Addons
-	l         *sync.RWMutex
+	name            string
+	project         string
+	location        string
+	jsonCreds       []byte
+	waitForTeardown bool
+	client          *kubernetes.Clientset
+	cfg             *rest.Config
+	addons          clusters.Addons
+	l               *sync.RWMutex
 }
 
 // NewFromExistingWithEnv provides a new clusters.Cluster backed by an existing GKE cluster,
@@ -111,13 +114,44 @@ func (c *Cluster) Cleanup(ctx context.Context) error {
 		}
 		defer mgrc.Close()
 
-		_, err = deleteCluster(ctx, mgrc, c.name, c.project, c.location)
-		return err
+		teardownOp, err := deleteCluster(ctx, mgrc, c.name, c.project, c.location)
+		if err != nil {
+			return err
+		}
+
+		if c.waitForTeardown {
+			fullTeardownOpName := fmt.Sprintf("projects/%s/locations/%s/operations/%s", c.project, c.location, teardownOp.Name)
+			if err := waitForTeardown(ctx, mgrc, fullTeardownOpName); err != nil {
+				return err
+			}
+		}
+
+		return nil
 	}
 
 	return nil
 }
 
+func waitForTeardown(ctx context.Context, mgrc *container.ClusterManagerClient, teardownOpName string) error {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+			op, err := mgrc.GetOperation(ctx, &containerpb.GetOperationRequest{Name: teardownOpName})
+			if err != nil {
+				return err
+			}
+			if op.Status == containerpb.Operation_DONE {
+				return nil
+			}
+		}
+	}
+}
+
 func (c *Cluster) Client() *kubernetes.Clientset {
 	return c.client
 }

test/e2e/gke_cluster_test.go

@@ -64,6 +64,7 @@ func testGKECluster(t *testing.T, createSubnet bool) {
 	t.Logf("configuring the GKE cluster PROJECT=(%s) LOCATION=(%s)", gkeProject, gkeLocation)
 	builder := gke.NewBuilder([]byte(gkeCreds), gkeProject, gkeLocation)
 	builder.WithClusterMinorVersion(1, 23)
+	builder.WithWaitForTeardown(true)
 	builder.WithCreateSubnet(createSubnet)
 
 	t.Logf("building cluster %s (this can take some time)", builder.Name)
@@ -72,6 +73,7 @@ func testGKECluster(t *testing.T, createSubnet bool) {
 
 	t.Logf("setting up cleanup for cluster %s", cluster.Name())
 	defer func() {
+		t.Logf("running cluster cleanup for %s", cluster.Name())
 		assert.NoError(t, cluster.Cleanup(ctx))
 	}()