Fix endless PVC await

CHANGELOG.md

@@ -12,6 +12,8 @@
 
 - Updated logic to accurately detect if a resource is a Patch variant. (https://github.com/pulumi/pulumi-kubernetes/pull/3102)
 - Added java as supported language for CustomResource overlays. (https://github.com/pulumi/pulumi-kubernetes/pull/3120)
+- `PersistentVolumeClaims` with a bind mode of `WaitForFirstConsumer` will no
+  long hang indefinitely. (https://github.com/pulumi/pulumi-kubernetes/pull/3130)
 
 ## 4.15.0 (July 9, 2024)
 

provider/pkg/await/awaiters.go

@@ -30,15 +30,13 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
 	logger "github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
+	corev1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/client-go/dynamic"
 )
 
-const (
-	statusAvailable = "Available"
-	statusBound     = "Bound"
-)
-
 // createAwaitConfig specifies on which conditions we are to consider a resource created and fully
 // initialized. For example, we might consider a `Deployment` created and initialized only when the
 // live number of Pods reaches the minimum liveness threshold. `pool` and `disco` are provided
@@ -244,7 +242,7 @@ var awaiters = map[string]awaitSpec{
 		awaitCreation: untilCoreV1PersistentVolumeInitialized,
 	},
 	coreV1PersistentVolumeClaim: {
-		awaitCreation: untilCoreV1PersistentVolumeClaimBound,
+		awaitCreation: untilCoreV1PersistentVolumeClaimReady,
 	},
 	coreV1Pod: {
 		awaitCreation: awaitPodInit,
@@ -458,15 +456,17 @@ func untilCoreV1NamespaceDeleted(config deleteAwaitConfig) error {
 // --------------------------------------------------------------------------
 
 func untilCoreV1PersistentVolumeInitialized(c createAwaitConfig) error {
+	available := string(corev1.VolumeAvailable)
+	bound := string(corev1.VolumeBound)
 	pvAvailableOrBound := func(pv *unstructured.Unstructured) bool {
-		statusPhase, _ := openapi.Pluck(pv.Object, "status", "phase")
-		logger.V(3).Infof("Persistent volume %q status received: %#v", pv.GetName(), statusPhase)
-		if statusPhase == statusAvailable {
-			c.logStatus(diag.Info, "✅ PVC marked available")
-		} else if statusPhase == statusBound {
-			c.logStatus(diag.Info, "✅ PVC has been bound")
+		phase, _ := openapi.Pluck(pv.Object, "status", "phase")
+		logger.V(3).Infof("Persistent volume %q status received: %#v", pv.GetName(), phase)
+		if phase == available {
+			c.logStatus(diag.Info, "✅ PV marked available")
+		} else if phase == bound {
+			c.logStatus(diag.Info, "✅ PV has been bound")
 		}
-		return statusPhase == statusAvailable || statusPhase == statusBound
+		return phase == available || phase == bound
 	}
 
 	client, err := c.clientSet.ResourceClientForObject(c.currentOutputs)
@@ -483,19 +483,57 @@ func untilCoreV1PersistentVolumeInitialized(c createAwaitConfig) error {
 
 // --------------------------------------------------------------------------
 
-func untilCoreV1PersistentVolumeClaimBound(c createAwaitConfig) error {
-	pvcBound := func(pvc *unstructured.Unstructured) bool {
-		statusPhase, _ := openapi.Pluck(pvc.Object, "status", "phase")
-		logger.V(3).Infof("Persistent volume claim %s status received: %#v", pvc.GetName(), statusPhase)
-		return statusPhase == statusBound
+func untilCoreV1PersistentVolumeClaimReady(c createAwaitConfig) error {
+	var bindMode string
+	pvcReady := func(pvc *unstructured.Unstructured) bool {
+		// Lookup the PVC's storage class once it's available.
+		if bindMode == "" {
+			b, err := pvcBindMode(c.ctx, c.clientSet, pvc)
+			if err != nil {
+				c.logStatus(diag.Warning, err.Error())
+			}
+			bindMode = b
+		}
+
+		phase, _ := openapi.Pluck(pvc.Object, "status", "phase")
+		logger.V(3).Infof("Persistent volume claim %s status received: %#v", pvc.GetName(), phase)
+
+		if bindMode == string(storagev1.VolumeBindingWaitForFirstConsumer) {
+			return phase == string(corev1.ClaimPending)
+		}
+		return phase == string(corev1.ClaimBound)
 	}
 
 	client, err := c.clientSet.ResourceClientForObject(c.currentOutputs)
 	if err != nil {
 		return err
 	}
 	return watcher.ForObject(c.ctx, client, c.currentOutputs.GetName()).
-		WatchUntil(pvcBound, 5*time.Minute)
+		WatchUntil(pvcReady, 5*time.Minute)
+}
+
+// pvcBindMode attempts to fetch a PVC's StorageClass and returns its
+// volumeBindingMode.
+func pvcBindMode(
+	ctx context.Context,
+	clientset *clients.DynamicClientSet,
+	pvc *unstructured.Unstructured,
+) (string, error) {
+	gvk := storagev1.SchemeGroupVersion.WithKind("StorageClass")
+	scClient, err := clientset.ResourceClient(gvk, "")
+	if err != nil {
+		return "", fmt.Errorf("getting storageclass client: %w", err)
+	}
+	name, _, err := unstructured.NestedString(pvc.Object, "spec", "storageClassName")
+	if err != nil {
+		return "", err
+	}
+	sc, err := scClient.Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		return "", fmt.Errorf("getting storagclass %q: %w", name, err)
+	}
+	bindMode, _, err := unstructured.NestedString(sc.Object, "volumeBindingMode")
+	return bindMode, err
 }
 
 // --------------------------------------------------------------------------

provider/pkg/await/deployment.go

@@ -28,6 +28,8 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
 	logger "github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -334,7 +336,6 @@ func (dia *deploymentInitAwaiter) await(
 	timeout,
 	aggregateErrorTicker <-chan time.Time,
 ) error {
-
 	// Before we start processing any ReplicaSet, PVC or Pod events, we need to wait until the Deployment controller
 	// has seen and updated the status of the Deployment object.
 	if err := dia.waitUntilDeploymentControllerReconciles(deploymentEvents, timeout); err != nil {
@@ -551,7 +552,7 @@ func (dia *deploymentInitAwaiter) processDeploymentEvent(event watch.Event) {
 			dia.replicaSetAvailable = condition["reason"] == "NewReplicaSetAvailable" && isProgressing
 		}
 
-		if condition["type"] == statusAvailable {
+		if condition["type"] == string(appsv1.DeploymentAvailable) {
 			dia.deploymentAvailable = condition["status"] == trueStatus
 			if !dia.deploymentAvailable {
 				rawReason, hasReason := condition["reason"]
@@ -749,7 +750,7 @@ func (dia *deploymentInitAwaiter) checkPersistentVolumeClaimStatus() {
 
 		// Success only occurs when there are no PersistentVolumeClaims
 		// defined, or when all PVCs have a status of 'Bound'
-		if phase != statusBound {
+		if phase != string(corev1.ClaimBound) {
 			allPVCsReady = false
 			message := fmt.Sprintf(
 				"PersistentVolumeClaim: [%s] is not ready. status.phase currently at: %s", pvc.GetName(), phase)
@@ -855,7 +856,7 @@ func (dia *deploymentInitAwaiter) getFailedPersistentValueClaims() []string {
 	failed := make([]string, 0)
 	for _, pvc := range dia.pvcs {
 		phase, _ := openapi.Pluck(pvc.Object, "status", "phase")
-		if phase != statusBound {
+		if phase != string(corev1.ClaimBound) {
 			failed = append(failed, pvc.GetName())
 		}
 	}

tests/sdk/java/await_test.go

@@ -72,3 +72,23 @@ func TestAwaitDaemonSet(t *testing.T) {
 	_, err := test.CurrentStack().Up(context.Background())
 	assert.ErrorContains(t, err, `the Kubernetes API server reported that "default/await-daemonset" failed to fully initialize or become live: timed out waiting for the condition`)
 }
+
+func TestAwaitPVC(t *testing.T) {
+	t.Parallel()
+
+	test := pulumitest.NewPulumiTest(t,
+		"testdata/await/pvc",
+		opttest.SkipInstall(),
+	)
+	t.Cleanup(func() {
+		test.Destroy()
+	})
+
+	// WaitUntilFirstConsumer PVC should still be Pending.
+	up := test.Up()
+	assert.Equal(t, "Pending", up.Outputs["status"].Value)
+
+	// Adding a Deployment to consume the PVC should succeed.
+	test.UpdateSource("testdata/await/pvc/step2")
+	up = test.Up()
+}

tests/sdk/java/testdata/await/pvc/Pulumi.yaml

@@ -0,0 +1,37 @@
+name: pvc
+runtime: yaml
+description: |
+  Tests PVC with WaitForFirstConsumer (https://github.com/pulumi/pulumi-kubernetes/issues/895)
+
+resources:
+  ns:
+    type: kubernetes:core/v1:Namespace
+
+  provider:
+    type: pulumi:providers:kubernetes
+    properties:
+      namespace: ${ns.metadata.name}
+
+  sc:
+    type: kubernetes:storage.k8s.io/v1:StorageClass
+    properties:
+      volumeBindingMode: WaitForFirstConsumer
+      provisioner: rancher.io/local-path
+    options:
+      provider: ${provider}
+
+  pvc:
+    type: kubernetes:core/v1:PersistentVolumeClaim
+    properties:
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        storageClassName: ${sc.metadata.name}
+        resources:
+          requests:
+            storage: 1Mi
+    options:
+      provider: ${provider}
+
+outputs:
+  status: ${pvc.status.phase}

tests/sdk/java/testdata/await/pvc/step2/Pulumi.yaml

@@ -0,0 +1,64 @@
+name: pvc
+runtime: yaml
+description: |
+  Tests PVC with WaitForFirstConsumer (https://github.com/pulumi/pulumi-kubernetes/issues/895)
+
+resources:
+  ns:
+    type: kubernetes:core/v1:Namespace
+
+  provider:
+    type: pulumi:providers:kubernetes
+    properties:
+      namespace: ${ns.metadata.name}
+
+  sc:
+    type: kubernetes:storage.k8s.io/v1:StorageClass
+    properties:
+      volumeBindingMode: WaitForFirstConsumer
+      provisioner: rancher.io/local-path
+    options:
+      provider: ${provider}
+
+  pvc:
+    type: kubernetes:core/v1:PersistentVolumeClaim
+    properties:
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        storageClassName: ${sc.metadata.name}
+        resources:
+          requests:
+            storage: 1Mi
+    options:
+      provider: ${provider}
+
+  # Add a deployment to consume the PVC.
+  deployment:
+    type: kubernetes:apps/v1:Deployment
+    properties:
+      spec:
+        replicas: 1
+        selector:
+          matchLabels:
+            app: nginx
+        template:
+          metadata:
+            labels:
+              app: nginx
+          spec:
+            containers:
+              - name: nginx
+                image: nginx:stable-alpine3.17-slim
+                volumeMounts:
+                  - name: local
+                    mountPath: /usr/share/nginx/html
+            volumes:
+              - name: local
+                persistentVolumeClaim:
+                  claimName: ${pvc.metadata.name}
+    options:
+      provider: ${provider}
+
+outputs:
+  status: ${pvc.status.phase}