Use Eventually instead of resetting resourceVersion when updating pod…

…'s status Signed-off-by: Yuki Iwai <yuki.iwai.tz@gmail.com>
kubeflow · Jul 17, 2023 · ecd3eca · ecd3eca
1 parent 4bb00b7
commit ecd3eca
Showing 1 changed file with 38 additions and 24 deletions.
diff --git a/pkg/controller.v1/pytorch/pytorchjob_controller_test.go b/pkg/controller.v1/pytorch/pytorchjob_controller_test.go
@@ -178,9 +178,11 @@ var _ = Describe("PyTorchJob controller", func() {
 			}))
 
 			// Test job status.
-			masterPod.Status.Phase = corev1.PodSucceeded
-			masterPod.ResourceVersion = ""
-			Expect(testK8sClient.Status().Update(ctx, masterPod)).Should(Succeed())
+			Eventually(func() error {
+				Expect(testK8sClient.Get(ctx, masterKey, masterPod)).Should(Succeed())
+				masterPod.Status.Phase = corev1.PodSucceeded
+				return testK8sClient.Status().Update(ctx, masterPod)
+			}, testutil.Timeout, testutil.Interval).Should(Succeed())
 			Eventually(func() bool {
 				err := testK8sClient.Get(ctx, jobKey, created)
 				if err != nil {
@@ -272,12 +274,16 @@ var _ = Describe("PyTorchJob controller", func() {
 			}, testutil.Timeout, testutil.Interval).Should(BeNil())
 
 			By("Updating Pod's condition with running")
-			masterPod.Status.Phase = corev1.PodRunning
-			masterPod.ResourceVersion = ""
-			Expect(testK8sClient.Status().Update(ctx, masterPod)).Should(Succeed())
-			workerPod.Status.Phase = corev1.PodRunning
-			workerPod.ResourceVersion = ""
-			Expect(testK8sClient.Status().Update(ctx, workerPod)).Should(Succeed())
+			Eventually(func() error {
+				Expect(testK8sClient.Get(ctx, masterKey, masterPod)).Should(Succeed())
+				masterPod.Status.Phase = corev1.PodRunning
+				return testK8sClient.Status().Update(ctx, masterPod)
+			}, testutil.Timeout, testutil.Interval).Should(Succeed())
+			Eventually(func() error {
+				Expect(testK8sClient.Get(ctx, worker0Key, workerPod)).Should(Succeed())
+				workerPod.Status.Phase = corev1.PodRunning
+				return testK8sClient.Status().Update(ctx, workerPod)
+			}, testutil.Timeout, testutil.Interval).Should(Succeed())
 
 			By("Checking the PyTorchJob has resumed conditions")
 			Eventually(func() []kubeflowv1.JobCondition {
@@ -343,12 +349,16 @@ var _ = Describe("PyTorchJob controller", func() {
 			}, testutil.Timeout, testutil.Interval).Should(BeTrue())
 
 			By("Updating the pod's phase with Running")
-			masterPod.Status.Phase = corev1.PodRunning
-			masterPod.ResourceVersion = ""
-			Expect(testK8sClient.Status().Update(ctx, masterPod)).Should(Succeed())
-			workerPod.Status.Phase = corev1.PodRunning
-			workerPod.ResourceVersion = ""
-			Expect(testK8sClient.Status().Update(ctx, workerPod)).Should(Succeed())
+			Eventually(func() error {
+				Expect(testK8sClient.Get(ctx, masterKey, masterPod)).Should(Succeed())
+				masterPod.Status.Phase = corev1.PodRunning
+				return testK8sClient.Status().Update(ctx, masterPod)
+			}, testutil.Timeout, testutil.Interval).Should(Succeed())
+			Eventually(func() error {
+				Expect(testK8sClient.Get(ctx, worker0Key, workerPod)).Should(Succeed())
+				workerPod.Status.Phase = corev1.PodRunning
+				return testK8sClient.Status().Update(ctx, workerPod)
+			}, testutil.Timeout, testutil.Interval).Should(Succeed())
 
 			By("Checking the PyTorchJob's condition")
 			Eventually(func() []kubeflowv1.JobCondition {
@@ -438,13 +448,15 @@ var _ = Describe("PyTorchJob controller", func() {
 			}, testutil.Timeout, testutil.Interval).Should(Succeed())
 
 			By("Updating the pod's phase with Running")
-			Eventually(func() bool {
+			Eventually(func() error {
 				errMaster := testK8sClient.Get(ctx, masterKey, masterPod)
-				return errMaster == nil
-			}, testutil.Timeout, testutil.Interval).Should(BeTrue())
-			masterPod.Status.Phase = corev1.PodRunning
-			masterPod.ResourceVersion = ""
-			Expect(testK8sClient.Status().Update(ctx, masterPod)).Should(Succeed())
+				return errMaster
+			}, testutil.Timeout, testutil.Interval).Should(Succeed())
+			Eventually(func() error {
+				Expect(testK8sClient.Get(ctx, masterKey, masterPod)).Should(Succeed())
+				masterPod.Status.Phase = corev1.PodRunning
+				return testK8sClient.Status().Update(ctx, masterPod)
+			}, testutil.Timeout, testutil.Interval).Should(Succeed())
 
 			By("Checking if the PyTorchJob has resumed conditions")
 			Eventually(func() []kubeflowv1.JobCondition {
@@ -619,9 +631,11 @@ var _ = Describe("PyTorchJob controller", func() {
 			}))
 
 			// Test job status.
-			pod.Status.Phase = corev1.PodSucceeded
-			pod.ResourceVersion = ""
-			Expect(testK8sClient.Status().Update(ctx, pod)).Should(Succeed())
+			Eventually(func() error {
+				Expect(testK8sClient.Get(ctx, workerKey, pod)).Should(Succeed())
+				pod.Status.Phase = corev1.PodSucceeded
+				return testK8sClient.Status().Update(ctx, pod)
+			}, testutil.Timeout, testutil.Interval).Should(Succeed())
 			Eventually(func() bool {
 				err := testK8sClient.Get(ctx, jobKey, created)
 				if err != nil {