better error handling for evicted pods (#711)

Netflix · Sep 22, 2021 · c784b16 · c784b16
1 parent f8a4c4e
commit c784b16
Showing 1 changed file with 12 additions and 1 deletion.
diff --git a/metaflow/plugins/aws/eks/kubernetes_client.py b/metaflow/plugins/aws/eks/kubernetes_client.py
@@ -580,8 +580,19 @@ def _done():
                     # If pod status is dirty, check for newer status
                     self._pod = self._fetch_pod()
                 if self._pod:
+                    pod_status = self._pod["status"]
+                    if pod_status.get("container_statuses") is None:
+                        # We're done, but no container_statuses is set
+                        # This can happen when the pod is evicted
+                        return None, ": ".join(
+                                filter(
+                                    None,
+                                    [pod_status.get("reason"), pod_status.get("message")],
+                                )
+                            )
+
                     for k, v in (
-                        self._pod["status"]
+                        pod_status
                         .get("container_statuses", [{}])[0]
                         .get("state", {})
                         .items()