better error handling for evicted pods

Netflix · Sep 21, 2021 · 92c82d4 · 92c82d4
1 parent f8a4c4e
commit 92c82d4
Showing 1 changed file with 29 additions and 13 deletions.
diff --git a/metaflow/plugins/aws/eks/kubernetes_client.py b/metaflow/plugins/aws/eks/kubernetes_client.py
@@ -579,19 +579,35 @@ def _done():
                 if not _done():
                     # If pod status is dirty, check for newer status
                     self._pod = self._fetch_pod()
-                if self._pod:
-                    for k, v in (
-                        self._pod["status"]
-                        .get("container_statuses", [{}])[0]
-                        .get("state", {})
-                        .items()
-                    ):
-                        if v is not None:
-                            return v.get("exit_code"), ": ".join(
-                                filter(
-                                    None,
-                                    [v.get("reason"), v.get("message")],
+                try:
+                    if self._pod:
+                        pod_status = self._pod["status"]
+                        if pod_status.get("container_statuses") is None:
+                            # We're done, but no container_statuses is set
+                            # This can happen when the pod is evicted
+                            return None, ": ".join(
+                                    filter(
+                                        None,
+                                        [pod_status.get("reason"), pod_status.get("message")],
+                                    )
                                 )
-                            )
+
+                        for k, v in (
+                            pod_status
+                            .get("container_statuses", [{}])[0]
+                            .get("state", {})
+                            .items()
+                        ):
+                            if v is not None:
+                                return v.get("exit_code"), ": ".join(
+                                    filter(
+                                        None,
+                                        [v.get("reason"), v.get("message")],
+                                    )
+                                )
+                except TypeError:
+                    import sys
+                    print("self._pod", self._pod, file=sys.stderr)
+                    raise
 
         return None, None