Handle unexpected exceptions from run_job (#911)

dstackai · Feb 19, 2024 · ad1966c · ad1966c
1 parent edda1d7
commit ad1966c
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 10 deletions.
diff --git a/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py b/src/dstack/_internal/server/background/tasks/process_submitted_jobs.py
@@ -195,15 +195,10 @@ async def _run_job(
     if run.run_spec.profile.backends is not None:
         backends = [b for b in backends if b.TYPE in run.run_spec.profile.backends]
 
-    try:
-        requirements = job.job_spec.requirements
-        offers = await backends_services.get_instance_offers(
-            backends, requirements, exclude_not_available=True
-        )
-    except BackendError as e:
-        logger.warning(*job_log("failed to get instance offers: %s", job_model, repr(e)))
-        return (None, None)
-
+    requirements = job.job_spec.requirements
+    offers = await backends_services.get_instance_offers(
+        backends, requirements, exclude_not_available=True
+    )
     # Limit number of offers tried to prevent long-running processing
     # in case all offers fail.
     for backend, offer in offers[:15]:
@@ -238,6 +233,17 @@ async def _run_job(
                 )
             )
             continue
+        except Exception:
+            logger.exception(
+                *job_log(
+                    "got exception when launching %s in %s/%s",
+                    job_model,
+                    offer.instance.name,
+                    offer.backend.value,
+                    offer.region,
+                )
+            )
+            continue
         else:
             job_provisioning_data = JobProvisioningData(
                 backend=backend.TYPE,

diff --git a/src/dstack/_internal/server/services/backends/__init__.py b/src/dstack/_internal/server/services/backends/__init__.py
@@ -8,6 +8,7 @@
 from dstack._internal.core.backends.base import Backend
 from dstack._internal.core.backends.local import LocalBackend
 from dstack._internal.core.errors import (
+    BackendError,
     BackendInvalidCredentialsError,
     BackendNotAvailable,
     ResourceExistsError,
@@ -300,7 +301,14 @@ async def get_instance_offers(
     tasks = [run_async(backend.compute().get_offers, requirements) for backend in backends]
     offers_by_backend = []
     for backend, result in zip(backends, await asyncio.gather(*tasks, return_exceptions=True)):
-        if isinstance(result, BaseException):
+        if isinstance(result, BackendError):
+            logger.warning(
+                "Failed to get offers from backend %s: %s",
+                backend.TYPE,
+                repr(result),
+            )
+            continue
+        elif isinstance(result, BaseException):
             logger.error(
                 "Got exception when requesting offers from backend %s",
                 backend.TYPE,