Merge pull request #562 from opsani/OPTSERV-1079-servo-stop-leaves-co…

…-in-unrecoverable-state

Fix servo run_servo being called twice preventing clean shutdown

servo/cli.py

@@ -1109,18 +1109,10 @@ def run(
             Run the servo
             """
 
-            def run_servo():
-                poll = not no_poll
-                servo.runner.AssemblyRunner(context.assembly).run(
-                    poll=poll,
-                    interactive=bool(interactive),
-                    debug=debug,
-                )
+            if not context.assembly:
+                raise typer.Abort("failed to assemble servo")
 
             if check or dry_run:
-                if isinstance(context, click.core.Context):
-                    context = context.parent
-
                 # Check all targeted servos
                 def print_callback(input: str) -> None:
                     typer.echo(input)
@@ -1156,18 +1148,16 @@ async def gather_checks():
                     )
                     raise typer.Exit(1) from e
 
-                if ready:
-                    if not dry_run:
-                        run_servo()
-                    else:
-                        raise typer.Exit(0)
-                else:
+                if not ready:
                     raise typer.Exit(1)
 
-            if context.assembly:
-                run_servo()
-            else:
-                raise typer.Abort("failed to assemble servo")
+            if not dry_run:
+                poll = not no_poll
+                servo.runner.AssemblyRunner(context.assembly).run(
+                    poll=poll,
+                    interactive=bool(interactive),
+                    debug=debug,
+                )
 
         @self.command(section=section)
         def describe(

servo/connectors/kubernetes.py

@@ -1415,10 +1415,11 @@ async def handle_error(self, error: Exception) -> bool:
                         self.shutdown(), timeout=self.timeout.total_seconds()
                     )
                 except asyncio.exceptions.TimeoutError:
-                    self.logger.exception(level="TRACE")
-                    raise RuntimeError(
+                    message = (
                         f"Time out after {self.timeout} waiting for tuning pod shutdown"
                     )
+                    self.logger.exception(message, level="TRACE")
+                    raise RuntimeError(message)
 
                 # create a new canary against baseline
                 self.logger.info(