✨ Support podman's --preserve-fds arg to container run/exec (#576)

python_on_whales/components/container/cli_wrapper.py

@@ -844,6 +844,7 @@ def execute(
         workdir: Optional[ValidPath] = None,
         stream: bool = False,
         detach_keys: Optional[str] = None,
+        preserve_fds: Optional[int] = None,
     ) -> Union[None, str, Iterable[Tuple[str, bytes]]]:
         """Execute a command inside a container
 
@@ -860,6 +861,8 @@ def execute(
                 to allow communication with the parent process.
                 Currently only works with `tty=True` for interactive use
                 on the terminal.
+            preserve_fds: The number of additional file descriptors to pass
+                through to the container. Only supported by podman.
             privileged: Give extended privileges to the container.
             tty: Allocate a pseudo-TTY. Allow the process to access your terminal
                 to write on it.
@@ -924,6 +927,7 @@ def execute(
             )
 
         full_cmd.add_flag("--interactive", interactive)
+        full_cmd.add_simple_arg("--preserve-fds", preserve_fds)
         full_cmd.add_flag("--privileged", privileged)
         full_cmd.add_flag("--tty", tty)
 
@@ -933,10 +937,18 @@ def execute(
         full_cmd.append(container)
         for arg in to_list(command):
             full_cmd.append(arg)
+
+        if preserve_fds:
+            # Pass through additional file descriptors (as well as 0-2,
+            # stdin, stdout, stderr, which are handled separately by the
+            # container runtime). See the podman documentation.
+            pass_fds = range(3, 3 + preserve_fds)
+        else:
+            pass_fds = ()
         if stream:
-            return stream_stdout_and_stderr(full_cmd)
+            return stream_stdout_and_stderr(full_cmd, pass_fds=pass_fds)
         else:
-            result = run(full_cmd, tty=tty)
+            result = run(full_cmd, tty=tty, pass_fds=pass_fds)
             if detach:
                 return None
             else:
@@ -1355,6 +1367,7 @@ def run(
         pid: Optional[str] = None,
         pids_limit: Optional[int] = None,
         platform: Optional[str] = None,
+        preserve_fds: Optional[int] = None,
         privileged: bool = False,
         publish: List[ValidPortMapping] = [],
         publish_all: bool = False,
@@ -1513,6 +1526,8 @@ def run(
             pid: PID namespace to use
             pids_limit: Tune container pids limit (set `-1` for unlimited)
             platform: Set platform if server is multi-platform capable.
+            preserve_fds: The number of additional file descriptors to pass
+                through to the container. Only supported by podman.
             privileged: Give extended privileges to this container.
             publish: Ports to publish, same as the `-p` argument in the Docker CLI.
                 example are `[(8000, 7000) , ("127.0.0.1:3000", 2000)]` or
@@ -1675,6 +1690,7 @@ def run(
         full_cmd.add_simple_arg("--pids-limit", pids_limit)
 
         full_cmd.add_simple_arg("--platform", platform)
+        full_cmd.add_simple_arg("--preserve-fds", preserve_fds)
         full_cmd.add_flag("--privileged", privileged)
 
         full_cmd.add_args_list("-p", [format_port_arg(p) for p in publish])
@@ -1725,12 +1741,20 @@ def run(
                     "It's not possible to stream and detach a container at "
                     "the same time."
                 )
+
+        if preserve_fds:
+            # Pass through additional file descriptors (as well as 0-2,
+            # stdin, stdout, stderr, which are handled separately by the
+            # container runtime). See the podman documentation.
+            pass_fds = range(3, 3 + preserve_fds)
+        else:
+            pass_fds = ()
         if detach:
-            return Container(self.client_config, run(full_cmd))
+            return Container(self.client_config, run(full_cmd, pass_fds=pass_fds))
         elif stream:
-            return stream_stdout_and_stderr(full_cmd)
+            return stream_stdout_and_stderr(full_cmd, pass_fds=pass_fds)
         else:
-            return run(full_cmd, tty=tty, capture_stderr=False)
+            return run(full_cmd, tty=tty, capture_stderr=False, pass_fds=pass_fds)
 
     def start(
         self,

python_on_whales/utils.py

@@ -8,7 +8,7 @@
 from queue import Queue
 from subprocess import PIPE, Popen
 from threading import Thread
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union, overload
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union, overload
 
 import pydantic
 from typing_extensions import Literal
@@ -98,6 +98,7 @@ def run(
     return_stderr: Literal[True] = ...,
     env: Dict[str, str] = ...,
     tty: bool = ...,
+    pass_fds: Sequence[int] = ...,
 ) -> Tuple[str, str]:
     ...
 
@@ -111,6 +112,7 @@ def run(
     return_stderr: Literal[False] = ...,
     env: Dict[str, str] = ...,
     tty: bool = ...,
+    pass_fds: Sequence[int] = ...,
 ) -> str:
     ...
 
@@ -123,6 +125,7 @@ def run(
     return_stderr: bool = False,
     env: Dict[str, str] = {},
     tty: bool = False,
+    pass_fds: Sequence[int] = (),
 ) -> Union[str, Tuple[str, str]]:
     args = [str(x) for x in args]
     subprocess_env = dict(os.environ)
@@ -147,7 +150,12 @@ def run(
         print(f"Env: {subprocess_env}")
         print("------------------------------")
     completed_process = subprocess.run(
-        args, input=input, stdout=stdout_dest, stderr=stderr_dest, env=subprocess_env
+        args,
+        input=input,
+        stdout=stdout_dest,
+        stderr=stderr_dest,
+        env=subprocess_env,
+        pass_fds=pass_fds,
     )
 
     if completed_process.returncode != 0:
@@ -255,7 +263,7 @@ def reader(pipe, pipe_name, queue):
 
 
 def stream_stdout_and_stderr(
-    full_cmd: list, env: Dict[str, str] = None
+    full_cmd: list, env: Dict[str, str] = None, pass_fds: Sequence[int] = ()
 ) -> Iterable[Tuple[str, bytes]]:
     if env is None:
         subprocess_env = None
@@ -264,7 +272,9 @@ def stream_stdout_and_stderr(
         subprocess_env.update(env)
 
     full_cmd = list(map(str, full_cmd))
-    process = Popen(full_cmd, stdout=PIPE, stderr=PIPE, env=subprocess_env)
+    process = Popen(
+        full_cmd, stdout=PIPE, stderr=PIPE, env=subprocess_env, pass_fds=pass_fds
+    )
     q = Queue()
     full_stderr = b""  # for the error message
     # we use deamon threads to avoid hanging if the user uses ctrl+c

tests/python_on_whales/components/test_container.py

@@ -218,6 +218,21 @@ def test_create_with_systemd_mode(podman_client: DockerClient):
         assert container.config.systemd_mode is True
 
 
+def test_run_with_preserve_fds(podman_client: DockerClient):
+    read_fd, write_fd = os.pipe()
+    # Pass through enough additional file descriptors (as well as 0-2, stdin,
+    # stdout, stderr, which are handled separately by the container runtime) to
+    # ensure the write fd is available to the container.
+    # See the podman documentation.
+    with podman_client.container.run(
+        "ubuntu",
+        ["bash", "-c", f"echo foobar >&{write_fd}"],
+        detach=True,
+        preserve_fds=write_fd - 2,
+    ):
+        assert os.read(read_fd, 7) == b"foobar\n"
+
+
 @pytest.mark.parametrize(
     "ctr_client",
     ["docker", pytest.param("podman", marks=pytest.mark.xfail)],
@@ -786,6 +801,7 @@ def test_exec_detach_keys(run_mock: Mock):
         docker.client_config.docker_cmd
         + ["exec", "--detach-keys", "a,b", "ctr_name", "cmd"],
         tty=False,
+        pass_fds=(),
     )
 
 
@@ -1164,6 +1180,7 @@ def test_run_default_pull(image_mock: Mock, _: Mock, run_mock: Mock):
         docker.client_config.docker_cmd + ["container", "run", test_image_name],
         tty=False,
         capture_stderr=False,
+        pass_fds=(),
     )
 
 
@@ -1184,6 +1201,7 @@ def test_run_missing_pull(image_mock: Mock, _: Mock, run_mock: Mock):
         docker.client_config.docker_cmd + ["container", "run", test_image_name],
         tty=False,
         capture_stderr=False,
+        pass_fds=(),
     )
 
 
@@ -1204,6 +1222,7 @@ def test_run_always_pull(image_mock: Mock, _: Mock, run_mock: Mock):
         docker.client_config.docker_cmd + ["container", "run", test_image_name],
         tty=False,
         capture_stderr=False,
+        pass_fds=(),
     )
 
 
@@ -1225,6 +1244,7 @@ def test_run_never_pull(image_mock: Mock, _: Mock, run_mock: Mock):
         + ["container", "run", "--pull", "never", test_image_name],
         tty=False,
         capture_stderr=False,
+        pass_fds=(),
     )