Add a retry mechanism when pulling the agent docker image (#16157)

* Add a retry mechanism when pulling the agent docker image

* add the attempts

* use stamina

* add tests

ddev/changelog.d/16157.fixed

@@ -0,0 +1 @@
+Add a retry mechanism when pulling the agent docker image

ddev/pyproject.toml

@@ -34,6 +34,7 @@ dependencies = [
     "jsonpointer",
     "pluggy",
     "rich>=12.5.1",
+    "stamina==23.2.0",
     "tomli; python_version < '3.11'",
     "tomli-w",
     "tomlkit",

ddev/src/ddev/e2e/agent/docker.py

@@ -9,6 +9,8 @@
 from functools import cache, cached_property
 from typing import TYPE_CHECKING, Callable
 
+import stamina
+
 from ddev.e2e.agent.interface import AgentInterface
 from ddev.utils.structures import EnvVars
 
@@ -165,9 +167,7 @@ def start(self, *, agent_build: str, local_packages: dict[Path, str], env_vars:
                     volumes[i] = f'/{vm_file}:{remaining}'
 
         if os.getenv('DDEV_E2E_DOCKER_NO_PULL') != '1':
-            process = self._run_command(['docker', 'pull', agent_build])
-            if process.returncode:
-                raise RuntimeError(f'Could not pull image {agent_build}')
+            self.__pull_image(agent_build)
 
         command = [
             'docker',
@@ -282,6 +282,12 @@ def invoke(self, args: list[str]) -> None:
     def enter_shell(self) -> None:
         self._run_command(self._format_command(['cmd' if self._is_windows_container else 'bash']), check=True)
 
+    @stamina.retry(on=RuntimeError, attempts=3)
+    def __pull_image(self, agent_build):
+        process = self._run_command(['docker', 'pull', agent_build])
+        if process.returncode:
+            raise RuntimeError(f'Could not pull image {agent_build}')
+
 
 @cache
 def _get_hostname():

ddev/tests/e2e/agent/test_docker.py

@@ -491,6 +491,70 @@ def test_docker_volumes_windows_running_windows(
             ),
         ]
 
+    def test_retry_pull_image(
+        self,
+        platform,
+        temp_dir,
+        default_hostname,
+        get_integration,
+        docker_path,
+        free_port,
+        mocker,
+    ):
+        run = mocker.patch(
+            'subprocess.run',
+            side_effect=[
+                mocker.MagicMock(returncode=1),
+                mocker.MagicMock(returncode=1),
+                mocker.MagicMock(returncode=0),
+                mocker.MagicMock(returncode=0),
+            ],
+        )
+
+        integration = 'postgres'
+        environment = 'py3.12'
+        metadata = {}
+
+        agent = DockerAgent(platform, get_integration(integration), environment, metadata, temp_dir / 'config.yaml')
+        agent.start(agent_build='', local_packages={}, env_vars={})
+
+        assert run.call_args_list == [
+            mocker.call([docker_path, 'pull', 'datadog/agent-dev:master-py3'], shell=False),
+            mocker.call([docker_path, 'pull', 'datadog/agent-dev:master-py3'], shell=False),
+            mocker.call([docker_path, 'pull', 'datadog/agent-dev:master-py3'], shell=False),
+            mocker.call(
+                [
+                    docker_path,
+                    'run',
+                    '-d',
+                    '--name',
+                    f'dd_{integration}_{environment}',
+                    '--network',
+                    'host',
+                    '-v',
+                    '/proc:/host/proc',
+                    '-e',
+                    'DD_API_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+                    '-e',
+                    'DD_APM_ENABLED=false',
+                    '-e',
+                    f'DD_CMD_PORT={free_port}',
+                    '-e',
+                    'DD_EXPVAR_PORT=5000',
+                    '-e',
+                    f'DD_HOSTNAME={default_hostname}',
+                    '-e',
+                    'DD_TELEMETRY_ENABLED=1',
+                    '-e',
+                    'PYTHONDONTWRITEBYTECODE=1',
+                    'datadog/agent-dev:master-py3',
+                ],
+                shell=False,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+            ),
+        ]
+
     def test_custom_hosts(
         self,
         platform,