Add retry for api_request() call (#2240) (#2273)

* Add retry for api_request() call (#2240)

* Add retry for api_request() call

* add optional retry_params to pass timeout and sleep_time

* updates based on review comments

* updates based on review comments

* fix missing default value

* move default retry params to where they are used

* Add typing

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

ocp_resources/constants.py

@@ -25,6 +25,7 @@
 PROTOCOL_ERROR_EXCEPTION_DICT: Dict[type[Exception], List[str]] = {ProtocolError: []}
 NOT_FOUND_ERROR_EXCEPTION_DICT: Dict[type[Exception], List[str]] = {NotFoundError: []}
 
+TIMEOUT_1SEC: int = 1
 TIMEOUT_5SEC: int = 5
 TIMEOUT_10SEC: int = 10
 TIMEOUT_30SEC: int = 30

ocp_resources/resource.py

@@ -37,6 +37,7 @@
     TIMEOUT_10SEC,
     TIMEOUT_30SEC,
     TIMEOUT_5SEC,
+    TIMEOUT_1SEC,
 )
 from ocp_resources.event import Event
 from timeout_sampler import (
@@ -1027,27 +1028,42 @@ def wait_for_condition(self, condition: str, status: str, timeout: int = 300) ->
                     if cond["type"] == condition and cond["status"] == status:
                         return
 
-    def api_request(self, method: str, action: str, url: str, **params: Any) -> Dict[str, Any]:
+    def api_request(
+        self, method: str, action: str, url: str, retry_params: dict[str, int] | None = None, **params: Any
+    ) -> dict[str, Any]:
         """
         Handle API requests to resource.
 
         Args:
             method (str): Request method (GET/PUT etc.).
             action (str): Action to perform (stop/start/guestosinfo etc.).
             url (str): URL of resource.
+            retry_params (dict): dict of timeout and sleep_time values for retrying the api request call
 
         Returns:
            data(dict): response data
 
         """
         client: DynamicClient = self.privileged_client or self.client
-        response = client.client.request(
-            method=method,
-            url=f"{url}/{action}",
-            headers=client.client.configuration.api_key,
-            **params,
-        )
 
+        api_request_params = {
+            "url": f"{url}/{action}",
+            "method": method,
+            "headers": client.client.configuration.api_key,
+        }
+        if retry_params:
+            response = self.retry_cluster_exceptions(
+                func=client.client.request,
+                timeout=retry_params.get("timeout", TIMEOUT_10SEC),
+                sleep_time=retry_params.get("sleep_time", TIMEOUT_1SEC),
+                **api_request_params,
+                **params,
+            )
+        else:
+            response = client.client.request(
+                **api_request_params,
+                **params,
+            )
         try:
             return json.loads(response.data)
         except json.decoder.JSONDecodeError:

ocp_resources/virtual_machine.py

@@ -1,10 +1,13 @@
-# -*- coding: utf-8 -*-
+from __future__ import annotations
+from typing import Any
 
 
 from ocp_resources.constants import (
     DEFAULT_CLUSTER_RETRY_EXCEPTIONS,
     PROTOCOL_ERROR_EXCEPTION_DICT,
     TIMEOUT_4MINUTES,
+    TIMEOUT_30SEC,
+    TIMEOUT_5SEC,
 )
 from ocp_resources.resource import NamespacedResource
 from timeout_sampler import TimeoutSampler
@@ -72,8 +75,17 @@ def _subresource_api_url(self):
             f"namespaces/{self.namespace}/virtualmachines/{self.name}"
         )
 
-    def api_request(self, method, action, **params):
-        return super().api_request(method=method, action=action, url=self._subresource_api_url, **params)
+    def api_request(
+        self, method: str, action: str, url: str = "", retry_params: dict[str, int] | None = None, **params: Any
+    ) -> dict[str, Any]:
+        default_vm_api_request_retry_params: dict[str, int] = {"timeout": TIMEOUT_30SEC, "sleep_time": TIMEOUT_5SEC}
+        return super().api_request(
+            method=method,
+            action=action,
+            url=url or self._subresource_api_url,
+            retry_params=retry_params or default_vm_api_request_retry_params,
+            **params,
+        )
 
     def to_dict(self) -> None:
         super().to_dict()

ocp_resources/virtual_machine_instance.py

@@ -1,9 +1,11 @@
+from __future__ import annotations
 import shlex
+from typing import Any
 
 import xmltodict
 from kubernetes.dynamic.exceptions import ResourceNotFoundError
 
-from ocp_resources.constants import PROTOCOL_ERROR_EXCEPTION_DICT, TIMEOUT_4MINUTES
+from ocp_resources.constants import PROTOCOL_ERROR_EXCEPTION_DICT, TIMEOUT_4MINUTES, TIMEOUT_30SEC, TIMEOUT_5SEC
 from ocp_resources.node import Node
 from ocp_resources.pod import Pod
 from ocp_resources.resource import NamespacedResource
@@ -49,8 +51,17 @@ def _subresource_api_url(self):
             f"namespaces/{self.namespace}/virtualmachineinstances/{self.name}"
         )
 
-    def api_request(self, method, action, **params):
-        return super().api_request(method=method, action=action, url=self._subresource_api_url, **params)
+    def api_request(
+        self, method: str, action: str, url: str = "", retry_params: dict[str, int] | None = None, **params: Any
+    ) -> dict[str, Any]:
+        default_vmi_api_request_retry_params: dict[str, int] = {"timeout": TIMEOUT_30SEC, "sleep_time": TIMEOUT_5SEC}
+        return super().api_request(
+            method=method,
+            action=action,
+            url=url or self._subresource_api_url,
+            retry_params=retry_params or default_vmi_api_request_retry_params,
+            **params,
+        )
 
     def pause(self, timeout=TIMEOUT_4MINUTES, wait=False):
         self.api_request(method="PUT", action="pause")