[DPE-5536][test_upgrades.py] CI - Gracefully fail refresh cmd error (#…

…455)

Currently, our CI is failing intermittently whenever we call `juju
refresh`, [as the controller is still busy downloading the
charm](https://github.com/juju/juju/blob/4584c53183a1b128ce0ffbe5f4e3c4b6235290bc/cmd/juju/application/refresh.go#L371)

To avoid failing due to this intermittent error, we add a retry loop at
refresh call.

Closes #454

tests/integration/upgrades/helpers.py

@@ -3,8 +3,11 @@
 # See LICENSE file for licensing details.
 
 import logging
+import subprocess
+from typing import Optional
 
 from pytest_operator.plugin import OpsTest
+from tenacity import Retrying, stop_after_attempt, wait_fixed
 
 from ..ha.continuous_writes import ContinuousWrites
 from ..ha.helpers import (
@@ -23,6 +26,41 @@
 logger = logging.getLogger(__name__)
 
 
+async def refresh(
+    ops_test: OpsTest,
+    app_name: str,
+    *,
+    revision: Optional[int] = None,
+    switch: Optional[str] = None,
+    channel: Optional[str] = None,
+    path: Optional[str] = None,
+) -> None:
+    # due to: https://github.com/juju/python-libjuju/issues/1057
+    # the following call does not work:
+    # application = ops_test.model.applications[APP_NAME]
+    # await application.refresh(
+    #     revision=rev,
+    # )
+
+    # Point to the right model, as we are calling the juju cli directly
+    args = [f"--model={ops_test.model.info.name}"]
+    if revision:
+        args.append(f"--revision={revision}")
+    if switch:
+        args.append(f"--switch={switch}")
+    if channel:
+        args.append(f"--channel={channel}")
+    if path:
+        args.append(f"--path={path}")
+
+    for attempt in Retrying(stop=stop_after_attempt(6), wait=wait_fixed(wait=30)):
+        with attempt:
+            cmd = ["juju", "refresh"]
+            cmd.extend(args)
+            cmd.append(app_name)
+            subprocess.check_output(cmd)
+
+
 async def assert_upgrade_to_local(
     ops_test: OpsTest, cwrites: ContinuousWrites, local_charm: str
 ) -> None:

tests/integration/upgrades/test_small_deployment_upgrades.py

@@ -3,7 +3,6 @@
 # See LICENSE file for licensing details.
 
 import logging
-import subprocess
 
 import pytest
 from pytest_operator.plugin import OpsTest
@@ -20,7 +19,7 @@
 )
 from ..helpers_deployments import get_application_units, wait_until
 from ..tls.test_tls import TLS_CERTIFICATES_APP_NAME
-from .helpers import assert_upgrade_to_local
+from .helpers import assert_upgrade_to_local, refresh
 
 logger = logging.getLogger(__name__)
 
@@ -61,8 +60,6 @@
 #  Auxiliary functions
 #
 #######################################################################
-
-
 @pytest.mark.runner(["self-hosted", "linux", "X64", "jammy", "large"])
 async def _build_env(ops_test: OpsTest, version: str) -> None:
     """Deploy OpenSearch cluster from a given revision."""
@@ -137,12 +134,7 @@ async def test_upgrade_between_versions(
 
         async with ops_test.fast_forward():
             logger.info("Refresh the charm")
-            # due to: https://github.com/juju/python-libjuju/issues/1057
-            # application = ops_test.model.applications[APP_NAME]
-            # await application.refresh(
-            #     revision=rev,
-            # )
-            subprocess.check_output(f"juju refresh opensearch --revision={rev}".split())
+            await refresh(ops_test, app, revision=rev)
 
             await wait_until(
                 ops_test,
@@ -238,12 +230,7 @@ async def test_upgrade_rollback_from_local(
 
     async with ops_test.fast_forward():
         logger.info("Refresh the charm")
-        # due to: https://github.com/juju/python-libjuju/issues/1057
-        # application = ops_test.model.applications[APP_NAME]
-        # await application.refresh(
-        #     revision=new_rev,
-        # )
-        subprocess.check_output(f"juju refresh opensearch --path={charm}".split())
+        await refresh(ops_test, app, path=charm)
 
         await wait_until(
             ops_test,
@@ -258,20 +245,12 @@ async def test_upgrade_rollback_from_local(
         )
 
         logger.info(f"Rolling back to {version}")
-        # due to: https://github.com/juju/python-libjuju/issues/1057
-        # await application.refresh(
-        #     revision=rev,
-        # )
-
-        # Rollback operation
-        # We must first switch back to the upstream charm, then rollback to the original
-        # revision we were using.
-        subprocess.check_output(
-            f"""juju refresh opensearch
-                 --switch={OPENSEARCH_ORIGINAL_CHARM_NAME}
-                 --channel={OPENSEARCH_CHANNEL}""".split()
+        await refresh(
+            ops_test,
+            app,
+            switch=OPENSEARCH_ORIGINAL_CHARM_NAME,
+            channel=OPENSEARCH_CHANNEL,
         )
-
         # Wait until we are set in an idle state and can rollback the revision.
         # app status blocked: that will happen if we are jumping N-2 versions in our test
         # app status active: that will happen if we are jumping N-1 in our test
@@ -286,9 +265,10 @@ async def test_upgrade_rollback_from_local(
             timeout=1400,
             idle_period=IDLE_PERIOD,
         )
-
-        subprocess.check_output(
-            f"juju refresh opensearch --revision={VERSION_TO_REVISION[version]}".split()
+        await refresh(
+            ops_test,
+            app,
+            revision=VERSION_TO_REVISION[version],
         )
 
         await wait_until(