ensure cluster metadata node addresses consistency (#328)

src/upgrade.py

@@ -15,6 +15,7 @@
 )
 from charms.mysql.v0.mysql import (
     MySQLGetMySQLVersionError,
+    MySQLRescanClusterError,
     MySQLSetClusterPrimaryError,
     MySQLSetVariableError,
 )
@@ -88,6 +89,17 @@ def _count_online_instances(status_dict: dict) -> int:
                 if not item.get("instanceerrors", [])
             ].count("online")
 
+        try:
+            # ensure cluster node addresses are consistent in cluster metadata
+            # https://github.com/canonical/mysql-k8s-operator/issues/327
+            self.charm._mysql.rescan_cluster()
+        except MySQLRescanClusterError:
+            raise ClusterNotReadyError(
+                message=fail_message,
+                cause="Failed to rescan cluster",
+                resolution="Check the cluster status",
+            )
+
         if cluster_status := self.charm._mysql.get_cluster_status(extended=True):
             if _count_online_instances(cluster_status) < self.charm.app.planned_units():
                 # case any not fully online unit is found

tests/unit/test_upgrade.py

@@ -61,14 +61,18 @@ def test_highest_ordinal(self):
         """Test the highest ordinal."""
         self.assertEqual(1, self.charm.upgrade.highest_ordinal)
 
+    @patch("mysql_k8s_helpers.MySQL.rescan_cluster")
     @patch("upgrade.MySQLK8sUpgrade._pre_upgrade_prepare")
     @patch("mysql_k8s_helpers.MySQL.get_cluster_status", return_value=MOCK_STATUS_ONLINE)
-    def test_pre_upgrade_check(self, mock_get_cluster_status, mock_pre_upgrade_prepare):
+    def test_pre_upgrade_check(
+        self, mock_get_cluster_status, mock_pre_upgrade_prepare, mock_rescan_cluster
+    ):
         """Test the pre upgrade check."""
         self.harness.set_leader(True)
         self.charm.on.config_changed.emit()
 
         self.charm.upgrade.pre_upgrade_check()
+        mock_rescan_cluster.assert_called_once()
         mock_pre_upgrade_prepare.assert_called_once()
         mock_get_cluster_status.assert_called_once()