[Core] Fix random port conflict from dashboard agent (#46437)

This PR fixes an issue where the dashboard agent default port 52365
could be taken by other components if unlucky. The changes both prevent
this specific case and is future proof now.

Signed-off-by: Superskyyy <yihaochen@apache.org>

python/ray/_private/node.py

@@ -293,16 +293,19 @@ def __init__(
                 logger.debug(f"Setting node ID to {node_id}")
                 self._node_id = node_id
 
+        # The dashboard agent port is assigned first to avoid
+        # other processes accidentally taking its default port
+        self._dashboard_agent_listen_port = self._get_cached_port(
+            "dashboard_agent_listen_port",
+            default_port=ray_params.dashboard_agent_listen_port,
+        )
+
         self.metrics_agent_port = self._get_cached_port(
             "metrics_agent_port", default_port=ray_params.metrics_agent_port
         )
         self._metrics_export_port = self._get_cached_port(
             "metrics_export_port", default_port=ray_params.metrics_export_port
         )
-        self._dashboard_agent_listen_port = self._get_cached_port(
-            "dashboard_agent_listen_port",
-            default_port=ray_params.dashboard_agent_listen_port,
-        )
         self._runtime_env_agent_port = self._get_cached_port(
             "runtime_env_agent_port",
             default_port=ray_params.runtime_env_agent_port,
@@ -1005,9 +1008,14 @@ def _get_cached_port(
                 port = int(ports_by_node[self.unique_id][port_name])
             else:
                 # Pick a new port to use and cache it at this node.
-                port = default_port or self._get_unused_port(
-                    set(ports_by_node[self.unique_id].values())
-                )
+                allocated_ports = set(ports_by_node[self.unique_id].values())
+
+                if default_port is not None and default_port in allocated_ports:
+                    # The default port is already in use, so don't use it.
+                    default_port = None
+
+                port = default_port or self._get_unused_port(allocated_ports)
+
                 ports_by_node[self.unique_id][port_name] = port
                 with open(file_path, "w") as f:
                     json.dump(ports_by_node, f)