From 90c02db38632bcfbea6086b0b1f1f1f19e645fb8 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Tue, 12 May 2020 13:23:08 -0400 Subject: [PATCH 1/2] track stats by hostname not remote host/ip * broadcast websockets have stats tracked (i.e. connection status, number of messages total, messages per minute, etc). Previous to this change, stats were tracked by ip address, if it was defined on the instance, XOR hostname. This changeset tracks stats by hostname. --- awx/main/wsbroadcast.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/awx/main/wsbroadcast.py b/awx/main/wsbroadcast.py index 243981a88513..6549d59e739c 100644 --- a/awx/main/wsbroadcast.py +++ b/awx/main/wsbroadcast.py @@ -37,7 +37,7 @@ def get_broadcast_hosts(): .order_by('hostname') \ .values('hostname', 'ip_address') \ .distinct() - return [i['ip_address'] or i['hostname'] for i in instances] + return {i['hostname']: i['ip_address'] or i['hostname'] for i in instances} def get_local_host(): @@ -149,15 +149,22 @@ async def run_loop(self, websocket: aiohttp.ClientWebSocketResponse): class BroadcastWebsocketManager(object): def __init__(self): self.event_loop = asyncio.get_event_loop() + ''' + { + 'hostname1': BroadcastWebsocketTask(), + 'hostname2': BroadcastWebsocketTask(), + 'hostname3': BroadcastWebsocketTask(), + } + ''' self.broadcast_tasks = dict() - # parallel dict to broadcast_tasks that tracks stats self.local_hostname = get_local_host() self.stats_mgr = BroadcastWebsocketStatsManager(self.event_loop, self.local_hostname) async def run_per_host_websocket(self): while True: - future_remote_hosts = get_broadcast_hosts() + known_hosts = get_broadcast_hosts() + future_remote_hosts = known_hosts.keys() current_remote_hosts = self.broadcast_tasks.keys() deleted_remote_hosts = set(current_remote_hosts) - set(future_remote_hosts) new_remote_hosts = set(future_remote_hosts) - set(current_remote_hosts) @@ -177,7 +184,7 @@ async def run_per_host_websocket(self): broadcast_task = BroadcastWebsocketTask(name=self.local_hostname, event_loop=self.event_loop, stats=stats, - remote_host=h) + remote_host=known_hosts[h]) broadcast_task.start() self.broadcast_tasks[h] = broadcast_task From 155faa01383cbd5cb868127e996bf2b8a8f517b0 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Wed, 13 May 2020 08:30:39 -0400 Subject: [PATCH 2/2] delete and re-add host when ip address changes * The websocket backplane interconnect is done via ip address for Kubernetes and OpenShift. On init run_wsbroadcast reads all Instances from the DB and makes a decision to use the ip address or the hostname based, with preference given to the ip address if defined. For Kubernetes and OpenShift the nodes can load the Instance before the ip_address is set. This would cause the connection to be tried by hostname rather than ip address. This changeset ensures that an ip address set after an Instance record is created will be detected and used. --- awx/main/wsbroadcast.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/awx/main/wsbroadcast.py b/awx/main/wsbroadcast.py index 6549d59e739c..a97baf45f42a 100644 --- a/awx/main/wsbroadcast.py +++ b/awx/main/wsbroadcast.py @@ -169,6 +169,13 @@ async def run_per_host_websocket(self): deleted_remote_hosts = set(current_remote_hosts) - set(future_remote_hosts) new_remote_hosts = set(future_remote_hosts) - set(current_remote_hosts) + remote_addresses = {k: v.remote_host for k, v in self.broadcast_tasks.items()} + for hostname, address in known_hosts.items(): + if hostname in self.broadcast_tasks and \ + address != remote_addresses[hostname]: + deleted_remote_hosts.add(hostname) + new_remote_hosts.add(hostname) + if deleted_remote_hosts: logger.warn(f"Removing {deleted_remote_hosts} from websocket broadcast list") if new_remote_hosts: