wsbroadcast better logging and behavior

awx/main/management/commands/run_wsbroadcast.py

@@ -4,9 +4,11 @@
 import asyncio
 import datetime
 import re
+import redis
 from datetime import datetime as dt
 
 from django.core.management.base import BaseCommand
+from django.db.models import Q
 
 from awx.main.analytics.broadcast_websocket import (
     BroadcastWebsocketStatsManager,
@@ -55,8 +57,8 @@ def get_connection_status(cls, me, hostnames, data):
         host_stats = [('hostname', 'state', 'start time', 'duration (sec)')]
         for h in hostnames:
             connection_color = '91'    # red
-            h = safe_name(h)
-            prefix = f'awx_{h}'
+            h_safe = safe_name(h)
+            prefix = f'awx_{h_safe}'
             connection_state = data.get(f'{prefix}_connection', 'N/A')
             connection_started = 'N/A'
             connection_duration = 'N/A'
@@ -67,7 +69,7 @@ def get_connection_status(cls, me, hostnames, data):
                 connection_started = data.get(f'{prefix}_connection_start', 'Error')
                 if connection_started != 'Error':
                     connection_started = datetime.datetime.fromtimestamp(connection_started)
-                    connection_duration = (dt.now() - connection_started).total_seconds()
+                    connection_duration = int((dt.now() - connection_started).total_seconds())
 
             connection_state = f'\033[{connection_color}m{connection_state}\033[0m'
 
@@ -79,18 +81,23 @@ def get_connection_status(cls, me, hostnames, data):
     def get_connection_stats(cls, me, hostnames, data):
         host_stats = [('hostname', 'total', 'per minute')]
         for h in hostnames:
-            h = safe_name(h)
-            prefix = f'awx_{h}'
-            messages_total = data.get(f'{prefix}_messages_received', 'N/A')
-            messages_per_minute = data.get(f'{prefix}_messages_received_per_minute', 'N/A')
+            h_safe = safe_name(h)
+            prefix = f'awx_{h_safe}'
+            messages_total = data.get(f'{prefix}_messages_received', '0')
+            messages_per_minute = data.get(f'{prefix}_messages_received_per_minute', '0')
 
             host_stats.append((h, str(int(messages_total)), str(int(messages_per_minute))))
 
         return host_stats
 
     def handle(self, *arg, **options):
         if options.get('status'):
-            stats_all = BroadcastWebsocketStatsManager.get_stats_sync()
+            try:
+                stats_all = BroadcastWebsocketStatsManager.get_stats_sync()
+            except redis.exceptions.ConnectionError as e:
+                print(f"Unable to get Broadcast Websocket Status. Failed to connect to redis {e}")
+                return
+
             data = {}
             for family in stats_all:
                 if family.type == 'gauge' and len(family.samples) > 1:
@@ -101,7 +108,7 @@ def handle(self, *arg, **options):
                 else:
                     data[family.name] = family.samples[0].value
             me = Instance.objects.me()
-            hostnames = [i.hostname for i in Instance.objects.exclude(hostname=me.hostname)]
+            hostnames = [i.hostname for i in Instance.objects.exclude(Q(hostname=me.hostname) | Q(rampart_groups__controller__isnull=False))]
 
             host_stats = Command.get_connection_status(me, hostnames, data)
             lines = Command._format_lines(host_stats)

awx/settings/defaults.py

@@ -1043,6 +1043,15 @@ def IS_TESTING(argv=None):
             'backupCount': 5,
             'formatter':'dispatcher',
         },
+        'wsbroadcast': {
+            # don't define a level here, it's set by settings.LOG_AGGREGATOR_LEVEL
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filters': ['require_debug_false', 'dynamic_level_filter'],
+            'filename': os.path.join(LOG_ROOT, 'wsbroadcast.log'),
+            'maxBytes': 1024 * 1024 * 5, # 5 MB
+            'backupCount': 5,
+            'formatter':'simple',
+        },
         'celery.beat': {
             'class':'logging.StreamHandler',
             'level': 'ERROR'
@@ -1130,6 +1139,9 @@ def IS_TESTING(argv=None):
         'awx.main.dispatch': {
             'handlers': ['dispatcher'],
         },
+        'awx.main.wsbroadcast': {
+            'handlers': ['wsbroadcast'],
+        },
         'awx.isolated.manager.playbooks': {
             'handlers': ['management_playbooks'],
             'propagate': False

awx/settings/production.py

@@ -55,6 +55,7 @@
 LOGGING['handlers']['tower_warnings']['filename'] = '/var/log/tower/tower.log'  # noqa
 LOGGING['handlers']['callback_receiver']['filename'] = '/var/log/tower/callback_receiver.log'  # noqa
 LOGGING['handlers']['dispatcher']['filename'] = '/var/log/tower/dispatcher.log'  # noqa
+LOGGING['handlers']['wsbroadcast']['filename'] = '/var/log/tower/wsbroadcast.log'  # noqa
 LOGGING['handlers']['task_system']['filename'] = '/var/log/tower/task_system.log'  # noqa
 LOGGING['handlers']['management_playbooks']['filename'] = '/var/log/tower/management_playbooks.log'  # noqa
 LOGGING['handlers']['system_tracking_migrations']['filename'] = '/var/log/tower/tower_system_tracking_migrations.log'  # noqa