Adjust metrics (tunnels and cache)

api/prometheus_metrics.py

@@ -11,37 +11,37 @@ class PrometheusMetrics:
     # Create, and initialise the metrics for this module
     ssh_tunnels = Counter(
         'fragalysis_ssh_tunnels',
-        'Total number of SSH tunnels created',
+        'Number of SSH tunnels successfully created',
     )
     ssh_tunnels.reset()
     ssh_tunnel_failures = Counter(
         'fragalysis_ssh_tunnel_failures',
-        'Total number of SSH tunnel failures',
+        'Number of SSH tunnel failures',
     )
     ssh_tunnel_failures.reset()
     ispyb_connections = Counter(
         'fragalysis_ispyb_connections',
-        'Total number of ISpyB connections',
+        'Number of ISpyB successful connections (excluding retries)',
     )
     ispyb_connections.reset()
     ispyb_connection_attempts = Counter(
         'fragalysis_ispyb_connection_attempts',
-        'Total number of ISpyB connection attempts (after initial failure)',
+        'Number of ISpyB connection retries (after initial failure)',
     )
     ispyb_connection_attempts.reset()
     ispyb_connection_failures = Counter(
         'fragalysis_ispyb_connection_failures',
-        'Total number of ISpyB connection failures',
+        'Number of ISpyB connection failures',
     )
     ispyb_connection_failures.reset()
     proposal_cache_hit = Counter(
         'fragalysis_proposal_cache_hit',
-        'Total number of proposal cache hits (avoiding new connections)',
+        'Number of proposal cache hits',
     )
     proposal_cache_hit.reset()
     proposal_cache_miss = Counter(
         'fragalysis_proposal_cache_miss',
-        'Total number of proposal cache misses (forcing a new connection)',
+        'Number of proposal cache misses',
     )
     proposal_cache_miss.reset()
 

api/remote_ispyb_connector.py

@@ -136,6 +136,7 @@ def remote_connect(
 
         logger.debug('Starting SSH server...')
         self.server.start()
+        PrometheusMetrics.new_tunnel()
         logger.debug('Started SSH server')
 
         # Try to connect to the database

api/security.py

@@ -48,12 +48,10 @@ def has_expired(username) -> bool:
                 # User's not known,
                 # initialise an entry that will automatically expire
                 CachedContent._timers[username] = now
-                PrometheusMetrics.new_proposal_cache_hit()
             if CachedContent._timers[username] <= now:
                 has_expired = True
                 # Expired, reset the expiry time
                 CachedContent._timers[username] = now + CachedContent._cache_period
-                PrometheusMetrics.new_proposal_cache_miss()
         return has_expired
 
     @staticmethod
@@ -103,17 +101,15 @@ def get_remote_conn(force_error_display=False) -> Optional[SSHConnector]:
     try:
         conn = SSHConnector(**credentials)
     except ISPyBConnectionException:
-        # Got SSH tunnel but the ISPyB connection failed
-        PrometheusMetrics.new_tunnel()
+        # The ISPyB connection failed.
+        # Nothing else to do here, metrics are already updated
+        pass
     except Exception:
         # Any other exception will be a problem with the SSH tunnel connection
         PrometheusMetrics.failed_tunnel()
         if logging.DEBUG >= logger.level or force_error_display:
             logger.info("credentials=%s", credentials)
             logger.exception("Got the following exception creating Connector...")
-    else:
-        # No exception - we must have created an SSH tunnel
-        PrometheusMetrics.new_tunnel()
 
     if conn:
         logger.debug("Got remote ISPyB connector")
@@ -246,12 +242,15 @@ def _run_query_with_connector(self, conn, user):
     def _get_proposals_for_user_from_ispyb(self, user):
         if CachedContent.has_expired(user.username):
             logger.info("Cache has expired for '%s'", user.username)
+            PrometheusMetrics.new_proposal_cache_miss()
             if conn := get_configured_connector():
                 logger.debug("Got a connector for '%s'", user.username)
                 self._get_proposals_from_connector(user, conn)
             else:
                 logger.warning("Failed to get a connector for '%s'", user.username)
                 self._mark_cache_collection_failure(user)
+        else:
+            PrometheusMetrics.new_proposal_cache_hit()
 
         # The cache has either been updated, has not changed or is empty.
         # Return what we have for the user. Public (open) proposals