From 17a85c5368b999e29695f0fd05fe1c631d24f1a2 Mon Sep 17 00:00:00 2001
From: Alan Christie <alan.christie@matildapeak.com>
Date: Thu, 23 May 2024 09:53:46 +0100
Subject: [PATCH] fix: Adjust metrics (tunnels and cache)

---
 api/prometheus_metrics.py     | 14 +++++++-------
 api/remote_ispyb_connector.py |  1 +
 api/security.py               | 13 ++++++-------
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/api/prometheus_metrics.py b/api/prometheus_metrics.py
index fc268a38..c2988760 100644
--- a/api/prometheus_metrics.py
+++ b/api/prometheus_metrics.py
@@ -11,37 +11,37 @@ class PrometheusMetrics:
     # Create, and initialise the metrics for this module
     ssh_tunnels = Counter(
         'fragalysis_ssh_tunnels',
-        'Total number of SSH tunnels created',
+        'Number of SSH tunnels successfully created',
     )
     ssh_tunnels.reset()
     ssh_tunnel_failures = Counter(
         'fragalysis_ssh_tunnel_failures',
-        'Total number of SSH tunnel failures',
+        'Number of SSH tunnel failures',
     )
     ssh_tunnel_failures.reset()
     ispyb_connections = Counter(
         'fragalysis_ispyb_connections',
-        'Total number of ISpyB connections',
+        'Number of ISpyB successful connections (excluding retries)',
     )
     ispyb_connections.reset()
     ispyb_connection_attempts = Counter(
         'fragalysis_ispyb_connection_attempts',
-        'Total number of ISpyB connection attempts (after initial failure)',
+        'Number of ISpyB connection retries (after initial failure)',
     )
     ispyb_connection_attempts.reset()
     ispyb_connection_failures = Counter(
         'fragalysis_ispyb_connection_failures',
-        'Total number of ISpyB connection failures',
+        'Number of ISpyB connection failures',
     )
     ispyb_connection_failures.reset()
     proposal_cache_hit = Counter(
         'fragalysis_proposal_cache_hit',
-        'Total number of proposal cache hits (avoiding new connections)',
+        'Number of proposal cache hits',
     )
     proposal_cache_hit.reset()
     proposal_cache_miss = Counter(
         'fragalysis_proposal_cache_miss',
-        'Total number of proposal cache misses (forcing a new connection)',
+        'Number of proposal cache misses',
     )
     proposal_cache_miss.reset()
 
diff --git a/api/remote_ispyb_connector.py b/api/remote_ispyb_connector.py
index d7b7a02e..c410a9fe 100644
--- a/api/remote_ispyb_connector.py
+++ b/api/remote_ispyb_connector.py
@@ -136,6 +136,7 @@ def remote_connect(
 
         logger.debug('Starting SSH server...')
         self.server.start()
+        PrometheusMetrics.new_tunnel()
         logger.debug('Started SSH server')
 
         # Try to connect to the database
diff --git a/api/security.py b/api/security.py
index 093b7b72..4b03a091 100644
--- a/api/security.py
+++ b/api/security.py
@@ -48,12 +48,10 @@ def has_expired(username) -> bool:
                 # User's not known,
                 # initialise an entry that will automatically expire
                 CachedContent._timers[username] = now
-                PrometheusMetrics.new_proposal_cache_hit()
             if CachedContent._timers[username] <= now:
                 has_expired = True
                 # Expired, reset the expiry time
                 CachedContent._timers[username] = now + CachedContent._cache_period
-                PrometheusMetrics.new_proposal_cache_miss()
         return has_expired
 
     @staticmethod
@@ -103,17 +101,15 @@ def get_remote_conn(force_error_display=False) -> Optional[SSHConnector]:
     try:
         conn = SSHConnector(**credentials)
     except ISPyBConnectionException:
-        # Got SSH tunnel but the ISPyB connection failed
-        PrometheusMetrics.new_tunnel()
+        # The ISPyB connection failed.
+        # Nothing else to do here, metrics are already updated
+        pass
     except Exception:
         # Any other exception will be a problem with the SSH tunnel connection
         PrometheusMetrics.failed_tunnel()
         if logging.DEBUG >= logger.level or force_error_display:
             logger.info("credentials=%s", credentials)
             logger.exception("Got the following exception creating Connector...")
-    else:
-        # No exception - we must have created an SSH tunnel
-        PrometheusMetrics.new_tunnel()
 
     if conn:
         logger.debug("Got remote ISPyB connector")
@@ -246,12 +242,15 @@ def _run_query_with_connector(self, conn, user):
     def _get_proposals_for_user_from_ispyb(self, user):
         if CachedContent.has_expired(user.username):
             logger.info("Cache has expired for '%s'", user.username)
+            PrometheusMetrics.new_proposal_cache_miss()
             if conn := get_configured_connector():
                 logger.debug("Got a connector for '%s'", user.username)
                 self._get_proposals_from_connector(user, conn)
             else:
                 logger.warning("Failed to get a connector for '%s'", user.username)
                 self._mark_cache_collection_failure(user)
+        else:
+            PrometheusMetrics.new_proposal_cache_hit()
 
         # The cache has either been updated, has not changed or is empty.
         # Return what we have for the user. Public (open) proposals