Add custom (security) metrics

api/prometheus_metrics.py

@@ -0,0 +1,74 @@
+"""Prometheus metrics used by the fragalysis API module.
+"""
+from prometheus_client import Counter
+
+
+class PrometheusMetrics:
+    """A static class to hold the Prometheus metrics for the fragalysis API module.
+    Each metric has its own static method to adjust it.
+    """
+
+    # Create, and initialise the metrics for this module
+    ssh_tunnels = Counter(
+        'fragalysis_ssh_tunnels',
+        'Total number of SSH tunnels created',
+    )
+    ssh_tunnels.reset()
+    ssh_tunnel_failures = Counter(
+        'fragalysis_ssh_tunnel_failures',
+        'Total number of SSH tunnel failures',
+    )
+    ssh_tunnel_failures.reset()
+    ispyb_connections = Counter(
+        'fragalysis_ispyb_connections',
+        'Total number of ISpyB connections',
+    )
+    ispyb_connections.reset()
+    ispyb_connection_attempts = Counter(
+        'fragalysis_ispyb_connection_attempts',
+        'Total number of ISpyB connection attempts (after initial failure)',
+    )
+    ispyb_connection_attempts.reset()
+    ispyb_connection_failures = Counter(
+        'fragalysis_ispyb_connection_failures',
+        'Total number of ISpyB connection failures',
+    )
+    ispyb_connection_failures.reset()
+    proposal_cache_hit = Counter(
+        'fragalysis_proposal_cache_hit',
+        'Total number of proposal cache hits (avoiding new connections)',
+    )
+    proposal_cache_hit.reset()
+    proposal_cache_miss = Counter(
+        'fragalysis_proposal_cache_miss',
+        'Total number of proposal cache misses (forcing a new connection)',
+    )
+    proposal_cache_miss.reset()
+
+    @staticmethod
+    def new_tunnel():
+        PrometheusMetrics.ssh_tunnels.inc()
+
+    @staticmethod
+    def failed_tunnel():
+        PrometheusMetrics.ssh_tunnel_failures.inc()
+
+    @staticmethod
+    def new_ispyb_connection():
+        PrometheusMetrics.ispyb_connections.inc()
+
+    @staticmethod
+    def new_ispyb_connection_attempt():
+        PrometheusMetrics.ispyb_connection_attempts.inc()
+
+    @staticmethod
+    def failed_ispyb_connection():
+        PrometheusMetrics.ispyb_connection_failures.inc()
+
+    @staticmethod
+    def new_proposal_cache_hit():
+        PrometheusMetrics.proposal_cache_hit.inc()
+
+    @staticmethod
+    def new_proposal_cache_miss():
+        PrometheusMetrics.proposal_cache_miss.inc()

api/remote_ispyb_connector.py

@@ -13,6 +13,8 @@
 )
 from pymysql.err import OperationalError
 
+from .prometheus_metrics import PrometheusMetrics
+
 logger: logging.Logger = logging.getLogger(__name__)
 
 # Timeout to allow the pymysql.connect() method to connect to the DB.
@@ -176,15 +178,18 @@ def remote_connect(
                     )
                 logger.warning('Unexpected %s', repr(e))
                 connect_attempts += 1
+                PrometheusMetrics.new_ispyb_connection_attempt()
                 time.sleep(PYMYSQL_EXCEPTION_RECONNECT_DELAY_S)
 
         if self.conn is not None:
             if connect_attempts > 0:
                 logger.info('Connected')
+            PrometheusMetrics.new_ispyb_connection()
             self.conn.autocommit = True
         else:
             if connect_attempts > 0:
                 logger.info('Failed to connect')
+            PrometheusMetrics.failed_ispyb_connection()
             self.server.stop()
             raise ISPyBConnectionException
         self.last_activity_ts = time.time()

api/security.py

@@ -17,6 +17,7 @@
 
 from viewer.models import Project
 
+from .prometheus_metrics import PrometheusMetrics
 from .remote_ispyb_connector import SSHConnector
 
 logger: logging.Logger = logging.getLogger(__name__)
@@ -47,10 +48,12 @@ def has_expired(username) -> bool:
                 # User's not known,
                 # initialise an entry that will automatically expire
                 CachedContent._timers[username] = now
+                PrometheusMetrics.new_proposal_cache_hit()
             if CachedContent._timers[username] <= now:
                 has_expired = True
                 # Expired, reset the expiry time
                 CachedContent._timers[username] = now + CachedContent._cache_period
+                PrometheusMetrics.new_proposal_cache_miss()
         return has_expired
 
     @staticmethod
@@ -105,8 +108,10 @@ def get_remote_conn(force_error_display=False) -> Optional[SSHConnector]:
             logger.exception("Got the following exception creating Connector...")
     if conn:
         logger.debug("Got remote connector")
+        PrometheusMetrics.new_tunnel()
     else:
         logger.debug("Failed to get a remote connector")
+        PrometheusMetrics.failed_tunnel()
 
     return conn
 
@@ -140,8 +145,10 @@ def get_conn(force_error_display=False) -> Optional[Connector]:
             logger.exception("Got the following exception creating Connector...")
     if conn:
         logger.debug("Got connector")
+        PrometheusMetrics.new_ispyb_connection()
     else:
         logger.debug("Did not get a connector")
+        PrometheusMetrics.failed_ispyb_connection()
 
     return conn
 

fragalysis/settings.py

@@ -67,7 +67,7 @@
 import os
 import sys
 from datetime import timedelta
-from typing import List
+from typing import List, Optional
 
 import sentry_sdk
 from sentry_sdk.integrations.celery import CeleryIntegration
@@ -535,6 +535,16 @@
 NEO4J_QUERY: str = os.environ.get("NEO4J_QUERY", "neo4j")
 NEO4J_AUTH: str = os.environ.get("NEO4J_AUTH", "neo4j/neo4j")
 
+# Does it look like we're running in Kubernetes?
+# If so, let's get the namespace we're in - it will provide
+# useful discrimination material in log/metrics messages.
+# If there is no apparent namespace the variable will be 'None'.
+OUR_KUBERNETES_NAMESPACE: Optional[str] = None
+_NS_FILENAME: str = '/var/run/secrets/kubernetes.io/serviceaccount/namespace'
+if os.path.isfile(_NS_FILENAME):
+    with open(_NS_FILENAME, 'rt', encoding='utf8') as ns_file:
+        OUR_KUBERNETES_NAMESPACE = ns_file.read().strip()
+
 # These flags are used in the upload_tset form as follows.
 # Proposal Supported | Proposal Required | Proposal / View fields
 # Y                  | Y                 | Shown / Required