Add socket backlog metric

docs/source/settings.rst

@@ -461,6 +461,19 @@ if not provided).
 
 .. versionadded:: 19.2
 
+.. _enable-backlog-metric:
+
+``enable_backlog_metric``
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Command line:** ``--enable-backlog-metric``
+
+**Default:** ``False``
+
+Enable socket backlog metric (only supported on Linux).
+
+.. versionadded:: 23.1
+
 Process Naming
 --------------
 

gunicorn/arbiter.py

@@ -583,6 +583,16 @@ def manage_workers(self):
                                   "value": active_worker_count,
                                   "mtype": "gauge"})
 
+        if self.cfg.enable_backlog_metric:
+            backlog = sum(sock.get_backlog() or 0
+                          for sock in self.LISTENERS)
+
+            if backlog >= 0:
+                self.log.debug("socket backlog: {0}".format(backlog),
+                               extra={"metric": "gunicorn.backlog",
+                                      "value": backlog,
+                                      "mtype": "histogram"})
+
     def spawn_worker(self):
         self.worker_age += 1
         worker = self.worker_class(self.worker_age, self.pid, self.LISTENERS,

gunicorn/config.py

@@ -1693,6 +1693,20 @@ class StatsdPrefix(Setting):
     """
 
 
+class BacklogMetric(Setting):
+    name = "enable_backlog_metric"
+    section = "Logging"
+    cli = ["--enable-backlog-metric"]
+    validator = validate_bool
+    default = False
+    action = "store_true"
+    desc = """\
+    Enable socket backlog metric (only supported on Linux).
+
+    .. versionadded:: 23.1
+    """
+
+
 class Procname(Setting):
     name = "proc_name"
     section = "Process Naming"

gunicorn/instrument/statsd.py

@@ -17,6 +17,7 @@
 GAUGE_TYPE = "gauge"
 COUNTER_TYPE = "counter"
 HISTOGRAM_TYPE = "histogram"
+TIMER_TYPE = "timer"
 
 
 class Statsd(Logger):
@@ -80,6 +81,8 @@ def log(self, lvl, msg, *args, **kwargs):
                         self.increment(metric, value)
                     elif typ == HISTOGRAM_TYPE:
                         self.histogram(metric, value)
+                    elif typ == TIMER_TYPE:
+                        self.timer(metric, value)
                     else:
                         pass
 
@@ -101,7 +104,7 @@ def access(self, resp, req, environ, request_time):
             status = status.decode('utf-8')
         if isinstance(status, str):
             status = int(status.split(None, 1)[0])
-        self.histogram("gunicorn.request.duration", duration_in_ms)
+        self.timer("gunicorn.request.duration", duration_in_ms)
         self.increment("gunicorn.requests", 1)
         self.increment("gunicorn.request.status.%d" % status, 1)
 
@@ -116,9 +119,12 @@ def increment(self, name, value, sampling_rate=1.0):
     def decrement(self, name, value, sampling_rate=1.0):
         self._sock_send("{0}{1}:-{2}|c|@{3}".format(self.prefix, name, value, sampling_rate))
 
-    def histogram(self, name, value):
+    def timer(self, name, value):
         self._sock_send("{0}{1}:{2}|ms".format(self.prefix, name, value))
 
+    def histogram(self, name, value):
+        self._sock_send("{0}{1}:{2}|h".format(self.prefix, name, value))
+
     def _sock_send(self, msg):
         try:
             if isinstance(msg, str):

gunicorn/sock.py

@@ -9,8 +9,10 @@
 import stat
 import sys
 import time
+import struct
 
 from gunicorn import util
+PLATFORM = sys.platform
 
 
 class BaseSocket:
@@ -70,6 +72,9 @@ def close(self):
 
         self.sock = None
 
+    def get_backlog(self):
+        return -1
+
 
 class TCPSocket(BaseSocket):
 
@@ -88,6 +93,23 @@ def set_options(self, sock, bound=False):
         sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
         return super().set_options(sock, bound=bound)
 
+    if PLATFORM == "linux":
+        def get_backlog(self):
+            if self.sock:
+                # tcp_info struct from include/uapi/linux/tcp.h
+                fmt = 'B' * 8 + 'I' * 24
+                try:
+                    tcp_info_struct = self.sock.getsockopt(socket.IPPROTO_TCP,
+                                                           socket.TCP_INFO, 104)
+                    # 12 is tcpi_unacked
+                    return struct.unpack(fmt, tcp_info_struct)[12]
+                except AttributeError:
+                    pass
+            return 0
+    else:
+        def get_backlog(self):
+            return -1
+
 
 class TCP6Socket(TCPSocket):