From b5737f47c1902945b676091a6996118514f940d9 Mon Sep 17 00:00:00 2001
From: cproudlock <cproudlock@localhost>
Date: Thu, 3 Sep 2026 17:35:48 -0400
Subject: [PATCH] Serve gunicorn with threaded workers so a retry storm cannot
 starve the UI

Taken from the other site, who hit it in production today (their commit
db75ca7): agents that fail the TLS handshake retry in a loop, and with four
blocking sync workers a handful of them pinned every worker. Real UI requests
queued for 20+ seconds while the CPU sat idle - the box was not busy, it was
blocked.

gthread, 4 workers x 8 threads, and a 30-second timeout rather than 60. A stuck
connection now holds one thread out of 32 instead of one worker out of four, and
it lets go in half the time, which is the difference between absorbing a retry
storm and queueing behind it.

Their server, our agent. The client behaviour that caused it is the endpoint
agent, so it will reach any site running one - which makes this a product fix
rather than something that belongs on one site's branch.

Applied to the Dockerfile as well as docker-compose.yml. A compose file
overrides the image CMD, but docker-compose.airgap.yml, .channel.yml and
.pull.yml do not specify a command at all, so they take the image default - and
so does anybody running the image by hand. Fixing only compose would have left
every air-gapped install on sync workers.

WJ production is unaffected either way: it runs waitress under IIS, already with
--threads=8, so it was never exposed to this. That is worth knowing before
anyone treats it as a prod hardening item that was missing.

Scope: product
---
 Dockerfile         |  8 +++++++-
 docker-compose.yml | 18 +++++++++++++++++-
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 91904fdb..7fd22a02 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -139,10 +139,16 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
         -H "Content-Type: application/json" -d '{}' \
         | grep -q "VALIDATION_ERROR" || exit 1
 
+# Threaded workers, matching docker-compose.yml - see the reasoning there. A
+# compose file overrides this CMD, but an image run without one (a plain
+# `docker run`, a site's own orchestrator) would otherwise still get the
+# blocking sync default and the same starvation.
 CMD ["gunicorn", \
      "--bind", "0.0.0.0:5001", \
+     "--worker-class", "gthread", \
      "--workers", "4", \
-     "--timeout", "60", \
+     "--threads", "8", \
+     "--timeout", "30", \
      "--access-logfile", "-", \
      "--error-logfile", "-", \
      "wsgi:app"]
diff --git a/docker-compose.yml b/docker-compose.yml
index 326966a5..8af86016 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -79,10 +79,26 @@ services:
       - gunicorn
       - --bind
       - 0.0.0.0:5001
+      # Threaded workers, not the blocking sync default. An agent that fails
+      # the TLS handshake retries in a loop, and with 4 sync workers a handful
+      # of those pinned every worker: real UI requests queued 20+ seconds while
+      # the CPU sat idle. A stuck connection now holds one THREAD, and 32
+      # (4 x 8) have to be exhausted before a person waits.
+      #
+      # Found in production at the other site (their commit db75ca7). Their
+      # server, our agent - so the same client behaviour will reach any site
+      # running the endpoint agent, and the fix belongs in the product rather
+      # than on one branch.
+      - --worker-class
+      - gthread
       - --workers
       - "4"
+      - --threads
+      - "8"
+      # 30, not 60: a hung connection frees its thread in half the time, which
+      # is what decides whether a retry storm is absorbed or queues.
       - --timeout
-      - "60"
+      - "30"
       - --access-logfile
       - "-"
       - --error-logfile