diff --git a/Dockerfile b/Dockerfile index 91904fdb..7fd22a02 100644 --- a/Dockerfile +++ b/Dockerfile @@ -139,10 +139,16 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ -H "Content-Type: application/json" -d '{}' \ | grep -q "VALIDATION_ERROR" || exit 1 +# Threaded workers, matching docker-compose.yml - see the reasoning there. A +# compose file overrides this CMD, but an image run without one (a plain +# `docker run`, a site's own orchestrator) would otherwise still get the +# blocking sync default and the same starvation. CMD ["gunicorn", \ "--bind", "0.0.0.0:5001", \ + "--worker-class", "gthread", \ "--workers", "4", \ - "--timeout", "60", \ + "--threads", "8", \ + "--timeout", "30", \ "--access-logfile", "-", \ "--error-logfile", "-", \ "wsgi:app"] diff --git a/docker-compose.yml b/docker-compose.yml index ec2628a8..03780a73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,18 +79,25 @@ services: - gunicorn - --bind - 0.0.0.0:5001 - # Threaded workers so one blocked request (e.g. an agent that fails the TLS - # handshake and retries in a loop) cannot pin a whole worker and starve the - # UI. With sync workers a handful of stuck clients exhausted all capacity. - - --worker-class - - gthread - - --workers - - "4" - - --threads - - "8" - # Shorter timeout so a hung connection frees its thread quickly instead of - # holding it for a full minute. - - --timeout +# Threaded workers, not the blocking sync default. An agent that fails +# the TLS handshake retries in a loop, and with 4 sync workers a handful +# of those pinned every worker: real UI requests queued 20+ seconds while +# the CPU sat idle. A stuck connection now holds one THREAD, and 32 +# (4 x 8) have to be exhausted before a person waits. +# +# Found in production at the other site (their commit db75ca7). Their +# server, our agent - so the same client behaviour will reach any site +# running the endpoint agent, and the fix belongs in the product rather +# than on one branch. +- --worker-class +- gthread +- --workers +- "4" +- --threads +- "8" +# 30, not 60: a hung connection frees its thread in half the time, which +# is what decides whether a retry storm is absorbed or queues. +- --timeout - "30" - --access-logfile - "-"