shopdb-flask/shopdb/plugins/recycle.py

"""Recycle a worker when the set of enabled plugins changes underneath it.

WHY THIS EXISTS. Enabling a plugin cannot make its routes live in a running
process: Flask refuses register_blueprint once the app has served a request, and
even if it did not, each gunicorn worker holds its own app object, so registering
in the worker that served the toggle would leave the others answering 404 for the
same URL. The operator therefore had to restart the app by hand, and the UI had
to tell them so.

The fix is the primitive gunicorn already uses for --reload: a worker does not
ask anyone to reload it, it EXITS, and the supervisor replaces it. Applied to a
single file that is cheap to stat, that is production-grade rather than a dev
convenience.

WHY NOT SIGNAL THE MASTER. Sending SIGHUP to os.getppid() assumes gunicorn, a
POSIX host, and that the parent is in fact the master; under the dev server that
parent is the operator's shell. Exiting instead needs none of those assumptions:
whatever started the process decides what replaces it. It is also ROLLING - each
worker recycles when it next serves a request, so the others keep serving - where
signalling the master drains every worker at once.

SCOPE. This runs under gunicorn only (see SUPERVISED). The Windows/IIS path
waitress behind HttpPlatformHandler is single-process and has never been
exercised here, so it keeps the manual restart and the UI keeps saying so.

WHAT IT FIXES BEYOND THE RESTART. Every worker converges on plugins.json within
one request of a change, so the per-worker registry skew that caused enables to
be lost and dependency checks to be refused stops being a standing condition.
"""

import hashlib
import logging
import os
import random
import signal
import threading
import time

logger = logging.getLogger(__name__)

# Do not recycle a worker younger than this. A worker that exits immediately on
# boot cannot serve anything, and a fingerprint that is somehow permanently
# "changed" would otherwise spin the supervisor respawning workers forever.
MIN_UPTIME_SECONDS = 15

# Ignore a change newer than this. plugins.json is written under an exclusive
# lock and replaced atomically, but a burst of enables (the wizard's "enable all
# recommended") rewrites it once per plugin. Waiting lets the burst settle so a
# worker recycles once at the end rather than after every plugin.
DEBOUNCE_SECONDS = 3

# Per-worker spread, on top of the debounce. Without it every worker that
# serves a request in the same second clears the same threshold and they all
# exit together, so the pool is briefly empty and the restart is not rolling at
# all - it is a synchronised outage. This is the same reason gunicorn pairs
# --max-requests with --max-requests-jitter. Chosen once per process.
JITTER_CEILING_SECONDS = 10

# Servers whose supervisor will bring the process back AND where the exit path
# below has been exercised. Anything else - the Werkzeug dev server above all -
# must never be exited from under the developer.
#
# WAITRESS IS DELIBERATELY ABSENT. The IIS path (HttpPlatformHandler running
# waitress) is single-process, so there is no worker to recycle: the process
# itself would have to exit and be relaunched. That is plausible but UNPROVEN -
# it has never run on Windows here - and HttpPlatformHandler stops relaunching
# after rapidFailsPerMinute (default 10), which would take the site down rather
# than reload it. Until it is tested on the Windows VM, IIS keeps the honest
# behaviour it always had: the operator restarts, and the UI says so. Claiming
# otherwise there would be worse than the manual step it replaces.
SUPERVISED = ('gunicorn/',)


def _fingerprint(path):
    """Content hash of plugins.json, or None when it is absent or unreadable.

    Content, not mtime: an atomic replace gives the new file its own mtime even
    when the contents are identical (every _save rewrites the whole file), so
    mtime alone would recycle workers over writes that changed nothing.
    """
    try:
        with open(path, 'rb') as handle:
            return hashlib.sha256(handle.read()).hexdigest()
    except FileNotFoundError:
        return None
    except OSError:
        # Unreadable is not "changed". Returning None on a transient error and
        # a hash the next time would recycle the worker for no reason.
        return 'unreadable'


def _supervised(environ):
    """True when something will restart this process after it exits."""
    server = str(environ.get('SERVER_SOFTWARE') or '')
    return server.startswith(SUPERVISED)


def _exit_gracefully():
    """Leave, letting in-flight requests finish, and let the supervisor return.

    SIGTERM to our own pid is exactly what the gunicorn master sends for a
    graceful worker shutdown: the worker stops accepting, drains what it is
    holding, exits, and is respawned. Only reachable under gunicorn, because
    _supervised() gates on it.

    Runs on a short timer rather than inline so the response that triggered it
    is already on its way out.
    """
    def _leave():
        # Not load-bearing under gunicorn, which finishes in-flight requests
        # during its graceful drain regardless. It only buys a moment for the
        # triggering response to reach the wire before anything changes.
        time.sleep(0.5)
        try:
            # SIGTERM to our own pid is precisely what the gunicorn master
            # sends for a graceful worker shutdown: stop accepting, drain what
            # we are holding, exit, and be respawned. _supervised() has already
            # established that we are under gunicorn, so this branch is the
            # only one that can be reached.
            os.kill(os.getpid(), signal.SIGTERM)
        except Exception:
            # Last resort only. sys.exit() would be wrong here: raising
            # SystemExit in a non-main thread ends the THREAD, not the process,
            # and the process would carry on believing it had recycled.
            os._exit(0)

    threading.Thread(target=_leave, daemon=True, name='plugin-recycle').start()


def init_recycle(app):
    """Watch plugins.json and recycle this worker when it changes.

    Registered as an after_request hook: the check is a stat and a hash of a
    file measured in kilobytes, and it only runs once the response is built.
    """
    if app.config.get('TESTING'):
        return

    state_file = os.path.join(app.instance_path, 'plugins.json')
    booted_fingerprint = _fingerprint(state_file)
    booted_at = time.monotonic()
    settle_after = DEBOUNCE_SECONDS + random.uniform(0, JITTER_CEILING_SECONDS)
    # One worker, one exit. Without this two concurrent requests can both see
    # the change and both start the timer.
    leaving = threading.Event()

    @app.after_request
    def _recycle_if_plugins_changed(response):
        if leaving.is_set():
            return response
        if time.monotonic() - booted_at < MIN_UPTIME_SECONDS:
            return response
        try:
            from flask import request
            if not _supervised(request.environ):
                return response
        except Exception:
            return response

        current = _fingerprint(state_file)
        if current == booted_fingerprint or current == 'unreadable':
            return response

        try:
            age = time.time() - os.path.getmtime(state_file)
        except OSError:
            return response
        if age < settle_after:
            return response

        if leaving.is_set():
            return response
        leaving.set()
        logger.warning(
            'Enabled plugins changed on disk; recycling this worker so the '
            'change takes effect. In-flight requests are finished first and '
            'the supervisor starts a replacement.')
        _exit_gracefully()
        return response