"""Recycle a worker when the set of enabled plugins changes underneath it. WHY THIS EXISTS. Enabling a plugin cannot make its routes live in a running process: Flask refuses register_blueprint once the app has served a request, and even if it did not, each gunicorn worker holds its own app object, so registering in the worker that served the toggle would leave the others answering 404 for the same URL. The operator therefore had to restart the app by hand, and the UI had to tell them so. The fix is the primitive gunicorn already uses for --reload: a worker does not ask anyone to reload it, it EXITS, and the supervisor replaces it. Applied to a single file that is cheap to stat, that is production-grade rather than a dev convenience. WHY NOT SIGNAL THE MASTER. Sending SIGHUP to os.getppid() assumes gunicorn, a POSIX host, and that the parent is in fact the master; under the dev server that parent is the operator's shell. Exiting instead needs none of those assumptions: whatever started the process decides what replaces it. It is also ROLLING - each worker recycles when it next serves a request, so the others keep serving - where signalling the master drains every worker at once. SCOPE. This runs under gunicorn only (see SUPERVISED). The Windows/IIS path waitress behind HttpPlatformHandler is single-process and has never been exercised here, so it keeps the manual restart and the UI keeps saying so. WHAT IT FIXES BEYOND THE RESTART. Every worker converges on plugins.json within one request of a change, so the per-worker registry skew that caused enables to be lost and dependency checks to be refused stops being a standing condition. """ import hashlib import logging import os import random import signal import threading import time logger = logging.getLogger(__name__) # Do not recycle a worker younger than this. A worker that exits immediately on # boot cannot serve anything, and a fingerprint that is somehow permanently # "changed" would otherwise spin the supervisor respawning workers forever. MIN_UPTIME_SECONDS = 15 # Ignore a change newer than this. plugins.json is written under an exclusive # lock and replaced atomically, but a burst of enables (the wizard's "enable all # recommended") rewrites it once per plugin. Waiting lets the burst settle so a # worker recycles once at the end rather than after every plugin. DEBOUNCE_SECONDS = 3 # Per-worker spread, on top of the debounce. Without it every worker that # serves a request in the same second clears the same threshold and they all # exit together, so the pool is briefly empty and the restart is not rolling at # all - it is a synchronised outage. This is the same reason gunicorn pairs # --max-requests with --max-requests-jitter. Chosen once per process. JITTER_CEILING_SECONDS = 10 # Servers whose supervisor will bring the process back AND where the exit path # below has been exercised. Anything else - the Werkzeug dev server above all - # must never be exited from under the developer. # # WAITRESS IS DELIBERATELY ABSENT. The IIS path (HttpPlatformHandler running # waitress) is single-process, so there is no worker to recycle: the process # itself would have to exit and be relaunched. That is plausible but UNPROVEN - # it has never run on Windows here - and HttpPlatformHandler stops relaunching # after rapidFailsPerMinute (default 10), which would take the site down rather # than reload it. Until it is tested on the Windows VM, IIS keeps the honest # behaviour it always had: the operator restarts, and the UI says so. Claiming # otherwise there would be worse than the manual step it replaces. SUPERVISED = ('gunicorn/',) def _fingerprint(path): """Content hash of plugins.json, or None when it is absent or unreadable. Content, not mtime: an atomic replace gives the new file its own mtime even when the contents are identical (every _save rewrites the whole file), so mtime alone would recycle workers over writes that changed nothing. """ try: with open(path, 'rb') as handle: return hashlib.sha256(handle.read()).hexdigest() except FileNotFoundError: return None except OSError: # Unreadable is not "changed". Returning None on a transient error and # a hash the next time would recycle the worker for no reason. return 'unreadable' def _supervised(environ): """True when something will restart this process after it exits.""" server = str(environ.get('SERVER_SOFTWARE') or '') return server.startswith(SUPERVISED) def _exit_gracefully(): """Leave, letting in-flight requests finish, and let the supervisor return. SIGTERM to our own pid is exactly what the gunicorn master sends for a graceful worker shutdown: the worker stops accepting, drains what it is holding, exits, and is respawned. Only reachable under gunicorn, because _supervised() gates on it. Runs on a short timer rather than inline so the response that triggered it is already on its way out. """ def _leave(): # Not load-bearing under gunicorn, which finishes in-flight requests # during its graceful drain regardless. It only buys a moment for the # triggering response to reach the wire before anything changes. time.sleep(0.5) try: # SIGTERM to our own pid is precisely what the gunicorn master # sends for a graceful worker shutdown: stop accepting, drain what # we are holding, exit, and be respawned. _supervised() has already # established that we are under gunicorn, so this branch is the # only one that can be reached. os.kill(os.getpid(), signal.SIGTERM) except Exception: # Last resort only. sys.exit() would be wrong here: raising # SystemExit in a non-main thread ends the THREAD, not the process, # and the process would carry on believing it had recycled. os._exit(0) threading.Thread(target=_leave, daemon=True, name='plugin-recycle').start() def init_recycle(app): """Watch plugins.json and recycle this worker when it changes. Registered as an after_request hook: the check is a stat and a hash of a file measured in kilobytes, and it only runs once the response is built. """ if app.config.get('TESTING'): return state_file = os.path.join(app.instance_path, 'plugins.json') booted_fingerprint = _fingerprint(state_file) booted_at = time.monotonic() settle_after = DEBOUNCE_SECONDS + random.uniform(0, JITTER_CEILING_SECONDS) # One worker, one exit. Without this two concurrent requests can both see # the change and both start the timer. leaving = threading.Event() @app.after_request def _recycle_if_plugins_changed(response): if leaving.is_set(): return response if time.monotonic() - booted_at < MIN_UPTIME_SECONDS: return response try: from flask import request if not _supervised(request.environ): return response except Exception: return response current = _fingerprint(state_file) if current == booted_fingerprint or current == 'unreadable': return response try: age = time.time() - os.path.getmtime(state_file) except OSError: return response if age < settle_after: return response if leaving.is_set(): return response leaving.set() logger.warning( 'Enabled plugins changed on disk; recycling this worker so the ' 'change takes effect. In-flight requests are finished first and ' 'the supervisor starts a replacement.') _exit_gracefully() return response