ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge wilm: gevent migration, endpoint telemetry phase 2, backup/restore

Browse Source 
Brings in 98 commits from wilm (origin/wilm @ 811e6ca61a):

Major features:
- Move API to gevent workers for better concurrency
- Endpoint agent startup jitter (prevents thundering herd)
- Complete backup/restore with restic
- Endpoint telemetry phase 2 (disk health, mapped drives, user context)
- PC Detail UI rework with new cards and tabs
- Dell Command actions and Corporate Tools tab
- Agent version column in Fleet view

Conflict resolution:
- Backup/restore: Accepted wilm implementation (10 files)
- Agent startup jitter: Accepted wilm v1.12.0
- PC Types list fix: Accepted wilm version
- Endpoint plugin domain clamping: Kept ours (defensive)
- Test files: Accepted wilm versions for consistency

Testing required:
- Verify gevent handles concurrent requests
- Test backup/restore functionality
- Verify agent startup jitter timing
- Test endpoint telemetry collection
- Validate PC Detail UI changes

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Proudlock
2026-09-06 21:23:42 -04:00
parent 95abe72729 811e6ca61a
commit a6af09858c
32 changed files with 986 additions and 710 deletions
Download Patch File Download Diff File Expand all files Collapse all files

287
shopdb/core/api/shopdbbackups.py
View File

@@ -17,8 +17,6 @@ load rows that reference a column the live schema has not applied yet.
import json
import os
import threading
import time
import subprocess
from datetime import datetime, timezone
@@ -87,30 +85,6 @@ def _backups_root():
return os.environ.get(BACKUPS_ROOT_ENV, DEFAULT_BACKUPS_ROOT)
def _backups_configured():
"""Whether this deployment has a backup store behind the feature.
The overlay is a compose profile: a third container writes the sets onto a
shared volume, and the api container only ever reads them and drops a
sentinel. A Windows/IIS install has no such container, and the default root
resolves there to a drive-letter path nothing creates - so every button
either 500s on PermissionError or reports 'queued' for a run that will
never happen. Answer the caller honestly instead.
"""
return os.path.isdir(_backups_root())
def _unavailable_response():
# 404 + NOT_FOUND is the established shape for "this feature is not
# enabled here" (the configuration-policy routes answer the same way), and
# the UI already tells a 404 apart from a real failure.
return error_response(
ErrorCodes.NOT_FOUND,
'Site Backups is not configured on this deployment. It needs the '
'backup overlay and a readable backups root (%s).' % BACKUPS_ROOT_ENV,
http_code=404)
def _read_json(path):
"""Load a JSON file, or None when it is absent or unreadable."""
try:
@@ -184,106 +158,6 @@ def _append_history(kind, outcome, setname, message):
pass
# The post-restore upgrade is claimed by renaming the sentinel. rename(2) is
# atomic, so exactly one claimant wins even with several gunicorn workers and a
# browser poll all racing - which matters because the loser would otherwise run
# Alembic concurrently against the same database.
UPGRADE_SENTINEL = '.upgrade-requested'
UPGRADE_CLAIM = '.upgrade-inprogress'
UPGRADE_FAILED = '.upgrade-failed'
def _claim_upgrade():
"""Take exclusive ownership of a pending upgrade, or return None.
Returns the set name on success. Losing the race is not an error: it means
another worker is already finalizing the same restore.
"""
root = _backups_root()
sentinel = os.path.join(root, UPGRADE_SENTINEL)
claim = os.path.join(root, UPGRADE_CLAIM)
try:
os.rename(sentinel, claim)
except OSError:
return None
try:
with open(claim, 'r', encoding='utf-8') as handle:
return handle.read().strip() or 'unknown'
except OSError:
return 'unknown'
def _finalize_claimed(setname):
"""Run the upgrade for a claimed restore and record how it ended.
A failure parks the claim as .upgrade-failed rather than putting it back:
re-arming it here would retry a migration that just failed, every poll,
forever. The operator re-arms it explicitly through the finalize endpoint.
"""
root = _backups_root()
claim = os.path.join(root, UPGRADE_CLAIM)
_write_status('restore', 'upgrading', setname,
'Upgrading restored schema to code')
try:
_run_schema_upgrade()
except Exception as error: # noqa: BLE001 - report any upgrade failure
message = 'Schema upgrade failed: ' + str(error)
try:
os.replace(claim, os.path.join(root, UPGRADE_FAILED))
except OSError:
pass
_write_status('restore', 'failed', setname, message)
_append_history('restore', 'failed', setname, message)
current_app.logger.error('Post-restore schema upgrade failed: %s', error)
return False
try:
os.remove(claim)
except OSError:
pass
_write_status('restore', 'done', setname, 'Restore complete')
_append_history('restore', 'success', setname, 'Restore complete')
current_app.logger.info('Restore finalized: %s', setname)
return True
def start_restore_finalizer(app):
"""Watch for a restore awaiting its schema upgrade, and run it.
WHY THIS EXISTS: the restore itself runs in the Flask-less backup sidecar,
which loads the dump and drops a sentinel because it cannot run
'flask db upgrade'. Until this watcher, the ONLY thing that acted on that
sentinel was the browser poll loop of the modal that queued the restore -
so closing the tab, or letting a token expire, during the minutes a load
takes left the data at the old revision under new code, with every query on
a newer column failing and nothing retrying.
Runs in the api container because that is where the app context and the
engine live, and out of band rather than in a request because Alembic can
outrun the gunicorn worker timeout and get killed mid-chain.
"""
interval = max(1, int(os.environ.get('SHOPDB_FINALIZE_POLL_SECONDS', '10')))
def _loop():
while True:
time.sleep(interval)
try:
with app.app_context():
if not _backups_configured():
continue
setname = _claim_upgrade()
if setname:
_finalize_claimed(setname)
except Exception: # noqa: BLE001 - a watcher must not die
try:
app.logger.exception('Restore finalizer pass failed')
except Exception: # noqa: BLE001
pass
thread = threading.Thread(target=_loop, name='restore-finalizer', daemon=True)
thread.start()
return thread
def _run_schema_upgrade():
"""Bring the just-restored database up to the running code, in-process.
@@ -383,63 +257,6 @@ def _known_core_revisions():
return set()
def _known_plugin_revisions():
"""Every revision each plugin chain knows, as {plugin: {revisions}}.
Read by text scan rather than through Alembic. Each plugin chain has its own
env and version table, so loading them properly would mean building 17
ScriptDirectories on a request that only needs to answer "is this id in
there" - and manifest.py already reads the same trees the same way, so the
two agree by construction.
"""
root = os.path.dirname(current_app.root_path)
plugins_dir = os.path.join(root, 'plugins')
known = {}
if not os.path.isdir(plugins_dir):
return known
for name in sorted(os.listdir(plugins_dir)):
versions = os.path.join(plugins_dir, name, 'migrations', 'versions')
if not os.path.isdir(versions):
continue
revisions = set()
for filename in os.listdir(versions):
if not filename.endswith('.py'):
continue
try:
with open(os.path.join(versions, filename), 'r',
encoding='utf-8') as handle:
for line in handle:
stripped = line.strip()
if stripped.startswith('revision ='):
revisions.add(
stripped.split('=', 1)[1].strip().strip('"\''))
break
except OSError:
continue
if revisions:
known[name] = revisions
return known
def _plugin_data_is_known(manifest):
"""Whether this build knows every plugin revision the set's data sits at.
A set with no `dataplugins` predates the stamp; its plugin revisions are
unknowable, so it keeps the old core-only verdict rather than being marked
unrestorable retroactively.
"""
stamped = manifest.get('dataplugins') or {}
if not isinstance(stamped, dict) or not stamped:
return True
known = _known_plugin_revisions()
for name, revision in stamped.items():
if not revision:
continue
if revision not in known.get(name, ()):
return False
return True
def _is_compatible(manifest, current):
"""Decide whether a set restores onto THIS build.
@@ -452,34 +269,17 @@ def _is_compatible(manifest, current):
revision, the dump is from a NEWER schema than this build and would 500 on
restored rows, so we block until the code is upgraded.
The same question is asked of every PLUGIN chain, because plugin schemas
move independently: code rolled back to an older plugin revision can sit at
an identical core head while the dump carries plugin tables it cannot
migrate.
Legacy sets predate `datarev`; fall back to the old head-subset check on
`schemarev` so they stay listable and restorable as before.
"""
if not manifest:
return False
# A manifest that CARRIES the key but left it empty is not a legacy set: it
# is a set whose data revision could not be read when it was taken. Falling
# through to the code-to-code check would badge it restorable on no
# evidence, using exactly the comparison the datarev work replaced. Only a
# manifest with no datarev key at all predates the stamp.
if 'datarev' in manifest and not manifest.get('datarev'):
return False
datarev = manifest.get('datarev')
if datarev:
known = _known_core_revisions()
if not known:
return False
# Core AND every plugin chain. Core alone said yes to a set whose plugin
# data the code had never heard of, which loads and then dies during the
# plugin upgrade with the old data already gone.
return datarev in known and _plugin_data_is_known(manifest)
return datarev in known
setrevs = set(manifest.get('schemarev', {}).get('core', []))
currevs = set(current.get('core', []))
if not setrevs or not currevs:
@@ -539,7 +339,6 @@ def list_backups():
'retention': RETENTION,
'lastgoodat': (laststgood or {}).get('takenat'),
'count': len(sets),
'configured': _backups_configured(),
}
return success_response(data=sets, meta=meta)
@@ -574,9 +373,6 @@ def trigger_backup():
up. Returning isrunning=false with a clear message keeps the UI honest
rather than pretending the run started inside this process.
"""
if not _backups_configured():
return _unavailable_response()
root = _backups_root()
try:
os.makedirs(root, exist_ok=True)
@@ -605,9 +401,6 @@ def restore_backup(snapshotid):
The actual restore is performed by scripts/backup/restore.sh in the backup
container; here we validate and record intent.
"""
if not _backups_configured():
return _unavailable_response()
root = _backups_root()
setdir = os.path.join(root, os.path.basename(snapshotid))
if not os.path.isdir(setdir):
@@ -645,53 +438,52 @@ def restore_backup(snapshotid):
@shopdbbackups_bp.route('/finalize-restore', methods=['POST'])
@require_role('admin')
def finalize_restore():
"""Make sure a restore awaiting its schema upgrade is queued to be finalized.
"""Run the post-restore schema upgrade in the app context.
This used to RUN the upgrade inline, which made two things true that should
not have been: a restore only completed while a browser tab stayed open on
it, and the upgrade ran inside a request, where Alembic can outrun the
gunicorn worker timeout and be killed part-way through the chain, leaving a
half-applied revision the next attempt trips over.
The restore itself runs in the Flask-less backup sidecar, which loads the
dump and then drops an .upgrade-requested sentinel because it cannot run
'flask db upgrade' itself. This endpoint is the api-side half: it brings the
just-restored DATA up to the running CODE with the exact idempotent commands
the deploy path uses, then stamps the final 'done'. Without it a restored
dump that sits at an older revision leaves every newer-migration table
throwing 1054 Unknown column.
The work now belongs to the finalizer watcher in this container
(start_restore_finalizer). This endpoint only ensures something is pending
for it to claim, so it is safe to call repeatedly, and it is how an operator
retries an upgrade that failed.
Idempotent and safe to call when nothing is pending: if there is no
.upgrade-requested sentinel it reports there was nothing to finalize.
"""
if not _backups_configured():
return _unavailable_response()
root = _backups_root()
sentinel = os.path.join(root, UPGRADE_SENTINEL)
claim = os.path.join(root, UPGRADE_CLAIM)
failed = os.path.join(root, UPGRADE_FAILED)
if os.path.exists(claim):
sentinel = os.path.join(root, '.upgrade-requested')
try:
with open(sentinel, 'r', encoding='utf-8') as handle:
setname = handle.read().strip()
except OSError:
return success_response(
data={'queued': True, 'inprogress': True},
message='The schema upgrade is already running.')
data={'finalized': False},
message='No restore is awaiting schema upgrade.')
if os.path.exists(sentinel):
return success_response(
data={'queued': True, 'inprogress': False},
message='The schema upgrade is queued and will run shortly.')
# A previous attempt failed and parked itself. Re-arming is the retry.
if os.path.exists(failed):
try:
os.replace(failed, sentinel)
except OSError as error:
return error_response(ErrorCodes.INTERNAL_ERROR,
'Could not re-queue the schema upgrade: ' + str(error),
http_code=500)
AuditLog.log('restore-finalize-retried', 'Backup')
return success_response(
data={'queued': True, 'inprogress': False, 'retried': True},
message='The failed schema upgrade has been queued to run again.')
_write_status('restore', 'upgrading', setname,
'Upgrading restored schema to code')
try:
_run_schema_upgrade()
except Exception as error: # noqa: BLE001 - report any upgrade failure
_write_status('restore', 'failed', setname,
'Schema upgrade failed: ' + str(error))
_append_history('restore', 'failed', setname,
'Schema upgrade failed: ' + str(error))
return error_response(ErrorCodes.INTERNAL_ERROR,
'Post-restore schema upgrade failed: ' + str(error),
http_code=500)
try:
os.remove(sentinel)
except OSError:
pass
_write_status('restore', 'done', setname, 'Restore complete')
_append_history('restore', 'success', setname, 'Restore complete')
AuditLog.log('restore-finalized', 'Backup', entityname=setname)
return success_response(
data={'queued': False},
message='No restore is awaiting schema upgrade.')
data={'finalized': True, 'snapshotid': setname},
message='Restore finalized: schema upgraded to the running code.')
@shopdbbackups_bp.route('/status', methods=['GET'])
@@ -704,7 +496,6 @@ def get_status():
value is None when that kind of run has never been requested.
"""
return success_response(data={
'configured': _backups_configured(),
'backup': _read_status('backup'),
'restore': _read_status('restore'),
})