debe1fce69
The watcher scripts now write live status and append-only run history onto the shared volume; the api exposes them via /shopdbbackups/status and /history. The restore modal shows an indeterminate spinner with phase labels, and two cards list backup and restore run records. Timestamps render in UTC to match the Zulu-stamped set ids, the history directory is excluded from the set list, and a stale-status guard stops a previous run's done phase from collapsing the spinner on the next restore.
443 lines
17 KiB
Python
443 lines
17 KiB
Python
"""Backups API routes.
|
|
|
|
Admin-only surface over the site's backup sets. The heavy lifting (mysqldump,
|
|
restic snapshot of the instance tree, retention prune, integrity check) runs in
|
|
the one-shot backup container defined in docker-compose.backup.yml; this
|
|
blueprint is the read-mostly control plane the UI talks to.
|
|
|
|
Two recovery products stay firewalled, per the design notes:
|
|
- code rollback lives in git (tags/releases), NOT here
|
|
- data restore lives here, gated on schema compatibility via each set's manifest
|
|
|
|
Every backup set carries a manifest.json stamped with the core + plugin Alembic
|
|
revisions and the running image tag at capture time. A restore is a CHECKED
|
|
operation: current code must be at least the set's schema revision, so we never
|
|
load rows that reference a column the live schema has not applied yet.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
from datetime import datetime, timezone
|
|
|
|
from flask import Blueprint, current_app, request
|
|
|
|
from shopdb.core.models import AuditLog
|
|
from shopdb.core.models.setting import Setting
|
|
from shopdb.utils.responses import success_response, error_response, ErrorCodes
|
|
from shopdb.utils.authz import require_role
|
|
|
|
shopdbbackups_bp = Blueprint('shopdbbackups', __name__)
|
|
|
|
# Supported backup destination kinds. 'local' needs no credentials; 'smb' and
|
|
# 'aws' are ground-wired here so operators can configure them now, but their
|
|
# client libraries are held back until the security review clears them, so the
|
|
# runner treats a non-local destination as configured-but-not-yet-armed.
|
|
DESTINATION_KINDS = ('local', 'smb', 'aws')
|
|
|
|
# The setting keys that make up each destination's config, grouped by kind so
|
|
# the GET can surface only the relevant fields and the PUT can validate what it
|
|
# is allowed to write. Secret keys (password/secret/token in the name) are
|
|
# masked by the shared settings serializer, same convention as SMTP/SAML.
|
|
DESTINATION_FIELDS = {
|
|
'local': [],
|
|
'smb': [
|
|
'backup_smb_host',
|
|
'backup_smb_share',
|
|
'backup_smb_path',
|
|
'backup_smb_username',
|
|
'backup_smb_password',
|
|
],
|
|
'aws': [
|
|
'backup_aws_region',
|
|
'backup_aws_bucket',
|
|
'backup_aws_prefix',
|
|
'backup_aws_accesskeyid',
|
|
'backup_aws_secretaccesskey',
|
|
],
|
|
}
|
|
|
|
# Keys whose value must never leave the API in the clear. Mirrors the shared
|
|
# settings convention (_is_secret) so the two surfaces agree on what is secret.
|
|
_SECRET_TOKENS = ('password', 'secret', 'token')
|
|
SECRET_MASK = '********'
|
|
|
|
|
|
def _is_secret_key(key):
|
|
return any(token in key for token in _SECRET_TOKENS)
|
|
|
|
# Where the backup container drops each set and its manifest. Mounted read-only
|
|
# for the api container so the UI can enumerate sets without being able to
|
|
# mutate them; writes happen only from the backup one-shot.
|
|
BACKUPS_ROOT_ENV = 'SHOPDB_BACKUPS_ROOT'
|
|
DEFAULT_BACKUPS_ROOT = '/backups'
|
|
|
|
# Retention the backup container enforces; surfaced read-only so the UI can show
|
|
# "day 1 of 5 retained" without re-deriving the policy.
|
|
RETENTION = {
|
|
'keepdaily': 5,
|
|
'keepweekly': 4,
|
|
'keepmonthly': 6,
|
|
}
|
|
|
|
|
|
def _backups_root():
|
|
return os.environ.get(BACKUPS_ROOT_ENV, DEFAULT_BACKUPS_ROOT)
|
|
|
|
|
|
def _read_json(path):
|
|
"""Load a JSON file, or None when it is absent or unreadable."""
|
|
try:
|
|
with open(path, 'r', encoding='utf-8') as handle:
|
|
return json.load(handle)
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
|
|
def _read_status(kind):
|
|
"""Return the live status dict for a run kind, or None when idle."""
|
|
root = _backups_root()
|
|
return _read_json(os.path.join(root, '.' + kind + '-status.json'))
|
|
|
|
|
|
def _read_history(kind):
|
|
"""Return the recorded run history for a kind, newest first."""
|
|
root = _backups_root()
|
|
entries = _read_json(os.path.join(root, 'history', kind + 's.json'))
|
|
if not isinstance(entries, list):
|
|
return []
|
|
return list(reversed(entries))
|
|
|
|
|
|
def _write_status(kind, phase, setname, message):
|
|
"""Seed a live status file so the UI shows movement the instant a run is
|
|
queued, before the watcher has picked up the sentinel."""
|
|
root = _backups_root()
|
|
stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z'
|
|
payload = {
|
|
'kind': kind,
|
|
'phase': phase,
|
|
'setname': setname,
|
|
'message': message,
|
|
'updatedat': stamp,
|
|
}
|
|
try:
|
|
path = os.path.join(root, '.' + kind + '-status.json')
|
|
with open(path, 'w', encoding='utf-8') as handle:
|
|
json.dump(payload, handle)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
|
|
def _read_manifest(setdir):
|
|
"""Load a set's manifest.json, or None when absent/unreadable.
|
|
|
|
A set without a readable manifest is not restorable (we cannot check its
|
|
schema), so callers treat None as "unverified" rather than trusting it.
|
|
"""
|
|
path = os.path.join(setdir, 'manifest.json')
|
|
try:
|
|
with open(path, 'r', encoding='utf-8') as handle:
|
|
return json.load(handle)
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
|
|
def _current_schema_revisions():
|
|
"""Return the live Alembic head revisions (core + plugins) for this build.
|
|
|
|
Compared against a set's stamped revisions to decide restore compatibility.
|
|
Imported lazily so a missing alembic at request time degrades to an empty
|
|
map (everything reads as "unknown") instead of 500ing the list endpoint.
|
|
"""
|
|
try:
|
|
from alembic.config import Config
|
|
from alembic.script import ScriptDirectory
|
|
except ImportError:
|
|
return {}
|
|
|
|
root = current_app.root_path
|
|
migrations = os.path.join(os.path.dirname(root), 'migrations')
|
|
ini = os.path.join(migrations, 'alembic.ini')
|
|
if not os.path.isfile(ini):
|
|
return {}
|
|
try:
|
|
config = Config(ini)
|
|
# This is a Flask-Migrate project: alembic.ini carries no
|
|
# script_location (it is set at runtime), so ScriptDirectory.from_config
|
|
# would raise and leave every set reading as incompatible. Point it at
|
|
# the migrations directory - the same versions tree manifest.py scans.
|
|
config.set_main_option('script_location', migrations)
|
|
script = ScriptDirectory.from_config(config)
|
|
return {'core': list(script.get_heads())}
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _is_compatible(manifest, current):
|
|
"""A set restores onto THIS build only when the build's schema is at least
|
|
the set's. We compare head-revision sets: if the set was taken on a schema
|
|
the current code has not reached, the code is too OLD and would 500 on the
|
|
restored rows, so restore is blocked until the code is upgraded.
|
|
"""
|
|
if not manifest:
|
|
return False
|
|
setrevs = set(manifest.get('schemarev', {}).get('core', []))
|
|
currevs = set(current.get('core', []))
|
|
if not setrevs or not currevs:
|
|
return False
|
|
# Compatible when the current heads cover every head the set was taken at.
|
|
return setrevs.issubset(currevs)
|
|
|
|
|
|
# Reserved entries under the backups root that are not backup sets: the run
|
|
# history store and any dotfile sentinels/status files the watcher drops.
|
|
_NON_SET_ENTRIES = ('history',)
|
|
|
|
|
|
def _list_sets():
|
|
"""Enumerate backup sets newest-first, each summarized for the list view."""
|
|
root = _backups_root()
|
|
if not os.path.isdir(root):
|
|
return []
|
|
current = _current_schema_revisions()
|
|
sets = []
|
|
for name in sorted(os.listdir(root), reverse=True):
|
|
if name in _NON_SET_ENTRIES or name.startswith('.'):
|
|
continue
|
|
setdir = os.path.join(root, name)
|
|
if not os.path.isdir(setdir):
|
|
continue
|
|
manifest = _read_manifest(setdir)
|
|
dbdump = os.path.join(setdir, 'db.sql.gz')
|
|
sizebytes = 0
|
|
for base, _dirs, files in os.walk(setdir):
|
|
for filename in files:
|
|
try:
|
|
sizebytes += os.path.getsize(os.path.join(base, filename))
|
|
except OSError:
|
|
pass
|
|
sets.append({
|
|
'snapshotid': name,
|
|
'takenat': (manifest or {}).get('takenat'),
|
|
'sizebytes': sizebytes,
|
|
'schemarev': (manifest or {}).get('schemarev', {}),
|
|
'imagetag': (manifest or {}).get('imagetag'),
|
|
'ischeckpassed': bool((manifest or {}).get('ischeckpassed')),
|
|
'hasmanifest': manifest is not None,
|
|
'hasdbdump': os.path.isfile(dbdump),
|
|
'iscompatible': _is_compatible(manifest, current),
|
|
})
|
|
return sets
|
|
|
|
|
|
@shopdbbackups_bp.route('', methods=['GET'])
|
|
@require_role('admin')
|
|
def list_backups():
|
|
"""List every retained backup set, newest first, with a compatibility flag."""
|
|
sets = _list_sets()
|
|
laststgood = next((s for s in sets if s['ischeckpassed']), None)
|
|
meta = {
|
|
'retention': RETENTION,
|
|
'lastgoodat': (laststgood or {}).get('takenat'),
|
|
'count': len(sets),
|
|
}
|
|
return success_response(data=sets, meta=meta)
|
|
|
|
|
|
@shopdbbackups_bp.route('/<snapshotid>', methods=['GET'])
|
|
@require_role('admin')
|
|
def get_backup(snapshotid):
|
|
"""Return one set's full manifest plus a resolved compatibility verdict."""
|
|
root = _backups_root()
|
|
setdir = os.path.join(root, os.path.basename(snapshotid))
|
|
if not os.path.isdir(setdir):
|
|
return error_response(ErrorCodes.NOT_FOUND, 'Backup set not found',
|
|
http_code=404)
|
|
manifest = _read_manifest(setdir)
|
|
current = _current_schema_revisions()
|
|
return success_response(data={
|
|
'snapshotid': os.path.basename(snapshotid),
|
|
'manifest': manifest,
|
|
'currentschemarev': current,
|
|
'iscompatible': _is_compatible(manifest, current),
|
|
'hasmanifest': manifest is not None,
|
|
})
|
|
|
|
|
|
@shopdbbackups_bp.route('', methods=['POST'])
|
|
@require_role('admin')
|
|
def trigger_backup():
|
|
"""Kick off an on-demand backup run.
|
|
|
|
The api container cannot run docker; it writes a sentinel the host-side
|
|
scheduler (or an operator running `docker compose run --rm backup`) picks
|
|
up. Returning isrunning=false with a clear message keeps the UI honest
|
|
rather than pretending the run started inside this process.
|
|
"""
|
|
root = _backups_root()
|
|
try:
|
|
os.makedirs(root, exist_ok=True)
|
|
sentinel = os.path.join(root, '.backup-requested')
|
|
stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z'
|
|
with open(sentinel, 'w', encoding='utf-8') as handle:
|
|
handle.write(stamp)
|
|
except OSError as error:
|
|
return error_response(ErrorCodes.INTERNAL_ERROR,
|
|
'Could not queue a backup: ' + str(error),
|
|
http_code=500)
|
|
_write_status('backup', 'queued', '', 'Backup queued')
|
|
AuditLog.log('requested', 'Backup', entityname=stamp)
|
|
return success_response(
|
|
data={'isrunning': False, 'isqueued': True, 'requestedat': stamp},
|
|
message='Backup queued. The backup service runs it on its next pass.')
|
|
|
|
|
|
@shopdbbackups_bp.route('/<snapshotid>/restore', methods=['POST'])
|
|
@require_role('admin')
|
|
def restore_backup(snapshotid):
|
|
"""Queue a restore of a set, gated on schema compatibility.
|
|
|
|
We refuse outright when the set is incompatible (current code older than the
|
|
set's schema) so an operator cannot foot-gun a schema mismatch from the UI.
|
|
The actual restore is performed by scripts/backup/restore.sh in the backup
|
|
container; here we validate and record intent.
|
|
"""
|
|
root = _backups_root()
|
|
setdir = os.path.join(root, os.path.basename(snapshotid))
|
|
if not os.path.isdir(setdir):
|
|
return error_response(ErrorCodes.NOT_FOUND, 'Backup set not found',
|
|
http_code=404)
|
|
manifest = _read_manifest(setdir)
|
|
current = _current_schema_revisions()
|
|
if not _is_compatible(manifest, current):
|
|
return error_response(
|
|
ErrorCodes.CONFLICT,
|
|
'This backup was taken on a newer schema than the running code. '
|
|
'Upgrade the code to at least the set\'s revision, run '
|
|
'`flask db upgrade` and `flask plugin upgrade-all`, then retry.',
|
|
details={'currentschemarev': current,
|
|
'setschemarev': (manifest or {}).get('schemarev', {})},
|
|
http_code=409)
|
|
try:
|
|
sentinel = os.path.join(root, '.restore-requested')
|
|
stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z'
|
|
with open(sentinel, 'w', encoding='utf-8') as handle:
|
|
handle.write(os.path.basename(snapshotid))
|
|
except OSError as error:
|
|
return error_response(ErrorCodes.INTERNAL_ERROR,
|
|
'Could not queue the restore: ' + str(error),
|
|
http_code=500)
|
|
_write_status('restore', 'queued', os.path.basename(snapshotid),
|
|
'Restore queued')
|
|
AuditLog.log('restore-requested', 'Backup',
|
|
entityname=os.path.basename(snapshotid))
|
|
return success_response(
|
|
data={'isqueued': True, 'snapshotid': os.path.basename(snapshotid)},
|
|
message='Restore queued and validated against the live schema.')
|
|
|
|
|
|
@shopdbbackups_bp.route('/status', methods=['GET'])
|
|
@require_role('admin')
|
|
def get_status():
|
|
"""Return the live status of the most recent backup and restore runs.
|
|
|
|
The watcher writes these files as it moves through each phase; the UI polls
|
|
this endpoint to drive the restore progress spinner and phase label. Either
|
|
value is None when that kind of run has never been requested.
|
|
"""
|
|
return success_response(data={
|
|
'backup': _read_status('backup'),
|
|
'restore': _read_status('restore'),
|
|
})
|
|
|
|
|
|
@shopdbbackups_bp.route('/history', methods=['GET'])
|
|
@require_role('admin')
|
|
def get_history():
|
|
"""Return the recorded history of backup and restore runs, newest first.
|
|
|
|
These are operational run logs written by the backup container onto the
|
|
shared volume, not the audit trail (AuditLog records the requested intent).
|
|
"""
|
|
return success_response(data={
|
|
'backups': _read_history('backup'),
|
|
'restores': _read_history('restore'),
|
|
})
|
|
|
|
|
|
@shopdbbackups_bp.route('/settings', methods=['GET'])
|
|
@require_role('admin')
|
|
def get_backup_settings():
|
|
"""Return destination + retention config for the settings pane.
|
|
|
|
The destination kind and its (non-secret) fields come from the settings
|
|
store so an operator can configure where backups ship without editing env
|
|
or redeploying. Secret fields (SMB password, AWS secret key) are never
|
|
returned in the clear: we send a mask when a value is set and an empty
|
|
string when it is not, matching the shared settings API convention.
|
|
|
|
'smb' and 'aws' are configurable now but their client libraries are held
|
|
back pending the security review, so we flag those destinations as armed:
|
|
false to tell the UI the config is stored but not yet acting.
|
|
"""
|
|
kind = Setting.get('backup_destination_kind', 'local')
|
|
if kind not in DESTINATION_KINDS:
|
|
kind = 'local'
|
|
|
|
fields = {}
|
|
for key in DESTINATION_FIELDS.get(kind, []):
|
|
raw = Setting.get(key, '')
|
|
if _is_secret_key(key):
|
|
fields[key] = SECRET_MASK if raw else ''
|
|
else:
|
|
fields[key] = raw or ''
|
|
|
|
return success_response(data={
|
|
'destinationkind': kind,
|
|
'destinationkinds': list(DESTINATION_KINDS),
|
|
'fields': fields,
|
|
'isarmed': kind == 'local',
|
|
'retention': RETENTION,
|
|
'backupsroot': _backups_root(),
|
|
})
|
|
|
|
|
|
@shopdbbackups_bp.route('/settings', methods=['PUT'])
|
|
@require_role('admin')
|
|
def update_backup_settings():
|
|
"""Persist the backup destination kind and its credential/config fields.
|
|
|
|
Only keys that belong to the chosen destination are accepted, so a stray
|
|
payload cannot write arbitrary settings rows through this surface. A secret
|
|
field submitted as the mask placeholder means "leave unchanged" - the
|
|
client only ever received the mask, so we never overwrite a real secret
|
|
with it.
|
|
"""
|
|
data = request.get_json(silent=True) or {}
|
|
kind = data.get('destinationkind', Setting.get('backup_destination_kind', 'local'))
|
|
if kind not in DESTINATION_KINDS:
|
|
return error_response(
|
|
ErrorCodes.VALIDATION_ERROR,
|
|
'destinationkind must be one of: ' + ', '.join(DESTINATION_KINDS))
|
|
|
|
Setting.set('backup_destination_kind', kind, valuetype='string',
|
|
category='backup')
|
|
|
|
incoming = data.get('fields', {}) or {}
|
|
allowed = set(DESTINATION_FIELDS.get(kind, []))
|
|
for key, value in incoming.items():
|
|
if key not in allowed:
|
|
continue
|
|
if _is_secret_key(key) and value == SECRET_MASK:
|
|
continue
|
|
Setting.set(key, value if value is not None else '',
|
|
valuetype='string', category='backup')
|
|
|
|
AuditLog.log('updated', 'BackupSettings', entityname=kind, changes={
|
|
'destinationkind': kind,
|
|
})
|
|
|
|
return get_backup_settings()
|