800 lines
31 KiB
Python
800 lines
31 KiB
Python
"""Backups API routes.
|
|
|
|
Admin-only surface over the site's backup sets. The heavy lifting (mysqldump,
|
|
restic snapshot of the instance tree, retention prune, integrity check) runs in
|
|
the one-shot backup container defined in docker-compose.backup.yml; this
|
|
blueprint is the read-mostly control plane the UI talks to.
|
|
|
|
Two recovery products stay firewalled, per the design notes:
|
|
- code rollback lives in git (tags/releases), NOT here
|
|
- data restore lives here, gated on schema compatibility via each set's manifest
|
|
|
|
Every backup set carries a manifest.json stamped with the core + plugin Alembic
|
|
revisions and the running image tag at capture time. A restore is a CHECKED
|
|
operation: current code must be at least the set's schema revision, so we never
|
|
load rows that reference a column the live schema has not applied yet.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import threading
|
|
import time
|
|
import subprocess
|
|
from datetime import datetime, timezone
|
|
|
|
from flask import Blueprint, current_app, request
|
|
|
|
from shopdb.core.models import AuditLog
|
|
from shopdb.core.models.setting import Setting
|
|
from shopdb.utils.responses import success_response, error_response, ErrorCodes
|
|
from shopdb.utils.authz import require_role
|
|
|
|
shopdbbackups_bp = Blueprint('shopdbbackups', __name__)
|
|
|
|
# Supported backup destination kinds. 'local' needs no credentials; 'smb' and
|
|
# 'aws' are ground-wired here so operators can configure them now, but their
|
|
# client libraries are held back until the security review clears them, so the
|
|
# runner treats a non-local destination as configured-but-not-yet-armed.
|
|
DESTINATION_KINDS = ('local', 'smb', 'aws')
|
|
|
|
# The setting keys that make up each destination's config, grouped by kind so
|
|
# the GET can surface only the relevant fields and the PUT can validate what it
|
|
# is allowed to write. Secret keys (password/secret/token in the name) are
|
|
# masked by the shared settings serializer, same convention as SMTP/SAML.
|
|
DESTINATION_FIELDS = {
|
|
'local': [],
|
|
'smb': [
|
|
'backup_smb_host',
|
|
'backup_smb_share',
|
|
'backup_smb_path',
|
|
'backup_smb_username',
|
|
'backup_smb_password',
|
|
],
|
|
'aws': [
|
|
'backup_aws_region',
|
|
'backup_aws_bucket',
|
|
'backup_aws_prefix',
|
|
'backup_aws_accesskeyid',
|
|
'backup_aws_secretaccesskey',
|
|
],
|
|
}
|
|
|
|
# Keys whose value must never leave the API in the clear. Mirrors the shared
|
|
# settings convention (_is_secret) so the two surfaces agree on what is secret.
|
|
_SECRET_TOKENS = ('password', 'secret', 'token')
|
|
SECRET_MASK = '********'
|
|
|
|
|
|
def _is_secret_key(key):
|
|
return any(token in key for token in _SECRET_TOKENS)
|
|
|
|
# Where the backup container drops each set and its manifest. Mounted read-only
|
|
# for the api container so the UI can enumerate sets without being able to
|
|
# mutate them; writes happen only from the backup one-shot.
|
|
BACKUPS_ROOT_ENV = 'SHOPDB_BACKUPS_ROOT'
|
|
DEFAULT_BACKUPS_ROOT = '/backups'
|
|
|
|
# Retention the backup container enforces; surfaced read-only so the UI can show
|
|
# "day 1 of 5 retained" without re-deriving the policy.
|
|
RETENTION = {
|
|
'keepdaily': 5,
|
|
'keepweekly': 4,
|
|
'keepmonthly': 6,
|
|
}
|
|
|
|
|
|
def _backups_root():
|
|
return os.environ.get(BACKUPS_ROOT_ENV, DEFAULT_BACKUPS_ROOT)
|
|
|
|
|
|
def _backups_configured():
|
|
"""Whether this deployment has a backup store behind the feature.
|
|
|
|
The overlay is a compose profile: a third container writes the sets onto a
|
|
shared volume, and the api container only ever reads them and drops a
|
|
sentinel. A Windows/IIS install has no such container, and the default root
|
|
resolves there to a drive-letter path nothing creates - so every button
|
|
either 500s on PermissionError or reports 'queued' for a run that will
|
|
never happen. Answer the caller honestly instead.
|
|
"""
|
|
return os.path.isdir(_backups_root())
|
|
|
|
|
|
def _unavailable_response():
|
|
# 404 + NOT_FOUND is the established shape for "this feature is not
|
|
# enabled here" (the configuration-policy routes answer the same way), and
|
|
# the UI already tells a 404 apart from a real failure.
|
|
return error_response(
|
|
ErrorCodes.NOT_FOUND,
|
|
'Site Backups is not configured on this deployment. It needs the '
|
|
'backup overlay and a readable backups root (%s).' % BACKUPS_ROOT_ENV,
|
|
http_code=404)
|
|
|
|
|
|
def _read_json(path):
|
|
"""Load a JSON file, or None when it is absent or unreadable."""
|
|
try:
|
|
with open(path, 'r', encoding='utf-8') as handle:
|
|
return json.load(handle)
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
|
|
def _read_status(kind):
|
|
"""Return the live status dict for a run kind, or None when idle."""
|
|
root = _backups_root()
|
|
return _read_json(os.path.join(root, '.' + kind + '-status.json'))
|
|
|
|
|
|
def _read_history(kind):
|
|
"""Return the recorded run history for a kind, newest first."""
|
|
root = _backups_root()
|
|
entries = _read_json(os.path.join(root, 'history', kind + 's.json'))
|
|
if not isinstance(entries, list):
|
|
return []
|
|
return list(reversed(entries))
|
|
|
|
|
|
def _write_status(kind, phase, setname, message):
|
|
"""Seed a live status file so the UI shows movement the instant a run is
|
|
queued, before the watcher has picked up the sentinel."""
|
|
root = _backups_root()
|
|
stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z'
|
|
payload = {
|
|
'kind': kind,
|
|
'phase': phase,
|
|
'setname': setname,
|
|
'message': message,
|
|
'updatedat': stamp,
|
|
}
|
|
try:
|
|
path = os.path.join(root, '.' + kind + '-status.json')
|
|
with open(path, 'w', encoding='utf-8') as handle:
|
|
json.dump(payload, handle)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _append_history(kind, outcome, setname, message):
|
|
"""Append one run record to the shared history the UI reads, newest last.
|
|
|
|
Mirrors status.sh's history_append so a finalize that completes in the api
|
|
(not the sidecar) still lands in the same history file the restore page
|
|
shows. Best-effort: history is a convenience log, never a gate.
|
|
"""
|
|
root = _backups_root()
|
|
stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z'
|
|
entry = {
|
|
'kind': kind,
|
|
'outcome': outcome,
|
|
'setname': setname,
|
|
'message': message,
|
|
'at': stamp,
|
|
}
|
|
path = os.path.join(root, 'history', kind + 's.json')
|
|
try:
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
entries = _read_json(path)
|
|
if not isinstance(entries, list):
|
|
entries = []
|
|
entries.append(entry)
|
|
with open(path, 'w', encoding='utf-8') as handle:
|
|
json.dump(entries, handle)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
# The post-restore upgrade is claimed by renaming the sentinel. rename(2) is
|
|
# atomic, so exactly one claimant wins even with several gunicorn workers and a
|
|
# browser poll all racing - which matters because the loser would otherwise run
|
|
# Alembic concurrently against the same database.
|
|
UPGRADE_SENTINEL = '.upgrade-requested'
|
|
UPGRADE_CLAIM = '.upgrade-inprogress'
|
|
UPGRADE_FAILED = '.upgrade-failed'
|
|
|
|
|
|
def _claim_upgrade():
|
|
"""Take exclusive ownership of a pending upgrade, or return None.
|
|
|
|
Returns the set name on success. Losing the race is not an error: it means
|
|
another worker is already finalizing the same restore.
|
|
"""
|
|
root = _backups_root()
|
|
sentinel = os.path.join(root, UPGRADE_SENTINEL)
|
|
claim = os.path.join(root, UPGRADE_CLAIM)
|
|
try:
|
|
os.rename(sentinel, claim)
|
|
except OSError:
|
|
return None
|
|
try:
|
|
with open(claim, 'r', encoding='utf-8') as handle:
|
|
return handle.read().strip() or 'unknown'
|
|
except OSError:
|
|
return 'unknown'
|
|
|
|
|
|
def _finalize_claimed(setname):
|
|
"""Run the upgrade for a claimed restore and record how it ended.
|
|
|
|
A failure parks the claim as .upgrade-failed rather than putting it back:
|
|
re-arming it here would retry a migration that just failed, every poll,
|
|
forever. The operator re-arms it explicitly through the finalize endpoint.
|
|
"""
|
|
root = _backups_root()
|
|
claim = os.path.join(root, UPGRADE_CLAIM)
|
|
_write_status('restore', 'upgrading', setname,
|
|
'Upgrading restored schema to code')
|
|
try:
|
|
_run_schema_upgrade()
|
|
except Exception as error: # noqa: BLE001 - report any upgrade failure
|
|
message = 'Schema upgrade failed: ' + str(error)
|
|
try:
|
|
os.replace(claim, os.path.join(root, UPGRADE_FAILED))
|
|
except OSError:
|
|
pass
|
|
_write_status('restore', 'failed', setname, message)
|
|
_append_history('restore', 'failed', setname, message)
|
|
current_app.logger.error('Post-restore schema upgrade failed: %s', error)
|
|
return False
|
|
try:
|
|
os.remove(claim)
|
|
except OSError:
|
|
pass
|
|
_write_status('restore', 'done', setname, 'Restore complete')
|
|
_append_history('restore', 'success', setname, 'Restore complete')
|
|
current_app.logger.info('Restore finalized: %s', setname)
|
|
return True
|
|
|
|
|
|
def start_restore_finalizer(app):
|
|
"""Watch for a restore awaiting its schema upgrade, and run it.
|
|
|
|
WHY THIS EXISTS: the restore itself runs in the Flask-less backup sidecar,
|
|
which loads the dump and drops a sentinel because it cannot run
|
|
'flask db upgrade'. Until this watcher, the ONLY thing that acted on that
|
|
sentinel was the browser poll loop of the modal that queued the restore -
|
|
so closing the tab, or letting a token expire, during the minutes a load
|
|
takes left the data at the old revision under new code, with every query on
|
|
a newer column failing and nothing retrying.
|
|
|
|
Runs in the api container because that is where the app context and the
|
|
engine live, and out of band rather than in a request because Alembic can
|
|
outrun the gunicorn worker timeout and get killed mid-chain.
|
|
"""
|
|
interval = max(1, int(os.environ.get('SHOPDB_FINALIZE_POLL_SECONDS', '10')))
|
|
|
|
def _loop():
|
|
while True:
|
|
time.sleep(interval)
|
|
try:
|
|
with app.app_context():
|
|
if not _backups_configured():
|
|
continue
|
|
setname = _claim_upgrade()
|
|
if setname:
|
|
_finalize_claimed(setname)
|
|
except Exception: # noqa: BLE001 - a watcher must not die
|
|
try:
|
|
app.logger.exception('Restore finalizer pass failed')
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
|
|
thread = threading.Thread(target=_loop, name='restore-finalizer', daemon=True)
|
|
thread.start()
|
|
return thread
|
|
|
|
|
|
def _run_schema_upgrade():
|
|
"""Bring the just-restored database up to the running code, in-process.
|
|
|
|
Runs the same two idempotent steps every deploy runs: the core Alembic
|
|
upgrade to head, then each plugin chain via the plugin manager. Both are
|
|
no-ops when the data already matches the code head, so this is safe to call
|
|
even when the restored dump was already current. Uses the live app context
|
|
and its configured engine, so no subprocess or extra Flask CLI is needed.
|
|
"""
|
|
from flask_migrate import upgrade as alembic_upgrade
|
|
|
|
alembic_upgrade()
|
|
|
|
manager = current_app.extensions.get('plugin_manager')
|
|
if manager is None:
|
|
raise RuntimeError('Plugin manager not initialized; cannot upgrade plugins')
|
|
results = manager.upgrade_all_plugins()
|
|
failed = {name: status for name, status in (results or {}).items()
|
|
if status not in ('ok', 'no-migrations')}
|
|
if failed:
|
|
raise RuntimeError('Plugin upgrade failed: ' + json.dumps(failed))
|
|
|
|
|
|
|
|
def _read_manifest(setdir):
|
|
"""Load a set's manifest.json, or None when absent/unreadable.
|
|
|
|
A set without a readable manifest is not restorable (we cannot check its
|
|
schema), so callers treat None as "unverified" rather than trusting it.
|
|
"""
|
|
path = os.path.join(setdir, 'manifest.json')
|
|
try:
|
|
with open(path, 'r', encoding='utf-8') as handle:
|
|
return json.load(handle)
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
|
|
def _current_schema_revisions():
|
|
"""Return the live Alembic head revisions (core + plugins) for this build.
|
|
|
|
Compared against a set's stamped revisions to decide restore compatibility.
|
|
Imported lazily so a missing alembic at request time degrades to an empty
|
|
map (everything reads as "unknown") instead of 500ing the list endpoint.
|
|
"""
|
|
try:
|
|
from alembic.config import Config
|
|
from alembic.script import ScriptDirectory
|
|
except ImportError:
|
|
return {}
|
|
|
|
root = current_app.root_path
|
|
migrations = os.path.join(os.path.dirname(root), 'migrations')
|
|
ini = os.path.join(migrations, 'alembic.ini')
|
|
if not os.path.isfile(ini):
|
|
return {}
|
|
try:
|
|
config = Config(ini)
|
|
# This is a Flask-Migrate project: alembic.ini carries no
|
|
# script_location (it is set at runtime), so ScriptDirectory.from_config
|
|
# would raise and leave every set reading as incompatible. Point it at
|
|
# the migrations directory - the same versions tree manifest.py scans.
|
|
config.set_main_option('script_location', migrations)
|
|
script = ScriptDirectory.from_config(config)
|
|
return {'core': list(script.get_heads())}
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _known_core_revisions():
|
|
"""Return every core Alembic revision this build knows, heads and ancestors.
|
|
|
|
Restore compatibility is now data-driven: a set restores when the running
|
|
code KNOWS the dump's data revision (so `flask db upgrade` can roll it
|
|
forward to head). Heads alone cannot answer that - a dump stamped at an
|
|
older-but-valid revision is perfectly restorable - so we walk the whole
|
|
script history. Lazy import + broad except keeps a missing/broken alembic
|
|
from 500ing the list endpoint (returns empty = everything reads unknown).
|
|
"""
|
|
try:
|
|
from alembic.config import Config
|
|
from alembic.script import ScriptDirectory
|
|
except ImportError:
|
|
return set()
|
|
|
|
root = current_app.root_path
|
|
migrations = os.path.join(os.path.dirname(root), 'migrations')
|
|
ini = os.path.join(migrations, 'alembic.ini')
|
|
if not os.path.isfile(ini):
|
|
return set()
|
|
try:
|
|
config = Config(ini)
|
|
config.set_main_option('script_location', migrations)
|
|
script = ScriptDirectory.from_config(config)
|
|
return {rev.revision for rev in script.walk_revisions()}
|
|
except Exception:
|
|
return set()
|
|
|
|
|
|
def _known_plugin_revisions():
|
|
"""Every revision each plugin chain knows, as {plugin: {revisions}}.
|
|
|
|
Read by text scan rather than through Alembic. Each plugin chain has its own
|
|
env and version table, so loading them properly would mean building 17
|
|
ScriptDirectories on a request that only needs to answer "is this id in
|
|
there" - and manifest.py already reads the same trees the same way, so the
|
|
two agree by construction.
|
|
"""
|
|
root = os.path.dirname(current_app.root_path)
|
|
plugins_dir = os.path.join(root, 'plugins')
|
|
known = {}
|
|
if not os.path.isdir(plugins_dir):
|
|
return known
|
|
for name in sorted(os.listdir(plugins_dir)):
|
|
versions = os.path.join(plugins_dir, name, 'migrations', 'versions')
|
|
if not os.path.isdir(versions):
|
|
continue
|
|
revisions = set()
|
|
for filename in os.listdir(versions):
|
|
if not filename.endswith('.py'):
|
|
continue
|
|
try:
|
|
with open(os.path.join(versions, filename), 'r',
|
|
encoding='utf-8') as handle:
|
|
for line in handle:
|
|
stripped = line.strip()
|
|
if stripped.startswith('revision ='):
|
|
revisions.add(
|
|
stripped.split('=', 1)[1].strip().strip('"\''))
|
|
break
|
|
except OSError:
|
|
continue
|
|
if revisions:
|
|
known[name] = revisions
|
|
return known
|
|
|
|
|
|
def _plugin_data_is_known(manifest):
|
|
"""Whether this build knows every plugin revision the set's data sits at.
|
|
|
|
A set with no `dataplugins` predates the stamp; its plugin revisions are
|
|
unknowable, so it keeps the old core-only verdict rather than being marked
|
|
unrestorable retroactively.
|
|
"""
|
|
stamped = manifest.get('dataplugins') or {}
|
|
if not isinstance(stamped, dict) or not stamped:
|
|
return True
|
|
known = _known_plugin_revisions()
|
|
for name, revision in stamped.items():
|
|
if not revision:
|
|
continue
|
|
if revision not in known.get(name, ()):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _is_compatible(manifest, current):
|
|
"""Decide whether a set restores onto THIS build.
|
|
|
|
Data-driven gate (the fix for the FBJXBDB4 outage): the authoritative
|
|
signal is the set's `datarev` - the actual `alembic_version` of the dumped
|
|
DATABASE, not the code head that captured it. A restore is safe when the
|
|
running code KNOWS that revision (it is somewhere in this build's migration
|
|
history), because then the post-restore finalize can run `flask db upgrade`
|
|
to roll the loaded data forward to head. If the code does not know the
|
|
revision, the dump is from a NEWER schema than this build and would 500 on
|
|
restored rows, so we block until the code is upgraded.
|
|
|
|
The same question is asked of every PLUGIN chain, because plugin schemas
|
|
move independently: code rolled back to an older plugin revision can sit at
|
|
an identical core head while the dump carries plugin tables it cannot
|
|
migrate.
|
|
|
|
Legacy sets predate `datarev`; fall back to the old head-subset check on
|
|
`schemarev` so they stay listable and restorable as before.
|
|
"""
|
|
if not manifest:
|
|
return False
|
|
|
|
# A manifest that CARRIES the key but left it empty is not a legacy set: it
|
|
# is a set whose data revision could not be read when it was taken. Falling
|
|
# through to the code-to-code check would badge it restorable on no
|
|
# evidence, using exactly the comparison the datarev work replaced. Only a
|
|
# manifest with no datarev key at all predates the stamp.
|
|
if 'datarev' in manifest and not manifest.get('datarev'):
|
|
return False
|
|
|
|
datarev = manifest.get('datarev')
|
|
if datarev:
|
|
known = _known_core_revisions()
|
|
if not known:
|
|
return False
|
|
# Core AND every plugin chain. Core alone said yes to a set whose plugin
|
|
# data the code had never heard of, which loads and then dies during the
|
|
# plugin upgrade with the old data already gone.
|
|
return datarev in known and _plugin_data_is_known(manifest)
|
|
setrevs = set(manifest.get('schemarev', {}).get('core', []))
|
|
currevs = set(current.get('core', []))
|
|
if not setrevs or not currevs:
|
|
return False
|
|
# Compatible when the current heads cover every head the set was taken at.
|
|
return setrevs.issubset(currevs)
|
|
|
|
|
|
# Reserved entries under the backups root that are not backup sets: the run
|
|
# history store and any dotfile sentinels/status files the watcher drops.
|
|
_NON_SET_ENTRIES = ('history',)
|
|
|
|
|
|
def _list_sets():
|
|
"""Enumerate backup sets newest-first, each summarized for the list view."""
|
|
root = _backups_root()
|
|
if not os.path.isdir(root):
|
|
return []
|
|
current = _current_schema_revisions()
|
|
sets = []
|
|
for name in sorted(os.listdir(root), reverse=True):
|
|
if name in _NON_SET_ENTRIES or name.startswith('.'):
|
|
continue
|
|
setdir = os.path.join(root, name)
|
|
if not os.path.isdir(setdir):
|
|
continue
|
|
manifest = _read_manifest(setdir)
|
|
dbdump = os.path.join(setdir, 'db.sql.gz')
|
|
sizebytes = 0
|
|
for base, _dirs, files in os.walk(setdir):
|
|
for filename in files:
|
|
try:
|
|
sizebytes += os.path.getsize(os.path.join(base, filename))
|
|
except OSError:
|
|
pass
|
|
sets.append({
|
|
'snapshotid': name,
|
|
'takenat': (manifest or {}).get('takenat'),
|
|
'sizebytes': sizebytes,
|
|
'schemarev': (manifest or {}).get('schemarev', {}),
|
|
'imagetag': (manifest or {}).get('imagetag'),
|
|
'ischeckpassed': bool((manifest or {}).get('ischeckpassed')),
|
|
'hasmanifest': manifest is not None,
|
|
'hasdbdump': os.path.isfile(dbdump),
|
|
'iscompatible': _is_compatible(manifest, current),
|
|
})
|
|
return sets
|
|
|
|
|
|
@shopdbbackups_bp.route('', methods=['GET'])
|
|
@require_role('admin')
|
|
def list_backups():
|
|
"""List every retained backup set, newest first, with a compatibility flag."""
|
|
sets = _list_sets()
|
|
laststgood = next((s for s in sets if s['ischeckpassed']), None)
|
|
meta = {
|
|
'retention': RETENTION,
|
|
'lastgoodat': (laststgood or {}).get('takenat'),
|
|
'count': len(sets),
|
|
'configured': _backups_configured(),
|
|
}
|
|
return success_response(data=sets, meta=meta)
|
|
|
|
|
|
@shopdbbackups_bp.route('/<snapshotid>', methods=['GET'])
|
|
@require_role('admin')
|
|
def get_backup(snapshotid):
|
|
"""Return one set's full manifest plus a resolved compatibility verdict."""
|
|
root = _backups_root()
|
|
setdir = os.path.join(root, os.path.basename(snapshotid))
|
|
if not os.path.isdir(setdir):
|
|
return error_response(ErrorCodes.NOT_FOUND, 'Backup set not found',
|
|
http_code=404)
|
|
manifest = _read_manifest(setdir)
|
|
current = _current_schema_revisions()
|
|
return success_response(data={
|
|
'snapshotid': os.path.basename(snapshotid),
|
|
'manifest': manifest,
|
|
'currentschemarev': current,
|
|
'iscompatible': _is_compatible(manifest, current),
|
|
'hasmanifest': manifest is not None,
|
|
})
|
|
|
|
|
|
@shopdbbackups_bp.route('', methods=['POST'])
|
|
@require_role('admin')
|
|
def trigger_backup():
|
|
"""Kick off an on-demand backup run.
|
|
|
|
The api container cannot run docker; it writes a sentinel the host-side
|
|
scheduler (or an operator running `docker compose run --rm backup`) picks
|
|
up. Returning isrunning=false with a clear message keeps the UI honest
|
|
rather than pretending the run started inside this process.
|
|
"""
|
|
if not _backups_configured():
|
|
return _unavailable_response()
|
|
|
|
root = _backups_root()
|
|
try:
|
|
os.makedirs(root, exist_ok=True)
|
|
sentinel = os.path.join(root, '.backup-requested')
|
|
stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z'
|
|
with open(sentinel, 'w', encoding='utf-8') as handle:
|
|
handle.write(stamp)
|
|
except OSError as error:
|
|
return error_response(ErrorCodes.INTERNAL_ERROR,
|
|
'Could not queue a backup: ' + str(error),
|
|
http_code=500)
|
|
_write_status('backup', 'queued', '', 'Backup queued')
|
|
AuditLog.log('requested', 'Backup', entityname=stamp)
|
|
return success_response(
|
|
data={'isrunning': False, 'isqueued': True, 'requestedat': stamp},
|
|
message='Backup queued. The backup service runs it on its next pass.')
|
|
|
|
|
|
@shopdbbackups_bp.route('/<snapshotid>/restore', methods=['POST'])
|
|
@require_role('admin')
|
|
def restore_backup(snapshotid):
|
|
"""Queue a restore of a set, gated on schema compatibility.
|
|
|
|
We refuse outright when the set is incompatible (current code older than the
|
|
set's schema) so an operator cannot foot-gun a schema mismatch from the UI.
|
|
The actual restore is performed by scripts/backup/restore.sh in the backup
|
|
container; here we validate and record intent.
|
|
"""
|
|
if not _backups_configured():
|
|
return _unavailable_response()
|
|
|
|
root = _backups_root()
|
|
setdir = os.path.join(root, os.path.basename(snapshotid))
|
|
if not os.path.isdir(setdir):
|
|
return error_response(ErrorCodes.NOT_FOUND, 'Backup set not found',
|
|
http_code=404)
|
|
manifest = _read_manifest(setdir)
|
|
current = _current_schema_revisions()
|
|
if not _is_compatible(manifest, current):
|
|
return error_response(
|
|
ErrorCodes.CONFLICT,
|
|
'This backup was taken on a newer schema than the running code. '
|
|
'Upgrade the code to at least the set\'s revision, run '
|
|
'`flask db upgrade` and `flask plugin upgrade-all`, then retry.',
|
|
details={'currentschemarev': current,
|
|
'setschemarev': (manifest or {}).get('schemarev', {})},
|
|
http_code=409)
|
|
try:
|
|
sentinel = os.path.join(root, '.restore-requested')
|
|
stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z'
|
|
with open(sentinel, 'w', encoding='utf-8') as handle:
|
|
handle.write(os.path.basename(snapshotid))
|
|
except OSError as error:
|
|
return error_response(ErrorCodes.INTERNAL_ERROR,
|
|
'Could not queue the restore: ' + str(error),
|
|
http_code=500)
|
|
_write_status('restore', 'queued', os.path.basename(snapshotid),
|
|
'Restore queued')
|
|
AuditLog.log('restore-requested', 'Backup',
|
|
entityname=os.path.basename(snapshotid))
|
|
return success_response(
|
|
data={'isqueued': True, 'snapshotid': os.path.basename(snapshotid)},
|
|
message='Restore queued and validated against the live schema.')
|
|
|
|
|
|
@shopdbbackups_bp.route('/finalize-restore', methods=['POST'])
|
|
@require_role('admin')
|
|
def finalize_restore():
|
|
"""Make sure a restore awaiting its schema upgrade is queued to be finalized.
|
|
|
|
This used to RUN the upgrade inline, which made two things true that should
|
|
not have been: a restore only completed while a browser tab stayed open on
|
|
it, and the upgrade ran inside a request, where Alembic can outrun the
|
|
gunicorn worker timeout and be killed part-way through the chain, leaving a
|
|
half-applied revision the next attempt trips over.
|
|
|
|
The work now belongs to the finalizer watcher in this container
|
|
(start_restore_finalizer). This endpoint only ensures something is pending
|
|
for it to claim, so it is safe to call repeatedly, and it is how an operator
|
|
retries an upgrade that failed.
|
|
"""
|
|
if not _backups_configured():
|
|
return _unavailable_response()
|
|
|
|
root = _backups_root()
|
|
sentinel = os.path.join(root, UPGRADE_SENTINEL)
|
|
claim = os.path.join(root, UPGRADE_CLAIM)
|
|
failed = os.path.join(root, UPGRADE_FAILED)
|
|
|
|
if os.path.exists(claim):
|
|
return success_response(
|
|
data={'queued': True, 'inprogress': True},
|
|
message='The schema upgrade is already running.')
|
|
|
|
if os.path.exists(sentinel):
|
|
return success_response(
|
|
data={'queued': True, 'inprogress': False},
|
|
message='The schema upgrade is queued and will run shortly.')
|
|
|
|
# A previous attempt failed and parked itself. Re-arming is the retry.
|
|
if os.path.exists(failed):
|
|
try:
|
|
os.replace(failed, sentinel)
|
|
except OSError as error:
|
|
return error_response(ErrorCodes.INTERNAL_ERROR,
|
|
'Could not re-queue the schema upgrade: ' + str(error),
|
|
http_code=500)
|
|
AuditLog.log('restore-finalize-retried', 'Backup')
|
|
return success_response(
|
|
data={'queued': True, 'inprogress': False, 'retried': True},
|
|
message='The failed schema upgrade has been queued to run again.')
|
|
|
|
return success_response(
|
|
data={'queued': False},
|
|
message='No restore is awaiting schema upgrade.')
|
|
|
|
|
|
@shopdbbackups_bp.route('/status', methods=['GET'])
|
|
@require_role('admin')
|
|
def get_status():
|
|
"""Return the live status of the most recent backup and restore runs.
|
|
|
|
The watcher writes these files as it moves through each phase; the UI polls
|
|
this endpoint to drive the restore progress spinner and phase label. Either
|
|
value is None when that kind of run has never been requested.
|
|
"""
|
|
return success_response(data={
|
|
'configured': _backups_configured(),
|
|
'backup': _read_status('backup'),
|
|
'restore': _read_status('restore'),
|
|
})
|
|
|
|
|
|
@shopdbbackups_bp.route('/history', methods=['GET'])
|
|
@require_role('admin')
|
|
def get_history():
|
|
"""Return the recorded history of backup and restore runs, newest first.
|
|
|
|
These are operational run logs written by the backup container onto the
|
|
shared volume, not the audit trail (AuditLog records the requested intent).
|
|
"""
|
|
return success_response(data={
|
|
'backups': _read_history('backup'),
|
|
'restores': _read_history('restore'),
|
|
})
|
|
|
|
|
|
@shopdbbackups_bp.route('/settings', methods=['GET'])
|
|
@require_role('admin')
|
|
def get_backup_settings():
|
|
"""Return destination + retention config for the settings pane.
|
|
|
|
The destination kind and its (non-secret) fields come from the settings
|
|
store so an operator can configure where backups ship without editing env
|
|
or redeploying. Secret fields (SMB password, AWS secret key) are never
|
|
returned in the clear: we send a mask when a value is set and an empty
|
|
string when it is not, matching the shared settings API convention.
|
|
|
|
'smb' and 'aws' are configurable now but their client libraries are held
|
|
back pending the security review, so we flag those destinations as armed:
|
|
false to tell the UI the config is stored but not yet acting.
|
|
"""
|
|
kind = Setting.get('backup_destination_kind', 'local')
|
|
if kind not in DESTINATION_KINDS:
|
|
kind = 'local'
|
|
|
|
fields = {}
|
|
for key in DESTINATION_FIELDS.get(kind, []):
|
|
raw = Setting.get(key, '')
|
|
if _is_secret_key(key):
|
|
fields[key] = SECRET_MASK if raw else ''
|
|
else:
|
|
fields[key] = raw or ''
|
|
|
|
return success_response(data={
|
|
'destinationkind': kind,
|
|
'destinationkinds': list(DESTINATION_KINDS),
|
|
'fields': fields,
|
|
'isarmed': kind == 'local',
|
|
'retention': RETENTION,
|
|
'backupsroot': _backups_root(),
|
|
})
|
|
|
|
|
|
@shopdbbackups_bp.route('/settings', methods=['PUT'])
|
|
@require_role('admin')
|
|
def update_backup_settings():
|
|
"""Persist the backup destination kind and its credential/config fields.
|
|
|
|
Only keys that belong to the chosen destination are accepted, so a stray
|
|
payload cannot write arbitrary settings rows through this surface. A secret
|
|
field submitted as the mask placeholder means "leave unchanged" - the
|
|
client only ever received the mask, so we never overwrite a real secret
|
|
with it.
|
|
"""
|
|
data = request.get_json(silent=True) or {}
|
|
kind = data.get('destinationkind', Setting.get('backup_destination_kind', 'local'))
|
|
if kind not in DESTINATION_KINDS:
|
|
return error_response(
|
|
ErrorCodes.VALIDATION_ERROR,
|
|
'destinationkind must be one of: ' + ', '.join(DESTINATION_KINDS))
|
|
|
|
Setting.set('backup_destination_kind', kind, valuetype='string',
|
|
category='backup')
|
|
|
|
incoming = data.get('fields', {}) or {}
|
|
allowed = set(DESTINATION_FIELDS.get(kind, []))
|
|
for key, value in incoming.items():
|
|
if key not in allowed:
|
|
continue
|
|
if _is_secret_key(key) and value == SECRET_MASK:
|
|
continue
|
|
Setting.set(key, value if value is not None else '',
|
|
valuetype='string', category='backup')
|
|
|
|
AuditLog.log('updated', 'BackupSettings', entityname=kind, changes={
|
|
'destinationkind': kind,
|
|
})
|
|
|
|
return get_backup_settings()
|