e2705d732a
The compatibility gate read alembic_version and nothing else, but plugin schemas move independently of core. Roll the code back to an older plugin revision and the core head can be identical, so a dump carrying plugin tables the code has never heard of passed the gate, loaded - destroying what was there - and only then failed in 'flask plugin upgrade-all' with "Can't locate revision". A failed restore that has already replaced what it was restoring over is the worst outcome this feature can produce, and it was reachable from an ordinary rollback. Backups now stamp the applied revision of every alembic_version_<plugin> table alongside the core one, read from the dumped database the same way. restore.sh checks each against that plugin's own versions directory and refuses before touching anything, naming the plugin and the revision to deploy past. A plugin in the dump that this build does not have at all is refused for the same reason. The list endpoint's compatibility badge asks the same question, so the UI cannot offer a set the restore will reject. Sets taken before the stamp existed carry no dataplugins and keep their old core-only verdict. Their plugin revisions are unknowable, and making every existing set unrestorable to close this would be a worse trade than the gap. Proven against a real stack, reproducing the exact case: core head identical, endpoints data stamped at a revision the build does not have. The restore refused, the live row was still there afterwards, and it stopped early enough that the pre-restore safety copy was never needed. The same set with its true revision restored normally. Suite 2605 passed / 9 skipped, with the gate covered both ways plus the legacy fallback. Scope: product
791 lines
30 KiB
Python
791 lines
30 KiB
Python
"""Backups API routes.
|
|
|
|
Admin-only surface over the site's backup sets. The heavy lifting (mysqldump,
|
|
restic snapshot of the instance tree, retention prune, integrity check) runs in
|
|
the one-shot backup container defined in docker-compose.backup.yml; this
|
|
blueprint is the read-mostly control plane the UI talks to.
|
|
|
|
Two recovery products stay firewalled, per the design notes:
|
|
- code rollback lives in git (tags/releases), NOT here
|
|
- data restore lives here, gated on schema compatibility via each set's manifest
|
|
|
|
Every backup set carries a manifest.json stamped with the core + plugin Alembic
|
|
revisions and the running image tag at capture time. A restore is a CHECKED
|
|
operation: current code must be at least the set's schema revision, so we never
|
|
load rows that reference a column the live schema has not applied yet.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import threading
|
|
import time
|
|
import subprocess
|
|
from datetime import datetime, timezone
|
|
|
|
from flask import Blueprint, current_app, request
|
|
|
|
from shopdb.core.models import AuditLog
|
|
from shopdb.core.models.setting import Setting
|
|
from shopdb.utils.responses import success_response, error_response, ErrorCodes
|
|
from shopdb.utils.authz import require_role
|
|
|
|
shopdbbackups_bp = Blueprint('shopdbbackups', __name__)
|
|
|
|
# Supported backup destination kinds. 'local' needs no credentials; 'smb' and
|
|
# 'aws' are ground-wired here so operators can configure them now, but their
|
|
# client libraries are held back until the security review clears them, so the
|
|
# runner treats a non-local destination as configured-but-not-yet-armed.
|
|
DESTINATION_KINDS = ('local', 'smb', 'aws')
|
|
|
|
# The setting keys that make up each destination's config, grouped by kind so
|
|
# the GET can surface only the relevant fields and the PUT can validate what it
|
|
# is allowed to write. Secret keys (password/secret/token in the name) are
|
|
# masked by the shared settings serializer, same convention as SMTP/SAML.
|
|
DESTINATION_FIELDS = {
|
|
'local': [],
|
|
'smb': [
|
|
'backup_smb_host',
|
|
'backup_smb_share',
|
|
'backup_smb_path',
|
|
'backup_smb_username',
|
|
'backup_smb_password',
|
|
],
|
|
'aws': [
|
|
'backup_aws_region',
|
|
'backup_aws_bucket',
|
|
'backup_aws_prefix',
|
|
'backup_aws_accesskeyid',
|
|
'backup_aws_secretaccesskey',
|
|
],
|
|
}
|
|
|
|
# Keys whose value must never leave the API in the clear. Mirrors the shared
|
|
# settings convention (_is_secret) so the two surfaces agree on what is secret.
|
|
_SECRET_TOKENS = ('password', 'secret', 'token')
|
|
SECRET_MASK = '********'
|
|
|
|
|
|
def _is_secret_key(key):
|
|
return any(token in key for token in _SECRET_TOKENS)
|
|
|
|
# Where the backup container drops each set and its manifest. Mounted read-only
|
|
# for the api container so the UI can enumerate sets without being able to
|
|
# mutate them; writes happen only from the backup one-shot.
|
|
BACKUPS_ROOT_ENV = 'SHOPDB_BACKUPS_ROOT'
|
|
DEFAULT_BACKUPS_ROOT = '/backups'
|
|
|
|
# Retention the backup container enforces; surfaced read-only so the UI can show
|
|
# "day 1 of 5 retained" without re-deriving the policy.
|
|
RETENTION = {
|
|
'keepdaily': 5,
|
|
'keepweekly': 4,
|
|
'keepmonthly': 6,
|
|
}
|
|
|
|
|
|
def _backups_root():
|
|
return os.environ.get(BACKUPS_ROOT_ENV, DEFAULT_BACKUPS_ROOT)
|
|
|
|
|
|
def _backups_configured():
|
|
"""Whether this deployment has a backup store behind the feature.
|
|
|
|
The overlay is a compose profile: a third container writes the sets onto a
|
|
shared volume, and the api container only ever reads them and drops a
|
|
sentinel. A Windows/IIS install has no such container, and the default root
|
|
resolves there to a drive-letter path nothing creates - so every button
|
|
either 500s on PermissionError or reports 'queued' for a run that will
|
|
never happen. Answer the caller honestly instead.
|
|
"""
|
|
return os.path.isdir(_backups_root())
|
|
|
|
|
|
def _unavailable_response():
|
|
# 404 + NOT_FOUND is the established shape for "this feature is not
|
|
# enabled here" (the configuration-policy routes answer the same way), and
|
|
# the UI already tells a 404 apart from a real failure.
|
|
return error_response(
|
|
ErrorCodes.NOT_FOUND,
|
|
'Site Backups is not configured on this deployment. It needs the '
|
|
'backup overlay and a readable backups root (%s).' % BACKUPS_ROOT_ENV,
|
|
http_code=404)
|
|
|
|
|
|
def _read_json(path):
|
|
"""Load a JSON file, or None when it is absent or unreadable."""
|
|
try:
|
|
with open(path, 'r', encoding='utf-8') as handle:
|
|
return json.load(handle)
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
|
|
def _read_status(kind):
|
|
"""Return the live status dict for a run kind, or None when idle."""
|
|
root = _backups_root()
|
|
return _read_json(os.path.join(root, '.' + kind + '-status.json'))
|
|
|
|
|
|
def _read_history(kind):
|
|
"""Return the recorded run history for a kind, newest first."""
|
|
root = _backups_root()
|
|
entries = _read_json(os.path.join(root, 'history', kind + 's.json'))
|
|
if not isinstance(entries, list):
|
|
return []
|
|
return list(reversed(entries))
|
|
|
|
|
|
def _write_status(kind, phase, setname, message):
|
|
"""Seed a live status file so the UI shows movement the instant a run is
|
|
queued, before the watcher has picked up the sentinel."""
|
|
root = _backups_root()
|
|
stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z'
|
|
payload = {
|
|
'kind': kind,
|
|
'phase': phase,
|
|
'setname': setname,
|
|
'message': message,
|
|
'updatedat': stamp,
|
|
}
|
|
try:
|
|
path = os.path.join(root, '.' + kind + '-status.json')
|
|
with open(path, 'w', encoding='utf-8') as handle:
|
|
json.dump(payload, handle)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _append_history(kind, outcome, setname, message):
|
|
"""Append one run record to the shared history the UI reads, newest last.
|
|
|
|
Mirrors status.sh's history_append so a finalize that completes in the api
|
|
(not the sidecar) still lands in the same history file the restore page
|
|
shows. Best-effort: history is a convenience log, never a gate.
|
|
"""
|
|
root = _backups_root()
|
|
stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z'
|
|
entry = {
|
|
'kind': kind,
|
|
'outcome': outcome,
|
|
'setname': setname,
|
|
'message': message,
|
|
'at': stamp,
|
|
}
|
|
path = os.path.join(root, 'history', kind + 's.json')
|
|
try:
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
entries = _read_json(path)
|
|
if not isinstance(entries, list):
|
|
entries = []
|
|
entries.append(entry)
|
|
with open(path, 'w', encoding='utf-8') as handle:
|
|
json.dump(entries, handle)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
# The post-restore upgrade is claimed by renaming the sentinel. rename(2) is
|
|
# atomic, so exactly one claimant wins even with several gunicorn workers and a
|
|
# browser poll all racing - which matters because the loser would otherwise run
|
|
# Alembic concurrently against the same database.
|
|
UPGRADE_SENTINEL = '.upgrade-requested'
|
|
UPGRADE_CLAIM = '.upgrade-inprogress'
|
|
UPGRADE_FAILED = '.upgrade-failed'
|
|
|
|
|
|
def _claim_upgrade():
|
|
"""Take exclusive ownership of a pending upgrade, or return None.
|
|
|
|
Returns the set name on success. Losing the race is not an error: it means
|
|
another worker is already finalizing the same restore.
|
|
"""
|
|
root = _backups_root()
|
|
sentinel = os.path.join(root, UPGRADE_SENTINEL)
|
|
claim = os.path.join(root, UPGRADE_CLAIM)
|
|
try:
|
|
os.rename(sentinel, claim)
|
|
except OSError:
|
|
return None
|
|
try:
|
|
with open(claim, 'r', encoding='utf-8') as handle:
|
|
return handle.read().strip() or 'unknown'
|
|
except OSError:
|
|
return 'unknown'
|
|
|
|
|
|
def _finalize_claimed(setname):
|
|
"""Run the upgrade for a claimed restore and record how it ended.
|
|
|
|
A failure parks the claim as .upgrade-failed rather than putting it back:
|
|
re-arming it here would retry a migration that just failed, every poll,
|
|
forever. The operator re-arms it explicitly through the finalize endpoint.
|
|
"""
|
|
root = _backups_root()
|
|
claim = os.path.join(root, UPGRADE_CLAIM)
|
|
_write_status('restore', 'upgrading', setname,
|
|
'Upgrading restored schema to code')
|
|
try:
|
|
_run_schema_upgrade()
|
|
except Exception as error: # noqa: BLE001 - report any upgrade failure
|
|
message = 'Schema upgrade failed: ' + str(error)
|
|
try:
|
|
os.replace(claim, os.path.join(root, UPGRADE_FAILED))
|
|
except OSError:
|
|
pass
|
|
_write_status('restore', 'failed', setname, message)
|
|
_append_history('restore', 'failed', setname, message)
|
|
current_app.logger.error('Post-restore schema upgrade failed: %s', error)
|
|
return False
|
|
try:
|
|
os.remove(claim)
|
|
except OSError:
|
|
pass
|
|
_write_status('restore', 'done', setname, 'Restore complete')
|
|
_append_history('restore', 'success', setname, 'Restore complete')
|
|
current_app.logger.info('Restore finalized: %s', setname)
|
|
return True
|
|
|
|
|
|
def start_restore_finalizer(app):
|
|
"""Watch for a restore awaiting its schema upgrade, and run it.
|
|
|
|
WHY THIS EXISTS: the restore itself runs in the Flask-less backup sidecar,
|
|
which loads the dump and drops a sentinel because it cannot run
|
|
'flask db upgrade'. Until this watcher, the ONLY thing that acted on that
|
|
sentinel was the browser poll loop of the modal that queued the restore -
|
|
so closing the tab, or letting a token expire, during the minutes a load
|
|
takes left the data at the old revision under new code, with every query on
|
|
a newer column failing and nothing retrying.
|
|
|
|
Runs in the api container because that is where the app context and the
|
|
engine live, and out of band rather than in a request because Alembic can
|
|
outrun the gunicorn worker timeout and get killed mid-chain.
|
|
"""
|
|
interval = max(1, int(os.environ.get('SHOPDB_FINALIZE_POLL_SECONDS', '10')))
|
|
|
|
def _loop():
|
|
while True:
|
|
time.sleep(interval)
|
|
try:
|
|
with app.app_context():
|
|
if not _backups_configured():
|
|
continue
|
|
setname = _claim_upgrade()
|
|
if setname:
|
|
_finalize_claimed(setname)
|
|
except Exception: # noqa: BLE001 - a watcher must not die
|
|
try:
|
|
app.logger.exception('Restore finalizer pass failed')
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
|
|
thread = threading.Thread(target=_loop, name='restore-finalizer', daemon=True)
|
|
thread.start()
|
|
return thread
|
|
|
|
|
|
def _run_schema_upgrade():
|
|
"""Bring the just-restored database up to the running code, in-process.
|
|
|
|
Runs the same two idempotent steps every deploy runs: the core Alembic
|
|
upgrade to head, then each plugin chain via the plugin manager. Both are
|
|
no-ops when the data already matches the code head, so this is safe to call
|
|
even when the restored dump was already current. Uses the live app context
|
|
and its configured engine, so no subprocess or extra Flask CLI is needed.
|
|
"""
|
|
from flask_migrate import upgrade as alembic_upgrade
|
|
|
|
alembic_upgrade()
|
|
|
|
manager = current_app.extensions.get('plugin_manager')
|
|
if manager is None:
|
|
raise RuntimeError('Plugin manager not initialized; cannot upgrade plugins')
|
|
results = manager.upgrade_all_plugins()
|
|
failed = {name: status for name, status in (results or {}).items()
|
|
if status not in ('ok', 'no-migrations')}
|
|
if failed:
|
|
raise RuntimeError('Plugin upgrade failed: ' + json.dumps(failed))
|
|
|
|
|
|
|
|
def _read_manifest(setdir):
|
|
"""Load a set's manifest.json, or None when absent/unreadable.
|
|
|
|
A set without a readable manifest is not restorable (we cannot check its
|
|
schema), so callers treat None as "unverified" rather than trusting it.
|
|
"""
|
|
path = os.path.join(setdir, 'manifest.json')
|
|
try:
|
|
with open(path, 'r', encoding='utf-8') as handle:
|
|
return json.load(handle)
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
|
|
def _current_schema_revisions():
|
|
"""Return the live Alembic head revisions (core + plugins) for this build.
|
|
|
|
Compared against a set's stamped revisions to decide restore compatibility.
|
|
Imported lazily so a missing alembic at request time degrades to an empty
|
|
map (everything reads as "unknown") instead of 500ing the list endpoint.
|
|
"""
|
|
try:
|
|
from alembic.config import Config
|
|
from alembic.script import ScriptDirectory
|
|
except ImportError:
|
|
return {}
|
|
|
|
root = current_app.root_path
|
|
migrations = os.path.join(os.path.dirname(root), 'migrations')
|
|
ini = os.path.join(migrations, 'alembic.ini')
|
|
if not os.path.isfile(ini):
|
|
return {}
|
|
try:
|
|
config = Config(ini)
|
|
# This is a Flask-Migrate project: alembic.ini carries no
|
|
# script_location (it is set at runtime), so ScriptDirectory.from_config
|
|
# would raise and leave every set reading as incompatible. Point it at
|
|
# the migrations directory - the same versions tree manifest.py scans.
|
|
config.set_main_option('script_location', migrations)
|
|
script = ScriptDirectory.from_config(config)
|
|
return {'core': list(script.get_heads())}
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _known_core_revisions():
|
|
"""Return every core Alembic revision this build knows, heads and ancestors.
|
|
|
|
Restore compatibility is now data-driven: a set restores when the running
|
|
code KNOWS the dump's data revision (so `flask db upgrade` can roll it
|
|
forward to head). Heads alone cannot answer that - a dump stamped at an
|
|
older-but-valid revision is perfectly restorable - so we walk the whole
|
|
script history. Lazy import + broad except keeps a missing/broken alembic
|
|
from 500ing the list endpoint (returns empty = everything reads unknown).
|
|
"""
|
|
try:
|
|
from alembic.config import Config
|
|
from alembic.script import ScriptDirectory
|
|
except ImportError:
|
|
return set()
|
|
|
|
root = current_app.root_path
|
|
migrations = os.path.join(os.path.dirname(root), 'migrations')
|
|
ini = os.path.join(migrations, 'alembic.ini')
|
|
if not os.path.isfile(ini):
|
|
return set()
|
|
try:
|
|
config = Config(ini)
|
|
config.set_main_option('script_location', migrations)
|
|
script = ScriptDirectory.from_config(config)
|
|
return {rev.revision for rev in script.walk_revisions()}
|
|
except Exception:
|
|
return set()
|
|
|
|
|
|
def _known_plugin_revisions():
|
|
"""Every revision each plugin chain knows, as {plugin: {revisions}}.
|
|
|
|
Read by text scan rather than through Alembic. Each plugin chain has its own
|
|
env and version table, so loading them properly would mean building 17
|
|
ScriptDirectories on a request that only needs to answer "is this id in
|
|
there" - and manifest.py already reads the same trees the same way, so the
|
|
two agree by construction.
|
|
"""
|
|
root = os.path.dirname(current_app.root_path)
|
|
plugins_dir = os.path.join(root, 'plugins')
|
|
known = {}
|
|
if not os.path.isdir(plugins_dir):
|
|
return known
|
|
for name in sorted(os.listdir(plugins_dir)):
|
|
versions = os.path.join(plugins_dir, name, 'migrations', 'versions')
|
|
if not os.path.isdir(versions):
|
|
continue
|
|
revisions = set()
|
|
for filename in os.listdir(versions):
|
|
if not filename.endswith('.py'):
|
|
continue
|
|
try:
|
|
with open(os.path.join(versions, filename), 'r',
|
|
encoding='utf-8') as handle:
|
|
for line in handle:
|
|
stripped = line.strip()
|
|
if stripped.startswith('revision ='):
|
|
revisions.add(
|
|
stripped.split('=', 1)[1].strip().strip('"\''))
|
|
break
|
|
except OSError:
|
|
continue
|
|
if revisions:
|
|
known[name] = revisions
|
|
return known
|
|
|
|
|
|
def _plugin_data_is_known(manifest):
|
|
"""Whether this build knows every plugin revision the set's data sits at.
|
|
|
|
A set with no `dataplugins` predates the stamp; its plugin revisions are
|
|
unknowable, so it keeps the old core-only verdict rather than being marked
|
|
unrestorable retroactively.
|
|
"""
|
|
stamped = manifest.get('dataplugins') or {}
|
|
if not isinstance(stamped, dict) or not stamped:
|
|
return True
|
|
known = _known_plugin_revisions()
|
|
for name, revision in stamped.items():
|
|
if not revision:
|
|
continue
|
|
if revision not in known.get(name, ()):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _is_compatible(manifest, current):
|
|
"""Decide whether a set restores onto THIS build.
|
|
|
|
Data-driven gate (the fix for the FBJXBDB4 outage): the authoritative
|
|
signal is the set's `datarev` - the actual `alembic_version` of the dumped
|
|
DATABASE, not the code head that captured it. A restore is safe when the
|
|
running code KNOWS that revision (it is somewhere in this build's migration
|
|
history), because then the post-restore finalize can run `flask db upgrade`
|
|
to roll the loaded data forward to head. If the code does not know the
|
|
revision, the dump is from a NEWER schema than this build and would 500 on
|
|
restored rows, so we block until the code is upgraded.
|
|
|
|
The same question is asked of every PLUGIN chain, because plugin schemas
|
|
move independently: code rolled back to an older plugin revision can sit at
|
|
an identical core head while the dump carries plugin tables it cannot
|
|
migrate.
|
|
|
|
Legacy sets predate `datarev`; fall back to the old head-subset check on
|
|
`schemarev` so they stay listable and restorable as before.
|
|
"""
|
|
if not manifest:
|
|
return False
|
|
datarev = manifest.get('datarev')
|
|
if datarev:
|
|
known = _known_core_revisions()
|
|
if not known:
|
|
return False
|
|
# Core AND every plugin chain. Core alone said yes to a set whose plugin
|
|
# data the code had never heard of, which loads and then dies during the
|
|
# plugin upgrade with the old data already gone.
|
|
return datarev in known and _plugin_data_is_known(manifest)
|
|
setrevs = set(manifest.get('schemarev', {}).get('core', []))
|
|
currevs = set(current.get('core', []))
|
|
if not setrevs or not currevs:
|
|
return False
|
|
# Compatible when the current heads cover every head the set was taken at.
|
|
return setrevs.issubset(currevs)
|
|
|
|
|
|
# Reserved entries under the backups root that are not backup sets: the run
|
|
# history store and any dotfile sentinels/status files the watcher drops.
|
|
_NON_SET_ENTRIES = ('history',)
|
|
|
|
|
|
def _list_sets():
|
|
"""Enumerate backup sets newest-first, each summarized for the list view."""
|
|
root = _backups_root()
|
|
if not os.path.isdir(root):
|
|
return []
|
|
current = _current_schema_revisions()
|
|
sets = []
|
|
for name in sorted(os.listdir(root), reverse=True):
|
|
if name in _NON_SET_ENTRIES or name.startswith('.'):
|
|
continue
|
|
setdir = os.path.join(root, name)
|
|
if not os.path.isdir(setdir):
|
|
continue
|
|
manifest = _read_manifest(setdir)
|
|
dbdump = os.path.join(setdir, 'db.sql.gz')
|
|
sizebytes = 0
|
|
for base, _dirs, files in os.walk(setdir):
|
|
for filename in files:
|
|
try:
|
|
sizebytes += os.path.getsize(os.path.join(base, filename))
|
|
except OSError:
|
|
pass
|
|
sets.append({
|
|
'snapshotid': name,
|
|
'takenat': (manifest or {}).get('takenat'),
|
|
'sizebytes': sizebytes,
|
|
'schemarev': (manifest or {}).get('schemarev', {}),
|
|
'imagetag': (manifest or {}).get('imagetag'),
|
|
'ischeckpassed': bool((manifest or {}).get('ischeckpassed')),
|
|
'hasmanifest': manifest is not None,
|
|
'hasdbdump': os.path.isfile(dbdump),
|
|
'iscompatible': _is_compatible(manifest, current),
|
|
})
|
|
return sets
|
|
|
|
|
|
@shopdbbackups_bp.route('', methods=['GET'])
|
|
@require_role('admin')
|
|
def list_backups():
|
|
"""List every retained backup set, newest first, with a compatibility flag."""
|
|
sets = _list_sets()
|
|
laststgood = next((s for s in sets if s['ischeckpassed']), None)
|
|
meta = {
|
|
'retention': RETENTION,
|
|
'lastgoodat': (laststgood or {}).get('takenat'),
|
|
'count': len(sets),
|
|
'configured': _backups_configured(),
|
|
}
|
|
return success_response(data=sets, meta=meta)
|
|
|
|
|
|
@shopdbbackups_bp.route('/<snapshotid>', methods=['GET'])
|
|
@require_role('admin')
|
|
def get_backup(snapshotid):
|
|
"""Return one set's full manifest plus a resolved compatibility verdict."""
|
|
root = _backups_root()
|
|
setdir = os.path.join(root, os.path.basename(snapshotid))
|
|
if not os.path.isdir(setdir):
|
|
return error_response(ErrorCodes.NOT_FOUND, 'Backup set not found',
|
|
http_code=404)
|
|
manifest = _read_manifest(setdir)
|
|
current = _current_schema_revisions()
|
|
return success_response(data={
|
|
'snapshotid': os.path.basename(snapshotid),
|
|
'manifest': manifest,
|
|
'currentschemarev': current,
|
|
'iscompatible': _is_compatible(manifest, current),
|
|
'hasmanifest': manifest is not None,
|
|
})
|
|
|
|
|
|
@shopdbbackups_bp.route('', methods=['POST'])
|
|
@require_role('admin')
|
|
def trigger_backup():
|
|
"""Kick off an on-demand backup run.
|
|
|
|
The api container cannot run docker; it writes a sentinel the host-side
|
|
scheduler (or an operator running `docker compose run --rm backup`) picks
|
|
up. Returning isrunning=false with a clear message keeps the UI honest
|
|
rather than pretending the run started inside this process.
|
|
"""
|
|
if not _backups_configured():
|
|
return _unavailable_response()
|
|
|
|
root = _backups_root()
|
|
try:
|
|
os.makedirs(root, exist_ok=True)
|
|
sentinel = os.path.join(root, '.backup-requested')
|
|
stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z'
|
|
with open(sentinel, 'w', encoding='utf-8') as handle:
|
|
handle.write(stamp)
|
|
except OSError as error:
|
|
return error_response(ErrorCodes.INTERNAL_ERROR,
|
|
'Could not queue a backup: ' + str(error),
|
|
http_code=500)
|
|
_write_status('backup', 'queued', '', 'Backup queued')
|
|
AuditLog.log('requested', 'Backup', entityname=stamp)
|
|
return success_response(
|
|
data={'isrunning': False, 'isqueued': True, 'requestedat': stamp},
|
|
message='Backup queued. The backup service runs it on its next pass.')
|
|
|
|
|
|
@shopdbbackups_bp.route('/<snapshotid>/restore', methods=['POST'])
|
|
@require_role('admin')
|
|
def restore_backup(snapshotid):
|
|
"""Queue a restore of a set, gated on schema compatibility.
|
|
|
|
We refuse outright when the set is incompatible (current code older than the
|
|
set's schema) so an operator cannot foot-gun a schema mismatch from the UI.
|
|
The actual restore is performed by scripts/backup/restore.sh in the backup
|
|
container; here we validate and record intent.
|
|
"""
|
|
if not _backups_configured():
|
|
return _unavailable_response()
|
|
|
|
root = _backups_root()
|
|
setdir = os.path.join(root, os.path.basename(snapshotid))
|
|
if not os.path.isdir(setdir):
|
|
return error_response(ErrorCodes.NOT_FOUND, 'Backup set not found',
|
|
http_code=404)
|
|
manifest = _read_manifest(setdir)
|
|
current = _current_schema_revisions()
|
|
if not _is_compatible(manifest, current):
|
|
return error_response(
|
|
ErrorCodes.CONFLICT,
|
|
'This backup was taken on a newer schema than the running code. '
|
|
'Upgrade the code to at least the set\'s revision, run '
|
|
'`flask db upgrade` and `flask plugin upgrade-all`, then retry.',
|
|
details={'currentschemarev': current,
|
|
'setschemarev': (manifest or {}).get('schemarev', {})},
|
|
http_code=409)
|
|
try:
|
|
sentinel = os.path.join(root, '.restore-requested')
|
|
stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z'
|
|
with open(sentinel, 'w', encoding='utf-8') as handle:
|
|
handle.write(os.path.basename(snapshotid))
|
|
except OSError as error:
|
|
return error_response(ErrorCodes.INTERNAL_ERROR,
|
|
'Could not queue the restore: ' + str(error),
|
|
http_code=500)
|
|
_write_status('restore', 'queued', os.path.basename(snapshotid),
|
|
'Restore queued')
|
|
AuditLog.log('restore-requested', 'Backup',
|
|
entityname=os.path.basename(snapshotid))
|
|
return success_response(
|
|
data={'isqueued': True, 'snapshotid': os.path.basename(snapshotid)},
|
|
message='Restore queued and validated against the live schema.')
|
|
|
|
|
|
@shopdbbackups_bp.route('/finalize-restore', methods=['POST'])
|
|
@require_role('admin')
|
|
def finalize_restore():
|
|
"""Make sure a restore awaiting its schema upgrade is queued to be finalized.
|
|
|
|
This used to RUN the upgrade inline, which made two things true that should
|
|
not have been: a restore only completed while a browser tab stayed open on
|
|
it, and the upgrade ran inside a request, where Alembic can outrun the
|
|
gunicorn worker timeout and be killed part-way through the chain, leaving a
|
|
half-applied revision the next attempt trips over.
|
|
|
|
The work now belongs to the finalizer watcher in this container
|
|
(start_restore_finalizer). This endpoint only ensures something is pending
|
|
for it to claim, so it is safe to call repeatedly, and it is how an operator
|
|
retries an upgrade that failed.
|
|
"""
|
|
if not _backups_configured():
|
|
return _unavailable_response()
|
|
|
|
root = _backups_root()
|
|
sentinel = os.path.join(root, UPGRADE_SENTINEL)
|
|
claim = os.path.join(root, UPGRADE_CLAIM)
|
|
failed = os.path.join(root, UPGRADE_FAILED)
|
|
|
|
if os.path.exists(claim):
|
|
return success_response(
|
|
data={'queued': True, 'inprogress': True},
|
|
message='The schema upgrade is already running.')
|
|
|
|
if os.path.exists(sentinel):
|
|
return success_response(
|
|
data={'queued': True, 'inprogress': False},
|
|
message='The schema upgrade is queued and will run shortly.')
|
|
|
|
# A previous attempt failed and parked itself. Re-arming is the retry.
|
|
if os.path.exists(failed):
|
|
try:
|
|
os.replace(failed, sentinel)
|
|
except OSError as error:
|
|
return error_response(ErrorCodes.INTERNAL_ERROR,
|
|
'Could not re-queue the schema upgrade: ' + str(error),
|
|
http_code=500)
|
|
AuditLog.log('restore-finalize-retried', 'Backup')
|
|
return success_response(
|
|
data={'queued': True, 'inprogress': False, 'retried': True},
|
|
message='The failed schema upgrade has been queued to run again.')
|
|
|
|
return success_response(
|
|
data={'queued': False},
|
|
message='No restore is awaiting schema upgrade.')
|
|
|
|
|
|
@shopdbbackups_bp.route('/status', methods=['GET'])
|
|
@require_role('admin')
|
|
def get_status():
|
|
"""Return the live status of the most recent backup and restore runs.
|
|
|
|
The watcher writes these files as it moves through each phase; the UI polls
|
|
this endpoint to drive the restore progress spinner and phase label. Either
|
|
value is None when that kind of run has never been requested.
|
|
"""
|
|
return success_response(data={
|
|
'configured': _backups_configured(),
|
|
'backup': _read_status('backup'),
|
|
'restore': _read_status('restore'),
|
|
})
|
|
|
|
|
|
@shopdbbackups_bp.route('/history', methods=['GET'])
|
|
@require_role('admin')
|
|
def get_history():
|
|
"""Return the recorded history of backup and restore runs, newest first.
|
|
|
|
These are operational run logs written by the backup container onto the
|
|
shared volume, not the audit trail (AuditLog records the requested intent).
|
|
"""
|
|
return success_response(data={
|
|
'backups': _read_history('backup'),
|
|
'restores': _read_history('restore'),
|
|
})
|
|
|
|
|
|
@shopdbbackups_bp.route('/settings', methods=['GET'])
|
|
@require_role('admin')
|
|
def get_backup_settings():
|
|
"""Return destination + retention config for the settings pane.
|
|
|
|
The destination kind and its (non-secret) fields come from the settings
|
|
store so an operator can configure where backups ship without editing env
|
|
or redeploying. Secret fields (SMB password, AWS secret key) are never
|
|
returned in the clear: we send a mask when a value is set and an empty
|
|
string when it is not, matching the shared settings API convention.
|
|
|
|
'smb' and 'aws' are configurable now but their client libraries are held
|
|
back pending the security review, so we flag those destinations as armed:
|
|
false to tell the UI the config is stored but not yet acting.
|
|
"""
|
|
kind = Setting.get('backup_destination_kind', 'local')
|
|
if kind not in DESTINATION_KINDS:
|
|
kind = 'local'
|
|
|
|
fields = {}
|
|
for key in DESTINATION_FIELDS.get(kind, []):
|
|
raw = Setting.get(key, '')
|
|
if _is_secret_key(key):
|
|
fields[key] = SECRET_MASK if raw else ''
|
|
else:
|
|
fields[key] = raw or ''
|
|
|
|
return success_response(data={
|
|
'destinationkind': kind,
|
|
'destinationkinds': list(DESTINATION_KINDS),
|
|
'fields': fields,
|
|
'isarmed': kind == 'local',
|
|
'retention': RETENTION,
|
|
'backupsroot': _backups_root(),
|
|
})
|
|
|
|
|
|
@shopdbbackups_bp.route('/settings', methods=['PUT'])
|
|
@require_role('admin')
|
|
def update_backup_settings():
|
|
"""Persist the backup destination kind and its credential/config fields.
|
|
|
|
Only keys that belong to the chosen destination are accepted, so a stray
|
|
payload cannot write arbitrary settings rows through this surface. A secret
|
|
field submitted as the mask placeholder means "leave unchanged" - the
|
|
client only ever received the mask, so we never overwrite a real secret
|
|
with it.
|
|
"""
|
|
data = request.get_json(silent=True) or {}
|
|
kind = data.get('destinationkind', Setting.get('backup_destination_kind', 'local'))
|
|
if kind not in DESTINATION_KINDS:
|
|
return error_response(
|
|
ErrorCodes.VALIDATION_ERROR,
|
|
'destinationkind must be one of: ' + ', '.join(DESTINATION_KINDS))
|
|
|
|
Setting.set('backup_destination_kind', kind, valuetype='string',
|
|
category='backup')
|
|
|
|
incoming = data.get('fields', {}) or {}
|
|
allowed = set(DESTINATION_FIELDS.get(kind, []))
|
|
for key, value in incoming.items():
|
|
if key not in allowed:
|
|
continue
|
|
if _is_secret_key(key) and value == SECRET_MASK:
|
|
continue
|
|
Setting.set(key, value if value is not None else '',
|
|
valuetype='string', category='backup')
|
|
|
|
AuditLog.log('updated', 'BackupSettings', entityname=kind, changes={
|
|
'destinationkind': kind,
|
|
})
|
|
|
|
return get_backup_settings()
|