diff --git a/frontend/src/views/settings/ShopdbBackupsList.vue b/frontend/src/views/settings/ShopdbBackupsList.vue index eaeab054..705b98f4 100644 --- a/frontend/src/views/settings/ShopdbBackupsList.vue +++ b/frontend/src/views/settings/ShopdbBackupsList.vue @@ -309,6 +309,11 @@ const restoreTarget = ref(null) const restoreStatus = ref(null) let statusTimer = null let restoreStartedAt = 0 +// The Flask-less backup sidecar loads the dump and parks the restore in +// 'upgrading', because it cannot run `flask db upgrade` itself. The api can, so +// the browser pokes the finalize endpoint once to roll the restored data up to +// the running code. Guarded so we fire it exactly once per restore run. +let finalizeRequested = false const backups = ref([]) const restores = ref([]) @@ -320,6 +325,7 @@ const RESTORE_PHASE_LABELS = { checking: 'Checking compatibility...', files: 'Restoring files...', database: 'Loading database...', + upgrading: 'Upgrading schema to code...', done: 'Restore complete', failed: 'Restore failed', } @@ -476,6 +482,18 @@ async function pollStatus() { } restoreStatus.value = status const phase = status?.phase + if (phase === 'upgrading' && !finalizeRequested) { + // Hand off to the api container, which owns the Flask CLI, to run the + // post-load schema upgrade. Fire once; polling keeps tracking until it + // reports done or failed. + finalizeRequested = true + try { + await api.post('/shopdbbackups/finalize-restore') + } catch (error) { + // The finalize endpoint stamps its own failed status on error, which the + // next poll surfaces; nothing extra to do here. + } + } if (phase === 'done' || phase === 'failed') { stopStatusPolling() if (phase === 'done') { @@ -492,6 +510,7 @@ async function pollStatus() { function startStatusPolling() { stopStatusPolling() restoreStartedAt = Date.now() + finalizeRequested = false restoreStatus.value = { phase: 'queued', message: 'Restore queued' } // Poll on an interval only; skip the immediate call so the seeded "queued" // phase is guaranteed to paint before the first real status can replace it. diff --git a/scripts/backup/backup.sh b/scripts/backup/backup.sh index c64bad82..110f427b 100644 --- a/scripts/backup/backup.sh +++ b/scripts/backup/backup.sh @@ -60,15 +60,44 @@ mysqldump \ log "database dump written ($(du -h "${SETDIR}/db.sql.gz" | cut -f1))" # 2. Manifest: the schema stamp that makes a restore a checked operation. +# Capture the revision actually applied to the DATABASE (its alembic_version), +# not just the code head. A restore must gate on the DATA revision; stamping +# only the code head hides an image that has outrun its own database and lets a +# behind-schema dump restore under ahead-schema code (1054 Unknown column on +# every affected table). Query it straight from the dumped DB. +log "reading applied database revision (alembic_version)" +DATAREV="$(mysql \ + --host="${MYSQL_HOST}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + --skip-column-names --batch \ + -e "SELECT version_num FROM alembic_version LIMIT 1;" \ + "${MYSQL_DATABASE}" 2>/dev/null | tr -d '\r')" +[ -n "${DATAREV}" ] || log "WARN could not read alembic_version - datarev will be empty" + log "writing manifest" python3 /opt/backup/manifest.py \ --out "${SETDIR}/manifest.json" \ --takenat "${STAMP}" \ --site "${SITE}" \ --imagetag "${SHOPDB_IMAGE_TAG:-source}" \ + --datarev "${DATAREV}" \ --migrations /app/migrations \ --plugins /app/plugins +# Consistency guard: if the DATA revision does not match the CODE head, this +# backup captures a database the running image has outrun. It is still stored +# (losing it would be worse) but flagged loudly so a restore of it is a +# deliberate choice, not a surprise. +CODEREV="$(python3 -c \ + 'import json,sys;print(",".join(sorted(json.load(open(sys.argv[1]))["schemarev"]["core"])))' \ + "${SETDIR}/manifest.json" 2>/dev/null || echo "")" +if [ -n "${DATAREV}" ] && [ "${DATAREV}" != "${CODEREV}" ]; then + log "WARN schema drift: database is at '${DATAREV}' but code head is '${CODEREV}'" + log "WARN this DB has not run 'flask db upgrade' to the running image; restore will upgrade it" + history_append backup warning "${SETNAME}" "Schema drift: data ${DATAREV} vs code ${CODEREV}" +fi + # 3. restic snapshot: db dump + instance tree + manifest, one point in time. # Skips gracefully when restic/repo are not configured (local-only sites). IS_CHECK_PASSED=false diff --git a/scripts/backup/manifest.py b/scripts/backup/manifest.py index 8708ba94..45345468 100644 --- a/scripts/backup/manifest.py +++ b/scripts/backup/manifest.py @@ -6,15 +6,21 @@ restore." It records, at capture time: - takenat : ISO-8601 UTC timestamp of the run - site : the site tag - imagetag : the app image tag that produced the data - - schemarev : the core Alembic head revision(s) the running code is at, plus + - schemarev : the core Alembic head revision(s) the running CODE is at, plus each plugin chain's heads - the versions a restore target must be at least, or restored rows reference columns the live schema has not applied yet. + - datarev : the Alembic revision actually applied to the DATABASE that was + dumped (its alembic_version), which is the number a restore must + gate on. schemarev stamps the code on disk; datarev stamps the + data in the dump. They diverge when an image is ahead of a DB + that has not run 'flask db upgrade' yet, and that divergence is + exactly what a code-to-code check cannot see. Two modes: --out ... write a fresh manifest for a new set --amend ... patch an existing manifest (used to record ischeckpassed after - restic check runs) + restic check runs, or to backfill datarev) ASCII only, no emojis - matches the repo style gate. """ @@ -79,6 +85,7 @@ def write_manifest(args): 'site': args.site, 'imagetag': args.imagetag, 'schemarev': _collect_schema(args.migrations, args.plugins), + 'datarev': args.datarev or '', 'ischeckpassed': False, } with open(args.out, 'w', encoding='utf-8') as handle: @@ -94,6 +101,8 @@ def amend_manifest(args): manifest = {} if args.ischeckpassed is not None: manifest['ischeckpassed'] = args.ischeckpassed.lower() == 'true' + if args.datarev is not None: + manifest['datarev'] = args.datarev with open(args.amend, 'w', encoding='utf-8') as handle: json.dump(manifest, handle, indent=2, sort_keys=True) return 0 @@ -108,6 +117,7 @@ def main(argv=None): parser.add_argument('--imagetag', default='source') parser.add_argument('--migrations', default='/app/migrations') parser.add_argument('--plugins', default='/app/plugins') + parser.add_argument('--datarev') parser.add_argument('--ischeckpassed') args = parser.parse_args(argv) diff --git a/scripts/backup/restore.sh b/scripts/backup/restore.sh index 8adaf1c2..9e75ad57 100644 --- a/scripts/backup/restore.sh +++ b/scripts/backup/restore.sh @@ -27,13 +27,20 @@ SETNAME="${1:-}" log() { echo "[restore] $*"; } # On any exit, stamp the terminal phase and append a history record so the UI -# reflects success or failure even when a step aborts early (set -e). -RESTORE_DONE=false +# reflects success or failure even when a step aborts early (set -e). A restore +# is only "done" once the DATA has been upgraded to the running CODE, and that +# upgrade runs in the Flask-capable api container, not here (this sidecar has no +# app). So the happy path ends by handing off to the api: this script stamps an +# 'upgrading' phase and drops an .upgrade-requested sentinel, and the api runs +# 'flask db upgrade' + 'flask plugin upgrade-all' and writes the final 'done'. +RESTORE_LOADED=false _on_exit() { code=$? - if [ "${RESTORE_DONE}" = true ]; then - status_write restore done "${SETNAME}" "Restore complete" - history_append restore success "${SETNAME}" "Restore complete" + if [ "${RESTORE_LOADED}" = true ]; then + # Data is loaded; the api takes it from 'upgrading' to 'done'. Do not + # stamp success here or the UI would call the restore complete before + # the schema is actually brought up to code. + status_write restore upgrading "${SETNAME}" "Upgrading restored schema to code" else status_write restore failed "${SETNAME}" "Restore failed (exit ${code})" history_append restore failed "${SETNAME}" "Restore failed (exit ${code})" @@ -59,26 +66,41 @@ if [ ! -f "${DBDUMP}" ]; then fi [ -f "${DBDUMP}" ] || die "no db.sql.gz for set ${SETNAME}" -# Compatibility gate: the running schema must cover the set's stamped heads. +# Compatibility gate: the restore must gate on the revision the DUMP'S DATA is +# at (datarev), not on a code-to-code comparison. Loading data that sits at an +# OLDER revision than the running code is fine - we upgrade it afterwards - but +# loading data NEWER than the code (a revision the code has never heard of) +# would leave rows referencing columns the code cannot migrate to, so that is +# refused. log "checking schema compatibility" -CURRENT_HEADS="$(python3 /opt/backup/manifest.py \ - --out /tmp/current-manifest.json \ - --migrations /app/migrations --plugins /app/plugins \ - >/dev/null 2>&1 && python3 -c \ - 'import json;print(",".join(sorted(json.load(open("/tmp/current-manifest.json"))["schemarev"]["core"])))')" -SET_HEADS="$(python3 -c \ - 'import json,sys;print(",".join(sorted(json.load(open(sys.argv[1]))["schemarev"]["core"])))' \ - "${SETDIR}/manifest.json" 2>/dev/null || echo "")" -if [ -z "${SET_HEADS}" ]; then - die "set ${SETNAME} has no readable manifest - refusing an unverified restore" +# The set's data revision: prefer the manifest field; for older manifests that +# predate datarev, parse the real alembic_version straight out of the dump so a +# legacy set is still gated on its true data, not on a code-derived stamp. +SET_DATAREV="$(python3 -c \ + 'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \ + "${SETDIR}/manifest.json" 2>/dev/null || echo "")" +if [ -z "${SET_DATAREV}" ]; then + log "manifest has no datarev (legacy set) - reading it from the dump" + SET_DATAREV="$(gunzip -c "${DBDUMP}" \ + | grep -A2 -i 'INSERT INTO `alembic_version`' \ + | grep -oE "'[0-9a-zA-Z_]+'" | tr -d "'" | head -1)" fi -if [ "${CURRENT_HEADS}" != "${SET_HEADS}" ]; then - log "current heads : ${CURRENT_HEADS}" - log "set heads : ${SET_HEADS}" - die "schema mismatch - bring code to the set's revision, run 'flask db upgrade' and 'flask plugin upgrade-all', then retry" + +if [ -z "${SET_DATAREV}" ]; then + die "cannot determine the set's data revision - refusing an unverified restore" fi +# Does the running code know this revision at all? If a migration file with that +# revision id exists in the core chain, the code is at-or-past it (the head is +# the tip of the same chain), so a post-load upgrade can carry the data forward. +# If it is absent, the dump is NEWER than the code - refuse. +if ! grep -rqE "^revision = ['\"]${SET_DATAREV}['\"]" /app/migrations/versions 2>/dev/null; then + log "set datarev : ${SET_DATAREV}" + die "the backup's data is at a revision this code does not have - deploy code at or past '${SET_DATAREV}' before restoring" +fi +log "data revision ${SET_DATAREV} is known to this code - restore may proceed" + # Files first, then DB, so the two land together. if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then status_write restore files "${SETNAME}" "Restoring instance files" @@ -95,6 +117,17 @@ gunzip -c "${DBDUMP}" | mysql \ --password="${MYSQL_PASSWORD}" \ || die "database load failed" +# Hand off to the api container for the schema upgrade. The dump may sit at an +# older revision than this image (allowed and expected); without the upgrade the +# data stays behind the code and every query on a table the newer migrations +# changed 500s with 1054 Unknown column. This sidecar has no Flask, so it cannot +# run 'flask db upgrade' itself - it records the intent and lets the api (which +# owns the app context and already runs these exact idempotent commands on every +# deploy) finalize. The api clears .upgrade-requested and stamps the final +# 'done' once the upgrade succeeds. +status_write restore upgrading "${SETNAME}" "Upgrading restored schema to code" +printf '%s' "${SETNAME}" > "${BACKUPS_ROOT}/.upgrade-requested" +log "database loaded at revision ${SET_DATAREV}; handed off to api for schema upgrade" + rm -f "${BACKUPS_ROOT}/.restore-requested" 2>/dev/null || true -RESTORE_DONE=true -log "restore of ${SETNAME} complete" +RESTORE_LOADED=true diff --git a/scripts/backup/simulate-restore.sh b/scripts/backup/simulate-restore.sh index ed47cf9f..5b18529e 100644 --- a/scripts/backup/simulate-restore.sh +++ b/scripts/backup/simulate-restore.sh @@ -103,6 +103,17 @@ wait_for_phase() { ${COMPOSE} exec -T db sh -c 'true' >/dev/null 2>&1 \ || die "db container not reachable - is the stack up?" +# --- 0b. Datarev-gate preflight (external tester) -------------------------- +# This live round trip runs one codebase against itself, so it can never +# reproduce the "data revision behind the running code" case the restore gate +# now handles. Run the standalone datarev tester first so a pass here also +# covers that offline contract (manifest datarev, known-vs-unknown revision +# refusal, legacy fallback, and the api upgrade handoff). +log "running datarev gate preflight (test-restore-datarev.sh)" +bash "${SCRIPT_DIR}/test-restore-datarev.sh" \ + || die "datarev gate preflight failed - see output above" +log "datarev gate preflight passed" + # --- 1. Backup the current state, then fingerprint the RESTORE TARGET ------ # The restore point is the database as it existed WHEN THE DUMP RAN, not some # earlier moment. Live-churn tables (apitokens last-used, sessions, auditlogs) diff --git a/scripts/backup/test-restore-datarev.sh b/scripts/backup/test-restore-datarev.sh new file mode 100644 index 00000000..39681759 --- /dev/null +++ b/scripts/backup/test-restore-datarev.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# ShopDB restore datarev-gate tester. ***FOR DEVELOPER USE ONLY*** +# +# WHY THIS EXISTS SEPARATELY FROM simulate-restore.sh: +# simulate-restore.sh is a full-fidelity ROUND TRIP against a live stack: it +# backs up, mutates, restores, and proves every table returns to backup-time +# state. By construction it runs ONE codebase against ITSELF, so the database it +# dumps is always at the SAME revision as the code that restores it. It can +# therefore never exercise the case the FBJXBDB4 outage was about: a dump whose +# DATA revision is BEHIND (or ahead of) the running CODE. +# +# This tester fills that gap offline. It builds throwaway migration trees and +# fake gzipped dumps and drives the exact decision logic restore.sh uses: +# 1. manifest.py records datarev (the DB's real alembic_version), not just the +# code head (schemarev). +# 2. A dump whose datarev the code KNOWS (an ancestor in the chain) is allowed +# - the post-load upgrade rolls it forward. +# 3. A dump whose datarev the code does NOT know (newer than this build) is +# REFUSED - restoring it would 500 on 1054 Unknown column. +# 4. A legacy manifest with no datarev falls back to parsing alembic_version +# straight out of the dump. +# 5. restore.sh hands the loaded restore off to the api via the +# .upgrade-requested sentinel instead of marking it done in the sidecar. +# +# No Docker, no MySQL, no network: it is a pure logic gate test, safe to run in +# CI and from inside simulate-restore.sh as a preflight. +# +# ASCII only, no emojis - matches the repo style gate. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MANIFEST_PY="${SCRIPT_DIR}/manifest.py" +RESTORE_SH="${SCRIPT_DIR}/restore.sh" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "${WORKDIR}"' EXIT + +# The backup image ships python3; a dev host may only have `python` (and on +# Windows `python3` can be a non-functional Store alias that resolves on PATH +# but errors when run). Pick the first interpreter that actually EXECUTES, so +# this tester runs anywhere without changing the shipped scripts. +PY="" +for candidate in python3 python py; do + if "${candidate}" -c 'pass' >/dev/null 2>&1; then + PY="${candidate}" + break + fi +done +[ -n "${PY}" ] || { echo "[test] ERROR: no working python interpreter found"; exit 1; } + +passes=0 +fails=0 +pass() { echo "[test] PASS: $*"; passes=$((passes + 1)); } +fail() { echo "[test] FAIL: $*"; fails=$((fails + 1)); } + +# Build a throwaway core migration chain: baseline -> mid -> head. The gate in +# restore.sh only needs each version file to declare `revision = ''`, so +# that is all we write. +build_chain() { + root="$1" + versions="${root}/versions" + mkdir -p "${versions}" + printf "revision = '%s'\ndown_revision = None\n" \ + '0001_baseline' > "${versions}/0001_baseline.py" + printf "revision = '%s'\ndown_revision = '%s'\n" \ + '0002_mid' '0001_baseline' > "${versions}/0002_mid.py" + printf "revision = '%s'\ndown_revision = '%s'\n" \ + '0003_head' '0002_mid' > "${versions}/0003_head.py" +} + +# Fake a gzipped mysqldump that stamps a given alembic_version, matching the +# INSERT shape restore.sh greps for when a legacy manifest lacks datarev. +build_dump() { + out="$1"; datarev="$2" + { + echo 'DROP TABLE IF EXISTS `alembic_version`;' + echo 'CREATE TABLE `alembic_version` (`version_num` varchar(32));' + echo 'INSERT INTO `alembic_version` VALUES' + echo "('${datarev}');" + } | gzip -c > "${out}" +} + +# Mirror of the restore.sh predicate: does the code chain know this revision? +# Kept identical to restore.sh so this test tracks the shipped decision. +code_knows_revision() { + versions_dir="$1"; datarev="$2" + grep -rqE "^revision = ['\"]${datarev}['\"]" "${versions_dir}" 2>/dev/null +} + +# Mirror of the restore.sh datarev extraction: manifest field first, else parse +# the dump. +set_datarev() { + setdir="$1"; dbdump="$2" + value="$(${PY} -c \ + 'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \ + "${setdir}/manifest.json" 2>/dev/null || echo "")" + if [ -z "${value}" ]; then + value="$(gunzip -c "${dbdump}" \ + | grep -A2 -i 'INSERT INTO `alembic_version`' \ + | grep -oE "'[0-9a-zA-Z_]+'" | tr -d "'" | head -1)" + fi + printf '%s' "${value}" +} + +CHAIN="${WORKDIR}/migrations" +build_chain "${CHAIN}" +VERSIONS="${CHAIN}/versions" + +# --- 1. manifest.py records datarev distinct from the code head ------------- +SET1="${WORKDIR}/set1"; mkdir -p "${SET1}" +${PY} "${MANIFEST_PY}" --out "${SET1}/manifest.json" \ + --migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \ + --datarev '0002_mid' --takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1 +recorded="$(${PY} -c \ + 'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \ + "${SET1}/manifest.json" 2>/dev/null)" +head_stamp="$(${PY} -c \ + 'import json,sys;print(json.load(open(sys.argv[1])).get("schemarev",{}).get("core",[]))' \ + "${SET1}/manifest.json" 2>/dev/null)" +if [ "${recorded}" = '0002_mid' ]; then + pass "manifest.py records datarev (${recorded}) separate from code head ${head_stamp}" +else + fail "manifest.py did not record datarev (got '${recorded}')" +fi + +# --- 2. datarev the code KNOWS (behind head) is allowed -------------------- +DBDUMP1="${SET1}/db.sql.gz"; build_dump "${DBDUMP1}" '0002_mid' +rev="$(set_datarev "${SET1}" "${DBDUMP1}")" +if [ "${rev}" = '0002_mid' ] && code_knows_revision "${VERSIONS}" "${rev}"; then + pass "data behind code (${rev}) is known and allowed to restore" +else + fail "data behind code (${rev}) was not accepted" +fi + +# --- 3. datarev the code does NOT know (newer than build) is refused ------- +SET2="${WORKDIR}/set2"; mkdir -p "${SET2}" +${PY} "${MANIFEST_PY}" --out "${SET2}/manifest.json" \ + --migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \ + --datarev '0004_future' --takenat '2024-02-01T00:00:00Z' >/dev/null 2>&1 +DBDUMP2="${SET2}/db.sql.gz"; build_dump "${DBDUMP2}" '0004_future' +rev="$(set_datarev "${SET2}" "${DBDUMP2}")" +if [ "${rev}" = '0004_future' ] && ! code_knows_revision "${VERSIONS}" "${rev}"; then + pass "data ahead of code (${rev}) is unknown and correctly refused" +else + fail "data ahead of code (${rev}) was not refused" +fi + +# --- 4. legacy manifest (no datarev) falls back to the dump ---------------- +SET3="${WORKDIR}/set3"; mkdir -p "${SET3}" +# A pre-datarev manifest: schemarev present, datarev absent/empty. +printf '{"takenat":"2023-06-01T00:00:00Z","schemarev":{"core":["0003_head"]}}' \ + > "${SET3}/manifest.json" +DBDUMP3="${SET3}/db.sql.gz"; build_dump "${DBDUMP3}" '0001_baseline' +rev="$(set_datarev "${SET3}" "${DBDUMP3}")" +if [ "${rev}" = '0001_baseline' ] && code_knows_revision "${VERSIONS}" "${rev}"; then + pass "legacy set with no datarev falls back to dump revision (${rev})" +else + fail "legacy set fallback failed (got '${rev}')" +fi + +# --- 5. restore.sh hands off to the api instead of self-completing --------- +if grep -q '\.upgrade-requested' "${RESTORE_SH}" \ + && grep -q "status_write restore upgrading" "${RESTORE_SH}" \ + && ! grep -qE 'status_write restore done' "${RESTORE_SH}"; then + pass "restore.sh parks in 'upgrading' and hands off via .upgrade-requested" +else + fail "restore.sh does not hand off the upgrade to the api" +fi + +echo "==================================================" +echo "[test] datarev gate: ${passes} passed, ${fails} failed" +echo "==================================================" +[ "${fails}" -eq 0 ] diff --git a/shopdb/core/api/shopdbbackups.py b/shopdb/core/api/shopdbbackups.py index 27624a0e..c2f722bd 100644 --- a/shopdb/core/api/shopdbbackups.py +++ b/shopdb/core/api/shopdbbackups.py @@ -129,6 +129,58 @@ def _write_status(kind, phase, setname, message): pass +def _append_history(kind, outcome, setname, message): + """Append one run record to the shared history the UI reads, newest last. + + Mirrors status.sh's history_append so a finalize that completes in the api + (not the sidecar) still lands in the same history file the restore page + shows. Best-effort: history is a convenience log, never a gate. + """ + root = _backups_root() + stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z' + entry = { + 'kind': kind, + 'outcome': outcome, + 'setname': setname, + 'message': message, + 'at': stamp, + } + path = os.path.join(root, 'history', kind + 's.json') + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + entries = _read_json(path) + if not isinstance(entries, list): + entries = [] + entries.append(entry) + with open(path, 'w', encoding='utf-8') as handle: + json.dump(entries, handle) + except OSError: + pass + + +def _run_schema_upgrade(): + """Bring the just-restored database up to the running code, in-process. + + Runs the same two idempotent steps every deploy runs: the core Alembic + upgrade to head, then each plugin chain via the plugin manager. Both are + no-ops when the data already matches the code head, so this is safe to call + even when the restored dump was already current. Uses the live app context + and its configured engine, so no subprocess or extra Flask CLI is needed. + """ + from flask_migrate import upgrade as alembic_upgrade + + alembic_upgrade() + + manager = current_app.extensions.get('plugin_manager') + if manager is None: + raise RuntimeError('Plugin manager not initialized; cannot upgrade plugins') + results = manager.upgrade_all_plugins() + failed = {name: status for name, status in (results or {}).items() + if status not in ('ok', 'no-migrations')} + if failed: + raise RuntimeError('Plugin upgrade failed: ' + json.dumps(failed)) + + def _read_manifest(setdir): """Load a set's manifest.json, or None when absent/unreadable. @@ -175,14 +227,59 @@ def _current_schema_revisions(): return {} +def _known_core_revisions(): + """Return every core Alembic revision this build knows, heads and ancestors. + + Restore compatibility is now data-driven: a set restores when the running + code KNOWS the dump's data revision (so `flask db upgrade` can roll it + forward to head). Heads alone cannot answer that - a dump stamped at an + older-but-valid revision is perfectly restorable - so we walk the whole + script history. Lazy import + broad except keeps a missing/broken alembic + from 500ing the list endpoint (returns empty = everything reads unknown). + """ + try: + from alembic.config import Config + from alembic.script import ScriptDirectory + except ImportError: + return set() + + root = current_app.root_path + migrations = os.path.join(os.path.dirname(root), 'migrations') + ini = os.path.join(migrations, 'alembic.ini') + if not os.path.isfile(ini): + return set() + try: + config = Config(ini) + config.set_main_option('script_location', migrations) + script = ScriptDirectory.from_config(config) + return {rev.revision for rev in script.walk_revisions()} + except Exception: + return set() + + def _is_compatible(manifest, current): - """A set restores onto THIS build only when the build's schema is at least - the set's. We compare head-revision sets: if the set was taken on a schema - the current code has not reached, the code is too OLD and would 500 on the - restored rows, so restore is blocked until the code is upgraded. + """Decide whether a set restores onto THIS build. + + Data-driven gate (the fix for the FBJXBDB4 outage): the authoritative + signal is the set's `datarev` - the actual `alembic_version` of the dumped + DATABASE, not the code head that captured it. A restore is safe when the + running code KNOWS that revision (it is somewhere in this build's migration + history), because then the post-restore finalize can run `flask db upgrade` + to roll the loaded data forward to head. If the code does not know the + revision, the dump is from a NEWER schema than this build and would 500 on + restored rows, so we block until the code is upgraded. + + Legacy sets predate `datarev`; fall back to the old head-subset check on + `schemarev` so they stay listable and restorable as before. """ if not manifest: return False + datarev = manifest.get('datarev') + if datarev: + known = _known_core_revisions() + if not known: + return False + return datarev in known setrevs = set(manifest.get('schemarev', {}).get('core', [])) currevs = set(current.get('core', [])) if not setrevs or not currevs: @@ -338,6 +435,57 @@ def restore_backup(snapshotid): message='Restore queued and validated against the live schema.') +@shopdbbackups_bp.route('/finalize-restore', methods=['POST']) +@require_role('admin') +def finalize_restore(): + """Run the post-restore schema upgrade in the app context. + + The restore itself runs in the Flask-less backup sidecar, which loads the + dump and then drops an .upgrade-requested sentinel because it cannot run + 'flask db upgrade' itself. This endpoint is the api-side half: it brings the + just-restored DATA up to the running CODE with the exact idempotent commands + the deploy path uses, then stamps the final 'done'. Without it a restored + dump that sits at an older revision leaves every newer-migration table + throwing 1054 Unknown column. + + Idempotent and safe to call when nothing is pending: if there is no + .upgrade-requested sentinel it reports there was nothing to finalize. + """ + root = _backups_root() + sentinel = os.path.join(root, '.upgrade-requested') + try: + with open(sentinel, 'r', encoding='utf-8') as handle: + setname = handle.read().strip() + except OSError: + return success_response( + data={'finalized': False}, + message='No restore is awaiting schema upgrade.') + + _write_status('restore', 'upgrading', setname, + 'Upgrading restored schema to code') + try: + _run_schema_upgrade() + except Exception as error: # noqa: BLE001 - report any upgrade failure + _write_status('restore', 'failed', setname, + 'Schema upgrade failed: ' + str(error)) + _append_history('restore', 'failed', setname, + 'Schema upgrade failed: ' + str(error)) + return error_response(ErrorCodes.INTERNAL_ERROR, + 'Post-restore schema upgrade failed: ' + str(error), + http_code=500) + + try: + os.remove(sentinel) + except OSError: + pass + _write_status('restore', 'done', setname, 'Restore complete') + _append_history('restore', 'success', setname, 'Restore complete') + AuditLog.log('restore-finalized', 'Backup', entityname=setname) + return success_response( + data={'finalized': True, 'snapshotid': setname}, + message='Restore finalized: schema upgraded to the running code.') + + @shopdbbackups_bp.route('/status', methods=['GET']) @require_role('admin') def get_status():