From b973561cb76a5f24662ca89b8535c1f4d1a071c3 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Fri, 4 Sep 2026 15:38:31 -0400 Subject: [PATCH] Gate restore on the dump's data revision, not the code head A restore could load a database dump whose schema was older than the running code. The manifest stamped schemarev from the code image on disk, so the compatibility check compared the code against itself and always passed. A set captured from a database at 7d37_import_batches_and_journal restored under code at 7d41_drop_guest_tools_secrets, leaving the data behind the code: every query on a table the newer migrations touched failed with 1054 Unknown column, which surfaced as empty configuration-policy dropdowns and a failure to load data on edit PC. The authoritative signal is the alembic_version of the database that was dumped, so manifests now carry datarev alongside schemarev, backup.sh reads it from the database and warns when it has drifted from the image, and the gate asks whether the running code knows that revision. Data behind the code is allowed because it can be rolled forward; data ahead of the code is refused because it cannot. Rolling it forward has to happen somewhere with an app context. The backup sidecar is mysql plus restic plus alembic only and has no Flask, so restore.sh loads the dump and parks the run in an upgrading phase behind an .upgrade-requested sentinel rather than declaring a success it cannot verify. The api container owns the Flask CLI and already runs these exact idempotent commands on every deploy, so a finalize endpoint there runs db upgrade plus plugin upgrade-all and stamps the terminal phase. The UI drives it on seeing the upgrading phase, so recovery needs no operator intervention. The live simulation runs one codebase against itself, so its dump is always at the same revision as the code restoring it and it can never reproduce this drift. The gate therefore gets a standalone tester that builds throwaway migration chains and dumps offline, invoked as a simulation preflight. It caught a real bug in the legacy dump-parse fallback, where the revision was matched loosely enough to return the alembic_version table name instead of the value it holds. --- .../src/views/settings/ShopdbBackupsList.vue | 19 ++ scripts/backup/backup.sh | 29 +++ scripts/backup/manifest.py | 14 +- scripts/backup/restore.sh | 77 +++++--- scripts/backup/simulate-restore.sh | 11 ++ scripts/backup/test-restore-datarev.sh | 172 ++++++++++++++++++ shopdb/core/api/shopdbbackups.py | 156 +++++++++++++++- 7 files changed, 450 insertions(+), 28 deletions(-) create mode 100644 scripts/backup/test-restore-datarev.sh diff --git a/frontend/src/views/settings/ShopdbBackupsList.vue b/frontend/src/views/settings/ShopdbBackupsList.vue index eaeab054..705b98f4 100644 --- a/frontend/src/views/settings/ShopdbBackupsList.vue +++ b/frontend/src/views/settings/ShopdbBackupsList.vue @@ -309,6 +309,11 @@ const restoreTarget = ref(null) const restoreStatus = ref(null) let statusTimer = null let restoreStartedAt = 0 +// The Flask-less backup sidecar loads the dump and parks the restore in +// 'upgrading', because it cannot run `flask db upgrade` itself. The api can, so +// the browser pokes the finalize endpoint once to roll the restored data up to +// the running code. Guarded so we fire it exactly once per restore run. +let finalizeRequested = false const backups = ref([]) const restores = ref([]) @@ -320,6 +325,7 @@ const RESTORE_PHASE_LABELS = { checking: 'Checking compatibility...', files: 'Restoring files...', database: 'Loading database...', + upgrading: 'Upgrading schema to code...', done: 'Restore complete', failed: 'Restore failed', } @@ -476,6 +482,18 @@ async function pollStatus() { } restoreStatus.value = status const phase = status?.phase + if (phase === 'upgrading' && !finalizeRequested) { + // Hand off to the api container, which owns the Flask CLI, to run the + // post-load schema upgrade. Fire once; polling keeps tracking until it + // reports done or failed. + finalizeRequested = true + try { + await api.post('/shopdbbackups/finalize-restore') + } catch (error) { + // The finalize endpoint stamps its own failed status on error, which the + // next poll surfaces; nothing extra to do here. + } + } if (phase === 'done' || phase === 'failed') { stopStatusPolling() if (phase === 'done') { @@ -492,6 +510,7 @@ async function pollStatus() { function startStatusPolling() { stopStatusPolling() restoreStartedAt = Date.now() + finalizeRequested = false restoreStatus.value = { phase: 'queued', message: 'Restore queued' } // Poll on an interval only; skip the immediate call so the seeded "queued" // phase is guaranteed to paint before the first real status can replace it. diff --git a/scripts/backup/backup.sh b/scripts/backup/backup.sh index c64bad82..110f427b 100644 --- a/scripts/backup/backup.sh +++ b/scripts/backup/backup.sh @@ -60,15 +60,44 @@ mysqldump \ log "database dump written ($(du -h "${SETDIR}/db.sql.gz" | cut -f1))" # 2. Manifest: the schema stamp that makes a restore a checked operation. +# Capture the revision actually applied to the DATABASE (its alembic_version), +# not just the code head. A restore must gate on the DATA revision; stamping +# only the code head hides an image that has outrun its own database and lets a +# behind-schema dump restore under ahead-schema code (1054 Unknown column on +# every affected table). Query it straight from the dumped DB. +log "reading applied database revision (alembic_version)" +DATAREV="$(mysql \ + --host="${MYSQL_HOST}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + --skip-column-names --batch \ + -e "SELECT version_num FROM alembic_version LIMIT 1;" \ + "${MYSQL_DATABASE}" 2>/dev/null | tr -d '\r')" +[ -n "${DATAREV}" ] || log "WARN could not read alembic_version - datarev will be empty" + log "writing manifest" python3 /opt/backup/manifest.py \ --out "${SETDIR}/manifest.json" \ --takenat "${STAMP}" \ --site "${SITE}" \ --imagetag "${SHOPDB_IMAGE_TAG:-source}" \ + --datarev "${DATAREV}" \ --migrations /app/migrations \ --plugins /app/plugins +# Consistency guard: if the DATA revision does not match the CODE head, this +# backup captures a database the running image has outrun. It is still stored +# (losing it would be worse) but flagged loudly so a restore of it is a +# deliberate choice, not a surprise. +CODEREV="$(python3 -c \ + 'import json,sys;print(",".join(sorted(json.load(open(sys.argv[1]))["schemarev"]["core"])))' \ + "${SETDIR}/manifest.json" 2>/dev/null || echo "")" +if [ -n "${DATAREV}" ] && [ "${DATAREV}" != "${CODEREV}" ]; then + log "WARN schema drift: database is at '${DATAREV}' but code head is '${CODEREV}'" + log "WARN this DB has not run 'flask db upgrade' to the running image; restore will upgrade it" + history_append backup warning "${SETNAME}" "Schema drift: data ${DATAREV} vs code ${CODEREV}" +fi + # 3. restic snapshot: db dump + instance tree + manifest, one point in time. # Skips gracefully when restic/repo are not configured (local-only sites). IS_CHECK_PASSED=false diff --git a/scripts/backup/manifest.py b/scripts/backup/manifest.py index 8708ba94..45345468 100644 --- a/scripts/backup/manifest.py +++ b/scripts/backup/manifest.py @@ -6,15 +6,21 @@ restore." It records, at capture time: - takenat : ISO-8601 UTC timestamp of the run - site : the site tag - imagetag : the app image tag that produced the data - - schemarev : the core Alembic head revision(s) the running code is at, plus + - schemarev : the core Alembic head revision(s) the running CODE is at, plus each plugin chain's heads - the versions a restore target must be at least, or restored rows reference columns the live schema has not applied yet. + - datarev : the Alembic revision actually applied to the DATABASE that was + dumped (its alembic_version), which is the number a restore must + gate on. schemarev stamps the code on disk; datarev stamps the + data in the dump. They diverge when an image is ahead of a DB + that has not run 'flask db upgrade' yet, and that divergence is + exactly what a code-to-code check cannot see. Two modes: --out ... write a fresh manifest for a new set --amend ... patch an existing manifest (used to record ischeckpassed after - restic check runs) + restic check runs, or to backfill datarev) ASCII only, no emojis - matches the repo style gate. """ @@ -79,6 +85,7 @@ def write_manifest(args): 'site': args.site, 'imagetag': args.imagetag, 'schemarev': _collect_schema(args.migrations, args.plugins), + 'datarev': args.datarev or '', 'ischeckpassed': False, } with open(args.out, 'w', encoding='utf-8') as handle: @@ -94,6 +101,8 @@ def amend_manifest(args): manifest = {} if args.ischeckpassed is not None: manifest['ischeckpassed'] = args.ischeckpassed.lower() == 'true' + if args.datarev is not None: + manifest['datarev'] = args.datarev with open(args.amend, 'w', encoding='utf-8') as handle: json.dump(manifest, handle, indent=2, sort_keys=True) return 0 @@ -108,6 +117,7 @@ def main(argv=None): parser.add_argument('--imagetag', default='source') parser.add_argument('--migrations', default='/app/migrations') parser.add_argument('--plugins', default='/app/plugins') + parser.add_argument('--datarev') parser.add_argument('--ischeckpassed') args = parser.parse_args(argv) diff --git a/scripts/backup/restore.sh b/scripts/backup/restore.sh index 8adaf1c2..9e75ad57 100644 --- a/scripts/backup/restore.sh +++ b/scripts/backup/restore.sh @@ -27,13 +27,20 @@ SETNAME="${1:-}" log() { echo "[restore] $*"; } # On any exit, stamp the terminal phase and append a history record so the UI -# reflects success or failure even when a step aborts early (set -e). -RESTORE_DONE=false +# reflects success or failure even when a step aborts early (set -e). A restore +# is only "done" once the DATA has been upgraded to the running CODE, and that +# upgrade runs in the Flask-capable api container, not here (this sidecar has no +# app). So the happy path ends by handing off to the api: this script stamps an +# 'upgrading' phase and drops an .upgrade-requested sentinel, and the api runs +# 'flask db upgrade' + 'flask plugin upgrade-all' and writes the final 'done'. +RESTORE_LOADED=false _on_exit() { code=$? - if [ "${RESTORE_DONE}" = true ]; then - status_write restore done "${SETNAME}" "Restore complete" - history_append restore success "${SETNAME}" "Restore complete" + if [ "${RESTORE_LOADED}" = true ]; then + # Data is loaded; the api takes it from 'upgrading' to 'done'. Do not + # stamp success here or the UI would call the restore complete before + # the schema is actually brought up to code. + status_write restore upgrading "${SETNAME}" "Upgrading restored schema to code" else status_write restore failed "${SETNAME}" "Restore failed (exit ${code})" history_append restore failed "${SETNAME}" "Restore failed (exit ${code})" @@ -59,26 +66,41 @@ if [ ! -f "${DBDUMP}" ]; then fi [ -f "${DBDUMP}" ] || die "no db.sql.gz for set ${SETNAME}" -# Compatibility gate: the running schema must cover the set's stamped heads. +# Compatibility gate: the restore must gate on the revision the DUMP'S DATA is +# at (datarev), not on a code-to-code comparison. Loading data that sits at an +# OLDER revision than the running code is fine - we upgrade it afterwards - but +# loading data NEWER than the code (a revision the code has never heard of) +# would leave rows referencing columns the code cannot migrate to, so that is +# refused. log "checking schema compatibility" -CURRENT_HEADS="$(python3 /opt/backup/manifest.py \ - --out /tmp/current-manifest.json \ - --migrations /app/migrations --plugins /app/plugins \ - >/dev/null 2>&1 && python3 -c \ - 'import json;print(",".join(sorted(json.load(open("/tmp/current-manifest.json"))["schemarev"]["core"])))')" -SET_HEADS="$(python3 -c \ - 'import json,sys;print(",".join(sorted(json.load(open(sys.argv[1]))["schemarev"]["core"])))' \ - "${SETDIR}/manifest.json" 2>/dev/null || echo "")" -if [ -z "${SET_HEADS}" ]; then - die "set ${SETNAME} has no readable manifest - refusing an unverified restore" +# The set's data revision: prefer the manifest field; for older manifests that +# predate datarev, parse the real alembic_version straight out of the dump so a +# legacy set is still gated on its true data, not on a code-derived stamp. +SET_DATAREV="$(python3 -c \ + 'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \ + "${SETDIR}/manifest.json" 2>/dev/null || echo "")" +if [ -z "${SET_DATAREV}" ]; then + log "manifest has no datarev (legacy set) - reading it from the dump" + SET_DATAREV="$(gunzip -c "${DBDUMP}" \ + | grep -A2 -i 'INSERT INTO `alembic_version`' \ + | grep -oE "'[0-9a-zA-Z_]+'" | tr -d "'" | head -1)" fi -if [ "${CURRENT_HEADS}" != "${SET_HEADS}" ]; then - log "current heads : ${CURRENT_HEADS}" - log "set heads : ${SET_HEADS}" - die "schema mismatch - bring code to the set's revision, run 'flask db upgrade' and 'flask plugin upgrade-all', then retry" + +if [ -z "${SET_DATAREV}" ]; then + die "cannot determine the set's data revision - refusing an unverified restore" fi +# Does the running code know this revision at all? If a migration file with that +# revision id exists in the core chain, the code is at-or-past it (the head is +# the tip of the same chain), so a post-load upgrade can carry the data forward. +# If it is absent, the dump is NEWER than the code - refuse. +if ! grep -rqE "^revision = ['\"]${SET_DATAREV}['\"]" /app/migrations/versions 2>/dev/null; then + log "set datarev : ${SET_DATAREV}" + die "the backup's data is at a revision this code does not have - deploy code at or past '${SET_DATAREV}' before restoring" +fi +log "data revision ${SET_DATAREV} is known to this code - restore may proceed" + # Files first, then DB, so the two land together. if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then status_write restore files "${SETNAME}" "Restoring instance files" @@ -95,6 +117,17 @@ gunzip -c "${DBDUMP}" | mysql \ --password="${MYSQL_PASSWORD}" \ || die "database load failed" +# Hand off to the api container for the schema upgrade. The dump may sit at an +# older revision than this image (allowed and expected); without the upgrade the +# data stays behind the code and every query on a table the newer migrations +# changed 500s with 1054 Unknown column. This sidecar has no Flask, so it cannot +# run 'flask db upgrade' itself - it records the intent and lets the api (which +# owns the app context and already runs these exact idempotent commands on every +# deploy) finalize. The api clears .upgrade-requested and stamps the final +# 'done' once the upgrade succeeds. +status_write restore upgrading "${SETNAME}" "Upgrading restored schema to code" +printf '%s' "${SETNAME}" > "${BACKUPS_ROOT}/.upgrade-requested" +log "database loaded at revision ${SET_DATAREV}; handed off to api for schema upgrade" + rm -f "${BACKUPS_ROOT}/.restore-requested" 2>/dev/null || true -RESTORE_DONE=true -log "restore of ${SETNAME} complete" +RESTORE_LOADED=true diff --git a/scripts/backup/simulate-restore.sh b/scripts/backup/simulate-restore.sh index ed47cf9f..5b18529e 100644 --- a/scripts/backup/simulate-restore.sh +++ b/scripts/backup/simulate-restore.sh @@ -103,6 +103,17 @@ wait_for_phase() { ${COMPOSE} exec -T db sh -c 'true' >/dev/null 2>&1 \ || die "db container not reachable - is the stack up?" +# --- 0b. Datarev-gate preflight (external tester) -------------------------- +# This live round trip runs one codebase against itself, so it can never +# reproduce the "data revision behind the running code" case the restore gate +# now handles. Run the standalone datarev tester first so a pass here also +# covers that offline contract (manifest datarev, known-vs-unknown revision +# refusal, legacy fallback, and the api upgrade handoff). +log "running datarev gate preflight (test-restore-datarev.sh)" +bash "${SCRIPT_DIR}/test-restore-datarev.sh" \ + || die "datarev gate preflight failed - see output above" +log "datarev gate preflight passed" + # --- 1. Backup the current state, then fingerprint the RESTORE TARGET ------ # The restore point is the database as it existed WHEN THE DUMP RAN, not some # earlier moment. Live-churn tables (apitokens last-used, sessions, auditlogs) diff --git a/scripts/backup/test-restore-datarev.sh b/scripts/backup/test-restore-datarev.sh new file mode 100644 index 00000000..39681759 --- /dev/null +++ b/scripts/backup/test-restore-datarev.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# ShopDB restore datarev-gate tester. ***FOR DEVELOPER USE ONLY*** +# +# WHY THIS EXISTS SEPARATELY FROM simulate-restore.sh: +# simulate-restore.sh is a full-fidelity ROUND TRIP against a live stack: it +# backs up, mutates, restores, and proves every table returns to backup-time +# state. By construction it runs ONE codebase against ITSELF, so the database it +# dumps is always at the SAME revision as the code that restores it. It can +# therefore never exercise the case the FBJXBDB4 outage was about: a dump whose +# DATA revision is BEHIND (or ahead of) the running CODE. +# +# This tester fills that gap offline. It builds throwaway migration trees and +# fake gzipped dumps and drives the exact decision logic restore.sh uses: +# 1. manifest.py records datarev (the DB's real alembic_version), not just the +# code head (schemarev). +# 2. A dump whose datarev the code KNOWS (an ancestor in the chain) is allowed +# - the post-load upgrade rolls it forward. +# 3. A dump whose datarev the code does NOT know (newer than this build) is +# REFUSED - restoring it would 500 on 1054 Unknown column. +# 4. A legacy manifest with no datarev falls back to parsing alembic_version +# straight out of the dump. +# 5. restore.sh hands the loaded restore off to the api via the +# .upgrade-requested sentinel instead of marking it done in the sidecar. +# +# No Docker, no MySQL, no network: it is a pure logic gate test, safe to run in +# CI and from inside simulate-restore.sh as a preflight. +# +# ASCII only, no emojis - matches the repo style gate. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MANIFEST_PY="${SCRIPT_DIR}/manifest.py" +RESTORE_SH="${SCRIPT_DIR}/restore.sh" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "${WORKDIR}"' EXIT + +# The backup image ships python3; a dev host may only have `python` (and on +# Windows `python3` can be a non-functional Store alias that resolves on PATH +# but errors when run). Pick the first interpreter that actually EXECUTES, so +# this tester runs anywhere without changing the shipped scripts. +PY="" +for candidate in python3 python py; do + if "${candidate}" -c 'pass' >/dev/null 2>&1; then + PY="${candidate}" + break + fi +done +[ -n "${PY}" ] || { echo "[test] ERROR: no working python interpreter found"; exit 1; } + +passes=0 +fails=0 +pass() { echo "[test] PASS: $*"; passes=$((passes + 1)); } +fail() { echo "[test] FAIL: $*"; fails=$((fails + 1)); } + +# Build a throwaway core migration chain: baseline -> mid -> head. The gate in +# restore.sh only needs each version file to declare `revision = ''`, so +# that is all we write. +build_chain() { + root="$1" + versions="${root}/versions" + mkdir -p "${versions}" + printf "revision = '%s'\ndown_revision = None\n" \ + '0001_baseline' > "${versions}/0001_baseline.py" + printf "revision = '%s'\ndown_revision = '%s'\n" \ + '0002_mid' '0001_baseline' > "${versions}/0002_mid.py" + printf "revision = '%s'\ndown_revision = '%s'\n" \ + '0003_head' '0002_mid' > "${versions}/0003_head.py" +} + +# Fake a gzipped mysqldump that stamps a given alembic_version, matching the +# INSERT shape restore.sh greps for when a legacy manifest lacks datarev. +build_dump() { + out="$1"; datarev="$2" + { + echo 'DROP TABLE IF EXISTS `alembic_version`;' + echo 'CREATE TABLE `alembic_version` (`version_num` varchar(32));' + echo 'INSERT INTO `alembic_version` VALUES' + echo "('${datarev}');" + } | gzip -c > "${out}" +} + +# Mirror of the restore.sh predicate: does the code chain know this revision? +# Kept identical to restore.sh so this test tracks the shipped decision. +code_knows_revision() { + versions_dir="$1"; datarev="$2" + grep -rqE "^revision = ['\"]${datarev}['\"]" "${versions_dir}" 2>/dev/null +} + +# Mirror of the restore.sh datarev extraction: manifest field first, else parse +# the dump. +set_datarev() { + setdir="$1"; dbdump="$2" + value="$(${PY} -c \ + 'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \ + "${setdir}/manifest.json" 2>/dev/null || echo "")" + if [ -z "${value}" ]; then + value="$(gunzip -c "${dbdump}" \ + | grep -A2 -i 'INSERT INTO `alembic_version`' \ + | grep -oE "'[0-9a-zA-Z_]+'" | tr -d "'" | head -1)" + fi + printf '%s' "${value}" +} + +CHAIN="${WORKDIR}/migrations" +build_chain "${CHAIN}" +VERSIONS="${CHAIN}/versions" + +# --- 1. manifest.py records datarev distinct from the code head ------------- +SET1="${WORKDIR}/set1"; mkdir -p "${SET1}" +${PY} "${MANIFEST_PY}" --out "${SET1}/manifest.json" \ + --migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \ + --datarev '0002_mid' --takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1 +recorded="$(${PY} -c \ + 'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \ + "${SET1}/manifest.json" 2>/dev/null)" +head_stamp="$(${PY} -c \ + 'import json,sys;print(json.load(open(sys.argv[1])).get("schemarev",{}).get("core",[]))' \ + "${SET1}/manifest.json" 2>/dev/null)" +if [ "${recorded}" = '0002_mid' ]; then + pass "manifest.py records datarev (${recorded}) separate from code head ${head_stamp}" +else + fail "manifest.py did not record datarev (got '${recorded}')" +fi + +# --- 2. datarev the code KNOWS (behind head) is allowed -------------------- +DBDUMP1="${SET1}/db.sql.gz"; build_dump "${DBDUMP1}" '0002_mid' +rev="$(set_datarev "${SET1}" "${DBDUMP1}")" +if [ "${rev}" = '0002_mid' ] && code_knows_revision "${VERSIONS}" "${rev}"; then + pass "data behind code (${rev}) is known and allowed to restore" +else + fail "data behind code (${rev}) was not accepted" +fi + +# --- 3. datarev the code does NOT know (newer than build) is refused ------- +SET2="${WORKDIR}/set2"; mkdir -p "${SET2}" +${PY} "${MANIFEST_PY}" --out "${SET2}/manifest.json" \ + --migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \ + --datarev '0004_future' --takenat '2024-02-01T00:00:00Z' >/dev/null 2>&1 +DBDUMP2="${SET2}/db.sql.gz"; build_dump "${DBDUMP2}" '0004_future' +rev="$(set_datarev "${SET2}" "${DBDUMP2}")" +if [ "${rev}" = '0004_future' ] && ! code_knows_revision "${VERSIONS}" "${rev}"; then + pass "data ahead of code (${rev}) is unknown and correctly refused" +else + fail "data ahead of code (${rev}) was not refused" +fi + +# --- 4. legacy manifest (no datarev) falls back to the dump ---------------- +SET3="${WORKDIR}/set3"; mkdir -p "${SET3}" +# A pre-datarev manifest: schemarev present, datarev absent/empty. +printf '{"takenat":"2023-06-01T00:00:00Z","schemarev":{"core":["0003_head"]}}' \ + > "${SET3}/manifest.json" +DBDUMP3="${SET3}/db.sql.gz"; build_dump "${DBDUMP3}" '0001_baseline' +rev="$(set_datarev "${SET3}" "${DBDUMP3}")" +if [ "${rev}" = '0001_baseline' ] && code_knows_revision "${VERSIONS}" "${rev}"; then + pass "legacy set with no datarev falls back to dump revision (${rev})" +else + fail "legacy set fallback failed (got '${rev}')" +fi + +# --- 5. restore.sh hands off to the api instead of self-completing --------- +if grep -q '\.upgrade-requested' "${RESTORE_SH}" \ + && grep -q "status_write restore upgrading" "${RESTORE_SH}" \ + && ! grep -qE 'status_write restore done' "${RESTORE_SH}"; then + pass "restore.sh parks in 'upgrading' and hands off via .upgrade-requested" +else + fail "restore.sh does not hand off the upgrade to the api" +fi + +echo "==================================================" +echo "[test] datarev gate: ${passes} passed, ${fails} failed" +echo "==================================================" +[ "${fails}" -eq 0 ] diff --git a/shopdb/core/api/shopdbbackups.py b/shopdb/core/api/shopdbbackups.py index 27624a0e..c2f722bd 100644 --- a/shopdb/core/api/shopdbbackups.py +++ b/shopdb/core/api/shopdbbackups.py @@ -129,6 +129,58 @@ def _write_status(kind, phase, setname, message): pass +def _append_history(kind, outcome, setname, message): + """Append one run record to the shared history the UI reads, newest last. + + Mirrors status.sh's history_append so a finalize that completes in the api + (not the sidecar) still lands in the same history file the restore page + shows. Best-effort: history is a convenience log, never a gate. + """ + root = _backups_root() + stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z' + entry = { + 'kind': kind, + 'outcome': outcome, + 'setname': setname, + 'message': message, + 'at': stamp, + } + path = os.path.join(root, 'history', kind + 's.json') + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + entries = _read_json(path) + if not isinstance(entries, list): + entries = [] + entries.append(entry) + with open(path, 'w', encoding='utf-8') as handle: + json.dump(entries, handle) + except OSError: + pass + + +def _run_schema_upgrade(): + """Bring the just-restored database up to the running code, in-process. + + Runs the same two idempotent steps every deploy runs: the core Alembic + upgrade to head, then each plugin chain via the plugin manager. Both are + no-ops when the data already matches the code head, so this is safe to call + even when the restored dump was already current. Uses the live app context + and its configured engine, so no subprocess or extra Flask CLI is needed. + """ + from flask_migrate import upgrade as alembic_upgrade + + alembic_upgrade() + + manager = current_app.extensions.get('plugin_manager') + if manager is None: + raise RuntimeError('Plugin manager not initialized; cannot upgrade plugins') + results = manager.upgrade_all_plugins() + failed = {name: status for name, status in (results or {}).items() + if status not in ('ok', 'no-migrations')} + if failed: + raise RuntimeError('Plugin upgrade failed: ' + json.dumps(failed)) + + def _read_manifest(setdir): """Load a set's manifest.json, or None when absent/unreadable. @@ -175,14 +227,59 @@ def _current_schema_revisions(): return {} +def _known_core_revisions(): + """Return every core Alembic revision this build knows, heads and ancestors. + + Restore compatibility is now data-driven: a set restores when the running + code KNOWS the dump's data revision (so `flask db upgrade` can roll it + forward to head). Heads alone cannot answer that - a dump stamped at an + older-but-valid revision is perfectly restorable - so we walk the whole + script history. Lazy import + broad except keeps a missing/broken alembic + from 500ing the list endpoint (returns empty = everything reads unknown). + """ + try: + from alembic.config import Config + from alembic.script import ScriptDirectory + except ImportError: + return set() + + root = current_app.root_path + migrations = os.path.join(os.path.dirname(root), 'migrations') + ini = os.path.join(migrations, 'alembic.ini') + if not os.path.isfile(ini): + return set() + try: + config = Config(ini) + config.set_main_option('script_location', migrations) + script = ScriptDirectory.from_config(config) + return {rev.revision for rev in script.walk_revisions()} + except Exception: + return set() + + def _is_compatible(manifest, current): - """A set restores onto THIS build only when the build's schema is at least - the set's. We compare head-revision sets: if the set was taken on a schema - the current code has not reached, the code is too OLD and would 500 on the - restored rows, so restore is blocked until the code is upgraded. + """Decide whether a set restores onto THIS build. + + Data-driven gate (the fix for the FBJXBDB4 outage): the authoritative + signal is the set's `datarev` - the actual `alembic_version` of the dumped + DATABASE, not the code head that captured it. A restore is safe when the + running code KNOWS that revision (it is somewhere in this build's migration + history), because then the post-restore finalize can run `flask db upgrade` + to roll the loaded data forward to head. If the code does not know the + revision, the dump is from a NEWER schema than this build and would 500 on + restored rows, so we block until the code is upgraded. + + Legacy sets predate `datarev`; fall back to the old head-subset check on + `schemarev` so they stay listable and restorable as before. """ if not manifest: return False + datarev = manifest.get('datarev') + if datarev: + known = _known_core_revisions() + if not known: + return False + return datarev in known setrevs = set(manifest.get('schemarev', {}).get('core', [])) currevs = set(current.get('core', [])) if not setrevs or not currevs: @@ -338,6 +435,57 @@ def restore_backup(snapshotid): message='Restore queued and validated against the live schema.') +@shopdbbackups_bp.route('/finalize-restore', methods=['POST']) +@require_role('admin') +def finalize_restore(): + """Run the post-restore schema upgrade in the app context. + + The restore itself runs in the Flask-less backup sidecar, which loads the + dump and then drops an .upgrade-requested sentinel because it cannot run + 'flask db upgrade' itself. This endpoint is the api-side half: it brings the + just-restored DATA up to the running CODE with the exact idempotent commands + the deploy path uses, then stamps the final 'done'. Without it a restored + dump that sits at an older revision leaves every newer-migration table + throwing 1054 Unknown column. + + Idempotent and safe to call when nothing is pending: if there is no + .upgrade-requested sentinel it reports there was nothing to finalize. + """ + root = _backups_root() + sentinel = os.path.join(root, '.upgrade-requested') + try: + with open(sentinel, 'r', encoding='utf-8') as handle: + setname = handle.read().strip() + except OSError: + return success_response( + data={'finalized': False}, + message='No restore is awaiting schema upgrade.') + + _write_status('restore', 'upgrading', setname, + 'Upgrading restored schema to code') + try: + _run_schema_upgrade() + except Exception as error: # noqa: BLE001 - report any upgrade failure + _write_status('restore', 'failed', setname, + 'Schema upgrade failed: ' + str(error)) + _append_history('restore', 'failed', setname, + 'Schema upgrade failed: ' + str(error)) + return error_response(ErrorCodes.INTERNAL_ERROR, + 'Post-restore schema upgrade failed: ' + str(error), + http_code=500) + + try: + os.remove(sentinel) + except OSError: + pass + _write_status('restore', 'done', setname, 'Restore complete') + _append_history('restore', 'success', setname, 'Restore complete') + AuditLog.log('restore-finalized', 'Backup', entityname=setname) + return success_response( + data={'finalized': True, 'snapshotid': setname}, + message='Restore finalized: schema upgraded to the running code.') + + @shopdbbackups_bp.route('/status', methods=['GET']) @require_role('admin') def get_status():