b973561cb7
A restore could load a database dump whose schema was older than the running code. The manifest stamped schemarev from the code image on disk, so the compatibility check compared the code against itself and always passed. A set captured from a database at 7d37_import_batches_and_journal restored under code at 7d41_drop_guest_tools_secrets, leaving the data behind the code: every query on a table the newer migrations touched failed with 1054 Unknown column, which surfaced as empty configuration-policy dropdowns and a failure to load data on edit PC. The authoritative signal is the alembic_version of the database that was dumped, so manifests now carry datarev alongside schemarev, backup.sh reads it from the database and warns when it has drifted from the image, and the gate asks whether the running code knows that revision. Data behind the code is allowed because it can be rolled forward; data ahead of the code is refused because it cannot. Rolling it forward has to happen somewhere with an app context. The backup sidecar is mysql plus restic plus alembic only and has no Flask, so restore.sh loads the dump and parks the run in an upgrading phase behind an .upgrade-requested sentinel rather than declaring a success it cannot verify. The api container owns the Flask CLI and already runs these exact idempotent commands on every deploy, so a finalize endpoint there runs db upgrade plus plugin upgrade-all and stamps the terminal phase. The UI drives it on seeing the upgrading phase, so recovery needs no operator intervention. The live simulation runs one codebase against itself, so its dump is always at the same revision as the code restoring it and it can never reproduce this drift. The gate therefore gets a standalone tester that builds throwaway migration chains and dumps offline, invoked as a simulation preflight. It caught a real bug in the legacy dump-parse fallback, where the revision was matched loosely enough to return the alembic_version table name instead of the value it holds.
173 lines
7.2 KiB
Bash
173 lines
7.2 KiB
Bash
#!/usr/bin/env bash
|
|
# ShopDB restore datarev-gate tester. ***FOR DEVELOPER USE ONLY***
|
|
#
|
|
# WHY THIS EXISTS SEPARATELY FROM simulate-restore.sh:
|
|
# simulate-restore.sh is a full-fidelity ROUND TRIP against a live stack: it
|
|
# backs up, mutates, restores, and proves every table returns to backup-time
|
|
# state. By construction it runs ONE codebase against ITSELF, so the database it
|
|
# dumps is always at the SAME revision as the code that restores it. It can
|
|
# therefore never exercise the case the FBJXBDB4 outage was about: a dump whose
|
|
# DATA revision is BEHIND (or ahead of) the running CODE.
|
|
#
|
|
# This tester fills that gap offline. It builds throwaway migration trees and
|
|
# fake gzipped dumps and drives the exact decision logic restore.sh uses:
|
|
# 1. manifest.py records datarev (the DB's real alembic_version), not just the
|
|
# code head (schemarev).
|
|
# 2. A dump whose datarev the code KNOWS (an ancestor in the chain) is allowed
|
|
# - the post-load upgrade rolls it forward.
|
|
# 3. A dump whose datarev the code does NOT know (newer than this build) is
|
|
# REFUSED - restoring it would 500 on 1054 Unknown column.
|
|
# 4. A legacy manifest with no datarev falls back to parsing alembic_version
|
|
# straight out of the dump.
|
|
# 5. restore.sh hands the loaded restore off to the api via the
|
|
# .upgrade-requested sentinel instead of marking it done in the sidecar.
|
|
#
|
|
# No Docker, no MySQL, no network: it is a pure logic gate test, safe to run in
|
|
# CI and from inside simulate-restore.sh as a preflight.
|
|
#
|
|
# ASCII only, no emojis - matches the repo style gate.
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
MANIFEST_PY="${SCRIPT_DIR}/manifest.py"
|
|
RESTORE_SH="${SCRIPT_DIR}/restore.sh"
|
|
WORKDIR="$(mktemp -d)"
|
|
trap 'rm -rf "${WORKDIR}"' EXIT
|
|
|
|
# The backup image ships python3; a dev host may only have `python` (and on
|
|
# Windows `python3` can be a non-functional Store alias that resolves on PATH
|
|
# but errors when run). Pick the first interpreter that actually EXECUTES, so
|
|
# this tester runs anywhere without changing the shipped scripts.
|
|
PY=""
|
|
for candidate in python3 python py; do
|
|
if "${candidate}" -c 'pass' >/dev/null 2>&1; then
|
|
PY="${candidate}"
|
|
break
|
|
fi
|
|
done
|
|
[ -n "${PY}" ] || { echo "[test] ERROR: no working python interpreter found"; exit 1; }
|
|
|
|
passes=0
|
|
fails=0
|
|
pass() { echo "[test] PASS: $*"; passes=$((passes + 1)); }
|
|
fail() { echo "[test] FAIL: $*"; fails=$((fails + 1)); }
|
|
|
|
# Build a throwaway core migration chain: baseline -> mid -> head. The gate in
|
|
# restore.sh only needs each version file to declare `revision = '<id>'`, so
|
|
# that is all we write.
|
|
build_chain() {
|
|
root="$1"
|
|
versions="${root}/versions"
|
|
mkdir -p "${versions}"
|
|
printf "revision = '%s'\ndown_revision = None\n" \
|
|
'0001_baseline' > "${versions}/0001_baseline.py"
|
|
printf "revision = '%s'\ndown_revision = '%s'\n" \
|
|
'0002_mid' '0001_baseline' > "${versions}/0002_mid.py"
|
|
printf "revision = '%s'\ndown_revision = '%s'\n" \
|
|
'0003_head' '0002_mid' > "${versions}/0003_head.py"
|
|
}
|
|
|
|
# Fake a gzipped mysqldump that stamps a given alembic_version, matching the
|
|
# INSERT shape restore.sh greps for when a legacy manifest lacks datarev.
|
|
build_dump() {
|
|
out="$1"; datarev="$2"
|
|
{
|
|
echo 'DROP TABLE IF EXISTS `alembic_version`;'
|
|
echo 'CREATE TABLE `alembic_version` (`version_num` varchar(32));'
|
|
echo 'INSERT INTO `alembic_version` VALUES'
|
|
echo "('${datarev}');"
|
|
} | gzip -c > "${out}"
|
|
}
|
|
|
|
# Mirror of the restore.sh predicate: does the code chain know this revision?
|
|
# Kept identical to restore.sh so this test tracks the shipped decision.
|
|
code_knows_revision() {
|
|
versions_dir="$1"; datarev="$2"
|
|
grep -rqE "^revision = ['\"]${datarev}['\"]" "${versions_dir}" 2>/dev/null
|
|
}
|
|
|
|
# Mirror of the restore.sh datarev extraction: manifest field first, else parse
|
|
# the dump.
|
|
set_datarev() {
|
|
setdir="$1"; dbdump="$2"
|
|
value="$(${PY} -c \
|
|
'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \
|
|
"${setdir}/manifest.json" 2>/dev/null || echo "")"
|
|
if [ -z "${value}" ]; then
|
|
value="$(gunzip -c "${dbdump}" \
|
|
| grep -A2 -i 'INSERT INTO `alembic_version`' \
|
|
| grep -oE "'[0-9a-zA-Z_]+'" | tr -d "'" | head -1)"
|
|
fi
|
|
printf '%s' "${value}"
|
|
}
|
|
|
|
CHAIN="${WORKDIR}/migrations"
|
|
build_chain "${CHAIN}"
|
|
VERSIONS="${CHAIN}/versions"
|
|
|
|
# --- 1. manifest.py records datarev distinct from the code head -------------
|
|
SET1="${WORKDIR}/set1"; mkdir -p "${SET1}"
|
|
${PY} "${MANIFEST_PY}" --out "${SET1}/manifest.json" \
|
|
--migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \
|
|
--datarev '0002_mid' --takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1
|
|
recorded="$(${PY} -c \
|
|
'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \
|
|
"${SET1}/manifest.json" 2>/dev/null)"
|
|
head_stamp="$(${PY} -c \
|
|
'import json,sys;print(json.load(open(sys.argv[1])).get("schemarev",{}).get("core",[]))' \
|
|
"${SET1}/manifest.json" 2>/dev/null)"
|
|
if [ "${recorded}" = '0002_mid' ]; then
|
|
pass "manifest.py records datarev (${recorded}) separate from code head ${head_stamp}"
|
|
else
|
|
fail "manifest.py did not record datarev (got '${recorded}')"
|
|
fi
|
|
|
|
# --- 2. datarev the code KNOWS (behind head) is allowed --------------------
|
|
DBDUMP1="${SET1}/db.sql.gz"; build_dump "${DBDUMP1}" '0002_mid'
|
|
rev="$(set_datarev "${SET1}" "${DBDUMP1}")"
|
|
if [ "${rev}" = '0002_mid' ] && code_knows_revision "${VERSIONS}" "${rev}"; then
|
|
pass "data behind code (${rev}) is known and allowed to restore"
|
|
else
|
|
fail "data behind code (${rev}) was not accepted"
|
|
fi
|
|
|
|
# --- 3. datarev the code does NOT know (newer than build) is refused -------
|
|
SET2="${WORKDIR}/set2"; mkdir -p "${SET2}"
|
|
${PY} "${MANIFEST_PY}" --out "${SET2}/manifest.json" \
|
|
--migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \
|
|
--datarev '0004_future' --takenat '2024-02-01T00:00:00Z' >/dev/null 2>&1
|
|
DBDUMP2="${SET2}/db.sql.gz"; build_dump "${DBDUMP2}" '0004_future'
|
|
rev="$(set_datarev "${SET2}" "${DBDUMP2}")"
|
|
if [ "${rev}" = '0004_future' ] && ! code_knows_revision "${VERSIONS}" "${rev}"; then
|
|
pass "data ahead of code (${rev}) is unknown and correctly refused"
|
|
else
|
|
fail "data ahead of code (${rev}) was not refused"
|
|
fi
|
|
|
|
# --- 4. legacy manifest (no datarev) falls back to the dump ----------------
|
|
SET3="${WORKDIR}/set3"; mkdir -p "${SET3}"
|
|
# A pre-datarev manifest: schemarev present, datarev absent/empty.
|
|
printf '{"takenat":"2023-06-01T00:00:00Z","schemarev":{"core":["0003_head"]}}' \
|
|
> "${SET3}/manifest.json"
|
|
DBDUMP3="${SET3}/db.sql.gz"; build_dump "${DBDUMP3}" '0001_baseline'
|
|
rev="$(set_datarev "${SET3}" "${DBDUMP3}")"
|
|
if [ "${rev}" = '0001_baseline' ] && code_knows_revision "${VERSIONS}" "${rev}"; then
|
|
pass "legacy set with no datarev falls back to dump revision (${rev})"
|
|
else
|
|
fail "legacy set fallback failed (got '${rev}')"
|
|
fi
|
|
|
|
# --- 5. restore.sh hands off to the api instead of self-completing ---------
|
|
if grep -q '\.upgrade-requested' "${RESTORE_SH}" \
|
|
&& grep -q "status_write restore upgrading" "${RESTORE_SH}" \
|
|
&& ! grep -qE 'status_write restore done' "${RESTORE_SH}"; then
|
|
pass "restore.sh parks in 'upgrading' and hands off via .upgrade-requested"
|
|
else
|
|
fail "restore.sh does not hand off the upgrade to the api"
|
|
fi
|
|
|
|
echo "=================================================="
|
|
echo "[test] datarev gate: ${passes} passed, ${fails} failed"
|
|
echo "=================================================="
|
|
[ "${fails}" -eq 0 ]
|