#!/usr/bin/env bash # ShopDB restore datarev-gate tester. ***FOR DEVELOPER USE ONLY*** # # WHY THIS EXISTS SEPARATELY FROM simulate-restore.sh: # simulate-restore.sh is a full-fidelity ROUND TRIP against a live stack: it # backs up, mutates, restores, and proves every table returns to backup-time # state. By construction it runs ONE codebase against ITSELF, so the database it # dumps is always at the SAME revision as the code that restores it. It can # therefore never exercise the case the FBJXBDB4 outage was about: a dump whose # DATA revision is BEHIND (or ahead of) the running CODE. # # This tester fills that gap offline. It builds throwaway migration trees and # fake gzipped dumps and drives the exact decision logic restore.sh uses: # 1. manifest.py records datarev (the DB's real alembic_version), not just the # code head (schemarev). # 2. A dump whose datarev the code KNOWS (an ancestor in the chain) is allowed # - the post-load upgrade rolls it forward. # 3. A dump whose datarev the code does NOT know (newer than this build) is # REFUSED - restoring it would 500 on 1054 Unknown column. # 4. A legacy manifest with no datarev falls back to parsing alembic_version # straight out of the dump. # 5. restore.sh hands the loaded restore off to the api via the # .upgrade-requested sentinel instead of marking it done in the sidecar. # # No Docker, no MySQL, no network: it is a pure logic gate test, safe to run in # CI and from inside simulate-restore.sh as a preflight. # # ASCII only, no emojis - matches the repo style gate. set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" MANIFEST_PY="${SCRIPT_DIR}/manifest.py" RESTORE_SH="${SCRIPT_DIR}/restore.sh" WORKDIR="$(mktemp -d)" trap 'rm -rf "${WORKDIR}"' EXIT # The backup image ships python3; a dev host may only have `python` (and on # Windows `python3` can be a non-functional Store alias that resolves on PATH # but errors when run). Pick the first interpreter that actually EXECUTES, so # this tester runs anywhere without changing the shipped scripts. PY="" for candidate in python3 python py; do if "${candidate}" -c 'pass' >/dev/null 2>&1; then PY="${candidate}" break fi done [ -n "${PY}" ] || { echo "[test] ERROR: no working python interpreter found"; exit 1; } passes=0 fails=0 pass() { echo "[test] PASS: $*"; passes=$((passes + 1)); } fail() { echo "[test] FAIL: $*"; fails=$((fails + 1)); } # Build a throwaway core migration chain: baseline -> mid -> head. The gate in # restore.sh only needs each version file to declare `revision = ''`, so # that is all we write. build_chain() { root="$1" versions="${root}/versions" mkdir -p "${versions}" printf "revision = '%s'\ndown_revision = None\n" \ '0001_baseline' > "${versions}/0001_baseline.py" printf "revision = '%s'\ndown_revision = '%s'\n" \ '0002_mid' '0001_baseline' > "${versions}/0002_mid.py" printf "revision = '%s'\ndown_revision = '%s'\n" \ '0003_head' '0002_mid' > "${versions}/0003_head.py" } # Fake a gzipped mysqldump that stamps a given alembic_version, matching the # INSERT shape restore.sh greps for when a legacy manifest lacks datarev. build_dump() { out="$1"; datarev="$2" { echo 'DROP TABLE IF EXISTS `alembic_version`;' echo 'CREATE TABLE `alembic_version` (`version_num` varchar(32));' echo 'INSERT INTO `alembic_version` VALUES' echo "('${datarev}');" } | gzip -c > "${out}" } # Mirror of the restore.sh predicate: does the code chain know this revision? # Kept identical to restore.sh so this test tracks the shipped decision. code_knows_revision() { versions_dir="$1"; datarev="$2" grep -rqE "^revision = ['\"]${datarev}['\"]" "${versions_dir}" 2>/dev/null } # Mirror of the restore.sh datarev extraction: manifest field first, else parse # the dump. set_datarev() { setdir="$1"; dbdump="$2" value="$(${PY} -c \ 'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \ "${setdir}/manifest.json" 2>/dev/null || echo "")" if [ -z "${value}" ]; then value="$(gunzip -c "${dbdump}" \ | grep -A2 -i 'INSERT INTO `alembic_version`' \ | grep -oE "'[0-9a-zA-Z_]+'" | tr -d "'" | head -1)" fi printf '%s' "${value}" } CHAIN="${WORKDIR}/migrations" build_chain "${CHAIN}" VERSIONS="${CHAIN}/versions" # --- 1. manifest.py records datarev distinct from the code head ------------- SET1="${WORKDIR}/set1"; mkdir -p "${SET1}" ${PY} "${MANIFEST_PY}" --out "${SET1}/manifest.json" \ --migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \ --datarev '0002_mid' --takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1 recorded="$(${PY} -c \ 'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \ "${SET1}/manifest.json" 2>/dev/null)" head_stamp="$(${PY} -c \ 'import json,sys;print(json.load(open(sys.argv[1])).get("schemarev",{}).get("core",[]))' \ "${SET1}/manifest.json" 2>/dev/null)" if [ "${recorded}" = '0002_mid' ]; then pass "manifest.py records datarev (${recorded}) separate from code head ${head_stamp}" else fail "manifest.py did not record datarev (got '${recorded}')" fi # --- 2. datarev the code KNOWS (behind head) is allowed -------------------- DBDUMP1="${SET1}/db.sql.gz"; build_dump "${DBDUMP1}" '0002_mid' rev="$(set_datarev "${SET1}" "${DBDUMP1}")" if [ "${rev}" = '0002_mid' ] && code_knows_revision "${VERSIONS}" "${rev}"; then pass "data behind code (${rev}) is known and allowed to restore" else fail "data behind code (${rev}) was not accepted" fi # --- 3. datarev the code does NOT know (newer than build) is refused ------- SET2="${WORKDIR}/set2"; mkdir -p "${SET2}" ${PY} "${MANIFEST_PY}" --out "${SET2}/manifest.json" \ --migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \ --datarev '0004_future' --takenat '2024-02-01T00:00:00Z' >/dev/null 2>&1 DBDUMP2="${SET2}/db.sql.gz"; build_dump "${DBDUMP2}" '0004_future' rev="$(set_datarev "${SET2}" "${DBDUMP2}")" if [ "${rev}" = '0004_future' ] && ! code_knows_revision "${VERSIONS}" "${rev}"; then pass "data ahead of code (${rev}) is unknown and correctly refused" else fail "data ahead of code (${rev}) was not refused" fi # --- 4. legacy manifest (no datarev) falls back to the dump ---------------- SET3="${WORKDIR}/set3"; mkdir -p "${SET3}" # A pre-datarev manifest: schemarev present, datarev absent/empty. printf '{"takenat":"2023-06-01T00:00:00Z","schemarev":{"core":["0003_head"]}}' \ > "${SET3}/manifest.json" DBDUMP3="${SET3}/db.sql.gz"; build_dump "${DBDUMP3}" '0001_baseline' rev="$(set_datarev "${SET3}" "${DBDUMP3}")" if [ "${rev}" = '0001_baseline' ] && code_knows_revision "${VERSIONS}" "${rev}"; then pass "legacy set with no datarev falls back to dump revision (${rev})" else fail "legacy set fallback failed (got '${rev}')" fi # --- 5. restore.sh hands off to the api instead of self-completing --------- if grep -q '\.upgrade-requested' "${RESTORE_SH}" \ && grep -q "status_write restore upgrading" "${RESTORE_SH}" \ && ! grep -qE 'status_write restore done' "${RESTORE_SH}"; then pass "restore.sh parks in 'upgrading' and hands off via .upgrade-requested" else fail "restore.sh does not hand off the upgrade to the api" fi echo "==================================================" echo "[test] datarev gate: ${passes} passed, ${fails} failed" echo "==================================================" [ "${fails}" -eq 0 ]