232 lines
9.6 KiB
Bash
Executable File
232 lines
9.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ShopDB restore datarev-gate tester. ***FOR DEVELOPER USE ONLY***
|
|
#
|
|
# WHY THIS EXISTS SEPARATELY FROM simulate-restore.sh:
|
|
# simulate-restore.sh is a full-fidelity ROUND TRIP against a live stack: it
|
|
# backs up, mutates, restores, and proves every table returns to backup-time
|
|
# state. By construction it runs ONE codebase against ITSELF, so the database it
|
|
# dumps is always at the SAME revision as the code that restores it. It can
|
|
# therefore never exercise the case the FBJXBDB4 outage was about: a dump whose
|
|
# DATA revision is BEHIND (or ahead of) the running CODE.
|
|
#
|
|
# This tester fills that gap offline. It builds throwaway migration trees and
|
|
# fake gzipped dumps and drives the exact decision logic restore.sh uses:
|
|
# 1. manifest.py records datarev (the DB's real alembic_version), not just the
|
|
# code head (schemarev).
|
|
# 2. A dump whose datarev the code KNOWS (an ancestor in the chain) is allowed
|
|
# - the post-load upgrade rolls it forward.
|
|
# 3. A dump whose datarev the code does NOT know (newer than this build) is
|
|
# REFUSED - restoring it would 500 on 1054 Unknown column.
|
|
# 4. A legacy manifest with no datarev falls back to parsing alembic_version
|
|
# straight out of the dump.
|
|
# 5. restore.sh hands the loaded restore off to the api via the
|
|
# .upgrade-requested sentinel instead of marking it done in the sidecar.
|
|
#
|
|
# No Docker, no MySQL, no network: it is a pure logic gate test, safe to run in
|
|
# CI and from inside simulate-restore.sh as a preflight.
|
|
#
|
|
# ASCII only, no emojis - matches the repo style gate.
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
MANIFEST_PY="${SCRIPT_DIR}/manifest.py"
|
|
RESTORE_SH="${SCRIPT_DIR}/restore.sh"
|
|
WORKDIR="$(mktemp -d)"
|
|
trap 'rm -rf "${WORKDIR}"' EXIT
|
|
|
|
# The backup image ships python3; a dev host may only have `python` (and on
|
|
# Windows `python3` can be a non-functional Store alias that resolves on PATH
|
|
# but errors when run). Pick the first interpreter that actually EXECUTES, so
|
|
# this tester runs anywhere without changing the shipped scripts.
|
|
PY=""
|
|
for candidate in python3 python py; do
|
|
if "${candidate}" -c 'pass' >/dev/null 2>&1; then
|
|
PY="${candidate}"
|
|
break
|
|
fi
|
|
done
|
|
[ -n "${PY}" ] || { echo "[test] ERROR: no working python interpreter found"; exit 1; }
|
|
|
|
passes=0
|
|
fails=0
|
|
pass() { echo "[test] PASS: $*"; passes=$((passes + 1)); }
|
|
fail() { echo "[test] FAIL: $*"; fails=$((fails + 1)); }
|
|
|
|
# Build a throwaway core migration chain: baseline -> mid -> head. The gate in
|
|
# restore.sh only needs each version file to declare `revision = '<id>'`, so
|
|
# that is all we write.
|
|
build_chain() {
|
|
root="$1"
|
|
versions="${root}/versions"
|
|
mkdir -p "${versions}"
|
|
printf "revision = '%s'\ndown_revision = None\n" \
|
|
'0001_baseline' > "${versions}/0001_baseline.py"
|
|
printf "revision = '%s'\ndown_revision = '%s'\n" \
|
|
'0002_mid' '0001_baseline' > "${versions}/0002_mid.py"
|
|
printf "revision = '%s'\ndown_revision = '%s'\n" \
|
|
'0003_head' '0002_mid' > "${versions}/0003_head.py"
|
|
}
|
|
|
|
# Fake a gzipped mysqldump that stamps a given alembic_version, matching the
|
|
# INSERT shape restore.sh greps for when a legacy manifest lacks datarev.
|
|
build_dump() {
|
|
out="$1"; datarev="$2"
|
|
{
|
|
echo 'DROP TABLE IF EXISTS `alembic_version`;'
|
|
echo 'CREATE TABLE `alembic_version` (`version_num` varchar(32));'
|
|
echo 'INSERT INTO `alembic_version` VALUES'
|
|
echo "('${datarev}');"
|
|
} | gzip -c > "${out}"
|
|
}
|
|
|
|
# Build a throwaway plugin chain, shaped like plugins/<name>/migrations/versions.
|
|
build_plugin_chain() {
|
|
root="$1"; name="$2"; rev="$3"
|
|
versions="${root}/${name}/migrations/versions"
|
|
mkdir -p "${versions}"
|
|
printf "revision = '%s'\ndown_revision = None\n" \
|
|
"${rev}" > "${versions}/${rev}.py"
|
|
}
|
|
|
|
# Drive the REAL restore.sh gate. This is the point of the rewrite: the test
|
|
# used to carry its own copy of the predicate, so it stayed green no matter how
|
|
# far restore.sh drifted from it. RESTORE_CHECK_ONLY stops the script right
|
|
# after the gate, so the verdict here IS the shipped verdict.
|
|
run_gate() {
|
|
backups_root="$1"; setname="$2"; chain="$3"; plugins="$4"
|
|
RESTORE_CHECK_ONLY=1 \
|
|
SHOPDB_BACKUPS_ROOT="${backups_root}" \
|
|
BACKUP_LIB_DIR="${SCRIPT_DIR}" \
|
|
APP_MIGRATIONS_DIR="${chain}" \
|
|
APP_PLUGINS_DIR="${plugins}" \
|
|
bash "${RESTORE_SH}" "${setname}" >"${WORKDIR}/gate.log" 2>&1
|
|
}
|
|
|
|
CHAIN="${WORKDIR}/migrations"
|
|
build_chain "${CHAIN}"
|
|
PLUGINS="${WORKDIR}/plugins"
|
|
build_plugin_chain "${PLUGINS}" 'endpoints' 'endpoints0002known'
|
|
ROOT="${WORKDIR}/backups"
|
|
mkdir -p "${ROOT}"
|
|
|
|
# Lay out one set the way a real backup does, under the backups root.
|
|
make_set() {
|
|
setname="$1"; datarev="$2"; dumprev="$3"; dataplugins="$4"
|
|
setdir="${ROOT}/${setname}"
|
|
mkdir -p "${setdir}"
|
|
if [ "${datarev}" = "__legacy__" ]; then
|
|
printf '{"takenat":"2023-06-01T00:00:00Z","schemarev":{"core":["0003_head"]}}' \
|
|
> "${setdir}/manifest.json"
|
|
else
|
|
${PY} "${MANIFEST_PY}" --out "${setdir}/manifest.json" \
|
|
--migrations "${CHAIN}" --plugins "${PLUGINS}" \
|
|
--datarev "${datarev}" --dataplugins "${dataplugins}" \
|
|
--takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1
|
|
fi
|
|
build_dump "${setdir}/db.sql.gz" "${dumprev}"
|
|
printf '%s' "${setdir}"
|
|
}
|
|
|
|
# --- 1. manifest.py records datarev separately from the code head ----------
|
|
make_set 'set-behind' '0002_mid' '0002_mid' '{}' >/dev/null
|
|
recorded="$(${PY} -c \
|
|
'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \
|
|
"${ROOT}/set-behind/manifest.json" 2>/dev/null)"
|
|
head_stamp="$(${PY} -c \
|
|
'import json,sys;print(json.load(open(sys.argv[1])).get("schemarev",{}).get("core",[]))' \
|
|
"${ROOT}/set-behind/manifest.json" 2>/dev/null)"
|
|
if [ "${recorded}" = '0002_mid' ]; then
|
|
pass "manifest.py records datarev (${recorded}) separate from code head ${head_stamp}"
|
|
else
|
|
fail "manifest.py did not record datarev (got '${recorded}')"
|
|
fi
|
|
|
|
# --- 2. datarev the code KNOWS (behind head) is allowed --------------------
|
|
if run_gate "${ROOT}" 'set-behind' "${CHAIN}" "${PLUGINS}"; then
|
|
pass "data behind code (0002_mid) is known and allowed to restore"
|
|
else
|
|
fail "data behind code was refused: $(tail -1 "${WORKDIR}/gate.log")"
|
|
fi
|
|
|
|
# --- 3. datarev the code does NOT know (newer than build) is refused -------
|
|
make_set 'set-ahead' '0004_future' '0004_future' '{}' >/dev/null
|
|
if run_gate "${ROOT}" 'set-ahead' "${CHAIN}" "${PLUGINS}"; then
|
|
fail "data ahead of code (0004_future) was NOT refused"
|
|
else
|
|
pass "data ahead of code (0004_future) is unknown and correctly refused"
|
|
fi
|
|
|
|
# --- 4. legacy manifest (no datarev) falls back to the dump ----------------
|
|
make_set 'set-legacy' '__legacy__' '0001_baseline' '{}' >/dev/null
|
|
if run_gate "${ROOT}" 'set-legacy' "${CHAIN}" "${PLUGINS}"; then
|
|
pass "legacy set with no datarev falls back to the dump revision"
|
|
else
|
|
fail "legacy set fallback failed: $(tail -1 "${WORKDIR}/gate.log")"
|
|
fi
|
|
|
|
# --- 5. restore.sh hands off to the api instead of self-completing ---------
|
|
if grep -q '\.upgrade-requested' "${RESTORE_SH}" \
|
|
&& grep -q "status_write restore upgrading" "${RESTORE_SH}" \
|
|
&& ! grep -qE 'status_write restore done' "${RESTORE_SH}"; then
|
|
pass "restore.sh parks in 'upgrading' and hands off via .upgrade-requested"
|
|
else
|
|
fail "restore.sh does not hand off the upgrade to the api"
|
|
fi
|
|
|
|
# --- 6. a PLUGIN chain the code does not know is refused -------------------
|
|
# The core head is identical in both of these; only the plugin revision differs.
|
|
# That is the whole failure: a core-only gate cannot tell them apart.
|
|
make_set 'set-plugin-known' '0002_mid' '0002_mid' '{"endpoints":"endpoints0002known"}' >/dev/null
|
|
if run_gate "${ROOT}" 'set-plugin-known' "${CHAIN}" "${PLUGINS}"; then
|
|
pass "a set whose plugin revision the code knows is allowed"
|
|
else
|
|
fail "known plugin revision was refused: $(tail -1 "${WORKDIR}/gate.log")"
|
|
fi
|
|
|
|
make_set 'set-plugin-future' '0002_mid' '0002_mid' '{"endpoints":"endpoints0099future"}' >/dev/null
|
|
if run_gate "${ROOT}" 'set-plugin-future' "${CHAIN}" "${PLUGINS}"; then
|
|
fail "plugin data ahead of code was NOT refused (core-only gate regression)"
|
|
else
|
|
pass "plugin data ahead of code is refused even though the core head matches"
|
|
fi
|
|
|
|
# --- 7. a plugin the build does not have at all is refused -----------------
|
|
make_set 'set-plugin-missing' '0002_mid' '0002_mid' '{"nosuchplugin":"whatever0001"}' >/dev/null
|
|
if run_gate "${ROOT}" 'set-plugin-missing' "${CHAIN}" "${PLUGINS}"; then
|
|
fail "a set carrying data for an absent plugin was NOT refused"
|
|
else
|
|
pass "a set carrying data for a plugin this build lacks is refused"
|
|
fi
|
|
|
|
# --- 8. a dump bound to another schema is refused --------------------------
|
|
# --databases bakes 'USE <name>' in, which silently loaded the rows beside the
|
|
# live database and reported success.
|
|
setdir="${ROOT}/set-otherschema"
|
|
mkdir -p "${setdir}"
|
|
${PY} "${MANIFEST_PY}" --out "${setdir}/manifest.json" \
|
|
--migrations "${CHAIN}" --plugins "${PLUGINS}" \
|
|
--datarev '0002_mid' --dataplugins '{}' \
|
|
--takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1
|
|
{
|
|
echo 'CREATE DATABASE `someothersite`;'
|
|
echo 'USE `someothersite`;'
|
|
echo 'INSERT INTO `alembic_version` VALUES'
|
|
echo "('0002_mid');"
|
|
} | gzip -c > "${setdir}/db.sql.gz"
|
|
# Now a gate, so check-only reaches it without needing a database.
|
|
out="$(RESTORE_CHECK_ONLY=1 SHOPDB_BACKUPS_ROOT="${ROOT}" \
|
|
BACKUP_LIB_DIR="${SCRIPT_DIR}" \
|
|
APP_MIGRATIONS_DIR="${CHAIN}" APP_PLUGINS_DIR="${PLUGINS}" \
|
|
MYSQL_DATABASE='shopdb_flask' \
|
|
bash "${RESTORE_SH}" 'set-otherschema' 2>&1)" || true
|
|
if printf '%s' "${out}" | grep -q "bound to schema 'someothersite'"; then
|
|
pass "a dump bound to another site's schema is refused"
|
|
else
|
|
fail "a schema-bound dump was not caught (got: $(printf '%s' "${out}" | tail -1))"
|
|
fi
|
|
|
|
echo "=================================================="
|
|
echo "[test] datarev gate: ${passes} passed, ${fails} failed"
|
|
echo "=================================================="
|
|
[ "${fails}" -eq 0 ]
|