Endpoint telemetry, PC Detail rework and a checked backup/restore path
This commit is contained in:
231
scripts/backup/test-restore-datarev.sh
Executable file
231
scripts/backup/test-restore-datarev.sh
Executable file
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env bash
|
||||
# ShopDB restore datarev-gate tester. ***FOR DEVELOPER USE ONLY***
|
||||
#
|
||||
# WHY THIS EXISTS SEPARATELY FROM simulate-restore.sh:
|
||||
# simulate-restore.sh is a full-fidelity ROUND TRIP against a live stack: it
|
||||
# backs up, mutates, restores, and proves every table returns to backup-time
|
||||
# state. By construction it runs ONE codebase against ITSELF, so the database it
|
||||
# dumps is always at the SAME revision as the code that restores it. It can
|
||||
# therefore never exercise the case the FBJXBDB4 outage was about: a dump whose
|
||||
# DATA revision is BEHIND (or ahead of) the running CODE.
|
||||
#
|
||||
# This tester fills that gap offline. It builds throwaway migration trees and
|
||||
# fake gzipped dumps and drives the exact decision logic restore.sh uses:
|
||||
# 1. manifest.py records datarev (the DB's real alembic_version), not just the
|
||||
# code head (schemarev).
|
||||
# 2. A dump whose datarev the code KNOWS (an ancestor in the chain) is allowed
|
||||
# - the post-load upgrade rolls it forward.
|
||||
# 3. A dump whose datarev the code does NOT know (newer than this build) is
|
||||
# REFUSED - restoring it would 500 on 1054 Unknown column.
|
||||
# 4. A legacy manifest with no datarev falls back to parsing alembic_version
|
||||
# straight out of the dump.
|
||||
# 5. restore.sh hands the loaded restore off to the api via the
|
||||
# .upgrade-requested sentinel instead of marking it done in the sidecar.
|
||||
#
|
||||
# No Docker, no MySQL, no network: it is a pure logic gate test, safe to run in
|
||||
# CI and from inside simulate-restore.sh as a preflight.
|
||||
#
|
||||
# ASCII only, no emojis - matches the repo style gate.
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
MANIFEST_PY="${SCRIPT_DIR}/manifest.py"
|
||||
RESTORE_SH="${SCRIPT_DIR}/restore.sh"
|
||||
WORKDIR="$(mktemp -d)"
|
||||
trap 'rm -rf "${WORKDIR}"' EXIT
|
||||
|
||||
# The backup image ships python3; a dev host may only have `python` (and on
|
||||
# Windows `python3` can be a non-functional Store alias that resolves on PATH
|
||||
# but errors when run). Pick the first interpreter that actually EXECUTES, so
|
||||
# this tester runs anywhere without changing the shipped scripts.
|
||||
PY=""
|
||||
for candidate in python3 python py; do
|
||||
if "${candidate}" -c 'pass' >/dev/null 2>&1; then
|
||||
PY="${candidate}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
[ -n "${PY}" ] || { echo "[test] ERROR: no working python interpreter found"; exit 1; }
|
||||
|
||||
passes=0
|
||||
fails=0
|
||||
pass() { echo "[test] PASS: $*"; passes=$((passes + 1)); }
|
||||
fail() { echo "[test] FAIL: $*"; fails=$((fails + 1)); }
|
||||
|
||||
# Build a throwaway core migration chain: baseline -> mid -> head. The gate in
|
||||
# restore.sh only needs each version file to declare `revision = '<id>'`, so
|
||||
# that is all we write.
|
||||
build_chain() {
|
||||
root="$1"
|
||||
versions="${root}/versions"
|
||||
mkdir -p "${versions}"
|
||||
printf "revision = '%s'\ndown_revision = None\n" \
|
||||
'0001_baseline' > "${versions}/0001_baseline.py"
|
||||
printf "revision = '%s'\ndown_revision = '%s'\n" \
|
||||
'0002_mid' '0001_baseline' > "${versions}/0002_mid.py"
|
||||
printf "revision = '%s'\ndown_revision = '%s'\n" \
|
||||
'0003_head' '0002_mid' > "${versions}/0003_head.py"
|
||||
}
|
||||
|
||||
# Fake a gzipped mysqldump that stamps a given alembic_version, matching the
|
||||
# INSERT shape restore.sh greps for when a legacy manifest lacks datarev.
|
||||
build_dump() {
|
||||
out="$1"; datarev="$2"
|
||||
{
|
||||
echo 'DROP TABLE IF EXISTS `alembic_version`;'
|
||||
echo 'CREATE TABLE `alembic_version` (`version_num` varchar(32));'
|
||||
echo 'INSERT INTO `alembic_version` VALUES'
|
||||
echo "('${datarev}');"
|
||||
} | gzip -c > "${out}"
|
||||
}
|
||||
|
||||
# Build a throwaway plugin chain, shaped like plugins/<name>/migrations/versions.
|
||||
build_plugin_chain() {
|
||||
root="$1"; name="$2"; rev="$3"
|
||||
versions="${root}/${name}/migrations/versions"
|
||||
mkdir -p "${versions}"
|
||||
printf "revision = '%s'\ndown_revision = None\n" \
|
||||
"${rev}" > "${versions}/${rev}.py"
|
||||
}
|
||||
|
||||
# Drive the REAL restore.sh gate. This is the point of the rewrite: the test
|
||||
# used to carry its own copy of the predicate, so it stayed green no matter how
|
||||
# far restore.sh drifted from it. RESTORE_CHECK_ONLY stops the script right
|
||||
# after the gate, so the verdict here IS the shipped verdict.
|
||||
run_gate() {
|
||||
backups_root="$1"; setname="$2"; chain="$3"; plugins="$4"
|
||||
RESTORE_CHECK_ONLY=1 \
|
||||
SHOPDB_BACKUPS_ROOT="${backups_root}" \
|
||||
BACKUP_LIB_DIR="${SCRIPT_DIR}" \
|
||||
APP_MIGRATIONS_DIR="${chain}" \
|
||||
APP_PLUGINS_DIR="${plugins}" \
|
||||
bash "${RESTORE_SH}" "${setname}" >"${WORKDIR}/gate.log" 2>&1
|
||||
}
|
||||
|
||||
CHAIN="${WORKDIR}/migrations"
|
||||
build_chain "${CHAIN}"
|
||||
PLUGINS="${WORKDIR}/plugins"
|
||||
build_plugin_chain "${PLUGINS}" 'endpoints' 'endpoints0002known'
|
||||
ROOT="${WORKDIR}/backups"
|
||||
mkdir -p "${ROOT}"
|
||||
|
||||
# Lay out one set the way a real backup does, under the backups root.
|
||||
make_set() {
|
||||
setname="$1"; datarev="$2"; dumprev="$3"; dataplugins="$4"
|
||||
setdir="${ROOT}/${setname}"
|
||||
mkdir -p "${setdir}"
|
||||
if [ "${datarev}" = "__legacy__" ]; then
|
||||
printf '{"takenat":"2023-06-01T00:00:00Z","schemarev":{"core":["0003_head"]}}' \
|
||||
> "${setdir}/manifest.json"
|
||||
else
|
||||
${PY} "${MANIFEST_PY}" --out "${setdir}/manifest.json" \
|
||||
--migrations "${CHAIN}" --plugins "${PLUGINS}" \
|
||||
--datarev "${datarev}" --dataplugins "${dataplugins}" \
|
||||
--takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1
|
||||
fi
|
||||
build_dump "${setdir}/db.sql.gz" "${dumprev}"
|
||||
printf '%s' "${setdir}"
|
||||
}
|
||||
|
||||
# --- 1. manifest.py records datarev separately from the code head ----------
|
||||
make_set 'set-behind' '0002_mid' '0002_mid' '{}' >/dev/null
|
||||
recorded="$(${PY} -c \
|
||||
'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \
|
||||
"${ROOT}/set-behind/manifest.json" 2>/dev/null)"
|
||||
head_stamp="$(${PY} -c \
|
||||
'import json,sys;print(json.load(open(sys.argv[1])).get("schemarev",{}).get("core",[]))' \
|
||||
"${ROOT}/set-behind/manifest.json" 2>/dev/null)"
|
||||
if [ "${recorded}" = '0002_mid' ]; then
|
||||
pass "manifest.py records datarev (${recorded}) separate from code head ${head_stamp}"
|
||||
else
|
||||
fail "manifest.py did not record datarev (got '${recorded}')"
|
||||
fi
|
||||
|
||||
# --- 2. datarev the code KNOWS (behind head) is allowed --------------------
|
||||
if run_gate "${ROOT}" 'set-behind' "${CHAIN}" "${PLUGINS}"; then
|
||||
pass "data behind code (0002_mid) is known and allowed to restore"
|
||||
else
|
||||
fail "data behind code was refused: $(tail -1 "${WORKDIR}/gate.log")"
|
||||
fi
|
||||
|
||||
# --- 3. datarev the code does NOT know (newer than build) is refused -------
|
||||
make_set 'set-ahead' '0004_future' '0004_future' '{}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-ahead' "${CHAIN}" "${PLUGINS}"; then
|
||||
fail "data ahead of code (0004_future) was NOT refused"
|
||||
else
|
||||
pass "data ahead of code (0004_future) is unknown and correctly refused"
|
||||
fi
|
||||
|
||||
# --- 4. legacy manifest (no datarev) falls back to the dump ----------------
|
||||
make_set 'set-legacy' '__legacy__' '0001_baseline' '{}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-legacy' "${CHAIN}" "${PLUGINS}"; then
|
||||
pass "legacy set with no datarev falls back to the dump revision"
|
||||
else
|
||||
fail "legacy set fallback failed: $(tail -1 "${WORKDIR}/gate.log")"
|
||||
fi
|
||||
|
||||
# --- 5. restore.sh hands off to the api instead of self-completing ---------
|
||||
if grep -q '\.upgrade-requested' "${RESTORE_SH}" \
|
||||
&& grep -q "status_write restore upgrading" "${RESTORE_SH}" \
|
||||
&& ! grep -qE 'status_write restore done' "${RESTORE_SH}"; then
|
||||
pass "restore.sh parks in 'upgrading' and hands off via .upgrade-requested"
|
||||
else
|
||||
fail "restore.sh does not hand off the upgrade to the api"
|
||||
fi
|
||||
|
||||
# --- 6. a PLUGIN chain the code does not know is refused -------------------
|
||||
# The core head is identical in both of these; only the plugin revision differs.
|
||||
# That is the whole failure: a core-only gate cannot tell them apart.
|
||||
make_set 'set-plugin-known' '0002_mid' '0002_mid' '{"endpoints":"endpoints0002known"}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-plugin-known' "${CHAIN}" "${PLUGINS}"; then
|
||||
pass "a set whose plugin revision the code knows is allowed"
|
||||
else
|
||||
fail "known plugin revision was refused: $(tail -1 "${WORKDIR}/gate.log")"
|
||||
fi
|
||||
|
||||
make_set 'set-plugin-future' '0002_mid' '0002_mid' '{"endpoints":"endpoints0099future"}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-plugin-future' "${CHAIN}" "${PLUGINS}"; then
|
||||
fail "plugin data ahead of code was NOT refused (core-only gate regression)"
|
||||
else
|
||||
pass "plugin data ahead of code is refused even though the core head matches"
|
||||
fi
|
||||
|
||||
# --- 7. a plugin the build does not have at all is refused -----------------
|
||||
make_set 'set-plugin-missing' '0002_mid' '0002_mid' '{"nosuchplugin":"whatever0001"}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-plugin-missing' "${CHAIN}" "${PLUGINS}"; then
|
||||
fail "a set carrying data for an absent plugin was NOT refused"
|
||||
else
|
||||
pass "a set carrying data for a plugin this build lacks is refused"
|
||||
fi
|
||||
|
||||
# --- 8. a dump bound to another schema is refused --------------------------
|
||||
# --databases bakes 'USE <name>' in, which silently loaded the rows beside the
|
||||
# live database and reported success.
|
||||
setdir="${ROOT}/set-otherschema"
|
||||
mkdir -p "${setdir}"
|
||||
${PY} "${MANIFEST_PY}" --out "${setdir}/manifest.json" \
|
||||
--migrations "${CHAIN}" --plugins "${PLUGINS}" \
|
||||
--datarev '0002_mid' --dataplugins '{}' \
|
||||
--takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1
|
||||
{
|
||||
echo 'CREATE DATABASE `someothersite`;'
|
||||
echo 'USE `someothersite`;'
|
||||
echo 'INSERT INTO `alembic_version` VALUES'
|
||||
echo "('0002_mid');"
|
||||
} | gzip -c > "${setdir}/db.sql.gz"
|
||||
# Now a gate, so check-only reaches it without needing a database.
|
||||
out="$(RESTORE_CHECK_ONLY=1 SHOPDB_BACKUPS_ROOT="${ROOT}" \
|
||||
BACKUP_LIB_DIR="${SCRIPT_DIR}" \
|
||||
APP_MIGRATIONS_DIR="${CHAIN}" APP_PLUGINS_DIR="${PLUGINS}" \
|
||||
MYSQL_DATABASE='shopdb_flask' \
|
||||
bash "${RESTORE_SH}" 'set-otherschema' 2>&1)" || true
|
||||
if printf '%s' "${out}" | grep -q "bound to schema 'someothersite'"; then
|
||||
pass "a dump bound to another site's schema is refused"
|
||||
else
|
||||
fail "a schema-bound dump was not caught (got: $(printf '%s' "${out}" | tail -1))"
|
||||
fi
|
||||
|
||||
echo "=================================================="
|
||||
echo "[test] datarev gate: ${passes} passed, ${fails} failed"
|
||||
echo "=================================================="
|
||||
[ "${fails}" -eq 0 ]
|
||||
Reference in New Issue
Block a user