shopdb-flask/scripts/backup/test-restore-datarev.sh

#!/usr/bin/env bash
# ShopDB restore datarev-gate tester.  ***FOR DEVELOPER USE ONLY***
#
# WHY THIS EXISTS SEPARATELY FROM simulate-restore.sh:
# simulate-restore.sh is a full-fidelity ROUND TRIP against a live stack: it
# backs up, mutates, restores, and proves every table returns to backup-time
# state. By construction it runs ONE codebase against ITSELF, so the database it
# dumps is always at the SAME revision as the code that restores it. It can
# therefore never exercise the case the FBJXBDB4 outage was about: a dump whose
# DATA revision is BEHIND (or ahead of) the running CODE.
#
# This tester fills that gap offline. It builds throwaway migration trees and
# fake gzipped dumps and drives the exact decision logic restore.sh uses:
#   1. manifest.py records datarev (the DB's real alembic_version), not just the
#      code head (schemarev).
#   2. A dump whose datarev the code KNOWS (an ancestor in the chain) is allowed
#      - the post-load upgrade rolls it forward.
#   3. A dump whose datarev the code does NOT know (newer than this build) is
#      REFUSED - restoring it would 500 on 1054 Unknown column.
#   4. A legacy manifest with no datarev falls back to parsing alembic_version
#      straight out of the dump.
#   5. restore.sh hands the loaded restore off to the api via the
#      .upgrade-requested sentinel instead of marking it done in the sidecar.
#
# No Docker, no MySQL, no network: it is a pure logic gate test, safe to run in
# CI and from inside simulate-restore.sh as a preflight.
#
# ASCII only, no emojis - matches the repo style gate.
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
MANIFEST_PY="${SCRIPT_DIR}/manifest.py"
RESTORE_SH="${SCRIPT_DIR}/restore.sh"
WORKDIR="$(mktemp -d)"
trap 'rm -rf "${WORKDIR}"' EXIT

# The backup image ships python3; a dev host may only have `python` (and on
# Windows `python3` can be a non-functional Store alias that resolves on PATH
# but errors when run). Pick the first interpreter that actually EXECUTES, so
# this tester runs anywhere without changing the shipped scripts.
PY=""
for candidate in python3 python py; do
	if "${candidate}" -c 'pass' >/dev/null 2>&1; then
		PY="${candidate}"
		break
	fi
done
[ -n "${PY}" ] || { echo "[test] ERROR: no working python interpreter found"; exit 1; }

passes=0
fails=0
pass() { echo "[test] PASS: $*"; passes=$((passes + 1)); }
fail() { echo "[test] FAIL: $*"; fails=$((fails + 1)); }

# Build a throwaway core migration chain: baseline -> mid -> head. The gate in
# restore.sh only needs each version file to declare `revision = '<id>'`, so
# that is all we write.
build_chain() {
	root="$1"
	versions="${root}/versions"
	mkdir -p "${versions}"
	printf "revision = '%s'\ndown_revision = None\n" \
		'0001_baseline' > "${versions}/0001_baseline.py"
	printf "revision = '%s'\ndown_revision = '%s'\n" \
		'0002_mid' '0001_baseline' > "${versions}/0002_mid.py"
	printf "revision = '%s'\ndown_revision = '%s'\n" \
		'0003_head' '0002_mid' > "${versions}/0003_head.py"
}

# Fake a gzipped mysqldump that stamps a given alembic_version, matching the
# INSERT shape restore.sh greps for when a legacy manifest lacks datarev.
build_dump() {
	out="$1"; datarev="$2"
	{
		echo 'DROP TABLE IF EXISTS `alembic_version`;'
		echo 'CREATE TABLE `alembic_version` (`version_num` varchar(32));'
		echo 'INSERT INTO `alembic_version` VALUES'
		echo "('${datarev}');"
	} | gzip -c > "${out}"
}

# Mirror of the restore.sh predicate: does the code chain know this revision?
# Kept identical to restore.sh so this test tracks the shipped decision.
code_knows_revision() {
	versions_dir="$1"; datarev="$2"
	grep -rqE "^revision = ['\"]${datarev}['\"]" "${versions_dir}" 2>/dev/null
}

# Mirror of the restore.sh datarev extraction: manifest field first, else parse
# the dump.
set_datarev() {
	setdir="$1"; dbdump="$2"
	value="$(${PY} -c \
		'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \
		"${setdir}/manifest.json" 2>/dev/null || echo "")"
	if [ -z "${value}" ]; then
		value="$(gunzip -c "${dbdump}" \
			| grep -A2 -i 'INSERT INTO `alembic_version`' \
			| grep -oE "'[0-9a-zA-Z_]+'" | tr -d "'" | head -1)"
	fi
	printf '%s' "${value}"
}

CHAIN="${WORKDIR}/migrations"
build_chain "${CHAIN}"
VERSIONS="${CHAIN}/versions"

# --- 1. manifest.py records datarev distinct from the code head -------------
SET1="${WORKDIR}/set1"; mkdir -p "${SET1}"
${PY} "${MANIFEST_PY}" --out "${SET1}/manifest.json" \
	--migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \
	--datarev '0002_mid' --takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1
recorded="$(${PY} -c \
	'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \
	"${SET1}/manifest.json" 2>/dev/null)"
head_stamp="$(${PY} -c \
	'import json,sys;print(json.load(open(sys.argv[1])).get("schemarev",{}).get("core",[]))' \
	"${SET1}/manifest.json" 2>/dev/null)"
if [ "${recorded}" = '0002_mid' ]; then
	pass "manifest.py records datarev (${recorded}) separate from code head ${head_stamp}"
else
	fail "manifest.py did not record datarev (got '${recorded}')"
fi

# --- 2. datarev the code KNOWS (behind head) is allowed --------------------
DBDUMP1="${SET1}/db.sql.gz"; build_dump "${DBDUMP1}" '0002_mid'
rev="$(set_datarev "${SET1}" "${DBDUMP1}")"
if [ "${rev}" = '0002_mid' ] && code_knows_revision "${VERSIONS}" "${rev}"; then
	pass "data behind code (${rev}) is known and allowed to restore"
else
	fail "data behind code (${rev}) was not accepted"
fi

# --- 3. datarev the code does NOT know (newer than build) is refused -------
SET2="${WORKDIR}/set2"; mkdir -p "${SET2}"
${PY} "${MANIFEST_PY}" --out "${SET2}/manifest.json" \
	--migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \
	--datarev '0004_future' --takenat '2024-02-01T00:00:00Z' >/dev/null 2>&1
DBDUMP2="${SET2}/db.sql.gz"; build_dump "${DBDUMP2}" '0004_future'
rev="$(set_datarev "${SET2}" "${DBDUMP2}")"
if [ "${rev}" = '0004_future' ] && ! code_knows_revision "${VERSIONS}" "${rev}"; then
	pass "data ahead of code (${rev}) is unknown and correctly refused"
else
	fail "data ahead of code (${rev}) was not refused"
fi

# --- 4. legacy manifest (no datarev) falls back to the dump ----------------
SET3="${WORKDIR}/set3"; mkdir -p "${SET3}"
# A pre-datarev manifest: schemarev present, datarev absent/empty.
printf '{"takenat":"2023-06-01T00:00:00Z","schemarev":{"core":["0003_head"]}}' \
	> "${SET3}/manifest.json"
DBDUMP3="${SET3}/db.sql.gz"; build_dump "${DBDUMP3}" '0001_baseline'
rev="$(set_datarev "${SET3}" "${DBDUMP3}")"
if [ "${rev}" = '0001_baseline' ] && code_knows_revision "${VERSIONS}" "${rev}"; then
	pass "legacy set with no datarev falls back to dump revision (${rev})"
else
	fail "legacy set fallback failed (got '${rev}')"
fi

# --- 5. restore.sh hands off to the api instead of self-completing ---------
if grep -q '\.upgrade-requested' "${RESTORE_SH}" \
	&& grep -q "status_write restore upgrading" "${RESTORE_SH}" \
	&& ! grep -qE 'status_write restore done' "${RESTORE_SH}"; then
	pass "restore.sh parks in 'upgrading' and hands off via .upgrade-requested"
else
	fail "restore.sh does not hand off the upgrade to the api"
fi

echo "=================================================="
echo "[test] datarev gate: ${passes} passed, ${fails} failed"
echo "=================================================="
[ "${fails}" -eq 0 ]