236 lines
11 KiB
Bash
Executable File
236 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ShopDB restore simulation harness. ***FOR DEVELOPER USE ONLY***
|
|
#
|
|
# This is a developer diagnostic, NOT an operator tool and NOT part of the
|
|
# shipped backup feature. It mutates a live database on purpose. Do not run it
|
|
# against production or any environment you are not prepared to disturb.
|
|
#
|
|
# WHY: the restore path reloads the WHOLE database in one pass (mysqldump
|
|
# --databases emits DROP TABLE + CREATE TABLE + full INSERTs, so restore is a
|
|
# full replace, not a row-level merge). The strongest real-world check is a
|
|
# full-fidelity round trip: fingerprint every table, take a backup, mutate
|
|
# across the table archetypes (core standalone, asset + plugin subtype, and
|
|
# consumable ledgers), restore, then prove every table's fingerprint returns to
|
|
# the backup-time state.
|
|
#
|
|
# It drives the exact sentinel path the UI uses (.backup-requested /
|
|
# .restore-requested on the shared volume), not a private code path, so a pass
|
|
# here is a pass for the operator-facing feature.
|
|
#
|
|
# KNOWN, EXPECTED, NOT-A-FAILURE behaviors (see LiveDatabaseTest notes below):
|
|
# - apitokens does not round-trip bit-for-bit: the app rewrites token
|
|
# last-used timestamps continuously, so restore reproduces backup-time
|
|
# state, which differs from any later fingerprint. This is correct.
|
|
# - endpoint-agent tables (computers heartbeat, endpointstates/events/
|
|
# applications) churn from the EXTERNAL fleet agents, which cannot be
|
|
# quiesced. Post-restore writes from them are expected drift, not a
|
|
# restore defect.
|
|
# - Asset ROWS cannot be deleted in a live/CMMC environment (RESTRICT and
|
|
# compliance controls). The cascade probe is therefore informational only;
|
|
# restore fidelity does not depend on it. A missing/corrupted asset is
|
|
# still fully restored because the dump drops and recreates the table.
|
|
#
|
|
# Usage (from a host with the stack up, git bash):
|
|
# bash scripts/backup/simulate-restore.sh
|
|
#
|
|
# ASCII only, no emojis - matches the repo style gate.
|
|
set -uo pipefail
|
|
|
|
# Silence the docker CLI "What's next" hints - they bleed into command
|
|
# substitution and corrupt captured SQL output.
|
|
export DOCKER_CLI_HINTS=false
|
|
|
|
COMPOSE="docker compose -f docker-compose.yml -f docker-compose.backup.yml --profile backup"
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
|
|
# Tables that legitimately differ after restore and MUST NOT be counted as
|
|
# failures:
|
|
# - App-churn: token last-used, sessions, audit trail.
|
|
# - Endpoint-agent churn: the external fleet agents keep writing to these
|
|
# after restore (heartbeats, state, events, inventory) and cannot be
|
|
# quiesced from here.
|
|
# Restore reproduces their state at dump time; later fingerprints differ by
|
|
# design, so they are reported informationally.
|
|
VOLATILE="apitokens sessions auditlogs computers endpointstates endpointevents endpointapplications"
|
|
WORKDIR="$(mktemp -d)"
|
|
BASELINE="${WORKDIR}/baseline.txt"
|
|
MUTATED="${WORKDIR}/mutated.txt"
|
|
RESTORED="${WORKDIR}/restored.txt"
|
|
MARKER="simrestore-$(date -u +%Y%m%dT%H%M%SZ)"
|
|
|
|
log() { echo "[sim] $*"; }
|
|
die() { echo "[sim] ERROR: $*" >&2; exit 1; }
|
|
|
|
# Run a SQL statement, returning tab-separated rows with no column headers.
|
|
sql() {
|
|
printf '%s\n' "$1" | ${COMPOSE} exec -T db sh -c \
|
|
'mysql -ushopdb -p"$MYSQL_PASSWORD" shopdb_flask -N 2>/dev/null'
|
|
}
|
|
|
|
# Fingerprint every base table with CHECKSUM TABLE into the given file.
|
|
fingerprint() {
|
|
out="$1"
|
|
tables="$(sql "SELECT table_name FROM information_schema.tables \
|
|
WHERE table_schema='shopdb_flask' AND table_type='BASE TABLE' \
|
|
ORDER BY table_name;" | tr -d '\r')"
|
|
: > "${out}"
|
|
for table in ${tables}; do
|
|
# CHECKSUM TABLE returns "dbname.table<TAB>checksum"; keep table+checksum.
|
|
line="$(sql "CHECKSUM TABLE \`${table}\`;" | tr -d '\r')"
|
|
checksum="$(printf '%s' "${line}" | awk '{print $2}')"
|
|
printf '%s\t%s\n' "${table}" "${checksum}" >> "${out}"
|
|
done
|
|
}
|
|
|
|
wait_for_phase() {
|
|
kind="$1"; want="$2"; timeout="${3:-120}"
|
|
waited=0
|
|
while [ "${waited}" -lt "${timeout}" ]; do
|
|
phase="$(${COMPOSE} exec -T backup-watcher sh -c \
|
|
"cat /backups/.${kind}-status.json 2>/dev/null" \
|
|
| sed -n 's/.*\"phase\":\"\([^\"]*\)\".*/\1/p')"
|
|
if [ "${phase}" = "${want}" ]; then return 0; fi
|
|
if [ "${phase}" = "failed" ]; then
|
|
die "${kind} reported failed status"
|
|
fi
|
|
sleep 3
|
|
waited=$((waited + 3))
|
|
done
|
|
die "${kind} did not reach '${want}' within ${timeout}s (last: ${phase:-none})"
|
|
}
|
|
|
|
# --- 0. Preconditions ------------------------------------------------------
|
|
${COMPOSE} exec -T db sh -c 'true' >/dev/null 2>&1 \
|
|
|| die "db container not reachable - is the stack up?"
|
|
|
|
# --- 0b. Datarev-gate preflight (external tester) --------------------------
|
|
# This live round trip runs one codebase against itself, so it can never
|
|
# reproduce the "data revision behind the running code" case the restore gate
|
|
# now handles. Run the standalone datarev tester first so a pass here also
|
|
# covers that offline contract (manifest datarev, known-vs-unknown revision
|
|
# refusal, legacy fallback, and the api upgrade handoff).
|
|
log "running datarev gate preflight (test-restore-datarev.sh)"
|
|
bash "${SCRIPT_DIR}/test-restore-datarev.sh" \
|
|
|| die "datarev gate preflight failed - see output above"
|
|
log "datarev gate preflight passed"
|
|
|
|
# --- 1. Backup the current state, then fingerprint the RESTORE TARGET ------
|
|
# The restore point is the database as it existed WHEN THE DUMP RAN, not some
|
|
# earlier moment. Live-churn tables (apitokens last-used, sessions, auditlogs)
|
|
# move under the running app, so the baseline must be captured immediately
|
|
# after the backup completes - that is the exact state restore must reproduce.
|
|
log "requesting a backup (sentinel .backup-requested)"
|
|
${COMPOSE} exec -T backup-watcher sh -c 'echo sim > /backups/.backup-requested'
|
|
wait_for_phase backup done 300
|
|
SETNAME="$(${COMPOSE} exec -T backup-watcher sh -c \
|
|
'cat /backups/.backup-status.json 2>/dev/null' \
|
|
| sed -n 's/.*\"setname\":\"\([^\"]*\)\".*/\1/p')"
|
|
[ -n "${SETNAME}" ] || die "could not read the completed backup set name"
|
|
log "backup complete: ${SETNAME}"
|
|
|
|
log "baseline fingerprint of all tables (restore target)"
|
|
fingerprint "${BASELINE}"
|
|
baseline_count="$(wc -l < "${BASELINE}")"
|
|
log "fingerprinted ${baseline_count} tables"
|
|
|
|
# Quiesce the app for the mutate/restore/verify window. A real DR runbook stops
|
|
# serving traffic before a restore; here it also stops the endpoint-agent write
|
|
# path (computers heartbeat, endpointstates/events/applications) that would
|
|
# otherwise churn the DB concurrently and defeat a bit-for-bit comparison.
|
|
log "stopping api to quiesce writers"
|
|
${COMPOSE} stop api >/dev/null 2>&1
|
|
resume_api() { log "resuming api"; ${COMPOSE} start api >/dev/null 2>&1; }
|
|
trap resume_api EXIT
|
|
|
|
# --- 2. Mutate across the table archetypes ---------------------------------
|
|
# Core standalone: rename a vendor and flip a setting.
|
|
# Ledger/consumable: bump a modelsupplies quantityonhand.
|
|
# NOTE: asset ROW deletion is intentionally NOT performed. In a live/CMMC
|
|
# environment assets cannot be removed (RESTRICT + compliance controls), which
|
|
# is the app behaving as designed. Restore fidelity does not depend on it: the
|
|
# dump drops and recreates every table, so a missing or corrupted asset is
|
|
# fully reconstructed regardless. The cascade edge is reported informationally.
|
|
log "mutating rows (marker ${MARKER})"
|
|
|
|
sql "UPDATE vendors SET vendorname=CONCAT('${MARKER}-', vendorname) \
|
|
ORDER BY vendorid LIMIT 1;"
|
|
sql "UPDATE settings SET settingvalue='${MARKER}' \
|
|
WHERE settingkey=(SELECT settingkey FROM (SELECT settingkey FROM settings \
|
|
ORDER BY settingkey LIMIT 1) AS s);"
|
|
|
|
# Cascade edge (informational only - no destructive delete): report the FK
|
|
# delete rule that would remove computer subtype rows if an asset were deleted.
|
|
CASCADE_RULE="$(sql "SELECT delete_rule FROM information_schema.referential_constraints \
|
|
WHERE constraint_schema='shopdb_flask' AND referenced_table_name='assets' \
|
|
AND table_name='computers' LIMIT 1;" | tr -d '\r')"
|
|
log "cascade edge computers->assets delete_rule=${CASCADE_RULE:-unknown} (live asset delete not permitted; not exercised)"
|
|
|
|
# Ledger: move a cached quantityonhand so the restore must revert it.
|
|
sql "UPDATE modelsupplies SET quantityonhand=quantityonhand+9999 \
|
|
ORDER BY id LIMIT 1;" 2>/dev/null \
|
|
|| sql "UPDATE modelsupplies SET quantityonhand=quantityonhand+9999 \
|
|
ORDER BY modelsupplyid LIMIT 1;"
|
|
|
|
log "post-mutation fingerprint"
|
|
fingerprint "${MUTATED}"
|
|
changed="$(diff <(sort "${BASELINE}") <(sort "${MUTATED}") | grep -c '^>')"
|
|
log "mutation changed ${changed} table fingerprint(s)"
|
|
|
|
# --- 3. Restore the backup we took -----------------------------------------
|
|
log "requesting restore of ${SETNAME}"
|
|
${COMPOSE} exec -T backup-watcher sh -c "printf '%s' '${SETNAME}' > /backups/.restore-requested"
|
|
wait_for_phase restore done 600
|
|
log "restore complete"
|
|
|
|
# --- 4. Verify every table returned to baseline ----------------------------
|
|
log "post-restore fingerprint"
|
|
fingerprint "${RESTORED}"
|
|
|
|
fails=0
|
|
volatile_diffs=0
|
|
while IFS="$(printf '\t')" read -r table base; do
|
|
restored="$(grep -P "^${table}\t" "${RESTORED}" | awk '{print $2}')"
|
|
if [ "${base}" != "${restored}" ]; then
|
|
case " ${VOLATILE} " in
|
|
*" ${table} "*)
|
|
echo "[sim] (volatile) ${table} differs as expected: baseline=${base} restored=${restored}"
|
|
volatile_diffs=$((volatile_diffs + 1))
|
|
;;
|
|
*)
|
|
echo "[sim] MISMATCH ${table}: baseline=${base} restored=${restored}"
|
|
fails=$((fails + 1))
|
|
;;
|
|
esac
|
|
fi
|
|
done < "${BASELINE}"
|
|
|
|
echo "=================================================="
|
|
RESULTS_FILE="${SCRIPT_DIR}/simulate-restore-results.txt"
|
|
{
|
|
echo "ShopDB restore simulation results (FOR DEVELOPER USE ONLY)"
|
|
echo "run (UTC) : $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
echo "marker : ${MARKER}"
|
|
echo "backup set : ${SETNAME}"
|
|
echo "tables checked : ${baseline_count}"
|
|
echo "cascade edge : computers->assets delete_rule=${CASCADE_RULE:-unknown} (not exercised - live asset delete not permitted)"
|
|
echo ""
|
|
echo "Expected, not-a-failure drift (excluded from strict equality):"
|
|
echo " - apitokens: app rewrites token last-used; restore is correct at backup time."
|
|
echo " - endpoint-agent tables (computers, endpointstates/events/applications):"
|
|
echo " external fleet agents keep writing post-restore and cannot be quiesced."
|
|
echo " - asset row deletion is not permitted live (RESTRICT + CMMC); the table is"
|
|
echo " still fully restored because the dump drops and recreates it."
|
|
echo ""
|
|
if [ "${fails}" -eq 0 ]; then
|
|
echo "RESULT: PASS - all ${baseline_count} tables restored to backup-time state"
|
|
echo " (${volatile_diffs} volatile table(s) excluded from strict equality)"
|
|
else
|
|
echo "RESULT: FAIL - ${fails} table(s) did not match after restore"
|
|
fi
|
|
} | tee "${RESULTS_FILE}"
|
|
echo "=================================================="
|
|
echo "[sim] results written to ${RESULTS_FILE}"
|
|
echo "[sim] work dir: ${WORKDIR}"
|
|
echo "=================================================="
|
|
[ "${fails}" -eq 0 ]
|