#!/usr/bin/env bash # ShopDB restore simulation harness. ***FOR DEVELOPER USE ONLY*** # # This is a developer diagnostic, NOT an operator tool and NOT part of the # shipped backup feature. It mutates a live database on purpose. Do not run it # against production or any environment you are not prepared to disturb. # # WHY: the restore path reloads the WHOLE database in one pass (mysqldump # --databases emits DROP TABLE + CREATE TABLE + full INSERTs, so restore is a # full replace, not a row-level merge). The strongest real-world check is a # full-fidelity round trip: fingerprint every table, take a backup, mutate # across the table archetypes (core standalone, asset + plugin subtype, and # consumable ledgers), restore, then prove every table's fingerprint returns to # the backup-time state. # # It drives the exact sentinel path the UI uses (.backup-requested / # .restore-requested on the shared volume), not a private code path, so a pass # here is a pass for the operator-facing feature. # # KNOWN, EXPECTED, NOT-A-FAILURE behaviors (see LiveDatabaseTest notes below): # - apitokens does not round-trip bit-for-bit: the app rewrites token # last-used timestamps continuously, so restore reproduces backup-time # state, which differs from any later fingerprint. This is correct. # - endpoint-agent tables (computers heartbeat, endpointstates/events/ # applications) churn from the EXTERNAL fleet agents, which cannot be # quiesced. Post-restore writes from them are expected drift, not a # restore defect. # - Asset ROWS cannot be deleted in a live/CMMC environment (RESTRICT and # compliance controls). The cascade probe is therefore informational only; # restore fidelity does not depend on it. A missing/corrupted asset is # still fully restored because the dump drops and recreates the table. # # Usage (from a host with the stack up, git bash): # bash scripts/backup/simulate-restore.sh # # ASCII only, no emojis - matches the repo style gate. set -uo pipefail # Silence the docker CLI "What's next" hints - they bleed into command # substitution and corrupt captured SQL output. export DOCKER_CLI_HINTS=false COMPOSE="docker compose -f docker-compose.yml -f docker-compose.backup.yml --profile backup" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" # Tables that legitimately differ after restore and MUST NOT be counted as # failures: # - App-churn: token last-used, sessions, audit trail. # - Endpoint-agent churn: the external fleet agents keep writing to these # after restore (heartbeats, state, events, inventory) and cannot be # quiesced from here. # Restore reproduces their state at dump time; later fingerprints differ by # design, so they are reported informationally. VOLATILE="apitokens sessions auditlogs computers endpointstates endpointevents endpointapplications" WORKDIR="$(mktemp -d)" BASELINE="${WORKDIR}/baseline.txt" MUTATED="${WORKDIR}/mutated.txt" RESTORED="${WORKDIR}/restored.txt" MARKER="simrestore-$(date -u +%Y%m%dT%H%M%SZ)" log() { echo "[sim] $*"; } die() { echo "[sim] ERROR: $*" >&2; exit 1; } # Run a SQL statement, returning tab-separated rows with no column headers. sql() { printf '%s\n' "$1" | ${COMPOSE} exec -T db sh -c \ 'mysql -ushopdb -p"$MYSQL_PASSWORD" shopdb_flask -N 2>/dev/null' } # Fingerprint every base table with CHECKSUM TABLE into the given file. fingerprint() { out="$1" tables="$(sql "SELECT table_name FROM information_schema.tables \ WHERE table_schema='shopdb_flask' AND table_type='BASE TABLE' \ ORDER BY table_name;" | tr -d '\r')" : > "${out}" for table in ${tables}; do # CHECKSUM TABLE returns "dbname.tablechecksum"; keep table+checksum. line="$(sql "CHECKSUM TABLE \`${table}\`;" | tr -d '\r')" checksum="$(printf '%s' "${line}" | awk '{print $2}')" printf '%s\t%s\n' "${table}" "${checksum}" >> "${out}" done } wait_for_phase() { kind="$1"; want="$2"; timeout="${3:-120}" waited=0 while [ "${waited}" -lt "${timeout}" ]; do phase="$(${COMPOSE} exec -T backup-watcher sh -c \ "cat /backups/.${kind}-status.json 2>/dev/null" \ | sed -n 's/.*\"phase\":\"\([^\"]*\)\".*/\1/p')" if [ "${phase}" = "${want}" ]; then return 0; fi if [ "${phase}" = "failed" ]; then die "${kind} reported failed status" fi sleep 3 waited=$((waited + 3)) done die "${kind} did not reach '${want}' within ${timeout}s (last: ${phase:-none})" } # --- 0. Preconditions ------------------------------------------------------ ${COMPOSE} exec -T db sh -c 'true' >/dev/null 2>&1 \ || die "db container not reachable - is the stack up?" # --- 0b. Datarev-gate preflight (external tester) -------------------------- # This live round trip runs one codebase against itself, so it can never # reproduce the "data revision behind the running code" case the restore gate # now handles. Run the standalone datarev tester first so a pass here also # covers that offline contract (manifest datarev, known-vs-unknown revision # refusal, legacy fallback, and the api upgrade handoff). log "running datarev gate preflight (test-restore-datarev.sh)" bash "${SCRIPT_DIR}/test-restore-datarev.sh" \ || die "datarev gate preflight failed - see output above" log "datarev gate preflight passed" # --- 1. Backup the current state, then fingerprint the RESTORE TARGET ------ # The restore point is the database as it existed WHEN THE DUMP RAN, not some # earlier moment. Live-churn tables (apitokens last-used, sessions, auditlogs) # move under the running app, so the baseline must be captured immediately # after the backup completes - that is the exact state restore must reproduce. log "requesting a backup (sentinel .backup-requested)" ${COMPOSE} exec -T backup-watcher sh -c 'echo sim > /backups/.backup-requested' wait_for_phase backup done 300 SETNAME="$(${COMPOSE} exec -T backup-watcher sh -c \ 'cat /backups/.backup-status.json 2>/dev/null' \ | sed -n 's/.*\"setname\":\"\([^\"]*\)\".*/\1/p')" [ -n "${SETNAME}" ] || die "could not read the completed backup set name" log "backup complete: ${SETNAME}" log "baseline fingerprint of all tables (restore target)" fingerprint "${BASELINE}" baseline_count="$(wc -l < "${BASELINE}")" log "fingerprinted ${baseline_count} tables" # Quiesce the app for the mutate/restore/verify window. A real DR runbook stops # serving traffic before a restore; here it also stops the endpoint-agent write # path (computers heartbeat, endpointstates/events/applications) that would # otherwise churn the DB concurrently and defeat a bit-for-bit comparison. log "stopping api to quiesce writers" ${COMPOSE} stop api >/dev/null 2>&1 resume_api() { log "resuming api"; ${COMPOSE} start api >/dev/null 2>&1; } trap resume_api EXIT # --- 2. Mutate across the table archetypes --------------------------------- # Core standalone: rename a vendor and flip a setting. # Ledger/consumable: bump a modelsupplies quantityonhand. # NOTE: asset ROW deletion is intentionally NOT performed. In a live/CMMC # environment assets cannot be removed (RESTRICT + compliance controls), which # is the app behaving as designed. Restore fidelity does not depend on it: the # dump drops and recreates every table, so a missing or corrupted asset is # fully reconstructed regardless. The cascade edge is reported informationally. log "mutating rows (marker ${MARKER})" sql "UPDATE vendors SET vendorname=CONCAT('${MARKER}-', vendorname) \ ORDER BY vendorid LIMIT 1;" sql "UPDATE settings SET settingvalue='${MARKER}' \ WHERE settingkey=(SELECT settingkey FROM (SELECT settingkey FROM settings \ ORDER BY settingkey LIMIT 1) AS s);" # Cascade edge (informational only - no destructive delete): report the FK # delete rule that would remove computer subtype rows if an asset were deleted. CASCADE_RULE="$(sql "SELECT delete_rule FROM information_schema.referential_constraints \ WHERE constraint_schema='shopdb_flask' AND referenced_table_name='assets' \ AND table_name='computers' LIMIT 1;" | tr -d '\r')" log "cascade edge computers->assets delete_rule=${CASCADE_RULE:-unknown} (live asset delete not permitted; not exercised)" # Ledger: move a cached quantityonhand so the restore must revert it. sql "UPDATE modelsupplies SET quantityonhand=quantityonhand+9999 \ ORDER BY id LIMIT 1;" 2>/dev/null \ || sql "UPDATE modelsupplies SET quantityonhand=quantityonhand+9999 \ ORDER BY modelsupplyid LIMIT 1;" log "post-mutation fingerprint" fingerprint "${MUTATED}" changed="$(diff <(sort "${BASELINE}") <(sort "${MUTATED}") | grep -c '^>')" log "mutation changed ${changed} table fingerprint(s)" # --- 3. Restore the backup we took ----------------------------------------- log "requesting restore of ${SETNAME}" ${COMPOSE} exec -T backup-watcher sh -c "printf '%s' '${SETNAME}' > /backups/.restore-requested" wait_for_phase restore done 600 log "restore complete" # --- 4. Verify every table returned to baseline ---------------------------- log "post-restore fingerprint" fingerprint "${RESTORED}" fails=0 volatile_diffs=0 while IFS="$(printf '\t')" read -r table base; do restored="$(grep -P "^${table}\t" "${RESTORED}" | awk '{print $2}')" if [ "${base}" != "${restored}" ]; then case " ${VOLATILE} " in *" ${table} "*) echo "[sim] (volatile) ${table} differs as expected: baseline=${base} restored=${restored}" volatile_diffs=$((volatile_diffs + 1)) ;; *) echo "[sim] MISMATCH ${table}: baseline=${base} restored=${restored}" fails=$((fails + 1)) ;; esac fi done < "${BASELINE}" echo "==================================================" RESULTS_FILE="${SCRIPT_DIR}/simulate-restore-results.txt" { echo "ShopDB restore simulation results (FOR DEVELOPER USE ONLY)" echo "run (UTC) : $(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "marker : ${MARKER}" echo "backup set : ${SETNAME}" echo "tables checked : ${baseline_count}" echo "cascade edge : computers->assets delete_rule=${CASCADE_RULE:-unknown} (not exercised - live asset delete not permitted)" echo "" echo "Expected, not-a-failure drift (excluded from strict equality):" echo " - apitokens: app rewrites token last-used; restore is correct at backup time." echo " - endpoint-agent tables (computers, endpointstates/events/applications):" echo " external fleet agents keep writing post-restore and cannot be quiesced." echo " - asset row deletion is not permitted live (RESTRICT + CMMC); the table is" echo " still fully restored because the dump drops and recreates it." echo "" if [ "${fails}" -eq 0 ]; then echo "RESULT: PASS - all ${baseline_count} tables restored to backup-time state" echo " (${volatile_diffs} volatile table(s) excluded from strict equality)" else echo "RESULT: FAIL - ${fails} table(s) did not match after restore" fi } | tee "${RESULTS_FILE}" echo "==================================================" echo "[sim] results written to ${RESULTS_FILE}" echo "[sim] work dir: ${WORKDIR}" echo "==================================================" [ "${fails}" -eq 0 ]