Merge wilm: gevent migration, endpoint telemetry phase 2, backup/restore
Brings in 98 commits from wilm (origin/wilm @ 811e6ca61a):
Major features:
- Move API to gevent workers for better concurrency
- Endpoint agent startup jitter (prevents thundering herd)
- Complete backup/restore with restic
- Endpoint telemetry phase 2 (disk health, mapped drives, user context)
- PC Detail UI rework with new cards and tabs
- Dell Command actions and Corporate Tools tab
- Agent version column in Fleet view
Conflict resolution:
- Backup/restore: Accepted wilm implementation (10 files)
- Agent startup jitter: Accepted wilm v1.12.0
- PC Types list fix: Accepted wilm version
- Endpoint plugin domain clamping: Kept ours (defensive)
- Test files: Accepted wilm versions for consistency
Testing required:
- Verify gevent handles concurrent requests
- Test backup/restore functionality
- Verify agent startup jitter timing
- Test endpoint telemetry collection
- Validate PC Detail UI changes
Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Proudlock
@@ -14,23 +14,8 @@ FROM mysql:8.4
|
||||
# bzip2/python3/alembic or a base image that carries them) so the release image
|
||||
# needs no outbound calls and no embedded cert. The cert is gitignored (*.pem)
|
||||
# and must be staged at scripts/backup/zscaler-root-ca.pem before building.
|
||||
#
|
||||
# Copied via the directory rather than by name, because the cert is OPTIONAL: it
|
||||
# is gitignored, so a plain `COPY scripts/backup/zscaler-root-ca.pem` fails the
|
||||
# build outright and nobody outside a TLS-intercepting network - another site,
|
||||
# or our own CI - could build this image at all. A trailing glob does not fix
|
||||
# that either: BuildKit fails a COPY whose pattern matches nothing, which is
|
||||
# precisely the case being handled. Copying a directory that always exists and
|
||||
# testing for the file inside it is the form that works both ways.
|
||||
COPY scripts/backup/ /tmp/buildctx/
|
||||
RUN if [ -f /tmp/buildctx/zscaler-root-ca.pem ]; then \
|
||||
echo "staging corporate root CA"; \
|
||||
cp /tmp/buildctx/zscaler-root-ca.pem /etc/pki/ca-trust/source/anchors/; \
|
||||
update-ca-trust; \
|
||||
else \
|
||||
echo "no corporate root CA staged - using the public trust store"; \
|
||||
fi \
|
||||
&& rm -rf /tmp/buildctx
|
||||
COPY scripts/backup/zscaler-root-ca.pem /etc/pki/ca-trust/source/anchors/zscaler-root-ca.pem
|
||||
RUN update-ca-trust
|
||||
|
||||
# restic from the distro is often stale; pull the pinned release binary so the
|
||||
# snapshot format and retention flags are predictable across sites.
|
||||
|
||||
0
scripts/backup/_probe_manifest.py
Normal file
0
scripts/backup/_probe_manifest.py
Normal file
@@ -47,12 +47,6 @@ mkdir -p "${SETDIR}"
|
||||
status_write backup database "${SETNAME}" "Dumping database"
|
||||
|
||||
# 1. Database: whole-DB logical dump, consistent and non-blocking on InnoDB.
|
||||
# Dumped WITHOUT --databases on purpose. That flag bakes CREATE DATABASE and USE
|
||||
# <name> into the file, which pins the dump to the schema name it was taken
|
||||
# from: restoring onto a site whose MYSQL_DATABASE differs loaded the rows into
|
||||
# the dump's schema, left the live one untouched, and reported success.
|
||||
# --no-tablespaces because dumping tablespace metadata needs the PROCESS
|
||||
# privilege, which the app user does not have on MySQL 8.
|
||||
log "dumping database ${MYSQL_DATABASE} from ${MYSQL_HOST}"
|
||||
mysqldump \
|
||||
--host="${MYSQL_HOST}" \
|
||||
@@ -61,8 +55,7 @@ mysqldump \
|
||||
--single-transaction \
|
||||
--routines \
|
||||
--triggers \
|
||||
--no-tablespaces \
|
||||
"${MYSQL_DATABASE}" \
|
||||
--databases "${MYSQL_DATABASE}" \
|
||||
| gzip -c > "${SETDIR}/db.sql.gz"
|
||||
log "database dump written ($(du -h "${SETDIR}/db.sql.gz" | cut -f1))"
|
||||
|
||||
@@ -80,48 +73,7 @@ DATAREV="$(mysql \
|
||||
--skip-column-names --batch \
|
||||
-e "SELECT version_num FROM alembic_version LIMIT 1;" \
|
||||
"${MYSQL_DATABASE}" 2>/dev/null | tr -d '\r')"
|
||||
if [ -z "${DATAREV}" ]; then
|
||||
# Recorded loudly rather than shrugged off: a set with no data revision
|
||||
# cannot be gated on restore, and the gate treats an empty stamp as "no
|
||||
# evidence" and refuses it. Better a visible warning now than a set that
|
||||
# looks restorable and is not.
|
||||
log "WARN could not read alembic_version - this set will NOT be restorable"
|
||||
history_append backup warning "${SETNAME}" \
|
||||
"Could not read alembic_version; set has no data revision and cannot be restored"
|
||||
fi
|
||||
|
||||
# The same question for every PLUGIN chain. Each plugin owns an
|
||||
# alembic_version_<plugin> table, and those move independently of the core one:
|
||||
# code rolled back to an older plugin revision can sit at an identical core
|
||||
# head while the dump carries plugin tables it has never heard of. Stamping only
|
||||
# the core revision let that combination pass the restore gate, load, and then
|
||||
# fail during the plugin upgrade with the data already replaced. Read straight
|
||||
# from the dumped DB, same as the core revision above.
|
||||
log "reading applied plugin revisions (alembic_version_*)"
|
||||
DATAPLUGINS="$(mysql \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--skip-column-names --batch \
|
||||
-e "SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = '${MYSQL_DATABASE}'
|
||||
AND table_name LIKE 'alembic\\_version\\_%';" 2>/dev/null \
|
||||
| tr -d '\r' \
|
||||
| while read -r table; do
|
||||
[ -n "${table}" ] || continue
|
||||
plugin="${table#alembic_version_}"
|
||||
rev="$(mysql \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--skip-column-names --batch \
|
||||
-e "SELECT version_num FROM \`${table}\` LIMIT 1;" \
|
||||
"${MYSQL_DATABASE}" 2>/dev/null | tr -d '\r')"
|
||||
[ -n "${rev}" ] && printf '%s\t%s\n' "${plugin}" "${rev}"
|
||||
done \
|
||||
| python3 -c 'import json,sys; print(json.dumps(dict(l.rstrip("\n").split("\t",1) for l in sys.stdin if "\t" in l)))')"
|
||||
[ -n "${DATAPLUGINS}" ] || DATAPLUGINS='{}'
|
||||
log "plugin revisions: ${DATAPLUGINS}"
|
||||
[ -n "${DATAREV}" ] || log "WARN could not read alembic_version - datarev will be empty"
|
||||
|
||||
log "writing manifest"
|
||||
python3 /opt/backup/manifest.py \
|
||||
@@ -130,7 +82,6 @@ python3 /opt/backup/manifest.py \
|
||||
--site "${SITE}" \
|
||||
--imagetag "${SHOPDB_IMAGE_TAG:-source}" \
|
||||
--datarev "${DATAREV}" \
|
||||
--dataplugins "${DATAPLUGINS}" \
|
||||
--migrations /app/migrations \
|
||||
--plugins /app/plugins
|
||||
|
||||
|
||||
0
scripts/backup/backup.sh.tmp
Normal file
0
scripts/backup/backup.sh.tmp
Normal file
@@ -10,19 +10,12 @@ restore." It records, at capture time:
|
||||
each plugin chain's heads - the versions a restore target must
|
||||
be at least, or restored rows reference columns the live schema
|
||||
has not applied yet.
|
||||
- datarev : the CORE Alembic revision actually applied to the DATABASE that
|
||||
was dumped (its alembic_version), which is the number a restore
|
||||
must gate on. schemarev stamps the code on disk; datarev stamps
|
||||
the data in the dump. They diverge when an image is ahead of a DB
|
||||
- datarev : the Alembic revision actually applied to the DATABASE that was
|
||||
dumped (its alembic_version), which is the number a restore must
|
||||
gate on. schemarev stamps the code on disk; datarev stamps the
|
||||
data in the dump. They diverge when an image is ahead of a DB
|
||||
that has not run 'flask db upgrade' yet, and that divergence is
|
||||
exactly what a code-to-code check cannot see.
|
||||
- dataplugins: the same thing for every PLUGIN chain - the applied revision of
|
||||
each alembic_version_<plugin> table. The core chain alone cannot
|
||||
answer compatibility: plugin schemas move independently, so code
|
||||
rolled back to an older plugin revision can sit at an identical
|
||||
core head while the dump carries plugin tables it has never heard
|
||||
of. Gating on core only let that pass, load, and then fail during
|
||||
the plugin upgrade with the data already replaced.
|
||||
|
||||
Two modes:
|
||||
--out ... write a fresh manifest for a new set
|
||||
@@ -74,25 +67,6 @@ def _heads_for(chain_dir):
|
||||
return sorted(rev for rev in revisions if rev not in down_refs)
|
||||
|
||||
|
||||
def _parse_dataplugins(raw):
|
||||
"""The applied per-plugin revisions, as a {plugin: revision} map.
|
||||
|
||||
Bad or absent JSON becomes an empty map rather than an error: a manifest
|
||||
that cannot say what the plugin chains were at is exactly the legacy case
|
||||
the restore gate already knows how to treat conservatively, and failing the
|
||||
backup here would lose the set over a stamp.
|
||||
"""
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except ValueError:
|
||||
return {}
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
return {str(k): str(v) for k, v in value.items() if k and v}
|
||||
|
||||
|
||||
def _collect_schema(migrations_dir, plugins_dir):
|
||||
"""Core heads plus each plugin's chain heads, keyed for compatibility checks."""
|
||||
schema = {'core': _heads_for(migrations_dir), 'plugins': {}}
|
||||
@@ -112,7 +86,6 @@ def write_manifest(args):
|
||||
'imagetag': args.imagetag,
|
||||
'schemarev': _collect_schema(args.migrations, args.plugins),
|
||||
'datarev': args.datarev or '',
|
||||
'dataplugins': _parse_dataplugins(args.dataplugins),
|
||||
'ischeckpassed': False,
|
||||
}
|
||||
with open(args.out, 'w', encoding='utf-8') as handle:
|
||||
@@ -130,8 +103,6 @@ def amend_manifest(args):
|
||||
manifest['ischeckpassed'] = args.ischeckpassed.lower() == 'true'
|
||||
if args.datarev is not None:
|
||||
manifest['datarev'] = args.datarev
|
||||
if args.dataplugins is not None:
|
||||
manifest['dataplugins'] = _parse_dataplugins(args.dataplugins)
|
||||
with open(args.amend, 'w', encoding='utf-8') as handle:
|
||||
json.dump(manifest, handle, indent=2, sort_keys=True)
|
||||
return 0
|
||||
@@ -147,8 +118,6 @@ def main(argv=None):
|
||||
parser.add_argument('--migrations', default='/app/migrations')
|
||||
parser.add_argument('--plugins', default='/app/plugins')
|
||||
parser.add_argument('--datarev')
|
||||
parser.add_argument('--dataplugins',
|
||||
help='JSON object of {plugin: applied revision}')
|
||||
parser.add_argument('--ischeckpassed')
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
|
||||
@@ -21,22 +21,8 @@ set -euo pipefail
|
||||
BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}"
|
||||
SETNAME="${1:-}"
|
||||
|
||||
# Overridable so the gate can be exercised outside a container against throwaway
|
||||
# migration trees. In the image these are exactly the mounted paths, so the
|
||||
# shipped behaviour is unchanged; test-restore-datarev.sh drives THIS script
|
||||
# rather than a copy of its logic, which is the only way the test can notice
|
||||
# this script drifting.
|
||||
BACKUP_LIB_DIR="${BACKUP_LIB_DIR:-/opt/backup}"
|
||||
APP_MIGRATIONS_DIR="${APP_MIGRATIONS_DIR:-/app/migrations}"
|
||||
APP_PLUGINS_DIR="${APP_PLUGINS_DIR:-/app/plugins}"
|
||||
APP_INSTANCE_DIR="${APP_INSTANCE_DIR:-/app/instance}"
|
||||
|
||||
# When set, stop after the compatibility gate and report the verdict as the exit
|
||||
# code, touching nothing. This is what makes the gate testable.
|
||||
RESTORE_CHECK_ONLY="${RESTORE_CHECK_ONLY:-}"
|
||||
|
||||
# shellcheck source=scripts/backup/status.sh
|
||||
. "${BACKUP_LIB_DIR}/status.sh"
|
||||
. /opt/backup/status.sh
|
||||
|
||||
log() { echo "[restore] $*"; }
|
||||
|
||||
@@ -75,7 +61,7 @@ DBDUMP="${SETDIR}/db.sql.gz"
|
||||
if [ ! -f "${DBDUMP}" ]; then
|
||||
log "local dump not found; restoring set from restic"
|
||||
command -v restic >/dev/null 2>&1 || die "restic not available and no local dump"
|
||||
restic restore latest --tag "${SETNAME}" --target / --include "${SETDIR}" \
|
||||
restic restore latest --target / --include "${SETDIR}" \
|
||||
|| die "restic restore of ${SETNAME} failed"
|
||||
fi
|
||||
[ -f "${DBDUMP}" ] || die "no db.sql.gz for set ${SETNAME}"
|
||||
@@ -105,173 +91,30 @@ if [ -z "${SET_DATAREV}" ]; then
|
||||
die "cannot determine the set's data revision - refusing an unverified restore"
|
||||
fi
|
||||
|
||||
|
||||
# Does the running code know this revision at all? If a migration file with that
|
||||
# revision id exists in the core chain, the code is at-or-past it (the head is
|
||||
# the tip of the same chain), so a post-load upgrade can carry the data forward.
|
||||
# If it is absent, the dump is NEWER than the code - refuse.
|
||||
if ! grep -rqE "^revision = ['\"]${SET_DATAREV}['\"]" "${APP_MIGRATIONS_DIR}/versions" 2>/dev/null; then
|
||||
if ! grep -rqE "^revision = ['\"]${SET_DATAREV}['\"]" /app/migrations/versions 2>/dev/null; then
|
||||
log "set datarev : ${SET_DATAREV}"
|
||||
die "the backup's data is at a revision this code does not have - deploy code at or past '${SET_DATAREV}' before restoring"
|
||||
fi
|
||||
log "data revision ${SET_DATAREV} is known to this code - restore may proceed"
|
||||
|
||||
# The same gate for every PLUGIN chain, because the core chain alone cannot
|
||||
# answer this. Plugin schemas move independently of core: roll the code back to
|
||||
# an older plugin revision and the core head can be identical while the dump
|
||||
# carries plugin tables the code has never heard of. Gating on core only let
|
||||
# that pass, load the dump, and then fail in 'flask plugin upgrade-all' with
|
||||
# "Can't locate revision" and the data already replaced - a failed restore that
|
||||
# has destroyed what it was restoring over.
|
||||
#
|
||||
# A manifest with no dataplugins is a legacy set taken before this was stamped.
|
||||
# Its plugin revisions are unknowable, so it is allowed through on the core gate
|
||||
# alone (the behaviour before this check existed) rather than being made
|
||||
# unrestorable retroactively.
|
||||
SET_DATAPLUGINS="$(python3 -c \
|
||||
'import json,sys
|
||||
try:
|
||||
data = json.load(open(sys.argv[1])).get("dataplugins") or {}
|
||||
except Exception:
|
||||
data = {}
|
||||
for name, rev in sorted(data.items()):
|
||||
print(name + " " + rev)' \
|
||||
"${SETDIR}/manifest.json" 2>/dev/null || echo "")"
|
||||
|
||||
if [ -z "${SET_DATAPLUGINS}" ]; then
|
||||
log "manifest stamps no plugin revisions (legacy set) - core gate only"
|
||||
else
|
||||
while read -r plugin rev; do
|
||||
[ -n "${plugin}" ] || continue
|
||||
chain="${APP_PLUGINS_DIR}/${plugin}/migrations/versions"
|
||||
if [ ! -d "${chain}" ]; then
|
||||
die "the backup carries data for plugin '${plugin}', which this build does not have - deploy a build that includes it before restoring"
|
||||
fi
|
||||
if ! grep -rqE "^revision = ['\"]${rev}['\"]" "${chain}" 2>/dev/null; then
|
||||
log "plugin ${plugin} data revision: ${rev}"
|
||||
die "plugin '${plugin}' data is at a revision this code does not have - deploy code at or past '${rev}' before restoring"
|
||||
fi
|
||||
log "plugin ${plugin} at ${rev} - known to this code"
|
||||
done <<PLUGINREVS
|
||||
${SET_DATAPLUGINS}
|
||||
PLUGINREVS
|
||||
fi
|
||||
|
||||
# Last gate: which schema does this dump target? A dump taken with --databases
|
||||
# carries CREATE DATABASE and USE <name>, pinning it to the schema it came from.
|
||||
# Loaded into a site whose MYSQL_DATABASE differs, the rows land in the dump's
|
||||
# schema, the live database is untouched, and the restore reports success. New
|
||||
# sets are dumped without it and load into whatever schema is named here; older
|
||||
# sets are still bound, so refuse rather than restore beside the live data.
|
||||
# Checked HERE, with the other gates, so it costs nothing and stops before the
|
||||
# safety copy rather than after it.
|
||||
DUMP_SCHEMA="$(gunzip -c "${DBDUMP}" \
|
||||
| grep -m1 -oE '^USE `[^`]+`' \
|
||||
| sed -E 's/^USE `(.*)`/\1/' || true)"
|
||||
if [ -n "${DUMP_SCHEMA}" ]; then
|
||||
if [ "${DUMP_SCHEMA}" != "${MYSQL_DATABASE}" ]; then
|
||||
die "this set is bound to schema '${DUMP_SCHEMA}' but this site's database is '${MYSQL_DATABASE}' - restoring it would load the data beside the live database and report success"
|
||||
fi
|
||||
log "legacy schema-bound dump for '${DUMP_SCHEMA}' - matches this site"
|
||||
fi
|
||||
|
||||
if [ -n "${RESTORE_CHECK_ONLY}" ]; then
|
||||
log "check-only: every gate passed, stopping before anything destructive"
|
||||
RESTORE_LOADED=false
|
||||
trap - EXIT
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Safety set FIRST. Everything below this point is destructive and none of it
|
||||
# is transactional: the dump carries DROP TABLE per table, and the instance tree
|
||||
# is replaced wholesale. Without a copy taken here, a restore of the wrong set -
|
||||
# or a failure part-way through the load - leaves nothing to go back to. This is
|
||||
# a local copy on the same volume, so it is cheap and it is NOT a substitute for
|
||||
# a real backup; it exists to make THIS operation reversible.
|
||||
SAFETY_STAMP="$(date -u +%Y-%m-%dT%H%M%SZ)"
|
||||
SAFETY_DIR="${BACKUPS_ROOT}/pre-restore-${SAFETY_STAMP}"
|
||||
status_write restore safety "${SETNAME}" "Copying the current state aside"
|
||||
log "taking a pre-restore safety copy into ${SAFETY_DIR}"
|
||||
mkdir -p "${SAFETY_DIR}" || die "could not create ${SAFETY_DIR}"
|
||||
|
||||
mysqldump \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--single-transaction \
|
||||
--routines \
|
||||
--triggers \
|
||||
--no-tablespaces \
|
||||
"${MYSQL_DATABASE}" \
|
||||
| gzip -c > "${SAFETY_DIR}/db.sql.gz" \
|
||||
|| die "pre-restore dump failed - refusing to touch the live database"
|
||||
|
||||
if [ -d /app/instance ]; then
|
||||
tar -czf "${SAFETY_DIR}/instance.tar.gz" -C /app instance \
|
||||
|| die "pre-restore instance copy failed - refusing to touch the live files"
|
||||
fi
|
||||
|
||||
# Named so an operator finding this directory months later knows what it is and
|
||||
# how to use it, without reading this script.
|
||||
cat > "${SAFETY_DIR}/README.txt" <<SAFETYNOTE
|
||||
Automatic pre-restore copy of the state this site was in immediately before
|
||||
restoring set '${SETNAME}' at ${SAFETY_STAMP}.
|
||||
|
||||
db.sql.gz mysqldump of ${MYSQL_DATABASE} as it was
|
||||
instance.tar.gz /app/instance as it was (absent if there was no such tree)
|
||||
|
||||
To go back to this state, load the dump the same way a restore does:
|
||||
|
||||
gunzip -c db.sql.gz | mysql --host=... --user=... --password=... \
|
||||
--database=${MYSQL_DATABASE}
|
||||
tar -xzf instance.tar.gz -C /app
|
||||
|
||||
This is a local copy on the backups volume, taken for reversibility of one
|
||||
restore. It is not a backup set: it has no manifest and no restic snapshot, and
|
||||
nothing prunes it.
|
||||
SAFETYNOTE
|
||||
log "safety copy written ($(du -sh "${SAFETY_DIR}" | cut -f1))"
|
||||
|
||||
# Files first, then DB, so the two land together.
|
||||
if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then
|
||||
status_write restore files "${SETNAME}" "Restoring instance files"
|
||||
# Scoped to THIS set's snapshot. A bare 'latest' restores whatever was
|
||||
# snapshotted most recently, so restoring anything other than the newest set
|
||||
# paired that set's DATABASE with the NEWEST snapshot's uploads and
|
||||
# plugins.json - a mismatch nothing downstream could detect.
|
||||
log "restoring instance files from the ${SETNAME} snapshot"
|
||||
STAGE="$(mktemp -d)"
|
||||
restic restore latest --tag "${SETNAME}" --target "${STAGE}" \
|
||||
--include /app/instance \
|
||||
|| die "restic restore of instance files for ${SETNAME} failed"
|
||||
|
||||
# Replace the CONTENTS of the instance tree rather than restoring over it,
|
||||
# so files created after the snapshot are gone afterwards - a restore has to
|
||||
# put the site back to the snapshot, not merge into it. Done as an explicit
|
||||
# staged swap rather than 'restic restore --delete --target /', because that
|
||||
# aims a delete pass at the container root and the blast radius of getting
|
||||
# its filter wrong is the whole filesystem. /app/instance is a mount, so the
|
||||
# directory itself has to survive; only what is inside it is replaced.
|
||||
if [ -d "${STAGE}/app/instance" ]; then
|
||||
find /app/instance -mindepth 1 -delete \
|
||||
|| die "could not clear the instance tree before restoring it"
|
||||
cp -a "${STAGE}/app/instance/." /app/instance/ \
|
||||
|| die "could not lay down the restored instance tree"
|
||||
else
|
||||
log "WARN snapshot for ${SETNAME} carried no /app/instance - leaving files alone"
|
||||
fi
|
||||
rm -rf "${STAGE}"
|
||||
log "restoring instance files from restic"
|
||||
restic restore latest --target / --include /app/instance \
|
||||
|| die "restic restore of instance files failed"
|
||||
fi
|
||||
|
||||
status_write restore database "${SETNAME}" "Loading database dump"
|
||||
|
||||
|
||||
log "loading database dump into ${MYSQL_DATABASE}"
|
||||
log "loading database dump"
|
||||
gunzip -c "${DBDUMP}" | mysql \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--database="${MYSQL_DATABASE}" \
|
||||
|| die "database load failed"
|
||||
|
||||
# Hand off to the api container for the schema upgrade. The dump may sit at an
|
||||
|
||||
16
scripts/backup/simulate-restore-results.txt
Normal file
16
scripts/backup/simulate-restore-results.txt
Normal file
@@ -0,0 +1,16 @@
|
||||
ShopDB restore simulation results (FOR DEVELOPER USE ONLY)
|
||||
run (UTC) : 2026-09-04T09:00:19Z
|
||||
marker : simrestore-20260904T085538Z
|
||||
backup set : wilm-2026-09-04T084716Z
|
||||
tables checked : 110
|
||||
cascade edge : computers->assets delete_rule=CASCADE (not exercised - live asset delete not permitted)
|
||||
|
||||
Expected, not-a-failure drift (excluded from strict equality):
|
||||
- apitokens: app rewrites token last-used; restore is correct at backup time.
|
||||
- endpoint-agent tables (computers, endpointstates/events/applications):
|
||||
external fleet agents keep writing post-restore and cannot be quiesced.
|
||||
- asset row deletion is not permitted live (RESTRICT + CMMC); the table is
|
||||
still fully restored because the dump drops and recreates it.
|
||||
|
||||
RESULT: PASS - all 110 tables restored to backup-time state
|
||||
(6 volatile table(s) excluded from strict equality)
|
||||
@@ -79,62 +79,43 @@ build_dump() {
|
||||
} | gzip -c > "${out}"
|
||||
}
|
||||
|
||||
# Build a throwaway plugin chain, shaped like plugins/<name>/migrations/versions.
|
||||
build_plugin_chain() {
|
||||
root="$1"; name="$2"; rev="$3"
|
||||
versions="${root}/${name}/migrations/versions"
|
||||
mkdir -p "${versions}"
|
||||
printf "revision = '%s'\ndown_revision = None\n" \
|
||||
"${rev}" > "${versions}/${rev}.py"
|
||||
# Mirror of the restore.sh predicate: does the code chain know this revision?
|
||||
# Kept identical to restore.sh so this test tracks the shipped decision.
|
||||
code_knows_revision() {
|
||||
versions_dir="$1"; datarev="$2"
|
||||
grep -rqE "^revision = ['\"]${datarev}['\"]" "${versions_dir}" 2>/dev/null
|
||||
}
|
||||
|
||||
# Drive the REAL restore.sh gate. This is the point of the rewrite: the test
|
||||
# used to carry its own copy of the predicate, so it stayed green no matter how
|
||||
# far restore.sh drifted from it. RESTORE_CHECK_ONLY stops the script right
|
||||
# after the gate, so the verdict here IS the shipped verdict.
|
||||
run_gate() {
|
||||
backups_root="$1"; setname="$2"; chain="$3"; plugins="$4"
|
||||
RESTORE_CHECK_ONLY=1 \
|
||||
SHOPDB_BACKUPS_ROOT="${backups_root}" \
|
||||
BACKUP_LIB_DIR="${SCRIPT_DIR}" \
|
||||
APP_MIGRATIONS_DIR="${chain}" \
|
||||
APP_PLUGINS_DIR="${plugins}" \
|
||||
bash "${RESTORE_SH}" "${setname}" >"${WORKDIR}/gate.log" 2>&1
|
||||
# Mirror of the restore.sh datarev extraction: manifest field first, else parse
|
||||
# the dump.
|
||||
set_datarev() {
|
||||
setdir="$1"; dbdump="$2"
|
||||
value="$(${PY} -c \
|
||||
'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \
|
||||
"${setdir}/manifest.json" 2>/dev/null || echo "")"
|
||||
if [ -z "${value}" ]; then
|
||||
value="$(gunzip -c "${dbdump}" \
|
||||
| grep -A2 -i 'INSERT INTO `alembic_version`' \
|
||||
| grep -oE "'[0-9a-zA-Z_]+'" | tr -d "'" | head -1)"
|
||||
fi
|
||||
printf '%s' "${value}"
|
||||
}
|
||||
|
||||
CHAIN="${WORKDIR}/migrations"
|
||||
build_chain "${CHAIN}"
|
||||
PLUGINS="${WORKDIR}/plugins"
|
||||
build_plugin_chain "${PLUGINS}" 'endpoints' 'endpoints0002known'
|
||||
ROOT="${WORKDIR}/backups"
|
||||
mkdir -p "${ROOT}"
|
||||
VERSIONS="${CHAIN}/versions"
|
||||
|
||||
# Lay out one set the way a real backup does, under the backups root.
|
||||
make_set() {
|
||||
setname="$1"; datarev="$2"; dumprev="$3"; dataplugins="$4"
|
||||
setdir="${ROOT}/${setname}"
|
||||
mkdir -p "${setdir}"
|
||||
if [ "${datarev}" = "__legacy__" ]; then
|
||||
printf '{"takenat":"2023-06-01T00:00:00Z","schemarev":{"core":["0003_head"]}}' \
|
||||
> "${setdir}/manifest.json"
|
||||
else
|
||||
${PY} "${MANIFEST_PY}" --out "${setdir}/manifest.json" \
|
||||
--migrations "${CHAIN}" --plugins "${PLUGINS}" \
|
||||
--datarev "${datarev}" --dataplugins "${dataplugins}" \
|
||||
--takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1
|
||||
fi
|
||||
build_dump "${setdir}/db.sql.gz" "${dumprev}"
|
||||
printf '%s' "${setdir}"
|
||||
}
|
||||
|
||||
# --- 1. manifest.py records datarev separately from the code head ----------
|
||||
make_set 'set-behind' '0002_mid' '0002_mid' '{}' >/dev/null
|
||||
# --- 1. manifest.py records datarev distinct from the code head -------------
|
||||
SET1="${WORKDIR}/set1"; mkdir -p "${SET1}"
|
||||
${PY} "${MANIFEST_PY}" --out "${SET1}/manifest.json" \
|
||||
--migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \
|
||||
--datarev '0002_mid' --takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1
|
||||
recorded="$(${PY} -c \
|
||||
'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \
|
||||
"${ROOT}/set-behind/manifest.json" 2>/dev/null)"
|
||||
"${SET1}/manifest.json" 2>/dev/null)"
|
||||
head_stamp="$(${PY} -c \
|
||||
'import json,sys;print(json.load(open(sys.argv[1])).get("schemarev",{}).get("core",[]))' \
|
||||
"${ROOT}/set-behind/manifest.json" 2>/dev/null)"
|
||||
"${SET1}/manifest.json" 2>/dev/null)"
|
||||
if [ "${recorded}" = '0002_mid' ]; then
|
||||
pass "manifest.py records datarev (${recorded}) separate from code head ${head_stamp}"
|
||||
else
|
||||
@@ -142,26 +123,38 @@ else
|
||||
fi
|
||||
|
||||
# --- 2. datarev the code KNOWS (behind head) is allowed --------------------
|
||||
if run_gate "${ROOT}" 'set-behind' "${CHAIN}" "${PLUGINS}"; then
|
||||
pass "data behind code (0002_mid) is known and allowed to restore"
|
||||
DBDUMP1="${SET1}/db.sql.gz"; build_dump "${DBDUMP1}" '0002_mid'
|
||||
rev="$(set_datarev "${SET1}" "${DBDUMP1}")"
|
||||
if [ "${rev}" = '0002_mid' ] && code_knows_revision "${VERSIONS}" "${rev}"; then
|
||||
pass "data behind code (${rev}) is known and allowed to restore"
|
||||
else
|
||||
fail "data behind code was refused: $(tail -1 "${WORKDIR}/gate.log")"
|
||||
fail "data behind code (${rev}) was not accepted"
|
||||
fi
|
||||
|
||||
# --- 3. datarev the code does NOT know (newer than build) is refused -------
|
||||
make_set 'set-ahead' '0004_future' '0004_future' '{}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-ahead' "${CHAIN}" "${PLUGINS}"; then
|
||||
fail "data ahead of code (0004_future) was NOT refused"
|
||||
SET2="${WORKDIR}/set2"; mkdir -p "${SET2}"
|
||||
${PY} "${MANIFEST_PY}" --out "${SET2}/manifest.json" \
|
||||
--migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \
|
||||
--datarev '0004_future' --takenat '2024-02-01T00:00:00Z' >/dev/null 2>&1
|
||||
DBDUMP2="${SET2}/db.sql.gz"; build_dump "${DBDUMP2}" '0004_future'
|
||||
rev="$(set_datarev "${SET2}" "${DBDUMP2}")"
|
||||
if [ "${rev}" = '0004_future' ] && ! code_knows_revision "${VERSIONS}" "${rev}"; then
|
||||
pass "data ahead of code (${rev}) is unknown and correctly refused"
|
||||
else
|
||||
pass "data ahead of code (0004_future) is unknown and correctly refused"
|
||||
fail "data ahead of code (${rev}) was not refused"
|
||||
fi
|
||||
|
||||
# --- 4. legacy manifest (no datarev) falls back to the dump ----------------
|
||||
make_set 'set-legacy' '__legacy__' '0001_baseline' '{}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-legacy' "${CHAIN}" "${PLUGINS}"; then
|
||||
pass "legacy set with no datarev falls back to the dump revision"
|
||||
SET3="${WORKDIR}/set3"; mkdir -p "${SET3}"
|
||||
# A pre-datarev manifest: schemarev present, datarev absent/empty.
|
||||
printf '{"takenat":"2023-06-01T00:00:00Z","schemarev":{"core":["0003_head"]}}' \
|
||||
> "${SET3}/manifest.json"
|
||||
DBDUMP3="${SET3}/db.sql.gz"; build_dump "${DBDUMP3}" '0001_baseline'
|
||||
rev="$(set_datarev "${SET3}" "${DBDUMP3}")"
|
||||
if [ "${rev}" = '0001_baseline' ] && code_knows_revision "${VERSIONS}" "${rev}"; then
|
||||
pass "legacy set with no datarev falls back to dump revision (${rev})"
|
||||
else
|
||||
fail "legacy set fallback failed: $(tail -1 "${WORKDIR}/gate.log")"
|
||||
fail "legacy set fallback failed (got '${rev}')"
|
||||
fi
|
||||
|
||||
# --- 5. restore.sh hands off to the api instead of self-completing ---------
|
||||
@@ -173,58 +166,6 @@ else
|
||||
fail "restore.sh does not hand off the upgrade to the api"
|
||||
fi
|
||||
|
||||
# --- 6. a PLUGIN chain the code does not know is refused -------------------
|
||||
# The core head is identical in both of these; only the plugin revision differs.
|
||||
# That is the whole failure: a core-only gate cannot tell them apart.
|
||||
make_set 'set-plugin-known' '0002_mid' '0002_mid' '{"endpoints":"endpoints0002known"}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-plugin-known' "${CHAIN}" "${PLUGINS}"; then
|
||||
pass "a set whose plugin revision the code knows is allowed"
|
||||
else
|
||||
fail "known plugin revision was refused: $(tail -1 "${WORKDIR}/gate.log")"
|
||||
fi
|
||||
|
||||
make_set 'set-plugin-future' '0002_mid' '0002_mid' '{"endpoints":"endpoints0099future"}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-plugin-future' "${CHAIN}" "${PLUGINS}"; then
|
||||
fail "plugin data ahead of code was NOT refused (core-only gate regression)"
|
||||
else
|
||||
pass "plugin data ahead of code is refused even though the core head matches"
|
||||
fi
|
||||
|
||||
# --- 7. a plugin the build does not have at all is refused -----------------
|
||||
make_set 'set-plugin-missing' '0002_mid' '0002_mid' '{"nosuchplugin":"whatever0001"}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-plugin-missing' "${CHAIN}" "${PLUGINS}"; then
|
||||
fail "a set carrying data for an absent plugin was NOT refused"
|
||||
else
|
||||
pass "a set carrying data for a plugin this build lacks is refused"
|
||||
fi
|
||||
|
||||
# --- 8. a dump bound to another schema is refused --------------------------
|
||||
# --databases bakes 'USE <name>' in, which silently loaded the rows beside the
|
||||
# live database and reported success.
|
||||
setdir="${ROOT}/set-otherschema"
|
||||
mkdir -p "${setdir}"
|
||||
${PY} "${MANIFEST_PY}" --out "${setdir}/manifest.json" \
|
||||
--migrations "${CHAIN}" --plugins "${PLUGINS}" \
|
||||
--datarev '0002_mid' --dataplugins '{}' \
|
||||
--takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1
|
||||
{
|
||||
echo 'CREATE DATABASE `someothersite`;'
|
||||
echo 'USE `someothersite`;'
|
||||
echo 'INSERT INTO `alembic_version` VALUES'
|
||||
echo "('0002_mid');"
|
||||
} | gzip -c > "${setdir}/db.sql.gz"
|
||||
# Now a gate, so check-only reaches it without needing a database.
|
||||
out="$(RESTORE_CHECK_ONLY=1 SHOPDB_BACKUPS_ROOT="${ROOT}" \
|
||||
BACKUP_LIB_DIR="${SCRIPT_DIR}" \
|
||||
APP_MIGRATIONS_DIR="${CHAIN}" APP_PLUGINS_DIR="${PLUGINS}" \
|
||||
MYSQL_DATABASE='shopdb_flask' \
|
||||
bash "${RESTORE_SH}" 'set-otherschema' 2>&1)" || true
|
||||
if printf '%s' "${out}" | grep -q "bound to schema 'someothersite'"; then
|
||||
pass "a dump bound to another site's schema is refused"
|
||||
else
|
||||
fail "a schema-bound dump was not caught (got: $(printf '%s' "${out}" | tail -1))"
|
||||
fi
|
||||
|
||||
echo "=================================================="
|
||||
echo "[test] datarev gate: ${passes} passed, ${fails} failed"
|
||||
echo "=================================================="
|
||||
|
||||
0
scripts/backup/watch.sh.tmp
Normal file
0
scripts/backup/watch.sh.tmp
Normal file
Reference in New Issue
Block a user