Merge wilm: gevent migration, endpoint telemetry phase 2, backup/restore
Brings in 98 commits from wilm (origin/wilm @ 811e6ca61a):
Major features:
- Move API to gevent workers for better concurrency
- Endpoint agent startup jitter (prevents thundering herd)
- Complete backup/restore with restic
- Endpoint telemetry phase 2 (disk health, mapped drives, user context)
- PC Detail UI rework with new cards and tabs
- Dell Command actions and Corporate Tools tab
- Agent version column in Fleet view
Conflict resolution:
- Backup/restore: Accepted wilm implementation (10 files)
- Agent startup jitter: Accepted wilm v1.12.0
- PC Types list fix: Accepted wilm version
- Endpoint plugin domain clamping: Kept ours (defensive)
- Test files: Accepted wilm versions for consistency
Testing required:
- Verify gevent handles concurrent requests
- Test backup/restore functionality
- Verify agent startup jitter timing
- Test endpoint telemetry collection
- Validate PC Detail UI changes
Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Proudlock
@@ -21,22 +21,8 @@ set -euo pipefail
|
||||
BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}"
|
||||
SETNAME="${1:-}"
|
||||
|
||||
# Overridable so the gate can be exercised outside a container against throwaway
|
||||
# migration trees. In the image these are exactly the mounted paths, so the
|
||||
# shipped behaviour is unchanged; test-restore-datarev.sh drives THIS script
|
||||
# rather than a copy of its logic, which is the only way the test can notice
|
||||
# this script drifting.
|
||||
BACKUP_LIB_DIR="${BACKUP_LIB_DIR:-/opt/backup}"
|
||||
APP_MIGRATIONS_DIR="${APP_MIGRATIONS_DIR:-/app/migrations}"
|
||||
APP_PLUGINS_DIR="${APP_PLUGINS_DIR:-/app/plugins}"
|
||||
APP_INSTANCE_DIR="${APP_INSTANCE_DIR:-/app/instance}"
|
||||
|
||||
# When set, stop after the compatibility gate and report the verdict as the exit
|
||||
# code, touching nothing. This is what makes the gate testable.
|
||||
RESTORE_CHECK_ONLY="${RESTORE_CHECK_ONLY:-}"
|
||||
|
||||
# shellcheck source=scripts/backup/status.sh
|
||||
. "${BACKUP_LIB_DIR}/status.sh"
|
||||
. /opt/backup/status.sh
|
||||
|
||||
log() { echo "[restore] $*"; }
|
||||
|
||||
@@ -75,7 +61,7 @@ DBDUMP="${SETDIR}/db.sql.gz"
|
||||
if [ ! -f "${DBDUMP}" ]; then
|
||||
log "local dump not found; restoring set from restic"
|
||||
command -v restic >/dev/null 2>&1 || die "restic not available and no local dump"
|
||||
restic restore latest --tag "${SETNAME}" --target / --include "${SETDIR}" \
|
||||
restic restore latest --target / --include "${SETDIR}" \
|
||||
|| die "restic restore of ${SETNAME} failed"
|
||||
fi
|
||||
[ -f "${DBDUMP}" ] || die "no db.sql.gz for set ${SETNAME}"
|
||||
@@ -105,173 +91,30 @@ if [ -z "${SET_DATAREV}" ]; then
|
||||
die "cannot determine the set's data revision - refusing an unverified restore"
|
||||
fi
|
||||
|
||||
|
||||
# Does the running code know this revision at all? If a migration file with that
|
||||
# revision id exists in the core chain, the code is at-or-past it (the head is
|
||||
# the tip of the same chain), so a post-load upgrade can carry the data forward.
|
||||
# If it is absent, the dump is NEWER than the code - refuse.
|
||||
if ! grep -rqE "^revision = ['\"]${SET_DATAREV}['\"]" "${APP_MIGRATIONS_DIR}/versions" 2>/dev/null; then
|
||||
if ! grep -rqE "^revision = ['\"]${SET_DATAREV}['\"]" /app/migrations/versions 2>/dev/null; then
|
||||
log "set datarev : ${SET_DATAREV}"
|
||||
die "the backup's data is at a revision this code does not have - deploy code at or past '${SET_DATAREV}' before restoring"
|
||||
fi
|
||||
log "data revision ${SET_DATAREV} is known to this code - restore may proceed"
|
||||
|
||||
# The same gate for every PLUGIN chain, because the core chain alone cannot
|
||||
# answer this. Plugin schemas move independently of core: roll the code back to
|
||||
# an older plugin revision and the core head can be identical while the dump
|
||||
# carries plugin tables the code has never heard of. Gating on core only let
|
||||
# that pass, load the dump, and then fail in 'flask plugin upgrade-all' with
|
||||
# "Can't locate revision" and the data already replaced - a failed restore that
|
||||
# has destroyed what it was restoring over.
|
||||
#
|
||||
# A manifest with no dataplugins is a legacy set taken before this was stamped.
|
||||
# Its plugin revisions are unknowable, so it is allowed through on the core gate
|
||||
# alone (the behaviour before this check existed) rather than being made
|
||||
# unrestorable retroactively.
|
||||
SET_DATAPLUGINS="$(python3 -c \
|
||||
'import json,sys
|
||||
try:
|
||||
data = json.load(open(sys.argv[1])).get("dataplugins") or {}
|
||||
except Exception:
|
||||
data = {}
|
||||
for name, rev in sorted(data.items()):
|
||||
print(name + " " + rev)' \
|
||||
"${SETDIR}/manifest.json" 2>/dev/null || echo "")"
|
||||
|
||||
if [ -z "${SET_DATAPLUGINS}" ]; then
|
||||
log "manifest stamps no plugin revisions (legacy set) - core gate only"
|
||||
else
|
||||
while read -r plugin rev; do
|
||||
[ -n "${plugin}" ] || continue
|
||||
chain="${APP_PLUGINS_DIR}/${plugin}/migrations/versions"
|
||||
if [ ! -d "${chain}" ]; then
|
||||
die "the backup carries data for plugin '${plugin}', which this build does not have - deploy a build that includes it before restoring"
|
||||
fi
|
||||
if ! grep -rqE "^revision = ['\"]${rev}['\"]" "${chain}" 2>/dev/null; then
|
||||
log "plugin ${plugin} data revision: ${rev}"
|
||||
die "plugin '${plugin}' data is at a revision this code does not have - deploy code at or past '${rev}' before restoring"
|
||||
fi
|
||||
log "plugin ${plugin} at ${rev} - known to this code"
|
||||
done <<PLUGINREVS
|
||||
${SET_DATAPLUGINS}
|
||||
PLUGINREVS
|
||||
fi
|
||||
|
||||
# Last gate: which schema does this dump target? A dump taken with --databases
|
||||
# carries CREATE DATABASE and USE <name>, pinning it to the schema it came from.
|
||||
# Loaded into a site whose MYSQL_DATABASE differs, the rows land in the dump's
|
||||
# schema, the live database is untouched, and the restore reports success. New
|
||||
# sets are dumped without it and load into whatever schema is named here; older
|
||||
# sets are still bound, so refuse rather than restore beside the live data.
|
||||
# Checked HERE, with the other gates, so it costs nothing and stops before the
|
||||
# safety copy rather than after it.
|
||||
DUMP_SCHEMA="$(gunzip -c "${DBDUMP}" \
|
||||
| grep -m1 -oE '^USE `[^`]+`' \
|
||||
| sed -E 's/^USE `(.*)`/\1/' || true)"
|
||||
if [ -n "${DUMP_SCHEMA}" ]; then
|
||||
if [ "${DUMP_SCHEMA}" != "${MYSQL_DATABASE}" ]; then
|
||||
die "this set is bound to schema '${DUMP_SCHEMA}' but this site's database is '${MYSQL_DATABASE}' - restoring it would load the data beside the live database and report success"
|
||||
fi
|
||||
log "legacy schema-bound dump for '${DUMP_SCHEMA}' - matches this site"
|
||||
fi
|
||||
|
||||
if [ -n "${RESTORE_CHECK_ONLY}" ]; then
|
||||
log "check-only: every gate passed, stopping before anything destructive"
|
||||
RESTORE_LOADED=false
|
||||
trap - EXIT
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Safety set FIRST. Everything below this point is destructive and none of it
|
||||
# is transactional: the dump carries DROP TABLE per table, and the instance tree
|
||||
# is replaced wholesale. Without a copy taken here, a restore of the wrong set -
|
||||
# or a failure part-way through the load - leaves nothing to go back to. This is
|
||||
# a local copy on the same volume, so it is cheap and it is NOT a substitute for
|
||||
# a real backup; it exists to make THIS operation reversible.
|
||||
SAFETY_STAMP="$(date -u +%Y-%m-%dT%H%M%SZ)"
|
||||
SAFETY_DIR="${BACKUPS_ROOT}/pre-restore-${SAFETY_STAMP}"
|
||||
status_write restore safety "${SETNAME}" "Copying the current state aside"
|
||||
log "taking a pre-restore safety copy into ${SAFETY_DIR}"
|
||||
mkdir -p "${SAFETY_DIR}" || die "could not create ${SAFETY_DIR}"
|
||||
|
||||
mysqldump \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--single-transaction \
|
||||
--routines \
|
||||
--triggers \
|
||||
--no-tablespaces \
|
||||
"${MYSQL_DATABASE}" \
|
||||
| gzip -c > "${SAFETY_DIR}/db.sql.gz" \
|
||||
|| die "pre-restore dump failed - refusing to touch the live database"
|
||||
|
||||
if [ -d /app/instance ]; then
|
||||
tar -czf "${SAFETY_DIR}/instance.tar.gz" -C /app instance \
|
||||
|| die "pre-restore instance copy failed - refusing to touch the live files"
|
||||
fi
|
||||
|
||||
# Named so an operator finding this directory months later knows what it is and
|
||||
# how to use it, without reading this script.
|
||||
cat > "${SAFETY_DIR}/README.txt" <<SAFETYNOTE
|
||||
Automatic pre-restore copy of the state this site was in immediately before
|
||||
restoring set '${SETNAME}' at ${SAFETY_STAMP}.
|
||||
|
||||
db.sql.gz mysqldump of ${MYSQL_DATABASE} as it was
|
||||
instance.tar.gz /app/instance as it was (absent if there was no such tree)
|
||||
|
||||
To go back to this state, load the dump the same way a restore does:
|
||||
|
||||
gunzip -c db.sql.gz | mysql --host=... --user=... --password=... \
|
||||
--database=${MYSQL_DATABASE}
|
||||
tar -xzf instance.tar.gz -C /app
|
||||
|
||||
This is a local copy on the backups volume, taken for reversibility of one
|
||||
restore. It is not a backup set: it has no manifest and no restic snapshot, and
|
||||
nothing prunes it.
|
||||
SAFETYNOTE
|
||||
log "safety copy written ($(du -sh "${SAFETY_DIR}" | cut -f1))"
|
||||
|
||||
# Files first, then DB, so the two land together.
|
||||
if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then
|
||||
status_write restore files "${SETNAME}" "Restoring instance files"
|
||||
# Scoped to THIS set's snapshot. A bare 'latest' restores whatever was
|
||||
# snapshotted most recently, so restoring anything other than the newest set
|
||||
# paired that set's DATABASE with the NEWEST snapshot's uploads and
|
||||
# plugins.json - a mismatch nothing downstream could detect.
|
||||
log "restoring instance files from the ${SETNAME} snapshot"
|
||||
STAGE="$(mktemp -d)"
|
||||
restic restore latest --tag "${SETNAME}" --target "${STAGE}" \
|
||||
--include /app/instance \
|
||||
|| die "restic restore of instance files for ${SETNAME} failed"
|
||||
|
||||
# Replace the CONTENTS of the instance tree rather than restoring over it,
|
||||
# so files created after the snapshot are gone afterwards - a restore has to
|
||||
# put the site back to the snapshot, not merge into it. Done as an explicit
|
||||
# staged swap rather than 'restic restore --delete --target /', because that
|
||||
# aims a delete pass at the container root and the blast radius of getting
|
||||
# its filter wrong is the whole filesystem. /app/instance is a mount, so the
|
||||
# directory itself has to survive; only what is inside it is replaced.
|
||||
if [ -d "${STAGE}/app/instance" ]; then
|
||||
find /app/instance -mindepth 1 -delete \
|
||||
|| die "could not clear the instance tree before restoring it"
|
||||
cp -a "${STAGE}/app/instance/." /app/instance/ \
|
||||
|| die "could not lay down the restored instance tree"
|
||||
else
|
||||
log "WARN snapshot for ${SETNAME} carried no /app/instance - leaving files alone"
|
||||
fi
|
||||
rm -rf "${STAGE}"
|
||||
log "restoring instance files from restic"
|
||||
restic restore latest --target / --include /app/instance \
|
||||
|| die "restic restore of instance files failed"
|
||||
fi
|
||||
|
||||
status_write restore database "${SETNAME}" "Loading database dump"
|
||||
|
||||
|
||||
log "loading database dump into ${MYSQL_DATABASE}"
|
||||
log "loading database dump"
|
||||
gunzip -c "${DBDUMP}" | mysql \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--database="${MYSQL_DATABASE}" \
|
||||
|| die "database load failed"
|
||||
|
||||
# Hand off to the api container for the schema upgrade. The dump may sit at an
|
||||
|
||||
Reference in New Issue
Block a user