Merge wilm: gevent migration, endpoint telemetry phase 2, backup/restore

Brings in 98 commits from wilm (origin/wilm @ 811e6ca61a):

Major features:
- Move API to gevent workers for better concurrency
- Endpoint agent startup jitter (prevents thundering herd)
- Complete backup/restore with restic
- Endpoint telemetry phase 2 (disk health, mapped drives, user context)
- PC Detail UI rework with new cards and tabs
- Dell Command actions and Corporate Tools tab
- Agent version column in Fleet view

Conflict resolution:
- Backup/restore: Accepted wilm implementation (10 files)
- Agent startup jitter: Accepted wilm v1.12.0
- PC Types list fix: Accepted wilm version
- Endpoint plugin domain clamping: Kept ours (defensive)
- Test files: Accepted wilm versions for consistency

Testing required:
- Verify gevent handles concurrent requests
- Test backup/restore functionality
- Verify agent startup jitter timing
- Test endpoint telemetry collection
- Validate PC Detail UI changes

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Proudlock
2026-09-06 21:23:42 -04:00
32 changed files with 986 additions and 710 deletions

View File

@@ -47,12 +47,6 @@ mkdir -p "${SETDIR}"
status_write backup database "${SETNAME}" "Dumping database"
# 1. Database: whole-DB logical dump, consistent and non-blocking on InnoDB.
# Dumped WITHOUT --databases on purpose. That flag bakes CREATE DATABASE and USE
# <name> into the file, which pins the dump to the schema name it was taken
# from: restoring onto a site whose MYSQL_DATABASE differs loaded the rows into
# the dump's schema, left the live one untouched, and reported success.
# --no-tablespaces because dumping tablespace metadata needs the PROCESS
# privilege, which the app user does not have on MySQL 8.
log "dumping database ${MYSQL_DATABASE} from ${MYSQL_HOST}"
mysqldump \
--host="${MYSQL_HOST}" \
@@ -61,8 +55,7 @@ mysqldump \
--single-transaction \
--routines \
--triggers \
--no-tablespaces \
"${MYSQL_DATABASE}" \
--databases "${MYSQL_DATABASE}" \
| gzip -c > "${SETDIR}/db.sql.gz"
log "database dump written ($(du -h "${SETDIR}/db.sql.gz" | cut -f1))"
@@ -80,48 +73,7 @@ DATAREV="$(mysql \
--skip-column-names --batch \
-e "SELECT version_num FROM alembic_version LIMIT 1;" \
"${MYSQL_DATABASE}" 2>/dev/null | tr -d '\r')"
if [ -z "${DATAREV}" ]; then
# Recorded loudly rather than shrugged off: a set with no data revision
# cannot be gated on restore, and the gate treats an empty stamp as "no
# evidence" and refuses it. Better a visible warning now than a set that
# looks restorable and is not.
log "WARN could not read alembic_version - this set will NOT be restorable"
history_append backup warning "${SETNAME}" \
"Could not read alembic_version; set has no data revision and cannot be restored"
fi
# The same question for every PLUGIN chain. Each plugin owns an
# alembic_version_<plugin> table, and those move independently of the core one:
# code rolled back to an older plugin revision can sit at an identical core
# head while the dump carries plugin tables it has never heard of. Stamping only
# the core revision let that combination pass the restore gate, load, and then
# fail during the plugin upgrade with the data already replaced. Read straight
# from the dumped DB, same as the core revision above.
log "reading applied plugin revisions (alembic_version_*)"
DATAPLUGINS="$(mysql \
--host="${MYSQL_HOST}" \
--user="${MYSQL_USER}" \
--password="${MYSQL_PASSWORD}" \
--skip-column-names --batch \
-e "SELECT table_name FROM information_schema.tables
WHERE table_schema = '${MYSQL_DATABASE}'
AND table_name LIKE 'alembic\\_version\\_%';" 2>/dev/null \
| tr -d '\r' \
| while read -r table; do
[ -n "${table}" ] || continue
plugin="${table#alembic_version_}"
rev="$(mysql \
--host="${MYSQL_HOST}" \
--user="${MYSQL_USER}" \
--password="${MYSQL_PASSWORD}" \
--skip-column-names --batch \
-e "SELECT version_num FROM \`${table}\` LIMIT 1;" \
"${MYSQL_DATABASE}" 2>/dev/null | tr -d '\r')"
[ -n "${rev}" ] && printf '%s\t%s\n' "${plugin}" "${rev}"
done \
| python3 -c 'import json,sys; print(json.dumps(dict(l.rstrip("\n").split("\t",1) for l in sys.stdin if "\t" in l)))')"
[ -n "${DATAPLUGINS}" ] || DATAPLUGINS='{}'
log "plugin revisions: ${DATAPLUGINS}"
[ -n "${DATAREV}" ] || log "WARN could not read alembic_version - datarev will be empty"
log "writing manifest"
python3 /opt/backup/manifest.py \
@@ -130,7 +82,6 @@ python3 /opt/backup/manifest.py \
--site "${SITE}" \
--imagetag "${SHOPDB_IMAGE_TAG:-source}" \
--datarev "${DATAREV}" \
--dataplugins "${DATAPLUGINS}" \
--migrations /app/migrations \
--plugins /app/plugins