Merge wilm: gevent migration, endpoint telemetry phase 2, backup/restore
Brings in 98 commits from wilm (origin/wilm @ 811e6ca61a):
Major features:
- Move API to gevent workers for better concurrency
- Endpoint agent startup jitter (prevents thundering herd)
- Complete backup/restore with restic
- Endpoint telemetry phase 2 (disk health, mapped drives, user context)
- PC Detail UI rework with new cards and tabs
- Dell Command actions and Corporate Tools tab
- Agent version column in Fleet view
Conflict resolution:
- Backup/restore: Accepted wilm implementation (10 files)
- Agent startup jitter: Accepted wilm v1.12.0
- PC Types list fix: Accepted wilm version
- Endpoint plugin domain clamping: Kept ours (defensive)
- Test files: Accepted wilm versions for consistency
Testing required:
- Verify gevent handles concurrent requests
- Test backup/restore functionality
- Verify agent startup jitter timing
- Test endpoint telemetry collection
- Validate PC Detail UI changes
Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Proudlock
@@ -79,62 +79,43 @@ build_dump() {
|
||||
} | gzip -c > "${out}"
|
||||
}
|
||||
|
||||
# Build a throwaway plugin chain, shaped like plugins/<name>/migrations/versions.
|
||||
build_plugin_chain() {
|
||||
root="$1"; name="$2"; rev="$3"
|
||||
versions="${root}/${name}/migrations/versions"
|
||||
mkdir -p "${versions}"
|
||||
printf "revision = '%s'\ndown_revision = None\n" \
|
||||
"${rev}" > "${versions}/${rev}.py"
|
||||
# Mirror of the restore.sh predicate: does the code chain know this revision?
|
||||
# Kept identical to restore.sh so this test tracks the shipped decision.
|
||||
code_knows_revision() {
|
||||
versions_dir="$1"; datarev="$2"
|
||||
grep -rqE "^revision = ['\"]${datarev}['\"]" "${versions_dir}" 2>/dev/null
|
||||
}
|
||||
|
||||
# Drive the REAL restore.sh gate. This is the point of the rewrite: the test
|
||||
# used to carry its own copy of the predicate, so it stayed green no matter how
|
||||
# far restore.sh drifted from it. RESTORE_CHECK_ONLY stops the script right
|
||||
# after the gate, so the verdict here IS the shipped verdict.
|
||||
run_gate() {
|
||||
backups_root="$1"; setname="$2"; chain="$3"; plugins="$4"
|
||||
RESTORE_CHECK_ONLY=1 \
|
||||
SHOPDB_BACKUPS_ROOT="${backups_root}" \
|
||||
BACKUP_LIB_DIR="${SCRIPT_DIR}" \
|
||||
APP_MIGRATIONS_DIR="${chain}" \
|
||||
APP_PLUGINS_DIR="${plugins}" \
|
||||
bash "${RESTORE_SH}" "${setname}" >"${WORKDIR}/gate.log" 2>&1
|
||||
# Mirror of the restore.sh datarev extraction: manifest field first, else parse
|
||||
# the dump.
|
||||
set_datarev() {
|
||||
setdir="$1"; dbdump="$2"
|
||||
value="$(${PY} -c \
|
||||
'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \
|
||||
"${setdir}/manifest.json" 2>/dev/null || echo "")"
|
||||
if [ -z "${value}" ]; then
|
||||
value="$(gunzip -c "${dbdump}" \
|
||||
| grep -A2 -i 'INSERT INTO `alembic_version`' \
|
||||
| grep -oE "'[0-9a-zA-Z_]+'" | tr -d "'" | head -1)"
|
||||
fi
|
||||
printf '%s' "${value}"
|
||||
}
|
||||
|
||||
CHAIN="${WORKDIR}/migrations"
|
||||
build_chain "${CHAIN}"
|
||||
PLUGINS="${WORKDIR}/plugins"
|
||||
build_plugin_chain "${PLUGINS}" 'endpoints' 'endpoints0002known'
|
||||
ROOT="${WORKDIR}/backups"
|
||||
mkdir -p "${ROOT}"
|
||||
VERSIONS="${CHAIN}/versions"
|
||||
|
||||
# Lay out one set the way a real backup does, under the backups root.
|
||||
make_set() {
|
||||
setname="$1"; datarev="$2"; dumprev="$3"; dataplugins="$4"
|
||||
setdir="${ROOT}/${setname}"
|
||||
mkdir -p "${setdir}"
|
||||
if [ "${datarev}" = "__legacy__" ]; then
|
||||
printf '{"takenat":"2023-06-01T00:00:00Z","schemarev":{"core":["0003_head"]}}' \
|
||||
> "${setdir}/manifest.json"
|
||||
else
|
||||
${PY} "${MANIFEST_PY}" --out "${setdir}/manifest.json" \
|
||||
--migrations "${CHAIN}" --plugins "${PLUGINS}" \
|
||||
--datarev "${datarev}" --dataplugins "${dataplugins}" \
|
||||
--takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1
|
||||
fi
|
||||
build_dump "${setdir}/db.sql.gz" "${dumprev}"
|
||||
printf '%s' "${setdir}"
|
||||
}
|
||||
|
||||
# --- 1. manifest.py records datarev separately from the code head ----------
|
||||
make_set 'set-behind' '0002_mid' '0002_mid' '{}' >/dev/null
|
||||
# --- 1. manifest.py records datarev distinct from the code head -------------
|
||||
SET1="${WORKDIR}/set1"; mkdir -p "${SET1}"
|
||||
${PY} "${MANIFEST_PY}" --out "${SET1}/manifest.json" \
|
||||
--migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \
|
||||
--datarev '0002_mid' --takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1
|
||||
recorded="$(${PY} -c \
|
||||
'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \
|
||||
"${ROOT}/set-behind/manifest.json" 2>/dev/null)"
|
||||
"${SET1}/manifest.json" 2>/dev/null)"
|
||||
head_stamp="$(${PY} -c \
|
||||
'import json,sys;print(json.load(open(sys.argv[1])).get("schemarev",{}).get("core",[]))' \
|
||||
"${ROOT}/set-behind/manifest.json" 2>/dev/null)"
|
||||
"${SET1}/manifest.json" 2>/dev/null)"
|
||||
if [ "${recorded}" = '0002_mid' ]; then
|
||||
pass "manifest.py records datarev (${recorded}) separate from code head ${head_stamp}"
|
||||
else
|
||||
@@ -142,26 +123,38 @@ else
|
||||
fi
|
||||
|
||||
# --- 2. datarev the code KNOWS (behind head) is allowed --------------------
|
||||
if run_gate "${ROOT}" 'set-behind' "${CHAIN}" "${PLUGINS}"; then
|
||||
pass "data behind code (0002_mid) is known and allowed to restore"
|
||||
DBDUMP1="${SET1}/db.sql.gz"; build_dump "${DBDUMP1}" '0002_mid'
|
||||
rev="$(set_datarev "${SET1}" "${DBDUMP1}")"
|
||||
if [ "${rev}" = '0002_mid' ] && code_knows_revision "${VERSIONS}" "${rev}"; then
|
||||
pass "data behind code (${rev}) is known and allowed to restore"
|
||||
else
|
||||
fail "data behind code was refused: $(tail -1 "${WORKDIR}/gate.log")"
|
||||
fail "data behind code (${rev}) was not accepted"
|
||||
fi
|
||||
|
||||
# --- 3. datarev the code does NOT know (newer than build) is refused -------
|
||||
make_set 'set-ahead' '0004_future' '0004_future' '{}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-ahead' "${CHAIN}" "${PLUGINS}"; then
|
||||
fail "data ahead of code (0004_future) was NOT refused"
|
||||
SET2="${WORKDIR}/set2"; mkdir -p "${SET2}"
|
||||
${PY} "${MANIFEST_PY}" --out "${SET2}/manifest.json" \
|
||||
--migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \
|
||||
--datarev '0004_future' --takenat '2024-02-01T00:00:00Z' >/dev/null 2>&1
|
||||
DBDUMP2="${SET2}/db.sql.gz"; build_dump "${DBDUMP2}" '0004_future'
|
||||
rev="$(set_datarev "${SET2}" "${DBDUMP2}")"
|
||||
if [ "${rev}" = '0004_future' ] && ! code_knows_revision "${VERSIONS}" "${rev}"; then
|
||||
pass "data ahead of code (${rev}) is unknown and correctly refused"
|
||||
else
|
||||
pass "data ahead of code (0004_future) is unknown and correctly refused"
|
||||
fail "data ahead of code (${rev}) was not refused"
|
||||
fi
|
||||
|
||||
# --- 4. legacy manifest (no datarev) falls back to the dump ----------------
|
||||
make_set 'set-legacy' '__legacy__' '0001_baseline' '{}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-legacy' "${CHAIN}" "${PLUGINS}"; then
|
||||
pass "legacy set with no datarev falls back to the dump revision"
|
||||
SET3="${WORKDIR}/set3"; mkdir -p "${SET3}"
|
||||
# A pre-datarev manifest: schemarev present, datarev absent/empty.
|
||||
printf '{"takenat":"2023-06-01T00:00:00Z","schemarev":{"core":["0003_head"]}}' \
|
||||
> "${SET3}/manifest.json"
|
||||
DBDUMP3="${SET3}/db.sql.gz"; build_dump "${DBDUMP3}" '0001_baseline'
|
||||
rev="$(set_datarev "${SET3}" "${DBDUMP3}")"
|
||||
if [ "${rev}" = '0001_baseline' ] && code_knows_revision "${VERSIONS}" "${rev}"; then
|
||||
pass "legacy set with no datarev falls back to dump revision (${rev})"
|
||||
else
|
||||
fail "legacy set fallback failed: $(tail -1 "${WORKDIR}/gate.log")"
|
||||
fail "legacy set fallback failed (got '${rev}')"
|
||||
fi
|
||||
|
||||
# --- 5. restore.sh hands off to the api instead of self-completing ---------
|
||||
@@ -173,58 +166,6 @@ else
|
||||
fail "restore.sh does not hand off the upgrade to the api"
|
||||
fi
|
||||
|
||||
# --- 6. a PLUGIN chain the code does not know is refused -------------------
|
||||
# The core head is identical in both of these; only the plugin revision differs.
|
||||
# That is the whole failure: a core-only gate cannot tell them apart.
|
||||
make_set 'set-plugin-known' '0002_mid' '0002_mid' '{"endpoints":"endpoints0002known"}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-plugin-known' "${CHAIN}" "${PLUGINS}"; then
|
||||
pass "a set whose plugin revision the code knows is allowed"
|
||||
else
|
||||
fail "known plugin revision was refused: $(tail -1 "${WORKDIR}/gate.log")"
|
||||
fi
|
||||
|
||||
make_set 'set-plugin-future' '0002_mid' '0002_mid' '{"endpoints":"endpoints0099future"}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-plugin-future' "${CHAIN}" "${PLUGINS}"; then
|
||||
fail "plugin data ahead of code was NOT refused (core-only gate regression)"
|
||||
else
|
||||
pass "plugin data ahead of code is refused even though the core head matches"
|
||||
fi
|
||||
|
||||
# --- 7. a plugin the build does not have at all is refused -----------------
|
||||
make_set 'set-plugin-missing' '0002_mid' '0002_mid' '{"nosuchplugin":"whatever0001"}' >/dev/null
|
||||
if run_gate "${ROOT}" 'set-plugin-missing' "${CHAIN}" "${PLUGINS}"; then
|
||||
fail "a set carrying data for an absent plugin was NOT refused"
|
||||
else
|
||||
pass "a set carrying data for a plugin this build lacks is refused"
|
||||
fi
|
||||
|
||||
# --- 8. a dump bound to another schema is refused --------------------------
|
||||
# --databases bakes 'USE <name>' in, which silently loaded the rows beside the
|
||||
# live database and reported success.
|
||||
setdir="${ROOT}/set-otherschema"
|
||||
mkdir -p "${setdir}"
|
||||
${PY} "${MANIFEST_PY}" --out "${setdir}/manifest.json" \
|
||||
--migrations "${CHAIN}" --plugins "${PLUGINS}" \
|
||||
--datarev '0002_mid' --dataplugins '{}' \
|
||||
--takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1
|
||||
{
|
||||
echo 'CREATE DATABASE `someothersite`;'
|
||||
echo 'USE `someothersite`;'
|
||||
echo 'INSERT INTO `alembic_version` VALUES'
|
||||
echo "('0002_mid');"
|
||||
} | gzip -c > "${setdir}/db.sql.gz"
|
||||
# Now a gate, so check-only reaches it without needing a database.
|
||||
out="$(RESTORE_CHECK_ONLY=1 SHOPDB_BACKUPS_ROOT="${ROOT}" \
|
||||
BACKUP_LIB_DIR="${SCRIPT_DIR}" \
|
||||
APP_MIGRATIONS_DIR="${CHAIN}" APP_PLUGINS_DIR="${PLUGINS}" \
|
||||
MYSQL_DATABASE='shopdb_flask' \
|
||||
bash "${RESTORE_SH}" 'set-otherschema' 2>&1)" || true
|
||||
if printf '%s' "${out}" | grep -q "bound to schema 'someothersite'"; then
|
||||
pass "a dump bound to another site's schema is refused"
|
||||
else
|
||||
fail "a schema-bound dump was not caught (got: $(printf '%s' "${out}" | tail -1))"
|
||||
fi
|
||||
|
||||
echo "=================================================="
|
||||
echo "[test] datarev gate: ${passes} passed, ${fails} failed"
|
||||
echo "=================================================="
|
||||
|
||||
Reference in New Issue
Block a user