From debe1fce690978816f556e2313a32ffe526555da Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Thu, 3 Sep 2026 20:43:42 -0400 Subject: [PATCH] it is risen - add restore progress and backup/restore history The watcher scripts now write live status and append-only run history onto the shared volume; the api exposes them via /shopdbbackups/status and /history. The restore modal shows an indeterminate spinner with phase labels, and two cards list backup and restore run records. Timestamps render in UTC to match the Zulu-stamped set ids, the history directory is excluded from the set list, and a stale-status guard stops a previous run's done phase from collapsing the spinner on the next restore. --- .../src/views/settings/ShopdbBackupsList.vue | 254 +++++++++++++++++- scripts/backup/backup.sh | 21 ++ scripts/backup/restore.sh | 24 ++ scripts/backup/status.sh | 100 +++++++ scripts/backup/watch.sh | 5 + shopdb/core/api/shopdbbackups.py | 84 ++++++ 6 files changed, 485 insertions(+), 3 deletions(-) create mode 100644 scripts/backup/status.sh diff --git a/frontend/src/views/settings/ShopdbBackupsList.vue b/frontend/src/views/settings/ShopdbBackupsList.vue index d8f541dd..eaeab054 100644 --- a/frontend/src/views/settings/ShopdbBackupsList.vue +++ b/frontend/src/views/settings/ShopdbBackupsList.vue @@ -185,6 +185,38 @@ + +
+
+

Backup history

+

No backup runs recorded yet.

+
    +
  • + + {{ outcomeLabel(entry.outcome) }} + + {{ entry.setname || 'unknown set' }} + {{ formatTaken(entry.finishedat) }} +
    • +
+
+ +
+

Restore history

+

No restore runs recorded yet.

+
    +
  • + + {{ outcomeLabel(entry.outcome) }} + + {{ entry.setname || 'unknown set' }} + {{ formatTaken(entry.finishedat) }} +
    • +
+
+
+
@@ -192,6 +224,34 @@

Restore this backup?

{{ formatTaken(restoreTarget.takenat || restoreTarget.snapshotid) }}

+ + + +
@@ -570,6 +739,85 @@ onMounted(() => { .compat.ok { background: rgba(56, 161, 105, 0.15); color: #9ae6b4; } .compat.warn { background: rgba(214, 158, 46, 0.15); color: #f6e05e; } +.restore-progress { + display: flex; + align-items: center; + gap: 0.6rem; + padding: 0.7rem 0.8rem; + border-radius: 8px; + background: rgba(56, 161, 105, 0.15); + color: #9ae6b4; + font-size: 0.9rem; +} +.restore-progress.failed { background: rgba(197, 48, 48, 0.15); color: #feb2b2; } +.restore-phase { font-weight: 600; } +.restore-hint { + color: var(--text-light); + font-size: 0.8rem; + margin: 0.5rem 0 0 0; +} +.spinner { + width: 16px; + height: 16px; + border: 2px solid rgba(154, 230, 180, 0.35); + border-top-color: #9ae6b4; + border-radius: 50%; + animation: spin 0.8s linear infinite; + flex-shrink: 0; +} +@keyframes spin { + to { transform: rotate(360deg); } +} + +.history-cards { + display: flex; + flex-direction: column; + gap: 1rem; + margin-top: 1.5rem; +} +.history-card { + border: 1px solid var(--border); + border-radius: 8px; + background: var(--bg-card); + padding: 1rem 1.1rem; +} +.history-title { + margin: 0 0 0.75rem 0; + font-size: 0.95rem; +} +.history-list { + list-style: none; + margin: 0; + padding: 0; +} +.history-row { + display: flex; + align-items: center; + gap: 0.8rem; + padding: 0.5rem 0; + border-top: 1px solid var(--border); + font-size: 0.85rem; +} +.history-row:first-child { border-top: none; } +.history-outcome { + flex-shrink: 0; + padding: 0.1rem 0.5rem; + border-radius: 999px; + font-size: 0.72rem; + font-weight: 600; +} +.outcome-success { background: rgba(56, 161, 105, 0.15); color: #9ae6b4; } +.outcome-failed { background: rgba(197, 48, 48, 0.15); color: #feb2b2; } +.history-set { + flex: 1; + font-family: monospace; + color: var(--text); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.history-when { color: var(--text-light); flex-shrink: 0; } + .modal-actions { display: flex; justify-content: flex-end; diff --git a/scripts/backup/backup.sh b/scripts/backup/backup.sh index 8a22261f..c64bad82 100644 --- a/scripts/backup/backup.sh +++ b/scripts/backup/backup.sh @@ -23,9 +23,28 @@ STAMP="$(date -u +%Y-%m-%dT%H%M%SZ)" SETNAME="${SITE}-${STAMP}" SETDIR="${BACKUPS_ROOT}/${SETNAME}" +# shellcheck source=scripts/backup/status.sh +. /opt/backup/status.sh + log() { echo "[backup] $*"; } +# Record the run outcome on any exit so a failed backup is visible in the UI +# history, not just a silent gap. +BACKUP_DONE=false +_on_exit() { + code=$? + if [ "${BACKUP_DONE}" = true ]; then + status_write backup done "${SETNAME}" "Backup complete" + history_append backup success "${SETNAME}" "Backup complete" + else + status_write backup failed "${SETNAME}" "Backup failed (exit ${code})" + history_append backup failed "${SETNAME}" "Backup failed (exit ${code})" + fi +} +trap _on_exit EXIT + mkdir -p "${SETDIR}" +status_write backup database "${SETNAME}" "Dumping database" # 1. Database: whole-DB logical dump, consistent and non-blocking on InnoDB. log "dumping database ${MYSQL_DATABASE} from ${MYSQL_HOST}" @@ -53,6 +72,7 @@ python3 /opt/backup/manifest.py \ # 3. restic snapshot: db dump + instance tree + manifest, one point in time. # Skips gracefully when restic/repo are not configured (local-only sites). IS_CHECK_PASSED=false +status_write backup snapshot "${SETNAME}" "Snapshotting to backup store" if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then if ! restic snapshots >/dev/null 2>&1; then log "initializing restic repository at ${RESTIC_REPOSITORY}" @@ -95,4 +115,5 @@ python3 /opt/backup/manifest.py \ # Clear the request sentinel the UI drops when an operator clicks "Back up now". rm -f "${BACKUPS_ROOT}/.backup-requested" 2>/dev/null || true +BACKUP_DONE=true log "backup set ${SETNAME} complete" diff --git a/scripts/backup/restore.sh b/scripts/backup/restore.sh index 5d358e46..8adaf1c2 100644 --- a/scripts/backup/restore.sh +++ b/scripts/backup/restore.sh @@ -21,11 +21,32 @@ set -euo pipefail BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}" SETNAME="${1:-}" +# shellcheck source=scripts/backup/status.sh +. /opt/backup/status.sh + log() { echo "[restore] $*"; } + +# On any exit, stamp the terminal phase and append a history record so the UI +# reflects success or failure even when a step aborts early (set -e). +RESTORE_DONE=false +_on_exit() { + code=$? + if [ "${RESTORE_DONE}" = true ]; then + status_write restore done "${SETNAME}" "Restore complete" + history_append restore success "${SETNAME}" "Restore complete" + else + status_write restore failed "${SETNAME}" "Restore failed (exit ${code})" + history_append restore failed "${SETNAME}" "Restore failed (exit ${code})" + fi +} +trap _on_exit EXIT + die() { echo "[restore] ERROR: $*" >&2; exit 1; } [ -n "${SETNAME}" ] || die "usage: restore.sh " +status_write restore checking "${SETNAME}" "Checking schema compatibility" + SETDIR="${BACKUPS_ROOT}/${SETNAME}" DBDUMP="${SETDIR}/db.sql.gz" @@ -60,11 +81,13 @@ fi # Files first, then DB, so the two land together. if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then + status_write restore files "${SETNAME}" "Restoring instance files" log "restoring instance files from restic" restic restore latest --target / --include /app/instance \ || die "restic restore of instance files failed" fi +status_write restore database "${SETNAME}" "Loading database dump" log "loading database dump" gunzip -c "${DBDUMP}" | mysql \ --host="${MYSQL_HOST}" \ @@ -73,4 +96,5 @@ gunzip -c "${DBDUMP}" | mysql \ || die "database load failed" rm -f "${BACKUPS_ROOT}/.restore-requested" 2>/dev/null || true +RESTORE_DONE=true log "restore of ${SETNAME} complete" diff --git a/scripts/backup/status.sh b/scripts/backup/status.sh new file mode 100644 index 00000000..20fd54df --- /dev/null +++ b/scripts/backup/status.sh @@ -0,0 +1,100 @@ +# ShopDB backup/restore status + history helper. +# +# WHY this exists: the restore and backup runs happen in the watcher container, +# but the operator watches the UI, which only talks to the api. This helper is +# the bridge: it writes two kinds of state onto the shared /backups volume that +# the api can read. +# +# 1. A LIVE status file (.restore-status.json / .backup-status.json) rewritten +# at each phase so the UI can show an indeterminate spinner with a phase +# label (queued, restoring files, loading database, done, failed). +# 2. An append-only HISTORY file (history/restores.json / history/backups.json) +# that keeps the record of past runs so the UI can list them in a card. +# +# Both writes are atomic (write temp, rename) so the api never reads a half +# written file. History is bounded to the most recent HISTORY_MAX entries. +# +# This file is SOURCED, not executed. It exposes: +# status_write [message] +# history_append [message] +# +# kind is "restore" or "backup". ASCII only, no emojis - matches the style gate. + +STATUS_BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}" +HISTORY_MAX="${BACKUP_HISTORY_MAX:-50}" + +_status_now() { date -u +%Y-%m-%dT%H%M%SZ; } + +_status_json_escape() { + # Escape backslashes and double quotes for embedding in a JSON string. + printf '%s' "${1:-}" | sed 's/\\/\\\\/g; s/"/\\"/g' +} + +# status_write [message] +# Rewrites the live status file for this run kind. +status_write() { + kind="$1" + phase="$2" + setname="${3:-}" + message="${4:-}" + statusfile="${STATUS_BACKUPS_ROOT}/.${kind}-status.json" + tmpfile="${statusfile}.tmp.$$" + { + printf '{' + printf '"kind":"%s",' "$(_status_json_escape "${kind}")" + printf '"phase":"%s",' "$(_status_json_escape "${phase}")" + printf '"setname":"%s",' "$(_status_json_escape "${setname}")" + printf '"message":"%s",' "$(_status_json_escape "${message}")" + printf '"updatedat":"%s"' "$(_status_now)" + printf '}\n' + } > "${tmpfile}" 2>/dev/null || return 0 + mv -f "${tmpfile}" "${statusfile}" 2>/dev/null || rm -f "${tmpfile}" 2>/dev/null + return 0 +} + +# history_append [message] +# Appends a completed run record and trims to HISTORY_MAX newest entries. +history_append() { + kind="$1" + outcome="$2" + setname="${3:-}" + message="${4:-}" + historydir="${STATUS_BACKUPS_ROOT}/history" + historyfile="${historydir}/${kind}s.json" + tmpfile="${historyfile}.tmp.$$" + mkdir -p "${historydir}" 2>/dev/null || return 0 + + record="$(printf '{"setname":"%s","outcome":"%s","message":"%s","finishedat":"%s"}' \ + "$(_status_json_escape "${setname}")" \ + "$(_status_json_escape "${outcome}")" \ + "$(_status_json_escape "${message}")" \ + "$(_status_now)")" + + if command -v python3 >/dev/null 2>&1; then + HISTORY_FILE="${historyfile}" HISTORY_RECORD="${record}" \ + HISTORY_MAX="${HISTORY_MAX}" python3 - "${tmpfile}" <<'PY' 2>/dev/null || return 0 +import json, os, sys + +path = os.environ["HISTORY_FILE"] +record = json.loads(os.environ["HISTORY_RECORD"]) +limit = int(os.environ.get("HISTORY_MAX", "50")) +tmp = sys.argv[1] + +try: + with open(path, "r", encoding="utf-8") as handle: + entries = json.load(handle) + if not isinstance(entries, list): + entries = [] +except (OSError, ValueError): + entries = [] + +entries.append(record) +entries = entries[-limit:] + +with open(tmp, "w", encoding="utf-8") as handle: + json.dump(entries, handle) +os.replace(tmp, path) +PY + fi + return 0 +} diff --git a/scripts/backup/watch.sh b/scripts/backup/watch.sh index 18c53827..a0a0b975 100644 --- a/scripts/backup/watch.sh +++ b/scripts/backup/watch.sh @@ -19,6 +19,9 @@ INTERVAL="${BACKUP_POLL_SECONDS:-10}" log() { echo "[backup-watch] $*"; } +# shellcheck source=scripts/backup/status.sh +. /opt/backup/status.sh + mkdir -p "${BACKUPS_ROOT}" # The api container runs unprivileged and only drops sentinel files here; it # cannot write into a root-owned 0755 directory. Make the backups root @@ -29,6 +32,7 @@ log "watcher up; polling ${SENTINEL} and ${RESTORE_SENTINEL} every ${INTERVAL}s" while true; do if [ -f "${SENTINEL}" ]; then log "backup requested - running one-shot" + status_write backup queued "" "Backup queued" if /opt/backup/backup.sh; then log "backup completed" else @@ -45,6 +49,7 @@ while true; do # fails so a bad set cannot pin the watcher in a tight retry loop. setname="$(head -n1 "${RESTORE_SENTINEL}" 2>/dev/null | tr -d '\r\n')" log "restore requested - running restore for ${setname}" + status_write restore queued "${setname}" "Restore queued" if /opt/backup/restore.sh "${setname}"; then log "restore completed" else diff --git a/shopdb/core/api/shopdbbackups.py b/shopdb/core/api/shopdbbackups.py index 4081cc80..27624a0e 100644 --- a/shopdb/core/api/shopdbbackups.py +++ b/shopdb/core/api/shopdbbackups.py @@ -85,6 +85,51 @@ def _backups_root(): return os.environ.get(BACKUPS_ROOT_ENV, DEFAULT_BACKUPS_ROOT) +def _read_json(path): + """Load a JSON file, or None when it is absent or unreadable.""" + try: + with open(path, 'r', encoding='utf-8') as handle: + return json.load(handle) + except (OSError, ValueError): + return None + + +def _read_status(kind): + """Return the live status dict for a run kind, or None when idle.""" + root = _backups_root() + return _read_json(os.path.join(root, '.' + kind + '-status.json')) + + +def _read_history(kind): + """Return the recorded run history for a kind, newest first.""" + root = _backups_root() + entries = _read_json(os.path.join(root, 'history', kind + 's.json')) + if not isinstance(entries, list): + return [] + return list(reversed(entries)) + + +def _write_status(kind, phase, setname, message): + """Seed a live status file so the UI shows movement the instant a run is + queued, before the watcher has picked up the sentinel.""" + root = _backups_root() + stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z' + payload = { + 'kind': kind, + 'phase': phase, + 'setname': setname, + 'message': message, + 'updatedat': stamp, + } + try: + path = os.path.join(root, '.' + kind + '-status.json') + with open(path, 'w', encoding='utf-8') as handle: + json.dump(payload, handle) + except OSError: + pass + + + def _read_manifest(setdir): """Load a set's manifest.json, or None when absent/unreadable. @@ -146,6 +191,11 @@ def _is_compatible(manifest, current): return setrevs.issubset(currevs) +# Reserved entries under the backups root that are not backup sets: the run +# history store and any dotfile sentinels/status files the watcher drops. +_NON_SET_ENTRIES = ('history',) + + def _list_sets(): """Enumerate backup sets newest-first, each summarized for the list view.""" root = _backups_root() @@ -154,6 +204,8 @@ def _list_sets(): current = _current_schema_revisions() sets = [] for name in sorted(os.listdir(root), reverse=True): + if name in _NON_SET_ENTRIES or name.startswith('.'): + continue setdir = os.path.join(root, name) if not os.path.isdir(setdir): continue @@ -235,6 +287,7 @@ def trigger_backup(): return error_response(ErrorCodes.INTERNAL_ERROR, 'Could not queue a backup: ' + str(error), http_code=500) + _write_status('backup', 'queued', '', 'Backup queued') AuditLog.log('requested', 'Backup', entityname=stamp) return success_response( data={'isrunning': False, 'isqueued': True, 'requestedat': stamp}, @@ -276,6 +329,8 @@ def restore_backup(snapshotid): return error_response(ErrorCodes.INTERNAL_ERROR, 'Could not queue the restore: ' + str(error), http_code=500) + _write_status('restore', 'queued', os.path.basename(snapshotid), + 'Restore queued') AuditLog.log('restore-requested', 'Backup', entityname=os.path.basename(snapshotid)) return success_response( @@ -283,6 +338,35 @@ def restore_backup(snapshotid): message='Restore queued and validated against the live schema.') +@shopdbbackups_bp.route('/status', methods=['GET']) +@require_role('admin') +def get_status(): + """Return the live status of the most recent backup and restore runs. + + The watcher writes these files as it moves through each phase; the UI polls + this endpoint to drive the restore progress spinner and phase label. Either + value is None when that kind of run has never been requested. + """ + return success_response(data={ + 'backup': _read_status('backup'), + 'restore': _read_status('restore'), + }) + + +@shopdbbackups_bp.route('/history', methods=['GET']) +@require_role('admin') +def get_history(): + """Return the recorded history of backup and restore runs, newest first. + + These are operational run logs written by the backup container onto the + shared volume, not the audit trail (AuditLog records the requested intent). + """ + return success_response(data={ + 'backups': _read_history('backup'), + 'restores': _read_history('restore'), + }) + + @shopdbbackups_bp.route('/settings', methods=['GET']) @require_role('admin') def get_backup_settings():