it is risen - add restore progress and backup/restore history

The watcher scripts now write live status and append-only run history onto the shared volume; the api exposes them via /shopdbbackups/status and /history. The restore modal shows an indeterminate spinner with phase labels, and two cards list backup and restore run records. Timestamps render in UTC to match the Zulu-stamped set ids, the history directory is excluded from the set list, and a stale-status guard stops a previous run's done phase from collapsing the spinner on the next restore.

scripts/backup/backup.sh

@@ -23,9 +23,28 @@ STAMP="$(date -u +%Y-%m-%dT%H%M%SZ)"
 SETNAME="${SITE}-${STAMP}"
 SETDIR="${BACKUPS_ROOT}/${SETNAME}"
 
+# shellcheck source=scripts/backup/status.sh
+. /opt/backup/status.sh
+
 log() { echo "[backup] $*"; }
 
+# Record the run outcome on any exit so a failed backup is visible in the UI
+# history, not just a silent gap.
+BACKUP_DONE=false
+_on_exit() {
+	code=$?
+	if [ "${BACKUP_DONE}" = true ]; then
+		status_write backup done "${SETNAME}" "Backup complete"
+		history_append backup success "${SETNAME}" "Backup complete"
+	else
+		status_write backup failed "${SETNAME}" "Backup failed (exit ${code})"
+		history_append backup failed "${SETNAME}" "Backup failed (exit ${code})"
+	fi
+}
+trap _on_exit EXIT
+
 mkdir -p "${SETDIR}"
+status_write backup database "${SETNAME}" "Dumping database"
 
 # 1. Database: whole-DB logical dump, consistent and non-blocking on InnoDB.
 log "dumping database ${MYSQL_DATABASE} from ${MYSQL_HOST}"
@@ -53,6 +72,7 @@ python3 /opt/backup/manifest.py \
 # 3. restic snapshot: db dump + instance tree + manifest, one point in time.
 #    Skips gracefully when restic/repo are not configured (local-only sites).
 IS_CHECK_PASSED=false
+status_write backup snapshot "${SETNAME}" "Snapshotting to backup store"
 if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then
 	if ! restic snapshots >/dev/null 2>&1; then
 		log "initializing restic repository at ${RESTIC_REPOSITORY}"
@@ -95,4 +115,5 @@ python3 /opt/backup/manifest.py \
 # Clear the request sentinel the UI drops when an operator clicks "Back up now".
 rm -f "${BACKUPS_ROOT}/.backup-requested" 2>/dev/null || true
 
+BACKUP_DONE=true
 log "backup set ${SETNAME} complete"

scripts/backup/restore.sh

@@ -21,11 +21,32 @@ set -euo pipefail
 BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}"
 SETNAME="${1:-}"
 
+# shellcheck source=scripts/backup/status.sh
+. /opt/backup/status.sh
+
 log() { echo "[restore] $*"; }
+
+# On any exit, stamp the terminal phase and append a history record so the UI
+# reflects success or failure even when a step aborts early (set -e).
+RESTORE_DONE=false
+_on_exit() {
+	code=$?
+	if [ "${RESTORE_DONE}" = true ]; then
+		status_write restore done "${SETNAME}" "Restore complete"
+		history_append restore success "${SETNAME}" "Restore complete"
+	else
+		status_write restore failed "${SETNAME}" "Restore failed (exit ${code})"
+		history_append restore failed "${SETNAME}" "Restore failed (exit ${code})"
+	fi
+}
+trap _on_exit EXIT
+
 die() { echo "[restore] ERROR: $*" >&2; exit 1; }
 
 [ -n "${SETNAME}" ] || die "usage: restore.sh <snapshot-set-name>"
 
+status_write restore checking "${SETNAME}" "Checking schema compatibility"
+
 SETDIR="${BACKUPS_ROOT}/${SETNAME}"
 DBDUMP="${SETDIR}/db.sql.gz"
 
@@ -60,11 +81,13 @@ fi
 
 # Files first, then DB, so the two land together.
 if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then
+	status_write restore files "${SETNAME}" "Restoring instance files"
 	log "restoring instance files from restic"
 	restic restore latest --target / --include /app/instance \
 		|| die "restic restore of instance files failed"
 fi
 
+status_write restore database "${SETNAME}" "Loading database dump"
 log "loading database dump"
 gunzip -c "${DBDUMP}" | mysql \
 	--host="${MYSQL_HOST}" \
@@ -73,4 +96,5 @@ gunzip -c "${DBDUMP}" | mysql \
 	|| die "database load failed"
 
 rm -f "${BACKUPS_ROOT}/.restore-requested" 2>/dev/null || true
+RESTORE_DONE=true
 log "restore of ${SETNAME} complete"

scripts/backup/status.sh

@@ -0,0 +1,100 @@
+# ShopDB backup/restore status + history helper.
+#
+# WHY this exists: the restore and backup runs happen in the watcher container,
+# but the operator watches the UI, which only talks to the api. This helper is
+# the bridge: it writes two kinds of state onto the shared /backups volume that
+# the api can read.
+#
+#   1. A LIVE status file (.restore-status.json / .backup-status.json) rewritten
+#      at each phase so the UI can show an indeterminate spinner with a phase
+#      label (queued, restoring files, loading database, done, failed).
+#   2. An append-only HISTORY file (history/restores.json / history/backups.json)
+#      that keeps the record of past runs so the UI can list them in a card.
+#
+# Both writes are atomic (write temp, rename) so the api never reads a half
+# written file. History is bounded to the most recent HISTORY_MAX entries.
+#
+# This file is SOURCED, not executed. It exposes:
+#   status_write   <kind> <phase> <setname> [message]
+#   history_append <kind> <outcome> <setname> [message]
+#
+# kind is "restore" or "backup". ASCII only, no emojis - matches the style gate.
+
+STATUS_BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}"
+HISTORY_MAX="${BACKUP_HISTORY_MAX:-50}"
+
+_status_now() { date -u +%Y-%m-%dT%H%M%SZ; }
+
+_status_json_escape() {
+	# Escape backslashes and double quotes for embedding in a JSON string.
+	printf '%s' "${1:-}" | sed 's/\\/\\\\/g; s/"/\\"/g'
+}
+
+# status_write <kind> <phase> <setname> [message]
+# Rewrites the live status file for this run kind.
+status_write() {
+	kind="$1"
+	phase="$2"
+	setname="${3:-}"
+	message="${4:-}"
+	statusfile="${STATUS_BACKUPS_ROOT}/.${kind}-status.json"
+	tmpfile="${statusfile}.tmp.$$"
+	{
+		printf '{'
+		printf '"kind":"%s",' "$(_status_json_escape "${kind}")"
+		printf '"phase":"%s",' "$(_status_json_escape "${phase}")"
+		printf '"setname":"%s",' "$(_status_json_escape "${setname}")"
+		printf '"message":"%s",' "$(_status_json_escape "${message}")"
+		printf '"updatedat":"%s"' "$(_status_now)"
+		printf '}\n'
+	} > "${tmpfile}" 2>/dev/null || return 0
+	mv -f "${tmpfile}" "${statusfile}" 2>/dev/null || rm -f "${tmpfile}" 2>/dev/null
+	return 0
+}
+
+# history_append <kind> <outcome> <setname> [message]
+# Appends a completed run record and trims to HISTORY_MAX newest entries.
+history_append() {
+	kind="$1"
+	outcome="$2"
+	setname="${3:-}"
+	message="${4:-}"
+	historydir="${STATUS_BACKUPS_ROOT}/history"
+	historyfile="${historydir}/${kind}s.json"
+	tmpfile="${historyfile}.tmp.$$"
+	mkdir -p "${historydir}" 2>/dev/null || return 0
+
+	record="$(printf '{"setname":"%s","outcome":"%s","message":"%s","finishedat":"%s"}' \
+		"$(_status_json_escape "${setname}")" \
+		"$(_status_json_escape "${outcome}")" \
+		"$(_status_json_escape "${message}")" \
+		"$(_status_now)")"
+
+	if command -v python3 >/dev/null 2>&1; then
+		HISTORY_FILE="${historyfile}" HISTORY_RECORD="${record}" \
+			HISTORY_MAX="${HISTORY_MAX}" python3 - "${tmpfile}" <<'PY' 2>/dev/null || return 0
+import json, os, sys
+
+path = os.environ["HISTORY_FILE"]
+record = json.loads(os.environ["HISTORY_RECORD"])
+limit = int(os.environ.get("HISTORY_MAX", "50"))
+tmp = sys.argv[1]
+
+try:
+	with open(path, "r", encoding="utf-8") as handle:
+		entries = json.load(handle)
+	if not isinstance(entries, list):
+		entries = []
+except (OSError, ValueError):
+	entries = []
+
+entries.append(record)
+entries = entries[-limit:]
+
+with open(tmp, "w", encoding="utf-8") as handle:
+	json.dump(entries, handle)
+os.replace(tmp, path)
+PY
+	fi
+	return 0
+}

scripts/backup/watch.sh

@@ -19,6 +19,9 @@ INTERVAL="${BACKUP_POLL_SECONDS:-10}"
 
 log() { echo "[backup-watch] $*"; }
 
+# shellcheck source=scripts/backup/status.sh
+. /opt/backup/status.sh
+
 mkdir -p "${BACKUPS_ROOT}"
 # The api container runs unprivileged and only drops sentinel files here; it
 # cannot write into a root-owned 0755 directory. Make the backups root
@@ -29,6 +32,7 @@ log "watcher up; polling ${SENTINEL} and ${RESTORE_SENTINEL} every ${INTERVAL}s"
 while true; do
 	if [ -f "${SENTINEL}" ]; then
 		log "backup requested - running one-shot"
+		status_write backup queued "" "Backup queued"
 		if /opt/backup/backup.sh; then
 			log "backup completed"
 		else
@@ -45,6 +49,7 @@ while true; do
 		# fails so a bad set cannot pin the watcher in a tight retry loop.
 		setname="$(head -n1 "${RESTORE_SENTINEL}" 2>/dev/null | tr -d '\r\n')"
 		log "restore requested - running restore for ${setname}"
+		status_write restore queued "${setname}" "Restore queued"
 		if /opt/backup/restore.sh "${setname}"; then
 			log "restore completed"
 		else