Add opt-in backup overlay with third-container restic store

Backups are per-site and not part of the base stack, so they ship as a separate compose overlay with an isolated restic REST server the app can only reach through the backup utility (no shell or db creds). The sidecar image trusts the corporate proxy root CA at build time so its Oracle yum, github, and PyPI fetches succeed behind TLS interception; shell scripts are pinned to LF via .gitattributes so the container entrypoints do not die on a CRLF shebang.

scripts/backup/.gitattributes

@@ -0,0 +1,4 @@
+# Keep shell scripts LF-only so they run inside Linux containers regardless of
+# the host checkout. A CRLF shebang makes the kernel look for "bash\r" and the
+# container dies with: env: 'bash\r': No such file or directory.
+*.sh text eol=lf

scripts/backup/Dockerfile.backup

@@ -0,0 +1,38 @@
+# Backup sidecar image: mysql:8.4 (for a version-matched mysqldump) plus restic
+# (for deduped, encrypted, destination-agnostic file snapshots) and a small
+# Python for the manifest writer.
+#
+# WHY mysql:8.4 as the base: the dump tool must match the server major/minor so
+# the SQL it emits restores cleanly onto the same engine. Reusing the exact
+# image the db service runs removes version drift as a failure mode.
+FROM mysql:8.4
+
+# Track 1 local build unblock: trust the corporate proxy (Zscaler/GE) root CA so
+# the build-time fetches below (Oracle yum, github.com, PyPI) succeed behind TLS
+# interception. This is a local convenience only; the shipping build vendors
+# these deps instead of fetching them (pinned restic + SHA256, vendored
+# bzip2/python3/alembic or a base image that carries them) so the release image
+# needs no outbound calls and no embedded cert. The cert is gitignored (*.pem)
+# and must be staged at scripts/backup/zscaler-root-ca.pem before building.
+COPY scripts/backup/zscaler-root-ca.pem /etc/pki/ca-trust/source/anchors/zscaler-root-ca.pem
+RUN update-ca-trust
+
+# restic from the distro is often stale; pull the pinned release binary so the
+# snapshot format and retention flags are predictable across sites.
+ARG RESTIC_VERSION=0.17.3
+RUN microdnf install -y bzip2 python3 python3-pip ca-certificates \
+	&& curl -fsSL -o /tmp/restic.bz2 \
+		"https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_amd64.bz2" \
+	&& bunzip2 /tmp/restic.bz2 \
+	&& install -m 0755 /tmp/restic /usr/local/bin/restic \
+	&& rm -f /tmp/restic \
+	&& microdnf clean all
+
+# alembic so the manifest writer can read the current schema head. pip does not
+# read the system trust store, so point it at the extracted system CA bundle
+# (now including the proxy root) to get through TLS interception.
+RUN python3 -m pip install --no-cache-dir \
+	--cert /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem \
+	alembic
+
+ENTRYPOINT ["/opt/backup/backup.sh"]

scripts/backup/backup.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# ShopDB backup one-shot.
+#
+# Produces one recoverable set per run and stores it two ways: a plain
+# db.sql.gz + manifest.json on the local /backups volume for a fast, repo-free
+# restore, and a restic snapshot (db dump + instance tree + manifest) at
+# RESTIC_REPOSITORY for a deduped, encrypted, offsite copy.
+#
+# Sequence:
+#   1. mysqldump --single-transaction (InnoDB-consistent, non-blocking)
+#   2. manifest.py stamps core+plugin Alembic revisions + image tag
+#   3. restic backup the dump + /app/instance + manifest as ONE snapshot
+#   4. restic forget --keep-daily 5 --keep-weekly 4 --keep-monthly 6 --prune
+#   5. restic check - fail loudly if the repo is unhealthy, so a bad repo is
+#      caught on backup night, not restore night.
+#
+# ASCII only, no emojis - matches the repo style gate.
+set -euo pipefail
+
+BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}"
+SITE="${SHOPDB_SITE:-shopdb}"
+STAMP="$(date -u +%Y-%m-%dT%H%M%SZ)"
+SETNAME="${SITE}-${STAMP}"
+SETDIR="${BACKUPS_ROOT}/${SETNAME}"
+
+log() { echo "[backup] $*"; }
+
+mkdir -p "${SETDIR}"
+
+# 1. Database: whole-DB logical dump, consistent and non-blocking on InnoDB.
+log "dumping database ${MYSQL_DATABASE} from ${MYSQL_HOST}"
+mysqldump \
+	--host="${MYSQL_HOST}" \
+	--user="${MYSQL_USER}" \
+	--password="${MYSQL_PASSWORD}" \
+	--single-transaction \
+	--routines \
+	--triggers \
+	--databases "${MYSQL_DATABASE}" \
+	| gzip -c > "${SETDIR}/db.sql.gz"
+log "database dump written ($(du -h "${SETDIR}/db.sql.gz" | cut -f1))"
+
+# 2. Manifest: the schema stamp that makes a restore a checked operation.
+log "writing manifest"
+python3 /opt/backup/manifest.py \
+	--out "${SETDIR}/manifest.json" \
+	--takenat "${STAMP}" \
+	--site "${SITE}" \
+	--imagetag "${SHOPDB_IMAGE_TAG:-source}" \
+	--migrations /app/migrations \
+	--plugins /app/plugins
+
+# 3. restic snapshot: db dump + instance tree + manifest, one point in time.
+#    Skips gracefully when restic/repo are not configured (local-only sites).
+IS_CHECK_PASSED=false
+if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then
+	if ! restic snapshots >/dev/null 2>&1; then
+		log "initializing restic repository at ${RESTIC_REPOSITORY}"
+		restic init
+	fi
+	log "restic backup (db dump + instance tree + manifest)"
+	restic backup \
+		--tag "${SITE}" \
+		--tag "${SETNAME}" \
+		"${SETDIR}/db.sql.gz" \
+		"${SETDIR}/manifest.json" \
+		/app/instance
+
+	# 4. Retention: any day in the last 5, plus weekly/monthly tail.
+	log "pruning per retention policy"
+	restic forget \
+		--tag "${SITE}" \
+		--keep-daily 5 \
+		--keep-weekly 4 \
+		--keep-monthly 6 \
+		--prune
+
+	# 5. Integrity: learn about corruption now, not at restore time.
+	log "verifying repository integrity"
+	if restic check; then
+		IS_CHECK_PASSED=true
+		log "restic check passed"
+	else
+		log "restic check FAILED - repository may be corrupt"
+	fi
+else
+	log "restic not configured - local db.sql.gz + manifest only"
+fi
+
+# Record the verification result into the manifest so the UI can badge the set.
+python3 /opt/backup/manifest.py \
+	--amend "${SETDIR}/manifest.json" \
+	--ischeckpassed "${IS_CHECK_PASSED}"
+
+# Clear the request sentinel the UI drops when an operator clicks "Back up now".
+rm -f "${BACKUPS_ROOT}/.backup-requested" 2>/dev/null || true
+
+log "backup set ${SETNAME} complete"

scripts/backup/manifest.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Backup manifest writer.
+
+The manifest is the stamp that turns "we have dumps" into "we can actually
+restore." It records, at capture time:
+  - takenat   : ISO-8601 UTC timestamp of the run
+  - site      : the site tag
+  - imagetag  : the app image tag that produced the data
+  - schemarev : the core Alembic head revision(s) the running code is at, plus
+                each plugin chain's heads - the versions a restore target must
+                be at least, or restored rows reference columns the live schema
+                has not applied yet.
+
+Two modes:
+  --out ...      write a fresh manifest for a new set
+  --amend ...    patch an existing manifest (used to record ischeckpassed after
+                 restic check runs)
+
+ASCII only, no emojis - matches the repo style gate.
+"""
+
+import argparse
+import json
+import os
+import sys
+
+
+def _heads_for(chain_dir):
+    """Return the Alembic head revision id(s) under a versions directory.
+
+    We read the revision graph the cheap way: a head is a revision no other
+    revision names as its down_revision. Done by text scan so this stays
+    dependency-light and works for both the core chain and each plugin chain.
+    """
+    versions = os.path.join(chain_dir, 'versions')
+    if not os.path.isdir(versions):
+        return []
+    revisions = {}
+    down_refs = set()
+    for name in os.listdir(versions):
+        if not name.endswith('.py'):
+            continue
+        path = os.path.join(versions, name)
+        revid = None
+        downid = None
+        try:
+            with open(path, 'r', encoding='utf-8') as handle:
+                for line in handle:
+                    stripped = line.strip()
+                    if stripped.startswith('revision =') and revid is None:
+                        revid = stripped.split('=', 1)[1].strip().strip('"\'')
+                    elif stripped.startswith('down_revision ='):
+                        value = stripped.split('=', 1)[1].strip()
+                        downid = None if value in ('None', 'null') else value.strip('"\'')
+        except OSError:
+            continue
+        if revid:
+            revisions[revid] = downid
+            if downid:
+                down_refs.add(downid)
+    return sorted(rev for rev in revisions if rev not in down_refs)
+
+
+def _collect_schema(migrations_dir, plugins_dir):
+    """Core heads plus each plugin's chain heads, keyed for compatibility checks."""
+    schema = {'core': _heads_for(migrations_dir), 'plugins': {}}
+    if plugins_dir and os.path.isdir(plugins_dir):
+        for name in sorted(os.listdir(plugins_dir)):
+            chain = os.path.join(plugins_dir, name, 'migrations')
+            heads = _heads_for(chain)
+            if heads:
+                schema['plugins'][name] = heads
+    return schema
+
+
+def write_manifest(args):
+    manifest = {
+        'takenat': args.takenat,
+        'site': args.site,
+        'imagetag': args.imagetag,
+        'schemarev': _collect_schema(args.migrations, args.plugins),
+        'ischeckpassed': False,
+    }
+    with open(args.out, 'w', encoding='utf-8') as handle:
+        json.dump(manifest, handle, indent=2, sort_keys=True)
+    return 0
+
+
+def amend_manifest(args):
+    try:
+        with open(args.amend, 'r', encoding='utf-8') as handle:
+            manifest = json.load(handle)
+    except (OSError, ValueError):
+        manifest = {}
+    if args.ischeckpassed is not None:
+        manifest['ischeckpassed'] = args.ischeckpassed.lower() == 'true'
+    with open(args.amend, 'w', encoding='utf-8') as handle:
+        json.dump(manifest, handle, indent=2, sort_keys=True)
+    return 0
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description='ShopDB backup manifest writer')
+    parser.add_argument('--out')
+    parser.add_argument('--amend')
+    parser.add_argument('--takenat', default='')
+    parser.add_argument('--site', default='shopdb')
+    parser.add_argument('--imagetag', default='source')
+    parser.add_argument('--migrations', default='/app/migrations')
+    parser.add_argument('--plugins', default='/app/plugins')
+    parser.add_argument('--ischeckpassed')
+    args = parser.parse_args(argv)
+
+    if args.amend:
+        return amend_manifest(args)
+    if args.out:
+        return write_manifest(args)
+    parser.error('one of --out or --amend is required')
+
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/backup/restore.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# ShopDB restore.
+#
+# The DANGEROUS half of the pair, so it is deliberately guarded. A restore is
+# only safe onto code whose schema is at least the set's - otherwise restored
+# rows reference columns the live schema has not applied yet and every query on
+# those tables 500s. This script refuses an incompatible restore rather than
+# leaving the site half-broken.
+#
+# Usage (from the backup container):
+#   docker compose -f docker-compose.yml -f docker-compose.backup.yml \
+#     run --rm --entrypoint /opt/backup/restore.sh backup <snapshot-set-name>
+#
+# It does NOT roll back code - that is git's job (git checkout <tag> && build).
+# Restore code FIRST, bring the schema up (flask db upgrade + plugin
+# upgrade-all), THEN run this to load the data.
+#
+# ASCII only, no emojis - matches the repo style gate.
+set -euo pipefail
+
+BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}"
+SETNAME="${1:-}"
+
+log() { echo "[restore] $*"; }
+die() { echo "[restore] ERROR: $*" >&2; exit 1; }
+
+[ -n "${SETNAME}" ] || die "usage: restore.sh <snapshot-set-name>"
+
+SETDIR="${BACKUPS_ROOT}/${SETNAME}"
+DBDUMP="${SETDIR}/db.sql.gz"
+
+# Prefer the local fast path; fall back to pulling the set out of restic.
+if [ ! -f "${DBDUMP}" ]; then
+	log "local dump not found; restoring set from restic"
+	command -v restic >/dev/null 2>&1 || die "restic not available and no local dump"
+	restic restore latest --target / --include "${SETDIR}" \
+		|| die "restic restore of ${SETNAME} failed"
+fi
+[ -f "${DBDUMP}" ] || die "no db.sql.gz for set ${SETNAME}"
+
+# Compatibility gate: the running schema must cover the set's stamped heads.
+log "checking schema compatibility"
+CURRENT_HEADS="$(python3 /opt/backup/manifest.py \
+	--out /tmp/current-manifest.json \
+	--migrations /app/migrations --plugins /app/plugins \
+	>/dev/null 2>&1 && python3 -c \
+	'import json;print(",".join(sorted(json.load(open("/tmp/current-manifest.json"))["schemarev"]["core"])))')"
+SET_HEADS="$(python3 -c \
+	'import json,sys;print(",".join(sorted(json.load(open(sys.argv[1]))["schemarev"]["core"])))' \
+	"${SETDIR}/manifest.json" 2>/dev/null || echo "")"
+
+if [ -z "${SET_HEADS}" ]; then
+	die "set ${SETNAME} has no readable manifest - refusing an unverified restore"
+fi
+if [ "${CURRENT_HEADS}" != "${SET_HEADS}" ]; then
+	log "current heads : ${CURRENT_HEADS}"
+	log "set heads     : ${SET_HEADS}"
+	die "schema mismatch - bring code to the set's revision, run 'flask db upgrade' and 'flask plugin upgrade-all', then retry"
+fi
+
+# Files first, then DB, so the two land together.
+if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then
+	log "restoring instance files from restic"
+	restic restore latest --target / --include /app/instance \
+		|| die "restic restore of instance files failed"
+fi
+
+log "loading database dump"
+gunzip -c "${DBDUMP}" | mysql \
+	--host="${MYSQL_HOST}" \
+	--user="${MYSQL_USER}" \
+	--password="${MYSQL_PASSWORD}" \
+	|| die "database load failed"
+
+rm -f "${BACKUPS_ROOT}/.restore-requested" 2>/dev/null || true
+log "restore of ${SETNAME} complete"

scripts/backup/watch.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# ShopDB backup WATCHER - the resident half of the in-app backup utility.
+#
+# WHY this exists: the api container cannot run docker, so when an operator
+# clicks "Back up now" the api only drops a sentinel file (.backup-requested)
+# on the shared backups volume. This watcher is the thing that actuates that
+# request: it polls for the sentinel and runs the one-shot backup.sh, which
+# ships a restic snapshot to the backup-store container and clears the sentinel
+# on success. That turns the UI button into a real, self-serve backup without
+# giving the api container any docker or host privileges.
+#
+# ASCII only, no emojis - matches the repo style gate.
+set -uo pipefail
+
+BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}"
+SENTINEL="${BACKUPS_ROOT}/.backup-requested"
+INTERVAL="${BACKUP_POLL_SECONDS:-10}"
+
+log() { echo "[backup-watch] $*"; }
+
+mkdir -p "${BACKUPS_ROOT}"
+log "watcher up; polling ${SENTINEL} every ${INTERVAL}s"
+
+while true; do
+	if [ -f "${SENTINEL}" ]; then
+		log "backup requested - running one-shot"
+		if /opt/backup/backup.sh; then
+			log "backup completed"
+		else
+			# backup.sh only clears the sentinel on success, so a failed run
+			# would otherwise loop forever. Clear it here and wait for the next
+			# explicit request rather than hammering a broken backup.
+			log "backup FAILED - clearing request, awaiting next"
+			rm -f "${SENTINEL}" 2>/dev/null || true
+		fi
+	fi
+	sleep "${INTERVAL}"
+done