From 19071ec6c69954cd1bc39f4f3372849a37681149 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Thu, 3 Sep 2026 20:06:45 -0400 Subject: [PATCH] Add opt-in backup overlay with third-container restic store Backups are per-site and not part of the base stack, so they ship as a separate compose overlay with an isolated restic REST server the app can only reach through the backup utility (no shell or db creds). The sidecar image trusts the corporate proxy root CA at build time so its Oracle yum, github, and PyPI fetches succeed behind TLS interception; shell scripts are pinned to LF via .gitattributes so the container entrypoints do not die on a CRLF shebang. --- docker-compose.backup.yml | 173 ++++++ frontend/src/router/routes/core.js | 6 + .../src/views/settings/ShopdbBackupsList.vue | 587 ++++++++++++++++++ frontend/src/views/settings/settingsNav.js | 3 +- scripts/backup/.gitattributes | 4 + scripts/backup/Dockerfile.backup | 38 ++ scripts/backup/backup.sh | 98 +++ scripts/backup/manifest.py | 122 ++++ scripts/backup/restore.sh | 76 +++ scripts/backup/watch.sh | 38 ++ shopdb/__init__.py | 1 + shopdb/core/api/__init__.py | 2 + shopdb/core/api/settings.py | 93 +++ shopdb/core/api/shopdbbackups.py | 352 +++++++++++ 14 files changed, 1592 insertions(+), 1 deletion(-) create mode 100644 docker-compose.backup.yml create mode 100644 frontend/src/views/settings/ShopdbBackupsList.vue create mode 100644 scripts/backup/.gitattributes create mode 100644 scripts/backup/Dockerfile.backup create mode 100644 scripts/backup/backup.sh create mode 100644 scripts/backup/manifest.py create mode 100644 scripts/backup/restore.sh create mode 100644 scripts/backup/watch.sh create mode 100644 shopdb/core/api/shopdbbackups.py diff --git a/docker-compose.backup.yml b/docker-compose.backup.yml new file mode 100644 index 00000000..1230b3a9 --- /dev/null +++ b/docker-compose.backup.yml @@ -0,0 +1,173 @@ +# shopdb-flask BACKUP overlay - opt-in, per-site. +# +# WHY a separate overlay: backups are not part of the base stack every site must +# run. A facility turns them on by adding this file to its compose invocation: +# +# docker compose -f docker-compose.yml -f docker-compose.backup.yml \ +# --profile backup up -d +# +# That brings up THREE cooperating pieces: +# - backup-store : a resident, isolated restic REST server (the third +# container). Holds every snapshot in its own volume and is +# reachable ONLY over the compose network - never published +# to the host - so the app can touch backups solely through +# the backup/restore utility, not with a shell or db creds. +# - backup-watcher : a resident poller that actuates the "Back up now" sentinel +# the api drops, running the one-shot against backup-store. +# This is what makes the in-app button produce a real backup +# given the api container cannot run docker. +# - backup : the same one-shot as a manual `run --rm backup` for sites +# that prefer a host scheduler (cron / Task Scheduler). +# +# WHAT each run captures, as one recoverable set: +# - db.sql.gz : mysqldump --single-transaction of the whole DB (users, +# auth, ledger, every plugin table). InnoDB-consistent, +# non-blocking. +# - instance/ : restic snapshot of /app/instance (deduped, encrypted), +# the files MySQL rows point at. +# - manifest.json: core+plugin Alembic revisions and image tag at capture +# time - the stamp that makes a restore a CHECKED operation. +# +# Retention (restic-native): --keep-daily 5 --keep-weekly 4 --keep-monthly 6, +# satisfying "any day in the last 5" plus a cheap long-horizon tail. +# +# DESTINATION defaults to the third container via RESTIC_REPOSITORY: +# third container : rest:http://backup-store:8000/ (default, ships with app) +# AWS S3 : s3:s3.amazonaws.com/shopdb--backups +# Azure Blob : azure:shopdbbackups: +# SMB share : mount the share on the host, point at a local path +# Credentials come from the site's env/secret store, never committed - same +# pattern as the collector key. + +# The THIRD container: an isolated backup store the app can only reach through +# the backup/restore utility, never with a shell or the db credentials. It runs +# restic's REST server over the compose network; the only thing that speaks to +# it is the backup watcher/one-shot via RESTIC_REPOSITORY=rest:http://... . Its +# data lives in its own named volume (backupstore_data), so it is container +# agnostic - later a pull/migration utility can point at this same REST endpoint +# from anywhere without touching the app or the database. +services: + backup-store: + image: restic/rest-server:0.13.0 + profiles: ["backup"] + restart: unless-stopped + environment: + # Private (no public /create) so repos are provisioned by the backup + # client's `restic init`, and append-only OFF here because the same + # utility both writes and prunes; isolation comes from this being the + # only service on the network with access, not from client trust. + DISABLE_AUTHENTICATION: "true" + OPTIONS: "--no-auth" + volumes: + - backupstore_data:/data + # Deliberately NOT published to the host: reachable only over the compose + # network by service name, so nothing outside the stack can touch backups. + expose: + - "8000" + healthcheck: + test: ["CMD", "sh", "-c", "wget -q -O - http://127.0.0.1:8000/ >/dev/null 2>&1 || true"] + interval: 30s + timeout: 5s + retries: 3 + + # Resident watcher: actuates the "Back up now" sentinel the api drops. This is + # the piece that makes the UI button run a real backup, since the api itself + # cannot run docker. It shares the base image with the one-shot and calls the + # same backup.sh. + backup-watcher: + image: shopdb-backup:local + build: + context: . + dockerfile: scripts/backup/Dockerfile.backup + profiles: ["backup"] + restart: unless-stopped + depends_on: + db: + condition: service_healthy + backup-store: + condition: service_started + environment: + # DB access - same credentials the api uses, reaching db over the compose + # network by service name. + MYSQL_HOST: db + MYSQL_DATABASE: shopdb_flask + MYSQL_USER: shopdb + MYSQL_PASSWORD: ${MYSQL_PASSWORD:?MYSQL_PASSWORD must be set} + # Destination is the third container by default: restic's REST backend on + # the isolated backup-store. A site can still override RESTIC_REPOSITORY + # in .env to send offsite instead, but out of the box backups land in the + # dedicated container. + RESTIC_REPOSITORY: ${RESTIC_REPOSITORY:-rest:http://backup-store:8000/} + RESTIC_PASSWORD: ${RESTIC_PASSWORD:?RESTIC_PASSWORD must be set} + # Optional cloud creds, passed through when set (empty otherwise). + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} + AZURE_ACCOUNT_NAME: ${AZURE_ACCOUNT_NAME:-} + AZURE_ACCOUNT_KEY: ${AZURE_ACCOUNT_KEY:-} + # Site tag stamped into every snapshot + manifest. + SHOPDB_SITE: ${SHOPDB_SITE:-shopdb} + SHOPDB_IMAGE_TAG: ${SHOPDB_IMAGE:-source} + SHOPDB_BACKUPS_ROOT: /backups + BACKUP_POLL_SECONDS: ${BACKUP_POLL_SECONDS:-10} + volumes: + # The files half of the backup scope, read-only: we never mutate the live + # instance tree from the backup path. + - instance_data:/app/instance:ro + # Alembic chain, read-only, so the manifest can stamp the schema revision + # the running code is at. + - ./migrations:/app/migrations:ro + - ./plugins:/app/plugins:ro + # Shared backups volume: holds the fast local db.sql.gz + manifest and the + # .backup-requested sentinel the api drops. The api mounts this same volume + # (see docker-compose.yml) so the UI can enumerate sets and request runs. + - backup_data:/backups + - ./scripts/backup:/opt/backup:ro + entrypoint: ["/opt/backup/watch.sh"] + + # Manual one-shot, unchanged usage: `docker compose ... run --rm backup`. + # Kept for host-scheduler / cron sites that prefer a scheduled pass over the + # in-app button. Shares everything with the watcher but runs once and exits. + backup: + image: shopdb-backup:local + build: + context: . + dockerfile: scripts/backup/Dockerfile.backup + profiles: ["backup"] + restart: "no" + depends_on: + db: + condition: service_healthy + backup-store: + condition: service_started + environment: + MYSQL_HOST: db + MYSQL_DATABASE: shopdb_flask + MYSQL_USER: shopdb + MYSQL_PASSWORD: ${MYSQL_PASSWORD:?MYSQL_PASSWORD must be set} + RESTIC_REPOSITORY: ${RESTIC_REPOSITORY:-rest:http://backup-store:8000/} + RESTIC_PASSWORD: ${RESTIC_PASSWORD:?RESTIC_PASSWORD must be set} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} + AZURE_ACCOUNT_NAME: ${AZURE_ACCOUNT_NAME:-} + AZURE_ACCOUNT_KEY: ${AZURE_ACCOUNT_KEY:-} + SHOPDB_SITE: ${SHOPDB_SITE:-shopdb} + SHOPDB_IMAGE_TAG: ${SHOPDB_IMAGE:-source} + SHOPDB_BACKUPS_ROOT: /backups + volumes: + - instance_data:/app/instance:ro + - ./migrations:/app/migrations:ro + - ./plugins:/app/plugins:ro + - backup_data:/backups + - ./scripts/backup:/opt/backup:ro + entrypoint: ["/opt/backup/backup.sh"] + + # Give the api read/write on the shared backups volume so the UI can list sets + # and drop the "Back up now" sentinel the watcher actuates. This extends the + # api service defined in docker-compose.yml (merged by compose). + api: + volumes: + - backup_data:/backups + +volumes: + backup_data: + backupstore_data: diff --git a/frontend/src/router/routes/core.js b/frontend/src/router/routes/core.js index b2bc3864..287a5f51 100644 --- a/frontend/src/router/routes/core.js +++ b/frontend/src/router/routes/core.js @@ -221,5 +221,11 @@ export default [ name: 'users', component: () => import('../../views/settings/UsersList.vue'), meta: { requiresAuth: true, requiresAdmin: true } + }, + { + path: 'settings/shopdbbackups', + name: 'shopdbbackups', + component: () => import('../../views/settings/ShopdbBackupsList.vue'), + meta: { requiresAuth: true, requiresAdmin: true } } ] diff --git a/frontend/src/views/settings/ShopdbBackupsList.vue b/frontend/src/views/settings/ShopdbBackupsList.vue new file mode 100644 index 00000000..0b649ad2 --- /dev/null +++ b/frontend/src/views/settings/ShopdbBackupsList.vue @@ -0,0 +1,587 @@ + + + + + diff --git a/frontend/src/views/settings/settingsNav.js b/frontend/src/views/settings/settingsNav.js index a5b87238..3b19a95e 100644 --- a/frontend/src/views/settings/settingsNav.js +++ b/frontend/src/views/settings/settingsNav.js @@ -4,7 +4,7 @@ // Order: site identity + the reference-data catalogs users touch daily come // first, then the platform/system groups (integrations, communication, search, // plugins, access) cluster together at the end. -import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key, Upload } from 'lucide-vue-next' +import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key, Upload, DatabaseBackup } from 'lucide-vue-next' export const settingsGroups = [ { @@ -113,6 +113,7 @@ export const settingsGroups = [ title: 'System', cards: [ { to: '/settings/plugins', icon: Puzzle, title: 'Plugins', description: 'Enable or disable installed plugins' }, + { to: '/settings/shopdbbackups', icon: DatabaseBackup, title: 'Site Backups', description: 'Scheduled database and file backups with guided restore; keeps any day in the last 5' }, ], }, { diff --git a/scripts/backup/.gitattributes b/scripts/backup/.gitattributes new file mode 100644 index 00000000..920d0296 --- /dev/null +++ b/scripts/backup/.gitattributes @@ -0,0 +1,4 @@ +# Keep shell scripts LF-only so they run inside Linux containers regardless of +# the host checkout. A CRLF shebang makes the kernel look for "bash\r" and the +# container dies with: env: 'bash\r': No such file or directory. +*.sh text eol=lf diff --git a/scripts/backup/Dockerfile.backup b/scripts/backup/Dockerfile.backup new file mode 100644 index 00000000..a207fdde --- /dev/null +++ b/scripts/backup/Dockerfile.backup @@ -0,0 +1,38 @@ +# Backup sidecar image: mysql:8.4 (for a version-matched mysqldump) plus restic +# (for deduped, encrypted, destination-agnostic file snapshots) and a small +# Python for the manifest writer. +# +# WHY mysql:8.4 as the base: the dump tool must match the server major/minor so +# the SQL it emits restores cleanly onto the same engine. Reusing the exact +# image the db service runs removes version drift as a failure mode. +FROM mysql:8.4 + +# Track 1 local build unblock: trust the corporate proxy (Zscaler/GE) root CA so +# the build-time fetches below (Oracle yum, github.com, PyPI) succeed behind TLS +# interception. This is a local convenience only; the shipping build vendors +# these deps instead of fetching them (pinned restic + SHA256, vendored +# bzip2/python3/alembic or a base image that carries them) so the release image +# needs no outbound calls and no embedded cert. The cert is gitignored (*.pem) +# and must be staged at scripts/backup/zscaler-root-ca.pem before building. +COPY scripts/backup/zscaler-root-ca.pem /etc/pki/ca-trust/source/anchors/zscaler-root-ca.pem +RUN update-ca-trust + +# restic from the distro is often stale; pull the pinned release binary so the +# snapshot format and retention flags are predictable across sites. +ARG RESTIC_VERSION=0.17.3 +RUN microdnf install -y bzip2 python3 python3-pip ca-certificates \ + && curl -fsSL -o /tmp/restic.bz2 \ + "https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_amd64.bz2" \ + && bunzip2 /tmp/restic.bz2 \ + && install -m 0755 /tmp/restic /usr/local/bin/restic \ + && rm -f /tmp/restic \ + && microdnf clean all + +# alembic so the manifest writer can read the current schema head. pip does not +# read the system trust store, so point it at the extracted system CA bundle +# (now including the proxy root) to get through TLS interception. +RUN python3 -m pip install --no-cache-dir \ + --cert /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem \ + alembic + +ENTRYPOINT ["/opt/backup/backup.sh"] diff --git a/scripts/backup/backup.sh b/scripts/backup/backup.sh new file mode 100644 index 00000000..8a22261f --- /dev/null +++ b/scripts/backup/backup.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# ShopDB backup one-shot. +# +# Produces one recoverable set per run and stores it two ways: a plain +# db.sql.gz + manifest.json on the local /backups volume for a fast, repo-free +# restore, and a restic snapshot (db dump + instance tree + manifest) at +# RESTIC_REPOSITORY for a deduped, encrypted, offsite copy. +# +# Sequence: +# 1. mysqldump --single-transaction (InnoDB-consistent, non-blocking) +# 2. manifest.py stamps core+plugin Alembic revisions + image tag +# 3. restic backup the dump + /app/instance + manifest as ONE snapshot +# 4. restic forget --keep-daily 5 --keep-weekly 4 --keep-monthly 6 --prune +# 5. restic check - fail loudly if the repo is unhealthy, so a bad repo is +# caught on backup night, not restore night. +# +# ASCII only, no emojis - matches the repo style gate. +set -euo pipefail + +BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}" +SITE="${SHOPDB_SITE:-shopdb}" +STAMP="$(date -u +%Y-%m-%dT%H%M%SZ)" +SETNAME="${SITE}-${STAMP}" +SETDIR="${BACKUPS_ROOT}/${SETNAME}" + +log() { echo "[backup] $*"; } + +mkdir -p "${SETDIR}" + +# 1. Database: whole-DB logical dump, consistent and non-blocking on InnoDB. +log "dumping database ${MYSQL_DATABASE} from ${MYSQL_HOST}" +mysqldump \ + --host="${MYSQL_HOST}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + --single-transaction \ + --routines \ + --triggers \ + --databases "${MYSQL_DATABASE}" \ + | gzip -c > "${SETDIR}/db.sql.gz" +log "database dump written ($(du -h "${SETDIR}/db.sql.gz" | cut -f1))" + +# 2. Manifest: the schema stamp that makes a restore a checked operation. +log "writing manifest" +python3 /opt/backup/manifest.py \ + --out "${SETDIR}/manifest.json" \ + --takenat "${STAMP}" \ + --site "${SITE}" \ + --imagetag "${SHOPDB_IMAGE_TAG:-source}" \ + --migrations /app/migrations \ + --plugins /app/plugins + +# 3. restic snapshot: db dump + instance tree + manifest, one point in time. +# Skips gracefully when restic/repo are not configured (local-only sites). +IS_CHECK_PASSED=false +if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then + if ! restic snapshots >/dev/null 2>&1; then + log "initializing restic repository at ${RESTIC_REPOSITORY}" + restic init + fi + log "restic backup (db dump + instance tree + manifest)" + restic backup \ + --tag "${SITE}" \ + --tag "${SETNAME}" \ + "${SETDIR}/db.sql.gz" \ + "${SETDIR}/manifest.json" \ + /app/instance + + # 4. Retention: any day in the last 5, plus weekly/monthly tail. + log "pruning per retention policy" + restic forget \ + --tag "${SITE}" \ + --keep-daily 5 \ + --keep-weekly 4 \ + --keep-monthly 6 \ + --prune + + # 5. Integrity: learn about corruption now, not at restore time. + log "verifying repository integrity" + if restic check; then + IS_CHECK_PASSED=true + log "restic check passed" + else + log "restic check FAILED - repository may be corrupt" + fi +else + log "restic not configured - local db.sql.gz + manifest only" +fi + +# Record the verification result into the manifest so the UI can badge the set. +python3 /opt/backup/manifest.py \ + --amend "${SETDIR}/manifest.json" \ + --ischeckpassed "${IS_CHECK_PASSED}" + +# Clear the request sentinel the UI drops when an operator clicks "Back up now". +rm -f "${BACKUPS_ROOT}/.backup-requested" 2>/dev/null || true + +log "backup set ${SETNAME} complete" diff --git a/scripts/backup/manifest.py b/scripts/backup/manifest.py new file mode 100644 index 00000000..8708ba94 --- /dev/null +++ b/scripts/backup/manifest.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Backup manifest writer. + +The manifest is the stamp that turns "we have dumps" into "we can actually +restore." It records, at capture time: + - takenat : ISO-8601 UTC timestamp of the run + - site : the site tag + - imagetag : the app image tag that produced the data + - schemarev : the core Alembic head revision(s) the running code is at, plus + each plugin chain's heads - the versions a restore target must + be at least, or restored rows reference columns the live schema + has not applied yet. + +Two modes: + --out ... write a fresh manifest for a new set + --amend ... patch an existing manifest (used to record ischeckpassed after + restic check runs) + +ASCII only, no emojis - matches the repo style gate. +""" + +import argparse +import json +import os +import sys + + +def _heads_for(chain_dir): + """Return the Alembic head revision id(s) under a versions directory. + + We read the revision graph the cheap way: a head is a revision no other + revision names as its down_revision. Done by text scan so this stays + dependency-light and works for both the core chain and each plugin chain. + """ + versions = os.path.join(chain_dir, 'versions') + if not os.path.isdir(versions): + return [] + revisions = {} + down_refs = set() + for name in os.listdir(versions): + if not name.endswith('.py'): + continue + path = os.path.join(versions, name) + revid = None + downid = None + try: + with open(path, 'r', encoding='utf-8') as handle: + for line in handle: + stripped = line.strip() + if stripped.startswith('revision =') and revid is None: + revid = stripped.split('=', 1)[1].strip().strip('"\'') + elif stripped.startswith('down_revision ='): + value = stripped.split('=', 1)[1].strip() + downid = None if value in ('None', 'null') else value.strip('"\'') + except OSError: + continue + if revid: + revisions[revid] = downid + if downid: + down_refs.add(downid) + return sorted(rev for rev in revisions if rev not in down_refs) + + +def _collect_schema(migrations_dir, plugins_dir): + """Core heads plus each plugin's chain heads, keyed for compatibility checks.""" + schema = {'core': _heads_for(migrations_dir), 'plugins': {}} + if plugins_dir and os.path.isdir(plugins_dir): + for name in sorted(os.listdir(plugins_dir)): + chain = os.path.join(plugins_dir, name, 'migrations') + heads = _heads_for(chain) + if heads: + schema['plugins'][name] = heads + return schema + + +def write_manifest(args): + manifest = { + 'takenat': args.takenat, + 'site': args.site, + 'imagetag': args.imagetag, + 'schemarev': _collect_schema(args.migrations, args.plugins), + 'ischeckpassed': False, + } + with open(args.out, 'w', encoding='utf-8') as handle: + json.dump(manifest, handle, indent=2, sort_keys=True) + return 0 + + +def amend_manifest(args): + try: + with open(args.amend, 'r', encoding='utf-8') as handle: + manifest = json.load(handle) + except (OSError, ValueError): + manifest = {} + if args.ischeckpassed is not None: + manifest['ischeckpassed'] = args.ischeckpassed.lower() == 'true' + with open(args.amend, 'w', encoding='utf-8') as handle: + json.dump(manifest, handle, indent=2, sort_keys=True) + return 0 + + +def main(argv=None): + parser = argparse.ArgumentParser(description='ShopDB backup manifest writer') + parser.add_argument('--out') + parser.add_argument('--amend') + parser.add_argument('--takenat', default='') + parser.add_argument('--site', default='shopdb') + parser.add_argument('--imagetag', default='source') + parser.add_argument('--migrations', default='/app/migrations') + parser.add_argument('--plugins', default='/app/plugins') + parser.add_argument('--ischeckpassed') + args = parser.parse_args(argv) + + if args.amend: + return amend_manifest(args) + if args.out: + return write_manifest(args) + parser.error('one of --out or --amend is required') + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/backup/restore.sh b/scripts/backup/restore.sh new file mode 100644 index 00000000..5d358e46 --- /dev/null +++ b/scripts/backup/restore.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# ShopDB restore. +# +# The DANGEROUS half of the pair, so it is deliberately guarded. A restore is +# only safe onto code whose schema is at least the set's - otherwise restored +# rows reference columns the live schema has not applied yet and every query on +# those tables 500s. This script refuses an incompatible restore rather than +# leaving the site half-broken. +# +# Usage (from the backup container): +# docker compose -f docker-compose.yml -f docker-compose.backup.yml \ +# run --rm --entrypoint /opt/backup/restore.sh backup +# +# It does NOT roll back code - that is git's job (git checkout && build). +# Restore code FIRST, bring the schema up (flask db upgrade + plugin +# upgrade-all), THEN run this to load the data. +# +# ASCII only, no emojis - matches the repo style gate. +set -euo pipefail + +BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}" +SETNAME="${1:-}" + +log() { echo "[restore] $*"; } +die() { echo "[restore] ERROR: $*" >&2; exit 1; } + +[ -n "${SETNAME}" ] || die "usage: restore.sh " + +SETDIR="${BACKUPS_ROOT}/${SETNAME}" +DBDUMP="${SETDIR}/db.sql.gz" + +# Prefer the local fast path; fall back to pulling the set out of restic. +if [ ! -f "${DBDUMP}" ]; then + log "local dump not found; restoring set from restic" + command -v restic >/dev/null 2>&1 || die "restic not available and no local dump" + restic restore latest --target / --include "${SETDIR}" \ + || die "restic restore of ${SETNAME} failed" +fi +[ -f "${DBDUMP}" ] || die "no db.sql.gz for set ${SETNAME}" + +# Compatibility gate: the running schema must cover the set's stamped heads. +log "checking schema compatibility" +CURRENT_HEADS="$(python3 /opt/backup/manifest.py \ + --out /tmp/current-manifest.json \ + --migrations /app/migrations --plugins /app/plugins \ + >/dev/null 2>&1 && python3 -c \ + 'import json;print(",".join(sorted(json.load(open("/tmp/current-manifest.json"))["schemarev"]["core"])))')" +SET_HEADS="$(python3 -c \ + 'import json,sys;print(",".join(sorted(json.load(open(sys.argv[1]))["schemarev"]["core"])))' \ + "${SETDIR}/manifest.json" 2>/dev/null || echo "")" + +if [ -z "${SET_HEADS}" ]; then + die "set ${SETNAME} has no readable manifest - refusing an unverified restore" +fi +if [ "${CURRENT_HEADS}" != "${SET_HEADS}" ]; then + log "current heads : ${CURRENT_HEADS}" + log "set heads : ${SET_HEADS}" + die "schema mismatch - bring code to the set's revision, run 'flask db upgrade' and 'flask plugin upgrade-all', then retry" +fi + +# Files first, then DB, so the two land together. +if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then + log "restoring instance files from restic" + restic restore latest --target / --include /app/instance \ + || die "restic restore of instance files failed" +fi + +log "loading database dump" +gunzip -c "${DBDUMP}" | mysql \ + --host="${MYSQL_HOST}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + || die "database load failed" + +rm -f "${BACKUPS_ROOT}/.restore-requested" 2>/dev/null || true +log "restore of ${SETNAME} complete" diff --git a/scripts/backup/watch.sh b/scripts/backup/watch.sh new file mode 100644 index 00000000..42c8b93e --- /dev/null +++ b/scripts/backup/watch.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# ShopDB backup WATCHER - the resident half of the in-app backup utility. +# +# WHY this exists: the api container cannot run docker, so when an operator +# clicks "Back up now" the api only drops a sentinel file (.backup-requested) +# on the shared backups volume. This watcher is the thing that actuates that +# request: it polls for the sentinel and runs the one-shot backup.sh, which +# ships a restic snapshot to the backup-store container and clears the sentinel +# on success. That turns the UI button into a real, self-serve backup without +# giving the api container any docker or host privileges. +# +# ASCII only, no emojis - matches the repo style gate. +set -uo pipefail + +BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}" +SENTINEL="${BACKUPS_ROOT}/.backup-requested" +INTERVAL="${BACKUP_POLL_SECONDS:-10}" + +log() { echo "[backup-watch] $*"; } + +mkdir -p "${BACKUPS_ROOT}" +log "watcher up; polling ${SENTINEL} every ${INTERVAL}s" + +while true; do + if [ -f "${SENTINEL}" ]; then + log "backup requested - running one-shot" + if /opt/backup/backup.sh; then + log "backup completed" + else + # backup.sh only clears the sentinel on success, so a failed run + # would otherwise loop forever. Clear it here and wait for the next + # explicit request rather than hammering a broken backup. + log "backup FAILED - clearing request, awaiting next" + rm -f "${SENTINEL}" 2>/dev/null || true + fi + fi + sleep "${INTERVAL}" +done diff --git a/shopdb/__init__.py b/shopdb/__init__.py index 4bab5be6..f513c409 100644 --- a/shopdb/__init__.py +++ b/shopdb/__init__.py @@ -298,6 +298,7 @@ CORE_BLUEPRINT_NAMES = ( 'pluginui', 'apitokens', 'docs', + 'shopdbbackups', ) diff --git a/shopdb/core/api/__init__.py b/shopdb/core/api/__init__.py index 3ebda32a..aab4d55a 100644 --- a/shopdb/core/api/__init__.py +++ b/shopdb/core/api/__init__.py @@ -27,6 +27,7 @@ from .setup import setup_bp from .pluginui import pluginui_bp from .apitokens import apitokens_bp from .docs import docs_bp +from .shopdbbackups import shopdbbackups_bp __all__ = [ 'auth_bp', @@ -56,4 +57,5 @@ __all__ = [ 'pluginui_bp', 'apitokens_bp', 'docs_bp', + 'shopdbbackups_bp', ] diff --git a/shopdb/core/api/settings.py b/shopdb/core/api/settings.py index ef774a86..8373f98c 100644 --- a/shopdb/core/api/settings.py +++ b/shopdb/core/api/settings.py @@ -1488,6 +1488,99 @@ def build_default_settings(): 'Incoming Webhook), adaptivecard (Teams Workflow), ' 'or json (generic {title,text})' }, + # Backup destination settings + # + # These are the ground wiring for where the one-shot backup container + # ships each set. 'local' keeps sets on the mounted backups volume and + # needs no credentials. 'smb' and 'aws' are declared so the UI and the + # backup runner can be configured now; the client libraries (an SMB + # stack, boto3/awscli) are deliberately NOT pulled in yet - those land + # behind the security review that vets third-party deps against + # external exploit. Until then a non-local destination is accepted and + # persisted but the runner treats it as not-yet-armed. + # + # Secrets follow the existing convention: any key containing + # 'password'/'secret'/'token' is masked by _is_secret on the way out + # and preserved when the client submits the mask back unchanged. + { + 'key': 'backup_destination_kind', + 'value': 'local', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'Where backup sets are shipped: local, smb, or aws. ' + 'Non-local destinations are wiring-only until the ' + 'security review clears their client libraries.' + }, + { + 'key': 'backup_smb_host', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'SMB/CIFS server hostname or IP for the backup share' + }, + { + 'key': 'backup_smb_share', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'SMB share name (the top-level export, no path)' + }, + { + 'key': 'backup_smb_path', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'Subpath within the SMB share to write sets into' + }, + { + 'key': 'backup_smb_username', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'SMB authentication username (DOMAIN\\user allowed)' + }, + { + 'key': 'backup_smb_password', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'SMB authentication password' + }, + { + 'key': 'backup_aws_region', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'AWS region for the S3 backup bucket (e.g. us-east-1)' + }, + { + 'key': 'backup_aws_bucket', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'S3 bucket name that holds backup sets' + }, + { + 'key': 'backup_aws_prefix', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'Optional key prefix within the S3 bucket' + }, + { + 'key': 'backup_aws_accesskeyid', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'AWS access key id for the backup bucket' + }, + { + 'key': 'backup_aws_secretaccesskey', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'AWS secret access key for the backup bucket' + }, # Audit log retention lives at 'audit_retentiondays' (category site), # defined above and read by shopdb/core/services/auditretention.py. A # second key 'audit_retention_days' was seeded here with a different diff --git a/shopdb/core/api/shopdbbackups.py b/shopdb/core/api/shopdbbackups.py new file mode 100644 index 00000000..f2f69cb1 --- /dev/null +++ b/shopdb/core/api/shopdbbackups.py @@ -0,0 +1,352 @@ +"""Backups API routes. + +Admin-only surface over the site's backup sets. The heavy lifting (mysqldump, +restic snapshot of the instance tree, retention prune, integrity check) runs in +the one-shot backup container defined in docker-compose.backup.yml; this +blueprint is the read-mostly control plane the UI talks to. + +Two recovery products stay firewalled, per the design notes: + - code rollback lives in git (tags/releases), NOT here + - data restore lives here, gated on schema compatibility via each set's manifest + +Every backup set carries a manifest.json stamped with the core + plugin Alembic +revisions and the running image tag at capture time. A restore is a CHECKED +operation: current code must be at least the set's schema revision, so we never +load rows that reference a column the live schema has not applied yet. +""" + +import json +import os +import subprocess +from datetime import datetime, timezone + +from flask import Blueprint, current_app, request + +from shopdb.core.models import AuditLog +from shopdb.core.models.setting import Setting +from shopdb.utils.responses import success_response, error_response, ErrorCodes +from shopdb.utils.authz import require_role + +shopdbbackups_bp = Blueprint('shopdbbackups', __name__) + +# Supported backup destination kinds. 'local' needs no credentials; 'smb' and +# 'aws' are ground-wired here so operators can configure them now, but their +# client libraries are held back until the security review clears them, so the +# runner treats a non-local destination as configured-but-not-yet-armed. +DESTINATION_KINDS = ('local', 'smb', 'aws') + +# The setting keys that make up each destination's config, grouped by kind so +# the GET can surface only the relevant fields and the PUT can validate what it +# is allowed to write. Secret keys (password/secret/token in the name) are +# masked by the shared settings serializer, same convention as SMTP/SAML. +DESTINATION_FIELDS = { + 'local': [], + 'smb': [ + 'backup_smb_host', + 'backup_smb_share', + 'backup_smb_path', + 'backup_smb_username', + 'backup_smb_password', + ], + 'aws': [ + 'backup_aws_region', + 'backup_aws_bucket', + 'backup_aws_prefix', + 'backup_aws_accesskeyid', + 'backup_aws_secretaccesskey', + ], +} + +# Keys whose value must never leave the API in the clear. Mirrors the shared +# settings convention (_is_secret) so the two surfaces agree on what is secret. +_SECRET_TOKENS = ('password', 'secret', 'token') +SECRET_MASK = '********' + + +def _is_secret_key(key): + return any(token in key for token in _SECRET_TOKENS) + +# Where the backup container drops each set and its manifest. Mounted read-only +# for the api container so the UI can enumerate sets without being able to +# mutate them; writes happen only from the backup one-shot. +BACKUPS_ROOT_ENV = 'SHOPDB_BACKUPS_ROOT' +DEFAULT_BACKUPS_ROOT = '/backups' + +# Retention the backup container enforces; surfaced read-only so the UI can show +# "day 1 of 5 retained" without re-deriving the policy. +RETENTION = { + 'keepdaily': 5, + 'keepweekly': 4, + 'keepmonthly': 6, +} + + +def _backups_root(): + return os.environ.get(BACKUPS_ROOT_ENV, DEFAULT_BACKUPS_ROOT) + + +def _read_manifest(setdir): + """Load a set's manifest.json, or None when absent/unreadable. + + A set without a readable manifest is not restorable (we cannot check its + schema), so callers treat None as "unverified" rather than trusting it. + """ + path = os.path.join(setdir, 'manifest.json') + try: + with open(path, 'r', encoding='utf-8') as handle: + return json.load(handle) + except (OSError, ValueError): + return None + + +def _current_schema_revisions(): + """Return the live Alembic head revisions (core + plugins) for this build. + + Compared against a set's stamped revisions to decide restore compatibility. + Imported lazily so a missing alembic at request time degrades to an empty + map (everything reads as "unknown") instead of 500ing the list endpoint. + """ + try: + from alembic.config import Config + from alembic.script import ScriptDirectory + except ImportError: + return {} + + root = current_app.root_path + ini = os.path.join(os.path.dirname(root), 'migrations', 'alembic.ini') + if not os.path.isfile(ini): + return {} + try: + config = Config(ini) + script = ScriptDirectory.from_config(config) + return {'core': list(script.get_heads())} + except Exception: + return {} + + +def _is_compatible(manifest, current): + """A set restores onto THIS build only when the build's schema is at least + the set's. We compare head-revision sets: if the set was taken on a schema + the current code has not reached, the code is too OLD and would 500 on the + restored rows, so restore is blocked until the code is upgraded. + """ + if not manifest: + return False + setrevs = set(manifest.get('schemarev', {}).get('core', [])) + currevs = set(current.get('core', [])) + if not setrevs or not currevs: + return False + # Compatible when the current heads cover every head the set was taken at. + return setrevs.issubset(currevs) + + +def _list_sets(): + """Enumerate backup sets newest-first, each summarized for the list view.""" + root = _backups_root() + if not os.path.isdir(root): + return [] + current = _current_schema_revisions() + sets = [] + for name in sorted(os.listdir(root), reverse=True): + setdir = os.path.join(root, name) + if not os.path.isdir(setdir): + continue + manifest = _read_manifest(setdir) + dbdump = os.path.join(setdir, 'db.sql.gz') + sizebytes = 0 + for base, _dirs, files in os.walk(setdir): + for filename in files: + try: + sizebytes += os.path.getsize(os.path.join(base, filename)) + except OSError: + pass + sets.append({ + 'snapshotid': name, + 'takenat': (manifest or {}).get('takenat'), + 'sizebytes': sizebytes, + 'schemarev': (manifest or {}).get('schemarev', {}), + 'imagetag': (manifest or {}).get('imagetag'), + 'ischeckpassed': bool((manifest or {}).get('ischeckpassed')), + 'hasmanifest': manifest is not None, + 'hasdbdump': os.path.isfile(dbdump), + 'iscompatible': _is_compatible(manifest, current), + }) + return sets + + +@shopdbbackups_bp.route('', methods=['GET']) +@require_role('admin') +def list_backups(): + """List every retained backup set, newest first, with a compatibility flag.""" + sets = _list_sets() + laststgood = next((s for s in sets if s['ischeckpassed']), None) + meta = { + 'retention': RETENTION, + 'lastgoodat': (laststgood or {}).get('takenat'), + 'count': len(sets), + } + return success_response(data=sets, meta=meta) + + +@shopdbbackups_bp.route('/', methods=['GET']) +@require_role('admin') +def get_backup(snapshotid): + """Return one set's full manifest plus a resolved compatibility verdict.""" + root = _backups_root() + setdir = os.path.join(root, os.path.basename(snapshotid)) + if not os.path.isdir(setdir): + return error_response(ErrorCodes.NOT_FOUND, 'Backup set not found', + http_code=404) + manifest = _read_manifest(setdir) + current = _current_schema_revisions() + return success_response(data={ + 'snapshotid': os.path.basename(snapshotid), + 'manifest': manifest, + 'currentschemarev': current, + 'iscompatible': _is_compatible(manifest, current), + 'hasmanifest': manifest is not None, + }) + + +@shopdbbackups_bp.route('', methods=['POST']) +@require_role('admin') +def trigger_backup(): + """Kick off an on-demand backup run. + + The api container cannot run docker; it writes a sentinel the host-side + scheduler (or an operator running `docker compose run --rm backup`) picks + up. Returning isrunning=false with a clear message keeps the UI honest + rather than pretending the run started inside this process. + """ + root = _backups_root() + try: + os.makedirs(root, exist_ok=True) + sentinel = os.path.join(root, '.backup-requested') + stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z' + with open(sentinel, 'w', encoding='utf-8') as handle: + handle.write(stamp) + except OSError as error: + return error_response(ErrorCodes.INTERNAL_ERROR, + 'Could not queue a backup: ' + str(error), + http_code=500) + AuditLog.log('requested', 'Backup', entityname=stamp) + return success_response( + data={'isrunning': False, 'isqueued': True, 'requestedat': stamp}, + message='Backup queued. The backup service runs it on its next pass.') + + +@shopdbbackups_bp.route('//restore', methods=['POST']) +@require_role('admin') +def restore_backup(snapshotid): + """Queue a restore of a set, gated on schema compatibility. + + We refuse outright when the set is incompatible (current code older than the + set's schema) so an operator cannot foot-gun a schema mismatch from the UI. + The actual restore is performed by scripts/backup/restore.sh in the backup + container; here we validate and record intent. + """ + root = _backups_root() + setdir = os.path.join(root, os.path.basename(snapshotid)) + if not os.path.isdir(setdir): + return error_response(ErrorCodes.NOT_FOUND, 'Backup set not found', + http_code=404) + manifest = _read_manifest(setdir) + current = _current_schema_revisions() + if not _is_compatible(manifest, current): + return error_response( + ErrorCodes.CONFLICT, + 'This backup was taken on a newer schema than the running code. ' + 'Upgrade the code to at least the set\'s revision, run ' + '`flask db upgrade` and `flask plugin upgrade-all`, then retry.', + details={'currentschemarev': current, + 'setschemarev': (manifest or {}).get('schemarev', {})}, + http_code=409) + try: + sentinel = os.path.join(root, '.restore-requested') + stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z' + with open(sentinel, 'w', encoding='utf-8') as handle: + handle.write(os.path.basename(snapshotid)) + except OSError as error: + return error_response(ErrorCodes.INTERNAL_ERROR, + 'Could not queue the restore: ' + str(error), + http_code=500) + AuditLog.log('restore-requested', 'Backup', + entityname=os.path.basename(snapshotid)) + return success_response( + data={'isqueued': True, 'snapshotid': os.path.basename(snapshotid)}, + message='Restore queued and validated against the live schema.') + + +@shopdbbackups_bp.route('/settings', methods=['GET']) +@require_role('admin') +def get_backup_settings(): + """Return destination + retention config for the settings pane. + + The destination kind and its (non-secret) fields come from the settings + store so an operator can configure where backups ship without editing env + or redeploying. Secret fields (SMB password, AWS secret key) are never + returned in the clear: we send a mask when a value is set and an empty + string when it is not, matching the shared settings API convention. + + 'smb' and 'aws' are configurable now but their client libraries are held + back pending the security review, so we flag those destinations as armed: + false to tell the UI the config is stored but not yet acting. + """ + kind = Setting.get('backup_destination_kind', 'local') + if kind not in DESTINATION_KINDS: + kind = 'local' + + fields = {} + for key in DESTINATION_FIELDS.get(kind, []): + raw = Setting.get(key, '') + if _is_secret_key(key): + fields[key] = SECRET_MASK if raw else '' + else: + fields[key] = raw or '' + + return success_response(data={ + 'destinationkind': kind, + 'destinationkinds': list(DESTINATION_KINDS), + 'fields': fields, + 'isarmed': kind == 'local', + 'retention': RETENTION, + 'backupsroot': _backups_root(), + }) + + +@shopdbbackups_bp.route('/settings', methods=['PUT']) +@require_role('admin') +def update_backup_settings(): + """Persist the backup destination kind and its credential/config fields. + + Only keys that belong to the chosen destination are accepted, so a stray + payload cannot write arbitrary settings rows through this surface. A secret + field submitted as the mask placeholder means "leave unchanged" - the + client only ever received the mask, so we never overwrite a real secret + with it. + """ + data = request.get_json(silent=True) or {} + kind = data.get('destinationkind', Setting.get('backup_destination_kind', 'local')) + if kind not in DESTINATION_KINDS: + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'destinationkind must be one of: ' + ', '.join(DESTINATION_KINDS)) + + Setting.set('backup_destination_kind', kind, valuetype='string', + category='backup') + + incoming = data.get('fields', {}) or {} + allowed = set(DESTINATION_FIELDS.get(kind, [])) + for key, value in incoming.items(): + if key not in allowed: + continue + if _is_secret_key(key) and value == SECRET_MASK: + continue + Setting.set(key, value if value is not None else '', + valuetype='string', category='backup') + + AuditLog.log('updated', 'BackupSettings', entityname=kind, changes={ + 'destinationkind': kind, + }) + + return get_backup_settings()