diff --git a/docs/PROJECT-MAP.md b/docs/PROJECT-MAP.md index f80658c2..1180d821 100644 --- a/docs/PROJECT-MAP.md +++ b/docs/PROJECT-MAP.md @@ -91,6 +91,6 @@ Manifest-less directories under `plugins/` are core frontend surface and always ## Size -- test functions defined: **1847** (parametrised cases collect higher) +- test functions defined: **1852** (parametrised cases collect higher) - documented API paths: **376** (`docs/openapi.json`, regenerate with `scripts/gen_openapi.py`) diff --git a/scripts/backup/backup.sh b/scripts/backup/backup.sh index 110f427b..95fee553 100755 --- a/scripts/backup/backup.sh +++ b/scripts/backup/backup.sh @@ -75,6 +75,39 @@ DATAREV="$(mysql \ "${MYSQL_DATABASE}" 2>/dev/null | tr -d '\r')" [ -n "${DATAREV}" ] || log "WARN could not read alembic_version - datarev will be empty" +# The same question for every PLUGIN chain. Each plugin owns an +# alembic_version_ table, and those move independently of the core one: +# code rolled back to an older plugin revision can sit at an identical core +# head while the dump carries plugin tables it has never heard of. Stamping only +# the core revision let that combination pass the restore gate, load, and then +# fail during the plugin upgrade with the data already replaced. Read straight +# from the dumped DB, same as the core revision above. +log "reading applied plugin revisions (alembic_version_*)" +DATAPLUGINS="$(mysql \ + --host="${MYSQL_HOST}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + --skip-column-names --batch \ + -e "SELECT table_name FROM information_schema.tables + WHERE table_schema = '${MYSQL_DATABASE}' + AND table_name LIKE 'alembic\\_version\\_%';" 2>/dev/null \ + | tr -d '\r' \ + | while read -r table; do + [ -n "${table}" ] || continue + plugin="${table#alembic_version_}" + rev="$(mysql \ + --host="${MYSQL_HOST}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + --skip-column-names --batch \ + -e "SELECT version_num FROM \`${table}\` LIMIT 1;" \ + "${MYSQL_DATABASE}" 2>/dev/null | tr -d '\r')" + [ -n "${rev}" ] && printf '%s\t%s\n' "${plugin}" "${rev}" + done \ + | python3 -c 'import json,sys; print(json.dumps(dict(l.rstrip("\n").split("\t",1) for l in sys.stdin if "\t" in l)))')" +[ -n "${DATAPLUGINS}" ] || DATAPLUGINS='{}' +log "plugin revisions: ${DATAPLUGINS}" + log "writing manifest" python3 /opt/backup/manifest.py \ --out "${SETDIR}/manifest.json" \ @@ -82,6 +115,7 @@ python3 /opt/backup/manifest.py \ --site "${SITE}" \ --imagetag "${SHOPDB_IMAGE_TAG:-source}" \ --datarev "${DATAREV}" \ + --dataplugins "${DATAPLUGINS}" \ --migrations /app/migrations \ --plugins /app/plugins diff --git a/scripts/backup/manifest.py b/scripts/backup/manifest.py index 45345468..511a8678 100644 --- a/scripts/backup/manifest.py +++ b/scripts/backup/manifest.py @@ -10,12 +10,19 @@ restore." It records, at capture time: each plugin chain's heads - the versions a restore target must be at least, or restored rows reference columns the live schema has not applied yet. - - datarev : the Alembic revision actually applied to the DATABASE that was - dumped (its alembic_version), which is the number a restore must - gate on. schemarev stamps the code on disk; datarev stamps the - data in the dump. They diverge when an image is ahead of a DB + - datarev : the CORE Alembic revision actually applied to the DATABASE that + was dumped (its alembic_version), which is the number a restore + must gate on. schemarev stamps the code on disk; datarev stamps + the data in the dump. They diverge when an image is ahead of a DB that has not run 'flask db upgrade' yet, and that divergence is exactly what a code-to-code check cannot see. + - dataplugins: the same thing for every PLUGIN chain - the applied revision of + each alembic_version_ table. The core chain alone cannot + answer compatibility: plugin schemas move independently, so code + rolled back to an older plugin revision can sit at an identical + core head while the dump carries plugin tables it has never heard + of. Gating on core only let that pass, load, and then fail during + the plugin upgrade with the data already replaced. Two modes: --out ... write a fresh manifest for a new set @@ -67,6 +74,25 @@ def _heads_for(chain_dir): return sorted(rev for rev in revisions if rev not in down_refs) +def _parse_dataplugins(raw): + """The applied per-plugin revisions, as a {plugin: revision} map. + + Bad or absent JSON becomes an empty map rather than an error: a manifest + that cannot say what the plugin chains were at is exactly the legacy case + the restore gate already knows how to treat conservatively, and failing the + backup here would lose the set over a stamp. + """ + if not raw: + return {} + try: + value = json.loads(raw) + except ValueError: + return {} + if not isinstance(value, dict): + return {} + return {str(k): str(v) for k, v in value.items() if k and v} + + def _collect_schema(migrations_dir, plugins_dir): """Core heads plus each plugin's chain heads, keyed for compatibility checks.""" schema = {'core': _heads_for(migrations_dir), 'plugins': {}} @@ -86,6 +112,7 @@ def write_manifest(args): 'imagetag': args.imagetag, 'schemarev': _collect_schema(args.migrations, args.plugins), 'datarev': args.datarev or '', + 'dataplugins': _parse_dataplugins(args.dataplugins), 'ischeckpassed': False, } with open(args.out, 'w', encoding='utf-8') as handle: @@ -103,6 +130,8 @@ def amend_manifest(args): manifest['ischeckpassed'] = args.ischeckpassed.lower() == 'true' if args.datarev is not None: manifest['datarev'] = args.datarev + if args.dataplugins is not None: + manifest['dataplugins'] = _parse_dataplugins(args.dataplugins) with open(args.amend, 'w', encoding='utf-8') as handle: json.dump(manifest, handle, indent=2, sort_keys=True) return 0 @@ -118,6 +147,8 @@ def main(argv=None): parser.add_argument('--migrations', default='/app/migrations') parser.add_argument('--plugins', default='/app/plugins') parser.add_argument('--datarev') + parser.add_argument('--dataplugins', + help='JSON object of {plugin: applied revision}') parser.add_argument('--ischeckpassed') args = parser.parse_args(argv) diff --git a/scripts/backup/restore.sh b/scripts/backup/restore.sh index 6cdae2ec..de6140f0 100755 --- a/scripts/backup/restore.sh +++ b/scripts/backup/restore.sh @@ -101,6 +101,47 @@ if ! grep -rqE "^revision = ['\"]${SET_DATAREV}['\"]" /app/migrations/versions 2 fi log "data revision ${SET_DATAREV} is known to this code - restore may proceed" +# The same gate for every PLUGIN chain, because the core chain alone cannot +# answer this. Plugin schemas move independently of core: roll the code back to +# an older plugin revision and the core head can be identical while the dump +# carries plugin tables the code has never heard of. Gating on core only let +# that pass, load the dump, and then fail in 'flask plugin upgrade-all' with +# "Can't locate revision" and the data already replaced - a failed restore that +# has destroyed what it was restoring over. +# +# A manifest with no dataplugins is a legacy set taken before this was stamped. +# Its plugin revisions are unknowable, so it is allowed through on the core gate +# alone (the behaviour before this check existed) rather than being made +# unrestorable retroactively. +SET_DATAPLUGINS="$(python3 -c \ + 'import json,sys +try: + data = json.load(open(sys.argv[1])).get("dataplugins") or {} +except Exception: + data = {} +for name, rev in sorted(data.items()): + print(name + " " + rev)' \ + "${SETDIR}/manifest.json" 2>/dev/null || echo "")" + +if [ -z "${SET_DATAPLUGINS}" ]; then + log "manifest stamps no plugin revisions (legacy set) - core gate only" +else + while read -r plugin rev; do + [ -n "${plugin}" ] || continue + chain="/app/plugins/${plugin}/migrations/versions" + if [ ! -d "${chain}" ]; then + die "the backup carries data for plugin '${plugin}', which this build does not have - deploy a build that includes it before restoring" + fi + if ! grep -rqE "^revision = ['\"]${rev}['\"]" "${chain}" 2>/dev/null; then + log "plugin ${plugin} data revision: ${rev}" + die "plugin '${plugin}' data is at a revision this code does not have - deploy code at or past '${rev}' before restoring" + fi + log "plugin ${plugin} at ${rev} - known to this code" + done <