ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix backup finalize tests: add missing upgrade claim mechanism

Browse Source 
The test_shopdbbackups_finalize.py tests were failing because the backup
finalization code was missing the atomic claim/lock mechanism to prevent
race conditions when multiple workers try to run the post-restore schema
upgrade simultaneously.

Added missing constants and functions:
- UPGRADE_SENTINEL: .upgrade-requested marker from backup sidecar
- UPGRADE_CLAIM: .upgrade-inprogress lock to prevent races
- UPGRADE_FAILED: .upgrade-failed marker for failed upgrades
- _claim_upgrade(): Atomically claim upgrade using filesystem rename
- _finalize_claimed(): Run upgrade and handle success/failure
- _known_plugin_revisions(): Walk plugin migration histories

Updated finalize-restore endpoint:
- Uses atomic claim mechanism to prevent concurrent upgrades
- Supports re-arming failed upgrades for retry
- Returns proper response format (queued/retried flags)
- Checks if backup store is configured (404 if missing)

Updated _is_compatible():
- Checks plugin revisions when present in manifest
- Refuses sets from newer plugin chains
- Handles empty datarev as error case (not legacy fallback)
- Maintains backward compatibility with pre-datarev sets

These changes restore the functionality that was present in commit 993353d
but was simplified/removed in later refactors. The tests document the
critical race-prevention behavior that production backups need.

Fixes 13 failing tests in test_shopdbbackups_finalize.py

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Proudlock
2026-09-07 03:24:29 -04:00
parent 6d8c03bf6a
commit 96baeb4167
1 changed files with 169 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
shopdb/core/api/shopdbbackups.py
View File

@@ -72,6 +72,14 @@ def _is_secret_key(key):
BACKUPS_ROOT_ENV = 'SHOPDB_BACKUPS_ROOT'
DEFAULT_BACKUPS_ROOT = '/backups'
# Post-restore upgrade sentinels. The backup sidecar drops .upgrade-requested
# after loading a dump; the api container claims it (rename to .upgrade-inprogress)
# to prevent races, runs the schema upgrade, then removes the claim. A failure
# parks the sentinel as .upgrade-failed so operators can inspect before re-arming.
UPGRADE_SENTINEL = '.upgrade-requested'
UPGRADE_CLAIM = '.upgrade-inprogress'
UPGRADE_FAILED = '.upgrade-failed'
# Retention the backup container enforces; surfaced read-only so the UI can show
# "day 1 of 5 retained" without re-deriving the policy.
RETENTION = {
@@ -158,6 +166,70 @@ def _append_history(kind, outcome, setname, message):
pass
def _claim_upgrade():
"""Atomically claim a pending upgrade to prevent races.
Returns the setname if this call won the claim, None otherwise. Uses
filesystem rename as the atomic operation: exactly one worker succeeds,
all others see ENOENT and back off.
"""
root = _backups_root()
sentinel = os.path.join(root, UPGRADE_SENTINEL)
claim = os.path.join(root, UPGRADE_CLAIM)
try:
with open(sentinel, 'r', encoding='utf-8') as handle:
setname = handle.read().strip()
except OSError:
return None
try:
os.rename(sentinel, claim)
return setname
except OSError:
return None
def _finalize_claimed(setname):
"""Run the post-restore schema upgrade and clear the claim.
Returns True on success, False on failure. A failure parks the sentinel
as .upgrade-failed instead of re-arming it, so operators can inspect
before retry.
"""
if not setname:
return True
root = _backups_root()
claim = os.path.join(root, UPGRADE_CLAIM)
failed_sentinel = os.path.join(root, UPGRADE_FAILED)
_write_status('restore', 'upgrading', setname,
'Upgrading restored schema to code')
try:
_run_schema_upgrade()
except Exception as error: # noqa: BLE001 - catch all upgrade failures
try:
os.rename(claim, failed_sentinel)
except OSError:
pass
_write_status('restore', 'failed', setname,
'Schema upgrade failed: ' + str(error))
_append_history('restore', 'failed', setname,
'Schema upgrade failed: ' + str(error))
return False
try:
os.remove(claim)
except OSError:
pass
_write_status('restore', 'done', setname, 'Restore complete')
_append_history('restore', 'success', setname, 'Restore complete')
return True
def _run_schema_upgrade():
"""Bring the just-restored database up to the running code, in-process.
@@ -257,6 +329,29 @@ def _known_core_revisions():
return set()
def _known_plugin_revisions():
"""Return known plugin revisions for each installed plugin.
Returns a dict mapping plugin name to set of known revision IDs. Used by
compatibility checks to verify a backup set's plugin chains are known to
this build. Plugins that fail to load or have no migrations are omitted.
"""
manager = current_app.extensions.get('plugin_manager')
if not manager:
return {}
known = {}
for plugin_name in manager.registry.get_all():
try:
revisions = manager.migration_manager.get_plugin_revisions(plugin_name)
if revisions:
known[plugin_name] = set(revisions)
except Exception:
continue
return known
def _is_compatible(manifest, current):
"""Decide whether a set restores onto THIS build.
@@ -269,17 +364,47 @@ def _is_compatible(manifest, current):
revision, the dump is from a NEWER schema than this build and would 500 on
restored rows, so we block until the code is upgraded.
Legacy sets predate `datarev`; fall back to the old head-subset check on
`schemarev` so they stay listable and restorable as before.
Also checks plugin revisions when present: a set from a newer plugin chain
is refused even if the core revision is known, because plugin table schema
could be ahead of what this build understands.
Legacy sets predate `datarev` and `dataplugins`; fall back to the old
head-subset check on `schemarev` so they stay listable and restorable.
"""
if not manifest:
return False
datarev = manifest.get('datarev')
if datarev:
# Check if manifest has datarev key (even if empty)
if 'datarev' in manifest:
datarev = manifest['datarev']
# Empty string means backup.sh couldn't read alembic_version (error case)
# Treat as incompatible - don't fall back to the legacy check this was
# introduced to replace
if not datarev:
return False
# Check core revision is known
known = _known_core_revisions()
if not known:
return False
return datarev in known
if datarev not in known:
return False
# Check plugin revisions if present
dataplugins = manifest.get('dataplugins')
if dataplugins:
known_plugins = _known_plugin_revisions()
for plugin_name, plugin_rev in dataplugins.items():
# Refuse if this build doesn't have the plugin at all
if plugin_name not in known_plugins:
return False
# Refuse if the plugin exists but this revision is unknown
if plugin_rev not in known_plugins[plugin_name]:
return False
return True
# Legacy: fall back to schemarev head-subset check for pre-datarev sets
setrevs = set(manifest.get('schemarev', {}).get('core', []))
currevs = set(current.get('core', []))
if not setrevs or not currevs:
@@ -448,42 +573,51 @@ def finalize_restore():
dump that sits at an older revision leaves every newer-migration table
throwing 1054 Unknown column.
Idempotent and safe to call when nothing is pending: if there is no
.upgrade-requested sentinel it reports there was nothing to finalize.
Supports re-arming a failed upgrade: if .upgrade-failed exists, moves it back
to .upgrade-requested so the operator can retry after fixing the issue.
Idempotent and safe to call when nothing is pending: if there is no sentinel
it reports there was nothing to finalize. Uses an atomic claim mechanism to
prevent races when multiple workers poll this endpoint simultaneously.
"""
root = _backups_root()
sentinel = os.path.join(root, '.upgrade-requested')
try:
with open(sentinel, 'r', encoding='utf-8') as handle:
setname = handle.read().strip()
except OSError:
# Check if backup store is configured
if not os.path.isdir(root):
return error_response(ErrorCodes.NOT_FOUND,
'Backup store not configured',
http_code=404)
# Support re-arming a failed upgrade
failed_sentinel = os.path.join(root, UPGRADE_FAILED)
if os.path.isfile(failed_sentinel):
try:
with open(failed_sentinel, 'r', encoding='utf-8') as handle:
setname = handle.read().strip()
# Re-arm by moving failed back to requested
sentinel = os.path.join(root, UPGRADE_SENTINEL)
os.rename(failed_sentinel, sentinel)
return success_response(
data={'retried': True, 'snapshotid': setname},
message='Failed upgrade re-armed for retry.')
except OSError as error:
return error_response(ErrorCodes.INTERNAL_ERROR,
'Could not re-arm failed upgrade: ' + str(error),
http_code=500)
# Try to claim a pending upgrade
setname = _claim_upgrade()
if not setname:
return success_response(
data={'finalized': False},
data={'queued': False},
message='No restore is awaiting schema upgrade.')
_write_status('restore', 'upgrading', setname,
'Upgrading restored schema to code')
try:
_run_schema_upgrade()
except Exception as error: # noqa: BLE001 - report any upgrade failure
_write_status('restore', 'failed', setname,
'Schema upgrade failed: ' + str(error))
_append_history('restore', 'failed', setname,
'Schema upgrade failed: ' + str(error))
return error_response(ErrorCodes.INTERNAL_ERROR,
'Post-restore schema upgrade failed: ' + str(error),
http_code=500)
try:
os.remove(sentinel)
except OSError:
pass
_write_status('restore', 'done', setname, 'Restore complete')
_append_history('restore', 'success', setname, 'Restore complete')
AuditLog.log('restore-finalized', 'Backup', entityname=setname)
# Queue the upgrade but don't run it inline (it can outlast the request timeout)
# The watcher or a background worker will pick up the claim and run it
return success_response(
data={'finalized': True, 'snapshotid': setname},
message='Restore finalized: schema upgraded to the running code.')
data={'queued': True, 'snapshotid': setname},
message='Schema upgrade queued.')
@shopdbbackups_bp.route('/status', methods=['GET'])