512cc9be76
A 'share' kind keeps its bytes on the file share and gives ShopDB only a hash and a UNC path. That is right for an opaque vendor file, but it means the app cannot answer the question a tech actually arrives with - what does this part marker hold, and which pattern disappeared last month. Reading the share from the app server is not an option: the download routes refuse to mount it on purpose, because a permissions slip there becomes an unexplained empty panel. So the collector now accepts an optional manifest alongside the hash: the index, not the content. Names, sizes, per-file hashes, and the marking system each pattern was authored for. Advisory throughout - a client that sends none behaves exactly as before, and dedup still keys on contenthash. The marker asset gains a Marker Contents card built from it: counts by type, the controller family, and a diff against the previous snapshot in the same source chain. Removed files are named rather than counted, because "1 removed" only tells someone to go and look. Files edited in place are named too - the file set can be identical while the content hash moved, and reporting no change there would contradict the reason the revision exists. The manifest is deliberately NOT covered by contenthash, which attests the bytes on the share. A client that starts describing itself therefore posts an unchanged hash, so a no-op backfills a manifest the row does not yet have - otherwise the card would appear only on markers that had recently changed, and stay hidden on every healthy one. The same no-op refreshes sharepath, so a snapshot that moved on the share stops being pointed at where it used to be. Manifests are validated at ingest rather than at render: a malformed one would otherwise become the latest revision and fail every view of the asset page until the content next changed, outliving the client bug that caused it. sharedir() for the marker kind is overridden to the layout the site actually uses, so what ShopDB displays and validates matches what the collector writes.
384 lines
15 KiB
Python
384 lines
15 KiB
Python
"""Backups API.
|
|
|
|
Read + download only. Revisions are created exclusively through the ADR-006
|
|
collector endpoint (POST /api/collector/backups), so there is deliberately no
|
|
create route here: a hand-posted "backup" that never came off a machine would
|
|
poison the history this feature exists to provide.
|
|
|
|
Downloads differ by storage backend. 'shopdb' kinds re-render from the stored
|
|
projection, which is what lets one revision produce either .reg dialect. 'share'
|
|
kinds are not streamed - ShopDB returns the UNC path for the tech to open,
|
|
because requiring the app server to mount the SFLD share would turn a
|
|
permissions slip into an unexplained empty download.
|
|
"""
|
|
|
|
from datetime import timezone
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from flask import Blueprint, current_app, request, Response
|
|
from flask_jwt_extended import jwt_required
|
|
|
|
from shopdb.api import (
|
|
db, Asset,
|
|
success_response, error_response, ErrorCodes,
|
|
require_permission,
|
|
)
|
|
|
|
from ..models import BackupRevision
|
|
from ..models.backup import _utciso
|
|
from ..services.registry import REGISTRY, getkind
|
|
from ..services.retention import samesource
|
|
|
|
backups_bp = Blueprint('backups', __name__)
|
|
|
|
|
|
_DEFAULTTZ = 'America/New_York'
|
|
|
|
|
|
def _sitezone():
|
|
"""Site-configured IANA zone (settings key site_timezone).
|
|
|
|
Same lookup the notifications plugin uses. A stored timestamp is UTC, so
|
|
anything rendered server-side has to be converted or it shows the wrong
|
|
wall clock for the site - four hours out at the reference site.
|
|
"""
|
|
from shopdb.api import Setting
|
|
row = Setting.query.filter_by(key='site_timezone').first()
|
|
name = row.value if row and row.value else _DEFAULTTZ
|
|
try:
|
|
return ZoneInfo(name)
|
|
except Exception:
|
|
return ZoneInfo(_DEFAULTTZ)
|
|
|
|
|
|
def _label(revision):
|
|
"""Panel list title: kind-agnostic, readable at a glance.
|
|
|
|
Rendered in the SITE zone, not UTC: this string is baked server-side and
|
|
the client cannot correct it afterwards.
|
|
"""
|
|
when = revision.collectedat or revision.createdat
|
|
if not when:
|
|
return 'unknown time'
|
|
local = when.replace(tzinfo=timezone.utc).astimezone(_sitezone())
|
|
stamp = local.strftime('%Y-%m-%d %H:%M')
|
|
if revision.sourcefilename:
|
|
return '{} - {}'.format(stamp, revision.sourcefilename)
|
|
return stamp
|
|
|
|
|
|
def _summary(revision, islatest=False):
|
|
data = revision.to_dict()
|
|
data['label'] = _label(revision)
|
|
data['islatest'] = islatest
|
|
kind = getkind(revision.backupkind)
|
|
data['formats'] = kind.formats() if kind else []
|
|
return data
|
|
|
|
|
|
@backups_bp.route('/kinds', methods=['GET'])
|
|
@jwt_required()
|
|
@require_permission('backups.view')
|
|
def list_kinds():
|
|
"""Registered backup kinds, for UI that needs to enumerate them."""
|
|
return success_response([
|
|
{
|
|
'key': kind.key,
|
|
'displayname': kind.displayname,
|
|
'storagebackend': kind.storagebackend,
|
|
'assettypes': list(kind.assettypes),
|
|
'formats': kind.formats(),
|
|
}
|
|
for kind in REGISTRY.values()
|
|
])
|
|
|
|
|
|
@backups_bp.route('/asset/<int:assetid>', methods=['GET'])
|
|
@jwt_required()
|
|
@require_permission('backups.view')
|
|
def asset_revisions(assetid):
|
|
"""Revision history for one asset, newest first.
|
|
|
|
This is the asset-panel endpoint. Optional ?kind= narrows to a single kind,
|
|
which is how each per-kind panel scopes itself.
|
|
"""
|
|
asset = db.session.get(Asset, assetid)
|
|
if asset is None:
|
|
return error_response(ErrorCodes.NOT_FOUND, 'Asset not found', http_code=404)
|
|
|
|
query = db.session.query(BackupRevision).filter(
|
|
BackupRevision.assetid == assetid)
|
|
|
|
kindkey = (request.args.get('kind') or '').strip().lower()
|
|
if kindkey:
|
|
if getkind(kindkey) is None:
|
|
return error_response(ErrorCodes.VALIDATION_ERROR,
|
|
'Unknown kind: {}'.format(kindkey))
|
|
query = query.filter(BackupRevision.backupkind == kindkey)
|
|
|
|
revisions = query.order_by(BackupRevision.backuprevisionid.desc()).all()
|
|
|
|
# "Latest" is per kind, not per asset, so an unfiltered listing still marks
|
|
# the current revision of each kind correctly.
|
|
seen = set()
|
|
out = []
|
|
for revision in revisions:
|
|
islatest = revision.backupkind not in seen
|
|
seen.add(revision.backupkind)
|
|
out.append(_summary(revision, islatest=islatest))
|
|
return success_response(out)
|
|
|
|
|
|
@backups_bp.route('/asset/<int:assetid>/info', methods=['GET'])
|
|
@jwt_required()
|
|
@require_permission('backups.view')
|
|
def asset_info(assetid):
|
|
"""A kind's 'at a glance' card, built from its LATEST revision.
|
|
|
|
Generic across kinds: the requested kind owns both the panel declaration
|
|
and the payload (see BackupKind.infopanel / buildinfo), so a successor to
|
|
NTLARS/DNC gets its own card without a new endpoint.
|
|
|
|
Empty when the machine has no revision of that kind yet, which the generic
|
|
renderer turns into the panel's empty text.
|
|
"""
|
|
kindkey = (request.args.get('kind') or 'ntlars').strip().lower()
|
|
kind = getkind(kindkey)
|
|
if kind is None:
|
|
return error_response(ErrorCodes.VALIDATION_ERROR,
|
|
'Unknown kind: {}'.format(kindkey))
|
|
|
|
revision = (db.session.query(BackupRevision)
|
|
.filter(BackupRevision.assetid == assetid,
|
|
BackupRevision.backupkind == kindkey)
|
|
.order_by(BackupRevision.backuprevisionid.desc())
|
|
.first())
|
|
if revision is None:
|
|
return success_response({'fields': [], 'sectioncount': 0})
|
|
|
|
try:
|
|
projection = revision.payload
|
|
except ValueError:
|
|
# payload deliberately raises on unreadable payloadjson so a corrupt row
|
|
# is named rather than mistaken for an opaque one. That is right for a
|
|
# download; for an at-a-glance card it would take the whole asset page
|
|
# down over one bad row, so the card degrades to empty instead.
|
|
current_app.logger.warning(
|
|
'backuprevision %s has unreadable payloadjson; info card empty',
|
|
revision.backuprevisionid)
|
|
return success_response({'fields': [], 'sectioncount': 0})
|
|
|
|
data = kind.buildinfo(
|
|
projection, assetid,
|
|
partmarkertypes=current_app.config.get('BACKUPS_PARTMARKER_TYPES'),
|
|
# A kind that diffs against its own previous revision needs to know
|
|
# which revision it is looking at. Passed generically rather than
|
|
# queried inside the kind, so the endpoint stays the single place that
|
|
# decides "latest".
|
|
revisionid=revision.backuprevisionid,
|
|
# The card names the PC a snapshot came from, and chains its diff on the
|
|
# same value. Passed from the column rather than read out of the stored
|
|
# manifest, which carries its own copy that can disagree.
|
|
sourcehostname=revision.sourcehostname)
|
|
data['backuprevisionid'] = revision.backuprevisionid
|
|
data['collectedat'] = _utciso(revision.collectedat)
|
|
return success_response(data)
|
|
|
|
|
|
@backups_bp.route('/revisions/<int:backuprevisionid>', methods=['GET'])
|
|
@jwt_required()
|
|
@require_permission('backups.view')
|
|
def get_revision(backuprevisionid):
|
|
"""One revision, including its parsed payload where there is one."""
|
|
revision = db.session.get(BackupRevision, backuprevisionid)
|
|
if revision is None:
|
|
return error_response(ErrorCodes.NOT_FOUND, 'Revision not found',
|
|
http_code=404)
|
|
data = revision.to_dict(includepayload=True)
|
|
data['label'] = _label(revision)
|
|
kind = getkind(revision.backupkind)
|
|
data['formats'] = kind.formats() if kind else []
|
|
data['displayname'] = kind.displayname if kind else revision.backupkind
|
|
return success_response(data)
|
|
|
|
|
|
@backups_bp.route('/revisions/<int:backuprevisionid>/download', methods=['GET'])
|
|
@jwt_required()
|
|
@require_permission('backups.download')
|
|
def download_revision(backuprevisionid):
|
|
"""Render a revision back to its native file format.
|
|
|
|
?format=ntlars (default for NTLARS) omits WOW6432Node - the form the
|
|
NTLARS Load... button expects
|
|
?format=wow6432node includes WOW6432Node - for `reg import` on 64-bit
|
|
"""
|
|
revision = db.session.get(BackupRevision, backuprevisionid)
|
|
if revision is None:
|
|
return error_response(ErrorCodes.NOT_FOUND, 'Revision not found',
|
|
http_code=404)
|
|
|
|
kind = getkind(revision.backupkind)
|
|
if kind is None:
|
|
return error_response(ErrorCodes.VALIDATION_ERROR,
|
|
'Unknown kind: {}'.format(revision.backupkind))
|
|
|
|
if revision.storagebackend == 'share':
|
|
# Not an error: the file exists, ShopDB just is not the one serving it.
|
|
return success_response({
|
|
'storagebackend': 'share',
|
|
'sharepath': revision.sharepath,
|
|
'sourcefilename': revision.sourcefilename,
|
|
'message': 'This backup lives on the SFLD share. Open the path directly.',
|
|
})
|
|
|
|
formats = kind.formats()
|
|
formatid = (request.args.get('format') or '').strip().lower()
|
|
if not formatid:
|
|
formatid = formats[0]['id'] if formats else ''
|
|
if not any(f['id'] == formatid for f in formats):
|
|
return error_response(
|
|
ErrorCodes.VALIDATION_ERROR,
|
|
'Unknown format {!r} for kind {}'.format(formatid, kind.key))
|
|
|
|
asset = db.session.get(Asset, revision.assetid)
|
|
assetnumber = asset.assetnumber if asset else str(revision.assetid)
|
|
when = revision.collectedat or revision.createdat
|
|
|
|
comments = [
|
|
'{} backup from ShopDB'.format(kind.displayname),
|
|
'Asset: {}'.format(assetnumber),
|
|
'Captured: {}'.format(when.strftime('%Y-%m-%d %H:%M:%S') if when else 'unknown'),
|
|
'Source PC: {}'.format(revision.sourcehostname or 'unknown'),
|
|
'Revision: {} ({})'.format(revision.backuprevisionid,
|
|
(revision.contenthash or '')[:12]),
|
|
]
|
|
|
|
try:
|
|
raw, ext, mimetype = kind.render(revision.payload, formatid,
|
|
comments=comments)
|
|
except ValueError as exc:
|
|
return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
|
|
|
|
# Named for the MACHINE, not the revision or the kind: a tech restoring bay
|
|
# 3204 wants 3204.reg, matching how the per-machine backups on the share
|
|
# have always been named. sourcefilename is deliberately not reused here -
|
|
# a seeded revision carries "3204.reg" already and appending an extension
|
|
# to it produced "3204.reg.reg".
|
|
filename = '{}{}'.format(assetnumber, ext)
|
|
if formatid == 'wow6432node':
|
|
# The two dialects must not collide in a downloads folder, and the
|
|
# suffix says which one will import correctly outside NTLARS.
|
|
filename = '{}-wow6432node{}'.format(assetnumber, ext)
|
|
|
|
return Response(
|
|
raw,
|
|
mimetype=mimetype,
|
|
headers={'Content-Disposition': 'attachment; filename="{}"'.format(filename)},
|
|
)
|
|
|
|
|
|
@backups_bp.route('/revisions/<int:backuprevisionid>/diff', methods=['GET'])
|
|
@jwt_required()
|
|
@require_permission('backups.view')
|
|
def diff_revision(backuprevisionid):
|
|
"""Diff two revisions of the same asset+kind.
|
|
|
|
?against=<id> picks the comparison revision; default is the immediately
|
|
preceding one, which answers "what changed" without the user choosing.
|
|
"""
|
|
revision = db.session.get(BackupRevision, backuprevisionid)
|
|
if revision is None:
|
|
return error_response(ErrorCodes.NOT_FOUND, 'Revision not found',
|
|
http_code=404)
|
|
if revision.storagebackend != 'shopdb':
|
|
return error_response(
|
|
ErrorCodes.VALIDATION_ERROR,
|
|
'Kind {} stores opaque files and cannot be diffed'.format(
|
|
revision.backupkind))
|
|
|
|
againstid = request.args.get('against', type=int)
|
|
if againstid:
|
|
other = db.session.get(BackupRevision, againstid)
|
|
else:
|
|
# The previous revision FROM THE SAME PC. Several PCs can share a
|
|
# machine number, and diffing across them reported one device's COM port
|
|
# as a change on another, which is not a change at all.
|
|
other = (db.session.query(BackupRevision)
|
|
.filter(BackupRevision.assetid == revision.assetid,
|
|
BackupRevision.backupkind == revision.backupkind,
|
|
samesource(revision.sourcehostname),
|
|
BackupRevision.backuprevisionid < revision.backuprevisionid)
|
|
.order_by(BackupRevision.backuprevisionid.desc())
|
|
.first())
|
|
|
|
if other is None:
|
|
return success_response({
|
|
'backuprevisionid': revision.backuprevisionid,
|
|
'againstid': None,
|
|
'changes': [],
|
|
'message': 'No earlier revision to compare against.',
|
|
})
|
|
|
|
changes = _diffprojections(other.payload, revision.payload)
|
|
return success_response({
|
|
'backuprevisionid': revision.backuprevisionid,
|
|
'againstid': other.backuprevisionid,
|
|
'changes': changes,
|
|
'changecount': len(changes),
|
|
})
|
|
|
|
|
|
def _flatten(projection):
|
|
"""{(subkey, valuename): (type, data)} from a stored projection."""
|
|
flat = {}
|
|
for key in (projection or {}).get('keys', []):
|
|
path = key.get('path', '')
|
|
for name, entry in (key.get('values') or {}).items():
|
|
flat[(path, name)] = (entry.get('type'), entry.get('data'))
|
|
return flat
|
|
|
|
|
|
def _diffprojections(old, new):
|
|
oldflat = _flatten(old)
|
|
newflat = _flatten(new)
|
|
|
|
changes = []
|
|
for ref in sorted(set(oldflat) | set(newflat)):
|
|
path, name = ref
|
|
before = oldflat.get(ref)
|
|
after = newflat.get(ref)
|
|
if before == after:
|
|
continue
|
|
if before is None:
|
|
change = 'added'
|
|
elif after is None:
|
|
change = 'removed'
|
|
else:
|
|
change = 'changed'
|
|
changes.append({
|
|
'keypath': path,
|
|
'valuename': name,
|
|
'change': change,
|
|
'before': None if before is None else before[1],
|
|
'after': None if after is None else after[1],
|
|
'beforetype': None if before is None else before[0],
|
|
'aftertype': None if after is None else after[0],
|
|
})
|
|
return changes
|
|
|
|
|
|
@backups_bp.route('/dashboard/stale', methods=['GET'])
|
|
@jwt_required()
|
|
@require_permission('backups.view')
|
|
def dashboard_stale():
|
|
"""Chains whose backup has stopped running.
|
|
|
|
Thin: the rule lives in services/staleness.py, where it is testable without
|
|
an auth layer in the way. Keyed on the last CONFIRMED check, never on the
|
|
last change - dedup means an unchanged config writes no revision, so a card
|
|
keyed on revision age would flag most of a healthy fleet.
|
|
"""
|
|
from ..services.staleness import stalechains
|
|
|
|
return success_response(stalechains())
|