From f1e265190eb90b724de56bdb2b06d8e258cee68e Mon Sep 17 00:00:00 2001 From: Cam Proudlock Date: Tue, 25 Aug 2026 13:52:06 -0400 Subject: [PATCH] Let a share-backed backup say what is in it --- plugins/backups/api/routes.py | 25 +- plugins/backups/models/backup.py | 18 +- plugins/backups/plugin.py | 65 ++++ plugins/backups/services/partmarkerinfo.py | 176 ++++++++++ plugins/backups/services/registry.py | 59 +++- tests/test_plugins/test_backups.py | 381 ++++++++++++++++++++- 6 files changed, 712 insertions(+), 12 deletions(-) create mode 100644 plugins/backups/services/partmarkerinfo.py diff --git a/plugins/backups/api/routes.py b/plugins/backups/api/routes.py index b707fa81..9bed80c6 100644 --- a/plugins/backups/api/routes.py +++ b/plugins/backups/api/routes.py @@ -156,9 +156,30 @@ def asset_info(assetid): if revision is None: return success_response({'fields': [], 'sectioncount': 0}) + try: + projection = revision.payload + except ValueError: + # payload deliberately raises on unreadable payloadjson so a corrupt row + # is named rather than mistaken for an opaque one. That is right for a + # download; for an at-a-glance card it would take the whole asset page + # down over one bad row, so the card degrades to empty instead. + current_app.logger.warning( + 'backuprevision %s has unreadable payloadjson; info card empty', + revision.backuprevisionid) + return success_response({'fields': [], 'sectioncount': 0}) + data = kind.buildinfo( - revision.payload, assetid, - partmarkertypes=current_app.config.get('BACKUPS_PARTMARKER_TYPES')) + projection, assetid, + partmarkertypes=current_app.config.get('BACKUPS_PARTMARKER_TYPES'), + # A kind that diffs against its own previous revision needs to know + # which revision it is looking at. Passed generically rather than + # queried inside the kind, so the endpoint stays the single place that + # decides "latest". + revisionid=revision.backuprevisionid, + # The card names the PC a snapshot came from, and chains its diff on the + # same value. Passed from the column rather than read out of the stored + # manifest, which carries its own copy that can disagree. + sourcehostname=revision.sourcehostname) data['backuprevisionid'] = revision.backuprevisionid data['collectedat'] = _utciso(revision.collectedat) return success_response(data) diff --git a/plugins/backups/models/backup.py b/plugins/backups/models/backup.py index 2550eb3f..89e06357 100644 --- a/plugins/backups/models/backup.py +++ b/plugins/backups/models/backup.py @@ -67,7 +67,10 @@ class BackupRevision(db.Model): # (assetid, backupkind). contenthash = db.Column(db.String(64), nullable=False, index=True) - # Populated for storagebackend='shopdb' only. MEDIUMTEXT holding serialized + # Populated for storagebackend='shopdb', and optionally for 'share' kinds + # that send a manifest describing what their snapshot contains (the bytes + # still live on the share; this is the index, not the content). + # MEDIUMTEXT holding serialized # JSON rather than a native JSON column, matching geenforce.manifestjson. # Text also means the bytes come back exactly as written, so the canonical # key ordering the codec produces survives the round trip - a native JSON @@ -105,7 +108,18 @@ class BackupRevision(db.Model): @property def payload(self): - """Decoded payloadjson, or None for opaque (share-backed) kinds.""" + """Decoded payloadjson, or None when the row stores nothing. + + A 'shopdb' kind always has one: the parsed projection ShopDB is + authoritative for. A 'share' kind has one only when its client sends a + manifest describing the snapshot's contents, and None otherwise - which + includes every share revision written before manifests existed. + + Note the manifest is NOT covered by contenthash: the hash attests the + bytes on the share, so two posts with identical content but different + manifests dedup to one row. apply_collector_payload backfills a missing + manifest on a no-op for that reason. + """ if not self.payloadjson: return None try: diff --git a/plugins/backups/plugin.py b/plugins/backups/plugin.py index b784b884..250f6b64 100644 --- a/plugins/backups/plugin.py +++ b/plugins/backups/plugin.py @@ -263,6 +263,15 @@ class BackupsPlugin(BasePlugin): 'type': 'string', 'description': "Full UNC path. Required for 'share' kinds.", }, + 'manifest': { + 'type': 'object', + 'description': "Optional description of a 'share' kind's " + "contents - what is in the snapshot, not " + "the bytes themselves. Lets an asset panel " + "list a snapshot and diff it against the " + "previous one without the app server " + "mounting the share.", + }, 'sourcefilename': {'type': 'string'}, 'sourcehostname': { 'type': 'string', @@ -325,6 +334,42 @@ class BackupsPlugin(BasePlugin): "contenthash and sharepath are required for kind '{}'".format( kindkey)) + # A share kind may also describe itself. The bytes stay on the + # share - that part is unchanged - but a manifest of what the + # snapshot CONTAINS is small, structured, and the only way an asset + # panel can list a part marker's patterns or say which one + # disappeared last month. The alternative is the app server + # mounting SFLD, which the read routes deliberately refuse to do. + # + # It is advisory: dedup still keys on contenthash, so a client that + # sends no manifest behaves exactly as before. + manifest = payload.get('manifest') + if manifest is not None: + # Validated at the door, not at render time. A shallow check + # here would let a malformed manifest become the LATEST + # revision, and every view of the asset panel would then fail + # until the content next changed - a client bug turning into a + # broken page that outlives it. + if not isinstance(manifest, dict): + raise ValueError('manifest must be an object') + files = manifest.get('files') + if files is not None: + if not isinstance(files, list): + raise ValueError('manifest files must be a list') + for entry in files: + if not isinstance(entry, dict): + raise ValueError( + 'each manifest file must be an object') + if not isinstance(entry.get('name'), str) or not entry['name']: + raise ValueError( + 'each manifest file needs a name') + size = entry.get('bytes') + if size is not None and not isinstance(size, int): + raise ValueError( + 'manifest file {!r} has a non-integer size'.format( + entry['name'])) + projection = manifest + # Dedup is against the LATEST row of THIS SOURCE's chain, not the # asset's. Several PCs legitimately share one machine number here - the # part markers on 0613, 0615 and WJPRT do, and their configs genuinely @@ -356,6 +401,26 @@ class BackupsPlugin(BasePlugin): # backup that died six months ago. Cheap: one column on a row that # already exists, no new history. latest.lastseenat = datetime.utcnow() + + # BACKFILL, on a no-op. The contenthash attests the BYTES, not the + # manifest, so a client that starts describing itself posts the same + # hash as before and would otherwise no-op forever - leaving the + # index unstored on exactly the markers that are healthy and + # unchanging, which is most of them. A revision that already has a + # manifest is left alone; this only fills a hole. + # + # Honest by construction: the hash proves the content is identical, + # so indexing it now describes the same snapshot. lastseenat + # semantics are untouched. + if projection is not None and latest.payload is None: + latest.payload = projection + # Same reasoning for the pointer: a snapshot that moved on the share + # (a changed layout, a renamed asset directory) reports a new + # sharepath with unchanged bytes, and the stored path would + # otherwise go on pointing at a directory nobody writes to. + if sharepath and latest.sharepath != sharepath: + latest.sharepath = sharepath + db.session.flush() return { 'action': 'noop', diff --git a/plugins/backups/services/partmarkerinfo.py b/plugins/backups/services/partmarkerinfo.py new file mode 100644 index 00000000..784da1f5 --- /dev/null +++ b/plugins/backups/services/partmarkerinfo.py @@ -0,0 +1,176 @@ +"""Marker Contents card. + +What a part marker holds right now, on the marker's own asset page, plus what +changed since the previous snapshot. + +WHY THIS EXISTS AT ALL + + A 'share' kind keeps its bytes on SFLD and only a UNC pointer in ShopDB, so + the obvious way to list a snapshot's contents - read the directory - is + exactly what the read routes refuse to do: mounting the share in the app + server turns a permissions slip into an unexplained empty panel. Instead the + collector sends a MANIFEST alongside the hash, and this builds the card from + that. Small, structured, and it works from a server that cannot see the + share at all. + +WHAT IT ANSWERS + + "What patterns are on this marker" - the question a tech arrives with, and + today the only way to answer it is to walk to the bay and read the panel. + + "What changed" - the added/removed diff against the previous snapshot. This + is the reason the snapshot layout exists: a pattern deleted from a + controller is invisible everywhere else until someone needs it. Here it is + dated, and the snapshot that still holds it is one row above in the history. + +The manifest is written by Backup-PartMarker.ps1 and carries, per file: name, +bytes, sha256, and the marking system it was authored for. +""" + + +def _previousmanifest(assetid, currentid): + """The manifest of the snapshot before this one, or None. + + Scoped to the same source host, matching how dedup chains revisions: two + PCs reporting the same marker keep separate histories, and diffing across + them would report every file as added and removed at once. + + The host comes from the REVISION's own column, not from a field inside the + manifest. Those can disagree - the column is what the collector recorded and + what retention chains on - and using the manifest's copy would silently diff + against the wrong chain, or against nothing at all. + """ + from shopdb.api import db + from ..models import BackupRevision + from .retention import samesource + + current = db.session.get(BackupRevision, currentid) + if current is None: + return None, None + sourcehostname = current.sourcehostname + + row = (db.session.query(BackupRevision) + .filter(BackupRevision.assetid == assetid, + BackupRevision.backupkind == 'partmarker', + BackupRevision.backuprevisionid < currentid, + samesource(sourcehostname)) + .order_by(BackupRevision.backuprevisionid.desc()) + .first()) + if row is None: + return None, None + when = row.collectedat or row.createdat + try: + return row.payload, when + except ValueError: + # A corrupt neighbour must not take the card down - the current + # snapshot is still worth showing without its diff. + return None, None + + +def _hashes(manifest): + """name -> sha256, for spotting a file edited in place.""" + return {entry.get('name'): entry.get('sha256') + for entry in (manifest or {}).get('files', []) + if entry.get('name') and entry.get('sha256')} + + +def _names(manifest): + return {entry.get('name') for entry in (manifest or {}).get('files', []) + if entry.get('name')} + + +def _summarise(files): + """Counts by extension, so 34 patterns and 12 settings read as two facts.""" + groups = {} + for entry in files: + name = entry.get('name') or '' + ext = name.rsplit('.', 1)[-1].upper() if '.' in name else 'OTHER' + groups[ext] = groups.get(ext, 0) + 1 + labels = {'PAT': 'Patterns', 'DAT': 'Settings files', 'TFF': 'Fonts'} + return [(labels.get(ext, ext), count) for ext, count in sorted(groups.items())] + + +def build(projection, assetid, revisionid=None, sourcehostname=None, + **options): + """Build the card from a stored partmarker manifest. + + Returns the keyvalue shape. An older revision predating manifests has no + projection at all, which renders as no card rather than an error - the + revision itself is still listed in the history panel. + """ + # "No manifest" and "a manifest listing nothing" are different facts. The + # first is an older revision predating manifests, which has no card. The + # second is a controller that has been wiped or replaced - precisely the + # disaster the Removed line exists to report - so it must render, with the + # diff intact. + if not projection: + return {'fields': [], 'sectioncount': 0} + manifest = projection + files = manifest.get('files') or [] + + fields = [] + + for label, count in _summarise(files): + fields.append({'label': label, 'value': str(count)}) + + total = sum(int(entry.get('bytes') or 0) for entry in files) + fields.append({'label': 'Total size', 'value': '{:,} bytes'.format(total)}) + + if manifest.get('family'): + fields.append({'label': 'Controller', 'value': manifest['family']}) + + # The heads patterns were authored for. Worth showing because they can + # DIFFER from the controller holding them - a TMC420 in the reference site + # holds both a TMP3200/470 and a TMP4210/420 pattern - and pushing a pattern + # back onto a system with a different head is a mistake worth catching + # before the restore, not after. + systems = manifest.get('systems') or [] + if systems: + fields.append({'label': 'Authored for', 'value': ', '.join(systems)}) + + # From the revision's own column where the caller passes it: the manifest + # carries its own copy, and the two can disagree. The column is what the + # collector recorded and what the diff above is chained on, so naming a + # different PC here would contradict the comparison shown beside it. + capturedfrom = sourcehostname or manifest.get('sourcehost') + if capturedfrom: + fields.append({'label': 'Captured from', 'value': capturedfrom}) + + # The diff. Only meaningful with a predecessor: on a marker's first snapshot + # every file would otherwise read as "added", which says nothing. + if revisionid: + previous, when = _previousmanifest(assetid, revisionid) + if previous is not None: + before, now = _names(previous), _names(manifest) + added = sorted(now - before) + removed = sorted(before - now) + + # Same name, different bytes. A revision exists only because the + # content hash moved, so if the file SET is identical then something + # was edited in place - and saying "no files added or removed" would + # be a flat contradiction of why this row exists. The per-file + # sha256 is in the manifest, so the edited files can be named. + beforehashes = _hashes(previous) + changed = sorted(name for name, digest in _hashes(manifest).items() + if name in beforehashes and beforehashes[name] != digest) + + # Date the window. Retention prunes middle revisions, so "previous" + # can be months older than the row before it in the history, and an + # undated diff would quietly attribute an old removal to this + # capture. + label = 'Since {}'.format(when.strftime('%Y-%m-%d')) if when else 'Since previous' + + if added: + fields.append({'label': 'Added', 'value': ', '.join(added)}) + if removed: + # The one the whole design exists for. Named, not counted - + # "1 removed" would send someone to the share to find out which. + fields.append({'label': 'Removed', 'value': ', '.join(removed)}) + if changed: + fields.append({'label': 'Changed', 'value': ', '.join(changed)}) + if not (added or removed or changed): + fields.append({'label': label, 'value': 'No files added, removed or changed'}) + else: + fields.append({'label': 'Compared with', 'value': label.replace('Since ', '')}) + + return {'fields': fields, 'sectioncount': 1} diff --git a/plugins/backups/services/registry.py b/plugins/backups/services/registry.py index 45df56db..3080a39b 100644 --- a/plugins/backups/services/registry.py +++ b/plugins/backups/services/registry.py @@ -279,11 +279,60 @@ class PartMarkerKind(BackupKind): key = 'partmarker' displayname = 'Part Marker Configuration' storagebackend = 'share' - # NO asset panel. The marker's DNC settings are a tab on the single DNC Info - # card, and a separate card that is empty on 145 of 147 machines earns - # nobody anything. The kind is still fully live - it stores, dedupes and - # serves revisions, which are listed on the backup history page. - assettypes = [] + # Marker assets are core type 'machine', like the other 144 machines - but + # only a marker ever has a partmarker revision, and the list renderer hides + # a panel with no rows and no empty text. So declaring 'machine' shows the + # panel on markers and nowhere else, with no special-casing. + # + # This was deliberately EMPTY while backups were filed against the operation + # number, when a partmarker card would have appeared on every machine and + # said nothing. Since a marker PC's backup is filed against its own marker + # asset (see markerforsource), the panel has a home of its own. + assettypes = ['machine'] + + def sharedir(self, machinetype, identifier, shareroot=None): + """\\partmarker\\\\ - the SITE's layout, which + inverts the base class's \\\\\\. + + Overridden rather than left to the default so what ShopDB displays and + validates matches what the collector actually writes. Still advisory: + the authoritative path is the sharepath on each revision, which is what + keeps older snapshots resolvable after a PC swap renames the asset. + + Snapshots live in timestamped subdirectories BELOW this - one per + capture, so a pattern deleted from the controller stays recoverable + from the snapshot that still holds it. + """ + root = shareroot or DEFAULTSHAREROOT + if not root: + return '' + return '{}\\{}\\{}'.format(root, self.key, identifier or 'unknown') + + def infopanel(self): + """Contents of the latest snapshot, on the marker's own asset page. + + A 'share' kind stores no bytes in ShopDB, so this is built from the + manifest the collector sends alongside the hash. Without it the app + server would have to mount SFLD to say what a snapshot contains, which + the read routes refuse to do on purpose. + """ + return { + 'id': 'backups-partmarker-contents', + 'title': 'Marker Contents', + 'assettypes': ['machine'], + 'endpoint': '/api/backups/asset/{assetid}/info?kind=partmarker', + 'render': 'keyvalue', + # Above the history panel: "what is on this marker" is the question + # a tech arrives with; the history is for the rarer restore. + 'position': 39, + } + + def buildinfo(self, projection, assetid, **options): + from . import partmarkerinfo + return partmarkerinfo.build( + projection, assetid, + revisionid=options.get('revisionid'), + sourcehostname=options.get('sourcehostname')) def resolveassetid(self, payload): from shopdb.api import db, Asset diff --git a/tests/test_plugins/test_backups.py b/tests/test_plugins/test_backups.py index e2123cc3..ff5f9155 100644 --- a/tests/test_plugins/test_backups.py +++ b/tests/test_plugins/test_backups.py @@ -207,10 +207,21 @@ def test_partmarker_kind_is_share_backed_and_not_renderable(): def test_partmarker_sharedir_is_under_the_configured_backups_root(): + """The marker kind keys its tree on the MARKER, not the machine type. + + It overrides the base layout (\\\\\\) + because the site files marker snapshots as \\partmarker\\\\, + with one timestamped snapshot directory per capture below that. The + override exists so what ShopDB displays matches what the collector writes. + """ root = r'\\fileserver\shopfloor\backups' path = registry.getkind('partmarker').sharedir('lathe', '3204', shareroot=root) assert path.startswith(root) - assert path.endswith(r'lathe\3204\partmarker') + assert path.endswith(r'partmarker\3204') + # The machine type is deliberately NOT in the path: a marker is identified + # by its own asset, and the type of the machine it serves is irrelevant to + # where its patterns live. + assert 'lathe' not in path def test_sharedir_is_empty_when_no_root_is_configured(): @@ -608,8 +619,19 @@ def test_infopanel_is_kind_owned_so_a_successor_ships_its_own_card(): panel = ntlarskind.infopanel() assert panel['id'] == 'backups-dncinfo' assert 'kind=ntlars' in panel['endpoint'] - # A kind with nothing to summarise contributes no panel at all. - assert registry.getkind('partmarker').infopanel() is None + + # The part marker kind now ships a card of its own, built from the manifest + # the collector sends beside the hash. It could not before: a share kind + # stored no description of its contents, and reading the share from the app + # server is refused on purpose. + markerpanel = registry.getkind('partmarker').infopanel() + assert markerpanel['id'] == 'backups-partmarker-contents' + assert 'kind=partmarker' in markerpanel['endpoint'] + # Marker assets are core type 'machine' like the other 144, and the panel + # is kept off those by having no rows and no empty text rather than by any + # special-casing. + assert markerpanel['assettypes'] == ['machine'] + assert 'empty' not in markerpanel def test_base_kind_buildinfo_is_an_empty_card(): @@ -1006,3 +1028,356 @@ def test_retention_reads_the_key_the_settings_page_writes(app, db): # The helper cannot see it, which is the whole defect: it looks under # plugin.backups.backups_retentioncount, which nothing writes. assert int(BackupsPlugin().get_setting('backups_retentioncount', 0) or 0) == 0 + + +# --------------------------------------------------------------------------- +# Share kinds that describe themselves +# +# A 'share' kind keeps its bytes on SFLD, so ShopDB knew a snapshot existed and +# where, but nothing about what was in it. The collector now accepts an optional +# manifest - the index, not the content - which is what lets a marker's asset +# page list its patterns and name the one that disappeared. Reading the share +# from the app server is refused on purpose, so this is the only route. +# --------------------------------------------------------------------------- + +def _markermanifest(names, sourcehost='FBYTNCX3', family='TMC470'): + return { + 'capturedat': '2026-08-25T12:00:00Z', + 'machine': '3204', + 'sourcehost': sourcehost, + 'family': family, + 'systems': ['TMP1700/470'], + 'files': [{'name': n, 'bytes': 266, 'sha256': 'ab' * 32, + 'system': 'TMP1700/470'} for n in names], + } + + +def test_a_share_kind_may_send_a_manifest_and_it_is_stored(bk_app, bk_plugin): + from plugins.backups.models import BackupRevision + + with bk_app.app_context(): + result = bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', + 'backupkind': 'partmarker', + 'contenthash': 'a' * 64, + 'sharepath': r'\\host\share\partmarker\3204\20260825-120000', + 'manifest': _markermanifest(['TESTPAT.PAT', 'NET.DAT']), + }) + assert result['action'] == 'created' + revision = _db.session.get(BackupRevision, result['backuprevisionid']) + # Bytes still live on the share; only the index is in the database. + assert revision.storagebackend == 'share' + assert revision.sharepath.endswith('20260825-120000') + assert [f['name'] for f in revision.payload['files']] == \ + ['TESTPAT.PAT', 'NET.DAT'] + + +def test_a_share_kind_without_a_manifest_still_works(bk_app, bk_plugin): + """The manifest is advisory. A client that sends none behaves as before.""" + from plugins.backups.models import BackupRevision + + with bk_app.app_context(): + result = bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', + 'backupkind': 'partmarker', + 'contenthash': 'b' * 64, + 'sharepath': r'\\host\share\partmarker\3204\20260825-130000', + }) + assert result['action'] == 'created' + assert _db.session.get( + BackupRevision, result['backuprevisionid']).payload is None + + +def test_a_manifest_that_is_not_an_object_is_rejected(bk_app, bk_plugin): + with bk_app.app_context(): + with pytest.raises(ValueError): + bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', + 'backupkind': 'partmarker', + 'contenthash': 'c' * 64, + 'sharepath': r'\\host\share\x', + 'manifest': 'not an object', + }) + + +def test_marker_card_counts_by_kind_and_names_what_was_removed(bk_app, bk_plugin): + """The removed line is the reason the snapshot layout exists. + + A pattern deleted from a controller is invisible everywhere else until + somebody needs it. It is NAMED rather than counted - "1 removed" would send + a tech to the share to find out which one. + """ + from plugins.backups.services import partmarkerinfo + from plugins.backups.models import BackupRevision + + with bk_app.app_context(): + first = bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': 'd' * 64, 'sharepath': r'\\host\s\1', + 'sourcehostname': 'FBYTNCX3', + 'manifest': _markermanifest(['A.PAT', 'B.PAT', 'NET.DAT']), + }) + _db.session.commit() + second = bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': 'e' * 64, 'sharepath': r'\\host\s\2', + 'sourcehostname': 'FBYTNCX3', + 'manifest': _markermanifest(['A.PAT', 'C.PAT', 'NET.DAT']), + }) + _db.session.commit() + assert first['action'] == 'created' and second['action'] == 'created' + + revision = _db.session.get(BackupRevision, second['backuprevisionid']) + card = partmarkerinfo.build(revision.payload, revision.assetid, + revisionid=revision.backuprevisionid) + fields = {f['label']: f['value'] for f in card['fields']} + + assert fields['Patterns'] == '2' + assert fields['Settings files'] == '1' + assert fields['Controller'] == 'TMC470' + assert fields['Added'] == 'C.PAT' + assert fields['Removed'] == 'B.PAT' + + +def test_the_first_snapshot_has_nothing_to_diff_against(bk_app, bk_plugin): + """Every file would otherwise read as 'added', which says nothing.""" + from plugins.backups.services import partmarkerinfo + from plugins.backups.models import BackupRevision + + with bk_app.app_context(): + result = bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': 'f' * 64, 'sharepath': r'\\host\s\1', + 'manifest': _markermanifest(['A.PAT']), + }) + _db.session.commit() + revision = _db.session.get(BackupRevision, result['backuprevisionid']) + card = partmarkerinfo.build(revision.payload, revision.assetid, + revisionid=revision.backuprevisionid) + labels = [f['label'] for f in card['fields']] + assert 'Added' not in labels and 'Removed' not in labels + + +def test_a_revision_with_no_manifest_renders_no_card(): + """Older revisions predate manifests; they must not error the panel.""" + from plugins.backups.services import partmarkerinfo + assert partmarkerinfo.build(None, 1)['sectioncount'] == 0 + assert partmarkerinfo.build({}, 1)['fields'] == [] + + +def test_a_noop_backfills_a_manifest_the_row_never_had(bk_app, bk_plugin): + """The hash attests the BYTES, not the manifest. + + A client that starts describing itself posts the same contenthash as + before, so without this the manifest would never be stored on exactly the + markers that are healthy and unchanging - which is most of them. The panel + would then be visible only on markers that had recently changed. + """ + from plugins.backups.models import BackupRevision + + with bk_app.app_context(): + bare = bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': '1' * 64, 'sharepath': r'\\host\s\old', + 'sourcehostname': 'FBYTNCX3', + }) + _db.session.commit() + assert _db.session.get(BackupRevision, bare['backuprevisionid']).payload is None + + described = bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': '1' * 64, 'sharepath': r'\\host\s\new', + 'sourcehostname': 'FBYTNCX3', + 'manifest': _markermanifest(['A.PAT']), + }) + _db.session.commit() + + # Still a no-op: nothing changed on the controller, so no new history. + assert described['action'] == 'noop' + assert described['backuprevisionid'] == bare['backuprevisionid'] + + revision = _db.session.get(BackupRevision, bare['backuprevisionid']) + assert [f['name'] for f in revision.payload['files']] == ['A.PAT'] + # And the pointer follows a snapshot that moved on the share. + assert revision.sharepath == r'\\host\s\new' + + +def test_a_noop_does_not_overwrite_a_manifest_that_is_already_there(bk_app, bk_plugin): + from plugins.backups.models import BackupRevision + + with bk_app.app_context(): + first = bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': '2' * 64, 'sharepath': r'\\host\s\1', + 'sourcehostname': 'FBYTNCX3', + 'manifest': _markermanifest(['ORIGINAL.PAT']), + }) + _db.session.commit() + bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': '2' * 64, 'sharepath': r'\\host\s\1', + 'sourcehostname': 'FBYTNCX3', + 'manifest': _markermanifest(['REPLACEMENT.PAT']), + }) + _db.session.commit() + revision = _db.session.get(BackupRevision, first['backuprevisionid']) + assert [f['name'] for f in revision.payload['files']] == ['ORIGINAL.PAT'] + + +def test_a_file_edited_in_place_is_named_as_changed(bk_app, bk_plugin): + """Same file set, different bytes. + + A revision exists only because the hash moved, so reporting "no files added + or removed" here would contradict the reason the row is there at all. + """ + from plugins.backups.services import partmarkerinfo + from plugins.backups.models import BackupRevision + + with bk_app.app_context(): + before = _markermanifest(['A.PAT', 'B.PAT']) + after = _markermanifest(['A.PAT', 'B.PAT']) + after['files'][1]['sha256'] = 'cd' * 32 + + bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': '3' * 64, 'sharepath': r'\\host\s\1', + 'sourcehostname': 'FBYTNCX3', 'manifest': before}) + _db.session.commit() + second = bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': '4' * 64, 'sharepath': r'\\host\s\2', + 'sourcehostname': 'FBYTNCX3', 'manifest': after}) + _db.session.commit() + + revision = _db.session.get(BackupRevision, second['backuprevisionid']) + card = partmarkerinfo.build(revision.payload, revision.assetid, + revisionid=revision.backuprevisionid) + fields = {f['label']: f['value'] for f in card['fields']} + assert fields['Changed'] == 'B.PAT' + assert 'Added' not in fields and 'Removed' not in fields + + +def test_two_pcs_reporting_one_marker_do_not_cross_diff(bk_app, bk_plugin): + """Dedup chains per source host, and the diff must follow the same chain. + + Diffing across chains would report every file as added and removed at once + the moment a second PC reported the same marker. + """ + from plugins.backups.services import partmarkerinfo + from plugins.backups.models import BackupRevision + + with bk_app.app_context(): + bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': '5' * 64, 'sharepath': r'\\host\s\a', + 'sourcehostname': 'PC-A', + 'manifest': _markermanifest(['ONLY-ON-A.PAT'])}) + _db.session.commit() + onb = bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': '6' * 64, 'sharepath': r'\\host\s\b', + 'sourcehostname': 'PC-B', + 'manifest': _markermanifest(['ONLY-ON-B.PAT'])}) + _db.session.commit() + + revision = _db.session.get(BackupRevision, onb['backuprevisionid']) + card = partmarkerinfo.build(revision.payload, revision.assetid, + revisionid=revision.backuprevisionid) + fields = {f['label']: f['value'] for f in card['fields']} + # PC-B's first snapshot has no predecessor OF ITS OWN, so no diff at all + # rather than a diff against PC-A's unrelated history. + assert 'Added' not in fields and 'Removed' not in fields + + +def test_a_predecessor_without_a_manifest_yields_no_diff(bk_app, bk_plugin): + """Mixed rollout: the previous snapshot predates manifests.""" + from plugins.backups.services import partmarkerinfo + from plugins.backups.models import BackupRevision + + with bk_app.app_context(): + bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': '7' * 64, 'sharepath': r'\\host\s\1', + 'sourcehostname': 'FBYTNCX3'}) + _db.session.commit() + second = bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': '8' * 64, 'sharepath': r'\\host\s\2', + 'sourcehostname': 'FBYTNCX3', + 'manifest': _markermanifest(['A.PAT'])}) + _db.session.commit() + + revision = _db.session.get(BackupRevision, second['backuprevisionid']) + card = partmarkerinfo.build(revision.payload, revision.assetid, + revisionid=revision.backuprevisionid) + labels = [f['label'] for f in card['fields']] + assert 'Added' not in labels and 'Removed' not in labels + assert 'Patterns' in labels # the card itself still renders + + +def test_a_wiped_controller_still_renders_and_reports_the_loss(bk_app, bk_plugin): + """files: [] is a fact, not an absence. + + A controller wiped or swapped is the disaster the Removed line exists for, + so it must not render as no panel at all. + """ + from plugins.backups.services import partmarkerinfo + from plugins.backups.models import BackupRevision + + with bk_app.app_context(): + bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': '9' * 64, 'sharepath': r'\\host\s\1', + 'sourcehostname': 'FBYTNCX3', + 'manifest': _markermanifest(['GONE.PAT'])}) + _db.session.commit() + empty = _markermanifest([]) + wiped = bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': 'a1' * 32, 'sharepath': r'\\host\s\2', + 'sourcehostname': 'FBYTNCX3', 'manifest': empty}) + _db.session.commit() + + revision = _db.session.get(BackupRevision, wiped['backuprevisionid']) + card = partmarkerinfo.build(revision.payload, revision.assetid, + revisionid=revision.backuprevisionid) + fields = {f['label']: f['value'] for f in card['fields']} + assert card['sectioncount'] == 1 + assert fields['Removed'] == 'GONE.PAT' + + +def test_a_malformed_manifest_is_refused_before_it_can_break_a_panel(bk_app, bk_plugin): + """A client bug must not become a page that fails until content changes.""" + bad = [ + {'files': ['A.PAT']}, # not objects + {'files': [{'bytes': 10}]}, # nameless + {'files': [{'name': 'A.PAT', 'bytes': '1,024'}]}, # size not an int + {'files': 'A.PAT'}, # not a list + ] + with bk_app.app_context(): + for manifest in bad: + with pytest.raises(ValueError): + bk_plugin.apply_collector_payload({ + 'machinenumber': '3204', 'backupkind': 'partmarker', + 'contenthash': 'b2' * 32, 'sharepath': r'\\host\s\x', + 'manifest': manifest}) + + +def test_buildinfo_kwargs_do_not_disturb_the_live_ntlars_card(bk_app, bk_plugin): + """The endpoint passes revisionid and sourcehostname to EVERY kind. + + NTLARS is live in production; a new keyword reaching its buildinfo must be + absorbed by **options rather than raising. + """ + with bk_app.app_context(): + result = bk_plugin.apply_collector_payload(_payload()) + _db.session.commit() + from plugins.backups.models import BackupRevision + revision = _db.session.get(BackupRevision, result['backuprevisionid']) + # The assertion is that this does not raise TypeError on the new + # keywords, not that this particular fixture produces sections. + card = registry.getkind('ntlars').buildinfo( + revision.payload, revision.assetid, + partmarkertypes=None, revisionid=revision.backuprevisionid, + sourcehostname=revision.sourcehostname) + assert 'sectioncount' in card