Let a share-backed backup say what is in it
A 'share' kind keeps its bytes on the file share and gives ShopDB only a hash and a UNC path. That is right for an opaque vendor file, but it means the app cannot answer the question a tech actually arrives with - what does this part marker hold, and which pattern disappeared last month. Reading the share from the app server is not an option: the download routes refuse to mount it on purpose, because a permissions slip there becomes an unexplained empty panel. So the collector now accepts an optional manifest alongside the hash: the index, not the content. Names, sizes, per-file hashes, and the marking system each pattern was authored for. Advisory throughout - a client that sends none behaves exactly as before, and dedup still keys on contenthash. The marker asset gains a Marker Contents card built from it: counts by type, the controller family, and a diff against the previous snapshot in the same source chain. Removed files are named rather than counted, because "1 removed" only tells someone to go and look. Files edited in place are named too - the file set can be identical while the content hash moved, and reporting no change there would contradict the reason the revision exists. The manifest is deliberately NOT covered by contenthash, which attests the bytes on the share. A client that starts describing itself therefore posts an unchanged hash, so a no-op backfills a manifest the row does not yet have - otherwise the card would appear only on markers that had recently changed, and stay hidden on every healthy one. The same no-op refreshes sharepath, so a snapshot that moved on the share stops being pointed at where it used to be. Manifests are validated at ingest rather than at render: a malformed one would otherwise become the latest revision and fail every view of the asset page until the content next changed, outliving the client bug that caused it. sharedir() for the marker kind is overridden to the layout the site actually uses, so what ShopDB displays and validates matches what the collector writes.
This commit is contained in:
cproudlock
@@ -207,10 +207,21 @@ def test_partmarker_kind_is_share_backed_and_not_renderable():
|
||||
|
||||
|
||||
def test_partmarker_sharedir_is_under_the_configured_backups_root():
|
||||
"""The marker kind keys its tree on the MARKER, not the machine type.
|
||||
|
||||
It overrides the base layout (<root>\\<machinetype>\\<identifier>\\<kind>)
|
||||
because the site files marker snapshots as <root>\\partmarker\\<asset>\\,
|
||||
with one timestamped snapshot directory per capture below that. The
|
||||
override exists so what ShopDB displays matches what the collector writes.
|
||||
"""
|
||||
root = r'\\fileserver\shopfloor\backups'
|
||||
path = registry.getkind('partmarker').sharedir('lathe', '3204', shareroot=root)
|
||||
assert path.startswith(root)
|
||||
assert path.endswith(r'lathe\3204\partmarker')
|
||||
assert path.endswith(r'partmarker\3204')
|
||||
# The machine type is deliberately NOT in the path: a marker is identified
|
||||
# by its own asset, and the type of the machine it serves is irrelevant to
|
||||
# where its patterns live.
|
||||
assert 'lathe' not in path
|
||||
|
||||
|
||||
def test_sharedir_is_empty_when_no_root_is_configured():
|
||||
@@ -608,8 +619,19 @@ def test_infopanel_is_kind_owned_so_a_successor_ships_its_own_card():
|
||||
panel = ntlarskind.infopanel()
|
||||
assert panel['id'] == 'backups-dncinfo'
|
||||
assert 'kind=ntlars' in panel['endpoint']
|
||||
# A kind with nothing to summarise contributes no panel at all.
|
||||
assert registry.getkind('partmarker').infopanel() is None
|
||||
|
||||
# The part marker kind now ships a card of its own, built from the manifest
|
||||
# the collector sends beside the hash. It could not before: a share kind
|
||||
# stored no description of its contents, and reading the share from the app
|
||||
# server is refused on purpose.
|
||||
markerpanel = registry.getkind('partmarker').infopanel()
|
||||
assert markerpanel['id'] == 'backups-partmarker-contents'
|
||||
assert 'kind=partmarker' in markerpanel['endpoint']
|
||||
# Marker assets are core type 'machine' like the other 144, and the panel
|
||||
# is kept off those by having no rows and no empty text rather than by any
|
||||
# special-casing.
|
||||
assert markerpanel['assettypes'] == ['machine']
|
||||
assert 'empty' not in markerpanel
|
||||
|
||||
|
||||
def test_base_kind_buildinfo_is_an_empty_card():
|
||||
@@ -1006,3 +1028,356 @@ def test_retention_reads_the_key_the_settings_page_writes(app, db):
|
||||
# The helper cannot see it, which is the whole defect: it looks under
|
||||
# plugin.backups.backups_retentioncount, which nothing writes.
|
||||
assert int(BackupsPlugin().get_setting('backups_retentioncount', 0) or 0) == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Share kinds that describe themselves
|
||||
#
|
||||
# A 'share' kind keeps its bytes on SFLD, so ShopDB knew a snapshot existed and
|
||||
# where, but nothing about what was in it. The collector now accepts an optional
|
||||
# manifest - the index, not the content - which is what lets a marker's asset
|
||||
# page list its patterns and name the one that disappeared. Reading the share
|
||||
# from the app server is refused on purpose, so this is the only route.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _markermanifest(names, sourcehost='FBYTNCX3', family='TMC470'):
|
||||
return {
|
||||
'capturedat': '2026-08-25T12:00:00Z',
|
||||
'machine': '3204',
|
||||
'sourcehost': sourcehost,
|
||||
'family': family,
|
||||
'systems': ['TMP1700/470'],
|
||||
'files': [{'name': n, 'bytes': 266, 'sha256': 'ab' * 32,
|
||||
'system': 'TMP1700/470'} for n in names],
|
||||
}
|
||||
|
||||
|
||||
def test_a_share_kind_may_send_a_manifest_and_it_is_stored(bk_app, bk_plugin):
|
||||
from plugins.backups.models import BackupRevision
|
||||
|
||||
with bk_app.app_context():
|
||||
result = bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204',
|
||||
'backupkind': 'partmarker',
|
||||
'contenthash': 'a' * 64,
|
||||
'sharepath': r'\\host\share\partmarker\3204\20260825-120000',
|
||||
'manifest': _markermanifest(['TESTPAT.PAT', 'NET.DAT']),
|
||||
})
|
||||
assert result['action'] == 'created'
|
||||
revision = _db.session.get(BackupRevision, result['backuprevisionid'])
|
||||
# Bytes still live on the share; only the index is in the database.
|
||||
assert revision.storagebackend == 'share'
|
||||
assert revision.sharepath.endswith('20260825-120000')
|
||||
assert [f['name'] for f in revision.payload['files']] == \
|
||||
['TESTPAT.PAT', 'NET.DAT']
|
||||
|
||||
|
||||
def test_a_share_kind_without_a_manifest_still_works(bk_app, bk_plugin):
|
||||
"""The manifest is advisory. A client that sends none behaves as before."""
|
||||
from plugins.backups.models import BackupRevision
|
||||
|
||||
with bk_app.app_context():
|
||||
result = bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204',
|
||||
'backupkind': 'partmarker',
|
||||
'contenthash': 'b' * 64,
|
||||
'sharepath': r'\\host\share\partmarker\3204\20260825-130000',
|
||||
})
|
||||
assert result['action'] == 'created'
|
||||
assert _db.session.get(
|
||||
BackupRevision, result['backuprevisionid']).payload is None
|
||||
|
||||
|
||||
def test_a_manifest_that_is_not_an_object_is_rejected(bk_app, bk_plugin):
|
||||
with bk_app.app_context():
|
||||
with pytest.raises(ValueError):
|
||||
bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204',
|
||||
'backupkind': 'partmarker',
|
||||
'contenthash': 'c' * 64,
|
||||
'sharepath': r'\\host\share\x',
|
||||
'manifest': 'not an object',
|
||||
})
|
||||
|
||||
|
||||
def test_marker_card_counts_by_kind_and_names_what_was_removed(bk_app, bk_plugin):
|
||||
"""The removed line is the reason the snapshot layout exists.
|
||||
|
||||
A pattern deleted from a controller is invisible everywhere else until
|
||||
somebody needs it. It is NAMED rather than counted - "1 removed" would send
|
||||
a tech to the share to find out which one.
|
||||
"""
|
||||
from plugins.backups.services import partmarkerinfo
|
||||
from plugins.backups.models import BackupRevision
|
||||
|
||||
with bk_app.app_context():
|
||||
first = bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': 'd' * 64, 'sharepath': r'\\host\s\1',
|
||||
'sourcehostname': 'FBYTNCX3',
|
||||
'manifest': _markermanifest(['A.PAT', 'B.PAT', 'NET.DAT']),
|
||||
})
|
||||
_db.session.commit()
|
||||
second = bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': 'e' * 64, 'sharepath': r'\\host\s\2',
|
||||
'sourcehostname': 'FBYTNCX3',
|
||||
'manifest': _markermanifest(['A.PAT', 'C.PAT', 'NET.DAT']),
|
||||
})
|
||||
_db.session.commit()
|
||||
assert first['action'] == 'created' and second['action'] == 'created'
|
||||
|
||||
revision = _db.session.get(BackupRevision, second['backuprevisionid'])
|
||||
card = partmarkerinfo.build(revision.payload, revision.assetid,
|
||||
revisionid=revision.backuprevisionid)
|
||||
fields = {f['label']: f['value'] for f in card['fields']}
|
||||
|
||||
assert fields['Patterns'] == '2'
|
||||
assert fields['Settings files'] == '1'
|
||||
assert fields['Controller'] == 'TMC470'
|
||||
assert fields['Added'] == 'C.PAT'
|
||||
assert fields['Removed'] == 'B.PAT'
|
||||
|
||||
|
||||
def test_the_first_snapshot_has_nothing_to_diff_against(bk_app, bk_plugin):
|
||||
"""Every file would otherwise read as 'added', which says nothing."""
|
||||
from plugins.backups.services import partmarkerinfo
|
||||
from plugins.backups.models import BackupRevision
|
||||
|
||||
with bk_app.app_context():
|
||||
result = bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': 'f' * 64, 'sharepath': r'\\host\s\1',
|
||||
'manifest': _markermanifest(['A.PAT']),
|
||||
})
|
||||
_db.session.commit()
|
||||
revision = _db.session.get(BackupRevision, result['backuprevisionid'])
|
||||
card = partmarkerinfo.build(revision.payload, revision.assetid,
|
||||
revisionid=revision.backuprevisionid)
|
||||
labels = [f['label'] for f in card['fields']]
|
||||
assert 'Added' not in labels and 'Removed' not in labels
|
||||
|
||||
|
||||
def test_a_revision_with_no_manifest_renders_no_card():
|
||||
"""Older revisions predate manifests; they must not error the panel."""
|
||||
from plugins.backups.services import partmarkerinfo
|
||||
assert partmarkerinfo.build(None, 1)['sectioncount'] == 0
|
||||
assert partmarkerinfo.build({}, 1)['fields'] == []
|
||||
|
||||
|
||||
def test_a_noop_backfills_a_manifest_the_row_never_had(bk_app, bk_plugin):
|
||||
"""The hash attests the BYTES, not the manifest.
|
||||
|
||||
A client that starts describing itself posts the same contenthash as
|
||||
before, so without this the manifest would never be stored on exactly the
|
||||
markers that are healthy and unchanging - which is most of them. The panel
|
||||
would then be visible only on markers that had recently changed.
|
||||
"""
|
||||
from plugins.backups.models import BackupRevision
|
||||
|
||||
with bk_app.app_context():
|
||||
bare = bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': '1' * 64, 'sharepath': r'\\host\s\old',
|
||||
'sourcehostname': 'FBYTNCX3',
|
||||
})
|
||||
_db.session.commit()
|
||||
assert _db.session.get(BackupRevision, bare['backuprevisionid']).payload is None
|
||||
|
||||
described = bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': '1' * 64, 'sharepath': r'\\host\s\new',
|
||||
'sourcehostname': 'FBYTNCX3',
|
||||
'manifest': _markermanifest(['A.PAT']),
|
||||
})
|
||||
_db.session.commit()
|
||||
|
||||
# Still a no-op: nothing changed on the controller, so no new history.
|
||||
assert described['action'] == 'noop'
|
||||
assert described['backuprevisionid'] == bare['backuprevisionid']
|
||||
|
||||
revision = _db.session.get(BackupRevision, bare['backuprevisionid'])
|
||||
assert [f['name'] for f in revision.payload['files']] == ['A.PAT']
|
||||
# And the pointer follows a snapshot that moved on the share.
|
||||
assert revision.sharepath == r'\\host\s\new'
|
||||
|
||||
|
||||
def test_a_noop_does_not_overwrite_a_manifest_that_is_already_there(bk_app, bk_plugin):
|
||||
from plugins.backups.models import BackupRevision
|
||||
|
||||
with bk_app.app_context():
|
||||
first = bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': '2' * 64, 'sharepath': r'\\host\s\1',
|
||||
'sourcehostname': 'FBYTNCX3',
|
||||
'manifest': _markermanifest(['ORIGINAL.PAT']),
|
||||
})
|
||||
_db.session.commit()
|
||||
bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': '2' * 64, 'sharepath': r'\\host\s\1',
|
||||
'sourcehostname': 'FBYTNCX3',
|
||||
'manifest': _markermanifest(['REPLACEMENT.PAT']),
|
||||
})
|
||||
_db.session.commit()
|
||||
revision = _db.session.get(BackupRevision, first['backuprevisionid'])
|
||||
assert [f['name'] for f in revision.payload['files']] == ['ORIGINAL.PAT']
|
||||
|
||||
|
||||
def test_a_file_edited_in_place_is_named_as_changed(bk_app, bk_plugin):
|
||||
"""Same file set, different bytes.
|
||||
|
||||
A revision exists only because the hash moved, so reporting "no files added
|
||||
or removed" here would contradict the reason the row is there at all.
|
||||
"""
|
||||
from plugins.backups.services import partmarkerinfo
|
||||
from plugins.backups.models import BackupRevision
|
||||
|
||||
with bk_app.app_context():
|
||||
before = _markermanifest(['A.PAT', 'B.PAT'])
|
||||
after = _markermanifest(['A.PAT', 'B.PAT'])
|
||||
after['files'][1]['sha256'] = 'cd' * 32
|
||||
|
||||
bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': '3' * 64, 'sharepath': r'\\host\s\1',
|
||||
'sourcehostname': 'FBYTNCX3', 'manifest': before})
|
||||
_db.session.commit()
|
||||
second = bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': '4' * 64, 'sharepath': r'\\host\s\2',
|
||||
'sourcehostname': 'FBYTNCX3', 'manifest': after})
|
||||
_db.session.commit()
|
||||
|
||||
revision = _db.session.get(BackupRevision, second['backuprevisionid'])
|
||||
card = partmarkerinfo.build(revision.payload, revision.assetid,
|
||||
revisionid=revision.backuprevisionid)
|
||||
fields = {f['label']: f['value'] for f in card['fields']}
|
||||
assert fields['Changed'] == 'B.PAT'
|
||||
assert 'Added' not in fields and 'Removed' not in fields
|
||||
|
||||
|
||||
def test_two_pcs_reporting_one_marker_do_not_cross_diff(bk_app, bk_plugin):
|
||||
"""Dedup chains per source host, and the diff must follow the same chain.
|
||||
|
||||
Diffing across chains would report every file as added and removed at once
|
||||
the moment a second PC reported the same marker.
|
||||
"""
|
||||
from plugins.backups.services import partmarkerinfo
|
||||
from plugins.backups.models import BackupRevision
|
||||
|
||||
with bk_app.app_context():
|
||||
bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': '5' * 64, 'sharepath': r'\\host\s\a',
|
||||
'sourcehostname': 'PC-A',
|
||||
'manifest': _markermanifest(['ONLY-ON-A.PAT'])})
|
||||
_db.session.commit()
|
||||
onb = bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': '6' * 64, 'sharepath': r'\\host\s\b',
|
||||
'sourcehostname': 'PC-B',
|
||||
'manifest': _markermanifest(['ONLY-ON-B.PAT'])})
|
||||
_db.session.commit()
|
||||
|
||||
revision = _db.session.get(BackupRevision, onb['backuprevisionid'])
|
||||
card = partmarkerinfo.build(revision.payload, revision.assetid,
|
||||
revisionid=revision.backuprevisionid)
|
||||
fields = {f['label']: f['value'] for f in card['fields']}
|
||||
# PC-B's first snapshot has no predecessor OF ITS OWN, so no diff at all
|
||||
# rather than a diff against PC-A's unrelated history.
|
||||
assert 'Added' not in fields and 'Removed' not in fields
|
||||
|
||||
|
||||
def test_a_predecessor_without_a_manifest_yields_no_diff(bk_app, bk_plugin):
|
||||
"""Mixed rollout: the previous snapshot predates manifests."""
|
||||
from plugins.backups.services import partmarkerinfo
|
||||
from plugins.backups.models import BackupRevision
|
||||
|
||||
with bk_app.app_context():
|
||||
bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': '7' * 64, 'sharepath': r'\\host\s\1',
|
||||
'sourcehostname': 'FBYTNCX3'})
|
||||
_db.session.commit()
|
||||
second = bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': '8' * 64, 'sharepath': r'\\host\s\2',
|
||||
'sourcehostname': 'FBYTNCX3',
|
||||
'manifest': _markermanifest(['A.PAT'])})
|
||||
_db.session.commit()
|
||||
|
||||
revision = _db.session.get(BackupRevision, second['backuprevisionid'])
|
||||
card = partmarkerinfo.build(revision.payload, revision.assetid,
|
||||
revisionid=revision.backuprevisionid)
|
||||
labels = [f['label'] for f in card['fields']]
|
||||
assert 'Added' not in labels and 'Removed' not in labels
|
||||
assert 'Patterns' in labels # the card itself still renders
|
||||
|
||||
|
||||
def test_a_wiped_controller_still_renders_and_reports_the_loss(bk_app, bk_plugin):
|
||||
"""files: [] is a fact, not an absence.
|
||||
|
||||
A controller wiped or swapped is the disaster the Removed line exists for,
|
||||
so it must not render as no panel at all.
|
||||
"""
|
||||
from plugins.backups.services import partmarkerinfo
|
||||
from plugins.backups.models import BackupRevision
|
||||
|
||||
with bk_app.app_context():
|
||||
bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': '9' * 64, 'sharepath': r'\\host\s\1',
|
||||
'sourcehostname': 'FBYTNCX3',
|
||||
'manifest': _markermanifest(['GONE.PAT'])})
|
||||
_db.session.commit()
|
||||
empty = _markermanifest([])
|
||||
wiped = bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': 'a1' * 32, 'sharepath': r'\\host\s\2',
|
||||
'sourcehostname': 'FBYTNCX3', 'manifest': empty})
|
||||
_db.session.commit()
|
||||
|
||||
revision = _db.session.get(BackupRevision, wiped['backuprevisionid'])
|
||||
card = partmarkerinfo.build(revision.payload, revision.assetid,
|
||||
revisionid=revision.backuprevisionid)
|
||||
fields = {f['label']: f['value'] for f in card['fields']}
|
||||
assert card['sectioncount'] == 1
|
||||
assert fields['Removed'] == 'GONE.PAT'
|
||||
|
||||
|
||||
def test_a_malformed_manifest_is_refused_before_it_can_break_a_panel(bk_app, bk_plugin):
|
||||
"""A client bug must not become a page that fails until content changes."""
|
||||
bad = [
|
||||
{'files': ['A.PAT']}, # not objects
|
||||
{'files': [{'bytes': 10}]}, # nameless
|
||||
{'files': [{'name': 'A.PAT', 'bytes': '1,024'}]}, # size not an int
|
||||
{'files': 'A.PAT'}, # not a list
|
||||
]
|
||||
with bk_app.app_context():
|
||||
for manifest in bad:
|
||||
with pytest.raises(ValueError):
|
||||
bk_plugin.apply_collector_payload({
|
||||
'machinenumber': '3204', 'backupkind': 'partmarker',
|
||||
'contenthash': 'b2' * 32, 'sharepath': r'\\host\s\x',
|
||||
'manifest': manifest})
|
||||
|
||||
|
||||
def test_buildinfo_kwargs_do_not_disturb_the_live_ntlars_card(bk_app, bk_plugin):
|
||||
"""The endpoint passes revisionid and sourcehostname to EVERY kind.
|
||||
|
||||
NTLARS is live in production; a new keyword reaching its buildinfo must be
|
||||
absorbed by **options rather than raising.
|
||||
"""
|
||||
with bk_app.app_context():
|
||||
result = bk_plugin.apply_collector_payload(_payload())
|
||||
_db.session.commit()
|
||||
from plugins.backups.models import BackupRevision
|
||||
revision = _db.session.get(BackupRevision, result['backuprevisionid'])
|
||||
# The assertion is that this does not raise TypeError on the new
|
||||
# keywords, not that this particular fixture produces sections.
|
||||
card = registry.getkind('ntlars').buildinfo(
|
||||
revision.payload, revision.assetid,
|
||||
partmarkertypes=None, revisionid=revision.backuprevisionid,
|
||||
sourcehostname=revision.sourcehostname)
|
||||
assert 'sectioncount' in card
|
||||
|
||||
Reference in New Issue
Block a user