ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into wilm

Browse Source 
# Conflicts:
#	shopdb/data/catalog.json
This commit is contained in:
benjamin.baker
2026-08-26 12:53:03 -04:00
parent 7cd3aadcd1 271d558f8b
commit 801dfd54ab
21 changed files with 1675 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

416
tests/test_plugins/test_backups.py
View File

@@ -207,10 +207,21 @@ def test_partmarker_kind_is_share_backed_and_not_renderable():
def test_partmarker_sharedir_is_under_the_configured_backups_root():
"""The marker kind keys its tree on the MARKER, not the machine type.
It overrides the base layout (<root>\\<machinetype>\\<identifier>\\<kind>)
because the site files marker snapshots as <root>\\partmarker\\<asset>\\,
with one timestamped snapshot directory per capture below that. The
override exists so what ShopDB displays matches what the collector writes.
"""
root = r'\\fileserver\shopfloor\backups'
path = registry.getkind('partmarker').sharedir('lathe', '3204', shareroot=root)
assert path.startswith(root)
assert path.endswith(r'lathe\3204\partmarker')
assert path.endswith(r'partmarker\3204')
# The machine type is deliberately NOT in the path: a marker is identified
# by its own asset, and the type of the machine it serves is irrelevant to
# where its patterns live.
assert 'lathe' not in path
def test_sharedir_is_empty_when_no_root_is_configured():
@@ -608,8 +619,19 @@ def test_infopanel_is_kind_owned_so_a_successor_ships_its_own_card():
panel = ntlarskind.infopanel()
assert panel['id'] == 'backups-dncinfo'
assert 'kind=ntlars' in panel['endpoint']
# A kind with nothing to summarise contributes no panel at all.
assert registry.getkind('partmarker').infopanel() is None
# The part marker kind now ships a card of its own, built from the manifest
# the collector sends beside the hash. It could not before: a share kind
# stored no description of its contents, and reading the share from the app
# server is refused on purpose.
markerpanel = registry.getkind('partmarker').infopanel()
assert markerpanel['id'] == 'backups-partmarker-contents'
assert 'kind=partmarker' in markerpanel['endpoint']
# Marker assets are core type 'machine' like the other 144, and the panel
# is kept off those by having no rows and no empty text rather than by any
# special-casing.
assert markerpanel['assettypes'] == ['machine']
assert 'empty' not in markerpanel
def test_base_kind_buildinfo_is_an_empty_card():
@@ -1006,3 +1028,391 @@ def test_retention_reads_the_key_the_settings_page_writes(app, db):
# The helper cannot see it, which is the whole defect: it looks under
# plugin.backups.backups_retentioncount, which nothing writes.
assert int(BackupsPlugin().get_setting('backups_retentioncount', 0) or 0) == 0
# ---------------------------------------------------------------------------
# Share kinds that describe themselves
#
# A 'share' kind keeps its bytes on SFLD, so ShopDB knew a snapshot existed and
# where, but nothing about what was in it. The collector now accepts an optional
# manifest - the index, not the content - which is what lets a marker's asset
# page list its patterns and name the one that disappeared. Reading the share
# from the app server is refused on purpose, so this is the only route.
# ---------------------------------------------------------------------------
def _markermanifest(names, sourcehost='FBYTNCX3', family='TMC470'):
return {
'capturedat': '2026-08-25T12:00:00Z',
'machine': '3204',
'sourcehost': sourcehost,
'family': family,
'systems': ['TMP1700/470'],
'files': [{'name': n, 'bytes': 266, 'sha256': 'ab' * 32,
'system': 'TMP1700/470'} for n in names],
}
def test_a_share_kind_may_send_a_manifest_and_it_is_stored(bk_app, bk_plugin):
from plugins.backups.models import BackupRevision
with bk_app.app_context():
result = bk_plugin.apply_collector_payload({
'machinenumber': '3204',
'backupkind': 'partmarker',
'contenthash': 'a' * 64,
'sharepath': r'\\host\share\partmarker\3204\20260825-120000',
'manifest': _markermanifest(['TESTPAT.PAT', 'NET.DAT']),
})
assert result['action'] == 'created'
revision = _db.session.get(BackupRevision, result['backuprevisionid'])
# Bytes still live on the share; only the index is in the database.
assert revision.storagebackend == 'share'
assert revision.sharepath.endswith('20260825-120000')
assert [f['name'] for f in revision.payload['files']] == \
['TESTPAT.PAT', 'NET.DAT']
def test_a_share_kind_without_a_manifest_still_works(bk_app, bk_plugin):
"""The manifest is advisory. A client that sends none behaves as before."""
from plugins.backups.models import BackupRevision
with bk_app.app_context():
result = bk_plugin.apply_collector_payload({
'machinenumber': '3204',
'backupkind': 'partmarker',
'contenthash': 'b' * 64,
'sharepath': r'\\host\share\partmarker\3204\20260825-130000',
})
assert result['action'] == 'created'
assert _db.session.get(
BackupRevision, result['backuprevisionid']).payload is None
def test_a_manifest_that_is_not_an_object_is_rejected(bk_app, bk_plugin):
with bk_app.app_context():
with pytest.raises(ValueError):
bk_plugin.apply_collector_payload({
'machinenumber': '3204',
'backupkind': 'partmarker',
'contenthash': 'c' * 64,
'sharepath': r'\\host\share\x',
'manifest': 'not an object',
})
def test_marker_card_counts_by_kind_and_names_what_was_removed(bk_app, bk_plugin):
"""The removed line is the reason the snapshot layout exists.
A pattern deleted from a controller is invisible everywhere else until
somebody needs it. It is NAMED rather than counted - "1 removed" would send
a tech to the share to find out which one.
"""
from plugins.backups.services import partmarkerinfo
from plugins.backups.models import BackupRevision
with bk_app.app_context():
first = bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': 'd' * 64, 'sharepath': r'\\host\s\1',
'sourcehostname': 'FBYTNCX3',
'manifest': _markermanifest(['A.PAT', 'B.PAT', 'NET.DAT']),
})
_db.session.commit()
second = bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': 'e' * 64, 'sharepath': r'\\host\s\2',
'sourcehostname': 'FBYTNCX3',
'manifest': _markermanifest(['A.PAT', 'C.PAT', 'NET.DAT']),
})
_db.session.commit()
assert first['action'] == 'created' and second['action'] == 'created'
revision = _db.session.get(BackupRevision, second['backuprevisionid'])
card = partmarkerinfo.build(revision.payload, revision.assetid,
revisionid=revision.backuprevisionid)
fields = {f['label']: f['value'] for f in card['fields']}
assert fields['Patterns'] == '2'
assert fields['Settings files'] == '1'
assert fields['Controller'] == 'TMC470'
assert fields['Added'] == 'C.PAT'
assert fields['Removed'] == 'B.PAT'
def test_the_first_snapshot_has_nothing_to_diff_against(bk_app, bk_plugin):
"""Every file would otherwise read as 'added', which says nothing."""
from plugins.backups.services import partmarkerinfo
from plugins.backups.models import BackupRevision
with bk_app.app_context():
result = bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': 'f' * 64, 'sharepath': r'\\host\s\1',
'manifest': _markermanifest(['A.PAT']),
})
_db.session.commit()
revision = _db.session.get(BackupRevision, result['backuprevisionid'])
card = partmarkerinfo.build(revision.payload, revision.assetid,
revisionid=revision.backuprevisionid)
labels = [f['label'] for f in card['fields']]
assert 'Added' not in labels and 'Removed' not in labels
def test_a_revision_with_no_manifest_renders_no_card():
"""Older revisions predate manifests; they must not error the panel."""
from plugins.backups.services import partmarkerinfo
assert partmarkerinfo.build(None, 1)['sectioncount'] == 0
assert partmarkerinfo.build({}, 1)['fields'] == []
def test_a_noop_backfills_a_manifest_the_row_never_had(bk_app, bk_plugin):
"""The hash attests the BYTES, not the manifest.
A client that starts describing itself posts the same contenthash as
before, so without this the manifest would never be stored on exactly the
markers that are healthy and unchanging - which is most of them. The panel
would then be visible only on markers that had recently changed.
"""
from plugins.backups.models import BackupRevision
with bk_app.app_context():
bare = bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': '1' * 64, 'sharepath': r'\\host\s\old',
'sourcehostname': 'FBYTNCX3',
})
_db.session.commit()
assert _db.session.get(BackupRevision, bare['backuprevisionid']).payload is None
described = bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': '1' * 64, 'sharepath': r'\\host\s\new',
'sourcehostname': 'FBYTNCX3',
'manifest': _markermanifest(['A.PAT']),
})
_db.session.commit()
# Still a no-op: nothing changed on the controller, so no new history.
assert described['action'] == 'noop'
assert described['backuprevisionid'] == bare['backuprevisionid']
revision = _db.session.get(BackupRevision, bare['backuprevisionid'])
assert [f['name'] for f in revision.payload['files']] == ['A.PAT']
# And the pointer follows a snapshot that moved on the share.
assert revision.sharepath == r'\\host\s\new'
def test_a_noop_does_not_overwrite_a_manifest_that_is_already_there(bk_app, bk_plugin):
from plugins.backups.models import BackupRevision
with bk_app.app_context():
first = bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': '2' * 64, 'sharepath': r'\\host\s\1',
'sourcehostname': 'FBYTNCX3',
'manifest': _markermanifest(['ORIGINAL.PAT']),
})
_db.session.commit()
bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': '2' * 64, 'sharepath': r'\\host\s\1',
'sourcehostname': 'FBYTNCX3',
'manifest': _markermanifest(['REPLACEMENT.PAT']),
})
_db.session.commit()
revision = _db.session.get(BackupRevision, first['backuprevisionid'])
assert [f['name'] for f in revision.payload['files']] == ['ORIGINAL.PAT']
def test_a_file_edited_in_place_is_named_as_changed(bk_app, bk_plugin):
"""Same file set, different bytes.
A revision exists only because the hash moved, so reporting "no files added
or removed" here would contradict the reason the row is there at all.
"""
from plugins.backups.services import partmarkerinfo
from plugins.backups.models import BackupRevision
with bk_app.app_context():
before = _markermanifest(['A.PAT', 'B.PAT'])
after = _markermanifest(['A.PAT', 'B.PAT'])
after['files'][1]['sha256'] = 'cd' * 32
bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': '3' * 64, 'sharepath': r'\\host\s\1',
'sourcehostname': 'FBYTNCX3', 'manifest': before})
_db.session.commit()
second = bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': '4' * 64, 'sharepath': r'\\host\s\2',
'sourcehostname': 'FBYTNCX3', 'manifest': after})
_db.session.commit()
revision = _db.session.get(BackupRevision, second['backuprevisionid'])
card = partmarkerinfo.build(revision.payload, revision.assetid,
revisionid=revision.backuprevisionid)
fields = {f['label']: f['value'] for f in card['fields']}
assert fields['Changed'] == 'B.PAT'
assert 'Added' not in fields and 'Removed' not in fields
def test_two_pcs_reporting_one_marker_do_not_cross_diff(bk_app, bk_plugin):
"""Dedup chains per source host, and the diff must follow the same chain.
Diffing across chains would report every file as added and removed at once
the moment a second PC reported the same marker.
"""
from plugins.backups.services import partmarkerinfo
from plugins.backups.models import BackupRevision
with bk_app.app_context():
bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': '5' * 64, 'sharepath': r'\\host\s\a',
'sourcehostname': 'PC-A',
'manifest': _markermanifest(['ONLY-ON-A.PAT'])})
_db.session.commit()
onb = bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': '6' * 64, 'sharepath': r'\\host\s\b',
'sourcehostname': 'PC-B',
'manifest': _markermanifest(['ONLY-ON-B.PAT'])})
_db.session.commit()
revision = _db.session.get(BackupRevision, onb['backuprevisionid'])
card = partmarkerinfo.build(revision.payload, revision.assetid,
revisionid=revision.backuprevisionid)
fields = {f['label']: f['value'] for f in card['fields']}
# PC-B's first snapshot has no predecessor OF ITS OWN, so no diff at all
# rather than a diff against PC-A's unrelated history.
assert 'Added' not in fields and 'Removed' not in fields
def test_a_predecessor_without_a_manifest_yields_no_diff(bk_app, bk_plugin):
"""Mixed rollout: the previous snapshot predates manifests."""
from plugins.backups.services import partmarkerinfo
from plugins.backups.models import BackupRevision
with bk_app.app_context():
bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': '7' * 64, 'sharepath': r'\\host\s\1',
'sourcehostname': 'FBYTNCX3'})
_db.session.commit()
second = bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': '8' * 64, 'sharepath': r'\\host\s\2',
'sourcehostname': 'FBYTNCX3',
'manifest': _markermanifest(['A.PAT'])})
_db.session.commit()
revision = _db.session.get(BackupRevision, second['backuprevisionid'])
card = partmarkerinfo.build(revision.payload, revision.assetid,
revisionid=revision.backuprevisionid)
labels = [f['label'] for f in card['fields']]
assert 'Added' not in labels and 'Removed' not in labels
assert 'Patterns' in labels # the card itself still renders
def test_a_wiped_controller_still_renders_and_reports_the_loss(bk_app, bk_plugin):
"""files: [] is a fact, not an absence.
A controller wiped or swapped is the disaster the Removed line exists for,
so it must not render as no panel at all.
"""
from plugins.backups.services import partmarkerinfo
from plugins.backups.models import BackupRevision
with bk_app.app_context():
bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': '9' * 64, 'sharepath': r'\\host\s\1',
'sourcehostname': 'FBYTNCX3',
'manifest': _markermanifest(['GONE.PAT'])})
_db.session.commit()
empty = _markermanifest([])
wiped = bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': 'a1' * 32, 'sharepath': r'\\host\s\2',
'sourcehostname': 'FBYTNCX3', 'manifest': empty})
_db.session.commit()
revision = _db.session.get(BackupRevision, wiped['backuprevisionid'])
card = partmarkerinfo.build(revision.payload, revision.assetid,
revisionid=revision.backuprevisionid)
fields = {f['label']: f['value'] for f in card['fields']}
assert card['sectioncount'] == 1
assert fields['Removed'] == 'GONE.PAT'
def test_a_malformed_manifest_is_refused_before_it_can_break_a_panel(bk_app, bk_plugin):
"""A client bug must not become a page that fails until content changes."""
bad = [
{'files': ['A.PAT']}, # not objects
{'files': [{'bytes': 10}]}, # nameless
{'files': [{'name': 'A.PAT', 'bytes': '1,024'}]}, # size not an int
{'files': 'A.PAT'}, # not a list
]
with bk_app.app_context():
for manifest in bad:
with pytest.raises(ValueError):
bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': 'b2' * 32, 'sharepath': r'\\host\s\x',
'manifest': manifest})
def test_buildinfo_kwargs_do_not_disturb_the_live_ntlars_card(bk_app, bk_plugin):
"""The endpoint passes revisionid and sourcehostname to EVERY kind.
NTLARS is live in production; a new keyword reaching its buildinfo must be
absorbed by **options rather than raising.
"""
with bk_app.app_context():
result = bk_plugin.apply_collector_payload(_payload())
_db.session.commit()
from plugins.backups.models import BackupRevision
revision = _db.session.get(BackupRevision, result['backuprevisionid'])
# The assertion is that this does not raise TypeError on the new
# keywords, not that this particular fixture produces sections.
card = registry.getkind('ntlars').buildinfo(
revision.payload, revision.assetid,
partmarkertypes=None, revisionid=revision.backuprevisionid,
sourcehostname=revision.sourcehostname)
assert 'sectioncount' in card
def test_the_card_shows_the_snapshot_path_it_was_actually_written_to(bk_app, bk_plugin):
"""Plain text, and from the REVISION - not composed from the setting.
A snapshot written before the share layout changed still lives where it was
put, so the stored path is the only honest answer. It is not a link: a
browser refuses to navigate from https to file://, so a clickable path
would look actionable and do nothing.
"""
from plugins.backups.services import partmarkerinfo
from plugins.backups.models import BackupRevision
with bk_app.app_context():
result = bk_plugin.apply_collector_payload({
'machinenumber': '3204', 'backupkind': 'partmarker',
'contenthash': 'c3' * 32,
'sharepath': r'\\host\share\partmarker\3204\20260825-144500',
'sourcehostname': 'FBYTNCX3',
'manifest': _markermanifest(['A.PAT'])})
_db.session.commit()
revision = _db.session.get(BackupRevision, result['backuprevisionid'])
card = partmarkerinfo.build(revision.payload, revision.assetid,
revisionid=revision.backuprevisionid,
sharepath=revision.sharepath)
snapshot = [f for f in card['fields'] if f['label'] == 'Snapshot']
assert snapshot and snapshot[0]['value'] == revision.sharepath
assert snapshot[0]['mono'] is True
def test_the_card_omits_the_snapshot_row_when_no_path_is_passed(bk_app, bk_plugin):
from plugins.backups.services import partmarkerinfo
card = partmarkerinfo.build(_markermanifest(['A.PAT']), 1)
assert 'Snapshot' not in [f['label'] for f in card['fields']]

22
tests/test_plugins/test_geenforce_publish_gate.py
View File

@@ -155,9 +155,20 @@ def test_hosts_below_libversion_reports_the_floor(db):
# -- backup state on the reports table ---------------------------------------
def test_backup_state_prefers_the_newest_confirmed_revision(db):
"""A host with several revisions shows the most recently CONFIRMED one, and
a revision that has never been confirmed (lastseenat NULL) never wins.
def test_backup_state_reports_the_stalest_measurable_kind(db):
"""A host is judged by the WORST backup it has, not the best.
This used to prefer the most recently confirmed revision. That was right
while a PC reported one kind, and became wrong when part markers arrived: a
marker bay reports ntlars AND partmarker, so a daily ntlars post kept the
badge green while the marker backup had been failing for a month. A host is
healthy only when everything it backs up is current, and the badge names the
kind that is not.
A revision that has never been confirmed (lastseenat NULL) still never wins.
It is not stale, it is unmeasurable - and letting it win hid a working
backup entirely, because the verdict for a NULL confirmation is tri-state
None, which renders as no badge at all.
Ordering happens in Python on purpose. Doing it in SQL needs
ORDER BY ... NULLS LAST, which SQLite accepts and MySQL rejects outright -
@@ -191,7 +202,10 @@ def test_backup_state_prefers_the_newest_confirmed_revision(db):
facts = {'wjpc01': {'backupkind': None, 'backuplastseen': None}}
_attach_backup_state(facts, {'WJPC01'})
assert facts['wjpc01']['backupkind'] == 'udc'
# ntlars is 9 days old, udc was confirmed just now: the host is judged by
# ntlars, and the badge names it so somebody knows WHICH backup stopped.
assert facts['wjpc01']['backupkind'] == 'ntlars'
# 'files' has never been confirmed, so it neither wins nor blanks the badge.
assert facts['wjpc01']['backuplastseen'] is not None

77
tests/test_plugins/test_geenforce_reporting.py
View File

@@ -486,3 +486,80 @@ def test_the_capture_date_is_reported_separately_from_the_check(client, db, app,
assert row['backupcollectedat'] is not None
assert row['backupcollectedat'] < row['backuplastseen'], \
'captured before it was last confirmed'
def _extra_revision(db, hostname, kind, ageda, contenthash=None):
"""A second backup kind for a host that already exists.
_backup_revision creates the asset and the Computer row, so it cannot be
called twice for one host.
"""
from datetime import timedelta
from plugins.backups.models import BackupRevision
from plugins.geenforce.service import _utcnow
from plugins.computers.models import Computer
computer = Computer.query.filter_by(hostname=hostname).first()
db.session.add(BackupRevision(
assetid=computer.assetid, backupkind=kind, sourcehostname=hostname,
contenthash=(contenthash or (kind[0] * 64))[:64],
collectedat=_utcnow() - timedelta(days=ageda + 30),
lastseenat=_utcnow() - timedelta(days=ageda)))
db.session.commit()
def test_a_stale_kind_is_not_hidden_by_a_healthy_one(client, db, app, auth_headers):
"""A marker bay reports TWO kinds, and the badge must not report the better.
ntlars posts daily for the machine's DNC settings; partmarker posts the
controller's patterns. Judging a host by its newest revision meant a
marker backup that had been failing for a week still showed green, because
ntlars kept the badge fed. The failure was then visible only on the
marker's own asset page, which is not where anyone looks to discover that
something stopped.
"""
_seed_and_publish(app)
secret = _token(client, auth_headers, ['geenforce.report'])
_backup_revision(db, 'WJBAK10', 'ntlars', ageda=1)
_extra_revision(db, 'WJBAK10', 'partmarker', ageda=9)
client.post('/api/geenforce/report', json={
'hostname': 'WJBAK10', 'scopename': 'gea-shopfloor-cmm', 'counts': {}},
headers={'X-API-Key': secret})
row = _row_for(client, auth_headers, 'WJBAK10')
assert row['backupok'] is False
# And it names the kind that is actually broken, not the healthy one.
assert row['backupkind'] == 'partmarker'
def test_a_host_is_good_only_when_every_kind_is_current(client, db, app, auth_headers):
_seed_and_publish(app)
secret = _token(client, auth_headers, ['geenforce.report'])
_backup_revision(db, 'WJBAK11', 'ntlars', ageda=1)
_extra_revision(db, 'WJBAK11', 'partmarker', ageda=1)
client.post('/api/geenforce/report', json={
'hostname': 'WJBAK11', 'scopename': 'gea-shopfloor-cmm', 'counts': {}},
headers={'X-API-Key': secret})
row = _row_for(client, auth_headers, 'WJBAK11')
assert row['backupok'] is True
def test_an_older_revision_of_a_kind_does_not_make_it_look_stale(client, db, app, auth_headers):
"""lastseenat only ever moves on the NEWEST revision of a chain.
Older revisions keep the value they had when they were superseded, so
judging on every row rather than the freshest per kind would report a
healthy, frequently changing config as permanently stale.
"""
_seed_and_publish(app)
secret = _token(client, auth_headers, ['geenforce.report'])
_backup_revision(db, 'WJBAK12', 'ntlars', ageda=1)
_extra_revision(db, 'WJBAK12', 'ntlars', ageda=40, contenthash='f' * 64)
client.post('/api/geenforce/report', json={
'hostname': 'WJBAK12', 'scopename': 'gea-shopfloor-cmm', 'counts': {}},
headers={'X-API-Key': secret})
row = _row_for(client, auth_headers, 'WJBAK12')
assert row['backupok'] is True
assert row['backupkind'] == 'ntlars'