diff --git a/CHANGELOG.md b/CHANGELOG.md index e68d271..c6f91d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,22 @@ ADR-007 and ADR-002. ## [Unreleased] +### Added + +- A backup now records that it was CHECKED, not only that it changed. Dedup + means an unchanged configuration writes no revision, so the stored timestamps + moved only on a change: a machine whose settings had been stable for six + months was indistinguishable from a machine whose backup died six months ago. + The only evidence a backup still ran was a line in a log file on the PC. + `lastseenat` is touched on every matching post, including the no-op, and is + backfilled from the timestamps that already exist. +- Dashboard cards for enforcement failures, PCs not reporting, and backups that + have stopped. The dashboard now renders cards plugins declare, which it never + did before - five plugins had been declaring widgets into a void, pointing at + components nobody wrote. Cards are ordered by severity, hide themselves when + there is nothing to report, gate on a permission, and each fetches + independently so one broken endpoint cannot blank the board. + ## [0.9.0] - 2026-08-11 Driven by a fleet that had been failing quietly. A bay had been returning 500 diff --git a/plugins/backups/api/routes.py b/plugins/backups/api/routes.py index 33baec2..f4533a8 100644 --- a/plugins/backups/api/routes.py +++ b/plugins/backups/api/routes.py @@ -344,3 +344,19 @@ def _diffprojections(old, new): 'aftertype': None if after is None else after[0], }) return changes + + +@backups_bp.route('/dashboard/stale', methods=['GET']) +@jwt_required() +@require_permission('backups.view') +def dashboard_stale(): + """Chains whose backup has stopped running. + + Thin: the rule lives in services/staleness.py, where it is testable without + an auth layer in the way. Keyed on the last CONFIRMED check, never on the + last change - dedup means an unchanged config writes no revision, so a card + keyed on revision age would flag most of a healthy fleet. + """ + from ..services.staleness import stalechains + + return success_response(stalechains()) diff --git a/plugins/backups/migrations/versions/0002_backups_lastseenat.py b/plugins/backups/migrations/versions/0002_backups_lastseenat.py new file mode 100644 index 0000000..b4f5c70 --- /dev/null +++ b/plugins/backups/migrations/versions/0002_backups_lastseenat.py @@ -0,0 +1,43 @@ +"""backups: record when a configuration was last confirmed unchanged. + +Dedup means an unchanged config writes NO revision, so `collectedat` moves only +when something changes. A machine whose settings have been stable for six months +therefore has a six-month-old newest revision and is entirely healthy - and +ShopDB had no way to tell it apart from a machine whose backup stopped running +six months ago. The evidence that a check happened existed only in a log file on +the PC. + +`lastseenat` records the check rather than the change. It is touched on every +matching post, including the no-op that writes nothing else. + +Backfilled from `collectedat` (falling back to `createdat`) so existing rows +start from the last moment we can actually prove the config was current, rather +than from now - claiming a fresh check that never happened would be worse than +saying nothing. +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'backups0002lastseenat' +down_revision = 'backups0001baseline' +branch_labels = None +depends_on = None + + +def upgrade(): + columns = {c['name'] for c in + sa.inspect(op.get_bind()).get_columns('backuprevisions')} + if 'lastseenat' not in columns: + op.add_column('backuprevisions', + sa.Column('lastseenat', sa.DateTime(), nullable=True)) + op.execute('UPDATE backuprevisions ' + 'SET lastseenat = COALESCE(collectedat, createdat)') + + +def downgrade(): + columns = {c['name'] for c in + sa.inspect(op.get_bind()).get_columns('backuprevisions')} + if 'lastseenat' in columns: + op.drop_column('backuprevisions', 'lastseenat') diff --git a/plugins/backups/models/backup.py b/plugins/backups/models/backup.py index 4b22aec..2550eb3 100644 --- a/plugins/backups/models/backup.py +++ b/plugins/backups/models/backup.py @@ -89,6 +89,16 @@ class BackupRevision(db.Model): createdat = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) + # When this configuration was last CONFIRMED still current, which is not the + # same as when it last changed. Dedup means an unchanged config writes no + # revision, so collectedat only ever moves on a change - a machine whose + # settings have been stable for six months has a six-month-old newest + # revision and is perfectly healthy. Without this column ShopDB cannot tell + # that machine from one whose backup stopped running six months ago, which + # is the one question a backup system has to be able to answer. Touched on + # every matching post, including the no-op that writes nothing else. + lastseenat = db.Column(db.DateTime, nullable=True) + __table_args__ = ( db.Index('ixbackuprevisionsassetkind', 'assetid', 'backupkind'), ) @@ -138,6 +148,10 @@ class BackupRevision(db.Model): # BROWSER-LOCAL and the timestamp silently shifts by the viewer's # offset before any site-timezone formatting is applied. 'collectedat': _utciso(self.collectedat), + # When the config was last CONFIRMED current, versus when it last + # changed. The history view needs both or a stable machine looks + # abandoned. + 'lastseenat': _utciso(self.lastseenat), 'createdat': _utciso(self.createdat), } if includepayload: diff --git a/plugins/backups/plugin.py b/plugins/backups/plugin.py index 69ea9e7..0e8b5b7 100644 --- a/plugins/backups/plugin.py +++ b/plugins/backups/plugin.py @@ -161,6 +161,17 @@ class BackupsPlugin(BasePlugin): 'description': 'UNC root that opaque (non-JSON) backups are ' 'written under by the collecting PC. Site-specific.', }, + { + 'key': 'backups_staledays', + 'value': '3', + 'valuetype': 'integer', + 'category': 'backups', + 'description': 'Days without a CONFIRMED check before a backup ' + 'is listed as stopped on the dashboard. Measured ' + 'from the last check, not the last change - an ' + 'unchanged config writes no revision. 0 disables ' + 'the card.', + }, { 'key': 'backups_retentioncount', 'value': '50', @@ -180,6 +191,36 @@ class BackupsPlugin(BasePlugin): }, ] + def get_dashboard_widgets(self) -> List[Dict]: + """Dashboard card: backups that have stopped running. + + Keyed on the last CONFIRMED check, never on the last change. Dedup means + an unchanged config writes no revision, so a card keyed on revision age + would flag most of a healthy fleet. + """ + return [ + { + 'id': 'backups-stale', + 'title': 'Backups stopped', + 'endpoint': '/api/backups/dashboard/stale', + 'render': 'exceptions', + 'severity': 'warning', + 'permission': 'backups.view', + 'empty': 'hide', + 'position': 30, + 'map': { + 'title': 'assetnumber', + 'detail': 'backupkind', + 'meta': [ + {'key': 'sourcehostname', 'label': 'from'}, + {'key': 'quietdays', 'label': 'last checked', + 'suffix': ' days ago'}, + ], + 'link': '/backups/asset/{assetid}', + }, + }, + ] + # ---- ADR-006 collector contract ------------------------------------- def get_collector_schema(self) -> Optional[dict]: @@ -299,6 +340,13 @@ class BackupsPlugin(BasePlugin): .first()) if latest is not None and latest.contenthash == contenthash: + # Unchanged, so no revision - but RECORD THE CHECK. Without this the + # only evidence a backup still runs is a line in a log on the PC, + # and a config stable for six months is indistinguishable from a + # backup that died six months ago. Cheap: one column on a row that + # already exists, no new history. + latest.lastseenat = datetime.utcnow() + db.session.flush() return { 'action': 'noop', 'assetid': assetid, @@ -330,6 +378,7 @@ class BackupsPlugin(BasePlugin): bytesize=bytesize, sourcehostname=payload.get('sourcehostname'), collectedat=collectedat or datetime.utcnow(), + lastseenat=datetime.utcnow(), ) revision.payload = projection db.session.add(revision) diff --git a/plugins/backups/services/staleness.py b/plugins/backups/services/staleness.py new file mode 100644 index 0000000..3831fe7 --- /dev/null +++ b/plugins/backups/services/staleness.py @@ -0,0 +1,85 @@ +"""Which backup chains have stopped running. + +Split from the route for the same reason retention.py is: the rule here is the +interesting part and it should be testable without an auth layer in the way. + +THE RULE, and it is not the obvious one. Dedup means an unchanged configuration +writes no revision, so `collectedat` moves only when something CHANGES. A +machine whose settings have been stable for six months has a six-month-old +newest revision and is entirely healthy. Staleness is therefore measured from +`lastseenat` - when the config was last CONFIRMED current, recorded on every +matching post including the no-op that writes nothing else. + +Keyed on the CHAIN (asset, kind, source PC), because that is the unit that fails +independently. A machine with two part markers can have one still reporting +while the other stopped, and a per-asset view would report the machine as fine. +""" + +from datetime import datetime, timedelta + +from shopdb.api import db, Asset + +from ..models import BackupRevision + +DEFAULTSTALEDAYS = 3 + + +def staledays(): + """Configured threshold; 0 or negative disables the card entirely.""" + from shopdb.api import Setting + + setting = Setting.query.filter_by(key='backups_staledays').first() + if setting and (setting.value or '').strip(): + try: + return int(setting.value) + except (TypeError, ValueError): + pass + return DEFAULTSTALEDAYS + + +def latestperchain(): + """Newest revision of every (asset, kind, source) chain.""" + latest = {} + for revision in (BackupRevision.query + .order_by(BackupRevision.backuprevisionid.asc()).all()): + latest[(revision.assetid, revision.backupkind, + revision.sourcehostname)] = revision + return latest + + +def stalechains(days=None, limit=50): + """Chains whose last confirmed check is older than the threshold. + + Deliberately silent about assets that have NEVER been backed up. Whether one + SHOULD be is a question only the manifest can answer - most machines carry + no NTLARS at all - and guessing would list a hundred healthy assets, which + is the noise that makes a board worth ignoring. That is the + desired-versus-observed card, which needs the manifest resolver. + """ + days = staledays() if days is None else days + if days <= 0: + return [] + + now = datetime.utcnow() + cutoff = now - timedelta(days=days) + + rows = [] + for revision in latestperchain().values(): + # Rows written before lastseenat existed fall back to the timestamps + # that do exist, so an old install reports something sane on day one + # rather than every chain at once. + seen = revision.lastseenat or revision.collectedat or revision.createdat + if seen is None or seen >= cutoff: + continue + asset = db.session.get(Asset, revision.assetid) + rows.append({ + 'assetid': revision.assetid, + 'assetnumber': asset.assetnumber if asset else str(revision.assetid), + 'backupkind': revision.backupkind, + 'sourcehostname': revision.sourcehostname, + 'lastseenat': seen.isoformat() + 'Z', + 'quietdays': (now - seen).days, + }) + + rows.sort(key=lambda r: -r['quietdays']) + return rows[:limit] diff --git a/tests/test_plugin_migrations.py b/tests/test_plugin_migrations.py index 5ebfba1..80bfe3f 100644 --- a/tests/test_plugin_migrations.py +++ b/tests/test_plugin_migrations.py @@ -46,8 +46,10 @@ CUTOVER_PLUGINS = ( # stamp '0001anchor'; measuringtools stamps its real baseline id. EXPECTED_HEAD_REVISION = {plugin: f'{plugin}0001anchor' for plugin in CUTOVER_PLUGINS} EXPECTED_HEAD_REVISION['measuringtools'] = 'measuringtools0001baseline' -# backups is also post-cutover: its 0001 really creates backuprevisions. -EXPECTED_HEAD_REVISION['backups'] = 'backups0001baseline' +# backups is also post-cutover: its 0001 really creates backuprevisions, and +# 0002 adds lastseenat - when a config was last CONFIRMED unchanged, which dedup +# otherwise throws away. +EXPECTED_HEAD_REVISION['backups'] = 'backups0002lastseenat' # geenforce adds the content-addressed blob store (manifestblobs) on top of its # baseline. EXPECTED_HEAD_REVISION['geenforce'] = 'geenforce0002blobs' diff --git a/tests/test_plugins/test_backups.py b/tests/test_plugins/test_backups.py index 3b3945e..acfa92a 100644 --- a/tests/test_plugins/test_backups.py +++ b/tests/test_plugins/test_backups.py @@ -13,6 +13,7 @@ Two layers: """ import base64 +from datetime import datetime, timedelta import json import pytest @@ -862,3 +863,100 @@ def test_the_collection_interval_is_readable_without_a_login(bk_app): BackupsPlugin().get_settings_defaults()} assert declared['backups_intervalhours'].get('public') is True assert not declared['backups_shareroot'].get('public') + + +# ============================================================================= +# Dashboard: backups that have STOPPED, which is not the same as unchanged +# ============================================================================= + +def test_an_unchanged_config_records_the_check(bk_app, bk_plugin): + """The fix that makes the card possible at all. Dedup writes no revision on + a no-op, so without recording the check there is no evidence a backup still + runs - a config stable for six months looks identical to a backup that died + six months ago.""" + from plugins.backups.models import BackupRevision + with bk_app.app_context(): + first = bk_plugin.apply_collector_payload(_payload()) + revision = _db.session.get(BackupRevision, first['backuprevisionid']) + _db.session.query(BackupRevision).update( + {'lastseenat': datetime(2026, 1, 1)}) + _db.session.commit() + + second = bk_plugin.apply_collector_payload(_payload()) + assert second['action'] == 'noop' + _db.session.refresh(revision) + assert revision.lastseenat > datetime(2026, 1, 2) + + +def test_a_stable_machine_is_not_called_stale(bk_app, bk_plugin): + """A machine whose settings have not changed in months is HEALTHY. Keying + the card on revision age would have flagged most of the fleet.""" + from plugins.backups.models import BackupRevision + with bk_app.app_context(): + bk_plugin.apply_collector_payload(_payload()) + # Changed long ago, checked just now: the normal steady state. + _db.session.query(BackupRevision).update({ + 'collectedat': datetime.utcnow() - timedelta(days=180), + 'lastseenat': datetime.utcnow(), + }) + _db.session.commit() + + from plugins.backups.services.staleness import stalechains + assert stalechains() == [] + + +def test_a_machine_that_stopped_checking_in_is_listed(bk_app, bk_plugin): + from plugins.backups.models import BackupRevision + with bk_app.app_context(): + bk_plugin.apply_collector_payload(_payload()) + _db.session.query(BackupRevision).update( + {'lastseenat': datetime.utcnow() - timedelta(days=10)}) + _db.session.commit() + + from plugins.backups.services.staleness import stalechains + [row] = stalechains() + assert row['quietdays'] == 10 + assert row['backupkind'] == 'ntlars' + + +def test_one_marker_stopping_is_not_hidden_by_its_healthy_sibling(bk_app, + bk_plugin): + """Per CHAIN, not per asset. Two part markers share one machine number, and + a per-asset view would report the machine as fine while one device has been + silent for a week.""" + from plugins.backups.models import BackupRevision + with bk_app.app_context(): + bk_plugin.apply_collector_payload( + _payload(reg=_markerreg('COM3'), sourcehostname='MARKERA')) + bk_plugin.apply_collector_payload( + _payload(reg=_markerreg('COM4'), sourcehostname='MARKERB')) + (_db.session.query(BackupRevision) + .filter_by(sourcehostname='MARKERA') + .update({'lastseenat': datetime.utcnow() - timedelta(days=9)})) + (_db.session.query(BackupRevision) + .filter_by(sourcehostname='MARKERB') + .update({'lastseenat': datetime.utcnow()})) + _db.session.commit() + + from plugins.backups.services.staleness import stalechains + rows = stalechains() + assert [r['sourcehostname'] for r in rows] == ['MARKERA'] + + +def test_the_threshold_is_a_setting_and_zero_disables_the_card(bk_app, bk_plugin): + from shopdb.core.models import Setting + from plugins.backups.models import BackupRevision + with bk_app.app_context(): + bk_plugin.apply_collector_payload(_payload()) + _db.session.query(BackupRevision).update( + {'lastseenat': datetime.utcnow() - timedelta(days=2)}) + _db.session.add(Setting(key='backups_staledays', value='1')) + _db.session.commit() + + from plugins.backups.services.staleness import stalechains + assert len(stalechains()) == 1 + + _db.session.query(Setting).filter_by(key='backups_staledays').update( + {'value': '0'}) + _db.session.commit() + assert stalechains() == []