backups: record that a config was checked, not only that it changed

The stale-backup card could not be built as designed, and the reason is more
important than the card. Dedup means an unchanged configuration writes no
revision, so collectedat moves only on a CHANGE. A machine stable for six
months has a six-month-old newest revision and is perfectly healthy. Keying a
staleness card on revision age would have flagged most of the fleet - exactly
the noise that makes a board worth ignoring.

Underneath that: ShopDB could not distinguish those cases at all. On a no-op
the server returned "unchanged" and wrote nothing, so "we checked yesterday and
it matched" was discarded. That fact is the one thing a backup system must be
able to prove, and the only record of it was a line in a log file on the PC.

lastseenat records the check rather than the change. Touched on every matching
post including the no-op; set on creation, since a new revision has by
definition just been seen; backfilled from collectedat or createdat so existing
rows start from the last moment the config can be PROVEN current, rather than
from now - claiming a check that never happened would be worse than silence.

The card keys on it, one row per CHAIN rather than per asset: a machine with
two part markers can have one still reporting while the other stopped, and a
per-asset view would report the machine as fine. It stays deliberately silent
about assets never backed up, because whether one SHOULD be is a question only
the manifest can answer, and guessing would list a hundred healthy machines.

The rule lives in services/staleness.py rather than the route, so it is
testable without an auth layer in the way - the same split retention.py uses.

Threshold is backups_staledays, default 3, and 0 disables the card.

tests/test_plugin_migrations.py

@@ -46,8 +46,10 @@ CUTOVER_PLUGINS = (
 # stamp '<plugin>0001anchor'; measuringtools stamps its real baseline id.
 EXPECTED_HEAD_REVISION = {plugin: f'{plugin}0001anchor' for plugin in CUTOVER_PLUGINS}
 EXPECTED_HEAD_REVISION['measuringtools'] = 'measuringtools0001baseline'
-# backups is also post-cutover: its 0001 really creates backuprevisions.
-EXPECTED_HEAD_REVISION['backups'] = 'backups0001baseline'
+# backups is also post-cutover: its 0001 really creates backuprevisions, and
+# 0002 adds lastseenat - when a config was last CONFIRMED unchanged, which dedup
+# otherwise throws away.
+EXPECTED_HEAD_REVISION['backups'] = 'backups0002lastseenat'
 # geenforce adds the content-addressed blob store (manifestblobs) on top of its
 # baseline.
 EXPECTED_HEAD_REVISION['geenforce'] = 'geenforce0002blobs'

tests/test_plugins/test_backups.py

@@ -13,6 +13,7 @@ Two layers:
 """
 
 import base64
+from datetime import datetime, timedelta
 import json
 
 import pytest
@@ -862,3 +863,100 @@ def test_the_collection_interval_is_readable_without_a_login(bk_app):
                 BackupsPlugin().get_settings_defaults()}
     assert declared['backups_intervalhours'].get('public') is True
     assert not declared['backups_shareroot'].get('public')
+
+
+# =============================================================================
+# Dashboard: backups that have STOPPED, which is not the same as unchanged
+# =============================================================================
+
+def test_an_unchanged_config_records_the_check(bk_app, bk_plugin):
+    """The fix that makes the card possible at all. Dedup writes no revision on
+    a no-op, so without recording the check there is no evidence a backup still
+    runs - a config stable for six months looks identical to a backup that died
+    six months ago."""
+    from plugins.backups.models import BackupRevision
+    with bk_app.app_context():
+        first = bk_plugin.apply_collector_payload(_payload())
+        revision = _db.session.get(BackupRevision, first['backuprevisionid'])
+        _db.session.query(BackupRevision).update(
+            {'lastseenat': datetime(2026, 1, 1)})
+        _db.session.commit()
+
+        second = bk_plugin.apply_collector_payload(_payload())
+        assert second['action'] == 'noop'
+        _db.session.refresh(revision)
+        assert revision.lastseenat > datetime(2026, 1, 2)
+
+
+def test_a_stable_machine_is_not_called_stale(bk_app, bk_plugin):
+    """A machine whose settings have not changed in months is HEALTHY. Keying
+    the card on revision age would have flagged most of the fleet."""
+    from plugins.backups.models import BackupRevision
+    with bk_app.app_context():
+        bk_plugin.apply_collector_payload(_payload())
+        # Changed long ago, checked just now: the normal steady state.
+        _db.session.query(BackupRevision).update({
+            'collectedat': datetime.utcnow() - timedelta(days=180),
+            'lastseenat': datetime.utcnow(),
+        })
+        _db.session.commit()
+
+        from plugins.backups.services.staleness import stalechains
+        assert stalechains() == []
+
+
+def test_a_machine_that_stopped_checking_in_is_listed(bk_app, bk_plugin):
+    from plugins.backups.models import BackupRevision
+    with bk_app.app_context():
+        bk_plugin.apply_collector_payload(_payload())
+        _db.session.query(BackupRevision).update(
+            {'lastseenat': datetime.utcnow() - timedelta(days=10)})
+        _db.session.commit()
+
+        from plugins.backups.services.staleness import stalechains
+        [row] = stalechains()
+        assert row['quietdays'] == 10
+        assert row['backupkind'] == 'ntlars'
+
+
+def test_one_marker_stopping_is_not_hidden_by_its_healthy_sibling(bk_app,
+                                                                  bk_plugin):
+    """Per CHAIN, not per asset. Two part markers share one machine number, and
+    a per-asset view would report the machine as fine while one device has been
+    silent for a week."""
+    from plugins.backups.models import BackupRevision
+    with bk_app.app_context():
+        bk_plugin.apply_collector_payload(
+            _payload(reg=_markerreg('COM3'), sourcehostname='MARKERA'))
+        bk_plugin.apply_collector_payload(
+            _payload(reg=_markerreg('COM4'), sourcehostname='MARKERB'))
+        (_db.session.query(BackupRevision)
+         .filter_by(sourcehostname='MARKERA')
+         .update({'lastseenat': datetime.utcnow() - timedelta(days=9)}))
+        (_db.session.query(BackupRevision)
+         .filter_by(sourcehostname='MARKERB')
+         .update({'lastseenat': datetime.utcnow()}))
+        _db.session.commit()
+
+        from plugins.backups.services.staleness import stalechains
+        rows = stalechains()
+        assert [r['sourcehostname'] for r in rows] == ['MARKERA']
+
+
+def test_the_threshold_is_a_setting_and_zero_disables_the_card(bk_app, bk_plugin):
+    from shopdb.core.models import Setting
+    from plugins.backups.models import BackupRevision
+    with bk_app.app_context():
+        bk_plugin.apply_collector_payload(_payload())
+        _db.session.query(BackupRevision).update(
+            {'lastseenat': datetime.utcnow() - timedelta(days=2)})
+        _db.session.add(Setting(key='backups_staledays', value='1'))
+        _db.session.commit()
+
+        from plugins.backups.services.staleness import stalechains
+        assert len(stalechains()) == 1
+
+        _db.session.query(Setting).filter_by(key='backups_staledays').update(
+            {'value': '0'})
+        _db.session.commit()
+        assert stalechains() == []