Backups: a revision chain belongs to a PC, not just a machine

plugins/backups/api/routes.py

@@ -27,6 +27,7 @@ from shopdb.api import (
 from ..models import BackupRevision
 from ..models.backup import _utciso
 from ..services.registry import REGISTRY, getkind
+from ..services.retention import samesource
 
 backups_bp = Blueprint('backups', __name__)
 
@@ -278,9 +279,13 @@ def diff_revision(backuprevisionid):
     if againstid:
         other = db.session.get(BackupRevision, againstid)
     else:
+        # The previous revision FROM THE SAME PC. Several PCs can share a
+        # machine number, and diffing across them reported one device's COM port
+        # as a change on another, which is not a change at all.
         other = (db.session.query(BackupRevision)
                  .filter(BackupRevision.assetid == revision.assetid,
                          BackupRevision.backupkind == revision.backupkind,
+                         samesource(revision.sourcehostname),
                          BackupRevision.backuprevisionid < revision.backuprevisionid)
                  .order_by(BackupRevision.backuprevisionid.desc())
                  .first())

plugins/backups/plugin.py

@@ -25,7 +25,7 @@ from .api import backups_bp
 from .models import BackupRevision
 from .services.registry import (REGISTRY, getkind, canonicalhash,
                                DEFAULTSHAREROOT)
-from .services.retention import prune
+from .services.retention import prune, samesource as _samesource
 
 logger = logging.getLogger(__name__)
 
@@ -266,15 +266,26 @@ class BackupsPlugin(BasePlugin):
                     "contenthash and sharepath are required for kind '{}'".format(
                         kindkey))
 
-        # with_for_update serializes concurrent posts for the same asset+kind.
-        # The whole fleet collects on the same GE-Enforce cycle, so a retry or
-        # two PCs claiming one machine number can otherwise both pass the hash
-        # check and insert duplicate identical revisions. No unique constraint
-        # can cover this: dedup is against the LATEST row only, because a config
+        # Dedup is against the LATEST row of THIS SOURCE's chain, not the
+        # asset's. Several PCs legitimately share one machine number here - the
+        # part markers on 0613, 0615 and WJPRT do, and their configs genuinely
+        # differ, typically by COM port. Keyed on
+        # asset alone, each marker's post differed from whichever marker posted
+        # last, so nothing ever deduped and the chain grew by one revision per
+        # PC per collection cycle. Keyed on the source, an unchanged config is a
+        # no-op again and each marker keeps its own history against the machine.
+        #
+        # Dedup is against the latest row only, not any row, because a config
         # reverting to an earlier state is a legitimate new revision.
+        #
+        # with_for_update serializes concurrent posts for the same chain: the
+        # fleet collects on one GE-Enforce cycle, so a retry could otherwise
+        # pass the hash check twice and insert the same revision.
+        sourcehostname = payload.get('sourcehostname')
         latest = (db.session.query(BackupRevision)
                   .filter(BackupRevision.assetid == assetid,
-                          BackupRevision.backupkind == kindkey)
+                          BackupRevision.backupkind == kindkey,
+                          _samesource(sourcehostname))
                   .order_by(BackupRevision.backuprevisionid.desc())
                   .with_for_update()
                   .first())

plugins/backups/services/retention.py

@@ -2,18 +2,23 @@
 
 Dedup already keeps growth low - a revision appears only when a setting really
 changed - so retention exists for the pathological case, not the normal one: a
-value that flaps (or two PCs alternately claiming one machine number) would
-otherwise append a revision every collection cycle forever.
+value that flaps would otherwise append a revision every collection cycle
+forever.
 
 Two independent limits, both off by default at 0:
 
-  retentioncount  keep at most N revisions per asset per kind
+  retentioncount  keep at most N revisions per chain
   retentiondays   additionally drop anything older than N days
 
-The NEWEST and the OLDEST revision are never pruned. The newest is the one a
-tech restores from; the oldest is the earliest known-good baseline, which is
-usually the most valuable row in the table and the one a naive "keep last N"
-would delete first.
+A CHAIN is one asset, one kind, one source hostname. Several PCs can share a
+machine number, and each keeps its own revision history against that machine;
+pruning per asset would let a busy marker's revisions push a quiet marker's
+only backup out of the table.
+
+The NEWEST and the OLDEST revision of each chain are never pruned. The newest
+is the one a tech restores from; the oldest is the earliest known-good
+baseline, which is usually the most valuable row in the table and the one a
+naive "keep last N" would delete first.
 """
 
 from datetime import datetime, timedelta
@@ -23,11 +28,45 @@ from shopdb.api import db
 from ..models import BackupRevision
 
 
+def samesource(sourcehostname):
+    """Filter matching one source's chain, treating NULL as its own source.
+
+    `sourcehostname == None` never matches in SQL, so a NULL-sourced row - every
+    row written before the column was populated, and any hand-loaded one - needs
+    IS NULL. Without this those rows form no chain at all and re-post forever.
+    """
+    if sourcehostname is None or sourcehostname == '':
+        return BackupRevision.sourcehostname.is_(None)
+    return BackupRevision.sourcehostname == sourcehostname
+
+
+def sourcesfor(assetid, backupkind):
+    """Every distinct source hostname with a revision for this asset+kind."""
+    rows = (db.session.query(BackupRevision.sourcehostname)
+            .filter(BackupRevision.assetid == assetid,
+                    BackupRevision.backupkind == backupkind)
+            .distinct()
+            .all())
+    return [row[0] for row in rows]
+
+
 def prune(assetid, backupkind, retentioncount=0, retentiondays=0):
     """Delete surplus revisions for one asset+kind. Returns the number removed.
 
-    Caller commits. Returns 0 when both limits are disabled.
+    Prunes each source's chain independently. Caller commits. Returns 0 when
+    both limits are disabled.
     """
+    if int(retentioncount or 0) <= 0 and int(retentiondays or 0) <= 0:
+        return 0
+    return sum(prunechain(assetid, backupkind, source,
+                          retentioncount=retentioncount,
+                          retentiondays=retentiondays)
+               for source in sourcesfor(assetid, backupkind))
+
+
+def prunechain(assetid, backupkind, sourcehostname, retentioncount=0,
+               retentiondays=0):
+    """Delete surplus revisions for one asset+kind+source chain."""
     retentioncount = int(retentioncount or 0)
     retentiondays = int(retentiondays or 0)
     if retentioncount <= 0 and retentiondays <= 0:
@@ -35,7 +74,8 @@ def prune(assetid, backupkind, retentioncount=0, retentiondays=0):
 
     revisions = (db.session.query(BackupRevision)
                  .filter(BackupRevision.assetid == assetid,
-                         BackupRevision.backupkind == backupkind)
+                         BackupRevision.backupkind == backupkind,
+                         samesource(sourcehostname))
                  .order_by(BackupRevision.backuprevisionid.desc())
                  .all())
     if len(revisions) <= 2: