109 lines
4.2 KiB
Python
109 lines
4.2 KiB
Python
"""Revision retention.
|
|
|
|
Dedup already keeps growth low - a revision appears only when a setting really
|
|
changed - so retention exists for the pathological case, not the normal one: a
|
|
value that flaps would otherwise append a revision every collection cycle
|
|
forever.
|
|
|
|
Two independent limits, both off by default at 0:
|
|
|
|
retentioncount keep at most N revisions per chain
|
|
retentiondays additionally drop anything older than N days
|
|
|
|
A CHAIN is one asset, one kind, one source hostname. Several PCs can share a
|
|
machine number, and each keeps its own revision history against that machine;
|
|
pruning per asset would let a busy marker's revisions push a quiet marker's
|
|
only backup out of the table.
|
|
|
|
The NEWEST and the OLDEST revision of each chain are never pruned. The newest
|
|
is the one a tech restores from; the oldest is the earliest known-good
|
|
baseline, which is usually the most valuable row in the table and the one a
|
|
naive "keep last N" would delete first.
|
|
"""
|
|
|
|
from datetime import datetime, timedelta
|
|
|
|
from shopdb.api import db
|
|
|
|
from ..models import BackupRevision
|
|
|
|
|
|
def samesource(sourcehostname):
|
|
"""Filter matching one source's chain, treating NULL as its own source.
|
|
|
|
`sourcehostname == None` never matches in SQL, so a NULL-sourced row - every
|
|
row written before the column was populated, and any hand-loaded one - needs
|
|
IS NULL. Without this those rows form no chain at all and re-post forever.
|
|
"""
|
|
if sourcehostname is None or sourcehostname == '':
|
|
return BackupRevision.sourcehostname.is_(None)
|
|
return BackupRevision.sourcehostname == sourcehostname
|
|
|
|
|
|
def sourcesfor(assetid, backupkind):
|
|
"""Every distinct source hostname with a revision for this asset+kind."""
|
|
rows = (db.session.query(BackupRevision.sourcehostname)
|
|
.filter(BackupRevision.assetid == assetid,
|
|
BackupRevision.backupkind == backupkind)
|
|
.distinct()
|
|
.all())
|
|
return [row[0] for row in rows]
|
|
|
|
|
|
def prune(assetid, backupkind, retentioncount=0, retentiondays=0):
|
|
"""Delete surplus revisions for one asset+kind. Returns the number removed.
|
|
|
|
Prunes each source's chain independently. Caller commits. Returns 0 when
|
|
both limits are disabled.
|
|
"""
|
|
if int(retentioncount or 0) <= 0 and int(retentiondays or 0) <= 0:
|
|
return 0
|
|
return sum(prunechain(assetid, backupkind, source,
|
|
retentioncount=retentioncount,
|
|
retentiondays=retentiondays)
|
|
for source in sourcesfor(assetid, backupkind))
|
|
|
|
|
|
def prunechain(assetid, backupkind, sourcehostname, retentioncount=0,
|
|
retentiondays=0):
|
|
"""Delete surplus revisions for one asset+kind+source chain."""
|
|
retentioncount = int(retentioncount or 0)
|
|
retentiondays = int(retentiondays or 0)
|
|
if retentioncount <= 0 and retentiondays <= 0:
|
|
return 0
|
|
|
|
revisions = (db.session.query(BackupRevision)
|
|
.filter(BackupRevision.assetid == assetid,
|
|
BackupRevision.backupkind == backupkind,
|
|
samesource(sourcehostname))
|
|
.order_by(BackupRevision.backuprevisionid.desc())
|
|
.all())
|
|
if len(revisions) <= 2:
|
|
return 0 # newest + oldest are both protected
|
|
|
|
newest = revisions[0]
|
|
oldest = revisions[-1]
|
|
protected = {newest.backuprevisionid, oldest.backuprevisionid}
|
|
|
|
doomed = []
|
|
|
|
if retentioncount > 0 and len(revisions) > retentioncount:
|
|
# Walk from the oldest END of the middle, so the surplus dropped is
|
|
# always the least recent, never the newest.
|
|
for revision in revisions[retentioncount:]:
|
|
if revision.backuprevisionid not in protected:
|
|
doomed.append(revision)
|
|
|
|
if retentiondays > 0:
|
|
cutoff = datetime.utcnow() - timedelta(days=retentiondays)
|
|
for revision in revisions:
|
|
if revision.backuprevisionid in protected:
|
|
continue
|
|
stamp = revision.collectedat or revision.createdat
|
|
if stamp and stamp < cutoff and revision not in doomed:
|
|
doomed.append(revision)
|
|
|
|
for revision in doomed:
|
|
db.session.delete(revision)
|
|
return len(doomed)
|