217 lines
9.1 KiB
Python
217 lines
9.1 KiB
Python
"""Collapse the duplicate backup revisions left by the shared-machine-number bug.
|
|
|
|
Dedup used to compare a posted config against the latest revision for the
|
|
ASSET. That is correct only when a machine number means one PC. Several PCs
|
|
legitimately share one here - the part markers on 0613, 0615 and WJPRT, whose
|
|
configs differ by COM port - so each marker's post differed from whichever
|
|
marker had posted last, nothing ever deduped, and the table grew by one row per
|
|
PC per collection cycle. Dedup is now per source hostname; this cleans up what
|
|
the old rule wrote.
|
|
|
|
A machine number that is NOT meant to be shared but shows two sources here is a
|
|
different fault - two PCs carrying the same machine number - and this script
|
|
will report it as two chains. Deal with that at the PC, not in the database.
|
|
|
|
WHAT IT REMOVES, and nothing else: within one chain (asset + kind + source
|
|
hostname), a revision whose content hash equals the revision immediately before
|
|
it from the same source. Those rows record no change and are exactly the rows
|
|
the fixed code would not have created.
|
|
|
|
WHAT IT KEEPS: every genuine change, in order; the newest revision of every
|
|
chain; the oldest revision of every chain; and every chain of every source, so
|
|
a quiet PC's only backup is never touched. A config that reverts to an earlier
|
|
state keeps both rows, because that is a real change and not a duplicate.
|
|
|
|
Share-backed kinds (partmarker) are metadata rows pointing at files on the
|
|
share. Deleting a row does not touch the file.
|
|
|
|
Usage (on the target instance, in the app dir):
|
|
venv\\Scripts\\python scripts\\collapse_duplicate_backup_revisions.py
|
|
venv\\Scripts\\python scripts\\collapse_duplicate_backup_revisions.py --commit
|
|
... --assetnumber 0615 # limit to one machine
|
|
... --kind ntlars # limit to one kind
|
|
|
|
Dry run by default: it prints what it would delete and changes nothing. Take a
|
|
database backup before --commit anyway; this deletes rows.
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from collections import defaultdict
|
|
|
|
# Run from anywhere: put the repo root (parent of scripts/) on the path.
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
from dotenv import load_dotenv
|
|
|
|
# Load .env before anything reads config. wsgi.py does this for the served app
|
|
# and the `flask` CLI does it for itself, but a plain `python scripts/...` run
|
|
# gets neither: DATABASE_URL stays unset, FLASK_ENV defaults to development,
|
|
# and config falls back to a hardcoded root@localhost URL. That fails on a real
|
|
# deployment with a bare 1045 Access denied for 'root', naming a user the site
|
|
# does not use - and on a box where those root credentials happen to work, it
|
|
# is worse, because the script silently reads or WRITES the wrong database.
|
|
load_dotenv(Path(__file__).resolve().parent.parent / '.env')
|
|
|
|
|
|
from shopdb import create_app
|
|
from shopdb.api import db
|
|
from shopdb.core.models import Asset
|
|
from plugins.backups.models import BackupRevision
|
|
|
|
|
|
def _chains(assetnumber=None, kind=None):
|
|
"""Every revision grouped into (assetid, kind, sourcehostname) chains."""
|
|
query = db.session.query(BackupRevision)
|
|
if assetnumber:
|
|
asset = db.session.query(Asset).filter(
|
|
Asset.assetnumber == assetnumber).first()
|
|
if asset is None:
|
|
raise SystemExit('no asset with assetnumber {!r}'.format(assetnumber))
|
|
query = query.filter(BackupRevision.assetid == asset.assetid)
|
|
if kind:
|
|
query = query.filter(BackupRevision.backupkind == kind)
|
|
|
|
chains = defaultdict(list)
|
|
for revision in query.order_by(BackupRevision.backuprevisionid.asc()).all():
|
|
key = (revision.assetid, revision.backupkind, revision.sourcehostname)
|
|
chains[key].append(revision)
|
|
return chains
|
|
|
|
|
|
def _redundant(revisions):
|
|
"""Revisions in one chain that repeat the hash of the one before them.
|
|
|
|
The chain is in id order, so this walks it forwards and keeps the first
|
|
appearance of each run. A later run of the same hash after a genuine change
|
|
is kept: that is a revert, which is a real event.
|
|
"""
|
|
doomed = []
|
|
previoushash = None
|
|
for revision in revisions:
|
|
if previoushash is not None and revision.contenthash == previoushash:
|
|
doomed.append(revision)
|
|
else:
|
|
previoushash = revision.contenthash
|
|
# Never drop the newest row of a chain, even if it repeats: it is the row
|
|
# the asset panel and a restore both read.
|
|
newestid = revisions[-1].backuprevisionid
|
|
return [r for r in doomed if r.backuprevisionid != newestid]
|
|
|
|
|
|
def _report(chains):
|
|
"""Say what grew each chain, so a real cause can be told from the shared
|
|
machine numbers.
|
|
|
|
Three findings, and they need different fixes:
|
|
|
|
several sources on one machine number - either a legitimately shared
|
|
number (0613, 0615, WJPRT) or, on any other number, two PCs carrying the
|
|
same machine number, which is a fault to fix on the PC.
|
|
|
|
one source, many revisions, few distinct hashes - the old asset-wide
|
|
dedup, or a value flapping between two states.
|
|
|
|
one source, many revisions, all distinct hashes - something in the
|
|
captured config changes on its own. The value names listed are the ones
|
|
that changed most often between consecutive revisions; a timestamp or a
|
|
session counter in the projection would show up here, and the fix is to
|
|
exclude it from the hash, not to delete rows.
|
|
"""
|
|
from plugins.backups.api.routes import _diffprojections
|
|
|
|
bymachine = defaultdict(list)
|
|
for key, revisions in chains.items():
|
|
bymachine[key[0], key[1]].append((key[2], revisions))
|
|
|
|
for (assetid, kind), sources in sorted(bymachine.items()):
|
|
asset = db.session.get(Asset, assetid)
|
|
name = asset.assetnumber if asset else str(assetid)
|
|
total = sum(len(r) for _, r in sources)
|
|
print('{} / {}: {} revisions from {} source(s)'.format(
|
|
name, kind, total, len(sources)))
|
|
|
|
for source, revisions in sorted(sources, key=lambda s: s[0] or ''):
|
|
hashes = {r.contenthash for r in revisions}
|
|
print(' {:<20} {:>5} revisions, {:>4} distinct'.format(
|
|
source or '(no source)', len(revisions), len(hashes)))
|
|
|
|
if len(revisions) < 3 or kind != 'ntlars':
|
|
continue
|
|
churn = defaultdict(int)
|
|
for older, newer in zip(revisions, revisions[1:]):
|
|
if older.contenthash == newer.contenthash:
|
|
continue
|
|
for change in _diffprojections(older.payload, newer.payload):
|
|
churn['{}\\{}'.format(change['keypath'],
|
|
change['valuename'])] += 1
|
|
for valuename, count in sorted(churn.items(),
|
|
key=lambda kv: -kv[1])[:5]:
|
|
print(' changed {:>4}x {}'.format(count, valuename))
|
|
print()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--commit', action='store_true',
|
|
help='apply the deletions (default is a dry run)')
|
|
parser.add_argument('--report', action='store_true',
|
|
help='explain what grew each chain and delete nothing')
|
|
parser.add_argument('--assetnumber', help='limit to one machine number')
|
|
parser.add_argument('--kind', help='limit to one backup kind')
|
|
args = parser.parse_args()
|
|
|
|
app = create_app(os.environ.get('FLASK_ENV', 'production'))
|
|
with app.app_context():
|
|
chains = _chains(args.assetnumber, args.kind)
|
|
if not chains:
|
|
print('No revisions matched.')
|
|
return
|
|
|
|
if args.report:
|
|
_report(chains)
|
|
return
|
|
|
|
assetnames = {}
|
|
total = 0
|
|
machines = defaultdict(int)
|
|
|
|
for (assetid, kind, source), revisions in sorted(chains.items(),
|
|
key=lambda kv: kv[0]):
|
|
doomed = _redundant(revisions)
|
|
if not doomed:
|
|
continue
|
|
if assetid not in assetnames:
|
|
asset = db.session.get(Asset, assetid)
|
|
assetnames[assetid] = (asset.assetnumber if asset
|
|
else str(assetid))
|
|
name = assetnames[assetid]
|
|
machines[name] += len(doomed)
|
|
total += len(doomed)
|
|
print('{:>10} {:<12} {:<20} {} of {} redundant'.format(
|
|
name, kind, source or '(no source)', len(doomed),
|
|
len(revisions)))
|
|
for revision in doomed:
|
|
db.session.delete(revision)
|
|
|
|
print()
|
|
for name in sorted(machines):
|
|
print('{:>10}: {} rows'.format(name, machines[name]))
|
|
print('{} chains, {} redundant revisions'.format(len(chains), total))
|
|
|
|
if not total:
|
|
print('Nothing to do.')
|
|
return
|
|
if args.commit:
|
|
db.session.commit()
|
|
print('DELETED {} revisions.'.format(total))
|
|
else:
|
|
db.session.rollback()
|
|
print('Dry run - nothing changed. Re-run with --commit to apply.')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|