"""Re-file existing backup revisions from an operation onto the part marker. Backups collected before part markers became assets were filed against the operation number the PC reported - several markers' configs landing on one asset, where they overwrote each other. The collector now files a marker PC's backup against that PC's own marker asset, but only for backups collected since the change. Everything already stored stays where it was put, so the new marker assets show an empty history while the operation shows a mixed one. This moves each historical revision to the marker it actually came from. The attribution is exact, not a guess: a revision records the PC it was read from (sourcehostname), and the collector records which marker each PC drives (an active `collector:partmarker` relationship). The move is that mapping applied. WHAT IT MOVES: a revision whose source hostname resolves to a PC that has a marker, and which is currently filed against an asset that is NOT that marker. WHAT IT LEAVES: revisions with no source hostname, since nothing says which marker they came from; revisions from PCs that drive no marker, which is every ordinary machine; and anything already filed correctly. Moving a revision can make it a duplicate of one already on the marker - the same config counted twice, once under the operation and once under the marker. Those are collapsed as part of the move, keeping the earliest, so the marker's history reads as one clean chain. Usage (on the target instance, in the app dir): venv\\Scripts\\python scripts\\refile_partmarker_backups.py # dry run venv\\Scripts\\python scripts\\refile_partmarker_backups.py --commit # apply ... --hostname FBYTNCX3 # one PC Dry run by default. Take a database backup before --commit. """ import argparse import os import sys from collections import defaultdict # Run from anywhere: put the repo root (parent of scripts/) on the path. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from shopdb import create_app from shopdb.api import db, Asset, AssetRelationship, RelationshipType from plugins.backups.models import BackupRevision from plugins.computers.models import Computer from plugins.computers.plugin import PARTMARKER_LINK_ORIGIN def _markerbyhostname(): """hostname (lowercased) -> marker asset, for every PC that drives one.""" controls = RelationshipType.query.filter_by( relationshiptype='controls').first() if controls is None: raise SystemExit("no 'controls' relationship type; " 'run flask seed reference-data') rows = (db.session.query(Computer, AssetRelationship) .join(AssetRelationship, AssetRelationship.sourceassetid == Computer.assetid) .filter(AssetRelationship.relationshiptypeid == controls.relationshiptypeid, AssetRelationship.label == PARTMARKER_LINK_ORIGIN, AssetRelationship.isactive.is_(True)) .all()) markers = {} for computer, link in rows: marker = db.session.get(Asset, link.targetassetid) if marker is not None and computer.hostname: markers[computer.hostname.strip().lower()] = marker return markers def _dedupe(revisions): """Within one chain, drop a revision repeating the hash before it. Applied after a move, because the same config can arrive twice: once as the row filed under the operation and once as a row the collector has already written against the marker. Keeps the earliest of each run, so the first time a config was seen is the date that survives. """ doomed = [] previoushash = None for revision in sorted(revisions, key=lambda r: r.backuprevisionid): if previoushash is not None and revision.contenthash == previoushash: doomed.append(revision) else: previoushash = revision.contenthash return doomed def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--commit', action='store_true', help='apply the moves (default is a dry run)') parser.add_argument('--hostname', help='limit to one source PC') args = parser.parse_args() app = create_app(os.environ.get('FLASK_ENV', 'production')) with app.app_context(): markers = _markerbyhostname() if not markers: print('No PC drives a part marker. Nothing to re-file.') print('If that is unexpected, the marker assets are minted by the ' 'computers collector when a PC reports with ' 'pctype=gea-shopfloor-partmarker.') return print('{} PC(s) drive a marker.'.format(len(markers))) query = db.session.query(BackupRevision).filter( BackupRevision.sourcehostname.isnot(None)) if args.hostname: query = query.filter( BackupRevision.sourcehostname.ilike(args.hostname)) moved = 0 touchedchains = defaultdict(list) for revision in query.order_by(BackupRevision.backuprevisionid).all(): marker = markers.get((revision.sourcehostname or '').strip().lower()) if marker is None or revision.assetid == marker.assetid: continue fromasset = db.session.get(Asset, revision.assetid) print(' {:<12} {:<10} rev {:<6} {} -> {}'.format( revision.sourcehostname, revision.backupkind, revision.backuprevisionid, fromasset.assetnumber if fromasset else revision.assetid, marker.assetnumber)) revision.assetid = marker.assetid moved += 1 touchedchains[(marker.assetid, revision.backupkind, revision.sourcehostname)].append(revision) # Collapse duplicates the move created, counting rows already on the # marker so an existing revision is compared against, not ignored. collapsed = 0 for (assetid, kind, source) in list(touchedchains): chain = (db.session.query(BackupRevision) .filter(BackupRevision.assetid == assetid, BackupRevision.backupkind == kind, BackupRevision.sourcehostname == source) .all()) for revision in _dedupe(chain): db.session.delete(revision) collapsed += 1 print() print('{} revision(s) re-filed, {} duplicate(s) collapsed.'.format( moved, collapsed)) if not moved: print('Nothing to do.') db.session.rollback() return if args.commit: db.session.commit() print('APPLIED.') else: db.session.rollback() print('Dry run - nothing changed. Re-run with --commit to apply.') if __name__ == '__main__': main()