Re-file historical part-marker backups
This commit is contained in:
166
scripts/refile_partmarker_backups.py
Normal file
166
scripts/refile_partmarker_backups.py
Normal file
@@ -0,0 +1,166 @@
|
||||
"""Re-file existing backup revisions from an operation onto the part marker.
|
||||
|
||||
Backups collected before part markers became assets were filed against the
|
||||
operation number the PC reported - several markers' configs landing on one
|
||||
asset, where they overwrote each other. The collector now files a marker PC's
|
||||
backup against that PC's own marker asset, but only for backups collected since
|
||||
the change. Everything already stored stays where it was put, so the new marker
|
||||
assets show an empty history while the operation shows a mixed one.
|
||||
|
||||
This moves each historical revision to the marker it actually came from. The
|
||||
attribution is exact, not a guess: a revision records the PC it was read from
|
||||
(sourcehostname), and the collector records which marker each PC drives (an
|
||||
active `collector:partmarker` relationship). The move is that mapping applied.
|
||||
|
||||
WHAT IT MOVES: a revision whose source hostname resolves to a PC that has a
|
||||
marker, and which is currently filed against an asset that is NOT that marker.
|
||||
|
||||
WHAT IT LEAVES: revisions with no source hostname, since nothing says which
|
||||
marker they came from; revisions from PCs that drive no marker, which is every
|
||||
ordinary machine; and anything already filed correctly.
|
||||
|
||||
Moving a revision can make it a duplicate of one already on the marker - the
|
||||
same config counted twice, once under the operation and once under the marker.
|
||||
Those are collapsed as part of the move, keeping the earliest, so the marker's
|
||||
history reads as one clean chain.
|
||||
|
||||
Usage (on the target instance, in the app dir):
|
||||
venv\\Scripts\\python scripts\\refile_partmarker_backups.py # dry run
|
||||
venv\\Scripts\\python scripts\\refile_partmarker_backups.py --commit # apply
|
||||
... --hostname FBYTNCX3 # one PC
|
||||
|
||||
Dry run by default. Take a database backup before --commit.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
# Run from anywhere: put the repo root (parent of scripts/) on the path.
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from shopdb import create_app
|
||||
from shopdb.api import db, Asset, AssetRelationship, RelationshipType
|
||||
from plugins.backups.models import BackupRevision
|
||||
from plugins.computers.models import Computer
|
||||
from plugins.computers.plugin import PARTMARKER_LINK_ORIGIN
|
||||
|
||||
|
||||
def _markerbyhostname():
|
||||
"""hostname (lowercased) -> marker asset, for every PC that drives one."""
|
||||
controls = RelationshipType.query.filter_by(
|
||||
relationshiptype='controls').first()
|
||||
if controls is None:
|
||||
raise SystemExit("no 'controls' relationship type; "
|
||||
'run flask seed reference-data')
|
||||
|
||||
rows = (db.session.query(Computer, AssetRelationship)
|
||||
.join(AssetRelationship,
|
||||
AssetRelationship.sourceassetid == Computer.assetid)
|
||||
.filter(AssetRelationship.relationshiptypeid ==
|
||||
controls.relationshiptypeid,
|
||||
AssetRelationship.label == PARTMARKER_LINK_ORIGIN,
|
||||
AssetRelationship.isactive.is_(True))
|
||||
.all())
|
||||
|
||||
markers = {}
|
||||
for computer, link in rows:
|
||||
marker = db.session.get(Asset, link.targetassetid)
|
||||
if marker is not None and computer.hostname:
|
||||
markers[computer.hostname.strip().lower()] = marker
|
||||
return markers
|
||||
|
||||
|
||||
def _dedupe(revisions):
|
||||
"""Within one chain, drop a revision repeating the hash before it.
|
||||
|
||||
Applied after a move, because the same config can arrive twice: once as the
|
||||
row filed under the operation and once as a row the collector has already
|
||||
written against the marker. Keeps the earliest of each run, so the first
|
||||
time a config was seen is the date that survives.
|
||||
"""
|
||||
doomed = []
|
||||
previoushash = None
|
||||
for revision in sorted(revisions, key=lambda r: r.backuprevisionid):
|
||||
if previoushash is not None and revision.contenthash == previoushash:
|
||||
doomed.append(revision)
|
||||
else:
|
||||
previoushash = revision.contenthash
|
||||
return doomed
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--commit', action='store_true',
|
||||
help='apply the moves (default is a dry run)')
|
||||
parser.add_argument('--hostname', help='limit to one source PC')
|
||||
args = parser.parse_args()
|
||||
|
||||
app = create_app(os.environ.get('FLASK_ENV', 'production'))
|
||||
with app.app_context():
|
||||
markers = _markerbyhostname()
|
||||
if not markers:
|
||||
print('No PC drives a part marker. Nothing to re-file.')
|
||||
print('If that is unexpected, the marker assets are minted by the '
|
||||
'computers collector when a PC reports with '
|
||||
'pctype=gea-shopfloor-partmarker.')
|
||||
return
|
||||
|
||||
print('{} PC(s) drive a marker.'.format(len(markers)))
|
||||
|
||||
query = db.session.query(BackupRevision).filter(
|
||||
BackupRevision.sourcehostname.isnot(None))
|
||||
if args.hostname:
|
||||
query = query.filter(
|
||||
BackupRevision.sourcehostname.ilike(args.hostname))
|
||||
|
||||
moved = 0
|
||||
touchedchains = defaultdict(list)
|
||||
|
||||
for revision in query.order_by(BackupRevision.backuprevisionid).all():
|
||||
marker = markers.get((revision.sourcehostname or '').strip().lower())
|
||||
if marker is None or revision.assetid == marker.assetid:
|
||||
continue
|
||||
fromasset = db.session.get(Asset, revision.assetid)
|
||||
print(' {:<12} {:<10} rev {:<6} {} -> {}'.format(
|
||||
revision.sourcehostname, revision.backupkind,
|
||||
revision.backuprevisionid,
|
||||
fromasset.assetnumber if fromasset else revision.assetid,
|
||||
marker.assetnumber))
|
||||
revision.assetid = marker.assetid
|
||||
moved += 1
|
||||
touchedchains[(marker.assetid, revision.backupkind,
|
||||
revision.sourcehostname)].append(revision)
|
||||
|
||||
# Collapse duplicates the move created, counting rows already on the
|
||||
# marker so an existing revision is compared against, not ignored.
|
||||
collapsed = 0
|
||||
for (assetid, kind, source) in list(touchedchains):
|
||||
chain = (db.session.query(BackupRevision)
|
||||
.filter(BackupRevision.assetid == assetid,
|
||||
BackupRevision.backupkind == kind,
|
||||
BackupRevision.sourcehostname == source)
|
||||
.all())
|
||||
for revision in _dedupe(chain):
|
||||
db.session.delete(revision)
|
||||
collapsed += 1
|
||||
|
||||
print()
|
||||
print('{} revision(s) re-filed, {} duplicate(s) collapsed.'.format(
|
||||
moved, collapsed))
|
||||
|
||||
if not moved:
|
||||
print('Nothing to do.')
|
||||
db.session.rollback()
|
||||
return
|
||||
if args.commit:
|
||||
db.session.commit()
|
||||
print('APPLIED.')
|
||||
else:
|
||||
db.session.rollback()
|
||||
print('Dry run - nothing changed. Re-run with --commit to apply.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user