shopdb-flask/scripts/refile_partmarker_backups.py

"""Re-file existing backup revisions from an operation onto the part marker.

Backups collected before part markers became assets were filed against the
operation number the PC reported - several markers' configs landing on one
asset, where they overwrote each other. The collector now files a marker PC's
backup against that PC's own marker asset, but only for backups collected since
the change. Everything already stored stays where it was put, so the new marker
assets show an empty history while the operation shows a mixed one.

This moves each historical revision to the marker it actually came from. The
attribution is exact, not a guess: a revision records the PC it was read from
(sourcehostname), and the collector records which marker each PC drives (an
active `collector:partmarker` relationship). The move is that mapping applied.

WHAT IT MOVES: a revision whose source hostname resolves to a PC that has a
marker, and which is currently filed against an asset that is NOT that marker.

WHAT IT LEAVES: revisions with no source hostname, since nothing says which
marker they came from; revisions from PCs that drive no marker, which is every
ordinary machine; and anything already filed correctly.

Moving a revision can make it a duplicate of one already on the marker - the
same config counted twice, once under the operation and once under the marker.
Those are collapsed as part of the move, keeping the earliest, so the marker's
history reads as one clean chain.

Usage (on the target instance, in the app dir):
    venv\\Scripts\\python scripts\\refile_partmarker_backups.py            # dry run
    venv\\Scripts\\python scripts\\refile_partmarker_backups.py --commit   # apply
    ... --hostname FBYTNCX3                                               # one PC

Dry run by default. Take a database backup before --commit.
"""

import argparse
import os
import sys
from collections import defaultdict

# Run from anywhere: put the repo root (parent of scripts/) on the path.
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from shopdb import create_app
from shopdb.api import db, Asset, AssetRelationship, RelationshipType
from plugins.backups.models import BackupRevision
from plugins.computers.models import Computer
from plugins.computers.plugin import PARTMARKER_LINK_ORIGIN


def _markerbyhostname():
    """hostname (lowercased) -> marker asset, for every PC that drives one."""
    controls = RelationshipType.query.filter_by(
        relationshiptype='controls').first()
    if controls is None:
        raise SystemExit("no 'controls' relationship type; "
                         'run flask seed reference-data')

    rows = (db.session.query(Computer, AssetRelationship)
            .join(AssetRelationship,
                  AssetRelationship.sourceassetid == Computer.assetid)
            .filter(AssetRelationship.relationshiptypeid ==
                    controls.relationshiptypeid,
                    AssetRelationship.label == PARTMARKER_LINK_ORIGIN,
                    AssetRelationship.isactive.is_(True))
            .all())

    markers = {}
    for computer, link in rows:
        marker = db.session.get(Asset, link.targetassetid)
        if marker is not None and computer.hostname:
            markers[computer.hostname.strip().lower()] = marker
    return markers


def _dedupe(revisions):
    """Within one chain, drop a revision repeating the hash before it.

    Applied after a move, because the same config can arrive twice: once as the
    row filed under the operation and once as a row the collector has already
    written against the marker. Keeps the earliest of each run, so the first
    time a config was seen is the date that survives.
    """
    doomed = []
    previoushash = None
    for revision in sorted(revisions, key=lambda r: r.backuprevisionid):
        if previoushash is not None and revision.contenthash == previoushash:
            doomed.append(revision)
        else:
            previoushash = revision.contenthash
    return doomed


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--commit', action='store_true',
                        help='apply the moves (default is a dry run)')
    parser.add_argument('--hostname', help='limit to one source PC')
    args = parser.parse_args()

    app = create_app(os.environ.get('FLASK_ENV', 'production'))
    with app.app_context():
        markers = _markerbyhostname()
        if not markers:
            print('No PC drives a part marker. Nothing to re-file.')
            print('If that is unexpected, the marker assets are minted by the '
                  'computers collector when a PC reports with '
                  'pctype=gea-shopfloor-partmarker.')
            return

        print('{} PC(s) drive a marker.'.format(len(markers)))

        query = db.session.query(BackupRevision).filter(
            BackupRevision.sourcehostname.isnot(None))
        if args.hostname:
            query = query.filter(
                BackupRevision.sourcehostname.ilike(args.hostname))

        moved = 0
        touchedchains = defaultdict(list)

        for revision in query.order_by(BackupRevision.backuprevisionid).all():
            marker = markers.get((revision.sourcehostname or '').strip().lower())
            if marker is None or revision.assetid == marker.assetid:
                continue
            fromasset = db.session.get(Asset, revision.assetid)
            print('  {:<12} {:<10} rev {:<6} {} -> {}'.format(
                revision.sourcehostname, revision.backupkind,
                revision.backuprevisionid,
                fromasset.assetnumber if fromasset else revision.assetid,
                marker.assetnumber))
            revision.assetid = marker.assetid
            moved += 1
            touchedchains[(marker.assetid, revision.backupkind,
                           revision.sourcehostname)].append(revision)

        # Collapse duplicates the move created, counting rows already on the
        # marker so an existing revision is compared against, not ignored.
        collapsed = 0
        for (assetid, kind, source) in list(touchedchains):
            chain = (db.session.query(BackupRevision)
                     .filter(BackupRevision.assetid == assetid,
                             BackupRevision.backupkind == kind,
                             BackupRevision.sourcehostname == source)
                     .all())
            for revision in _dedupe(chain):
                db.session.delete(revision)
                collapsed += 1

        print()
        print('{} revision(s) re-filed, {} duplicate(s) collapsed.'.format(
            moved, collapsed))

        if not moved:
            print('Nothing to do.')
            db.session.rollback()
            return
        if args.commit:
            db.session.commit()
            print('APPLIED.')
        else:
            db.session.rollback()
            print('Dry run - nothing changed. Re-run with --commit to apply.')


if __name__ == '__main__':
    main()