shopdb-flask/scripts/import_ntlars_backups.py

#!/usr/bin/env python
"""Seed backup history from a directory of existing NTLARS .reg files.

The share already holds ~150 per-machine backups collected by hand over years
(S:\\DT\\RegFiles\\Dnc\\Backup Copies\\<machinenumber>.reg and the pxe-images
mirrors). Importing them gives every machine a baseline revision on day one,
so the feature is useful before the fleet collector has run even once.

Files are keyed by machine number in the FILENAME, which is how the share names
them. The MachineNo embedded in the file is compared against it and any
disagreement is reported: in the curated set the two always agree, so a
mismatch means the file is misfiled and should not be trusted as that machine's
baseline.

Dry run by default. Nothing is written without --commit.

    venv/bin/python scripts/import_ntlars_backups.py /home/camp/pxe-images/ntlars-deploy
    venv/bin/python scripts/import_ntlars_backups.py /path/to/regs --commit
"""

import argparse
import os
import sys
from datetime import datetime

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from shopdb import create_app                                     # noqa: E402
from shopdb.api import db                                         # noqa: E402


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('directory', help='directory holding <machinenumber>.reg files')
    parser.add_argument('--commit', action='store_true',
                        help='actually write (default is a dry run)')
    parser.add_argument('--hostname', default='imported',
                        help="sourcehostname to record (default 'imported')")
    args = parser.parse_args()

    if not os.path.isdir(args.directory):
        parser.error('not a directory: {}'.format(args.directory))

    regs = sorted(f for f in os.listdir(args.directory) if f.lower().endswith('.reg'))
    if not regs:
        parser.error('no .reg files in {}'.format(args.directory))

    app = create_app()
    with app.app_context():
        from shopdb.core.models import Asset
        from plugins.backups.models import BackupRevision
        from plugins.backups.services.registry import getkind, canonicalhash

        kind = getkind('ntlars')

        created = skipped = unresolved = rejected = duplicate = 0
        mismatches = []

        for filename in regs:
            path = os.path.join(args.directory, filename)
            machinenumber = os.path.splitext(filename)[0]

            with open(path, 'rb') as handle:
                raw = handle.read()

            try:
                projection = kind.parse(raw)
            except ValueError as exc:
                # Unconfigured NTLARS installs are in the corpus; they are not
                # a usable baseline for anyone.
                print('  REJECT  {:<12} {}'.format(machinenumber, exc))
                rejected += 1
                continue

            embedded = kind.embeddedmachineno(projection)
            if embedded and embedded != machinenumber:
                mismatches.append((machinenumber, embedded))

            asset = db.session.query(Asset).filter(
                Asset.assetnumber == machinenumber).first()
            if asset is None:
                print('  NOASSET {:<12} no asset with that assetnumber'.format(
                    machinenumber))
                unresolved += 1
                continue

            contenthash = canonicalhash(projection)
            latest = (db.session.query(BackupRevision)
                      .filter(BackupRevision.assetid == asset.assetid,
                              BackupRevision.backupkind == 'ntlars')
                      .order_by(BackupRevision.backuprevisionid.desc())
                      .first())
            if latest is not None and latest.contenthash == contenthash:
                duplicate += 1
                continue

            if args.commit:
                revision = BackupRevision(
                    assetid=asset.assetid,
                    backupkind='ntlars',
                    storagebackend='shopdb',
                    contenthash=contenthash,
                    sourcefilename=filename,
                    bytesize=len(raw),
                    sourcehostname=args.hostname,
                    # The file's mtime is the closest thing to when the config
                    # was actually captured; createdat records the import.
                    collectedat=datetime.utcfromtimestamp(os.path.getmtime(path)),
                )
                revision.payload = projection
                db.session.add(revision)
            created += 1

        if args.commit:
            db.session.commit()

        print()
        print('files            : {}'.format(len(regs)))
        print('would create     : {}'.format(created) if not args.commit
              else 'created          : {}'.format(created))
        print('already current  : {}'.format(duplicate))
        print('no matching asset: {}'.format(unresolved))
        print('rejected (blank) : {}'.format(rejected))
        if mismatches:
            print()
            print('filename/embedded MachineNo disagreements ({}):'.format(
                len(mismatches)))
            for filename_no, embedded_no in mismatches[:20]:
                print('  file={:<10} embedded={}'.format(filename_no, embedded_no))
            print('  These are likely misfiled; review before trusting them.')
        if not args.commit:
            print()
            print('DRY RUN - nothing written. Re-run with --commit to apply.')


if __name__ == '__main__':
    main()