shopdb-flask/scripts/backfill_app_fingerprints.py

#!/usr/bin/env python3
"""Backfill appfingerprint column for existing EndpointApplication records.

Run this ONCE after deploying the 0020_app_delta_fingerprints migration and
before deploying the updated C# agent that sends fingerprints. This populates
the fingerprint column for existing records so the delta logic can function
correctly during the transition period.

Usage:
    flask run-script scripts/backfill_app_fingerprints.py

or:
    python scripts/backfill_app_fingerprints.py
"""

import sys
import hashlib
from pathlib import Path

# Add project root to Python path
sys.path.insert(0, str(Path(__file__).parent.parent))

from dotenv import load_dotenv

# Load .env before anything reads config: a plain `python scripts/...` run
# otherwise falls back to the hardcoded root@localhost URL and could write the
# wrong database.
load_dotenv(Path(__file__).resolve().parent.parent / '.env')

from shopdb.extensions import db
from plugins.endpoints.models import EndpointApplication


def compute_fingerprint(displayname: str, displayversion: str, publisher: str) -> str:
    """Compute SHA256 fingerprint from app metadata.

    This must match the agent's fingerprint algorithm exactly:
    SHA256(displayname|displayversion|publisher)

    Args:
        displayname: Application display name
        displayversion: Application version (may be None)
        publisher: Application publisher (may be None)

    Returns:
        64-character lowercase hex SHA256 hash
    """
    # Normalize None to empty string (matches agent behavior)
    name = displayname or ''
    version = displayversion or ''
    pub = publisher or ''

    data = f"{name}|{version}|{pub}"
    return hashlib.sha256(data.encode('utf-8')).hexdigest()


def verify_fingerprint_algorithm():
    """Verify backend fingerprint algorithm matches expected behavior.

    Fixed test vectors: SHA256 of the literal 'name|version|publisher' strings,
    which is what the agent's ComputeApplicationFingerprint hashes. Any
    mismatch means compute_fingerprint drifted from the agent, which would make
    the delta logic delete and re-add every app on every check-in.
    """
    print("Verifying fingerprint algorithm...")

    test_cases = [
        ('Google Chrome', '117.0.5938.132', 'Google LLC',
         'a22de9ae7e641954872208b21aa52ac948898b78bcc8d7de5746af3a324f49ae'),
        ('Microsoft Office', '16.0.16827.20166', 'Microsoft Corporation',
         'a180bfd16f34c52c94d01317044f7d3112bd9b88c6be570ee5cca4eff9199ae4'),
        ('Test App No Version', None, 'Test Publisher',
         'aae1e046f90b39da80b2356713d8e6dc8579746d378ce7bc515800a4fbb85cda'),
        ('Test App Empty Strings', '', '',
         '1a29cd2e3bd856ac3143d2675376b92eedf1563f5e8eb78f235d252943580ceb'),
    ]

    failures = []
    for name, version, publisher, expected in test_cases:
        computed = compute_fingerprint(name, version, publisher)
        if computed == expected:
            print(f"  OK {name}: {computed[:16]}...")
        else:
            print(f"  FAIL MISMATCH: {name}")
            print(f"    Expected: {expected}")
            print(f"    Got:      {computed}")
            failures.append(name)

    if failures:
        raise ValueError(
            "Fingerprint algorithm verification FAILED for: "
            + ", ".join(failures)
            + ". Backend algorithm does not match the agent; the delta logic "
            "would delete+re-add all apps on every check-in."
        )

    print("OK Fingerprint algorithm verified\n")
    return True


def _fingerprint_batch(apps):
    """Assign fingerprints to one batch, removing duplicate (assetid, fingerprint) rows.

    The unique index on (assetid, appfingerprint) means two rows for the same
    install on one asset cannot both carry the fingerprint. Of each duplicate
    set the lowest endpointapplicationid is kept (fingerprinted rows already in
    the table included) and the rest are deleted.

    Returns the number of duplicate rows deleted.
    """
    duplicates = 0
    claimed = {}
    for app in sorted(apps, key=lambda row: row.endpointapplicationid):
        fingerprint = compute_fingerprint(app.displayname, app.displayversion,
                                          app.publisher)
        key = (app.assetid, fingerprint)

        holder = claimed.get(key)
        if holder is None:
            holder = (EndpointApplication.query
                      .filter(EndpointApplication.assetid == app.assetid,
                              EndpointApplication.appfingerprint == fingerprint,
                              EndpointApplication.endpointapplicationid
                              != app.endpointapplicationid)
                      .first())

        if holder is None:
            app.appfingerprint = fingerprint
            claimed[key] = app
        elif holder.endpointapplicationid < app.endpointapplicationid:
            db.session.delete(app)
            duplicates += 1
        else:
            # An already-fingerprinted row with a HIGHER id: drop it and let the
            # older row carry the fingerprint. Flush the delete first so the
            # unique index is free before the update lands.
            db.session.delete(holder)
            db.session.flush()
            duplicates += 1
            app.appfingerprint = fingerprint
            claimed[key] = app
    return duplicates


def backfill_fingerprints(batch_size: int = 1000, dry_run: bool = False):
    """Backfill fingerprints for existing EndpointApplication records.

    Each batch commits on its own. A batch that fails is rolled back and the run
    stops, so nothing half-applied is left in the session.

    Args:
        batch_size: Number of records to update per commit
        dry_run: If True, show what would be done without making changes

    Returns:
        dict with processed, duplicatesremoved and failed counts.
    """
    verify_fingerprint_algorithm()

    summary = {'processed': 0, 'duplicatesremoved': 0, 'failed': 0}

    total = EndpointApplication.query.filter(
        EndpointApplication.appfingerprint.is_(None)
    ).count()

    if total == 0:
        print("No records need backfilling (all have fingerprints).")
        return summary

    print(f"Found {total:,} records without fingerprints")

    if dry_run:
        print("\nDRY RUN MODE - showing sample records:")
        sample = EndpointApplication.query.filter(
            EndpointApplication.appfingerprint.is_(None)
        ).limit(5).all()

        for app in sample:
            fp = compute_fingerprint(app.displayname, app.displayversion, app.publisher)
            print(f"  {app.displayname:50s} -> {fp[:16]}...")

        print(f"\nWould backfill {total:,} records in batches of {batch_size}")
        return summary

    print(f"Backfilling in batches of {batch_size}...")

    batch = 0
    while True:
        apps = (EndpointApplication.query
                .filter(EndpointApplication.appfingerprint.is_(None))
                .order_by(EndpointApplication.endpointapplicationid)
                .limit(batch_size)
                .all())

        if not apps:
            break

        batch += 1
        try:
            duplicates = _fingerprint_batch(apps)
            db.session.commit()
        except Exception as error:
            db.session.rollback()
            summary['failed'] = len(apps)
            print(f"  Batch {batch} FAILED and was rolled back: {error}")
            break

        summary['processed'] += len(apps)
        summary['duplicatesremoved'] += duplicates
        print(f"  Batch {batch}: {summary['processed']:,} / {total:,} "
              f"({summary['processed'] * 100 // total}%), "
              f"{duplicates} duplicate(s) removed")

    print(f"\nBackfill finished. Processed {summary['processed']:,} records; "
          f"removed {summary['duplicatesremoved']:,} duplicate row(s).")
    if summary['failed']:
        print("A batch failed; fix the error above and re-run (completed batches "
              "are kept, the run resumes from the remaining null rows).")
        return summary

    print("\nNext steps:")
    print("1. Deploy updated C# agent that computes and sends fingerprints")
    print("2. Monitor collector warnings for agents without fingerprint support")
    print("3. After all agents updated, consider making appfingerprint NOT NULL")
    return summary

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(description='Backfill appfingerprint column')
    parser.add_argument('--dry-run', action='store_true',
                        help='Show what would be done without making changes')
    parser.add_argument('--batch-size', type=int, default=1000,
                        help='Number of records per batch (default: 1000)')

    args = parser.parse_args()

    # Need Flask app context for database access
    try:
        from shopdb import create_app
        app = create_app()
        with app.app_context():
            result = backfill_fingerprints(
                batch_size=args.batch_size,
                dry_run=args.dry_run
            )
    except Exception as e:
        print(f"Error: {e}")
        sys.exit(1)
    if result['failed']:
        sys.exit(1)