shopdb-flask/scripts/backfill_app_fingerprints.py

#!/usr/bin/env python3
"""Backfill app_fingerprint column for existing EndpointApplication records.

Run this ONCE after deploying the 0020_app_delta_fingerprints migration and
before deploying the updated C# agent that sends fingerprints. This populates
the fingerprint column for existing records so the delta logic can function
correctly during the transition period.

Usage:
    flask run-script scripts/backfill_app_fingerprints.py

or:
    python scripts/backfill_app_fingerprints.py
"""

import sys
import hashlib
from pathlib import Path

# Add project root to Python path
sys.path.insert(0, str(Path(__file__).parent.parent))

from shopdb.extensions import db
from plugins.endpoints.models import EndpointApplication


def compute_fingerprint(displayname: str, displayversion: str, publisher: str) -> str:
    """Compute SHA256 fingerprint from app metadata.

    This must match the agent's fingerprint algorithm exactly:
    SHA256(displayname|displayversion|publisher)

    Args:
        displayname: Application display name
        displayversion: Application version (may be None)
        publisher: Application publisher (may be None)

    Returns:
        64-character lowercase hex SHA256 hash
    """
    # Normalize None to empty string (matches agent behavior)
    name = displayname or ''
    version = displayversion or ''
    pub = publisher or ''

    data = f"{name}|{version}|{pub}"
    return hashlib.sha256(data.encode('utf-8')).hexdigest()


def verify_fingerprint_algorithm():
    """Verify backend fingerprint algorithm matches expected behavior.

    Uses test vectors to ensure consistency with agent implementation.
    Any mismatch indicates algorithm drift and will cause delta logic to fail.
    """
    print("Verifying fingerprint algorithm...")

    # Test vectors: known inputs and their expected SHA256 outputs
    # These verify the algorithm matches the C# agent implementation
    test_cases = [
        {
            'name': 'Google Chrome',
            'version': '117.0.5938.132',
            'publisher': 'Google LLC',
            # SHA256("Google Chrome|117.0.5938.132|Google LLC")
            'expected_fp': None  # Prints computed value for verification
        },
        {
            'name': 'Microsoft Office',
            'version': '16.0.16827.20166',
            'publisher': 'Microsoft Corporation',
            # SHA256("Microsoft Office|16.0.16827.20166|Microsoft Corporation")
            'expected_fp': None  # Prints computed value for verification
        },
        {
            'name': 'Test App No Version',
            'version': None,
            'publisher': 'Test Publisher',
            # SHA256("Test App No Version||Test Publisher")
            'expected_fp': None  # Prints computed value for verification
        },
        {
            'name': 'Test App Empty Strings',
            'version': '',
            'publisher': '',
            # SHA256("Test App Empty Strings||")
            'expected_fp': None  # Prints computed value for verification
        },
    ]

    all_passed = True
    for tc in test_cases:
        computed = compute_fingerprint(tc['name'], tc['version'], tc['publisher'])

        # Validate format
        if len(computed) != 64 or not all(c in '0123456789abcdef' for c in computed):
            print(f"  FAIL INVALID FORMAT: {tc['name']}")
            print(f"    Got: {computed}")
            all_passed = False
            continue

        # Check against expected (if we have one)
        if tc['expected_fp']:
            if computed == tc['expected_fp']:
                print(f"  OK {tc['name']}: {computed[:16]}...")
            else:
                print(f"  FAIL MISMATCH: {tc['name']}")
                print(f"    Expected: {tc['expected_fp']}")
                print(f"    Got:      {computed}")
                all_passed = False
        else:
            # First run - show computed value for test vector creation
            print(f"  ? {tc['name']}: {computed[:16]}... (computed, no expected value)")

    if not all_passed:
        raise ValueError(
            "Fingerprint algorithm verification FAILED! "
            "Backend algorithm does not match expected behavior. "
            "This will cause delta logic to delete+re-add all apps on every check-in."
        )

    print("OK Fingerprint algorithm verified\n")


def backfill_fingerprints(batch_size: int = 1000, dry_run: bool = False):
    """Backfill fingerprints for existing EndpointApplication records.

    Args:
        batch_size: Number of records to update per commit
        dry_run: If True, show what would be done without making changes
    """
    # SAFETY: Verify algorithm before touching database
    verify_fingerprint_algorithm()

    # Count total records needing backfill
    total = EndpointApplication.query.filter(
        EndpointApplication.app_fingerprint.is_(None)
    ).count()

    if total == 0:
        print("No records need backfilling (all have fingerprints).")
        return

    print(f"Found {total:,} records without fingerprints")

    if dry_run:
        print("\nDRY RUN MODE - showing sample records:")
        sample = EndpointApplication.query.filter(
            EndpointApplication.app_fingerprint.is_(None)
        ).limit(5).all()

        for app in sample:
            fp = compute_fingerprint(app.displayname, app.displayversion, app.publisher)
            print(f"  {app.displayname:50s} -> {fp[:16]}...")

        print(f"\nWould backfill {total:,} records in batches of {batch_size}")
        return

    print(f"Backfilling in batches of {batch_size}...")

    processed = 0
    batch = 0

    while True:
        # Fetch next batch
        apps = (EndpointApplication.query
                .filter(EndpointApplication.app_fingerprint.is_(None))
                .limit(batch_size)
                .all())

        if not apps:
            break  # No more records

        # Update fingerprints
        for app in apps:
            app.app_fingerprint = compute_fingerprint(
                app.displayname,
                app.displayversion,
                app.publisher
            )

        # Commit batch
        db.session.commit()

        batch += 1
        processed += len(apps)
        print(f"  Batch {batch}: {processed:,} / {total:,} ({processed*100//total}%)")

    print(f"\nBackfill complete! Updated {processed:,} records.")
    print("\nNext steps:")
    print("1. Deploy updated C# agent that computes and sends fingerprints")
    print("2. Monitor collector warnings for agents without fingerprint support")
    print("3. After all agents updated, consider making app_fingerprint NOT NULL")


if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(description='Backfill app_fingerprint column')
    parser.add_argument('--dry-run', action='store_true',
                        help='Show what would be done without making changes')
    parser.add_argument('--batch-size', type=int, default=1000,
                        help='Number of records per batch (default: 1000)')

    args = parser.parse_args()

    # Need Flask app context for database access
    try:
        from shopdb import create_app
        app = create_app()
        with app.app_context():
            backfill_fingerprints(
                batch_size=args.batch_size,
                dry_run=args.dry_run
            )
    except Exception as e:
        print(f"Error: {e}")
        sys.exit(1)