#!/usr/bin/env python3 """Backfill app_fingerprint column for existing EndpointApplication records. Run this ONCE after deploying the 0020_app_delta_fingerprints migration and before deploying the updated C# agent that sends fingerprints. This populates the fingerprint column for existing records so the delta logic can function correctly during the transition period. Usage: flask run-script scripts/backfill_app_fingerprints.py or: python scripts/backfill_app_fingerprints.py """ import sys import hashlib from pathlib import Path # Add project root to Python path sys.path.insert(0, str(Path(__file__).parent.parent)) from shopdb.extensions import db from plugins.endpoints.models import EndpointApplication def compute_fingerprint(displayname: str, displayversion: str, publisher: str) -> str: """Compute SHA256 fingerprint from app metadata. This must match the agent's fingerprint algorithm exactly: SHA256(displayname|displayversion|publisher) Args: displayname: Application display name displayversion: Application version (may be None) publisher: Application publisher (may be None) Returns: 64-character lowercase hex SHA256 hash """ # Normalize None to empty string (matches agent behavior) name = displayname or '' version = displayversion or '' pub = publisher or '' data = f"{name}|{version}|{pub}" return hashlib.sha256(data.encode('utf-8')).hexdigest() def verify_fingerprint_algorithm(): """Verify backend fingerprint algorithm matches expected behavior. Uses test vectors to ensure consistency with agent implementation. Any mismatch indicates algorithm drift and will cause delta logic to fail. """ print("Verifying fingerprint algorithm...") # Test vectors: known inputs and their expected SHA256 outputs # These verify the algorithm matches the C# agent implementation test_cases = [ { 'name': 'Google Chrome', 'version': '117.0.5938.132', 'publisher': 'Google LLC', # SHA256("Google Chrome|117.0.5938.132|Google LLC") 'expected_fp': None # Prints computed value for verification }, { 'name': 'Microsoft Office', 'version': '16.0.16827.20166', 'publisher': 'Microsoft Corporation', # SHA256("Microsoft Office|16.0.16827.20166|Microsoft Corporation") 'expected_fp': None # Prints computed value for verification }, { 'name': 'Test App No Version', 'version': None, 'publisher': 'Test Publisher', # SHA256("Test App No Version||Test Publisher") 'expected_fp': None # Prints computed value for verification }, { 'name': 'Test App Empty Strings', 'version': '', 'publisher': '', # SHA256("Test App Empty Strings||") 'expected_fp': None # Prints computed value for verification }, ] all_passed = True for tc in test_cases: computed = compute_fingerprint(tc['name'], tc['version'], tc['publisher']) # Validate format if len(computed) != 64 or not all(c in '0123456789abcdef' for c in computed): print(f" FAIL INVALID FORMAT: {tc['name']}") print(f" Got: {computed}") all_passed = False continue # Check against expected (if we have one) if tc['expected_fp']: if computed == tc['expected_fp']: print(f" OK {tc['name']}: {computed[:16]}...") else: print(f" FAIL MISMATCH: {tc['name']}") print(f" Expected: {tc['expected_fp']}") print(f" Got: {computed}") all_passed = False else: # First run - show computed value for test vector creation print(f" ? {tc['name']}: {computed[:16]}... (computed, no expected value)") if not all_passed: raise ValueError( "Fingerprint algorithm verification FAILED! " "Backend algorithm does not match expected behavior. " "This will cause delta logic to delete+re-add all apps on every check-in." ) print("OK Fingerprint algorithm verified\n") def backfill_fingerprints(batch_size: int = 1000, dry_run: bool = False): """Backfill fingerprints for existing EndpointApplication records. Args: batch_size: Number of records to update per commit dry_run: If True, show what would be done without making changes """ # SAFETY: Verify algorithm before touching database verify_fingerprint_algorithm() # Count total records needing backfill total = EndpointApplication.query.filter( EndpointApplication.app_fingerprint.is_(None) ).count() if total == 0: print("No records need backfilling (all have fingerprints).") return print(f"Found {total:,} records without fingerprints") if dry_run: print("\nDRY RUN MODE - showing sample records:") sample = EndpointApplication.query.filter( EndpointApplication.app_fingerprint.is_(None) ).limit(5).all() for app in sample: fp = compute_fingerprint(app.displayname, app.displayversion, app.publisher) print(f" {app.displayname:50s} -> {fp[:16]}...") print(f"\nWould backfill {total:,} records in batches of {batch_size}") return print(f"Backfilling in batches of {batch_size}...") processed = 0 batch = 0 while True: # Fetch next batch apps = (EndpointApplication.query .filter(EndpointApplication.app_fingerprint.is_(None)) .limit(batch_size) .all()) if not apps: break # No more records # Update fingerprints for app in apps: app.app_fingerprint = compute_fingerprint( app.displayname, app.displayversion, app.publisher ) # Commit batch db.session.commit() batch += 1 processed += len(apps) print(f" Batch {batch}: {processed:,} / {total:,} ({processed*100//total}%)") print(f"\nBackfill complete! Updated {processed:,} records.") print("\nNext steps:") print("1. Deploy updated C# agent that computes and sends fingerprints") print("2. Monitor collector warnings for agents without fingerprint support") print("3. After all agents updated, consider making app_fingerprint NOT NULL") if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description='Backfill app_fingerprint column') parser.add_argument('--dry-run', action='store_true', help='Show what would be done without making changes') parser.add_argument('--batch-size', type=int, default=1000, help='Number of records per batch (default: 1000)') args = parser.parse_args() # Need Flask app context for database access try: from shopdb import create_app app = create_app() with app.app_context(): backfill_fingerprints( batch_size=args.batch_size, dry_run=args.dry_run ) except Exception as e: print(f"Error: {e}") sys.exit(1)