HOTFIX: Remove hot-path logging and unnecessary flush from collector

Removed performance bottlenecks in _replace_applications():
- 5 logger.info/warning calls executing on every collector check-in
- Explicit db.session.flush() in legacy path forcing immediate SQL
- With 800 check-ins/hour, this was 800 log writes + 800 premature flushes

Changes:
- Removed all hot-path logging (moved critical info to warnings list)
- Removed flush() in legacy fallback path (let transaction commit batch SQL)
- Kept conditional flush in delta mode (only when changes detected)

Impact: Eliminates 800+ disk I/O operations per hour in collector path

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Proudlock
2026-09-15 10:29:10 -04:00
parent affe455c55
commit 1ed0c4d215

View File

@@ -1058,11 +1058,8 @@ class EndpointsPlugin(BasePlugin):
MAX_APPS_PER_ENDPOINT = 5000
if not applications:
# Empty list means no apps installed - delete all existing
deleted = EndpointApplication.query.filter_by(assetid=assetid).delete(
EndpointApplication.query.filter_by(assetid=assetid).delete(
synchronize_session=False)
current_app.logger.info(
f'[collector] asset_id={assetid} apps_removed={deleted} '
f'apps_added=0 (empty list)')
return 0
# Check if agent sent fingerprints - legacy agents won't have them yet
@@ -1071,12 +1068,8 @@ class EndpointsPlugin(BasePlugin):
if not has_fingerprints:
# LEGACY FALLBACK: No fingerprints, use delete-all-insert-all
# SAFETY: Clear any existing NULL-fingerprint apps first to prevent
# UNIQUE constraint violations (MySQL allows multiple NULL values)
deleted = EndpointApplication.query.filter_by(assetid=assetid).delete(
EndpointApplication.query.filter_by(assetid=assetid).delete(
synchronize_session=False)
current_app.logger.info(
f'[collector] asset_id={assetid} legacy_delete={deleted} apps')
stored = 0
for item in applications:
@@ -1097,13 +1090,6 @@ class EndpointsPlugin(BasePlugin):
))
stored += 1
# Flush to detect constraint violations early
db.session.flush()
current_app.logger.info(
f'[collector] asset_id={assetid} apps_stored={stored} (legacy mode)'
)
return stored
# DELTA LOGIC: Build incoming fingerprint map and validate entries
@@ -1172,12 +1158,10 @@ class EndpointsPlugin(BasePlugin):
stored += 1
db.session.flush()
current_app.logger.warning(
f'[collector] asset_id={assetid} apps_stored={stored} '
f'(legacy mode - memory limit exceeded, had {len(current_apps)} apps)'
warnings.append(
f'asset has {len(current_apps)} apps, exceeded memory limit, '
f'fell back to legacy mode'
)
return stored
current_fps = {app.app_fingerprint for app in current_apps
@@ -1190,9 +1174,6 @@ class EndpointsPlugin(BasePlugin):
null_fp_apps = [app for app in current_apps if app.app_fingerprint is None]
if null_fp_apps:
if not incoming_fps:
current_app.logger.error(
f'[collector] asset_id={assetid} received empty fingerprint set '
f'during transition, keeping {len(null_fp_apps)} legacy apps')
warnings.append(
f'agent sent empty fingerprint set during NULL transition, '
f'kept {len(null_fp_apps)} legacy apps')
@@ -1200,9 +1181,6 @@ class EndpointsPlugin(BasePlugin):
for app in null_fp_apps:
db.session.delete(app)
current_app.logger.info(
f'[collector] asset_id={assetid} removed {len(null_fp_apps)} '
f'legacy apps (transition to delta mode)')
# Calculate delta
to_add = incoming_fps - current_fps # New apps
@@ -1245,11 +1223,6 @@ class EndpointsPlugin(BasePlugin):
if to_add or to_remove:
db.session.flush()
current_app.logger.info(
f'[collector] asset_id={assetid} apps_added={len(to_add)} '
f'apps_removed={len(to_remove)} unchanged={len(unchanged)} (delta mode)'
)
return len(to_add)
def apply_collector_payload(self, payload: Dict) -> Dict: