HOTFIX: Remove hot-path logging and unnecessary flush from collector
Removed performance bottlenecks in _replace_applications(): - 5 logger.info/warning calls executing on every collector check-in - Explicit db.session.flush() in legacy path forcing immediate SQL - With 800 check-ins/hour, this was 800 log writes + 800 premature flushes Changes: - Removed all hot-path logging (moved critical info to warnings list) - Removed flush() in legacy fallback path (let transaction commit batch SQL) - Kept conditional flush in delta mode (only when changes detected) Impact: Eliminates 800+ disk I/O operations per hour in collector path Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1058,11 +1058,8 @@ class EndpointsPlugin(BasePlugin):
|
||||
MAX_APPS_PER_ENDPOINT = 5000
|
||||
if not applications:
|
||||
# Empty list means no apps installed - delete all existing
|
||||
deleted = EndpointApplication.query.filter_by(assetid=assetid).delete(
|
||||
EndpointApplication.query.filter_by(assetid=assetid).delete(
|
||||
synchronize_session=False)
|
||||
current_app.logger.info(
|
||||
f'[collector] asset_id={assetid} apps_removed={deleted} '
|
||||
f'apps_added=0 (empty list)')
|
||||
return 0
|
||||
|
||||
# Check if agent sent fingerprints - legacy agents won't have them yet
|
||||
@@ -1071,12 +1068,8 @@ class EndpointsPlugin(BasePlugin):
|
||||
|
||||
if not has_fingerprints:
|
||||
# LEGACY FALLBACK: No fingerprints, use delete-all-insert-all
|
||||
# SAFETY: Clear any existing NULL-fingerprint apps first to prevent
|
||||
# UNIQUE constraint violations (MySQL allows multiple NULL values)
|
||||
deleted = EndpointApplication.query.filter_by(assetid=assetid).delete(
|
||||
EndpointApplication.query.filter_by(assetid=assetid).delete(
|
||||
synchronize_session=False)
|
||||
current_app.logger.info(
|
||||
f'[collector] asset_id={assetid} legacy_delete={deleted} apps')
|
||||
|
||||
stored = 0
|
||||
for item in applications:
|
||||
@@ -1097,13 +1090,6 @@ class EndpointsPlugin(BasePlugin):
|
||||
))
|
||||
stored += 1
|
||||
|
||||
# Flush to detect constraint violations early
|
||||
db.session.flush()
|
||||
|
||||
current_app.logger.info(
|
||||
f'[collector] asset_id={assetid} apps_stored={stored} (legacy mode)'
|
||||
)
|
||||
|
||||
return stored
|
||||
|
||||
# DELTA LOGIC: Build incoming fingerprint map and validate entries
|
||||
@@ -1172,12 +1158,10 @@ class EndpointsPlugin(BasePlugin):
|
||||
stored += 1
|
||||
|
||||
db.session.flush()
|
||||
|
||||
current_app.logger.warning(
|
||||
f'[collector] asset_id={assetid} apps_stored={stored} '
|
||||
f'(legacy mode - memory limit exceeded, had {len(current_apps)} apps)'
|
||||
warnings.append(
|
||||
f'asset has {len(current_apps)} apps, exceeded memory limit, '
|
||||
f'fell back to legacy mode'
|
||||
)
|
||||
|
||||
return stored
|
||||
|
||||
current_fps = {app.app_fingerprint for app in current_apps
|
||||
@@ -1190,9 +1174,6 @@ class EndpointsPlugin(BasePlugin):
|
||||
null_fp_apps = [app for app in current_apps if app.app_fingerprint is None]
|
||||
if null_fp_apps:
|
||||
if not incoming_fps:
|
||||
current_app.logger.error(
|
||||
f'[collector] asset_id={assetid} received empty fingerprint set '
|
||||
f'during transition, keeping {len(null_fp_apps)} legacy apps')
|
||||
warnings.append(
|
||||
f'agent sent empty fingerprint set during NULL transition, '
|
||||
f'kept {len(null_fp_apps)} legacy apps')
|
||||
@@ -1200,9 +1181,6 @@ class EndpointsPlugin(BasePlugin):
|
||||
|
||||
for app in null_fp_apps:
|
||||
db.session.delete(app)
|
||||
current_app.logger.info(
|
||||
f'[collector] asset_id={assetid} removed {len(null_fp_apps)} '
|
||||
f'legacy apps (transition to delta mode)')
|
||||
|
||||
# Calculate delta
|
||||
to_add = incoming_fps - current_fps # New apps
|
||||
@@ -1245,11 +1223,6 @@ class EndpointsPlugin(BasePlugin):
|
||||
if to_add or to_remove:
|
||||
db.session.flush()
|
||||
|
||||
current_app.logger.info(
|
||||
f'[collector] asset_id={assetid} apps_added={len(to_add)} '
|
||||
f'apps_removed={len(to_remove)} unchanged={len(unchanged)} (delta mode)'
|
||||
)
|
||||
|
||||
return len(to_add)
|
||||
|
||||
def apply_collector_payload(self, payload: Dict) -> Dict:
|
||||
|
||||
Reference in New Issue
Block a user