04616fce8b
Fleet ranked endpoints by raw critical-event counts, so a machine with a noisy but harmless event log outranked one with a mapped drive down or a blocking service stopped. Volume is not severity, and the aggregation ran across the whole fleet on every page load. Rank operational blockers above degraded conditions above event noise, and cache the verdict per asset so Fleet reads a row instead of recomputing 48h of history. Transitions are written only when a verdict changes, capped at 50 rows per asset. Absence of a policy binding must never suppress a verdict: drives and events are not policy-gated, and services fall back to a built-in default impact when no mapping applies. Event 1001 drops to informational because the WER bucket pairs with 1000 and double-counted every crash. Parallelize the Fleet first-paint loaders that were awaiting serially.
321 lines
12 KiB
Python
321 lines
12 KiB
Python
"""Health rollup: conditions outrank event counts, and no policy still works.
|
|
|
|
The bug this locks down: the Fleet Health column ranked a raw Windows event
|
|
count above every other signal, so a box with a crash loop reported
|
|
"Application error x5861" while its dead eDNC service was structurally
|
|
unreachable - the event branch returned before services were ever considered.
|
|
|
|
The other half is the no-policy path. An endpoint with no policies, no PC type,
|
|
and no monitor impact rows is the NORMAL case here, not an edge case, and it
|
|
must still produce a verdict. A policy binding may only RAISE impact; it may
|
|
never gate whether health is computed at all.
|
|
"""
|
|
|
|
import json
|
|
from datetime import datetime, timedelta
|
|
|
|
import pytest
|
|
|
|
from shopdb.core.models import Asset, AssetType
|
|
from plugins.endpoints.models import (
|
|
EndpointState, EndpointEvent, EndpointMonitor,
|
|
EndpointHealthState, EndpointHealthTransition, EndpointMonitorImpact,
|
|
)
|
|
from plugins.endpoints.models.health import (
|
|
IMPACT_BLOCKER, IMPACT_DEGRADED, SOURCE_DRIVE, SOURCE_EVENT,
|
|
SOURCE_SERVICE,
|
|
)
|
|
from plugins.endpoints.services.health import (
|
|
HEALTH_WINDOW_HOURS, TRANSITION_HIGHWATER, TRANSITION_RETENTION,
|
|
evaluate_endpoint_health,
|
|
)
|
|
|
|
NOW = datetime(2025, 1, 15, 12, 0, 0)
|
|
|
|
# A curated CRITICAL id, so the event path is genuinely exercised rather than
|
|
# filtered out by the catalog. 1000 is 'Application error'; note 1001 was
|
|
# deliberately demoted to informational and would not surface here.
|
|
CRASH_EVENTID = 1000
|
|
|
|
|
|
@pytest.fixture
|
|
def endpoint(db):
|
|
"""One computer asset with a state row. No policies, no PC type."""
|
|
assettype = AssetType.query.filter_by(assettype='computer').first()
|
|
if not assettype:
|
|
assettype = AssetType(assettype='computer')
|
|
db.session.add(assettype)
|
|
db.session.flush()
|
|
|
|
asset = Asset(assetnumber='HEALTH-PC-1', assettypeid=assettype.assettypeid)
|
|
db.session.add(asset)
|
|
db.session.flush()
|
|
|
|
state = EndpointState(assetid=asset.assetid, lastreportedat=NOW)
|
|
db.session.add(state)
|
|
db.session.commit()
|
|
return state
|
|
|
|
|
|
def _services(*entries):
|
|
return json.dumps(list(entries))
|
|
|
|
|
|
def _drives(*statuses):
|
|
"""A usercontext payload with one netuse drive per (letter, status)."""
|
|
return json.dumps({
|
|
'netuse': [{'localname': letter, 'remotename': r'\\server\share',
|
|
'status': status}
|
|
for letter, status in statuses]
|
|
})
|
|
|
|
|
|
def _events(db, assetid, count, hoursago=1):
|
|
for index in range(count):
|
|
db.session.add(EndpointEvent(
|
|
assetid=assetid,
|
|
eventid=CRASH_EVENTID,
|
|
eventrecordid=index + 1,
|
|
eventchannel='Application',
|
|
eventtime=NOW - timedelta(hours=hoursago),
|
|
))
|
|
db.session.commit()
|
|
|
|
|
|
def _monitor(db, servicename):
|
|
monitor = EndpointMonitor(kind='service', displayname=servicename,
|
|
servicename=servicename)
|
|
db.session.add(monitor)
|
|
db.session.flush()
|
|
return monitor
|
|
|
|
|
|
def test_no_policy_endpoint_still_gets_a_verdict(db, endpoint):
|
|
"""Zero policies, zero monitors, zero impact rows - still resolves.
|
|
|
|
The whole no-policy path in one assertion: absence of configuration is the
|
|
default, not a failure, and it must not read as unknown either.
|
|
"""
|
|
assert EndpointMonitorImpact.query.count() == 0
|
|
assert EndpointMonitor.query.count() == 0
|
|
|
|
result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
|
|
db.session.commit()
|
|
|
|
assert result.level == 'ok'
|
|
assert result.impactclass is None
|
|
|
|
|
|
def test_stopped_service_falls_back_to_degraded_without_an_impact_row(
|
|
db, endpoint):
|
|
"""No EndpointMonitorImpact row means the built-in default applies.
|
|
|
|
This is the verdict the old rollup gave a stopped watched service, kept
|
|
deliberately so shipping the mapping table changes nothing until someone
|
|
actually maps something.
|
|
"""
|
|
endpoint.servicesjson = _services(
|
|
{'name': 'eDNC', 'state': 'Stopped', 'monitorid': 4242})
|
|
db.session.commit()
|
|
|
|
result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
|
|
db.session.commit()
|
|
|
|
assert result.impactclass == IMPACT_DEGRADED
|
|
assert result.level == 'warning'
|
|
assert result.label == 'eDNC stopped'
|
|
|
|
|
|
def test_impact_is_a_property_of_the_monitor_in_context(db, endpoint):
|
|
"""Policy may RAISE impact, and only for the PC type it was mapped to.
|
|
|
|
Same service, same telemetry, two PC types: a blocker on the shopfloor
|
|
machine that runs parts, merely degraded on one that does not.
|
|
"""
|
|
monitor = _monitor(db, 'eDNC')
|
|
db.session.add(EndpointMonitorImpact(
|
|
endpointmonitorid=monitor.endpointmonitorid,
|
|
computertypeid=7,
|
|
impactclass=IMPACT_BLOCKER,
|
|
createdat=NOW))
|
|
endpoint.servicesjson = _services(
|
|
{'name': 'eDNC', 'state': 'Stopped',
|
|
'monitorid': monitor.endpointmonitorid})
|
|
db.session.commit()
|
|
|
|
raised = evaluate_endpoint_health(endpoint.assetid, endpoint,
|
|
computertypeid=7, now=NOW)
|
|
db.session.commit()
|
|
assert raised.impactclass == IMPACT_BLOCKER
|
|
assert raised.level == 'critical'
|
|
|
|
# PC type 9 has no mapping for this monitor, so the default applies.
|
|
default = evaluate_endpoint_health(endpoint.assetid, endpoint,
|
|
computertypeid=9, now=NOW)
|
|
db.session.commit()
|
|
assert default.impactclass == IMPACT_DEGRADED
|
|
|
|
|
|
def test_a_dead_drive_outranks_a_large_crash_count(db, endpoint):
|
|
"""THE REGRESSION. 200 crashes must not hide a disconnected drive."""
|
|
endpoint.usercontextjson = _drives(('S:', 'Disconnected'))
|
|
db.session.commit()
|
|
_events(db, endpoint.assetid, 200)
|
|
|
|
result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
|
|
db.session.commit()
|
|
|
|
assert result.sourcekind == SOURCE_DRIVE
|
|
assert result.impactclass == IMPACT_BLOCKER
|
|
assert 'S:' in result.label
|
|
|
|
|
|
def test_a_stopped_service_outranks_an_event_at_equal_impact(db, endpoint):
|
|
"""A live condition beats a report of something that already finished."""
|
|
endpoint.servicesjson = _services({'name': 'eDNC', 'state': 'Stopped'})
|
|
db.session.commit()
|
|
_events(db, endpoint.assetid, 50)
|
|
|
|
result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
|
|
db.session.commit()
|
|
|
|
assert result.sourcekind == SOURCE_SERVICE
|
|
|
|
|
|
def test_events_still_surface_when_nothing_is_structurally_wrong(db, endpoint):
|
|
"""Demoting events below conditions must not mean discarding them."""
|
|
_events(db, endpoint.assetid, 3)
|
|
|
|
result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
|
|
db.session.commit()
|
|
|
|
assert result.sourcekind == SOURCE_EVENT
|
|
assert result.label.endswith('x3')
|
|
|
|
|
|
def test_events_outside_the_window_are_not_counted(db, endpoint):
|
|
"""The window is a window, not a running total since install."""
|
|
_events(db, endpoint.assetid, 5, hoursago=HEALTH_WINDOW_HOURS + 1)
|
|
|
|
result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
|
|
db.session.commit()
|
|
|
|
assert result.level == 'ok'
|
|
|
|
|
|
def test_a_stalled_service_is_distinguished_from_a_stopped_one(db, endpoint):
|
|
"""stop-pending is not the same fault as cleanly down, and now says so."""
|
|
endpoint.servicesjson = _services(
|
|
{'name': 'eDNC', 'state': 'Stop Pending'})
|
|
db.session.commit()
|
|
|
|
result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
|
|
db.session.commit()
|
|
|
|
assert result.label == 'eDNC stalled'
|
|
|
|
|
|
def test_malformed_services_reads_as_unknown_not_as_healthy(db, endpoint):
|
|
"""Unparseable telemetry means we cannot tell - never claim ok."""
|
|
endpoint.servicesjson = '{not json'
|
|
db.session.commit()
|
|
|
|
result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
|
|
db.session.commit()
|
|
|
|
assert result.level == 'unknown'
|
|
|
|
|
|
def test_a_ticking_event_count_is_the_same_condition(db, endpoint):
|
|
"""x2 -> x3 must not churn history, reset firstseenat, or flap.
|
|
|
|
Identity is what and where, not the rendered label. If the label drove
|
|
this, a crash loop would open a new transition every single check-in.
|
|
"""
|
|
_events(db, endpoint.assetid, 2)
|
|
evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
|
|
db.session.commit()
|
|
|
|
db.session.add(EndpointEvent(
|
|
assetid=endpoint.assetid, eventid=CRASH_EVENTID, eventrecordid=99,
|
|
eventchannel='Application', eventtime=NOW - timedelta(hours=1)))
|
|
db.session.commit()
|
|
|
|
# Still inside the 48h window: the condition genuinely persists, so this
|
|
# isolates "same condition" from "events aged out".
|
|
later = NOW + timedelta(hours=1)
|
|
result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=later)
|
|
db.session.commit()
|
|
|
|
assert result.label.endswith('x3')
|
|
assert result.firstseenat == NOW
|
|
assert EndpointHealthTransition.query.filter_by(
|
|
assetid=endpoint.assetid).count() == 1
|
|
|
|
|
|
def test_recovery_resolves_the_open_transition(db, endpoint):
|
|
"""A condition is over only when something observes it to be over."""
|
|
endpoint.servicesjson = _services({'name': 'eDNC', 'state': 'Stopped'})
|
|
db.session.commit()
|
|
evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
|
|
db.session.commit()
|
|
|
|
endpoint.servicesjson = _services({'name': 'eDNC', 'state': 'Running'})
|
|
later = NOW + timedelta(hours=1)
|
|
db.session.commit()
|
|
result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=later)
|
|
db.session.commit()
|
|
|
|
assert result.level == 'ok'
|
|
opening = (EndpointHealthTransition.query
|
|
.filter_by(assetid=endpoint.assetid, sourcekind=SOURCE_SERVICE)
|
|
.first())
|
|
assert opening.resolvedat == later
|
|
|
|
|
|
def test_transitions_are_capped_but_unresolved_rows_are_never_trimmed(
|
|
db, endpoint):
|
|
"""History is capped; the row explaining what is on screen still stays.
|
|
|
|
Flap the service past the high-water mark, then leave it down. Retention
|
|
must bound the history AND keep the open transition, because that row is
|
|
what the UI reads to say how long the current fault has been running.
|
|
"""
|
|
for index in range(TRANSITION_HIGHWATER + 6):
|
|
running = index % 2 == 0
|
|
endpoint.servicesjson = _services(
|
|
{'name': 'eDNC', 'state': 'Running' if running else 'Stopped'})
|
|
db.session.commit()
|
|
evaluate_endpoint_health(
|
|
endpoint.assetid, endpoint, now=NOW + timedelta(minutes=index))
|
|
db.session.commit()
|
|
|
|
rows = EndpointHealthTransition.query.filter_by(
|
|
assetid=endpoint.assetid).all()
|
|
assert TRANSITION_RETENTION <= len(rows) <= TRANSITION_HIGHWATER
|
|
|
|
unresolved = [row for row in rows if row.resolvedat is None]
|
|
assert len(unresolved) == 1
|
|
|
|
# Flapping is counted, and there is exactly one live verdict throughout.
|
|
state = EndpointHealthState.query.filter_by(
|
|
assetid=endpoint.assetid).one()
|
|
assert state.flapcount > 1
|
|
|
|
|
|
def test_lastconfirmedat_does_not_advance_without_an_observation(db, endpoint):
|
|
"""A pinned blocker on a silent endpoint must not look freshly verified."""
|
|
endpoint.servicesjson = _services({'name': 'eDNC', 'state': 'Stopped'})
|
|
db.session.commit()
|
|
evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
|
|
db.session.commit()
|
|
|
|
# Re-evaluated two days later with no new check-in: lastreportedat is
|
|
# untouched, so confirmation must not move either.
|
|
result = evaluate_endpoint_health(endpoint.assetid, endpoint,
|
|
now=NOW + timedelta(days=2))
|
|
db.session.commit()
|
|
|
|
assert result.lastconfirmedat == NOW
|
|
assert result.lastevaluatedat == NOW + timedelta(days=2)
|