shopdb-flask/tests/test_plugins/test_endpoints_health.py

"""Health rollup: conditions outrank event counts, and no policy still works.

The bug this locks down: the Fleet Health column ranked a raw Windows event
count above every other signal, so a box with a crash loop reported
"Application error x5861" while its dead eDNC service was structurally
unreachable - the event branch returned before services were ever considered.

The other half is the no-policy path. An endpoint with no policies, no PC type,
and no monitor impact rows is the NORMAL case here, not an edge case, and it
must still produce a verdict. A policy binding may only RAISE impact; it may
never gate whether health is computed at all.
"""

import json
from datetime import datetime, timedelta

import pytest

from shopdb.core.models import Asset, AssetType
from plugins.endpoints.models import (
    EndpointState, EndpointEvent, EndpointMonitor,
    EndpointHealthState, EndpointHealthTransition, EndpointMonitorImpact,
)
from plugins.endpoints.models.health import (
    IMPACT_BLOCKER, IMPACT_DEGRADED, SOURCE_DRIVE, SOURCE_EVENT,
    SOURCE_SERVICE,
)
from plugins.endpoints.services.health import (
    HEALTH_WINDOW_HOURS, TRANSITION_HIGHWATER, TRANSITION_RETENTION,
    evaluate_endpoint_health,
)

NOW = datetime(2025, 1, 15, 12, 0, 0)

# A curated CRITICAL id, so the event path is genuinely exercised rather than
# filtered out by the catalog. 1000 is 'Application error'; note 1001 was
# deliberately demoted to informational and would not surface here.
CRASH_EVENTID = 1000


@pytest.fixture
def endpoint(db):
    """One computer asset with a state row. No policies, no PC type."""
    assettype = AssetType.query.filter_by(assettype='computer').first()
    if not assettype:
        assettype = AssetType(assettype='computer')
        db.session.add(assettype)
        db.session.flush()

    asset = Asset(assetnumber='HEALTH-PC-1', assettypeid=assettype.assettypeid)
    db.session.add(asset)
    db.session.flush()

    state = EndpointState(assetid=asset.assetid, lastreportedat=NOW)
    db.session.add(state)
    db.session.commit()
    return state


def _services(*entries):
    return json.dumps(list(entries))


def _drives(*statuses):
    """A usercontext payload with one netuse drive per (letter, status)."""
    return json.dumps({
        'netuse': [{'localname': letter, 'remotename': r'\\server\share',
                    'status': status}
                   for letter, status in statuses]
    })


def _events(db, assetid, count, hoursago=1):
    for index in range(count):
        db.session.add(EndpointEvent(
            assetid=assetid,
            eventid=CRASH_EVENTID,
            eventrecordid=index + 1,
            eventchannel='Application',
            eventtime=NOW - timedelta(hours=hoursago),
        ))
    db.session.commit()


def _monitor(db, servicename):
    monitor = EndpointMonitor(kind='service', displayname=servicename,
                              servicename=servicename)
    db.session.add(monitor)
    db.session.flush()
    return monitor


def test_no_policy_endpoint_still_gets_a_verdict(db, endpoint):
    """Zero policies, zero monitors, zero impact rows - still resolves.

    The whole no-policy path in one assertion: absence of configuration is the
    default, not a failure, and it must not read as unknown either.
    """
    assert EndpointMonitorImpact.query.count() == 0
    assert EndpointMonitor.query.count() == 0

    result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
    db.session.commit()

    assert result.level == 'ok'
    assert result.impactclass is None


def test_stopped_service_falls_back_to_degraded_without_an_impact_row(
        db, endpoint):
    """No EndpointMonitorImpact row means the built-in default applies.

    This is the verdict the old rollup gave a stopped watched service, kept
    deliberately so shipping the mapping table changes nothing until someone
    actually maps something.
    """
    endpoint.servicesjson = _services(
        {'name': 'eDNC', 'state': 'Stopped', 'monitorid': 4242})
    db.session.commit()

    result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
    db.session.commit()

    assert result.impactclass == IMPACT_DEGRADED
    assert result.level == 'warning'
    assert result.label == 'eDNC stopped'


def test_impact_is_a_property_of_the_monitor_in_context(db, endpoint):
    """Policy may RAISE impact, and only for the PC type it was mapped to.

    Same service, same telemetry, two PC types: a blocker on the shopfloor
    machine that runs parts, merely degraded on one that does not.
    """
    monitor = _monitor(db, 'eDNC')
    db.session.add(EndpointMonitorImpact(
        endpointmonitorid=monitor.endpointmonitorid,
        computertypeid=7,
        impactclass=IMPACT_BLOCKER,
        createdat=NOW))
    endpoint.servicesjson = _services(
        {'name': 'eDNC', 'state': 'Stopped',
         'monitorid': monitor.endpointmonitorid})
    db.session.commit()

    raised = evaluate_endpoint_health(endpoint.assetid, endpoint,
                                      computertypeid=7, now=NOW)
    db.session.commit()
    assert raised.impactclass == IMPACT_BLOCKER
    assert raised.level == 'critical'

    # PC type 9 has no mapping for this monitor, so the default applies.
    default = evaluate_endpoint_health(endpoint.assetid, endpoint,
                                       computertypeid=9, now=NOW)
    db.session.commit()
    assert default.impactclass == IMPACT_DEGRADED


def test_a_dead_drive_outranks_a_large_crash_count(db, endpoint):
    """THE REGRESSION. 200 crashes must not hide a disconnected drive."""
    endpoint.usercontextjson = _drives(('S:', 'Disconnected'))
    db.session.commit()
    _events(db, endpoint.assetid, 200)

    result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
    db.session.commit()

    assert result.sourcekind == SOURCE_DRIVE
    assert result.impactclass == IMPACT_BLOCKER
    assert 'S:' in result.label


def test_a_stopped_service_outranks_an_event_at_equal_impact(db, endpoint):
    """A live condition beats a report of something that already finished."""
    endpoint.servicesjson = _services({'name': 'eDNC', 'state': 'Stopped'})
    db.session.commit()
    _events(db, endpoint.assetid, 50)

    result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
    db.session.commit()

    assert result.sourcekind == SOURCE_SERVICE


def test_events_still_surface_when_nothing_is_structurally_wrong(db, endpoint):
    """Demoting events below conditions must not mean discarding them."""
    _events(db, endpoint.assetid, 3)

    result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
    db.session.commit()

    assert result.sourcekind == SOURCE_EVENT
    assert result.label.endswith('x3')


def test_events_outside_the_window_are_not_counted(db, endpoint):
    """The window is a window, not a running total since install."""
    _events(db, endpoint.assetid, 5, hoursago=HEALTH_WINDOW_HOURS + 1)

    result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
    db.session.commit()

    assert result.level == 'ok'


def test_a_stalled_service_is_distinguished_from_a_stopped_one(db, endpoint):
    """stop-pending is not the same fault as cleanly down, and now says so."""
    endpoint.servicesjson = _services(
        {'name': 'eDNC', 'state': 'Stop Pending'})
    db.session.commit()

    result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
    db.session.commit()

    assert result.label == 'eDNC stalled'


def test_malformed_services_reads_as_unknown_not_as_healthy(db, endpoint):
    """Unparseable telemetry means we cannot tell - never claim ok."""
    endpoint.servicesjson = '{not json'
    db.session.commit()

    result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
    db.session.commit()

    assert result.level == 'unknown'


def test_a_ticking_event_count_is_the_same_condition(db, endpoint):
    """x2 -> x3 must not churn history, reset firstseenat, or flap.

    Identity is what and where, not the rendered label. If the label drove
    this, a crash loop would open a new transition every single check-in.
    """
    _events(db, endpoint.assetid, 2)
    evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
    db.session.commit()

    db.session.add(EndpointEvent(
        assetid=endpoint.assetid, eventid=CRASH_EVENTID, eventrecordid=99,
        eventchannel='Application', eventtime=NOW - timedelta(hours=1)))
    db.session.commit()

    # Still inside the 48h window: the condition genuinely persists, so this
    # isolates "same condition" from "events aged out".
    later = NOW + timedelta(hours=1)
    result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=later)
    db.session.commit()

    assert result.label.endswith('x3')
    assert result.firstseenat == NOW
    assert EndpointHealthTransition.query.filter_by(
        assetid=endpoint.assetid).count() == 1


def test_recovery_resolves_the_open_transition(db, endpoint):
    """A condition is over only when something observes it to be over."""
    endpoint.servicesjson = _services({'name': 'eDNC', 'state': 'Stopped'})
    db.session.commit()
    evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
    db.session.commit()

    endpoint.servicesjson = _services({'name': 'eDNC', 'state': 'Running'})
    later = NOW + timedelta(hours=1)
    db.session.commit()
    result = evaluate_endpoint_health(endpoint.assetid, endpoint, now=later)
    db.session.commit()

    assert result.level == 'ok'
    opening = (EndpointHealthTransition.query
               .filter_by(assetid=endpoint.assetid, sourcekind=SOURCE_SERVICE)
               .first())
    assert opening.resolvedat == later


def test_transitions_are_capped_but_unresolved_rows_are_never_trimmed(
        db, endpoint):
    """History is capped; the row explaining what is on screen still stays.

    Flap the service past the high-water mark, then leave it down. Retention
    must bound the history AND keep the open transition, because that row is
    what the UI reads to say how long the current fault has been running.
    """
    for index in range(TRANSITION_HIGHWATER + 6):
        running = index % 2 == 0
        endpoint.servicesjson = _services(
            {'name': 'eDNC', 'state': 'Running' if running else 'Stopped'})
        db.session.commit()
        evaluate_endpoint_health(
            endpoint.assetid, endpoint, now=NOW + timedelta(minutes=index))
        db.session.commit()

    rows = EndpointHealthTransition.query.filter_by(
        assetid=endpoint.assetid).all()
    assert TRANSITION_RETENTION <= len(rows) <= TRANSITION_HIGHWATER

    unresolved = [row for row in rows if row.resolvedat is None]
    assert len(unresolved) == 1

    # Flapping is counted, and there is exactly one live verdict throughout.
    state = EndpointHealthState.query.filter_by(
        assetid=endpoint.assetid).one()
    assert state.flapcount > 1


def test_lastconfirmedat_does_not_advance_without_an_observation(db, endpoint):
    """A pinned blocker on a silent endpoint must not look freshly verified."""
    endpoint.servicesjson = _services({'name': 'eDNC', 'state': 'Stopped'})
    db.session.commit()
    evaluate_endpoint_health(endpoint.assetid, endpoint, now=NOW)
    db.session.commit()

    # Re-evaluated two days later with no new check-in: lastreportedat is
    # untouched, so confirmation must not move either.
    result = evaluate_endpoint_health(endpoint.assetid, endpoint,
                                      now=NOW + timedelta(days=2))
    db.session.commit()

    assert result.lastconfirmedat == NOW
    assert result.lastevaluatedat == NOW + timedelta(days=2)