Merge origin/main into wilm: dashboard/fleet console, vuln upgrades, colours

plugins/endpoints/api/routes.py

@@ -47,6 +47,17 @@ def _service_running(entry):
     return raw in _RUNNING_STATES
 
 
+# Paused, plus the four transitional states Windows reports while a service is
+# on its way somewhere.
+_TRANSITIONAL_STATES = ('paused', 'start pending', 'stop pending',
+                        'continue pending', 'pause pending', 'starting',
+                        'stopping', 'pausing')
+
+
+def _service_transitional(raw):
+    return (raw or '').strip().lower() in _TRANSITIONAL_STATES
+
+
 def _service_name(entry):
     return entry.get('name') or entry.get('service') or 'Unknown'
 
@@ -178,7 +189,7 @@ def list_endpoints():
         row['uptimedisplay'] = _format_uptime(state.uptimeseconds)
         row['status'] = ('never', 'quiet', 'reporting')[rank]
         row['machines'] = machines.get(state.assetid, [])
-        row['health'] = _health_for(state, critical.get(state.assetid, 0))
+        row['health'] = _health_for(state, critical.get(state.assetid))
         data.append(row)
 
     return paginated_response(data, page, per_page, total)
@@ -261,30 +272,48 @@ def _machines_for_assets(assetids):
 
 
 def _critical_counts_for_assets(assetids):
-    """{assetid: count} of critical events in the stability window."""
+    """{assetid: (total, dominant_eventid)} of critical events in the window.
+
+    Broken down BY EVENT ID, not just totalled. A bare count says "418
+    critical", which tells an operator nothing about what to do; naming the
+    event that dominates - service timeout, application crash - is the
+    difference between a number and a lead.
+    """
     if not assetids:
         return {}
     cutoff = _naive_utcnow() - timedelta(hours=48)
-    rows = (db.session.query(EndpointEvent.assetid,
+    rows = (db.session.query(EndpointEvent.assetid, EndpointEvent.eventid,
                              db.func.count(EndpointEvent.endpointeventid))
             .filter(EndpointEvent.assetid.in_(assetids),
                     EndpointEvent.eventid.in_(CRITICAL_EVENT_IDS),
                     EndpointEvent.eventtime.isnot(None),
                     EndpointEvent.eventtime >= cutoff)
-            .group_by(EndpointEvent.assetid)
+            .group_by(EndpointEvent.assetid, EndpointEvent.eventid)
             .all())
-    return dict(rows)
+
+    perasset = {}
+    for assetid, eventid, count in rows:
+        total, dominant, best = perasset.get(assetid, (0, None, 0))
+        total += count
+        if count > best:
+            dominant, best = eventid, count
+        perasset[assetid] = (total, dominant, best)
+    return {assetid: (total, dominant)
+            for assetid, (total, dominant, _best) in perasset.items()}
 
 
-def _health_for(state, criticalcount):
+def _health_for(state, critical):
     """Worst-of rollup: a stopped watched service, or a recent critical event.
 
     Deliberately a PHRASE, not a score. "eDNC stopped" tells an operator what
-    to do; a percentage does not.
+    to do; a percentage does not - and neither does "418 critical", which is
+    why the dominant event is named rather than merely counted.
     """
-    if criticalcount:
-        return {'level': 'critical',
-                'label': f'{criticalcount} critical' if criticalcount > 1 else '1 critical'}
+    if critical:
+        total, dominant = critical
+        title = event_title(dominant) if dominant else 'critical'
+        label = title if total == 1 else '{} x{}'.format(title, total)
+        return {'level': 'critical', 'label': label}
 
     stopped = []
     try:
@@ -450,7 +479,14 @@ def panel_services(asset_id: int):
             'name': name,
             'kind': (kind or '').lower() or None,
             'statelabel': 'Running' if running else (raw.title() or 'Unknown'),
-            'statecolor': 'green' if running else ('red' if raw else 'gray'),
+            # Amber for paused and the four Windows transitional states. A
+            # service stuck in Start Pending is failing to start on a retry
+            # cycle - which is what raises event 7011 - and a paused one was
+            # paused deliberately. Painting either the same red as Stopped
+            # loses the distinction that tells you which to chase.
+            'statecolor': ('green' if running
+                           else 'yellow' if _service_transitional(raw)
+                           else 'red' if raw else 'gray'),
         }
         # Only offer Start/Stop to a caller who could actually run it. The
         # emission never consulted the caller, so a read-only viewer was shown
@@ -520,6 +556,20 @@ def panel_activity(asset_id: int):
               .limit(15)
               .all())
 
+    # The newest 15 of ANY severity can be entirely routine logons while the
+    # health rollup is shouting about a burst of crashes underneath them - the
+    # page then contradicts itself for no visible reason. Pull the newest
+    # criticals separately and merge, so what health is reporting is always
+    # present in the feed that is supposed to explain it.
+    criticals = (EndpointEvent.query
+                 .filter(EndpointEvent.assetid == asset_id,
+                         EndpointEvent.eventid.in_(CRITICAL_EVENT_IDS))
+                 .order_by(EndpointEvent.eventtime.desc())
+                 .limit(5)
+                 .all())
+    seen = {e.endpointeventid for e in events}
+    events.extend(e for e in criticals if e.endpointeventid not in seen)
+
     commands = (EndpointCommand.query
                 .filter_by(assetid=asset_id)
                 .order_by(EndpointCommand.createdat.desc())
@@ -563,9 +613,24 @@ def panel_activity(asset_id: int):
     # Undated rows sort last rather than crashing the comparison.
     rows.sort(key=lambda r: (r['eventtime'] is not None, r['eventtime']), reverse=True)
 
+    # Trim to the cap without dropping the criticals that were fetched on
+    # purpose: they keep their place in time, but they survive the cut.
+    if len(rows) > 15:
+        keep = [r for r in rows if r['severity'] == 'critical'][:5]
+        rest = [r for r in rows if r not in keep]
+        rows = sorted(keep + rest[:15 - len(keep)],
+                      key=lambda r: (r['eventtime'] is not None, r['eventtime']),
+                      reverse=True)
+
+    # The list renderer reads a badge colour from a ROW FIELD, so without this
+    # every badge falls back to grey and the feed loses the one signal it is
+    # there to carry.
+    colours = {'critical': 'red', 'warning': 'yellow', 'info': 'gray'}
+
     return success_response([{
         'title': r['title'],
         'severity': r['severity'],
+        'severitycolor': colours.get(r['severity'], 'gray'),
         'detail': r['detail'],
         'eventtime': r['eventtime'].isoformat() + 'Z' if r['eventtime'] else None,
     } for r in rows[:15]])