ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge origin/main into wilm: reconcile endpoint agent dev with release channel

Browse Source 
# Conflicts:
#	plugins/endpoints/api/routes.py
This commit is contained in:
benjamin.baker
2026-08-30 08:46:04 -04:00
parent 6ff7594996 8098a5e82e
commit 345c7cefff
15 changed files with 1396 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

363
plugins/endpoints/api/routes.py
View File

@@ -16,8 +16,9 @@ from flask_jwt_extended import current_user
from shopdb.api import (
db, Asset, Setting, success_response, error_response, paginated_response,
ErrorCodes, get_pagination_params, paginate_query, require_permission,
require_role, mint_service_token, list_service_tokens, revoke_service_tokens,
AuditLog, collector_authorized,
require_role, caller_holds, mint_service_token, list_service_tokens,
revoke_service_tokens,
AuditLog, collector_authorized, User,
)
from ..models import EndpointState, EndpointEvent, EndpointCommand
@@ -31,6 +32,22 @@ def _naive_utcnow():
return datetime.now(timezone.utc).replace(tzinfo=None)
# A watched service or process reports a STATE STRING, not a boolean, and the
# vocabulary varies by source. Both the services panel and the fleet health
# rollup have to agree on what counts as running, so the rule lives here once -
# reading it wrong makes every service look stopped.
_RUNNING_STATES = ('running', 'started', 'up', 'ok', 'active')
def _service_running(entry):
raw = (entry.get('state') or entry.get('status') or '').strip().lower()
return raw in _RUNNING_STATES
def _service_name(entry):
return entry.get('name') or entry.get('service') or 'Unknown'
def _format_uptime(seconds):
"""Whole-seconds uptime -> 'Nd Nh Nm', or None."""
if seconds is None:
@@ -55,44 +72,247 @@ def _format_uptime(seconds):
# Endpoint state
# =============================================================================
@endpoints_bp.route('/', methods=['GET'], strict_slashes=False)
@require_permission('endpoints.view')
@endpoints_bp.route('', methods=['GET'])
@require_permission('endpoints.fleet')
def list_endpoints():
"""List reported endpoint states.
Registered as '' so the rule is '/api/endpoints', with no trailing slash.
CONTRIBUTING.md's API section spells endpoints that way ('/api/machines'),
and every other plugin's list route follows it - this was the only route in
331 paths that did not.
It was not cosmetic. A rule ending in a slash makes Flask answer the
unslashed URL with a 308 to an ABSOLUTE url. On dev that resolves and
nobody notices. Behind the IIS subpath it does not: with no ProxyFix,
Werkzeug builds that Location from the scheme and host it believes it has,
so it emitted http:// and dropped the /shopdb prefix. The browser refuses
to follow an https page to http, the list request failed, and the page
rendered "No endpoints found" - indistinguishable from a fleet with nothing
in it. The agent had been reporting correctly the whole time.
Joins through Computer so each row carries computerid and can deep-link
into the existing PC detail page (/pcs/{computerid}). Endpoint state only
exists for hostnames resolved to a Computer, so the join is not lossy.
Sorted WORST FIRST by default: never-reported, then quiet, then reporting.
It used to sort by lastreportedat descending, which floated the healthiest
endpoints to the top and buried the ones somebody needs to look at.
Each row also carries what the fleet view is actually for: which MACHINE
depends on this PC, its PC type, and a health rollup. On a shop floor a PC
matters because a machine stops without it, so that is the column an
operator reads first.
"""
from plugins.computers.models import Computer
from plugins.computers.models import Computer, ComputerType
page, per_page = get_pagination_params(request)
cutoff = _naive_utcnow() - timedelta(hours=_quiet_hours())
query = (db.session.query(EndpointState, Computer.computerid)
# Status as a sortable expression rather than a post-hoc label, so
# worst-first survives pagination. 0 never reported, 1 quiet, 2 reporting.
statusrank = db.case(
(EndpointState.lastreportedat.is_(None), 0),
(EndpointState.lastreportedat < cutoff, 1),
else_=2,
)
query = (db.session.query(EndpointState, Computer.computerid,
Computer.computertypeid, ComputerType.computertype,
Asset.assetnumber, statusrank.label('statusrank'))
.join(Asset, Asset.assetid == EndpointState.assetid)
.join(Computer, Computer.assetid == EndpointState.assetid)
.outerjoin(ComputerType,
ComputerType.computertypeid == Computer.computertypeid)
.filter(Asset.isactive.is_(True)))
if search := request.args.get('search'):
query = query.filter(EndpointState.hostname.ilike(f'%{search}%'))
# Search covers hostname, asset number and the machine number a PC serves.
# Hostname alone was too narrow: people look a PC up by the bay it runs.
if search := (request.args.get('search') or '').strip():
like = f'%{search}%'
machinehits = _assetids_serving_machine_like(like)
clauses = [EndpointState.hostname.ilike(like), Asset.assetnumber.ilike(like)]
if machinehits:
clauses.append(EndpointState.assetid.in_(machinehits))
query = query.filter(db.or_(*clauses))
query = query.order_by(EndpointState.lastreportedat.desc())
if status := request.args.get('status'):
wanted = {'never': 0, 'quiet': 1, 'reporting': 2}.get(status.lower())
if wanted is not None:
query = query.filter(statusrank == wanted)
if computertypeid := request.args.get('computertypeid'):
query = query.filter(Computer.computertypeid == computertypeid)
if locationid := request.args.get('locationid'):
query = query.filter(Asset.locationid == locationid)
if businessunitid := request.args.get('businessunitid'):
query = query.filter(Asset.businessunitid == businessunitid)
sort = (request.args.get('sort') or 'status').lower()
sorts = {
'status': (statusrank.asc(), EndpointState.hostname.asc()),
'hostname': (EndpointState.hostname.asc(),),
'lastreported': (EndpointState.lastreportedat.desc(),),
'uptime': (EndpointState.uptimeseconds.desc(),),
}
query = query.order_by(*sorts.get(sort, sorts['status']))
items, total = paginate_query(query, page, per_page)
# Batch the per-row extras for this PAGE only. Doing them inline would be
# one query per endpoint for machines and another for events.
assetids = [state.assetid for state, *_ in items]
machines = _machines_for_assets(assetids)
critical = _critical_counts_for_assets(assetids)
data = []
for state, computerid in items:
for state, computerid, computertypeid, computertype, assetnumber, rank in items:
row = state.to_dict()
row['computerid'] = computerid
row['assetnumber'] = assetnumber
row['computertypeid'] = computertypeid
row['computertype'] = computertype
row['uptimedisplay'] = _format_uptime(state.uptimeseconds)
row['status'] = ('never', 'quiet', 'reporting')[rank]
row['machines'] = machines.get(state.assetid, [])
row['health'] = _health_for(state, critical.get(state.assetid, 0))
data.append(row)
return paginated_response(data, page, per_page, total)
def _assetids_serving_machine_like(like):
"""Asset ids of PCs whose controlled machine matches a search term.
Checked in BOTH directions on purpose. The canonical edge is PC controls
machine, but production also holds reciprocal rows, so matching only
source->target silently drops half the fleet from a machine-number search.
"""
from shopdb.api import AssetRelationship, RelationshipType, AssetType
controls = (RelationshipType.query
.filter(db.func.lower(RelationshipType.relationshiptype) == 'controls')
.first())
if controls is None:
return []
# Only MACHINE assets. A PC also 'controls' printers and other kit, and
# matching any controlled asset put office locations in the machine column.
other = db.aliased(Asset)
othertype = db.aliased(AssetType)
forward = (db.session.query(AssetRelationship.sourceassetid)
.join(other, other.assetid == AssetRelationship.targetassetid)
.join(othertype, othertype.assettypeid == other.assettypeid)
.filter(AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
db.func.lower(othertype.assettype) == 'machine',
other.assetnumber.ilike(like)))
backward = (db.session.query(AssetRelationship.targetassetid)
.join(other, other.assetid == AssetRelationship.sourceassetid)
.join(othertype, othertype.assettypeid == other.assettypeid)
.filter(AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
db.func.lower(othertype.assettype) == 'machine',
other.assetnumber.ilike(like)))
return [r[0] for r in forward.union(backward).all()]
def _machines_for_assets(assetids):
"""{assetid: [{assetid, assetnumber, name}]} for the machines each PC serves.
Both edge directions again, deduplicated by machine asset id. One query for
the whole page rather than one per row.
"""
from shopdb.api import AssetRelationship, RelationshipType, AssetType
if not assetids:
return {}
controls = (RelationshipType.query
.filter(db.func.lower(RelationshipType.relationshiptype) == 'controls')
.first())
if controls is None:
return {}
other = db.aliased(Asset)
othertype = db.aliased(AssetType)
forward = (db.session.query(AssetRelationship.sourceassetid.label('pc'),
other.assetid, other.assetnumber, other.name)
.join(other, other.assetid == AssetRelationship.targetassetid)
.join(othertype, othertype.assettypeid == other.assettypeid)
.filter(AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
db.func.lower(othertype.assettype) == 'machine',
AssetRelationship.sourceassetid.in_(assetids)))
backward = (db.session.query(AssetRelationship.targetassetid.label('pc'),
other.assetid, other.assetnumber, other.name)
.join(other, other.assetid == AssetRelationship.sourceassetid)
.join(othertype, othertype.assettypeid == other.assettypeid)
.filter(AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
db.func.lower(othertype.assettype) == 'machine',
AssetRelationship.targetassetid.in_(assetids)))
out = {}
for pc, machineid, assetnumber, name in forward.union(backward).all():
seen = out.setdefault(pc, [])
if not any(m['assetid'] == machineid for m in seen):
seen.append({'assetid': machineid, 'assetnumber': assetnumber, 'name': name})
return out
def _critical_counts_for_assets(assetids):
"""{assetid: count} of critical events in the stability window."""
if not assetids:
return {}
cutoff = _naive_utcnow() - timedelta(hours=48)
rows = (db.session.query(EndpointEvent.assetid,
db.func.count(EndpointEvent.endpointeventid))
.filter(EndpointEvent.assetid.in_(assetids),
EndpointEvent.eventid.in_(CRITICAL_EVENT_IDS),
EndpointEvent.eventtime.isnot(None),
EndpointEvent.eventtime >= cutoff)
.group_by(EndpointEvent.assetid)
.all())
return dict(rows)
def _health_for(state, criticalcount):
"""Worst-of rollup: a stopped watched service, or a recent critical event.
Deliberately a PHRASE, not a score. "eDNC stopped" tells an operator what
to do; a percentage does not.
"""
if criticalcount:
return {'level': 'critical',
'label': f'{criticalcount} critical' if criticalcount > 1 else '1 critical'}
stopped = []
try:
for entry in json.loads(state.servicesjson or '[]'):
if not _service_running(entry):
stopped.append(_service_name(entry))
except (TypeError, ValueError):
# Malformed payload is not a health verdict - say nothing rather than
# claiming the endpoint is unhealthy.
return {'level': 'unknown', 'label': None}
if stopped:
first = stopped[0]
extra = f' +{len(stopped) - 1}' if len(stopped) > 1 else ''
return {'level': 'warning', 'label': f'{first} stopped{extra}'}
if state.lastreportedat is None:
return {'level': 'unknown', 'label': 'no agent'}
return {'level': 'ok', 'label': 'ok'}
@endpoints_bp.route('/<int:asset_id>', methods=['GET'])
@require_permission('endpoints.view')
def get_endpoint(asset_id: int):
"""Get the current state for one endpoint by asset id."""
"""Get the current state for one endpoint by asset id.
Carries the same reporting/quiet/never verdict the fleet list uses. The
quiet window is a site setting, so the SERVER decides it: a caller that
guessed a threshold in the browser would disagree with the fleet list on
the same endpoint the moment a site changed the setting.
"""
state = EndpointState.query.filter_by(assetid=asset_id).first()
if not state:
@@ -102,7 +322,18 @@ def get_endpoint(asset_id: int):
http_code=404,
)
return success_response(state.to_dict())
cutoff = _naive_utcnow() - timedelta(hours=_quiet_hours())
if state.lastreportedat is None:
status = 'never'
elif state.lastreportedat < cutoff:
status = 'quiet'
else:
status = 'reporting'
data = state.to_dict()
data['status'] = status
data['uptimedisplay'] = _format_uptime(state.uptimeseconds)
return success_response(data)
@endpoints_bp.route('/<int:asset_id>/events', methods=['GET'])
@@ -195,6 +426,8 @@ def panel_services(asset_id: int):
if not state or not state.servicesjson:
return success_response([])
maycommand = caller_holds('endpoints.command')
try:
services = json.loads(state.servicesjson)
except (TypeError, ValueError):
@@ -209,14 +442,18 @@ def panel_services(asset_id: int):
kind = entry.get('kind')
else:
name, raw, app, kind = str(entry), '', None, None
running = raw in ('running', 'started', 'up', 'ok', 'active')
running = _service_running(entry)
row = {
'name': name,
'kind': (kind or '').lower() or None,
'statelabel': 'Running' if running else (raw.title() or 'Unknown'),
'statecolor': 'green' if running else ('red' if raw else 'gray'),
}
targets = row_actions_for(app, kind, name)
# Only offer Start/Stop to a caller who could actually run it. The
# emission never consulted the caller, so a read-only viewer was shown
# controls on a shop-floor PC that 403 on click - wrong signal, and it
# gets worse the moment guests can read this panel.
targets = row_actions_for(app, kind, name) if maycommand else None
if targets:
row['action'] = {
'running': running,
@@ -253,6 +490,84 @@ def panel_recent_events(asset_id: int):
return success_response(rows)
@endpoints_bp.route('/asset/<int:asset_id>/activity', methods=['GET'])
@require_permission('endpoints.view')
def panel_activity(asset_id: int):
"""List panel: one chronological feed of what happened to this endpoint.
Curated Windows events and operator commands merged and sorted newest
first. They answer the same question - what happened to this box - and
reading them as two separate cards means interleaving them by eye.
Both sources are already keyed by assetid, so this is two indexed reads and
a merge sort, not a join. The core AuditLog is deliberately NOT a third
source: its rows carry the action as the entity name and the asset only
inside a details string, so pulling it in would mean parsing prose.
The requester's name is withheld from callers who cannot themselves issue
commands. A read-only viewer - a guest, on a site that grants this panel -
has no need for the roster of who restarted what, and the feed is just as
useful without it.
"""
naming = caller_holds('endpoints.command')
events = (EndpointEvent.query
.filter_by(assetid=asset_id)
.order_by(EndpointEvent.eventtime.desc())
.limit(15)
.all())
commands = (EndpointCommand.query
.filter_by(assetid=asset_id)
.order_by(EndpointCommand.createdat.desc())
.limit(15)
.all())
# requestedby holds a user ID, not a name. One batched lookup rather than a
# query per row, and only when the caller is allowed to see the names at all.
names = {}
if naming:
userids = {c.requestedby for c in commands if c.requestedby}
if userids:
names = {u.userid: (u.username or '')
for u in User.query.filter(User.userid.in_(userids)).all()}
rows = []
for event in events:
rows.append({
'title': event_title(event.eventid),
'severity': event_severity(event.eventid),
'detail': 'Windows event {}'.format(event.eventid),
'eventtime': event.eventtime,
})
for command in commands:
label = COMMAND_ALLOWLIST.get(command.action, {}).get('label') or command.action
target = ' on {}'.format(command.target) if command.target else ''
who = ''
if naming and command.requestedby:
username = names.get(command.requestedby)
if username:
who = ' by {}'.format(username)
rows.append({
'title': '{}{}'.format(label, target),
# A failed command is the one an operator needs to notice.
'severity': 'critical' if command.status == EndpointCommand.STATUS_FAILED else 'info',
'detail': '{}{}'.format((command.status or 'queued').title(), who),
'eventtime': command.createdat,
})
# Undated rows sort last rather than crashing the comparison.
rows.sort(key=lambda r: (r['eventtime'] is not None, r['eventtime']), reverse=True)
return success_response([{
'title': r['title'],
'severity': r['severity'],
'detail': r['detail'],
'eventtime': r['eventtime'].isoformat() + 'Z' if r['eventtime'] else None,
} for r in rows[:15]])
@endpoints_bp.route('/asset/<int:asset_id>/interfaces', methods=['GET'])
@require_permission('endpoints.view')
def panel_interfaces(asset_id: int):
@@ -299,7 +614,7 @@ def panel_interfaces(asset_id: int):
# =============================================================================
@endpoints_bp.route('/dashboard/summary', methods=['GET'])
@require_permission('endpoints.view')
@require_permission('endpoints.fleet')
def dashboard_summary():
"""High-level fleet counts for the endpoints dashboard."""
total = db.session.query(EndpointState).join(
@@ -317,16 +632,28 @@ def dashboard_summary():
EndpointState.lastreportedat >= cutoff,
).count()
# Counted separately from quiet. An endpoint that has NEVER reported is a
# different problem from one that reported and stopped: the first is an
# enrolment or key failure, the second is a machine that went down. Folding
# them together (quiet = total - reporting) hid the enrolment failures.
never = db.session.query(EndpointState).join(
Asset, Asset.assetid == EndpointState.assetid
).filter(
Asset.isactive.is_(True),
EndpointState.lastreportedat.is_(None),
).count()
return success_response({
'total': total,
'reporting': reporting,
'quiet': total - reporting,
'quiet': total - reporting - never,
'never': never,
'quiethours': hours,
})
@endpoints_bp.route('/dashboard/quiet', methods=['GET'])
@require_permission('endpoints.view')
@require_permission('endpoints.fleet')
def dashboard_quiet():
"""Endpoints that reported before, then went quiet past the window.
@@ -361,7 +688,7 @@ def dashboard_quiet():
@endpoints_bp.route('/dashboard/stability', methods=['GET'])
@require_permission('endpoints.view')
@require_permission('endpoints.fleet')
def dashboard_stability():
"""Endpoints with recent crash/shutdown/corruption events.