ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge the wilm delivery work into main

Browse Source 
Ben Baker's installed-software collection, software delivery (run_script and
update_app), agentless adoption tool and command-queue hardening, with his
authorship preserved.

Reviewed before promotion. Fixed: the endpoints plugin tests were never collected
by pytest, the SBOM snapshot table's key would exhaust on MySQL, ingest did not
truncate two columns strict MySQL would reject, and update_app's version branch
pinned a digest for a file that route never serves. Agent packaging moved off
require_role('admin') onto a new endpoints.agentpackage permission.

Contract 0.27.0. Full suite 2231, frontend 314.
This commit is contained in:
Cameron Proudlock
2026-08-30 19:41:15 -04:00
parent 8f4fe74dcb
commit 6680c3c5de
59 changed files with 5557 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
plugins/endpoints/plugin.py
View File

@@ -26,7 +26,9 @@ from shopdb.api import db
from .api import endpoints_bp
from .eventcatalog import EVENT_CATALOG
from .models import EndpointState, EndpointEvent, EndpointCommand
from .models import (
EndpointState, EndpointEvent, EndpointCommand, EndpointApplication,
)
logger = logging.getLogger(__name__)
@@ -37,6 +39,21 @@ logger = logging.getLogger(__name__)
# of a raw event firehose.
CURATED_EVENT_IDS = frozenset(EVENT_CATALOG)
# Top-level payload keys the ingest path knows about today - either persisted on
# EndpointState, appended as events, or relayed to the computers collector.
# Anything the agent sends outside this set is not stored yet; it is logged (see
# apply_collector_payload) so new agent telemetry - disks, bios, memory, etc. -
# is observable in the logs before the server grows columns for it. Keep this in
# lockstep with the keys consumed in apply_collector_payload and the computers
# handoff subset.
KNOWN_PAYLOAD_KEYS = frozenset({
'hostname', 'adopt', 'loggedinuser', 'serialnumber', 'pctype',
'machinenumber', 'vendorname', 'modelnumber', 'osname', 'ipaddress',
'agentversion', 'lastboottime', 'uptimeseconds', 'idleseconds',
'cpupercent', 'memorypercent', 'services', 'ports', 'interfaces',
'scheduledtasks', 'applications', 'events',
})
def _parse_dt(value):
"""Best-effort parse of an ISO-8601 string to naive UTC, or None."""
@@ -61,6 +78,20 @@ def _as_json(value):
return json.dumps(value, separators=(',', ':'), sort_keys=True)
def _clamp(value, length):
"""A string bounded to its column width, or None.
Ingest writes whatever an endpoint reports. A registry value longer than its
column is not a data-quality nuisance on MySQL: in strict mode it raises
Data too long and fails the whole check-in transaction, so one machine with
a malformed Uninstall entry stops reporting telemetry and events entirely.
"""
if value is None:
return None
text = str(value).strip()
return text[:length] or None
class EndpointsPlugin(BasePlugin):
"""Endpoint RMM telemetry ingest plus read/dashboard surface."""
@@ -91,7 +122,8 @@ class EndpointsPlugin(BasePlugin):
return endpoints_bp
def get_models(self) -> List[Type]:
return [EndpointState, EndpointEvent, EndpointCommand]
return [EndpointState, EndpointEvent, EndpointCommand,
EndpointApplication]
def get_permissions(self) -> List:
"""RBAC for the human-facing read routes.
@@ -129,6 +161,23 @@ class EndpointsPlugin(BasePlugin):
# remote reboot is a strictly heavier grant than a routine toggle.
('endpoints.restart',
'Restart endpoint computers (destructive)', 'endpoints'),
# Elevated grant for software delivery (run_script, update_app).
# Required IN ADDITION to endpoints.command by the enqueue route, so
# running a staged script or pushing an installer is a strictly
# heavier grant than a routine service/process toggle.
('endpoints.deploy',
'Run staged scripts and push application installers to endpoints', 'endpoints'),
# Packaging the agent is a heavier grant than deploying to one, and
# it is not the same job either: the generated ZIP carries a live
# collector key stamped into its install script, so whoever holds
# this can mint a distributable artefact containing the fleet-wide
# command-channel credential. It was require_role('admin'), which
# this codebase otherwise reserves for the few genuinely
# role-shaped checks - and a role cannot be granted to one person
# for one job the way a permission can.
('endpoints.agentpackage',
'Generate and download an agent or adoption-tool package '
'(contains a live collector key)', 'endpoints'),
# Collector service scope for the endpoint agent. A managed token
# scoped to ONLY this authorizes the endpoints collector and nothing
# else, so rotating/revoking it stops endpoint-agent traffic without
@@ -208,6 +257,19 @@ class EndpointsPlugin(BasePlugin):
'permission': 'endpoints.fleet',
'section': 'assets',
},
# TEMPORARY / DEV ONLY. A manual bench to exercise the update_app and
# run_script delivery path while Cameron builds the production UI.
# Gated on endpoints.deploy so only delivery operators see it. Remove
# this entry (and the DeliveryTestBench view/route) once the real
# delivery UI ships.
{
'name': 'Delivery test bench',
'icon': 'send',
'route': '/endpoints/delivery-test',
'position': 17,
'permission': 'endpoints.deploy',
'section': 'assets',
},
]
def init_app(self, app: Flask, db_instance) -> None:
@@ -374,6 +436,26 @@ class EndpointsPlugin(BasePlugin):
'empty': 'No network interfaces reported.',
'position': 16,
},
{
# Installed-software (SBOM) inventory. Low-priority reference
# data, not something looked at on every visit, so it starts
# collapsed and sits near the bottom. Raw discovered list for
# now: no managed-application linkage is surfaced yet.
'id': 'endpoints-applications',
'title': 'Installed software',
'assettypes': ['computer'],
'endpoint': '/api/endpoints/asset/{assetid}/applications',
'render': 'table',
'collapsible': True,
'collapsedDefault': True,
'columns': [
{'key': 'displayname', 'label': 'Name'},
{'key': 'displayversion', 'label': 'Version'},
{'key': 'publisher', 'label': 'Publisher'},
],
'empty': 'No installed software reported.',
'position': 30,
},
]
def get_dashboard_widgets(self) -> List[Dict]:
@@ -468,6 +550,24 @@ class EndpointsPlugin(BasePlugin):
'type': 'array',
'description': 'Scheduled task inventory + last run.',
},
'applications': {
'type': 'array',
'description': 'Installed-software SBOM from both '
'Windows Uninstall registry hives. Each '
'item is {displayname, displayversion, '
'publisher}, de-duped on name+version. '
'Stored as a per-asset snapshot, '
'replaced on every report.',
'items': {
'type': 'object',
'required': ['displayname'],
'properties': {
'displayname': {'type': 'string'},
'displayversion': {'type': 'string'},
'publisher': {'type': 'string'},
},
},
},
'events': {
'type': 'array',
'description': 'Curated Windows Event Log records. Only '
@@ -540,6 +640,48 @@ class EndpointsPlugin(BasePlugin):
db.session.flush()
warnings.append('adopted new computer for hostname: {!r}'.format(hostname))
def _replace_applications(self, assetid: int, applications,
warnings: List[str]) -> int:
"""Replace an asset's SBOM snapshot with the reported set.
The agent reports the full installed-software list each cycle, so the
current inventory is the newest report, not an accumulation. Delete the
asset's existing rows and insert the reported ones in the same
transaction the caller flushes. appid is left null: no
discovery-to-catalog ("Managed Applications") matcher is wired yet.
Malformed items (no displayname) are skipped, not fatal.
"""
EndpointApplication.query.filter_by(assetid=assetid).delete(
synchronize_session=False)
if not applications:
return 0
stored = 0
for item in applications:
if not isinstance(item, dict):
warnings.append('skipped non-object application entry')
continue
displayname = (item.get('displayname') or '').strip()
if not displayname:
warnings.append('skipped application with no displayname')
continue
# TRUNCATE EVERY STRING, not just the name. On MySQL in strict
# mode an over-long value raises Data too long and takes down the
# WHOLE check-in transaction, so one machine with a malformed
# Uninstall registry entry stops reporting entirely - telemetry,
# events and all - until somebody uninstalls the offender. SQLite
# does not enforce declared lengths, so no test here would ever see
# it. displayname was already clamped; these two were not.
db.session.add(EndpointApplication(
assetid=assetid,
displayname=displayname[:255],
displayversion=_clamp(item.get('displayversion'), 128),
publisher=_clamp(item.get('publisher'), 255),
))
stored += 1
return stored
def apply_collector_payload(self, payload: Dict) -> Dict:
"""Idempotently upsert one endpoint's state and append its events.
@@ -616,8 +758,36 @@ class EndpointsPlugin(BasePlugin):
if 'scheduledtasks' in payload:
state.scheduledtasksjson = _as_json(payload.get('scheduledtasks'))
# Installed-software SBOM: stored as a per-asset SNAPSHOT in its own
# table (endpointapplications), not a JSON blob on the state, so it can
# later join the applications catalog ("Managed Applications"). Only
# replace when the key is present: a report that omits applications must
# not wipe the last-known inventory. The delete + re-insert happens in
# this same transaction (the dispatcher owns the commit), so a reader
# never sees a half-written list. displayname is required; malformed
# rows are skipped rather than failing the whole report.
if 'applications' in payload:
self._replace_applications(assetid, payload.get('applications'),
warnings)
state.lastreportedat = datetime.utcnow()
# Log any keys the agent sent that the ingest path does not consume yet.
# New agent telemetry (disks, bios, memory, ...) lands here first: it is
# not persisted until the server grows columns for it, but logging the
# keys and a compact preview makes incoming telemetry visible in the
# logs so we can confirm the agent is sending it before wiring the UI.
unknown_keys = [key for key in payload if key not in KNOWN_PAYLOAD_KEYS]
if unknown_keys:
preview = {key: payload.get(key) for key in sorted(unknown_keys)}
try:
preview_text = json.dumps(preview, default=str)[:2000]
except (TypeError, ValueError):
preview_text = repr(preview)[:2000]
logger.info(
'endpoint %s sent %d unpersisted telemetry key(s): %s | preview=%s',
hostname, len(unknown_keys), sorted(unknown_keys), preview_text)
stored_events = 0
for record in (payload.get('events') or []):
try: