ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Endpoint telemetry, PC Detail rework and a checked backup/restore path

Browse Source
This commit is contained in:
Cameron Proudlock
2026-09-06 18:53:11 -04:00
parent 32b8745d10
commit 993353dfc0
97 changed files with 13340 additions and 756 deletions
Download Patch File Download Diff File Expand all files Collapse all files

375
plugins/endpoints/plugin.py
View File

@@ -52,6 +52,9 @@ KNOWN_PAYLOAD_KEYS = frozenset({
'agentversion', 'trayversion', 'lastboottime', 'uptimeseconds', 'idleseconds',
'cpupercent', 'memorypercent', 'services', 'ports', 'interfaces',
'scheduledtasks', 'applications', 'events',
'disks', 'bios', 'memory', 'processor', 'osinfo', 'domain',
'pendingreboot', 'usercontext', 'diskhealth',
'rebootreasons', 'bootinfo', 'defender', 'dellupdates',
})
@@ -179,12 +182,32 @@ class EndpointsPlugin(BasePlugin):
# remote reboot is a strictly heavier grant than a routine toggle.
('endpoints.restart',
'Restart endpoint computers (destructive)', 'endpoints'),
# Elevated grant for software delivery (run_script, update_app).
# Elevated grant for software delivery (run_script, update_app) and
# for starting approved scheduled tasks (scheduledtask_run).
# Required IN ADDITION to endpoints.command by the enqueue route, so
# running a staged script or pushing an installer is a strictly
# heavier grant than a routine service/process toggle.
#
# NOTE FOR THE ROLES WORK - this wire is pre-built for you.
# scheduledtask_run was deliberately mapped onto this EXISTING grant
# rather than minting a new permission, so nothing needs migrating.
# Be aware that WIDENS what endpoints.deploy means: it used to cover
# run_script + update_app, and now also covers starting any task on
# the agent's approved-task allowlist. Splitting it out is a one-line
# change (add endpoints.corptools here and remap it in
# commands.ELEVATED_ACTION_PERMISSIONS) - the enqueue route reads
# that table, so no other code changes.
#
# If you do split it, note the three are NOT equal risk:
# scheduledtask_run only starts something already registered on the
# endpoint and is allowlisted agent-side; update_app runs a pinned
# installer; run_script is arbitrary code that runs as SYSTEM unless
# its descriptor pins the user context. run_script is the reason this
# grant's description has to stay blunt.
('endpoints.deploy',
'Run staged scripts and push application installers to endpoints', 'endpoints'),
'Run staged scripts, start approved scheduled tasks, and push '
'application installers to endpoints (scripts run as SYSTEM; '
'grant only to fully trusted operators)', 'endpoints'),
# Packaging the agent is a heavier grant than deploying to one, and
# it is not the same job either: the generated ZIP carries a live
# collector key stamped into its install script, so whoever holds
@@ -391,7 +414,9 @@ class EndpointsPlugin(BasePlugin):
'position': 12,
# Service state is what is being watched; the Start/Stop
# controls belong beside it, not in a tab of their own.
'group': 'monitors',
# Lives in serviceprocess rather than monitors: that tab is the
# box's own health, this is what runs on it.
'group': 'serviceprocess',
},
{
'id': 'endpoints-tracked-processes',
@@ -431,7 +456,7 @@ class EndpointsPlugin(BasePlugin):
'empty': 'No monitored processes reported.',
'position': 13,
# observed process state, not something you act on here
'group': 'monitors',
'group': 'serviceprocess',
},
{
# One feed instead of two cards. Events and commands answer the
@@ -472,6 +497,11 @@ class EndpointsPlugin(BasePlugin):
{'key': 'detail'},
{'key': 'eventtime', 'label': 'At', 'format': 'date'},
],
# Expands in place on click. The message body is several
# hundred words of Windows boilerplate around the one or
# two lines that matter, so it stays collapsed until asked
# for rather than pushing every other row off the card.
'body': 'body',
},
'empty': 'Nothing recorded for this endpoint yet.',
'position': 20,
@@ -493,10 +523,46 @@ class EndpointsPlugin(BasePlugin):
{'key': 'subnet', 'label': 'Subnet'},
{'key': 'dhcp', 'label': 'Assignment'},
{'key': 'role', 'label': 'Role'},
{'key': 'link', 'label': 'Link'},
],
'empty': 'No network interfaces reported.',
'position': 16,
# adapter state is watched, not something you act on
# adapter state is watched, not something you act on.
# Declared in monitors but RENDERED at the top of the Details
# tab (PCDetail.vue lifts it by id and excludes it here): which
# adapter is up and what address it holds is asked at the same
# moment as "what is this box".
'group': 'monitors',
},
{
# What this box has been on the network, carved out of the
# hand-written Network card that used to sit on Details and
# show only current addresses with a Primary badge.
#
# The history is ACCIDENTAL: _record_interfaces matches on MAC
# then IP and never deletes, so old addresses accumulate. That
# turned out to be worth reading rather than worth cleaning
# away, so it is presented as what it already is.
'id': 'endpoints-iphistory',
'title': 'IP History',
'assettypes': ['computer'],
'endpoint': '/api/endpoints/asset/{assetid}/iphistory',
'render': 'table',
'collapsible': True,
'columns': [
{'key': 'ipaddress', 'label': 'IP address', 'mono': True},
{'key': 'macaddress', 'label': 'MAC', 'mono': True},
{'key': 'status', 'label': 'Status'},
{'key': 'firstseen', 'label': 'First seen',
'format': 'date'},
# NOT "last seen": modifieddate only moves when a field on
# the row changes, so an address still held and reported
# unchanged keeps an old stamp here.
{'key': 'lastchanged', 'label': 'Last changed',
'format': 'date'},
],
'empty': 'No addresses on record for this endpoint.',
'position': 16.5,
'group': 'monitors',
},
{
@@ -524,10 +590,123 @@ class EndpointsPlugin(BasePlugin):
# the installed-software inventory
'group': 'software',
},
{
'id': 'endpoints-hardware',
'title': 'Hardware',
'assettypes': ['computer'],
'endpoint': '/api/endpoints/asset/{assetid}/hardware',
'render': 'keyvalue',
'collapsible': True,
'empty': 'No hardware inventory reported.',
'position': 14,
'group': 'monitors',
},
{
'id': 'endpoints-osinfo',
'title': 'Operating system',
'assettypes': ['computer'],
'endpoint': '/api/endpoints/asset/{assetid}/osinfo',
'render': 'keyvalue',
'collapsible': True,
'empty': 'No OS inventory reported.',
'position': 15,
'group': 'monitors',
},
{
# Session-scoped, so it sits with the other things watched
# rather than acted on. The user context card still carries a
# one-line summary; this is the same data with room to show a
# remote path and a per-drive badge, and both are built from
# _mapped_drive_rows so they cannot disagree.
#
# Room reserved here for the future one-click reconnect: the
# list render already supports a row action, so adding it is a
# map/commandendpoint change rather than a new card.
'id': 'endpoints-mappeddrives',
'title': 'Mapped drives',
'assettypes': ['computer'],
'endpoint': '/api/endpoints/asset/{assetid}/mappeddrives',
'render': 'list',
'collapsible': True,
'status': {
'key': 'statecolor',
'levels': {
'green': 'ok',
'yellow': 'warning',
'red': 'critical',
},
},
'map': {
'title': 'drive',
'meta': [{'key': 'remote', 'mono': True}],
'badge': {'label': 'statuslabel', 'color': 'statecolor'},
},
'empty': 'No mapped drives reported.',
'position': 21,
'group': 'monitors',
},
{
# Volumes and physical drive health in one card, as two
# sections. VISUAL GROUPING ONLY: the two tables share no key
# (storage is per volume, health is per physical device) and
# nothing is reconciled across them. They are one card because
# the operator reading a full disk and the one reading a dying
# disk are usually the same person in the same minute.
'id': 'endpoints-storage',
'title': 'Storage',
'assettypes': ['computer'],
'endpoint': '/api/endpoints/asset/{assetid}/storage',
'render': 'tabs',
'collapsible': True,
'empty': 'No disks reported.',
'position': 17,
'group': 'monitors',
},
{
# The listening set is the box's exposed network surface. The
# full connection table is deliberately not here: it is an
# on-demand command, because a per-cycle capture of every
# connection is a lot of rows nobody reads.
'id': 'endpoints-listeners',
'title': 'Listening ports',
'assettypes': ['computer'],
'endpoint': '/api/endpoints/asset/{assetid}/listeners',
'render': 'table',
'collapsible': True,
'columns': [
{'key': 'localport', 'label': 'Port'},
{'key': 'protocol', 'label': 'Protocol'},
{'key': 'localaddress', 'label': 'Address'},
{'key': 'processname', 'label': 'Process'},
{'key': 'processid', 'label': 'PID'},
],
'empty': 'No listening ports reported.',
'position': 19,
'group': 'monitors',
},
{
# Session-scoped data collected by a probe in the interactive
# user's session, so it is absent whenever nobody is signed in.
# Last on the page: it is the least reliably present card, and
# an empty one at the bottom reads as "not reported" rather
# than pushing the populated cards down.
'id': 'endpoints-usercontext',
'title': 'User context',
'assettypes': ['computer'],
'endpoint': '/api/endpoints/asset/{assetid}/usercontext',
'render': 'keyvalue',
'collapsible': True,
'empty': 'No user session data reported.',
'position': 40,
'group': 'monitors',
},
{
# Operator-submitted issue reports: a free-text note and an
# optional screenshot the operator captured from the tray. Sits
# in the Monitors tab beside the other watch-and-observe panels.
# optional screenshot the operator captured from the tray. Has
# a tab of its own rather than sitting in System Monitors: that
# tab is machine-collected health, this is a human telling you
# something, and the thumbnail grid is far bulkier than the
# fact cards it used to sit between.
# A custom render carries the thumbnail grid and the expanding
# lightbox with left/right cycling the generic modes cannot.
'id': 'endpoints-operator-reports',
@@ -539,7 +718,7 @@ class EndpointsPlugin(BasePlugin):
'collapsible': True,
'empty': 'No operator reports for this endpoint.',
'position': 18,
'group': 'monitors',
'group': 'reports',
},
]
@@ -636,6 +815,80 @@ class EndpointsPlugin(BasePlugin):
'type': 'array',
'description': 'Scheduled task inventory + last run.',
},
'disks': {
'type': 'array',
'description': 'Fixed local disks: {drive, label, '
'filesystem, sizebytes, freebytes, '
'usedbytes, freepercent}. Newest '
'snapshot, replaced on every report.',
},
'bios': {
'type': 'object',
'description': 'Firmware identity: vendor, version, '
'release date.',
},
'memory': {
'type': 'object',
'description': 'Physical memory totals in bytes.',
},
'processor': {
'type': 'object',
'description': 'CPU identity, core and logical '
'processor counts, max clock.',
},
'osinfo': {
'type': 'object',
'description': 'OS build, version, and install date.',
},
'domain': {
'type': 'string',
'description': 'Windows domain or workgroup membership.',
},
'pendingreboot': {
'type': 'boolean',
'description': 'True when the endpoint is holding a '
'reboot. Absent means never reported, '
'which is not the same as false.',
},
'diskhealth': {
'type': 'array',
'description': 'Per-drive health from two sources: '
'physicaldisk (the only one reporting '
'NVMe) and diskdrive (ATA/SATA and '
'fallback). Each entry carries a '
'precomputed predictfailure; null there '
'means the drive did not answer.',
},
'rebootreasons': {
'type': 'array',
'description': 'Which signal(s) drove pendingreboot: '
'componentbasedservicing, windowsupdate, '
'pendingfilerename. Empty array means '
'checked and nothing pending; omit when '
'the check failed.',
},
'bootinfo': {
'type': 'object',
'description': 'hiberbootenabled (fast startup is '
'configured) and lastboottype (what the '
'last boot actually did). Uptime is not '
'trustworthy when fast startup is in '
'play.',
},
'defender': {
'type': 'object',
'description': 'Defender service/realtime state and '
'signature and scan ages in days. '
'Persisted but not surfaced.',
},
'usercontext': {
'type': 'object',
'description': 'Session-scoped data collected by a probe '
'running as the signed-in user: mapped '
'drives (netuse), user-owned listeners, '
'and idle seconds. Absent when nobody is '
'signed in.',
},
'applications': {
'type': 'array',
'description': 'Installed-software SBOM from both '
@@ -663,6 +916,12 @@ class EndpointsPlugin(BasePlugin):
'required': ['eventid'],
'properties': {
'eventid': {'type': 'integer'},
'eventrecordid': {
'type': 'integer',
'description': 'Windows event record id; '
'unique per channel per '
'machine, used to dedup.',
},
'eventchannel': {'type': 'string'},
'eventsource': {'type': 'string'},
'eventtime': {'type': 'string',
@@ -846,6 +1105,51 @@ class EndpointsPlugin(BasePlugin):
if 'scheduledtasks' in payload:
state.scheduledtasksjson = _as_json(payload.get('scheduledtasks'))
# Hardware/OS inventory. Same presence guard as above: a report that
# omits a key must not wipe the last-known value. These change rarely
# (a RAM upgrade, a BIOS flash), so a transient collection failure on
# one cycle should leave the previous snapshot standing rather than
# blank the asset record.
if 'disks' in payload:
state.disksjson = _as_json(payload.get('disks'))
if 'bios' in payload:
state.biosjson = _as_json(payload.get('bios'))
if 'memory' in payload:
state.memoryjson = _as_json(payload.get('memory'))
if 'processor' in payload:
state.processorjson = _as_json(payload.get('processor'))
if 'osinfo' in payload:
state.osinfojson = _as_json(payload.get('osinfo'))
if 'domain' in payload:
# Clamped like every other String column here: an over-long value
# aborts the whole collector transaction under STRICT_TRANS_TABLES,
# so one bad agent report would drop a machine's entire telemetry.
state.domain = _clamp(payload.get('domain'), 255)
if 'pendingreboot' in payload:
state.pendingreboot = payload.get('pendingreboot')
if 'diskhealth' in payload:
state.diskhealthjson = _as_json(payload.get('diskhealth'))
if 'rebootreasons' in payload:
state.rebootreasonsjson = _as_json(payload.get('rebootreasons'))
if 'bootinfo' in payload:
state.bootinfojson = _as_json(payload.get('bootinfo'))
if 'defender' in payload:
state.defenderjson = _as_json(payload.get('defender'))
# Presence-guarded like the probe output below: the agent omits the key
# entirely when the collector utility is not installed, and overwriting
# to null there would discard a still-valid scan result from before the
# utility was removed.
if 'dellupdates' in payload:
state.dellupdatesjson = _as_json(payload.get('dellupdates'))
# Session-scoped probe output. The agent repeats its last result between
# probe runs and omits the key entirely when nobody is signed in, so the
# presence guard preserves the last known mapped drives across an
# unattended period instead of flapping the field to null.
if 'usercontext' in payload:
state.usercontextjson = _as_json(payload.get('usercontext'))
# Installed-software SBOM: stored as a per-asset SNAPSHOT in its own
# table (endpointapplications), not a JSON blob on the state, so it can
# later join the applications catalog ("Managed Applications"). Only
@@ -897,6 +1201,19 @@ class EndpointsPlugin(BasePlugin):
hostname, len(unknown_keys), sorted(unknown_keys), preview_text)
stored_events = 0
skipped_events = 0
# Records the agent already sent must not be appended a second time.
# The agent carries a per-channel watermark so the steady state is
# "only what is new", but a reinstalled agent, a restored image, or two
# overlapping reports can all replay records. Resolve that here rather
# than relying on the unique index to raise, because an IntegrityError
# would abort the whole collector transaction and lose the state upsert
# along with it.
#
# One query for the whole batch, not one per record: a seeding agent
# sends up to the per-channel cap at once.
candidates = []
for record in (payload.get('events') or []):
try:
eventid = int(record.get('eventid'))
@@ -906,10 +1223,45 @@ class EndpointsPlugin(BasePlugin):
if eventid not in CURATED_EVENT_IDS:
# Outside the catalog: ignored on purpose, not an error.
continue
try:
eventrecordid = record.get('eventrecordid')
eventrecordid = (int(eventrecordid)
if eventrecordid is not None else None)
except (TypeError, ValueError):
eventrecordid = None
candidates.append((eventid, eventrecordid, record))
keyed = [(record.get('eventchannel'), eventrecordid)
for _e, eventrecordid, record in candidates
if eventrecordid is not None]
already_stored = set()
if keyed:
rows = (db.session.query(EndpointEvent.eventchannel,
EndpointEvent.eventrecordid)
.filter(EndpointEvent.assetid == assetid,
EndpointEvent.eventrecordid.in_(
[recordid for _c, recordid in keyed]))
.all())
already_stored = {(channel, recordid) for channel, recordid in rows}
for eventid, eventrecordid, record in candidates:
eventchannel = record.get('eventchannel')
if eventrecordid is not None:
key = (eventchannel, eventrecordid)
if key in already_stored:
skipped_events += 1
continue
# Guard against the same record appearing twice inside one
# payload as well as against rows already in the table.
already_stored.add(key)
db.session.add(EndpointEvent(
assetid=assetid,
eventid=eventid,
eventchannel=record.get('eventchannel'),
eventrecordid=eventrecordid,
eventchannel=eventchannel,
eventsource=record.get('eventsource'),
eventtime=_parse_dt(record.get('eventtime')),
eventmessage=record.get('eventmessage'),
@@ -917,6 +1269,11 @@ class EndpointsPlugin(BasePlugin):
))
stored_events += 1
if skipped_events:
logger.debug(
'endpoint %s replayed %d already-stored event record(s)',
hostname, skipped_events)
# flush, not commit: the collector dispatcher owns the transaction.
db.session.flush()