2c4d8ab4cb
Adds rebootreasonsjson, bootinfojson and defenderjson with their migration in the same change, so the model never references a column the schema has not applied. rebootreasons is not redundant with the existing pendingreboot column. That column is a single bool OR-ed from three registry signals and cannot say which fired, and the difference matters: a staged file rename means the box is mid-servicing, while a lone servicing key can sit set for weeks on a machine nobody rebooted. Null means never reported and is deliberately distinct from an empty list, which means checked and nothing pending. Backfilling [] would assert every pre-existing endpoint was confirmed clean.
438 lines
20 KiB
Python
438 lines
20 KiB
Python
"""Endpoint RMM telemetry models.
|
|
|
|
Two shapes, one per concern:
|
|
|
|
EndpointState The cached CURRENT snapshot for an asset - one row per asset,
|
|
overwritten in place on every agent report. Holds the fast-
|
|
moving metrics (uptime, idle, last boot, logged-in user) and
|
|
the latest inventory snapshots (services/processes, open ports,
|
|
interfaces, scheduled tasks) as serialized JSON. This is the
|
|
cached-current-value half of the ledger pattern: it moves in
|
|
the SAME commit as any EndpointEvent rows the report carries.
|
|
|
|
EndpointEvent Append-only record of a curated Windows Event Log entry. The
|
|
agent only forwards a fixed catalog of IDs (logon/logoff,
|
|
privilege use, service install, unexpected shutdown, DCOM,
|
|
crash) so this stays a readable security/stability history
|
|
rather than a raw event firehose.
|
|
|
|
Both link to the core Asset via assetid (FK, ON DELETE CASCADE), so an endpoint
|
|
is the same physical thing as its Computer/Asset - no duplicate identity. The
|
|
agent reports a hostname; the plugin resolves it to the existing Computer.
|
|
"""
|
|
|
|
from datetime import datetime
|
|
|
|
from shopdb.api import db
|
|
|
|
|
|
def _utciso(value):
|
|
"""Naive-UTC datetime -> ISO string marked UTC, or None.
|
|
|
|
Every datetime this plugin stores is naive UTC. Serializing without the Z
|
|
leaves the receiver to guess, and JavaScript guesses browser-local.
|
|
"""
|
|
return value.isoformat() + 'Z' if value else None
|
|
|
|
|
|
class EndpointState(db.Model):
|
|
"""Latest reported state for a single endpoint (one row per asset)."""
|
|
|
|
__tablename__ = 'endpointstates'
|
|
|
|
endpointstateid = db.Column(db.Integer, primary_key=True)
|
|
|
|
# The asset this state belongs to. Unique: exactly one current snapshot per
|
|
# endpoint. Resolved from the reported hostname against the computers plugin.
|
|
assetid = db.Column(
|
|
db.Integer,
|
|
db.ForeignKey('assets.assetid', ondelete='CASCADE'),
|
|
unique=True,
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
|
|
# Read-only view of the owning asset. No cascade or backref: the asset side
|
|
# must not gain a dependency on this plugin.
|
|
asset = db.relationship('Asset', lazy='joined', viewonly=True)
|
|
|
|
hostname = db.Column(db.String(100), nullable=True)
|
|
loggedinuser = db.Column(db.String(100), nullable=True)
|
|
agentversion = db.Column(db.String(40), nullable=True)
|
|
# The interactive tray app versions independently of the agent service, so
|
|
# it is reported and cached as its own field (AGENT-CONTRACT.md 2). Null
|
|
# when the endpoint has no tray installed or an older agent that predates
|
|
# the field.
|
|
trayversion = db.Column(db.String(40), nullable=True)
|
|
|
|
# Fast-moving metrics
|
|
lastboottime = db.Column(db.DateTime, nullable=True)
|
|
uptimeseconds = db.Column(db.BigInteger, nullable=True)
|
|
idleseconds = db.Column(db.BigInteger, nullable=True)
|
|
cpupercent = db.Column(db.Float, nullable=True)
|
|
memorypercent = db.Column(db.Float, nullable=True)
|
|
|
|
# Latest inventory snapshots, serialized JSON (MEDIUMTEXT). Kept as the
|
|
# newest snapshot rather than one row per sample: the agent reports the full
|
|
# list each cycle and history of these lists is not what operators ask for.
|
|
servicesjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
portsjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
interfacesjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
scheduledtasksjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
|
|
# Hardware and OS inventory, same newest-snapshot rule as above. These
|
|
# arrived on the wire well before this and were logged as unknown keys, so
|
|
# the shapes are already proven against real endpoint traffic.
|
|
disksjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
biosjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
memoryjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
processorjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
osinfojson = db.Column(db.Text(length=16777215), nullable=True)
|
|
|
|
# Session-scoped data collected by a probe running as the signed-in user
|
|
# (mapped drives, user-owned listeners). Null when nobody is signed in,
|
|
# which is a normal unattended state rather than a fault.
|
|
usercontextjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
|
|
# Windows domain or workgroup membership. A scalar, not JSON: it is a single
|
|
# value the fleet view filters and groups on.
|
|
domain = db.Column(db.String(255), nullable=True)
|
|
|
|
# True when the endpoint is holding a reboot. Null = never reported, which
|
|
# is deliberately distinct from a confident False.
|
|
pendingreboot = db.Column(db.Boolean, nullable=True)
|
|
|
|
# Per-drive health from the agent's two disk sources. JSON because a machine
|
|
# has a variable number of drives and each carries its own verdict. Kept
|
|
# separate from disksjson: that is capacity, this is whether the hardware is
|
|
# dying. Null = never reported.
|
|
diskhealthjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
|
|
# Which signal(s) drove pendingreboot. That column is a single bool OR-ed
|
|
# from three sources and so cannot say WHY, and the difference matters: a
|
|
# pendingfilerename means files are staged for replacement on next boot, so
|
|
# the box is mid-servicing, while a lone servicing key can sit set for weeks
|
|
# on a machine nobody rebooted. Empty list = checked, nothing pending.
|
|
# Null = never reported or the check failed.
|
|
rebootreasonsjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
|
|
# Fast startup configuration and what the last boot actually did. Kept
|
|
# because uptime is not trustworthy under fast startup: a box can report
|
|
# days of uptime after what the operator experienced as a shutdown, and a
|
|
# "reboot" that was a hybrid resume does not clear the state a support call
|
|
# is usually trying to clear.
|
|
bootinfojson = db.Column(db.Text(length=16777215), nullable=True)
|
|
|
|
# Defender health. Persisted but deliberately NOT surfaced anywhere yet:
|
|
# this environment produces known false positives, and wiring it to a card
|
|
# now would train operators to ignore that card. Stored so the history is
|
|
# already accumulating when someone is ready to interpret it.
|
|
defenderjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
|
|
# Warn-only reconciler output, refreshed on every agent check-in in the same
|
|
# commit as servicesjson. Each entry names a policy/shipped-attached monitor
|
|
# the endpoint is expected to be running but the latest snapshot does not
|
|
# report as running, so an operator sees the drift and intervenes manually.
|
|
# This is warn-only: nothing here is auto-remediated. Null = never computed.
|
|
driftjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
|
|
# Last-known MTConnect agent run state, cached so the WebUI toggle can show
|
|
# a real state between command polls. Set by the agent's mtconnect_status
|
|
# result. Null = never reported (unknown).
|
|
mtconnectrunning = db.Column(db.Boolean, nullable=True)
|
|
mtconnectcheckedat = db.Column(db.DateTime, nullable=True)
|
|
|
|
# When the agent last successfully reported. There is no heartbeat separate
|
|
# from the report, so silence past the quiet-hours window is the only
|
|
# evidence an endpoint has stopped checking in.
|
|
lastreportedat = db.Column(db.DateTime, nullable=True, index=True)
|
|
|
|
createdat = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
|
|
|
|
def __repr__(self):
|
|
return f"<EndpointState {self.hostname or self.assetid}>"
|
|
|
|
def to_dict(self):
|
|
return {
|
|
'endpointstateid': self.endpointstateid,
|
|
'assetid': self.assetid,
|
|
'hostname': self.hostname,
|
|
'loggedinuser': self.loggedinuser,
|
|
'agentversion': self.agentversion,
|
|
'trayversion': self.trayversion,
|
|
'lastboottime': _utciso(self.lastboottime),
|
|
'uptimeseconds': self.uptimeseconds,
|
|
'idleseconds': self.idleseconds,
|
|
'cpupercent': self.cpupercent,
|
|
'memorypercent': self.memorypercent,
|
|
'domain': self.domain,
|
|
'pendingreboot': self.pendingreboot,
|
|
'mtconnectrunning': self.mtconnectrunning,
|
|
'mtconnectcheckedat': _utciso(self.mtconnectcheckedat),
|
|
'lastreportedat': _utciso(self.lastreportedat),
|
|
'createdat': _utciso(self.createdat),
|
|
}
|
|
|
|
|
|
class EndpointCommand(db.Model):
|
|
"""A queued operator command targeting one endpoint agent.
|
|
|
|
The WebUI enqueues a named action (server allowlist only, never an arbitrary
|
|
command string) for an asset. The agent claims the next pending command on
|
|
its long-poll, executes it locally, and reports the outcome back, which
|
|
moves the row to a terminal status in the same request. This is a small
|
|
at-most-one-in-flight queue, not a general job system.
|
|
"""
|
|
|
|
__tablename__ = 'endpointcommands'
|
|
|
|
# Lifecycle: pending -> running -> succeeded|failed. A command is claimed
|
|
# (pending -> running) by exactly one agent poll, then finalized by its
|
|
# result post. A command that is claimed but whose agent never reports back
|
|
# (agent killed/uninstalled/offline mid-flight) would otherwise pin the
|
|
# at-most-one guard forever; the enqueue path reaps such rows to STATUS_EXPIRED
|
|
# so a dead in-flight command cannot wedge the endpoint's command queue.
|
|
STATUS_PENDING = 'pending'
|
|
STATUS_RUNNING = 'running'
|
|
STATUS_SUCCEEDED = 'succeeded'
|
|
STATUS_FAILED = 'failed'
|
|
STATUS_EXPIRED = 'expired'
|
|
|
|
# In-flight staleness ceilings. A RUNNING command is expected to finish in
|
|
# seconds to a couple of minutes; the generous ceiling tolerates a slow
|
|
# app install while still freeing the queue when the agent is gone. A
|
|
# PENDING command is claimed within one poll (~25s) by a live agent, so a
|
|
# long-unclaimed row means the agent is offline and the queue should not
|
|
# stay wedged behind it.
|
|
RUNNING_STALE_SECONDS = 15 * 60
|
|
PENDING_STALE_SECONDS = 60 * 60
|
|
|
|
endpointcommandid = db.Column(db.Integer, primary_key=True)
|
|
|
|
assetid = db.Column(
|
|
db.Integer,
|
|
db.ForeignKey('assets.assetid', ondelete='CASCADE'),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
|
|
asset = db.relationship('Asset', lazy='joined', viewonly=True)
|
|
|
|
# The allowlisted action name (for example 'mtconnect_start'). Validated
|
|
# against the server allowlist at enqueue time, so only known names land.
|
|
action = db.Column(db.String(64), nullable=False)
|
|
|
|
# Generic actions (service_* / process_*) carry their subject over the wire:
|
|
# target is a Windows service name or a process image (for example
|
|
# 'DncMain.exe'); targetkind is 'service' or 'process'. Fixed-target actions
|
|
# (mtconnect_*) leave both null. This is what lets a site build the agent
|
|
# once and drive every future policy/app purely from server data.
|
|
target = db.Column(db.String(255), nullable=True)
|
|
targetkind = db.Column(db.String(16), nullable=True)
|
|
|
|
status = db.Column(db.String(20), nullable=False,
|
|
default=STATUS_PENDING, index=True)
|
|
|
|
# Who queued it (userid) for audit; the agent never sees this.
|
|
requestedby = db.Column(db.Integer, nullable=True)
|
|
|
|
# The content digest as it stood WHEN AN OPERATOR APPROVED THIS PUSH.
|
|
#
|
|
# Delivery descriptors are re-resolved at claim time, minutes or hours
|
|
# later, so without this the bytes an operator approved and the bytes the
|
|
# agent runs are two different lookups against mutable content. Whoever can
|
|
# replace a script body or an installer package could swap it in the gap
|
|
# and the digest would faithfully describe the new bytes.
|
|
#
|
|
# Set at enqueue for run_script and update_app; null for actions that
|
|
# deliver no content. The claim path refuses a command whose descriptor no
|
|
# longer matches, so a swap fails the command instead of shipping.
|
|
pinnedsha256 = db.Column(db.String(64), nullable=True)
|
|
|
|
# Agent-reported outcome. resultjson holds structured detail (for example
|
|
# the observed MTConnect running state); resultmessage is a short summary.
|
|
resultmessage = db.Column(db.Text, nullable=True)
|
|
resultjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
|
|
createdat = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
|
|
claimedat = db.Column(db.DateTime, nullable=True)
|
|
completedat = db.Column(db.DateTime, nullable=True)
|
|
|
|
__table_args__ = (
|
|
db.Index('idx_endpointcommands_assetstatus', 'assetid', 'status'),
|
|
)
|
|
|
|
def __repr__(self):
|
|
return f"<EndpointCommand {self.action} asset={self.assetid} {self.status}>"
|
|
|
|
def to_dict(self):
|
|
return {
|
|
'endpointcommandid': self.endpointcommandid,
|
|
'assetid': self.assetid,
|
|
'action': self.action,
|
|
'target': self.target,
|
|
'targetkind': self.targetkind,
|
|
'status': self.status,
|
|
'requestedby': self.requestedby,
|
|
'resultmessage': self.resultmessage,
|
|
'createdat': _utciso(self.createdat),
|
|
'claimedat': _utciso(self.claimedat),
|
|
'completedat': _utciso(self.completedat),
|
|
}
|
|
|
|
|
|
class EndpointEvent(db.Model):
|
|
"""A single curated Windows Event Log record forwarded by the agent."""
|
|
|
|
__tablename__ = 'endpointevents'
|
|
|
|
endpointeventid = db.Column(db.Integer, primary_key=True)
|
|
|
|
assetid = db.Column(
|
|
db.Integer,
|
|
db.ForeignKey('assets.assetid', ondelete='CASCADE'),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
|
|
asset = db.relationship('Asset', lazy='joined', viewonly=True)
|
|
|
|
# The Windows Event ID (for example 4625 failed logon, 41 unexpected
|
|
# shutdown, 1001 application crash). Indexed because alerting queries filter
|
|
# on it.
|
|
eventid = db.Column(db.Integer, nullable=False, index=True)
|
|
|
|
# Event Log channel the record came from: Security, System, Application.
|
|
eventchannel = db.Column(db.String(50), nullable=True)
|
|
eventsource = db.Column(db.String(255), nullable=True)
|
|
|
|
# The Windows event record id: monotonic and unique within its channel on a
|
|
# given machine. This is the dedup key. The agent also uses it as its
|
|
# harvest watermark, so in normal operation a record is sent once; the
|
|
# unique index below is what guarantees it even when an agent is reinstalled
|
|
# (losing its watermark) or two reports overlap. Nullable because an agent
|
|
# older than this field still reports, and MySQL permits repeated NULLs in a
|
|
# unique index, so those degrade to the previous append-only behavior rather
|
|
# than failing.
|
|
eventrecordid = db.Column(db.BigInteger, nullable=True)
|
|
|
|
# When the event happened on the endpoint (from the record), not when it
|
|
# reached ShopDB.
|
|
eventtime = db.Column(db.DateTime, nullable=True, index=True)
|
|
|
|
eventmessage = db.Column(db.Text, nullable=True)
|
|
|
|
# Structured fields the agent extracted for this ID (for example the target
|
|
# account on a 4625). Advisory index only, not part of dedup.
|
|
payloadjson = db.Column(db.Text(length=16777215), nullable=True)
|
|
|
|
recordedat = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
|
|
|
|
__table_args__ = (
|
|
db.Index('idx_endpointevents_assetevent', 'assetid', 'eventtime'),
|
|
db.Index('idx_endpointevents_assetrecord',
|
|
'assetid', 'eventchannel', 'eventrecordid', unique=True),
|
|
)
|
|
|
|
def __repr__(self):
|
|
return f"<EndpointEvent {self.eventid} asset={self.assetid}>"
|
|
|
|
def to_dict(self):
|
|
return {
|
|
'endpointeventid': self.endpointeventid,
|
|
'assetid': self.assetid,
|
|
'eventid': self.eventid,
|
|
'eventrecordid': self.eventrecordid,
|
|
'eventchannel': self.eventchannel,
|
|
'eventsource': self.eventsource,
|
|
'eventtime': _utciso(self.eventtime),
|
|
'eventmessage': self.eventmessage,
|
|
'recordedat': _utciso(self.recordedat),
|
|
}
|
|
|
|
|
|
class EndpointApplication(db.Model):
|
|
"""One installed-software (SBOM) entry reported for an endpoint.
|
|
|
|
The agent enumerates both Windows Uninstall registry hives and emits a
|
|
per-item {displayname, displayversion, publisher}, de-duped on
|
|
name+version. The server stores that as a per-asset SNAPSHOT: on each
|
|
report the existing rows for the asset are replaced with the reported set,
|
|
mirroring how the services/ports inventories are kept as the newest list
|
|
rather than an append-only history. History of installs is not what
|
|
operators ask for; the current inventory is.
|
|
|
|
appid is groundwork for a future link to the core applications catalog
|
|
(the "Managed Applications" concept): a nullable, unenforced-by-match FK
|
|
that a later matcher can populate to associate a discovered install with a
|
|
managed Application row. No matching logic exists yet - the column and the
|
|
relationship are laid down here so the association can be added without a
|
|
second migration.
|
|
"""
|
|
|
|
__tablename__ = 'endpointapplications'
|
|
|
|
# BigInteger on MySQL, Integer on SQLite. This table is rewritten wholesale
|
|
# on every check-in, so the key never stops climbing and a 32-bit column
|
|
# eventually stops accepting inserts - see the migration for the arithmetic.
|
|
#
|
|
# The variant is not decoration: SQLite only autoincrements a column typed
|
|
# exactly INTEGER PRIMARY KEY, so a plain BigInteger makes every insert fail
|
|
# with a NOT NULL violation on the test database. Nothing caught that,
|
|
# because no test inserts into this table.
|
|
endpointapplicationid = db.Column(
|
|
db.BigInteger().with_variant(db.Integer, 'sqlite'), primary_key=True)
|
|
|
|
# The asset this install belongs to. Not unique: many applications per
|
|
# endpoint. CASCADE so removing the asset clears its SBOM snapshot.
|
|
assetid = db.Column(
|
|
db.Integer,
|
|
db.ForeignKey('assets.assetid', ondelete='CASCADE'),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
|
|
asset = db.relationship('Asset', lazy='joined', viewonly=True)
|
|
|
|
# Wire fields, stored under the same lowercase/no-underscore names the agent
|
|
# emits. displayname is always present (the agent skips entries without it);
|
|
# displayversion and publisher default to '' on the agent side.
|
|
displayname = db.Column(db.String(255), nullable=False, index=True)
|
|
displayversion = db.Column(db.String(128), nullable=True)
|
|
publisher = db.Column(db.String(255), nullable=True)
|
|
|
|
# Groundwork link to the core applications catalog ("Managed Applications").
|
|
# Nullable and unpopulated for now: no discovery-to-catalog matcher is wired
|
|
# yet. SET NULL so retiring a managed Application does not delete discovered
|
|
# SBOM rows - the install is still on the machine, it is just no longer
|
|
# managed.
|
|
appid = db.Column(
|
|
db.Integer,
|
|
db.ForeignKey('applications.appid', ondelete='SET NULL'),
|
|
nullable=True,
|
|
index=True,
|
|
)
|
|
|
|
application = db.relationship('Application', lazy='select', viewonly=True)
|
|
|
|
# When this snapshot row was written (the report that produced it).
|
|
recordedat = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
|
|
|
|
__table_args__ = (
|
|
db.Index('idx_endpointapplications_assetname', 'assetid', 'displayname'),
|
|
)
|
|
|
|
def __repr__(self):
|
|
return f"<EndpointApplication {self.displayname!r} asset={self.assetid}>"
|
|
|
|
def to_dict(self):
|
|
return {
|
|
'endpointapplicationid': self.endpointapplicationid,
|
|
'assetid': self.assetid,
|
|
'displayname': self.displayname,
|
|
'displayversion': self.displayversion,
|
|
'publisher': self.publisher,
|
|
'appid': self.appid,
|
|
'recordedat': _utciso(self.recordedat),
|
|
}
|