Report each Windows event once, and persist the inventory already sent
Curated events were re-read from the newest 25 entries per channel on every collect, and the server appended whatever arrived. One real event was therefore stored again on every cadence tick for as long as it stayed in that window, so event counts measured how long a box had been up rather than what had happened to it. The agent now keeps a per-channel watermark of the last Windows event record id it reported, persisted under ProgramData so a service restart does not replay history, and sends eventrecordid with each event. The server dedupes on (assetid, eventchannel, eventrecordid) behind a unique index, so a replay from an older agent cannot double-store either. Six inventory keys the agent had been sending all along - bios, disks, domain, memory, osinfo, processor - were parsed, logged as unpersisted, and dropped. They now have columns and are written on check-in, along with user context, pending reboot, and the cpupercent/memorypercent/ idleseconds fields that were declared in the contract but always null. Session-scoped data cannot be read from the agent's LocalSystem session 0 at all: mapped drives and user-owned listeners do not exist there. A probe runs in the interactive user's session and hands results back through a spool file. It is not yet producing output on a live endpoint; the column and panel are in place and read empty until that is fixed. Also bounds the user-context script wait. It was WaitForSingleObject with INFINITE, so a hung script pinned the command worker permanently.
This commit is contained in:
benjamin.baker
@@ -51,9 +51,44 @@ arrays. `adopt` is first-contact only.
|
||||
}
|
||||
],
|
||||
"scheduledtasks": [ /* opaque array */ ],
|
||||
"disks": [
|
||||
{
|
||||
"drive": "C:",
|
||||
"label": "OS",
|
||||
"filesystem": "NTFS",
|
||||
"sizebytes": 511000000000,
|
||||
"freebytes": 128000000000,
|
||||
"usedbytes": 383000000000,
|
||||
"freepercent": 25.0 // precomputed; a threshold is a direct compare
|
||||
}
|
||||
],
|
||||
"bios": { /* opaque object: vendor, version, release date */ },
|
||||
"memory": { /* opaque object: totalbytes, freebytes */ },
|
||||
"processor": { /* opaque object: name, cores, logicalprocessors, maxclockmhz */ },
|
||||
"osinfo": { /* opaque object: build, version, install date */ },
|
||||
"domain": "shopfloor.example.com", // domain or workgroup membership
|
||||
"pendingreboot": false, // omit when unknown; absent != false
|
||||
"usercontext": { // omitted entirely when nobody is signed in
|
||||
"netuse": [
|
||||
{
|
||||
"localname": "Z:",
|
||||
"remotename": "\\\\fileserver\\programs",
|
||||
"status": "OK",
|
||||
"username": "SHOPFLOOR\\operator",
|
||||
"persistent": true
|
||||
}
|
||||
],
|
||||
"userlisteners": [
|
||||
{ "localaddress": "0.0.0.0", "localport": 7878, "processid": 4512, "processname": "DncMain" }
|
||||
],
|
||||
"idleseconds": 45,
|
||||
"username": "operator",
|
||||
"collectedat": "2025-01-14T06:12:00Z"
|
||||
},
|
||||
"events": [
|
||||
{
|
||||
"eventid": 6008, // required per item; non-catalog IDs are dropped, not errored
|
||||
"eventrecordid": 184412, // Windows record id; dedupe key, see notes
|
||||
"eventchannel": "System",
|
||||
"eventsource": "EventLog",
|
||||
"eventtime": "2025-01-14T06:11:58Z",
|
||||
@@ -81,6 +116,31 @@ Notes:
|
||||
(`HKLM\SOFTWARE\[WOW6432Node\]GE Aircraft Engines\DNC\General:MachineNo`) or
|
||||
the UDC `udc_settings_*.json` (`GeneralSettings.MachineNumber`), in that order
|
||||
(DESIGN-DECISIONS-agent.md 9.4/9.5). `""` or `"9999"` means none.
|
||||
- `eventrecordid` is the Windows event record id: monotonic and unique within a
|
||||
channel on a machine. The agent uses it as a per-channel harvest watermark and
|
||||
the server enforces a unique index on
|
||||
`(assetid, eventchannel, eventrecordid)`, so one real event is stored exactly
|
||||
once. It is nullable: an older agent omits it, and MySQL permits repeated
|
||||
NULLs in a unique index, so those reports keep the previous append behavior
|
||||
rather than failing ingest. Send it whenever the source record exposes one.
|
||||
- `disks`/`bios`/`memory`/`processor`/`osinfo`/`usercontext` are stored verbatim
|
||||
as JSON snapshots on the endpoint state, newest-wins. `domain` and
|
||||
`pendingreboot` are scalar columns because the fleet view filters and alerts
|
||||
on them.
|
||||
- OMITTING a key is not the same as sending an empty value. Every one of these
|
||||
fields is applied only when the key is PRESENT, so a report that omits it
|
||||
preserves the last-known value. Send `null`/`[]` only when you mean "this is
|
||||
genuinely empty now"; omit when you simply did not collect it this cycle.
|
||||
- `pendingreboot` follows that rule strictly: omit it when the check failed. A
|
||||
literal `false` asserts the endpoint is confirmed NOT holding a reboot.
|
||||
- `usercontext` is gathered by a probe the agent runs inside the signed-in
|
||||
user's session, because mapped drives, user-owned listeners and idle time do
|
||||
not exist in the agent's own LocalSystem session 0. It is omitted when no one
|
||||
is signed in, which is a normal unattended state, not a fault.
|
||||
- `ports` carries LISTENING sockets only. The full connection table is available
|
||||
on demand via the `network_connections` command instead, because established
|
||||
connections churn constantly and would change every report for no diagnostic
|
||||
gain.
|
||||
|
||||
## 3. Response
|
||||
|
||||
@@ -137,13 +197,23 @@ public sealed class EndpointReport
|
||||
[JsonPropertyName("ports")] public object[]? Ports { get; set; }
|
||||
[JsonPropertyName("interfaces")] public object[]? Interfaces { get; set; }
|
||||
[JsonPropertyName("scheduledtasks")]public object[]? ScheduledTasks { get; set; }
|
||||
[JsonPropertyName("disks")] public object[]? Disks { get; set; }
|
||||
[JsonPropertyName("bios")] public object? Bios { get; set; }
|
||||
[JsonPropertyName("memory")] public object? Memory { get; set; }
|
||||
[JsonPropertyName("processor")] public object? Processor { get; set; }
|
||||
[JsonPropertyName("osinfo")] public object? OsInfo { get; set; }
|
||||
[JsonPropertyName("domain")] public string? Domain { get; set; }
|
||||
[JsonPropertyName("pendingreboot")] public bool? PendingReboot { get; set; }
|
||||
[JsonPropertyName("usercontext")] public object? UserContext { get; set; }
|
||||
[JsonPropertyName("applications")] public object[]? Applications { get; set; }
|
||||
[JsonPropertyName("events")] public EndpointEventRecord[]? Events { get; set; }
|
||||
}
|
||||
|
||||
public sealed class EndpointEventRecord
|
||||
{
|
||||
[JsonPropertyName("eventid")] public int EventId { get; set; }
|
||||
[JsonPropertyName("eventchannel")] public string? EventChannel { get; set; }
|
||||
[JsonPropertyName("eventid")] public int EventId { get; set; }
|
||||
[JsonPropertyName("eventrecordid")] public long? EventRecordId { get; set; }
|
||||
[JsonPropertyName("eventchannel")] public string? EventChannel { get; set; }
|
||||
[JsonPropertyName("eventsource")] public string? EventSource { get; set; }
|
||||
[JsonPropertyName("eventtime")] public DateTimeOffset? EventTime { get; set; }
|
||||
[JsonPropertyName("eventmessage")] public string? EventMessage { get; set; }
|
||||
@@ -230,6 +300,11 @@ Allowlisted actions (server source of truth: `plugins/endpoints/commands.py`):
|
||||
(`C:\ProgramData\UDC\MTConnect_UDC\Agent\run.bat`).
|
||||
- `mtconnect_stop` - stop the running MTConnect UDC agent process.
|
||||
- `mtconnect_status` - report whether the MTConnect UDC agent is running.
|
||||
- `network_connections` - return the full TCP connection table with owning
|
||||
process ids and names. Read-only diagnostic. The result is capped at 500
|
||||
connections with a `truncated` flag in `detail`, because an oversized payload
|
||||
rejected by the server returns nothing at all and a partial answer is more
|
||||
useful than a failed one. Standard reports carry listening ports only.
|
||||
|
||||
### 7.1 Claim the next command (long-poll)
|
||||
|
||||
|
||||
@@ -77,6 +77,12 @@ COMMAND_ALLOWLIST = {
|
||||
'label': 'Collect now',
|
||||
'description': 'Force the endpoint agent to report fresh telemetry off-cadence.',
|
||||
},
|
||||
'network_connections': {
|
||||
'label': 'Show network connections',
|
||||
'description': 'Return the full TCP connection table with owning processes. '
|
||||
'Read-only diagnostic; the standard report carries only '
|
||||
'listening ports because established connections churn.',
|
||||
},
|
||||
'run_script': {
|
||||
'label': 'Run script',
|
||||
'description': 'Run a ShopDB-hosted script on the endpoint. The agent '
|
||||
|
||||
@@ -24,7 +24,14 @@ EVENT_CATALOG = {
|
||||
55: ('File system corruption', 'critical', 'System'),
|
||||
7011: ('Service timeout', 'critical', 'System'),
|
||||
10016: ('DCOM permission error', 'warning', 'System'),
|
||||
# Windows Error Reporting trio on the Application channel. 1000 is the
|
||||
# faulting-application record (app name, version, faulting module, exception
|
||||
# code) and is the one that says what actually broke; 1001 is the WER bucket
|
||||
# record Windows writes alongside it; 1002 is a hang rather than a crash, so
|
||||
# it is a warning - the app is unresponsive, not gone.
|
||||
1000: ('Application error', 'critical', 'Application'),
|
||||
1001: ('Application crash', 'critical', 'Application'),
|
||||
1002: ('Application hang', 'warning', 'Application'),
|
||||
}
|
||||
|
||||
# The IDs a tech should see surfaced fleet-wide on the dashboard stability card.
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Dedup curated Windows events on the Windows event record id.
|
||||
|
||||
Adds endpointevents.eventrecordid and a unique index over
|
||||
(assetid, eventchannel, eventrecordid).
|
||||
|
||||
Why: the agent used to re-read the newest window of every curated channel on
|
||||
each cadence tick, and this ingest appended those rows without a dedup check, so
|
||||
one real event was stored once per report for as long as it stayed in that
|
||||
window - up to 96 identical rows a day. The agent now carries a per-channel
|
||||
watermark keyed on the Windows record id, which is monotonic and unique within a
|
||||
channel on a machine; this index is the server-side guarantee that survives an
|
||||
agent reinstall, a restored image, or two overlapping reports, none of which
|
||||
preserve the agent's watermark.
|
||||
|
||||
Nullable on purpose: an agent older than this field reports no record id, and
|
||||
MySQL permits repeated NULLs inside a unique index, so those rows keep the
|
||||
previous append-only behavior instead of failing ingest.
|
||||
|
||||
Existing rows are left alone. They all predate the column and so carry NULL,
|
||||
which cannot collide with the new index. Purging the historical duplicates is a
|
||||
separate, deliberate data cleanup - not something a schema migration should do
|
||||
irreversibly.
|
||||
|
||||
Idempotent: both the column-add and the index-create check the live schema
|
||||
first, so a db.create_all() site and a re-run both stay clean on strict MySQL 8.
|
||||
"""
|
||||
from alembic import op
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'endpoints0012eventrecordid'
|
||||
down_revision = 'endpoints0011trayversion'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
INDEX_NAME = 'idx_endpointevents_assetrecord'
|
||||
|
||||
|
||||
def _has_column(table_name, column_name):
|
||||
from sqlalchemy import inspect
|
||||
|
||||
inspector = inspect(op.get_bind())
|
||||
if not inspector.has_table(table_name):
|
||||
return False
|
||||
return any(col['name'] == column_name
|
||||
for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def _has_index(table_name, index_name):
|
||||
from sqlalchemy import inspect
|
||||
|
||||
inspector = inspect(op.get_bind())
|
||||
if not inspector.has_table(table_name):
|
||||
return False
|
||||
return any(index['name'] == index_name
|
||||
for index in inspector.get_indexes(table_name))
|
||||
|
||||
|
||||
def upgrade():
|
||||
if not _has_column('endpointevents', 'eventrecordid'):
|
||||
op.add_column(
|
||||
'endpointevents',
|
||||
sa.Column('eventrecordid', sa.BigInteger(), nullable=True))
|
||||
|
||||
if not _has_index('endpointevents', INDEX_NAME):
|
||||
op.create_index(
|
||||
INDEX_NAME,
|
||||
'endpointevents',
|
||||
['assetid', 'eventchannel', 'eventrecordid'],
|
||||
unique=True)
|
||||
|
||||
|
||||
def downgrade():
|
||||
if _has_index('endpointevents', INDEX_NAME):
|
||||
op.drop_index(INDEX_NAME, table_name='endpointevents')
|
||||
|
||||
if _has_column('endpointevents', 'eventrecordid'):
|
||||
op.drop_column('endpointevents', 'eventrecordid')
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Persist the hardware, OS, and session inventory the agent already reports.
|
||||
|
||||
Adds to endpointstates: disksjson, biosjson, memoryjson, processorjson,
|
||||
osinfojson, usercontextjson, domain, and pendingreboot.
|
||||
|
||||
Why now: the agent has been sending disks, bios, memory, processor, osinfo and
|
||||
domain on every report for several versions. The ingest logged them as unknown
|
||||
payload keys and threw them away, which was the intended staging step - the
|
||||
shapes are now proven against real endpoint traffic, so they get columns.
|
||||
usercontext and pendingreboot are new in this same change.
|
||||
|
||||
Shape follows the existing servicesjson/portsjson convention: newest snapshot
|
||||
per endpoint as MEDIUMTEXT, not one row per sample. The agent resends the full
|
||||
structure every cycle and operators ask "what is in this PC now", not "what was
|
||||
in it in March". History would be pure write amplification on a fleet-sized
|
||||
table.
|
||||
|
||||
domain and pendingreboot are real columns rather than JSON because they are
|
||||
single values the fleet view filters, groups, and alerts on; burying a scalar in
|
||||
a blob makes it unqueryable.
|
||||
|
||||
All columns are nullable. Every existing row predates them, and there is no
|
||||
correct backfill value - a machine whose reboot state has never been reported is
|
||||
NOT the same as one confirmed to have no reboot pending, and inventing False
|
||||
would silently create a fleet of "healthy" endpoints.
|
||||
|
||||
Idempotent: each column-add checks the live schema first, so a db.create_all()
|
||||
site and a re-run both stay clean on strict MySQL 8.
|
||||
"""
|
||||
from alembic import op
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'endpoints0013inventorycolumns'
|
||||
down_revision = 'endpoints0012eventrecordid'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
TABLE_NAME = 'endpointstates'
|
||||
|
||||
# MEDIUMTEXT. An installed-application or disk list on a busy engineering
|
||||
# workstation comfortably exceeds the 64KB TEXT ceiling, and a silent truncation
|
||||
# there would corrupt the JSON rather than merely shorten it.
|
||||
_SNAPSHOT_COLUMNS = (
|
||||
'disksjson',
|
||||
'biosjson',
|
||||
'memoryjson',
|
||||
'processorjson',
|
||||
'osinfojson',
|
||||
'usercontextjson',
|
||||
)
|
||||
|
||||
|
||||
def _has_column(table_name, column_name):
|
||||
from sqlalchemy import inspect
|
||||
|
||||
inspector = inspect(op.get_bind())
|
||||
if not inspector.has_table(table_name):
|
||||
return False
|
||||
return any(col['name'] == column_name
|
||||
for col in inspector.get_columns(table_name))
|
||||
|
||||
|
||||
def upgrade():
|
||||
for column_name in _SNAPSHOT_COLUMNS:
|
||||
if not _has_column(TABLE_NAME, column_name):
|
||||
op.add_column(
|
||||
TABLE_NAME,
|
||||
sa.Column(column_name, sa.Text(length=16777215), nullable=True))
|
||||
|
||||
if not _has_column(TABLE_NAME, 'domain'):
|
||||
op.add_column(
|
||||
TABLE_NAME,
|
||||
sa.Column('domain', sa.String(length=255), nullable=True))
|
||||
|
||||
if not _has_column(TABLE_NAME, 'pendingreboot'):
|
||||
op.add_column(
|
||||
TABLE_NAME,
|
||||
sa.Column('pendingreboot', sa.Boolean(), nullable=True))
|
||||
|
||||
|
||||
def downgrade():
|
||||
if _has_column(TABLE_NAME, 'pendingreboot'):
|
||||
op.drop_column(TABLE_NAME, 'pendingreboot')
|
||||
|
||||
if _has_column(TABLE_NAME, 'domain'):
|
||||
op.drop_column(TABLE_NAME, 'domain')
|
||||
|
||||
for column_name in reversed(_SNAPSHOT_COLUMNS):
|
||||
if _has_column(TABLE_NAME, column_name):
|
||||
op.drop_column(TABLE_NAME, column_name)
|
||||
@@ -80,6 +80,28 @@ class EndpointState(db.Model):
|
||||
interfacesjson = db.Column(db.Text(length=16777215), nullable=True)
|
||||
scheduledtasksjson = db.Column(db.Text(length=16777215), nullable=True)
|
||||
|
||||
# Hardware and OS inventory, same newest-snapshot rule as above. These
|
||||
# arrived on the wire well before this and were logged as unknown keys, so
|
||||
# the shapes are already proven against real endpoint traffic.
|
||||
disksjson = db.Column(db.Text(length=16777215), nullable=True)
|
||||
biosjson = db.Column(db.Text(length=16777215), nullable=True)
|
||||
memoryjson = db.Column(db.Text(length=16777215), nullable=True)
|
||||
processorjson = db.Column(db.Text(length=16777215), nullable=True)
|
||||
osinfojson = db.Column(db.Text(length=16777215), nullable=True)
|
||||
|
||||
# Session-scoped data collected by a probe running as the signed-in user
|
||||
# (mapped drives, user-owned listeners). Null when nobody is signed in,
|
||||
# which is a normal unattended state rather than a fault.
|
||||
usercontextjson = db.Column(db.Text(length=16777215), nullable=True)
|
||||
|
||||
# Windows domain or workgroup membership. A scalar, not JSON: it is a single
|
||||
# value the fleet view filters and groups on.
|
||||
domain = db.Column(db.String(255), nullable=True)
|
||||
|
||||
# True when the endpoint is holding a reboot. Null = never reported, which
|
||||
# is deliberately distinct from a confident False.
|
||||
pendingreboot = db.Column(db.Boolean, nullable=True)
|
||||
|
||||
# Warn-only reconciler output, refreshed on every agent check-in in the same
|
||||
# commit as servicesjson. Each entry names a policy/shipped-attached monitor
|
||||
# the endpoint is expected to be running but the latest snapshot does not
|
||||
@@ -116,6 +138,8 @@ class EndpointState(db.Model):
|
||||
'idleseconds': self.idleseconds,
|
||||
'cpupercent': self.cpupercent,
|
||||
'memorypercent': self.memorypercent,
|
||||
'domain': self.domain,
|
||||
'pendingreboot': self.pendingreboot,
|
||||
'mtconnectrunning': self.mtconnectrunning,
|
||||
'mtconnectcheckedat': _utciso(self.mtconnectcheckedat),
|
||||
'lastreportedat': _utciso(self.lastreportedat),
|
||||
@@ -255,6 +279,16 @@ class EndpointEvent(db.Model):
|
||||
eventchannel = db.Column(db.String(50), nullable=True)
|
||||
eventsource = db.Column(db.String(255), nullable=True)
|
||||
|
||||
# The Windows event record id: monotonic and unique within its channel on a
|
||||
# given machine. This is the dedup key. The agent also uses it as its
|
||||
# harvest watermark, so in normal operation a record is sent once; the
|
||||
# unique index below is what guarantees it even when an agent is reinstalled
|
||||
# (losing its watermark) or two reports overlap. Nullable because an agent
|
||||
# older than this field still reports, and MySQL permits repeated NULLs in a
|
||||
# unique index, so those degrade to the previous append-only behavior rather
|
||||
# than failing.
|
||||
eventrecordid = db.Column(db.BigInteger, nullable=True)
|
||||
|
||||
# When the event happened on the endpoint (from the record), not when it
|
||||
# reached ShopDB.
|
||||
eventtime = db.Column(db.DateTime, nullable=True, index=True)
|
||||
@@ -269,6 +303,8 @@ class EndpointEvent(db.Model):
|
||||
|
||||
__table_args__ = (
|
||||
db.Index('idx_endpointevents_assetevent', 'assetid', 'eventtime'),
|
||||
db.Index('idx_endpointevents_assetrecord',
|
||||
'assetid', 'eventchannel', 'eventrecordid', unique=True),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
@@ -279,6 +315,7 @@ class EndpointEvent(db.Model):
|
||||
'endpointeventid': self.endpointeventid,
|
||||
'assetid': self.assetid,
|
||||
'eventid': self.eventid,
|
||||
'eventrecordid': self.eventrecordid,
|
||||
'eventchannel': self.eventchannel,
|
||||
'eventsource': self.eventsource,
|
||||
'eventtime': _utciso(self.eventtime),
|
||||
|
||||
Reference in New Issue
Block a user