Endpoint telemetry, PC Detail rework and a checked backup/restore path

plugins/endpoints/migrations/versions/0012_endpoint_eventrecordid.py

@@ -0,0 +1,80 @@
+"""Dedup curated Windows events on the Windows event record id.
+
+Adds endpointevents.eventrecordid and a unique index over
+(assetid, eventchannel, eventrecordid).
+
+Why: the agent used to re-read the newest window of every curated channel on
+each cadence tick, and this ingest appended those rows without a dedup check, so
+one real event was stored once per report for as long as it stayed in that
+window - up to 96 identical rows a day. The agent now carries a per-channel
+watermark keyed on the Windows record id, which is monotonic and unique within a
+channel on a machine; this index is the server-side guarantee that survives an
+agent reinstall, a restored image, or two overlapping reports, none of which
+preserve the agent's watermark.
+
+Nullable on purpose: an agent older than this field reports no record id, and
+MySQL permits repeated NULLs inside a unique index, so those rows keep the
+previous append-only behavior instead of failing ingest.
+
+Existing rows are left alone. They all predate the column and so carry NULL,
+which cannot collide with the new index. Purging the historical duplicates is a
+separate, deliberate data cleanup - not something a schema migration should do
+irreversibly.
+
+Idempotent: both the column-add and the index-create check the live schema
+first, so a db.create_all() site and a re-run both stay clean on strict MySQL 8.
+"""
+from alembic import op
+
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'endpoints0012eventrecordid'
+down_revision = 'endpoints0011trayversion'
+branch_labels = None
+depends_on = None
+
+INDEX_NAME = 'idx_endpointevents_assetrecord'
+
+
+def _has_column(table_name, column_name):
+    from sqlalchemy import inspect
+
+    inspector = inspect(op.get_bind())
+    if not inspector.has_table(table_name):
+        return False
+    return any(col['name'] == column_name
+               for col in inspector.get_columns(table_name))
+
+
+def _has_index(table_name, index_name):
+    from sqlalchemy import inspect
+
+    inspector = inspect(op.get_bind())
+    if not inspector.has_table(table_name):
+        return False
+    return any(index['name'] == index_name
+               for index in inspector.get_indexes(table_name))
+
+
+def upgrade():
+    if not _has_column('endpointevents', 'eventrecordid'):
+        op.add_column(
+            'endpointevents',
+            sa.Column('eventrecordid', sa.BigInteger(), nullable=True))
+
+    if not _has_index('endpointevents', INDEX_NAME):
+        op.create_index(
+            INDEX_NAME,
+            'endpointevents',
+            ['assetid', 'eventchannel', 'eventrecordid'],
+            unique=True)
+
+
+def downgrade():
+    if _has_index('endpointevents', INDEX_NAME):
+        op.drop_index(INDEX_NAME, table_name='endpointevents')
+
+    if _has_column('endpointevents', 'eventrecordid'):
+        op.drop_column('endpointevents', 'eventrecordid')

plugins/endpoints/migrations/versions/0013_endpoint_inventory_columns.py

@@ -0,0 +1,93 @@
+"""Persist the hardware, OS, and session inventory the agent already reports.
+
+Adds to endpointstates: disksjson, biosjson, memoryjson, processorjson,
+osinfojson, usercontextjson, domain, and pendingreboot.
+
+Why now: the agent has been sending disks, bios, memory, processor, osinfo and
+domain on every report for several versions. The ingest logged them as unknown
+payload keys and threw them away, which was the intended staging step - the
+shapes are now proven against real endpoint traffic, so they get columns.
+usercontext and pendingreboot are new in this same change.
+
+Shape follows the existing servicesjson/portsjson convention: newest snapshot
+per endpoint as MEDIUMTEXT, not one row per sample. The agent resends the full
+structure every cycle and operators ask "what is in this PC now", not "what was
+in it in March". History would be pure write amplification on a fleet-sized
+table.
+
+domain and pendingreboot are real columns rather than JSON because they are
+single values the fleet view filters, groups, and alerts on; burying a scalar in
+a blob makes it unqueryable.
+
+All columns are nullable. Every existing row predates them, and there is no
+correct backfill value - a machine whose reboot state has never been reported is
+NOT the same as one confirmed to have no reboot pending, and inventing False
+would silently create a fleet of "healthy" endpoints.
+
+Idempotent: each column-add checks the live schema first, so a db.create_all()
+site and a re-run both stay clean on strict MySQL 8.
+"""
+from alembic import op
+
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'endpoints0013inventorycolumns'
+down_revision = 'endpoints0012eventrecordid'
+branch_labels = None
+depends_on = None
+
+TABLE_NAME = 'endpointstates'
+
+# MEDIUMTEXT. An installed-application or disk list on a busy engineering
+# workstation comfortably exceeds the 64KB TEXT ceiling, and a silent truncation
+# there would corrupt the JSON rather than merely shorten it.
+_SNAPSHOT_COLUMNS = (
+    'disksjson',
+    'biosjson',
+    'memoryjson',
+    'processorjson',
+    'osinfojson',
+    'usercontextjson',
+)
+
+
+def _has_column(table_name, column_name):
+    from sqlalchemy import inspect
+
+    inspector = inspect(op.get_bind())
+    if not inspector.has_table(table_name):
+        return False
+    return any(col['name'] == column_name
+               for col in inspector.get_columns(table_name))
+
+
+def upgrade():
+    for column_name in _SNAPSHOT_COLUMNS:
+        if not _has_column(TABLE_NAME, column_name):
+            op.add_column(
+                TABLE_NAME,
+                sa.Column(column_name, sa.Text(length=16777215), nullable=True))
+
+    if not _has_column(TABLE_NAME, 'domain'):
+        op.add_column(
+            TABLE_NAME,
+            sa.Column('domain', sa.String(length=255), nullable=True))
+
+    if not _has_column(TABLE_NAME, 'pendingreboot'):
+        op.add_column(
+            TABLE_NAME,
+            sa.Column('pendingreboot', sa.Boolean(), nullable=True))
+
+
+def downgrade():
+    if _has_column(TABLE_NAME, 'pendingreboot'):
+        op.drop_column(TABLE_NAME, 'pendingreboot')
+
+    if _has_column(TABLE_NAME, 'domain'):
+        op.drop_column(TABLE_NAME, 'domain')
+
+    for column_name in reversed(_SNAPSHOT_COLUMNS):
+        if _has_column(TABLE_NAME, column_name):
+            op.drop_column(TABLE_NAME, column_name)

plugins/endpoints/migrations/versions/0014_endpoint_diskhealth.py

@@ -0,0 +1,56 @@
+"""Add diskhealthjson to endpointstates.
+
+Per-drive health, kept separate from disksjson. That column answers "how full
+is this drive"; this one answers "is this drive dying". Merging them would mean
+a capacity collection failure and a health collection failure could not be told
+apart, and the two come from different WMI sources with different failure modes.
+
+JSON rather than scalar columns because a machine has a variable number of
+drives and each carries its own verdict, and because the agent reports the same
+drive from two sources - MSFT_PhysicalDisk for NVMe and Win32_DiskDrive for
+ATA/SATA - which the server keeps side by side rather than collapsing.
+
+MEDIUMTEXT for consistency with the other snapshot columns on this table.
+
+Nullable with no backfill. A drive whose health has never been reported is not
+a healthy drive, and defaulting to an empty list would read as "no disks have
+problems" on every endpoint that predates this column.
+
+Idempotent: the column-add checks the live schema first, so a re-run and a
+db.create_all() site both stay clean on strict MySQL 8.
+"""
+from alembic import op
+
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'endpoints0014diskhealth'
+down_revision = 'endpoints0013inventorycolumns'
+branch_labels = None
+depends_on = None
+
+TABLE_NAME = 'endpointstates'
+COLUMN_NAME = 'diskhealthjson'
+
+
+def _has_column(table_name, column_name):
+    from sqlalchemy import inspect
+
+    inspector = inspect(op.get_bind())
+    if not inspector.has_table(table_name):
+        return False
+    return any(col['name'] == column_name
+               for col in inspector.get_columns(table_name))
+
+
+def upgrade():
+    if not _has_column(TABLE_NAME, COLUMN_NAME):
+        op.add_column(
+            TABLE_NAME,
+            sa.Column(COLUMN_NAME, sa.Text(length=16777215), nullable=True))
+
+
+def downgrade():
+    if _has_column(TABLE_NAME, COLUMN_NAME):
+        op.drop_column(TABLE_NAME, COLUMN_NAME)

plugins/endpoints/migrations/versions/0015_endpoint_system_telemetry.py

@@ -0,0 +1,78 @@
+"""Add rebootreasonsjson, bootinfojson and defenderjson to endpointstates.
+
+Phase 2 system-side telemetry. All three are collect-and-persist; none is
+surfaced in the UI by this change.
+
+rebootreasonsjson is NOT redundant with the existing pendingreboot column. That
+column is a single boolean OR-ed from three registry signals, so it cannot say
+which one fired, and the difference is operationally significant: a
+PendingFileRenameOperations entry means files are staged for replacement on the
+next boot, so the machine is mid-servicing, whereas a lone Component Based
+Servicing or Windows Update key can sit set for weeks on a box nobody has
+rebooted. Same boolean, very different urgency.
+
+bootinfojson exists because uptime lies under fast startup - a machine reports
+days of uptime after what the operator experienced as a shutdown, and a "reboot"
+that was really a hybrid resume does not clear the driver or kernel state a
+support call is usually trying to clear.
+
+defenderjson is stored deliberately unsurfaced. Defender is known to produce
+false positives in this environment, so wiring it to a card now would train
+operators to ignore that card; persisting it starts the history accumulating for
+whenever someone is ready to interpret it.
+
+JSON rather than scalars: reboot reasons is a variable-length list, and the boot
+and Defender payloads are small related groups that are read together and would
+otherwise be six sparse columns.
+
+MEDIUMTEXT for consistency with the other snapshot columns on this table.
+
+Nullable with no backfill. Null means never reported, which is deliberately
+distinct from an empty list (checked, nothing pending) - defaulting the reboot
+reasons to [] would assert every pre-existing endpoint was confirmed clean.
+
+Idempotent: each column-add checks the live schema first, so a re-run and a
+db.create_all() site both stay clean on strict MySQL 8.
+"""
+from alembic import op
+
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'endpoints0015systemtelemetry'
+down_revision = 'endpoints0014diskhealth'
+branch_labels = None
+depends_on = None
+
+TABLE_NAME = 'endpointstates'
+
+_COLUMNS = (
+    'rebootreasonsjson',
+    'bootinfojson',
+    'defenderjson',
+)
+
+
+def _has_column(table_name, column_name):
+    from sqlalchemy import inspect
+
+    inspector = inspect(op.get_bind())
+    if not inspector.has_table(table_name):
+        return False
+    return any(col['name'] == column_name
+               for col in inspector.get_columns(table_name))
+
+
+def upgrade():
+    for column_name in _COLUMNS:
+        if not _has_column(TABLE_NAME, column_name):
+            op.add_column(
+                TABLE_NAME,
+                sa.Column(column_name, sa.Text(length=16777215), nullable=True))
+
+
+def downgrade():
+    for column_name in reversed(_COLUMNS):
+        if _has_column(TABLE_NAME, column_name):
+            op.drop_column(TABLE_NAME, column_name)

plugins/endpoints/migrations/versions/0016_endpoint_dellupdates.py

@@ -0,0 +1,51 @@
+"""Add dellupdatesjson to endpointstates.
+
+Stores the pending Dell Command | Update summary reported by the
+shopdb-dellupdates agent utility. MEDIUMTEXT for consistency with the other
+snapshot columns on this table.
+
+Nullable with no backfill. Null means the endpoint has never reported a Dell
+scan result, which is deliberately distinct from a stored payload whose status
+is 'ok' with an empty list (scanned, nothing pending). The status card renders
+those two states differently, so defaulting this to an empty summary would
+assert every pre-existing endpoint had been scanned and found clean.
+
+Idempotent: the column-add checks the live schema first, so a re-run and a
+db.create_all() site both stay clean on strict MySQL 8.
+"""
+from alembic import op
+
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'endpoints0016dellupdates'
+down_revision = 'endpoints0015systemtelemetry'
+branch_labels = None
+depends_on = None
+
+TABLE_NAME = 'endpointstates'
+
+COLUMN_NAME = 'dellupdatesjson'
+
+
+def _has_column(table_name, column_name):
+    from sqlalchemy import inspect
+
+    inspector = inspect(op.get_bind())
+    if not inspector.has_table(table_name):
+        return False
+    return any(col['name'] == column_name
+               for col in inspector.get_columns(table_name))
+
+
+def upgrade():
+    if not _has_column(TABLE_NAME, COLUMN_NAME):
+        op.add_column(
+            TABLE_NAME,
+            sa.Column(COLUMN_NAME, sa.Text(length=16777215), nullable=True))
+
+
+def downgrade():
+    if _has_column(TABLE_NAME, COLUMN_NAME):
+        op.drop_column(TABLE_NAME, COLUMN_NAME)