ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

log ignored agent collector keys for later UI enhancements

Browse Source 
The endpoint agent now sends disk, bios, memory, processor, osinfo, and domain telemetry. The server does not persist these yet, so ingest logs any unpersisted payload keys with a compact preview. This makes incoming telemetry observable in the logs before any UI or schema work, so we can confirm the agent is sending it and enhance the UI later.
This commit is contained in:
benjamin.baker
2026-08-29 17:00:49 -04:00
committed by cproudlock
parent 2a4b1bb040
commit 57845e1a9a
4 changed files with 262 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
agent/ShopDbEndpointAgent.Core/Contracts.cs
View File

@@ -29,6 +29,16 @@ public sealed class EndpointReport
[JsonPropertyName("interfaces")] public object[]? Interfaces { get; set; }
[JsonPropertyName("scheduledtasks")] public object[]? ScheduledTasks { get; set; }
[JsonPropertyName("events")] public EndpointEventRecord[]? Events { get; set; }
// Additive inventory telemetry. The server does not persist these yet; its
// ingest logs them as unknown keys (plugins/endpoints/plugin.py) so the
// incoming values are observable before any DB column or UI card exists.
// Keys stay lowercase/no-underscore to match AGENT-CONTRACT.md.
[JsonPropertyName("disks")] public object[]? Disks { get; set; }
[JsonPropertyName("bios")] public object? Bios { get; set; }
[JsonPropertyName("memory")] public object? Memory { get; set; }
[JsonPropertyName("processor")] public object? Processor { get; set; }
[JsonPropertyName("osinfo")] public object? OsInfo { get; set; }
[JsonPropertyName("domain")] public string? Domain { get; set; }
}
public sealed class EndpointEventRecord

194
agent/ShopDbEndpointAgent.Core/EndpointCollector.cs
View File

@@ -50,6 +50,12 @@ public sealed class EndpointCollector : ICollector
Services = GetMonitoredTargets(),
Interfaces = GetInterfaces(),
Events = GetCuratedEvents(),
Disks = GetDisks(),
Bios = GetBios(),
Memory = GetMemory(),
Processor = GetProcessor(),
OsInfo = GetOsInfo(),
Domain = GetDomain(),
};
if (report.LastBootTime is DateTimeOffset boot)
@@ -460,6 +466,194 @@ public sealed class EndpointCollector : ICollector
return records.ToArray();
}
// Fixed local disks with capacity/free so the server can flag a PC that is
// running out of space. Win32_LogicalDisk DriveType 3 = local fixed disk;
// network/removable drives are skipped. Bytes stay as longs; the server can
// derive percentages. Defensive: a locked WMI hive yields an empty list.
private object[] GetDisks()
{
var result = new List<object>();
try
{
using var searcher = new ManagementObjectSearcher(
"SELECT DeviceID, VolumeName, FileSystem, Size, FreeSpace " +
"FROM Win32_LogicalDisk WHERE DriveType = 3");
foreach (var mo in searcher.Get())
{
var size = ToLong(mo["Size"]);
var free = ToLong(mo["FreeSpace"]);
result.Add(new
{
drive = mo["DeviceID"]?.ToString()?.Trim() ?? "",
label = mo["VolumeName"]?.ToString()?.Trim() ?? "",
filesystem = mo["FileSystem"]?.ToString()?.Trim() ?? "",
sizebytes = size,
freebytes = free,
usedbytes = size is long total && free is long available
? total - available : (long?)null,
});
}
}
catch (Exception error)
{
logger.LogDebug(error, "Disk enumeration blocked");
}
return result.ToArray();
}
// Firmware identity from Win32_BIOS: vendor, version string, and release
// date. Lets the server track BIOS/UEFI levels for a fleet update campaign.
private object? GetBios()
{
try
{
using var searcher = new ManagementObjectSearcher(
"SELECT Manufacturer, SMBIOSBIOSVersion, ReleaseDate FROM Win32_BIOS");
foreach (var mo in searcher.Get())
{
return new
{
vendor = mo["Manufacturer"]?.ToString()?.Trim() ?? "",
version = mo["SMBIOSBIOSVersion"]?.ToString()?.Trim() ?? "",
releasedate = ToBiosDate(mo["ReleaseDate"]?.ToString()),
};
}
}
catch (Exception error)
{
logger.LogDebug(error, "BIOS query failed");
}
return null;
}
// Physical memory totals. TotalVisibleMemorySize/FreePhysicalMemory come
// from Win32_OperatingSystem in kilobytes; normalize to bytes so the wire
// stays in one unit with the disk figures.
private object? GetMemory()
{
try
{
using var searcher = new ManagementObjectSearcher(
"SELECT TotalVisibleMemorySize, FreePhysicalMemory " +
"FROM Win32_OperatingSystem");
foreach (var mo in searcher.Get())
{
var totalKb = ToLong(mo["TotalVisibleMemorySize"]);
var freeKb = ToLong(mo["FreePhysicalMemory"]);
return new
{
totalbytes = totalKb is long total ? total * 1024 : (long?)null,
freebytes = freeKb is long free ? free * 1024 : (long?)null,
};
}
}
catch (Exception error)
{
logger.LogDebug(error, "Memory query failed");
}
return null;
}
// CPU identity and core counts from Win32_Processor (first socket).
private object? GetProcessor()
{
try
{
using var searcher = new ManagementObjectSearcher(
"SELECT Name, NumberOfCores, NumberOfLogicalProcessors, " +
"MaxClockSpeed FROM Win32_Processor");
foreach (var mo in searcher.Get())
{
return new
{
name = mo["Name"]?.ToString()?.Trim() ?? "",
cores = ToLong(mo["NumberOfCores"]),
logicalprocessors = ToLong(mo["NumberOfLogicalProcessors"]),
maxclockmhz = ToLong(mo["MaxClockSpeed"]),
};
}
}
catch (Exception error)
{
logger.LogDebug(error, "Processor query failed");
}
return null;
}
// OS build/version detail beyond the caption already sent as osname. Lets
// the server track patch level (build number) across the fleet.
private object? GetOsInfo()
{
try
{
using var searcher = new ManagementObjectSearcher(
"SELECT Version, BuildNumber, OSArchitecture, InstallDate " +
"FROM Win32_OperatingSystem");
foreach (var mo in searcher.Get())
{
return new
{
version = mo["Version"]?.ToString()?.Trim() ?? "",
build = mo["BuildNumber"]?.ToString()?.Trim() ?? "",
architecture = mo["OSArchitecture"]?.ToString()?.Trim() ?? "",
installdate = ToBiosDate(mo["InstallDate"]?.ToString()),
};
}
}
catch (Exception error)
{
logger.LogDebug(error, "OS info query failed");
}
return null;
}
// AD domain or workgroup this PC is joined to. PartOfDomain distinguishes a
// domain-joined box from a workgroup one.
private string? GetDomain()
{
try
{
using var searcher = new ManagementObjectSearcher(
"SELECT Domain, PartOfDomain FROM Win32_ComputerSystem");
foreach (var mo in searcher.Get())
{
var domain = mo["Domain"]?.ToString()?.Trim();
if (!string.IsNullOrWhiteSpace(domain))
return domain;
}
}
catch (Exception error)
{
logger.LogDebug(error, "Domain query failed");
}
return null;
}
private static long? ToLong(object? value)
{
if (value is null)
return null;
try { return Convert.ToInt64(value); }
catch { return null; }
}
// Win32 CIM datetimes (yyyymmddHHMMSS.ffffff+zzz) render as an ISO-8601 UTC
// string; anything unparseable passes through untouched so nothing is lost.
private string? ToBiosDate(string? raw)
{
if (string.IsNullOrWhiteSpace(raw))
return null;
try
{
return ManagementDateTimeConverter.ToDateTime(raw)
.ToUniversalTime().ToString("o");
}
catch
{
return raw.Trim();
}
}
private string? QueryFirst(string wql, string property)
{
try

31
plugins/endpoints/plugin.py
View File

@@ -37,6 +37,21 @@ logger = logging.getLogger(__name__)
# of a raw event firehose.
CURATED_EVENT_IDS = frozenset(EVENT_CATALOG)
# Top-level payload keys the ingest path knows about today - either persisted on
# EndpointState, appended as events, or relayed to the computers collector.
# Anything the agent sends outside this set is not stored yet; it is logged (see
# apply_collector_payload) so new agent telemetry - disks, bios, memory, etc. -
# is observable in the logs before the server grows columns for it. Keep this in
# lockstep with the keys consumed in apply_collector_payload and the computers
# handoff subset.
KNOWN_PAYLOAD_KEYS = frozenset({
'hostname', 'adopt', 'loggedinuser', 'serialnumber', 'pctype',
'machinenumber', 'vendorname', 'modelnumber', 'osname', 'ipaddress',
'agentversion', 'lastboottime', 'uptimeseconds', 'idleseconds',
'cpupercent', 'memorypercent', 'services', 'ports', 'interfaces',
'scheduledtasks', 'events',
})
def _parse_dt(value):
"""Best-effort parse of an ISO-8601 string to naive UTC, or None."""
@@ -618,6 +633,22 @@ class EndpointsPlugin(BasePlugin):
state.lastreportedat = datetime.utcnow()
# Log any keys the agent sent that the ingest path does not consume yet.
# New agent telemetry (disks, bios, memory, ...) lands here first: it is
# not persisted until the server grows columns for it, but logging the
# keys and a compact preview makes incoming telemetry visible in the
# logs so we can confirm the agent is sending it before wiring the UI.
unknown_keys = [key for key in payload if key not in KNOWN_PAYLOAD_KEYS]
if unknown_keys:
preview = {key: payload.get(key) for key in sorted(unknown_keys)}
try:
preview_text = json.dumps(preview, default=str)[:2000]
except (TypeError, ValueError):
preview_text = repr(preview)[:2000]
logger.info(
'endpoint %s sent %d unpersisted telemetry key(s): %s | preview=%s',
hostname, len(unknown_keys), sorted(unknown_keys), preview_text)
stored_events = 0
for record in (payload.get('events') or []):
try:

27
plugins/endpoints/tests/test_plugin.py
View File

@@ -160,6 +160,33 @@ def test_collect_now_is_allowlisted_targetless_and_routine():
assert elevated_permission_for('collect_now') is None
def test_known_payload_keys_cover_persisted_and_relayed_fields():
"""The ingest allow-set must list every key the handler consumes today, so
those keys are never mislogged as 'unpersisted'. Anything the agent sends
outside this set is logged for observability before columns exist."""
from plugins.endpoints.plugin import KNOWN_PAYLOAD_KEYS
persisted_or_relayed = {
'hostname', 'adopt', 'loggedinuser', 'serialnumber', 'pctype',
'machinenumber', 'vendorname', 'modelnumber', 'osname', 'ipaddress',
'agentversion', 'lastboottime', 'uptimeseconds', 'idleseconds',
'cpupercent', 'memorypercent', 'services', 'ports', 'interfaces',
'scheduledtasks', 'events',
}
assert persisted_or_relayed <= KNOWN_PAYLOAD_KEYS
def test_new_hardware_telemetry_keys_are_logged_not_silently_known():
"""Disk/bios/memory inventory is collected at the agent before the server
persists it. Those keys must be OUTSIDE the known set so ingest logs them
rather than dropping them silently."""
from plugins.endpoints.plugin import KNOWN_PAYLOAD_KEYS
for key in ('disks', 'bios', 'memory', 'processor', 'osinfo', 'domain'):
assert key not in KNOWN_PAYLOAD_KEYS
def test_telemetrymeta_route_is_registered_for_freshness_gating():
"""The WebUI gates panel refreshes on lastreportedat advancing, so the
lightweight freshness probe must be exposed on the plugin blueprint."""