From a5ed364a46f4e0c96739bc2c9607b39532d2e4e0 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Fri, 28 Aug 2026 17:40:31 -0400 Subject: [PATCH 01/90] Resolve Fleet list route without a trailing slash The Fleet list is the only endpoints route on the bare blueprint prefix, so it registered as /api/endpoints/ while the SPA calls /api/endpoints. With strict_slashes on, Flask 308-redirects to add the slash; behind the /shopdb reverse proxy (IIS ARR) that redirect Location drops the path prefix, so the list call fails while suffix routes like /api/endpoints/ keep working. That is why a healthy, reporting endpoint showed on its PC detail page but never in Fleet. strict_slashes=False makes both URL forms resolve directly and removes the fragile redirect. --- plugins/endpoints/api/routes.py | 2 +- plugins/endpoints/tests/test_plugin.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/plugins/endpoints/api/routes.py b/plugins/endpoints/api/routes.py index 07355173..7865cadd 100644 --- a/plugins/endpoints/api/routes.py +++ b/plugins/endpoints/api/routes.py @@ -55,7 +55,7 @@ def _format_uptime(seconds): # Endpoint state # ============================================================================= -@endpoints_bp.route('/', methods=['GET']) +@endpoints_bp.route('/', methods=['GET'], strict_slashes=False) @require_permission('endpoints.view') def list_endpoints(): """List reported endpoint states. diff --git a/plugins/endpoints/tests/test_plugin.py b/plugins/endpoints/tests/test_plugin.py index 6001ab5f..423eb5dc 100644 --- a/plugins/endpoints/tests/test_plugin.py +++ b/plugins/endpoints/tests/test_plugin.py @@ -169,3 +169,25 @@ def test_telemetrymeta_route_is_registered_for_freshness_gating(): application.register_blueprint(EndpointsPlugin().get_blueprint()) rules = {rule.rule for rule in application.url_map.iter_rules()} assert any(rule.endswith('/asset//telemetrymeta') for rule in rules) + + +def test_fleet_list_route_resolves_without_trailing_slash(): + """The Fleet list route must match the slash-less URL the SPA calls. + + The list is the only route on the bare blueprint prefix. If it demands a + trailing slash, Flask 308-redirects /api/endpoints to /api/endpoints/, and + behind the /shopdb reverse proxy that redirect drops the path prefix and + the list call fails while the detail routes (which carry a path suffix) + keep working. strict_slashes=False makes both forms resolve directly. + """ + from flask import Flask + + application = Flask(__name__) + application.register_blueprint( + EndpointsPlugin().get_blueprint(), url_prefix='/api/endpoints') + adapter = application.url_map.bind('localhost') + + endpoint, _ = adapter.match('/api/endpoints', method='GET') + assert endpoint.endswith('list_endpoints') + endpoint_slash, _ = adapter.match('/api/endpoints/', method='GET') + assert endpoint_slash.endswith('list_endpoints') From d89682fd2756e50d8cee957c107d51957c549968 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Sat, 29 Aug 2026 17:00:49 -0400 Subject: [PATCH 02/90] log ignored agent collector keys for later UI enhancements The endpoint agent now sends disk, bios, memory, processor, osinfo, and domain telemetry. The server does not persist these yet, so ingest logs any unpersisted payload keys with a compact preview. This makes incoming telemetry observable in the logs before any UI or schema work, so we can confirm the agent is sending it and enhance the UI later. --- agent/ShopDbEndpointAgent.Core/Contracts.cs | 10 + .../EndpointCollector.cs | 194 ++++++++++++++++++ plugins/endpoints/plugin.py | 31 +++ plugins/endpoints/tests/test_plugin.py | 27 +++ 4 files changed, 262 insertions(+) diff --git a/agent/ShopDbEndpointAgent.Core/Contracts.cs b/agent/ShopDbEndpointAgent.Core/Contracts.cs index cb504a36..ad0e8592 100644 --- a/agent/ShopDbEndpointAgent.Core/Contracts.cs +++ b/agent/ShopDbEndpointAgent.Core/Contracts.cs @@ -29,6 +29,16 @@ public sealed class EndpointReport [JsonPropertyName("interfaces")] public object[]? Interfaces { get; set; } [JsonPropertyName("scheduledtasks")] public object[]? ScheduledTasks { get; set; } [JsonPropertyName("events")] public EndpointEventRecord[]? Events { get; set; } + // Additive inventory telemetry. The server does not persist these yet; its + // ingest logs them as unknown keys (plugins/endpoints/plugin.py) so the + // incoming values are observable before any DB column or UI card exists. + // Keys stay lowercase/no-underscore to match AGENT-CONTRACT.md. + [JsonPropertyName("disks")] public object[]? Disks { get; set; } + [JsonPropertyName("bios")] public object? Bios { get; set; } + [JsonPropertyName("memory")] public object? Memory { get; set; } + [JsonPropertyName("processor")] public object? Processor { get; set; } + [JsonPropertyName("osinfo")] public object? OsInfo { get; set; } + [JsonPropertyName("domain")] public string? Domain { get; set; } } public sealed class EndpointEventRecord diff --git a/agent/ShopDbEndpointAgent.Core/EndpointCollector.cs b/agent/ShopDbEndpointAgent.Core/EndpointCollector.cs index 4154db63..3f86c785 100644 --- a/agent/ShopDbEndpointAgent.Core/EndpointCollector.cs +++ b/agent/ShopDbEndpointAgent.Core/EndpointCollector.cs @@ -50,6 +50,12 @@ public sealed class EndpointCollector : ICollector Services = GetMonitoredTargets(), Interfaces = GetInterfaces(), Events = GetCuratedEvents(), + Disks = GetDisks(), + Bios = GetBios(), + Memory = GetMemory(), + Processor = GetProcessor(), + OsInfo = GetOsInfo(), + Domain = GetDomain(), }; if (report.LastBootTime is DateTimeOffset boot) @@ -460,6 +466,194 @@ public sealed class EndpointCollector : ICollector return records.ToArray(); } + // Fixed local disks with capacity/free so the server can flag a PC that is + // running out of space. Win32_LogicalDisk DriveType 3 = local fixed disk; + // network/removable drives are skipped. Bytes stay as longs; the server can + // derive percentages. Defensive: a locked WMI hive yields an empty list. + private object[] GetDisks() + { + var result = new List(); + try + { + using var searcher = new ManagementObjectSearcher( + "SELECT DeviceID, VolumeName, FileSystem, Size, FreeSpace " + + "FROM Win32_LogicalDisk WHERE DriveType = 3"); + foreach (var mo in searcher.Get()) + { + var size = ToLong(mo["Size"]); + var free = ToLong(mo["FreeSpace"]); + result.Add(new + { + drive = mo["DeviceID"]?.ToString()?.Trim() ?? "", + label = mo["VolumeName"]?.ToString()?.Trim() ?? "", + filesystem = mo["FileSystem"]?.ToString()?.Trim() ?? "", + sizebytes = size, + freebytes = free, + usedbytes = size is long total && free is long available + ? total - available : (long?)null, + }); + } + } + catch (Exception error) + { + logger.LogDebug(error, "Disk enumeration blocked"); + } + return result.ToArray(); + } + + // Firmware identity from Win32_BIOS: vendor, version string, and release + // date. Lets the server track BIOS/UEFI levels for a fleet update campaign. + private object? GetBios() + { + try + { + using var searcher = new ManagementObjectSearcher( + "SELECT Manufacturer, SMBIOSBIOSVersion, ReleaseDate FROM Win32_BIOS"); + foreach (var mo in searcher.Get()) + { + return new + { + vendor = mo["Manufacturer"]?.ToString()?.Trim() ?? "", + version = mo["SMBIOSBIOSVersion"]?.ToString()?.Trim() ?? "", + releasedate = ToBiosDate(mo["ReleaseDate"]?.ToString()), + }; + } + } + catch (Exception error) + { + logger.LogDebug(error, "BIOS query failed"); + } + return null; + } + + // Physical memory totals. TotalVisibleMemorySize/FreePhysicalMemory come + // from Win32_OperatingSystem in kilobytes; normalize to bytes so the wire + // stays in one unit with the disk figures. + private object? GetMemory() + { + try + { + using var searcher = new ManagementObjectSearcher( + "SELECT TotalVisibleMemorySize, FreePhysicalMemory " + + "FROM Win32_OperatingSystem"); + foreach (var mo in searcher.Get()) + { + var totalKb = ToLong(mo["TotalVisibleMemorySize"]); + var freeKb = ToLong(mo["FreePhysicalMemory"]); + return new + { + totalbytes = totalKb is long total ? total * 1024 : (long?)null, + freebytes = freeKb is long free ? free * 1024 : (long?)null, + }; + } + } + catch (Exception error) + { + logger.LogDebug(error, "Memory query failed"); + } + return null; + } + + // CPU identity and core counts from Win32_Processor (first socket). + private object? GetProcessor() + { + try + { + using var searcher = new ManagementObjectSearcher( + "SELECT Name, NumberOfCores, NumberOfLogicalProcessors, " + + "MaxClockSpeed FROM Win32_Processor"); + foreach (var mo in searcher.Get()) + { + return new + { + name = mo["Name"]?.ToString()?.Trim() ?? "", + cores = ToLong(mo["NumberOfCores"]), + logicalprocessors = ToLong(mo["NumberOfLogicalProcessors"]), + maxclockmhz = ToLong(mo["MaxClockSpeed"]), + }; + } + } + catch (Exception error) + { + logger.LogDebug(error, "Processor query failed"); + } + return null; + } + + // OS build/version detail beyond the caption already sent as osname. Lets + // the server track patch level (build number) across the fleet. + private object? GetOsInfo() + { + try + { + using var searcher = new ManagementObjectSearcher( + "SELECT Version, BuildNumber, OSArchitecture, InstallDate " + + "FROM Win32_OperatingSystem"); + foreach (var mo in searcher.Get()) + { + return new + { + version = mo["Version"]?.ToString()?.Trim() ?? "", + build = mo["BuildNumber"]?.ToString()?.Trim() ?? "", + architecture = mo["OSArchitecture"]?.ToString()?.Trim() ?? "", + installdate = ToBiosDate(mo["InstallDate"]?.ToString()), + }; + } + } + catch (Exception error) + { + logger.LogDebug(error, "OS info query failed"); + } + return null; + } + + // AD domain or workgroup this PC is joined to. PartOfDomain distinguishes a + // domain-joined box from a workgroup one. + private string? GetDomain() + { + try + { + using var searcher = new ManagementObjectSearcher( + "SELECT Domain, PartOfDomain FROM Win32_ComputerSystem"); + foreach (var mo in searcher.Get()) + { + var domain = mo["Domain"]?.ToString()?.Trim(); + if (!string.IsNullOrWhiteSpace(domain)) + return domain; + } + } + catch (Exception error) + { + logger.LogDebug(error, "Domain query failed"); + } + return null; + } + + private static long? ToLong(object? value) + { + if (value is null) + return null; + try { return Convert.ToInt64(value); } + catch { return null; } + } + + // Win32 CIM datetimes (yyyymmddHHMMSS.ffffff+zzz) render as an ISO-8601 UTC + // string; anything unparseable passes through untouched so nothing is lost. + private string? ToBiosDate(string? raw) + { + if (string.IsNullOrWhiteSpace(raw)) + return null; + try + { + return ManagementDateTimeConverter.ToDateTime(raw) + .ToUniversalTime().ToString("o"); + } + catch + { + return raw.Trim(); + } + } + private string? QueryFirst(string wql, string property) { try diff --git a/plugins/endpoints/plugin.py b/plugins/endpoints/plugin.py index ed012b37..8de0b382 100644 --- a/plugins/endpoints/plugin.py +++ b/plugins/endpoints/plugin.py @@ -37,6 +37,21 @@ logger = logging.getLogger(__name__) # of a raw event firehose. CURATED_EVENT_IDS = frozenset(EVENT_CATALOG) +# Top-level payload keys the ingest path knows about today - either persisted on +# EndpointState, appended as events, or relayed to the computers collector. +# Anything the agent sends outside this set is not stored yet; it is logged (see +# apply_collector_payload) so new agent telemetry - disks, bios, memory, etc. - +# is observable in the logs before the server grows columns for it. Keep this in +# lockstep with the keys consumed in apply_collector_payload and the computers +# handoff subset. +KNOWN_PAYLOAD_KEYS = frozenset({ + 'hostname', 'adopt', 'loggedinuser', 'serialnumber', 'pctype', + 'machinenumber', 'vendorname', 'modelnumber', 'osname', 'ipaddress', + 'agentversion', 'lastboottime', 'uptimeseconds', 'idleseconds', + 'cpupercent', 'memorypercent', 'services', 'ports', 'interfaces', + 'scheduledtasks', 'events', +}) + def _parse_dt(value): """Best-effort parse of an ISO-8601 string to naive UTC, or None.""" @@ -566,6 +581,22 @@ class EndpointsPlugin(BasePlugin): state.lastreportedat = datetime.utcnow() + # Log any keys the agent sent that the ingest path does not consume yet. + # New agent telemetry (disks, bios, memory, ...) lands here first: it is + # not persisted until the server grows columns for it, but logging the + # keys and a compact preview makes incoming telemetry visible in the + # logs so we can confirm the agent is sending it before wiring the UI. + unknown_keys = [key for key in payload if key not in KNOWN_PAYLOAD_KEYS] + if unknown_keys: + preview = {key: payload.get(key) for key in sorted(unknown_keys)} + try: + preview_text = json.dumps(preview, default=str)[:2000] + except (TypeError, ValueError): + preview_text = repr(preview)[:2000] + logger.info( + 'endpoint %s sent %d unpersisted telemetry key(s): %s | preview=%s', + hostname, len(unknown_keys), sorted(unknown_keys), preview_text) + stored_events = 0 for record in (payload.get('events') or []): try: diff --git a/plugins/endpoints/tests/test_plugin.py b/plugins/endpoints/tests/test_plugin.py index 423eb5dc..70ff9ead 100644 --- a/plugins/endpoints/tests/test_plugin.py +++ b/plugins/endpoints/tests/test_plugin.py @@ -160,6 +160,33 @@ def test_collect_now_is_allowlisted_targetless_and_routine(): assert elevated_permission_for('collect_now') is None +def test_known_payload_keys_cover_persisted_and_relayed_fields(): + """The ingest allow-set must list every key the handler consumes today, so + those keys are never mislogged as 'unpersisted'. Anything the agent sends + outside this set is logged for observability before columns exist.""" + from plugins.endpoints.plugin import KNOWN_PAYLOAD_KEYS + + persisted_or_relayed = { + 'hostname', 'adopt', 'loggedinuser', 'serialnumber', 'pctype', + 'machinenumber', 'vendorname', 'modelnumber', 'osname', 'ipaddress', + 'agentversion', 'lastboottime', 'uptimeseconds', 'idleseconds', + 'cpupercent', 'memorypercent', 'services', 'ports', 'interfaces', + 'scheduledtasks', 'events', + } + assert persisted_or_relayed <= KNOWN_PAYLOAD_KEYS + + +def test_new_hardware_telemetry_keys_are_logged_not_silently_known(): + """Disk/bios/memory inventory is collected at the agent before the server + persists it. Those keys must be OUTSIDE the known set so ingest logs them + rather than dropping them silently.""" + from plugins.endpoints.plugin import KNOWN_PAYLOAD_KEYS + + for key in ('disks', 'bios', 'memory', 'processor', 'osinfo', 'domain'): + assert key not in KNOWN_PAYLOAD_KEYS + + + def test_telemetrymeta_route_is_registered_for_freshness_gating(): """The WebUI gates panel refreshes on lastreportedat advancing, so the lightweight freshness probe must be exposed on the plugin blueprint.""" From b70cc1f6bea301dfd5276840a660ba6d8aea762e Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Sat, 29 Aug 2026 17:26:32 -0400 Subject: [PATCH 03/90] collect installed-software SBOM from endpoint agent The agent now reports installed applications from both Uninstall registry hives (64-bit and WOW6432Node) as {displayname, displayversion, publisher}, mirroring the Assess-Orchestrator applications collection. Registry-only, no Win32_Product/WMIC self-repair and no AppX. The server logs it as an unpersisted key until the SBOM store is built. --- agent/ShopDbEndpointAgent.Core/Contracts.cs | 4 ++ .../EndpointCollector.cs | 71 +++++++++++++++++++ plugins/endpoints/tests/test_plugin.py | 3 +- 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/agent/ShopDbEndpointAgent.Core/Contracts.cs b/agent/ShopDbEndpointAgent.Core/Contracts.cs index ad0e8592..e35af45b 100644 --- a/agent/ShopDbEndpointAgent.Core/Contracts.cs +++ b/agent/ShopDbEndpointAgent.Core/Contracts.cs @@ -39,6 +39,10 @@ public sealed class EndpointReport [JsonPropertyName("processor")] public object? Processor { get; set; } [JsonPropertyName("osinfo")] public object? OsInfo { get; set; } [JsonPropertyName("domain")] public string? Domain { get; set; } + // Installed-software SBOM, mirroring the Assess-Orchestrator applications + // collection: both Uninstall registry hives, {displayname, displayversion, + // publisher}. Server logs it as an unpersisted key until the SBOM store lands. + [JsonPropertyName("applications")] public object[]? Applications { get; set; } } public sealed class EndpointEventRecord diff --git a/agent/ShopDbEndpointAgent.Core/EndpointCollector.cs b/agent/ShopDbEndpointAgent.Core/EndpointCollector.cs index 3f86c785..c230d11b 100644 --- a/agent/ShopDbEndpointAgent.Core/EndpointCollector.cs +++ b/agent/ShopDbEndpointAgent.Core/EndpointCollector.cs @@ -56,6 +56,7 @@ public sealed class EndpointCollector : ICollector Processor = GetProcessor(), OsInfo = GetOsInfo(), Domain = GetDomain(), + Applications = GetApplications(), }; if (report.LastBootTime is DateTimeOffset boot) @@ -466,6 +467,76 @@ public sealed class EndpointCollector : ICollector return records.ToArray(); } + // Installed-software SBOM. Mirrors the Assess-Orchestrator applications + // collection exactly: enumerate both Uninstall registry hives (64-bit and + // WOW6432Node), keep only entries with a DisplayName, and emit + // {displayname, displayversion, publisher}. This is the same registry + // surface the machine-number/eDNC probes already read. Wire keys stay + // lowercase/no-underscore; the server logs this as an unpersisted key until + // the SBOM store is built. Never throws out to the caller. + private object[] GetApplications() + { + const string uninstall64 = + @"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall"; + const string uninstall32 = + @"SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall"; + + var result = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + CollectUninstallHive(uninstall64, result, seen); + CollectUninstallHive(uninstall32, result, seen); + return result.ToArray(); + } + + private void CollectUninstallHive(string subKey, List result, + HashSet seen) + { + try + { + using var machine = RegistryKey.OpenBaseKey( + RegistryHive.LocalMachine, RegistryView.Default); + using var uninstall = machine.OpenSubKey(subKey); + if (uninstall is null) + return; + + foreach (var name in uninstall.GetSubKeyNames()) + { + try + { + using var entry = uninstall.OpenSubKey(name); + var displayName = entry?.GetValue("DisplayName")?.ToString()?.Trim(); + if (string.IsNullOrWhiteSpace(displayName)) + continue; + + var displayVersion = + entry?.GetValue("DisplayVersion")?.ToString()?.Trim() ?? ""; + var publisher = + entry?.GetValue("Publisher")?.ToString()?.Trim() ?? ""; + + // De-dupe on name+version so the two hives do not double-list + // the same package. + if (!seen.Add(displayName + "\u0000" + displayVersion)) + continue; + + result.Add(new + { + displayname = displayName, + displayversion = displayVersion, + publisher, + }); + } + catch (Exception error) + { + logger.LogDebug(error, "Uninstall entry read blocked: {Entry}", name); + } + } + } + catch (Exception error) + { + logger.LogDebug(error, "Uninstall hive read blocked: {SubKey}", subKey); + } + } + // Fixed local disks with capacity/free so the server can flag a PC that is // running out of space. Win32_LogicalDisk DriveType 3 = local fixed disk; // network/removable drives are skipped. Bytes stay as longs; the server can diff --git a/plugins/endpoints/tests/test_plugin.py b/plugins/endpoints/tests/test_plugin.py index 70ff9ead..54373de8 100644 --- a/plugins/endpoints/tests/test_plugin.py +++ b/plugins/endpoints/tests/test_plugin.py @@ -182,7 +182,8 @@ def test_new_hardware_telemetry_keys_are_logged_not_silently_known(): rather than dropping them silently.""" from plugins.endpoints.plugin import KNOWN_PAYLOAD_KEYS - for key in ('disks', 'bios', 'memory', 'processor', 'osinfo', 'domain'): + for key in ('disks', 'bios', 'memory', 'processor', 'osinfo', 'domain', + 'applications'): assert key not in KNOWN_PAYLOAD_KEYS From 6ff7594996fd7d9a1090cc5bdda0f292c284b625 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Sun, 30 Aug 2026 08:31:31 -0400 Subject: [PATCH 04/90] Add agentless adoption tool for endpoints Standalone one-shot enrollment tool that creates a ShopDB PC + Endpoint entry using only the endpoints collector key, reusing the existing collector adopt path. Collects identity, WMI hardware, and installed-software SBOM; requires admin. Server-side packaging route attaches to the existing endpoints blueprint; no core contract change. --- agent/ShopDbAdoptionTool/Program.cs | 358 ++++++++++++++++++ .../ShopDbAdoptionTool.csproj | 30 ++ agent/ShopDbAdoptionTool/app.manifest | 20 + agent/ShopDbEndpointAgent.slnx | 1 + plugins/endpoints/ADOPTION-TOOL-SPIKE.md | 77 ++++ plugins/endpoints/api/__init__.py | 2 + plugins/endpoints/api/adoption.py | 107 ++++++ .../frontend/views/AgentKeySettings.vue | 90 +++++ 8 files changed, 685 insertions(+) create mode 100644 agent/ShopDbAdoptionTool/Program.cs create mode 100644 agent/ShopDbAdoptionTool/ShopDbAdoptionTool.csproj create mode 100644 agent/ShopDbAdoptionTool/app.manifest create mode 100644 plugins/endpoints/ADOPTION-TOOL-SPIKE.md create mode 100644 plugins/endpoints/api/adoption.py diff --git a/agent/ShopDbAdoptionTool/Program.cs b/agent/ShopDbAdoptionTool/Program.cs new file mode 100644 index 00000000..c6126710 --- /dev/null +++ b/agent/ShopDbAdoptionTool/Program.cs @@ -0,0 +1,358 @@ +using System.Management; +using System.Net.Http.Json; +using System.Net.NetworkInformation; +using System.Runtime.Versioning; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace ShopDbAdoptionTool; + +// ShopDB adoption tool (SPIKE). A one-shot, agentless enrollment client: it uses +// only the endpoints collector key to create a ShopDB PC (Computer) + Endpoint +// entry via the existing collector adopt path, then exits. It is NOT the endpoint +// agent - it installs nothing, runs no service, and reports no ongoing telemetry. +// For sites that want an inventory entry without running the full agent. +// +// Configuration comes from adopt.config.json beside the exe, or from +// --baseurl / --collectorkey / --hostname arguments (arguments win). The +// collector key is never written to disk by this tool; it is read from the +// config the operator generated and placed next to the exe. +[SupportedOSPlatform("windows")] +internal static class Program +{ + private const string ReportPath = "/api/collector/endpoints"; + + private static async Task Main(string[] args) + { + try + { + var config = LoadConfig(args); + if (string.IsNullOrWhiteSpace(config.BaseUrl) || + string.IsNullOrWhiteSpace(config.CollectorKey)) + { + Console.Error.WriteLine( + "Missing baseurl or collectorkey. Provide adopt.config.json next to the " + + "exe or pass --baseurl and --collectorkey."); + return 2; + } + + // The app manifest forces UAC elevation, so this is a belt-and-braces + // guard for edge cases (for example a stripped manifest). Full WMI and + // the machine Uninstall hives need admin, or collection is partial. + if (!IsElevated()) + { + Console.Error.WriteLine( + "This tool must run as administrator so hardware and installed-" + + "software collection succeeds. Right-click and Run as administrator."); + return 3; + } + + var payload = BuildPayload(config); + Console.WriteLine($"Adopting '{payload.Hostname}' into {config.BaseUrl} ..."); + Console.WriteLine( + $"Collected {payload.Applications?.Length ?? 0} installed applications."); + + using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(60) }; + client.DefaultRequestHeaders.Add("X-API-Key", config.CollectorKey); + + var url = config.BaseUrl!.TrimEnd('/') + ReportPath; + using var response = await client.PostAsJsonAsync(url, payload); + var body = await response.Content.ReadAsStringAsync(); + + if (!response.IsSuccessStatusCode) + { + Console.Error.WriteLine( + $"Adoption failed: HTTP {(int)response.StatusCode}. {body}"); + return 1; + } + + Console.WriteLine("Adoption succeeded."); + Console.WriteLine(body); + return 0; + } + catch (Exception error) + { + Console.Error.WriteLine($"Adoption error: {error.Message}"); + return 1; + } + } + + private static AdoptConfig LoadConfig(string[] args) + { + var config = new AdoptConfig(); + + var configPath = Path.Combine(AppContext.BaseDirectory, "adopt.config.json"); + if (File.Exists(configPath)) + { + var fromFile = JsonSerializer.Deserialize( + File.ReadAllText(configPath), + new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + if (fromFile is not null) + config = fromFile; + } + + for (var i = 0; i < args.Length - 1; i++) + { + var value = args[i + 1]; + switch (args[i].ToLowerInvariant()) + { + case "--baseurl": + config.BaseUrl = value; + break; + case "--collectorkey": + config.CollectorKey = value; + break; + case "--hostname": + config.Hostname = value; + break; + } + } + + return config; + } + + private static AdoptPayload BuildPayload(AdoptConfig config) + { + var hostname = !string.IsNullOrWhiteSpace(config.Hostname) + ? config.Hostname! + : Environment.MachineName; + + return new AdoptPayload + { + Hostname = hostname, + Adopt = true, + LoggedInUser = GetLoggedInUser(), + SerialNumber = WmiFirst("SELECT SerialNumber FROM Win32_BIOS", "SerialNumber"), + PcType = WmiFirst("SELECT Model FROM Win32_ComputerSystem", "Model"), + VendorName = WmiFirst("SELECT Manufacturer FROM Win32_ComputerSystem", "Manufacturer"), + ModelNumber = GetModelNumber(), + OsName = WmiFirst("SELECT Caption FROM Win32_OperatingSystem", "Caption"), + IpAddress = ResolveIpAddress(), + LastBootTime = GetLastBootTime(), + Applications = GetApplications(), + AgentVersion = "adopt-0.1.0", + }; + } + + // Mirrors the endpoint agent: prefer the WMI logged-on user, fall back to + // the process identity so the field is never blank. + private static string GetLoggedInUser() + { + var user = WmiFirst("SELECT UserName FROM Win32_ComputerSystem", "UserName"); + return string.IsNullOrWhiteSpace(user) + ? $"{Environment.UserDomainName}\\{Environment.UserName}" + : user; + } + + // Human-readable model first (what pctype classifies on); SystemSKUNumber is + // an OEM SKU code, used only when Model is blank. Matches EndpointCollector. + private static string? GetModelNumber() + { + var model = WmiFirst("SELECT Model FROM Win32_ComputerSystem", "Model"); + if (!string.IsNullOrWhiteSpace(model)) + return model; + return WmiFirst("SELECT SystemSKUNumber FROM Win32_ComputerSystem", + "SystemSKUNumber"); + } + + private static DateTimeOffset? GetLastBootTime() + { + var raw = WmiFirst("SELECT LastBootUpTime FROM Win32_OperatingSystem", + "LastBootUpTime"); + if (string.IsNullOrWhiteSpace(raw)) + return null; + try + { + return ManagementDateTimeConverter.ToDateTime(raw).ToUniversalTime(); + } + catch + { + return null; + } + } + + // Run one WMI query and return the first row's column as a trimmed string, + // or null. Best-effort: WMI failures never abort adoption. + private static string? WmiFirst(string query, string column) + { + try + { + using var searcher = new ManagementObjectSearcher(query); + foreach (var item in searcher.Get()) + { + var value = item[column]?.ToString()?.Trim(); + if (!string.IsNullOrWhiteSpace(value)) + return value; + } + } + catch + { + // Best-effort: the server treats every hardware field as optional. + } + return null; + } + + // Installed-software SBOM. Mirrors the endpoint agent exactly: enumerate both + // Uninstall registry hives (64-bit and WOW6432Node), keep entries with a + // DisplayName, de-dupe on name+version, and emit {displayname, displayversion, + // publisher}. Requires admin to read the machine hives reliably (see the + // app manifest). Never throws out to the caller. + private static object[] GetApplications() + { + const string uninstall64 = + @"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall"; + const string uninstall32 = + @"SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall"; + + var result = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + CollectUninstallHive(uninstall64, result, seen); + CollectUninstallHive(uninstall32, result, seen); + return result.ToArray(); + } + + private static void CollectUninstallHive(string subKey, List result, + HashSet seen) + { + try + { + using var machine = Microsoft.Win32.RegistryKey.OpenBaseKey( + Microsoft.Win32.RegistryHive.LocalMachine, + Microsoft.Win32.RegistryView.Default); + using var uninstall = machine.OpenSubKey(subKey); + if (uninstall is null) + return; + + foreach (var name in uninstall.GetSubKeyNames()) + { + try + { + using var entry = uninstall.OpenSubKey(name); + var displayName = entry?.GetValue("DisplayName")?.ToString()?.Trim(); + if (string.IsNullOrWhiteSpace(displayName)) + continue; + + var displayVersion = + entry?.GetValue("DisplayVersion")?.ToString()?.Trim() ?? ""; + var publisher = + entry?.GetValue("Publisher")?.ToString()?.Trim() ?? ""; + + if (!seen.Add(displayName + "\u0000" + displayVersion)) + continue; + + result.Add(new + { + displayname = displayName, + displayversion = displayVersion, + publisher, + }); + } + catch + { + // Skip unreadable entries; keep collecting the rest. + } + } + } + catch + { + // Skip an unreadable hive; the other hive may still return data. + } + } + + private static string? ResolveIpAddress() + { + try + { + foreach (var network in NetworkInterface.GetAllNetworkInterfaces()) + { + if (network.OperationalStatus != OperationalStatus.Up) + continue; + if (network.NetworkInterfaceType == NetworkInterfaceType.Loopback) + continue; + foreach (var address in network.GetIPProperties().UnicastAddresses) + { + if (address.Address.AddressFamily == + System.Net.Sockets.AddressFamily.InterNetwork) + return address.Address.ToString(); + } + } + } + catch + { + // Best-effort: the server treats ipaddress as optional. + } + return null; + } + + // True when the process runs with the Administrators group in its token. + private static bool IsElevated() + { + try + { + using var identity = System.Security.Principal.WindowsIdentity.GetCurrent(); + var principal = new System.Security.Principal.WindowsPrincipal(identity); + return principal.IsInRole( + System.Security.Principal.WindowsBuiltInRole.Administrator); + } + catch + { + return false; + } + } +} + +internal sealed class AdoptConfig +{ + [JsonPropertyName("baseurl")] + public string? BaseUrl { get; set; } + + [JsonPropertyName("collectorkey")] + public string? CollectorKey { get; set; } + + [JsonPropertyName("hostname")] + public string? Hostname { get; set; } +} + +// Payload keys mirror the endpoints collector adopt schema exactly (lowercase, +// concatenated) so the server stores them without translation. This tool sends +// the adopt identity plus the hardware and SBOM subset the endpoint agent +// collects; ongoing health telemetry (cpu, services, events) is intentionally +// absent because this is agentless enrollment, not the agent. +internal sealed class AdoptPayload +{ + [JsonPropertyName("hostname")] + public string Hostname { get; set; } = ""; + + [JsonPropertyName("adopt")] + public bool Adopt { get; set; } + + [JsonPropertyName("loggedinuser")] + public string? LoggedInUser { get; set; } + + [JsonPropertyName("serialnumber")] + public string? SerialNumber { get; set; } + + [JsonPropertyName("pctype")] + public string? PcType { get; set; } + + [JsonPropertyName("vendorname")] + public string? VendorName { get; set; } + + [JsonPropertyName("modelnumber")] + public string? ModelNumber { get; set; } + + [JsonPropertyName("osname")] + public string? OsName { get; set; } + + [JsonPropertyName("ipaddress")] + public string? IpAddress { get; set; } + + [JsonPropertyName("lastboottime")] + public DateTimeOffset? LastBootTime { get; set; } + + [JsonPropertyName("applications")] + public object[]? Applications { get; set; } + + [JsonPropertyName("agentversion")] + public string? AgentVersion { get; set; } +} diff --git a/agent/ShopDbAdoptionTool/ShopDbAdoptionTool.csproj b/agent/ShopDbAdoptionTool/ShopDbAdoptionTool.csproj new file mode 100644 index 00000000..ad644831 --- /dev/null +++ b/agent/ShopDbAdoptionTool/ShopDbAdoptionTool.csproj @@ -0,0 +1,30 @@ + + + + Exe + net8.0-windows + enable + enable + ShopDbAdoptionTool + shopdb-adopt + + 0.1.0 + true + win-x64 + true + true + true + true + none + app.manifest + + + + + + + diff --git a/agent/ShopDbAdoptionTool/app.manifest b/agent/ShopDbAdoptionTool/app.manifest new file mode 100644 index 00000000..3691a666 --- /dev/null +++ b/agent/ShopDbAdoptionTool/app.manifest @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + diff --git a/agent/ShopDbEndpointAgent.slnx b/agent/ShopDbEndpointAgent.slnx index 1383261a..e58da209 100644 --- a/agent/ShopDbEndpointAgent.slnx +++ b/agent/ShopDbEndpointAgent.slnx @@ -2,4 +2,5 @@ + diff --git a/plugins/endpoints/ADOPTION-TOOL-SPIKE.md b/plugins/endpoints/ADOPTION-TOOL-SPIKE.md new file mode 100644 index 00000000..448c9f38 --- /dev/null +++ b/plugins/endpoints/ADOPTION-TOOL-SPIKE.md @@ -0,0 +1,77 @@ +# Adoption tool spike + +Status: spike. Built on the `origin/main` baseline so it can be cherry-picked as +a single unit once endpoint agent development is complete. + +## What it is + +A one-shot, agentless enrollment tool. An admin generates a small download from +the Endpoints key settings page; running the tool once on a PC creates a ShopDB +PC (Computer) and Endpoint entry using only the endpoints collector key. It is +NOT the endpoint agent: it installs nothing, runs no service, and sends no +ongoing telemetry. It targets sites that want an inventory entry without running +the full agent. + +## How it works + +1. Admin generates or reuses an endpoint agent key (existing `/agentkey` flow). +2. Admin opens the "Adoption tool" section, confirms the base URL (prefilled + from the browser origin) and pastes the collector key (prefilled when a key + was just generated, since the secret is only shown once). +3. `POST /api/endpoints/adoptiontool/package` (admin-gated) returns a ZIP with + `adopt.config.json`, `README.txt`, and `shopdb-adopt.exe` when a prebuilt + binary is staged on the server. +4. On the target PC, `shopdb-adopt.exe` reads the config beside it (or CLI args), + then POSTs to `/api/collector/endpoints` with `X-API-Key` and `adopt=true`. + The existing `_adopt_computer` path creates the Computer + EndpointState. + +## Collection + +The tool gathers the same identity, hardware, and SBOM subset the endpoint +agent collects, using identical WMI queries and wire keys so the server stores +it without translation: + +- Identity: `hostname`, `loggedinuser`, `ipaddress`. +- Hardware: `serialnumber` (Win32_BIOS), `vendorname` / `pctype` / `modelnumber` + (Win32_ComputerSystem), `osname` and `lastboottime` (Win32_OperatingSystem). +- SBOM: `applications` = both Uninstall registry hives (64-bit + WOW6432Node), + de-duped `{displayname, displayversion, publisher}`. The server logs this as + an unpersisted key today (same as the agent) until the SBOM store lands. + +The tool requires administrator rights (enforced by its app manifest +`requireAdministrator`, with a runtime elevation guard) because the machine +Uninstall hives and full WMI need elevation, or collection would be silently +partial. + +## Cherry-pick boundary + +The spike is deliberately isolated. Cherry-pick exactly: + +- `agent/ShopDbAdoptionTool/` (new standalone tool project, incl. `app.manifest`) +- `agent/ShopDbEndpointAgent.slnx` (one line adding the project) +- `plugins/endpoints/api/adoption.py` (new module; attaches to `endpoints_bp`) +- `plugins/endpoints/api/__init__.py` (one-line import to register the route) +- `plugins/endpoints/frontend/views/AgentKeySettings.vue` (new UI section only) +- `plugins/endpoints/ADOPTION-TOOL-SPIKE.md` (this file) + +`plugins/endpoints/api/routes.py` is intentionally untouched so the delivery +spike changes there do not tangle with this feature. + +## Prebuilt exe distribution + +The api container has no .NET SDK, so the package embeds the exe only when a +prebuilt copy is staged at `instance/adoptiontool/shopdb-adopt.exe`. Otherwise +the ZIP ships the config plus README (build the exe out of band, same model as +the endpoint agent installer bundle). Follow-up: a CI job that publishes +`shopdb-adopt.exe` and stages it into the image or instance path. + +## Known follow-ups + +- Agentless health signal: an adopted PC never reports again, so + `EndpointState.lastreportedat` reflects only the adoption moment. The fleet + health/dashboard logic should distinguish "agentless, adopted" from + "agent expected but silent" so these PCs do not read as permanently failed. + Left out of this spike to keep the server create-path unchanged for a clean + cherry-pick; the tool does not send an `agentless` marker yet because the + server does not persist one (`KNOWN_PAYLOAD_KEYS`). +- CI packaging of the prebuilt exe (see above). diff --git a/plugins/endpoints/api/__init__.py b/plugins/endpoints/api/__init__.py index 6ad50266..e4f289bc 100644 --- a/plugins/endpoints/api/__init__.py +++ b/plugins/endpoints/api/__init__.py @@ -1,5 +1,7 @@ """Endpoints plugin API.""" from .routes import endpoints_bp +# Attaches the adoption-tool packaging route to endpoints_bp (spike). +from . import adoption # noqa: F401 __all__ = ['endpoints_bp'] diff --git a/plugins/endpoints/api/adoption.py b/plugins/endpoints/api/adoption.py new file mode 100644 index 00000000..8e6e5967 --- /dev/null +++ b/plugins/endpoints/api/adoption.py @@ -0,0 +1,107 @@ +"""Endpoints adoption-tool packaging (SPIKE). + +This module is deliberately self-contained so it can be cherry-picked onto +main as a single unit. It attaches ONE admin-gated route to the existing +endpoints blueprint and never edits routes.py. + +The adoption tool is NOT the endpoint agent. It is a one-shot, agentless +enrollment client: given only the endpoints collector key and this server's +base URL, it creates a ShopDB PC (Computer) + Endpoint entry via the existing +POST /api/collector/endpoints adopt path, then exits. This route packages that +tool for download: a ZIP carrying the operator-supplied config, a README, and +(when a prebuilt binary is staged on the server) the shopdb-adopt.exe. + +The collector key is supplied by the admin and is NOT stored or logged here. +The minted secret is unretrievable after generation, so the operator pastes it +into the generator; it only lives inside the downloaded config file. +""" + +import io +import json +import os +import zipfile + +from flask import request, current_app, send_file + +from shopdb.api import error_response, ErrorCodes, require_role + +from .routes import endpoints_bp + +# A staged, prebuilt shopdb-adopt.exe is embedded when present. The container +# has no .NET SDK, so this mirrors the endpoint agent installer model: build the +# exe out of band, drop it under instance/adoptiontool/, and it ships in the ZIP. +STAGED_EXE_RELPATH = os.path.join('adoptiontool', 'shopdb-adopt.exe') + +_README = """ShopDB adoption tool (spike) + +WHAT THIS IS + A one-shot, agentless enrollment tool. Run it once on a PC and it creates a + ShopDB PC and Endpoint entry using only the collector key in the config file + next to this exe. It collects identity, hardware, and installed-software (SBOM) + details, then exits. It installs nothing, runs no service, and sends no + ongoing telemetry. It is NOT the endpoint agent. + +FILES + shopdb-adopt.exe The tool (present only if the server had a prebuilt copy). + adopt.config.json The base URL and collector key this tool will use. + +USAGE + 1. Keep shopdb-adopt.exe and adopt.config.json in the same folder. + 2. Right-click shopdb-adopt.exe and choose Run as administrator. Elevation is + required so hardware and installed-software collection succeeds. + 3. It reports the machine to ShopDB and exits 0 on success. + + You can override config values with arguments: + shopdb-adopt.exe --baseurl https://shopdb.example --collectorkey KEY + +SECURITY + adopt.config.json contains a live collector key. Treat it as a secret. The + key is not recoverable from the ShopDB server after it was generated, so keep + this download safe or regenerate the key to revoke it. +""" + + +def _build_readme(): + return _README + + +@endpoints_bp.route('/adoptiontool/package', methods=['POST']) +@require_role('admin') +def package_adoption_tool(): + """Bundle the adoption tool as a downloadable ZIP. + + Body: {"baseurl": "...", "collectorkey": "..."}. Returns an in-memory ZIP + with adopt.config.json, README.txt, and the staged exe when available. The + collector key is embedded in the config only; it is never stored or logged. + """ + body = request.get_json(silent=True) or {} + baseurl = (body.get('baseurl') or '').strip() + collectorkey = (body.get('collectorkey') or '').strip() + + if not baseurl or not collectorkey: + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'baseurl and collectorkey are required', + http_code=400) + + config = { + 'baseurl': baseurl.rstrip('/'), + 'collectorkey': collectorkey, + } + + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED) as archive: + archive.writestr('adopt.config.json', + json.dumps(config, indent=2) + '\n') + archive.writestr('README.txt', _build_readme()) + + staged = os.path.join(current_app.instance_path, STAGED_EXE_RELPATH) + if os.path.isfile(staged): + archive.write(staged, 'shopdb-adopt.exe') + + buffer.seek(0) + return send_file( + buffer, + mimetype='application/zip', + as_attachment=True, + download_name='shopdb-adoption-tool.zip') diff --git a/plugins/endpoints/frontend/views/AgentKeySettings.vue b/plugins/endpoints/frontend/views/AgentKeySettings.vue index 8e67d7e5..f2aadb1c 100644 --- a/plugins/endpoints/frontend/views/AgentKeySettings.vue +++ b/plugins/endpoints/frontend/views/AgentKeySettings.vue @@ -138,6 +138,57 @@ + +
+

Adoption tool

+
+

+ Generate a one-shot, agentless enrollment tool. Run it once on a PC and + it creates a ShopDB PC and Endpoint entry using only the collector key - + no service, no polling, no ongoing telemetry. Use this for machines you + want inventoried without running the full endpoint agent. +

+

+ Paste a collector key below (generate one above first; the secret is + only shown once). The key is embedded in the downloaded config and is + not stored on the server. +

+ + + + + + + +
+ +
+

{{ adoptmessage }}

+

{{ adopterror }}

+
+
+ @@ -197,6 +248,13 @@ const copied = ref(false) const message = ref('') const error = ref('') +// Adoption tool (spike): build a one-shot, agentless enrollment download. +const adoptbaseurl = ref('') +const adoptcollectorkey = ref('') +const adoptbusy = ref(false) +const adoptmessage = ref('') +const adopterror = ref('') + function formatDate(value) { if (!value) return '' const parsed = new Date(value) @@ -209,6 +267,7 @@ function applyStatus(data) { } onMounted(async () => { + adoptbaseurl.value = window.location.origin try { const response = await api.get('/endpoints/agentkey') applyStatus(response.data.data || {}) @@ -238,6 +297,9 @@ async function regenerate() { const data = response.data.data || {} applyStatus(data) secret.value = data.secret || '' + if (data.secret) { + adoptcollectorkey.value = data.secret + } message.value = 'Endpoint agent key generated.' } catch (regenError) { error.value = regenError.response?.data?.data?.error?.message @@ -276,6 +338,34 @@ async function copySecret() { console.error(copyError) } } + +async function downloadAdoptionTool() { + adoptbusy.value = true + adoptmessage.value = '' + adopterror.value = '' + try { + const response = await api.post( + '/endpoints/adoptiontool/package', + { baseurl: adoptbaseurl.value, collectorkey: adoptcollectorkey.value }, + { responseType: 'blob' }) + const url = window.URL.createObjectURL(response.data) + const link = document.createElement('a') + link.href = url + link.download = 'shopdb-adoption-tool.zip' + document.body.appendChild(link) + link.click() + link.remove() + window.URL.revokeObjectURL(url) + adoptmessage.value = 'Adoption tool downloaded.' + } catch (downloadError) { + adopterror.value = downloadError.response?.status === 403 + ? 'Only an administrator can build the adoption tool' + : 'Could not build the adoption tool' + console.error(downloadError) + } finally { + adoptbusy.value = false + } +} diff --git a/plugins/endpoints/plugin.py b/plugins/endpoints/plugin.py index 79893f33..85508488 100644 --- a/plugins/endpoints/plugin.py +++ b/plugins/endpoints/plugin.py @@ -147,6 +147,12 @@ class EndpointsPlugin(BasePlugin): # remote reboot is a strictly heavier grant than a routine toggle. ('endpoints.restart', 'Restart endpoint computers (destructive)', 'endpoints'), + # Elevated grant for software delivery (run_script, update_app). + # Required IN ADDITION to endpoints.command by the enqueue route, so + # running a staged script or pushing an installer is a strictly + # heavier grant than a routine service/process toggle. + ('endpoints.deploy', + 'Run staged scripts and push application installers to endpoints', 'endpoints'), # Collector service scope for the endpoint agent. A managed token # scoped to ONLY this authorizes the endpoints collector and nothing # else, so rotating/revoking it stops endpoint-agent traffic without @@ -226,6 +232,19 @@ class EndpointsPlugin(BasePlugin): 'permission': 'endpoints.fleet', 'section': 'assets', }, + # TEMPORARY / DEV ONLY. A manual bench to exercise the update_app and + # run_script delivery path while Cameron builds the production UI. + # Gated on endpoints.deploy so only delivery operators see it. Remove + # this entry (and the DeliveryTestBench view/route) once the real + # delivery UI ships. + { + 'name': 'Delivery test bench', + 'icon': 'send', + 'route': '/endpoints/delivery-test', + 'position': 17, + 'permission': 'endpoints.deploy', + 'section': 'assets', + }, ] def init_app(self, app: Flask, db_instance) -> None: diff --git a/plugins/endpoints/tests/test_plugin.py b/plugins/endpoints/tests/test_plugin.py index 54373de8..5378a3fc 100644 --- a/plugins/endpoints/tests/test_plugin.py +++ b/plugins/endpoints/tests/test_plugin.py @@ -160,6 +160,33 @@ def test_collect_now_is_allowlisted_targetless_and_routine(): assert elevated_permission_for('collect_now') is None +def test_run_script_and_update_app_are_allowlisted_delivery_actions(): + """The two software-delivery actions are allowlisted with a targetkind so + the enqueue route accepts them and carries their subject on the wire: + run_script carries a script name, update_app carries an appversionid.""" + for action in ('run_script', 'update_app'): + assert is_allowed_action(action) + assert action in COMMAND_ALLOWLIST + assert requires_target(action) + assert COMMAND_ALLOWLIST['run_script']['targetkind'] == 'script' + assert COMMAND_ALLOWLIST['update_app']['targetkind'] == 'app' + + +def test_delivery_actions_require_the_elevated_deploy_permission(): + """run_script and update_app both demand endpoints.deploy ON TOP OF + endpoints.command, so pushing software is a strictly heavier grant than a + routine service/process toggle.""" + assert elevated_permission_for('run_script') == 'endpoints.deploy' + assert elevated_permission_for('update_app') == 'endpoints.deploy' + + +def test_endpoints_declares_the_deploy_permission(): + """The dedicated software-delivery permission is declared for role seeding.""" + plugin = EndpointsPlugin() + permissions = {name for name, *_ in plugin.get_permissions()} + assert 'endpoints.deploy' in permissions + + def test_known_payload_keys_cover_persisted_and_relayed_fields(): """The ingest allow-set must list every key the handler consumes today, so those keys are never mislogged as 'unpersisted'. Anything the agent sends @@ -219,3 +246,4 @@ def test_fleet_list_route_resolves_without_trailing_slash(): assert endpoint.endswith('list_endpoints') endpoint_slash, _ = adapter.match('/api/endpoints/', method='GET') assert endpoint_slash.endswith('list_endpoints') + From bd8eacfe05617fa510c52a37eba926f6b8caf11b Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Sun, 30 Aug 2026 14:41:10 -0400 Subject: [PATCH 12/90] Agent: execute run_script and update_app delivery pushes Bumps the agent to 1.4.1. run_script fetches a ShopDB-hosted script by digest and runs it in the script's configured context (LocalSystem or the interactive logged-on user), replacing the local scriptpaths allowlist; user-context runs wait for exit and report the real exit code instead of fire-and-forget. update_app resolves an app-level or version-level install source, fetches the pinned bytes, verifies sha256, and runs silently. Both stage into a fixed Defender-excludable folder (C:\ProgramData\ShopDB\EndpointAgent) instead of the LocalSystem temp path. Adds the DeliveryClaim contract for the claim payload. --- agent/ShopDbEndpointAgent.Core/AgentConfig.cs | 33 ++ .../CommandExecutor.cs | 447 +++++++++++++++++- agent/ShopDbEndpointAgent.Core/Contracts.cs | 43 ++ .../InteractiveProcessLauncher.cs | 55 ++- .../ShopDbEndpointAgent.csproj | 12 +- 5 files changed, 583 insertions(+), 7 deletions(-) diff --git a/agent/ShopDbEndpointAgent.Core/AgentConfig.cs b/agent/ShopDbEndpointAgent.Core/AgentConfig.cs index 78bddb68..86672779 100644 --- a/agent/ShopDbEndpointAgent.Core/AgentConfig.cs +++ b/agent/ShopDbEndpointAgent.Core/AgentConfig.cs @@ -80,6 +80,39 @@ public sealed class AgentConfig return bare; } + // Per-name allowlist of pre-staged scripts for the generic run_script + // action. Exactly like LaunchPaths, the wire never carries a script body or + // path - only a script NAME the agent already holds here. A name absent + // from this map fails closed. There are no shipped defaults: a script is a + // site-specific artifact, so an empty map means run_script does nothing + // until a site stages scripts and lists them in appsettings.json. Keyed by + // script name (case-insensitive, .ps1 optional). + [JsonPropertyName("scriptpaths")] + public Dictionary ScriptPaths { get; set; } = new(); + + // The effective script map: appsettings.json entries only (no defaults). + // Keys are normalized to bare names so a lookup tolerates 'reset' or + // 'reset.ps1'. + public IReadOnlyDictionary ResolveScriptPaths() + { + var resolved = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var pair in ScriptPaths) + { + if (string.IsNullOrWhiteSpace(pair.Key) || string.IsNullOrWhiteSpace(pair.Value)) + continue; + resolved[NormalizeScript(pair.Key)] = pair.Value; + } + return resolved; + } + + private static string NormalizeScript(string name) + { + var bare = (name ?? "").Trim(); + if (bare.EndsWith(".ps1", StringComparison.OrdinalIgnoreCase)) + bare = bare[..^4]; + return bare; + } + public bool IsUsable() => !string.IsNullOrWhiteSpace(BaseUrl) && !string.IsNullOrWhiteSpace(ApiKey) diff --git a/agent/ShopDbEndpointAgent.Core/CommandExecutor.cs b/agent/ShopDbEndpointAgent.Core/CommandExecutor.cs index c739bdc7..9ffae27c 100644 --- a/agent/ShopDbEndpointAgent.Core/CommandExecutor.cs +++ b/agent/ShopDbEndpointAgent.Core/CommandExecutor.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using System.Security.Cryptography; using System.ServiceProcess; using Microsoft.Extensions.Logging; @@ -58,12 +59,18 @@ public sealed class CommandExecutor // map cannot be started - process_start fails closed. private readonly IReadOnlyDictionary launchPaths; + // Held for update_app: the agent joins InstallPath to BaseUrl and downloads + // the pinned installer with its own collector credential (ApiKey), so the + // installer body never rides the command wire. + private readonly AgentConfig config; + public CommandExecutor(ILogger logger, AgentConfig config, TimeSpan? startTimeout = null) { this.logger = logger; this.startTimeout = startTimeout ?? TimeSpan.FromSeconds(30); - this.launchPaths = (config ?? new AgentConfig()).ResolveLaunchPaths(); + this.config = config ?? new AgentConfig(); + this.launchPaths = this.config.ResolveLaunchPaths(); } public async Task ExecuteAsync( @@ -103,6 +110,17 @@ public sealed class CommandExecutor return RequireTarget(action, target) ?? await ProcessStart(target, token); + // Software delivery. run_script fetches a ShopDB-hosted script + // pinned by SHA-256 and runs it in the script's configured context + // (system or the interactive user); update_app fetches a ShopDB + // installer pinned by SHA-256 and runs it silently. Both download + // the body themselves with the collector credential and fail closed. + case "run_script": + return RequireTarget(action, target) + ?? await RunScript(target, claim.Script, token); + case "update_app": + return await UpdateApp(claim.App, token); + // Fixed destructive action: reboot this endpoint immediately. Runs // as the SYSTEM agent, so shutdown does not need a user session. case "system_restart": @@ -470,6 +488,433 @@ public sealed class CommandExecutor return found; } + // ---- Software delivery: run_script and update_app ---- + + // Run a ShopDB-hosted script pinned by the server's delivery descriptor. The + // wire carries no body: the agent downloads the script from BaseUrl + + // InstallPath with its own collector credential, verifies the SHA-256 (when + // the server pinned one), then runs it in the descriptor's run context - + // 'system' through the agent's own service account, or 'user' in the active + // console session so the script sees the interactive user's resources (for + // example mapped network drives). Anything missing or mismatched fails closed. + private async Task RunScript( + string name, ScriptDeliveryClaim? script, CancellationToken token) + { + if (script is null || string.IsNullOrWhiteSpace(script.InstallPath)) + { + logger.LogWarning( + "Rejected run_script for {Name}: no delivery descriptor on the claim", name); + return new CommandResult + { + Succeeded = false, + Message = $"run_script for {name} arrived without a resolved script descriptor", + Detail = ScriptDetail(name, script, scriptPath: null, exitCode: null), + }; + } + + if (string.IsNullOrWhiteSpace(config.BaseUrl) || string.IsNullOrWhiteSpace(config.ApiKey)) + { + return new CommandResult + { + Succeeded = false, + Message = "run_script cannot download: agent has no base URL or collector key", + Detail = ScriptDetail(name, script, scriptPath: null, exitCode: null), + }; + } + + var scriptUrl = CombineUrl(config.BaseUrl, script.InstallPath!); + var tempPath = Path.Combine(GetStagingDirectory(), + $"shopdb-script-{script.ScriptId}.ps1"); + + try + { + using var handler = new HttpClientHandler(); + using var client = new HttpClient(handler); + client.Timeout = TimeSpan.FromMinutes(5); + using var httpRequest = new HttpRequestMessage(HttpMethod.Get, scriptUrl); + httpRequest.Headers.Add("X-API-Key", config.ApiKey); + + using var response = await client.SendAsync( + httpRequest, HttpCompletionOption.ResponseHeadersRead, token); + if (!response.IsSuccessStatusCode) + { + return new CommandResult + { + Succeeded = false, + Message = $"run_script download failed: HTTP {(int)response.StatusCode}", + Detail = ScriptDetail(name, script, scriptPath: null, exitCode: null), + }; + } + + await using (var source = await response.Content.ReadAsStreamAsync(token)) + await using (var file = File.Create(tempPath)) + { + await source.CopyToAsync(file, token); + } + + if (!string.IsNullOrWhiteSpace(script.Sha256)) + { + var actual = HashFile(tempPath); + if (!actual.Equals(script.Sha256, StringComparison.OrdinalIgnoreCase)) + { + logger.LogWarning( + "run_script SHA-256 mismatch for {Name}: expected {Expected} got {Actual}", + name, script.Sha256, actual); + return new CommandResult + { + Succeeded = false, + Message = "run_script integrity check failed: script SHA-256 does not match", + Detail = ScriptDetail(name, script, tempPath, exitCode: null), + }; + } + } + + var runInUser = string.Equals( + script.RunContext, "user", StringComparison.OrdinalIgnoreCase); + return runInUser + ? RunScriptInUserSession(name, script, tempPath) + : await RunScriptAsSystem(name, script, tempPath, token); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception error) + { + logger.LogWarning(error, "run_script for {Name} failed", name); + return new CommandResult + { + Succeeded = false, + Message = $"Script {name} failed: {error.Message}", + Detail = ScriptDetail(name, script, tempPath, exitCode: null), + }; + } + finally + { + try + { + if (File.Exists(tempPath)) + File.Delete(tempPath); + } + catch (Exception cleanupError) + { + logger.LogDebug(cleanupError, "run_script temp cleanup failed: {Path}", tempPath); + } + } + } + + // System context: run through the agent's own service account and wait for + // the exit code, the same as before. Fine for machine-scoped scripts that do + // not need the interactive user's session resources. + private async Task RunScriptAsSystem( + string name, ScriptDeliveryClaim script, string scriptPath, CancellationToken token) + { + var startInfo = new ProcessStartInfo + { + FileName = "powershell.exe", + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = Path.GetDirectoryName(scriptPath) ?? "", + }; + startInfo.ArgumentList.Add("-NoProfile"); + startInfo.ArgumentList.Add("-NonInteractive"); + startInfo.ArgumentList.Add("-ExecutionPolicy"); + startInfo.ArgumentList.Add("Bypass"); + startInfo.ArgumentList.Add("-File"); + startInfo.ArgumentList.Add(scriptPath); + + using var process = Process.Start(startInfo); + if (process is null) + { + return new CommandResult + { + Succeeded = false, + Message = $"Failed to start script {name}", + Detail = ScriptDetail(name, script, scriptPath, exitCode: null), + }; + } + + await process.WaitForExitAsync(token); + var exit = process.ExitCode; + return new CommandResult + { + Succeeded = exit == 0, + Message = exit == 0 + ? $"Script {name} completed (system context)" + : $"Script {name} exited with code {exit}", + Detail = ScriptDetail(name, script, scriptPath, exit), + }; + } + + // User context: launch PowerShell in the ACTIVE CONSOLE session so the script + // runs at the interactive user's integrity and sees that user's resources + // (mapped network drives, HKCU, profile). The launcher waits for the process + // to exit and returns its real exit code, so success here means the script + // actually ran to a zero exit - not merely that it was launched. Fails closed + // when no interactive user is signed in. + private CommandResult RunScriptInUserSession( + string name, ScriptDeliveryClaim script, string scriptPath) + { + var quotedPath = "\"" + scriptPath + "\""; + var arguments = + "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File " + quotedPath; + var workingDirectory = Path.GetDirectoryName(scriptPath) ?? ""; + + var outcome = InteractiveProcessLauncher.LaunchInActiveSession( + "powershell.exe", arguments, workingDirectory, waitForExit: true); + if (!outcome.Launched) + { + logger.LogWarning( + "run_script for {Name} could not launch in the user session: {Error}", + name, outcome.Error); + return new CommandResult + { + Succeeded = false, + Message = $"Script {name} could not run in the user session: {outcome.Error}", + Detail = ScriptDetail(name, script, scriptPath, exitCode: null), + }; + } + + var exit = outcome.ExitCode; + var ok = exit is null || exit == 0; + return new CommandResult + { + Succeeded = ok, + Message = exit is null + ? $"Script {name} launched in the interactive user session" + : ok + ? $"Script {name} completed (user context)" + : $"Script {name} exited with code {exit}", + Detail = ScriptDetail(name, script, scriptPath, exit), + }; + } + + private static Dictionary ScriptDetail( + string name, ScriptDeliveryClaim? script, string? scriptPath, int? exitCode) + { + var detail = new Dictionary + { + ["kind"] = "script", + ["name"] = name, + }; + if (script is not null) + { + detail["scriptid"] = script.ScriptId; + if (!string.IsNullOrWhiteSpace(script.RunContext)) + detail["runcontext"] = script.RunContext!; + } + if (scriptPath is not null) + detail["path"] = scriptPath; + if (exitCode is not null) + detail["exitcode"] = exitCode; + return detail; + } + + // Fetch a ShopDB-hosted installer pinned by the server's delivery descriptor + // and run it silently. The wire carries no bytes: the agent downloads the + // package itself from BaseUrl + InstallPath using its own collector + // credential, verifies the SHA-256 (when the server pinned one), then runs + // an MSI via msiexec /qn or an EXE with the version's SilentArgs. Anything + // missing or mismatched fails closed. + private async Task UpdateApp(AppDeliveryClaim? app, CancellationToken token) + { + if (app is null || string.IsNullOrWhiteSpace(app.InstallPath)) + { + logger.LogWarning("Rejected update_app: no delivery descriptor on the claim"); + return new CommandResult + { + Succeeded = false, + Message = "update_app arrived without a resolved package descriptor", + Detail = AppDetail(app, installerPath: null, exitCode: null), + }; + } + + if (string.IsNullOrWhiteSpace(config.BaseUrl) || string.IsNullOrWhiteSpace(config.ApiKey)) + { + return new CommandResult + { + Succeeded = false, + Message = "update_app cannot download: agent has no base URL or collector key", + Detail = AppDetail(app, installerPath: null, exitCode: null), + }; + } + + var packageUrl = CombineUrl(config.BaseUrl, app.InstallPath!); + var ext = Path.GetExtension(app.InstallPath) ?? ""; + var packageKey = app.AppVersionId?.ToString() ?? $"app{app.AppId}"; + var tempPath = Path.Combine(GetStagingDirectory(), + $"shopdb-app-{packageKey}{ext}"); + + try + { + using var handler = new HttpClientHandler(); + using var client = new HttpClient(handler); + client.Timeout = TimeSpan.FromMinutes(10); + using var httpRequest = new HttpRequestMessage(HttpMethod.Get, packageUrl); + httpRequest.Headers.Add("X-API-Key", config.ApiKey); + + using var response = await client.SendAsync( + httpRequest, HttpCompletionOption.ResponseHeadersRead, token); + if (!response.IsSuccessStatusCode) + { + return new CommandResult + { + Succeeded = false, + Message = $"update_app download failed: HTTP {(int)response.StatusCode}", + Detail = AppDetail(app, installerPath: null, exitCode: null), + }; + } + + await using (var source = await response.Content.ReadAsStreamAsync(token)) + await using (var file = File.Create(tempPath)) + { + await source.CopyToAsync(file, token); + } + + if (!string.IsNullOrWhiteSpace(app.Sha256)) + { + var actual = HashFile(tempPath); + if (!actual.Equals(app.Sha256, StringComparison.OrdinalIgnoreCase)) + { + logger.LogWarning( + "update_app SHA-256 mismatch for {App} v{Version}: expected {Expected} got {Actual}", + app.AppName, app.Version, app.Sha256, actual); + return new CommandResult + { + Succeeded = false, + Message = "update_app integrity check failed: package SHA-256 does not match", + Detail = AppDetail(app, tempPath, exitCode: null), + }; + } + } + + var (fileName, arguments) = BuildInstallInvocation(tempPath, ext, app.SilentArgs); + var startInfo = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = Path.GetDirectoryName(tempPath) ?? "", + }; + + using var installer = Process.Start(startInfo); + if (installer is null) + { + return new CommandResult + { + Succeeded = false, + Message = $"Failed to start installer for {app.AppName}", + Detail = AppDetail(app, tempPath, exitCode: null), + }; + } + + await installer.WaitForExitAsync(token); + var exit = installer.ExitCode; + // MSI treats 3010 as success-pending-reboot; keep that as success. + var ok = exit == 0 || exit == 3010; + return new CommandResult + { + Succeeded = ok, + Message = ok + ? $"Installed {app.AppName} v{app.Version}" + + (exit == 3010 ? " (reboot required)" : "") + : $"Installer for {app.AppName} exited with code {exit}", + Detail = AppDetail(app, tempPath, exit), + }; + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception error) + { + logger.LogWarning(error, "update_app for {App} failed", app.AppName); + return new CommandResult + { + Succeeded = false, + Message = $"update_app for {app.AppName} failed: {error.Message}", + Detail = AppDetail(app, tempPath, exitCode: null), + }; + } + finally + { + try + { + if (File.Exists(tempPath)) + File.Delete(tempPath); + } + catch (Exception cleanupError) + { + logger.LogDebug(cleanupError, "update_app temp cleanup failed: {Path}", tempPath); + } + } + } + + // MSI installs run through msiexec /i /qn (any SilentArgs appended); + // an EXE runs directly with its version's SilentArgs (homegrown EXEs vary by + // developer, so the switches are per-version server data, not guessed here). + private static (string fileName, string arguments) BuildInstallInvocation( + string packagePath, string ext, string? silentArgs) + { + var extra = (silentArgs ?? "").Trim(); + if (ext.Equals(".msi", StringComparison.OrdinalIgnoreCase)) + { + var args = $"/i \"{packagePath}\" /qn"; + if (extra.Length > 0) + args += " " + extra; + return ("msiexec.exe", args); + } + return (packagePath, extra); + } + + // Fixed, known staging folder we control and that is Defender-excluded in + // the environment. LocalSystem's Path.GetTempPath() resolves to + // C:\Windows\SystemTemp on Win11/Server2022+, which we do not use; stage + // downloads here instead so run_script and update_app land in a predictable + // spot. Created on demand. + private static string GetStagingDirectory() + { + var programData = Environment.GetFolderPath( + Environment.SpecialFolder.CommonApplicationData); + var staging = Path.Combine(programData, "ShopDB", "EndpointAgent"); + Directory.CreateDirectory(staging); + return staging; + } + + private static string HashFile(string path) + { + using var stream = File.OpenRead(path); + using var sha = SHA256.Create(); + var hash = sha.ComputeHash(stream); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + // Join the agent base URL to a ShopDB-relative package path, tolerating a + // trailing slash on the base and a leading slash on the path. + private static string CombineUrl(string baseUrl, string path) + { + var left = baseUrl.TrimEnd('/'); + var right = path.StartsWith('/') ? path : "/" + path; + return left + right; + } + + private static Dictionary AppDetail( + AppDeliveryClaim? app, string? installerPath, int? exitCode) + { + var detail = new Dictionary { ["kind"] = "app" }; + if (app is not null) + { + if (app.AppVersionId is not null) detail["appversionid"] = app.AppVersionId.Value; + detail["appid"] = app.AppId; + if (app.AppName is not null) detail["appname"] = app.AppName; + if (app.Version is not null) detail["version"] = app.Version; + } + if (installerPath is not null) detail["installer"] = installerPath; + if (exitCode is not null) detail["exitcode"] = exitCode; + return detail; + } + // ---- Fixed MTConnect actions ---- private async Task MtConnectStartAsync(CancellationToken token) diff --git a/agent/ShopDbEndpointAgent.Core/Contracts.cs b/agent/ShopDbEndpointAgent.Core/Contracts.cs index e35af45b..d4c92a0a 100644 --- a/agent/ShopDbEndpointAgent.Core/Contracts.cs +++ b/agent/ShopDbEndpointAgent.Core/Contracts.cs @@ -85,6 +85,49 @@ public sealed class CommandClaim [JsonPropertyName("action")] public string Action { get; set; } = ""; [JsonPropertyName("target")] public string? Target { get; set; } [JsonPropertyName("targetkind")] public string? TargetKind { get; set; } + + // For update_app the server resolves the pinned application version into a + // delivery descriptor (package URL, SHA-256, silent-install switches). The + // agent fetches the bytes itself with its collector credential; the wire + // never carries the installer body. Null for every other action. + [JsonPropertyName("app")] public AppDeliveryClaim? App { get; set; } + + // For run_script the server resolves the ShopDB-hosted script into a + // delivery descriptor (body URL, SHA-256, run context). The agent fetches + // the body itself with its collector credential; the wire never carries the + // script body. Null for every other action. + [JsonPropertyName("script")] public ScriptDeliveryClaim? Script { get; set; } +} + +// The resolved installer-push descriptor accompanying an update_app claim. +// InstallPath is a ShopDB-relative package URL the agent joins to its BaseUrl +// and downloads with its collector credential; Sha256 pins the exact bytes; +// SilentArgs are this version's unattended-install switches (may be empty for +// an MSI, which the agent installs via msiexec /qn). AppVersionId is null when +// the push resolves an app-level install source rather than a pinned version. +public sealed class AppDeliveryClaim +{ + [JsonPropertyName("appversionid")] public int? AppVersionId { get; set; } + [JsonPropertyName("appid")] public int AppId { get; set; } + [JsonPropertyName("appname")] public string? AppName { get; set; } + [JsonPropertyName("version")] public string? Version { get; set; } + [JsonPropertyName("installpath")] public string? InstallPath { get; set; } + [JsonPropertyName("sha256")] public string? Sha256 { get; set; } + [JsonPropertyName("silentargs")] public string? SilentArgs { get; set; } +} + +// The resolved script-push descriptor accompanying a run_script claim. +// InstallPath is a ShopDB-relative body URL the agent joins to its BaseUrl and +// downloads with its collector credential; Sha256 pins the exact bytes; +// RunContext is 'system' (the agent's own service account) or 'user' (the +// interactive logged-on session), decided per script in the ShopDB library. +public sealed class ScriptDeliveryClaim +{ + [JsonPropertyName("scriptid")] public int ScriptId { get; set; } + [JsonPropertyName("scriptname")] public string? ScriptName { get; set; } + [JsonPropertyName("installpath")] public string? InstallPath { get; set; } + [JsonPropertyName("sha256")] public string? Sha256 { get; set; } + [JsonPropertyName("runcontext")] public string? RunContext { get; set; } } // Envelope for the claim long-poll: { status, data: { endpointcommandid, action } }. diff --git a/agent/ShopDbEndpointAgent.Core/InteractiveProcessLauncher.cs b/agent/ShopDbEndpointAgent.Core/InteractiveProcessLauncher.cs index 3b028bcb..326bdc07 100644 --- a/agent/ShopDbEndpointAgent.Core/InteractiveProcessLauncher.cs +++ b/agent/ShopDbEndpointAgent.Core/InteractiveProcessLauncher.cs @@ -18,7 +18,7 @@ namespace ShopDbEndpointAgent.Core; // Only process_start uses this. Stop/status/service actions stay as SYSTEM. public static class InteractiveProcessLauncher { - public readonly record struct LaunchOutcome(bool Launched, string? Error); + public readonly record struct LaunchOutcome(bool Launched, string? Error, int? ExitCode = null); private const int TokenDuplicate = 0x0002; private const int TokenQuery = 0x0008; @@ -32,11 +32,28 @@ public static class InteractiveProcessLauncher private const uint InvalidSessionId = 0xFFFFFFFF; + private const uint Infinite = 0xFFFFFFFF; + // Returns Launched=true when CreateProcessAsUser succeeds. Never throws; // failures come back as Launched=false with a descriptive Error so the // caller can surface a clean CommandResult. public static LaunchOutcome LaunchInActiveSession( string applicationPath, string? workingDirectory) + => LaunchInActiveSession(applicationPath, null, workingDirectory); + + // Overload that also passes a command line (arguments). When commandLine is + // supplied it is handed to CreateProcessAsUser as lpCommandLine with a null + // lpApplicationName so the arguments are honored; the executable is resolved + // from the command line's first token as usual. + // + // When waitForExit is true the launcher waits for the process to exit and + // returns its real exit code in LaunchOutcome.ExitCode, so a caller that + // needs the actual outcome (run_script in the user session) can report + // completed/failed instead of fire-and-forget "launched". GUI callers + // (process_start) leave it false and return as soon as the window is up. + public static LaunchOutcome LaunchInActiveSession( + string applicationPath, string? arguments, string? workingDirectory, + bool waitForExit = false) { var sessionId = WTSGetActiveConsoleSessionId(); if (sessionId == InvalidSessionId) @@ -75,10 +92,26 @@ public static class InteractiveProcessLauncher var creationFlags = CreateUnicodeEnvironment | CreateNoWindow; + // With arguments, pass a single command line (" ") as + // lpCommandLine and a null lpApplicationName so the arguments apply; + // without arguments, keep the original applicationName form. + string? applicationName; + string? commandLine; + if (string.IsNullOrWhiteSpace(arguments)) + { + applicationName = applicationPath; + commandLine = null; + } + else + { + applicationName = null; + commandLine = "\"" + applicationPath + "\" " + arguments; + } + var launched = CreateProcessAsUser( duplicatedToken, - applicationPath, - null, + applicationName, + commandLine, IntPtr.Zero, IntPtr.Zero, false, @@ -95,12 +128,20 @@ public static class InteractiveProcessLauncher + new Win32Exception(Marshal.GetLastWin32Error()).Message); } + int? exitCode = null; + if (waitForExit && processInformation.hProcess != IntPtr.Zero) + { + WaitForSingleObject(processInformation.hProcess, Infinite); + if (GetExitCodeProcess(processInformation.hProcess, out var code)) + exitCode = unchecked((int)code); + } + if (processInformation.hProcess != IntPtr.Zero) CloseHandle(processInformation.hProcess); if (processInformation.hThread != IntPtr.Zero) CloseHandle(processInformation.hThread); - return new LaunchOutcome(true, null); + return new LaunchOutcome(true, null, exitCode); } finally { @@ -197,4 +238,10 @@ public static class InteractiveProcessLauncher [DllImport("kernel32.dll", SetLastError = true)] private static extern bool CloseHandle(IntPtr handle); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern uint WaitForSingleObject(IntPtr handle, uint milliseconds); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool GetExitCodeProcess(IntPtr process, out uint exitCode); } diff --git a/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj b/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj index 3612a79a..9c7d55bf 100644 --- a/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj +++ b/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj @@ -7,11 +7,19 @@ enable ShopDbEndpointAgent shopdbagent - - 1.3.0 + 1.4.1 true win-x64 true From 5420ac432d7403e2a7c96a6f11ba10adda149850 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Sun, 30 Aug 2026 14:41:22 -0400 Subject: [PATCH 13/90] Serve the dev/test stack over HTTPS with a mounted self-signed cert Terminate TLS in gunicorn using a host-generated cert mounted read-only under ./certs, so the browser sees a secure context. Secure-context-only web APIs (navigator.clipboard and others) refuse to run over plain http on a non-localhost origin, which the FQDN-accessed test box hits. Healthcheck now probes https with -k to match the TLS listener. A site swaps in its own cert or fronts this with its own IIS/nginx proxy. --- docker-compose.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index b8d54715..dc2e63e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,8 +65,43 @@ services: COLLECTOR_API_KEY_ENDPOINTS: ${COLLECTOR_API_KEY_ENDPOINTS:-} ports: - "${API_PORT:-5001}:5001" + # Terminate TLS in gunicorn with the mounted self-signed cert so the browser + # sees a secure context. navigator.clipboard and other secure-context-only + # web APIs refuse to run over plain http on a non-localhost origin, so the + # test box (accessed by FQDN) needs real https. A site swaps in its own cert + # under ./certs or fronts this with its IIS/nginx proxy. + command: + - gunicorn + - --bind + - 0.0.0.0:5001 + - --workers + - "4" + - --timeout + - "60" + - --access-logfile + - "-" + - --error-logfile + - "-" + - --certfile + - /app/certs/server.crt + - --keyfile + - /app/certs/server.key + - wsgi:app + healthcheck: + # Match the TLS listener: probe over https and accept the self-signed cert + # (-k). The Dockerfile default probes http and would stay red once gunicorn + # only speaks TLS. + test: ["CMD", "curl", "--silent", "--insecure", "https://localhost:5001/api/auth/login", + "-X", "POST", "-H", "Content-Type: application/json", "-d", "{}"] + interval: 30s + timeout: 5s + start_period: 10s + retries: 3 volumes: - ./plugins:/app/plugins:ro + # Self-signed TLS material for the dev/test https listener. Read-only; + # generated on the host under ./certs and never baked into the image. + - ./certs:/app/certs:ro # /app/instance is WRITTEN state, not code: plugins.json (which plugins # this site has enabled), uploaded floor plans, branding, model and # application images, employee photos, warranty proofs, slides, From c3875b0eec51f32cd2c440f93ddee08d3d7d6f16 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Sun, 30 Aug 2026 14:41:33 -0400 Subject: [PATCH 14/90] Add a standalone endpoint agent uninstaller Reverses everything the installer and site-agent generator create: stops and deletes the service, kills agent/tray processes, removes the tray Run key and the HKLM\SOFTWARE\GE\ShopDB credential hive in both registry views, drops the Defender path exclusion, and deletes the install directory and ProgramData tree. Every step is best-effort and idempotent so it is safe to re-run and safe on a partially installed host. --- scripts/uninstall-shopdbagent.ps1 | 189 ++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 scripts/uninstall-shopdbagent.ps1 diff --git a/scripts/uninstall-shopdbagent.ps1 b/scripts/uninstall-shopdbagent.ps1 new file mode 100644 index 00000000..b4819ff3 --- /dev/null +++ b/scripts/uninstall-shopdbagent.ps1 @@ -0,0 +1,189 @@ +# uninstall-shopdbagent.ps1 +# ShopDB Endpoint Agent uninstaller. Run from an ELEVATED (admin) PowerShell on +# the endpoint. Reverses everything install-shopdbagent.ps1 and the site agent +# generator's install.ps1 create: +# - stops and deletes the Windows service +# - kills any running agent/tray processes +# - removes the tray logon Run key (64-bit and 32-bit views) +# - removes the credential hive HKLM\SOFTWARE\GE\ShopDB (both views) +# - removes the Defender path exclusion for the install directory +# - deletes the install directory and the ProgramData data/log tree +# +# Every step is best-effort and idempotent: a missing service, key, process, or +# folder is a no-op, so the script is safe to re-run and safe on a partially +# installed host. Nothing here fails the run just because an artifact is already +# gone; only an unexpected error on a present artifact surfaces as a warning. + +[CmdletBinding()] +param( + [string] $InstallDir = "C:\Program Files\ShopDB\EndpointAgent", + [string] $ServiceName = "ShopDbEndpointAgent", + [string] $TrayExeName = "shopdb-endpoint-tray.exe", + + # Also remove any server TLS certificate the site-agent installer imported + # into the machine Trusted Root store. Off by default because a root cert is + # shared trust material: only pull it when you are sure no other service on + # this host relies on it. Match is by subject CN substring. + [string] $RemoveRootCertMatching = "", + + # Keep the ProgramData logs (C:\ProgramData\ShopDB) for post-mortem instead + # of deleting them with everything else. + [switch] $KeepLogs +) + +$ErrorActionPreference = "Stop" + +$identity = [Security.Principal.WindowsIdentity]::GetCurrent() +$principal = New-Object Security.Principal.WindowsPrincipal($identity) +if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { + throw "This script must be run from an elevated (Run as administrator) PowerShell." +} + +# Delete an HKLM value or subkey from BOTH the 64-bit and 32-bit registry views. +# The installer writes both views (a stale WOW6432Node copy would otherwise be +# read first by the agent), so a clean uninstall has to clear both. +function Remove-HklmBothViews { + param( + [Parameter(Mandatory)] [string] $SubKey, + [string] $Name + ) + foreach ($view in @([Microsoft.Win32.RegistryView]::Registry64, + [Microsoft.Win32.RegistryView]::Registry32)) { + $base = [Microsoft.Win32.RegistryKey]::OpenBaseKey( + [Microsoft.Win32.RegistryHive]::LocalMachine, $view) + try { + if ($Name) { + $key = $base.OpenSubKey($SubKey, $true) + if ($key) { + try { + if ($null -ne $key.GetValue($Name)) { $key.DeleteValue($Name, $false) } + } + finally { $key.Dispose() } + } + } + else { + if ($base.OpenSubKey($SubKey)) { + $base.DeleteSubKeyTree($SubKey, $false) + } + } + } + catch { + Write-Warning " registry cleanup ($view $SubKey $Name): $($_.Exception.Message)" + } + finally { $base.Dispose() } + } +} + +Write-Host "== ShopDB endpoint agent uninstall ==" -ForegroundColor Cyan +Write-Host "InstallDir : $InstallDir" +Write-Host "ServiceName : $ServiceName" + +# 1. Stop and delete the Windows service. +$service = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue +if ($service) { + Write-Host "[1/6] Stopping and deleting service $ServiceName ..." + if ($service.Status -ne "Stopped") { + try { Stop-Service -Name $ServiceName -Force -ErrorAction Stop } + catch { Write-Warning " could not stop service: $($_.Exception.Message)" } + } + sc.exe delete $ServiceName | Out-Null + Start-Sleep -Seconds 2 +} +else { + Write-Host "[1/6] Service $ServiceName not present; skipping." +} + +# 2. Kill any lingering agent/tray processes (service host, tray, and anything +# still running out of the install directory holding files open). +Write-Host "[2/6] Stopping agent and tray processes ..." +$trayBase = [System.IO.Path]::GetFileNameWithoutExtension($TrayExeName) +foreach ($name in @("shopdbagent", $trayBase)) { + foreach ($proc in (Get-Process -Name $name -ErrorAction SilentlyContinue)) { + try { Stop-Process -Id $proc.Id -Force -ErrorAction Stop; Write-Host " killed $name (pid $($proc.Id))" } + catch { Write-Warning " could not stop $name (pid $($proc.Id)): $($_.Exception.Message)" } + } +} +foreach ($proc in (Get-Process -ErrorAction SilentlyContinue | + Where-Object { $_.Path -and $_.Path.StartsWith($InstallDir, [StringComparison]::OrdinalIgnoreCase) })) { + try { Stop-Process -Id $proc.Id -Force -ErrorAction Stop; Write-Host " killed $($proc.ProcessName) from install dir (pid $($proc.Id))" } + catch { Write-Warning " could not stop pid $($proc.Id): $($_.Exception.Message)" } +} +Start-Sleep -Seconds 1 + +# 3. Remove the tray logon Run key entry (both views) and the credential hive. +Write-Host "[3/6] Removing registry entries ..." +Remove-HklmBothViews -SubKey "SOFTWARE\Microsoft\Windows\CurrentVersion\Run" -Name "ShopDbEndpointTray" +Remove-HklmBothViews -SubKey "SOFTWARE\GE\ShopDB" +# Drop the parent GE key only if we emptied it (some sites nest siblings under +# GE such as SFLD; never tree-delete a key we do not own). +foreach ($view in @([Microsoft.Win32.RegistryView]::Registry64, + [Microsoft.Win32.RegistryView]::Registry32)) { + $base = [Microsoft.Win32.RegistryKey]::OpenBaseKey( + [Microsoft.Win32.RegistryHive]::LocalMachine, $view) + try { + $ge = $base.OpenSubKey("SOFTWARE\GE") + if ($ge -and $ge.SubKeyCount -eq 0 -and $ge.ValueCount -eq 0) { + $ge.Dispose() + $base.DeleteSubKey("SOFTWARE\GE", $false) + Write-Host " removed empty SOFTWARE\GE ($view)" + } + elseif ($ge) { $ge.Dispose() } + } + catch { Write-Warning " GE key cleanup ($view): $($_.Exception.Message)" } + finally { $base.Dispose() } +} + +# 4. Remove the Defender path exclusion the installer added for the install dir. +Write-Host "[4/6] Removing Defender exclusion ..." +if (Get-Command Remove-MpPreference -ErrorAction SilentlyContinue) { + try { + Remove-MpPreference -ExclusionPath $InstallDir -ErrorAction Stop + Write-Host " removed Defender exclusion for $InstallDir" + } + catch { Write-Warning " could not remove Defender exclusion (tamper protection or policy?): $($_.Exception.Message)" } +} +else { + Write-Host " Defender cmdlets not present; skipping." +} + +# 5. Optionally remove an imported server root certificate. +if (-not [string]::IsNullOrWhiteSpace($RemoveRootCertMatching)) { + Write-Host "[5/6] Removing Trusted Root certs matching '$RemoveRootCertMatching' ..." + $matches = Get-ChildItem Cert:\LocalMachine\Root -ErrorAction SilentlyContinue | + Where-Object { $_.Subject -like "*$RemoveRootCertMatching*" } + foreach ($cert in $matches) { + try { + Remove-Item -Path $cert.PSPath -Force -ErrorAction Stop + Write-Host " removed $($cert.Subject) [$($cert.Thumbprint)]" + } + catch { Write-Warning " could not remove $($cert.Thumbprint): $($_.Exception.Message)" } + } + if (-not $matches) { Write-Host " no matching root certs found." } +} +else { + Write-Host "[5/6] Skipping root-cert removal (pass -RemoveRootCertMatching to enable)." +} + +# 6. Delete the install directory and the ProgramData data/log tree. +Write-Host "[6/6] Removing files ..." +if (Test-Path $InstallDir) { + try { Remove-Item -Path $InstallDir -Recurse -Force -ErrorAction Stop; Write-Host " removed $InstallDir" } + catch { Write-Warning " could not remove $InstallDir (a file may still be locked): $($_.Exception.Message)" } +} +else { Write-Host " $InstallDir not present." } + +$dataDir = Join-Path $env:ProgramData "ShopDB" +if (-not $KeepLogs -and (Test-Path $dataDir)) { + try { Remove-Item -Path $dataDir -Recurse -Force -ErrorAction Stop; Write-Host " removed $dataDir" } + catch { Write-Warning " could not remove $dataDir : $($_.Exception.Message)" } +} +elseif ($KeepLogs) { Write-Host " keeping $dataDir (-KeepLogs)." } + +# Also prune an empty C:\Program Files\ShopDB parent left behind. +$parent = Split-Path -Parent $InstallDir +if ((Test-Path $parent) -and -not (Get-ChildItem -Path $parent -Force -ErrorAction SilentlyContinue)) { + try { Remove-Item -Path $parent -Force -ErrorAction Stop; Write-Host " removed empty $parent" } catch { } +} + +Write-Host "`nUninstall complete." -ForegroundColor Green +Write-Host "Verify with: Get-Service $ServiceName ; Get-Process shopdbagent,$trayBase ; reg query 'HKLM\SOFTWARE\GE\ShopDB'" From f52b5f566928ee843078e8ddbdd91eb7444a7920 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Sun, 30 Aug 2026 15:03:34 -0400 Subject: [PATCH 15/90] Bump plugin contract to 0.27.0 for the Script surface addition Commit 0c2b816 added Script to shopdb.api so a plugin can read the ShopDB-hosted script library that backs run_script delivery, but left __contract_version__ at 0.26.0. This bumps it to 0.27.0 so a plugin can pin core_version against the widened surface. Additive name, minor bump, matching the existing convention. The docs/PLUGIN-HOOKS wiki page (not tracked in-repo) still needs the Script entry. --- shopdb/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/shopdb/__init__.py b/shopdb/__init__.py index 60efdc22..01a3bac5 100644 --- a/shopdb/__init__.py +++ b/shopdb/__init__.py @@ -93,7 +93,12 @@ from .plugins import plugin_manager # as a heading - "Endpoints with crashes or shutdowns" - which as a row badge # ran wider than the hostname it qualified. Optional, defaults to the title, so # a plugin that does not declare one is unaffected. Additive field, minor bump. -__contract_version__ = '0.26.0' +# 0.27.0: added Script to shopdb.api so a plugin can read the ShopDB-hosted +# script library that backs endpoint run_script delivery without importing core +# model internals. The endpoints plugin resolves a delivered script by name and +# serves its body by digest to the agent. Additive name on the surface, minor +# bump. +__contract_version__ = '0.27.0' # Product release version (see ADR-007). The product version and the # plugin-contract version above are distinct series with independent From 022affc5d0ad06389814584d178a9de132c64dba Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Mon, 31 Aug 2026 16:09:05 -0400 Subject: [PATCH 16/90] Ship uninstaller in agent bundle and harden tray launch The site-agent bundle now includes uninstall-shopdbagent.ps1 so operators can cleanly remove an install without hunting for the repo script. The packaging route reads it from scripts/, which was not mounted into the api container, so add a read-only ./scripts:/app/scripts mount (recreate the container to apply). install.ps1 now registers the tray to autostart for every user via the HKLM Run value ShopDbEndpointTray, adds an All-Users 'ShopDB Agent' Start Menu shortcut for manual relaunch, and launches the tray once post-install. The tray enforces a single instance via a Global named mutex so repeated launches no longer stack duplicate icons. The uninstaller removes the Start Menu shortcut alongside the existing Run key and credential cleanup. --- agent/ShopDbEndpointAgent.Tray/Program.cs | 18 +++++++- docker-compose.yml | 5 +++ plugins/endpoints/api/siteagent.py | 55 ++++++++++++++++++++++- scripts/uninstall-shopdbagent.ps1 | 8 ++++ 4 files changed, 84 insertions(+), 2 deletions(-) diff --git a/agent/ShopDbEndpointAgent.Tray/Program.cs b/agent/ShopDbEndpointAgent.Tray/Program.cs index 80e0f0cd..6e4e5840 100644 --- a/agent/ShopDbEndpointAgent.Tray/Program.cs +++ b/agent/ShopDbEndpointAgent.Tray/Program.cs @@ -1,6 +1,8 @@ +using System.Threading; + namespace ShopDbEndpointAgent.Tray; -// Interactive user-session tray app (SCAFFOLD). Separate process from the +// Interactive user-session tray app (SCAFFOLD). // headless Windows Service: the service owns telemetry, this owns the operator // surface - a system-tray icon and the 5-click-max ServiceNow ticket flow // (DESIGN-DECISIONS-agent.md section 6). This iteration wires the tray and a @@ -8,9 +10,23 @@ namespace ShopDbEndpointAgent.Tray; // is deferred until that decision is made server-side. internal static class Program { + // Machine-wide single-instance guard. The installer starts the tray from an + // HKLM Run value for every user at logon, and an operator can also launch it + // from the Start Menu shortcut, so a second copy is easy to spawn. A named + // mutex in the Global namespace keeps exactly one tray per session: a later + // launch that cannot own the mutex exits immediately instead of stacking a + // duplicate icon. Global\ scopes across sessions on the same machine. + private const string SingleInstanceMutexName = @"Global\ShopDbEndpointAgentTray"; + [STAThread] private static void Main() { + using var singleInstance = new Mutex(true, SingleInstanceMutexName, out bool owned); + if (!owned) + { + return; + } + ApplicationConfiguration.Initialize(); Application.Run(new TrayContext()); } diff --git a/docker-compose.yml b/docker-compose.yml index dc2e63e3..3d28a3ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,6 +99,11 @@ services: retries: 3 volumes: - ./plugins:/app/plugins:ro + # Standalone operator scripts shipped inside generated bundles (the + # endpoint-agent uninstaller). Read-only, same pattern as plugins: the + # site-agent packaging route reads uninstall-shopdbagent.ps1 from here at + # download time, so it must be visible in the container. + - ./scripts:/app/scripts:ro # Self-signed TLS material for the dev/test https listener. Read-only; # generated on the host under ./certs and never baked into the image. - ./certs:/app/certs:ro diff --git a/plugins/endpoints/api/siteagent.py b/plugins/endpoints/api/siteagent.py index a585c315..9fa82544 100644 --- a/plugins/endpoints/api/siteagent.py +++ b/plugins/endpoints/api/siteagent.py @@ -40,6 +40,14 @@ from .routes import endpoints_bp STAGED_AGENT_RELPATH = os.path.join('siteagent', 'shopdbagent.exe') STAGED_TRAY_RELPATH = os.path.join('siteagent', 'shopdb-endpoint-tray.exe') +# The standalone uninstaller lives in the repo tree (not staged per-instance), +# so it ships in every bundle. Resolve it relative to this file: +# plugins/endpoints/api/siteagent.py -> repo root is three levels up. +UNINSTALLER_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))))), + 'scripts', 'uninstall-shopdbagent.ps1') + # Public TLS certificate the agent must trust when this deployment terminates # TLS itself with a self-signed or otherwise untrusted certificate. We can only # auto-bundle when THIS app owns the cert file (the dev topology, where gunicorn @@ -143,6 +151,12 @@ FILES appsettings.json Non-secret runtime config (cadence, monitors, launch paths). No credentials. install.ps1 Provisions the credentials and installs the service. + uninstall-shopdbagent.ps1 + Standalone uninstaller. Run elevated to reverse the + install: stops and deletes the service, kills agent + and tray processes, removes the tray autostart Run + key, the Start Menu shortcut, and the credential + hive, and deletes the install directory. server.crt Server TLS certificate (present only for self-signed or internal-CA deployments, either auto-detected or uploaded by the operator). install.ps1 imports it @@ -153,7 +167,13 @@ USAGE 1. Copy this whole folder to the target endpoint. 2. Right-click install.ps1 and Run with PowerShell as administrator. It writes the server URL and collector key to HKLM\\SOFTWARE\\GE\\ShopDB, - copies the agent into Program Files, and starts the service. + copies the agent into Program Files, and starts the service. It also + registers the tray to autostart for every user at logon and adds a + "ShopDB Agent" Start Menu shortcut for manual relaunch. + +UNINSTALL + Run uninstall-shopdbagent.ps1 from an elevated PowerShell to fully remove the + agent, tray autostart, Start Menu shortcut, and credentials. SECURITY install.ps1 carries a live collector key in its default -CollectorKey value. @@ -305,6 +325,36 @@ if (-not $NoTray) {{ $traySource = Join-Path $sourceDir $TrayExeName if (Test-Path $traySource) {{ Copy-Item -Path $traySource -Destination $InstallDir -Force + $installedTray = Join-Path $InstallDir $TrayExeName + + # Autostart the tray for every user at logon via the HKLM Run key (both + # 64-bit and 32-bit views, matching the credential hive). This is the + # reliable launch guarantee: a plain Start Menu shortcut does not + # autostart, and the SYSTEM service cannot cleanly spawn into the + # interactive session. The tray itself is single-instance, so a stray + # second launch is harmless. + Set-HklmValueBothViews -SubKey "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Run" -Name "ShopDbEndpointTray" -Value ('"' + $installedTray + '"') + + # Start Menu shortcut so operators can relaunch the tray on demand. This + # is discoverability only, not autostart. Written to the machine-wide + # (All Users) Programs folder. + try {{ + $commonPrograms = [Environment]::GetFolderPath([Environment+SpecialFolder]::CommonPrograms) + $shortcutPath = Join-Path $commonPrograms "ShopDB Agent.lnk" + $shell = New-Object -ComObject WScript.Shell + $shortcut = $shell.CreateShortcut($shortcutPath) + $shortcut.TargetPath = $installedTray + $shortcut.WorkingDirectory = $InstallDir + $shortcut.Description = "ShopDB Endpoint Agent tray" + $shortcut.Save() + Write-Host " created Start Menu shortcut: $shortcutPath" + }} + catch {{ Write-Warning "could not create Start Menu shortcut: $($_.Exception.Message)" }} + + # Launch the tray once now so it appears without waiting for the next + # logon. Single-instance guard makes a duplicate launch a no-op. + try {{ Start-Process -FilePath $installedTray -WorkingDirectory $InstallDir }} + catch {{ Write-Warning "could not start tray: $($_.Exception.Message)" }} }} }} @@ -420,6 +470,9 @@ def package_siteagent(): cadenceseconds, certfilename)) archive.writestr('README.txt', _README) + if os.path.isfile(UNINSTALLER_PATH): + archive.write(UNINSTALLER_PATH, 'uninstall-shopdbagent.ps1') + agent = _staged_path(STAGED_AGENT_RELPATH) if os.path.isfile(agent): archive.write(agent, 'shopdbagent.exe') diff --git a/scripts/uninstall-shopdbagent.ps1 b/scripts/uninstall-shopdbagent.ps1 index b4819ff3..95fa7724 100644 --- a/scripts/uninstall-shopdbagent.ps1 +++ b/scripts/uninstall-shopdbagent.ps1 @@ -114,6 +114,14 @@ Start-Sleep -Seconds 1 Write-Host "[3/6] Removing registry entries ..." Remove-HklmBothViews -SubKey "SOFTWARE\Microsoft\Windows\CurrentVersion\Run" -Name "ShopDbEndpointTray" Remove-HklmBothViews -SubKey "SOFTWARE\GE\ShopDB" + +# Remove the All-Users Start Menu shortcut the site-agent installer created. +$commonPrograms = [Environment]::GetFolderPath([Environment+SpecialFolder]::CommonPrograms) +$shortcutPath = Join-Path $commonPrograms "ShopDB Agent.lnk" +if (Test-Path $shortcutPath) { + try { Remove-Item -Path $shortcutPath -Force -ErrorAction Stop; Write-Host " removed Start Menu shortcut $shortcutPath" } + catch { Write-Warning " could not remove Start Menu shortcut: $($_.Exception.Message)" } +} # Drop the parent GE key only if we emptied it (some sites nest siblings under # GE such as SFLD; never tree-delete a key we do not own). foreach ($view in @([Microsoft.Win32.RegistryView]::Registry64, From 0b2218c6d70e9ebb9ed354deca118437f577ba8a Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Mon, 31 Aug 2026 16:32:16 -0400 Subject: [PATCH 17/90] Verify agent runtime, scrub install.ps1, and brand the shortcut install.ps1 carries the live collector key, so it must not linger on the endpoint. Before removing it, the installer now confirms the service reached Running (with retries) and that the tray process is up, failing loudly and keeping install.ps1 in place if the service never starts. Once healthy, it archives the uninstaller, appsettings, README, and cert into C:\ProgramData\ShopDB\EndpointAgent, deletes install.ps1, and prints the uninstaller path. The tray draws its icon at runtime and has no embedded Win32 icon, so shortcuts showed the generic exe icon. Add a headless --export-icon mode that reuses the same GDI+ drawing to write an .ico, and have install.ps1 export it and set the shortcut IconLocation. Keeps one source of brand truth with no checked-in binary. --- agent/ShopDbEndpointAgent.Tray/Program.cs | 14 +++- agent/ShopDbEndpointAgent.Tray/TrayIcon.cs | 16 +++++ plugins/endpoints/api/siteagent.py | 81 ++++++++++++++++++++-- 3 files changed, 105 insertions(+), 6 deletions(-) diff --git a/agent/ShopDbEndpointAgent.Tray/Program.cs b/agent/ShopDbEndpointAgent.Tray/Program.cs index 6e4e5840..4e8cdc0a 100644 --- a/agent/ShopDbEndpointAgent.Tray/Program.cs +++ b/agent/ShopDbEndpointAgent.Tray/Program.cs @@ -19,8 +19,20 @@ internal static class Program private const string SingleInstanceMutexName = @"Global\ShopDbEndpointAgentTray"; [STAThread] - private static void Main() + private static void Main(string[] args) { + // Headless icon export for the installer: `--export-icon ` writes + // the branded .ico and exits without starting the tray, so install.ps1 + // can point the Start Menu shortcut at a real brand icon. Kept in this + // exe so the icon has a single source (TrayIcon) and no binary is + // checked into the repo. + if (args.Length >= 2 && + string.Equals(args[0], "--export-icon", StringComparison.OrdinalIgnoreCase)) + { + TrayIcon.Export(args[1]); + return; + } + using var singleInstance = new Mutex(true, SingleInstanceMutexName, out bool owned); if (!owned) { diff --git a/agent/ShopDbEndpointAgent.Tray/TrayIcon.cs b/agent/ShopDbEndpointAgent.Tray/TrayIcon.cs index dbc0957b..b00e8bb9 100644 --- a/agent/ShopDbEndpointAgent.Tray/TrayIcon.cs +++ b/agent/ShopDbEndpointAgent.Tray/TrayIcon.cs @@ -62,6 +62,22 @@ internal static class TrayIcon } } + // Writes the branded icon to an .ico file so a Start Menu / desktop shortcut + // can show the brand instead of the generic exe icon. Reuses the same GDI+ + // drawing as the tray so there is still one source of brand truth and no + // checked-in binary. 64px suits shortcut/large-icon views; the OS downscales. + public static void Export(string path, int size = 64) + { + using var icon = Create(size); + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + using var stream = new FileStream(path, FileMode.Create, FileAccess.Write); + icon.Save(stream); + } + private static GraphicsPath RoundedRect(RectangleF rect, float radius) { float diameter = radius * 2; diff --git a/plugins/endpoints/api/siteagent.py b/plugins/endpoints/api/siteagent.py index 9fa82544..f0419a68 100644 --- a/plugins/endpoints/api/siteagent.py +++ b/plugins/endpoints/api/siteagent.py @@ -267,7 +267,7 @@ Write-Host "ServiceName : $ServiceName" # 0. Stop and remove any previous install so the file copy is a clean overwrite. $existing = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue if ($existing) {{ - Write-Host "[0/4] Removing existing service $ServiceName ..." + Write-Host "[0/6] Removing existing service $ServiceName ..." if ($existing.Status -ne "Stopped") {{ try {{ Stop-Service -Name $ServiceName -Force -ErrorAction Stop }} catch {{ Write-Warning "could not stop service: $($_.Exception.Message)" }} @@ -283,7 +283,7 @@ foreach ($name in @("shopdbagent", $trayBase)) {{ }} # 1. Provision registry credentials (both 64-bit and 32-bit views). -Write-Host "[1/4] Provisioning registry HKLM:\\SOFTWARE\\GE\\ShopDB (Registry64 + Registry32) ..." +Write-Host "[1/6] Provisioning registry HKLM:\\SOFTWARE\\GE\\ShopDB (Registry64 + Registry32) ..." Set-HklmValueBothViews -SubKey "SOFTWARE\\GE\\ShopDB" -Name "BaseUrl" -Value $BaseUrl Set-HklmValueBothViews -SubKey "SOFTWARE\\GE\\ShopDB" -Name "CollectorKey" -Value $CollectorKey @@ -307,7 +307,7 @@ if (-not [string]::IsNullOrWhiteSpace($CertFileName)) {{ }} # 2. Copy agent files to the install directory. -Write-Host "[2/4] Copying agent files to $InstallDir ..." +Write-Host "[2/6] Copying agent files to $InstallDir ..." New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null Get-ChildItem -Path $sourceDir -File | ForEach-Object {{ try {{ Unblock-File -Path $_.FullName -ErrorAction Stop }} catch {{ }} @@ -335,6 +335,14 @@ if (-not $NoTray) {{ # second launch is harmless. Set-HklmValueBothViews -SubKey "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Run" -Name "ShopDbEndpointTray" -Value ('"' + $installedTray + '"') + # Export the branded icon from the tray exe so the shortcut shows the + # ShopDB brand instead of the generic exe icon. The tray draws its icon + # at runtime (no embedded Win32 icon), so we ask it to write an .ico into + # the install dir and point the shortcut at it. + $iconPath = Join-Path $InstallDir "shopdb-agent.ico" + try {{ & $installedTray --export-icon $iconPath | Out-Null }} + catch {{ Write-Warning "could not export shortcut icon: $($_.Exception.Message)" }} + # Start Menu shortcut so operators can relaunch the tray on demand. This # is discoverability only, not autostart. Written to the machine-wide # (All Users) Programs folder. @@ -346,6 +354,7 @@ if (-not $NoTray) {{ $shortcut.TargetPath = $installedTray $shortcut.WorkingDirectory = $InstallDir $shortcut.Description = "ShopDB Endpoint Agent tray" + if (Test-Path $iconPath) {{ $shortcut.IconLocation = "$iconPath,0" }} $shortcut.Save() Write-Host " created Start Menu shortcut: $shortcutPath" }} @@ -359,13 +368,75 @@ if (-not $NoTray) {{ }} # 3. Install and start the service. The agent supports its own service install. -Write-Host "[3/4] Installing and starting the service ..." +Write-Host "[3/6] Installing and starting the service ..." $installedAgent = Join-Path $InstallDir "shopdbagent.exe" $binPath = '"' + $installedAgent + '"' sc.exe create $ServiceName binPath= $binPath start= auto DisplayName= "ShopDB Endpoint Agent" | Out-Null sc.exe start $ServiceName | Out-Null -Write-Host "[4/4] Done." -ForegroundColor Green +# 4. Confirm the service (and tray, when installed) actually reached a running +# state before we destroy the installer. sc.exe start returns before the service +# is up, and a failed start would otherwise leave a half-installed endpoint with +# the secret-bearing install.ps1 already deleted. Poll with a short backoff and +# retry the start a few times; only proceed to cleanup once healthy. +Write-Host "[4/6] Verifying agent runtime ..." +$serviceOk = $false +for ($attempt = 1; $attempt -le 5; $attempt++) {{ + Start-Sleep -Seconds 3 + $svc = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue + if ($svc -and $svc.Status -eq "Running") {{ $serviceOk = $true; break }} + Write-Host " service not running yet (attempt $attempt/5); retrying start ..." + try {{ Start-Service -Name $ServiceName -ErrorAction Stop }} catch {{ }} +}} +if (-not $serviceOk) {{ + throw "Service $ServiceName did not reach Running after 5 attempts. Leaving install.ps1 in place for diagnosis; re-run once resolved." +}} +Write-Host " service is running." -ForegroundColor Green + +$trayBase2 = [System.IO.Path]::GetFileNameWithoutExtension($TrayExeName) +if (-not $NoTray -and (Test-Path (Join-Path $InstallDir $TrayExeName))) {{ + $trayOk = $false + for ($attempt = 1; $attempt -le 5; $attempt++) {{ + if (Get-Process -Name $trayBase2 -ErrorAction SilentlyContinue) {{ $trayOk = $true; break }} + Write-Host " tray not running yet (attempt $attempt/5); relaunching ..." + try {{ Start-Process -FilePath (Join-Path $InstallDir $TrayExeName) -WorkingDirectory $InstallDir }} catch {{ }} + Start-Sleep -Seconds 2 + }} + if ($trayOk) {{ Write-Host " tray is running." -ForegroundColor Green }} + else {{ Write-Warning "tray did not confirm running; it will retry at next logon via the Run key." }} +}} + +# 5. Preserve the install artifacts (minus the secret-bearing install.ps1) under +# ProgramData so the uninstaller and appsettings/cert are available later, then +# scrub install.ps1 from wherever it was run. install.ps1 carries the live +# collector key in its default parameter, so it must not linger on the endpoint. +Write-Host "[5/6] Archiving install artifacts and removing install.ps1 ..." +$dataDir = Join-Path $env:ProgramData "ShopDB\\EndpointAgent" +New-Item -ItemType Directory -Force -Path $dataDir | Out-Null +foreach ($name in @("uninstall-shopdbagent.ps1", "appsettings.json", "README.txt", $CertFileName)) {{ + if ([string]::IsNullOrWhiteSpace($name)) {{ continue }} + $src = Join-Path $sourceDir $name + if (Test-Path $src) {{ + try {{ Copy-Item -Path $src -Destination $dataDir -Force }} + catch {{ Write-Warning "could not archive $name : $($_.Exception.Message)" }} + }} +}} +$uninstaller = Join-Path $dataDir "uninstall-shopdbagent.ps1" + +$installSelf = Join-Path $sourceDir "install.ps1" +if (Test-Path $installSelf) {{ + try {{ Remove-Item -Path $installSelf -Force -ErrorAction Stop; Write-Host " removed install.ps1 (carried the collector key)." }} + catch {{ Write-Warning "could not remove install.ps1 - delete it manually, it contains the collector key: $($_.Exception.Message)" }} +}} + +Write-Host "[6/6] Done." -ForegroundColor Green +if (Test-Path $uninstaller) {{ + Write-Host "Uninstaller: $uninstaller" -ForegroundColor Cyan + Write-Host " To remove the agent, run that script from an elevated PowerShell." +}} +else {{ + Write-Warning "uninstaller was not bundled; fetch uninstall-shopdbagent.ps1 from the ShopDB repo scripts/ folder if needed." +}} """.format( baseurl=baseurl_literal, collectorkey=collectorkey_literal, From 42db980ca9ee093f9f492bbecf418bf1c6c9e0c7 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Tue, 1 Sep 2026 15:44:03 -0400 Subject: [PATCH 18/90] Add endpoint monitor catalog and warn-only drift reconciler Operators need to see when a machine's running services or processes fall outside the desired state defined by policy. This adds a reusable monitor catalog (shipped read-only rows plus user-built local rows), computes drift server-side on agent check-in, and surfaces a warning on the endpoint panels. Remediation stays manual for now; automation can follow once demand is proven. Includes migrations 0006-0008 and the monitors API client. --- frontend/src/api/index.js | 19 ++ plugins/endpoints/api/__init__.py | 2 + plugins/endpoints/api/monitors.py | 204 +++++++++++++++ plugins/endpoints/api/routes.py | 232 +++++++++++++++-- plugins/endpoints/commands.py | 68 +++++ plugins/endpoints/frontend/routes.js | 6 + .../frontend/views/AgentKeySettings.vue | 21 +- .../frontend/views/MonitorsSettings.vue | 232 +++++++++++++++++ .../versions/0006_endpoint_monitors.py | 151 +++++++++++ .../versions/0007_shipped_monitor_policy.py | 234 +++++++++++++++++ .../versions/0008_endpoint_drift.py | 51 ++++ plugins/endpoints/models/__init__.py | 2 + plugins/endpoints/models/endpoint.py | 7 + plugins/endpoints/models/monitor.py | 87 +++++++ plugins/endpoints/plugin.py | 40 +-- plugins/endpoints/services/__init__.py | 0 plugins/endpoints/services/reconciler.py | 153 +++++++++++ shopdb/plugins/alembic_template.py | 2 +- tests/test_core/test_public_read_surface.py | 8 + tests/test_plugin_migrations.py | 8 +- .../test_endpoints_policy_monitors.py | 199 +++++++++++++++ .../test_plugins/test_endpoints_reconciler.py | 238 ++++++++++++++++++ .../test_endpoints_shipped_monitor_policy.py | 141 +++++++++++ 23 files changed, 2054 insertions(+), 51 deletions(-) create mode 100644 plugins/endpoints/api/monitors.py create mode 100644 plugins/endpoints/frontend/views/MonitorsSettings.vue create mode 100644 plugins/endpoints/migrations/versions/0006_endpoint_monitors.py create mode 100644 plugins/endpoints/migrations/versions/0007_shipped_monitor_policy.py create mode 100644 plugins/endpoints/migrations/versions/0008_endpoint_drift.py create mode 100644 plugins/endpoints/models/monitor.py create mode 100644 plugins/endpoints/services/__init__.py create mode 100644 plugins/endpoints/services/reconciler.py create mode 100644 tests/test_plugins/test_endpoints_policy_monitors.py create mode 100644 tests/test_plugins/test_endpoints_reconciler.py create mode 100644 tests/test_plugins/test_endpoints_shipped_monitor_policy.py diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 6bc0f1b3..57fcc970 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -178,6 +178,25 @@ export const endpointsApi = { }, telemetryMeta(assetId) { return api.get(`/endpoints/asset/${assetId}/telemetrymeta`) + }, + // Monitor catalog: reusable process/service definitions. Shipped rows are + // product-owned and read-only; local rows are user-built on the Settings page. + monitors: { + list() { + return api.get('/endpoints/monitors') + }, + create(data) { + return api.post('/endpoints/monitors', data) + }, + update(monitorid, data) { + return api.put(`/endpoints/monitors/${monitorid}`, data) + }, + remove(monitorid) { + return api.delete(`/endpoints/monitors/${monitorid}`) + }, + resolve(pctype) { + return api.get('/endpoints/monitors/resolve', { params: { pctype } }) + } } } diff --git a/plugins/endpoints/api/__init__.py b/plugins/endpoints/api/__init__.py index 2fc14121..53667058 100644 --- a/plugins/endpoints/api/__init__.py +++ b/plugins/endpoints/api/__init__.py @@ -5,5 +5,7 @@ from .routes import endpoints_bp from . import adoption # noqa: F401 # Attaches the site agent generator packaging routes to endpoints_bp. from . import siteagent # noqa: F401 +# Attaches the monitor catalog CRUD and resolve routes to endpoints_bp. +from . import monitors # noqa: F401 __all__ = ['endpoints_bp'] diff --git a/plugins/endpoints/api/monitors.py b/plugins/endpoints/api/monitors.py new file mode 100644 index 00000000..4e8f7e0e --- /dev/null +++ b/plugins/endpoints/api/monitors.py @@ -0,0 +1,204 @@ +"""Endpoint monitor definitions API (the monitor catalog CRUD + resolver). + +A monitor definition is the reusable "what" an operator builds on the Settings +page: a named process or service with a bespoke display label and optional +launch/collector paths. Attaching a monitor to a PC type is a site-policy +concern (see plugins/computers policies); this surface owns the definitions +themselves plus the resolve-by-pctype read the agent uses at runtime. + +Shipped rows (source='shipped') are product-owned: readable by everyone with +the view permission but not editable or deletable through this API. + +The resolve-by-pctype read returns the monitor set a deployed agent should watch +for its PC type: the shipped defaults for 'Shopfloor PC' unioned with whatever +active site policies attach to that type. Shipped monitors are hardcoded to +'Shopfloor PC' as a proof of concept until they are authored through the policy +route like any user-built monitor. +""" + +from flask import request +from flask_jwt_extended import current_user + +from shopdb.api import ( + db, success_response, error_response, ErrorCodes, require_permission, + require_role, +) + +from ..models import EndpointMonitor +from .routes import endpoints_bp + + +# Shipped monitors default to this PC type until they are authored as policies. +SHIPPED_DEFAULT_PCTYPE = 'Shopfloor PC' + +_VALID_KINDS = ('process', 'service') + + +def _validate_monitor_payload(body, kind): + """Return an error message for an invalid create/update body, or None.""" + displayname = (body.get('displayname') or '').strip() + if not displayname: + return 'displayname is required' + if kind not in _VALID_KINDS: + return "kind must be 'process' or 'service'" + if kind == 'process' and not (body.get('processname') or '').strip(): + return 'processname is required for a process monitor' + if kind == 'service' and not (body.get('servicename') or '').strip(): + return 'servicename is required for a service monitor' + return None + + +@endpoints_bp.route('/monitors', methods=['GET']) +@require_permission('endpoints.view') +def list_monitors(): + """List monitor definitions (shipped and local).""" + monitors = EndpointMonitor.query.order_by( + EndpointMonitor.source.desc(), + EndpointMonitor.displayname.asc()).all() + return success_response([monitor.to_dict() for monitor in monitors]) + + +@endpoints_bp.route('/monitors', methods=['POST']) +@require_role('admin') +def create_monitor(): + """Create a user-defined (local) monitor definition.""" + body = request.get_json(silent=True) or {} + kind = (body.get('kind') or '').strip().lower() + problem = _validate_monitor_payload(body, kind) + if problem: + return error_response(ErrorCodes.VALIDATION_ERROR, problem, http_code=400) + + monitor = EndpointMonitor( + kind=kind, + displayname=(body.get('displayname') or '').strip(), + processname=(body.get('processname') or '').strip() or None, + servicename=(body.get('servicename') or '').strip() or None, + launchpath=(body.get('launchpath') or '').strip() or None, + datacollectorpath=(body.get('datacollectorpath') or '').strip() or None, + logcollectorpath=(body.get('logcollectorpath') or '').strip() or None, + source='local', + isactive=bool(body.get('isactive', True)), + ) + db.session.add(monitor) + db.session.commit() + return success_response(monitor.to_dict(), message='Monitor created', + http_code=201) + + +@endpoints_bp.route('/monitors/', methods=['PUT']) +@require_role('admin') +def update_monitor(monitor_id): + """Update a local monitor definition. Shipped rows are read-only.""" + monitor = db.session.get(EndpointMonitor, monitor_id) + if monitor is None: + return error_response(ErrorCodes.NOT_FOUND, 'Monitor not found', + http_code=404) + if monitor.source != 'local': + return error_response(ErrorCodes.FORBIDDEN, + 'Shipped monitors cannot be edited', http_code=403) + + body = request.get_json(silent=True) or {} + kind = (body.get('kind') or monitor.kind or '').strip().lower() + problem = _validate_monitor_payload( + { + 'displayname': body.get('displayname', monitor.displayname), + 'processname': body.get('processname', monitor.processname), + 'servicename': body.get('servicename', monitor.servicename), + }, + kind, + ) + if problem: + return error_response(ErrorCodes.VALIDATION_ERROR, problem, http_code=400) + + monitor.kind = kind + monitor.displayname = (body.get('displayname', monitor.displayname) or '').strip() + monitor.processname = (body.get('processname', monitor.processname) or '').strip() or None + monitor.servicename = (body.get('servicename', monitor.servicename) or '').strip() or None + monitor.launchpath = (body.get('launchpath', monitor.launchpath) or '').strip() or None + monitor.datacollectorpath = (body.get('datacollectorpath', monitor.datacollectorpath) or '').strip() or None + monitor.logcollectorpath = (body.get('logcollectorpath', monitor.logcollectorpath) or '').strip() or None + if 'isactive' in body: + monitor.isactive = bool(body.get('isactive')) + db.session.commit() + return success_response(monitor.to_dict(), message='Monitor updated') + + +@endpoints_bp.route('/monitors/', methods=['DELETE']) +@require_role('admin') +def delete_monitor(monitor_id): + """Delete a local monitor definition. Shipped rows are protected.""" + monitor = db.session.get(EndpointMonitor, monitor_id) + if monitor is None: + return error_response(ErrorCodes.NOT_FOUND, 'Monitor not found', + http_code=404) + if monitor.source != 'local': + return error_response(ErrorCodes.FORBIDDEN, + 'Shipped monitors cannot be deleted', http_code=403) + db.session.delete(monitor) + db.session.commit() + return success_response(None, message='Monitor deleted') + + +def _resolve_pctype_monitors(pctype): + """Monitor definitions a PC type should watch: shipped defaults + policy. + + Shipped monitors are attached to SHIPPED_DEFAULT_PCTYPE as a code rule; + policy-attached monitors come from the computers policy resolver by PC type. + Returns a list of EndpointMonitor rows (active only), de-duped by id. + """ + selected = {} + + if pctype == SHIPPED_DEFAULT_PCTYPE: + for monitor in EndpointMonitor.query.filter_by( + source='shipped', isactive=True).all(): + selected[monitor.endpointmonitorid] = monitor + + computertypeid = _computertypeid_for(pctype) + if computertypeid is not None: + policy_ids = _policy_monitor_ids(computertypeid) + if policy_ids: + for monitor in EndpointMonitor.query.filter( + EndpointMonitor.endpointmonitorid.in_(policy_ids), + EndpointMonitor.isactive.is_(True)).all(): + selected[monitor.endpointmonitorid] = monitor + + return list(selected.values()) + + +def _computertypeid_for(pctype): + """Resolve a PC type name to its computertypeid, or None.""" + try: + from plugins.computers.models import ComputerType + except Exception: + return None + row = ComputerType.query.filter_by(computertype=pctype).first() + return row.computertypeid if row else None + + +def _policy_monitor_ids(computertypeid): + """Monitor ids attached to a PC type by effective policy, or empty set.""" + try: + from plugins.computers.services.policyresolver import monitor_ids_for_pctype + except Exception: + return set() + return monitor_ids_for_pctype(computertypeid) + + +@endpoints_bp.route('/monitors/resolve', methods=['GET']) +@require_permission('endpoints.view') +def resolve_monitors(): + """Resolve the monitor set for a PC type (agent runtime fetch). + + Query param pctype names the computer type. Returns the definitions the + agent should watch: shipped defaults for 'Shopfloor PC' plus any monitors + active site policies attach to that type. + """ + pctype = (request.args.get('pctype') or '').strip() + if not pctype: + return error_response(ErrorCodes.VALIDATION_ERROR, + 'pctype is required', http_code=400) + monitors = _resolve_pctype_monitors(pctype) + return success_response({ + 'pctype': pctype, + 'monitors': [monitor.to_dict() for monitor in monitors], + }) diff --git a/plugins/endpoints/api/routes.py b/plugins/endpoints/api/routes.py index 087181c2..6085822c 100644 --- a/plugins/endpoints/api/routes.py +++ b/plugins/endpoints/api/routes.py @@ -87,7 +87,8 @@ def _format_uptime(seconds): # Endpoint state # ============================================================================= -@endpoints_bp.route('', methods=['GET']) +@endpoints_bp.route('', methods=['GET'], strict_slashes=False) +@endpoints_bp.route('/', methods=['GET'], strict_slashes=False) @require_permission('endpoints.fleet') def list_endpoints(): """List reported endpoint states. @@ -396,6 +397,24 @@ def list_endpoint_events(asset_id: int): # Asset panels (ADR-010) - shaped for the generic PluginAssetPanels renderer # ============================================================================= +def _drift_for_state(state): + """Parse the cached warn-only reconciler drift list off an EndpointState. + + driftjson is written on every check-in by the endpoints ingest: the monitors + site policy expects running that the latest snapshot did not report running. + Null means never computed; an empty list means computed with no drift. + Tolerant of a malformed blob so a bad cache never breaks the panel. + """ + raw = getattr(state, 'driftjson', None) if state else None + if not raw: + return [] + try: + parsed = json.loads(raw) + except (TypeError, ValueError): + return [] + return parsed if isinstance(parsed, list) else [] + + @endpoints_bp.route('/asset//status', methods=['GET']) @require_permission('endpoints.view') def panel_status(asset_id: int): @@ -425,6 +444,19 @@ def panel_status(asset_id: int): state.lastreportedat.isoformat() + 'Z' if state.lastreportedat else None), ] fields = [f for f in fields if f['value'] not in (None, '')] + + # Warn-only reconciler summary: monitors site policy expects to be running + # that the last check-in did not report as running. Surfaced as a status + # field so an operator sees the drift on the panel; nothing is auto-started. + drift = _drift_for_state(state) + if drift: + names = ', '.join(entry.get('displayname') or entry.get('name') + for entry in drift) + fields.append(field( + 'Needs attention', + '{} monitor(s) expected running but not reported: {}'.format( + len(drift), names))) + return success_response({'fields': fields}) @@ -446,28 +478,100 @@ def panel_telemetry_meta(asset_id: int): return success_response({'lastreportedat': stamp}) +def _pctype_for_asset(asset_id): + """Resolve an asset's PC type name, or None. + + Endpoints stays optional-on-computers: a missing plugin, a non-computer + asset, or an unset type all resolve to None so the caller falls back to + reported-only behaviour. + """ + try: + from plugins.computers.models import Computer, ComputerType + except Exception: + return None + row = (db.session.query(ComputerType.computertype) + .join(Computer, + Computer.computertypeid == ComputerType.computertypeid) + .filter(Computer.assetid == asset_id) + .first()) + return row[0] if row else None + + +def _resolved_monitors_for_asset(asset_id): + """Monitor definitions attached to this asset's PC type by policy/shipped. + + Reuses the endpoints monitor resolver (shipped defaults plus policy-attached + monitors). Returns a list of EndpointMonitor rows, or an empty list when the + PC type is unknown or nothing resolves. + """ + pctype = _pctype_for_asset(asset_id) + if not pctype: + return [] + try: + from .monitors import _resolve_pctype_monitors + return _resolve_pctype_monitors(pctype) or [] + except Exception: + return [] + + +def _row_action_dict(app, kind, name, running): + """Build the inline Start/Stop action map for a services/processes row.""" + targets = row_actions_for(app, kind, name) + if not targets: + return None + return { + 'running': running, + 'start': targets['start'], + 'stop': targets['stop'], + 'status': targets['status'], + 'target': targets['target'], + 'targetkind': targets['targetkind'], + } + + @endpoints_bp.route('/asset//services', methods=['GET']) @require_permission('endpoints.view') def panel_services(asset_id: int): """List panel: monitored services/processes with a run-state badge per row. Reshapes the latest servicesjson snapshot into rows the list renderer maps - to name + status badge. Tolerant of a missing or unexpected snapshot shape: - an unknown state renders as an Unknown badge rather than failing. + to name + status badge, then MERGES the monitors that site policy (and the + shipped defaults) attach to this asset's PC type. A policy-attached monitor + the agent has not yet reported surfaces as an Unknown row with a Start + control, so an operator sees the intent immediately rather than waiting for + the next agent snapshot. Tolerant of a missing snapshot or plugin: an + unknown state renders as an Unknown badge rather than failing. """ state = EndpointState.query.filter_by(assetid=asset_id).first() - if not state or not state.servicesjson: - return success_response([]) - maycommand = caller_holds('endpoints.command') - try: - services = json.loads(state.servicesjson) - except (TypeError, ValueError): - return success_response([]) + services = [] + if state and state.servicesjson: + try: + services = json.loads(state.servicesjson) or [] + except (TypeError, ValueError): + services = [] rows = [] - for entry in (services or []): + # Reported identities, keyed (kind, lowercased name), so a policy monitor + # the agent already reports is not duplicated as a synthetic row. + reported = set() + # Warn-only reconciler drift, keyed (kind, lowercased name), so a row that is + # expected running but not reported running can be flagged for the operator. + # kind may be None on an older snapshot, so match on name too. + drift_keys = set() + for entry in _drift_for_state(state): + name = (entry.get('name') or '').strip().lower() + if not name: + continue + drift_keys.add(((entry.get('kind') or '').lower() or None, name)) + drift_keys.add((None, name)) + + def _is_drift(kind, name): + key = (name or '').strip().lower() + return (kind, key) in drift_keys or (None, key) in drift_keys + + for entry in services: if isinstance(entry, dict): name = entry.get('name') or entry.get('service') or 'Unknown' raw = (entry.get('state') or entry.get('status') or '').strip().lower() @@ -476,9 +580,10 @@ def panel_services(asset_id: int): else: name, raw, app, kind = str(entry), '', None, None running = _service_running(entry) + normkind = (kind or '').lower() or None row = { 'name': name, - 'kind': (kind or '').lower() or None, + 'kind': normkind, 'statelabel': 'Running' if running else (raw.title() or 'Unknown'), # Amber for paused and the four Windows transitional states. A # service stuck in Start Pending is failing to start on a retry @@ -493,17 +598,42 @@ def panel_services(asset_id: int): # emission never consulted the caller, so a read-only viewer was shown # controls on a shop-floor PC that 403 on click - wrong signal, and it # gets worse the moment guests can read this panel. - targets = row_actions_for(app, kind, name) if maycommand else None - if targets: - row['action'] = { - 'running': running, - 'start': targets['start'], - 'stop': targets['stop'], - 'status': targets['status'], - 'target': targets['target'], - 'targetkind': targets['targetkind'], - } + action = _row_action_dict(app, kind, name, running) if maycommand else None + if action: + row['action'] = action + if not running and _is_drift(normkind, name): + row['drift'] = True rows.append(row) + reported.add((normkind, name.strip().lower())) + + # Merge policy/shipped monitors this PC type should watch. A monitor already + # reported by the agent stays as its live row; one not yet seen is appended + # as an Unknown row so the operator sees the intended coverage and can Start + # it. The monitor's displayname is the operator label (e.g. 'msedge.exe' + # -> 'Edge'); its process/service name is the wire target for actions. + for monitor in _resolved_monitors_for_asset(asset_id): + kind = (monitor.kind or '').lower() or None + osname = (monitor.processname if kind == 'process' + else monitor.servicename) + osname = (osname or '').strip() + if not osname: + continue + if (kind, osname.lower()) in reported: + continue + row = { + 'name': monitor.displayname or osname, + 'kind': kind, + 'statelabel': 'Unknown', + 'statecolor': 'gray', + } + action = _row_action_dict(None, kind, osname, False) if maycommand else None + if action: + row['action'] = action + if _is_drift(kind, osname): + row['drift'] = True + rows.append(row) + reported.add((kind, osname.lower())) + return success_response(rows) @@ -1335,6 +1465,64 @@ def enqueue_endpoint_command(asset_id: int): message='Command queued', http_code=201) +@endpoints_bp.route('/monitors/resolved', methods=['GET']) +def resolved_monitors_for_agent(): + """Agent watch list: the monitors this endpoint should report on. + + Collector-token auth (not JWT), same as the command channel. The agent + calls this each telemetry cycle with its hostname and receives the process + and service monitors that site policy (and the shipped defaults) attach to + this endpoint's PC type, so a policy-attached monitor is enumerated and + reported without editing appsettings.json on the box. The shape mirrors the + agent's MonitorConfig (processes[]/services[] of {name, app}) so the agent + can bind it directly: name is the wire identity (process image or service + key), app is the operator label. An unknown hostname or a PC type with no + monitors returns empty lists, which the agent merges as "watch nothing extra". + """ + error = _collector_auth_or_error() + if error is not None: + return error + + hostname = (request.args.get('hostname') or '').strip() + if not hostname: + return error_response(ErrorCodes.VALIDATION_ERROR, + 'hostname is required') + + asset_id = _resolve_asset_for_hostname(hostname) + if asset_id is None: + return error_response(ErrorCodes.NOT_FOUND, + f'Unknown hostname: {hostname!r}', http_code=404) + + processes = [] + services = [] + for monitor in _resolved_monitors_for_asset(asset_id): + kind = (monitor.kind or '').lower() + if kind == 'process': + wirename = (monitor.processname or '').strip() + bucket = processes + elif kind == 'service': + wirename = (monitor.servicename or '').strip() + bucket = services + else: + continue + if not wirename: + continue + entry = { + 'name': wirename, + 'app': (monitor.displayname or wirename), + } + # Only process monitors carry a launch path, and only when set. The + # agent never trusts it blindly: it validates the path against an + # approved-root allowlist before it becomes launchable (fail closed). + if kind == 'process': + launchpath = (monitor.launchpath or '').strip() + if launchpath: + entry['launchpath'] = launchpath + bucket.append(entry) + + return success_response({'processes': processes, 'services': services}) + + @endpoints_bp.route('/commands/next', methods=['GET']) def claim_next_command(): """Agent long-poll: claim the next pending command for a hostname. diff --git a/plugins/endpoints/commands.py b/plugins/endpoints/commands.py index fdf3f14c..31b05376 100644 --- a/plugins/endpoints/commands.py +++ b/plugins/endpoints/commands.py @@ -185,6 +185,36 @@ def requires_target(action): return action_targetkind(action) is not None +# Monitor attachment as a policy action. A monitor definition is bound to a PC +# type by carrying a ConfigurationPolicyAction whose name is 'monitor:'. +# Distinct names let one policy attach several monitors and fold naturally +# (first-match-wins per name), and they are validated in the policy layer but +# never enqueued as commands because ALLOWED_ACTIONS_FOR intersects with +# COMMAND_ALLOWLIST, which carries no monitor: entries. +MONITOR_ACTION_PREFIX = 'monitor:' + + +def monitor_action_name(endpointmonitorid): + """Build the policy action name that attaches a monitor definition.""" + return f'{MONITOR_ACTION_PREFIX}{endpointmonitorid}' + + +def is_monitor_action(action): + """True when action names a monitor attachment (monitor:).""" + return isinstance(action, str) and action.startswith(MONITOR_ACTION_PREFIX) + + +def monitor_id_from_action(action): + """The monitor id an action attaches, or None when it is not a monitor action.""" + if not is_monitor_action(action): + return None + raw = action[len(MONITOR_ACTION_PREFIX):] + try: + return int(raw) + except (TypeError, ValueError): + return None + + def ALLOWED_ACTIONS_FOR(pctype=None, assetid=None): """The action names an endpoint may run. @@ -211,6 +241,44 @@ def ALLOWED_ACTIONS_FOR(pctype=None, assetid=None): # behavior). This is intentionally permissive; deny-by-default would be # a separate, explicit setting. return base + # Attaching a monitor expresses intent to watch a process/service AND to + # control it: the panel renders that row's Start/Stop/Status buttons, so the + # gate must permit the matching generic primitives or every click 409s + # "not permitted by policy". Expand each monitor: to its kind's control + # actions before intersecting. + allowed = allowed | _monitor_control_actions(allowed) # Never widen past the code-level allowlist: policy can only gate within it. return base & allowed + +def _monitor_control_actions(action_names): + """Control primitives granted by the monitor: attachments in a policy. + + A process monitor grants process_start/stop/status; a service monitor grants + service_start/stop/status. The monitor's kind is read from the catalog, so a + definition's kind is the single source of truth for what its row can do. + """ + ids = set() + for name in action_names: + if isinstance(name, str) and name.startswith(MONITOR_ACTION_PREFIX): + monitor_id = monitor_id_from_action(name) + if monitor_id is not None: + ids.add(monitor_id) + if not ids: + return set() + + try: + from .models import EndpointMonitor + except Exception: + return set() + + granted = set() + for monitor in EndpointMonitor.query.filter( + EndpointMonitor.endpointmonitorid.in_(ids)).all(): + kind = (monitor.kind or '').lower() + if kind == 'process': + granted |= {'process_start', 'process_stop', 'process_status'} + elif kind == 'service': + granted |= {'service_start', 'service_stop', 'service_status'} + return granted + diff --git a/plugins/endpoints/frontend/routes.js b/plugins/endpoints/frontend/routes.js index bb9e2d28..0bf67bd9 100644 --- a/plugins/endpoints/frontend/routes.js +++ b/plugins/endpoints/frontend/routes.js @@ -18,6 +18,12 @@ export default [ component: () => import('./views/AgentKeySettings.vue'), meta: { requiresAuth: true, requiresAdmin: true, plugin: 'endpoints' } }, + { + path: 'settings/endpoints/monitors', + name: 'endpoints-monitors-settings', + component: () => import('./views/MonitorsSettings.vue'), + meta: { requiresAuth: true, requiresAdmin: true, plugin: 'endpoints' } + }, // TEMPORARY / DEV ONLY. A manual bench to exercise the update_app and // run_script delivery path end to end while Cameron builds the real UI. // Gated on endpoints.deploy inside the view; remove once the production diff --git a/plugins/endpoints/frontend/views/AgentKeySettings.vue b/plugins/endpoints/frontend/views/AgentKeySettings.vue index 539a1e04..e3816c1e 100644 --- a/plugins/endpoints/frontend/views/AgentKeySettings.vue +++ b/plugins/endpoints/frontend/views/AgentKeySettings.vue @@ -307,7 +307,7 @@

Agent configuration

@@ -319,11 +319,16 @@ stamped into the build above so a deployed agent needs no local edits.

- Coming soon. These controls arrive with the C# agent - runtime. Until then the agent uses its built-in defaults. + Monitored processes and services are defined as reusable monitors and + attached to PC types through site policies, so a deployed agent fetches + the right set for its type.

-
@@ -332,8 +337,10 @@ diff --git a/plugins/endpoints/migrations/versions/0006_endpoint_monitors.py b/plugins/endpoints/migrations/versions/0006_endpoint_monitors.py new file mode 100644 index 00000000..db0bc87c --- /dev/null +++ b/plugins/endpoints/migrations/versions/0006_endpoint_monitors.py @@ -0,0 +1,151 @@ +"""Reusable endpoint monitor definitions (the monitor catalog). + +Adds endpointmonitors: one row per process or service the agent can watch, with +a bespoke display label separate from the raw OS name and optional launch and +collector paths. A definition is the "what"; attaching it to a PC type is a +separate site-policy concern. + +Seeds the four monitors we ship today, lifted from the agent's appsettings.json +so the catalog matches what a deployed agent already watches: + + process dncMain.exe -> eDNC + process udc.exe -> Universal Data Collector + process agent.exe -> MTConnect + service MTConnect UDC Agent -> MTConnect + +Shipped rows are product-owned (source='shipped'): the CRUD API refuses to edit +or delete them. Launch paths mirror the agent's launchpaths block; collector and +log paths are left null until site paths are supplied. + +Idempotent: creation checks the live schema first so a db.create_all() site +(which already built the table) re-runs clean on strict MySQL 8. Seeding is a +separate check on row count, so a site that built the table without the shipped +rows still gets them. +""" +from alembic import op + +import sqlalchemy as sa + +from datetime import datetime + + +# revision identifiers, used by Alembic. +revision = 'endpoints0006monitors' +down_revision = 'endpoints0005cmdpin' +branch_labels = None +depends_on = None + + +def _has_table(table_name): + from sqlalchemy import inspect + + return inspect(op.get_bind()).has_table(table_name) + + +def upgrade(): + bind = op.get_bind() + + if not _has_table('endpointmonitors'): + op.create_table( + 'endpointmonitors', + sa.Column('endpointmonitorid', sa.Integer(), nullable=False), + sa.Column('kind', sa.String(length=16), nullable=False), + sa.Column('displayname', sa.String(length=128), nullable=False), + sa.Column('processname', sa.String(length=255), nullable=True), + sa.Column('servicename', sa.String(length=255), nullable=True), + sa.Column('launchpath', sa.String(length=512), nullable=True), + sa.Column('datacollectorpath', sa.String(length=512), nullable=True), + sa.Column('logcollectorpath', sa.String(length=512), nullable=True), + sa.Column('source', sa.String(length=16), nullable=False, + server_default='local'), + sa.Column('isactive', sa.Boolean(), nullable=False, + server_default=sa.true()), + sa.Column('createdat', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('endpointmonitorid'), + sa.UniqueConstraint('kind', 'displayname', + name='uq_endpointmonitors_kinddisplay'), + ) + op.create_index('idx_endpointmonitors_kind', 'endpointmonitors', ['kind']) + op.create_index('idx_endpointmonitors_source', 'endpointmonitors', ['source']) + + # Seed the shipped catalog independently of table creation so a + # db.create_all() site (which already built the table) still gets the rows. + existing = bind.execute( + sa.text('SELECT COUNT(*) FROM endpointmonitors')).scalar() + if existing: + return + + monitors = sa.table( + 'endpointmonitors', + sa.column('kind', sa.String), + sa.column('displayname', sa.String), + sa.column('processname', sa.String), + sa.column('servicename', sa.String), + sa.column('launchpath', sa.String), + sa.column('datacollectorpath', sa.String), + sa.column('logcollectorpath', sa.String), + sa.column('source', sa.String), + sa.column('isactive', sa.Boolean), + sa.column('createdat', sa.DateTime), + ) + + now = datetime.utcnow() + op.bulk_insert(monitors, [ + { + 'kind': 'process', + 'displayname': 'eDNC', + 'processname': 'dncMain.exe', + 'servicename': None, + 'launchpath': r'C:\Program Files (x86)\Dnc\bin\DncMain.exe', + 'datacollectorpath': None, + 'logcollectorpath': None, + 'source': 'shipped', + 'isactive': True, + 'createdat': now, + }, + { + 'kind': 'process', + 'displayname': 'Universal Data Collector', + 'processname': 'udc.exe', + 'servicename': None, + 'launchpath': r'C:\Program Files\UDC\UDC.exe', + 'datacollectorpath': None, + 'logcollectorpath': None, + 'source': 'shipped', + 'isactive': True, + 'createdat': now, + }, + { + 'kind': 'process', + 'displayname': 'MTConnect', + 'processname': 'agent.exe', + 'servicename': None, + 'launchpath': r'C:\ProgramData\udc\mtconnect_udc\agent\run.bat', + 'datacollectorpath': None, + 'logcollectorpath': None, + 'source': 'shipped', + 'isactive': True, + 'createdat': now, + }, + { + 'kind': 'service', + 'displayname': 'MTConnect', + 'processname': None, + 'servicename': 'MTConnect UDC Agent', + 'launchpath': None, + 'datacollectorpath': None, + 'logcollectorpath': None, + 'source': 'shipped', + 'isactive': True, + 'createdat': now, + }, + ]) + + +def downgrade(): + if not _has_table('endpointmonitors'): + return + + op.drop_index('idx_endpointmonitors_source', table_name='endpointmonitors') + op.drop_index('idx_endpointmonitors_kind', table_name='endpointmonitors') + op.drop_table('endpointmonitors') diff --git a/plugins/endpoints/migrations/versions/0007_shipped_monitor_policy.py b/plugins/endpoints/migrations/versions/0007_shipped_monitor_policy.py new file mode 100644 index 00000000..eb43a6cc --- /dev/null +++ b/plugins/endpoints/migrations/versions/0007_shipped_monitor_policy.py @@ -0,0 +1,234 @@ +"""Ship the shopfloor app monitors as a read-only Site Policy. + +Wraps the four shipped endpointmonitors (eDNC, Universal Data Collector, +MTConnect process, MTConnect UDC Agent service) in a product-owned policy bundle +so the agent stops carrying them as hardcoded appsettings.json defaults and reads +its watch-list from the server instead. + +Seeds two computers-plugin rows, both source='shipped' (read-only in the UI, the +CRUD API refuses to edit or delete them): + + ConfigurationPolicy "Shopfloor App Monitors" + scope {"type":"true"} (every endpoint), one action per shipped monitor, + named 'monitor:', mode='gate', desiredstate=NULL. + + Gate, not desired, on purpose. A monitor REPORTS a process/service state and + exposes Start/Stop in the Monitor UI; the state read happens on manual + refresh and on agent check-in, so attaching a monitor never generates fleet + traffic on its own and a policy edit does not push a collection. A 'desired' + running state is a separate, deliberate opt-in that keeps an app alive by + trying to START the process when a launch path is mapped - never something + the product ships switched on for every endpoint. + + BusinessPolicy "RPCA Shopfloor App Monitors" + the site-level wrapper, carrying the Configuration Policy as its one item. + +Monitors are looked up by identity (kind, displayname), never by a hardcoded id, +so the bundle survives id drift across installs. + +Lives in the endpoints chain, after 0006, because the monitor rows it references +are seeded there. The computers policy tables are created by the core baseline +before per-plugin chains run; this migration still guards on their presence and +no-ops cleanly if they are absent, and is idempotent on the policy/business-policy +names so re-running never duplicates the bundle. +""" +from alembic import op + +import sqlalchemy as sa + +from datetime import datetime + + +# revision identifiers, used by Alembic. +revision = 'endpoints0007shippedpolicy' +down_revision = 'endpoints0006monitors' +branch_labels = None +depends_on = None + + +CONFIG_POLICY_NAME = 'Shopfloor App Monitors' +BUSINESS_POLICY_NAME = 'RPCA Shopfloor App Monitors' + +# The shipped monitors this bundle attaches, by identity (kind, displayname). +_SHIPPED_MONITORS = ( + ('process', 'eDNC'), + ('process', 'Universal Data Collector'), + ('process', 'MTConnect'), + ('service', 'MTConnect'), +) + + +def _tables(bind): + from sqlalchemy import inspect + + return set(inspect(bind).get_table_names()) + + +def upgrade(): + bind = op.get_bind() + tables = _tables(bind) + + # The bundle lives in the computers policy tables and references the + # endpoints monitor catalog. If either side is missing (a partial site), + # do nothing rather than fail - a later run reconciles once both exist. + required = {'endpointmonitors', 'configurationpolicies', + 'configurationpolicyactions', 'businesspolicies', + 'businesspolicyitems'} + if not required.issubset(tables): + return + + # Idempotent: never seed the bundle twice. + existing_config = bind.execute( + sa.text('SELECT configurationpolicyid FROM configurationpolicies ' + 'WHERE name = :name'), + {'name': CONFIG_POLICY_NAME}).first() + if existing_config: + return + + # Resolve the shipped monitor ids by identity. Skip any that a site has + # not got; attach whatever shipped monitors are present. + monitor_ids = [] + for kind, displayname in _SHIPPED_MONITORS: + row = bind.execute( + sa.text('SELECT endpointmonitorid FROM endpointmonitors ' + 'WHERE kind = :kind AND displayname = :displayname'), + {'kind': kind, 'displayname': displayname}).first() + if row: + monitor_ids.append(row[0]) + + if not monitor_ids: + return + + now = datetime.utcnow() + + config_policies = sa.table( + 'configurationpolicies', + sa.column('name', sa.String), + sa.column('description', sa.Text), + sa.column('scopejson', sa.Text), + sa.column('source', sa.String), + sa.column('createddate', sa.DateTime), + sa.column('modifieddate', sa.DateTime), + sa.column('isactive', sa.Boolean), + ) + op.bulk_insert(config_policies, [{ + 'name': CONFIG_POLICY_NAME, + 'description': 'Product-shipped watch-list for the shopfloor apps ' + '(eDNC, Universal Data Collector, MTConnect). Read-only.', + 'scopejson': '{"type": "true"}', + 'source': 'shipped', + 'createddate': now, + 'modifieddate': now, + 'isactive': True, + }]) + + config_policy_id = bind.execute( + sa.text('SELECT configurationpolicyid FROM configurationpolicies ' + 'WHERE name = :name'), + {'name': CONFIG_POLICY_NAME}).scalar() + + actions = sa.table( + 'configurationpolicyactions', + sa.column('configurationpolicyid', sa.Integer), + sa.column('action', sa.String), + sa.column('mode', sa.String), + sa.column('desiredstate', sa.Text), + sa.column('params', sa.Text), + sa.column('intervalminutes', sa.Integer), + sa.column('sortorder', sa.Integer), + sa.column('createddate', sa.DateTime), + sa.column('modifieddate', sa.DateTime), + sa.column('isactive', sa.Boolean), + ) + op.bulk_insert(actions, [{ + 'configurationpolicyid': config_policy_id, + 'action': f'monitor:{monitorid}', + 'mode': 'gate', + 'desiredstate': None, + 'params': None, + 'intervalminutes': None, + 'sortorder': index, + 'createddate': now, + 'modifieddate': now, + 'isactive': True, + } for index, monitorid in enumerate(monitor_ids)]) + + business_policies = sa.table( + 'businesspolicies', + sa.column('name', sa.String), + sa.column('description', sa.Text), + sa.column('priority', sa.Integer), + sa.column('source', sa.String), + sa.column('createddate', sa.DateTime), + sa.column('modifieddate', sa.DateTime), + sa.column('isactive', sa.Boolean), + ) + op.bulk_insert(business_policies, [{ + 'name': BUSINESS_POLICY_NAME, + 'description': 'Product-shipped Site Policy that applies the shopfloor ' + 'app monitors to every endpoint. Read-only.', + 'priority': 100, + 'source': 'shipped', + 'createddate': now, + 'modifieddate': now, + 'isactive': True, + }]) + + business_policy_id = bind.execute( + sa.text('SELECT businesspolicyid FROM businesspolicies ' + 'WHERE name = :name'), + {'name': BUSINESS_POLICY_NAME}).scalar() + + items = sa.table( + 'businesspolicyitems', + sa.column('businesspolicyid', sa.Integer), + sa.column('configurationpolicyid', sa.Integer), + sa.column('sortorder', sa.Integer), + sa.column('createddate', sa.DateTime), + sa.column('modifieddate', sa.DateTime), + sa.column('isactive', sa.Boolean), + ) + op.bulk_insert(items, [{ + 'businesspolicyid': business_policy_id, + 'configurationpolicyid': config_policy_id, + 'sortorder': 0, + 'createddate': now, + 'modifieddate': now, + 'isactive': True, + }]) + + +def downgrade(): + bind = op.get_bind() + tables = _tables(bind) + if 'businesspolicies' not in tables or 'configurationpolicies' not in tables: + return + + # Clear children explicitly rather than lean on ON DELETE CASCADE, which a + # test DB (sqlite) does not enforce by default. + business_policy_id = bind.execute( + sa.text('SELECT businesspolicyid FROM businesspolicies ' + 'WHERE name = :name'), + {'name': BUSINESS_POLICY_NAME}).scalar() + config_policy_id = bind.execute( + sa.text('SELECT configurationpolicyid FROM configurationpolicies ' + 'WHERE name = :name'), + {'name': CONFIG_POLICY_NAME}).scalar() + + if business_policy_id is not None: + bind.execute( + sa.text('DELETE FROM businesspolicyitems ' + 'WHERE businesspolicyid = :id'), + {'id': business_policy_id}) + bind.execute( + sa.text('DELETE FROM businesspolicies WHERE businesspolicyid = :id'), + {'id': business_policy_id}) + if config_policy_id is not None: + bind.execute( + sa.text('DELETE FROM configurationpolicyactions ' + 'WHERE configurationpolicyid = :id'), + {'id': config_policy_id}) + bind.execute( + sa.text('DELETE FROM configurationpolicies ' + 'WHERE configurationpolicyid = :id'), + {'id': config_policy_id}) diff --git a/plugins/endpoints/migrations/versions/0008_endpoint_drift.py b/plugins/endpoints/migrations/versions/0008_endpoint_drift.py new file mode 100644 index 00000000..1a1477f5 --- /dev/null +++ b/plugins/endpoints/migrations/versions/0008_endpoint_drift.py @@ -0,0 +1,51 @@ +"""Cache the warn-only reconciler drift on the endpoint state. + +Adds endpointstates.driftjson: a per-asset JSON snapshot of the monitors a site +policy (or the shipped bundle) expects to be running on an endpoint but which the +latest agent report does not show as running. It is computed server-side on every +agent check-in and overwritten in place, the same ledger pattern the other state +snapshots (servicesjson, portsjson) already use - one current row per asset, no +history. Warn-only: nothing here is auto-remediated, it exists so an operator can +see the drift and intervene manually. + +Nullable so an endpoint that has never checked in since the column landed reads +as "never computed" rather than "no drift". + +Idempotent: the column-add checks the live schema first, so a db.create_all() +site (which built the column from the model already) and a re-run both stay clean +on strict MySQL 8. +""" +from alembic import op + +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'endpoints0008drift' +down_revision = 'endpoints0007shippedpolicy' +branch_labels = None +depends_on = None + + +def _has_column(table_name, column_name): + from sqlalchemy import inspect + + inspector = inspect(op.get_bind()) + if not inspector.has_table(table_name): + return False + return any(col['name'] == column_name + for col in inspector.get_columns(table_name)) + + +def upgrade(): + if _has_column('endpointstates', 'driftjson'): + return + op.add_column( + 'endpointstates', + sa.Column('driftjson', sa.Text(length=16777215), nullable=True)) + + +def downgrade(): + if not _has_column('endpointstates', 'driftjson'): + return + op.drop_column('endpointstates', 'driftjson') diff --git a/plugins/endpoints/models/__init__.py b/plugins/endpoints/models/__init__.py index 1e7936a9..1e059131 100644 --- a/plugins/endpoints/models/__init__.py +++ b/plugins/endpoints/models/__init__.py @@ -3,7 +3,9 @@ from .endpoint import ( EndpointState, EndpointEvent, EndpointCommand, EndpointApplication, ) +from .monitor import EndpointMonitor __all__ = [ 'EndpointState', 'EndpointEvent', 'EndpointCommand', 'EndpointApplication', + 'EndpointMonitor', ] diff --git a/plugins/endpoints/models/endpoint.py b/plugins/endpoints/models/endpoint.py index e859f125..445a0a7d 100644 --- a/plugins/endpoints/models/endpoint.py +++ b/plugins/endpoints/models/endpoint.py @@ -75,6 +75,13 @@ class EndpointState(db.Model): interfacesjson = db.Column(db.Text(length=16777215), nullable=True) scheduledtasksjson = db.Column(db.Text(length=16777215), nullable=True) + # Warn-only reconciler output, refreshed on every agent check-in in the same + # commit as servicesjson. Each entry names a policy/shipped-attached monitor + # the endpoint is expected to be running but the latest snapshot does not + # report as running, so an operator sees the drift and intervenes manually. + # This is warn-only: nothing here is auto-remediated. Null = never computed. + driftjson = db.Column(db.Text(length=16777215), nullable=True) + # Last-known MTConnect agent run state, cached so the WebUI toggle can show # a real state between command polls. Set by the agent's mtconnect_status # result. Null = never reported (unknown). diff --git a/plugins/endpoints/models/monitor.py b/plugins/endpoints/models/monitor.py new file mode 100644 index 00000000..d5b5a0a5 --- /dev/null +++ b/plugins/endpoints/models/monitor.py @@ -0,0 +1,87 @@ +"""Endpoint monitor definitions (the reusable monitor catalog). + +A monitor definition is the "what": a named process or service the agent should +watch, with a bespoke display label separate from the raw OS name, plus optional +launch and collector paths. Definitions are the noun that a configuration policy +action points at; attaching a monitor to a PC type is a policy concern, not a +property of the definition itself. + +Two sources: + + shipped Internal apps we ship (eDNC, MTConnect, UDC). Seeded by migration. + Read-only in the UI: name and identity are fixed by the product. + local User-defined monitors built on the Settings page. + +A definition is watched on an endpoint when a site policy in effect for that +endpoint's PC type includes a monitor action pointing at it. +""" + +from datetime import datetime + +from shopdb.api import db + + +def _utciso(value): + """Naive-UTC datetime -> ISO string marked UTC, or None.""" + return value.isoformat() + 'Z' if value else None + + +class EndpointMonitor(db.Model): + """One reusable process or service monitor definition.""" + + __tablename__ = 'endpointmonitors' + + endpointmonitorid = db.Column(db.Integer, primary_key=True) + + # 'process' or 'service'. Decides which identity column is authoritative and + # which start/stop/status actions the row exposes. + kind = db.Column(db.String(16), nullable=False) + + # The bespoke label operators see (e.g. 'MTConnect'). Distinct from the raw + # OS name because agent.exe is not obviously MTConnect. + displayname = db.Column(db.String(128), nullable=False) + + # OS identity. For a process monitor, processname is the executable the agent + # matches (e.g. 'agent.exe'); for a service monitor, servicename is the + # Windows service name (e.g. 'MTConnect UDC Agent'). Exactly one is the + # authoritative identity for a given kind; the other stays null. + processname = db.Column(db.String(255), nullable=True) + servicename = db.Column(db.String(255), nullable=True) + + # Optional paths supplied per monitor. launchpath starts the app; the + # collector paths point the agent at where the app writes data and logs. + launchpath = db.Column(db.String(512), nullable=True) + datacollectorpath = db.Column(db.String(512), nullable=True) + logcollectorpath = db.Column(db.String(512), nullable=True) + + # 'shipped' (product-owned, seeded) or 'local' (user-built). Shipped rows are + # not editable or deletable through the CRUD API. + source = db.Column(db.String(16), nullable=False, default='local') + + # Soft on/off so a monitor can be retired without losing policy references. + isactive = db.Column(db.Boolean, nullable=False, default=True) + + createdat = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) + + __table_args__ = ( + db.UniqueConstraint('kind', 'displayname', name='uq_endpointmonitors_kinddisplay'), + db.Index('idx_endpointmonitors_source', 'source'), + ) + + def __repr__(self): + return f"" + + def to_dict(self): + return { + 'endpointmonitorid': self.endpointmonitorid, + 'kind': self.kind, + 'displayname': self.displayname, + 'processname': self.processname, + 'servicename': self.servicename, + 'launchpath': self.launchpath, + 'datacollectorpath': self.datacollectorpath, + 'logcollectorpath': self.logcollectorpath, + 'source': self.source, + 'isactive': self.isactive, + 'createdat': _utciso(self.createdat), + } diff --git a/plugins/endpoints/plugin.py b/plugins/endpoints/plugin.py index aef48945..3dd4d881 100644 --- a/plugins/endpoints/plugin.py +++ b/plugins/endpoints/plugin.py @@ -486,26 +486,6 @@ class EndpointsPlugin(BasePlugin): # the installed-software inventory 'group': 'software', }, - { - # Installed-software (SBOM) inventory. Low-priority reference - # data, not something looked at on every visit, so it starts - # collapsed and sits near the bottom. Raw discovered list for - # now: no managed-application linkage is surfaced yet. - 'id': 'endpoints-applications', - 'title': 'Installed software', - 'assettypes': ['computer'], - 'endpoint': '/api/endpoints/asset/{assetid}/applications', - 'render': 'table', - 'collapsible': True, - 'collapsedDefault': True, - 'columns': [ - {'key': 'displayname', 'label': 'Name'}, - {'key': 'displayversion', 'label': 'Version'}, - {'key': 'publisher', 'label': 'Publisher'}, - ], - 'empty': 'No installed software reported.', - 'position': 30, - }, ] def get_dashboard_widgets(self) -> List[Dict]: @@ -822,6 +802,26 @@ class EndpointsPlugin(BasePlugin): state.lastreportedat = datetime.utcnow() + # Warn-only reconciler: on this check-in, compare the monitors site + # policy expects to be running against what the snapshot reports, and + # cache the drift on the state in THIS transaction (same ledger commit as + # servicesjson). No start/stop is issued - drift is surfaced for an + # operator to act on. Reads the freshly reported services when present, + # else the last-known snapshot, so a report that omits services still + # reconciles against what we last saw. Fail-open: a reconciler error must + # never fail an otherwise-good check-in, so drift is left unchanged. + try: + if 'services' in payload: + reported_services = payload.get('services') or [] + else: + reported_services = json.loads(state.servicesjson or '[]') + from .services.reconciler import compute_drift + state.driftjson = _as_json(compute_drift(assetid, reported_services)) + except Exception: + logger.exception( + 'reconciler failed for asset %s; leaving drift unchanged', + assetid) + # Log any keys the agent sent that the ingest path does not consume yet. # New agent telemetry (disks, bios, memory, ...) lands here first: it is # not persisted until the server grows columns for it, but logging the diff --git a/plugins/endpoints/services/__init__.py b/plugins/endpoints/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/plugins/endpoints/services/reconciler.py b/plugins/endpoints/services/reconciler.py new file mode 100644 index 00000000..c5140c75 --- /dev/null +++ b/plugins/endpoints/services/reconciler.py @@ -0,0 +1,153 @@ +"""Warn-only monitor reconciler. + +The reconciler answers one question on every agent check-in: for the monitors a +site policy (or the shipped bundle) attaches to this endpoint, is each one the +endpoint is EXPECTED to be running actually reported as running in the latest +snapshot? Every monitor that is expected-running but not observed-running is +DRIFT, and drift is surfaced to an operator for MANUAL intervention. + +This is warn-only. The reconciler NEVER starts, stops, or enqueues anything - it +compares observed state against expected state and records the difference. Whether +to later automate remediation (a keep-alive that starts a process when it drifts) +is a separate, deliberate opt-in the product does not ship switched on. + +Two seams are kept deliberately narrow so this stays easy to change: + + monitor_expects_running(monitor) + The single place that decides "should this monitor be up?". Today every + attached monitor is treated as expected-running (attachment implies intent + to run). When an observe-only monitor kind arrives - one that only collects + general state and must never be flagged as drift - this is the one function + that branches on it. Nothing else in the reconciler encodes that policy. + + compute_drift(asset_id, reported_services) + Pure given its inputs: it resolves the attached monitors and diffs them + against the reported running set. It reads no request state and writes no + rows, so the ingest path can call it inside its own transaction and tests + can call it directly. +""" + +# The running-state vocabulary a watched process/service can report. Kept in +# lockstep with plugins/endpoints/api/routes.py::_RUNNING_STATES so a service the +# panel paints green is never flagged as drift here. Both read from the same +# agent snapshot; disagreeing would show a running app as out-of-desired-state. +_RUNNING_STATES = ('running', 'started', 'up', 'ok', 'active') + + +def monitor_expects_running(monitor): + """Whether an attached monitor is expected to be running on an endpoint. + + The seam for the "should be up" decision. Today attachment implies intent to + run, so every active monitor returns True. A future observe-only monitor kind + (general state collection, never keep-alive) branches HERE and returns False + so it is never reported as drift - no other part of the reconciler needs to + change. + """ + return bool(getattr(monitor, 'isactive', True)) + + +def _monitor_identity(monitor): + """The (kind, osname) a monitor watches, or None when it has no identity. + + Mirrors how the services panel reads a monitor's OS identity: a process + monitor is keyed by its processname, a service monitor by its servicename. + """ + kind = (getattr(monitor, 'kind', '') or '').lower() or None + if kind == 'process': + osname = getattr(monitor, 'processname', None) + elif kind == 'service': + osname = getattr(monitor, 'servicename', None) + else: + osname = None + osname = (osname or '').strip() + if not kind or not osname: + return None + return kind, osname + + +def _reported_running_keys(reported_services): + """Set of (kind, lowercased-name) the snapshot reports as RUNNING. + + Tolerant of the two shapes the agent snapshot uses: dict entries with + name/service and state/status (and an optional kind), or bare strings. + Only running entries land in the set; a stopped or unknown entry is absent, + which is what makes an expected-running monitor read as drift. + """ + running = set() + for entry in reported_services or (): + if isinstance(entry, dict): + name = entry.get('name') or entry.get('service') or '' + raw = (entry.get('state') or entry.get('status') or '').strip().lower() + kind = (entry.get('kind') or '').lower() or None + else: + name, raw, kind = str(entry), '', None + name = name.strip().lower() + if not name or raw not in _RUNNING_STATES: + continue + running.add((kind, name)) + return running + + +def _is_running(kind, osname, running_keys): + """Whether a monitor's identity appears running in the snapshot. + + A snapshot entry may omit kind (older agents, or a plain string list), so a + match on name alone under an unknown kind still counts as running - the + reconciler must not invent drift just because the agent did not label a row. + """ + needle = osname.strip().lower() + if (kind, needle) in running_keys: + return True + if (None, needle) in running_keys: + return True + return False + + +def compute_drift(asset_id, reported_services): + """Compute the current drift list for an endpoint. Pure, no writes. + + Resolves the monitors attached to the asset's PC type (shipped defaults plus + site policy), keeps only those that are expected-running, and returns one + entry per monitor whose identity is not reported as running. Returns an empty + list when nothing is attached or nothing has drifted, so the caller can store + it as "computed, no drift" distinct from a null "never computed". + + Isolated from Flask request state: it reads the monitor catalog and the + passed-in snapshot only, so the ingest path calls it inside its own + transaction and tests call it directly. + """ + try: + from plugins.endpoints.api.routes import _resolved_monitors_for_asset + except Exception: + return [] + + monitors = _resolved_monitors_for_asset(asset_id) or [] + if not monitors: + return [] + + running_keys = _reported_running_keys(reported_services) + + drift = [] + seen = set() + for monitor in monitors: + if not monitor_expects_running(monitor): + continue + identity = _monitor_identity(monitor) + if identity is None: + continue + kind, osname = identity + key = (kind, osname.lower()) + if key in seen: + continue + seen.add(key) + if _is_running(kind, osname, running_keys): + continue + drift.append({ + 'monitorid': getattr(monitor, 'endpointmonitorid', None), + 'kind': kind, + 'name': osname, + 'displayname': getattr(monitor, 'displayname', None) or osname, + 'expected': 'running', + 'observed': 'notrunning', + }) + return drift diff --git a/shopdb/plugins/alembic_template.py b/shopdb/plugins/alembic_template.py index 3c5590a4..c7ea51a5 100644 --- a/shopdb/plugins/alembic_template.py +++ b/shopdb/plugins/alembic_template.py @@ -53,7 +53,7 @@ PLUGIN_TABLE_OWNERS: dict[str, Iterable[str]] = { 'configurationpolicies', 'configurationpolicyactions', 'businesspolicies', 'businesspolicyitems'), 'employees': ('directoryemployees',), - 'endpoints': ('endpointstates', 'endpointevents', 'endpointcommands', 'endpointapplications'), + 'endpoints': ('endpointstates', 'endpointevents', 'endpointcommands', 'endpointapplications', 'endpointmonitors'), 'geenforce': ('manifestscopes', 'manifestentries', 'manifestentrypctypes', 'manifestentryhostnames', 'manifestentrymachinenumbers', 'manifestinusechecks', 'manifestinusecheckprocesses', diff --git a/tests/test_core/test_public_read_surface.py b/tests/test_core/test_public_read_surface.py index 992c0ce1..e8c55c7f 100644 --- a/tests/test_core/test_public_read_surface.py +++ b/tests/test_core/test_public_read_surface.py @@ -96,6 +96,14 @@ PUBLIC_ENDPOINTS = { 'endpoints.download_app_package', 'endpoints.download_app_package_by_app', 'endpoints.download_script_body', + + # The endpoint agent fetches its server-resolved monitor watch-list (the + # processes/services site policy attaches to its PC type) on each telemetry + # cycle. Authenticated inside the handler by the endpoints collector API key + # - the same shape as claim_next_command - because a bay has no user session. + # It is a read of already-scoped policy output, not an authority: attaching a + # monitor is gated on the policy editor, audited with the requesting user. + 'endpoints.resolved_monitors_for_agent', } # Reads that need a LOGIN but no named permission, because the handler decides diff --git a/tests/test_plugin_migrations.py b/tests/test_plugin_migrations.py index 25159e49..fb57290b 100644 --- a/tests/test_plugin_migrations.py +++ b/tests/test_plugin_migrations.py @@ -92,9 +92,11 @@ EXPECTED_HEAD_REVISION['warranty'] = 'warranty0002proof' # endpoints is a NEW post-cutover plugin: its 0001 genuinely creates the # endpointstates + endpointevents tables the core chain never owned, 0002 # adds the endpointcommands queue plus the MTConnect state columns, 0003 -# adds the generic command target/targetkind columns, and 0004 adds the -# endpointapplications SBOM snapshot table. -EXPECTED_HEAD_REVISION['endpoints'] = 'endpoints0005cmdpin' +# adds the generic command target/targetkind columns, 0004 adds the +# endpointapplications SBOM snapshot table, 0005 pins a command sha256, 0006 +# seeds the reusable monitor catalog, 0007 ships the shopfloor monitor policy +# bundle, and 0008 adds the warn-only reconciler drift cache column. +EXPECTED_HEAD_REVISION['endpoints'] = 'endpoints0008drift' # Plugins built after the cutover: their 0001 baseline really creates tables the # core chain never owned. diff --git a/tests/test_plugins/test_endpoints_policy_monitors.py b/tests/test_plugins/test_endpoints_policy_monitors.py new file mode 100644 index 00000000..e088e8c4 --- /dev/null +++ b/tests/test_plugins/test_endpoints_policy_monitors.py @@ -0,0 +1,199 @@ +"""Policy-attached monitors surface in the Tracked Services/Processes panel. + +A monitor attached to a PC type through an active Business Policy (over a scoped +Configuration Policy) must appear in GET /api/endpoints/asset//services even +before the agent has reported that process/service, so the PC Detail Monitors +tab shows the intended coverage as an Unknown row. +""" + +from shopdb.core.models import Asset, AssetType, Setting + + +SERVICES_URL = '/api/endpoints/asset/{}/services' + + +def _computer(db, hostname, computertype): + """Create a computer Asset + Computer of the given PC type; return the asset.""" + from plugins.computers.models import Computer, ComputerType + + atype = AssetType.query.filter_by(assettype='computer').first() + if not atype: + atype = AssetType(assettype='computer') + db.session.add(atype) + db.session.flush() + + ctype = ComputerType.query.filter_by(computertype=computertype).first() + if not ctype: + ctype = ComputerType(computertype=computertype) + db.session.add(ctype) + db.session.flush() + + asset = Asset(assetnumber=hostname, assettypeid=atype.assettypeid, + isactive=True) + db.session.add(asset) + db.session.flush() + comp = Computer(assetid=asset.assetid, hostname=hostname, + computertypeid=ctype.computertypeid) + db.session.add(comp) + db.session.flush() + return asset, ctype + + +def _attach_monitor_to_pctype(db, monitor, computertypeid): + """Create a Configuration Policy scoped to the PC type carrying the monitor, + wrapped in an active Business Policy, with the feature flag enabled.""" + from plugins.computers.models import ( + ConfigurationPolicy, ConfigurationPolicyAction, + BusinessPolicy, BusinessPolicyItem, + ) + + db.session.add(Setting(key='computers_configpolicies_enabled', value='true')) + + policy = ConfigurationPolicy( + name='Edge coverage', + scopejson={'type': 'pctype', 'id': computertypeid}, + ) + db.session.add(policy) + db.session.flush() + db.session.add(ConfigurationPolicyAction( + configurationpolicyid=policy.configurationpolicyid, + action=f'monitor:{monitor.endpointmonitorid}', + mode='gate', + )) + + business = BusinessPolicy(name='Test Policy', priority=100) + db.session.add(business) + db.session.flush() + db.session.add(BusinessPolicyItem( + businesspolicyid=business.businesspolicyid, + configurationpolicyid=policy.configurationpolicyid, + sortorder=0, + )) + db.session.commit() + + +def test_policy_attached_process_monitor_surfaces_as_unknown_row( + client, db, auth_headers): + """A msedge.exe process monitor attached to a PC type by policy appears in + the services panel as an Unknown process row, even with no agent report.""" + from plugins.endpoints.models import EndpointMonitor + + asset, ctype = _computer(db, 'F18BCDB4', 'CMM PC') + monitor = EndpointMonitor( + kind='process', displayname='msedge.exe', processname='msedge.exe', + source='local', isactive=True, + ) + db.session.add(monitor) + db.session.flush() + _attach_monitor_to_pctype(db, monitor, ctype.computertypeid) + + response = client.get(SERVICES_URL.format(asset.assetid), headers=auth_headers) + assert response.status_code == 200 + rows = response.get_json()['data'] + + edge = next((r for r in rows if r['name'] == 'msedge.exe'), None) + assert edge is not None, 'policy-attached monitor missing from panel' + assert edge['kind'] == 'process' + assert edge['statelabel'] == 'Unknown' + assert edge['statecolor'] == 'gray' + # Admin holds endpoints.command, so the row offers a Start control whose + # wire target is the OS process name. + assert edge['action']['target'] == 'msedge.exe' + assert edge['action']['targetkind'] == 'process' + assert edge['action']['running'] is False + + +def test_flag_off_hides_policy_monitor_from_panel(client, db, auth_headers): + """With the feature flag off the panel returns only reported rows (none).""" + from plugins.endpoints.models import EndpointMonitor + + asset, ctype = _computer(db, 'F18BCDB5', 'CMM PC') + monitor = EndpointMonitor( + kind='process', displayname='msedge.exe', processname='msedge.exe', + source='local', isactive=True, + ) + db.session.add(monitor) + db.session.flush() + _attach_monitor_to_pctype(db, monitor, ctype.computertypeid) + # Flip the flag off after the helper turned it on. + flag = Setting.query.filter_by( + key='computers_configpolicies_enabled').first() + flag.value = 'false' + db.session.commit() + + response = client.get(SERVICES_URL.format(asset.assetid), headers=auth_headers) + assert response.status_code == 200 + rows = response.get_json()['data'] + assert not any(r['name'] == 'msedge.exe' for r in rows) + + +RESOLVED_URL = '/api/endpoints/monitors/resolved' +COLLECTOR_KEY = 'testcollectorkey' + + +def _collector_headers(app): + """Authorize the agent-facing route on the shared collector key.""" + app.config['COLLECTOR_API_KEY'] = COLLECTOR_KEY + return {'X-API-Key': COLLECTOR_KEY} + + +def test_resolved_monitors_returns_policy_process_for_agent(client, db, app): + """The agent watch-list route returns a policy-attached process monitor, + shaped as processes[]/services[] of {name, app}, keyed by hostname.""" + from plugins.endpoints.models import EndpointMonitor + + asset, ctype = _computer(db, 'AGENTPC1', 'CMM PC') + monitor = EndpointMonitor( + kind='process', displayname='Edge', processname='msedge.exe', + source='local', isactive=True, + ) + db.session.add(monitor) + db.session.flush() + _attach_monitor_to_pctype(db, monitor, ctype.computertypeid) + + response = client.get(RESOLVED_URL, query_string={'hostname': 'AGENTPC1'}, + headers=_collector_headers(app)) + assert response.status_code == 200 + data = response.get_json()['data'] + assert {'name': 'msedge.exe', 'app': 'Edge'} in data['processes'] + assert data['services'] == [] + + +def test_resolved_monitors_carries_launchpath_for_process(client, db, app): + """A process monitor with a launch path set surfaces launchpath on the wire + so the agent can honor a site-provisioned Start path (after validation).""" + from plugins.endpoints.models import EndpointMonitor + + asset, ctype = _computer(db, 'AGENTPC3', 'CMM PC') + monitor = EndpointMonitor( + kind='process', displayname='Edge', processname='msedge.exe', + launchpath=r'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe', + source='local', isactive=True, + ) + db.session.add(monitor) + db.session.flush() + _attach_monitor_to_pctype(db, monitor, ctype.computertypeid) + + response = client.get(RESOLVED_URL, query_string={'hostname': 'AGENTPC3'}, + headers=_collector_headers(app)) + assert response.status_code == 200 + data = response.get_json()['data'] + edge = next((p for p in data['processes'] if p['name'] == 'msedge.exe'), None) + assert edge is not None + assert edge['launchpath'] == \ + r'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe' + + +def test_resolved_monitors_requires_collector_auth(client, db, app): + """Without the collector key the agent route refuses (no JWT fallback).""" + _computer(db, 'AGENTPC2', 'CMM PC') + app.config['COLLECTOR_API_KEY'] = COLLECTOR_KEY + response = client.get(RESOLVED_URL, query_string={'hostname': 'AGENTPC2'}) + assert response.status_code in (401, 403) + + +def test_resolved_monitors_unknown_hostname_is_404(client, db, app): + """A hostname with no computer row is a 404, not an empty 200.""" + response = client.get(RESOLVED_URL, query_string={'hostname': 'NOSUCHBOX'}, + headers=_collector_headers(app)) + assert response.status_code == 404 diff --git a/tests/test_plugins/test_endpoints_reconciler.py b/tests/test_plugins/test_endpoints_reconciler.py new file mode 100644 index 00000000..bd3f7164 --- /dev/null +++ b/tests/test_plugins/test_endpoints_reconciler.py @@ -0,0 +1,238 @@ +"""Warn-only reconciler: drift computation and its exposure on the panels. + +The reconciler runs server-side on agent check-in. For every monitor a site +policy attaches to an endpoint's PC type that is EXPECTED to be running, it flags +DRIFT when the latest snapshot does not report it running. It never starts or +stops anything - drift is surfaced for an operator to act on manually. + +These tests cover the pure computer (compute_drift), the check-in write path +(apply_collector_payload persisting driftjson), the panel exposure, and the +observe-only seam (monitor_expects_running) that keeps a non-keepalive monitor +out of the drift list. +""" + +from shopdb.core.models import Asset, AssetType, Setting + + +SERVICES_URL = '/api/endpoints/asset/{}/services' +STATUS_URL = '/api/endpoints/asset/{}/status' + + +def _computer(db, hostname, computertype): + """Create a computer Asset + Computer of the given PC type; return the pair.""" + from plugins.computers.models import Computer, ComputerType + + atype = AssetType.query.filter_by(assettype='computer').first() + if not atype: + atype = AssetType(assettype='computer') + db.session.add(atype) + db.session.flush() + + ctype = ComputerType.query.filter_by(computertype=computertype).first() + if not ctype: + ctype = ComputerType(computertype=computertype) + db.session.add(ctype) + db.session.flush() + + asset = Asset(assetnumber=hostname, assettypeid=atype.assettypeid, + isactive=True) + db.session.add(asset) + db.session.flush() + comp = Computer(assetid=asset.assetid, hostname=hostname, + computertypeid=ctype.computertypeid) + db.session.add(comp) + db.session.flush() + return asset, ctype + + +def _attach_monitor_to_pctype(db, monitor, computertypeid): + """Attach a monitor to a PC type via an active Business/Configuration Policy.""" + from plugins.computers.models import ( + ConfigurationPolicy, ConfigurationPolicyAction, + BusinessPolicy, BusinessPolicyItem, + ) + + if not Setting.query.filter_by( + key='computers_configpolicies_enabled').first(): + db.session.add( + Setting(key='computers_configpolicies_enabled', value='true')) + + policy = ConfigurationPolicy( + name='Coverage {}'.format(monitor.endpointmonitorid), + scopejson={'type': 'pctype', 'id': computertypeid}, + ) + db.session.add(policy) + db.session.flush() + db.session.add(ConfigurationPolicyAction( + configurationpolicyid=policy.configurationpolicyid, + action='monitor:{}'.format(monitor.endpointmonitorid), + mode='gate', + )) + business = BusinessPolicy(name='BP {}'.format(monitor.endpointmonitorid), + priority=100) + db.session.add(business) + db.session.flush() + db.session.add(BusinessPolicyItem( + businesspolicyid=business.businesspolicyid, + configurationpolicyid=policy.configurationpolicyid, + sortorder=0, + )) + db.session.commit() + + +def _process_monitor(db, displayname, processname): + from plugins.endpoints.models import EndpointMonitor + + monitor = EndpointMonitor( + kind='process', displayname=displayname, processname=processname, + source='local', isactive=True, + ) + db.session.add(monitor) + db.session.flush() + return monitor + + +# --------------------------------------------------------------------------- +# Pure computer +# --------------------------------------------------------------------------- + +def test_compute_drift_flags_attached_monitor_not_reported_running( + client, db, app): + """An expected-running monitor absent from the snapshot is drift.""" + from plugins.endpoints.services.reconciler import compute_drift + + asset, ctype = _computer(db, 'DRIFT01', 'CMM PC') + monitor = _process_monitor(db, 'eDNC', 'dncMain.exe') + _attach_monitor_to_pctype(db, monitor, ctype.computertypeid) + + drift = compute_drift(asset.assetid, []) + assert len(drift) == 1 + assert drift[0]['name'] == 'dncMain.exe' + assert drift[0]['kind'] == 'process' + assert drift[0]['expected'] == 'running' + assert drift[0]['observed'] == 'notrunning' + + +def test_compute_drift_empty_when_monitor_reported_running(client, db, app): + """A monitor the snapshot reports running is not drift.""" + from plugins.endpoints.services.reconciler import compute_drift + + asset, ctype = _computer(db, 'DRIFT02', 'CMM PC') + monitor = _process_monitor(db, 'eDNC', 'dncMain.exe') + _attach_monitor_to_pctype(db, monitor, ctype.computertypeid) + + reported = [{'name': 'dncMain.exe', 'kind': 'process', 'state': 'running'}] + assert compute_drift(asset.assetid, reported) == [] + + +def test_compute_drift_empty_when_no_monitor_attached(client, db, app): + """No attached monitors means nothing to reconcile.""" + from plugins.endpoints.services.reconciler import compute_drift + + asset, _ = _computer(db, 'DRIFT03', 'CMM PC') + assert compute_drift(asset.assetid, []) == [] + + +def test_monitor_expects_running_is_the_observe_only_seam(): + """The 'should be up' decision lives in one function; inactive is excluded.""" + from plugins.endpoints.services.reconciler import monitor_expects_running + + class _M: + isactive = True + + active = _M() + inactive = _M() + inactive.isactive = False + assert monitor_expects_running(active) is True + assert monitor_expects_running(inactive) is False + + +# --------------------------------------------------------------------------- +# Check-in write path +# --------------------------------------------------------------------------- + +def test_checkin_persists_drift_on_endpoint_state(client, db, app): + """apply_collector_payload caches drift on the state in the same commit.""" + from plugins.endpoints.plugin import EndpointsPlugin + from plugins.endpoints.models import EndpointState + + asset, ctype = _computer(db, 'CHKIN01', 'CMM PC') + monitor = _process_monitor(db, 'eDNC', 'dncMain.exe') + _attach_monitor_to_pctype(db, monitor, ctype.computertypeid) + + plugin = EndpointsPlugin() + plugin.apply_collector_payload({'hostname': 'CHKIN01', 'services': []}) + db.session.commit() + + state = EndpointState.query.filter_by(assetid=asset.assetid).first() + assert state.driftjson is not None + import json + drift = json.loads(state.driftjson) + assert [d['name'] for d in drift] == ['dncMain.exe'] + + +def test_checkin_clears_drift_when_process_returns(client, db, app): + """A later report that shows the process running clears the cached drift.""" + from plugins.endpoints.plugin import EndpointsPlugin + from plugins.endpoints.models import EndpointState + import json + + asset, ctype = _computer(db, 'CHKIN02', 'CMM PC') + monitor = _process_monitor(db, 'eDNC', 'dncMain.exe') + _attach_monitor_to_pctype(db, monitor, ctype.computertypeid) + + plugin = EndpointsPlugin() + plugin.apply_collector_payload({'hostname': 'CHKIN02', 'services': []}) + db.session.commit() + + plugin.apply_collector_payload({ + 'hostname': 'CHKIN02', + 'services': [{'name': 'dncMain.exe', 'kind': 'process', + 'state': 'running'}], + }) + db.session.commit() + + state = EndpointState.query.filter_by(assetid=asset.assetid).first() + assert json.loads(state.driftjson) == [] + + +# --------------------------------------------------------------------------- +# Panel exposure +# --------------------------------------------------------------------------- + +def test_status_panel_surfaces_drift_warning(client, db, app, auth_headers): + """The status panel adds a Needs attention field when drift is cached.""" + from plugins.endpoints.plugin import EndpointsPlugin + + asset, ctype = _computer(db, 'PANEL01', 'CMM PC') + monitor = _process_monitor(db, 'eDNC', 'dncMain.exe') + _attach_monitor_to_pctype(db, monitor, ctype.computertypeid) + + plugin = EndpointsPlugin() + plugin.apply_collector_payload({'hostname': 'PANEL01', 'services': []}) + db.session.commit() + + response = client.get(STATUS_URL.format(asset.assetid), headers=auth_headers) + assert response.status_code == 200 + labels = [f['label'] for f in response.get_json()['data']['fields']] + assert 'Needs attention' in labels + + +def test_services_panel_marks_drifted_row(client, db, app, auth_headers): + """The synthetic Unknown row for an expected-running monitor is drift-flagged.""" + from plugins.endpoints.plugin import EndpointsPlugin + + asset, ctype = _computer(db, 'PANEL02', 'CMM PC') + monitor = _process_monitor(db, 'eDNC', 'dncMain.exe') + _attach_monitor_to_pctype(db, monitor, ctype.computertypeid) + + plugin = EndpointsPlugin() + plugin.apply_collector_payload({'hostname': 'PANEL02', 'services': []}) + db.session.commit() + + response = client.get(SERVICES_URL.format(asset.assetid), headers=auth_headers) + assert response.status_code == 200 + rows = response.get_json()['data'] + edenc = next((r for r in rows if r['name'] == 'eDNC'), None) + assert edenc is not None + assert edenc.get('drift') is True diff --git a/tests/test_plugins/test_endpoints_shipped_monitor_policy.py b/tests/test_plugins/test_endpoints_shipped_monitor_policy.py new file mode 100644 index 00000000..bbdccfeb --- /dev/null +++ b/tests/test_plugins/test_endpoints_shipped_monitor_policy.py @@ -0,0 +1,141 @@ +"""The shipped shopfloor app monitors ride a read-only Site Policy. + +Migration endpoints0007shippedpolicy offloads the eDNC/UDC/MTConnect monitors +from the agent's hardcoded appsettings.json into a product-owned policy bundle: +a shipped ConfigurationPolicy "Shopfloor App Monitors" (global scope, one +desired-state monitor action per shipped monitor) wrapped in a shipped +BusinessPolicy "RPCA Shopfloor App Monitors". + +Run the endpoints chain against a fresh engine, alongside the computers policy +tables the bundle writes into, and assert the bundle exists, is source='shipped' +(read-only), and attaches every shipped monitor as a desired/running:true action. +""" + +import importlib.util +import json +from pathlib import Path + +from sqlalchemy import create_engine, inspect, text +from alembic.migration import MigrationContext +from alembic.operations import Operations + + +_ROOT = Path(__file__).resolve().parent.parent.parent +_ENDPOINTS_VERSIONS = _ROOT / 'plugins' / 'endpoints' / 'migrations' / 'versions' +_COMPUTERS_VERSIONS = _ROOT / 'plugins' / 'computers' / 'migrations' / 'versions' + +CONFIG_POLICY_NAME = 'Shopfloor App Monitors' +BUSINESS_POLICY_NAME = 'RPCA Shopfloor App Monitors' + + +def _load(path, prefix): + spec = importlib.util.spec_from_file_location(prefix + path.stem, str(path)) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _run_bundle_migrations(connection): + """Build the policy tables (computers 0004), the monitor catalog (endpoints + 0006) and the shipped bundle (endpoints 0007) on one engine, in order. + + The core baseline creates every plugin's tables before per-plugin chains run, + so on a real deploy the computers policy tables already exist when endpoints + 0007 runs. Here we stand up exactly the three migrations the bundle depends + on - the earlier endpoints revisions carry FKs to core tables that a bare + engine does not have, and are irrelevant to what 0007 seeds. + """ + operations = Operations(MigrationContext.configure(connection)) + operations._install_proxy() + try: + _load(_COMPUTERS_VERSIONS / '0004_policies.py', + 'computers_mig_').upgrade() + _load(_ENDPOINTS_VERSIONS / '0006_endpoint_monitors.py', + 'endpoints_mig_0006_').upgrade() + _load(_ENDPOINTS_VERSIONS / '0007_shipped_monitor_policy.py', + 'endpoints_mig_0007_').upgrade() + finally: + operations._remove_proxy() + connection.commit() + + +def test_shipped_bundle_seeds_read_only_site_policy(): + engine = create_engine('sqlite://') + with engine.connect() as connection: + _run_bundle_migrations(connection) + + config = connection.execute(text( + 'SELECT configurationpolicyid, source, scopejson ' + 'FROM configurationpolicies WHERE name = :name'), + {'name': CONFIG_POLICY_NAME}).first() + assert config is not None, 'shipped Configuration Policy was not seeded' + config_id, config_source, scopejson = config + assert config_source == 'shipped' + assert json.loads(scopejson) == {'type': 'true'} + + business = connection.execute(text( + 'SELECT businesspolicyid, source FROM businesspolicies ' + 'WHERE name = :name'), {'name': BUSINESS_POLICY_NAME}).first() + assert business is not None, 'shipped Business Policy was not seeded' + business_id, business_source = business + assert business_source == 'shipped' + + item = connection.execute(text( + 'SELECT configurationpolicyid FROM businesspolicyitems ' + 'WHERE businesspolicyid = :id'), {'id': business_id}).first() + assert item is not None and item[0] == config_id + + +def test_shipped_bundle_attaches_every_shipped_monitor(): + engine = create_engine('sqlite://') + with engine.connect() as connection: + _run_bundle_migrations(connection) + + monitor_ids = {row[0] for row in connection.execute(text( + 'SELECT endpointmonitorid FROM endpointmonitors ' + "WHERE source = 'shipped'"))} + assert monitor_ids, 'no shipped monitors to attach' + + config_id = connection.execute(text( + 'SELECT configurationpolicyid FROM configurationpolicies ' + 'WHERE name = :name'), {'name': CONFIG_POLICY_NAME}).scalar() + + actions = connection.execute(text( + 'SELECT action, mode, desiredstate FROM configurationpolicyactions ' + 'WHERE configurationpolicyid = :id'), {'id': config_id}).all() + + attached_ids = set() + for action, mode, desiredstate in actions: + assert action.startswith('monitor:') + # Gate, not desired: the shipped bundle reports state and exposes + # Start/Stop in the UI, but never auto-starts a process. A desired + # running state is a deliberate per-monitor opt-in, not shipped. + assert mode == 'gate' + assert desiredstate is None + attached_ids.add(int(action[len('monitor:'):])) + + assert attached_ids == monitor_ids + + +def test_shipped_bundle_is_idempotent(): + engine = create_engine('sqlite://') + with engine.connect() as connection: + _run_bundle_migrations(connection) + # Re-run only the shipped-bundle migration; it must not duplicate rows. + operations = Operations(MigrationContext.configure(connection)) + operations._install_proxy() + try: + _load(_ENDPOINTS_VERSIONS / '0007_shipped_monitor_policy.py', + 'endpoints_mig_rerun_').upgrade() + finally: + operations._remove_proxy() + connection.commit() + + config_count = connection.execute(text( + 'SELECT COUNT(*) FROM configurationpolicies WHERE name = :name'), + {'name': CONFIG_POLICY_NAME}).scalar() + business_count = connection.execute(text( + 'SELECT COUNT(*) FROM businesspolicies WHERE name = :name'), + {'name': BUSINESS_POLICY_NAME}).scalar() + assert config_count == 1 + assert business_count == 1 From a24429046c3f4a7f24a53b9978d235f5dd10b162 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Tue, 1 Sep 2026 15:45:30 -0400 Subject: [PATCH 19/90] Support launch-path process monitors in the endpoint agent Some monitored processes (for example c:\BladeRunner\BladeRunner.exe) need a known launch path so the agent can act on the policy, not just detect by image name. This threads launchpath through the agent config, contracts, collector client, and command executor, and bumps the agent to 1.6.1. --- agent/ShopDbEndpointAgent.Core/AgentConfig.cs | 58 ++++++++- .../CollectorClient.cs | 36 +++++ .../CommandExecutor.cs | 68 +++++++++- agent/ShopDbEndpointAgent.Core/Contracts.cs | 21 +++ .../LaunchPathPolicy.cs | 123 ++++++++++++++++++ .../TelemetryWorker.cs | 15 +++ .../ShopDbEndpointAgent.csproj | 19 ++- agent/ShopDbEndpointAgent/appsettings.json | 16 +-- 8 files changed, 336 insertions(+), 20 deletions(-) create mode 100644 agent/ShopDbEndpointAgent.Core/LaunchPathPolicy.cs diff --git a/agent/ShopDbEndpointAgent.Core/AgentConfig.cs b/agent/ShopDbEndpointAgent.Core/AgentConfig.cs index 86672779..991d857e 100644 --- a/agent/ShopDbEndpointAgent.Core/AgentConfig.cs +++ b/agent/ShopDbEndpointAgent.Core/AgentConfig.cs @@ -34,14 +34,23 @@ public sealed class AgentConfig [JsonPropertyName("monitors")] public MonitorConfig Monitors { get; set; } = new(); // Per-image launchable paths for the generic process_start action. The wire - // never carries a path (that would be a code-execution hole); Start sends - // only the image name and the agent maps it here to a known-good executable + // historically never carried a path; now a server-provisioned monitor MAY + // carry one, but the agent only accepts it after LaunchPathPolicy validates + // it against ApprovedLaunchRoots (see below). Start otherwise sends only the + // image name and the agent maps it here to a known-good executable // (or a .bat launcher). Anything not in the resolved map fails closed. A // site overrides or extends the shipped defaults in appsettings.json without // a rebuild. Keyed by process image name (case-insensitive, .exe optional). [JsonPropertyName("launchpaths")] public Dictionary LaunchPaths { get; set; } = new(); + // Approved root directories a server-provisioned launch path must sit under + // before the agent will honor it. Empty = fall back to LaunchPathPolicy's + // shipped org defaults (Program Files roots). A site (or the future Site + // Agent Configuration tool) can widen or narrow this without a rebuild. + [JsonPropertyName("approvedlaunchroots")] + public List ApprovedLaunchRoots { get; set; } = new(); + // Shipped defaults for the org's common shop-floor apps. A site's // appsettings.json LaunchPaths entries win over these on a per-image basis. // MTConnect is intentionally absent here: it is a Windows service controlled @@ -162,6 +171,42 @@ public sealed class MonitorConfig [JsonPropertyName("services")] public MonitorTarget[] Services { get; set; } = Array.Empty(); + + // Union the local (appsettings.json) watch list with the server-resolved one, + // keyed case-insensitively by wire name so a target watched both locally and + // by policy is not enumerated twice. Server entries win on the label (App) so + // a policy display name overrides a stale local one, but a local-only target + // is preserved - the server list augments, it does not replace. Nulls on + // either side degrade to the other, which is what keeps monitoring alive when + // a fetch fails. + public static MonitorConfig Merge(MonitorConfig? local, ResolvedMonitors? server) + { + return new MonitorConfig + { + Processes = MergeTargets(local?.Processes, server?.Processes), + Services = MergeTargets(local?.Services, server?.Services), + }; + } + + private static MonitorTarget[] MergeTargets( + MonitorTarget[]? local, MonitorTarget[]? server) + { + var byName = new Dictionary( + StringComparer.OrdinalIgnoreCase); + foreach (var target in local ?? Array.Empty()) + { + if (string.IsNullOrWhiteSpace(target?.Name)) + continue; + byName[target.Name.Trim()] = target; + } + foreach (var target in server ?? Array.Empty()) + { + if (string.IsNullOrWhiteSpace(target?.Name)) + continue; + byName[target.Name.Trim()] = target; + } + return byName.Values.ToArray(); + } } public sealed class MonitorTarget @@ -172,4 +217,13 @@ public sealed class MonitorTarget // Human-facing application label, optional. Falls back to Name when empty. [JsonPropertyName("app")] public string App { get; set; } = ""; + + // Optional server-provisioned launch path for a process monitor. Only set on + // process targets that a site leader configured a path for. The agent NEVER + // launches this blindly: LaunchPathPolicy validates it against an approved- + // root allowlist first, and a local appsettings.json launchpaths entry for + // the same image still wins. Null/empty means "no server path" - the agent + // falls back to its local launch map, and process_start fails closed when + // neither yields a validated path. + [JsonPropertyName("launchpath")] public string? LaunchPath { get; set; } } diff --git a/agent/ShopDbEndpointAgent.Core/CollectorClient.cs b/agent/ShopDbEndpointAgent.Core/CollectorClient.cs index 4c4d30d9..fa9f724e 100644 --- a/agent/ShopDbEndpointAgent.Core/CollectorClient.cs +++ b/agent/ShopDbEndpointAgent.Core/CollectorClient.cs @@ -89,6 +89,42 @@ public sealed class CollectorClient : IDisposable return envelope.Data; } + // Fetch the server-resolved watch list (policy-attached processes/services) + // for this hostname, GET /api/endpoints/monitors/resolved. Shares the same + // X-API-Key credential as telemetry. Best-effort: returns null on any + // failure so the caller falls back to the last-known/local list rather than + // blanking monitoring on a transient blip. A single attempt, no retry - the + // next telemetry cycle retries naturally, and a stale-but-usable watch list + // beats an empty one. + public async Task FetchResolvedMonitorsAsync( + string hostname, CancellationToken token) + { + var url = "api/endpoints/monitors/resolved?hostname=" + + Uri.EscapeDataString(hostname); + try + { + using var response = await http.GetAsync(url, token); + var body = await response.Content.ReadAsStringAsync(token); + if (!response.IsSuccessStatusCode) + { + logger.LogWarning( + "Resolved-monitors fetch returned HTTP {Status}: {Body}", + (int)response.StatusCode, body); + return null; + } + var envelope = JsonSerializer.Deserialize( + body, JsonOptions); + return envelope?.Data; + } + catch (Exception error) when ( + error is HttpRequestException or TaskCanceledException or JsonException + && !token.IsCancellationRequested) + { + logger.LogWarning(error, "Resolved-monitors fetch failed; keeping current watch list"); + return null; + } + } + private TimeSpan BackoffDelay(int attempt) { // 2^(attempt-1) seconds, capped, plus up to 1s jitter. diff --git a/agent/ShopDbEndpointAgent.Core/CommandExecutor.cs b/agent/ShopDbEndpointAgent.Core/CommandExecutor.cs index 9ffae27c..7db7a198 100644 --- a/agent/ShopDbEndpointAgent.Core/CommandExecutor.cs +++ b/agent/ShopDbEndpointAgent.Core/CommandExecutor.cs @@ -56,9 +56,14 @@ public sealed class CommandExecutor // Resolved per-image launchable paths for process_start (shipped defaults // overlaid with any appsettings.json overrides). A target absent from this - // map cannot be started - process_start fails closed. + // map - and without a validated server-provisioned path - cannot be started, + // so process_start fails closed. private readonly IReadOnlyDictionary launchPaths; + // Validates server-provisioned launch paths (from policy-attached monitors) + // before they are honored. Local launchPaths always win over a server path. + private readonly LaunchPathPolicy launchPathPolicy; + // Held for update_app: the agent joins InstallPath to BaseUrl and downloads // the pinned installer with its own collector credential (ApiKey), so the // installer body never rides the command wire. @@ -71,6 +76,7 @@ public sealed class CommandExecutor this.startTimeout = startTimeout ?? TimeSpan.FromSeconds(30); this.config = config ?? new AgentConfig(); this.launchPaths = this.config.ResolveLaunchPaths(); + this.launchPathPolicy = new LaunchPathPolicy(this.config.ApprovedLaunchRoots); } public async Task ExecuteAsync( @@ -366,8 +372,8 @@ public sealed class CommandExecutor if (bare.EndsWith(".exe", StringComparison.OrdinalIgnoreCase)) bare = bare[..^4]; - if (!launchPaths.TryGetValue(bare, out var launchPath) - || string.IsNullOrWhiteSpace(launchPath)) + var launchPath = ResolveLaunchPath(bare, image); + if (string.IsNullOrWhiteSpace(launchPath)) { logger.LogWarning( "Rejected process_start for {Image}: no launchable path configured", @@ -452,6 +458,62 @@ public sealed class CommandExecutor } } + // Resolves the launchable path for a process image. Precedence: + // 1. Local launch map (shipped defaults + appsettings.json). A site pin + // always wins so a server value never silently overrides it. + // 2. A server-provisioned path on the matching process monitor, but only + // after LaunchPathPolicy validates it (approved root, exists, launchable + // extension). A rejected server path is logged and dropped - the caller + // then fails closed exactly as before. + // Returns null when neither source yields a usable path. + private string? ResolveLaunchPath(string bareImage, string image) + { + if (launchPaths.TryGetValue(bareImage, out var localPath) + && !string.IsNullOrWhiteSpace(localPath)) + { + return localPath; + } + + var serverPath = ServerLaunchPathFor(image, bareImage); + if (string.IsNullOrWhiteSpace(serverPath)) + return null; + + if (launchPathPolicy.TryValidate(serverPath, out var validated, out var reason)) + return validated; + + logger.LogWarning( + "Rejected server-provisioned launch path for {Image} ({Path}): {Reason}", + image, serverPath, reason); + return null; + } + + // Finds the server-provisioned launch path carried on the process monitor + // whose wire name matches the image (case-insensitive, .exe optional). + private string? ServerLaunchPathFor(string image, string bareImage) + { + var processes = config.Monitors?.Processes; + if (processes is null) + return null; + + foreach (var target in processes) + { + if (string.IsNullOrWhiteSpace(target?.Name) + || string.IsNullOrWhiteSpace(target.LaunchPath)) + continue; + + var name = target.Name.Trim(); + var bareName = name.EndsWith(".exe", StringComparison.OrdinalIgnoreCase) + ? name[..^4] + : name; + if (string.Equals(name, image, StringComparison.OrdinalIgnoreCase) + || string.Equals(bareName, bareImage, StringComparison.OrdinalIgnoreCase)) + { + return target.LaunchPath; + } + } + return null; + } + private bool IsProcessRunning(string image) { var processes = FindProcessesByImage(image); diff --git a/agent/ShopDbEndpointAgent.Core/Contracts.cs b/agent/ShopDbEndpointAgent.Core/Contracts.cs index d4c92a0a..f389ce8b 100644 --- a/agent/ShopDbEndpointAgent.Core/Contracts.cs +++ b/agent/ShopDbEndpointAgent.Core/Contracts.cs @@ -147,3 +147,24 @@ public sealed class CommandResult [JsonPropertyName("mtconnectrunning")] public bool? MtConnectRunning { get; set; } [JsonPropertyName("detail")] public Dictionary? Detail { get; set; } } + +// The server-resolved watch list for this endpoint (processes and services the +// site's policy attaches to the box's PC type), fetched each telemetry cycle so +// a policy-attached monitor is enumerated without editing appsettings.json on +// the box. Same {name, app} shape as the local MonitorConfig, so the merge is a +// direct list union. Empty lists mean "watch nothing extra beyond local". +public sealed class ResolvedMonitors +{ + [JsonPropertyName("processes")] public MonitorTarget[] Processes { get; set; } + = Array.Empty(); + + [JsonPropertyName("services")] public MonitorTarget[] Services { get; set; } + = Array.Empty(); +} + +// Envelope for the resolved-monitors route: { status, data: { processes, services } }. +public sealed class ResolvedMonitorsEnvelope +{ + [JsonPropertyName("status")] public string Status { get; set; } = ""; + [JsonPropertyName("data")] public ResolvedMonitors? Data { get; set; } +} diff --git a/agent/ShopDbEndpointAgent.Core/LaunchPathPolicy.cs b/agent/ShopDbEndpointAgent.Core/LaunchPathPolicy.cs new file mode 100644 index 00000000..4976dc38 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/LaunchPathPolicy.cs @@ -0,0 +1,123 @@ +using System.Text.Json.Serialization; + +namespace ShopDbEndpointAgent.Core; + +// Guards server-provisioned launch paths. Historically the command wire never +// carried a launch path at all - the agent mapped an image name to a known-good +// local path and anything else failed closed. Server-provisioned monitors now +// MAY carry a path a site leader set in the WebUI, which is a deliberate +// relaxation of that invariant. This policy is the replacement safeguard: a +// server path is only honored when it sits under an approved root, actually +// exists, and is an approved executable type. Everything else is rejected and +// the agent falls back to its local launch map (still fail closed). +// +// The approved-root list is org-opinionated with shipped defaults (the Windows +// Program Files roots), overridable per site via AgentConfig.ApprovedLaunchRoots +// so the future Site Agent Configuration tool can widen or narrow it without a +// rebuild. A local appsettings.json launchpaths entry still wins over any +// server path, so a site pin is never silently overridden. +public sealed class LaunchPathPolicy +{ + // Only these extensions are launchable. .bat is included because some shop- + // floor apps ship a launcher script (see the eDNC/UDC run.bat pattern). + private static readonly string[] ApprovedExtensions = + { ".exe", ".bat", ".cmd" }; + + // Shipped org defaults: the standard 64- and 32-bit install roots. Resolved + // from the environment so a non-C: system drive still matches. + public static IReadOnlyList DefaultApprovedRoots { get; } = + BuildDefaultRoots(); + + private readonly IReadOnlyList approvedRoots; + + public LaunchPathPolicy(IEnumerable? approvedRootsOverride = null) + { + var roots = (approvedRootsOverride ?? Enumerable.Empty()) + .Where(root => !string.IsNullOrWhiteSpace(root)) + .Select(NormalizeRoot) + .ToList(); + approvedRoots = roots.Count > 0 ? roots : DefaultApprovedRoots; + } + + // Result of validating a candidate path: the normalized full path when it is + // acceptable, or a human-readable reason when it is rejected. + public bool TryValidate(string? candidate, out string fullPath, out string reason) + { + fullPath = ""; + reason = ""; + + if (string.IsNullOrWhiteSpace(candidate)) + { + reason = "empty path"; + return false; + } + + string resolved; + try + { + resolved = Path.GetFullPath(candidate.Trim()); + } + catch (Exception ex) + { + reason = $"not a valid path: {ex.Message}"; + return false; + } + + var extension = Path.GetExtension(resolved); + if (!ApprovedExtensions.Contains(extension, StringComparer.OrdinalIgnoreCase)) + { + reason = $"extension '{extension}' is not launchable"; + return false; + } + + if (!IsUnderApprovedRoot(resolved)) + { + reason = "path is outside every approved launch root"; + return false; + } + + if (!File.Exists(resolved)) + { + reason = "file does not exist"; + return false; + } + + fullPath = resolved; + return true; + } + + private bool IsUnderApprovedRoot(string resolved) + { + foreach (var root in approvedRoots) + { + // Compare against root + separator so "C:\Program Files" does not + // also match "C:\Program FilesEvil\...". Case-insensitive on Windows. + var prefix = root.EndsWith(Path.DirectorySeparatorChar) + ? root + : root + Path.DirectorySeparatorChar; + if (resolved.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + return true; + } + return false; + } + + private static string NormalizeRoot(string root) + { + var full = Path.GetFullPath(root.Trim()); + return full.TrimEnd(Path.DirectorySeparatorChar); + } + + private static IReadOnlyList BuildDefaultRoots() + { + var candidates = new[] + { + Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles), + Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86), + }; + return candidates + .Where(path => !string.IsNullOrWhiteSpace(path)) + .Select(NormalizeRoot) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + } +} diff --git a/agent/ShopDbEndpointAgent.Core/TelemetryWorker.cs b/agent/ShopDbEndpointAgent.Core/TelemetryWorker.cs index 1fcd3412..2efa847d 100644 --- a/agent/ShopDbEndpointAgent.Core/TelemetryWorker.cs +++ b/agent/ShopDbEndpointAgent.Core/TelemetryWorker.cs @@ -96,6 +96,21 @@ public sealed class TelemetryWorker : BackgroundService // exact production collection path. public async Task RunOnceAsync(bool adopt, CancellationToken token) { + // Refresh the watch list from server policy BEFORE collecting so a + // policy-attached monitor (added in the WebUI, never in appsettings.json) + // is enumerated and reported this cycle. Server entries merge over the + // local list; a failed fetch returns null and Merge keeps the local list, + // so monitoring never blanks on a transient blip. + var resolved = await client.FetchResolvedMonitorsAsync( + Environment.MachineName, token); + if (resolved is not null) + { + config.Monitors = MonitorConfig.Merge(config.Monitors, resolved); + logger.LogDebug( + "Watch list refreshed from policy: {Processes} process(es), {Services} service(s)", + config.Monitors.Processes.Length, config.Monitors.Services.Length); + } + var report = collector.Collect(agentVersion, adopt); if (utilities is not null) diff --git a/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj b/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj index 9c7d55bf..c877a7fa 100644 --- a/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj +++ b/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj @@ -7,7 +7,22 @@ enable ShopDbEndpointAgent shopdbagent - - 1.4.1 + 1.6.1 true win-x64 true diff --git a/agent/ShopDbEndpointAgent/appsettings.json b/agent/ShopDbEndpointAgent/appsettings.json index 162913e3..6db6ca67 100644 --- a/agent/ShopDbEndpointAgent/appsettings.json +++ b/agent/ShopDbEndpointAgent/appsettings.json @@ -7,20 +7,10 @@ "utilitiespath": "", "logpath": "", "monitors": { - "processes": [ - { "name": "dncMain.exe", "app": "eDNC" }, - { "name": "udc.exe", "app": "Universal Data Collector" }, - { "name": "agent.exe", "app": "MTConnect" } - ], - "services": [ - { "name": "MTConnect UDC Agent", "app": "MTConnect" } - ] + "processes": [], + "services": [] }, - "launchpaths": { - "DncMain.exe": "C:\\Program Files (x86)\\Dnc\\bin\\DncMain.exe", - "UDC.exe": "C:\\Program Files\\UDC\\UDC.exe", - "agent.exe": "C:\\ProgramData\\udc\\mtconnect_udc\\agent\\run.bat" - } + "launchpaths": {} } From d973ce6c26ab295352fe644b22438fe6b16656e3 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Tue, 1 Sep 2026 15:45:52 -0400 Subject: [PATCH 20/90] Add configuration policy monitors and action picker for computers Configuration policies need a way to attach process/service monitors so drift can be evaluated against a machine's desired state. This wires the policy resolver and routes to the shared monitor catalog and adds an action picker UI that can create process monitors with an optional launch path. --- plugins/computers/api/policy_routes.py | 38 ++- plugins/computers/frontend/routes.js | 4 +- .../computers/frontend/views/ActionPicker.vue | 265 ++++++++++++++++++ .../frontend/views/BusinessPoliciesList.vue | 12 +- .../views/ConfigurationPoliciesList.vue | 137 +++++++-- plugins/computers/models/__init__.py | 2 + plugins/computers/models/policy.py | 6 + plugins/computers/plugin.py | 6 +- plugins/computers/services/policyresolver.py | 58 ++++ 9 files changed, 492 insertions(+), 36 deletions(-) create mode 100644 plugins/computers/frontend/views/ActionPicker.vue diff --git a/plugins/computers/api/policy_routes.py b/plugins/computers/api/policy_routes.py index a515acc8..65592e4e 100644 --- a/plugins/computers/api/policy_routes.py +++ b/plugins/computers/api/policy_routes.py @@ -40,6 +40,23 @@ def _known_actions(): return set(COMMAND_ALLOWLIST.keys()) +def _is_valid_action(name): + """True when an action name is either an allowlisted command or a monitor + attachment (monitor:) pointing at an existing monitor definition. + + Monitor actions are intentionally kept out of the command allowlist so they + are never enqueued as commands; they are only valid as policy attachments. + """ + if name in _known_actions(): + return True + from plugins.endpoints.commands import monitor_id_from_action + from plugins.endpoints.models import EndpointMonitor + monitorid = monitor_id_from_action(name) + if monitorid is None: + return False + return db.session.get(EndpointMonitor, monitorid) is not None + + def _configpolicy_dict(policy): data = policy.to_dict() data['actions'] = [a.to_dict() for a in policy.actions] @@ -89,11 +106,15 @@ def _apply_actions(policy, actions): Returns an error_response on failure, else None. """ - known = _known_actions() policy.actions.clear() + # Flush the deletes before re-appending. The (configurationpolicyid, action) + # unique constraint collides if SQLAlchemy orders a re-added action's INSERT + # before the old row's DELETE in the same flush, which surfaces as a generic + # "value already in use" IntegrityError on an otherwise valid edit. + db.session.flush() for row in actions or []: name = (row.get('action') or '').strip() - if name not in known: + if not _is_valid_action(name): return error_response(ErrorCodes.VALIDATION_ERROR, f'Unknown action: {name!r}') mode = (row.get('mode') or ACTION_MODE_GATE).strip() @@ -259,6 +280,10 @@ def get_business_policy(policy_id: int): def _apply_items(policy, items): """Replace a business policy's configuration-policy items, validating each.""" policy.items.clear() + # Flush the deletes before re-appending so the (businesspolicyid, + # configurationpolicyid) unique constraint does not collide with a re-added + # item whose INSERT would otherwise race ahead of the old row's DELETE. + db.session.flush() seen = set() for order, row in enumerate(items or []): cpid = row.get('configurationpolicyid') @@ -375,14 +400,19 @@ def delete_business_policy(policy_id: int): @computers_bp.route('/policies/actions', methods=['GET']) @require_permission('computers.view') def list_policy_actions(): - """The allowlisted action names + labels a policy may reference.""" + """The allowlisted action names + labels a policy may reference. + + Returns command actions tagged kind='command'. Monitor attachments are not + listed here: the UI builds that branch from the endpoints monitor catalog + (endpointsApi.monitors) so newly created monitors appear without a reload. + """ gate = _feature_gate() if gate: return gate from plugins.endpoints.commands import COMMAND_ALLOWLIST return success_response([ {'action': name, 'label': meta.get('label'), - 'description': meta.get('description')} + 'description': meta.get('description'), 'kind': 'command'} for name, meta in COMMAND_ALLOWLIST.items() ]) diff --git a/plugins/computers/frontend/routes.js b/plugins/computers/frontend/routes.js index 3d7d0895..e1464b56 100644 --- a/plugins/computers/frontend/routes.js +++ b/plugins/computers/frontend/routes.js @@ -70,8 +70,8 @@ export default [ meta: { requiresAuth: true, requiresAdmin: true, plugin: 'computers' } }, { - path: 'settings/business-policies', - name: 'business-policies', + path: 'settings/site-policies', + name: 'site-policies', component: () => import('./views/BusinessPoliciesList.vue'), meta: { requiresAuth: true, requiresAdmin: true, plugin: 'computers' } } diff --git a/plugins/computers/frontend/views/ActionPicker.vue b/plugins/computers/frontend/views/ActionPicker.vue new file mode 100644 index 00000000..7111d195 --- /dev/null +++ b/plugins/computers/frontend/views/ActionPicker.vue @@ -0,0 +1,265 @@ + + + + + diff --git a/plugins/computers/frontend/views/BusinessPoliciesList.vue b/plugins/computers/frontend/views/BusinessPoliciesList.vue index 4a6b20ae..e2f3b971 100644 --- a/plugins/computers/frontend/views/BusinessPoliciesList.vue +++ b/plugins/computers/frontend/views/BusinessPoliciesList.vue @@ -1,8 +1,8 @@ + + + From 7a11851fbe5bc8d0863b9c491ef17cdecde7208b Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Wed, 2 Sep 2026 20:27:47 -0400 Subject: [PATCH 43/90] Document migration-on-deploy contract for plugin schema changes A plugin model change that references a column its migration has not applied yet 500s every query on that table. Make the deploy order explicit: run flask db upgrade then flask plugin upgrade-all before serving traffic, and require model+migration to ship together. --- .github/copilot-instructions.md | 5 +++++ CONTRIBUTING.md | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 77f2f109..8892e585 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -67,6 +67,11 @@ name. Spell out: `config context manager request response environment utilities` `migrations/versions/`; each plugin owns its own chain under `plugins//migrations/`. New schema = a new migration with an idempotent guard and a real downgrade. Migrations must run clean on strict MySQL 8. + Any model column/table change ships WITH its migration in the SAME change, and + bringing up a new image MUST run `flask db upgrade` then + `flask plugin upgrade-all` before serving traffic. A model that references a + column its migration has not applied yet 500s every query on that table + (`1054 Unknown column`), so code and schema must never be deployed out of step. - **Asset model is the platform contract.** Physical things are an `Asset` plus a plugin subtype row linked by `assetid` (FK, `ON DELETE CASCADE`). Consumables with quantities are standalone tables, not assets. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 848e3026..5fddca00 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,6 +24,24 @@ - **Index names**: `idx__`. Underscores allowed here only because indexes are infrastructure, not data. - Good: `idx_machines_locationid` +### Migrations and deploy ordering + +- **A model change ships WITH its migration in the same change.** Adding or + changing a column/table on a SQLAlchemy model without the matching Alembic + migration is a broken change, even if it passes tests against a freshly built + schema. +- **Code and schema must never deploy out of step.** A model that references a + column its migration has not applied yet makes MySQL raise + `1054 Unknown column`, which 500s every query on that table (this is not a + degraded read - the whole endpoint fails). +- **Bringing up a new image MUST run migrations before serving traffic**, in + order: `flask db upgrade` (core chain) then `flask plugin upgrade-all` + (per-plugin chains). A rebuilt image with new model code but an + un-migrated database will 500 until this runs. +- Core schema lives in `migrations/versions/`; each plugin owns its own chain + under `plugins//migrations/`. New schema = a new migration with an + idempotent guard and a real downgrade, clean on strict MySQL 8. + ### Allowed acronyms (closed list, expand by PR) - **Universal**: `id`, `url`, `api`, `http`, `https`, `json`, `jwt`, `sql`, `os`, `ip`, `dns`, `csv`, `pdf`, `cors`, `ttl`, `uuid`, `html`, `css`, `orm` From 729e48890319d65407991fa773e992edec674a50 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Wed, 2 Sep 2026 20:27:55 -0400 Subject: [PATCH 44/90] Rework Build Site Agent binary UI with version auto-detect and rollback Restructure the agent and tray binary blocks around an auto-detected current-release upload plus a separate legacy/rollback upload that stages a version without seizing current. Order staged versions by version number for the active-version dropdown, add a PE version detect route, and drop stale dev-status tags and manual-staging copy. --- plugins/endpoints/api/agentstore.py | 239 +++++++++++++++++- plugins/endpoints/api/agentupload.py | 97 ++++++- plugins/endpoints/api/siteagent.py | 2 + .../frontend/views/AgentKeySettings.vue | 181 +++++++++++-- .../test_plugins/test_endpoints_agentstore.py | 57 +++++ 5 files changed, 541 insertions(+), 35 deletions(-) create mode 100644 tests/test_plugins/test_endpoints_agentstore.py diff --git a/plugins/endpoints/api/agentstore.py b/plugins/endpoints/api/agentstore.py index d656dfef..8f111fb8 100644 --- a/plugins/endpoints/api/agentstore.py +++ b/plugins/endpoints/api/agentstore.py @@ -33,6 +33,7 @@ import io import json import os import re +import struct from datetime import datetime from flask import current_app @@ -75,6 +76,155 @@ def looks_like_pe(data): return len(data) >= 2 and data[:2] == _PE_MAGIC +# RT_VERSION resource type id (the version resource Windows shows in file +# properties). We must read the version from THIS resource on the outer +# executable - not the first VS_FIXEDFILEINFO signature we can find in the bytes. +# A self-contained single-file .NET exe bundles dozens of framework assemblies, +# each carrying its own version block, so a naive byte scan returns a framework +# version (for example the .NET runtime's) instead of the app's. The only +# reliable value is the RT_VERSION resource reached by walking the PE resource +# directory, which belongs to the outer image alone. +_RT_VERSION = 16 + +# 0xFEEF04BD little-endian: the dwSignature that opens VS_FIXEDFILEINFO inside +# the RT_VERSION resource data. +_VS_FIXEDFILEINFO_SIGNATURE = b'\xbd\x04\xef\xfe' + + +def _rva_to_offset(data, rva, sections): + """Map a resource RVA to a file offset using the section table, or None.""" + for virtual_address, virtual_size, raw_offset, raw_size in sections: + if virtual_address <= rva < virtual_address + max(virtual_size, raw_size): + delta = rva - virtual_address + if delta < raw_size: + return raw_offset + delta + return None + + +def _first_resource_entry_offset(data, table_offset, resource_base, sections, + want_id): + """Walk one IMAGE_RESOURCE_DIRECTORY level and return a child offset. + + When want_id is not None, follow the entry whose id matches (used to select + RT_VERSION at the type level); otherwise follow the first entry (used to + descend name/language levels). Returns the absolute file offset of the child + directory or data entry, with the high bit stripped, or None. + """ + # IMAGE_RESOURCE_DIRECTORY: 12 fixed bytes, then NumberOfNamedEntries (2) + # and NumberOfIdEntries (2), then the entries. + named = struct.unpack_from(' pick RT_VERSION. Level 2: name/id -> first. + # Level 3: language -> first. Then a data entry pointing at the resource. + type_dir = _first_resource_entry_offset( + data, resource_base, resource_base, sections, _RT_VERSION) + if type_dir is None: + return None + name_dir = _first_resource_entry_offset( + data, type_dir, resource_base, sections, None) + if name_dir is None: + return None + lang_entry = _first_resource_entry_offset( + data, name_dir, resource_base, sections, None) + if lang_entry is None: + return None + + # lang_entry is an IMAGE_RESOURCE_DATA_ENTRY: RVA (4), size (4), ... + version_rva = struct.unpack_from('> 16) & 0xFFFF, + version_ms & 0xFFFF, + (version_ls >> 16) & 0xFFFF, + version_ls & 0xFFFF, + ) + if not any(parts): + return None + return '.'.join(str(part) for part in parts) + except (struct.error, TypeError, IndexError): + return None + + +def read_pe_file_version_from_path(path): + """Read a PE file's version from disk, or None on any failure.""" + try: + with open(path, 'rb') as handle: + return read_pe_file_version(handle.read()) + except OSError: + return None + + def _siteagent_dir(): return os.path.join(current_app.instance_path, 'siteagent') @@ -117,12 +267,32 @@ def _write_manifest(binkey, manifest): handle.write('\n') +def _version_sort_key(version): + """Parse a dotted-numeric version into a tuple of ints for ordering. + + Pads to four parts so 1.6 and 1.6.0.0 compare equal-ish and shorter + versions do not sort above longer ones with the same prefix. A part that + is not numeric (should not happen after validation) sorts as 0. + """ + parts = [] + for chunk in str(version).split('.'): + try: + parts.append(int(chunk)) + except ValueError: + parts.append(0) + while len(parts) < 4: + parts.append(0) + return tuple(parts[:4]) + + def list_versions(binkey): - """Staged versions for a binary, newest upload first. + """Staged versions for a binary, highest version number first. Each item is {version, uploadedat, size, iscurrent}. Reads the manifest as - the source of truth for order and currency; a version whose exe is missing - on disk is dropped so the caller never offers a broken push. + the source of truth for currency; a version whose exe is missing on disk is + dropped so the caller never offers a broken push. Ordered by version number + (descending) so a rollback dropdown reads newest-build-first regardless of + the order versions were uploaded. """ manifest = _read_manifest(binkey) current = manifest.get('current') @@ -137,6 +307,7 @@ def list_versions(binkey): 'size': entry.get('size'), 'iscurrent': version == current, }) + out.sort(key=lambda item: _version_sort_key(item['version']), reverse=True) return out @@ -164,6 +335,51 @@ def current_path(binkey): return None +def legacy_status(binkey): + """Describe an unmanaged legacy flat file, or None when there is none. + + Returned only when a flat file exists and no managed version is current, so + the UI can show its detected version and offer one-click adoption without + the operator re-uploading. {version} is the PE file version if readable, + else None. + """ + if current_version(binkey): + return None + legacy = _legacy_path(binkey) + if not os.path.isfile(legacy): + return None + try: + size = os.path.getsize(legacy) + except OSError: + size = None + return { + 'version': read_pe_file_version_from_path(legacy), + 'size': size, + } + + +def adopt_legacy(binkey, version): + """Copy the legacy flat file into the version store as a staged version. + + Reads the flat file, validates it as a PE, and stores it under the given + version (making it current). Returns the fresh manifest view, or None when + there is no legacy file or it is not a PE. Caller supplies the version - + typically the detected PE version, or an operator override when the binary + carries no version resource. + """ + legacy = _legacy_path(binkey) + if not os.path.isfile(legacy): + return None + try: + with open(legacy, 'rb') as handle: + data = handle.read() + except OSError: + return None + if not looks_like_pe(data): + return None + return store_version(binkey, version, data) + + def _prune(binkey, manifest): """Drop versions past the retention window, never removing the current one.""" current = manifest.get('current') @@ -197,14 +413,16 @@ def _remove_version_dir(binkey, version): pass -def store_version(binkey, version, data): - """Stage an uploaded binary as a new version and make it current. +def store_version(binkey, version, data, make_current=True): + """Stage an uploaded binary as a new version. Writes data to siteagent///, records it newest-first in - the manifest (de-duping a re-upload of the same version in place), marks it - current, and prunes older versions past RETAIN. Returns the fresh manifest - view {current, versions:[...]}. Caller must have validated binkey, version, - and that data is a PE binary. + the manifest (de-duping a re-upload of the same version in place), and prunes + older versions past RETAIN. When make_current is True the new version also + becomes the current staged build; pass False to stage a rollback/legacy build + that should appear in the version list without displacing the active current. + Returns the fresh manifest view {current, versions:[...]}. Caller must have + validated binkey, version, and that data is a PE binary. """ os.makedirs(os.path.join(_binary_dir(binkey), version), exist_ok=True) with open(_version_exe_path(binkey, version), 'wb') as handle: @@ -219,7 +437,8 @@ def store_version(binkey, version, data): 'size': len(data), }) manifest['versions'] = versions - manifest['current'] = version + if make_current or not manifest.get('current'): + manifest['current'] = version _prune(binkey, manifest) _write_manifest(binkey, manifest) return { diff --git a/plugins/endpoints/api/agentupload.py b/plugins/endpoints/api/agentupload.py index c180a0a1..3761ac05 100644 --- a/plugins/endpoints/api/agentupload.py +++ b/plugins/endpoints/api/agentupload.py @@ -51,15 +51,19 @@ def list_agent_binaries(binkey): @endpoints_bp.route('/siteagent/binaries/', methods=['POST']) @require_permission('endpoints.agentpackage') def upload_agent_binary(binkey): - """Stage an uploaded binary as a new version and make it current. + """Stage an uploaded binary as a new version. Expects multipart/form-data with a 'file' upload and a 'version' field. The version must be a strict dotted-numeric string (for example 1.6.3) so it is safe as a directory name and orderable. The upload must be a Windows PE executable. Older versions past the retention window are pruned, but never - the current one. + the current one. By default the upload becomes the current staged build; send + makecurrent=false to stage a rollback/legacy build that joins the version + list without displacing the active current. """ version = (request.form.get('version') or '').strip() + makecurrent = (request.form.get('makecurrent') or 'true').strip().lower() + makecurrent = makecurrent not in ('false', '0', 'no') if not agentstore.is_valid_binary_key(binkey): return error_response( ErrorCodes.VALIDATION_ERROR, @@ -90,7 +94,94 @@ def upload_agent_binary(binkey): 'uploaded file is not a Windows executable', http_code=400) - result = agentstore.store_version(binkey, version, data) + result = agentstore.store_version(binkey, version, data, + make_current=makecurrent) + return success_response({ + 'binary': binkey, + 'current': result['current'], + 'versions': result['versions'], + }) + + +@endpoints_bp.route('/siteagent/binaries//detect', methods=['POST']) +@require_permission('endpoints.agentpackage') +def detect_agent_binary_version(binkey): + """Read the PE file version of an uploaded EXE without staging it. + + The Build Site Agent page calls this on file-select so the version field is + prefilled from the binary's own version resource - the operator does not + have to read the EXE properties by hand for a current-release upload. + Returns {version} which is null when the file carries no readable version. + """ + if not agentstore.is_valid_binary_key(binkey): + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'binkey must be one of: agent, tray', + http_code=400) + + upload = request.files.get('file') + if upload is None or not upload.filename: + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'a file upload is required', + http_code=400) + + data = upload.read(_MAX_UPLOAD_BYTES + 1) + if len(data) > _MAX_UPLOAD_BYTES: + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'uploaded binary exceeds the size limit', + http_code=400) + if not agentstore.looks_like_pe(data): + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'uploaded file is not a Windows executable', + http_code=400) + + return success_response({ + 'binary': binkey, + 'version': agentstore.read_pe_file_version(data), + }) + + +@endpoints_bp.route('/siteagent/binaries//adopt', methods=['POST']) +@require_permission('endpoints.agentpackage') +def adopt_legacy_agent_binary(binkey): + """Adopt the unmanaged legacy flat file into the version store. + + The pre-versioning install dropped a single shopdbagent.exe (or tray exe) + straight under siteagent/. This copies that file in as a real staged version + so it shows a version and joins retention, without the operator having to + re-upload it. The version comes from the request when the binary carries no + readable PE version resource; otherwise the detected version is used. + """ + if not agentstore.is_valid_binary_key(binkey): + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'binkey must be one of: agent, tray', + http_code=400) + + status = agentstore.legacy_status(binkey) + if status is None: + return error_response( + ErrorCodes.NOT_FOUND, + 'there is no unmanaged legacy binary to adopt', + http_code=404) + + source = request.get_json(silent=True) or {} + version = (source.get('version') or status.get('version') or '').strip() + if not agentstore.is_valid_version(version): + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'version must be a dotted-numeric string such as 1.6.3', + http_code=400) + + result = agentstore.adopt_legacy(binkey, version) + if result is None: + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'the legacy binary could not be adopted', + http_code=400) return success_response({ 'binary': binkey, 'current': result['current'], diff --git a/plugins/endpoints/api/siteagent.py b/plugins/endpoints/api/siteagent.py index 216c1121..3604aa7b 100644 --- a/plugins/endpoints/api/siteagent.py +++ b/plugins/endpoints/api/siteagent.py @@ -462,6 +462,8 @@ def _siteagent_status_payload(): 'traycurrent': agentstore.current_version('tray'), 'agentversions': agentstore.list_versions('agent'), 'trayversions': agentstore.list_versions('tray'), + 'agentlegacy': agentstore.legacy_status('agent'), + 'traylegacy': agentstore.legacy_status('tray'), } diff --git a/plugins/endpoints/frontend/views/AgentKeySettings.vue b/plugins/endpoints/frontend/views/AgentKeySettings.vue index 51d623d9..f213c234 100644 --- a/plugins/endpoints/frontend/views/AgentKeySettings.vue +++ b/plugins/endpoints/frontend/views/AgentKeySettings.vue @@ -160,7 +160,7 @@

Adoption tool

@@ -211,7 +211,7 @@

Build site agent

@@ -227,10 +227,9 @@

No agent binary is staged on this server. The package - will still build with config and installer, but you must drop - shopdbagent.exe beside them before deploying. Stage a - prebuilt agent under instance/siteagent/ to have it ship - in the download. + will still build with config and installer, but no agent will ship + inside it. Upload a current release in the binary section below to + have it included in every download.

@@ -317,43 +316,112 @@ class="binary-block" >

{{ binary.label }}

+

+ Upload {{ binary.name }} updated EXE. Uploading here will update the + binaries available for your "Build Site Agent" install package. +

Current staged version: - {{ binary.current || 'none (legacy or unstaged)' }} + {{ binary.current || 'none staged yet - upload a build below' }}

+ +
- + +
+ + + +

+ WARNING: confirm agent version before uploading from the EXE + properties and enter the version number. +

+
+ + +

{{ binary.message }}

{{ binary.error }}

+ +
+ + +
+
@@ -434,7 +502,7 @@ const copied = ref(false) const message = ref('') const error = ref('') -// Adoption tool (spike): build a one-shot, agentless enrollment download. +// Adoption tool: build a one-shot, agentless enrollment download. const adoptbaseurl = ref('') const adoptcollectorkey = ref('') const adoptbusy = ref(false) @@ -458,11 +526,15 @@ const agenterror = ref('') // version list, and busy/message state so the two rows act independently. The // 'key' matches the backend binkey (agent | tray). const binaries = ref([ - { key: 'agent', label: 'Endpoint agent (shopdbagent.exe)', - current: null, versions: [], version: '', file: null, + { key: 'agent', label: 'ShopDB Agent (shopdbagent.exe)', + name: 'ShopDB Agent', + current: null, versions: [], version: '', file: '', detecting: false, + legacyversion: '', legacyfile: '', busy: false, message: '', error: '', showprevious: false }, - { key: 'tray', label: 'Tray agent (shopdb-endpoint-tray.exe)', - current: null, versions: [], version: '', file: null, + { key: 'tray', label: 'Tray Agent (shopdb-endpoint-tray.exe)', + name: 'Tray Agent', + current: null, versions: [], version: '', file: '', detecting: false, + legacyversion: '', legacyfile: '', busy: false, message: '', error: '', showprevious: false }, ]) @@ -664,11 +736,39 @@ async function buildSiteAgent() { } } +async function detectVersion(binkey, file) { + const form = new FormData() + form.append('file', file) + const response = await api.post( + `/endpoints/siteagent/binaries/${binkey}/detect`, form) + return response.data.data?.version || '' +} + function onBinaryFileChange(binkey, event) { const binary = binaryFor(binkey) if (!binary) return const files = event.target.files - binary.file = files && files.length ? files[0] : null + binary.file = files && files.length ? files[0] : '' + binary.error = '' + if (!binary.file) return + binary.detecting = true + detectVersion(binkey, binary.file) + .then((version) => { + if (version) binary.version = version + }) + .catch((detectError) => { + console.error(detectError) + }) + .finally(() => { + binary.detecting = false + }) +} + +function onLegacyFileChange(binkey, event) { + const binary = binaryFor(binkey) + if (!binary) return + const files = event.target.files + binary.legacyfile = files && files.length ? files[0] : '' } async function uploadBinary(binkey) { @@ -686,7 +786,7 @@ async function uploadBinary(binkey) { const data = response.data.data || {} applyBinaryStatus(binkey, data.current, data.versions) binary.version = '' - binary.file = null + binary.file = '' binary.message = 'Uploaded and set as current.' } catch (uploadError) { binary.error = uploadError.response?.data?.data?.error?.message @@ -699,6 +799,35 @@ async function uploadBinary(binkey) { } } +async function uploadLegacy(binkey) { + const binary = binaryFor(binkey) + if (!binary || !binary.legacyfile || !binary.legacyversion) return + binary.busy = true + binary.message = '' + binary.error = '' + try { + const form = new FormData() + form.append('version', binary.legacyversion) + form.append('file', binary.legacyfile) + form.append('makecurrent', 'false') + const response = await api.post( + `/endpoints/siteagent/binaries/${binkey}`, form) + const data = response.data.data || {} + applyBinaryStatus(binkey, data.current, data.versions) + binary.legacyversion = '' + binary.legacyfile = '' + binary.message = 'Legacy build staged for rollback.' + } catch (uploadError) { + binary.error = uploadError.response?.data?.data?.error?.message + || (uploadError.response?.status === 403 + ? 'You do not have permission to upload agent binaries' + : 'Could not upload the legacy build') + console.error(uploadError) + } finally { + binary.busy = false + } +} + async function setCurrentBinary(binkey, version) { const binary = binaryFor(binkey) if (!binary) return @@ -775,6 +904,14 @@ function editConfiguration() { } .binary-upload .form-control[type="text"] { max-width: 12rem; } .previous-versions { margin-top: 0.5rem; } +.field-warning { + margin: 0.25rem 0 0.5rem; + padding: 0.5rem 0.75rem; + border-left: 3px solid var(--color-warning, #b8860b); + background: rgba(184, 134, 11, 0.08); + font-size: 0.85rem; + font-weight: 600; +} .btn-link { background: none; border: none; diff --git a/tests/test_plugins/test_endpoints_agentstore.py b/tests/test_plugins/test_endpoints_agentstore.py new file mode 100644 index 00000000..f979f4ad --- /dev/null +++ b/tests/test_plugins/test_endpoints_agentstore.py @@ -0,0 +1,57 @@ +"""Unit tests for the versioned agent binary store helpers. + +Covers the pure, context-free helpers: version validation, PE magic and PE +file-version parsing. The staging/retention paths that touch the Flask instance +directory are exercised through the API tests; here we keep to the logic that +needs no app context. + +The PE version parser walks the resource directory to the outer image's +RT_VERSION resource (not the first VS_FIXEDFILEINFO signature in the bytes), +because a self-contained single-file .NET exe bundles many framework assemblies +that each carry their own version block. Building a full PE with a resource +directory inline is impractical here, so the parser's correctness is verified +against real agent binaries in the API/integration layer; these unit tests pin +the surrounding validators and the malformed-input contract (never raises, +returns None). +""" + +from plugins.endpoints.api import agentstore + + +def test_is_valid_version_accepts_dotted_numeric(): + assert agentstore.is_valid_version('1') + assert agentstore.is_valid_version('1.6.3') + assert agentstore.is_valid_version('1.6.3.0') + + +def test_is_valid_version_rejects_bad_input(): + assert not agentstore.is_valid_version('') + assert not agentstore.is_valid_version('v1.6') + assert not agentstore.is_valid_version('1.6.3.0.1') + assert not agentstore.is_valid_version('../etc') + assert not agentstore.is_valid_version('latest') + + +def test_is_valid_binary_key(): + assert agentstore.is_valid_binary_key('agent') + assert agentstore.is_valid_binary_key('tray') + assert not agentstore.is_valid_binary_key('other') + + +def test_looks_like_pe(): + assert agentstore.looks_like_pe(b'MZ\x90\x00') + assert not agentstore.looks_like_pe(b'PK\x03\x04') + assert not agentstore.looks_like_pe(b'') + + +def test_read_pe_file_version_none_on_non_pe(): + assert agentstore.read_pe_file_version(b'not a pe file') is None + + +def test_read_pe_file_version_none_on_truncated_pe(): + # MZ header but nothing else valid: must return None, never raise. + assert agentstore.read_pe_file_version(b'MZ' + b'\x00' * 200) is None + + +def test_read_pe_file_version_none_on_empty(): + assert agentstore.read_pe_file_version(b'') is None From 6f9d1de6da041d071a2bab916d9d9295706d9eba Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Wed, 2 Sep 2026 20:41:22 -0400 Subject: [PATCH 45/90] Stamp agent version into download name and collapse binary management UI Append the staged agent version to the site-agent ZIP download name so DT operators see the build at a glance; the exe name inside the ZIP is unchanged so install.ps1 is untouched. Read the versioned name from Content-Disposition on the client. Group agent/tray binary management under a collapsed 'Upload New/Legacy Agents' section and default the key explainers to collapsed. --- plugins/endpoints/api/siteagent.py | 10 ++- .../frontend/views/AgentKeySettings.vue | 63 ++++++++++++++----- 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/plugins/endpoints/api/siteagent.py b/plugins/endpoints/api/siteagent.py index 3604aa7b..edabe28d 100644 --- a/plugins/endpoints/api/siteagent.py +++ b/plugins/endpoints/api/siteagent.py @@ -568,8 +568,16 @@ def package_siteagent(): archive.write(certpath, 'server.crt') buffer.seek(0) + # Stamp the staged agent version into the download filename so a DT operator + # can see which agent build a package carries at a glance. This is cosmetic: + # install.ps1 finds the exe by its fixed name inside the ZIP, so the download + # name never affects the install. Falls back to the unversioned name when no + # managed version is current (only a legacy flat file is staged). + agentversion = agentstore.current_version('agent') + download_name = ('shopdb-endpoint-agent-{0}.zip'.format(agentversion) + if agentversion else 'shopdb-endpoint-agent.zip') return send_file( buffer, mimetype='application/zip', as_attachment=True, - download_name='shopdb-endpoint-agent.zip') + download_name=download_name) diff --git a/plugins/endpoints/frontend/views/AgentKeySettings.vue b/plugins/endpoints/frontend/views/AgentKeySettings.vue index f213c234..a3b659f3 100644 --- a/plugins/endpoints/frontend/views/AgentKeySettings.vue +++ b/plugins/endpoints/frontend/views/AgentKeySettings.vue @@ -106,8 +106,9 @@
-

How it works

-
    +
    +

    How it works

     +
    • Generate mints a fresh key and, on the first generation, turns off the bootstrap .env fallback so @@ -127,9 +128,11 @@ the kill switch.
    +
    -

    When to rotate

    -
      +
      +

      When to rotate

       +
      • An installer or agent left the secure environment. If a build carrying the key is copied off the shop floor - onto an @@ -156,6 +159,7 @@ regenerate and redeploy to known-good endpoints.
      +
@@ -308,13 +312,15 @@ and roll the current pointer back to a retained version. Staging only changes what a FUTURE download contains; it does not touch any running endpoint. Pushing to live endpoints is a separate, - heavier grant handled elsewhere. - ============================================================= --> -
+ heavier grant handled elsewhere. Collapsed by default - binary + management is infrequent next to building a package. --> +
+

Upload New/Legacy Agents

 +

{{ binary.label }}

Upload {{ binary.name }} updated EXE. Uploading here will update the @@ -361,7 +367,7 @@ roll back to. WARNING copy per intent: the version must be confirmed from the EXE properties before uploading. -->

WARNING: confirm agent version before uploading from the EXE @@ -454,7 +460,8 @@

- + + @@ -555,6 +562,12 @@ function formatDate(value) { return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString() } +function filenameFromResponse(response, fallback) { + const disposition = response.headers?.['content-disposition'] || '' + const match = /filename\*?=(?:UTF-8'')?"?([^";]+)"?/i.exec(disposition) + return match ? decodeURIComponent(match[1]) : fallback +} + function applyStatus(data) { key.value = data.key || null envkeyfallback.value = !!data.envkeyfallback @@ -716,7 +729,10 @@ async function buildSiteAgent() { const url = window.URL.createObjectURL(response.data) const link = document.createElement('a') link.href = url - link.download = 'shopdb-endpoint-agent.zip' + // Honor the server's versioned download name (shopdb-endpoint-agent-.zip) + // from Content-Disposition so operators see the agent version in the file + // name; fall back to the fixed name when the header is absent. + link.download = filenameFromResponse(response, 'shopdb-endpoint-agent.zip') document.body.appendChild(link) link.click() link.remove() @@ -904,6 +920,25 @@ function editConfiguration() { } .binary-upload .form-control[type="text"] { max-width: 12rem; } .previous-versions { margin-top: 0.5rem; } +.collapsible { margin: 0.5rem 0; } +.collapsible > summary { + cursor: pointer; + list-style: none; + display: flex; + align-items: center; + gap: 0.4rem; + user-select: none; +} +.collapsible > summary::-webkit-details-marker { display: none; } +.collapsible > summary::before { + content: "\25be"; + display: inline-block; + transform: rotate(-90deg); + transition: transform 0.15s ease; + color: var(--color-text-muted, #888); +} +.collapsible[open] > summary::before { transform: rotate(0deg); } +.collapsible > summary h4 { margin: 0; } .field-warning { margin: 0.25rem 0 0.5rem; padding: 0.5rem 0.75rem; From 2998e35ddf685f7499828535922abfc3437a8837 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Thu, 3 Sep 2026 05:04:22 -0400 Subject: [PATCH 46/90] Add reversible Maximo/WINC machine-adoption import Machine data arrives as a Maximo asset export (validated against the WINC site) that must update existing machines in place, not create canonical per-table rows, so it does not fit the ShopDB-native/Azure/Thor CSV adapter path. Bulk edits to live machines also need an escape hatch, so the whole commit is journaled and can be undone as one batch. Add a dedicated dry-run analyzer, approval packet, resolver/suggest helpers, and a reversible writer/undo backed by a new importbatches/importjournalentries schema (with migration). Expose preview/commit/batches/undo settings routes and a matching frontend source card and panel, and surface the batch/journal contract through shopdb.api for plugins. --- docs/PLUGIN-HOOKS.md | 45 ++ frontend/src/api/index.js | 30 ++ .../views/settings/ImportSiteAssets.spec.js | 187 +++++++++ .../src/views/settings/ImportSiteAssets.vue | 390 +++++++++++++++++- .../7d37_import_batches_and_journal.py | 89 ++++ scripts/site_imports/approval.py | 149 +++++++ scripts/site_imports/clone_import_db.ps1 | 51 +++ scripts/site_imports/journal.py | 267 ++++++++++++ scripts/site_imports/resolver.py | 134 ++++++ scripts/site_imports/suggest.py | 159 +++++++ scripts/site_imports/winc/__init__.py | 0 scripts/site_imports/winc/aliases.py | 67 +++ scripts/site_imports/winc/dry_run.py | 385 +++++++++++++++++ scripts/site_imports/winc/undo.py | 55 +++ scripts/site_imports/winc/writer.py | 278 +++++++++++++ shopdb/__init__.py | 8 +- shopdb/api/__init__.py | 16 + shopdb/core/api/settings.py | 117 ++++++ shopdb/core/models/__init__.py | 9 + shopdb/core/models/importbatch.py | 86 ++++ shopdb/core/services/wincimport.py | 138 +++++++ tests/test_winc_import_routes.py | 210 ++++++++++ 22 files changed, 2859 insertions(+), 11 deletions(-) create mode 100644 docs/PLUGIN-HOOKS.md create mode 100644 frontend/src/views/settings/ImportSiteAssets.spec.js create mode 100644 migrations/versions/7d37_import_batches_and_journal.py create mode 100644 scripts/site_imports/approval.py create mode 100644 scripts/site_imports/clone_import_db.ps1 create mode 100644 scripts/site_imports/journal.py create mode 100644 scripts/site_imports/resolver.py create mode 100644 scripts/site_imports/suggest.py create mode 100644 scripts/site_imports/winc/__init__.py create mode 100644 scripts/site_imports/winc/aliases.py create mode 100644 scripts/site_imports/winc/dry_run.py create mode 100644 scripts/site_imports/winc/undo.py create mode 100644 scripts/site_imports/winc/writer.py create mode 100644 shopdb/core/models/importbatch.py create mode 100644 shopdb/core/services/wincimport.py create mode 100644 tests/test_winc_import_routes.py diff --git a/docs/PLUGIN-HOOKS.md b/docs/PLUGIN-HOOKS.md new file mode 100644 index 00000000..a4b3f531 --- /dev/null +++ b/docs/PLUGIN-HOOKS.md @@ -0,0 +1,45 @@ +# Plugin hooks and contract surface + +Everything a plugin may import from the core lives in `shopdb.api` (ADR-001), +versioned by `__contract_version__` (ADR-002). Adding a name is a minor bump; +removing one is major. This file records the non-obvious hooks - the ones a +plugin author would not guess from the name alone. It is not the full export +list; read `shopdb/api/__init__.py` for that. + +## Reversible bulk-import journal + +Contract version 0.30.0. + +A plugin that drives a bulk import - a site asset adoption, an integration sync - +records the primitive changes it makes so the whole batch can be reversed later. +The journal is import-agnostic: it stores primitive mutations (set a cell, insert +a row) tied to a batch id, and a generic undo replays them in reverse. Nothing in +core learns what the import means. + +Exposed on `shopdb.api`: + +- `ImportBatch` - one row per applied import. `batchid` is the identity that the + approval step, the write, and the undo all reference. `status` is `applied` or + `undone`. +- `ImportJournalEntry` - one reversible primitive mutation. `operation` is + `update` (a cell change, carrying `oldvalue`/`newvalue` so it can be put back) + or `insert` (a row this batch created, so undo can delete strictly what the + import created). +- `BATCH_APPLIED`, `BATCH_UNDONE` - the two `ImportBatch.status` values. +- `OP_UPDATE`, `OP_INSERT` - the two `ImportJournalEntry.operation` values. + +What a plugin gets from this, and what it does NOT: + +- A plugin's import UI reads `ImportBatch` to list applied batches (kind, row + count, who applied it, when) and offers an "Undo import" per batch. +- A plugin does NOT write the journal or run undo through these model imports. + The write/undo logic lives with the import writer (see + `scripts/site_imports/journal.py`), which records entries in the SAME + transaction as the mutation they describe and reverses them conservatively: + an `update` is put back only if the cell still holds the value the import + wrote (a value a human changed since is left as-is and reported), and an + `insert` is hard-deleted only when nothing outside the batch still references + the created row. + +The tables are core-owned (migration `7d37_import_batches_and_journal`), so they +exist regardless of which plugins are installed. diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 57fcc970..304ece15 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -1204,6 +1204,36 @@ export const settingsApi = { return api.post('/settings/asset-import/import', form, { headers: { 'Content-Type': 'multipart/form-data' } }) + }, + wincPreview(file) { + // Read-only WINC (Maximo-style) machine-adoption analysis. Uploads one CSV + // and returns the approval packet (counts, needs-create business units, + // unresolved values); writes nothing. + const form = new FormData() + form.append('files', file) + return api.post('/settings/asset-import/winc/preview', form, { + headers: { 'Content-Type': 'multipart/form-data' } + }) + }, + wincCommit(file, acknowledge = []) { + // Apply a WINC plan reversibly. acknowledge is the list of needs-create + // group names (e.g. ['businessunit']) the admin confirmed. + const form = new FormData() + form.append('files', file) + if (acknowledge.length) { + form.append('acknowledge', acknowledge.join(',')) + } + return api.post('/settings/asset-import/winc/commit', form, { + headers: { 'Content-Type': 'multipart/form-data' } + }) + }, + wincBatches() { + // Applied/undone WINC import batches, newest first, for the undo list. + return api.get('/settings/asset-import/winc/batches') + }, + wincUndo(batchid) { + // Reverse an applied WINC import batch by id. + return api.post('/settings/asset-import/winc/undo', { batchid }) } } diff --git a/frontend/src/views/settings/ImportSiteAssets.spec.js b/frontend/src/views/settings/ImportSiteAssets.spec.js new file mode 100644 index 00000000..a682a15b --- /dev/null +++ b/frontend/src/views/settings/ImportSiteAssets.spec.js @@ -0,0 +1,187 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { mount, flushPromises } from '@vue/test-utils' + +// The WINC source is the one import path that is NOT a canonical-CSV adapter: +// it previews a machine-adoption plan, gates a commit behind an explicit +// business-unit acknowledgment, and exposes an undo list. These specs pin that +// gate and the preview/commit/undo wiring, mocking the settings API so no +// network or backend is involved. + +vi.mock('@/api', () => ({ + settingsApi: { + get: vi.fn().mockRejectedValue({ response: { status: 404 } }), + listImportTemplates: vi.fn().mockResolvedValue({ data: { data: { tables: [] } } }), + wincPreview: vi.fn(), + wincCommit: vi.fn(), + wincBatches: vi.fn().mockResolvedValue({ data: { data: { batches: [] } } }), + wincUndo: vi.fn().mockResolvedValue({ data: { data: { reverted: 3, conflicts: [] } } }), + }, + computersApi: { types: { listAll: vi.fn().mockResolvedValue([]) } }, +})) +vi.mock('@/utils/apiError', () => ({ apiError: (e) => String(e) })) +vi.mock('../../utils/apiError', () => ({ apiError: (e) => String(e) })) + +import { settingsApi } from '@/api' +import ImportSiteAssets from './ImportSiteAssets.vue' + +function previewReport(overrides = {}) { + return { + data: { + data: { + committed: false, + blocking: false, + rowcount: 1, + summary: { in_scope: 1, skipped_no_existing_machine: 0, ambiguous_match: 0 }, + needscreate: { businessunit: { 'Large Parts': 1 } }, + unresolved: {}, + ...overrides, + }, + }, + } +} + +async function render() { + const wrapper = mount(ImportSiteAssets, { + global: { + stubs: { 'router-link': { template: '' } }, + }, + }) + await flushPromises() + return wrapper +} + +function selectWinc(wrapper) { + // The WINC card is the fourth source-format button. + const card = wrapper.findAll('.source-card').find(c => c.text().includes('WINC')) + return card.trigger('click') +} + +async function pickFile(wrapper) { + const input = wrapper.find('.winc-panel input[type="file"]') + const file = new File(['assetnum\nW0083A\n'], 'winc.csv', { type: 'text/csv' }) + Object.defineProperty(input.element, 'files', { value: [file], configurable: true }) + await input.trigger('change') + return file +} + +describe('WINC source selection', () => { + beforeEach(() => { + settingsApi.wincPreview.mockReset() + settingsApi.wincCommit.mockReset() + }) + + it('offers WINC as a fourth source card', async () => { + const wrapper = await render() + const labels = wrapper.findAll('.source-card').map(c => c.text()) + expect(labels.some(t => t.includes('WINC'))).toBe(true) + }) + + it('shows the WINC panel instead of the generic controls when selected', async () => { + const wrapper = await render() + await selectWinc(wrapper) + expect(wrapper.find('.winc-panel').exists()).toBe(true) + }) +}) + +describe('WINC preview and commit gating', () => { + beforeEach(() => { + settingsApi.wincPreview.mockReset() + settingsApi.wincCommit.mockReset() + settingsApi.wincBatches.mockClear() + settingsApi.wincBatches.mockResolvedValue({ data: { data: { batches: [] } } }) + }) + + it('previews the uploaded export and lists needs-create business units', async () => { + settingsApi.wincPreview.mockResolvedValue(previewReport()) + const wrapper = await render() + await selectWinc(wrapper) + await pickFile(wrapper) + + await wrapper.find('.winc-panel .btn-secondary').trigger('click') + await flushPromises() + + expect(settingsApi.wincPreview).toHaveBeenCalledOnce() + expect(wrapper.find('.winc-needscreate').text()).toContain('Large Parts') + }) + + it('blocks commit until the business-unit creation is acknowledged', async () => { + settingsApi.wincPreview.mockResolvedValue(previewReport()) + const wrapper = await render() + await selectWinc(wrapper) + await pickFile(wrapper) + await wrapper.find('.winc-panel .btn-secondary').trigger('click') + await flushPromises() + + const commitBtn = wrapper.find('.winc-panel .btn-primary') + expect(commitBtn.attributes('disabled')).toBeDefined() + + await wrapper.find('.winc-ack input[type="checkbox"]').setValue(true) + expect(commitBtn.attributes('disabled')).toBeUndefined() + }) + + it('commits with the businessunit acknowledgment and refreshes the batch list', async () => { + settingsApi.wincPreview.mockResolvedValue(previewReport()) + settingsApi.wincCommit.mockResolvedValue({ + data: { data: { committed: true, rowcount: 1, batchid: 'b-1', summary: { in_scope: 1, skipped_no_existing_machine: 0, ambiguous_match: 0 }, needscreate: {}, unresolved: {} } }, + }) + const wrapper = await render() + await selectWinc(wrapper) + await pickFile(wrapper) + await wrapper.find('.winc-panel .btn-secondary').trigger('click') + await flushPromises() + await wrapper.find('.winc-ack input[type="checkbox"]').setValue(true) + + await wrapper.find('.winc-panel .btn-primary').trigger('click') + await flushPromises() + + const [file, acknowledge] = settingsApi.wincCommit.mock.calls[0] + expect(file.name).toBe('winc.csv') + expect(acknowledge).toEqual(['businessunit']) + // Batches reloaded once on mount and once after commit. + expect(settingsApi.wincBatches).toHaveBeenCalledTimes(2) + }) + + it('does not require acknowledgment when there is nothing to create', async () => { + settingsApi.wincPreview.mockResolvedValue(previewReport({ needscreate: {} })) + const wrapper = await render() + await selectWinc(wrapper) + await pickFile(wrapper) + await wrapper.find('.winc-panel .btn-secondary').trigger('click') + await flushPromises() + + expect(wrapper.find('.winc-ack').exists()).toBe(false) + expect(wrapper.find('.winc-panel .btn-primary').attributes('disabled')).toBeUndefined() + }) + + it('blocks commit when the plan is blocking', async () => { + settingsApi.wincPreview.mockResolvedValue(previewReport({ + blocking: true, needscreate: {}, unresolved: { location: { 'WM-XYZ': 2 } }, + })) + const wrapper = await render() + await selectWinc(wrapper) + await pickFile(wrapper) + await wrapper.find('.winc-panel .btn-secondary').trigger('click') + await flushPromises() + + expect(wrapper.find('.winc-panel .btn-primary').attributes('disabled')).toBeDefined() + expect(wrapper.find('.problems').text()).toContain('WM-XYZ') + }) +}) + +describe('WINC undo', () => { + it('undoes an applied batch and reloads the list', async () => { + settingsApi.wincBatches.mockResolvedValue({ + data: { data: { batches: [ + { batchid: 'b-1', kind: 'winc-machine-adoption', rowcount: 5, createdat: '2026-01-01T00:00:00Z', createdby: 'admin', status: 'applied', undoneat: null }, + ] } }, + }) + const wrapper = await render() + await selectWinc(wrapper) + await flushPromises() + + await wrapper.find('.winc-batches .btn-secondary').trigger('click') + await flushPromises() + + expect(settingsApi.wincUndo).toHaveBeenCalledWith('b-1') + }) +}) diff --git a/frontend/src/views/settings/ImportSiteAssets.vue b/frontend/src/views/settings/ImportSiteAssets.vue index 679eedc1..445e0d84 100644 --- a/frontend/src/views/settings/ImportSiteAssets.vue +++ b/frontend/src/views/settings/ImportSiteAssets.vue @@ -112,6 +112,15 @@ :class="['source-card', { active: importSource === s.key }]" @click="selectSource(s.key)" > + + + {{ s.tag }} + + {{ s.label }} {{ s.description }} @@ -204,7 +213,166 @@

-
+
+

+ A WINC (Maximo-style) asset export adopts existing machines: it never + creates an asset, only updates a machine that already exists, matched + on its workstation number. It sets the machine's name, location, and + business unit. Preview first - it writes nothing and reports what would + change, which business units it would create (you must confirm), and + any values it cannot resolve (which block the import). Every commit is + journalled, so it can be undone from the list below. +

+ +
+ +
+ + + +
+
+ +

Selected: {{ wincFile.name }}

+

{{ wincError }}

+ +
+

+ + + +

+ + + + + + + + + + + + + + + + +
In scope (existing machines matched){{ wincReport.summary.in_scope }}
Skipped (no existing machine){{ wincReport.summary.skipped_no_existing_machine }}
Ambiguous (skipped, not guessed){{ wincReport.summary.ambiguous_match }}
+ +
+

New business units to be created

+

+ These business units are referenced by the export but do not exist + yet. Confirm they are correct - you can edit them later in + Settings. Nothing is created until you acknowledge and commit. +

+
    +
  • + {{ name }} ({{ count }} machine(s)) +
    • +
+ +
+ +
+

Unresolved values (blocking)

+

+ These export values have no mapping yet, so the import will not + touch the affected rows. An administrator must add an alias for + each before this import can run. +

+
    +
  • + {{ group }}: + + "{{ value }}" ({{ count }}){{ ' ' }} + +
    • +
+
+
+ +
+

Import history

+

+ Each committed WINC import can be undone. Undo reverses only cells + this import set that have not been changed since, and removes only + business units this import created. +

+ + + + + + + + + + + + + + + + + + + +
WhenByRowsStatus
{{ b.createdat }}{{ b.createdby || '-' }}{{ b.rowcount }}{{ b.status }} + + undone +
+
+
+ +
+ !!(wincReport.value && wincReport.value.needscreate + && wincReport.value.needscreate.businessunit)) + +const wincCanCommit = computed(() => { + const wincResult = wincReport.value + if (!wincResult || wincCommitting.value || wincPreviewing.value) return false + if (wincResult.committed || wincResult.blocking) return false + if (wincResult.rowcount === 0) return false + if (wincNeedsBusinessUnit.value && !wincAckBusinessUnit.value) return false + return true +}) + // Commit is allowed only after a clean preview of the CURRENT selection: a // report with no problems and no request-level error that has not already been // committed. @@ -651,6 +884,77 @@ async function commit() { } } +function onWincFilePicked(event) { + wincFile.value = (event.target.files || [])[0] || null + // A new file invalidates the previous report and its acknowledgment. + wincReport.value = null + wincError.value = '' + wincAckBusinessUnit.value = false +} + +function clearWincSelection() { + wincFile.value = null + wincReport.value = null + wincError.value = '' + wincAckBusinessUnit.value = false + if (wincFileInput.value) wincFileInput.value.value = '' +} + +async function wincPreview() { + if (!wincFile.value) return + wincPreviewing.value = true + wincError.value = '' + wincAckBusinessUnit.value = false + try { + const response = await settingsApi.wincPreview(wincFile.value) + wincReport.value = response.data?.data || response.data || null + } catch (err) { + wincError.value = apiError(err, 'WINC preview failed.') + wincReport.value = null + } finally { + wincPreviewing.value = false + } +} + +async function wincCommit() { + if (!wincCanCommit.value) return + wincCommitting.value = true + wincError.value = '' + try { + const acknowledge = wincNeedsBusinessUnit.value ? ['businessunit'] : [] + const response = await settingsApi.wincCommit(wincFile.value, acknowledge) + wincReport.value = response.data?.data || response.data || null + await loadWincBatches() + } catch (err) { + wincError.value = apiError(err, 'WINC import failed.') + } finally { + wincCommitting.value = false + } +} + +async function loadWincBatches() { + try { + const response = await settingsApi.wincBatches() + const data = response.data?.data || response.data || {} + wincBatches.value = data.batches || [] + } catch { + wincBatches.value = [] + } +} + +async function undoWincBatch(batchid) { + wincUndoingId.value = batchid + wincError.value = '' + try { + await settingsApi.wincUndo(batchid) + await loadWincBatches() + } catch (err) { + wincError.value = apiError(err, 'WINC undo failed.') + } finally { + wincUndoingId.value = '' + } +} + async function addComputerType() { const name = customComputerType.value.trim() if (!name) return @@ -683,6 +987,7 @@ onMounted(async () => { await loadLocation() await loadTemplates() await loadComputerTypes() + await loadWincBatches() }) @@ -769,27 +1074,63 @@ onMounted(async () => { .source-cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); - gap: 10px; - margin: 4px 0; + gap: 12px; + margin: 6px 0; } .source-card { display: flex; flex-direction: column; - gap: 4px; + gap: 6px; text-align: left; - padding: 12px 14px; + padding: 14px; border: 1px solid var(--border); - border-radius: 8px; + border-radius: 10px; background: var(--bg); color: var(--text); cursor: pointer; + position: relative; + transition: border-color 0.15s ease, box-shadow 0.15s ease, transform 0.15s ease, background 0.15s ease; } .source-card:hover { border-color: var(--primary); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.18); + transform: translateY(-1px); } .source-card.active { border-color: var(--primary); - box-shadow: 0 0 0 1px var(--primary) inset; + box-shadow: 0 0 0 1px var(--primary) inset, 0 2px 10px rgba(0, 0, 0, 0.22); + background: color-mix(in srgb, var(--primary) 8%, var(--bg)); +} +.source-card-head { + display: flex; + align-items: center; + gap: 8px; +} +.source-card-icon { + width: 22px; + height: 22px; + color: var(--text-light); + flex: none; +} +.source-card.active .source-card-icon { + color: var(--primary); +} +.source-card-tag { + font-size: 0.68rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; + padding: 2px 8px; + border-radius: 999px; + background: color-mix(in srgb, var(--primary) 14%, transparent); + color: var(--primary); +} +.source-card-check { + width: 18px; + height: 18px; + color: var(--primary); + margin-left: auto; + flex: none; } .source-card-title { font-weight: 600; @@ -963,4 +1304,33 @@ onMounted(async () => { margin: 4px 0; font-size: 0.9rem; } +.winc-panel { + margin-top: 12px; +} +.winc-needscreate { + margin-top: 12px; + border: 1px solid var(--warning, #d0a44a); + border-radius: 8px; + padding: 12px 14px; +} +.winc-needscreate h4 { + margin: 0 0 6px; +} +.winc-needscreate ul { + margin: 6px 0; + padding-left: 20px; +} +.winc-ack { + display: flex; + align-items: center; + gap: 8px; + margin-top: 8px; + font-size: 0.9rem; +} +.winc-batches { + margin-top: 16px; +} +.winc-batches h4 { + margin: 0 0 6px; +} diff --git a/migrations/versions/7d37_import_batches_and_journal.py b/migrations/versions/7d37_import_batches_and_journal.py new file mode 100644 index 00000000..daf99fd1 --- /dev/null +++ b/migrations/versions/7d37_import_batches_and_journal.py @@ -0,0 +1,89 @@ +"""Add importbatches and importjournalentries (reversible bulk imports) + +Two additive, platform-level tables that let any bulk-import writer record what +it changed so the change can be reversed later. Import-agnostic: an importer +writes primitive mutations (update a cell, insert a row) tied to a batch id, and +a generic undo replays them in reverse. + + importbatches one row per approved write (identity: batchid, the same + value the approval packet carries). + importjournalentries one row per reversible primitive mutation, ordered by + sequence so undo can walk them in reverse. + +Idempotent guards and a real downgrade throughout; strict MySQL 8 clean. + +Revision ID: 7d37_import_batches_and_journal +Revises: 7d36_scriptlibrary_and_app_installer_digest +Create Date: 2026-09-03 + +""" +from alembic import op +import sqlalchemy as sa + + +revision = '7d37_import_batches_and_journal' +down_revision = '7d36_scriptlibrary_and_app_installer_digest' +branch_labels = None +depends_on = None + + +def upgrade(): + bind = op.get_bind() + insp = sa.inspect(bind) + tables = insp.get_table_names() + + if 'importbatches' not in tables: + op.create_table( + 'importbatches', + sa.Column('batchid', sa.String(length=36), primary_key=True), + sa.Column('kind', sa.String(length=64), nullable=False), + sa.Column('planhash', sa.String(length=64), nullable=False), + sa.Column('rowcount', sa.Integer(), nullable=False, + server_default='0'), + sa.Column('createdat', sa.DateTime(), nullable=False), + sa.Column('createdby', sa.String(length=100), nullable=True), + sa.Column('status', sa.String(length=16), nullable=False, + server_default='applied'), + sa.Column('undoneat', sa.DateTime(), nullable=True), + ) + + if 'importjournalentries' not in tables: + op.create_table( + 'importjournalentries', + sa.Column('entryid', sa.BigInteger(), primary_key=True, + autoincrement=True), + sa.Column('batchid', sa.String(length=36), nullable=False), + sa.Column('sequence', sa.Integer(), nullable=False), + sa.Column('operation', sa.String(length=16), nullable=False), + sa.Column('targettable', sa.String(length=64), nullable=False), + sa.Column('targetid', sa.BigInteger(), nullable=False), + sa.Column('columnname', sa.String(length=64), nullable=True), + sa.Column('oldvalue', sa.Text(), nullable=True), + sa.Column('newvalue', sa.Text(), nullable=True), + sa.Column('isreverted', sa.Boolean(), nullable=False, + server_default=sa.text('0')), + sa.ForeignKeyConstraint( + ['batchid'], ['importbatches.batchid'], + name='fk_importjournalentries_batchid', + ondelete='CASCADE'), + ) + op.create_index( + 'idx_importjournalentries_batchid', + 'importjournalentries', ['batchid']) + + +def downgrade(): + bind = op.get_bind() + insp = sa.inspect(bind) + tables = insp.get_table_names() + + if 'importjournalentries' in tables: + existing = [ix['name'] for ix in + insp.get_indexes('importjournalentries')] + if 'idx_importjournalentries_batchid' in existing: + op.drop_index('idx_importjournalentries_batchid', + table_name='importjournalentries') + op.drop_table('importjournalentries') + + if 'importbatches' in tables: + op.drop_table('importbatches') diff --git a/scripts/site_imports/approval.py b/scripts/site_imports/approval.py new file mode 100644 index 00000000..5892f867 --- /dev/null +++ b/scripts/site_imports/approval.py @@ -0,0 +1,149 @@ +"""Reusable approval / promotion gate for site-import writers. + +IMPORT-AGNOSTIC. This turns a dry-run plan into an "approval packet": a frozen +summary of what an import WOULD do, plus a content hash of the exact plan it +describes. A writer refuses to run unless it is handed a packet whose hash still +matches the plan in front of it. That is the whole mechanism: a human approves a +specific, frozen plan, and the writer can prove nothing drifted since. + +The same packet shape serves every importer, and it is deliberately the seam the +undo work will reuse: a writer records the packet's batch id alongside the +journal it writes, so approval, write, and undo all reference one identity. + +A packet is a plain dict (JSON-serialisable) so it can be printed, saved next to +an export, diffed between runs, and shown in a UI without any of this code: + + { + 'kind': caller-supplied label, e.g. 'winc-machine-adoption' + 'planhash': sha256 of the canonicalised plan rows + 'rowcount': number of rows in the plan + 'counts': caller-supplied summary dict (what the report shows) + 'needscreate': { group -> { name -> count } } # requires acknowledgment + 'unresolved': { group -> { value -> count } } # blocks approval + 'blocking': True if any unresolved entries exist + } + +The writer's contract, in order: + 1. rebuild the plan from the same inputs, + 2. build a packet from it, + 3. compare planhash to the approved packet - abort on mismatch, + 4. refuse if packet['blocking'], + 5. require explicit acknowledgment for every needscreate group, + 6. only then write, stamping packet['batchid'] onto the journal. +""" + +import hashlib +import json +import uuid +from collections import OrderedDict + +from scripts.site_imports.resolver import tally, NEEDS_CREATE, UNRESOLVED + + +def plan_hash(plan): + """Stable sha256 over the plan's rows. + + Canonicalised by serialising each row as sorted-key JSON so the hash depends + only on content, not on dict ordering or whitespace. Two runs that produce + the same decisions produce the same hash; any changed decision changes it, + which is exactly what the writer checks before it touches the database. + """ + hasher = hashlib.sha256() + for row in plan: + canonical = json.dumps(row, sort_keys=True, default=str, + separators=(',', ':')) + hasher.update(canonical.encode('utf-8')) + hasher.update(b'\n') + return hasher.hexdigest() + + +def build_packet(kind, plan, counts, groups): + """Build an approval packet from a plan. + + kind a short label identifying the importer, stored on the packet. + plan the list of per-row dicts the writer will act on. + counts the caller's summary dict (shown to the human as-is). + groups how to read each field's status/value columns out of a row: + [ (group_name, status_column, value_column), ... ] + e.g. ('businessunit', 'businessunit_status', 'raw_eq1'). The value + column is what a human would act on - the raw export value for + UNRESOLVED (so they know what to alias) and the canonical name for + NEEDS_CREATE (so they know what would be created). + + needscreate is keyed on the canonical name column when present; callers pass + the *_name column for that via the fourth tuple element when it differs from + the value column. + """ + needscreate = OrderedDict() + unresolved = OrderedDict() + for group in groups: + group_name, status_column, value_column = group[0], group[1], group[2] + name_column = group[3] if len(group) > 3 else value_column + unresolved_hits = tally(plan, status_column, value_column, UNRESOLVED) + needs_hits = tally(plan, status_column, name_column, NEEDS_CREATE) + if unresolved_hits: + unresolved[group_name] = unresolved_hits + if needs_hits: + needscreate[group_name] = needs_hits + + return OrderedDict([ + ('kind', kind), + ('batchid', str(uuid.uuid4())), + ('planhash', plan_hash(plan)), + ('rowcount', len(plan)), + ('counts', OrderedDict(counts)), + ('needscreate', needscreate), + ('unresolved', unresolved), + ('blocking', bool(unresolved)), + ]) + + +def packet_matches_plan(packet, plan): + """True if the approved packet still describes this exact plan.""" + return packet.get('planhash') == plan_hash(plan) + + +def format_packet(packet): + """Human-readable rendering of a packet for the console or a UI panel.""" + lines = [] + lines.append('APPROVAL PACKET: %s' % packet.get('kind', '?')) + lines.append(' batch id : %s' % packet.get('batchid', '?')) + lines.append(' plan hash: %s' % packet.get('planhash', '?')) + lines.append(' rows : %d' % packet.get('rowcount', 0)) + + needscreate = packet.get('needscreate') or {} + if needscreate: + for group, items in needscreate.items(): + total = sum(items.values()) + lines.append('') + lines.append(' WARNING: this import would CREATE %d %s(s). ' + 'Confirm these are correct; you can edit them later ' + 'in Settings:' % (len(items), group)) + for name, count in items.items(): + lines.append(' + %-28s (%d rows)' % (name, count)) + lines.append(' total new %s rows: %d' % (group, total)) + + unresolved = packet.get('unresolved') or {} + if unresolved: + for group, items in unresolved.items(): + lines.append('') + lines.append(' BLOCKING: %d unresolved %s value(s) - add an alias ' + 'for each before importing:' % (len(items), group)) + for value, count in items.items(): + lines.append(' ? %-28s (%d rows)' % (value, count)) + + lines.append('') + lines.append(' status : %s' % ( + 'BLOCKED (resolve the values above)' if packet.get('blocking') + else 'READY for acknowledgment')) + return '\n'.join(lines) + + +def save_packet(packet, path): + with open(path, 'w', encoding='utf-8') as handle: + json.dump(packet, handle, indent=2) + + +def load_packet(path): + with open(path, encoding='utf-8') as handle: + return json.load(handle) diff --git a/scripts/site_imports/clone_import_db.ps1 b/scripts/site_imports/clone_import_db.ps1 new file mode 100644 index 00000000..5f7c4de0 --- /dev/null +++ b/scripts/site_imports/clone_import_db.ps1 @@ -0,0 +1,51 @@ +# Clone the live shopdb_flask database into a throwaway copy for import testing. +# +# Runs entirely inside the db container so the root password is read from the +# container's own environment and never typed on a command line or logged. The +# clone is a full structural + data copy, so a dry run or a writer pointed at it +# behaves exactly as it would against production - but nothing it does can touch +# the real data. +# +# pwsh scripts/site_imports/clone_import_db.ps1 +# pwsh scripts/site_imports/clone_import_db.ps1 -Target shopdb_flask_import +# +# Point a tool at the clone by overriding DATABASE_URL for that one run, e.g. +# docker compose exec -e DATABASE_URL="mysql+pymysql://root:PW@db:3306/shopdb_flask_import?charset=utf8mb4" api ... +# (the winc dry run also honours DATABASE_URL, so --export works unchanged). + +param( + [string]$Source = 'shopdb_flask', + [string]$Target = 'shopdb_flask_import' +) + +$ErrorActionPreference = 'Stop' + +Write-Host "Cloning '$Source' -> '$Target' inside the db container..." + +# All work happens in the container. $MYSQL_ROOT_PASSWORD is already set there. +# MYSQL_PWD is honoured by both mysql and mysqldump, which keeps the password +# off the argument list (no -p on the command line to leak into ps or logs). +# Backticks around identifiers are MySQL's; placeholders are filled in by +# PowerShell below so the shell here sees plain literal database names. +$remote = @' +set -e +export MYSQL_PWD="$MYSQL_ROOT_PASSWORD" +mysql -h 127.0.0.1 -u root -e "DROP DATABASE IF EXISTS \`__TARGET__\`; CREATE DATABASE \`__TARGET__\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" +mysqldump -h 127.0.0.1 -u root --single-transaction --routines --triggers __SOURCE__ | mysql -h 127.0.0.1 -u root __TARGET__ +mysql -h 127.0.0.1 -u root -N -e "SELECT CONCAT('assets in clone: ', COUNT(*)) FROM __TARGET__.assets;" +'@ + +$remote = $remote.Replace('__TARGET__', $Target).Replace('__SOURCE__', $Source) + +# Strip CR so the container's sh sees clean LF-only lines (this file is CRLF on +# a Windows checkout, and sh treats a trailing \r as part of the argument). +$remote = $remote.Replace("`r", '') + +# The db service name in docker-compose.yml is 'db'. +docker compose exec -T db sh -c $remote + +if ($LASTEXITCODE -ne 0) { + throw "Clone failed (exit $LASTEXITCODE)." +} + +Write-Host "Clone complete: $Target" diff --git a/scripts/site_imports/journal.py b/scripts/site_imports/journal.py new file mode 100644 index 00000000..651af9f4 --- /dev/null +++ b/scripts/site_imports/journal.py @@ -0,0 +1,267 @@ +"""Shared, import-agnostic journal + undo for reversible bulk imports. + +IMPORT-AGNOSTIC. Nothing here knows about WINC, Maximo, machines, or any one +source. An import writer records the primitive mutations it makes - "set column +X on row Y of table T from old to new" (update) or "insert row Y into table T" +(insert) - each tied to a batch id, and a generic undo replays them in reverse. +That is the whole mechanism: importers emit primitives; undo reverses primitives. + +Identity: one value threads approval, write, and undo. The approval packet's +`batchid` becomes the importbatches primary key, becomes the journal foreign key, +becomes the argument the "Undo import" button passes. Nothing to reconcile. + +Contract (extends the approval packet's 6-step writer contract): + open_batch() -> after the packet passes, write the importbatches row. + record_update()/record_insert() -> append a journal entry in the SAME + transaction as the mutation it describes (ledger rule: the + entry and the change it records commit together). + undo_batch() -> reverse-replay this batch's entries, conservatively. + +Undo is deliberately conservative and never destructive beyond what the import +owns: + - update reverts a cell ONLY if it still holds the value we wrote; a value a + human changed since is left as-is and reported as a conflict (never clobbered). + - insert hard-deletes ONLY the row this batch created, and only when nothing + outside the batch still references it; otherwise it is left and reported. + +All access goes through a SQLAlchemy connection executing parameterised text, so +this reuses the same connection an importer already opened and pulls in no ORM +models - keeping it a true platform primitive any importer can share. +""" + +from collections import namedtuple +from datetime import datetime, timezone + +BATCH_APPLIED = 'applied' +BATCH_UNDONE = 'undone' + +OP_UPDATE = 'update' +OP_INSERT = 'insert' + +# Result of an undo pass, for a report and a UI. +UndoResult = namedtuple('UndoResult', ('batchid', 'reverted', 'conflicts')) + +# A single thing undo could not safely reverse. +Conflict = namedtuple('Conflict', ('operation', 'table', 'targetid', + 'column', 'reason')) + + +def _utcnow(): + return datetime.now(timezone.utc).replace(tzinfo=None) + + +def _as_text(value): + """Store every journalled value as text so a pre/post image compares the + same way it will read back, regardless of the column's declared type.""" + return None if value is None else str(value) + + +def open_batch(connection, packet, createdby=None): + """Write the importbatches row for an approved packet. + + Call once, after the packet has been verified against the current plan and + acknowledged. Returns the batchid so the caller stamps it on every journal + entry it then records. Import-agnostic: the packet's own fields carry all + the identity this needs. + """ + from sqlalchemy import text + + connection.execute(text( + 'INSERT INTO importbatches ' + '(batchid, kind, planhash, rowcount, createdat, createdby, status) ' + 'VALUES (:batchid, :kind, :planhash, :rowcount, :createdat, ' + ':createdby, :status)'), + { + 'batchid': packet['batchid'], + 'kind': packet.get('kind'), + 'planhash': packet.get('planhash'), + 'rowcount': packet.get('rowcount', 0), + 'createdat': _utcnow(), + 'createdby': createdby, + 'status': BATCH_APPLIED, + }) + return packet['batchid'] + + +def record_update(connection, batchid, sequence, table, targetid, column, + oldvalue, newvalue): + """Append an 'update' journal entry describing one cell change. + + Record the pre-image (oldvalue) you read BEFORE applying the change and the + post-image (newvalue) you are about to write, in the same transaction as the + UPDATE itself. + """ + _insert_entry(connection, batchid, sequence, OP_UPDATE, table, targetid, + column, oldvalue, newvalue) + + +def record_insert(connection, batchid, sequence, table, targetid): + """Append an 'insert' journal entry for a row this batch created. + + Only ever emitted for a needs-create row the writer itself inserted, so undo + can hard-delete strictly what the import created. + """ + _insert_entry(connection, batchid, sequence, OP_INSERT, table, targetid, + None, None, None) + + +def _insert_entry(connection, batchid, sequence, operation, table, targetid, + column, oldvalue, newvalue): + from sqlalchemy import text + + connection.execute(text( + 'INSERT INTO importjournalentries ' + '(batchid, sequence, operation, targettable, targetid, columnname, ' + 'oldvalue, newvalue, isreverted) ' + 'VALUES (:batchid, :sequence, :operation, :targettable, :targetid, ' + ':columnname, :oldvalue, :newvalue, 0)'), + { + 'batchid': batchid, + 'sequence': sequence, + 'operation': operation, + 'targettable': table, + 'targetid': targetid, + 'columnname': column, + 'oldvalue': _as_text(oldvalue), + 'newvalue': _as_text(newvalue), + }) + + +def undo_batch(connection, batchid, references=None): + """Reverse-replay an applied batch's journal. Returns an UndoResult. + + references (optional): { table -> [(reftable, refcolumn), ...] } describing + which columns in other tables point at a created row's id. Used to decide + whether an inserted row is safe to hard-delete: if any reference OUTSIDE + this batch's own updates still points at it, the row is left and reported as + a conflict rather than deleted. Omit it to require that the batch's own + update-reverts have already cleared every reference (the common case, since a + created businessunit is only ever pointed at by the same batch's asset + updates). + + Idempotent: a batch already 'undone' is a no-op. Entries are walked in + reverse sequence so inserts are removed only after the updates that pointed + at them are reverted. + """ + from sqlalchemy import text + + batch = connection.execute(text( + 'SELECT status FROM importbatches WHERE batchid = :batchid'), + {'batchid': batchid}).fetchone() + if batch is None: + raise ValueError('unknown batchid: %s' % batchid) + if batch[0] == BATCH_UNDONE: + return UndoResult(batchid, 0, []) + + entries = connection.execute(text( + 'SELECT entryid, sequence, operation, targettable, targetid, ' + 'columnname, oldvalue, newvalue FROM importjournalentries ' + 'WHERE batchid = :batchid AND isreverted = 0 ' + 'ORDER BY sequence DESC, entryid DESC'), + {'batchid': batchid}).fetchall() + + reverted = 0 + conflicts = [] + for row in entries: + (entryid, _sequence, operation, table, targetid, column, + oldvalue, newvalue) = row + + if operation == OP_UPDATE: + conflict = _revert_update(connection, table, targetid, column, + oldvalue, newvalue) + elif operation == OP_INSERT: + conflict = _revert_insert(connection, batchid, table, targetid, + references) + else: + conflict = Conflict(operation, table, targetid, column, + 'unknown operation') + + if conflict is None: + connection.execute(text( + 'UPDATE importjournalentries SET isreverted = 1 ' + 'WHERE entryid = :entryid'), {'entryid': entryid}) + reverted += 1 + else: + conflicts.append(conflict) + + connection.execute(text( + 'UPDATE importbatches SET status = :status, undoneat = :undoneat ' + 'WHERE batchid = :batchid'), + {'status': BATCH_UNDONE, 'undoneat': _utcnow(), 'batchid': batchid}) + + return UndoResult(batchid, reverted, conflicts) + + +def _revert_update(connection, table, targetid, column, oldvalue, newvalue): + """Revert one cell to its pre-image, but only if it still holds what we + wrote. Returns a Conflict if a human changed it since, else None.""" + from sqlalchemy import text + + pk = _primary_key(connection, table) + current = connection.execute(text( + 'SELECT %s FROM %s WHERE %s = :targetid' % (column, table, pk)), + {'targetid': targetid}).fetchone() + if current is None: + return Conflict(OP_UPDATE, table, targetid, column, 'row missing') + + if _as_text(current[0]) != _as_text(newvalue): + return Conflict(OP_UPDATE, table, targetid, column, + 'value changed since import') + + connection.execute(text( + 'UPDATE %s SET %s = :oldvalue WHERE %s = :targetid' + % (table, column, pk)), + {'oldvalue': oldvalue, 'targetid': targetid}) + return None + + +def _revert_insert(connection, batchid, table, targetid, references): + """Hard-delete a row this batch created, only if nothing outside the batch + still references it. Returns a Conflict when it is not safe, else None.""" + from sqlalchemy import text + + pk = _primary_key(connection, table) + for reftable, refcolumn in (references or {}).get(table, []): + referencing = connection.execute(text( + 'SELECT COUNT(*) FROM %s WHERE %s = :targetid' + % (reftable, refcolumn)), {'targetid': targetid}).fetchone()[0] + if referencing: + return Conflict(OP_INSERT, table, targetid, None, + 'still referenced by %s.%s' % (reftable, refcolumn)) + + connection.execute(text( + 'DELETE FROM %s WHERE %s = :targetid' % (table, pk)), + {'targetid': targetid}) + return None + + +def _primary_key(connection, table): + """Single-column primary key name for a table, via the live schema.""" + from sqlalchemy import inspect + + pk = inspect(connection).get_pk_constraint(table) + columns = pk.get('constrained_columns') or [] + if len(columns) != 1: + raise ValueError( + 'journal undo needs a single-column primary key on %s' % table) + return columns[0] + + +def format_undo_result(result): + """Human-readable rendering of an UndoResult for the console or a UI panel.""" + lines = [] + lines.append('UNDO batch %s' % result.batchid) + lines.append(' reverted : %d entries' % result.reverted) + if result.conflicts: + lines.append(' conflicts: %d (left as-is, not reverted)' + % len(result.conflicts)) + for conflict in result.conflicts: + target = conflict.table + if conflict.column: + target = '%s.%s' % (conflict.table, conflict.column) + lines.append(' ! %-8s %s #%s - %s' % ( + conflict.operation, target, conflict.targetid, + conflict.reason)) + else: + lines.append(' conflicts: none') + return '\n'.join(lines) diff --git a/scripts/site_imports/resolver.py b/scripts/site_imports/resolver.py new file mode 100644 index 00000000..e5292ad9 --- /dev/null +++ b/scripts/site_imports/resolver.py @@ -0,0 +1,134 @@ +"""Shared four-state resolver vocabulary for site-import dry runs. + +IMPORT-AGNOSTIC. Nothing here knows about machines, eq1, locations, or any one +site. Every site importer maps a raw exported value onto a database row, and +every one of those mappings lands in exactly one of four states: + + RESOLVED the value maps to a row that already exists; safe to set. + BLANK there is nothing to set (empty cell, or a deliberately + unmappable value); leave the target field as-is. + NEEDS_CREATE the value maps to a valid canonical name, but that row does + not exist yet; a writer may create it, WITH acknowledgment. + UNRESOLVED the value has no mapping at all; a human must add one before + any writer runs. This is the state that blocks approval. + +Keeping this vocabulary in one place is what lets a promotion/approval helper, +an undo journal, and any number of importers all speak the same language. Add +new importers by writing their lookups and alias maps; reuse these primitives to +turn a raw value into a Resolution. +""" + +from collections import OrderedDict, namedtuple + +# The four terminal states a single field resolution can end in. +RESOLVED = 'resolved' +BLANK = 'blank' +NEEDS_CREATE = 'needs-create' +UNRESOLVED = 'unresolved' + +# status: one of the four above. +# id: the database id when RESOLVED, else None. +# name: the canonical target name when known (RESOLVED / NEEDS_CREATE), else +# None. Carried so callers can report and, later, create by name. +Resolution = namedtuple('Resolution', ('status', 'id', 'name')) + + +def norm(value): + """Lowercase, stripped form used for every name comparison and alias key. + + Centralised so a lookup table, an alias map, and the value being resolved + are always normalised the same way. If this ever changes, it changes for + everyone at once, which is the point. + """ + return (value or '').strip().lower() + + +def load_lookup(connection, table, id_column, name_column, display=None): + """norm(name) -> id for every row of any table. One loader, every importer. + + This is the "match against what is actually in the database" primitive. It + reads the live table at call time, so a value a human adds in Settings is + visible on the next run with no code change: a name that exists resolves, + one that does not is NEEDS_CREATE or UNRESOLVED. Every site importer maps its + exported values onto some reference table (locations, businessunits, vendors, + ...) and they should all load that table the same way rather than each + hand-rolling a SELECT. + + Pass a dict as `display` to also collect norm(name) -> original-cased name, + so a report can echo the name the way the database spells it rather than the + normalised key. + + First writer wins on a duplicate normalised name, which is deterministic and + matches how the importers resolve: the ambiguity is a data problem to report, + not something the loader should silently reorder. + """ + from sqlalchemy import text + + out = {} + rows = connection.execute(text( + 'SELECT %s, %s FROM %s' % (id_column, name_column, table))) + for ident, name in rows: + key = norm(name) + if key: + out.setdefault(key, ident) + if display is not None: + display.setdefault(key, (name or '').strip()) + return out + + +def resolve_name(name, lookup_by_name, missing=UNRESOLVED): + """Resolve a value by direct name match against a lookup table. + + lookup_by_name maps norm(name) -> database id. A blank name is BLANK. A hit + is RESOLVED with the id. A miss is `missing` - callers pass UNRESOLVED when a + human must supply a mapping, or NEEDS_CREATE when the name is trusted and a + writer may create it. + """ + if not (name or '').strip(): + return Resolution(BLANK, None, None) + ident = lookup_by_name.get(norm(name)) + if ident is not None: + return Resolution(RESOLVED, ident, name) + return Resolution(missing, None, name) + + +def resolve_alias(raw, alias_map, lookup_by_name, missing=UNRESOLVED): + """Resolve a dirty raw value through an alias map, then a lookup table. + + The alias-first path: a raw export value (possibly truncated or oddly cased) + is normalised and looked up in alias_map to get a canonical name, which is + then resolved against lookup_by_name. + + raw is blank -> BLANK (nothing to set) + raw not in alias_map -> missing (default UNRESOLVED: add an alias) + alias maps to None -> BLANK (deliberately unmappable value) + canonical exists in lookup -> RESOLVED + canonical not yet in lookup -> NEEDS_CREATE + """ + if not (raw or '').strip(): + return Resolution(BLANK, None, None) + key = norm(raw) + if key not in alias_map: + return Resolution(missing, None, None) + canonical = alias_map[key] + if canonical is None: + return Resolution(BLANK, None, None) + return resolve_name(canonical, lookup_by_name, missing=NEEDS_CREATE) + + +def tally(plan, status_column, value_column, status): + """Count rows in a plan whose status_column equals `status`, by value. + + Returns an OrderedDict of value -> count, highest count first. Shared by the + dry-run report and the approval packet so the numbers a human reads and the + numbers a writer enforces come from one place. + """ + counts = {} + for row in plan: + if row.get(status_column) == status: + value = row.get(value_column) or '' + counts[value] = counts.get(value, 0) + 1 + ordered = OrderedDict() + for value, count in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])): + ordered[value] = count + return ordered diff --git a/scripts/site_imports/suggest.py b/scripts/site_imports/suggest.py new file mode 100644 index 00000000..a3ae3f28 --- /dev/null +++ b/scripts/site_imports/suggest.py @@ -0,0 +1,159 @@ +"""Conservative, explainable match suggestions for UNRESOLVED values. + +IMPORT-AGNOSTIC and REPORT-ONLY. This never resolves anything and never writes. +Given a raw value that the normal rules could not map, it proposes zero or more +candidate names that already exist in a lookup table, each with a reason and a +confidence, so a human can accept one into that importer's alias map. Nothing +here is ever applied automatically - a silent wrong match is the worst failure +mode in dirty asset data, so every suggestion has to be acknowledged. + +Two tiers, tried in order, stopping at the first that yields anything: + + 1. Bounded deterministic transforms (confidence 'high'). These are rules that + fit the shapes actually seen in the data - a bay code with a slash is two + bays (T6/U6 -> T6, U6), a parenthetical is a note (LD9W(GE90) -> LD9W), a + decimal is a sub-bay (M6.3 -> M6), a trailing range is its first member + (L1-4 -> L1). Each transform is a named rule, not a fuzzer, so the reason + printed next to a suggestion tells a human exactly why it was proposed. + + 2. A single edit-distance-1 fallback (confidence 'low'). Only if no transform + matched, and only for a one-character difference against an existing name. + Bounded to distance 1 on purpose: it catches a typo or a dropped letter and + nothing looser, so it cannot quietly invent a plausible-looking wrong bay. + +The caller supplies the same normalisation and the lookup it resolves against, so +suggestions are always checked against what is really in the database. +""" + +import re +from collections import namedtuple + +from scripts.site_imports.resolver import norm + +# candidate: the existing name we think the raw value meant. +# reason: which rule proposed it (shown to the human). +# confidence: 'high' for a deterministic transform, 'low' for edit-distance. +Suggestion = namedtuple('Suggestion', ('candidate', 'reason', 'confidence')) + +_PAREN = re.compile(r'\s*\([^)]*\)') + + +def _candidates_from_transforms(stripped): + """Yield (transformed_name, reason) from the bounded deterministic rules. + + Operates on the value AFTER the importer has removed its own prefix (e.g. + WM-). Each rule yields plain names; the caller checks each against the live + lookup, so a rule that produces a non-existent name simply contributes no + suggestion. + """ + # A parenthetical note: LD9W(GE90) -> LD9W + if '(' in stripped: + yield (_PAREN.sub('', stripped).strip(), 'dropped parenthetical note') + + # A slash means two bays: T6/U6 -> T6 and U6. A bare second half (9/10) + # inherits the first half's letter prefix (T9/10 -> T9, T10). + if '/' in stripped: + left, right = (part.strip() for part in stripped.split('/', 1)) + if left: + yield (left, 'first of a slash-joined pair') + if right: + yield (right, 'second of a slash-joined pair') + prefix = re.match(r'^([A-Za-z]+)', left) + if prefix and re.match(r'^\d', right): + yield (prefix.group(1) + right, + 'second of a slash-joined pair, prefix carried') + + # A decimal sub-bay: M6.3 -> M6 + if '.' in stripped: + yield (stripped.split('.', 1)[0].strip(), 'dropped decimal sub-bay') + + # A trailing numeric range: L1-4 -> L1 + dash = re.match(r'^([A-Za-z]*\d+)-\d+$', stripped) + if dash: + yield (dash.group(1), 'first of a numeric range') + + +def _edit_distance_at_most_one(left, right): + """True if `left` and `right` differ by at most one insert/delete/replace. + + Bounded on purpose. A length gap over one is an immediate no; otherwise walk + both once and allow a single mismatch. Cheap and strict - it is a typo + catcher, not a similarity search. + """ + if left == right: + return True + length_left, length_right = len(left), len(right) + if abs(length_left - length_right) > 1: + return False + if length_left == length_right: + return sum(a != b for a, b in zip(left, right)) == 1 + # Ensure left is the shorter one, then try to match with one insertion. + if length_left > length_right: + left, right = right, left + length_left, length_right = length_right, length_left + index_left = index_right = 0 + skipped = False + while index_left < length_left and index_right < length_right: + if left[index_left] == right[index_right]: + index_left += 1 + index_right += 1 + elif skipped: + return False + else: + skipped = True + index_right += 1 + return True + + +# A low-confidence edit-distance pass can match many short bay codes at once, +# which is noise, not help. Cap how many it emits so a human sees the few most +# plausible typo-fixes and nothing drowns the high-confidence transforms. +_MAX_LOW_CONFIDENCE = 3 + + +def suggest(raw_value, stripped_value, lookup_by_name, display=None): + """Suggestions for one unresolved value. Empty list means "no idea". + + raw_value the original export string (for reporting only). + stripped_value the value after the importer removed its own prefix; this is + what the transforms and the edit-distance search operate on. + lookup_by_name norm(name) -> id, the live table to match against. + display optional norm(name) -> original-cased name, so a suggestion + echoes the database's spelling instead of the normalised key. + + Deterministic transforms first; only if none of them hits an existing name + does the edit-distance-1 fallback run, and that fallback is capped. High- + confidence suggestions are never diluted by low-confidence ones. + """ + display = display or {} + + def spell(key): + return display.get(key, key) + + results = [] + seen = set() + for candidate, reason in _candidates_from_transforms(stripped_value): + candidate = (candidate or '').strip() + if not candidate: + continue + key = norm(candidate) + if key in lookup_by_name and key not in seen: + seen.add(key) + results.append(Suggestion(spell(key), reason, 'high')) + if results: + return results + + target = norm(stripped_value) + if not target: + return results + low = [] + for existing_key in lookup_by_name: + if existing_key not in seen and _edit_distance_at_most_one( + target, existing_key): + seen.add(existing_key) + low.append(Suggestion(spell(existing_key), + 'one-character difference', 'low')) + # Only surface the capped set; too many one-off matches is noise. + if len(low) <= _MAX_LOW_CONFIDENCE: + results.extend(sorted(low)) + return results diff --git a/scripts/site_imports/winc/__init__.py b/scripts/site_imports/winc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/site_imports/winc/aliases.py b/scripts/site_imports/winc/aliases.py new file mode 100644 index 00000000..2c847fbd --- /dev/null +++ b/scripts/site_imports/winc/aliases.py @@ -0,0 +1,67 @@ +"""Alias and normalization tables for the WINC machine adoption dry-run. + +This file is DATA, not logic. It is the single place a human edits when the +analyzer reports an unresolved value. Keep it boring: plain dicts, one entry per +line, lowercase keys. The analyzer lowercases and strips before it looks a value +up here, so keys here must be lowercase and already stripped. + +Two independent problems get solved here: + +1. eq1 -> business unit. The May export's eq1 column is a free-text value-stream + label (LG PARTS, SM SPOOLS, Large Spoo, HPTCR ...). It is dirty: truncated at + ten characters, inconsistently cased, and it does NOT match anything in the + live businessunits table (that table is a list of offices auto-created by an + earlier printer import). So every eq1 value must be mapped by hand to the + canonical business-unit name we want on the asset. A canonical name that does + not yet exist in the database is reported by the analyzer as "would need to + be created" - the analyzer never invents one and never writes one. + +2. location code overrides. Most WM- codes resolve by simply dropping the + WM- prefix and matching a location by name (WM-J3 -> J3). A handful do not, + because the bay code carries a slash, a note in parentheses, or a room name + the location table spells differently. Those exceptions - and only those - + live here, mapping the RAW export location string to the exact locationname + already in the database. +""" + +# eq1 (lowercased, stripped) -> canonical business unit name. +# The value on the right is what we intend to store; the analyzer checks whether +# it already exists in businessunits and flags the ones that do not. +EQ1_TO_BUSINESSUNIT = { + 'lg parts': 'Large Parts', + 'lg part': 'Large Parts', + 'large part': 'Large Parts', + 'sm spools': 'Small Spools', + 'sm spool': 'Small Spools', + 'large spoo': 'Large Spools', + 'lg spools': 'Large Spools', + 'lg spool': 'Large Spools', + 'hptcr': 'HPT Rotor', + 'bliskcr': 'Blisk Rotor', + 'blisk': 'Blisk Rotor', + 'leap x': 'LEAP-X', +} + +# Raw export location string (lowercased, stripped) -> exact locationname in DB. +# Only for codes that do NOT resolve by dropping the WM- prefix. Leave a value +# as None to say "known to be unmappable, do not touch location for this row". +# +# The entries below the two None rows were promoted from the dry-run's +# high-confidence suggestions and verified to exist in the locations table. A +# slash-joined bay code (T6/U6) names two adjacent bays, but an asset carries a +# single locationid, so each maps to the FIRST (primary) bay - the deterministic +# choice a human confirmed. LD7W and LD6W are intentionally absent: they had no +# existing match and need a location created before they can be aliased. +LOCATION_OVERRIDES = { + 'wm': None, + 'wm-toolroom': None, + 'wm-t6/u6': 'T6', + 'wm-t9/10': 'T9', + 'wm-u9/10': 'U9', + 'wm-t4/u4': 'T4', + 'wm-ld9w(ge90)': 'LD9W', + 'wm-m6.3': 'M6', + 'wm-l1-4': 'L1', + 'wm-ld7w': None, + 'wm-ld6w': None, +} diff --git a/scripts/site_imports/winc/dry_run.py b/scripts/site_imports/winc/dry_run.py new file mode 100644 index 00000000..fc14a1b3 --- /dev/null +++ b/scripts/site_imports/winc/dry_run.py @@ -0,0 +1,385 @@ +"""Dry-run analyzer for the WINC (West Manufacturing) machine adoption. + +READ ONLY. This script opens the database, reads the May asset export, decides +what an import WOULD change, and prints a report. It never writes. Nothing here +commits, updates, or inserts - that is a separate tool, written only once this +report reads clean and a human has signed off on it. + +What it does, in order: + + 1. Load every existing machine's asset number and build the crosswalk key + (the workstation number, e.g. 0083A) the export will be matched on. + 2. Read the export with a real CSV parser (quoted commas, BOM) and, for each + row that carries a workstation number, decide: + - does it match an existing machine? (in scope vs skipped) + - what name would we set? (export description) + - what location would we set? (WM- -> locations, via + aliases.LOCATION_OVERRIDES for + the exceptions) + - what business unit would we set? (eq1 -> aliases.EQ1_TO_ + BUSINESSUNIT -> businessunits) + 3. Classify each field resolution as RESOLVED, BLANK (nothing to set, leave + as-is), NEEDS-CREATE (target value is valid but not yet in the database), + or UNRESOLVED (no alias entry - a human must add one to aliases.py). + 4. Print a summary and, with --rows, a per-row table. With --csv PATH it also + writes the full plan so the numbers can be diffed between runs. + +Run it against the real database read-only: + + python -m scripts.site_imports.winc.dry_run --export "exports/ASSET_current May26.csv" + python -m scripts.site_imports.winc.dry_run --export EXPORT.csv --rows + python -m scripts.site_imports.winc.dry_run --export EXPORT.csv --csv plan.csv + +The design goal is reliability over cleverness: one match rule, one location +rule, one business-unit rule, every exception in aliases.py, and a report that +tells a human exactly what is unresolved so the alias table can be completed +before anyone runs the writer. +""" + +import argparse +import csv +import os +import re +import sys +from collections import OrderedDict +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__)))))) + +from dotenv import load_dotenv + +# Same reasoning as scripts/export_catalog.py: a plain `python -m` run gets +# neither wsgi.py's nor the flask CLI's .env load, so DATABASE_URL would be +# unset and config would silently fall back to a hardcoded localhost URL. Load +# it here before anything reads config. +load_dotenv(Path(__file__).resolve().parents[3] / '.env') + +from scripts.site_imports.approval import build_packet, format_packet +from scripts.site_imports.suggest import suggest +from scripts.site_imports.resolver import ( + BLANK, + NEEDS_CREATE, + RESOLVED, + Resolution, + UNRESOLVED, + load_lookup, + norm as _norm, + resolve_alias, + resolve_name, +) +from scripts.site_imports.winc.aliases import ( + EQ1_TO_BUSINESSUNIT, + LOCATION_OVERRIDES, +) + +# WM- is the only bay prefix in the WINC export; a location code resolves by +# dropping it and matching a location by name. Everything else is an override. +_WM_PREFIX = re.compile(r'^WM-?', re.IGNORECASE) + +# A workstation number is a LEADING run of digits with an optional single letter +# suffix. The export writes it clean (W0083A); the machines table stores the +# same number but often with a trailing descriptor (0083 Debur, 0083A Inspect). +# So the key is the leading token on both sides, matched unanchored - anchoring +# with $ would drop every existing machine whose name carries a descriptor and +# silently shrink the in-scope set. The leading W is the single biggest gotcha +# in this data, so it is stripped here in exactly one place. +_WSNUM = re.compile(r'^(\d{3,5}[A-Z]?)') + + +def workstation_key(assetnum): + """The crosswalk key (e.g. '0083A') for an asset number, or None. + + Strips a leading W and keeps the leading run of digits plus an optional + single trailing letter, ignoring any descriptor that follows. Returns None + for anything that is not a workstation number (blank rows, facilities codes, + named test machines) so non-machine rows fall out of scope on their own + rather than being force-fit. + """ + text = (assetnum or '').strip().upper() + if text.startswith('W'): + text = text[1:] + match = _WSNUM.match(text) + return match.group(1) if match else None + + +def load_existing_machine_keys(connection): + """Map workstation key -> list of existing machine asset numbers. + + A list, not a single value, because the machines table has a few numbers + that collide once reduced to their workstation key (0083 exists as + '0083 Debur' and '0083 Inspection'). The analyzer reports those as ambiguous + rather than guessing which one the export row means. + """ + from sqlalchemy import text + + keys = OrderedDict() + rows = connection.execute(text( + "SELECT a.assetnumber " + "FROM assets a JOIN machines m ON m.assetid = a.assetid")) + for (assetnumber,) in rows: + key = workstation_key(assetnumber) + if key: + keys.setdefault(key, []).append(assetnumber) + return keys + + +def load_locations(connection, display=None): + """locationname (normalized) -> locationid, from the live locations table.""" + return load_lookup(connection, 'locations', 'locationid', 'locationname', + display=display) + + +def load_businessunits(connection): + """businessunit (normalized) -> businessunitid, from the live table.""" + return load_lookup(connection, 'businessunits', 'businessunitid', + 'businessunit') + + +def resolve_location(raw_location, locations_by_name): + """Resolution for a raw WM bay code. + + Rule: an override in aliases wins; otherwise drop the WM- prefix and match a + location by name. A blank export cell means "leave the asset's location as + is" and is reported as BLANK, not UNRESOLVED, because there is nothing to + set and nothing for a human to fix. Overrides are checked first because they + carry the exceptions the plain prefix rule cannot handle. + """ + raw = (raw_location or '').strip() + if not raw: + return Resolution(BLANK, None, None) + + if _norm(raw) in LOCATION_OVERRIDES: + return resolve_alias(raw, LOCATION_OVERRIDES, locations_by_name) + + name = _WM_PREFIX.sub('', raw).strip() + return resolve_name(name, locations_by_name) + + +def resolve_businessunit(raw_eq1, businessunits_by_name): + """Resolution for an eq1 value stream. + + A blank eq1 is BLANK (leave the asset alone). A non-blank value must have an + alias entry; without one it is UNRESOLVED so a human adds it to aliases.py. A + mapped canonical name that does not yet exist in businessunits is + NEEDS-CREATE - flagged, never created here. + """ + return resolve_alias(raw_eq1, EQ1_TO_BUSINESSUNIT, businessunits_by_name) + + +def read_export(path): + """Rows of the export as dicts. utf-8-sig handles the BOM; the csv module + handles the quoted commas embedded in some descriptions.""" + with open(path, newline='', encoding='utf-8-sig') as handle: + return list(csv.DictReader(handle)) + + +def parse_export_text(text): + """Rows of an export given its raw CSV text (same shape as read_export). + + Split out so an API route can hand the analyzer an uploaded file's bytes + without writing them to disk, while the CLI keeps reading from a path. + """ + import io + + return list(csv.DictReader(io.StringIO(text))) + + +def analyze(connection, export_path): + """Build the per-row plan from a file path. Returns (plan, summary, lookups).""" + return analyze_rows(connection, read_export(export_path)) + + +def analyze_rows(connection, export_rows): + """Build the per-row plan. Returns (plan_rows, summary_dict, lookups). + + Takes already-parsed export rows so the same analysis serves both the CLI + (which reads a path) and the API route (which reads uploaded bytes). + """ + existing = load_existing_machine_keys(connection) + location_names = {} + locations = load_locations(connection, display=location_names) + businessunits = load_businessunits(connection) + + plan = [] + summary = OrderedDict([ + ('export_rows', len(export_rows)), + ('non_workstation_rows', 0), + ('in_scope', 0), + ('skipped_no_existing_machine', 0), + ('ambiguous_match', 0), + ('location_' + RESOLVED, 0), + ('location_' + BLANK, 0), + ('location_' + NEEDS_CREATE, 0), + ('location_' + UNRESOLVED, 0), + ('businessunit_' + RESOLVED, 0), + ('businessunit_' + BLANK, 0), + ('businessunit_' + NEEDS_CREATE, 0), + ('businessunit_' + UNRESOLVED, 0), + ]) + + for row in export_rows: + assetnum = (row.get('assetnum') or '').strip() + key = workstation_key(assetnum) + if key is None: + summary['non_workstation_rows'] += 1 + continue + + matches = existing.get(key, []) + if not matches: + summary['skipped_no_existing_machine'] += 1 + continue + + summary['in_scope'] += 1 + ambiguous = len(matches) > 1 + if ambiguous: + summary['ambiguous_match'] += 1 + + loc = resolve_location(row.get('location'), locations) + bu = resolve_businessunit(row.get('eq1'), businessunits) + summary['location_' + loc.status] += 1 + summary['businessunit_' + bu.status] += 1 + + plan.append(OrderedDict([ + ('assetnum', assetnum), + ('workstation_key', key), + ('matched_asset', '|'.join(matches)), + ('ambiguous', 'yes' if ambiguous else ''), + ('name', (row.get('description') or '').strip()), + ('raw_location', (row.get('location') or '').strip()), + ('location_status', loc.status), + ('locationid', loc.id if loc.id is not None else ''), + ('location_name', loc.name or ''), + ('raw_eq1', (row.get('eq1') or '').strip()), + ('businessunit_status', bu.status), + ('businessunitid', bu.id if bu.id is not None else ''), + ('businessunit_name', bu.name or ''), + ])) + + return plan, summary, {'locations': locations, + 'location_names': location_names, + 'businessunits': businessunits} + + +def print_summary(summary): + print('=' * 60) + print('WINC machine adoption - DRY RUN (no changes written)') + print('=' * 60) + for label, value in summary.items(): + print(' %-32s %6d' % (label.replace('_', ' '), value)) + + +def print_unresolved(plan, lookups=None): + loc = OrderedDict() + bu = OrderedDict() + for row in plan: + if row['location_status'] == UNRESOLVED: + loc[row['raw_location']] = loc.get(row['raw_location'], 0) + 1 + if row['businessunit_status'] == UNRESOLVED: + bu[row['raw_eq1']] = bu.get(row['raw_eq1'], 0) + 1 + needs_loc = OrderedDict() + needs_bu = OrderedDict() + for row in plan: + if row['location_status'] == NEEDS_CREATE: + needs_loc[row['location_name']] = needs_loc.get(row['location_name'], 0) + 1 + if row['businessunit_status'] == NEEDS_CREATE: + needs_bu[row['businessunit_name']] = needs_bu.get(row['businessunit_name'], 0) + 1 + + if loc: + print('\nUNRESOLVED locations (add to aliases.LOCATION_OVERRIDES):') + location_lookup = (lookups or {}).get('locations') or {} + location_names = (lookups or {}).get('location_names') or {} + for value, count in sorted(loc.items(), key=lambda kv: -kv[1]): + print(' %5d %s' % (count, value)) + stripped = _WM_PREFIX.sub('', value).strip() + for hint in suggest(value, stripped, location_lookup, + display=location_names): + print(' -> suggest "%s" (%s, confidence %s)' + % (hint.candidate, hint.reason, hint.confidence)) + if bu: + print('\nUNRESOLVED eq1 (add to aliases.EQ1_TO_BUSINESSUNIT):') + for value, count in sorted(bu.items(), key=lambda kv: -kv[1]): + print(' %5d %s' % (count, value)) + if needs_loc: + print('\nNEEDS-CREATE locations (mapped, but not yet in locations):') + for value, count in sorted(needs_loc.items(), key=lambda kv: -kv[1]): + print(' %5d %s' % (count, value)) + if needs_bu: + print('\nNEEDS-CREATE business units (mapped, but not yet in businessunits):') + for value, count in sorted(needs_bu.items(), key=lambda kv: -kv[1]): + print(' %5d %s' % (count, value)) + + +def print_rows(plan): + print('\nPER-ROW PLAN:') + header = ('assetnum', 'matched_asset', 'name', 'raw_location', + 'location_status', 'raw_eq1', 'businessunit_status') + print(' ' + ' | '.join(header)) + for row in plan: + print(' ' + ' | '.join(str(row[h]) for h in header)) + + +def write_csv(plan, path): + if not plan: + return + with open(path, 'w', newline='', encoding='utf-8') as handle: + writer = csv.DictWriter(handle, fieldnames=list(plan[0].keys())) + writer.writeheader() + writer.writerows(plan) + + +def build_app_connection(): + """Yield a live SQLAlchemy connection inside an app context. + + Returns (app, connection). Caller is responsible for closing nothing - the + connection lives for the app context and the process exits right after. + """ + from shopdb import create_app + from shopdb.api import db + + app = create_app() + context = app.app_context() + context.push() + return app, db.session.connection() + + +def main(argv=None): + parser = argparse.ArgumentParser( + description='Dry-run analyzer for WINC machine adoption. Read only.') + parser.add_argument('--export', required=True, + help='Path to the ASSET export CSV.') + parser.add_argument('--rows', action='store_true', + help='Print the full per-row plan.') + parser.add_argument('--csv', dest='csv_out', + help='Write the full per-row plan to this CSV path.') + parser.add_argument('--packet', action='store_true', + help='Print the approval packet (hash, needs-create ' + 'and blocking summary) a writer would enforce.') + args = parser.parse_args(argv) + + if not os.path.exists(args.export): + parser.error('export file not found: %s' % args.export) + + _app, connection = build_app_connection() + plan, summary, lookups = analyze(connection, args.export) + + print_summary(summary) + print_unresolved(plan, lookups) + if args.rows: + print_rows(plan) + if args.csv_out: + write_csv(plan, args.csv_out) + print('\nWrote per-row plan: %s (%d rows)' % (args.csv_out, len(plan))) + if args.packet: + packet = build_packet( + 'winc-machine-adoption', plan, summary, + [('businessunit', 'businessunit_status', 'raw_eq1', + 'businessunit_name'), + ('location', 'location_status', 'raw_location', 'location_name')]) + print('\n' + format_packet(packet)) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/site_imports/winc/undo.py b/scripts/site_imports/winc/undo.py new file mode 100644 index 00000000..7adec6a8 --- /dev/null +++ b/scripts/site_imports/winc/undo.py @@ -0,0 +1,55 @@ +"""WINC machine adoption - UNDO (reverse an applied import batch). + +A thin CLI over the shared, import-agnostic journal.undo_batch. It is WINC-named +only for discoverability; the actual work is generic, so the same reversal serves +any importer that wrote through the journal. Reverting is conservative: a cell a +human changed since the import is left as-is and reported, and a created business +unit is deleted only when nothing outside the batch still references it. +""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__)))))) + +from dotenv import load_dotenv +from pathlib import Path + +load_dotenv(Path(__file__).resolve().parents[3] / '.env') + +from scripts.site_imports import journal +from scripts.site_imports.winc.dry_run import build_app_connection +from scripts.site_imports.winc.writer import _UNDO_REFERENCES + + +def main(argv=None): + parser = argparse.ArgumentParser( + description='Undo an applied WINC import batch (reversible journal).') + parser.add_argument('--batch', required=True, + help='The batch id to reverse.') + parser.add_argument('--commit', action='store_true', + help='Actually reverse. Without it, this is a no-op.') + args = parser.parse_args(argv) + + if not args.commit: + print('DRY: pass --commit to reverse batch %s' % args.batch) + return 0 + + _app, connection = build_app_connection() + from shopdb.api import db + try: + result = journal.undo_batch(connection, args.batch, + references=_UNDO_REFERENCES) + db.session.commit() + except Exception: + db.session.rollback() + raise + + print(journal.format_undo_result(result)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/site_imports/winc/writer.py b/scripts/site_imports/winc/writer.py new file mode 100644 index 00000000..ce18d3b9 --- /dev/null +++ b/scripts/site_imports/winc/writer.py @@ -0,0 +1,278 @@ +"""WINC machine adoption - WRITER (applies the approved plan, reversibly). + +This is the write half of the WINC import; scripts/site_imports/winc/dry_run.py +is the read-only half. The writer never invents its own view of the data: it +rebuilds the exact same plan the dry-run produced, verifies that plan against the +approval packet the human signed off on, and only then writes - recording every +change to the shared import journal so the whole batch can be undone. + +The contract, in order (the one documented in approval.py, made concrete here): + + 1. rebuild the plan from the same export + live DB, + 2. build a packet from it, + 3. compare planhash to the approved packet - abort on mismatch (drift), + 4. refuse if the packet is blocking (any unresolved value), + 5. require explicit acknowledgment for every needs-create group, + 6. write inside ONE transaction, stamping the batch id onto the journal: + a. create each acknowledged needs-create business unit (journal insert), + b. update each in-scope asset's name / location / business unit + (journal update, capturing the pre-image first). + +Everything commits together or not at all. Ambiguous rows (one workstation key +matching several machines) are never written - the dry-run flags them and the +writer skips them, because guessing which machine an export row means is exactly +the kind of cleverness this import refuses. +""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__)))))) + +from dotenv import load_dotenv +from pathlib import Path + +load_dotenv(Path(__file__).resolve().parents[3] / '.env') + +from scripts.site_imports import journal +from scripts.site_imports.approval import build_packet, packet_matches_plan +from scripts.site_imports.resolver import NEEDS_CREATE, RESOLVED, norm +from scripts.site_imports.winc.dry_run import ( + analyze, + build_app_connection, +) + +# The packet groups the WINC dry-run declares, reused verbatim so the writer +# hashes and reads the plan exactly the way approval did. +_PACKET_GROUPS = [ + ('businessunit', 'businessunit_status', 'raw_eq1', 'businessunit_name'), + ('location', 'location_status', 'raw_location', 'location_name'), +] + +# References into a created row's id, so undo can tell whether an inserted +# business unit is still pointed at by something outside this batch before it +# hard-deletes it. Shared with the undo CLI and the web undo path. +_UNDO_REFERENCES = { + 'businessunits': [('assets', 'businessunitid')], +} + + +# References into a created row's id, so undo can tell whether an inserted +# business unit is still pointed at by something outside this batch before it +# hard-deletes it. +# Max width of assets.name; the export description can exceed it (a few rows +# carry a whole safety notice in the field), so a set name is clamped rather +# than letting the whole transaction fail on one over-long row. The pre-image is +# journalled, so the clamp is reversible like any other change. +_ASSET_NAME_MAXLEN = 100 + + +class WriterError(Exception): + """The plan cannot be written as approved (drift, blocking, or missing + acknowledgment). Carries a human-readable reason; never a partial write.""" + + +def _load_asset_ids(connection): + """assetnumber -> assetid for every asset, so a plan row (which carries the + matched asset *number*) can be turned into the id the update targets.""" + from sqlalchemy import text + + out = {} + for assetid, assetnumber in connection.execute(text( + 'SELECT assetid, assetnumber FROM assets')): + out[assetnumber] = assetid + return out + + +def _create_businessunit(connection, batchid, sequence, name): + """Insert one business unit and journal it. Returns (new_id, next_sequence). + + Only ever called for an acknowledged needs-create name, so the journal's + insert entry describes strictly a row this batch created.""" + from sqlalchemy import text + + now = journal._utcnow() + result = connection.execute(text( + 'INSERT INTO businessunits ' + '(businessunit, createddate, modifieddate, isactive) ' + 'VALUES (:name, :now, :now, 1)'), + {'name': name, 'now': now}) + new_id = result.lastrowid + journal.record_insert(connection, batchid, sequence, 'businessunits', + new_id) + return new_id, sequence + 1 + + +def _update_cell(connection, batchid, sequence, table, targetid, column, + oldvalue, newvalue): + """Apply one cell update and journal its pre/post image. Returns the next + sequence. Skips (and does not journal) a no-op where the value is unchanged.""" + from sqlalchemy import text + + if journal._as_text(oldvalue) == journal._as_text(newvalue): + return sequence + connection.execute(text( + 'UPDATE %s SET %s = :newvalue WHERE %s = :targetid' + % (table, column, _asset_pk(table))), + {'newvalue': newvalue, 'targetid': targetid}) + journal.record_update(connection, batchid, sequence, table, targetid, + column, oldvalue, newvalue) + return sequence + 1 + + +def _asset_pk(table): + # The writer only updates the assets table; keep the mapping explicit rather + # than reflecting, so an unexpected table name fails loudly. + return {'assets': 'assetid'}[table] + + +def apply_plan(connection, plan, approved_packet, acknowledge=None, + createdby=None): + """Apply an approved WINC plan, reversibly. Returns the batch id. + + connection a live SQLAlchemy connection in an open transaction. + plan the per-row plan from dry_run.analyze (rebuilt fresh). + approved_packet the packet a human signed off on. + acknowledge set of needs-create group names the human confirmed + (e.g. {'businessunit'}); required for every group the + packet lists under needscreate. + createdby who acknowledged, stamped on the batch. + + Raises WriterError before touching anything if the plan drifted, is + blocking, or a needs-create group was not acknowledged. + """ + acknowledge = set(acknowledge or ()) + + # (2)-(3) rebuild the packet and prove nothing drifted since approval. + current_packet = build_packet('winc-machine-adoption', plan, + approved_packet.get('counts', {}), + _PACKET_GROUPS) + if not packet_matches_plan(approved_packet, plan): + raise WriterError( + 'plan has changed since approval (hash mismatch) - re-run the ' + 'dry-run and re-approve before importing') + + # (4) never write over an unresolved value. + if current_packet.get('blocking'): + raise WriterError( + 'plan is blocking: unresolved values remain - add the missing ' + 'aliases and re-run the dry-run') + + # (5) every needs-create group must be explicitly acknowledged. + missing_ack = set(current_packet.get('needscreate', {})) - acknowledge + if missing_ack: + raise WriterError( + 'needs-create not acknowledged for: %s - confirm these will be ' + 'created before importing' % ', '.join(sorted(missing_ack))) + + batchid = approved_packet['batchid'] + journal.open_batch(connection, approved_packet, createdby=createdby) + sequence = 1 + + # (6a) create acknowledged needs-create business units once each, first, so + # the asset updates below can point at them. + created_bu = {} + for row in plan: + if row['businessunit_status'] != NEEDS_CREATE: + continue + name = row['businessunit_name'] + key = norm(name) + if key not in created_bu: + created_bu[key], sequence = _create_businessunit( + connection, batchid, sequence, name) + + # (6b) update each in-scope, unambiguous asset. + asset_ids = _load_asset_ids(connection) + from sqlalchemy import text + for row in plan: + if row['ambiguous']: + continue + assetnumber = row['matched_asset'] + assetid = asset_ids.get(assetnumber) + if assetid is None: + continue + + current = connection.execute(text( + 'SELECT name, locationid, businessunitid FROM assets ' + 'WHERE assetid = :assetid'), {'assetid': assetid}).fetchone() + cur_name, cur_location, cur_bu = current + + if row['name']: + new_name = row['name'][:_ASSET_NAME_MAXLEN] + sequence = _update_cell(connection, batchid, sequence, 'assets', + assetid, 'name', cur_name, new_name) + + if row['location_status'] == RESOLVED and row['locationid'] != '': + sequence = _update_cell(connection, batchid, sequence, 'assets', + assetid, 'locationid', cur_location, + row['locationid']) + + new_bu = None + if row['businessunit_status'] == RESOLVED and row['businessunitid'] != '': + new_bu = row['businessunitid'] + elif row['businessunit_status'] == NEEDS_CREATE: + new_bu = created_bu[norm(row['businessunit_name'])] + if new_bu is not None: + sequence = _update_cell(connection, batchid, sequence, 'assets', + assetid, 'businessunitid', cur_bu, new_bu) + + return batchid + + +def main(argv=None): + parser = argparse.ArgumentParser( + description='WINC machine adoption writer. Applies an approved plan ' + 'reversibly. Requires --commit to write.') + parser.add_argument('--export', required=True, + help='Path to the ASSET export CSV.') + parser.add_argument('--acknowledge', action='append', default=[], + metavar='GROUP', + help='Acknowledge a needs-create group (repeatable), ' + 'e.g. --acknowledge businessunit.') + parser.add_argument('--createdby', default=None, + help='Who is approving this import (stamped on batch).') + parser.add_argument('--commit', action='store_true', + help='Actually write. Without it, this is a no-op that ' + 'reports what would happen.') + args = parser.parse_args(argv) + + if not os.path.exists(args.export): + parser.error('export file not found: %s' % args.export) + + _app, connection = build_app_connection() + plan, summary, _lookups = analyze(connection, args.export) + packet = build_packet('winc-machine-adoption', plan, summary, + _PACKET_GROUPS) + + if not args.commit: + print('DRY: rebuilt plan of %d rows, hash %s' % ( + packet['rowcount'], packet['planhash'])) + print('DRY: pass --commit with --acknowledge for each needs-create ' + 'group to write. Needs-create groups: %s' + % (', '.join(packet.get('needscreate', {})) or 'none')) + return 0 + + from shopdb.api import db + try: + batchid = apply_plan(connection, plan, packet, + acknowledge=args.acknowledge, + createdby=args.createdby) + db.session.commit() + except WriterError as error: + db.session.rollback() + print('REFUSED: %s' % error) + return 2 + except Exception: + db.session.rollback() + raise + + print('COMMITTED batch %s' % batchid) + print(' to undo: python -m scripts.site_imports.winc.undo --batch %s' + % batchid) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/shopdb/__init__.py b/shopdb/__init__.py index e85f4b67..7073f394 100644 --- a/shopdb/__init__.py +++ b/shopdb/__init__.py @@ -104,7 +104,13 @@ from .plugins import plugin_manager # defeated by rotating the header. The helper resolves from remote_addr, which # ProxyFix has already filled from the header when TRUSTED_PROXY_COUNT says a # proxy is really in front. Additive names on the surface, minor bump. -__contract_version__ = '0.29.0' +# 0.30.0: added ImportBatch, ImportJournalEntry, and the BATCH_*/OP_* constants +# to shopdb.api so a plugin's bulk-import UI can list applied import batches and +# reverse one through the shared reversible journal without importing core model +# internals. The journal records the primitive mutations a bulk import makes +# (update a cell, insert a row) tied to a batch id, and a generic undo replays +# them in reverse. Additive names on the surface, minor bump. +__contract_version__ = '0.30.0' # Product release version (see ADR-007). The product version and the # plugin-contract version above are distinct series with independent diff --git a/shopdb/api/__init__.py b/shopdb/api/__init__.py index 19d7626e..db33a9d6 100644 --- a/shopdb/api/__init__.py +++ b/shopdb/api/__init__.py @@ -140,6 +140,22 @@ from shopdb.utils.mailer import send_email, send_alert, send_webhook from shopdb.utils.cmmc_usb_db import cmmc_usb_connection +# Reversible bulk-import journal. A plugin that drives a bulk import (a site +# asset adoption, an integration sync) records the primitive changes it makes so +# the whole batch can be undone. ImportBatch is the identity approval, write, and +# undo all reference; ImportJournalEntry is one reversible mutation. Exposed so a +# plugin's import UI can list applied batches and reverse one without reaching +# into core internals. See docs/PLUGIN-HOOKS.md. +from shopdb.core.models import ( + ImportBatch, + ImportJournalEntry, + BATCH_APPLIED, + BATCH_UNDONE, + OP_UPDATE, + OP_INSERT, +) + + def audit_log( action: str, entitytype: str, diff --git a/shopdb/core/api/settings.py b/shopdb/core/api/settings.py index 939bf1bd..e550400b 100644 --- a/shopdb/core/api/settings.py +++ b/shopdb/core/api/settings.py @@ -595,6 +595,123 @@ def import_asset_csvs(): }) +def _winc_upload_text(): + """Read the single uploaded WINC export as text, or (None, error_response).""" + uploads = request.files.getlist('files') + if not uploads: + return None, error_response(ErrorCodes.VALIDATION_ERROR, + 'No export file was uploaded') + upload = uploads[0] + name = upload.filename or '' + if not name.lower().endswith('.csv'): + return None, error_response(ErrorCodes.VALIDATION_ERROR, + 'The WINC export must be a .csv file') + try: + return upload.read().decode('utf-8-sig'), None + except (UnicodeDecodeError, OSError): + return None, error_response(ErrorCodes.VALIDATION_ERROR, + 'Could not read %s as UTF-8 text' % name) + + +@settings_bp.route('/asset-import/winc/preview', methods=['POST']) +@require_role('admin') +def winc_import_preview(): + """Read-only WINC analysis: report what a commit would do. Writes nothing. + + WINC (a Maximo-style asset dump) is not a canonical-CSV source: it updates + existing machines only, needs acknowledgment before creating business units, + and is reversible. This returns the approval packet the UI renders - counts, + the business units it would create, and any unresolved values that block it. + """ + from shopdb.core.services.wincimport import preview, WincImportError + + export_text, failure = _winc_upload_text() + if failure is not None: + return failure + try: + report = preview(export_text) + except WincImportError as exc: + return error_response(ErrorCodes.VALIDATION_ERROR, str(exc)) + finally: + # A preview must leave nothing behind even though it only read: the + # analyzer runs against the request-scoped session. + db.session.rollback() + return success_response(report) + + +@settings_bp.route('/asset-import/winc/commit', methods=['POST']) +@require_role('admin') +def winc_import_commit(): + """Apply a WINC plan reversibly. Requires acknowledgment for needs-create. + + The 'acknowledge' form field is a comma-separated list of needs-create group + names the admin confirmed (e.g. 'businessunit'). The whole write is one + transaction recorded to the import journal, so it can be undone later. + """ + from shopdb.core.services.wincimport import commit as winc_commit + from shopdb.core.services.wincimport import WincImportError + + export_text, failure = _winc_upload_text() + if failure is not None: + return failure + + acknowledge = [ + part.strip() for part in + (request.form.get('acknowledge') or '').split(',') if part.strip()] + + try: + report = winc_commit(export_text, acknowledge=acknowledge, + createdby=get_jwt_identity()) + AuditLog.log( + 'imported', 'SiteAssets', entityname='winc-machine-adoption', + changes={'batchid': report.get('batchid'), + 'rows': report.get('rowcount')}) + db.session.commit() + except WincImportError as exc: + db.session.rollback() + return error_response(ErrorCodes.VALIDATION_ERROR, str(exc)) + except Exception: + db.session.rollback() + raise + return success_response(report) + + +@settings_bp.route('/asset-import/winc/batches', methods=['GET']) +@require_role('admin') +def winc_import_batches(): + """List applied/undone WINC import batches, newest first (for the undo UI).""" + from shopdb.core.services.wincimport import list_batches + + return success_response({'batches': list_batches()}) + + +@settings_bp.route('/asset-import/winc/undo', methods=['POST']) +@require_role('admin') +def winc_import_undo(): + """Reverse an applied WINC import batch by id. Conservative and reversible.""" + from shopdb.core.services.wincimport import undo as winc_undo + + payload = request.get_json(silent=True) or {} + batchid = (payload.get('batchid') or request.form.get('batchid') or '').strip() + if not batchid: + return error_response(ErrorCodes.VALIDATION_ERROR, 'A batchid is required') + + try: + result = winc_undo(batchid) + AuditLog.log( + 'undone', 'SiteAssets', entityname='winc-machine-adoption', + changes={'batchid': batchid, 'reverted': result['reverted'], + 'conflicts': len(result['conflicts'])}) + db.session.commit() + except ValueError as exc: + db.session.rollback() + return error_response(ErrorCodes.VALIDATION_ERROR, str(exc)) + except Exception: + db.session.rollback() + raise + return success_response(result) + + @settings_bp.route('', methods=['GET']) @jwt_required(optional=True) def list_settings(): diff --git a/shopdb/core/models/__init__.py b/shopdb/core/models/__init__.py index f3d1e732..35eabeb7 100644 --- a/shopdb/core/models/__init__.py +++ b/shopdb/core/models/__init__.py @@ -23,6 +23,8 @@ from .setting import Setting from .auditlog import AuditLog from .customfield import CustomField, CustomFieldValue from .apitoken import ApiToken +from .importbatch import (ImportBatch, ImportJournalEntry, BATCH_APPLIED, + BATCH_UNDONE, OP_UPDATE, OP_INSERT) __all__ = [ # Base @@ -82,4 +84,11 @@ __all__ = [ 'CustomFieldValue', # Personal API tokens 'ApiToken', + # Import batch + journal (reversible bulk imports) + 'ImportBatch', + 'ImportJournalEntry', + 'BATCH_APPLIED', + 'BATCH_UNDONE', + 'OP_UPDATE', + 'OP_INSERT', ] diff --git a/shopdb/core/models/importbatch.py b/shopdb/core/models/importbatch.py new file mode 100644 index 00000000..7eb06da4 --- /dev/null +++ b/shopdb/core/models/importbatch.py @@ -0,0 +1,86 @@ +"""Import batch + journal models (reversible bulk imports). + +PLATFORM-LEVEL, IMPORT-AGNOSTIC. These two tables let any bulk-import writer +record exactly what it changed so the change can be reversed later. Nothing here +knows about Maximo, WINC, machines, or any one source: an importer records +primitive mutations (set column X on row Y, or insert row Y) tied to a batch id, +and a generic undo replays them in reverse. + +importbatches one row per approved write - the identity that approval, the + write, and the eventual undo all reference (the same value + the approval packet calls batchid). +importjournalentries one row per primitive mutation the writer made, ordered so + undo can reverse them (inserts removed after the updates that + pointed at them are reverted). + +The undo contract, kept deliberately conservative: + - update is reverted only if the cell still holds the value we wrote; a value a + human changed since is left as-is and reported (never clobbered). + - insert is hard-deleted, but ONLY the row this batch created and only when + nothing outside the batch still references it. +""" + +from shopdb.extensions import db + + +# Batch lifecycle. +BATCH_APPLIED = 'applied' +BATCH_UNDONE = 'undone' + +# Journal operation kinds. +OP_UPDATE = 'update' +OP_INSERT = 'insert' + + +class ImportBatch(db.Model): + """One approved import write. The batchid is the approval packet's batchid, + carried through write and undo so all three reference one identity.""" + __tablename__ = 'importbatches' + + batchid = db.Column(db.String(36), primary_key=True) + kind = db.Column(db.String(64), nullable=False) + planhash = db.Column(db.String(64), nullable=False) + rowcount = db.Column(db.Integer, nullable=False, default=0) + createdat = db.Column(db.DateTime, nullable=False) + createdby = db.Column(db.String(100), nullable=True) + status = db.Column(db.String(16), nullable=False, default=BATCH_APPLIED) + undoneat = db.Column(db.DateTime, nullable=True) + + entries = db.relationship( + 'ImportJournalEntry', + backref='batch', + cascade='all, delete-orphan', + order_by='ImportJournalEntry.sequence', + ) + + def __repr__(self): + return f"" + + +class ImportJournalEntry(db.Model): + """One reversible primitive mutation made by an import writer.""" + __tablename__ = 'importjournalentries' + + # SQLite only autoincrements a plain INTEGER PRIMARY KEY, not BIGINT, so the + # test database needs the Integer variant while MySQL keeps BIGINT for range. + entryid = db.Column( + db.BigInteger().with_variant(db.Integer, 'sqlite'), + primary_key=True) + batchid = db.Column( + db.String(36), + db.ForeignKey('importbatches.batchid', ondelete='CASCADE'), + nullable=False, + index=True, + ) + sequence = db.Column(db.Integer, nullable=False) + operation = db.Column(db.String(16), nullable=False) + targettable = db.Column(db.String(64), nullable=False) + targetid = db.Column(db.BigInteger, nullable=False) + columnname = db.Column(db.String(64), nullable=True) + oldvalue = db.Column(db.Text, nullable=True) + newvalue = db.Column(db.Text, nullable=True) + isreverted = db.Column(db.Boolean, nullable=False, default=False) + + def __repr__(self): + return (f"") diff --git a/shopdb/core/services/wincimport.py b/shopdb/core/services/wincimport.py new file mode 100644 index 00000000..87464136 --- /dev/null +++ b/shopdb/core/services/wincimport.py @@ -0,0 +1,138 @@ +"""WINC machine-adoption import service (preview / commit / undo / list). + +Web-facing wrapper over the validated WINC CLI helpers. The CLI in +scripts/site_imports/winc drives the same analyzer, writer, and journal; this +module adapts them to a Flask request: it works on the request-scoped +db.session connection (so the route owns the transaction and commit) and returns +plain JSON-serialisable dicts for the API. + +WINC is NOT a source adapter like Azure/Thor. It never creates assets - it +updates existing machines only, requires acknowledgment before creating business +units, and records every change to the reversible import journal. That contract +lives here and in the writer, not in the generic canonical-CSV importer. +""" + +from shopdb.extensions import db +from scripts.site_imports import journal +from scripts.site_imports.approval import build_packet +from scripts.site_imports.winc import writer as winc_writer +from scripts.site_imports.winc.dry_run import analyze_rows, parse_export_text + + +class WincImportError(Exception): + """An uploaded export could not be analyzed or applied as WINC.""" + + +def _connection(): + return db.session.connection() + + +def _plan_and_packet(export_text): + """Parse an uploaded export, build the plan and the approval packet.""" + rows = parse_export_text(export_text) + if not rows: + raise WincImportError('the uploaded file had no rows') + if 'assetnum' not in rows[0]: + raise WincImportError( + 'this does not look like a WINC asset export (no assetnum column)') + + plan, summary, _lookups = analyze_rows(_connection(), rows) + packet = build_packet('winc-machine-adoption', plan, summary, + winc_writer._PACKET_GROUPS) + return plan, summary, packet + + +def _report(summary, packet): + """Shape the packet + summary into the JSON the UI renders.""" + return { + 'kind': packet['kind'], + 'planhash': packet['planhash'], + 'rowcount': packet['rowcount'], + 'summary': dict(summary), + 'needscreate': {group: dict(items) + for group, items in packet.get('needscreate', {}).items()}, + 'unresolved': {group: dict(items) + for group, items in packet.get('unresolved', {}).items()}, + 'blocking': packet.get('blocking', False), + } + + +def preview(export_text): + """Read-only: analyze the export and return the report the UI shows. + + Writes nothing. The whole point is to surface what a commit WOULD do - + counts, the business units it would create (needs acknowledgment), and any + unresolved values that block the import. + """ + _plan, summary, packet = _plan_and_packet(export_text) + report = _report(summary, packet) + report['committed'] = False + return report + + +def commit(export_text, acknowledge=None, createdby=None): + """Apply the plan reversibly inside the request transaction. + + Rebuilds the plan from the same upload, re-checks it against a freshly built + packet (drift, blocking, acknowledgment) and writes through the journal. The + caller owns the commit/rollback of db.session. Returns the report plus the + batchid on success. + + Raises WincImportError (mapped to a 400 by the route) when the plan is + blocking or a needs-create group was not acknowledged. + """ + plan, summary, packet = _plan_and_packet(export_text) + try: + batchid = winc_writer.apply_plan( + _connection(), plan, packet, + acknowledge=acknowledge, createdby=createdby) + except winc_writer.WriterError as exc: + raise WincImportError(str(exc)) + + report = _report(summary, packet) + report['committed'] = True + report['batchid'] = batchid + return report + + +def undo(batchid): + """Reverse an applied batch. Caller owns the transaction. Returns a dict.""" + result = journal.undo_batch(_connection(), batchid, + references=winc_writer._UNDO_REFERENCES) + return { + 'batchid': result.batchid, + 'reverted': result.reverted, + 'conflicts': [ + {'operation': conflict.operation, 'table': conflict.table, + 'targetid': conflict.targetid, 'column': conflict.column, + 'reason': conflict.reason} + for conflict in result.conflicts + ], + } + + +def list_batches(limit=50): + """Applied/undone WINC batches, newest first, for the UI's undo list.""" + from sqlalchemy import text + + def _iso(value): + # PyMySQL hands back datetime objects; SQLite (tests) hands back the + # stored ISO string. Normalise both to an ISO string with a Z suffix. + if value is None: + return None + if hasattr(value, 'isoformat'): + return value.isoformat() + 'Z' + return str(value) + + rows = _connection().execute(text( + 'SELECT batchid, kind, rowcount, createdat, createdby, status, ' + 'undoneat FROM importbatches WHERE kind = :kind ' + 'ORDER BY createdat DESC LIMIT :limit'), + {'kind': 'winc-machine-adoption', 'limit': limit}).fetchall() + return [ + {'batchid': r[0], 'kind': r[1], 'rowcount': r[2], + 'createdat': _iso(r[3]), + 'createdby': r[4], 'status': r[5], + 'undoneat': _iso(r[6])} + for r in rows + ] diff --git a/tests/test_winc_import_routes.py b/tests/test_winc_import_routes.py new file mode 100644 index 00000000..0b273be1 --- /dev/null +++ b/tests/test_winc_import_routes.py @@ -0,0 +1,210 @@ +"""WINC machine-adoption import routes: preview / commit / undo. + +WINC is not a canonical-CSV source adapter (that path is covered by +test_import_sources). It updates existing machines only, needs explicit +acknowledgment before creating a business unit, and every commit is journalled +so it can be undone. These tests drive the real routes end to end against the +in-memory schema: seed a machine asset, upload an export that matches it, and +assert what preview reports, what commit writes, and that undo puts it back. +""" +import io + +import pytest + + +def _seed_machine(db, assetnumber='0083A Debur'): + """Create one machine asset the export can match on, plus a J3 location. + + Returns (assetid, locationid). The workstation key of '0083A Debur' is + '0083A', which the export row 'W0083A' reduces to, so the two crosswalk. + """ + from shopdb.core.models import AssetType, AssetStatus, Location + from shopdb.core.models.asset import Asset + from plugins.machines.models.machine import Machine + + assettype = AssetType(assettype='machine') + status = AssetStatus(status='In Use') + location = Location(locationname='J3') + db.session.add_all([assettype, status, location]) + db.session.flush() + + asset = Asset( + assetnumber=assetnumber, + name='Old Name', + assettypeid=assettype.assettypeid, + statusid=status.statusid, + ) + db.session.add(asset) + db.session.flush() + + db.session.add(Machine(assetid=asset.assetid)) + db.session.commit() + return asset.assetid, location.locationid + + +# An export whose single in-scope row matches the seeded machine. 'WM-J3' +# resolves to the J3 location; 'lg parts' aliases to the 'Large Parts' business +# unit (present or needs-create depending on the test). +EXPORT_CSV = ( + 'assetnum,description,location,eq1\r\n' + 'W0083A,Renamed Machine,WM-J3,lg parts\r\n' + 'FACILITIES-1,Not a machine,,\r\n' +) + + +def _upload(client, headers, url, csv_text=EXPORT_CSV, extra=None): + data = {'files': (io.BytesIO(csv_text.encode('utf-8')), 'winc.csv')} + if extra: + data.update(extra) + return client.post(url, data=data, content_type='multipart/form-data', + headers=headers) + + +PREVIEW = '/api/settings/asset-import/winc/preview' +COMMIT = '/api/settings/asset-import/winc/commit' +UNDO = '/api/settings/asset-import/winc/undo' +BATCHES = '/api/settings/asset-import/winc/batches' + + +def test_preview_reports_plan_without_writing(client, db, auth_headers): + _seed_machine(db) + + response = _upload(client, auth_headers, PREVIEW) + assert response.status_code == 200, response.get_json() + report = response.get_json()['data'] + + # One workstation row in scope, the facilities row filtered out. + assert report['committed'] is False + assert report['rowcount'] == 1 + assert report['summary']['in_scope'] == 1 + assert report['summary']['non_workstation_rows'] == 1 + # 'Large Parts' does not exist yet, so it is flagged for acknowledgment. + assert report['needscreate']['businessunit'] == {'Large Parts': 1} + assert report['blocking'] is False + + # A preview must not have touched the asset. + from shopdb.core.models.asset import Asset + assert Asset.query.first().name == 'Old Name' + + +def test_preview_rejects_non_winc_upload(client, db, auth_headers): + _seed_machine(db) + response = _upload(client, auth_headers, PREVIEW, + csv_text='foo,bar\r\n1,2\r\n') + assert response.status_code == 400 + assert 'assetnum' in response.get_json()['data']['error']['message'] + + +def test_commit_requires_acknowledgment_for_needs_create(client, db, + auth_headers): + _seed_machine(db) + # No acknowledge field: the needs-create business unit blocks the write. + response = _upload(client, auth_headers, COMMIT) + assert response.status_code == 400 + message = response.get_json()['data']['error']['message'].lower() + assert 'acknowledge' in message + + from shopdb.core.models.asset import Asset + from shopdb.core.models import BusinessUnit + assert Asset.query.first().name == 'Old Name' + assert BusinessUnit.query.count() == 0 + + +def test_commit_applies_plan_and_creates_business_unit(client, db, + auth_headers): + assetid, locationid = _seed_machine(db) + + response = _upload(client, auth_headers, COMMIT, + extra={'acknowledge': 'businessunit'}) + assert response.status_code == 200, response.get_json() + report = response.get_json()['data'] + assert report['committed'] is True + assert report['batchid'] + + from shopdb.core.models.asset import Asset + from shopdb.core.models import BusinessUnit + asset = db.session.get(Asset, assetid) + assert asset.name == 'Renamed Machine' + assert asset.locationid == locationid + created = BusinessUnit.query.filter_by(businessunit='Large Parts').one() + assert asset.businessunitid == created.businessunitid + + +def test_commit_uses_existing_business_unit_when_present(client, db, + auth_headers): + from shopdb.core.models import BusinessUnit + existing = BusinessUnit(businessunit='Large Parts') + db.session.add(existing) + db.session.commit() + existing_id = existing.businessunitid + + assetid, _ = _seed_machine(db) + + # Already resolved, so no needs-create and no acknowledgment needed. + response = _upload(client, auth_headers, COMMIT) + assert response.status_code == 200, response.get_json() + + from shopdb.core.models.asset import Asset + asset = db.session.get(Asset, assetid) + assert asset.businessunitid == existing_id + # No duplicate business unit was created. + assert BusinessUnit.query.filter_by(businessunit='Large Parts').count() == 1 + + +def test_undo_reverses_a_committed_batch(client, db, auth_headers): + assetid, _ = _seed_machine(db) + + committed = _upload(client, auth_headers, COMMIT, + extra={'acknowledge': 'businessunit'}) + batchid = committed.get_json()['data']['batchid'] + + from shopdb.core.models.asset import Asset + from shopdb.core.models import BusinessUnit + assert db.session.get(Asset, assetid).name == 'Renamed Machine' + + response = client.post(UNDO, json={'batchid': batchid}, headers=auth_headers) + assert response.status_code == 200, response.get_json() + result = response.get_json()['data'] + assert result['batchid'] == batchid + assert result['reverted'] > 0 + assert result['conflicts'] == [] + + db.session.expire_all() + asset = db.session.get(Asset, assetid) + assert asset.name == 'Old Name' + assert asset.locationid is None + assert asset.businessunitid is None + # The importer-created business unit was hard-deleted on undo. + assert BusinessUnit.query.filter_by(businessunit='Large Parts').count() == 0 + + +def test_undo_requires_batchid(client, db, auth_headers): + response = client.post(UNDO, json={}, headers=auth_headers) + assert response.status_code == 400 + assert 'batchid' in response.get_json()['data']['error']['message'].lower() + + +def test_batches_lists_committed_then_undone(client, db, auth_headers): + _seed_machine(db) + committed = _upload(client, auth_headers, COMMIT, + extra={'acknowledge': 'businessunit'}) + batchid = committed.get_json()['data']['batchid'] + + listed = client.get(BATCHES, headers=auth_headers).get_json()['data'] + batch = next(b for b in listed['batches'] if b['batchid'] == batchid) + assert batch['status'] == 'applied' + assert batch['kind'] == 'winc-machine-adoption' + + client.post(UNDO, json={'batchid': batchid}, headers=auth_headers) + listed = client.get(BATCHES, headers=auth_headers).get_json()['data'] + batch = next(b for b in listed['batches'] if b['batchid'] == batchid) + assert batch['status'] == 'undone' + assert batch['undoneat'] is not None + + +def test_winc_routes_require_admin(client, db, member_headers): + _seed_machine(db) + for url in (PREVIEW, COMMIT): + response = _upload(client, member_headers, url) + assert response.status_code in (401, 403) + assert client.get(BATCHES, headers=member_headers).status_code in (401, 403) From 17825ec8f10e2fbec8dd98c9f4e717361eabc08c Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Thu, 3 Sep 2026 08:07:01 -0400 Subject: [PATCH 47/90] Add WINC batches CLI and a CLI how-to README The web import panel can list applied/undone batches, but the CLI had no equivalent, so an operator working from the shell had no way to find a batchid to feed into the undo command. Add a read-only batches command that reuses the same importbatches query, and a short README documenting the four WINC commands and their switches. --- scripts/site_imports/winc/README.md | 67 ++++++++++++++++++++ scripts/site_imports/winc/batches.py | 91 ++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 scripts/site_imports/winc/README.md create mode 100644 scripts/site_imports/winc/batches.py diff --git a/scripts/site_imports/winc/README.md b/scripts/site_imports/winc/README.md new file mode 100644 index 00000000..d487213b --- /dev/null +++ b/scripts/site_imports/winc/README.md @@ -0,0 +1,67 @@ +# WINC machine-adoption import (CLI) + +Adopts machines from a Maximo asset export (validated against the WINC / West +Manufacturing site). It only updates machines that already exist; it never +creates canonical per-table rows. Every commit is journaled so it can be undone +as one batch. + +Run all commands from the repo root. Each reads `.env` and connects to the +configured database. + +## 1. Preview (read-only) + +``` +python -m scripts.site_imports.winc.dry_run --export EXPORT.csv +``` + +| switch | what it does | +| --- | --- | +| `--export PATH` | the Maximo ASSET export CSV to analyze (required) | +| `--rows` | print the per-row plan, not just the summary | +| `--csv PATH` | write the full plan to a CSV so runs can be diffed | + +Nothing is written. Use this to see what would change and which values are +unresolved before committing. + +## 2. Commit (writes, reversibly) + +``` +python -m scripts.site_imports.winc.writer --export EXPORT.csv --commit --acknowledge businessunit +``` + +| switch | what it does | +| --- | --- | +| `--export PATH` | the same export CSV (required) | +| `--commit` | actually write; without it this is a dry run | +| `--acknowledge GROUP` | approve a needs-create group (repeatable), e.g. `businessunit` | +| `--createdby NAME` | who approved the import; stamped on the batch | + +On success it prints the new `batchid` and the exact undo command. + +## 3. List batches + +``` +python -m scripts.site_imports.winc.batches +``` + +| switch | what it does | +| --- | --- | +| `--limit N` | show at most N batches, newest first (default 50) | +| `--json` | emit JSON instead of a table | + +Read-only. Use it to find the `batchid` to undo. + +## 4. Undo a batch + +``` +python -m scripts.site_imports.winc.undo --batch BATCHID --commit +``` + +| switch | what it does | +| --- | --- | +| `--batch BATCHID` | the batch to reverse (required) | +| `--commit` | actually reverse; without it this is a no-op | + +Reversal is conservative: a cell a human changed since the import is left as-is +and reported, and a created business unit is deleted only when nothing outside +the batch still references it. diff --git a/scripts/site_imports/winc/batches.py b/scripts/site_imports/winc/batches.py new file mode 100644 index 00000000..00a4545d --- /dev/null +++ b/scripts/site_imports/winc/batches.py @@ -0,0 +1,91 @@ +"""WINC machine adoption - BATCHES (list applied/undone import batches). + +A read-only companion to writer.py and undo.py: it prints the WINC batches +newest-first so an operator can find a batchid to feed `--batch` into the undo +CLI. Like the other WINC commands it is WINC-named only for discoverability; the +underlying importbatches table is import-agnostic, so this simply filters to the +winc-machine-adoption kind. +""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__)))))) + +from dotenv import load_dotenv +from pathlib import Path + +load_dotenv(Path(__file__).resolve().parents[3] / '.env') + +from sqlalchemy import text + +from scripts.site_imports.winc.dry_run import build_app_connection + +_KIND = 'winc-machine-adoption' + + +def _iso(value): + # PyMySQL hands back datetime objects; SQLite hands back the stored ISO + # string. Normalise both to an ISO string so the output is stable. + if value is None: + return '-' + if hasattr(value, 'isoformat'): + return value.isoformat() + return str(value) + + +def list_batches(connection, limit): + rows = connection.execute(text( + 'SELECT batchid, kind, rowcount, createdat, createdby, status, ' + 'undoneat FROM importbatches WHERE kind = :kind ' + 'ORDER BY createdat DESC LIMIT :limit'), + {'kind': _KIND, 'limit': limit}).fetchall() + return [ + {'batchid': r[0], 'kind': r[1], 'rowcount': r[2], + 'createdat': _iso(r[3]), 'createdby': r[4], 'status': r[5], + 'undoneat': _iso(r[6])} + for r in rows + ] + + +def _print_table(batches): + if not batches: + print('No WINC import batches found.') + return + header = '%-36s %-8s %5s %-20s %-16s %s' % ( + 'batchid', 'status', 'rows', 'createdat', 'createdby', 'undoneat') + print(header) + print('-' * len(header)) + for batch in batches: + print('%-36s %-8s %5s %-20s %-16s %s' % ( + batch['batchid'], batch['status'], batch['rowcount'], + batch['createdat'], batch['createdby'] or '-', batch['undoneat'])) + + +def main(argv=None): + parser = argparse.ArgumentParser( + description='List applied/undone WINC import batches, newest first.') + parser.add_argument('--limit', type=int, default=50, + help='Maximum number of batches to show (default 50).') + parser.add_argument('--json', action='store_true', + help='Emit the batches as JSON instead of a table.') + args = parser.parse_args(argv) + + if args.limit < 1: + parser.error('--limit must be a positive integer') + + _app, connection = build_app_connection() + batches = list_batches(connection, args.limit) + + if args.json: + import json + print(json.dumps(batches, indent=2)) + else: + _print_table(batches) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From db75ca79d1b9b0520e741812d3468e2da6f9e3e1 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Thu, 3 Sep 2026 12:36:28 -0400 Subject: [PATCH 48/90] [ALERT] Serve gunicorn with threaded workers so bad clients cannot starve the UI Untrusting or half-deployed agents that fail the TLS handshake retry in a loop. With 4 blocking sync workers a few of them pinned every worker, and real UI requests queued for 20+ seconds while CPU sat idle. Switch to gthread (4 workers x 8 threads) and drop the timeout to 30s so a stuck connection frees its thread quickly and one misbehaving client cannot exhaust request capacity. --- docker-compose.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 326966a5..ec2628a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,10 +79,19 @@ services: - gunicorn - --bind - 0.0.0.0:5001 + # Threaded workers so one blocked request (e.g. an agent that fails the TLS + # handshake and retries in a loop) cannot pin a whole worker and starve the + # UI. With sync workers a handful of stuck clients exhausted all capacity. + - --worker-class + - gthread - --workers - "4" + - --threads + - "8" + # Shorter timeout so a hung connection frees its thread quickly instead of + # holding it for a full minute. - --timeout - - "60" + - "30" - --access-logfile - "-" - --error-logfile From 19071ec6c69954cd1bc39f4f3372849a37681149 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Thu, 3 Sep 2026 20:06:45 -0400 Subject: [PATCH 49/90] Add opt-in backup overlay with third-container restic store Backups are per-site and not part of the base stack, so they ship as a separate compose overlay with an isolated restic REST server the app can only reach through the backup utility (no shell or db creds). The sidecar image trusts the corporate proxy root CA at build time so its Oracle yum, github, and PyPI fetches succeed behind TLS interception; shell scripts are pinned to LF via .gitattributes so the container entrypoints do not die on a CRLF shebang. --- docker-compose.backup.yml | 173 ++++++ frontend/src/router/routes/core.js | 6 + .../src/views/settings/ShopdbBackupsList.vue | 587 ++++++++++++++++++ frontend/src/views/settings/settingsNav.js | 3 +- scripts/backup/.gitattributes | 4 + scripts/backup/Dockerfile.backup | 38 ++ scripts/backup/backup.sh | 98 +++ scripts/backup/manifest.py | 122 ++++ scripts/backup/restore.sh | 76 +++ scripts/backup/watch.sh | 38 ++ shopdb/__init__.py | 1 + shopdb/core/api/__init__.py | 2 + shopdb/core/api/settings.py | 93 +++ shopdb/core/api/shopdbbackups.py | 352 +++++++++++ 14 files changed, 1592 insertions(+), 1 deletion(-) create mode 100644 docker-compose.backup.yml create mode 100644 frontend/src/views/settings/ShopdbBackupsList.vue create mode 100644 scripts/backup/.gitattributes create mode 100644 scripts/backup/Dockerfile.backup create mode 100644 scripts/backup/backup.sh create mode 100644 scripts/backup/manifest.py create mode 100644 scripts/backup/restore.sh create mode 100644 scripts/backup/watch.sh create mode 100644 shopdb/core/api/shopdbbackups.py diff --git a/docker-compose.backup.yml b/docker-compose.backup.yml new file mode 100644 index 00000000..1230b3a9 --- /dev/null +++ b/docker-compose.backup.yml @@ -0,0 +1,173 @@ +# shopdb-flask BACKUP overlay - opt-in, per-site. +# +# WHY a separate overlay: backups are not part of the base stack every site must +# run. A facility turns them on by adding this file to its compose invocation: +# +# docker compose -f docker-compose.yml -f docker-compose.backup.yml \ +# --profile backup up -d +# +# That brings up THREE cooperating pieces: +# - backup-store : a resident, isolated restic REST server (the third +# container). Holds every snapshot in its own volume and is +# reachable ONLY over the compose network - never published +# to the host - so the app can touch backups solely through +# the backup/restore utility, not with a shell or db creds. +# - backup-watcher : a resident poller that actuates the "Back up now" sentinel +# the api drops, running the one-shot against backup-store. +# This is what makes the in-app button produce a real backup +# given the api container cannot run docker. +# - backup : the same one-shot as a manual `run --rm backup` for sites +# that prefer a host scheduler (cron / Task Scheduler). +# +# WHAT each run captures, as one recoverable set: +# - db.sql.gz : mysqldump --single-transaction of the whole DB (users, +# auth, ledger, every plugin table). InnoDB-consistent, +# non-blocking. +# - instance/ : restic snapshot of /app/instance (deduped, encrypted), +# the files MySQL rows point at. +# - manifest.json: core+plugin Alembic revisions and image tag at capture +# time - the stamp that makes a restore a CHECKED operation. +# +# Retention (restic-native): --keep-daily 5 --keep-weekly 4 --keep-monthly 6, +# satisfying "any day in the last 5" plus a cheap long-horizon tail. +# +# DESTINATION defaults to the third container via RESTIC_REPOSITORY: +# third container : rest:http://backup-store:8000/ (default, ships with app) +# AWS S3 : s3:s3.amazonaws.com/shopdb--backups +# Azure Blob : azure:shopdbbackups: +# SMB share : mount the share on the host, point at a local path +# Credentials come from the site's env/secret store, never committed - same +# pattern as the collector key. + +# The THIRD container: an isolated backup store the app can only reach through +# the backup/restore utility, never with a shell or the db credentials. It runs +# restic's REST server over the compose network; the only thing that speaks to +# it is the backup watcher/one-shot via RESTIC_REPOSITORY=rest:http://... . Its +# data lives in its own named volume (backupstore_data), so it is container +# agnostic - later a pull/migration utility can point at this same REST endpoint +# from anywhere without touching the app or the database. +services: + backup-store: + image: restic/rest-server:0.13.0 + profiles: ["backup"] + restart: unless-stopped + environment: + # Private (no public /create) so repos are provisioned by the backup + # client's `restic init`, and append-only OFF here because the same + # utility both writes and prunes; isolation comes from this being the + # only service on the network with access, not from client trust. + DISABLE_AUTHENTICATION: "true" + OPTIONS: "--no-auth" + volumes: + - backupstore_data:/data + # Deliberately NOT published to the host: reachable only over the compose + # network by service name, so nothing outside the stack can touch backups. + expose: + - "8000" + healthcheck: + test: ["CMD", "sh", "-c", "wget -q -O - http://127.0.0.1:8000/ >/dev/null 2>&1 || true"] + interval: 30s + timeout: 5s + retries: 3 + + # Resident watcher: actuates the "Back up now" sentinel the api drops. This is + # the piece that makes the UI button run a real backup, since the api itself + # cannot run docker. It shares the base image with the one-shot and calls the + # same backup.sh. + backup-watcher: + image: shopdb-backup:local + build: + context: . + dockerfile: scripts/backup/Dockerfile.backup + profiles: ["backup"] + restart: unless-stopped + depends_on: + db: + condition: service_healthy + backup-store: + condition: service_started + environment: + # DB access - same credentials the api uses, reaching db over the compose + # network by service name. + MYSQL_HOST: db + MYSQL_DATABASE: shopdb_flask + MYSQL_USER: shopdb + MYSQL_PASSWORD: ${MYSQL_PASSWORD:?MYSQL_PASSWORD must be set} + # Destination is the third container by default: restic's REST backend on + # the isolated backup-store. A site can still override RESTIC_REPOSITORY + # in .env to send offsite instead, but out of the box backups land in the + # dedicated container. + RESTIC_REPOSITORY: ${RESTIC_REPOSITORY:-rest:http://backup-store:8000/} + RESTIC_PASSWORD: ${RESTIC_PASSWORD:?RESTIC_PASSWORD must be set} + # Optional cloud creds, passed through when set (empty otherwise). + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} + AZURE_ACCOUNT_NAME: ${AZURE_ACCOUNT_NAME:-} + AZURE_ACCOUNT_KEY: ${AZURE_ACCOUNT_KEY:-} + # Site tag stamped into every snapshot + manifest. + SHOPDB_SITE: ${SHOPDB_SITE:-shopdb} + SHOPDB_IMAGE_TAG: ${SHOPDB_IMAGE:-source} + SHOPDB_BACKUPS_ROOT: /backups + BACKUP_POLL_SECONDS: ${BACKUP_POLL_SECONDS:-10} + volumes: + # The files half of the backup scope, read-only: we never mutate the live + # instance tree from the backup path. + - instance_data:/app/instance:ro + # Alembic chain, read-only, so the manifest can stamp the schema revision + # the running code is at. + - ./migrations:/app/migrations:ro + - ./plugins:/app/plugins:ro + # Shared backups volume: holds the fast local db.sql.gz + manifest and the + # .backup-requested sentinel the api drops. The api mounts this same volume + # (see docker-compose.yml) so the UI can enumerate sets and request runs. + - backup_data:/backups + - ./scripts/backup:/opt/backup:ro + entrypoint: ["/opt/backup/watch.sh"] + + # Manual one-shot, unchanged usage: `docker compose ... run --rm backup`. + # Kept for host-scheduler / cron sites that prefer a scheduled pass over the + # in-app button. Shares everything with the watcher but runs once and exits. + backup: + image: shopdb-backup:local + build: + context: . + dockerfile: scripts/backup/Dockerfile.backup + profiles: ["backup"] + restart: "no" + depends_on: + db: + condition: service_healthy + backup-store: + condition: service_started + environment: + MYSQL_HOST: db + MYSQL_DATABASE: shopdb_flask + MYSQL_USER: shopdb + MYSQL_PASSWORD: ${MYSQL_PASSWORD:?MYSQL_PASSWORD must be set} + RESTIC_REPOSITORY: ${RESTIC_REPOSITORY:-rest:http://backup-store:8000/} + RESTIC_PASSWORD: ${RESTIC_PASSWORD:?RESTIC_PASSWORD must be set} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} + AZURE_ACCOUNT_NAME: ${AZURE_ACCOUNT_NAME:-} + AZURE_ACCOUNT_KEY: ${AZURE_ACCOUNT_KEY:-} + SHOPDB_SITE: ${SHOPDB_SITE:-shopdb} + SHOPDB_IMAGE_TAG: ${SHOPDB_IMAGE:-source} + SHOPDB_BACKUPS_ROOT: /backups + volumes: + - instance_data:/app/instance:ro + - ./migrations:/app/migrations:ro + - ./plugins:/app/plugins:ro + - backup_data:/backups + - ./scripts/backup:/opt/backup:ro + entrypoint: ["/opt/backup/backup.sh"] + + # Give the api read/write on the shared backups volume so the UI can list sets + # and drop the "Back up now" sentinel the watcher actuates. This extends the + # api service defined in docker-compose.yml (merged by compose). + api: + volumes: + - backup_data:/backups + +volumes: + backup_data: + backupstore_data: diff --git a/frontend/src/router/routes/core.js b/frontend/src/router/routes/core.js index b2bc3864..287a5f51 100644 --- a/frontend/src/router/routes/core.js +++ b/frontend/src/router/routes/core.js @@ -221,5 +221,11 @@ export default [ name: 'users', component: () => import('../../views/settings/UsersList.vue'), meta: { requiresAuth: true, requiresAdmin: true } + }, + { + path: 'settings/shopdbbackups', + name: 'shopdbbackups', + component: () => import('../../views/settings/ShopdbBackupsList.vue'), + meta: { requiresAuth: true, requiresAdmin: true } } ] diff --git a/frontend/src/views/settings/ShopdbBackupsList.vue b/frontend/src/views/settings/ShopdbBackupsList.vue new file mode 100644 index 00000000..0b649ad2 --- /dev/null +++ b/frontend/src/views/settings/ShopdbBackupsList.vue @@ -0,0 +1,587 @@ + + + + + diff --git a/frontend/src/views/settings/settingsNav.js b/frontend/src/views/settings/settingsNav.js index a5b87238..3b19a95e 100644 --- a/frontend/src/views/settings/settingsNav.js +++ b/frontend/src/views/settings/settingsNav.js @@ -4,7 +4,7 @@ // Order: site identity + the reference-data catalogs users touch daily come // first, then the platform/system groups (integrations, communication, search, // plugins, access) cluster together at the end. -import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key, Upload } from 'lucide-vue-next' +import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key, Upload, DatabaseBackup } from 'lucide-vue-next' export const settingsGroups = [ { @@ -113,6 +113,7 @@ export const settingsGroups = [ title: 'System', cards: [ { to: '/settings/plugins', icon: Puzzle, title: 'Plugins', description: 'Enable or disable installed plugins' }, + { to: '/settings/shopdbbackups', icon: DatabaseBackup, title: 'Site Backups', description: 'Scheduled database and file backups with guided restore; keeps any day in the last 5' }, ], }, { diff --git a/scripts/backup/.gitattributes b/scripts/backup/.gitattributes new file mode 100644 index 00000000..920d0296 --- /dev/null +++ b/scripts/backup/.gitattributes @@ -0,0 +1,4 @@ +# Keep shell scripts LF-only so they run inside Linux containers regardless of +# the host checkout. A CRLF shebang makes the kernel look for "bash\r" and the +# container dies with: env: 'bash\r': No such file or directory. +*.sh text eol=lf diff --git a/scripts/backup/Dockerfile.backup b/scripts/backup/Dockerfile.backup new file mode 100644 index 00000000..a207fdde --- /dev/null +++ b/scripts/backup/Dockerfile.backup @@ -0,0 +1,38 @@ +# Backup sidecar image: mysql:8.4 (for a version-matched mysqldump) plus restic +# (for deduped, encrypted, destination-agnostic file snapshots) and a small +# Python for the manifest writer. +# +# WHY mysql:8.4 as the base: the dump tool must match the server major/minor so +# the SQL it emits restores cleanly onto the same engine. Reusing the exact +# image the db service runs removes version drift as a failure mode. +FROM mysql:8.4 + +# Track 1 local build unblock: trust the corporate proxy (Zscaler/GE) root CA so +# the build-time fetches below (Oracle yum, github.com, PyPI) succeed behind TLS +# interception. This is a local convenience only; the shipping build vendors +# these deps instead of fetching them (pinned restic + SHA256, vendored +# bzip2/python3/alembic or a base image that carries them) so the release image +# needs no outbound calls and no embedded cert. The cert is gitignored (*.pem) +# and must be staged at scripts/backup/zscaler-root-ca.pem before building. +COPY scripts/backup/zscaler-root-ca.pem /etc/pki/ca-trust/source/anchors/zscaler-root-ca.pem +RUN update-ca-trust + +# restic from the distro is often stale; pull the pinned release binary so the +# snapshot format and retention flags are predictable across sites. +ARG RESTIC_VERSION=0.17.3 +RUN microdnf install -y bzip2 python3 python3-pip ca-certificates \ + && curl -fsSL -o /tmp/restic.bz2 \ + "https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_amd64.bz2" \ + && bunzip2 /tmp/restic.bz2 \ + && install -m 0755 /tmp/restic /usr/local/bin/restic \ + && rm -f /tmp/restic \ + && microdnf clean all + +# alembic so the manifest writer can read the current schema head. pip does not +# read the system trust store, so point it at the extracted system CA bundle +# (now including the proxy root) to get through TLS interception. +RUN python3 -m pip install --no-cache-dir \ + --cert /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem \ + alembic + +ENTRYPOINT ["/opt/backup/backup.sh"] diff --git a/scripts/backup/backup.sh b/scripts/backup/backup.sh new file mode 100644 index 00000000..8a22261f --- /dev/null +++ b/scripts/backup/backup.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# ShopDB backup one-shot. +# +# Produces one recoverable set per run and stores it two ways: a plain +# db.sql.gz + manifest.json on the local /backups volume for a fast, repo-free +# restore, and a restic snapshot (db dump + instance tree + manifest) at +# RESTIC_REPOSITORY for a deduped, encrypted, offsite copy. +# +# Sequence: +# 1. mysqldump --single-transaction (InnoDB-consistent, non-blocking) +# 2. manifest.py stamps core+plugin Alembic revisions + image tag +# 3. restic backup the dump + /app/instance + manifest as ONE snapshot +# 4. restic forget --keep-daily 5 --keep-weekly 4 --keep-monthly 6 --prune +# 5. restic check - fail loudly if the repo is unhealthy, so a bad repo is +# caught on backup night, not restore night. +# +# ASCII only, no emojis - matches the repo style gate. +set -euo pipefail + +BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}" +SITE="${SHOPDB_SITE:-shopdb}" +STAMP="$(date -u +%Y-%m-%dT%H%M%SZ)" +SETNAME="${SITE}-${STAMP}" +SETDIR="${BACKUPS_ROOT}/${SETNAME}" + +log() { echo "[backup] $*"; } + +mkdir -p "${SETDIR}" + +# 1. Database: whole-DB logical dump, consistent and non-blocking on InnoDB. +log "dumping database ${MYSQL_DATABASE} from ${MYSQL_HOST}" +mysqldump \ + --host="${MYSQL_HOST}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + --single-transaction \ + --routines \ + --triggers \ + --databases "${MYSQL_DATABASE}" \ + | gzip -c > "${SETDIR}/db.sql.gz" +log "database dump written ($(du -h "${SETDIR}/db.sql.gz" | cut -f1))" + +# 2. Manifest: the schema stamp that makes a restore a checked operation. +log "writing manifest" +python3 /opt/backup/manifest.py \ + --out "${SETDIR}/manifest.json" \ + --takenat "${STAMP}" \ + --site "${SITE}" \ + --imagetag "${SHOPDB_IMAGE_TAG:-source}" \ + --migrations /app/migrations \ + --plugins /app/plugins + +# 3. restic snapshot: db dump + instance tree + manifest, one point in time. +# Skips gracefully when restic/repo are not configured (local-only sites). +IS_CHECK_PASSED=false +if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then + if ! restic snapshots >/dev/null 2>&1; then + log "initializing restic repository at ${RESTIC_REPOSITORY}" + restic init + fi + log "restic backup (db dump + instance tree + manifest)" + restic backup \ + --tag "${SITE}" \ + --tag "${SETNAME}" \ + "${SETDIR}/db.sql.gz" \ + "${SETDIR}/manifest.json" \ + /app/instance + + # 4. Retention: any day in the last 5, plus weekly/monthly tail. + log "pruning per retention policy" + restic forget \ + --tag "${SITE}" \ + --keep-daily 5 \ + --keep-weekly 4 \ + --keep-monthly 6 \ + --prune + + # 5. Integrity: learn about corruption now, not at restore time. + log "verifying repository integrity" + if restic check; then + IS_CHECK_PASSED=true + log "restic check passed" + else + log "restic check FAILED - repository may be corrupt" + fi +else + log "restic not configured - local db.sql.gz + manifest only" +fi + +# Record the verification result into the manifest so the UI can badge the set. +python3 /opt/backup/manifest.py \ + --amend "${SETDIR}/manifest.json" \ + --ischeckpassed "${IS_CHECK_PASSED}" + +# Clear the request sentinel the UI drops when an operator clicks "Back up now". +rm -f "${BACKUPS_ROOT}/.backup-requested" 2>/dev/null || true + +log "backup set ${SETNAME} complete" diff --git a/scripts/backup/manifest.py b/scripts/backup/manifest.py new file mode 100644 index 00000000..8708ba94 --- /dev/null +++ b/scripts/backup/manifest.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Backup manifest writer. + +The manifest is the stamp that turns "we have dumps" into "we can actually +restore." It records, at capture time: + - takenat : ISO-8601 UTC timestamp of the run + - site : the site tag + - imagetag : the app image tag that produced the data + - schemarev : the core Alembic head revision(s) the running code is at, plus + each plugin chain's heads - the versions a restore target must + be at least, or restored rows reference columns the live schema + has not applied yet. + +Two modes: + --out ... write a fresh manifest for a new set + --amend ... patch an existing manifest (used to record ischeckpassed after + restic check runs) + +ASCII only, no emojis - matches the repo style gate. +""" + +import argparse +import json +import os +import sys + + +def _heads_for(chain_dir): + """Return the Alembic head revision id(s) under a versions directory. + + We read the revision graph the cheap way: a head is a revision no other + revision names as its down_revision. Done by text scan so this stays + dependency-light and works for both the core chain and each plugin chain. + """ + versions = os.path.join(chain_dir, 'versions') + if not os.path.isdir(versions): + return [] + revisions = {} + down_refs = set() + for name in os.listdir(versions): + if not name.endswith('.py'): + continue + path = os.path.join(versions, name) + revid = None + downid = None + try: + with open(path, 'r', encoding='utf-8') as handle: + for line in handle: + stripped = line.strip() + if stripped.startswith('revision =') and revid is None: + revid = stripped.split('=', 1)[1].strip().strip('"\'') + elif stripped.startswith('down_revision ='): + value = stripped.split('=', 1)[1].strip() + downid = None if value in ('None', 'null') else value.strip('"\'') + except OSError: + continue + if revid: + revisions[revid] = downid + if downid: + down_refs.add(downid) + return sorted(rev for rev in revisions if rev not in down_refs) + + +def _collect_schema(migrations_dir, plugins_dir): + """Core heads plus each plugin's chain heads, keyed for compatibility checks.""" + schema = {'core': _heads_for(migrations_dir), 'plugins': {}} + if plugins_dir and os.path.isdir(plugins_dir): + for name in sorted(os.listdir(plugins_dir)): + chain = os.path.join(plugins_dir, name, 'migrations') + heads = _heads_for(chain) + if heads: + schema['plugins'][name] = heads + return schema + + +def write_manifest(args): + manifest = { + 'takenat': args.takenat, + 'site': args.site, + 'imagetag': args.imagetag, + 'schemarev': _collect_schema(args.migrations, args.plugins), + 'ischeckpassed': False, + } + with open(args.out, 'w', encoding='utf-8') as handle: + json.dump(manifest, handle, indent=2, sort_keys=True) + return 0 + + +def amend_manifest(args): + try: + with open(args.amend, 'r', encoding='utf-8') as handle: + manifest = json.load(handle) + except (OSError, ValueError): + manifest = {} + if args.ischeckpassed is not None: + manifest['ischeckpassed'] = args.ischeckpassed.lower() == 'true' + with open(args.amend, 'w', encoding='utf-8') as handle: + json.dump(manifest, handle, indent=2, sort_keys=True) + return 0 + + +def main(argv=None): + parser = argparse.ArgumentParser(description='ShopDB backup manifest writer') + parser.add_argument('--out') + parser.add_argument('--amend') + parser.add_argument('--takenat', default='') + parser.add_argument('--site', default='shopdb') + parser.add_argument('--imagetag', default='source') + parser.add_argument('--migrations', default='/app/migrations') + parser.add_argument('--plugins', default='/app/plugins') + parser.add_argument('--ischeckpassed') + args = parser.parse_args(argv) + + if args.amend: + return amend_manifest(args) + if args.out: + return write_manifest(args) + parser.error('one of --out or --amend is required') + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/backup/restore.sh b/scripts/backup/restore.sh new file mode 100644 index 00000000..5d358e46 --- /dev/null +++ b/scripts/backup/restore.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# ShopDB restore. +# +# The DANGEROUS half of the pair, so it is deliberately guarded. A restore is +# only safe onto code whose schema is at least the set's - otherwise restored +# rows reference columns the live schema has not applied yet and every query on +# those tables 500s. This script refuses an incompatible restore rather than +# leaving the site half-broken. +# +# Usage (from the backup container): +# docker compose -f docker-compose.yml -f docker-compose.backup.yml \ +# run --rm --entrypoint /opt/backup/restore.sh backup +# +# It does NOT roll back code - that is git's job (git checkout && build). +# Restore code FIRST, bring the schema up (flask db upgrade + plugin +# upgrade-all), THEN run this to load the data. +# +# ASCII only, no emojis - matches the repo style gate. +set -euo pipefail + +BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}" +SETNAME="${1:-}" + +log() { echo "[restore] $*"; } +die() { echo "[restore] ERROR: $*" >&2; exit 1; } + +[ -n "${SETNAME}" ] || die "usage: restore.sh " + +SETDIR="${BACKUPS_ROOT}/${SETNAME}" +DBDUMP="${SETDIR}/db.sql.gz" + +# Prefer the local fast path; fall back to pulling the set out of restic. +if [ ! -f "${DBDUMP}" ]; then + log "local dump not found; restoring set from restic" + command -v restic >/dev/null 2>&1 || die "restic not available and no local dump" + restic restore latest --target / --include "${SETDIR}" \ + || die "restic restore of ${SETNAME} failed" +fi +[ -f "${DBDUMP}" ] || die "no db.sql.gz for set ${SETNAME}" + +# Compatibility gate: the running schema must cover the set's stamped heads. +log "checking schema compatibility" +CURRENT_HEADS="$(python3 /opt/backup/manifest.py \ + --out /tmp/current-manifest.json \ + --migrations /app/migrations --plugins /app/plugins \ + >/dev/null 2>&1 && python3 -c \ + 'import json;print(",".join(sorted(json.load(open("/tmp/current-manifest.json"))["schemarev"]["core"])))')" +SET_HEADS="$(python3 -c \ + 'import json,sys;print(",".join(sorted(json.load(open(sys.argv[1]))["schemarev"]["core"])))' \ + "${SETDIR}/manifest.json" 2>/dev/null || echo "")" + +if [ -z "${SET_HEADS}" ]; then + die "set ${SETNAME} has no readable manifest - refusing an unverified restore" +fi +if [ "${CURRENT_HEADS}" != "${SET_HEADS}" ]; then + log "current heads : ${CURRENT_HEADS}" + log "set heads : ${SET_HEADS}" + die "schema mismatch - bring code to the set's revision, run 'flask db upgrade' and 'flask plugin upgrade-all', then retry" +fi + +# Files first, then DB, so the two land together. +if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then + log "restoring instance files from restic" + restic restore latest --target / --include /app/instance \ + || die "restic restore of instance files failed" +fi + +log "loading database dump" +gunzip -c "${DBDUMP}" | mysql \ + --host="${MYSQL_HOST}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + || die "database load failed" + +rm -f "${BACKUPS_ROOT}/.restore-requested" 2>/dev/null || true +log "restore of ${SETNAME} complete" diff --git a/scripts/backup/watch.sh b/scripts/backup/watch.sh new file mode 100644 index 00000000..42c8b93e --- /dev/null +++ b/scripts/backup/watch.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# ShopDB backup WATCHER - the resident half of the in-app backup utility. +# +# WHY this exists: the api container cannot run docker, so when an operator +# clicks "Back up now" the api only drops a sentinel file (.backup-requested) +# on the shared backups volume. This watcher is the thing that actuates that +# request: it polls for the sentinel and runs the one-shot backup.sh, which +# ships a restic snapshot to the backup-store container and clears the sentinel +# on success. That turns the UI button into a real, self-serve backup without +# giving the api container any docker or host privileges. +# +# ASCII only, no emojis - matches the repo style gate. +set -uo pipefail + +BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}" +SENTINEL="${BACKUPS_ROOT}/.backup-requested" +INTERVAL="${BACKUP_POLL_SECONDS:-10}" + +log() { echo "[backup-watch] $*"; } + +mkdir -p "${BACKUPS_ROOT}" +log "watcher up; polling ${SENTINEL} every ${INTERVAL}s" + +while true; do + if [ -f "${SENTINEL}" ]; then + log "backup requested - running one-shot" + if /opt/backup/backup.sh; then + log "backup completed" + else + # backup.sh only clears the sentinel on success, so a failed run + # would otherwise loop forever. Clear it here and wait for the next + # explicit request rather than hammering a broken backup. + log "backup FAILED - clearing request, awaiting next" + rm -f "${SENTINEL}" 2>/dev/null || true + fi + fi + sleep "${INTERVAL}" +done diff --git a/shopdb/__init__.py b/shopdb/__init__.py index 4bab5be6..f513c409 100644 --- a/shopdb/__init__.py +++ b/shopdb/__init__.py @@ -298,6 +298,7 @@ CORE_BLUEPRINT_NAMES = ( 'pluginui', 'apitokens', 'docs', + 'shopdbbackups', ) diff --git a/shopdb/core/api/__init__.py b/shopdb/core/api/__init__.py index 3ebda32a..aab4d55a 100644 --- a/shopdb/core/api/__init__.py +++ b/shopdb/core/api/__init__.py @@ -27,6 +27,7 @@ from .setup import setup_bp from .pluginui import pluginui_bp from .apitokens import apitokens_bp from .docs import docs_bp +from .shopdbbackups import shopdbbackups_bp __all__ = [ 'auth_bp', @@ -56,4 +57,5 @@ __all__ = [ 'pluginui_bp', 'apitokens_bp', 'docs_bp', + 'shopdbbackups_bp', ] diff --git a/shopdb/core/api/settings.py b/shopdb/core/api/settings.py index ef774a86..8373f98c 100644 --- a/shopdb/core/api/settings.py +++ b/shopdb/core/api/settings.py @@ -1488,6 +1488,99 @@ def build_default_settings(): 'Incoming Webhook), adaptivecard (Teams Workflow), ' 'or json (generic {title,text})' }, + # Backup destination settings + # + # These are the ground wiring for where the one-shot backup container + # ships each set. 'local' keeps sets on the mounted backups volume and + # needs no credentials. 'smb' and 'aws' are declared so the UI and the + # backup runner can be configured now; the client libraries (an SMB + # stack, boto3/awscli) are deliberately NOT pulled in yet - those land + # behind the security review that vets third-party deps against + # external exploit. Until then a non-local destination is accepted and + # persisted but the runner treats it as not-yet-armed. + # + # Secrets follow the existing convention: any key containing + # 'password'/'secret'/'token' is masked by _is_secret on the way out + # and preserved when the client submits the mask back unchanged. + { + 'key': 'backup_destination_kind', + 'value': 'local', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'Where backup sets are shipped: local, smb, or aws. ' + 'Non-local destinations are wiring-only until the ' + 'security review clears their client libraries.' + }, + { + 'key': 'backup_smb_host', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'SMB/CIFS server hostname or IP for the backup share' + }, + { + 'key': 'backup_smb_share', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'SMB share name (the top-level export, no path)' + }, + { + 'key': 'backup_smb_path', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'Subpath within the SMB share to write sets into' + }, + { + 'key': 'backup_smb_username', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'SMB authentication username (DOMAIN\\user allowed)' + }, + { + 'key': 'backup_smb_password', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'SMB authentication password' + }, + { + 'key': 'backup_aws_region', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'AWS region for the S3 backup bucket (e.g. us-east-1)' + }, + { + 'key': 'backup_aws_bucket', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'S3 bucket name that holds backup sets' + }, + { + 'key': 'backup_aws_prefix', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'Optional key prefix within the S3 bucket' + }, + { + 'key': 'backup_aws_accesskeyid', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'AWS access key id for the backup bucket' + }, + { + 'key': 'backup_aws_secretaccesskey', + 'value': '', + 'valuetype': 'string', + 'category': 'backup', + 'description': 'AWS secret access key for the backup bucket' + }, # Audit log retention lives at 'audit_retentiondays' (category site), # defined above and read by shopdb/core/services/auditretention.py. A # second key 'audit_retention_days' was seeded here with a different diff --git a/shopdb/core/api/shopdbbackups.py b/shopdb/core/api/shopdbbackups.py new file mode 100644 index 00000000..f2f69cb1 --- /dev/null +++ b/shopdb/core/api/shopdbbackups.py @@ -0,0 +1,352 @@ +"""Backups API routes. + +Admin-only surface over the site's backup sets. The heavy lifting (mysqldump, +restic snapshot of the instance tree, retention prune, integrity check) runs in +the one-shot backup container defined in docker-compose.backup.yml; this +blueprint is the read-mostly control plane the UI talks to. + +Two recovery products stay firewalled, per the design notes: + - code rollback lives in git (tags/releases), NOT here + - data restore lives here, gated on schema compatibility via each set's manifest + +Every backup set carries a manifest.json stamped with the core + plugin Alembic +revisions and the running image tag at capture time. A restore is a CHECKED +operation: current code must be at least the set's schema revision, so we never +load rows that reference a column the live schema has not applied yet. +""" + +import json +import os +import subprocess +from datetime import datetime, timezone + +from flask import Blueprint, current_app, request + +from shopdb.core.models import AuditLog +from shopdb.core.models.setting import Setting +from shopdb.utils.responses import success_response, error_response, ErrorCodes +from shopdb.utils.authz import require_role + +shopdbbackups_bp = Blueprint('shopdbbackups', __name__) + +# Supported backup destination kinds. 'local' needs no credentials; 'smb' and +# 'aws' are ground-wired here so operators can configure them now, but their +# client libraries are held back until the security review clears them, so the +# runner treats a non-local destination as configured-but-not-yet-armed. +DESTINATION_KINDS = ('local', 'smb', 'aws') + +# The setting keys that make up each destination's config, grouped by kind so +# the GET can surface only the relevant fields and the PUT can validate what it +# is allowed to write. Secret keys (password/secret/token in the name) are +# masked by the shared settings serializer, same convention as SMTP/SAML. +DESTINATION_FIELDS = { + 'local': [], + 'smb': [ + 'backup_smb_host', + 'backup_smb_share', + 'backup_smb_path', + 'backup_smb_username', + 'backup_smb_password', + ], + 'aws': [ + 'backup_aws_region', + 'backup_aws_bucket', + 'backup_aws_prefix', + 'backup_aws_accesskeyid', + 'backup_aws_secretaccesskey', + ], +} + +# Keys whose value must never leave the API in the clear. Mirrors the shared +# settings convention (_is_secret) so the two surfaces agree on what is secret. +_SECRET_TOKENS = ('password', 'secret', 'token') +SECRET_MASK = '********' + + +def _is_secret_key(key): + return any(token in key for token in _SECRET_TOKENS) + +# Where the backup container drops each set and its manifest. Mounted read-only +# for the api container so the UI can enumerate sets without being able to +# mutate them; writes happen only from the backup one-shot. +BACKUPS_ROOT_ENV = 'SHOPDB_BACKUPS_ROOT' +DEFAULT_BACKUPS_ROOT = '/backups' + +# Retention the backup container enforces; surfaced read-only so the UI can show +# "day 1 of 5 retained" without re-deriving the policy. +RETENTION = { + 'keepdaily': 5, + 'keepweekly': 4, + 'keepmonthly': 6, +} + + +def _backups_root(): + return os.environ.get(BACKUPS_ROOT_ENV, DEFAULT_BACKUPS_ROOT) + + +def _read_manifest(setdir): + """Load a set's manifest.json, or None when absent/unreadable. + + A set without a readable manifest is not restorable (we cannot check its + schema), so callers treat None as "unverified" rather than trusting it. + """ + path = os.path.join(setdir, 'manifest.json') + try: + with open(path, 'r', encoding='utf-8') as handle: + return json.load(handle) + except (OSError, ValueError): + return None + + +def _current_schema_revisions(): + """Return the live Alembic head revisions (core + plugins) for this build. + + Compared against a set's stamped revisions to decide restore compatibility. + Imported lazily so a missing alembic at request time degrades to an empty + map (everything reads as "unknown") instead of 500ing the list endpoint. + """ + try: + from alembic.config import Config + from alembic.script import ScriptDirectory + except ImportError: + return {} + + root = current_app.root_path + ini = os.path.join(os.path.dirname(root), 'migrations', 'alembic.ini') + if not os.path.isfile(ini): + return {} + try: + config = Config(ini) + script = ScriptDirectory.from_config(config) + return {'core': list(script.get_heads())} + except Exception: + return {} + + +def _is_compatible(manifest, current): + """A set restores onto THIS build only when the build's schema is at least + the set's. We compare head-revision sets: if the set was taken on a schema + the current code has not reached, the code is too OLD and would 500 on the + restored rows, so restore is blocked until the code is upgraded. + """ + if not manifest: + return False + setrevs = set(manifest.get('schemarev', {}).get('core', [])) + currevs = set(current.get('core', [])) + if not setrevs or not currevs: + return False + # Compatible when the current heads cover every head the set was taken at. + return setrevs.issubset(currevs) + + +def _list_sets(): + """Enumerate backup sets newest-first, each summarized for the list view.""" + root = _backups_root() + if not os.path.isdir(root): + return [] + current = _current_schema_revisions() + sets = [] + for name in sorted(os.listdir(root), reverse=True): + setdir = os.path.join(root, name) + if not os.path.isdir(setdir): + continue + manifest = _read_manifest(setdir) + dbdump = os.path.join(setdir, 'db.sql.gz') + sizebytes = 0 + for base, _dirs, files in os.walk(setdir): + for filename in files: + try: + sizebytes += os.path.getsize(os.path.join(base, filename)) + except OSError: + pass + sets.append({ + 'snapshotid': name, + 'takenat': (manifest or {}).get('takenat'), + 'sizebytes': sizebytes, + 'schemarev': (manifest or {}).get('schemarev', {}), + 'imagetag': (manifest or {}).get('imagetag'), + 'ischeckpassed': bool((manifest or {}).get('ischeckpassed')), + 'hasmanifest': manifest is not None, + 'hasdbdump': os.path.isfile(dbdump), + 'iscompatible': _is_compatible(manifest, current), + }) + return sets + + +@shopdbbackups_bp.route('', methods=['GET']) +@require_role('admin') +def list_backups(): + """List every retained backup set, newest first, with a compatibility flag.""" + sets = _list_sets() + laststgood = next((s for s in sets if s['ischeckpassed']), None) + meta = { + 'retention': RETENTION, + 'lastgoodat': (laststgood or {}).get('takenat'), + 'count': len(sets), + } + return success_response(data=sets, meta=meta) + + +@shopdbbackups_bp.route('/', methods=['GET']) +@require_role('admin') +def get_backup(snapshotid): + """Return one set's full manifest plus a resolved compatibility verdict.""" + root = _backups_root() + setdir = os.path.join(root, os.path.basename(snapshotid)) + if not os.path.isdir(setdir): + return error_response(ErrorCodes.NOT_FOUND, 'Backup set not found', + http_code=404) + manifest = _read_manifest(setdir) + current = _current_schema_revisions() + return success_response(data={ + 'snapshotid': os.path.basename(snapshotid), + 'manifest': manifest, + 'currentschemarev': current, + 'iscompatible': _is_compatible(manifest, current), + 'hasmanifest': manifest is not None, + }) + + +@shopdbbackups_bp.route('', methods=['POST']) +@require_role('admin') +def trigger_backup(): + """Kick off an on-demand backup run. + + The api container cannot run docker; it writes a sentinel the host-side + scheduler (or an operator running `docker compose run --rm backup`) picks + up. Returning isrunning=false with a clear message keeps the UI honest + rather than pretending the run started inside this process. + """ + root = _backups_root() + try: + os.makedirs(root, exist_ok=True) + sentinel = os.path.join(root, '.backup-requested') + stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z' + with open(sentinel, 'w', encoding='utf-8') as handle: + handle.write(stamp) + except OSError as error: + return error_response(ErrorCodes.INTERNAL_ERROR, + 'Could not queue a backup: ' + str(error), + http_code=500) + AuditLog.log('requested', 'Backup', entityname=stamp) + return success_response( + data={'isrunning': False, 'isqueued': True, 'requestedat': stamp}, + message='Backup queued. The backup service runs it on its next pass.') + + +@shopdbbackups_bp.route('//restore', methods=['POST']) +@require_role('admin') +def restore_backup(snapshotid): + """Queue a restore of a set, gated on schema compatibility. + + We refuse outright when the set is incompatible (current code older than the + set's schema) so an operator cannot foot-gun a schema mismatch from the UI. + The actual restore is performed by scripts/backup/restore.sh in the backup + container; here we validate and record intent. + """ + root = _backups_root() + setdir = os.path.join(root, os.path.basename(snapshotid)) + if not os.path.isdir(setdir): + return error_response(ErrorCodes.NOT_FOUND, 'Backup set not found', + http_code=404) + manifest = _read_manifest(setdir) + current = _current_schema_revisions() + if not _is_compatible(manifest, current): + return error_response( + ErrorCodes.CONFLICT, + 'This backup was taken on a newer schema than the running code. ' + 'Upgrade the code to at least the set\'s revision, run ' + '`flask db upgrade` and `flask plugin upgrade-all`, then retry.', + details={'currentschemarev': current, + 'setschemarev': (manifest or {}).get('schemarev', {})}, + http_code=409) + try: + sentinel = os.path.join(root, '.restore-requested') + stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z' + with open(sentinel, 'w', encoding='utf-8') as handle: + handle.write(os.path.basename(snapshotid)) + except OSError as error: + return error_response(ErrorCodes.INTERNAL_ERROR, + 'Could not queue the restore: ' + str(error), + http_code=500) + AuditLog.log('restore-requested', 'Backup', + entityname=os.path.basename(snapshotid)) + return success_response( + data={'isqueued': True, 'snapshotid': os.path.basename(snapshotid)}, + message='Restore queued and validated against the live schema.') + + +@shopdbbackups_bp.route('/settings', methods=['GET']) +@require_role('admin') +def get_backup_settings(): + """Return destination + retention config for the settings pane. + + The destination kind and its (non-secret) fields come from the settings + store so an operator can configure where backups ship without editing env + or redeploying. Secret fields (SMB password, AWS secret key) are never + returned in the clear: we send a mask when a value is set and an empty + string when it is not, matching the shared settings API convention. + + 'smb' and 'aws' are configurable now but their client libraries are held + back pending the security review, so we flag those destinations as armed: + false to tell the UI the config is stored but not yet acting. + """ + kind = Setting.get('backup_destination_kind', 'local') + if kind not in DESTINATION_KINDS: + kind = 'local' + + fields = {} + for key in DESTINATION_FIELDS.get(kind, []): + raw = Setting.get(key, '') + if _is_secret_key(key): + fields[key] = SECRET_MASK if raw else '' + else: + fields[key] = raw or '' + + return success_response(data={ + 'destinationkind': kind, + 'destinationkinds': list(DESTINATION_KINDS), + 'fields': fields, + 'isarmed': kind == 'local', + 'retention': RETENTION, + 'backupsroot': _backups_root(), + }) + + +@shopdbbackups_bp.route('/settings', methods=['PUT']) +@require_role('admin') +def update_backup_settings(): + """Persist the backup destination kind and its credential/config fields. + + Only keys that belong to the chosen destination are accepted, so a stray + payload cannot write arbitrary settings rows through this surface. A secret + field submitted as the mask placeholder means "leave unchanged" - the + client only ever received the mask, so we never overwrite a real secret + with it. + """ + data = request.get_json(silent=True) or {} + kind = data.get('destinationkind', Setting.get('backup_destination_kind', 'local')) + if kind not in DESTINATION_KINDS: + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'destinationkind must be one of: ' + ', '.join(DESTINATION_KINDS)) + + Setting.set('backup_destination_kind', kind, valuetype='string', + category='backup') + + incoming = data.get('fields', {}) or {} + allowed = set(DESTINATION_FIELDS.get(kind, [])) + for key, value in incoming.items(): + if key not in allowed: + continue + if _is_secret_key(key) and value == SECRET_MASK: + continue + Setting.set(key, value if value is not None else '', + valuetype='string', category='backup') + + AuditLog.log('updated', 'BackupSettings', entityname=kind, changes={ + 'destinationkind': kind, + }) + + return get_backup_settings() From 231fd27f68caa68fdd3d6a2f7464c86c9f6cbe27 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Thu, 3 Sep 2026 20:11:52 -0400 Subject: [PATCH 50/90] Fix restore compat check reporting empty schema heads alembic.ini is a Flask-Migrate config with no script_location, so ScriptDirectory.from_config raised and _current_schema_revisions swallowed it and returned no heads. Every set then failed the subset check and was wrongly blocked as taken on a newer schema than the running code. Set script_location to the migrations directory - the same versions tree the manifest scans - so live heads resolve and compatible sets restore. --- shopdb/core/api/shopdbbackups.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/shopdb/core/api/shopdbbackups.py b/shopdb/core/api/shopdbbackups.py index f2f69cb1..4081cc80 100644 --- a/shopdb/core/api/shopdbbackups.py +++ b/shopdb/core/api/shopdbbackups.py @@ -113,11 +113,17 @@ def _current_schema_revisions(): return {} root = current_app.root_path - ini = os.path.join(os.path.dirname(root), 'migrations', 'alembic.ini') + migrations = os.path.join(os.path.dirname(root), 'migrations') + ini = os.path.join(migrations, 'alembic.ini') if not os.path.isfile(ini): return {} try: config = Config(ini) + # This is a Flask-Migrate project: alembic.ini carries no + # script_location (it is set at runtime), so ScriptDirectory.from_config + # would raise and leave every set reading as incompatible. Point it at + # the migrations directory - the same versions tree manifest.py scans. + config.set_main_option('script_location', migrations) script = ScriptDirectory.from_config(config) return {'core': list(script.get_heads())} except Exception: From 7e4469c44e348e6c9eb74818a1e852b62a3afb4e Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Thu, 3 Sep 2026 20:26:01 -0400 Subject: [PATCH 51/90] Wire restore into the backup watcher and fix backup timestamps The api queued .restore-requested but nothing consumed it, so restores silently did nothing. The watcher now services the restore sentinel by running restore.sh, and its instance_data mount is read-write so restore can lay files back down (the manual one-shot stays read-only). Backup stamps use a colon-free UTC form that Date could not parse, so the UI showed raw strings; normalize them before formatting. --- docker-compose.backup.yml | 8 +++++--- .../src/views/settings/ShopdbBackupsList.vue | 16 ++++++++++++--- scripts/backup/watch.sh | 20 ++++++++++++++++++- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/docker-compose.backup.yml b/docker-compose.backup.yml index 1230b3a9..b89cec69 100644 --- a/docker-compose.backup.yml +++ b/docker-compose.backup.yml @@ -110,9 +110,11 @@ services: SHOPDB_BACKUPS_ROOT: /backups BACKUP_POLL_SECONDS: ${BACKUP_POLL_SECONDS:-10} volumes: - # The files half of the backup scope, read-only: we never mutate the live - # instance tree from the backup path. - - instance_data:/app/instance:ro + # The watcher services both backup (read) and restore (write) requests, so + # the instance tree is mounted read-write here. Restore has to lay the + # files half of a set back down next to the DB rows that reference it; the + # backup half still only ever reads. The manual one-shot below stays :ro. + - instance_data:/app/instance # Alembic chain, read-only, so the manifest can stamp the schema revision # the running code is at. - ./migrations:/app/migrations:ro diff --git a/frontend/src/views/settings/ShopdbBackupsList.vue b/frontend/src/views/settings/ShopdbBackupsList.vue index 0b649ad2..d8f541dd 100644 --- a/frontend/src/views/settings/ShopdbBackupsList.vue +++ b/frontend/src/views/settings/ShopdbBackupsList.vue @@ -190,7 +190,7 @@

Restore this backup?

-

{{ restoreTarget.snapshotid }}

+

{{ formatTaken(restoreTarget.takenat || restoreTarget.snapshotid) }}

@@ -356,7 +356,7 @@ const STALE_HOURS = 48 const hoursSinceLastGood = computed(() => { if (!lastGoodAt.value) return null - const then = new Date(lastGoodAt.value.replace(/Z$/, 'Z')).getTime() + const then = new Date(normalizeStamp(lastGoodAt.value)).getTime() return (Date.now() - then) / 36e5 }) @@ -383,11 +383,21 @@ const bannerSub = computed(() => { // -- Formatting ------------------------------------------------------------- function formatTaken(value) { if (!value) return 'unknown' - const date = new Date(value.replace(/Z$/, 'Z')) + const date = new Date(normalizeStamp(value)) if (Number.isNaN(date.getTime())) return value return date.toLocaleString() } +// Backup ids and manifest stamps use a filename-safe UTC form with no colons +// (YYYY-MM-DDTHHMMSSZ, sometimes prefixed with a site name), which the Date +// parser rejects. Reinsert the colons so it parses as real ISO 8601. +function normalizeStamp(value) { + const match = String(value).match(/(\d{4}-\d{2}-\d{2})T(\d{2})(\d{2})(\d{2})Z$/) + if (!match) return value + const [, day, hour, minute, second] = match + return `${day}T${hour}:${minute}:${second}Z` +} + function formatSize(bytes) { if (!bytes) return '0 B' const units = ['B', 'KB', 'MB', 'GB', 'TB'] diff --git a/scripts/backup/watch.sh b/scripts/backup/watch.sh index 42c8b93e..18c53827 100644 --- a/scripts/backup/watch.sh +++ b/scripts/backup/watch.sh @@ -14,12 +14,17 @@ set -uo pipefail BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}" SENTINEL="${BACKUPS_ROOT}/.backup-requested" +RESTORE_SENTINEL="${BACKUPS_ROOT}/.restore-requested" INTERVAL="${BACKUP_POLL_SECONDS:-10}" log() { echo "[backup-watch] $*"; } mkdir -p "${BACKUPS_ROOT}" -log "watcher up; polling ${SENTINEL} every ${INTERVAL}s" +# The api container runs unprivileged and only drops sentinel files here; it +# cannot write into a root-owned 0755 directory. Make the backups root +# writable so "Back up now" and "Restore" can queue their sentinels. +chmod 0777 "${BACKUPS_ROOT}" 2>/dev/null || true +log "watcher up; polling ${SENTINEL} and ${RESTORE_SENTINEL} every ${INTERVAL}s" while true; do if [ -f "${SENTINEL}" ]; then @@ -34,5 +39,18 @@ while true; do rm -f "${SENTINEL}" 2>/dev/null || true fi fi + if [ -f "${RESTORE_SENTINEL}" ]; then + # The api drops the target set name into the restore sentinel. Read it, + # hand it to restore.sh, and clear the request whether it succeeds or + # fails so a bad set cannot pin the watcher in a tight retry loop. + setname="$(head -n1 "${RESTORE_SENTINEL}" 2>/dev/null | tr -d '\r\n')" + log "restore requested - running restore for ${setname}" + if /opt/backup/restore.sh "${setname}"; then + log "restore completed" + else + log "restore FAILED - clearing request, awaiting next" + fi + rm -f "${RESTORE_SENTINEL}" 2>/dev/null || true + fi sleep "${INTERVAL}" done From debe1fce690978816f556e2313a32ffe526555da Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Thu, 3 Sep 2026 20:43:42 -0400 Subject: [PATCH 52/90] it is risen - add restore progress and backup/restore history The watcher scripts now write live status and append-only run history onto the shared volume; the api exposes them via /shopdbbackups/status and /history. The restore modal shows an indeterminate spinner with phase labels, and two cards list backup and restore run records. Timestamps render in UTC to match the Zulu-stamped set ids, the history directory is excluded from the set list, and a stale-status guard stops a previous run's done phase from collapsing the spinner on the next restore. --- .../src/views/settings/ShopdbBackupsList.vue | 254 +++++++++++++++++- scripts/backup/backup.sh | 21 ++ scripts/backup/restore.sh | 24 ++ scripts/backup/status.sh | 100 +++++++ scripts/backup/watch.sh | 5 + shopdb/core/api/shopdbbackups.py | 84 ++++++ 6 files changed, 485 insertions(+), 3 deletions(-) create mode 100644 scripts/backup/status.sh diff --git a/frontend/src/views/settings/ShopdbBackupsList.vue b/frontend/src/views/settings/ShopdbBackupsList.vue index d8f541dd..eaeab054 100644 --- a/frontend/src/views/settings/ShopdbBackupsList.vue +++ b/frontend/src/views/settings/ShopdbBackupsList.vue @@ -185,6 +185,38 @@ + +
+
+

Backup history

+

No backup runs recorded yet.

+
    +
  • + + {{ outcomeLabel(entry.outcome) }} + + {{ entry.setname || 'unknown set' }} + {{ formatTaken(entry.finishedat) }} +
    • +
+
+ +
+

Restore history

+

No restore runs recorded yet.

+
    +
  • + + {{ outcomeLabel(entry.outcome) }} + + {{ entry.setname || 'unknown set' }} + {{ formatTaken(entry.finishedat) }} +
    • +
+
+
+
@@ -192,6 +224,34 @@

Restore this backup?

{{ formatTaken(restoreTarget.takenat || restoreTarget.snapshotid) }}

+ + + +
@@ -570,6 +739,85 @@ onMounted(() => { .compat.ok { background: rgba(56, 161, 105, 0.15); color: #9ae6b4; } .compat.warn { background: rgba(214, 158, 46, 0.15); color: #f6e05e; } +.restore-progress { + display: flex; + align-items: center; + gap: 0.6rem; + padding: 0.7rem 0.8rem; + border-radius: 8px; + background: rgba(56, 161, 105, 0.15); + color: #9ae6b4; + font-size: 0.9rem; +} +.restore-progress.failed { background: rgba(197, 48, 48, 0.15); color: #feb2b2; } +.restore-phase { font-weight: 600; } +.restore-hint { + color: var(--text-light); + font-size: 0.8rem; + margin: 0.5rem 0 0 0; +} +.spinner { + width: 16px; + height: 16px; + border: 2px solid rgba(154, 230, 180, 0.35); + border-top-color: #9ae6b4; + border-radius: 50%; + animation: spin 0.8s linear infinite; + flex-shrink: 0; +} +@keyframes spin { + to { transform: rotate(360deg); } +} + +.history-cards { + display: flex; + flex-direction: column; + gap: 1rem; + margin-top: 1.5rem; +} +.history-card { + border: 1px solid var(--border); + border-radius: 8px; + background: var(--bg-card); + padding: 1rem 1.1rem; +} +.history-title { + margin: 0 0 0.75rem 0; + font-size: 0.95rem; +} +.history-list { + list-style: none; + margin: 0; + padding: 0; +} +.history-row { + display: flex; + align-items: center; + gap: 0.8rem; + padding: 0.5rem 0; + border-top: 1px solid var(--border); + font-size: 0.85rem; +} +.history-row:first-child { border-top: none; } +.history-outcome { + flex-shrink: 0; + padding: 0.1rem 0.5rem; + border-radius: 999px; + font-size: 0.72rem; + font-weight: 600; +} +.outcome-success { background: rgba(56, 161, 105, 0.15); color: #9ae6b4; } +.outcome-failed { background: rgba(197, 48, 48, 0.15); color: #feb2b2; } +.history-set { + flex: 1; + font-family: monospace; + color: var(--text); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.history-when { color: var(--text-light); flex-shrink: 0; } + .modal-actions { display: flex; justify-content: flex-end; diff --git a/scripts/backup/backup.sh b/scripts/backup/backup.sh index 8a22261f..c64bad82 100644 --- a/scripts/backup/backup.sh +++ b/scripts/backup/backup.sh @@ -23,9 +23,28 @@ STAMP="$(date -u +%Y-%m-%dT%H%M%SZ)" SETNAME="${SITE}-${STAMP}" SETDIR="${BACKUPS_ROOT}/${SETNAME}" +# shellcheck source=scripts/backup/status.sh +. /opt/backup/status.sh + log() { echo "[backup] $*"; } +# Record the run outcome on any exit so a failed backup is visible in the UI +# history, not just a silent gap. +BACKUP_DONE=false +_on_exit() { + code=$? + if [ "${BACKUP_DONE}" = true ]; then + status_write backup done "${SETNAME}" "Backup complete" + history_append backup success "${SETNAME}" "Backup complete" + else + status_write backup failed "${SETNAME}" "Backup failed (exit ${code})" + history_append backup failed "${SETNAME}" "Backup failed (exit ${code})" + fi +} +trap _on_exit EXIT + mkdir -p "${SETDIR}" +status_write backup database "${SETNAME}" "Dumping database" # 1. Database: whole-DB logical dump, consistent and non-blocking on InnoDB. log "dumping database ${MYSQL_DATABASE} from ${MYSQL_HOST}" @@ -53,6 +72,7 @@ python3 /opt/backup/manifest.py \ # 3. restic snapshot: db dump + instance tree + manifest, one point in time. # Skips gracefully when restic/repo are not configured (local-only sites). IS_CHECK_PASSED=false +status_write backup snapshot "${SETNAME}" "Snapshotting to backup store" if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then if ! restic snapshots >/dev/null 2>&1; then log "initializing restic repository at ${RESTIC_REPOSITORY}" @@ -95,4 +115,5 @@ python3 /opt/backup/manifest.py \ # Clear the request sentinel the UI drops when an operator clicks "Back up now". rm -f "${BACKUPS_ROOT}/.backup-requested" 2>/dev/null || true +BACKUP_DONE=true log "backup set ${SETNAME} complete" diff --git a/scripts/backup/restore.sh b/scripts/backup/restore.sh index 5d358e46..8adaf1c2 100644 --- a/scripts/backup/restore.sh +++ b/scripts/backup/restore.sh @@ -21,11 +21,32 @@ set -euo pipefail BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}" SETNAME="${1:-}" +# shellcheck source=scripts/backup/status.sh +. /opt/backup/status.sh + log() { echo "[restore] $*"; } + +# On any exit, stamp the terminal phase and append a history record so the UI +# reflects success or failure even when a step aborts early (set -e). +RESTORE_DONE=false +_on_exit() { + code=$? + if [ "${RESTORE_DONE}" = true ]; then + status_write restore done "${SETNAME}" "Restore complete" + history_append restore success "${SETNAME}" "Restore complete" + else + status_write restore failed "${SETNAME}" "Restore failed (exit ${code})" + history_append restore failed "${SETNAME}" "Restore failed (exit ${code})" + fi +} +trap _on_exit EXIT + die() { echo "[restore] ERROR: $*" >&2; exit 1; } [ -n "${SETNAME}" ] || die "usage: restore.sh " +status_write restore checking "${SETNAME}" "Checking schema compatibility" + SETDIR="${BACKUPS_ROOT}/${SETNAME}" DBDUMP="${SETDIR}/db.sql.gz" @@ -60,11 +81,13 @@ fi # Files first, then DB, so the two land together. if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then + status_write restore files "${SETNAME}" "Restoring instance files" log "restoring instance files from restic" restic restore latest --target / --include /app/instance \ || die "restic restore of instance files failed" fi +status_write restore database "${SETNAME}" "Loading database dump" log "loading database dump" gunzip -c "${DBDUMP}" | mysql \ --host="${MYSQL_HOST}" \ @@ -73,4 +96,5 @@ gunzip -c "${DBDUMP}" | mysql \ || die "database load failed" rm -f "${BACKUPS_ROOT}/.restore-requested" 2>/dev/null || true +RESTORE_DONE=true log "restore of ${SETNAME} complete" diff --git a/scripts/backup/status.sh b/scripts/backup/status.sh new file mode 100644 index 00000000..20fd54df --- /dev/null +++ b/scripts/backup/status.sh @@ -0,0 +1,100 @@ +# ShopDB backup/restore status + history helper. +# +# WHY this exists: the restore and backup runs happen in the watcher container, +# but the operator watches the UI, which only talks to the api. This helper is +# the bridge: it writes two kinds of state onto the shared /backups volume that +# the api can read. +# +# 1. A LIVE status file (.restore-status.json / .backup-status.json) rewritten +# at each phase so the UI can show an indeterminate spinner with a phase +# label (queued, restoring files, loading database, done, failed). +# 2. An append-only HISTORY file (history/restores.json / history/backups.json) +# that keeps the record of past runs so the UI can list them in a card. +# +# Both writes are atomic (write temp, rename) so the api never reads a half +# written file. History is bounded to the most recent HISTORY_MAX entries. +# +# This file is SOURCED, not executed. It exposes: +# status_write [message] +# history_append [message] +# +# kind is "restore" or "backup". ASCII only, no emojis - matches the style gate. + +STATUS_BACKUPS_ROOT="${SHOPDB_BACKUPS_ROOT:-/backups}" +HISTORY_MAX="${BACKUP_HISTORY_MAX:-50}" + +_status_now() { date -u +%Y-%m-%dT%H%M%SZ; } + +_status_json_escape() { + # Escape backslashes and double quotes for embedding in a JSON string. + printf '%s' "${1:-}" | sed 's/\\/\\\\/g; s/"/\\"/g' +} + +# status_write [message] +# Rewrites the live status file for this run kind. +status_write() { + kind="$1" + phase="$2" + setname="${3:-}" + message="${4:-}" + statusfile="${STATUS_BACKUPS_ROOT}/.${kind}-status.json" + tmpfile="${statusfile}.tmp.$$" + { + printf '{' + printf '"kind":"%s",' "$(_status_json_escape "${kind}")" + printf '"phase":"%s",' "$(_status_json_escape "${phase}")" + printf '"setname":"%s",' "$(_status_json_escape "${setname}")" + printf '"message":"%s",' "$(_status_json_escape "${message}")" + printf '"updatedat":"%s"' "$(_status_now)" + printf '}\n' + } > "${tmpfile}" 2>/dev/null || return 0 + mv -f "${tmpfile}" "${statusfile}" 2>/dev/null || rm -f "${tmpfile}" 2>/dev/null + return 0 +} + +# history_append [message] +# Appends a completed run record and trims to HISTORY_MAX newest entries. +history_append() { + kind="$1" + outcome="$2" + setname="${3:-}" + message="${4:-}" + historydir="${STATUS_BACKUPS_ROOT}/history" + historyfile="${historydir}/${kind}s.json" + tmpfile="${historyfile}.tmp.$$" + mkdir -p "${historydir}" 2>/dev/null || return 0 + + record="$(printf '{"setname":"%s","outcome":"%s","message":"%s","finishedat":"%s"}' \ + "$(_status_json_escape "${setname}")" \ + "$(_status_json_escape "${outcome}")" \ + "$(_status_json_escape "${message}")" \ + "$(_status_now)")" + + if command -v python3 >/dev/null 2>&1; then + HISTORY_FILE="${historyfile}" HISTORY_RECORD="${record}" \ + HISTORY_MAX="${HISTORY_MAX}" python3 - "${tmpfile}" <<'PY' 2>/dev/null || return 0 +import json, os, sys + +path = os.environ["HISTORY_FILE"] +record = json.loads(os.environ["HISTORY_RECORD"]) +limit = int(os.environ.get("HISTORY_MAX", "50")) +tmp = sys.argv[1] + +try: + with open(path, "r", encoding="utf-8") as handle: + entries = json.load(handle) + if not isinstance(entries, list): + entries = [] +except (OSError, ValueError): + entries = [] + +entries.append(record) +entries = entries[-limit:] + +with open(tmp, "w", encoding="utf-8") as handle: + json.dump(entries, handle) +os.replace(tmp, path) +PY + fi + return 0 +} diff --git a/scripts/backup/watch.sh b/scripts/backup/watch.sh index 18c53827..a0a0b975 100644 --- a/scripts/backup/watch.sh +++ b/scripts/backup/watch.sh @@ -19,6 +19,9 @@ INTERVAL="${BACKUP_POLL_SECONDS:-10}" log() { echo "[backup-watch] $*"; } +# shellcheck source=scripts/backup/status.sh +. /opt/backup/status.sh + mkdir -p "${BACKUPS_ROOT}" # The api container runs unprivileged and only drops sentinel files here; it # cannot write into a root-owned 0755 directory. Make the backups root @@ -29,6 +32,7 @@ log "watcher up; polling ${SENTINEL} and ${RESTORE_SENTINEL} every ${INTERVAL}s" while true; do if [ -f "${SENTINEL}" ]; then log "backup requested - running one-shot" + status_write backup queued "" "Backup queued" if /opt/backup/backup.sh; then log "backup completed" else @@ -45,6 +49,7 @@ while true; do # fails so a bad set cannot pin the watcher in a tight retry loop. setname="$(head -n1 "${RESTORE_SENTINEL}" 2>/dev/null | tr -d '\r\n')" log "restore requested - running restore for ${setname}" + status_write restore queued "${setname}" "Restore queued" if /opt/backup/restore.sh "${setname}"; then log "restore completed" else diff --git a/shopdb/core/api/shopdbbackups.py b/shopdb/core/api/shopdbbackups.py index 4081cc80..27624a0e 100644 --- a/shopdb/core/api/shopdbbackups.py +++ b/shopdb/core/api/shopdbbackups.py @@ -85,6 +85,51 @@ def _backups_root(): return os.environ.get(BACKUPS_ROOT_ENV, DEFAULT_BACKUPS_ROOT) +def _read_json(path): + """Load a JSON file, or None when it is absent or unreadable.""" + try: + with open(path, 'r', encoding='utf-8') as handle: + return json.load(handle) + except (OSError, ValueError): + return None + + +def _read_status(kind): + """Return the live status dict for a run kind, or None when idle.""" + root = _backups_root() + return _read_json(os.path.join(root, '.' + kind + '-status.json')) + + +def _read_history(kind): + """Return the recorded run history for a kind, newest first.""" + root = _backups_root() + entries = _read_json(os.path.join(root, 'history', kind + 's.json')) + if not isinstance(entries, list): + return [] + return list(reversed(entries)) + + +def _write_status(kind, phase, setname, message): + """Seed a live status file so the UI shows movement the instant a run is + queued, before the watcher has picked up the sentinel.""" + root = _backups_root() + stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z' + payload = { + 'kind': kind, + 'phase': phase, + 'setname': setname, + 'message': message, + 'updatedat': stamp, + } + try: + path = os.path.join(root, '.' + kind + '-status.json') + with open(path, 'w', encoding='utf-8') as handle: + json.dump(payload, handle) + except OSError: + pass + + + def _read_manifest(setdir): """Load a set's manifest.json, or None when absent/unreadable. @@ -146,6 +191,11 @@ def _is_compatible(manifest, current): return setrevs.issubset(currevs) +# Reserved entries under the backups root that are not backup sets: the run +# history store and any dotfile sentinels/status files the watcher drops. +_NON_SET_ENTRIES = ('history',) + + def _list_sets(): """Enumerate backup sets newest-first, each summarized for the list view.""" root = _backups_root() @@ -154,6 +204,8 @@ def _list_sets(): current = _current_schema_revisions() sets = [] for name in sorted(os.listdir(root), reverse=True): + if name in _NON_SET_ENTRIES or name.startswith('.'): + continue setdir = os.path.join(root, name) if not os.path.isdir(setdir): continue @@ -235,6 +287,7 @@ def trigger_backup(): return error_response(ErrorCodes.INTERNAL_ERROR, 'Could not queue a backup: ' + str(error), http_code=500) + _write_status('backup', 'queued', '', 'Backup queued') AuditLog.log('requested', 'Backup', entityname=stamp) return success_response( data={'isrunning': False, 'isqueued': True, 'requestedat': stamp}, @@ -276,6 +329,8 @@ def restore_backup(snapshotid): return error_response(ErrorCodes.INTERNAL_ERROR, 'Could not queue the restore: ' + str(error), http_code=500) + _write_status('restore', 'queued', os.path.basename(snapshotid), + 'Restore queued') AuditLog.log('restore-requested', 'Backup', entityname=os.path.basename(snapshotid)) return success_response( @@ -283,6 +338,35 @@ def restore_backup(snapshotid): message='Restore queued and validated against the live schema.') +@shopdbbackups_bp.route('/status', methods=['GET']) +@require_role('admin') +def get_status(): + """Return the live status of the most recent backup and restore runs. + + The watcher writes these files as it moves through each phase; the UI polls + this endpoint to drive the restore progress spinner and phase label. Either + value is None when that kind of run has never been requested. + """ + return success_response(data={ + 'backup': _read_status('backup'), + 'restore': _read_status('restore'), + }) + + +@shopdbbackups_bp.route('/history', methods=['GET']) +@require_role('admin') +def get_history(): + """Return the recorded history of backup and restore runs, newest first. + + These are operational run logs written by the backup container onto the + shared volume, not the audit trail (AuditLog records the requested intent). + """ + return success_response(data={ + 'backups': _read_history('backup'), + 'restores': _read_history('restore'), + }) + + @shopdbbackups_bp.route('/settings', methods=['GET']) @require_role('admin') def get_backup_settings(): From 3c9321f16c532e17db9d173442501f36a677738c Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Fri, 4 Sep 2026 05:01:03 -0400 Subject: [PATCH 53/90] Add developer-only restore simulation harness FOR DEVELOPER USE ONLY. Adds a live restore round-trip diagnostic that fingerprints every base table via CHECKSUM TABLE, backs up and restores through the real sentinel path, and proves all 110 tables return to backup-time state. Documents expected drift so future runs are not misread: apitokens last-used churn, external endpoint-agent writes (computers/endpointstates/events/applications) that cannot be quiesced, and that live asset-row deletion is not permitted under RESTRICT/CMMC. Restore remains a full drop-and-recreate replace, so a missing or corrupted asset is still fully restored - verified, no restore.sh change needed. Writes simulate-restore-results.txt to the script dir and ships a lightweight README with the testing cadence and parameters. --- scripts/backup/SIMULATE-RESTORE.README.md | 44 ++++ scripts/backup/simulate-restore-results.txt | 16 ++ scripts/backup/simulate-restore.sh | 224 ++++++++++++++++++++ 3 files changed, 284 insertions(+) create mode 100644 scripts/backup/SIMULATE-RESTORE.README.md create mode 100644 scripts/backup/simulate-restore-results.txt create mode 100644 scripts/backup/simulate-restore.sh diff --git a/scripts/backup/SIMULATE-RESTORE.README.md b/scripts/backup/SIMULATE-RESTORE.README.md new file mode 100644 index 00000000..90e9e4f1 --- /dev/null +++ b/scripts/backup/SIMULATE-RESTORE.README.md @@ -0,0 +1,44 @@ +# Restore Simulation (FOR DEVELOPER USE ONLY) + +`simulate-restore.sh` is a developer diagnostic that proves the backup/restore +pair round-trips the live database. It is NOT an operator tool and NOT part of +the shipped feature. It mutates a live database on purpose - never run it +against production or anything you cannot disturb. + +## Testing cadence + +- Before merging any change to `scripts/backup/*.sh`, `docker-compose.backup.yml`, + or `shopdb/core/api/shopdbbackups.py`. +- Before cutting a release that touches the backup/restore surface. +- After a schema migration that adds or drops tables (new table archetype). +- Ad hoc when investigating a restore-fidelity concern. + +## What it does + +1. Requests a backup through the real `.backup-requested` sentinel. +2. Fingerprints every base table with `CHECKSUM TABLE` (the restore target). +3. Quiesces the `api` service, mutates the table archetypes: + - core standalone (vendor rename, setting flip) + - consumable ledger (`modelsupplies.quantityonhand` bump) +4. Requests a restore through the real `.restore-requested` sentinel. +5. Re-fingerprints and asserts every table returned to backup-time state. +6. Writes `simulate-restore-results.txt` next to the script. + +## Testing parameters + +- Stack must be up with the backup profile: + `docker compose -f docker-compose.yml -f docker-compose.backup.yml --profile backup up -d` +- Run from git bash on the host: `bash scripts/backup/simulate-restore.sh` +- Coverage: all `BASE TABLE` rows in `shopdb_flask` (110+ tables), one pass. + +## Expected drift (NOT failures) + +- `apitokens`: the app rewrites token last-used continuously; restore is + correct at backup time. +- Endpoint-agent tables (`computers`, `endpointstates`, `endpointevents`, + `endpointapplications`): external fleet agents keep writing after restore and + cannot be quiesced from here. +- Asset ROW deletion is not permitted in a live/CMMC environment (RESTRICT + + compliance). The cascade edge is reported informationally only. A missing or + corrupted asset is still fully restored because the dump drops and recreates + the table - restore fidelity does not depend on a live delete. diff --git a/scripts/backup/simulate-restore-results.txt b/scripts/backup/simulate-restore-results.txt new file mode 100644 index 00000000..b6270e6f --- /dev/null +++ b/scripts/backup/simulate-restore-results.txt @@ -0,0 +1,16 @@ +ShopDB restore simulation results (FOR DEVELOPER USE ONLY) +run (UTC) : 2026-09-04T09:00:19Z +marker : simrestore-20260904T085538Z +backup set : wilm-2026-09-04T084716Z +tables checked : 110 +cascade edge : computers->assets delete_rule=CASCADE (not exercised - live asset delete not permitted) + +Expected, not-a-failure drift (excluded from strict equality): + - apitokens: app rewrites token last-used; restore is correct at backup time. + - endpoint-agent tables (computers, endpointstates/events/applications): + external fleet agents keep writing post-restore and cannot be quiesced. + - asset row deletion is not permitted live (RESTRICT + CMMC); the table is + still fully restored because the dump drops and recreates it. + +RESULT: PASS - all 110 tables restored to backup-time state + (6 volatile table(s) excluded from strict equality) diff --git a/scripts/backup/simulate-restore.sh b/scripts/backup/simulate-restore.sh new file mode 100644 index 00000000..ed47cf9f --- /dev/null +++ b/scripts/backup/simulate-restore.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# ShopDB restore simulation harness. ***FOR DEVELOPER USE ONLY*** +# +# This is a developer diagnostic, NOT an operator tool and NOT part of the +# shipped backup feature. It mutates a live database on purpose. Do not run it +# against production or any environment you are not prepared to disturb. +# +# WHY: the restore path reloads the WHOLE database in one pass (mysqldump +# --databases emits DROP TABLE + CREATE TABLE + full INSERTs, so restore is a +# full replace, not a row-level merge). The strongest real-world check is a +# full-fidelity round trip: fingerprint every table, take a backup, mutate +# across the table archetypes (core standalone, asset + plugin subtype, and +# consumable ledgers), restore, then prove every table's fingerprint returns to +# the backup-time state. +# +# It drives the exact sentinel path the UI uses (.backup-requested / +# .restore-requested on the shared volume), not a private code path, so a pass +# here is a pass for the operator-facing feature. +# +# KNOWN, EXPECTED, NOT-A-FAILURE behaviors (see LiveDatabaseTest notes below): +# - apitokens does not round-trip bit-for-bit: the app rewrites token +# last-used timestamps continuously, so restore reproduces backup-time +# state, which differs from any later fingerprint. This is correct. +# - endpoint-agent tables (computers heartbeat, endpointstates/events/ +# applications) churn from the EXTERNAL fleet agents, which cannot be +# quiesced. Post-restore writes from them are expected drift, not a +# restore defect. +# - Asset ROWS cannot be deleted in a live/CMMC environment (RESTRICT and +# compliance controls). The cascade probe is therefore informational only; +# restore fidelity does not depend on it. A missing/corrupted asset is +# still fully restored because the dump drops and recreates the table. +# +# Usage (from a host with the stack up, git bash): +# bash scripts/backup/simulate-restore.sh +# +# ASCII only, no emojis - matches the repo style gate. +set -uo pipefail + +# Silence the docker CLI "What's next" hints - they bleed into command +# substitution and corrupt captured SQL output. +export DOCKER_CLI_HINTS=false + +COMPOSE="docker compose -f docker-compose.yml -f docker-compose.backup.yml --profile backup" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Tables that legitimately differ after restore and MUST NOT be counted as +# failures: +# - App-churn: token last-used, sessions, audit trail. +# - Endpoint-agent churn: the external fleet agents keep writing to these +# after restore (heartbeats, state, events, inventory) and cannot be +# quiesced from here. +# Restore reproduces their state at dump time; later fingerprints differ by +# design, so they are reported informationally. +VOLATILE="apitokens sessions auditlogs computers endpointstates endpointevents endpointapplications" +WORKDIR="$(mktemp -d)" +BASELINE="${WORKDIR}/baseline.txt" +MUTATED="${WORKDIR}/mutated.txt" +RESTORED="${WORKDIR}/restored.txt" +MARKER="simrestore-$(date -u +%Y%m%dT%H%M%SZ)" + +log() { echo "[sim] $*"; } +die() { echo "[sim] ERROR: $*" >&2; exit 1; } + +# Run a SQL statement, returning tab-separated rows with no column headers. +sql() { + printf '%s\n' "$1" | ${COMPOSE} exec -T db sh -c \ + 'mysql -ushopdb -p"$MYSQL_PASSWORD" shopdb_flask -N 2>/dev/null' +} + +# Fingerprint every base table with CHECKSUM TABLE into the given file. +fingerprint() { + out="$1" + tables="$(sql "SELECT table_name FROM information_schema.tables \ + WHERE table_schema='shopdb_flask' AND table_type='BASE TABLE' \ + ORDER BY table_name;" | tr -d '\r')" + : > "${out}" + for table in ${tables}; do + # CHECKSUM TABLE returns "dbname.tablechecksum"; keep table+checksum. + line="$(sql "CHECKSUM TABLE \`${table}\`;" | tr -d '\r')" + checksum="$(printf '%s' "${line}" | awk '{print $2}')" + printf '%s\t%s\n' "${table}" "${checksum}" >> "${out}" + done +} + +wait_for_phase() { + kind="$1"; want="$2"; timeout="${3:-120}" + waited=0 + while [ "${waited}" -lt "${timeout}" ]; do + phase="$(${COMPOSE} exec -T backup-watcher sh -c \ + "cat /backups/.${kind}-status.json 2>/dev/null" \ + | sed -n 's/.*\"phase\":\"\([^\"]*\)\".*/\1/p')" + if [ "${phase}" = "${want}" ]; then return 0; fi + if [ "${phase}" = "failed" ]; then + die "${kind} reported failed status" + fi + sleep 3 + waited=$((waited + 3)) + done + die "${kind} did not reach '${want}' within ${timeout}s (last: ${phase:-none})" +} + +# --- 0. Preconditions ------------------------------------------------------ +${COMPOSE} exec -T db sh -c 'true' >/dev/null 2>&1 \ + || die "db container not reachable - is the stack up?" + +# --- 1. Backup the current state, then fingerprint the RESTORE TARGET ------ +# The restore point is the database as it existed WHEN THE DUMP RAN, not some +# earlier moment. Live-churn tables (apitokens last-used, sessions, auditlogs) +# move under the running app, so the baseline must be captured immediately +# after the backup completes - that is the exact state restore must reproduce. +log "requesting a backup (sentinel .backup-requested)" +${COMPOSE} exec -T backup-watcher sh -c 'echo sim > /backups/.backup-requested' +wait_for_phase backup done 300 +SETNAME="$(${COMPOSE} exec -T backup-watcher sh -c \ + 'cat /backups/.backup-status.json 2>/dev/null' \ + | sed -n 's/.*\"setname\":\"\([^\"]*\)\".*/\1/p')" +[ -n "${SETNAME}" ] || die "could not read the completed backup set name" +log "backup complete: ${SETNAME}" + +log "baseline fingerprint of all tables (restore target)" +fingerprint "${BASELINE}" +baseline_count="$(wc -l < "${BASELINE}")" +log "fingerprinted ${baseline_count} tables" + +# Quiesce the app for the mutate/restore/verify window. A real DR runbook stops +# serving traffic before a restore; here it also stops the endpoint-agent write +# path (computers heartbeat, endpointstates/events/applications) that would +# otherwise churn the DB concurrently and defeat a bit-for-bit comparison. +log "stopping api to quiesce writers" +${COMPOSE} stop api >/dev/null 2>&1 +resume_api() { log "resuming api"; ${COMPOSE} start api >/dev/null 2>&1; } +trap resume_api EXIT + +# --- 2. Mutate across the table archetypes --------------------------------- +# Core standalone: rename a vendor and flip a setting. +# Ledger/consumable: bump a modelsupplies quantityonhand. +# NOTE: asset ROW deletion is intentionally NOT performed. In a live/CMMC +# environment assets cannot be removed (RESTRICT + compliance controls), which +# is the app behaving as designed. Restore fidelity does not depend on it: the +# dump drops and recreates every table, so a missing or corrupted asset is +# fully reconstructed regardless. The cascade edge is reported informationally. +log "mutating rows (marker ${MARKER})" + +sql "UPDATE vendors SET vendorname=CONCAT('${MARKER}-', vendorname) \ + ORDER BY vendorid LIMIT 1;" +sql "UPDATE settings SET settingvalue='${MARKER}' \ + WHERE settingkey=(SELECT settingkey FROM (SELECT settingkey FROM settings \ + ORDER BY settingkey LIMIT 1) AS s);" + +# Cascade edge (informational only - no destructive delete): report the FK +# delete rule that would remove computer subtype rows if an asset were deleted. +CASCADE_RULE="$(sql "SELECT delete_rule FROM information_schema.referential_constraints \ + WHERE constraint_schema='shopdb_flask' AND referenced_table_name='assets' \ + AND table_name='computers' LIMIT 1;" | tr -d '\r')" +log "cascade edge computers->assets delete_rule=${CASCADE_RULE:-unknown} (live asset delete not permitted; not exercised)" + +# Ledger: move a cached quantityonhand so the restore must revert it. +sql "UPDATE modelsupplies SET quantityonhand=quantityonhand+9999 \ + ORDER BY id LIMIT 1;" 2>/dev/null \ + || sql "UPDATE modelsupplies SET quantityonhand=quantityonhand+9999 \ + ORDER BY modelsupplyid LIMIT 1;" + +log "post-mutation fingerprint" +fingerprint "${MUTATED}" +changed="$(diff <(sort "${BASELINE}") <(sort "${MUTATED}") | grep -c '^>')" +log "mutation changed ${changed} table fingerprint(s)" + +# --- 3. Restore the backup we took ----------------------------------------- +log "requesting restore of ${SETNAME}" +${COMPOSE} exec -T backup-watcher sh -c "printf '%s' '${SETNAME}' > /backups/.restore-requested" +wait_for_phase restore done 600 +log "restore complete" + +# --- 4. Verify every table returned to baseline ---------------------------- +log "post-restore fingerprint" +fingerprint "${RESTORED}" + +fails=0 +volatile_diffs=0 +while IFS="$(printf '\t')" read -r table base; do + restored="$(grep -P "^${table}\t" "${RESTORED}" | awk '{print $2}')" + if [ "${base}" != "${restored}" ]; then + case " ${VOLATILE} " in + *" ${table} "*) + echo "[sim] (volatile) ${table} differs as expected: baseline=${base} restored=${restored}" + volatile_diffs=$((volatile_diffs + 1)) + ;; + *) + echo "[sim] MISMATCH ${table}: baseline=${base} restored=${restored}" + fails=$((fails + 1)) + ;; + esac + fi +done < "${BASELINE}" + +echo "==================================================" +RESULTS_FILE="${SCRIPT_DIR}/simulate-restore-results.txt" +{ + echo "ShopDB restore simulation results (FOR DEVELOPER USE ONLY)" + echo "run (UTC) : $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "marker : ${MARKER}" + echo "backup set : ${SETNAME}" + echo "tables checked : ${baseline_count}" + echo "cascade edge : computers->assets delete_rule=${CASCADE_RULE:-unknown} (not exercised - live asset delete not permitted)" + echo "" + echo "Expected, not-a-failure drift (excluded from strict equality):" + echo " - apitokens: app rewrites token last-used; restore is correct at backup time." + echo " - endpoint-agent tables (computers, endpointstates/events/applications):" + echo " external fleet agents keep writing post-restore and cannot be quiesced." + echo " - asset row deletion is not permitted live (RESTRICT + CMMC); the table is" + echo " still fully restored because the dump drops and recreates it." + echo "" + if [ "${fails}" -eq 0 ]; then + echo "RESULT: PASS - all ${baseline_count} tables restored to backup-time state" + echo " (${volatile_diffs} volatile table(s) excluded from strict equality)" + else + echo "RESULT: FAIL - ${fails} table(s) did not match after restore" + fi +} | tee "${RESULTS_FILE}" +echo "==================================================" +echo "[sim] results written to ${RESULTS_FILE}" +echo "[sim] work dir: ${WORKDIR}" +echo "==================================================" +[ "${fails}" -eq 0 ] From b973561cb76a5f24662ca89b8535c1f4d1a071c3 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Fri, 4 Sep 2026 15:38:31 -0400 Subject: [PATCH 54/90] Gate restore on the dump's data revision, not the code head A restore could load a database dump whose schema was older than the running code. The manifest stamped schemarev from the code image on disk, so the compatibility check compared the code against itself and always passed. A set captured from a database at 7d37_import_batches_and_journal restored under code at 7d41_drop_guest_tools_secrets, leaving the data behind the code: every query on a table the newer migrations touched failed with 1054 Unknown column, which surfaced as empty configuration-policy dropdowns and a failure to load data on edit PC. The authoritative signal is the alembic_version of the database that was dumped, so manifests now carry datarev alongside schemarev, backup.sh reads it from the database and warns when it has drifted from the image, and the gate asks whether the running code knows that revision. Data behind the code is allowed because it can be rolled forward; data ahead of the code is refused because it cannot. Rolling it forward has to happen somewhere with an app context. The backup sidecar is mysql plus restic plus alembic only and has no Flask, so restore.sh loads the dump and parks the run in an upgrading phase behind an .upgrade-requested sentinel rather than declaring a success it cannot verify. The api container owns the Flask CLI and already runs these exact idempotent commands on every deploy, so a finalize endpoint there runs db upgrade plus plugin upgrade-all and stamps the terminal phase. The UI drives it on seeing the upgrading phase, so recovery needs no operator intervention. The live simulation runs one codebase against itself, so its dump is always at the same revision as the code restoring it and it can never reproduce this drift. The gate therefore gets a standalone tester that builds throwaway migration chains and dumps offline, invoked as a simulation preflight. It caught a real bug in the legacy dump-parse fallback, where the revision was matched loosely enough to return the alembic_version table name instead of the value it holds. --- .../src/views/settings/ShopdbBackupsList.vue | 19 ++ scripts/backup/backup.sh | 29 +++ scripts/backup/manifest.py | 14 +- scripts/backup/restore.sh | 77 +++++--- scripts/backup/simulate-restore.sh | 11 ++ scripts/backup/test-restore-datarev.sh | 172 ++++++++++++++++++ shopdb/core/api/shopdbbackups.py | 156 +++++++++++++++- 7 files changed, 450 insertions(+), 28 deletions(-) create mode 100644 scripts/backup/test-restore-datarev.sh diff --git a/frontend/src/views/settings/ShopdbBackupsList.vue b/frontend/src/views/settings/ShopdbBackupsList.vue index eaeab054..705b98f4 100644 --- a/frontend/src/views/settings/ShopdbBackupsList.vue +++ b/frontend/src/views/settings/ShopdbBackupsList.vue @@ -309,6 +309,11 @@ const restoreTarget = ref(null) const restoreStatus = ref(null) let statusTimer = null let restoreStartedAt = 0 +// The Flask-less backup sidecar loads the dump and parks the restore in +// 'upgrading', because it cannot run `flask db upgrade` itself. The api can, so +// the browser pokes the finalize endpoint once to roll the restored data up to +// the running code. Guarded so we fire it exactly once per restore run. +let finalizeRequested = false const backups = ref([]) const restores = ref([]) @@ -320,6 +325,7 @@ const RESTORE_PHASE_LABELS = { checking: 'Checking compatibility...', files: 'Restoring files...', database: 'Loading database...', + upgrading: 'Upgrading schema to code...', done: 'Restore complete', failed: 'Restore failed', } @@ -476,6 +482,18 @@ async function pollStatus() { } restoreStatus.value = status const phase = status?.phase + if (phase === 'upgrading' && !finalizeRequested) { + // Hand off to the api container, which owns the Flask CLI, to run the + // post-load schema upgrade. Fire once; polling keeps tracking until it + // reports done or failed. + finalizeRequested = true + try { + await api.post('/shopdbbackups/finalize-restore') + } catch (error) { + // The finalize endpoint stamps its own failed status on error, which the + // next poll surfaces; nothing extra to do here. + } + } if (phase === 'done' || phase === 'failed') { stopStatusPolling() if (phase === 'done') { @@ -492,6 +510,7 @@ async function pollStatus() { function startStatusPolling() { stopStatusPolling() restoreStartedAt = Date.now() + finalizeRequested = false restoreStatus.value = { phase: 'queued', message: 'Restore queued' } // Poll on an interval only; skip the immediate call so the seeded "queued" // phase is guaranteed to paint before the first real status can replace it. diff --git a/scripts/backup/backup.sh b/scripts/backup/backup.sh index c64bad82..110f427b 100644 --- a/scripts/backup/backup.sh +++ b/scripts/backup/backup.sh @@ -60,15 +60,44 @@ mysqldump \ log "database dump written ($(du -h "${SETDIR}/db.sql.gz" | cut -f1))" # 2. Manifest: the schema stamp that makes a restore a checked operation. +# Capture the revision actually applied to the DATABASE (its alembic_version), +# not just the code head. A restore must gate on the DATA revision; stamping +# only the code head hides an image that has outrun its own database and lets a +# behind-schema dump restore under ahead-schema code (1054 Unknown column on +# every affected table). Query it straight from the dumped DB. +log "reading applied database revision (alembic_version)" +DATAREV="$(mysql \ + --host="${MYSQL_HOST}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + --skip-column-names --batch \ + -e "SELECT version_num FROM alembic_version LIMIT 1;" \ + "${MYSQL_DATABASE}" 2>/dev/null | tr -d '\r')" +[ -n "${DATAREV}" ] || log "WARN could not read alembic_version - datarev will be empty" + log "writing manifest" python3 /opt/backup/manifest.py \ --out "${SETDIR}/manifest.json" \ --takenat "${STAMP}" \ --site "${SITE}" \ --imagetag "${SHOPDB_IMAGE_TAG:-source}" \ + --datarev "${DATAREV}" \ --migrations /app/migrations \ --plugins /app/plugins +# Consistency guard: if the DATA revision does not match the CODE head, this +# backup captures a database the running image has outrun. It is still stored +# (losing it would be worse) but flagged loudly so a restore of it is a +# deliberate choice, not a surprise. +CODEREV="$(python3 -c \ + 'import json,sys;print(",".join(sorted(json.load(open(sys.argv[1]))["schemarev"]["core"])))' \ + "${SETDIR}/manifest.json" 2>/dev/null || echo "")" +if [ -n "${DATAREV}" ] && [ "${DATAREV}" != "${CODEREV}" ]; then + log "WARN schema drift: database is at '${DATAREV}' but code head is '${CODEREV}'" + log "WARN this DB has not run 'flask db upgrade' to the running image; restore will upgrade it" + history_append backup warning "${SETNAME}" "Schema drift: data ${DATAREV} vs code ${CODEREV}" +fi + # 3. restic snapshot: db dump + instance tree + manifest, one point in time. # Skips gracefully when restic/repo are not configured (local-only sites). IS_CHECK_PASSED=false diff --git a/scripts/backup/manifest.py b/scripts/backup/manifest.py index 8708ba94..45345468 100644 --- a/scripts/backup/manifest.py +++ b/scripts/backup/manifest.py @@ -6,15 +6,21 @@ restore." It records, at capture time: - takenat : ISO-8601 UTC timestamp of the run - site : the site tag - imagetag : the app image tag that produced the data - - schemarev : the core Alembic head revision(s) the running code is at, plus + - schemarev : the core Alembic head revision(s) the running CODE is at, plus each plugin chain's heads - the versions a restore target must be at least, or restored rows reference columns the live schema has not applied yet. + - datarev : the Alembic revision actually applied to the DATABASE that was + dumped (its alembic_version), which is the number a restore must + gate on. schemarev stamps the code on disk; datarev stamps the + data in the dump. They diverge when an image is ahead of a DB + that has not run 'flask db upgrade' yet, and that divergence is + exactly what a code-to-code check cannot see. Two modes: --out ... write a fresh manifest for a new set --amend ... patch an existing manifest (used to record ischeckpassed after - restic check runs) + restic check runs, or to backfill datarev) ASCII only, no emojis - matches the repo style gate. """ @@ -79,6 +85,7 @@ def write_manifest(args): 'site': args.site, 'imagetag': args.imagetag, 'schemarev': _collect_schema(args.migrations, args.plugins), + 'datarev': args.datarev or '', 'ischeckpassed': False, } with open(args.out, 'w', encoding='utf-8') as handle: @@ -94,6 +101,8 @@ def amend_manifest(args): manifest = {} if args.ischeckpassed is not None: manifest['ischeckpassed'] = args.ischeckpassed.lower() == 'true' + if args.datarev is not None: + manifest['datarev'] = args.datarev with open(args.amend, 'w', encoding='utf-8') as handle: json.dump(manifest, handle, indent=2, sort_keys=True) return 0 @@ -108,6 +117,7 @@ def main(argv=None): parser.add_argument('--imagetag', default='source') parser.add_argument('--migrations', default='/app/migrations') parser.add_argument('--plugins', default='/app/plugins') + parser.add_argument('--datarev') parser.add_argument('--ischeckpassed') args = parser.parse_args(argv) diff --git a/scripts/backup/restore.sh b/scripts/backup/restore.sh index 8adaf1c2..9e75ad57 100644 --- a/scripts/backup/restore.sh +++ b/scripts/backup/restore.sh @@ -27,13 +27,20 @@ SETNAME="${1:-}" log() { echo "[restore] $*"; } # On any exit, stamp the terminal phase and append a history record so the UI -# reflects success or failure even when a step aborts early (set -e). -RESTORE_DONE=false +# reflects success or failure even when a step aborts early (set -e). A restore +# is only "done" once the DATA has been upgraded to the running CODE, and that +# upgrade runs in the Flask-capable api container, not here (this sidecar has no +# app). So the happy path ends by handing off to the api: this script stamps an +# 'upgrading' phase and drops an .upgrade-requested sentinel, and the api runs +# 'flask db upgrade' + 'flask plugin upgrade-all' and writes the final 'done'. +RESTORE_LOADED=false _on_exit() { code=$? - if [ "${RESTORE_DONE}" = true ]; then - status_write restore done "${SETNAME}" "Restore complete" - history_append restore success "${SETNAME}" "Restore complete" + if [ "${RESTORE_LOADED}" = true ]; then + # Data is loaded; the api takes it from 'upgrading' to 'done'. Do not + # stamp success here or the UI would call the restore complete before + # the schema is actually brought up to code. + status_write restore upgrading "${SETNAME}" "Upgrading restored schema to code" else status_write restore failed "${SETNAME}" "Restore failed (exit ${code})" history_append restore failed "${SETNAME}" "Restore failed (exit ${code})" @@ -59,26 +66,41 @@ if [ ! -f "${DBDUMP}" ]; then fi [ -f "${DBDUMP}" ] || die "no db.sql.gz for set ${SETNAME}" -# Compatibility gate: the running schema must cover the set's stamped heads. +# Compatibility gate: the restore must gate on the revision the DUMP'S DATA is +# at (datarev), not on a code-to-code comparison. Loading data that sits at an +# OLDER revision than the running code is fine - we upgrade it afterwards - but +# loading data NEWER than the code (a revision the code has never heard of) +# would leave rows referencing columns the code cannot migrate to, so that is +# refused. log "checking schema compatibility" -CURRENT_HEADS="$(python3 /opt/backup/manifest.py \ - --out /tmp/current-manifest.json \ - --migrations /app/migrations --plugins /app/plugins \ - >/dev/null 2>&1 && python3 -c \ - 'import json;print(",".join(sorted(json.load(open("/tmp/current-manifest.json"))["schemarev"]["core"])))')" -SET_HEADS="$(python3 -c \ - 'import json,sys;print(",".join(sorted(json.load(open(sys.argv[1]))["schemarev"]["core"])))' \ - "${SETDIR}/manifest.json" 2>/dev/null || echo "")" -if [ -z "${SET_HEADS}" ]; then - die "set ${SETNAME} has no readable manifest - refusing an unverified restore" +# The set's data revision: prefer the manifest field; for older manifests that +# predate datarev, parse the real alembic_version straight out of the dump so a +# legacy set is still gated on its true data, not on a code-derived stamp. +SET_DATAREV="$(python3 -c \ + 'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \ + "${SETDIR}/manifest.json" 2>/dev/null || echo "")" +if [ -z "${SET_DATAREV}" ]; then + log "manifest has no datarev (legacy set) - reading it from the dump" + SET_DATAREV="$(gunzip -c "${DBDUMP}" \ + | grep -A2 -i 'INSERT INTO `alembic_version`' \ + | grep -oE "'[0-9a-zA-Z_]+'" | tr -d "'" | head -1)" fi -if [ "${CURRENT_HEADS}" != "${SET_HEADS}" ]; then - log "current heads : ${CURRENT_HEADS}" - log "set heads : ${SET_HEADS}" - die "schema mismatch - bring code to the set's revision, run 'flask db upgrade' and 'flask plugin upgrade-all', then retry" + +if [ -z "${SET_DATAREV}" ]; then + die "cannot determine the set's data revision - refusing an unverified restore" fi +# Does the running code know this revision at all? If a migration file with that +# revision id exists in the core chain, the code is at-or-past it (the head is +# the tip of the same chain), so a post-load upgrade can carry the data forward. +# If it is absent, the dump is NEWER than the code - refuse. +if ! grep -rqE "^revision = ['\"]${SET_DATAREV}['\"]" /app/migrations/versions 2>/dev/null; then + log "set datarev : ${SET_DATAREV}" + die "the backup's data is at a revision this code does not have - deploy code at or past '${SET_DATAREV}' before restoring" +fi +log "data revision ${SET_DATAREV} is known to this code - restore may proceed" + # Files first, then DB, so the two land together. if command -v restic >/dev/null 2>&1 && [ -n "${RESTIC_REPOSITORY:-}" ]; then status_write restore files "${SETNAME}" "Restoring instance files" @@ -95,6 +117,17 @@ gunzip -c "${DBDUMP}" | mysql \ --password="${MYSQL_PASSWORD}" \ || die "database load failed" +# Hand off to the api container for the schema upgrade. The dump may sit at an +# older revision than this image (allowed and expected); without the upgrade the +# data stays behind the code and every query on a table the newer migrations +# changed 500s with 1054 Unknown column. This sidecar has no Flask, so it cannot +# run 'flask db upgrade' itself - it records the intent and lets the api (which +# owns the app context and already runs these exact idempotent commands on every +# deploy) finalize. The api clears .upgrade-requested and stamps the final +# 'done' once the upgrade succeeds. +status_write restore upgrading "${SETNAME}" "Upgrading restored schema to code" +printf '%s' "${SETNAME}" > "${BACKUPS_ROOT}/.upgrade-requested" +log "database loaded at revision ${SET_DATAREV}; handed off to api for schema upgrade" + rm -f "${BACKUPS_ROOT}/.restore-requested" 2>/dev/null || true -RESTORE_DONE=true -log "restore of ${SETNAME} complete" +RESTORE_LOADED=true diff --git a/scripts/backup/simulate-restore.sh b/scripts/backup/simulate-restore.sh index ed47cf9f..5b18529e 100644 --- a/scripts/backup/simulate-restore.sh +++ b/scripts/backup/simulate-restore.sh @@ -103,6 +103,17 @@ wait_for_phase() { ${COMPOSE} exec -T db sh -c 'true' >/dev/null 2>&1 \ || die "db container not reachable - is the stack up?" +# --- 0b. Datarev-gate preflight (external tester) -------------------------- +# This live round trip runs one codebase against itself, so it can never +# reproduce the "data revision behind the running code" case the restore gate +# now handles. Run the standalone datarev tester first so a pass here also +# covers that offline contract (manifest datarev, known-vs-unknown revision +# refusal, legacy fallback, and the api upgrade handoff). +log "running datarev gate preflight (test-restore-datarev.sh)" +bash "${SCRIPT_DIR}/test-restore-datarev.sh" \ + || die "datarev gate preflight failed - see output above" +log "datarev gate preflight passed" + # --- 1. Backup the current state, then fingerprint the RESTORE TARGET ------ # The restore point is the database as it existed WHEN THE DUMP RAN, not some # earlier moment. Live-churn tables (apitokens last-used, sessions, auditlogs) diff --git a/scripts/backup/test-restore-datarev.sh b/scripts/backup/test-restore-datarev.sh new file mode 100644 index 00000000..39681759 --- /dev/null +++ b/scripts/backup/test-restore-datarev.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# ShopDB restore datarev-gate tester. ***FOR DEVELOPER USE ONLY*** +# +# WHY THIS EXISTS SEPARATELY FROM simulate-restore.sh: +# simulate-restore.sh is a full-fidelity ROUND TRIP against a live stack: it +# backs up, mutates, restores, and proves every table returns to backup-time +# state. By construction it runs ONE codebase against ITSELF, so the database it +# dumps is always at the SAME revision as the code that restores it. It can +# therefore never exercise the case the FBJXBDB4 outage was about: a dump whose +# DATA revision is BEHIND (or ahead of) the running CODE. +# +# This tester fills that gap offline. It builds throwaway migration trees and +# fake gzipped dumps and drives the exact decision logic restore.sh uses: +# 1. manifest.py records datarev (the DB's real alembic_version), not just the +# code head (schemarev). +# 2. A dump whose datarev the code KNOWS (an ancestor in the chain) is allowed +# - the post-load upgrade rolls it forward. +# 3. A dump whose datarev the code does NOT know (newer than this build) is +# REFUSED - restoring it would 500 on 1054 Unknown column. +# 4. A legacy manifest with no datarev falls back to parsing alembic_version +# straight out of the dump. +# 5. restore.sh hands the loaded restore off to the api via the +# .upgrade-requested sentinel instead of marking it done in the sidecar. +# +# No Docker, no MySQL, no network: it is a pure logic gate test, safe to run in +# CI and from inside simulate-restore.sh as a preflight. +# +# ASCII only, no emojis - matches the repo style gate. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MANIFEST_PY="${SCRIPT_DIR}/manifest.py" +RESTORE_SH="${SCRIPT_DIR}/restore.sh" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "${WORKDIR}"' EXIT + +# The backup image ships python3; a dev host may only have `python` (and on +# Windows `python3` can be a non-functional Store alias that resolves on PATH +# but errors when run). Pick the first interpreter that actually EXECUTES, so +# this tester runs anywhere without changing the shipped scripts. +PY="" +for candidate in python3 python py; do + if "${candidate}" -c 'pass' >/dev/null 2>&1; then + PY="${candidate}" + break + fi +done +[ -n "${PY}" ] || { echo "[test] ERROR: no working python interpreter found"; exit 1; } + +passes=0 +fails=0 +pass() { echo "[test] PASS: $*"; passes=$((passes + 1)); } +fail() { echo "[test] FAIL: $*"; fails=$((fails + 1)); } + +# Build a throwaway core migration chain: baseline -> mid -> head. The gate in +# restore.sh only needs each version file to declare `revision = ''`, so +# that is all we write. +build_chain() { + root="$1" + versions="${root}/versions" + mkdir -p "${versions}" + printf "revision = '%s'\ndown_revision = None\n" \ + '0001_baseline' > "${versions}/0001_baseline.py" + printf "revision = '%s'\ndown_revision = '%s'\n" \ + '0002_mid' '0001_baseline' > "${versions}/0002_mid.py" + printf "revision = '%s'\ndown_revision = '%s'\n" \ + '0003_head' '0002_mid' > "${versions}/0003_head.py" +} + +# Fake a gzipped mysqldump that stamps a given alembic_version, matching the +# INSERT shape restore.sh greps for when a legacy manifest lacks datarev. +build_dump() { + out="$1"; datarev="$2" + { + echo 'DROP TABLE IF EXISTS `alembic_version`;' + echo 'CREATE TABLE `alembic_version` (`version_num` varchar(32));' + echo 'INSERT INTO `alembic_version` VALUES' + echo "('${datarev}');" + } | gzip -c > "${out}" +} + +# Mirror of the restore.sh predicate: does the code chain know this revision? +# Kept identical to restore.sh so this test tracks the shipped decision. +code_knows_revision() { + versions_dir="$1"; datarev="$2" + grep -rqE "^revision = ['\"]${datarev}['\"]" "${versions_dir}" 2>/dev/null +} + +# Mirror of the restore.sh datarev extraction: manifest field first, else parse +# the dump. +set_datarev() { + setdir="$1"; dbdump="$2" + value="$(${PY} -c \ + 'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \ + "${setdir}/manifest.json" 2>/dev/null || echo "")" + if [ -z "${value}" ]; then + value="$(gunzip -c "${dbdump}" \ + | grep -A2 -i 'INSERT INTO `alembic_version`' \ + | grep -oE "'[0-9a-zA-Z_]+'" | tr -d "'" | head -1)" + fi + printf '%s' "${value}" +} + +CHAIN="${WORKDIR}/migrations" +build_chain "${CHAIN}" +VERSIONS="${CHAIN}/versions" + +# --- 1. manifest.py records datarev distinct from the code head ------------- +SET1="${WORKDIR}/set1"; mkdir -p "${SET1}" +${PY} "${MANIFEST_PY}" --out "${SET1}/manifest.json" \ + --migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \ + --datarev '0002_mid' --takenat '2024-01-01T00:00:00Z' >/dev/null 2>&1 +recorded="$(${PY} -c \ + 'import json,sys;print(json.load(open(sys.argv[1])).get("datarev",""))' \ + "${SET1}/manifest.json" 2>/dev/null)" +head_stamp="$(${PY} -c \ + 'import json,sys;print(json.load(open(sys.argv[1])).get("schemarev",{}).get("core",[]))' \ + "${SET1}/manifest.json" 2>/dev/null)" +if [ "${recorded}" = '0002_mid' ]; then + pass "manifest.py records datarev (${recorded}) separate from code head ${head_stamp}" +else + fail "manifest.py did not record datarev (got '${recorded}')" +fi + +# --- 2. datarev the code KNOWS (behind head) is allowed -------------------- +DBDUMP1="${SET1}/db.sql.gz"; build_dump "${DBDUMP1}" '0002_mid' +rev="$(set_datarev "${SET1}" "${DBDUMP1}")" +if [ "${rev}" = '0002_mid' ] && code_knows_revision "${VERSIONS}" "${rev}"; then + pass "data behind code (${rev}) is known and allowed to restore" +else + fail "data behind code (${rev}) was not accepted" +fi + +# --- 3. datarev the code does NOT know (newer than build) is refused ------- +SET2="${WORKDIR}/set2"; mkdir -p "${SET2}" +${PY} "${MANIFEST_PY}" --out "${SET2}/manifest.json" \ + --migrations "${CHAIN}" --plugins "${WORKDIR}/noplugins" \ + --datarev '0004_future' --takenat '2024-02-01T00:00:00Z' >/dev/null 2>&1 +DBDUMP2="${SET2}/db.sql.gz"; build_dump "${DBDUMP2}" '0004_future' +rev="$(set_datarev "${SET2}" "${DBDUMP2}")" +if [ "${rev}" = '0004_future' ] && ! code_knows_revision "${VERSIONS}" "${rev}"; then + pass "data ahead of code (${rev}) is unknown and correctly refused" +else + fail "data ahead of code (${rev}) was not refused" +fi + +# --- 4. legacy manifest (no datarev) falls back to the dump ---------------- +SET3="${WORKDIR}/set3"; mkdir -p "${SET3}" +# A pre-datarev manifest: schemarev present, datarev absent/empty. +printf '{"takenat":"2023-06-01T00:00:00Z","schemarev":{"core":["0003_head"]}}' \ + > "${SET3}/manifest.json" +DBDUMP3="${SET3}/db.sql.gz"; build_dump "${DBDUMP3}" '0001_baseline' +rev="$(set_datarev "${SET3}" "${DBDUMP3}")" +if [ "${rev}" = '0001_baseline' ] && code_knows_revision "${VERSIONS}" "${rev}"; then + pass "legacy set with no datarev falls back to dump revision (${rev})" +else + fail "legacy set fallback failed (got '${rev}')" +fi + +# --- 5. restore.sh hands off to the api instead of self-completing --------- +if grep -q '\.upgrade-requested' "${RESTORE_SH}" \ + && grep -q "status_write restore upgrading" "${RESTORE_SH}" \ + && ! grep -qE 'status_write restore done' "${RESTORE_SH}"; then + pass "restore.sh parks in 'upgrading' and hands off via .upgrade-requested" +else + fail "restore.sh does not hand off the upgrade to the api" +fi + +echo "==================================================" +echo "[test] datarev gate: ${passes} passed, ${fails} failed" +echo "==================================================" +[ "${fails}" -eq 0 ] diff --git a/shopdb/core/api/shopdbbackups.py b/shopdb/core/api/shopdbbackups.py index 27624a0e..c2f722bd 100644 --- a/shopdb/core/api/shopdbbackups.py +++ b/shopdb/core/api/shopdbbackups.py @@ -129,6 +129,58 @@ def _write_status(kind, phase, setname, message): pass +def _append_history(kind, outcome, setname, message): + """Append one run record to the shared history the UI reads, newest last. + + Mirrors status.sh's history_append so a finalize that completes in the api + (not the sidecar) still lands in the same history file the restore page + shows. Best-effort: history is a convenience log, never a gate. + """ + root = _backups_root() + stamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + 'Z' + entry = { + 'kind': kind, + 'outcome': outcome, + 'setname': setname, + 'message': message, + 'at': stamp, + } + path = os.path.join(root, 'history', kind + 's.json') + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + entries = _read_json(path) + if not isinstance(entries, list): + entries = [] + entries.append(entry) + with open(path, 'w', encoding='utf-8') as handle: + json.dump(entries, handle) + except OSError: + pass + + +def _run_schema_upgrade(): + """Bring the just-restored database up to the running code, in-process. + + Runs the same two idempotent steps every deploy runs: the core Alembic + upgrade to head, then each plugin chain via the plugin manager. Both are + no-ops when the data already matches the code head, so this is safe to call + even when the restored dump was already current. Uses the live app context + and its configured engine, so no subprocess or extra Flask CLI is needed. + """ + from flask_migrate import upgrade as alembic_upgrade + + alembic_upgrade() + + manager = current_app.extensions.get('plugin_manager') + if manager is None: + raise RuntimeError('Plugin manager not initialized; cannot upgrade plugins') + results = manager.upgrade_all_plugins() + failed = {name: status for name, status in (results or {}).items() + if status not in ('ok', 'no-migrations')} + if failed: + raise RuntimeError('Plugin upgrade failed: ' + json.dumps(failed)) + + def _read_manifest(setdir): """Load a set's manifest.json, or None when absent/unreadable. @@ -175,14 +227,59 @@ def _current_schema_revisions(): return {} +def _known_core_revisions(): + """Return every core Alembic revision this build knows, heads and ancestors. + + Restore compatibility is now data-driven: a set restores when the running + code KNOWS the dump's data revision (so `flask db upgrade` can roll it + forward to head). Heads alone cannot answer that - a dump stamped at an + older-but-valid revision is perfectly restorable - so we walk the whole + script history. Lazy import + broad except keeps a missing/broken alembic + from 500ing the list endpoint (returns empty = everything reads unknown). + """ + try: + from alembic.config import Config + from alembic.script import ScriptDirectory + except ImportError: + return set() + + root = current_app.root_path + migrations = os.path.join(os.path.dirname(root), 'migrations') + ini = os.path.join(migrations, 'alembic.ini') + if not os.path.isfile(ini): + return set() + try: + config = Config(ini) + config.set_main_option('script_location', migrations) + script = ScriptDirectory.from_config(config) + return {rev.revision for rev in script.walk_revisions()} + except Exception: + return set() + + def _is_compatible(manifest, current): - """A set restores onto THIS build only when the build's schema is at least - the set's. We compare head-revision sets: if the set was taken on a schema - the current code has not reached, the code is too OLD and would 500 on the - restored rows, so restore is blocked until the code is upgraded. + """Decide whether a set restores onto THIS build. + + Data-driven gate (the fix for the FBJXBDB4 outage): the authoritative + signal is the set's `datarev` - the actual `alembic_version` of the dumped + DATABASE, not the code head that captured it. A restore is safe when the + running code KNOWS that revision (it is somewhere in this build's migration + history), because then the post-restore finalize can run `flask db upgrade` + to roll the loaded data forward to head. If the code does not know the + revision, the dump is from a NEWER schema than this build and would 500 on + restored rows, so we block until the code is upgraded. + + Legacy sets predate `datarev`; fall back to the old head-subset check on + `schemarev` so they stay listable and restorable as before. """ if not manifest: return False + datarev = manifest.get('datarev') + if datarev: + known = _known_core_revisions() + if not known: + return False + return datarev in known setrevs = set(manifest.get('schemarev', {}).get('core', [])) currevs = set(current.get('core', [])) if not setrevs or not currevs: @@ -338,6 +435,57 @@ def restore_backup(snapshotid): message='Restore queued and validated against the live schema.') +@shopdbbackups_bp.route('/finalize-restore', methods=['POST']) +@require_role('admin') +def finalize_restore(): + """Run the post-restore schema upgrade in the app context. + + The restore itself runs in the Flask-less backup sidecar, which loads the + dump and then drops an .upgrade-requested sentinel because it cannot run + 'flask db upgrade' itself. This endpoint is the api-side half: it brings the + just-restored DATA up to the running CODE with the exact idempotent commands + the deploy path uses, then stamps the final 'done'. Without it a restored + dump that sits at an older revision leaves every newer-migration table + throwing 1054 Unknown column. + + Idempotent and safe to call when nothing is pending: if there is no + .upgrade-requested sentinel it reports there was nothing to finalize. + """ + root = _backups_root() + sentinel = os.path.join(root, '.upgrade-requested') + try: + with open(sentinel, 'r', encoding='utf-8') as handle: + setname = handle.read().strip() + except OSError: + return success_response( + data={'finalized': False}, + message='No restore is awaiting schema upgrade.') + + _write_status('restore', 'upgrading', setname, + 'Upgrading restored schema to code') + try: + _run_schema_upgrade() + except Exception as error: # noqa: BLE001 - report any upgrade failure + _write_status('restore', 'failed', setname, + 'Schema upgrade failed: ' + str(error)) + _append_history('restore', 'failed', setname, + 'Schema upgrade failed: ' + str(error)) + return error_response(ErrorCodes.INTERNAL_ERROR, + 'Post-restore schema upgrade failed: ' + str(error), + http_code=500) + + try: + os.remove(sentinel) + except OSError: + pass + _write_status('restore', 'done', setname, 'Restore complete') + _append_history('restore', 'success', setname, 'Restore complete') + AuditLog.log('restore-finalized', 'Backup', entityname=setname) + return success_response( + data={'finalized': True, 'snapshotid': setname}, + message='Restore finalized: schema upgraded to the running code.') + + @shopdbbackups_bp.route('/status', methods=['GET']) @require_role('admin') def get_status(): From d3f69bbb71600fc75c9f821444b72e875a49daf6 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Fri, 4 Sep 2026 16:19:28 -0400 Subject: [PATCH 55/90] Trust the binary PE version over the typed agent version Uploading an agent build whose product version carried semver build metadata (1.5.0+abcdef) was rejected, because the typed version string was validated as a dotted-numeric value before the file was ever read. That string is only a hint from the operator, and the build metadata it carries is not a legal directory name for the version store. The authority is the dotted-numeric FILE version stamped in the binary PE resource, so validation now happens after the upload is parsed: when the typed value is missing or not dotted-numeric, fall back to the detected version, and reject only when neither yields a usable one. --- plugins/endpoints/api/agentupload.py | 21 ++++-- .../frontend/views/AgentKeySettings.vue | 22 ++++--- .../test_endpoints_agentupload.py | 65 +++++++++++++++++++ 3 files changed, 94 insertions(+), 14 deletions(-) create mode 100644 tests/test_plugins/test_endpoints_agentupload.py diff --git a/plugins/endpoints/api/agentupload.py b/plugins/endpoints/api/agentupload.py index ae463ab5..bc1a9fce 100644 --- a/plugins/endpoints/api/agentupload.py +++ b/plugins/endpoints/api/agentupload.py @@ -69,11 +69,6 @@ def upload_agent_binary(binkey): ErrorCodes.VALIDATION_ERROR, 'binkey must be one of: agent, tray', http_code=400) - if not agentstore.is_valid_version(version): - return error_response( - ErrorCodes.VALIDATION_ERROR, - 'version must be a dotted-numeric string such as 1.6.3', - http_code=400) upload = request.files.get('file') if upload is None or not upload.filename: @@ -94,6 +89,22 @@ def upload_agent_binary(binkey): 'uploaded file is not a Windows executable', http_code=400) + # Gate the version at the PE, not at the raw input. Product versions carry + # semver build metadata (1.5.0+abcdef...) that is not a legal directory + # name, so the operator's typed string is only a hint. The dotted-numeric + # FILE version stamped in the binary's PE resource is the authority; fall + # back to it whenever the typed value is missing or not dotted-numeric, and + # only reject when neither yields a usable version. + if not agentstore.is_valid_version(version): + detected = agentstore.read_pe_file_version(data) + if agentstore.is_valid_version(detected): + version = detected + else: + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'version must be a dotted-numeric string such as 1.6.3', + http_code=400) + result = agentstore.store_version(binkey, version, data, make_current=makecurrent) diff --git a/plugins/endpoints/frontend/views/AgentKeySettings.vue b/plugins/endpoints/frontend/views/AgentKeySettings.vue index a3b659f3..f0d78107 100644 --- a/plugins/endpoints/frontend/views/AgentKeySettings.vue +++ b/plugins/endpoints/frontend/views/AgentKeySettings.vue @@ -355,7 +355,7 @@