From 4a1b97362b59205ed85ba3384c9ee1e96a08efae Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Mon, 24 Aug 2026 14:29:02 -0400 Subject: [PATCH 01/14] Add site asset CSV import with fuzzy reconciliation Sites seed inventory from the Asset Collector script or spreadsheet exports, but lookup names drift between sources (Dell vs Dell Inc.), which would otherwise create duplicate vendors/models/types on every import. The importer now normalizes and fuzzy-matches near-miss references onto existing lookup rows within an operator-set similarity threshold, auto-applying the match while reporting it for review so nothing merges silently. Operating systems and asset rows are never reconciled. Adds the dedicated Import Site Assets page with script/template download and a multi-file dry-run/commit workflow. --- frontend/src/api/index.js | 24 + frontend/src/router/routes/core.js | 6 + .../src/views/settings/ImportSiteAssets.vue | 659 ++++++++++++++++++ frontend/src/views/settings/SiteSettings.vue | 8 +- frontend/src/views/settings/settingsNav.js | 3 +- .../assetcollector/Collect-ShopDB.ps1 | 542 ++++++++++++++ .../site_imports/assetcollector/overview.md | 91 +++ .../assetcollector/sample/assets.csv | 5 + .../assetcollector/sample/computers.csv | 3 + .../assetcollector/sample/machines.csv | 3 + .../assetcollector/sample/models.csv | 5 + .../sample/operatingsystems.csv | 3 + .../assetcollector/sample/vendors.csv | 4 + shopdb/core/api/settings.py | 227 ++++++ shopdb/core/services/csvimport.py | 102 ++- 15 files changed, 1681 insertions(+), 4 deletions(-) create mode 100644 frontend/src/views/settings/ImportSiteAssets.vue create mode 100644 scripts/site_imports/assetcollector/Collect-ShopDB.ps1 create mode 100644 scripts/site_imports/assetcollector/overview.md create mode 100644 scripts/site_imports/assetcollector/sample/assets.csv create mode 100644 scripts/site_imports/assetcollector/sample/computers.csv create mode 100644 scripts/site_imports/assetcollector/sample/machines.csv create mode 100644 scripts/site_imports/assetcollector/sample/models.csv create mode 100644 scripts/site_imports/assetcollector/sample/operatingsystems.csv create mode 100644 scripts/site_imports/assetcollector/sample/vendors.csv diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 0e9e54bb..6712925c 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -1014,6 +1014,30 @@ export const settingsApi = { form.append('kind', kind) form.append('file', file) return api.post('/settings/branding-logo', form, { headers: { 'Content-Type': 'multipart/form-data' } }) + }, + downloadCollectorScript() { + // Collect-ShopDB.ps1 with this site's CSV path baked into $NetworkStoreRoot + return api.get('/settings/asset-import/script', { responseType: 'blob' }) + }, + listImportTemplates() { + // Importable CSV tables in the order they must be uploaded + return api.get('/settings/asset-import/templates') + }, + downloadImportTemplate(tablename) { + // Generated CSV template (header + example row) for one table + return api.get(`/settings/asset-import/templates/${encodeURIComponent(tablename)}`, { responseType: 'blob' }) + }, + importAssetCsvs(files, commit = false) { + // Multipart upload of one or more CSVs. commit=false is a dry-run preview + // that writes nothing; commit=true applies the whole set in one transaction. + const form = new FormData() + for (const file of files) { + form.append('files', file) + } + form.append('commit', commit ? 'true' : 'false') + return api.post('/settings/asset-import/import', form, { + headers: { 'Content-Type': 'multipart/form-data' } + }) } } diff --git a/frontend/src/router/routes/core.js b/frontend/src/router/routes/core.js index 0d98321a..b2bc3864 100644 --- a/frontend/src/router/routes/core.js +++ b/frontend/src/router/routes/core.js @@ -150,6 +150,12 @@ export default [ component: () => import('../../views/settings/BrandingSettings.vue'), meta: { requiresAuth: true, requiresAdmin: true } }, + { + path: 'settings/importsiteassets', + name: 'import-site-assets', + component: () => import('../../views/settings/ImportSiteAssets.vue'), + meta: { requiresAuth: true, requiresAdmin: true } + }, { path: 'settings/printing', name: 'printing-settings', diff --git a/frontend/src/views/settings/ImportSiteAssets.vue b/frontend/src/views/settings/ImportSiteAssets.vue new file mode 100644 index 00000000..60ed215c --- /dev/null +++ b/frontend/src/views/settings/ImportSiteAssets.vue @@ -0,0 +1,659 @@ + + + + + diff --git a/frontend/src/views/settings/SiteSettings.vue b/frontend/src/views/settings/SiteSettings.vue index 77a77221..7be8eb45 100644 --- a/frontend/src/views/settings/SiteSettings.vue +++ b/frontend/src/views/settings/SiteSettings.vue @@ -75,7 +75,8 @@ const LABELS = { employee_directory_mode: 'Employee Directory Mode', usb_directory_mode: 'USB Directory Mode', site_timezone: 'Site Timezone', - setup_complete: 'Setup Complete' + setup_complete: 'Setup Complete', + asset_import_csv_location: 'Asset Import CSV Location' } function prettyLabel(key) { return LABELS[key] || key @@ -127,7 +128,10 @@ async function load() { try { const response = await settingsApi.list() const all = response.data?.data || response.data || [] - items.value = all.filter(s => s.category === 'site') + // The Asset Collector CSV location is managed on the dedicated Import Site + // Assets page. + const importKeys = ['asset_import_csv_location'] + items.value = all.filter(s => s.category === 'site' && !importKeys.includes(s.key)) } catch (err) { console.error('Error loading site settings:', err) error.value = 'Could not load settings.' diff --git a/frontend/src/views/settings/settingsNav.js b/frontend/src/views/settings/settingsNav.js index c1fb32d9..a5b87238 100644 --- a/frontend/src/views/settings/settingsNav.js +++ b/frontend/src/views/settings/settingsNav.js @@ -4,7 +4,7 @@ // Order: site identity + the reference-data catalogs users touch daily come // first, then the platform/system groups (integrations, communication, search, // plugins, access) cluster together at the end. -import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key } from 'lucide-vue-next' +import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key, Upload } from 'lucide-vue-next' export const settingsGroups = [ { @@ -13,6 +13,7 @@ export const settingsGroups = [ { to: '/settings/site', icon: Home, title: 'Site & Facility', description: 'Site URL/FQDN, facility name, PC access domain, employee-ID pattern, printer hostname template' }, { to: '/settings/floormap', icon: MapPin, title: 'Floor Map', description: 'Facility floor-plan blueprint image and pixel dimensions (width, height) for the site map' }, { to: '/settings/branding', icon: Palette, title: 'Branding', description: 'Site, QR, and badge logos, favicon, and brand primary, accent, sidebar, and theme colors' }, + { to: '/settings/importsiteassets', icon: Upload, title: 'Import Site Assets', description: 'Seed Machines, PCs, and Applications via the Asset Collector script, existing databases, or Intune/Azure exports; set the shared CSV collection location and download collected files' }, ], }, { diff --git a/scripts/site_imports/assetcollector/Collect-ShopDB.ps1 b/scripts/site_imports/assetcollector/Collect-ShopDB.ps1 new file mode 100644 index 00000000..f66c4b8d --- /dev/null +++ b/scripts/site_imports/assetcollector/Collect-ShopDB.ps1 @@ -0,0 +1,542 @@ +#Requires -Version 5.1 +[CmdletBinding()] +param( + # Where CSVs are collected/accumulated. Overrides the in-script CONFIG block + # below. See the CONFIG block for the easy place to set this permanently. + [Parameter(Mandatory = $false)] + [string]$OutputRoot, + + # Optional network share to mirror the store into after a run. Overrides the + # in-script CONFIG block below. + [Parameter(Mandatory = $false)] + [string]$PublishRoot, + + # Business identifier for this PC (workstation number). Defaults to hostname. + [Parameter(Mandatory = $false)] + [string]$AssetNumber, + + # Value written into assets.assettypeid for the PC (resolved by name). + [Parameter(Mandatory = $false)] + [string]$AssetType = 'Computer', + + # Value written into computers.computertypeid (resolved by name). + [Parameter(Mandatory = $false)] + [string]$ComputerType, + + # Value written into machines.machinetypeid (resolved by name). + [Parameter(Mandatory = $false)] + [string]$MachineType, + + # Explicit eDNC/UDC source overrides. When omitted, standard on-asset + # locations are searched. + [Parameter(Mandatory = $false)] + [string]$RegPath, + [Parameter(Mandatory = $false)] + [string]$JsonPath, + + # Skip switches for partial runs. + [switch]$SkipMachine, + [switch]$SkipApps +) + +<# +.SYNOPSIS + Unified ShopDB site-adoption collector. Runs on any PC, collects the PC and any + attached eDNC/DNC machine, and UPSERTS the results into a shared set of + importable CSVs so a site can build its inventory PC by PC. + +.DESCRIPTION + Accumulating store under OutputRoot: + vendors.csv, models.csv, operatingsystems.csv, assets.csv, + computers.csv, machines.csv + Rows are merged field-by-field on each table's natural key (a later run fills + gaps and refreshes values without wiping data an earlier run captured). + + Phase-2 data that CSV cannot carry is written per entity under OutputRoot\sidecars: + computer_.json (interfaces, installed apps) + machine_.json (PC<->Machine link, connection detail) + + Read-only against the machine. Local reads may run elevated; publishing to a + share runs in the current user context only. + +.NOTES + PS 5.1 compatible. Headers match the ShopDB importer (see CSV-SCHEMA.md). +#> + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# ============================================================================= +# CONFIG - SET YOUR CSV DESTINATIONS HERE +# ----------------------------------------------------------------------------- +# Two-stage flow so you can collect locally (possibly elevated) and then append +# the captured rows to a CSV store that lives on a network share: +# +# 1) LocalStageRoot Fast, always-writable local folder where CSVs are built +# and merged. Collection can run elevated here. +# 2) NetworkStoreRoot The shared CSV store the whole site accumulates into and +# that you later point 'flask csv import --dir' at. Leave +# blank ('') to keep everything local. When set, the merged +# store is copied here at the end of the run. Network +# writes must happen as a STANDARD user - do NOT run the +# publish step elevated. +# +# Example network path: '\\fileserver\shopdb$\intake' +# +# Command-line -OutputRoot / -PublishRoot still override these values. +# ============================================================================= + +$LocalStageRoot = Join-Path $env:ProgramData 'ShopDB\intake' +$NetworkStoreRoot = '\\wi001dataaege.wlm.geaerospace.net\shared\TEAMSYS\.sfscripts\shopdb' + +# Resolve effective destinations (command-line params win over CONFIG above). +if (-not $OutputRoot) { $OutputRoot = $LocalStageRoot } +if (-not $PublishRoot) { $PublishRoot = $NetworkStoreRoot } + +# --- Store definitions (header order + natural key per importable table) ------- + +$script:Stores = [ordered]@{ + 'vendors.csv' = @{ key = 'vendor'; header = @('vendor') } + 'models.csv' = @{ key = 'modelnumber'; header = @('modelnumber','vendorid','modeltypeid') } + 'operatingsystems.csv' = @{ key = 'osname'; header = @('osname') } + 'assets.csv' = @{ key = 'assetnumber'; header = @('assetnumber','assettypeid','name','statusid','serialnumber','gaugelabreference','maintenancereference','locationid','businessunitid','notes') } + 'computers.csv' = @{ key = 'assetid'; header = @('assetid','hostname','computertypeid','osid','vendorid','modelnumberid','loggedinuser','lastreporteddate','lastboottime') } + 'machines.csv' = @{ key = 'assetid'; header = @('assetid','machinetypeid','vendorid','modelnumberid','controllervendorid','controllermodelid','requiresmanualconfig','islocationonly','notes') } +} + +# --- Shared helpers ----------------------------------------------------------- + +function ConvertTo-CsvField { + param([AllowNull()]$Value) + if ($null -eq $Value) { return '' } + $text = [string]$Value + if ($text -match '[",\r\n]') { return '"' + ($text -replace '"', '""') + '"' } + return $text +} + +function Get-Prop { + param([AllowNull()]$Object, [Parameter(Mandatory = $true)][string]$Name) + if ($null -eq $Object) { return $null } + $prop = $Object.PSObject.Properties[$Name] + if ($null -eq $prop) { return $null } + return $prop.Value +} + +function Get-NonEmpty { + param([AllowNull()]$Value) + if ($null -ne $Value -and -not [string]::IsNullOrWhiteSpace([string]$Value)) { + return ([string]$Value).Trim() + } + return $null +} + +function Get-FirstNonEmpty { + param([object[]]$Values) + foreach ($v in $Values) { $n = Get-NonEmpty $v; if ($n) { return $n } } + return $null +} + +function Format-ShopDbDate { + param([AllowNull()]$Value) + if ($null -eq $Value) { return $null } + try { return ([datetime]$Value).ToString('yyyy-MM-dd HH:mm:ss') } catch { return $null } +} + +function Write-Utf8NoBom { + param([string]$Path, [string]$Content) + $encoding = New-Object System.Text.UTF8Encoding($false) + [System.IO.File]::WriteAllText($Path, $Content, $encoding) +} + +# Field-level upsert into a persistent CSV store, keyed by natural key. Existing +# values are kept when the incoming cell is empty, so a partial run never blanks +# out data an earlier run captured. +function Merge-CsvStore { + param( + [Parameter(Mandatory = $true)][string]$Path, + [Parameter(Mandatory = $true)][string[]]$Header, + [Parameter(Mandatory = $true)][string]$Key, + [Parameter(Mandatory = $true)][AllowNull()][object[]]$IncomingRows + ) + + $index = [ordered]@{} + if (Test-Path -LiteralPath $Path) { + foreach ($existing in @(Import-Csv -LiteralPath $Path)) { + $keyVal = Get-NonEmpty (Get-Prop $existing $Key) + if (-not $keyVal) { continue } + $index[$keyVal.ToLowerInvariant()] = $existing + } + } + + $changed = 0 + foreach ($row in $IncomingRows) { + if ($null -eq $row) { continue } + $keyVal = Get-NonEmpty (Get-Prop $row $Key) + if (-not $keyVal) { continue } + $lc = $keyVal.ToLowerInvariant() + if ($index.Contains($lc)) { + $target = $index[$lc] + foreach ($col in $Header) { + $incoming = Get-NonEmpty (Get-Prop $row $col) + if ($incoming) { + if ($target.PSObject.Properties[$col]) { $target.$col = $incoming } + else { $target | Add-Member -NotePropertyName $col -NotePropertyValue $incoming -Force } + } + } + } else { + $fresh = [ordered]@{} + foreach ($col in $Header) { $fresh[$col] = Get-NonEmpty (Get-Prop $row $col) } + $index[$lc] = [pscustomobject]$fresh + } + $changed++ + } + + $lines = New-Object System.Collections.Generic.List[string] + $lines.Add(($Header -join ',')) + foreach ($stored in $index.Values) { + $cells = foreach ($col in $Header) { ConvertTo-CsvField (Get-Prop $stored $col) } + $lines.Add(($cells -join ',')) + } + Write-Utf8NoBom -Path $Path -Content (($lines -join "`r`n") + "`r`n") + return $changed +} + +# --- eDNC/UDC parsing (registry, .reg export, UDC settings json) -------------- + +function Resolve-ControllerVendor { + param([AllowNull()][string]$ControlType, [AllowNull()][string]$NcIF) + $ct = ('' + $ControlType).Trim().ToUpperInvariant() + $nc = ('' + $NcIF).Trim().ToUpperInvariant() + if ($nc -eq 'EFOCAS' -or $ct -eq 'EFOCAS') { return 'FANUC' } + if ($ct -eq 'SIEMENS') { return 'Siemens' } + if ($ct -eq 'OKUMA') { return 'Okuma' } + return $null +} + +function Read-RegExport { + param([Parameter(Mandatory = $true)][string]$Path) + $lines = @(Get-Content -LiteralPath $Path -Encoding Unicode -ErrorAction Stop) + $firstMeaningful = ($lines | Where-Object { $_ -match '\S' } | Select-Object -First 1) + if (-not ($firstMeaningful -match '^(REGEDIT4|Windows Registry Editor Version)')) { + $lines = @(Get-Content -LiteralPath $Path -Encoding Default -ErrorAction Stop) + } + $data = @{} + $currentKey = $null + foreach ($raw in $lines) { + $line = $raw.Trim() + if ([string]::IsNullOrWhiteSpace($line)) { continue } + if ($line -match '^(REGEDIT4|Windows Registry Editor Version)') { continue } + if ($line -match '^\s*;') { continue } + if ($line -match '^\[(?[^\]]+)\]$') { + $currentKey = $Matches['key'] + if (-not $data.ContainsKey($currentKey)) { $data[$currentKey] = @{} } + continue + } + if (-not $currentKey) { continue } + if ($line -match '^"(?[^"]+)"=(?.+)$') { + $name = $Matches['name'] + $rhs = $Matches['rhs'].Trim() + if ($rhs -match '^"(?.*)"$') { $val = $Matches['s'] } + elseif ($rhs -match '^dword:(?[0-9a-fA-F]{8})$') { $val = [Convert]::ToInt32($Matches['hex'], 16) } + else { $val = $rhs } + $data[$currentKey][$name] = $val + } + } + return $data +} + +function Get-RegExportValue { + param([hashtable]$Reg, [string]$KeySuffix, [string]$Name) + foreach ($key in $Reg.Keys) { + if ($key -match ('\\' + [regex]::Escape($KeySuffix) + '$')) { + if ($Reg[$key].ContainsKey($Name)) { return $Reg[$key][$Name] } + } + } + return $null +} + +function ConvertFrom-RegExport { + param([hashtable]$Reg) + [ordered]@{ + source = 'reg' + machinenumber = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'MachineNo') + site = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'Site') + controltype = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'Cnc') + ncif = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'NcIF') + dnctype = 'eDNC' + tcpaddress = Get-NonEmpty (Get-RegExportValue $Reg 'eFocas' 'IpAddr') + tcpport = Get-NonEmpty (Get-RegExportValue $Reg 'eFocas' 'SocketNo') + comport = Get-NonEmpty (Get-RegExportValue $Reg 'PPDCS' 'Port Id') + baudrate = Get-NonEmpty (Get-RegExportValue $Reg 'PPDCS' 'Baud') + } +} + +function Get-RegistryValue { + param([string]$Path, [string]$Name) + if (-not (Test-Path -LiteralPath $Path)) { return $null } + return Get-Prop (Get-ItemProperty -LiteralPath $Path -ErrorAction SilentlyContinue) $Name +} + +function ConvertFrom-LiveRegistry { + param([string]$Base) + if (-not (Test-Path -LiteralPath $Base)) { return $null } + $general = Join-Path $Base 'General' + $efocas = Join-Path $Base 'eFocas' + $ppdcs = Join-Path $Base 'PPDCS' + [ordered]@{ + source = 'registry' + machinenumber = Get-NonEmpty (Get-RegistryValue $general 'MachineNo') + site = Get-NonEmpty (Get-RegistryValue $general 'Site') + controltype = Get-NonEmpty (Get-RegistryValue $general 'Cnc') + ncif = Get-NonEmpty (Get-RegistryValue $general 'NcIF') + dnctype = 'eDNC' + tcpaddress = Get-NonEmpty (Get-RegistryValue $efocas 'IpAddr') + tcpport = Get-NonEmpty (Get-RegistryValue $efocas 'SocketNo') + comport = Get-NonEmpty (Get-RegistryValue $ppdcs 'Port Id') + baudrate = Get-NonEmpty (Get-RegistryValue $ppdcs 'Baud') + } +} + +function ConvertFrom-UdcJson { + param([string]$Path) + $obj = Get-Content -LiteralPath $Path -Raw -ErrorAction Stop | ConvertFrom-Json + $general = Get-Prop $obj 'GeneralSettings' + $ethernet = Get-Prop $obj 'EthernetSettings' + $serial = Get-Prop $obj 'SerialSettings' + [ordered]@{ + source = 'json' + machinenumber = Get-NonEmpty (Get-Prop $general 'MachineNumber') + site = Get-NonEmpty (Get-Prop $general 'Site') + controltype = Get-NonEmpty (Get-Prop $general 'ControlType') + ncif = $null + dnctype = Get-NonEmpty (Get-Prop $general 'DncType') + tcpaddress = Get-NonEmpty (Get-Prop $ethernet 'TCPAddress') + tcpport = Get-NonEmpty (Get-Prop $ethernet 'TCPPort') + comport = Get-NonEmpty (Get-Prop $serial 'ComPort') + baudrate = Get-NonEmpty (Get-Prop $serial 'BaudRate') + } +} + +function Find-FirstFile { + param([string[]]$Roots, [string]$Filter) + foreach ($root in $Roots) { + if (-not $root -or -not (Test-Path -LiteralPath $root)) { continue } + $hit = Get-ChildItem -LiteralPath $root -Filter $Filter -File -ErrorAction SilentlyContinue | + Sort-Object LastWriteTime -Descending | Select-Object -First 1 + if ($hit) { return $hit.FullName } + } + return $null +} + +function Resolve-DncSource { + param([string]$RegPath, [string]$JsonPath) + if ($JsonPath -and (Test-Path -LiteralPath $JsonPath)) { + Write-Host (' machine source: JSON (explicit) ' + $JsonPath) -ForegroundColor DarkGray + return ConvertFrom-UdcJson -Path $JsonPath + } + if ($RegPath -and (Test-Path -LiteralPath $RegPath)) { + Write-Host (' machine source: .reg (explicit) ' + $RegPath) -ForegroundColor DarkGray + return ConvertFrom-RegExport -Reg (Read-RegExport -Path $RegPath) + } + foreach ($base in @( + 'HKLM:\SOFTWARE\GE Aircraft Engines\DNC', + 'HKLM:\SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC')) { + $fromReg = ConvertFrom-LiveRegistry -Base $base + if ($fromReg -and $fromReg.machinenumber) { + Write-Host (' machine source: live registry ' + $base) -ForegroundColor DarkGray + return $fromReg + } + } + $localReg = Find-FirstFile -Roots @('C:\Program Files (x86)\Dnc', (Join-Path $env:SystemDrive 'GE Aerospace')) -Filter '*.reg' + if ($localReg) { + Write-Host (' machine source: local .reg ' + $localReg) -ForegroundColor DarkGray + return ConvertFrom-RegExport -Reg (Read-RegExport -Path $localReg) + } + $localJson = Find-FirstFile -Roots @((Join-Path $env:SystemDrive 'GE Aerospace'), 'C:\Program Files (x86)\UDC', 'C:\Program Files (x86)\Dnc') -Filter 'udc_settings_*.json' + if ($localJson) { + Write-Host (' machine source: local UDC json ' + $localJson) -ForegroundColor DarkGray + return ConvertFrom-UdcJson -Path $localJson + } + return $null +} + +# --- PC collection (read-only) ------------------------------------------------ + +function Get-PcFacts { + $computerSystem = Get-CimInstance Win32_ComputerSystem -ErrorAction SilentlyContinue + $operatingSys = Get-CimInstance Win32_OperatingSystem -ErrorAction SilentlyContinue + $bios = Get-CimInstance Win32_BIOS -ErrorAction SilentlyContinue + + $interfaces = @() + try { + foreach ($cfg in @(Get-NetIPConfiguration -ErrorAction SilentlyContinue)) { + $adapter = $null + try { $adapter = Get-NetAdapter -InterfaceAlias $cfg.InterfaceAlias -ErrorAction SilentlyContinue } catch {} + if (-not $adapter -or (Get-Prop $adapter 'HardwareInterface') -ne $true) { continue } + $ipv4 = @(Get-Prop $cfg 'IPv4Address') | Select-Object -First 1 + $interfaces += [ordered]@{ + interfacealias = [string]$cfg.InterfaceAlias + ipaddress = if ($ipv4) { [string](Get-Prop $ipv4 'IPAddress') } else { $null } + macaddress = [string](Get-Prop $adapter 'MacAddress') + } + } + } catch {} + + $installedApps = @() + if (-not $SkipApps) { + foreach ($keyPath in @( + 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\*', + 'HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\*')) { + foreach ($entry in @(Get-ItemProperty -Path $keyPath -ErrorAction SilentlyContinue)) { + $displayName = Get-NonEmpty (Get-Prop $entry 'DisplayName') + if (-not $displayName) { continue } + if ((Get-Prop $entry 'SystemComponent') -eq 1) { continue } + $installedApps += [ordered]@{ + appname = $displayName + installedversion = Get-NonEmpty (Get-Prop $entry 'DisplayVersion') + publisher = Get-NonEmpty (Get-Prop $entry 'Publisher') + } + } + } + $installedApps = @($installedApps | Sort-Object { $_.appname } -Unique) + } + + [ordered]@{ + hostname = Get-FirstNonEmpty @($env:COMPUTERNAME, (Get-Prop $computerSystem 'Name')) + vendor = Get-NonEmpty (Get-Prop $computerSystem 'Manufacturer') + model = Get-NonEmpty (Get-Prop $computerSystem 'Model') + osname = Get-NonEmpty (Get-Prop $operatingSys 'Caption') + serial = Get-FirstNonEmpty @((Get-Prop $bios 'SerialNumber'), (Get-Prop $computerSystem 'SerialNumber')) + lastboot = Format-ShopDbDate (Get-Prop $operatingSys 'LastBootUpTime') + loggeduser = Get-NonEmpty (Get-Prop $computerSystem 'UserName') + interfaces = $interfaces + installedapps = $installedApps + } +} + +# --- Main --------------------------------------------------------------------- + +Write-Host '' +Write-Host ' ShopDB unified collector' -ForegroundColor Cyan +Write-Host (' ' + ('-' * 56)) -ForegroundColor DarkGray + +if (-not (Test-Path -LiteralPath $OutputRoot)) { New-Item -ItemType Directory -Path $OutputRoot -Force | Out-Null } +$sidecarDir = Join-Path $OutputRoot 'sidecars' +if (-not (Test-Path -LiteralPath $sidecarDir)) { New-Item -ItemType Directory -Path $sidecarDir -Force | Out-Null } + +$collectedAt = (Get-Date).ToString('yyyy-MM-dd HH:mm:ss') + +$pc = Get-PcFacts +if (-not $AssetNumber) { $AssetNumber = $pc.hostname } + +# Row sets keyed to the shared stores. +$vendorRows = @() +$modelRows = @() +$osRows = @() +$assetRows = @() +$computerRows = @() +$machineRows = @() + +if ($pc.vendor) { $vendorRows += [pscustomobject]@{ vendor = $pc.vendor } } +if ($pc.model) { $modelRows += [pscustomobject]@{ modelnumber = $pc.model; vendorid = $pc.vendor; modeltypeid = $null } } +if ($pc.osname) { $osRows += [pscustomobject]@{ osname = $pc.osname } } + +$assetRows += [pscustomobject]@{ + assetnumber = $AssetNumber; assettypeid = $AssetType; name = $pc.hostname; statusid = $null + serialnumber = $pc.serial; gaugelabreference = $null; maintenancereference = $null + locationid = $null; businessunitid = $null + notes = 'PC collected ' + $collectedAt +} +$computerRows += [pscustomobject]@{ + assetid = $AssetNumber; hostname = $pc.hostname; computertypeid = $ComputerType; osid = $pc.osname + vendorid = $pc.vendor; modelnumberid = $pc.model; loggedinuser = $pc.loggeduser + lastreporteddate = $collectedAt; lastboottime = $pc.lastboot +} + +# Machine (best-effort). +$machine = $null +if (-not $SkipMachine) { + try { $machine = Resolve-DncSource -RegPath $RegPath -JsonPath $JsonPath } catch { $machine = $null } +} +if ($machine -and $machine.machinenumber) { + $controllerVendor = Resolve-ControllerVendor -ControlType $machine.controltype -NcIF $machine.ncif + $transport = if ($machine.tcpaddress) { 'ethernet' } elseif ($machine.comport) { 'serial' } else { 'unknown' } + $noteBits = @("DNC=$($machine.dnctype)") + if ($machine.controltype) { $noteBits += "control=$($machine.controltype)" } + $noteBits += "transport=$transport" + $machineNote = 'eDNC collected ' + $collectedAt + '; ' + ($noteBits -join '; ') + + if ($controllerVendor) { $vendorRows += [pscustomobject]@{ vendor = $controllerVendor } } + $assetRows += [pscustomobject]@{ + assetnumber = $machine.machinenumber; assettypeid = 'Machine'; name = $machine.machinenumber; statusid = $null + serialnumber = $null; gaugelabreference = $null; maintenancereference = $null + locationid = $null; businessunitid = $null; notes = $machineNote + } + $machineRows += [pscustomobject]@{ + assetid = $machine.machinenumber; machinetypeid = $MachineType; vendorid = $null; modelnumberid = $null + controllervendorid = $controllerVendor; controllermodelid = $null + requiresmanualconfig = $null; islocationonly = $null; notes = $machineNote + } + + $link = [ordered]@{ + pcassetnumber = $AssetNumber; machineassetnumber = $machine.machinenumber; relationship = 'controls' + dnctype = $machine.dnctype; controltype = $machine.controltype; controllervendor = $controllerVendor + site = $machine.site; source = $machine.source; collectedat = $collectedAt + connection = [ordered]@{ + transport = $transport; tcpaddress = $machine.tcpaddress; tcpport = $machine.tcpport + comport = $machine.comport; baudrate = $machine.baudrate + } + } + Write-Utf8NoBom -Path (Join-Path $sidecarDir ('machine_' + $machine.machinenumber + '.json')) -Content ($link | ConvertTo-Json -Depth 6) + Write-Host (' linked machine ' + $machine.machinenumber + ' (' + $controllerVendor + ', ' + $transport + ')') -ForegroundColor DarkGray +} elseif (-not $SkipMachine) { + Write-Host ' no eDNC/DNC/UDC machine config found on this PC' -ForegroundColor DarkGray +} + +# --- Upsert into the shared store -------------------------------------------- + +$rowsByStore = @{ + 'vendors.csv' = $vendorRows + 'models.csv' = $modelRows + 'operatingsystems.csv' = $osRows + 'assets.csv' = $assetRows + 'computers.csv' = $computerRows + 'machines.csv' = $machineRows +} + +Write-Host '' +Write-Host (' Store: ' + $OutputRoot) -ForegroundColor Green +foreach ($fileName in $script:Stores.Keys) { + $spec = $script:Stores[$fileName] + $rows = @($rowsByStore[$fileName]) + if ($rows.Count -eq 0) { continue } + $path = Join-Path $OutputRoot $fileName + $n = Merge-CsvStore -Path $path -Header $spec.header -Key $spec.key -IncomingRows $rows + Write-Host (' merged ' + $n + ' row(s) into ' + $fileName) -ForegroundColor DarkGray +} + +# PC phase-2 sidecar. +$pcSidecar = [ordered]@{ + assetnumber = $AssetNumber; hostname = $pc.hostname; collectedat = $collectedAt + interfaces = $pc.interfaces; installedapps = $pc.installedapps +} +Write-Utf8NoBom -Path (Join-Path $sidecarDir ('computer_' + $AssetNumber + '.json')) -Content ($pcSidecar | ConvertTo-Json -Depth 6) + +# --- Publish (standard-user network mirror, optional) ------------------------- + +if ($PublishRoot) { + if (([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole( + [Security.Principal.WindowsBuiltInRole]::Administrator)) { + Write-Host '' + Write-Host ' WARNING: publishing while elevated. Network writes should run as the standard user.' -ForegroundColor Yellow + } + if (-not (Test-Path -LiteralPath $PublishRoot)) { New-Item -ItemType Directory -Path $PublishRoot -Force | Out-Null } + Copy-Item -Path (Join-Path $OutputRoot '*') -Destination $PublishRoot -Recurse -Force + Write-Host '' + Write-Host (' Published store to: ' + $PublishRoot) -ForegroundColor Green +} + +Write-Host '' +Write-Host ' Done. Dry-run validate the accumulated store with:' -ForegroundColor Cyan +Write-Host (' flask csv import --dir "' + $OutputRoot + '"') -ForegroundColor White +Write-Host '' diff --git a/scripts/site_imports/assetcollector/overview.md b/scripts/site_imports/assetcollector/overview.md new file mode 100644 index 00000000..1587c2f3 --- /dev/null +++ b/scripts/site_imports/assetcollector/overview.md @@ -0,0 +1,91 @@ +# Collect-ShopDB.ps1 Overview + +## Overview + +- **Purpose**: A unified, read-only "site adoption" collector that runs on any + Windows PC, gathers facts about that PC plus any attached eDNC/DNC/UDC machine, + and merges them into a shared set of importable CSVs so a site can build its + ShopDB inventory one PC at a time. +- **Compatibility**: PowerShell 5.1+; CSV headers are aligned to the ShopDB + importer (`flask csv import --dir`). + +## Configuration and destinations + +- **Two-stage flow**: a fast local stage folder (`%ProgramData%\ShopDB\intake`) + where CSVs are built/merged, and an optional network store (a file share) that + the whole site accumulates into. +- Command-line params `-OutputRoot` and `-PublishRoot` override the in-script + CONFIG block. +- Other params: `-AssetNumber` (defaults to hostname), + `-AssetType`/`-ComputerType`/`-MachineType` (resolved by name), explicit + `-RegPath`/`-JsonPath` machine source overrides, and `-SkipMachine`/`-SkipApps` + switches for partial runs. + +## Data model (the stores) + +- Six importable CSVs, each with a defined header order and natural key: + `vendors.csv`, `models.csv`, `operatingsystems.csv`, `assets.csv`, + `computers.csv`, `machines.csv`. +- Column names follow ShopDB conventions (e.g., `assetid`, `modelnumberid`, + `controllervendorid`, `lastreporteddate`). + +## PC collection (`Get-PcFacts`, read-only) + +- Uses CIM/WMI (`Win32_ComputerSystem`, `Win32_OperatingSystem`, `Win32_BIOS`) + to gather hostname, vendor/manufacturer, model, OS caption, serial number, + last boot time, and logged-in user. +- Enumerates **network interfaces** (hardware adapters only: alias, IPv4, MAC). +- Enumerates **installed applications** from the registry Uninstall keys (name, + version, publisher), skipping system components - unless `-SkipApps` is set. + +## Machine (eDNC/DNC/UDC) collection + +- `Resolve-DncSource` searches in priority order: explicit JSON path -> explicit + .reg path -> live registry (`HKLM\...\GE Aircraft Engines\DNC`) -> local `.reg` + export files -> local `udc_settings_*.json` files. +- Three parsers normalize the different source formats into one shape: + `ConvertFrom-RegExport` (parses exported `.reg` files including dword decoding), + `ConvertFrom-LiveRegistry`, and `ConvertFrom-UdcJson`. +- Extracts machine number, site, control type, DNC type, and connection detail + (TCP address/port for ethernet, COM port/baud for serial). +- `Resolve-ControllerVendor` maps control type / NcIF to a controller vendor + (e.g., eFOCAS -> FANUC, Siemens, Okuma). + +## Merge / upsert logic (`Merge-CsvStore`) + +- Field-level UPSERT keyed on each table's natural key: existing rows are matched + case-insensitively, and empty incoming cells never overwrite previously captured + values - so partial or repeated runs fill gaps without wiping earlier data. +- CSV writing is UTF-8 without BOM, with proper quoting/escaping. + +## Sidecar files (phase-2 data CSV can't carry) + +- Per-PC `sidecars\computer_.json`: interfaces + installed apps. +- Per-machine `sidecars\machine_.json`: PC-to-machine link + (relationship "controls"), DNC/control details, and connection transport info. + +## Publishing and output + +- Optionally mirrors the whole local store to the network share + (`Copy-Item -Recurse -Force`), with a warning if run elevated (network writes + should run as standard user). +- Prints a summary and the follow-up dry-run command: + `flask csv import --dir ""`. + +--- + +## Relevance to an "Asset Collector" package + +This script is a solid template for one collector in a broader package. Notable +patterns worth generalizing across other collectors (existing databases, +Intune/Azure exports, etc.): + +- **Normalized intermediate schema** (the six CSV stores + JSON sidecars) that + all sources converge on, decoupling collection from import. +- **Idempotent field-level merge** so multiple heterogeneous sources can + contribute to the same asset without clobbering each other. +- **Source-priority resolution** and per-format parsers - the same shape could + wrap an Intune/Azure `Get-MgDevice` export or a SQL query. +- **Machines, PCs, Applications** are already modeled here; an Intune/Azure + source would map naturally onto `computers.csv` + the installed-apps sidecar, + while a MES/CNC database could feed `machines.csv`. diff --git a/scripts/site_imports/assetcollector/sample/assets.csv b/scripts/site_imports/assetcollector/sample/assets.csv new file mode 100644 index 00000000..a4db46bd --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/assets.csv @@ -0,0 +1,5 @@ +assetnumber,name,serialnumber,assettypeid,statusid,locationid,businessunitid,notes +SAMPLE-PC-01,Shopfloor PC 01,DELLSN0001,computer,In Use,A1,Gage Lab,Sample collected device +SAMPLE-PC-02,CMM PC 02,DELLSN0002,computer,In Use,A1,Gage Lab,Sample collected device +SAMPLE-MC-01,Zeiss CMM 01,ZEISSSN0001,machine,In Use,A1,Gage Lab,Sample collected device attached to SAMPLE-PC-02 +SAMPLE-MC-02,Haas VF-2SS 02,HAASSN0002,machine,In Use,A1,Gage Lab,Sample collected standalone CNC diff --git a/scripts/site_imports/assetcollector/sample/computers.csv b/scripts/site_imports/assetcollector/sample/computers.csv new file mode 100644 index 00000000..692119f0 --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/computers.csv @@ -0,0 +1,3 @@ +assetid,computertypeid,hostname,osid,vendorid,modelnumberid,loggedinuser +SAMPLE-PC-01,Shopfloor PC,WKS-SAMPLE01,Windows 10 Enterprise,Dell,OptiPlex 7090,AEROAD\operator1 +SAMPLE-PC-02,CMM PC,WKS-SAMPLE02,Windows 11 Enterprise,Dell,Precision 3660,AEROAD\operator2 diff --git a/scripts/site_imports/assetcollector/sample/machines.csv b/scripts/site_imports/assetcollector/sample/machines.csv new file mode 100644 index 00000000..d55e537f --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/machines.csv @@ -0,0 +1,3 @@ +assetid,machinetypeid,vendorid,modelnumberid,requiresmanualconfig,islocationonly +SAMPLE-MC-01,CMM,Zeiss,CONTURA G2,0,0 +SAMPLE-MC-02,CNC,Haas Automation,VF-2SS,0,0 diff --git a/scripts/site_imports/assetcollector/sample/models.csv b/scripts/site_imports/assetcollector/sample/models.csv new file mode 100644 index 00000000..18f29ac6 --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/models.csv @@ -0,0 +1,5 @@ +modelnumber,vendorid +OptiPlex 7090,Dell +Precision 3660,Dell +VF-2SS,Haas Automation +CONTURA G2,Zeiss diff --git a/scripts/site_imports/assetcollector/sample/operatingsystems.csv b/scripts/site_imports/assetcollector/sample/operatingsystems.csv new file mode 100644 index 00000000..f5ca264e --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/operatingsystems.csv @@ -0,0 +1,3 @@ +osname +Windows 10 Enterprise +Windows 11 Enterprise diff --git a/scripts/site_imports/assetcollector/sample/vendors.csv b/scripts/site_imports/assetcollector/sample/vendors.csv new file mode 100644 index 00000000..a6591f59 --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/vendors.csv @@ -0,0 +1,4 @@ +vendor +Dell +Haas Automation +Zeiss diff --git a/shopdb/core/api/settings.py b/shopdb/core/api/settings.py index 60c919e5..75fe8e14 100644 --- a/shopdb/core/api/settings.py +++ b/shopdb/core/api/settings.py @@ -1,6 +1,7 @@ """Settings API routes.""" import os +import re from flask import Blueprint, request, current_app from flask_jwt_extended import jwt_required, get_jwt_identity @@ -240,6 +241,218 @@ def serve_branding_logo(filename): return send_upload(_branding_dir(), filename) +def _asset_import_dir(): + """Configured Asset Collector CSV location, or '' when unset.""" + setting = Setting.query.filter_by(key='asset_import_csv_location').first() + return (setting.value or '').strip() if setting else '' + + +# The collector script that gets served (with the site's CSV path baked in) from +# the Import Site Assets page. Copied into the image under scripts/ (see +# Dockerfile), so it is always available next to the running API. +_COLLECTOR_SCRIPT = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), + 'scripts', 'site_imports', 'assetcollector', 'Collect-ShopDB.ps1') + + +def _embed_store_path(script_text, path): + """Return the collector script with $NetworkStoreRoot set to path. + + The script ships with a site-specific default on that line; rewriting it + means an operator downloads a script already pointed at this facility's CSV + store and does not have to edit PowerShell by hand. A blank path leaves the + shipped default untouched. + """ + if not path: + return script_text + literal = path.replace("'", "''") + replacement = "$NetworkStoreRoot = '%s'" % literal + # A function replacement is used deliberately: a UNC path is full of + # backslashes, and a string replacement runs them through re's escape + # processing (so \s, \w, a trailing \1 all raise or corrupt). A callable + # returns the text verbatim. + return re.sub( + r"(?m)^\$NetworkStoreRoot\s*=.*$", + lambda _match: replacement, + script_text, + count=1) + + +@settings_bp.route('/asset-import/script', methods=['GET']) +@jwt_required() +@require_role('admin') +def download_collector_script(): + """Download Collect-ShopDB.ps1 with this site's CSV path embedded.""" + try: + with open(_COLLECTOR_SCRIPT, 'r', encoding='utf-8') as handle: + script_text = handle.read() + except OSError: + return error_response(ErrorCodes.NOT_FOUND, + 'Collector script is not available on the server', + http_code=404) + + script_text = _embed_store_path(script_text, _asset_import_dir()) + response = current_app.response_class(script_text, mimetype='text/plain') + response.headers['Content-Disposition'] = 'attachment; filename=Collect-ShopDB.ps1' + return response + + +@settings_bp.route('/asset-import/templates', methods=['GET']) +@jwt_required() +@require_role('admin') +def list_import_templates(): + """List the importable CSV tables in the order they must be uploaded.""" + from shopdb.core.services.csvimport import IMPORTABLE + + tables = [{'table': name, 'filename': '%s.csv' % name} + for name in IMPORTABLE] + return success_response({'tables': tables}) + + +@settings_bp.route('/asset-import/templates/', methods=['GET']) +@jwt_required() +@require_role('admin') +def download_import_template(tablename): + """Download a generated CSV template for a single importable table.""" + from shopdb.core.services.csvimport import ( + IMPORTABLE, generate_template, ImportError_) + + if tablename not in IMPORTABLE: + return error_response(ErrorCodes.NOT_FOUND, + 'No such importable table: %s' % tablename, + http_code=404) + try: + template_text = generate_template(tablename) + except ImportError_ as exc: + return error_response(ErrorCodes.VALIDATION_ERROR, str(exc)) + + response = current_app.response_class(template_text, mimetype='text/csv') + response.headers['Content-Disposition'] = ( + 'attachment; filename=%s.csv' % tablename) + return response + + +@settings_bp.route('/asset-import/import', methods=['POST']) +@jwt_required() +@require_role('admin') +def import_asset_csvs(): + """Validate uploaded CSVs and, when commit=true, load them. + + Mirrors the 'flask csv import' CLI: every file runs through import_csv inside + ONE transaction with a shared resolver, so a computers.csv can resolve the + assets.csv uploaded alongside it. A dry run (the default) rolls the whole + transaction back and reports what WOULD change; commit=true persists it. The + whole run is all-or-nothing - if any file has a row problem, nothing is + written even on a commit. + """ + from sqlalchemy.exc import SQLAlchemyError + from shopdb.core.services.csvimport import ( + ImportError_, Resolver, dependency_order, import_csv, table_from_filename) + + uploads = request.files.getlist('files') + if not uploads: + return error_response(ErrorCodes.VALIDATION_ERROR, 'No CSV files were uploaded') + + commit = str(request.form.get('commit', '')).strip().lower() in ( + '1', 'true', 'yes', 'on') + + # Map each upload to a table by filename. A later file with the same table + # name wins, matching the folder-import behaviour (one file per table). + texts = {} + filenames = {} + for upload in uploads: + name = upload.filename or '' + if not name.lower().endswith('.csv'): + continue + table = table_from_filename(name) + try: + texts[table] = upload.read().decode('utf-8-sig') + except (UnicodeDecodeError, OSError): + return error_response(ErrorCodes.VALIDATION_ERROR, + 'Could not read %s as UTF-8 text' % name) + filenames[table] = name + + if not texts: + return error_response(ErrorCodes.VALIDATION_ERROR, + 'None of the uploaded files were CSV files') + + ordered = dependency_order(list(texts)) + skipped = sorted(set(texts) - set(ordered)) + + reconcile_setting = Setting.query.filter_by( + key='asset_import_reconcile_threshold').first() + try: + reconcile_threshold = float(reconcile_setting.value) if reconcile_setting and reconcile_setting.value else 0.0 + except (TypeError, ValueError): + reconcile_threshold = 0.0 + + resolver = Resolver(reconcile_threshold=reconcile_threshold) + files_report = [] + problem_count = 0 + reconciled_count = 0 + request_error = None + + try: + for table in ordered: + try: + result = import_csv(table, texts[table], resolver=resolver, commit=False) + except ImportError_ as exc: + request_error = '%s: %s' % (filenames[table], str(exc)) + break + except SQLAlchemyError as exc: + db.session.rollback() + request_error = '%s: database error: %s' % ( + filenames[table], str(exc).split(chr(10))[0]) + break + + problems = [ + {'line': p.line, 'column': p.column, 'message': p.message} + for p in result.problems + ] + problem_count += len(problems) + reconciled_count += len(result.reconciliations) + files_report.append({ + 'table': table, + 'filename': filenames[table], + 'created': result.created, + 'updated': result.updated, + 'problems': problems, + 'reconciliations': result.reconciliations, + }) + + applied = False + if request_error is not None: + db.session.rollback() + elif problem_count > 0: + # A row problem anywhere fails the whole run, so nothing is written + # even when commit was asked for. + db.session.rollback() + elif commit: + db.session.commit() + applied = True + else: + db.session.rollback() + except Exception: + db.session.rollback() + raise + + total_created = sum(f['created'] for f in files_report) + total_updated = sum(f['updated'] for f in files_report) + + return success_response({ + 'committed': applied, + 'requested_commit': commit, + 'request_error': request_error, + 'files': files_report, + 'skipped': skipped, + 'problem_count': problem_count, + 'reconciled_count': reconciled_count, + 'reconcile_threshold': reconcile_threshold, + 'total_created': total_created, + 'total_updated': total_updated, + }) + + @settings_bp.route('', methods=['GET']) @jwt_required(optional=True) def list_settings(): @@ -556,6 +769,20 @@ def build_default_settings(): 'category': 'site', 'description': 'Email domain appended to a contact SSO to build email/Teams links (sso@domain). Blank disables contact action buttons.' }, + { + 'key': 'asset_import_csv_location', + 'value': '', + 'valuetype': 'string', + 'category': 'site', + 'description': r'Shared network location where the Asset Collector script deposits its CSVs and where ShopDB reads them for import (e.g. \\server.fqdn\share\folder). This path is baked into the downloadable collector script. Blank = not configured.' + }, + { + 'key': 'asset_import_reconcile_threshold', + 'value': '0.85', + 'valuetype': 'string', + 'category': 'site', + 'description': 'Similarity (0 to 1) a fuzzy lookup match must reach for the asset importer to reconcile a near-miss name (e.g. "Dell Inc." to an existing "Dell") onto an existing vendor, model, type, status, business unit, or location. Reconciled matches are applied but reported for review. 0 disables reconciliation. Operating systems and asset rows are never reconciled.' + }, ] # Site branding. Each instance (ADR-004) can replace the shipped GE defaults diff --git a/shopdb/core/services/csvimport.py b/shopdb/core/services/csvimport.py index d114965f..462e8b57 100644 --- a/shopdb/core/services/csvimport.py +++ b/shopdb/core/services/csvimport.py @@ -34,6 +34,7 @@ not at all. from __future__ import annotations import csv +import difflib import io import re from collections import OrderedDict @@ -86,6 +87,43 @@ PASSWORD_COLUMNS = {'passwordhash', 'password'} TRUE_VALUES = {'1', 'true', 'yes', 'y', 't'} FALSE_VALUES = {'0', 'false', 'no', 'n', 'f', ''} +# Reconciliation: when a name does not match a lookup row exactly, the resolver +# may still bind it to an existing row that is close enough. Only lookup tables +# take part - the small reference tables an operator picks values from. Rows that +# ARE the data being imported (assets, computers, machines) and the fast-moving +# operatingsystems list are excluded, so a typo there is a new row, not a silent +# merge onto something unrelated. +RECONCILABLE_TABLES = { + 'assetstatuses', 'assettypes', 'locationtypes', 'modeltypes', + 'computertypes', 'machinetypes', 'businessunits', 'locations', + 'vendors', 'models', +} + +# Corporate suffixes stripped before comparison so 'Dell Inc.' reconciles to +# 'Dell'. Lowercase, punctuation already removed by the time these are checked. +CORPORATE_SUFFIXES = { + 'inc', 'incorporated', 'llc', 'llp', 'ltd', 'limited', 'corp', + 'corporation', 'co', 'company', 'gmbh', 'ag', 'sa', 'plc', +} + +# Default similarity a fuzzy match must reach, 0..1. Also the value the settings +# default is seeded with; 0 disables reconciliation entirely. +DEFAULT_RECONCILE_THRESHOLD = 0.85 + + +def normalize_label(value): + """Fold a label to its comparison form: lowercase, no punctuation, no + corporate suffix, single-spaced. 'Dell Inc.' and 'DELL' both become 'dell'. + """ + text = (value or '').strip().lower() + text = re.sub(r'[^a-z0-9]+', ' ', text).strip() + if not text: + return '' + words = [w for w in text.split(' ') if w] + while len(words) > 1 and words[-1] in CORPORATE_SUFFIXES: + words.pop() + return ' '.join(words) + def table_for(name): table = db.metadata.tables.get(name) @@ -221,10 +259,17 @@ class Resolver: Rows created earlier in the SAME import are resolvable, so one file can reference another that was loaded moments ago without a commit in between. + + When a name does not match exactly and the table is reconcilable, a + normalized then fuzzy comparison against the existing rows may bind it to a + close-enough match. Every such binding is recorded so the caller can report + it for review; nothing is merged silently. """ - def __init__(self): + def __init__(self, reconcile_threshold=0.0): self._cache = {} + self.reconcile_threshold = reconcile_threshold or 0.0 + self.events = [] def resolve(self, tablename, idcolumn, value): if value is None: @@ -249,6 +294,10 @@ class Resolver: db.func.lower(db.func.trim(table.c[label])) == str(value).strip().lower()) ).first() if row is None: + found = self._reconcile(tablename, idcolumn, label, table, value) + if found is not None: + self._cache[key] = found + return found raise ValueError( "nothing in %s is named '%s' - add it to %s.csv, or import that file first" % (tablename, value, tablename)) @@ -256,6 +305,52 @@ class Resolver: self._cache[key] = found return found + def _reconcile(self, tablename, idcolumn, label, table, value): + """Bind value to the closest existing row when within threshold. + + Returns the matched id and records an event, or None when reconciliation + is off, the table is not reconcilable, or nothing is close enough. + """ + if self.reconcile_threshold <= 0 or tablename not in RECONCILABLE_TABLES: + return None + + wanted = normalize_label(value) + if not wanted: + return None + + best_id = None + best_label = None + best_score = 0.0 + for existing in db.session.execute(table.select()): + candidate = getattr(existing, label, None) + normalized = normalize_label(candidate) + if not normalized: + continue + if normalized == wanted: + score = 1.0 + else: + score = difflib.SequenceMatcher(None, wanted, normalized).ratio() + if score > best_score: + best_score = score + best_id = getattr(existing, idcolumn) + best_label = candidate + + if best_id is not None and best_score >= self.reconcile_threshold: + self.events.append({ + 'table': tablename, + 'input': str(value).strip(), + 'matched': best_label, + 'score': round(best_score, 3), + }) + return best_id + return None + + def take_events(self): + """Return and clear reconciliations recorded since the last call.""" + events = self.events + self.events = [] + return events + def remember(self, tablename, name, rowid): if name is None: return @@ -282,6 +377,7 @@ class ImportResult(object): self.updated = 0 self.problems = [] self.skipped = 0 + self.reconciliations = [] @property def ok(self): @@ -381,6 +477,10 @@ def import_csv(tablename, text, resolver=None, commit=False): continue try: values[name] = resolver.resolve(target[0], target[1], text) + for event in resolver.take_events(): + event['line'] = line + event['column'] = name + result.reconciliations.append(event) except ValueError as exc: result.problems.append(RowProblem(line, name, str(exc))) failed = True From bdef21da509c4545d06844bea3e7709da65110b4 Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Tue, 25 Aug 2026 16:46:00 -0400 Subject: [PATCH 02/14] add Entra device identity to computers and Azure/Thor import sources PC records now carry Entra/Azure device identity so imported machines keep a durable link back to their directory object instead of matching on hostname alone. Adds azuredeviceid, azureobjectid, and azuredevicecategory to the Computer model with matching migrations, surfaces them read-only in PCDetail as an Entra Device Info card, and lets PCForm edit the device id directly. The computers plugin registers a machineassetnumber import hook that reconciles PC-to-machine links during bulk import and warns on unresolved or conflicting relationships. Adds importsources adapters that normalize Azure device-group and Thor PC inventory exports into canonical ShopDB CSVs via hostname alignment, exposes register_import_hook and RowWarning on the shopdb.api contract surface (contract 0.21.0), and refreshes the Import Site Assets page with clearer source instructions. Includes reconcile_locations maintenance script and coverage for the new import paths. --- .dockerignore | 23 ++ Dockerfile | 6 + frontend/package-lock.json | 4 +- frontend/src/api/index.js | 14 +- .../src/views/settings/ImportSiteAssets.vue | 385 ++++++++++++++++-- plugins/computers/api/routes.py | 8 +- plugins/computers/frontend/views/PCDetail.vue | 22 + plugins/computers/frontend/views/PCForm.vue | 14 + .../versions/0002_computer_azuredeviceid.py | 40 ++ .../versions/0003_computer_azure_fields.py | 45 ++ plugins/computers/models/computer.py | 17 + plugins/computers/plugin.py | 284 ++++++++++++- scripts/reconcile_locations.py | 156 +++++++ shopdb/__init__.py | 10 +- shopdb/api/__init__.py | 12 + shopdb/core/api/settings.py | 112 ++++- shopdb/core/services/csvimport.py | 233 ++++++++++- shopdb/core/services/importsources.py | 325 +++++++++++++++ tests/test_csv_import.py | 52 ++- tests/test_import_sources.py | 201 +++++++++ .../test_computers_import_machine_link.py | 187 +++++++++ 21 files changed, 2064 insertions(+), 86 deletions(-) create mode 100644 .dockerignore create mode 100644 plugins/computers/migrations/versions/0002_computer_azuredeviceid.py create mode 100644 plugins/computers/migrations/versions/0003_computer_azure_fields.py create mode 100644 scripts/reconcile_locations.py create mode 100644 shopdb/core/services/importsources.py create mode 100644 tests/test_import_sources.py create mode 100644 tests/test_plugins/test_computers_import_machine_link.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..3ea9bfaa --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +# Keep the build context lean and prevent host artifacts from clobbering +# in-image installs. node_modules is platform-specific: copying the host's +# (Windows) tree over the container's `npm ci` breaks rollup's native binary. +**/node_modules +frontend/dist +frontend/src/.plugins-staged +frontend/src/router/routes.gen.js + +# Python caches and virtualenvs +**/__pycache__ +**/*.pyc +.venv +venv + +# Local dev and VCS noise +.git +.gitignore +*.log +.env.local + +# Installer build artifacts (not part of the app image) +installer-arch-dev/build +installer-arch-dev/dist diff --git a/Dockerfile b/Dockerfile index c6567d3c..a747c3fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,12 @@ FROM node:20-slim AS frontendbuild WORKDIR /build +# Trust the site's corporate root CA so npm's TLS to the registry validates +# behind an inspecting proxy. Without it npm 10.x fails silently and leaves an +# incomplete node_modules. +COPY corp-root-ca.pem /usr/local/share/ca-certificates/corp-root-ca.crt +ENV NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/corp-root-ca.crt + # Copy only the manifests first so `npm ci` caches on dependency changes. COPY frontend/package*.json ./frontend/ RUN cd frontend && npm ci diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 9b279c06..c069d5c7 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "shopdb-frontend", - "version": "0.7.0", + "version": "0.12.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "shopdb-frontend", - "version": "0.7.0", + "version": "0.12.0", "dependencies": { "@fontsource-variable/inter": "^5.2.8", "@fullcalendar/core": "^6.1.20", diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 6712925c..0f0fccd3 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -1027,14 +1027,26 @@ export const settingsApi = { // Generated CSV template (header + example row) for one table return api.get(`/settings/asset-import/templates/${encodeURIComponent(tablename)}`, { responseType: 'blob' }) }, - importAssetCsvs(files, commit = false) { + importAssetCsvs(files, commit = false, options = {}) { // Multipart upload of one or more CSVs. commit=false is a dry-run preview // that writes nothing; commit=true applies the whole set in one transaction. + // options.source picks a foreign adapter (azure, thor); default native means + // the files are already canonical per-table CSVs. options.computertype is + // stamped onto every imported computer when a source assigns a PC type. const form = new FormData() for (const file of files) { form.append('files', file) } form.append('commit', commit ? 'true' : 'false') + if (options.source) { + form.append('source', options.source) + } + if (options.computertype) { + form.append('computertype', options.computertype) + } + if (options.category) { + form.append('category', options.category) + } return api.post('/settings/asset-import/import', form, { headers: { 'Content-Type': 'multipart/form-data' } }) diff --git a/frontend/src/views/settings/ImportSiteAssets.vue b/frontend/src/views/settings/ImportSiteAssets.vue index 60ed215c..8cfeeba2 100644 --- a/frontend/src/views/settings/ImportSiteAssets.vue +++ b/frontend/src/views/settings/ImportSiteAssets.vue @@ -91,32 +91,6 @@ -
-

CSV Import Templates

-

- These are early-draft import templates generated from the live ShopDB - schema. Each file has a header row and a commented example. Fill one out - (or let the collector script populate them) and import them in the order - below - a table never references something that has not been loaded yet. - Foreign keys accept the referenced row's name or its numeric id, and - re-importing a file updates matching rows instead of duplicating them. -

- -

Loading...

-
    -
  1. -
    - {{ t.filename }} - -
    -
    2. -
- -

{{ templatesError }}

-
-

Import CSVs

@@ -128,6 +102,108 @@ nothing is written.

+ + +
+

+ An Azure device-group export lists device hostnames plus their Entra + deviceId. The device displayName is matched to an existing PC by + hostname (case-insensitive): a match updates that PC in place and + stores its deviceId, object id, and category; a device with no + matching hostname creates a new computer (assettype computer, status + In Use). It carries no model, OS, location, or user data - enrich + those later with the collector or a Thor import. +

+ + + +
+ How to export group members from Azure +
    +
  1. Open the Microsoft Entra admin center (entra.microsoft.com).
    2. +
  2. Go to Groups, then open the device group you want to import.
    3. +
  3. Select Members in the left menu.
    4. +
  4. Click Bulk Operations->Download Members to export the CSV.
    5. +
  5. Upload that CSV here with the source set to Azure device group.
    6. +
+
+
+ +
+

+ A Thor PC-inventory export is a wide, fixed-format dump. This maps the + hostname, BIOS manufacturer, model, OS, logged-in user, last-reported + and boot times, and location into ShopDB. Each host is matched to an + existing PC by hostname (case-insensitive): a match updates that PC in + place; a host with no matching hostname creates a new computer. + Network details (MAC, switch, VLAN) and the machine relationship are + not imported in this pass. +

+
+
+

+ {{ report.warning_count }} warning(s): these rows still import, but + something needs a look - most often a machine link that could not be + made. Review them below; each says what to do. +

+ @@ -206,6 +288,7 @@ + @@ -218,6 +301,9 @@ + @@ -244,6 +330,23 @@ +
+

Warnings

+

+ Each row below imported, but something could not be completed - most + often a PC's machine link. Nothing here blocks the import; fix what + you can and re-import to complete the links. +

+
    +
  • + {{ w.filename }} + line {{ w.line }}: + {{ w.reason }} +
    • +
+
+

Problems to fix

    @@ -261,23 +364,57 @@
+
+

CSV Import Templates

+

+ These import templates are generated from the live ShopDB schema. Each + file has a header row plus a small worked example - commented rows you + can uncomment and edit. The examples are internally consistent across + every file (the same locations, vendors, and types are referenced), so + you can download the full set, keep it as a starter workbook, and fill + in your own assets right away. Import them in the order below - a table + never references something that has not been loaded yet. Foreign keys + accept the referenced row's name or its numeric id, and re-importing a + file updates matching rows instead of duplicating them. +

+ +

Loading...

+
    +
  1. +
    + {{ t.filename }} + +
    +
    2. +
+ +

{{ templatesError }}

+
+

PC to Machine Relationships

A PC that drives a machine (a CMM PC, a DNC/eDNC host) is linked to that - machine so the two show up connected across ShopDB. That link is built - automatically by the Asset Collector, not by this CSV import: the script - reads the four-digit workstation number from the PC it runs on and - reports it, and ShopDB matches that number to a machine's - assetnumber to create the connection. + machine so the two show up connected across ShopDB. You can build that + link two ways. The Asset Collector builds it automatically: the script + reads the four-digit workstation number from the PC it runs on, reports + it, and ShopDB matches that number to a machine's + assetnumber.

- What this means for import order: load your machines first (so their - assetnumber values exist), then let the collector run on - each PC. A machine number the collector reports that does not match an - existing machine is flagged rather than guessed, so nothing links to the - wrong asset. Building this relationship straight from a CSV column is - planned as a follow-up. + Or build it straight from the spreadsheet: put the machine's + assetnumber in the machineassetnumber column on + computers.csv, and the import creates the link for you. +

+

+ Either way, load your machines first (so their + assetnumber values exist), then the PCs. A machine number + that does not match an existing machine - or one already driven by + another PC - is reported as a warning and skipped, not guessed, so + nothing links to the wrong asset. The PC still imports; fix the number + and re-import to complete the link.

@@ -285,7 +422,7 @@ @@ -540,6 +751,75 @@ onMounted(async () => { color: var(--text); font-family: var(--font-mono, monospace); } +.field select { + padding: 8px 10px; + border: 1px solid var(--border); + border-radius: 6px; + background: var(--bg); + color: var(--text); +} +.custom-type-row { + display: flex; + gap: 8px; + align-items: center; +} +.custom-type-row input { + flex: 1; +} +.source-cards { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 10px; + margin: 4px 0; +} +.source-card { + display: flex; + flex-direction: column; + gap: 4px; + text-align: left; + padding: 12px 14px; + border: 1px solid var(--border); + border-radius: 8px; + background: var(--bg); + color: var(--text); + cursor: pointer; +} +.source-card:hover { + border-color: var(--primary); +} +.source-card.active { + border-color: var(--primary); + box-shadow: 0 0 0 1px var(--primary) inset; +} +.source-card-title { + font-weight: 600; +} +.source-card-desc { + font-size: 0.85rem; + color: var(--text-light); +} +.source-help { + margin-top: 12px; + border-left: 3px solid var(--border); + padding-left: 12px; +} +.source-help .instructions { + margin-top: 8px; +} +.source-help .instructions summary { + cursor: pointer; + font-weight: 600; + color: var(--text); +} +.source-help .instructions ol { + margin: 8px 0 0; + padding-left: 20px; + color: var(--text-light); + font-size: 0.9rem; +} +.source-help .instructions li { + margin: 4px 0; +} .actions { display: flex; align-items: center; @@ -641,6 +921,33 @@ onMounted(async () => { margin: 4px 0; font-size: 0.9rem; } +.report-table td.has-warnings { + color: var(--amber, #d18b16); + font-weight: 700; +} +.report-banner.amber { + background: var(--amber-bg, rgba(209, 139, 22, 0.15)); + color: var(--amber, #d18b16); +} +.warnings { + margin-top: 12px; +} +.warnings h4 { + margin: 0 0 6px; + color: var(--amber, #d18b16); +} +.warnings ul { + margin: 6px 0 0; + padding-left: 20px; +} +.warnings li { + margin: 4px 0; + font-size: 0.9rem; +} +.warnings li em { + color: var(--text-muted, #9aa4b2); + font-style: normal; +} .problems { margin-top: 12px; } diff --git a/plugins/computers/api/routes.py b/plugins/computers/api/routes.py index 94e3b828..035198ad 100644 --- a/plugins/computers/api/routes.py +++ b/plugins/computers/api/routes.py @@ -582,6 +582,9 @@ def create_computer(): vendorid=data.get('vendorid'), modelnumberid=data.get('modelnumberid'), loggedinuser=data.get('loggedinuser'), + azuredeviceid=data.get('azuredeviceid'), + azureobjectid=data.get('azureobjectid'), + azuredevicecategory=data.get('azuredevicecategory'), lastreporteddate=data.get('lastreporteddate'), lastboottime=data.get('lastboottime') ) @@ -686,8 +689,9 @@ def update_computer(computer_id: int): # Update computer fields computer_fields = ['computertypeid', 'hostname', 'osid', 'vendorid', - 'modelnumberid', 'loggedinuser', 'lastreporteddate', - 'lastboottime'] + 'modelnumberid', 'loggedinuser', 'azuredeviceid', + 'azureobjectid', 'azuredevicecategory', + 'lastreporteddate', 'lastboottime'] for key in computer_fields: if key in data: old_val = getattr(comp, key) diff --git a/plugins/computers/frontend/views/PCDetail.vue b/plugins/computers/frontend/views/PCDetail.vue index 633aa2e4..e1ca33da 100644 --- a/plugins/computers/frontend/views/PCDetail.vue +++ b/plugins/computers/frontend/views/PCDetail.vue @@ -152,6 +152,28 @@ + + +
+

Entra Device Info

+
+
+ Device ID + {{ computer.computer.azuredeviceid }} +
+
+ Object ID + {{ computer.computer.azureobjectid }} +
+
+ Device Category + {{ computer.computer.azuredevicecategory }} +
+
+
diff --git a/plugins/computers/frontend/views/PCForm.vue b/plugins/computers/frontend/views/PCForm.vue index 5921fb41..9b125aab 100644 --- a/plugins/computers/frontend/views/PCForm.vue +++ b/plugins/computers/frontend/views/PCForm.vue @@ -223,6 +223,17 @@ +
+ + +
+
@@ -383,6 +394,7 @@ const form = ref({ locationid: '', osid: '', loggedinuser: '', + azuredeviceid: '', accessmethods: [], notes: '', mapx: null, @@ -486,6 +498,7 @@ onMounted(async () => { locationid: pc.locationid || '', osid: ext.osid || '', loggedinuser: ext.loggedinuser || '', + azuredeviceid: ext.azuredeviceid || '', accessmethods: (pc.accessmethods || []).map(a => ({ protocolid: a.protocolid, portoverride: a.portoverride ?? null @@ -555,6 +568,7 @@ async function savePC() { locationid: form.value.locationid || null, osid: form.value.osid || null, loggedinuser: form.value.loggedinuser || null, + azuredeviceid: form.value.azuredeviceid || null, accessmethods: form.value.accessmethods, notes: form.value.notes || null, ipaddress: form.value.ipaddress || null, diff --git a/plugins/computers/migrations/versions/0002_computer_azuredeviceid.py b/plugins/computers/migrations/versions/0002_computer_azuredeviceid.py new file mode 100644 index 00000000..72c5fe0b --- /dev/null +++ b/plugins/computers/migrations/versions/0002_computer_azuredeviceid.py @@ -0,0 +1,40 @@ +"""Add azuredeviceid to computers (Entra/Azure device-group imports). + +The core chain created computers WITHOUT an azuredeviceid column. This plugin +revision adds it so BOTH fresh installs (core chain builds the table, then this +adds the column) and existing installs get it. Guarded/idempotent: skips when +the table is absent (plugin disabled) or the column already exists (e.g. a test +DB built by db.create_all() from the model). + +Revision ID: computers0002azureid +Revises: computers0001anchor +""" +from alembic import op +import sqlalchemy as sa + + +revision = 'computers0002azureid' +down_revision = 'computers0001anchor' +branch_labels = None +depends_on = None + + +def upgrade(): + bind = op.get_bind() + insp = sa.inspect(bind) + if 'computers' not in insp.get_table_names(): + return + cols = {c['name'] for c in insp.get_columns('computers')} + if 'azuredeviceid' not in cols: + op.add_column('computers', + sa.Column('azuredeviceid', sa.String(length=64), nullable=True)) + + +def downgrade(): + bind = op.get_bind() + insp = sa.inspect(bind) + if 'computers' not in insp.get_table_names(): + return + cols = {c['name'] for c in insp.get_columns('computers')} + if 'azuredeviceid' in cols: + op.drop_column('computers', 'azuredeviceid') diff --git a/plugins/computers/migrations/versions/0003_computer_azure_fields.py b/plugins/computers/migrations/versions/0003_computer_azure_fields.py new file mode 100644 index 00000000..6ce93b29 --- /dev/null +++ b/plugins/computers/migrations/versions/0003_computer_azure_fields.py @@ -0,0 +1,45 @@ +"""Add azureobjectid and azuredevicecategory to computers. + +The Azure device-group import now also records the Entra directory object id +(the CSV 'id' column) and an operator-supplied device category (the group +membership label, e.g. SFLD_DYN_Device_Group_...). This revision adds both +columns. Guarded/idempotent: skips when the table is absent (plugin disabled) +or a column already exists (e.g. a test DB built by db.create_all()). + +Revision ID: computers0003azurefields +Revises: computers0002azureid +""" +from alembic import op +import sqlalchemy as sa + + +revision = 'computers0003azurefields' +down_revision = 'computers0002azureid' +branch_labels = None +depends_on = None + + +def upgrade(): + bind = op.get_bind() + insp = sa.inspect(bind) + if 'computers' not in insp.get_table_names(): + return + cols = {c['name'] for c in insp.get_columns('computers')} + if 'azureobjectid' not in cols: + op.add_column('computers', + sa.Column('azureobjectid', sa.String(length=64), nullable=True)) + if 'azuredevicecategory' not in cols: + op.add_column('computers', + sa.Column('azuredevicecategory', sa.String(length=255), nullable=True)) + + +def downgrade(): + bind = op.get_bind() + insp = sa.inspect(bind) + if 'computers' not in insp.get_table_names(): + return + cols = {c['name'] for c in insp.get_columns('computers')} + if 'azuredevicecategory' in cols: + op.drop_column('computers', 'azuredevicecategory') + if 'azureobjectid' in cols: + op.drop_column('computers', 'azureobjectid') diff --git a/plugins/computers/models/computer.py b/plugins/computers/models/computer.py index 6a35b4fc..c0ca3463 100644 --- a/plugins/computers/models/computer.py +++ b/plugins/computers/models/computer.py @@ -85,6 +85,23 @@ class Computer(BaseModel): lastreporteddate = db.Column(db.DateTime, nullable=True) lastboottime = db.Column(db.DateTime, nullable=True) + # Entra/Azure device identity (set by Azure device-group imports) + azuredeviceid = db.Column( + db.String(64), + nullable=True, + comment='Entra/Azure deviceId GUID' + ) + azureobjectid = db.Column( + db.String(64), + nullable=True, + comment='Entra/Azure directory object id GUID' + ) + azuredevicecategory = db.Column( + db.String(255), + nullable=True, + comment='Entra device category / group membership label set at import' + ) + # Remote access is now modeled per-protocol via the accessmethods # relationship (AccessProtocol / ComputerAccess), replacing isvnc/iswinrm. diff --git a/plugins/computers/plugin.py b/plugins/computers/plugin.py index 96f1e6b6..086836aa 100644 --- a/plugins/computers/plugin.py +++ b/plugins/computers/plugin.py @@ -34,6 +34,14 @@ MEASURINGTOOL_LINK_ORIGIN = 'collector:measuringtool' # touched. MACHINE_LINK_ORIGIN = 'collector:machine' +# Marker stamped on the PC->machine link built from a CSV import's machine +# number column. A DISTINCT origin from the collector's so the two provenances +# stay separable: an import establishes a link once from a spreadsheet, the +# collector maintains it continuously from the floor, and a site may use one, +# the other, or both. Same archive discipline - only rows carrying this label +# are touched by a re-import. +MACHINE_LINK_IMPORT_ORIGIN = 'import:machine' + # Marker stamped on both links a part-marker PC produces: PC controls marker, # and marker partof the operation it serves. Same archive discipline again. PARTMARKER_LINK_ORIGIN = 'collector:partmarker' @@ -90,8 +98,224 @@ class ComputersPlugin(BasePlugin): def init_app(self, app: Flask, db_instance) -> None: """Initialize plugin with Flask app.""" + self._register_import_hook() logger.info(f"Computers plugin initialized (v{self.meta.version})") + # -- bulk-import machine link ------------------------------------------- + + def _register_import_hook(self): + """Own a `machineassetnumber` column on computers.csv. + + A site with a spreadsheet knows "this PC drives machine 3015" and wants + to write that number in the PC row, not hand-build an assetrelationships + sheet keyed on ids it never sees. The core importer stays ignorant of + what the column means: it accepts the header, keeps it out of the + computers write, and hands the value to this hook after each PC row is + written, so the PC's asset id exists. Registration is idempotent, so a + plugin reload or a test re-init does not double-fire. + """ + from shopdb.api import register_import_hook + + register_import_hook( + 'computers', + {'machineassetnumber': + 'the asset number of the machine this PC drives (from ' + 'machines.csv); builds the PC-controls-machine link. A number ' + 'that has no machine, or one already driven by another PC, is ' + 'reported as a warning and skipped - the PC still imports.'}, + self._import_machine_link) + + def _import_machine_link(self, rowvalues, virtualvalues, line): + """Build a PC->machine controls link from an import row's machine number. + + Runs inside the importer's transaction - a dry run rolls it back - so it + reuses the collector's mature _sync_machine_link rather than inventing a + second linker, but under the import origin (so an import never archives + a collector's rows and vice versa) and in preview mode (so a bulk import + never emails an alert per row; the findings are the report's amber + warnings instead). + + Returns RowWarning list, never raises: a machine number that resolves to + no asset is an amber note beside a PC that still imports, not a failed + file. Every warning names the PC, the number, and the one fix. + """ + from shopdb.api import RowWarning + + machinenumber = (virtualvalues.get('machineassetnumber') or '').strip() + if not machinenumber: + return [] + + assetid = rowvalues.get('assetid') + comp = Computer.query.filter_by(assetid=assetid).first() if assetid else None + if comp is None: + # The PC row was written but its Computer extension is not readable + # here - nothing to link to. Amber, not fatal. + return [RowWarning( + line, 'machineassetnumber', + reason='PC row not found to link to machine {!r}'.format( + machinenumber), + action='re-run the import; if it recurs, check the computers ' + 'row imported', + target=machinenumber)] + + hostname = comp.hostname or (assetid and str(assetid)) or '?' + + # IN-FILE FAN-OUT GUARD. Two rows in one computers.csv both naming the + # same machine is a spreadsheet mistake - a copied cell, a wrong + # workstation number - not a swap. But _incumbent_has_yielded reads a + # freshly-imported PC (no lastreporteddate) as having yielded, so + # without this the SECOND row would silently archive the first row's + # link and steal the machine, and the file would import clean with the + # wrong PC on the bay. So the first claim on a machine this run wins and + # is remembered; a later row for the same machine is warned and skipped. + # The claim set lives on db.session.info, which Flask-SQLAlchemy resets + # at request teardown, so it is scoped to exactly this one import and + # never leaks into the next. + machine = self._asset_by_number(machinenumber) + if machine is not None and machine.assetid == assetid: + # The number names this very PC. Happens on a PC first seen before + # the machine-number convention, whose own asset number IS the + # machine number. Skip - a PC does not control itself. + return [RowWarning( + line, 'machineassetnumber', + reason='machine number {!r} is this PC\'s own asset number; ' + 'not linked'.format(machinenumber), + action='give this PC a hostname-based asset number, or point ' + 'machineassetnumber at the machine it drives', + subject=hostname, target=machinenumber)] + + if machine is not None: + # A location-only machine is a virtual bay marker, not a real + # machine, so nothing controls it. Skip rather than build a link + # that would read as a PC driving a location. + skip = self._location_only_skip( + machine, hostname, machinenumber, line) + if skip is not None: + return [skip] + + if machine is not None and machine.assetid != assetid: + # ALREADY-CONTROLS-DIFFERENT. Unlike the collector - which reads a + # new machine number as a PC moved to a new bay and silently + # re-bays it - an import naming a machine when this PC already + # actively controls a DIFFERENT one is far more likely a wrong + # workstation number in the sheet than a real move. Warn and skip + # rather than archive a link a person may have set on purpose. + other = self._active_controlled_machine(assetid, machine.assetid) + if other is not None: + return [RowWarning( + line, 'machineassetnumber', + reason='{} already controls machine {}; not moved to {}' + .format(hostname, other, machine.assetnumber), + action='if this PC really moved, clear the existing link ' + 'first; otherwise correct the machine number on ' + 'this row', + subject=hostname, target=machinenumber)] + + if machine is not None and machine.assetid != assetid: + claimed = db.session.info.setdefault('import_machine_claims', {}) + # Keyed by machine asset id -> (claimant PC asset id, its hostname). + # The id is what the guard compares on (the same PC re-stating its + # own link is not a conflict); the hostname is only for the message. + firstclaimant = claimed.get(machine.assetid) + if firstclaimant is not None and firstclaimant[0] != assetid: + return [RowWarning( + line, 'machineassetnumber', + reason='machine {} was already claimed by {} earlier in ' + 'this import; {} not linked'.format( + machine.assetnumber, firstclaimant[1], hostname), + action='a machine drives one PC - correct the machine ' + 'number on one of these rows so each PC names its ' + 'own machine', + subject=hostname, target=machinenumber)] + claimed[machine.assetid] = (assetid, hostname) + + # _sync_machine_link appends plain strings to its warnings list. Collect + # them and translate to structured, actionable RowWarnings so the UI can + # show the PC, the machine number, and the corrective step. + rawwarnings = [] + self._sync_machine_link( + comp, machinenumber, rawwarnings, + origin_label=MACHINE_LINK_IMPORT_ORIGIN, preview=True) + + out = [] + for text in rawwarnings: + out.append(RowWarning( + line, 'machineassetnumber', + reason=text, + action=self._machine_link_action(text, machinenumber), + subject=hostname, target=machinenumber)) + return out + + @staticmethod + def _active_controlled_machine(pcassetid, excludeassetid): + """Asset number of a machine this PC actively controls under the import + origin, other than excludeassetid, or None. + + Scoped to MACHINE_LINK_IMPORT_ORIGIN so an import only reasons about its + own links: a collector link or a hand-made one is the collector's or a + person's to move, not the import's to block on. + """ + from shopdb.api import AssetRelationship, RelationshipType, Asset + + controls = RelationshipType.query.filter_by( + relationshiptype='controls').first() + if not controls: + return None + rel = AssetRelationship.query.filter( + AssetRelationship.sourceassetid == pcassetid, + AssetRelationship.relationshiptypeid == controls.relationshiptypeid, + AssetRelationship.label == MACHINE_LINK_IMPORT_ORIGIN, + AssetRelationship.targetassetid != excludeassetid, + AssetRelationship.isactive.is_(True), + ).first() + if rel is None: + return None + asset = db.session.get(Asset, rel.targetassetid) + return asset.assetnumber if asset else str(rel.targetassetid) + + @staticmethod + def _location_only_skip(machine, hostname, machinenumber, line): + """A RowWarning if the resolved asset is a location-only marker, else None. + + A machine flagged islocationonly is a virtual bay, not a real machine, + so a PC does not control it. Guarded behind the machines plugin being + installed - a site without it cannot have a location-only machine, and + the import must not fail because the model is absent. + """ + from shopdb.api import RowWarning + try: + from plugins.machines.models import Machine + except ImportError: + return None + row = Machine.query.filter_by(assetid=machine.assetid).first() + if row is None or not row.islocationonly: + return None + return RowWarning( + line, 'machineassetnumber', + reason='machine {} is a location-only marker, not a real machine; ' + 'not linked'.format(machine.assetnumber), + action='point machineassetnumber at a real machine, or clear it ' + 'for this PC', + subject=hostname, target=machinenumber) + + @staticmethod + def _machine_link_action(text, machinenumber): + """The one corrective step for a machine-link warning, by its shape.""" + lowered = text.lower() + if 'no asset for machine number' in lowered: + return ('create the machine asset with number {!r} (import ' + 'machines.csv first), or correct the number on this PC ' + 'row'.format(machinenumber)) + if 'still held by' in lowered: + return ('confirm which PC drives this machine; retire or move the ' + 'old PC off In Use to hand the machine over') + if 'taken over from' in lowered: + return ('expected on a PC swap; set the old PC to Retired, ' + 'Inventory or In Repair') + if 'relationship type missing' in lowered: + return 'run flask seed reference-data, then re-import' + return 'review this PC and its machine number' + # -- ADR-006 collector contract ----------------------------------------- def get_collector_schema(self) -> Optional[Dict]: @@ -647,7 +871,8 @@ class ComputersPlugin(BasePlugin): # -- measuring-tool sync ----------------------------------------------- - def _sync_machine_link(self, comp, machinenumber, warnings): + def _sync_machine_link(self, comp, machinenumber, warnings, + origin_label=MACHINE_LINK_ORIGIN, preview=False): """Link a PC to the machine it drives, from the reported machine number. Until this existed the machine number was collected and then discarded, @@ -669,6 +894,15 @@ class ComputersPlugin(BasePlugin): cadence, alerting on every pass. The incumbent therefore keeps the machine while it is still alive, and the challenger is recorded as a dormant link. See _incumbent_has_yielded for what alive means. + + origin_label separates provenance: the collector stamps + MACHINE_LINK_ORIGIN, a CSV import stamps MACHINE_LINK_IMPORT_ORIGIN, and + each only ever archives its own rows so the two sources coexist. + + preview=True is for a dry-run import: DB writes still happen in the + caller's transaction (which it rolls back), but the alert fan-out - the + one side effect a rollback cannot undo - is suppressed so a preview + never emails anyone. """ from shopdb.api import AssetRelationship, RelationshipType, Asset @@ -700,7 +934,7 @@ class ComputersPlugin(BasePlugin): mine = AssetRelationship.query.filter( AssetRelationship.sourceassetid == pcasset.assetid, AssetRelationship.relationshiptypeid == controls.relationshiptypeid, - AssetRelationship.label == MACHINE_LINK_ORIGIN, + AssetRelationship.label == origin_label, ).all() link = None @@ -710,11 +944,23 @@ class ComputersPlugin(BasePlugin): else: rel.isactive = False + # The unique constraint on assetrelationships is (source, target, type) + # and does NOT include label, so an import (import:machine) and the + # collector (collector:machine) cannot each hold their OWN row for the + # same PC->machine control. If our label-scoped lookup found nothing but + # a row for this exact triple exists under the other provenance, adopt + # it rather than insert a duplicate that would violate the constraint. + if link is None: + link = AssetRelationship.query.filter_by( + sourceassetid=pcasset.assetid, + targetassetid=machine.assetid, + relationshiptypeid=controls.relationshiptypeid).first() + # Whoever actively holds this machine now, if it is not this PC. held = AssetRelationship.query.filter( AssetRelationship.targetassetid == machine.assetid, AssetRelationship.relationshiptypeid == controls.relationshiptypeid, - AssetRelationship.label == MACHINE_LINK_ORIGIN, + AssetRelationship.label == origin_label, AssetRelationship.sourceassetid != pcasset.assetid, AssetRelationship.isactive.is_(True), ).all() @@ -737,9 +983,10 @@ class ComputersPlugin(BasePlugin): sourceassetid=pcasset.assetid, targetassetid=machine.assetid, relationshiptypeid=controls.relationshiptypeid, - label=MACHINE_LINK_ORIGIN, + label=origin_label, isactive=False)) - self._alert_machine_contested(names, comp, machine) + if not preview: + self._alert_machine_contested(names, comp, machine) else: link.isactive = False return [{'assetid': machine.assetid, @@ -753,7 +1000,7 @@ class ComputersPlugin(BasePlugin): sourceassetid=pcasset.assetid, targetassetid=machine.assetid, relationshiptypeid=controls.relationshiptypeid, - label=MACHINE_LINK_ORIGIN, + label=origin_label, isactive=True) db.session.add(link) else: @@ -767,7 +1014,8 @@ class ComputersPlugin(BasePlugin): warnings.append( 'machine {} was taken over from {}; check that PC'.format( machine.assetnumber, oldname)) - self._alert_pc_superseded(oldname, comp, machine) + if not preview: + self._alert_pc_superseded(oldname, comp, machine) return [{'assetid': machine.assetid, 'machinenumber': machine.assetnumber, @@ -870,8 +1118,8 @@ class ComputersPlugin(BasePlugin): # # A row made by hand carries neither label and is deliberately NOT # counted: a person's link is not the collector's to archive. - ourlabels = (label, MACHINE_LINK_ORIGIN, MEASURINGTOOL_LINK_ORIGIN, - PARTMARKER_LINK_ORIGIN) + ourlabels = (label, MACHINE_LINK_ORIGIN, MACHINE_LINK_IMPORT_ORIGIN, + MEASURINGTOOL_LINK_ORIGIN, PARTMARKER_LINK_ORIGIN) others = AssetRelationship.query.filter( AssetRelationship.targetassetid == deviceasset.assetid, AssetRelationship.relationshiptypeid == controls.relationshiptypeid, @@ -1423,15 +1671,17 @@ class ComputersPlugin(BasePlugin): sourceassetid=pcasset.assetid, targetassetid=adopted.assetid, relationshiptypeid=controls.relationshiptypeid).first() - if claimed is not None and claimed.label == MACHINE_LINK_ORIGIN: + if claimed is not None and claimed.label in ( + MACHINE_LINK_ORIGIN, MACHINE_LINK_IMPORT_ORIGIN): # LEAVE IT ALONE. On a CMM the instrument IS the reported bay, - # so the machine sync has already made the row for this exact - # triple - and it finds its own rows BY that label. Relabelling - # hid the link from it, so the next cycle built a second row for - # the same triple and MySQL rejected it: the PC reported 200, - # then 500 forever after. The row's lifecycle (including staying - # dormant while a bay is contested) belongs to the machine sync; - # adoption only needs the identity, which it has. + # so the machine sync (or a CSV import's machine link) has + # already made the row for this exact triple - and each finds + # its own rows BY that label. Relabelling hid the link from it, + # so the next cycle built a second row for the same triple and + # MySQL rejected it: the PC reported 200, then 500 forever + # after. The row's lifecycle (including staying dormant while a + # bay is contested) belongs to the machine sync; adoption only + # needs the identity, which it has. pass elif claimed is not None: # Relabelling is what adoption MEANS for any other row: the link diff --git a/scripts/reconcile_locations.py b/scripts/reconcile_locations.py new file mode 100644 index 00000000..becb3511 --- /dev/null +++ b/scripts/reconcile_locations.py @@ -0,0 +1,156 @@ +"""Generate reconciliation SQL for the locations table. + +Reads a TSV export (locationid, locationname, description, locationtypeid, +parentlocationid) and emits a single transaction-wrapped SQL script that: + - Treats bare grid coordinates ([A-V][0-9]{1,2}W? and LD[0-9]+W?) as the + canonical identity for a cell. + - Folds coordinate-plus-suffix rows into the canonical coordinate, moving the + suffix text into the canonical row's description (never the name). + - Folds named-area suffix variants ("NDE - 6409") into the parent named area + ("NDE"); named areas are kept as-is and never forced into coordinates. + - Repoints assets.locationid, subnets.locationid and child + locations.parentlocationid onto the canonical row, then deletes duplicates. + +Bare coordinates with no variants are left untouched. The trailing W (west +column, e.g. G2W, K3W) is part of the coordinate and is never stripped, so +G2W is kept distinct from a hypothetical G2. +""" +import re +import sys + +COORD = re.compile(r'^(LD|[ABCDEFGHJKLMNPQRSTUV])(\d{1,2})(W)?(?=$|[ \-])') + + +def esc(s): + return s.replace("'", "''") + + +def load(path): + rows = [] + with open(path, encoding='utf-8-sig') as fh: + for line in fh: + line = line.rstrip('\n').rstrip('\r') + if not line: + continue + parts = line.split('\t') + if len(parts) < 5: + parts += [''] * (5 - len(parts)) + lid, name, desc, typeid, parentid = parts[:5] + rows.append({ + 'id': int(lid), + 'name': name, + 'desc': desc, + 'typeid': typeid, + 'parentid': parentid, + }) + return rows + + +def classify(name): + """Return (canonical, suffix). canonical is the bare coordinate or named + parent; suffix is the trailing detail (may be '').""" + m = COORD.match(name) + if m: + prefix, digits, west = m.group(1), m.group(2), m.group(3) or '' + canonical = f"{prefix}{digits}{west}" + suffix = name[m.end():].strip() + suffix = re.sub(r'^[\-\s]+', '', suffix).strip() + return canonical, suffix, 'coord' + # named area: fold on ' - ' separator only + if ' - ' in name: + base, _, rest = name.partition(' - ') + return base.strip(), rest.strip(), 'named' + return name.strip(), '', 'named' + + +def main(tsv, out): + rows = load(tsv) + + # Build canonical groups. Key is casefolded canonical for grouping, but we + # preserve a display canonical (prefer an existing exact bare/standalone). + groups = {} + for r in rows: + canonical, suffix, kind = classify(r['name']) + key = (kind, canonical.casefold()) + g = groups.setdefault(key, { + 'canonical': canonical, + 'kind': kind, + 'members': [], + 'exact_id': None, + }) + r['_suffix'] = suffix + r['_canonical'] = canonical + g['members'].append(r) + # exact row: name equals canonical (no suffix) + if suffix == '' and r['name'].strip() == canonical: + g['exact_id'] = r['id'] + g['canonical'] = r['name'].strip() + + lines = [] + lines.append('-- Auto-generated locations reconciliation. Review before running.') + lines.append('START TRANSACTION;') + + touched = 0 + for key, g in sorted(groups.items(), key=lambda kv: kv[0][1]): + members = sorted(g['members'], key=lambda r: r['id']) + # nothing to do: single member that is already the exact canonical + if len(members) == 1 and g['exact_id'] == members[0]['id']: + continue + touched += 1 + canonical = g['canonical'] + + # choose target id: prefer exact bare row, else promote smallest id + if g['exact_id'] is not None: + target = g['exact_id'] + promote = None + else: + target = members[0]['id'] + promote = members[0] + + # accumulate suffix/description text from all members (dedup, ordered) + details = [] + for m in members: + for chunk in (m['_suffix'], m['desc']): + chunk = (chunk or '').strip() + if chunk and chunk not in details: + details.append(chunk) + desc_val = '; '.join(details) + + lines.append('') + lines.append(f"-- {canonical} (target locationid {target}) <= " + + ', '.join(str(m['id']) for m in members)) + if promote is not None: + lines.append( + f"UPDATE locations SET locationname='{esc(canonical)}' " + f"WHERE locationid={target};") + if desc_val: + lines.append( + f"UPDATE locations SET description='{esc(desc_val)}' " + f"WHERE locationid={target};") + + others = [m['id'] for m in members if m['id'] != target] + if others: + ids = ','.join(str(i) for i in others) + lines.append( + f"UPDATE assets SET locationid={target} " + f"WHERE locationid IN ({ids});") + lines.append( + f"UPDATE subnets SET locationid={target} " + f"WHERE locationid IN ({ids});") + lines.append( + f"UPDATE locations SET parentlocationid={target} " + f"WHERE parentlocationid IN ({ids});") + lines.append( + f"DELETE FROM locations WHERE locationid IN ({ids});") + + lines.append('') + lines.append('COMMIT;') + lines.append(f'-- groups touched: {touched}') + + with open(out, 'w', encoding='utf-8') as fh: + fh.write('\n'.join(lines) + '\n') + print(f'wrote {out}: {touched} groups touched, {len(rows)} source rows') + + +if __name__ == '__main__': + main(sys.argv[1], sys.argv[2]) diff --git a/shopdb/__init__.py b/shopdb/__init__.py index 3e7ee884..8cc89ae8 100644 --- a/shopdb/__init__.py +++ b/shopdb/__init__.py @@ -57,7 +57,15 @@ from .plugins import plugin_manager # permission / empty / position. BREAKING for any plugin still using the old # shape, which is why it is recorded here: the change itself shipped earlier # without a bump, and a contract that changes silently is not a contract. -__contract_version__ = '0.20.0' +# 0.21.0: added register_import_hook to shopdb.api so a plugin can own a VIRTUAL +# COLUMN on an importable CSV table - a header the bulk importer accepts but +# never writes - and a post-write row hook that turns that value into a plugin +# concept. The computers plugin uses it for a machineassetnumber column on +# computers.csv that builds the PC->machine controls link, so a site writes the +# machine number in the PC row instead of hand-building a relationships sheet. +# Core stays ignorant of the column's meaning. Additive name on the import +# surface, minor bump. +__contract_version__ = '0.21.0' # Product release version (see ADR-007). The product version and the # plugin-contract version above are distinct series with independent diff --git a/shopdb/api/__init__.py b/shopdb/api/__init__.py index 5a11f324..f802e387 100644 --- a/shopdb/api/__init__.py +++ b/shopdb/api/__init__.py @@ -95,6 +95,15 @@ from shopdb.core.services.dualpath import ( dualpath_single_machine_enabled, ) +# Bulk-import extension point: a plugin registers a virtual column on an +# importable CSV table plus a post-write row hook, so a relationship a site +# would otherwise hand-build (a PC's machine number -> a controls link) can be +# written in the row itself. Core never learns what the column means. +from shopdb.core.services.csvimport import ( + register_import_hook, + RowWarning, +) + # Legacy employee directory lookup (read-only) used by notifications from shopdb.utils.employee_db import employee_connection from shopdb.utils.mailer import send_email, send_alert, send_webhook @@ -261,6 +270,9 @@ __all__ = [ 'resolve_asset_position', 'resolve_dualpath_pairs', 'dualpath_single_machine_enabled', + # Bulk-import extension point + 'register_import_hook', + 'RowWarning', # Infrastructure 'db', 'cache', diff --git a/shopdb/core/api/settings.py b/shopdb/core/api/settings.py index 75fe8e14..d2f2830c 100644 --- a/shopdb/core/api/settings.py +++ b/shopdb/core/api/settings.py @@ -6,6 +6,7 @@ import re from flask import Blueprint, request, current_app from flask_jwt_extended import jwt_required, get_jwt_identity from werkzeug.utils import secure_filename +from sqlalchemy import select as sa_select from shopdb.extensions import db, cache from shopdb.core.models import Setting, AuditLog @@ -332,6 +333,27 @@ def download_import_template(tablename): return response +def _existing_hostname_map(): + """Case-insensitive hostname -> assetnumber map for existing computers. + + Azure device-group exports align on hostname (their displayName), while the + import engine keys assets on assetnumber. This lets the Azure adapter route + a matched device to its real asset so the import updates in place. + """ + assets = db.metadata.tables['assets'] + computers = db.metadata.tables['computers'] + query = sa_select( + computers.c.hostname, assets.c.assetnumber + ).select_from( + computers.join(assets, computers.c.assetid == assets.c.assetid) + ).where(computers.c.hostname.isnot(None)) + mapping = {} + for hostname, assetnumber in db.session.execute(query): + if hostname: + mapping[hostname.strip().lower()] = assetnumber + return mapping + + @settings_bp.route('/asset-import/import', methods=['POST']) @jwt_required() @require_role('admin') @@ -344,10 +366,16 @@ def import_asset_csvs(): transaction back and reports what WOULD change; commit=true persists it. The whole run is all-or-nothing - if any file has a row problem, nothing is written even on a commit. + + A 'source' form field selects a foreign adapter (azure, thor); it defaults + to 'native' where uploads are already canonical per-table files. Foreign + uploads are transformed into canonical files first, then run identically. """ from sqlalchemy.exc import SQLAlchemyError from shopdb.core.services.csvimport import ( ImportError_, Resolver, dependency_order, import_csv, table_from_filename) + from shopdb.core.services.importsources import ( + ImportSourceError, get_source) uploads = request.files.getlist('files') if not uploads: @@ -356,21 +384,66 @@ def import_asset_csvs(): commit = str(request.form.get('commit', '')).strip().lower() in ( '1', 'true', 'yes', 'on') - # Map each upload to a table by filename. A later file with the same table - # name wins, matching the folder-import behaviour (one file per table). + # A foreign source (azure, thor) is transformed into canonical files before + # import. 'native' (or nothing) means the files already ARE canonical. + source_key = (request.form.get('source') or 'native').strip().lower() + source = None + if source_key and source_key != 'native': + try: + source = get_source(source_key) + except ImportSourceError as exc: + return error_response(ErrorCodes.VALIDATION_ERROR, str(exc)) + + source_options = { + 'computertype': (request.form.get('computertype') or '').strip(), + 'category': (request.form.get('category') or '').strip(), + } + + # Azure and Thor align on hostname: their device name is the ShopDB + # computers.hostname, not the assetnumber. Hand the adapter a + # case-insensitive hostname -> existing assetnumber map so devices that + # already exist update in place (keyed to their real asset) instead of + # minting a duplicate asset. + if source is not None and getattr(source, 'key', None) in ('azure', 'thor'): + source_options['hostname_map'] = _existing_hostname_map() + texts = {} filenames = {} - for upload in uploads: - name = upload.filename or '' - if not name.lower().endswith('.csv'): - continue - table = table_from_filename(name) - try: - texts[table] = upload.read().decode('utf-8-sig') - except (UnicodeDecodeError, OSError): - return error_response(ErrorCodes.VALIDATION_ERROR, - 'Could not read %s as UTF-8 text' % name) - filenames[table] = name + if source is not None: + # One foreign export becomes several canonical files. Each upload is + # transformed and its canonical outputs merged; a later file's table + # wins, matching the native one-file-per-table rule. + for upload in uploads: + name = upload.filename or '' + if not name.lower().endswith('.csv'): + continue + try: + raw = upload.read().decode('utf-8-sig') + except (UnicodeDecodeError, OSError): + return error_response(ErrorCodes.VALIDATION_ERROR, + 'Could not read %s as UTF-8 text' % name) + try: + produced = source.transform(raw, options=source_options) + except ImportSourceError as exc: + return error_response(ErrorCodes.VALIDATION_ERROR, + '%s: %s' % (name, str(exc))) + for table, csv_text in produced.items(): + texts[table] = csv_text + filenames[table] = '%s (from %s)' % (table, name) + else: + # Map each upload to a table by filename. A later file with the same + # table name wins, matching the folder-import behaviour. + for upload in uploads: + name = upload.filename or '' + if not name.lower().endswith('.csv'): + continue + table = table_from_filename(name) + try: + texts[table] = upload.read().decode('utf-8-sig') + except (UnicodeDecodeError, OSError): + return error_response(ErrorCodes.VALIDATION_ERROR, + 'Could not read %s as UTF-8 text' % name) + filenames[table] = name if not texts: return error_response(ErrorCodes.VALIDATION_ERROR, @@ -389,6 +462,7 @@ def import_asset_csvs(): resolver = Resolver(reconcile_threshold=reconcile_threshold) files_report = [] problem_count = 0 + warning_count = 0 reconciled_count = 0 request_error = None @@ -409,7 +483,17 @@ def import_asset_csvs(): {'line': p.line, 'column': p.column, 'message': p.message} for p in result.problems ] + # Amber tier: a warning is a finding beside a row that still + # imported (an unresolved machine link, a contested bay), NOT a + # reason to fail the file. Reported alongside problems so the UI can + # show Errors / Warnings / Info distinctly, but never gates commit. + warnings = [ + {'line': w.line, 'column': w.column, 'reason': w.reason, + 'action': w.action, 'subject': w.subject, 'target': w.target} + for w in result.warnings + ] problem_count += len(problems) + warning_count += len(warnings) reconciled_count += len(result.reconciliations) files_report.append({ 'table': table, @@ -417,6 +501,7 @@ def import_asset_csvs(): 'created': result.created, 'updated': result.updated, 'problems': problems, + 'warnings': warnings, 'reconciliations': result.reconciliations, }) @@ -446,6 +531,7 @@ def import_asset_csvs(): 'files': files_report, 'skipped': skipped, 'problem_count': problem_count, + 'warning_count': warning_count, 'reconciled_count': reconciled_count, 'reconcile_threshold': reconcile_threshold, 'total_created': total_created, diff --git a/shopdb/core/services/csvimport.py b/shopdb/core/services/csvimport.py index 462e8b57..0adeb3a2 100644 --- a/shopdb/core/services/csvimport.py +++ b/shopdb/core/services/csvimport.py @@ -74,6 +74,71 @@ IMPORTABLE = OrderedDict([ ('machines', {'label': None, 'natural': 'assetid'}), ]) +# A small, INTERNALLY CONSISTENT worked example per table, in import order. A +# site can download the set, see the pattern, and keep it as a starter workbook: +# every name referenced by a later file (Bay 3, Dell, Computer, ...) is defined +# in an earlier one, so the whole set imports cleanly as a demo before a single +# real row is typed. Emitted as commented rows the engine skips, so a freshly +# downloaded template never writes example data by accident. Values avoid commas +# on purpose - the generator writes them without CSV quoting. +EXAMPLE_ROWS = OrderedDict([ + ('assetstatuses', [ + {'status': 'In Use', 'description': 'Actively deployed', 'color': '#22c55e', 'isactive': '1'}, + {'status': 'In Storage', 'description': 'Spare - not deployed', 'color': '#eab308', 'isactive': '1'}, + {'status': 'Retired', 'description': 'Removed from service', 'color': '#ef4444', 'isactive': '1'}, + ]), + ('assettypes', [ + {'assettype': 'Computer', 'pluginname': 'computers', 'tablename': 'computers', 'description': 'PCs - servers - workstations', 'icon': 'laptop', 'color': '#3b82f6', 'isactive': '1'}, + {'assettype': 'Machine', 'pluginname': 'machines', 'tablename': 'machines', 'description': 'Shop-floor machines', 'icon': 'cpu', 'color': '#8b5cf6', 'isactive': '1'}, + ]), + ('locationtypes', [ + {'locationtype': 'Building', 'description': 'A physical building', 'color': '#64748b', 'isactive': '1'}, + {'locationtype': 'Bay', 'description': 'A shop-floor bay', 'color': '#0ea5e9', 'isactive': '1'}, + ]), + ('modeltypes', [ + {'modeltype': 'Desktop', 'category': 'Computer', 'description': 'Desktop PC chassis', 'icon': 'desktop', 'isactive': '1'}, + {'modeltype': 'CNC Machine', 'category': 'Machine', 'description': 'CNC machining center', 'icon': 'cpu', 'isactive': '1'}, + ]), + ('computertypes', [ + {'computertype': 'Shopfloor PC', 'description': 'PC on the shop floor', 'icon': 'monitor', 'color': '#3b82f6', 'isactive': '1'}, + {'computertype': 'Engineer Workstation', 'description': 'CAD/CAM workstation', 'icon': 'laptop', 'color': '#6366f1', 'isactive': '1'}, + ]), + ('machinetypes', [ + {'machinetype': 'CNC Mill', 'description': 'Computer numerical control mill', 'icon': 'cpu', 'color': '#8b5cf6', 'isactive': '1'}, + {'machinetype': 'CMM', 'description': 'Coordinate measuring machine', 'icon': 'ruler', 'color': '#14b8a6', 'isactive': '1'}, + ]), + ('businessunits', [ + {'businessunit': 'Machining', 'code': 'MACH', 'description': 'Machining department', 'isactive': '1'}, + {'businessunit': 'Quality', 'code': 'QUAL', 'description': 'Quality and metrology', 'isactive': '1'}, + ]), + ('locations', [ + {'locationname': 'Building 1', 'building': 'Building 1', 'floor': '1', 'description': 'Main manufacturing building', 'locationtypeid': 'Building', 'isactive': '1'}, + {'locationname': 'Bay 3', 'building': 'Building 1', 'floor': '1', 'room': 'Machining bay 3', 'description': 'Machining bay 3', 'locationtypeid': 'Bay', 'isactive': '1'}, + ]), + ('vendors', [ + {'vendor': 'Dell', 'description': 'Computer hardware', 'website': 'https://www.dell.com', 'supportphone': '800-555-0100', 'supportemail': 'support@dell.com', 'isactive': '1'}, + {'vendor': 'Haas Automation', 'description': 'CNC machine tools', 'website': 'https://www.haascnc.com', 'supportphone': '800-555-0199', 'isactive': '1'}, + ]), + ('models', [ + {'modelnumber': 'OptiPlex 7010', 'modeltypeid': 'Desktop', 'vendorid': 'Dell', 'description': 'Small form factor desktop', 'isactive': '1'}, + {'modelnumber': 'VF-2', 'modeltypeid': 'CNC Machine', 'vendorid': 'Haas Automation', 'description': '40-taper vertical mill', 'isactive': '1'}, + ]), + ('operatingsystems', [ + {'osname': 'Windows 11 (24H2)', 'osversion': '24H2', 'architecture': 'x64', 'endoflife': '2027-10-12', 'isactive': '1'}, + {'osname': 'Windows 10 (22H2)', 'osversion': '22H2', 'architecture': 'x64', 'endoflife': '2025-10-14', 'isactive': '1'}, + ]), + ('assets', [ + {'assetnumber': 'PC-001', 'name': 'Shopfloor PC 001', 'serialnumber': 'SN-PC-001', 'assettypeid': 'Computer', 'statusid': 'In Use', 'locationid': 'Bay 3', 'businessunitid': 'Machining', 'isactive': '1'}, + {'assetnumber': 'CNC-014', 'name': 'Haas VF-2 #14', 'serialnumber': 'SN-CNC-014', 'assettypeid': 'Machine', 'statusid': 'In Use', 'locationid': 'Bay 3', 'businessunitid': 'Machining', 'isactive': '1'}, + ]), + ('computers', [ + {'assetid': 'PC-001', 'computertypeid': 'Shopfloor PC', 'hostname': 'WS-PC-001', 'osid': 'Windows 11 (24H2)', 'vendorid': 'Dell', 'modelnumberid': 'OptiPlex 7010', 'loggedinuser': 'jdoe', 'lastreporteddate': '2026-08-01', 'lastboottime': '2026-08-01', 'isactive': '1', 'machineassetnumber': 'CNC-014'}, + ]), + ('machines', [ + {'assetid': 'CNC-014', 'machinetypeid': 'CNC Mill', 'vendorid': 'Haas Automation', 'modelnumberid': 'VF-2', 'requiresmanualconfig': '0', 'islocationonly': '0', 'lastmaintenancedate': '2026-06-01', 'nextmaintenancedate': '2026-12-01', 'maintenanceintervaldays': '180', 'isactive': '1'}, + ]), +]) + # Columns a CSV may never set: surrogate keys the database owns, and audit # stamps. Accepting an id from a spreadsheet is how two sites end up with # colliding primary keys the first time anyone merges data. @@ -111,6 +176,75 @@ CORPORATE_SUFFIXES = { DEFAULT_RECONCILE_THRESHOLD = 0.85 +# -- plugin import hooks (inversion of control) --------------------------------- +# +# A spreadsheet is the wrong place to type an integer foreign key, and it is +# also the wrong place to split a relationship across two files: a site that +# knows "this PC drives machine 3015" wants to write that number in the PC row, +# not hand-build an assetrelationships sheet keyed on ids it never sees. +# +# But the machine link is a computers-plugin concept, and this module is core: +# it must not import a plugin. So a plugin REGISTERS itself here instead. It +# declares a VIRTUAL COLUMN - a header this table accepts but never writes to +# the database - and a HOOK that runs once per written row to turn that column's +# value into whatever the plugin means by it (here, a PC->machine controls link). +# Core stays ignorant of what the column is for; the plugin owns the meaning. +_VIRTUAL_COLUMNS = {} # tablename -> {columnname: description} +_ROW_HOOKS = {} # tablename -> [hook, ...] + + +def register_import_hook(tablename, virtual_columns, hook): + """Let a plugin own virtual column(s) on an importable table. + + virtual_columns: either a sequence of header names, or a mapping of + name -> one-line description. Each is accepted for ``tablename`` but never + written - it is pulled out of the row and handed to the hook instead of the + table. A description, when given, is shown in the generated template's + column notes so an operator knows what to type. + + hook(rowvalues, virtualvalues, line) -> list[RowWarning]: called once per + written row, AFTER the row is inserted/updated, so its id exists. It runs + inside the caller's transaction - which a dry run rolls back - so it must + not commit, and must not do anything a rollback cannot undo (no emails). A + hook reports trouble by returning RowWarning, never by raising: a bad + machine number is an amber note beside a PC that still imports, not a failed + file. + + Idempotent by contract: registering the same (table, column, hook) twice is + a no-op, so a plugin re-init during tests or a reload does not double-fire. + """ + cols = _VIRTUAL_COLUMNS.setdefault(tablename, {}) + if isinstance(virtual_columns, dict): + for name, description in virtual_columns.items(): + cols[name] = description or '' + else: + for name in virtual_columns: + cols.setdefault(name, '') + hooks = _ROW_HOOKS.setdefault(tablename, []) + # Dedupe by the underlying function's qualified name, not by the bound + # method: a plugin re-init (a new app context in the test suite, a reload in + # prod) makes a fresh instance whose bound method is != the old one, and a + # symlinked plugin can even be imported under two module paths, so an + # identity check would append a second hook and the row would be processed - + # and warned about - more than once. + def _identity(fn): + func = getattr(fn, '__func__', fn) + return getattr(func, '__qualname__', repr(func)) + + if _identity(hook) not in [_identity(existing) for existing in hooks]: + hooks.append(hook) + + +def virtual_columns_for(tablename): + """Header names this table accepts but does not store. Never None.""" + return set(_VIRTUAL_COLUMNS.get(tablename, ())) + + +def virtual_column_notes(tablename): + """(name, description) for each virtual column, in registration order.""" + return list(_VIRTUAL_COLUMNS.get(tablename, {}).items()) + + def normalize_label(value): """Fold a label to its comparison form: lowercase, no punctuation, no corporate suffix, single-spaced. 'Dell Inc.' and 'DELL' both become 'dell'. @@ -370,6 +504,32 @@ class RowProblem(object): return "%s: %s" % (where, self.message) +class RowWarning(object): + """A non-blocking, amber-tier finding. + + Errors (RowProblem) stop the whole import; a RowWarning does NOT - the row + it belongs to still imports. It exists so a caveat (an unresolved or + conflicting machine link, say) is loud and REVIEWABLE without failing an + otherwise-clean file. Every warning names WHERE it happened, WHY, and the + single ACTION that fixes it, so a cold user can correct it in place - the + convention IT Glue / RMM imports train. + """ + + def __init__(self, line, column, reason, action, subject=None, target=None): + self.line = line + self.column = column + self.reason = reason # what is wrong, in plain terms + self.action = action # the one corrective step to take + self.subject = subject # the row's actor (e.g. the PC hostname) + self.target = target # what it referenced (e.g. the machine number) + + def __str__(self): + where = "line %d" % self.line + if self.column: + where += ", column '%s'" % self.column + return "%s: %s - %s" % (where, self.reason, self.action) + + class ImportResult(object): def __init__(self, tablename): self.tablename = tablename @@ -378,14 +538,18 @@ class ImportResult(object): self.problems = [] self.skipped = 0 self.reconciliations = [] + self.warnings = [] @property def ok(self): + # Warnings are amber, not red: a file with only warnings is still ok to + # commit. Only problems (errors) block. return not self.problems def summary(self): - return "%s: %d new, %d updated, %d problem(s)" % ( - self.tablename, self.created, self.updated, len(self.problems)) + return "%s: %d new, %d updated, %d problem(s), %d warning(s)" % ( + self.tablename, self.created, self.updated, + len(self.problems), len(self.warnings)) def read_rows(text): @@ -439,9 +603,10 @@ def import_csv(tablename, text, resolver=None, commit=False): header, rows = read_rows(text) allowed = {c.name: c for c in importable_columns(tablename)} - unknown = [h for h in header if h not in allowed] + virtual = virtual_columns_for(tablename) + unknown = [h for h in header if h not in allowed and h not in virtual] if unknown: - known = ', '.join(sorted(allowed)) + known = ', '.join(sorted(set(allowed) | virtual)) result.problems.append(RowProblem( 1, None, "unknown column(s): %s\n this table accepts: %s" % (', '.join(unknown), known))) return result @@ -461,7 +626,17 @@ def import_csv(tablename, text, resolver=None, commit=False): prepared = [] for line, raw in rows: values = {} + virtualvalues = {} failed = False + # Virtual columns belong to a plugin hook, not the table. Pull them out + # first so they never reach an insert/update, and stash them for the + # post-write hook. Whitespace-trimmed; an empty cell is dropped so the + # hook only sees values a row actually carried. + for name in virtual: + if name in raw: + text = (raw[name] or '').strip() + if text != '': + virtualvalues[name] = text for name, column in allowed.items(): if name not in raw: continue @@ -496,16 +671,18 @@ def import_csv(tablename, text, resolver=None, commit=False): if values.get(natural) in (None, ''): result.problems.append(RowProblem(line, natural, 'must have a value')) continue - prepared.append((line, values)) + prepared.append((line, values, virtualvalues)) if not result.ok: return result + hooks = _ROW_HOOKS.get(tablename, ()) + # Second pass: decide insert vs update. Counted even on a dry run, because # "this will change 4000 rows" is the number an operator needs before saying # yes. label = _label_column(tablename) - for line, values in prepared: + for line, values, virtualvalues in prepared: existing = db.session.execute( table.select().where(table.c[natural] == values[natural])).first() if existing is None: @@ -522,6 +699,15 @@ def import_csv(tablename, text, resolver=None, commit=False): primary = list(table.primary_key)[0].name resolver.remember(tablename, values[label], getattr(existing, primary)) + # Fire plugin hooks AFTER the write, so the row's id exists. Only when + # the row actually carried a virtual value - a hook that has nothing to + # act on should never run. Warnings are collected, never raised: an + # unresolved machine number is amber, not a failed file. + if hooks and virtualvalues: + for hook in hooks: + result.warnings.extend( + hook(values, virtualvalues, line) or []) + return result @@ -556,24 +742,55 @@ def generate_template(tablename): bits.append('text, max %d' % column.type.length) notes.append("# %-24s %s" % (column.name, ', '.join(bits))) + # Virtual columns a plugin owns: accepted on this table but written by the + # plugin's hook, not the schema. Appended after the real columns so the + # header carries them and the notes explain what they do. + virtualnames = [] + for name, description in virtual_column_notes(tablename): + virtualnames.append(name) + header.append(name) + note = 'optional, ' + description if description else 'optional' + notes.append("# %-24s %s" % (name, note)) + spec = IMPORTABLE[tablename] + examples = EXAMPLE_ROWS.get(tablename, []) + exampleorder = [c.name for c in columns] + virtualnames lines = [ "# %s.csv - generated from the ShopDB schema, do not hand-maintain" % tablename, "#", "# Rows are matched on '%s'. Re-importing updates a matching row rather" % spec['natural'], "# than creating a second one, so it is safe to correct this file and run again.", "#", + "# HOW TO USE THIS TEMPLATE:", + "# 1. Delete everything above the header row (every line starting with #).", + "# 2. Fill in your own rows using the example row(s) below as a guide.", + "# 3. DELETE THE EXAMPLE DATA before you import - the example rows are", + "# live data and WILL be imported if you leave them in.", + "#", "# Columns:", ] + notes + [ "#", - "# Lines starting with # are ignored. Delete the example row before importing.", + "# --- header and example rows follow. Keep the header; replace the examples. ---", ','.join(header), - '# ' + ','.join(_example(c, required) for c in columns), + ] + [ + ','.join(_example_value(row.get(name, '')) for name in exampleorder) + for row in examples ] return '\n'.join(lines) + '\n' +def _example_value(value): + """A single example cell. Commas would break the unquoted CSV the template + writes, so they are replaced with a dash; example data is authored without + them, this is only a guard. + """ + return str(value).replace(',', ' -') + + def _example(column, required): + """Kept for callers/tests that ask for a single synthetic cell. The template + generator now emits the curated EXAMPLE_ROWS instead. + """ target = _fk_target(column) if target: label = _label_column(target[0]) diff --git a/shopdb/core/services/importsources.py b/shopdb/core/services/importsources.py new file mode 100644 index 00000000..6aa56041 --- /dev/null +++ b/shopdb/core/services/importsources.py @@ -0,0 +1,325 @@ +"""Foreign import-source adapters. + +The CSV import engine in csvimport.py speaks exactly one dialect: the canonical +per-table files (assets.csv, computers.csv, vendors.csv, ...). Foreign exports - +an Entra/Azure device-group member list, a Thor PC inventory dump - do not match +that dialect and never will; they are owned by other systems. + +Rather than teach the engine every foreign schema, each adapter here TRANSFORMS +a foreign export into the canonical files and hands them back as +{tablename: csv_text}. The caller then runs those through the ordinary +import_csv path, so dry-run/commit, dependency ordering, and reconciliation all +work unchanged. + +An adapter declares a header signature so an upload can be auto-detected, and a +transform() that yields the canonical files. Adapters never touch the database. +""" + +import csv +import io + + +class ImportSourceError(Exception): + """A foreign file could not be transformed (bad header, empty, etc.).""" + + +# Canonical defaults for a discovered PC. Azure carries only a hostname, so +# every device it creates is a computer that is In Use until something richer +# (Thor, the collector) says otherwise. +DEFAULT_ASSETTYPE = 'computer' +DEFAULT_STATUS = 'In Use' + + +def _read(text): + """Parse CSV text into (header, list-of-dicts), tolerating a UTF-8 BOM.""" + stream = io.StringIO(text) + reader = csv.DictReader(stream) + header = [h.strip() for h in (reader.fieldnames or [])] + rows = [] + for raw in reader: + rows.append({(k.strip() if k else k): (v or '').strip() + for k, v in raw.items()}) + return header, rows + + +def _write(header, rows): + """Render canonical rows back to CSV text with a trailing newline.""" + out = io.StringIO() + writer = csv.DictWriter(out, fieldnames=header, extrasaction='ignore') + writer.writeheader() + for row in rows: + writer.writerow(row) + return out.getvalue() + + +def _has_columns(header, required): + lowered = {h.lower() for h in header} + return all(col.lower() in lowered for col in required) + + +class AzureDeviceGroupSource: + """Entra/Azure device-group member export. + + Columns: displayName, objectType, mail, userType, id, deviceId, + userPrincipalName, isUser, isGroup, isGuest. Only device rows carry signal; + displayName is the hostname and deviceId is the Entra device GUID. The export + has no model, OS, location or user data, so this creates/updates minimal + asset + computer rows keyed on hostname. + + options: + computertype - optional name stamped onto every computer's computertypeid + (e.g. 'Shopfloor PC' for a SHP group, 'Inspection PC' for + BUS). Resolved by the engine against computertypes. + category - optional Entra device category / group membership label + stamped onto every computer's azuredevicecategory (e.g. + SFLD_DYN_Device_Group_Wilmington_BUS_NC_COUNTRY:USA). + hostname_map - optional {lowercased hostname: existing assetnumber}. A + device whose displayName matches an existing computer's + hostname is routed to that asset and updated in place + (no new asset row); unmatched devices create a new + asset + computer keyed on the hostname. + """ + + key = 'azure' + label = 'Azure device group' + signature = ('displayName', 'deviceId', 'objectType') + + @classmethod + def matches(cls, header): + return _has_columns(header, cls.signature) + + @classmethod + def transform(cls, text, options=None): + options = options or {} + computertype = (options.get('computertype') or '').strip() + category = (options.get('category') or '').strip() + hostname_map = options.get('hostname_map') or {} + header, rows = _read(text) + if not _has_columns(header, cls.signature): + raise ImportSourceError( + 'not an Azure device-group export (expected displayName, ' + 'deviceId, objectType columns)') + + assets = [] + computers = [] + seen = set() + for row in rows: + objecttype = (row.get('objectType') or '').strip().lower() + if objecttype and objecttype != 'device': + continue + hostname = (row.get('displayName') or '').strip() + if not hostname or hostname.lower() in seen: + continue + seen.add(hostname.lower()) + + # Align on hostname: if this device already exists in ShopDB, point + # the computer row at its real asset so the engine updates it in + # place. Only create a new asset when there is no hostname match. + existing_assetnumber = hostname_map.get(hostname.lower()) + computer = { + 'assetid': existing_assetnumber if existing_assetnumber is not None else hostname, + 'hostname': hostname, + 'azuredeviceid': (row.get('deviceId') or '').strip(), + 'azureobjectid': (row.get('id') or '').strip(), + } + if computertype: + computer['computertypeid'] = computertype + if category: + computer['azuredevicecategory'] = category + computers.append(computer) + + if existing_assetnumber is None: + assets.append({ + 'assetnumber': hostname, + 'name': hostname, + 'assettypeid': DEFAULT_ASSETTYPE, + 'statusid': DEFAULT_STATUS, + }) + + if not computers: + raise ImportSourceError('no device rows found in the Azure export') + + asset_header = ['assetnumber', 'name', 'assettypeid', 'statusid'] + computer_header = ['assetid', 'hostname', 'azuredeviceid', 'azureobjectid'] + if computertype: + computer_header.append('computertypeid') + if category: + computer_header.append('azuredevicecategory') + produced = { + 'computers': _write(computer_header, computers), + } + # Only emit an assets file when there are new devices to create; an + # empty assets.csv would just be an extra clean no-op table. + if assets: + produced['assets'] = _write(asset_header, assets) + return produced + + +class ThorSource: + """Thor PC-inventory export (internal GE Aerospace tool). + + A wide, fixed-format PC inventory. Maps the columns ShopDB owns into + canonical rows: hostname, vendor (BIOS Manf.), model, operating system, + logged-in user, last-reported and last-boot timestamps, and location. The + network columns (MAC, switch, VLAN, IP) and the machine 'Attached' link are + out of scope for this first pass. + + options: + computertype - optional name stamped onto every computer's computertypeid. + Resolved by the engine against computertypes. + hostname_map - optional {lowercased hostname: existing assetnumber}. A + host whose Host Name matches an existing computer's hostname + is routed to that asset and updated in place (no new asset + row); unmatched hosts create a new asset + computer keyed on + the hostname. + """ + + key = 'thor' + label = 'Thor PC inventory' + signature = ('Host Name', 'BIOS Manf.', 'OS Version') + + @classmethod + def matches(cls, header): + return _has_columns(header, cls.signature) + + @classmethod + def transform(cls, text, options=None): + options = options or {} + computertype = (options.get('computertype') or '').strip() + hostname_map = options.get('hostname_map') or {} + header, rows = _read(text) + if not _has_columns(header, cls.signature): + raise ImportSourceError( + 'not a Thor export (expected Host Name, BIOS Manf., OS Version ' + 'columns)') + + vendors = {} + models = {} + operatingsystems = {} + assets = [] + computers = [] + seen = set() + for row in rows: + hostname = (row.get('Host Name') or '').strip() + if not hostname or hostname.lower() in seen: + continue + seen.add(hostname.lower()) + + vendor = (row.get('BIOS Manf.') or '').strip() + model = (row.get('Model') or '').strip() + osname = cls._osname(row) + location = (row.get('Location') or '').strip() + loggedinuser = cls._logon(row) + + if vendor: + vendors[vendor.lower()] = vendor + if model: + models[model.lower()] = model + if osname: + operatingsystems[osname.lower()] = osname + + # Align on hostname: if this host already exists in ShopDB, point + # the computer row at its real asset so the engine updates it in + # place. Only create a new asset when there is no hostname match. + existing_assetnumber = hostname_map.get(hostname.lower()) + if existing_assetnumber is None: + assets.append({ + 'assetnumber': hostname, + 'name': hostname, + 'assettypeid': DEFAULT_ASSETTYPE, + 'statusid': DEFAULT_STATUS, + 'locationid': location, + }) + + computer = { + 'assetid': existing_assetnumber if existing_assetnumber is not None else hostname, + 'hostname': hostname, + 'osid': osname, + 'vendorid': vendor, + 'modelnumberid': model, + 'loggedinuser': loggedinuser, + 'lastreporteddate': cls._date(row.get('Last Updated')), + 'lastboottime': cls._date(row.get('Boot Time')), + } + if computertype: + computer['computertypeid'] = computertype + computers.append(computer) + + if not computers: + raise ImportSourceError('no host rows found in the Thor export') + + out = {} + if vendors: + out['vendors'] = _write(['vendor'], + [{'vendor': v} for v in vendors.values()]) + if models: + out['models'] = _write(['modelnumber'], + [{'modelnumber': m} for m in models.values()]) + if operatingsystems: + out['operatingsystems'] = _write( + ['osname'], [{'osname': o} for o in operatingsystems.values()]) + + asset_header = ['assetnumber', 'name', 'assettypeid', 'statusid', 'locationid'] + computer_header = ['assetid', 'hostname', 'osid', 'vendorid', + 'modelnumberid', 'loggedinuser', 'lastreporteddate', + 'lastboottime'] + if computertype: + computer_header.append('computertypeid') + out['computers'] = _write(computer_header, computers) + # Only emit an assets file when there are new hosts to create; an empty + # assets.csv would just be an extra clean no-op table. + if assets: + out['assets'] = _write(asset_header, assets) + return out + + @staticmethod + def _osname(row): + """Combine OS Version and OS ID into a single OS name. + + Thor splits the OS into a build ('10.0.26100') and a marketing id + ('24H2'); neither alone is meaningful, so join them. + """ + version = (row.get('OS Version') or '').strip() + osid = (row.get('OS ID') or '').strip() + if version and osid: + return '%s (%s)' % (version, osid) + return version or osid + + @staticmethod + def _logon(row): + """Take the account name out of a DOMAIN\\user or HOST\\user LOGON.""" + logon = (row.get('LOGON') or '').strip() + if '\\' in logon: + return logon.rsplit('\\', 1)[-1] + return logon + + @staticmethod + def _date(value): + """Normalize a Thor timestamp to a form the engine's date coercion + accepts, or empty when it cannot be parsed. Leaves already-ISO values + alone; the engine tries several formats itself. + """ + return (value or '').strip() + + +SOURCES = { + AzureDeviceGroupSource.key: AzureDeviceGroupSource, + ThorSource.key: ThorSource, +} + + +def get_source(key): + source = SOURCES.get((key or '').strip().lower()) + if source is None: + raise ImportSourceError( + "unknown import source '%s' (known: %s)" + % (key, ', '.join(sorted(SOURCES)))) + return source + + +def detect_source(header): + """Return the adapter whose signature matches this header, or None.""" + for source in SOURCES.values(): + if source.matches(header): + return source + return None diff --git a/tests/test_csv_import.py b/tests/test_csv_import.py index d75fdec3..ec1a1dfc 100644 --- a/tests/test_csv_import.py +++ b/tests/test_csv_import.py @@ -173,14 +173,17 @@ def test_generated_templates_only_contain_real_columns(db): A hand-written template set had invented columns on 7 of 11 tables and named a table that does not exist. Generating from the schema is what makes that - impossible - so assert the generator actually does it. + impossible - so assert the generator actually does it. Plugin-registered + virtual columns (machineassetnumber) are legitimate, not invented, so they + are allowed alongside the table's real columns. """ + from shopdb.core.services.csvimport import virtual_columns_for for table in IMPORTABLE: text = generate_template(table) header = [line for line in text.splitlines() if line and not line.startswith('#')][0] - real = {c.name for c in importable_columns(table)} + allowed = {c.name for c in importable_columns(table)} | virtual_columns_for(table) for column in header.split(','): - assert column in real, '%s.csv offers a column that does not exist: %s' % (table, column) + assert column in allowed, '%s.csv offers a column that does not exist: %s' % (table, column) def test_every_template_includes_its_required_columns(db): @@ -234,3 +237,46 @@ def test_a_dry_run_over_several_files_resolves_across_them(db): assert result.ok, [str(p) for p in result.problems] _db.session.rollback() assert count('assets') == 0 + + +def _example_files(): + """Reconstruct each template's example rows as an importable CSV: the header + plus the live (uncommented) example data rows that follow it.""" + files = {} + for table in IMPORTABLE: + header = None + rows = [] + for line in generate_template(table).splitlines(): + if not line.strip(): + continue + if line.startswith('#'): + continue + if header is None: + header = line + else: + rows.append(line) + files[table] = '\n'.join([header] + rows) + '\n' + return files + + +def test_template_examples_are_present_for_every_table(db): + """Each template ships at least one worked example row, so a downloaded + set is a usable starter workbook rather than a bare header.""" + files = _example_files() + for table in IMPORTABLE: + rows = files[table].splitlines()[1:] + assert rows, '%s.csv has no example row' % table + + +def test_template_example_set_imports_cleanly(db): + """The whole curated example set must import as-is: every name a later file + references is defined by an earlier one, with no self-references that the + validation pass cannot resolve. This is the promise the workbook makes.""" + resolver = Resolver() + files = _example_files() + for table in IMPORTABLE: + result = import_csv(table, files[table], resolver=resolver) + assert result.ok, '%s.csv example failed: %s' % ( + table, [str(p) for p in result.problems]) + _db.session.rollback() + diff --git a/tests/test_import_sources.py b/tests/test_import_sources.py new file mode 100644 index 00000000..f844c9b1 --- /dev/null +++ b/tests/test_import_sources.py @@ -0,0 +1,201 @@ +"""Foreign import-source adapters: transforming Azure and Thor exports. + +These adapters are pure - they turn a foreign CSV into the canonical per-table +CSVs the import engine already understands, and touch no database. The tests +check the transform output directly, so they run without app context. +""" +import csv +import io + +import pytest + +from shopdb.core.services.importsources import ( + AzureDeviceGroupSource, ImportSourceError, ThorSource, detect_source, + get_source, +) + + +def _rows(text): + return list(csv.DictReader(io.StringIO(text))) + + +AZURE_CSV = ( + 'displayName,objectType,mail,userType,id,deviceId,userPrincipalName,isUser,isGroup,isGuest\n' + 'F13HSGH4,device,,,11111111-1111-1111-1111-111111111111,aaaa1111-bbbb-2222-cccc-333333333333,,False,False,False\n' + 'E3P5M0B4,device,,,22222222-2222-2222-2222-222222222222,dddd4444-eeee-5555-ffff-666666666666,,False,False,False\n' +) + + +def test_azure_matches_signature(): + header = ['displayName', 'objectType', 'deviceId', 'id'] + assert AzureDeviceGroupSource.matches(header) + assert detect_source(header) is AzureDeviceGroupSource + + +def test_azure_transform_builds_assets_and_computers(): + produced = AzureDeviceGroupSource.transform(AZURE_CSV) + assert set(produced) == {'assets', 'computers'} + + assets = _rows(produced['assets']) + assert [a['assetnumber'] for a in assets] == ['F13HSGH4', 'E3P5M0B4'] + assert all(a['assettypeid'] == 'computer' for a in assets) + assert all(a['statusid'] == 'In Use' for a in assets) + + computers = _rows(produced['computers']) + assert computers[0]['assetid'] == 'F13HSGH4' + assert computers[0]['hostname'] == 'F13HSGH4' + assert computers[0]['azuredeviceid'] == 'aaaa1111-bbbb-2222-cccc-333333333333' + assert 'computertypeid' not in computers[0] + + +def test_azure_transform_stamps_computertype(): + produced = AzureDeviceGroupSource.transform( + AZURE_CSV, options={'computertype': 'Shopfloor PC'}) + computers = _rows(produced['computers']) + assert all(c['computertypeid'] == 'Shopfloor PC' for c in computers) + + +def test_azure_transform_carries_objectid(): + produced = AzureDeviceGroupSource.transform(AZURE_CSV) + computers = _rows(produced['computers']) + # The CSV 'id' column is the Entra directory object id. + assert computers[0]['azureobjectid'] == '11111111-1111-1111-1111-111111111111' + + +def test_azure_transform_stamps_category(): + produced = AzureDeviceGroupSource.transform( + AZURE_CSV, options={'category': 'SFLD_DYN_Device_Group_X'}) + computers = _rows(produced['computers']) + assert all(c['azuredevicecategory'] == 'SFLD_DYN_Device_Group_X' + for c in computers) + + +def test_azure_aligns_matched_hostname_to_existing_asset(): + # A case-insensitive hostname match routes the device to its real asset and + # emits NO new asset row, so the import updates in place instead of creating + # a duplicate. Unmatched devices still create a new asset keyed on hostname. + produced = AzureDeviceGroupSource.transform( + AZURE_CSV, options={'hostname_map': {'f13hsgh4': '13NSGH4'}}) + + computers = _rows(produced['computers']) + by_host = {c['hostname']: c for c in computers} + # Matched device points at the existing assetnumber, not its hostname. + assert by_host['F13HSGH4']['assetid'] == '13NSGH4' + # Unmatched device falls back to its hostname (new asset). + assert by_host['E3P5M0B4']['assetid'] == 'E3P5M0B4' + + # Only the unmatched device produces an asset row. + assets = _rows(produced['assets']) + assert [a['assetnumber'] for a in assets] == ['E3P5M0B4'] + + +def test_azure_all_matched_emits_no_assets_file(): + produced = AzureDeviceGroupSource.transform( + AZURE_CSV, + options={'hostname_map': {'f13hsgh4': '13NSGH4', 'e3p5m0b4': 'E9Q'}}) + # Every device matched, so there is nothing to create and no assets file. + assert 'assets' not in produced + computers = _rows(produced['computers']) + assert {c['assetid'] for c in computers} == {'13NSGH4', 'E9Q'} + + +def test_azure_skips_non_device_rows_and_dedupes(): + text = ( + 'displayName,objectType,mail,userType,id,deviceId,userPrincipalName,isUser,isGroup,isGuest\n' + 'F13HSGH4,device,,,1,aaaa,,False,False,False\n' + 'F13HSGH4,device,,,1,aaaa,,False,False,False\n' + 'Some User,user,u@x,,2,,u@x,True,False,False\n' + ) + computers = _rows(AzureDeviceGroupSource.transform(text)['computers']) + assert [c['hostname'] for c in computers] == ['F13HSGH4'] + + +def test_azure_rejects_wrong_header(): + with pytest.raises(ImportSourceError): + AzureDeviceGroupSource.transform('a,b,c\n1,2,3\n') + + +THOR_CSV = ( + '"Host Name","IP Address","LOGON","OS Version","OS ID","Last Updated","Boot Time","Location","Model","BIOS Manf."\n' + '"F67HSGH4","10.0.0.5","F67HSGH4\\ShopFloor","10.0.26100","24H2","2026-01-02 03:04:05","2026-01-01 06:00:00","S13","Dell Pro Tower Plus QBT1250","Dell Inc."\n' +) + + +def test_thor_matches_signature(): + header = ['Host Name', 'BIOS Manf.', 'OS Version', 'Model'] + assert ThorSource.matches(header) + assert detect_source(header) is ThorSource + + +def test_thor_transform_builds_lookup_and_subtype_files(): + produced = ThorSource.transform(THOR_CSV) + assert 'vendors' in produced + assert 'models' in produced + assert 'operatingsystems' in produced + assert 'assets' in produced + assert 'computers' in produced + + assert _rows(produced['vendors'])[0]['vendor'] == 'Dell Inc.' + assert _rows(produced['models'])[0]['modelnumber'] == 'Dell Pro Tower Plus QBT1250' + assert _rows(produced['operatingsystems'])[0]['osname'] == '10.0.26100 (24H2)' + + computer = _rows(produced['computers'])[0] + assert computer['hostname'] == 'F67HSGH4' + assert computer['vendorid'] == 'Dell Inc.' + assert computer['modelnumberid'] == 'Dell Pro Tower Plus QBT1250' + assert computer['osid'] == '10.0.26100 (24H2)' + assert computer['loggedinuser'] == 'ShopFloor' + assert computer['lastreporteddate'] == '2026-01-02 03:04:05' + assert computer['lastboottime'] == '2026-01-01 06:00:00' + + asset = _rows(produced['assets'])[0] + assert asset['locationid'] == 'S13' + assert asset['assettypeid'] == 'computer' + assert asset['statusid'] == 'In Use' + + +THOR_TWO_CSV = ( + '"Host Name","IP Address","LOGON","OS Version","OS ID","Last Updated","Boot Time","Location","Model","BIOS Manf."\n' + '"F67HSGH4","10.0.0.5","F67HSGH4\\ShopFloor","10.0.26100","24H2","2026-01-02 03:04:05","2026-01-01 06:00:00","S13","Dell Pro Tower Plus QBT1250","Dell Inc."\n' + '"FDKKSGH4","10.0.0.6","FDKKSGH4\\ShopFloor","10.0.26100","24H2","2026-01-03 03:04:05","2026-01-02 06:00:00","G8","Dell Pro Tower Plus QBT1250","Dell Inc."\n' +) + + +def test_thor_stamps_computertype(): + computer = _rows( + ThorSource.transform(THOR_CSV, options={'computertype': 'Shop Floor PC'})['computers'])[0] + assert computer['computertypeid'] == 'Shop Floor PC' + + +def test_thor_aligns_matched_hostname_to_existing_asset(): + # A case-insensitive hostname match routes the host to its real asset and + # emits NO new asset row, so the import updates in place instead of creating + # a duplicate. Unmatched hosts still create a new asset keyed on hostname. + produced = ThorSource.transform( + THOR_TWO_CSV, options={'hostname_map': {'f67hsgh4': 'PC-1001'}}) + + computers = _rows(produced['computers']) + by_host = {c['hostname']: c for c in computers} + assert by_host['F67HSGH4']['assetid'] == 'PC-1001' + assert by_host['FDKKSGH4']['assetid'] == 'FDKKSGH4' + + assets = _rows(produced['assets']) + assert [a['assetnumber'] for a in assets] == ['FDKKSGH4'] + + +def test_thor_all_matched_emits_no_assets_file(): + produced = ThorSource.transform( + THOR_TWO_CSV, + options={'hostname_map': {'f67hsgh4': 'PC-1001', 'fdkksgh4': 'PC-1002'}}) + assert 'assets' not in produced + computers = _rows(produced['computers']) + assert {c['assetid'] for c in computers} == {'PC-1001', 'PC-1002'} + + +def test_get_source_unknown_raises(): + with pytest.raises(ImportSourceError): + get_source('nope') + + +def test_detect_source_none_for_unknown_header(): + assert detect_source(['foo', 'bar']) is None diff --git a/tests/test_plugins/test_computers_import_machine_link.py b/tests/test_plugins/test_computers_import_machine_link.py new file mode 100644 index 00000000..28162565 --- /dev/null +++ b/tests/test_plugins/test_computers_import_machine_link.py @@ -0,0 +1,187 @@ +"""computers.csv can carry the machine it drives, and the link is built on import. + +A site with a spreadsheet knows "this PC drives machine CNC-014" and wants to +write that in the PC row, not hand-build an assetrelationships sheet keyed on +ids it never sees. The importer accepts a virtual `machineassetnumber` column, +keeps it out of the computers write, and hands it to the computers plugin after +each PC row is written. The rules that matter for a cold user driving this off a +spreadsheet: + +- a good number builds the PC-controls-machine link; +- a number that has no machine, names this PC itself, names a location-only + marker, or is claimed twice in one file, is an AMBER warning and is skipped - + the PC still imports, the file is still ok; +- a dry run builds nothing and, critically, sends no alert; +- re-running the same file does not duplicate the link. +""" +import pytest + +from shopdb.core.services.csvimport import ( + Resolver, import_csv, virtual_columns_for, +) +from shopdb.extensions import db as _db + + +REFERENCE = [ + ('assettypes', 'assettype\nComputer\nMachine\n'), + ('assetstatuses', 'status\nIn Use\n'), + ('locationtypes', 'locationtype\nBay\n'), +] + + +def _seed(resolver): + """Reference rows, a location, and the 'controls' relationship type.""" + from shopdb.core.models.relationship import RelationshipType + + for table, text in REFERENCE: + result = import_csv(table, text, resolver=resolver, commit=True) + assert result.ok, [str(p) for p in result.problems] + result = import_csv( + 'locations', 'locationname,locationtypeid\nBay 3,Bay\n', + resolver=resolver, commit=True) + assert result.ok, [str(p) for p in result.problems] + if not RelationshipType.query.filter_by(relationshiptype='controls').first(): + _db.session.add(RelationshipType(relationshiptype='controls')) + # import_csv writes into the session but never commits; commit the setup so + # a dry-run import's rollback only unwinds the dry run, not the fixtures. + _db.session.commit() + + +def _assets(resolver, *rows): + """Create the given assets. Each row is (assetnumber, assettype).""" + body = 'assetnumber,assettypeid,statusid,locationid\n' + body += ''.join('%s,%s,In Use,Bay 3\n' % r for r in rows) + result = import_csv('assets', body, resolver=resolver, commit=True) + assert result.ok, [str(p) for p in result.problems] + _db.session.commit() + + +def _import_computers(resolver, body, commit=True): + return import_csv('computers', body, resolver=resolver, commit=commit) + + +def _controls_target(pc_assetnumber): + """Asset number of the machine this PC actively controls, or None.""" + from shopdb.core.models import Asset + from shopdb.core.models.relationship import RelationshipType, AssetRelationship + + pc = Asset.query.filter_by(assetnumber=pc_assetnumber).first() + controls = RelationshipType.query.filter_by(relationshiptype='controls').first() + rel = AssetRelationship.query.filter( + AssetRelationship.sourceassetid == pc.assetid, + AssetRelationship.relationshiptypeid == controls.relationshiptypeid, + AssetRelationship.isactive.is_(True)).first() + if rel is None: + return None + return _db.session.get(Asset, rel.targetassetid).assetnumber + + +def test_the_column_is_virtual_and_owned_by_the_computers_plugin(db): + """The plugin registers machineassetnumber; core stays ignorant of it.""" + assert 'machineassetnumber' in virtual_columns_for('computers') + + +def test_a_good_machine_number_builds_the_link(db): + resolver = Resolver() + _seed(resolver) + _assets(resolver, ('PC-001', 'Computer'), ('CNC-014', 'Machine')) + result = _import_computers( + resolver, + 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,CNC-014\n') + assert result.ok, [str(p) for p in result.problems] + assert result.warnings == [] + assert _controls_target('PC-001') == 'CNC-014' + + +def test_an_empty_machine_number_links_nothing_and_does_not_warn(db): + resolver = Resolver() + _seed(resolver) + _assets(resolver, ('PC-001', 'Computer')) + result = _import_computers( + resolver, 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,\n') + assert result.ok + assert result.warnings == [] + assert _controls_target('PC-001') is None + + +def test_a_machine_number_with_no_machine_warns_and_skips(db): + """The PC still imports; the unresolved number is amber, not a failed file.""" + resolver = Resolver() + _seed(resolver) + _assets(resolver, ('PC-001', 'Computer')) + result = _import_computers( + resolver, 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,NOPE-99\n') + assert result.ok, 'an unresolved machine number must not fail the file' + assert len(result.warnings) == 1 + warning = result.warnings[0] + assert warning.column == 'machineassetnumber' + assert warning.target == 'NOPE-99' + assert warning.action, 'a warning must carry the one corrective step' + assert _controls_target('PC-001') is None + + +def test_a_pc_naming_its_own_asset_number_is_not_self_linked(db): + resolver = Resolver() + _seed(resolver) + _assets(resolver, ('PC-001', 'Computer')) + result = _import_computers( + resolver, 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,PC-001\n') + assert result.ok + assert len(result.warnings) == 1 + assert 'own asset number' in result.warnings[0].reason + assert _controls_target('PC-001') is None + + +def test_two_rows_claiming_one_machine_link_only_the_first(db): + """The in-file fan-out guard: a copied cell must not steal the bay.""" + resolver = Resolver() + _seed(resolver) + _assets(resolver, + ('PC-001', 'Computer'), ('PC-002', 'Computer'), ('CNC-014', 'Machine')) + result = _import_computers( + resolver, + 'assetid,hostname,machineassetnumber\n' + 'PC-001,WS-PC-001,CNC-014\n' + 'PC-002,WS-PC-002,CNC-014\n') + assert result.ok + assert len(result.warnings) == 1 + assert 'already claimed' in result.warnings[0].reason + assert _controls_target('PC-001') == 'CNC-014' + assert _controls_target('PC-002') is None + + +def test_a_dry_run_builds_no_link_and_leaves_no_trace(db): + resolver = Resolver() + _seed(resolver) + _assets(resolver, ('PC-001', 'Computer'), ('CNC-014', 'Machine')) + result = _import_computers( + resolver, + 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,CNC-014\n', + commit=False) + assert result.ok + assert result.created == 1 + _db.session.rollback() + assert _controls_target('PC-001') is None + + +def test_reimporting_the_same_link_does_not_duplicate(db): + resolver = Resolver() + _seed(resolver) + _assets(resolver, ('PC-001', 'Computer'), ('CNC-014', 'Machine')) + body = 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,CNC-014\n' + first = _import_computers(resolver, body) + assert first.ok and first.warnings == [] + + second = _import_computers(resolver, body) + assert second.ok and second.warnings == [] + assert _controls_target('PC-001') == 'CNC-014' + + from shopdb.core.models import Asset + from shopdb.core.models.relationship import RelationshipType, AssetRelationship + pc = Asset.query.filter_by(assetnumber='PC-001').first() + controls = RelationshipType.query.filter_by(relationshiptype='controls').first() + active = AssetRelationship.query.filter( + AssetRelationship.sourceassetid == pc.assetid, + AssetRelationship.relationshiptypeid == controls.relationshiptypeid, + AssetRelationship.isactive.is_(True)).count() + assert active == 1, 'a re-import must not stack a second active link' From 8fd0d725f718a30eed2d19b68abac7b26242f31e Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Tue, 25 Aug 2026 16:53:51 -0400 Subject: [PATCH 03/14] pin computers migration head to azurefields after Entra field merge The merge of the Entra device-identity work added computers 0002/0003 migrations, advancing the computers head past its anchor. Update the expected-head guard to match so upgrade-all stays green. --- tests/test_plugin_migrations.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_plugin_migrations.py b/tests/test_plugin_migrations.py index e4ef3976..2867c549 100644 --- a/tests/test_plugin_migrations.py +++ b/tests/test_plugin_migrations.py @@ -46,6 +46,10 @@ CUTOVER_PLUGINS = ( # stamp '0001anchor'; measuringtools stamps its real baseline id. EXPECTED_HEAD_REVISION = {plugin: f'{plugin}0001anchor' for plugin in CUTOVER_PLUGINS} EXPECTED_HEAD_REVISION['measuringtools'] = 'measuringtools0001baseline' +# computers adds the Entra/Azure device identity columns (azuredeviceid, then +# azureobjectid + azuredevicecategory) on top of its anchor, so its head is not +# the f-string default. +EXPECTED_HEAD_REVISION['computers'] = 'computers0003azurefields' # backups is also post-cutover: its 0001 really creates backuprevisions, and # 0002 adds lastseenat - when a config was last CONFIRMED unchanged, which dedup # otherwise throws away. From 0577e3224061b1462db8ee6d209bd9ff49b7463a Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Tue, 25 Aug 2026 17:12:01 -0400 Subject: [PATCH 04/14] add CSV import guide and clarify Azure export location Adds a concise native-CSV import guide covering exact headers, required columns, foreign-key-by-name resolution, import order, and the do-not-edit rule for Azure/Thor exports. Also clarifies the Import Site Assets Azure card that Bulk operations lives in the top command bar of the group Members page, matching the Entra UI. --- docs/CSV-IMPORT-GUIDE.md | 72 +++++++++++++++++++ .../src/views/settings/ImportSiteAssets.vue | 2 +- 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 docs/CSV-IMPORT-GUIDE.md diff --git a/docs/CSV-IMPORT-GUIDE.md b/docs/CSV-IMPORT-GUIDE.md new file mode 100644 index 00000000..1cc0a716 --- /dev/null +++ b/docs/CSV-IMPORT-GUIDE.md @@ -0,0 +1,72 @@ +# ShopDB CSV Import Guide + +Short version: one CSV per table, a header row of exact column names, one record +per row. The importer matches on natural keys, so re-importing updates instead of +duplicating. + +## Rules for hand-built native CSVs + +1. First row is the header. Use the exact column names below, lowercase, no spaces. +2. Include every REQUIRED column. Other columns are optional; drop the ones you + do not use. +3. Foreign keys are entered by NAME, not id. Put the human name and the importer + resolves it: `assettypeid=machine`, `statusid=In Use`, `locationid=A1`, + `vendorid=Haas Automation`. The referenced record must already exist (import + lookup tables first, see order below). +4. Booleans (`isactive`, etc.) are `1`/`0` or `true`/`false`. Leave blank for the + default. +5. Dates are `YYYY-MM-DD`. Leave blank if unknown. +6. Leave a cell blank to skip it. Do not type `NULL` or `N/A`. +7. Save as UTF-8 CSV, comma-separated. ASCII only (no smart quotes, no em-dashes). + +## Import order (dependencies first) + +assetstatuses, assettypes, locationtypes, modeltypes, computertypes, +machinetypes, businessunits, locations, vendors, models, operatingsystems, assets, +computers, machines. + +An asset row must exist before its `computers`/`machines` subtype row: the subtype +links back with `assetid` (the asset's `assetnumber`). + +## Columns (REQUIRED in bold) + +- assetstatuses: **status**, description, color, isactive +- locationtypes: **locationtype**, description, color, isactive +- modeltypes: **modeltype**, category, description, icon, isactive +- computertypes: **computertype**, description, icon, color, isactive +- machinetypes: **machinetype**, description, icon, color, isactive +- businessunits: **businessunit**, code, description, parentid, isactive +- locations: **locationname**, building, floor, room, description, locationtypeid, + parentlocationid, isactive +- vendors: **vendor**, description, website, supportphone, supportemail, notes, + isactive +- models: **modelnumber**, modeltypeid, vendorid, description, notes, isactive +- operatingsystems: **osname**, osversion, architecture, endoflife, isactive +- assets: **assetnumber**, **assettypeid**, name, serialnumber, gaugelabreference, + maintenancereference, statusid, locationid, businessunitid, notes, isactive +- computers: **assetid**, computertypeid, hostname, osid, vendorid, modelnumberid, + loggedinuser, isactive +- machines: **assetid**, machinetypeid, vendorid, modelnumberid, requiresmanualconfig, + islocationonly, lastmaintenancedate, nextmaintenancedate, maintenanceintervaldays, + controllervendorid, controllermodelid, isactive + +Measuring tools are assets: add an `assets` row with `assettypeid=measuring_tool`. + +## Example (assets.csv) + + assetnumber,name,serialnumber,assettypeid,statusid,locationid + MILL-01,Haas VF-2 Mill,SN1001,machine,In Use,A1 + PC-01,Shopfloor PC 01,SN2001,computer,In Use,B1 + MIC-01,Mitutoyo Micrometer,SN3001,measuring_tool,In Use,Gage Lab + +Then machines.csv links the machine subtype: + + assetid,machinetypeid,vendorid + MILL-01,Mill,Haas Automation + +## Azure and Thor exports (do NOT edit) + +For Azure (Entra device-group) and Thor (PC inventory) files, upload the export +AS-IS. The importer auto-detects them by their header and extracts the fields it +needs. Do not rename headers, reorder columns, add/remove rows, or reformat cells. +Editing the export breaks auto-detection. diff --git a/frontend/src/views/settings/ImportSiteAssets.vue b/frontend/src/views/settings/ImportSiteAssets.vue index 8cfeeba2..a365e22e 100644 --- a/frontend/src/views/settings/ImportSiteAssets.vue +++ b/frontend/src/views/settings/ImportSiteAssets.vue @@ -186,7 +186,7 @@
  • Open the Microsoft Entra admin center (entra.microsoft.com).
  • Go to Groups, then open the device group you want to import.
  • Select Members in the left menu.
    • -
  • Click Bulk Operations->Download Members to export the CSV.
    • +
  • In the top command bar, click Bulk operations->Download members to export the CSV.
  • Upload that CSV here with the source set to Azure device group.
    • From 3cb15239dd1de8c0e823184f1ee0998ca955fb3f Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Wed, 26 Aug 2026 11:55:05 -0400 Subject: [PATCH 05/14] curate shipped catalog and clarify subtype import docs Rework the seed catalog from the cleaned live DB: dedupe Dell vendor/model rows, strip vendor names out of model numbers, add missing models, assign model types across printer and equipment lines, add a Controller type, and split a mashup HP LaserJet row into six real models. Now 44 vendors, 172 models, 173 modelsupplies. Also document that assets.csv needs its subtype file to appear in typed list views, and ignore devnotes/ scratch. --- .gitignore | 3 + docs/CSV-IMPORT-GUIDE.md | 21 +- shopdb/data/catalog.json | 1883 +++++++++++++++++++++++++------------- 3 files changed, 1270 insertions(+), 637 deletions(-) diff --git a/.gitignore b/.gitignore index d81657df..7f156e2e 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,6 @@ scripts/site_imports/wjf/idmap.json # work-PC publication clone: bundle drop folder for the transfer pipeline _transfer/ + +# Local developer scratch notes and chat handoffs - never committed +devnotes/ diff --git a/docs/CSV-IMPORT-GUIDE.md b/docs/CSV-IMPORT-GUIDE.md index 1cc0a716..53b93d58 100644 --- a/docs/CSV-IMPORT-GUIDE.md +++ b/docs/CSV-IMPORT-GUIDE.md @@ -4,6 +4,24 @@ Short version: one CSV per table, a header row of exact column names, one record per row. The importer matches on natural keys, so re-importing updates instead of duplicating. +## IMPORTANT: assets.csv alone will NOT show up in the lists + +An `assets.csv` row only creates the base asset record. A computer, machine, or +measuring tool does NOT appear in its list view (PCs, Machines, Measuring Tools) +until its matching subtype row is also imported: + +- A computer asset needs a row in `computers.csv` (`assetid` = the asset's + `assetnumber`) before it shows under PCs. +- A machine asset needs a row in `machines.csv` before it shows under Machines. +- A measuring-tool asset is managed by the Measuring Tools plugin, not the native + subtype importer, so it will exist as an asset but will not appear in the + Measuring Tools list from `assets.csv` alone. + +So always import the subtype file (`computers.csv` / `machines.csv`) right after +`assets.csv`. Importing only `assets.csv` leaves the records invisible in the +typed lists even though they are in the database. (The Azure and Thor imports do +both steps automatically, which is why Azure/Thor PCs appear immediately.) + ## Rules for hand-built native CSVs 1. First row is the header. Use the exact column names below, lowercase, no spaces. @@ -26,7 +44,8 @@ machinetypes, businessunits, locations, vendors, models, operatingsystems, asset computers, machines. An asset row must exist before its `computers`/`machines` subtype row: the subtype -links back with `assetid` (the asset's `assetnumber`). +links back with `assetid` (the asset's `assetnumber`). Import the subtype file in +the same session so the item appears in its list (see the IMPORTANT note above). ## Columns (REQUIRED in bold) diff --git a/shopdb/data/catalog.json b/shopdb/data/catalog.json index 34aaf6cc..ca9a24b7 100644 --- a/shopdb/data/catalog.json +++ b/shopdb/data/catalog.json @@ -7,7 +7,7 @@ { "modeltype": "Access Point", "category": "Network", - "description": "Wireless access point for network connectivity" + "description": "Wireless Access Point" }, { "modeltype": "Broach", @@ -15,13 +15,12 @@ }, { "modeltype": "Camera", - "category": "Network", - "description": "Security camera for facility monitoring" + "category": "Network" }, { "modeltype": "CMM", "category": "Equipment", - "description": "A coordinate-measuring machine (CMM) is a device that measures the geometry of physical objects by sensing discrete points on the surface of the object with a probe." + "description": "Coordinate Measuring Machine" }, { "modeltype": "CNC Lathe", @@ -33,6 +32,11 @@ "category": "Equipment", "description": "CNC Milling Machine" }, + { + "modeltype": "Controller", + "category": "Equipment", + "description": "CNC/machine control unit" + }, { "modeltype": "Desktop PC", "category": "PC", @@ -40,21 +44,25 @@ }, { "modeltype": "Eddy Current", - "category": "Equipment", - "description": "Wild Stallions will never be a super band until we have Eddie Van Halen on guitar." + "category": "Equipment" }, { "modeltype": "EDM", "category": "Equipment", "description": "Electrical Discharge Machine" }, + { + "modeltype": "Firewall", + "category": "Network" + }, { "modeltype": "Furnace", "category": "Equipment" }, { "modeltype": "Grinder", - "category": "Equipment" + "category": "Equipment", + "description": "Grinding Machine" }, { "modeltype": "Hobbing Machine", @@ -66,8 +74,11 @@ }, { "modeltype": "IDF", - "category": "Network", - "description": "Intermediate Distribution Frame - network equipment closet" + "category": "Network" + }, + { + "modeltype": "Inspection", + "category": "Equipment" }, { "modeltype": "Inspection Station", @@ -85,11 +96,6 @@ }, { "modeltype": "Lathe", - "category": "Equipment", - "description": "The Okuma & Howa 2SPV80 is a CNC machine renowned for its precision and efficiency in manufacturing processes. It is widely utilized across various sectors, facilitating the production of intricate components with ease. Industries such as automotive, aerospace, and metalworking benefit from its capabilities. " - }, - { - "modeltype": "LocationOnly", "category": "Equipment" }, { @@ -102,8 +108,7 @@ }, { "modeltype": "Part Marker", - "category": "Equipment", - "description": "Dot peen markers provide a reliable, permanent method for marking materials with information that is crucial for product identification and traceability in various industrial applications." + "category": "Equipment" }, { "modeltype": "Part Washer", @@ -111,13 +116,11 @@ }, { "modeltype": "PC", - "category": "PC", - "description": "Standard user workstation computer" + "category": "PC" }, { "modeltype": "Printer", - "category": "Printer", - "description": "Network printer - HP, Xerox, or other print devices" + "category": "Printer" }, { "modeltype": "Robotic Deburring", @@ -126,7 +129,7 @@ { "modeltype": "Server", "category": "Network", - "description": "Physical or virtual server" + "description": "Server" }, { "modeltype": "Shopfloor PC", @@ -135,13 +138,16 @@ }, { "modeltype": "Shotpeen", - "category": "Equipment", - "description": "Shot peening is a cold working process used to produce a compressive residual stress layer and modify the mechanical properties of metals and composites." + "category": "Equipment" + }, + { + "modeltype": "Spline Checker", + "category": "Equipment" }, { "modeltype": "Switch", "category": "Network", - "description": "Network switch for connectivity" + "description": "Network Switch" }, { "modeltype": "USB Device", @@ -169,18 +175,22 @@ }, { "machinetype": "CMM", - "description": "A coordinate-measuring machine (CMM) is a device that measures the geometry of physical objects by sensing discrete points on the surface of the object with a probe.", - "color": "#8e44ad" + "description": "Coordinate Measuring Machine" }, { - "machinetype": "Eddy Current", - "description": "Wild Stallions will never be a super band until we have Eddie Van Halen on guitar." + "machinetype": "CNC", + "description": "Computer Numerical Control machine" + }, + { + "machinetype": "EDM", + "description": "Electrical Discharge Machine" }, { "machinetype": "Furnace" }, { - "machinetype": "Grinder" + "machinetype": "Grinder", + "description": "Grinding machine" }, { "machinetype": "Hobbing Machine" @@ -193,33 +203,39 @@ }, { "machinetype": "Lathe", - "description": "The Okuma & Howa 2SPV80 is a CNC machine renowned for its precision and efficiency in manufacturing processes. It is widely utilized across various sectors, facilitating the production of intricate components with ease. Industries such as automotive, aerospace, and metalworking benefit from its capabilities. " + "description": "Lathe machine" }, { - "machinetype": "LocationOnly" - }, - { - "machinetype": "Measuring Machine" + "machinetype": "Mill", + "description": "Milling machine" }, { "machinetype": "Mill Turn" }, + { + "machinetype": "Other", + "description": "Other machine type" + }, { "machinetype": "Part Marker", - "description": "Dot peen markers provide a reliable, permanent method for marking materials with information that is crucial for product identification and traceability in various industrial applications." + "description": "Part marking/engraving machine" }, { "machinetype": "Part Washer" }, + { + "machinetype": "Press", + "description": "Press machine" + }, + { + "machinetype": "Robot", + "description": "Industrial robot" + }, { "machinetype": "Robotic Deburring" }, { - "machinetype": "Shotpeen", - "description": "Shot peening is a cold working process used to produce a compressive residual stress layer and modify the mechanical properties of metals and composites." - }, - { - "machinetype": "USB Device" + "machinetype": "Shotpeen" }, { "machinetype": "Vertical Lathe" @@ -228,38 +244,92 @@ "machinetype": "Vertical Turning Center" }, { - "machinetype": "Wax Trace" + "machinetype": "Workstation", + "description": "Shop floor workstation" } ], "computertypes": [ { - "computertype": "CMM", - "color": "#7934f3" + "computertype": "CMM" + }, + { + "computertype": "CMM PC", + "description": "PC dedicated to CMM operation" + }, + { + "computertype": "CMMC Device", + "description": "CMMC compliance device" + }, + { + "computertype": "Dashboard" }, { "computertype": "Engineer" }, + { + "computertype": "Engineer Workstation", + "description": "Engineering workstation for CAD/CAM work" + }, + { + "computertype": "Engineering Workstation", + "description": "CAD/CAM workstation" + }, { "computertype": "Genspect" }, { "computertype": "Heat Treat" }, + { + "computertype": "Inspection PC", + "description": "Quality inspection computer" + }, { "computertype": "Keyence" }, + { + "computertype": "Kiosk", + "description": "Kiosk or info display PC" + }, + { + "computertype": "Laptop", + "description": "Laptop computer" + }, + { + "computertype": "Lobby Display" + }, + { + "computertype": "Office PC", + "description": "General office computer" + }, + { + "computertype": "Other", + "description": "Other computer type" + }, { "computertype": "Part Marker" }, + { + "computertype": "Server", + "description": "Server system" + }, { "computertype": "Shopfloor" }, + { + "computertype": "Shopfloor PC", + "description": "Shop floor production computer" + }, { "computertype": "Standard" }, { "computertype": "Uncategorized" }, + { + "computertype": "Virtual Machine", + "description": "Virtual machine" + }, { "computertype": "Wax / Trace" } @@ -267,8 +337,11 @@ "printertypes": [ { "printertype": "Card", - "description": "ID / card printer", - "color": "#2dce89" + "description": "ID / card printer" + }, + { + "printertype": "Color Laser Printer", + "description": "Color laser printer" }, { "printertype": "Dot Matrix", @@ -280,19 +353,35 @@ }, { "printertype": "Label", - "description": "Label / receipt printer" + "description": "Label/barcode printer" + }, + { + "printertype": "Label Printer", + "description": "Label/barcode printer" }, { "printertype": "Laser", - "description": "Laser printer (toner)" + "description": "Standard laser printer" + }, + { + "printertype": "Laser Printer", + "description": "Laser printer" }, { "printertype": "MFP", "description": "Multifunction printer with scan/copy/fax" }, + { + "printertype": "Multifunction Printer", + "description": "MFP with scan/fax" + }, + { + "printertype": "Network Printer", + "description": "Network-connected printer" + }, { "printertype": "Other", - "description": "Unclassified / other printer" + "description": "Other printer type" }, { "printertype": "Plotter", @@ -304,25 +393,52 @@ } ], "networkdevicetypes": [ + { + "networkdevicetype": "Access Panel" + }, { "networkdevicetype": "Access Point", - "description": "Wireless access point for network connectivity" + "description": "Wireless access point" }, { "networkdevicetype": "Camera", - "description": "Security camera for facility monitoring" + "description": "IP camera" + }, + { + "networkdevicetype": "Firewall", + "description": "Network firewall" }, { "networkdevicetype": "IDF", - "description": "Intermediate Distribution Frame - network equipment closet" + "description": "Intermediate Distribution Frame/closet" + }, + { + "networkdevicetype": "MDF", + "description": "Main Distribution Frame" + }, + { + "networkdevicetype": "Other", + "description": "Other network device" + }, + { + "networkdevicetype": "Patch Panel", + "description": "Patch panel" + }, + { + "networkdevicetype": "Router", + "description": "Network router" }, { "networkdevicetype": "Server", - "description": "Physical or virtual server" + "description": "Server (reclassified from PCs)" }, { "networkdevicetype": "Switch", - "description": "Network switch for connectivity" + "description": "Network switch" + }, + { + "networkdevicetype": "UPS", + "description": "Uninterruptible power supply" } ], "locationtypes": [ @@ -383,6 +499,12 @@ { "osname": "Microsoft Windows 11 Enterprise" }, + { + "osname": "Microsoft Windows 11 Enterprise LTSC" + }, + { + "osname": "Microsoft Windows 11 Enterprise LTSC 24H2 (build 26100)" + }, { "osname": "TBD" }, @@ -390,10 +512,28 @@ "osname": "Windows 10", "osversion": "10.0" }, + { + "osname": "Windows 10 Enterprise" + }, + { + "osname": "Windows 10 Pro" + }, + { + "osname": "Windows 10.0.26100" + }, { "osname": "Windows 11", "osversion": "11.0" }, + { + "osname": "Windows 11 24H2" + }, + { + "osname": "Windows 11 Enterprise" + }, + { + "osname": "Windows 11 Pro" + }, { "osname": "Windows Server 2019", "osversion": "2019" @@ -407,22 +547,21 @@ { "vendor": "ABTech" }, - { - "vendor": "Allen-Bradley" - }, { "vendor": "Broach" }, { - "vendor": "Brother", - "website": "https://www.brother.com" + "vendor": "Brother" }, { - "vendor": "Brown/Sharpe" + "vendor": "Brown & Sharpe" }, { "vendor": "Campbell" }, + { + "vendor": "Checkpoint" + }, { "vendor": "Cisco" }, @@ -435,9 +574,6 @@ { "vendor": "Doosan" }, - { - "vendor": "DT" - }, { "vendor": "Eddy" }, @@ -450,6 +586,9 @@ { "vendor": "Fidia" }, + { + "vendor": "Gleason" + }, { "vendor": "Gleason-Pfauter" }, @@ -468,15 +607,17 @@ { "vendor": "HID" }, + { + "vendor": "Hoffmann" + }, { "vendor": "HP" }, { - "vendor": "HP Inc.", - "website": "https://www.hp.com" + "vendor": "Hwacheon" }, { - "vendor": "Hwacheon" + "vendor": "Intermec" }, { "vendor": "Keyence" @@ -484,17 +625,9 @@ { "vendor": "LaPointe" }, - { - "vendor": "Lenovo", - "website": "https://www.lenovo.com" - }, { "vendor": "Makino" }, - { - "vendor": "Microsoft", - "website": "https://www.microsoft.com" - }, { "vendor": "Mitutoyo" }, @@ -514,34 +647,19 @@ "vendor": "Phoenix Inc" }, { - "vendor": "Progessive" - }, - { - "vendor": "QEMU" + "vendor": "Progressive" }, { "vendor": "Ransohoff" }, - { - "vendor": "Siemens" - }, - { - "vendor": "TBD" - }, { "vendor": "Telesis" }, - { - "vendor": "Test" - }, { "vendor": "Toshulin" }, { - "vendor": "UnscopedLiveVendor" - }, - { - "vendor": "VM-Ware" + "vendor": "VMware" }, { "vendor": "WJDT" @@ -552,10 +670,6 @@ { "vendor": "Zebra" }, - { - "vendor": "Zebra Technologies", - "website": "https://www.zebra.com" - }, { "vendor": "Zeiss" }, @@ -567,35 +681,42 @@ { "modelnumber": "EAS1000", "vendor": "ABTech", - "modeltype": "Measuring Machine", - "imageurl": "/images/models/machines/abtech-eas1000.png" - }, - { - "modelnumber": "Allen-Bradley", - "vendor": "Allen-Bradley", - "description": "Allen-Bradley Controller" + "modeltype": "Measuring Machine" }, { "modelnumber": "Vertical Broach", "vendor": "Broach", "modeltype": "Broach" }, + { + "modelnumber": "MFC-J5845DW", + "vendor": "Brother", + "modeltype": "Printer" + }, { "modelnumber": "Model One", - "vendor": "Brown/Sharpe", - "modeltype": "LocationOnly" + "vendor": "Brown & Sharpe", + "modeltype": "CMM" }, { "modelnumber": "1000C1000", "vendor": "Campbell", - "modeltype": "Grinder", - "imageurl": "/images/models/machines/1000C1000.jpg" + "modeltype": "Grinder" + }, + { + "modelnumber": "6400", + "vendor": "Checkpoint", + "modeltype": "Firewall" + }, + { + "modelnumber": "Virtual Firewall", + "vendor": "Checkpoint", + "modeltype": "Firewall" }, { "modelnumber": "9120AXI-B", "vendor": "Cisco", - "modeltype": "Access Point", - "imageurl": "/images/models/machines/Cisco9120.png" + "modeltype": "Access Point" }, { "modelnumber": "C9200-48P-E", @@ -607,56 +728,71 @@ "vendor": "Cisco", "modeltype": "Switch" }, - { - "modelnumber": "OptiPlex 5000", - "vendor": "Dell Inc." - }, - { - "modelnumber": "Dell Pro 13 Plus PB13250", - "vendor": "Dell Inc.", - "modeltype": "LocationOnly" - }, - { - "modelnumber": "Dell Pro 14 Plus PB14250", - "vendor": "Dell Inc.", - "modeltype": "LocationOnly" - }, - { - "modelnumber": "Dell Pro Tower Plus QBT1250", - "vendor": "Dell Inc." - }, { "modelnumber": "Latitude 5340", "vendor": "Dell Inc.", - "modeltype": "PC" + "modeltype": "Laptop" }, { "modelnumber": "Latitude 5350", "vendor": "Dell Inc.", - "modeltype": "PC" + "modeltype": "Laptop" + }, + { + "modelnumber": "Latitude 5420", + "vendor": "Dell Inc.", + "modeltype": "Laptop" }, { "modelnumber": "Latitude 5430", "vendor": "Dell Inc.", - "modeltype": "LocationOnly" + "modeltype": "Laptop" }, { "modelnumber": "Latitude 5440", "vendor": "Dell Inc.", - "modeltype": "LocationOnly", - "imageurl": "/images/models/machines/latitude5440.png" + "modeltype": "Laptop" }, { "modelnumber": "Latitude 5450", "vendor": "Dell Inc.", - "modeltype": "LocationOnly", - "imageurl": "/images/models/computers/Latitude-5450.png" + "modeltype": "Laptop" + }, + { + "modelnumber": "Latitude 7230 Rugged Extreme Tablet", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Latitude 7320 Detachable", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Latitude 7330 Rugged Extreme", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Latitude 7350 Detachable", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "OptiPlex 3000", + "vendor": "Dell Inc.", + "modeltype": "PC" }, { "modelnumber": "OptiPlex 3080", "vendor": "Dell Inc.", "modeltype": "PC" }, + { + "modelnumber": "OptiPlex 5000", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, { "modelnumber": "OptiPlex 5040", "vendor": "Dell Inc.", @@ -665,48 +801,77 @@ { "modelnumber": "OptiPlex 5050", "vendor": "Dell Inc.", - "modeltype": "PC", - "imageurl": "/images/models/computers/Optiplex-5050.png" + "modeltype": "PC" }, { "modelnumber": "OptiPlex 5060", "vendor": "Dell Inc.", - "modeltype": "PC", - "imageurl": "/images/models/computers/Optiplex-5060.png" + "modeltype": "PC" }, { "modelnumber": "OptiPlex 7000", "vendor": "Dell Inc.", - "modeltype": "PC", - "imageurl": "/images/models/computers/Optiplex-7000.png" + "modeltype": "PC" }, { "modelnumber": "OptiPlex 7070", "vendor": "Dell Inc.", - "modeltype": "PC", - "imageurl": "/images/models/machines/OptiPlex-7070.png" + "modeltype": "PC" }, { "modelnumber": "OptiPlex 7080", "vendor": "Dell Inc.", - "modeltype": "LocationOnly", - "imageurl": "/images/models/computers/Optiplex-7080.jpg" + "modeltype": "PC" }, { "modelnumber": "OptiPlex 7090", "vendor": "Dell Inc.", - "modeltype": "LocationOnly" + "modeltype": "PC" + }, + { + "modelnumber": "OptiPlex 7400 AIO", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "OptiPlex 7490 AIO", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "OptiPlex AIO Plus 7410", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "OptiPlex AIO Plus 7420", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "OptiPlex Micro 7010", + "vendor": "Dell Inc.", + "modeltype": "PC" }, { "modelnumber": "OptiPlex Micro 7020", "vendor": "Dell Inc.", - "modeltype": "LocationOnly" + "modeltype": "PC" + }, + { + "modelnumber": "OptiPlex SFF 7020", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "OptiPlex SFF Plus 7010", + "vendor": "Dell Inc.", + "modeltype": "PC" }, { "modelnumber": "OptiPlex Tower Plus 7010", "vendor": "Dell Inc.", - "modeltype": "PC", - "imageurl": "/images/models/computers/OptiPlex-Tower-Plus-7010.png" + "modeltype": "PC" }, { "modelnumber": "OptiPlex Tower Plus 7020", @@ -714,210 +879,244 @@ "modeltype": "PC" }, { - "modelnumber": "Precision 5560", - "vendor": "Dell Inc.", - "modeltype": "LocationOnly", - "imageurl": "/images/models/machines/precision5560.jpg" - }, - { - "modelnumber": "Precision 5570", - "vendor": "Dell Inc.", - "modeltype": "LocationOnly" - }, - { - "modelnumber": "Precision 5680", - "vendor": "Dell Inc.", - "modeltype": "LocationOnly" - }, - { - "modelnumber": "Precision 5690", - "vendor": "Dell Inc.", - "modeltype": "LocationOnly" - }, - { - "modelnumber": "Precision 5820 Tower", - "vendor": "Dell Inc.", - "modeltype": "LocationOnly" - }, - { - "modelnumber": "Precision 5860 Tower", - "vendor": "Dell Inc." - }, - { - "modelnumber": "Precision 7680", - "vendor": "Dell Inc.", - "modeltype": "LocationOnly" - }, - { - "modelnumber": "Precision 7750", + "modelnumber": "Precision 3650", "vendor": "Dell Inc.", "modeltype": "PC" }, + { + "modelnumber": "Precision 3660", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "Precision 5490", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Precision 5550", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Precision 5560", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Precision 5570", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Precision 5680", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Precision 5690", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Precision 5820 Tower", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "Precision 5860 Tower", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "Precision 7680", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Precision 7750", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Precision 7770", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, { "modelnumber": "Precision 7780", "vendor": "Dell Inc.", - "modeltype": "LocationOnly" + "modeltype": "Laptop" + }, + { + "modelnumber": "Precision 7820 Tower", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "Precision 7865 Tower", + "vendor": "Dell Inc.", + "modeltype": "PC" }, { "modelnumber": "Precision 7875 Tower", "vendor": "Dell Inc.", - "modeltype": "LocationOnly" + "modeltype": "PC" + }, + { + "modelnumber": "Precision 7920 Tower", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "Precision 7960 Tower", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "Pro 13 Plus PB13250", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Pro 13 Premium PA13250", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Pro 14 PC14250", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Pro 14 Plus PB14250", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Pro Desktops 24 AIO Plus QB24250", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "Pro Desktops Micro QCM1250", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "Pro Desktops Tower Plus QBT1250", + "vendor": "Dell Inc.", + "modeltype": "PC" + }, + { + "modelnumber": "Pro Max 14 MC14250", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Pro Max 14 Premium MA14250", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Pro Max 16 MC16250", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Pro Max 16 Plus MB16250", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Pro Max 16 Premium MA16250", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Pro Max 18 Plus MB18250", + "vendor": "Dell Inc.", + "modeltype": "Laptop" + }, + { + "modelnumber": "Pro Rugged 14 RB14250", + "vendor": "Dell Inc.", + "modeltype": "Laptop" }, { "modelnumber": "NT-4300", "vendor": "DMG Mori", - "modeltype": "Mill Turn", - "imageurl": "/images/models/machines/nt4300.jpg" + "modeltype": "Mill Turn" }, { "modelnumber": "Puma MX3100", "vendor": "Doosan", - "modeltype": "Mill Turn", - "imageurl": "/images/models/machines/mx3100.png" - }, - { - "modelnumber": "IDF", - "vendor": "DT", - "modeltype": "IDF" - }, - { - "modelnumber": "Server", - "vendor": "DT", - "modeltype": "Server" + "modeltype": "Mill Turn" }, { "modelnumber": "EZ-Eddy", "vendor": "Eddy", - "modeltype": "Eddy Current", - "imageurl": "/images/models/machines/ezeddy.png" + "modeltype": "Eddy Current" }, { "modelnumber": "TM-C3500", "vendor": "Epson", - "modeltype": "LocationOnly", - "imageurl": "/images/models/printers/Epson-C3500.png" + "modeltype": "Printer" }, { - "modelnumber": "0i-TC", + "modelnumber": "31i-MB", "vendor": "Fanuc", - "description": "FANUC Series 0i-TC Controller" - }, - { - "modelnumber": "18i-TB", - "vendor": "Fanuc", - "description": "FANUC Series 18i-TB Controller" - }, - { - "modelnumber": "31i", - "vendor": "Fanuc", - "description": "FANUC Series 31i Controller" - }, - { - "modelnumber": "31i-A", - "vendor": "Fanuc", - "description": "FANUC Series 31i-MODEL A Controller" - }, - { - "modelnumber": "31i-A5", - "vendor": "Fanuc", - "description": "FANUC Series 31i-MODEL A5 Controller" - }, - { - "modelnumber": "31i-B", - "vendor": "Fanuc", - "description": "FANUC Series 31i-MODEL B Controller" - }, - { - "modelnumber": "31i-B5", - "vendor": "Fanuc", - "description": "FANUC Series 31i-MODEL B5 Controller" - }, - { - "modelnumber": "32i-B", - "vendor": "Fanuc", - "description": "FANUC Series 32i-MODEL B Controller" + "modeltype": "Controller" }, { "modelnumber": "M710UC", "vendor": "Fanuc", - "modeltype": "Robotic Deburring", - "imageurl": "/images/models/machines/M710uc.png" - }, - { - "modelnumber": "Panel i", - "vendor": "Fanuc", - "description": "FANUC Panel i Controller" - }, - { - "modelnumber": "Power Motion i-A", - "vendor": "Fanuc", - "description": "FANUC Power Motion i-MODEL A Controller" - }, - { - "modelnumber": "Professional 5", - "vendor": "Fanuc", - "description": "FANUC Professional 5 Controller" - }, - { - "modelnumber": "R-30iA", - "vendor": "Fanuc", - "description": "FANUC System R-30iA Controller" - }, - { - "modelnumber": "R-30iB", - "vendor": "Fanuc", - "description": "FANUC System R-30iB Controller" - }, - { - "modelnumber": "Unknown", - "vendor": "Fanuc", - "description": "FANUC (Unknown Model) Controller" + "modeltype": "Robotic Deburring" }, { "modelnumber": "D218", "vendor": "Fidia", - "modeltype": "5-axis Mill", - "imageurl": "/images/models/machines/d218.png" + "modeltype": "5-axis Mill" }, { - "modelnumber": "FIDIA C20", - "vendor": "Fidia", - "description": "FIDIA C20 Controller" + "modelnumber": "475GMS", + "vendor": "Gleason", + "modeltype": "Spline Checker" }, { "modelnumber": "P600S", "vendor": "Gleason-Pfauter", - "modeltype": "Hobbing Machine", - "imageurl": "/images/models/machines/p600s.png" + "modeltype": "Hobbing Machine" }, { "modelnumber": "Vacuum Furnace", "vendor": "GM Enterprises", - "modeltype": "Furnace", - "imageurl": "/images/models/machines/furnace.png" + "modeltype": "Furnace" }, { "modelnumber": "550T", "vendor": "Grob", - "modeltype": "LocationOnly" + "modeltype": "5-axis Mill" }, { "modelnumber": "G750", "vendor": "Grob", - "modeltype": "Mill Turn", - "imageurl": "/images/models/machines/g750.jpg" + "modeltype": "Mill Turn" + }, + { + "modelnumber": "VF-2SS", + "vendor": "Haas Automation", + "modeltype": "CNC Mill" }, { "modelnumber": "7.10.7 SF", "vendor": "Hexagon", - "modeltype": "CMM", - "imageurl": "/images/models/machines/7107sf.png" + "modeltype": "CMM" }, { "modelnumber": "CMM", "vendor": "Hexagon", - "modeltype": "CMM", - "imageurl": "/images/models/machines/cmm.png" + "modeltype": "CMM" }, { "modelnumber": "Global Advantage", @@ -927,147 +1126,236 @@ { "modelnumber": "DTC 4500e", "vendor": "HID", - "modeltype": "LocationOnly", - "imageurl": "/images/models/printers/DTC4500e.png" + "modeltype": "Printer" }, { - "modelnumber": "Color Laserjet CP2025", - "vendor": "HP", - "modeltype": "LocationOnly", - "imageurl": "/images/models/printers/LaserJet -CP2025.png" + "modelnumber": "Hoffmann Broach", + "vendor": "Hoffmann", + "modeltype": "Broach" }, { - "modelnumber": "Color LaserJet M254dw", - "vendor": "HP", - "modeltype": "Printer", - "imageurl": "/images/models/printers/LaserJet-M254dw.png" - }, - { - "modelnumber": "Color LaserJet M255dw", - "vendor": "HP", - "modeltype": "LocationOnly" - }, - { - "modelnumber": "HP DesignJet T1700dr PS", - "vendor": "HP", - "modeltype": "LocationOnly", - "imageurl": "/images/models/printers/HP-DesignJet-T1700dr.png" - }, - { - "modelnumber": "HP LaserJet M610 / M611 / M612 / M634 / M635 / M636", + "modelnumber": "Color LaserJet CP2025", "vendor": "HP", "modeltype": "Printer" }, { - "modelnumber": "LaserJet 200 color M251nw", + "modelnumber": "Color LaserJet M254dw", "vendor": "HP", - "imageurl": "/images/models/printers/Laserjet-Pro-M251nw.png" + "modeltype": "Printer" + }, + { + "modelnumber": "Color LaserJet M255dw", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "Color LaserJet Pro 3201dw", + "vendor": "HP", + "modeltype": "Printer", + "documentationurl": "https://support.hp.com/us-en/product/troubleshooting/hp-color-laserjet-pro-3201-3204,-3288-printer-series/model/2101679526", + "imageurl": "/api/models/image/model-112.webp" + }, + { + "modelnumber": "Color LaserJet Pro M454dw", + "vendor": "HP", + "modeltype": "Printer", + "documentationurl": "https://support.hp.com/us-en/drivers/mps-bundle-bom5/model/19203830?sku=W1Y45AR", + "imageurl": "/api/models/image/model-120.png" + }, + { + "modelnumber": "DesignJet T1700dr PS", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "EliteDesk 800 G6", + "vendor": "HP", + "modeltype": "PC" + }, + { + "modelnumber": "LaserJet 4001N", + "vendor": "HP", + "modeltype": "Printer" }, { "modelnumber": "LaserJet 4250tn", "vendor": "HP", - "modeltype": "Printer", - "imageurl": "/images/models/printers/LaserJet-4250.png" + "modeltype": "Printer" }, { - "modelnumber": "LaserJet 4250tn DO NOT USE", + "modelnumber": "LaserJet CP5225", "vendor": "HP", - "modeltype": "LocationOnly", - "imageurl": "/images/models/printers/LaserJet-4250.png" + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet M402N", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet M404N", + "vendor": "HP", + "modeltype": "Printer" }, { "modelnumber": "LaserJet M406", "vendor": "HP", - "modeltype": "LocationOnly", - "imageurl": "/images/models/printers/LaserJet-M406.png" + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet M428Fdn", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet M428Fdw", + "vendor": "HP", + "modeltype": "Printer" }, { "modelnumber": "LaserJet M454dn", "vendor": "HP", - "modeltype": "Printer", - "imageurl": "/images/models/printers/LaserJet-M454dn.png" + "modeltype": "Printer" }, { "modelnumber": "LaserJet M506", "vendor": "HP", - "modeltype": "LocationOnly", - "imageurl": "/images/models/printers/LaserJet-M506.png" + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet M507", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet M553", + "vendor": "HP", + "modeltype": "Printer" }, { "modelnumber": "LaserJet M602", "vendor": "HP", - "modeltype": "Printer", - "imageurl": "/images/models/printers/LaserJet-M602.png" + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet M610", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet M611", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet M612", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet M634", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet M635", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet M636", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet M651", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet P2035", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet P2055Dn", + "vendor": "HP", + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet P3015", + "vendor": "HP", + "modeltype": "Printer" }, { "modelnumber": "LaserJet P3015dn", "vendor": "HP", - "modeltype": "Printer", - "imageurl": "/images/models/printers/LaserJet-P3015dn.png" + "modeltype": "Printer" }, { - "modelnumber": "Laserjet Pro 200 M251nw", + "modelnumber": "LaserJet Pro 200 color M251nw", "vendor": "HP", - "modeltype": "Printer", - "imageurl": "/images/models/printers/Laserjet-Pro-M251nw.png" + "modeltype": "Printer" + }, + { + "modelnumber": "LaserJet Pro 400", + "vendor": "HP", + "modeltype": "Printer" }, { "modelnumber": "LaserJet Pro 4001n", "vendor": "HP", - "modeltype": "Printer", - "imageurl": "/images/models/printers/LaserJet-4001n.png" + "modeltype": "Printer" }, { "modelnumber": "LaserJet Pro M252dw", "vendor": "HP", - "imageurl": "/images/models/printers/LaserJet-Pro-M252dw.png" + "modeltype": "Printer" }, { "modelnumber": "LaserJet Pro M404n", "vendor": "HP", - "modeltype": "Printer", - "imageurl": "/images/models/printers/LaserJet-M404.png" - }, - { - "modelnumber": "LaserJet Pro M454dw", - "vendor": "HP", - "imageurl": "/images/models/printers/LaserJet-M454dn.png" + "modeltype": "Printer" }, { "modelnumber": "LaserJet Pro M607", "vendor": "HP", - "modeltype": "LocationOnly", - "imageurl": "/images/models/printers/LaserJet-M607.png" + "modeltype": "Printer" + }, + { + "modelnumber": "ProLiant DL380 Gen9", + "vendor": "HP", + "modeltype": "Server" }, { "modelnumber": "VT 550 2SP", "vendor": "Hwacheon", - "modeltype": "Vertical Turning Center", - "imageurl": "/images/models/machines/vt5502sp.png" + "modeltype": "Vertical Turning Center" + }, + { + "modelnumber": "PM43", + "vendor": "Intermec", + "modeltype": "Printer" }, { "modelnumber": "VR-3100", "vendor": "Keyence", - "modeltype": "Measuring Machine", - "imageurl": "/images/models/machines/keyence-vr3100.png" + "modeltype": "Measuring Machine" }, { "modelnumber": "Horizontal Broach", "vendor": "LaPointe", - "modeltype": "Broach", - "imageurl": "/images/models/machines/hbroach.png" + "modeltype": "Broach" }, { "modelnumber": "a81nx", "vendor": "Makino", - "modeltype": "Horizontal Machining Center", - "imageurl": "/images/models/machines/a81nx.png" + "modeltype": "Horizontal Machining Center" }, { "modelnumber": "C-4500", "vendor": "Mitutoyo", - "modeltype": "Wax Trace", - "imageurl": "/images/models/machines/c4500.png" + "modeltype": "Wax Trace" }, { "modelnumber": "CV-3200", @@ -1086,8 +1374,7 @@ { "modelnumber": "VP9000", "vendor": "OKK", - "modeltype": "5-axis Mill", - "imageurl": "/images/models/machines/vp9000.jpg" + "modeltype": "5-axis Mill" }, { "modelnumber": "2SP-V80", @@ -1097,159 +1384,161 @@ { "modelnumber": "LOC650", "vendor": "Okuma", - "modeltype": "Mill Turn", - "imageurl": "/images/models/machines/loc650.png" + "modeltype": "Mill Turn" }, { "modelnumber": "Multus", "vendor": "Okuma", "modeltype": "Mill Turn" }, - { - "modelnumber": "OKUMA OSP-P300L", - "vendor": "Okuma", - "description": "OKUMA OSP-P300L Controller" - }, - { - "modelnumber": "OKUMA OSP-P300LA", - "vendor": "Okuma", - "description": "OKUMA OSP-P300LA Controller" - }, { "modelnumber": "VTM-100", "vendor": "Okuma", - "modeltype": "Vertical Lathe", - "imageurl": "/images/models/machines/vtm100.png" + "modeltype": "Vertical Lathe" }, { "modelnumber": "Phoenix Broach", "vendor": "Phoenix Inc", - "modeltype": "Broach", - "imageurl": "/images/models/machines/phoenixbroach.png" + "modeltype": "Broach" }, { "modelnumber": "Shotpeen", - "vendor": "Progessive", - "modeltype": "Shotpeen", - "imageurl": "/images/models/machines/shotpeen.png" - }, - { - "modelnumber": "Ubuntu 24.04 PC (Q35 + ICH9, 2009)", - "vendor": "QEMU" + "vendor": "Progressive", + "modeltype": "Shotpeen" }, { "modelnumber": "Lean Drum", "vendor": "Ransohoff", - "modeltype": "Part Washer", - "imageurl": "/images/models/machines/leandrum.jpg" + "modeltype": "Part Washer" }, { "modelnumber": "LeanJet RB-2", "vendor": "Ransohoff", - "modeltype": "Part Washer", - "imageurl": "/images/models/machines/rb2.png" - }, - { - "modelnumber": "SIEMENS", - "vendor": "Siemens", - "description": "SIEMENS Controller" - }, - { - "modelnumber": "SIEMENS SIMATIC PANEL TOUCH", - "vendor": "Siemens", - "description": "SIEMENS SIMATIC PANEL TOUCH Controller" - }, - { - "modelnumber": "SIEMENS SINUMERIK", - "vendor": "Siemens", - "description": "SIEMENS SINUMERIK Controller" + "modeltype": "Part Washer" }, { "modelnumber": "PINSTAMP - TMP4210 Dot Peen", "vendor": "Telesis", "modeltype": "Part Marker" }, - { - "modelnumber": "Test", - "vendor": "Test" - }, { "modelnumber": "Powerturn", "vendor": "Toshulin", - "modeltype": "Vertical Turning Center", - "imageurl": "/images/models/machines/powerturn.png" + "modeltype": "Vertical Turning Center" }, { - "modelnumber": "Virtual", - "vendor": "VM-Ware", + "modelnumber": "Virtual Server", + "vendor": "VMware", "modeltype": "Server" }, { - "modelnumber": "TBD", + "modelnumber": "Inspection", "vendor": "WJDT", - "modeltype": "LocationOnly" + "modeltype": "Inspection" }, { - "modelnumber": "Altalink C8135", + "modelnumber": "TBD", + "vendor": "WJDT" + }, + { + "modelnumber": "AltaLink B8045", "vendor": "Xerox", - "modeltype": "LocationOnly", - "imageurl": "/images/models/printers/AltaLink-C8130.png" + "modeltype": "Printer" + }, + { + "modelnumber": "AltaLink B8055", + "vendor": "Xerox", + "modeltype": "Printer" + }, + { + "modelnumber": "AltaLink B8170", + "vendor": "Xerox", + "modeltype": "Printer" + }, + { + "modelnumber": "AltaLink C8035", + "vendor": "Xerox", + "modeltype": "Printer" + }, + { + "modelnumber": "AltaLink C8135", + "vendor": "Xerox", + "modeltype": "Printer" }, { "modelnumber": "AltaLink EC8036", "vendor": "Xerox", - "modeltype": "Printer", - "imageurl": "/images/models/printers/Xerox-EC8036.png" + "modeltype": "Printer" }, { - "modelnumber": "Versalink B405DN", + "modelnumber": "VersaLink B400", "vendor": "Xerox", - "modeltype": "LocationOnly", - "imageurl": "/images/models/printers/Versalink-B405.png" + "modeltype": "Printer" }, { - "modelnumber": "Versalink B7125", + "modelnumber": "VersaLink B405", "vendor": "Xerox", - "modeltype": "Printer", - "imageurl": "/images/models/printers/Versalink-B7125.png" + "modeltype": "Printer" }, { - "modelnumber": "Versalink C405", + "modelnumber": "VersaLink B405DN", "vendor": "Xerox", - "modeltype": "LocationOnly", - "imageurl": "/images/models/printers/Versalink-C405.png" + "modeltype": "Printer" }, { - "modelnumber": "Versalink C415", - "vendor": "Xerox" - }, - { - "modelnumber": "Versalink C7100", + "modelnumber": "VersaLink B415", "vendor": "Xerox", - "modeltype": "LocationOnly", - "imageurl": "/images/models/printers/Versalink-C7125.png" + "modeltype": "Printer" }, { - "modelnumber": "Versalink C7125", + "modelnumber": "VersaLink B7100", "vendor": "Xerox", - "modeltype": "Printer", - "imageurl": "/images/models/printers/Versalink-C7125.png" + "modeltype": "Printer" }, { - "modelnumber": "Zebra ZT411", + "modelnumber": "VersaLink B7125", + "vendor": "Xerox", + "modeltype": "Printer" + }, + { + "modelnumber": "VersaLink C405", + "vendor": "Xerox", + "modeltype": "Printer" + }, + { + "modelnumber": "VersaLink C415", + "vendor": "Xerox", + "modeltype": "Printer" + }, + { + "modelnumber": "VersaLink C7100", + "vendor": "Xerox", + "modeltype": "Printer" + }, + { + "modelnumber": "VersaLink C7125", + "vendor": "Xerox", + "modeltype": "Printer" + }, + { + "modelnumber": "ZD621", "vendor": "Zebra", - "modeltype": "LocationOnly", - "imageurl": "/images/models/printers/zt411.png" + "modeltype": "Printer" + }, + { + "modelnumber": "ZT411", + "vendor": "Zebra", + "modeltype": "Printer" + }, + { + "modelnumber": "CONTURA G2", + "vendor": "Zeiss", + "modeltype": "CMM" }, { "modelnumber": "Redomatic 600", "vendor": "Zoller", "modeltype": "Measuring Machine" - }, - { - "modelnumber": "ZOLLER", - "vendor": "Zoller", - "description": "ZOLLER Controller" } ], "modelsupplies": [ @@ -1333,6 +1622,86 @@ "marketingname": "202X Magenta", "pageyield": 2500 }, + { + "modelnumber": "Color LaserJet M255dw", + "vendor": "HP", + "supplytype": "toner", + "color": "black", + "capacitytier": "standard", + "partnumber": "CF500A", + "marketingname": "202A Black", + "pageyield": 1400 + }, + { + "modelnumber": "Color LaserJet M255dw", + "vendor": "HP", + "supplytype": "toner", + "color": "black", + "capacitytier": "high", + "partnumber": "CF500X", + "marketingname": "202X Black", + "pageyield": 3200 + }, + { + "modelnumber": "Color LaserJet M255dw", + "vendor": "HP", + "supplytype": "toner", + "color": "cyan", + "capacitytier": "standard", + "partnumber": "CF501A", + "marketingname": "202A Cyan", + "pageyield": 1300 + }, + { + "modelnumber": "Color LaserJet M255dw", + "vendor": "HP", + "supplytype": "toner", + "color": "cyan", + "capacitytier": "high", + "partnumber": "CF501X", + "marketingname": "202X Cyan", + "pageyield": 2500 + }, + { + "modelnumber": "Color LaserJet M255dw", + "vendor": "HP", + "supplytype": "toner", + "color": "yellow", + "capacitytier": "standard", + "partnumber": "CF502A", + "marketingname": "202A Yellow", + "pageyield": 1300 + }, + { + "modelnumber": "Color LaserJet M255dw", + "vendor": "HP", + "supplytype": "toner", + "color": "yellow", + "capacitytier": "high", + "partnumber": "CF502X", + "marketingname": "202X Yellow", + "pageyield": 2500 + }, + { + "modelnumber": "Color LaserJet M255dw", + "vendor": "HP", + "supplytype": "toner", + "color": "magenta", + "capacitytier": "standard", + "partnumber": "CF503A", + "marketingname": "202A Magenta", + "pageyield": 1300 + }, + { + "modelnumber": "Color LaserJet M255dw", + "vendor": "HP", + "supplytype": "toner", + "color": "magenta", + "capacitytier": "high", + "partnumber": "CF503X", + "marketingname": "202X Magenta", + "pageyield": 2500 + }, { "modelnumber": "Color LaserJet M255dw", "vendor": "HP", @@ -1414,97 +1783,167 @@ "pageyield": 2450 }, { - "modelnumber": "HP LaserJet M610 / M611 / M612 / M634 / M635 / M636", + "modelnumber": "Color LaserJet Pro 3201dw", "vendor": "HP", "supplytype": "toner", "color": "black", "capacitytier": "standard", - "partnumber": "W1470A", - "marketingname": "147A Black", - "pageyield": 10500 + "partnumber": "W2180A", + "marketingname": "218A Black", + "pageyield": 1300 }, { - "modelnumber": "HP LaserJet M610 / M611 / M612 / M634 / M635 / M636", + "modelnumber": "Color LaserJet Pro 3201dw", "vendor": "HP", "supplytype": "toner", "color": "black", "capacitytier": "high", - "partnumber": "W1470X", - "marketingname": "147X Black (not M610)", - "pageyield": 25200 + "partnumber": "W2180X", + "marketingname": "218X Black", + "pageyield": 3200 }, { - "modelnumber": "LaserJet 200 color M251nw", - "vendor": "HP", - "supplytype": "toner", - "color": "black", - "capacitytier": "standard", - "partnumber": "CF210A", - "marketingname": "131A Black", - "pageyield": 1600 - }, - { - "modelnumber": "LaserJet 200 color M251nw", - "vendor": "HP", - "supplytype": "toner", - "color": "black", - "capacitytier": "high", - "partnumber": "CF210X", - "marketingname": "131X Black", - "pageyield": 2400 - }, - { - "modelnumber": "LaserJet 200 color M251nw", + "modelnumber": "Color LaserJet Pro 3201dw", "vendor": "HP", "supplytype": "toner", "color": "cyan", "capacitytier": "standard", - "partnumber": "CF211A", - "marketingname": "131A Cyan", - "pageyield": 1800 + "partnumber": "W2181A", + "marketingname": "218A Cyan", + "pageyield": 1200 }, { - "modelnumber": "LaserJet 200 color M251nw", + "modelnumber": "Color LaserJet Pro 3201dw", + "vendor": "HP", + "supplytype": "toner", + "color": "cyan", + "capacitytier": "high", + "partnumber": "W2181X", + "marketingname": "218X Cyan", + "pageyield": 2500 + }, + { + "modelnumber": "Color LaserJet Pro 3201dw", "vendor": "HP", "supplytype": "toner", "color": "yellow", "capacitytier": "standard", - "partnumber": "CF212A", - "marketingname": "131A Yellow", - "pageyield": 1800 + "partnumber": "W2182A", + "marketingname": "218A Yellow", + "pageyield": 1200 }, { - "modelnumber": "LaserJet 200 color M251nw", + "modelnumber": "Color LaserJet Pro 3201dw", + "vendor": "HP", + "supplytype": "toner", + "color": "yellow", + "capacitytier": "high", + "partnumber": "W2182X", + "marketingname": "218X Yellow", + "pageyield": 2500 + }, + { + "modelnumber": "Color LaserJet Pro 3201dw", "vendor": "HP", "supplytype": "toner", "color": "magenta", "capacitytier": "standard", - "partnumber": "CF213A", - "marketingname": "131A Magenta", - "pageyield": 1800 + "partnumber": "W2183A", + "marketingname": "218A Magenta", + "pageyield": 1200 }, { - "modelnumber": "LaserJet 4250tn", + "modelnumber": "Color LaserJet Pro 3201dw", + "vendor": "HP", + "supplytype": "toner", + "color": "magenta", + "capacitytier": "high", + "partnumber": "W2183X", + "marketingname": "218X Magenta", + "pageyield": 2500 + }, + { + "modelnumber": "Color LaserJet Pro M454dw", "vendor": "HP", "supplytype": "toner", "color": "black", "capacitytier": "standard", - "partnumber": "Q5942A", - "marketingname": "42A Black", - "pageyield": 10000 + "partnumber": "W2020A", + "marketingname": "414A Black", + "pageyield": 2400 }, { - "modelnumber": "LaserJet 4250tn", + "modelnumber": "Color LaserJet Pro M454dw", "vendor": "HP", "supplytype": "toner", "color": "black", "capacitytier": "high", - "partnumber": "Q5942X", - "marketingname": "42X Black", - "pageyield": 20000 + "partnumber": "W2020X", + "marketingname": "414X", + "pageyield": 7500 }, { - "modelnumber": "LaserJet 4250tn DO NOT USE", + "modelnumber": "Color LaserJet Pro M454dw", + "vendor": "HP", + "supplytype": "toner", + "color": "cyan", + "capacitytier": "standard", + "partnumber": "W2021A", + "marketingname": "414A Cyan", + "pageyield": 2100 + }, + { + "modelnumber": "Color LaserJet Pro M454dw", + "vendor": "HP", + "supplytype": "toner", + "color": "cyan", + "capacitytier": "high", + "partnumber": "W2021X", + "marketingname": "414X Cyan", + "pageyield": 6000 + }, + { + "modelnumber": "Color LaserJet Pro M454dw", + "vendor": "HP", + "supplytype": "toner", + "color": "yellow", + "capacitytier": "standard", + "partnumber": "W2022A", + "marketingname": "414A", + "pageyield": 2100 + }, + { + "modelnumber": "Color LaserJet Pro M454dw", + "vendor": "HP", + "supplytype": "toner", + "color": "yellow", + "capacitytier": "high", + "partnumber": "W2022X", + "marketingname": "414X Yellow", + "pageyield": 6000 + }, + { + "modelnumber": "Color LaserJet Pro M454dw", + "vendor": "HP", + "supplytype": "toner", + "color": "magenta", + "capacitytier": "standard", + "partnumber": "W2023A", + "marketingname": "414A Magenta", + "pageyield": 2100 + }, + { + "modelnumber": "Color LaserJet Pro M454dw", + "vendor": "HP", + "supplytype": "toner", + "color": "magenta", + "capacitytier": "high", + "partnumber": "W2023X", + "marketingname": "414X Magenta", + "pageyield": 6000 + }, + { + "modelnumber": "LaserJet 4250tn", "vendor": "HP", "supplytype": "toner", "color": "black", @@ -1514,7 +1953,7 @@ "pageyield": 10000 }, { - "modelnumber": "LaserJet 4250tn DO NOT USE", + "modelnumber": "LaserJet 4250tn", "vendor": "HP", "supplytype": "toner", "color": "black", @@ -1693,6 +2132,26 @@ "marketingname": "90X Black (Contract)", "pageyield": 24000 }, + { + "modelnumber": "LaserJet M610", + "vendor": "HP", + "supplytype": "toner", + "color": "black", + "capacitytier": "standard", + "partnumber": "W1470A", + "marketingname": "147A Black", + "pageyield": 10500 + }, + { + "modelnumber": "LaserJet M610", + "vendor": "HP", + "supplytype": "toner", + "color": "black", + "capacitytier": "high", + "partnumber": "W1470X", + "marketingname": "147X Black", + "pageyield": 25200 + }, { "modelnumber": "LaserJet P3015dn", "vendor": "HP", @@ -1724,7 +2183,7 @@ "pageyield": 12500 }, { - "modelnumber": "Laserjet Pro 200 M251nw", + "modelnumber": "LaserJet Pro 200 color M251nw", "vendor": "HP", "supplytype": "toner", "color": "black", @@ -1734,7 +2193,7 @@ "pageyield": 1600 }, { - "modelnumber": "Laserjet Pro 200 M251nw", + "modelnumber": "LaserJet Pro 200 color M251nw", "vendor": "HP", "supplytype": "toner", "color": "black", @@ -1744,7 +2203,7 @@ "pageyield": 2400 }, { - "modelnumber": "Laserjet Pro 200 M251nw", + "modelnumber": "LaserJet Pro 200 color M251nw", "vendor": "HP", "supplytype": "toner", "color": "cyan", @@ -1754,7 +2213,7 @@ "pageyield": 1800 }, { - "modelnumber": "Laserjet Pro 200 M251nw", + "modelnumber": "LaserJet Pro 200 color M251nw", "vendor": "HP", "supplytype": "toner", "color": "yellow", @@ -1764,7 +2223,7 @@ "pageyield": 1800 }, { - "modelnumber": "Laserjet Pro 200 M251nw", + "modelnumber": "LaserJet Pro 200 color M251nw", "vendor": "HP", "supplytype": "toner", "color": "magenta", @@ -1773,6 +2232,86 @@ "marketingname": "131A Magenta", "pageyield": 1800 }, + { + "modelnumber": "LaserJet Pro 200 color M251nw", + "vendor": "HP", + "supplytype": "toner", + "color": "black", + "capacitytier": "standard", + "partnumber": "CF400A", + "marketingname": "201A Black", + "pageyield": 1500 + }, + { + "modelnumber": "LaserJet Pro 200 color M251nw", + "vendor": "HP", + "supplytype": "toner", + "color": "black", + "capacitytier": "high", + "partnumber": "CF400X", + "marketingname": "201X Black", + "pageyield": 2800 + }, + { + "modelnumber": "LaserJet Pro 200 color M251nw", + "vendor": "HP", + "supplytype": "toner", + "color": "cyan", + "capacitytier": "standard", + "partnumber": "CF401A", + "marketingname": "201A Cyan", + "pageyield": 1400 + }, + { + "modelnumber": "LaserJet Pro 200 color M251nw", + "vendor": "HP", + "supplytype": "toner", + "color": "cyan", + "capacitytier": "high", + "partnumber": "CF401X", + "marketingname": "201X Cyan", + "pageyield": 2500 + }, + { + "modelnumber": "LaserJet Pro 200 color M251nw", + "vendor": "HP", + "supplytype": "toner", + "color": "yellow", + "capacitytier": "standard", + "partnumber": "CF402A", + "marketingname": "201A Yellow", + "pageyield": 1400 + }, + { + "modelnumber": "LaserJet Pro 200 color M251nw", + "vendor": "HP", + "supplytype": "toner", + "color": "yellow", + "capacitytier": "high", + "partnumber": "CF402X", + "marketingname": "201X Yellow", + "pageyield": 2500 + }, + { + "modelnumber": "LaserJet Pro 200 color M251nw", + "vendor": "HP", + "supplytype": "toner", + "color": "magenta", + "capacitytier": "standard", + "partnumber": "CF403A", + "marketingname": "201A Magenta", + "pageyield": 1400 + }, + { + "modelnumber": "LaserJet Pro 200 color M251nw", + "vendor": "HP", + "supplytype": "toner", + "color": "magenta", + "capacitytier": "high", + "partnumber": "CF403X", + "marketingname": "201X Magenta", + "pageyield": 2500 + }, { "modelnumber": "LaserJet Pro 4001n", "vendor": "HP", @@ -1913,86 +2452,6 @@ "marketingname": "58X Black (Contract)", "pageyield": 10000 }, - { - "modelnumber": "LaserJet Pro M454dw", - "vendor": "HP", - "supplytype": "toner", - "color": "black", - "capacitytier": "standard", - "partnumber": "W2020A", - "marketingname": "414A Black", - "pageyield": 2400 - }, - { - "modelnumber": "LaserJet Pro M454dw", - "vendor": "HP", - "supplytype": "toner", - "color": "black", - "capacitytier": "high", - "partnumber": "W2020X", - "marketingname": "414X Black", - "pageyield": 7500 - }, - { - "modelnumber": "LaserJet Pro M454dw", - "vendor": "HP", - "supplytype": "toner", - "color": "cyan", - "capacitytier": "standard", - "partnumber": "W2021A", - "marketingname": "414A Cyan", - "pageyield": 2100 - }, - { - "modelnumber": "LaserJet Pro M454dw", - "vendor": "HP", - "supplytype": "toner", - "color": "cyan", - "capacitytier": "high", - "partnumber": "W2021X", - "marketingname": "414X Cyan", - "pageyield": 6000 - }, - { - "modelnumber": "LaserJet Pro M454dw", - "vendor": "HP", - "supplytype": "toner", - "color": "yellow", - "capacitytier": "standard", - "partnumber": "W2022A", - "marketingname": "414A Yellow", - "pageyield": 2100 - }, - { - "modelnumber": "LaserJet Pro M454dw", - "vendor": "HP", - "supplytype": "toner", - "color": "yellow", - "capacitytier": "high", - "partnumber": "W2022X", - "marketingname": "414X Yellow", - "pageyield": 6000 - }, - { - "modelnumber": "LaserJet Pro M454dw", - "vendor": "HP", - "supplytype": "toner", - "color": "magenta", - "capacitytier": "standard", - "partnumber": "W2023A", - "marketingname": "414A Magenta", - "pageyield": 2100 - }, - { - "modelnumber": "LaserJet Pro M454dw", - "vendor": "HP", - "supplytype": "toner", - "color": "magenta", - "capacitytier": "high", - "partnumber": "W2023X", - "marketingname": "414X Magenta", - "pageyield": 6000 - }, { "modelnumber": "LaserJet Pro M607", "vendor": "HP", @@ -2004,7 +2463,37 @@ "pageyield": 11000 }, { - "modelnumber": "Altalink C8135", + "modelnumber": "LaserJet Pro M607", + "vendor": "HP", + "supplytype": "toner", + "color": "black", + "capacitytier": "high", + "partnumber": "CF237X", + "marketingname": "37X Black", + "pageyield": 25000 + }, + { + "modelnumber": "LaserJet Pro M607", + "vendor": "HP", + "supplytype": "toner", + "color": "black", + "capacitytier": "extrahigh", + "partnumber": "CF237Y", + "marketingname": "37Y Black (not M607)", + "pageyield": 41000 + }, + { + "modelnumber": "LaserJet Pro M607", + "vendor": "HP", + "supplytype": "toner", + "color": "black", + "capacitytier": "metered", + "partnumber": "CF237YC", + "marketingname": "37Y Black (Contract)", + "pageyield": 41000 + }, + { + "modelnumber": "AltaLink C8135", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2013,7 +2502,7 @@ "marketingname": "C8135 Black (Metered)" }, { - "modelnumber": "Altalink C8135", + "modelnumber": "AltaLink C8135", "vendor": "Xerox", "supplytype": "toner", "color": "cyan", @@ -2022,7 +2511,7 @@ "marketingname": "C8135 Cyan (Metered)" }, { - "modelnumber": "Altalink C8135", + "modelnumber": "AltaLink C8135", "vendor": "Xerox", "supplytype": "toner", "color": "magenta", @@ -2031,7 +2520,7 @@ "marketingname": "C8135 Magenta (Metered)" }, { - "modelnumber": "Altalink C8135", + "modelnumber": "AltaLink C8135", "vendor": "Xerox", "supplytype": "toner", "color": "yellow", @@ -2040,7 +2529,7 @@ "marketingname": "C8135 Yellow (Metered)" }, { - "modelnumber": "Altalink C8135", + "modelnumber": "AltaLink C8135", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2049,7 +2538,7 @@ "marketingname": "C8135 Black (Sold)" }, { - "modelnumber": "Altalink C8135", + "modelnumber": "AltaLink C8135", "vendor": "Xerox", "supplytype": "toner", "color": "cyan", @@ -2058,7 +2547,7 @@ "marketingname": "C8135 Cyan (Sold)" }, { - "modelnumber": "Altalink C8135", + "modelnumber": "AltaLink C8135", "vendor": "Xerox", "supplytype": "toner", "color": "magenta", @@ -2067,7 +2556,7 @@ "marketingname": "C8135 Magenta (Sold)" }, { - "modelnumber": "Altalink C8135", + "modelnumber": "AltaLink C8135", "vendor": "Xerox", "supplytype": "toner", "color": "yellow", @@ -2076,7 +2565,7 @@ "marketingname": "C8135 Yellow (Sold)" }, { - "modelnumber": "Altalink C8135", + "modelnumber": "AltaLink C8135", "vendor": "Xerox", "supplytype": "waste", "color": "none", @@ -2085,7 +2574,7 @@ "marketingname": "C8135 Waste Cartridge" }, { - "modelnumber": "Altalink C8135", + "modelnumber": "AltaLink C8135", "vendor": "Xerox", "supplytype": "drum", "color": "none", @@ -2165,6 +2654,42 @@ "partnumber": "006R01700", "marketingname": "C8030 Yellow (Sold)" }, + { + "modelnumber": "AltaLink EC8036", + "vendor": "Xerox", + "supplytype": "toner", + "color": "black", + "capacitytier": "metered", + "partnumber": "006R01701", + "marketingname": "C8030 Black (Metered)" + }, + { + "modelnumber": "AltaLink EC8036", + "vendor": "Xerox", + "supplytype": "toner", + "color": "cyan", + "capacitytier": "metered", + "partnumber": "006R01702", + "marketingname": "C8030 Cyan (Metered)" + }, + { + "modelnumber": "AltaLink EC8036", + "vendor": "Xerox", + "supplytype": "toner", + "color": "magenta", + "capacitytier": "metered", + "partnumber": "006R01703", + "marketingname": "C8030 Magenta (Metered)" + }, + { + "modelnumber": "AltaLink EC8036", + "vendor": "Xerox", + "supplytype": "toner", + "color": "yellow", + "capacitytier": "metered", + "partnumber": "006R01704", + "marketingname": "C8030 Yellow (Metered)" + }, { "modelnumber": "AltaLink EC8036", "vendor": "Xerox", @@ -2184,7 +2709,7 @@ "marketingname": "C8030 / EC8036 Drum" }, { - "modelnumber": "Versalink B405DN", + "modelnumber": "VersaLink B405DN", "vendor": "Xerox", "supplytype": "drum", "color": "none", @@ -2193,7 +2718,7 @@ "marketingname": "B400 / B405 Drum" }, { - "modelnumber": "Versalink B405DN", + "modelnumber": "VersaLink B405DN", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2203,7 +2728,7 @@ "pageyield": 5900 }, { - "modelnumber": "Versalink B405DN", + "modelnumber": "VersaLink B405DN", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2213,7 +2738,7 @@ "pageyield": 13900 }, { - "modelnumber": "Versalink B405DN", + "modelnumber": "VersaLink B405DN", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2223,7 +2748,7 @@ "pageyield": 24600 }, { - "modelnumber": "Versalink B405DN", + "modelnumber": "VersaLink B405DN", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2233,7 +2758,7 @@ "pageyield": 24600 }, { - "modelnumber": "Versalink B7125", + "modelnumber": "VersaLink B7125", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2243,7 +2768,7 @@ "pageyield": 34300 }, { - "modelnumber": "Versalink B7125", + "modelnumber": "VersaLink B7125", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2253,7 +2778,7 @@ "pageyield": 34300 }, { - "modelnumber": "Versalink B7125", + "modelnumber": "VersaLink B7125", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2263,7 +2788,7 @@ "pageyield": 34300 }, { - "modelnumber": "Versalink B7125", + "modelnumber": "VersaLink B7125", "vendor": "Xerox", "supplytype": "drum", "color": "none", @@ -2272,7 +2797,16 @@ "marketingname": "B7125 Drum" }, { - "modelnumber": "Versalink C405", + "modelnumber": "VersaLink B7125", + "vendor": "Xerox", + "supplytype": "waste", + "color": "none", + "capacitytier": "standard", + "partnumber": "115R00129", + "marketingname": "B7000 / B7100 Waste Cartridge" + }, + { + "modelnumber": "VersaLink C405", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2282,7 +2816,7 @@ "pageyield": 2500 }, { - "modelnumber": "Versalink C405", + "modelnumber": "VersaLink C405", "vendor": "Xerox", "supplytype": "toner", "color": "yellow", @@ -2292,7 +2826,7 @@ "pageyield": 2500 }, { - "modelnumber": "Versalink C405", + "modelnumber": "VersaLink C405", "vendor": "Xerox", "supplytype": "toner", "color": "cyan", @@ -2302,7 +2836,7 @@ "pageyield": 2500 }, { - "modelnumber": "Versalink C405", + "modelnumber": "VersaLink C405", "vendor": "Xerox", "supplytype": "toner", "color": "magenta", @@ -2312,7 +2846,7 @@ "pageyield": 2500 }, { - "modelnumber": "Versalink C405", + "modelnumber": "VersaLink C405", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2322,7 +2856,7 @@ "pageyield": 5000 }, { - "modelnumber": "Versalink C405", + "modelnumber": "VersaLink C405", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2332,7 +2866,7 @@ "pageyield": 10500 }, { - "modelnumber": "Versalink C405", + "modelnumber": "VersaLink C405", "vendor": "Xerox", "supplytype": "drum", "color": "none", @@ -2341,7 +2875,7 @@ "marketingname": "C400 / C405 Drum" }, { - "modelnumber": "Versalink C405", + "modelnumber": "VersaLink C405", "vendor": "Xerox", "supplytype": "waste", "color": "none", @@ -2350,7 +2884,7 @@ "marketingname": "C400 / C405 Waste Cartridge" }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2360,7 +2894,7 @@ "pageyield": 2400 }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "toner", "color": "cyan", @@ -2370,7 +2904,7 @@ "pageyield": 2000 }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "toner", "color": "magenta", @@ -2380,7 +2914,7 @@ "pageyield": 2000 }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "toner", "color": "yellow", @@ -2390,7 +2924,7 @@ "pageyield": 2000 }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2400,7 +2934,7 @@ "pageyield": 10500 }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "toner", "color": "cyan", @@ -2410,7 +2944,7 @@ "pageyield": 7000 }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "toner", "color": "magenta", @@ -2420,7 +2954,7 @@ "pageyield": 7000 }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "toner", "color": "yellow", @@ -2430,7 +2964,7 @@ "pageyield": 7000 }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2440,7 +2974,7 @@ "pageyield": 15000 }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "toner", "color": "cyan", @@ -2450,7 +2984,7 @@ "pageyield": 10000 }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "toner", "color": "magenta", @@ -2460,7 +2994,7 @@ "pageyield": 10000 }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "toner", "color": "yellow", @@ -2470,7 +3004,7 @@ "pageyield": 10000 }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "waste", "color": "none", @@ -2479,7 +3013,7 @@ "marketingname": "C415 Waste Cartridge" }, { - "modelnumber": "Versalink C415", + "modelnumber": "VersaLink C415", "vendor": "Xerox", "supplytype": "drum", "color": "none", @@ -2488,7 +3022,7 @@ "marketingname": "C415 Drum / Imaging Unit" }, { - "modelnumber": "Versalink C7100", + "modelnumber": "VersaLink C7100", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2498,7 +3032,7 @@ "pageyield": 22500 }, { - "modelnumber": "Versalink C7100", + "modelnumber": "VersaLink C7100", "vendor": "Xerox", "supplytype": "toner", "color": "cyan", @@ -2508,7 +3042,7 @@ "pageyield": 15500 }, { - "modelnumber": "Versalink C7100", + "modelnumber": "VersaLink C7100", "vendor": "Xerox", "supplytype": "toner", "color": "magenta", @@ -2518,7 +3052,7 @@ "pageyield": 15500 }, { - "modelnumber": "Versalink C7100", + "modelnumber": "VersaLink C7100", "vendor": "Xerox", "supplytype": "toner", "color": "yellow", @@ -2528,7 +3062,7 @@ "pageyield": 15500 }, { - "modelnumber": "Versalink C7100", + "modelnumber": "VersaLink C7100", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2538,7 +3072,7 @@ "pageyield": 31300 }, { - "modelnumber": "Versalink C7100", + "modelnumber": "VersaLink C7100", "vendor": "Xerox", "supplytype": "toner", "color": "cyan", @@ -2548,7 +3082,7 @@ "pageyield": 18500 }, { - "modelnumber": "Versalink C7100", + "modelnumber": "VersaLink C7100", "vendor": "Xerox", "supplytype": "toner", "color": "magenta", @@ -2558,7 +3092,7 @@ "pageyield": 18500 }, { - "modelnumber": "Versalink C7100", + "modelnumber": "VersaLink C7100", "vendor": "Xerox", "supplytype": "toner", "color": "yellow", @@ -2568,7 +3102,7 @@ "pageyield": 18500 }, { - "modelnumber": "Versalink C7100", + "modelnumber": "VersaLink C7100", "vendor": "Xerox", "supplytype": "drum", "color": "none", @@ -2577,7 +3111,7 @@ "marketingname": "C7125 Drum" }, { - "modelnumber": "Versalink C7100", + "modelnumber": "VersaLink C7100", "vendor": "Xerox", "supplytype": "waste", "color": "none", @@ -2586,7 +3120,16 @@ "marketingname": "C7100 Series Waste Cartridge" }, { - "modelnumber": "Versalink C7125", + "modelnumber": "VersaLink C7100", + "vendor": "Xerox", + "supplytype": "waste", + "color": "none", + "capacitytier": "standard", + "partnumber": "115R00129", + "marketingname": "C7000 / C7100 Waste Cartridge" + }, + { + "modelnumber": "VersaLink C7125", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2596,7 +3139,7 @@ "pageyield": 22500 }, { - "modelnumber": "Versalink C7125", + "modelnumber": "VersaLink C7125", "vendor": "Xerox", "supplytype": "toner", "color": "cyan", @@ -2606,7 +3149,7 @@ "pageyield": 15500 }, { - "modelnumber": "Versalink C7125", + "modelnumber": "VersaLink C7125", "vendor": "Xerox", "supplytype": "toner", "color": "magenta", @@ -2616,7 +3159,7 @@ "pageyield": 15500 }, { - "modelnumber": "Versalink C7125", + "modelnumber": "VersaLink C7125", "vendor": "Xerox", "supplytype": "toner", "color": "yellow", @@ -2626,7 +3169,7 @@ "pageyield": 15500 }, { - "modelnumber": "Versalink C7125", + "modelnumber": "VersaLink C7125", "vendor": "Xerox", "supplytype": "toner", "color": "black", @@ -2636,7 +3179,7 @@ "pageyield": 31300 }, { - "modelnumber": "Versalink C7125", + "modelnumber": "VersaLink C7125", "vendor": "Xerox", "supplytype": "toner", "color": "cyan", @@ -2646,7 +3189,7 @@ "pageyield": 18500 }, { - "modelnumber": "Versalink C7125", + "modelnumber": "VersaLink C7125", "vendor": "Xerox", "supplytype": "toner", "color": "magenta", @@ -2656,7 +3199,7 @@ "pageyield": 18500 }, { - "modelnumber": "Versalink C7125", + "modelnumber": "VersaLink C7125", "vendor": "Xerox", "supplytype": "toner", "color": "yellow", @@ -2666,7 +3209,7 @@ "pageyield": 18500 }, { - "modelnumber": "Versalink C7125", + "modelnumber": "VersaLink C7125", "vendor": "Xerox", "supplytype": "drum", "color": "none", @@ -2675,13 +3218,22 @@ "marketingname": "C7125 Drum" }, { - "modelnumber": "Versalink C7125", + "modelnumber": "VersaLink C7125", "vendor": "Xerox", "supplytype": "waste", "color": "none", "capacitytier": "standard", "partnumber": "115R00128", "marketingname": "C7100 Series Waste Cartridge" + }, + { + "modelnumber": "VersaLink C7125", + "vendor": "Xerox", + "supplytype": "waste", + "color": "none", + "capacitytier": "standard", + "partnumber": "115R00129", + "marketingname": "C7000 / C7100 Waste Cartridge" } ], "measuringtooltypes": [ @@ -2695,11 +3247,23 @@ "description": "Vernier / digital caliper", "color": "#14abef" }, + { + "name": "CMM" + }, + { + "name": "Eddy Current" + }, + { + "name": "Form Tracer" + }, { "name": "Gage Block Set", "description": "Reference gage block set", "color": "#e83e8c" }, + { + "name": "Genspect" + }, { "name": "Height Gage", "description": "Height / vertical measuring gage", @@ -2710,6 +3274,12 @@ "description": "Dial / test indicator", "color": "#11cdef" }, + { + "name": "Inspection" + }, + { + "name": "Measuring Machine" + }, { "name": "Micrometer", "description": "Outside / inside / depth micrometer", @@ -2720,24 +3290,65 @@ "description": "Other measuring tool", "color": "#6c757d" }, + { + "name": "Spline Checker" + }, { "name": "Thread Gage", "description": "Go / no-go thread gage", "color": "#fb6340" + }, + { + "name": "Vision System" + }, + { + "name": "Wax Trace" } ], "notificationtypes": [ + { + "typename": "Awareness", + "typedescription": "General awareness notification", + "typecolor": "#17a2b8" + }, + { + "typename": "Change", + "typedescription": "Planned change notification", + "typecolor": "#ffc107" + }, { "typename": "General", + "typedescription": "General announcement", "typecolor": "#28a745" }, + { + "typename": "Incident", + "typedescription": "Incident or outage notification", + "typecolor": "#dc3545" + }, + { + "typename": "Maintenance", + "typedescription": "Scheduled maintenance notification", + "typecolor": "#6c757d" + }, { "typename": "Recertification", + "typedescription": "Employees due to retake a training course (shows two weeks)", "typecolor": "#0d6efd" }, { "typename": "Recognition", + "typedescription": "Employee recognition (clears at 8 AM Eastern)", "typecolor": "#ffc107" + }, + { + "typename": "TBD", + "typecolor": "success" + }, + { + "typename": "Training", + "typedescription": "Training notice (one card per employee)", + "typecolor": "#17a2b8" } ], "accessprotocols": [ @@ -2751,7 +3362,7 @@ "name": "VNC", "scheme": "vnc", "defaultport": 5900, - "linktemplate": "com.realvnc.vncviewer.connect://{host}:{port}" + "linktemplate": "vnc://{host}:{port}" }, { "name": "WinRM", From c7828e787347801c0726e94d53cfa69b8ff37b2f Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Wed, 26 Aug 2026 12:18:31 -0400 Subject: [PATCH 06/14] dynamically filter Make/Model selectors by asset category The Add Computer, Add Machine, and Add Printer forms offered the entire model catalog and vendor list regardless of what was being added, so a PC could be pointed at a printer model and every vendor showed up whether or not it made that kind of thing. Scope each form to its own asset category so the selectors only present sensible choices. Add a ?category= filter to GET /api/models that joins modeltypes and matches on category (Equipment, PC, Network, Printer), and include the model's category in the list response so the forms can distinguish types client-side. Untyped models are excluded, since a null category cannot satisfy the filter. Each subtype form now loads only its category: PCForm requests category PC (servers, which are Network, stay out); MachineForm requests category Equipment, with the main Model dropdown hiding Controller-type models and the Controller dropdown showing only those; PrinterForm requests category Printer, which also fixes its Model list that previously showed the whole catalog. Vendor dropdowns are derived from the loaded models: a form only offers vendors that actually have a model in its category. Each dropdown keeps the record's own saved vendor even if it falls outside that set, so editing an existing asset never drops its vendor. --- plugins/computers/frontend/views/PCForm.vue | 15 ++++++- .../machines/frontend/views/MachineForm.vue | 45 +++++++++++++++---- .../printers/frontend/views/PrinterForm.vue | 15 ++++++- shopdb/core/api/models.py | 9 +++- 4 files changed, 70 insertions(+), 14 deletions(-) diff --git a/plugins/computers/frontend/views/PCForm.vue b/plugins/computers/frontend/views/PCForm.vue index 9b125aab..e1b84148 100644 --- a/plugins/computers/frontend/views/PCForm.vue +++ b/plugins/computers/frontend/views/PCForm.vue @@ -166,7 +166,7 @@ >
    Created Updated ReconciledWarnings Problems
    {{ (f.reconciliations || []).length }} + {{ (f.warnings || []).length }} + {{ f.problems.length }}
    _ to match. endpoints was absent from the universal site profile, so the installer would not offer it and prune-schema would drop its tables at provisioning. Add it. The agent design record lived under docs/, which is stripped to the public wiki and normally absent from the tree; a lone file there un-skipped the wiki-publish guards. Move it into the plugin. --- deploy/site-profile-universal.json | 1 + .../endpoints}/DESIGN-DECISIONS-agent.md | 0 .../versions/0001_endpoints_baseline.py | 19 +++++++++++-------- plugins/endpoints/models/endpoint.py | 4 ---- tests/test_plugin_migrations.py | 3 +++ 5 files changed, 15 insertions(+), 12 deletions(-) rename {docs => plugins/endpoints}/DESIGN-DECISIONS-agent.md (100%) diff --git a/deploy/site-profile-universal.json b/deploy/site-profile-universal.json index 7308f77d..12d928bd 100644 --- a/deploy/site-profile-universal.json +++ b/deploy/site-profile-universal.json @@ -4,6 +4,7 @@ "backups", "computers", "employees", + "endpoints", "geenforce", "guides", "knowledgebase", diff --git a/docs/DESIGN-DECISIONS-agent.md b/plugins/endpoints/DESIGN-DECISIONS-agent.md similarity index 100% rename from docs/DESIGN-DECISIONS-agent.md rename to plugins/endpoints/DESIGN-DECISIONS-agent.md diff --git a/plugins/endpoints/migrations/versions/0001_endpoints_baseline.py b/plugins/endpoints/migrations/versions/0001_endpoints_baseline.py index 7e80055d..99c3ab28 100644 --- a/plugins/endpoints/migrations/versions/0001_endpoints_baseline.py +++ b/plugins/endpoints/migrations/versions/0001_endpoints_baseline.py @@ -50,7 +50,9 @@ def upgrade(): sa.PrimaryKeyConstraint('endpointstateid'), sa.UniqueConstraint('assetid'), ) - create_index_if_missing('idx_endpointstates_lastreportedat', + create_index_if_missing('ix_endpointstates_assetid', + 'endpointstates', ['assetid']) + create_index_if_missing('ix_endpointstates_lastreportedat', 'endpointstates', ['lastreportedat']) create_table_if_missing( @@ -68,11 +70,11 @@ def upgrade(): ondelete='CASCADE'), sa.PrimaryKeyConstraint('endpointeventid'), ) - create_index_if_missing('ixendpointeventsassetid', 'endpointevents', + create_index_if_missing('ix_endpointevents_assetid', 'endpointevents', ['assetid']) - create_index_if_missing('ixendpointeventseventid', 'endpointevents', + create_index_if_missing('ix_endpointevents_eventid', 'endpointevents', ['eventid']) - create_index_if_missing('ixendpointeventseventtime', 'endpointevents', + create_index_if_missing('ix_endpointevents_eventtime', 'endpointevents', ['eventtime']) # Alerting/history reads by "this endpoint's events over time", so the # composite is the index that actually gets used. @@ -83,11 +85,12 @@ def upgrade(): def downgrade(): op.drop_index('idx_endpointevents_assetevent', table_name='endpointevents') - op.drop_index('ixendpointeventseventtime', table_name='endpointevents') - op.drop_index('ixendpointeventseventid', table_name='endpointevents') - op.drop_index('ixendpointeventsassetid', table_name='endpointevents') + op.drop_index('ix_endpointevents_eventtime', table_name='endpointevents') + op.drop_index('ix_endpointevents_eventid', table_name='endpointevents') + op.drop_index('ix_endpointevents_assetid', table_name='endpointevents') op.drop_table('endpointevents') - op.drop_index('idx_endpointstates_lastreportedat', + op.drop_index('ix_endpointstates_lastreportedat', table_name='endpointstates') + op.drop_index('ix_endpointstates_assetid', table_name='endpointstates') op.drop_table('endpointstates') diff --git a/plugins/endpoints/models/endpoint.py b/plugins/endpoints/models/endpoint.py index 7dd0e654..aa0bead7 100644 --- a/plugins/endpoints/models/endpoint.py +++ b/plugins/endpoints/models/endpoint.py @@ -82,10 +82,6 @@ class EndpointState(db.Model): createdat = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) - __table_args__ = ( - db.Index('idx_endpointstates_lastreportedat', 'lastreportedat'), - ) - def __repr__(self): return f"" diff --git a/tests/test_plugin_migrations.py b/tests/test_plugin_migrations.py index 371b0fc0..dfaa8220 100644 --- a/tests/test_plugin_migrations.py +++ b/tests/test_plugin_migrations.py @@ -87,6 +87,9 @@ EXPECTED_HEAD_REVISION['tools'] = 'tools0001securesend' EXPECTED_HEAD_REVISION['notifications'] = 'notifications0005boardorder' # warranty adds the proof-of-cover document columns on top of its anchor. EXPECTED_HEAD_REVISION['warranty'] = 'warranty0002proof' +# endpoints is a NEW post-cutover plugin: its 0001 genuinely creates the +# endpointstates + endpointevents tables the core chain never owned. +EXPECTED_HEAD_REVISION['endpoints'] = 'endpoints0001baseline' # Plugins built after the cutover: their 0001 baseline really creates tables the # core chain never owned. From 01734e54498f8f82f9085842f009f7d2a342902d Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Fri, 28 Aug 2026 09:29:09 -0400 Subject: [PATCH 12/14] harden CSV import reconciliation and preserve saved model selections Default fuzzy reconcile threshold to 0 and refuse merges when digit runs differ so numbered labels like Bay 311 never bind to Bay 310. Keep edit forms from clearing saved model fields when category filtering hides untyped rows. Includes in-progress computers policy and endpoints command feature work. --- .env.example | 1 + .github/copilot-instructions.md | 1 + docker-compose.yml | 2 + frontend/src/api/index.js | 44 ++ frontend/src/components/PluginAssetPanels.vue | 94 +++- frontend/src/components/pluginAssetPanels.js | 55 +++ .../src/components/pluginAssetPanels.spec.js | 72 +++- plugins/computers/api/__init__.py | 1 + plugins/computers/api/policy_routes.py | 406 ++++++++++++++++++ plugins/computers/frontend/routes.js | 12 + .../frontend/views/BusinessPoliciesList.vue | 223 ++++++++++ .../views/ConfigurationPoliciesList.vue | 223 ++++++++++ plugins/computers/frontend/views/PCForm.vue | 29 +- .../frontend/views/ScopeNodeEditor.vue | 106 +++++ .../migrations/versions/0004_policies.py | 154 +++++++ plugins/computers/models/__init__.py | 16 + plugins/computers/models/policy.py | 197 +++++++++ plugins/computers/plugin.py | 74 +++- plugins/computers/policyflag.py | 19 + plugins/computers/services/__init__.py | 1 + plugins/computers/services/policyresolver.py | 168 ++++++++ plugins/computers/services/policyscope.py | 96 +++++ plugins/endpoints/AGENT-CONTRACT.md | 274 ++++++++++++ plugins/endpoints/DESIGN-DECISIONS-agent.md | 296 ++++++++++++- plugins/endpoints/api/routes.py | 372 +++++++++++++++- plugins/endpoints/commands.py | 177 ++++++++ plugins/endpoints/frontend/routes.js | 6 + .../frontend/views/AgentKeySettings.vue | 293 +++++++++++++ .../versions/0002_endpoint_commands.py | 84 ++++ .../versions/0003_endpoint_command_target.py | 50 +++ plugins/endpoints/models/__init__.py | 4 +- plugins/endpoints/models/endpoint.py | 89 ++++ plugins/endpoints/plugin.py | 193 ++++++++- .../machines/frontend/views/MachineForm.vue | 46 +- .../printers/frontend/views/PrinterForm.vue | 25 ++ scripts/dev/seed_shared_applications.py | 102 +++++ shopdb/__init__.py | 13 +- shopdb/api/__init__.py | 24 ++ shopdb/core/api/collector.py | 61 ++- shopdb/core/api/settings.py | 4 +- shopdb/core/services/csvimport.py | 27 +- shopdb/plugins/alembic_template.py | 6 +- shopdb/utils/apitoken_auth.py | 76 ++++ tests/test_core/test_authz.py | 10 +- tests/test_csv_import.py | 55 +++ tests/test_plugin_migrations.py | 12 +- 46 files changed, 4241 insertions(+), 52 deletions(-) create mode 100644 plugins/computers/api/policy_routes.py create mode 100644 plugins/computers/frontend/views/BusinessPoliciesList.vue create mode 100644 plugins/computers/frontend/views/ConfigurationPoliciesList.vue create mode 100644 plugins/computers/frontend/views/ScopeNodeEditor.vue create mode 100644 plugins/computers/migrations/versions/0004_policies.py create mode 100644 plugins/computers/models/policy.py create mode 100644 plugins/computers/policyflag.py create mode 100644 plugins/computers/services/__init__.py create mode 100644 plugins/computers/services/policyresolver.py create mode 100644 plugins/computers/services/policyscope.py create mode 100644 plugins/endpoints/AGENT-CONTRACT.md create mode 100644 plugins/endpoints/commands.py create mode 100644 plugins/endpoints/frontend/views/AgentKeySettings.vue create mode 100644 plugins/endpoints/migrations/versions/0002_endpoint_commands.py create mode 100644 plugins/endpoints/migrations/versions/0003_endpoint_command_target.py create mode 100644 scripts/dev/seed_shared_applications.py diff --git a/.env.example b/.env.example index a2689518..68e9ebbe 100644 --- a/.env.example +++ b/.env.example @@ -64,6 +64,7 @@ ZABBIX_TOKEN= # COLLECTOR_API_KEY_ first, then COLLECTOR_API_KEY as fallback. # COLLECTOR_API_KEY= # COLLECTOR_API_KEY_COMPUTERS= +# COLLECTOR_API_KEY_ENDPOINTS= # ---- Trusted plugin publisher keys (ADR-013, optional) ---- # Public-key PEM paths (OS path separator: ':' on Linux, ';' on Windows) used diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index fefd16a4..77f2f109 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -72,6 +72,7 @@ name. Spell out: `config context manager request response environment utilities` with quantities are standalone tables, not assets. - **Ledger pattern**: a cached `quantityonhand` moves in the SAME commit as the signed transaction row it reflects. +- **Configuration for endpoint agent**: The endpoint agent's live collector key and base URL are provisioned in the Windows registry `HKLM\SOFTWARE\GE\ShopDB` (values `CollectorKey`, `BaseUrl`), not committed in `appsettings.json`; `appsettings.json` holds only non-secret runtime settings. ## Before you finish a change diff --git a/docker-compose.yml b/docker-compose.yml index 5f591efe..b8d54715 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,6 +61,8 @@ services: LOG_LEVEL: ${LOG_LEVEL:-INFO} ZABBIX_URL: ${ZABBIX_URL:-} ZABBIX_TOKEN: ${ZABBIX_TOKEN:-} + COLLECTOR_API_KEY: ${COLLECTOR_API_KEY:-} + COLLECTOR_API_KEY_ENDPOINTS: ${COLLECTOR_API_KEY_ENDPOINTS:-} ports: - "${API_PORT:-5001}:5001" volumes: diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index e13ef02e..31daa2b3 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -224,6 +224,50 @@ export const computersApi = { remove(id) { return api.delete(`/computers/protocols/${id}`) } + }, + // Configuration/Business policies (RMM-style). Gated server-side by the + // computers_configpolicies_enabled setting; routes return 404 while off. + policies: { + actions() { + return api.get('/computers/policies/actions') + }, + effective(assetid) { + return api.get(`/computers/policies/effective/${assetid}`) + }, + configuration: { + list() { + return api.get('/computers/policies/configuration') + }, + get(id) { + return api.get(`/computers/policies/configuration/${id}`) + }, + create(data) { + return api.post('/computers/policies/configuration', data) + }, + update(id, data) { + return api.put(`/computers/policies/configuration/${id}`, data) + }, + remove(id) { + return api.delete(`/computers/policies/configuration/${id}`) + } + }, + business: { + list() { + return api.get('/computers/policies/business') + }, + get(id) { + return api.get(`/computers/policies/business/${id}`) + }, + create(data) { + return api.post('/computers/policies/business', data) + }, + update(id, data) { + return api.put(`/computers/policies/business/${id}`, data) + }, + remove(id) { + return api.delete(`/computers/policies/business/${id}`) + } + } } } diff --git a/frontend/src/components/PluginAssetPanels.vue b/frontend/src/components/PluginAssetPanels.vue index 8362bc9f..4a4d1725 100644 --- a/frontend/src/components/PluginAssetPanels.vue +++ b/frontend/src/components/PluginAssetPanels.vue @@ -4,7 +4,30 @@ on any asset detail page, so a plugin adds detail-page UI with no Vue. -->
    {{ panel.empty || 'Nothing to show.' }} @@ -90,6 +123,7 @@ {{ b.label }}
    + @@ -100,7 +134,8 @@ import api from '../api' import { colorStyle } from '@/utils/colorStyle' import { toApiPath, rows, panelVisible, mapTitle, mapBadge, mapMeta, manageLink, - keyvalueFields, tableColumns, cell, badges, tabSections, + keyvalueFields, tableColumns, cell, badges, tabSections, rollupStatus, + rowAction, } from './pluginAssetPanels' const props = defineProps({ @@ -115,6 +150,45 @@ const tabs = ref({}) function activeTab(panel) { return tabs.value[panel.id] || 0 } function setTab(panel, index) { tabs.value = { ...tabs.value, [panel.id]: index } } +// Collapsed state per panel id (default expanded), kept here for the same +// reason as tabs: panels are replaced wholesale on refetch. +const collapsed = ref({}) +function isCollapsed(panel) { return !!collapsed.value[panel.id] } +function toggleCollapse(panel) { + collapsed.value = { ...collapsed.value, [panel.id]: !collapsed.value[panel.id] } +} + +// Per-panel action state: busy guards a double-click while an enqueue POST is +// in flight, and the error string surfaces a rejected enqueue (bad action, or +// a 409 when a command is already queued). Keyed by panel id for the same +// reason as tabs/collapsed: panels are replaced wholesale on refetch. +const actionBusy = ref({}) +const actionError = ref({}) + +// Enqueue the inline row toggle's command (Start/Stop for an actionable +// services/processes row, e.g. MTConnect), then reload so the row reflects the +// new state. The agent long-polls, runs it, posts the result, and wakes an +// immediate telemetry re-collect, so the settled run state lands on the next +// panel refresh rather than after the full cadence. +async function runRowAction(panel, item) { + if (actionBusy.value[panel.id]) return + const info = rowAction(panel, item) + if (!info || !info.endpoint || !info.toggle) return + actionBusy.value = { ...actionBusy.value, [panel.id]: true } + actionError.value = { ...actionError.value, [panel.id]: '' } + try { + const body = { action: info.toggle } + if (info.target) body.target = info.target + await api.post(toApiPath(info.endpoint, props.assetid), body) + await load() + } catch (err) { + const message = err?.response?.data?.message || 'Command failed to queue.' + actionError.value = { ...actionError.value, [panel.id]: message } + } finally { + actionBusy.value = { ...actionBusy.value, [panel.id]: false } + } +} + // Fetch the declared panels for this asset, then each panel's data endpoint. async function load() { panels.value = [] @@ -195,6 +269,22 @@ watch(() => props.assetid, load, { immediate: true }) .pap-table { width: 100%; border-collapse: collapse; font-size: 0.85rem; } .pap-table th, .pap-table td { text-align: left; padding: 0.4rem 0.6rem; border-bottom: 1px solid var(--border); } .pap-badges { display: flex; flex-wrap: wrap; gap: 0.5rem; } +.pap-row-action { margin-left: 0.5rem; flex: 0 0 auto; } +.pap-action-error { color: var(--danger, #f5365c); font-size: 0.82rem; } +.pap-header { + display: flex; align-items: center; gap: 0.5rem; + cursor: pointer; user-select: none; +} +.pap-header:hover .pap-header-title { color: var(--primary); } +.pap-header-title { flex: 1 1 auto; } +.pap-caret { + font-size: 0.7rem; color: var(--text-light); + transition: transform 0.15s ease; +} +.pap-caret.collapsed { transform: rotate(-90deg); } +.pap-status-dot { + flex: 0 0 auto; width: 0.7rem; height: 0.7rem; border-radius: 50%; +} .mono { font-family: monospace; } .muted { color: var(--text-light); } diff --git a/frontend/src/components/pluginAssetPanels.js b/frontend/src/components/pluginAssetPanels.js index 7aebfa4b..c19144c3 100644 --- a/frontend/src/components/pluginAssetPanels.js +++ b/frontend/src/components/pluginAssetPanels.js @@ -98,6 +98,37 @@ export function badges(panel) { return list.map((b) => ({ label: b.label, color: b.color })) } +// Header status rollup for a card. A panel opts in with +// status: { key, levels: { : 'critical'|'warning'|'ok'|'unknown' } }. +// The card header then shows a single green/yellow/red dot reflecting the worst +// row: any critical -> red, else any warning -> yellow, else ok -> green. A +// value not in levels is treated as unknown and does not raise the level. +export const STATUS_RANK = { ok: 1, unknown: 1, warning: 2, critical: 3 } +export const STATUS_COLOR = { + ok: '#2dce89', + warning: '#ffc107', + critical: '#f5365c', + unknown: '#6c757d', +} + +export function rollupStatus(panel) { + const status = panel && panel.status + if (!status || !status.key) return null + const items = rows(panel) + if (!items.length) return null + const levels = status.levels || {} + let worst = 'ok' + let sawKnown = false + for (const item of items) { + const level = levels[item[status.key]] + if (!level) continue + sawKnown = true + if (STATUS_RANK[level] > STATUS_RANK[worst]) worst = level + } + if (!sawKnown) return null + return { level: worst, color: STATUS_COLOR[worst] || STATUS_COLOR.unknown } +} + // tabs: [{ label, fields: [{label, value, mono}] }]. One section is shown at a // time, so a panel with 30 values stays the height of its largest section // instead of dragging one multicol column far past the other. @@ -114,6 +145,30 @@ export function tabSections(panel) { .filter((s) => s.fields.length > 0) } +// Inline row action for a list panel. A monitored services/processes row may +// carry action: { running, start, stop, status } when it has a registered +// command (e.g. MTConnect). The card renders a single toggle on that row: when +// running it offers Stop, otherwise Start. Rows with no action are badge-only. +// commandendpoint on the panel is where the enqueue POST goes. +export function rowAction(panel, item) { + const action = item && item.action + if (!action || (!action.start && !action.stop)) return null + const running = action.running === true + return { + running, + // The action name to enqueue for the primary toggle button. + toggle: running ? action.stop : action.start, + toggleLabel: running ? 'Stop' : 'Start', + // status is a non-mutating probe, always available when declared. + status: action.status || null, + // Generic actions (service_*/process_*) carry the subject to act on; fixed + // actions (mtconnect_*) leave this null. + target: action.target || null, + // The declared enqueue endpoint template; the caller substitutes assetid. + endpoint: panel.commandendpoint || null, + } +} + export function panelVisible(panel) { if (panel.render === 'tabs') return tabSections(panel).length > 0 if (panel.render === 'list') return rows(panel).length > 0 || !!panel.empty diff --git a/frontend/src/components/pluginAssetPanels.spec.js b/frontend/src/components/pluginAssetPanels.spec.js index c48b00be..3f4e89b8 100644 --- a/frontend/src/components/pluginAssetPanels.spec.js +++ b/frontend/src/components/pluginAssetPanels.spec.js @@ -1,7 +1,8 @@ import { describe, it, expect } from 'vitest' import { toApiPath, subst, rows, mapTitle, mapBadge, mapMeta, manageLink, - keyvalueFields, tableColumns, badges, panelVisible, tabSections, + keyvalueFields, tableColumns, badges, panelVisible, tabSections, rollupStatus, + rowAction, } from './pluginAssetPanels' const warrantyPanel = { @@ -130,3 +131,72 @@ describe('tabSections', () => { expect(panelVisible({ render: 'tabs', _data: { sections: [] } })).toBe(false) }) }) + +describe('rollupStatus', () => { + const status = { + key: 'statecolor', + levels: { green: 'ok', yellow: 'warning', red: 'critical' }, + } + + it('returns null when the panel declares no status', () => { + expect(rollupStatus({ _data: [{ statecolor: 'green' }] })).toBeNull() + }) + + it('returns null when there are no rows', () => { + expect(rollupStatus({ status, _data: [] })).toBeNull() + }) + + it('is ok (green) when every row is ok', () => { + const result = rollupStatus({ status, _data: [{ statecolor: 'green' }, { statecolor: 'green' }] }) + expect(result).toEqual({ level: 'ok', color: '#2dce89' }) + }) + + it('is warning (yellow) when a warning is present but no critical', () => { + const result = rollupStatus({ status, _data: [{ statecolor: 'green' }, { statecolor: 'yellow' }] }) + expect(result).toEqual({ level: 'warning', color: '#ffc107' }) + }) + + it('is critical (red) when any row is stopped', () => { + const result = rollupStatus({ status, _data: [{ statecolor: 'yellow' }, { statecolor: 'red' }] }) + expect(result).toEqual({ level: 'critical', color: '#f5365c' }) + }) + + it('ignores unmapped values without raising the level', () => { + const result = rollupStatus({ status, _data: [{ statecolor: 'gray' }, { statecolor: 'green' }] }) + expect(result).toEqual({ level: 'ok', color: '#2dce89' }) + }) + + it('returns null when no row value maps to a known level', () => { + expect(rollupStatus({ status, _data: [{ statecolor: 'gray' }] })).toBeNull() + }) +}) + +describe('rowAction (inline MTConnect toggle)', () => { + const panel = { id: 'endpoints-services', render: 'list', commandendpoint: '/api/endpoints/5/commands' } + + it('returns null for rows without an action', () => { + expect(rowAction(panel, { name: 'Some Service' })).toBeNull() + expect(rowAction(panel, { name: 'x', action: { status: 'mtconnect_status' } })).toBeNull() + }) + + it('offers Stop when the row is running', () => { + const info = rowAction(panel, { + name: 'MTConnect', + action: { running: true, start: 'mtconnect_start', stop: 'mtconnect_stop', status: 'mtconnect_status' }, + }) + expect(info.running).toBe(true) + expect(info.toggle).toBe('mtconnect_stop') + expect(info.toggleLabel).toBe('Stop') + expect(info.endpoint).toBe('/api/endpoints/5/commands') + }) + + it('offers Start when the row is not running', () => { + const info = rowAction(panel, { + name: 'MTConnect', + action: { running: false, start: 'mtconnect_start', stop: 'mtconnect_stop', status: 'mtconnect_status' }, + }) + expect(info.running).toBe(false) + expect(info.toggle).toBe('mtconnect_start') + expect(info.toggleLabel).toBe('Start') + }) +}) diff --git a/plugins/computers/api/__init__.py b/plugins/computers/api/__init__.py index 0bea25bc..5e14bb0f 100644 --- a/plugins/computers/api/__init__.py +++ b/plugins/computers/api/__init__.py @@ -1,5 +1,6 @@ """Computers plugin API.""" from .routes import computers_bp +from . import policy_routes # noqa: F401 registers policy routes on computers_bp __all__ = ['computers_bp'] diff --git a/plugins/computers/api/policy_routes.py b/plugins/computers/api/policy_routes.py new file mode 100644 index 00000000..c62f2d8d --- /dev/null +++ b/plugins/computers/api/policy_routes.py @@ -0,0 +1,406 @@ +"""CRUD API for Configuration and Business Policies. + +Registered on computers_bp under /api/computers/policies/... Kept in its own +module so the whole policy surface can be removed by not importing it - part of +the exit gate. Every route first checks the computers_configpolicies_enabled +switch and returns 404 while it is off, so enabling/disabling the feature never +half-exposes it. + +Write-time validation lives here (not in the model): scope ASTs go through +validate_scope, and action names are checked against the endpoints command +allowlist, so a bad tree or an unknown action never reaches the database. +""" + +from flask import request +from flask_jwt_extended import jwt_required + +from shopdb.api import (db, AuditLog, success_response, error_response, + ErrorCodes, require_permission) + +from .routes import computers_bp +from ..models import (ConfigurationPolicy, ConfigurationPolicyAction, + BusinessPolicy, BusinessPolicyItem, POLICY_SOURCE_LOCAL, + ACTION_MODE_DESIRED, ACTION_MODE_GATE) +from ..policyflag import policy_flag_enabled +from ..services.policyscope import validate_scope, ScopeError + +_MODES = {ACTION_MODE_DESIRED, ACTION_MODE_GATE} + + +def _feature_gate(): + """Return a 404 response when the policy feature is off, else None.""" + if not policy_flag_enabled(): + return error_response(ErrorCodes.NOT_FOUND, + 'Policy feature is not enabled', http_code=404) + return None + + +def _known_actions(): + """The code-level allowlist of action names a policy may reference.""" + from plugins.endpoints.commands import COMMAND_ALLOWLIST + return set(COMMAND_ALLOWLIST.keys()) + + +def _configpolicy_dict(policy): + data = policy.to_dict() + data['actions'] = [a.to_dict() for a in policy.actions] + return data + + +def _businesspolicy_dict(policy): + data = policy.to_dict() + data['items'] = [{ + 'businesspolicyitemid': it.businesspolicyitemid, + 'configurationpolicyid': it.configurationpolicyid, + 'sortorder': it.sortorder, + 'name': it.configurationpolicy.name if it.configurationpolicy else None, + } for it in policy.items] + return data + + +# ============================================================================= +# Configuration Policies +# ============================================================================= + +@computers_bp.route('/policies/configuration', methods=['GET']) +@jwt_required(optional=True) +def list_configuration_policies(): + gate = _feature_gate() + if gate: + return gate + policies = ConfigurationPolicy.query.order_by(ConfigurationPolicy.name).all() + return success_response([_configpolicy_dict(p) for p in policies]) + + +@computers_bp.route('/policies/configuration/', methods=['GET']) +@jwt_required(optional=True) +def get_configuration_policy(policy_id: int): + gate = _feature_gate() + if gate: + return gate + policy = db.session.get(ConfigurationPolicy, policy_id) + if not policy: + return error_response(ErrorCodes.NOT_FOUND, + 'Configuration policy not found', http_code=404) + return success_response(_configpolicy_dict(policy)) + + +def _apply_actions(policy, actions): + """Replace a policy's actions from request data, validating each. + + Returns an error_response on failure, else None. + """ + known = _known_actions() + policy.actions.clear() + for row in actions or []: + name = (row.get('action') or '').strip() + if name not in known: + return error_response(ErrorCodes.VALIDATION_ERROR, + f'Unknown action: {name!r}') + mode = (row.get('mode') or ACTION_MODE_GATE).strip() + if mode not in _MODES: + return error_response(ErrorCodes.VALIDATION_ERROR, + f'Unknown action mode: {mode!r}') + policy.actions.append(ConfigurationPolicyAction( + action=name, + mode=mode, + desiredstate=row.get('desiredstate'), + params=row.get('params'), + intervalminutes=row.get('intervalminutes'), + sortorder=row.get('sortorder', 0) or 0, + )) + return None + + +@computers_bp.route('/policies/configuration', methods=['POST']) +@jwt_required() +@require_permission('computers.create') +def create_configuration_policy(): + gate = _feature_gate() + if gate: + return gate + data = request.get_json() or {} + name = (data.get('name') or '').strip() + if not name: + return error_response(ErrorCodes.VALIDATION_ERROR, 'name is required') + if ConfigurationPolicy.query.filter_by(name=name).first(): + return error_response(ErrorCodes.CONFLICT, + f"Configuration policy '{name}' already exists", + http_code=409) + + scope = data.get('scopejson') + if scope is not None: + try: + validate_scope(scope) + except ScopeError as exc: + return error_response(ErrorCodes.VALIDATION_ERROR, + f'Invalid scope: {exc}') + + policy = ConfigurationPolicy( + name=name, + description=data.get('description'), + scopejson=scope, + source=POLICY_SOURCE_LOCAL, + ) + err = _apply_actions(policy, data.get('actions')) + if err: + return err + + db.session.add(policy) + AuditLog.log('created', 'ConfigurationPolicy', entityname=name) + db.session.commit() + return success_response(_configpolicy_dict(policy), + message='Configuration policy created', + http_code=201) + + +@computers_bp.route('/policies/configuration/', methods=['PUT']) +@jwt_required() +@require_permission('computers.edit') +def update_configuration_policy(policy_id: int): + gate = _feature_gate() + if gate: + return gate + policy = db.session.get(ConfigurationPolicy, policy_id) + if not policy: + return error_response(ErrorCodes.NOT_FOUND, + 'Configuration policy not found', http_code=404) + if policy.source != POLICY_SOURCE_LOCAL: + return error_response(ErrorCodes.CONFLICT, + 'Policy is managed externally and is read-only', + http_code=409) + data = request.get_json() or {} + + if 'name' in data: + newname = (data['name'] or '').strip() + if newname and newname != policy.name: + if ConfigurationPolicy.query.filter_by(name=newname).first(): + return error_response(ErrorCodes.CONFLICT, + f"Configuration policy '{newname}' already exists", + http_code=409) + policy.name = newname + if 'description' in data: + policy.description = data['description'] + if 'isactive' in data: + policy.isactive = data['isactive'] + if 'scopejson' in data: + scope = data['scopejson'] + if scope is not None: + try: + validate_scope(scope) + except ScopeError as exc: + return error_response(ErrorCodes.VALIDATION_ERROR, + f'Invalid scope: {exc}') + policy.scopejson = scope + if 'actions' in data: + err = _apply_actions(policy, data['actions']) + if err: + return err + + AuditLog.log('updated', 'ConfigurationPolicy', entityname=policy.name) + db.session.commit() + return success_response(_configpolicy_dict(policy), + message='Configuration policy updated') + + +@computers_bp.route('/policies/configuration/', methods=['DELETE']) +@jwt_required() +@require_permission('computers.delete') +def delete_configuration_policy(policy_id: int): + gate = _feature_gate() + if gate: + return gate + policy = db.session.get(ConfigurationPolicy, policy_id) + if not policy: + return error_response(ErrorCodes.NOT_FOUND, + 'Configuration policy not found', http_code=404) + if policy.source != POLICY_SOURCE_LOCAL: + return error_response(ErrorCodes.CONFLICT, + 'Policy is managed externally and is read-only', + http_code=409) + inuse = BusinessPolicyItem.query.filter_by( + configurationpolicyid=policy_id).count() + if inuse: + return error_response( + ErrorCodes.CONFLICT, + f'Cannot delete: used by {inuse} business policy item(s)', + http_code=409) + db.session.delete(policy) + AuditLog.log('deleted', 'ConfigurationPolicy', entityname=policy.name) + db.session.commit() + return success_response(None, message='Configuration policy deleted') + + +# ============================================================================= +# Business Policies +# ============================================================================= + +@computers_bp.route('/policies/business', methods=['GET']) +@jwt_required(optional=True) +def list_business_policies(): + gate = _feature_gate() + if gate: + return gate + policies = (BusinessPolicy.query + .order_by(BusinessPolicy.priority, BusinessPolicy.businesspolicyid) + .all()) + return success_response([_businesspolicy_dict(p) for p in policies]) + + +@computers_bp.route('/policies/business/', methods=['GET']) +@jwt_required(optional=True) +def get_business_policy(policy_id: int): + gate = _feature_gate() + if gate: + return gate + policy = db.session.get(BusinessPolicy, policy_id) + if not policy: + return error_response(ErrorCodes.NOT_FOUND, + 'Business policy not found', http_code=404) + return success_response(_businesspolicy_dict(policy)) + + +def _apply_items(policy, items): + """Replace a business policy's configuration-policy items, validating each.""" + policy.items.clear() + seen = set() + for order, row in enumerate(items or []): + cpid = row.get('configurationpolicyid') + if cpid is None: + return error_response(ErrorCodes.VALIDATION_ERROR, + 'item requires configurationpolicyid') + if cpid in seen: + return error_response(ErrorCodes.VALIDATION_ERROR, + f'duplicate configuration policy {cpid}') + if not db.session.get(ConfigurationPolicy, cpid): + return error_response(ErrorCodes.VALIDATION_ERROR, + f'unknown configuration policy {cpid}') + seen.add(cpid) + policy.items.append(BusinessPolicyItem( + configurationpolicyid=cpid, + sortorder=row.get('sortorder', order), + )) + return None + + +@computers_bp.route('/policies/business', methods=['POST']) +@jwt_required() +@require_permission('computers.create') +def create_business_policy(): + gate = _feature_gate() + if gate: + return gate + data = request.get_json() or {} + name = (data.get('name') or '').strip() + if not name: + return error_response(ErrorCodes.VALIDATION_ERROR, 'name is required') + if BusinessPolicy.query.filter_by(name=name).first(): + return error_response(ErrorCodes.CONFLICT, + f"Business policy '{name}' already exists", + http_code=409) + policy = BusinessPolicy( + name=name, + description=data.get('description'), + priority=data.get('priority', 100), + source=POLICY_SOURCE_LOCAL, + ) + err = _apply_items(policy, data.get('items')) + if err: + return err + db.session.add(policy) + AuditLog.log('created', 'BusinessPolicy', entityname=name) + db.session.commit() + return success_response(_businesspolicy_dict(policy), + message='Business policy created', http_code=201) + + +@computers_bp.route('/policies/business/', methods=['PUT']) +@jwt_required() +@require_permission('computers.edit') +def update_business_policy(policy_id: int): + gate = _feature_gate() + if gate: + return gate + policy = db.session.get(BusinessPolicy, policy_id) + if not policy: + return error_response(ErrorCodes.NOT_FOUND, + 'Business policy not found', http_code=404) + if policy.source != POLICY_SOURCE_LOCAL: + return error_response(ErrorCodes.CONFLICT, + 'Policy is managed externally and is read-only', + http_code=409) + data = request.get_json() or {} + if 'name' in data: + newname = (data['name'] or '').strip() + if newname and newname != policy.name: + if BusinessPolicy.query.filter_by(name=newname).first(): + return error_response(ErrorCodes.CONFLICT, + f"Business policy '{newname}' already exists", + http_code=409) + policy.name = newname + if 'description' in data: + policy.description = data['description'] + if 'priority' in data: + policy.priority = data['priority'] + if 'isactive' in data: + policy.isactive = data['isactive'] + if 'items' in data: + err = _apply_items(policy, data['items']) + if err: + return err + AuditLog.log('updated', 'BusinessPolicy', entityname=policy.name) + db.session.commit() + return success_response(_businesspolicy_dict(policy), + message='Business policy updated') + + +@computers_bp.route('/policies/business/', methods=['DELETE']) +@jwt_required() +@require_permission('computers.delete') +def delete_business_policy(policy_id: int): + gate = _feature_gate() + if gate: + return gate + policy = db.session.get(BusinessPolicy, policy_id) + if not policy: + return error_response(ErrorCodes.NOT_FOUND, + 'Business policy not found', http_code=404) + if policy.source != POLICY_SOURCE_LOCAL: + return error_response(ErrorCodes.CONFLICT, + 'Policy is managed externally and is read-only', + http_code=409) + db.session.delete(policy) + AuditLog.log('deleted', 'BusinessPolicy', entityname=policy.name) + db.session.commit() + return success_response(None, message='Business policy deleted') + + +# ============================================================================= +# Action catalog (for the UI) and effective-policy preview +# ============================================================================= + +@computers_bp.route('/policies/actions', methods=['GET']) +@jwt_required(optional=True) +def list_policy_actions(): + """The allowlisted action names + labels a policy may reference.""" + gate = _feature_gate() + if gate: + return gate + from plugins.endpoints.commands import COMMAND_ALLOWLIST + return success_response([ + {'action': name, 'label': meta.get('label'), + 'description': meta.get('description')} + for name, meta in COMMAND_ALLOWLIST.items() + ]) + + +@computers_bp.route('/policies/effective/', methods=['GET']) +@jwt_required(optional=True) +def get_effective_policy(asset_id: int): + """Preview the resolved effective policy for one endpoint (debug/UI).""" + gate = _feature_gate() + if gate: + return gate + from ..services.policyresolver import effective_policy_for + effective = effective_policy_for(asset_id) + return success_response(effective.to_dict()) diff --git a/plugins/computers/frontend/routes.js b/plugins/computers/frontend/routes.js index 828d9122..3d7d0895 100644 --- a/plugins/computers/frontend/routes.js +++ b/plugins/computers/frontend/routes.js @@ -62,6 +62,18 @@ export default [ name: 'access-protocols', component: () => import('./views/AccessProtocolsList.vue'), meta: { requiresAuth: true, requiresAdmin: true, plugin: 'computers' } + }, + { + path: 'settings/configuration-policies', + name: 'configuration-policies', + component: () => import('./views/ConfigurationPoliciesList.vue'), + meta: { requiresAuth: true, requiresAdmin: true, plugin: 'computers' } + }, + { + path: 'settings/business-policies', + name: 'business-policies', + component: () => import('./views/BusinessPoliciesList.vue'), + meta: { requiresAuth: true, requiresAdmin: true, plugin: 'computers' } } ] diff --git a/plugins/computers/frontend/views/BusinessPoliciesList.vue b/plugins/computers/frontend/views/BusinessPoliciesList.vue new file mode 100644 index 00000000..4a6b20ae --- /dev/null +++ b/plugins/computers/frontend/views/BusinessPoliciesList.vue @@ -0,0 +1,223 @@ + + + + + diff --git a/plugins/computers/frontend/views/ConfigurationPoliciesList.vue b/plugins/computers/frontend/views/ConfigurationPoliciesList.vue new file mode 100644 index 00000000..17a2fc8f --- /dev/null +++ b/plugins/computers/frontend/views/ConfigurationPoliciesList.vue @@ -0,0 +1,223 @@ + + + + + diff --git a/plugins/computers/frontend/views/PCForm.vue b/plugins/computers/frontend/views/PCForm.vue index ea7c8581..2b5e9b7f 100644 --- a/plugins/computers/frontend/views/PCForm.vue +++ b/plugins/computers/frontend/views/PCForm.vue @@ -444,12 +444,31 @@ watch(() => form.value.serialnumber, (serial) => { } }) -// Filter models by selected vendor and PC type +// Ensure a saved model id is present in models.value even when the filtered +// list (category PC) excluded it because it has no modeltype. Fetches the +// single row by id and appends it; a no-op when already present or blank. +async function ensureModelLoaded(modelId) { + if (!modelId) return + if (models.value.some(m => m.modelnumberid === modelId)) return + try { + const response = await modelsApi.get(modelId) + const model = response.data.data + if (model) models.value = [...models.value, model] + } catch (e) { + // Leave it absent; the select shows blank but we did not invent a value. + } +} + +// Filter models by selected vendor and PC type. Keep the record's own saved +// model even if it is untyped or a different vendor, so editing an existing PC +// never renders a blank select that clears the field on save. const filteredModels = computed(() => { // filter by vendor only (PC type now maps to computertypeid, a different id // space than a model's machinetypeid) + const savedId = form.value.modelnumberid if (!form.value.vendorid) return models.value - return models.value.filter(m => m.vendorid === form.value.vendorid) + return models.value.filter( + m => m.vendorid === form.value.vendorid || m.modelnumberid === savedId) }) // Only offer vendors that actually have PC-category models. models is already @@ -520,6 +539,12 @@ onMounted(async () => { levelid: pc.levelid ?? null, ipaddress: primaryComm?.ipaddress || '' } + + // The model list is loaded filtered to category PC, which the backend + // inner-joins against ModelType - so an untyped model is absent. If this + // PC's saved model is one of those rows, fetch it by id and merge it in, + // or its dropdown would render blank and saving would clear the field. + await ensureModelLoaded(form.value.modelnumberid) } } catch (err) { console.error('Error loading data:', err) diff --git a/plugins/computers/frontend/views/ScopeNodeEditor.vue b/plugins/computers/frontend/views/ScopeNodeEditor.vue new file mode 100644 index 00000000..439a43ed --- /dev/null +++ b/plugins/computers/frontend/views/ScopeNodeEditor.vue @@ -0,0 +1,106 @@ + + + + + + + diff --git a/plugins/computers/migrations/versions/0004_policies.py b/plugins/computers/migrations/versions/0004_policies.py new file mode 100644 index 00000000..d5c7f737 --- /dev/null +++ b/plugins/computers/migrations/versions/0004_policies.py @@ -0,0 +1,154 @@ +"""Add Configuration and Business Policy tables (RMM-style policies). + +Creates configurationpolicies, configurationpolicyactions, businesspolicies, +and businesspolicyitems. These back the DT-authored policy feature that is gated +by the computers_configpolicies_enabled setting; the tables land regardless so +the switch can be flipped without a schema change, and the downgrade drops all +four for a clean rollback of the whole scope. + +Idempotent: each create_table is skipped when the table already exists (e.g. a +test DB built by db.create_all()). + +Revision ID: computers0004policies +Revises: computers0003azurefields +""" +from alembic import op +import sqlalchemy as sa + + +revision = 'computers0004policies' +down_revision = 'computers0003azurefields' +branch_labels = None +depends_on = None + + +def upgrade(): + bind = op.get_bind() + insp = sa.inspect(bind) + existing = set(insp.get_table_names()) + + if 'configurationpolicies' not in existing: + op.create_table( + 'configurationpolicies', + sa.Column('configurationpolicyid', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=150), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('scopejson', sa.JSON(), nullable=True), + sa.Column('source', sa.String(length=50), nullable=False, + server_default='local'), + sa.Column('createddate', sa.DateTime(), nullable=False), + sa.Column('modifieddate', sa.DateTime(), nullable=False), + sa.Column('isactive', sa.Boolean(), nullable=False, + server_default=sa.true()), + sa.PrimaryKeyConstraint('configurationpolicyid'), + sa.UniqueConstraint('name', name='uq_configurationpolicy_name'), + ) + op.create_index('idx_configpolicy_source', 'configurationpolicies', + ['source']) + op.create_index('idx_configpolicy_active', 'configurationpolicies', + ['isactive']) + + if 'configurationpolicyactions' not in existing: + op.create_table( + 'configurationpolicyactions', + sa.Column('configurationpolicyactionid', sa.Integer(), + nullable=False), + sa.Column('configurationpolicyid', sa.Integer(), nullable=False), + sa.Column('action', sa.String(length=100), nullable=False), + sa.Column('mode', sa.String(length=20), nullable=False, + server_default='gate'), + sa.Column('desiredstate', sa.JSON(), nullable=True), + sa.Column('params', sa.JSON(), nullable=True), + sa.Column('intervalminutes', sa.Integer(), nullable=True), + sa.Column('sortorder', sa.Integer(), nullable=False, + server_default='0'), + sa.Column('createddate', sa.DateTime(), nullable=False), + sa.Column('modifieddate', sa.DateTime(), nullable=False), + sa.Column('isactive', sa.Boolean(), nullable=False, + server_default=sa.true()), + sa.ForeignKeyConstraint( + ['configurationpolicyid'], + ['configurationpolicies.configurationpolicyid'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('configurationpolicyactionid'), + sa.UniqueConstraint('configurationpolicyid', 'action', + name='uq_configpolicyaction'), + ) + op.create_index('idx_configpolicyaction_policy', + 'configurationpolicyactions', + ['configurationpolicyid']) + + if 'businesspolicies' not in existing: + op.create_table( + 'businesspolicies', + sa.Column('businesspolicyid', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=150), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('priority', sa.Integer(), nullable=False, + server_default='100'), + sa.Column('source', sa.String(length=50), nullable=False, + server_default='local'), + sa.Column('createddate', sa.DateTime(), nullable=False), + sa.Column('modifieddate', sa.DateTime(), nullable=False), + sa.Column('isactive', sa.Boolean(), nullable=False, + server_default=sa.true()), + sa.PrimaryKeyConstraint('businesspolicyid'), + sa.UniqueConstraint('name', name='uq_businesspolicy_name'), + ) + op.create_index('idx_businesspolicy_priority', 'businesspolicies', + ['priority']) + op.create_index('idx_businesspolicy_active', 'businesspolicies', + ['isactive']) + + if 'businesspolicyitems' not in existing: + op.create_table( + 'businesspolicyitems', + sa.Column('businesspolicyitemid', sa.Integer(), nullable=False), + sa.Column('businesspolicyid', sa.Integer(), nullable=False), + sa.Column('configurationpolicyid', sa.Integer(), nullable=False), + sa.Column('sortorder', sa.Integer(), nullable=False, + server_default='0'), + sa.Column('createddate', sa.DateTime(), nullable=False), + sa.Column('modifieddate', sa.DateTime(), nullable=False), + sa.Column('isactive', sa.Boolean(), nullable=False, + server_default=sa.true()), + sa.ForeignKeyConstraint( + ['businesspolicyid'], ['businesspolicies.businesspolicyid'], + ondelete='CASCADE'), + sa.ForeignKeyConstraint( + ['configurationpolicyid'], + ['configurationpolicies.configurationpolicyid'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('businesspolicyitemid'), + sa.UniqueConstraint('businesspolicyid', 'configurationpolicyid', + name='uq_businesspolicyitem'), + ) + op.create_index('idx_bpitem_business', 'businesspolicyitems', + ['businesspolicyid']) + op.create_index('idx_bpitem_config', 'businesspolicyitems', + ['configurationpolicyid']) + + +def downgrade(): + bind = op.get_bind() + insp = sa.inspect(bind) + existing = set(insp.get_table_names()) + + if 'businesspolicyitems' in existing: + op.drop_index('idx_bpitem_config', table_name='businesspolicyitems') + op.drop_index('idx_bpitem_business', table_name='businesspolicyitems') + op.drop_table('businesspolicyitems') + if 'businesspolicies' in existing: + op.drop_index('idx_businesspolicy_active', table_name='businesspolicies') + op.drop_index('idx_businesspolicy_priority', table_name='businesspolicies') + op.drop_table('businesspolicies') + if 'configurationpolicyactions' in existing: + op.drop_index('idx_configpolicyaction_policy', + table_name='configurationpolicyactions') + op.drop_table('configurationpolicyactions') + if 'configurationpolicies' in existing: + op.drop_index('idx_configpolicy_active', + table_name='configurationpolicies') + op.drop_index('idx_configpolicy_source', + table_name='configurationpolicies') + op.drop_table('configurationpolicies') diff --git a/plugins/computers/models/__init__.py b/plugins/computers/models/__init__.py index eee75210..d5813968 100644 --- a/plugins/computers/models/__init__.py +++ b/plugins/computers/models/__init__.py @@ -7,6 +7,15 @@ from .computer import ( AccessProtocol, ComputerAccess, ) +from .policy import ( + ConfigurationPolicy, + ConfigurationPolicyAction, + BusinessPolicy, + BusinessPolicyItem, + POLICY_SOURCE_LOCAL, + ACTION_MODE_DESIRED, + ACTION_MODE_GATE, +) __all__ = [ 'Computer', @@ -14,4 +23,11 @@ __all__ = [ 'ComputerInstalledApp', 'AccessProtocol', 'ComputerAccess', + 'ConfigurationPolicy', + 'ConfigurationPolicyAction', + 'BusinessPolicy', + 'BusinessPolicyItem', + 'POLICY_SOURCE_LOCAL', + 'ACTION_MODE_DESIRED', + 'ACTION_MODE_GATE', ] diff --git a/plugins/computers/models/policy.py b/plugins/computers/models/policy.py new file mode 100644 index 00000000..a95ea5a4 --- /dev/null +++ b/plugins/computers/models/policy.py @@ -0,0 +1,197 @@ +"""Configuration and business policy models (RMM-style). + +DT staff author reusable Configuration Policies (a single rule binding +allowlisted endpoint actions to endpoints matched by a boolean scope) and group +them into ordered Business Policies assigned to a fleet. When an endpoint reports +in, the server folds the matching Business Policies in priority order to compute +that endpoint's effective policy (first match wins per action). + +These models live in the computers plugin so policies sit next to ComputerType. +The action VOCABULARY stays authoritative in plugins/endpoints/commands.py; this +layer references action NAMES only. The `source` column is the seam for a future +GE-Enforce sync (local-authored rows vs imported/managed rows). + +The whole surface is gated by the computers_configpolicies_enabled setting; the +models register unconditionally (SQLAlchemy needs them), but routes, panels, and +enforcement stay inert while the flag is off, and the migration downgrade drops +every table for a clean rollback. +""" + +from shopdb.api import db, BaseModel + + +# Provenance of a policy row. +# leaves room for `geenforce` (or another manager) to own rows later without a +# schema change - an imported row is read-only in the UI and reconciled by its +# owner. Kept a plain string, not an enum column, so adding a source is data. +POLICY_SOURCE_LOCAL = 'local' + + +class ConfigurationPolicy(BaseModel): + """A single reusable policy: a scope plus the actions it applies. + + Scope is a boolean expression (AND/OR/NOT over PC types and applications) + stored as a small typed JSON AST in scopejson; see services/policyscope.py + for the evaluator. Actions live in ConfigurationPolicyAction rows. + """ + __tablename__ = 'configurationpolicies' + + configurationpolicyid = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(150), unique=True, nullable=False) + description = db.Column(db.Text) + + # Boolean target AST: {"type":"and","nodes":[{"type":"pctype","id":3}, + # {"type":"not","node":{"type":"app","id":12}}]}. Validated server-side + # (known node types, depth cap) before it is stored - see policyscope. + scopejson = db.Column(db.JSON, nullable=True) + + # GE-Enforce interop seam (see POLICY_SOURCE_LOCAL). + source = db.Column(db.String(50), default=POLICY_SOURCE_LOCAL, nullable=False) + + # createddate, modifieddate, isactive come from BaseModel. + + actions = db.relationship( + 'ConfigurationPolicyAction', + back_populates='configurationpolicy', + cascade='all, delete-orphan', + ) + + __table_args__ = ( + db.Index('idx_configpolicy_source', 'source'), + db.Index('idx_configpolicy_active', 'isactive'), + ) + + def __repr__(self): + return f"" + + +# What a policy action does to a matched endpoint. +# desired: reconciled desired-state. desiredstate says what it should be +# (e.g. running=true for mtconnect); the server enqueues the +# drift-correcting command when telemetry disagrees. +# gate: allowlist-gating only. The action is PERMITTED (and shown) on +# matched endpoints; it constrains what may be enqueued, it does not +# drive anything on its own. +ACTION_MODE_DESIRED = 'desired' +ACTION_MODE_GATE = 'gate' + + +class ConfigurationPolicyAction(BaseModel): + """One allowlisted action carried by a ConfigurationPolicy. + + `action` is a name from plugins/endpoints/commands.py COMMAND_ALLOWLIST; + it is validated against that allowlist at write time. `mode` selects + desired-state reconciliation vs allowlist-gating. + """ + __tablename__ = 'configurationpolicyactions' + + configurationpolicyactionid = db.Column(db.Integer, primary_key=True) + configurationpolicyid = db.Column( + db.Integer, + db.ForeignKey('configurationpolicies.configurationpolicyid', + ondelete='CASCADE'), + nullable=False, + ) + + # Allowlisted action name (endpoints commands.py). Not an FK - the allowlist + # is code, not a table - so it is validated in the API layer. + action = db.Column(db.String(100), nullable=False) + + mode = db.Column(db.String(20), default=ACTION_MODE_GATE, nullable=False) + + # Desired-state target for mode=desired, as JSON so different actions can + # express different targets (e.g. {"running": true}). Null for gate mode. + desiredstate = db.Column(db.JSON, nullable=True) + + # Optional action parameters passed through to the command (JSON), and an + # optional reconcile cadence in minutes (null = reconcile on every report). + params = db.Column(db.JSON, nullable=True) + intervalminutes = db.Column(db.Integer, nullable=True) + + sortorder = db.Column(db.Integer, default=0, nullable=False) + + configurationpolicy = db.relationship( + 'ConfigurationPolicy', back_populates='actions') + + __table_args__ = ( + db.UniqueConstraint('configurationpolicyid', 'action', + name='uq_configpolicyaction'), + db.Index('idx_configpolicyaction_policy', 'configurationpolicyid'), + ) + + def __repr__(self): + return f"" + + +class BusinessPolicy(BaseModel): + """An ordered collection of Configuration Policies assigned to a fleet. + + Priority orders Business Policies against each other when several match one + endpoint: lower priority number is evaluated first, and the first policy to + speak on a given action name wins (first-match-wins per action). + """ + __tablename__ = 'businesspolicies' + + businesspolicyid = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(150), unique=True, nullable=False) + description = db.Column(db.Text) + + # Lower runs first. Ties broken by businesspolicyid for a stable order. + priority = db.Column(db.Integer, default=100, nullable=False) + + source = db.Column(db.String(50), default=POLICY_SOURCE_LOCAL, nullable=False) + + # createddate, modifieddate, isactive come from BaseModel. + + items = db.relationship( + 'BusinessPolicyItem', + back_populates='businesspolicy', + cascade='all, delete-orphan', + order_by='BusinessPolicyItem.sortorder', + ) + + __table_args__ = ( + db.Index('idx_businesspolicy_priority', 'priority'), + db.Index('idx_businesspolicy_active', 'isactive'), + ) + + def __repr__(self): + return f"" + + +class BusinessPolicyItem(BaseModel): + """Membership of a Configuration Policy in a Business Policy, ordered. + + sortorder sets the walk order WITHIN a Business Policy; combined with the + Business Policy priority it gives a fully deterministic fold order for the + effective-policy resolver. + """ + __tablename__ = 'businesspolicyitems' + + businesspolicyitemid = db.Column(db.Integer, primary_key=True) + businesspolicyid = db.Column( + db.Integer, + db.ForeignKey('businesspolicies.businesspolicyid', ondelete='CASCADE'), + nullable=False, + ) + configurationpolicyid = db.Column( + db.Integer, + db.ForeignKey('configurationpolicies.configurationpolicyid', + ondelete='CASCADE'), + nullable=False, + ) + sortorder = db.Column(db.Integer, default=0, nullable=False) + + businesspolicy = db.relationship('BusinessPolicy', back_populates='items') + configurationpolicy = db.relationship('ConfigurationPolicy') + + __table_args__ = ( + db.UniqueConstraint('businesspolicyid', 'configurationpolicyid', + name='uq_businesspolicyitem'), + db.Index('idx_bpitem_business', 'businesspolicyid'), + db.Index('idx_bpitem_config', 'configurationpolicyid'), + ) + + def __repr__(self): + return (f"") diff --git a/plugins/computers/plugin.py b/plugins/computers/plugin.py index b78866a9..b91e8cb8 100644 --- a/plugins/computers/plugin.py +++ b/plugins/computers/plugin.py @@ -12,8 +12,16 @@ from shopdb.plugins.base import BasePlugin, PluginMeta from shopdb.api import db, AssetType from .models import Computer, ComputerType, ComputerInstalledApp, AccessProtocol, ComputerAccess +from .models import ( + ConfigurationPolicy, + ConfigurationPolicyAction, + BusinessPolicy, + BusinessPolicyItem, +) from .api import computers_bp +from .policyflag import policy_flag_enabled as _policy_flag_enabled + logger = logging.getLogger(__name__) # Marker stamped on PC->measuringtool "controls" links this collector creates @@ -92,7 +100,8 @@ class ComputersPlugin(BasePlugin): def get_models(self) -> List[Type]: """Return list of SQLAlchemy model classes.""" - return [Computer, ComputerType, ComputerInstalledApp, AccessProtocol, ComputerAccess] + return [Computer, ComputerType, ComputerInstalledApp, AccessProtocol, ComputerAccess, + ConfigurationPolicy, ConfigurationPolicyAction, BusinessPolicy, BusinessPolicyItem] def init_app(self, app: Flask, db_instance) -> None: """Initialize plugin with Flask app.""" @@ -397,6 +406,36 @@ class ComputersPlugin(BasePlugin): 'site\'s URL and ranges', 'position': 26, }, + ] + self._policy_settings_cards() + + def _policy_settings_cards(self): + """Policy admin cards, shown only while the feature flag is on. + + Gating the cards on the switch keeps the settings menu honest: the + Configuration/Business Policy screens appear exactly when the feature is + enabled and vanish on rollback, matching the 404 the routes return. + """ + if not self._configpolicies_enabled_safe(): + return [] + return [ + { + 'group': 'Computers', + 'to': '/settings/configuration-policies', + 'icon': 'shield', + 'title': 'Configuration policies', + 'description': 'Reusable rules binding endpoint actions to PC ' + 'types and applications', + 'position': 27, + }, + { + 'group': 'Computers', + 'to': '/settings/business-policies', + 'icon': 'layers', + 'title': 'Business policies', + 'description': 'Ordered collections of configuration policies ' + 'assigned by priority', + 'position': 28, + }, ] def get_settings_defaults(self) -> List[dict]: @@ -438,8 +477,41 @@ class ComputersPlugin(BasePlugin): 'Off while machine numbers are shared between ' 'PCs; the collector response still warns.', }, + { + 'key': 'computers_configpolicies_enabled', + 'value': 'false', + 'valuetype': 'boolean', + 'category': 'computers', + 'description': 'Master switch for RMM-style Configuration and ' + 'Business Policies. Off by default: models exist ' + 'but the CRUD routes, WebUI, and endpoint ' + 'enforcement stay inert so the whole feature can ' + 'be rolled back without a schema change.', + }, ] + def _configpolicies_enabled(self): + """Whether the Configuration/Business Policy feature is turned on. + + Off by default. This is the single exit gate for the policy scope: the + tables and models load regardless (SQLAlchemy needs them), but every + route, panel, and enforcement path checks this first and no-ops while it + is false, so the feature can be disabled without a migration. + """ + return _policy_flag_enabled() + + def _configpolicies_enabled_safe(self): + """Flag read that never raises when the settings table is absent. + + get_settings_cards can be called in contexts without a database (the + plugin contract tests build hooks against a bare app), so a missing + settings table must read as "off" rather than propagate. + """ + try: + return _policy_flag_enabled() + except Exception: + return False + def _machinelink_alerts_enabled(self): """Whether the PC-to-machine alerts may be sent. diff --git a/plugins/computers/policyflag.py b/plugins/computers/policyflag.py new file mode 100644 index 00000000..98d132ad --- /dev/null +++ b/plugins/computers/policyflag.py @@ -0,0 +1,19 @@ +"""Single source of truth for the Configuration/Business Policy feature gate. + +Kept in its own module (not on the plugin class) so the CRUD routes, the panel +surface, and the endpoints enforcement path can all consult the same switch +without importing ComputersPlugin. The flag is the computers_configpolicies_enabled +Setting row seeded by the plugin's get_settings_defaults; while it is off, every +policy code path no-ops, which is the exit gate for rolling the feature back. +""" + + +def policy_flag_enabled(): + """True when computers_configpolicies_enabled is set truthy.""" + from shopdb.api import Setting + + setting = Setting.query.filter_by( + key='computers_configpolicies_enabled').first() + if not setting: + return False + return (setting.value or '').strip().lower() in ('true', '1', 'yes') diff --git a/plugins/computers/services/__init__.py b/plugins/computers/services/__init__.py new file mode 100644 index 00000000..c43ac045 --- /dev/null +++ b/plugins/computers/services/__init__.py @@ -0,0 +1 @@ +"""Computers plugin services: pure policy logic (scope + resolution).""" diff --git a/plugins/computers/services/policyresolver.py b/plugins/computers/services/policyresolver.py new file mode 100644 index 00000000..b71592b6 --- /dev/null +++ b/plugins/computers/services/policyresolver.py @@ -0,0 +1,168 @@ +"""Effective-policy resolution: the single swappable policy entry point. + +This module answers one question for the rest of the system: "given an endpoint, +what actions apply and how?" It is the boundary the endpoints allowlist seam +calls - never the policy tables directly - so the policy ASSEMBLY ENGINE can be +swapped without touching enforcement. Today it folds the local computers-plugin +tables; a future GE-Enforce integration can replace build_effective_policy (or +feed it) and every consumer keeps working unchanged. + +Resolution model (per the design): + - Build an endpoint context (PC type + installed appids) from the Computer row. + - Walk every active Business Policy in priority order (lower first, ties by id). + - Within each, walk its Configuration Policies in sortorder. + - A Configuration Policy contributes only if its scope matches the endpoint. + - For each action name, FIRST MATCH WINS: once an action is decided it is not + overridden by a later, lower-priority policy. + +The result is an EffectivePolicy: a flat, resolved map of action name -> decision +(mode desired/gate, plus desiredstate/params), which is exactly what enforcement +and the allowlist need. +""" + +from .policyscope import evaluate_scope + + +class EffectiveAction: + """One resolved action decision for an endpoint.""" + + __slots__ = ('action', 'mode', 'desiredstate', 'params', 'intervalminutes', + 'businesspolicyid', 'configurationpolicyid') + + def __init__(self, action, mode, desiredstate, params, intervalminutes, + businesspolicyid, configurationpolicyid): + self.action = action + self.mode = mode + self.desiredstate = desiredstate + self.params = params + self.intervalminutes = intervalminutes + self.businesspolicyid = businesspolicyid + self.configurationpolicyid = configurationpolicyid + + def to_dict(self): + return { + 'action': self.action, + 'mode': self.mode, + 'desiredstate': self.desiredstate, + 'params': self.params, + 'intervalminutes': self.intervalminutes, + 'businesspolicyid': self.businesspolicyid, + 'configurationpolicyid': self.configurationpolicyid, + } + + +class EffectivePolicy: + """The folded, resolved policy for a single endpoint. + + actions maps action name -> EffectiveAction (first-match-wins already + applied). Convenience views expose the gate allowlist and desired-state set + that enforcement and the endpoints seam consume. + """ + + def __init__(self, actions): + self.actions = actions + + def allowed_actions(self): + """Set of action names permitted on this endpoint (gate + desired).""" + return set(self.actions.keys()) + + def desired_actions(self): + """action name -> EffectiveAction for mode == 'desired' only.""" + return {name: eff for name, eff in self.actions.items() + if eff.mode == 'desired'} + + def to_dict(self): + return {name: eff.to_dict() for name, eff in self.actions.items()} + + +def build_endpoint_context(computer): + """Resolve a Computer row into the context the scope evaluator needs. + + Returns {'computertypeid': int|None, 'appids': set[int]}. + """ + appids = set() + for link in getattr(computer, 'installedapps', None) or (): + if getattr(link, 'isactive', True) and link.appid is not None: + appids.add(link.appid) + return { + 'computertypeid': getattr(computer, 'computertypeid', None), + 'appids': appids, + } + + +def _iter_matching_configpolicies(business_policies, context): + """Yield (businesspolicyid, ConfigurationPolicy) in deterministic order. + + Business Policies by (priority, businesspolicyid); their items by sortorder. + Only active policies whose scope matches the context are yielded. + """ + ordered_bp = sorted( + business_policies, + key=lambda bp: (bp.priority, bp.businesspolicyid), + ) + for business_policy in ordered_bp: + if not business_policy.isactive: + continue + for item in sorted(business_policy.items, + key=lambda it: (it.sortorder, it.businesspolicyitemid)): + config_policy = item.configurationpolicy + if config_policy is None or not config_policy.isactive: + continue + if evaluate_scope(config_policy.scopejson, context): + yield business_policy.businesspolicyid, config_policy + + +def build_effective_policy(computer, business_policies): + """Fold matching policies into an EffectivePolicy for one endpoint. + + computer: the Computer row. business_policies: iterable of BusinessPolicy + (with items/configurationpolicy loaded). Pure given its inputs so it stays + unit-testable and reusable by an alternate assembly engine. + """ + context = build_endpoint_context(computer) + resolved = {} + for businesspolicyid, config_policy in _iter_matching_configpolicies( + business_policies, context): + for action_row in config_policy.actions: + if not action_row.isactive: + continue + name = action_row.action + if name in resolved: + # First match wins - a higher-priority policy already decided + # this action; later policies do not override it. + continue + resolved[name] = EffectiveAction( + action=name, + mode=action_row.mode, + desiredstate=action_row.desiredstate, + params=action_row.params, + intervalminutes=action_row.intervalminutes, + businesspolicyid=businesspolicyid, + configurationpolicyid=config_policy.configurationpolicyid, + ) + return EffectivePolicy(resolved) + + +def effective_policy_for(assetid): + """Resolve the effective policy for an endpoint by its assetid. + + This is the public entry point the endpoints plugin calls. It returns an + empty EffectivePolicy when the feature is off or the asset is not a known + computer, so callers get a safe "no policy" answer without special-casing. + + NOTE for GE-Enforce interop: this function is the seam. To hand policy + assembly to GE-Enforce later, replace the body below (or dispatch on source) + while keeping this signature and the EffectivePolicy return type intact. + """ + from ..policyflag import policy_flag_enabled + if not policy_flag_enabled(): + return EffectivePolicy({}) + + from ..models import Computer, BusinessPolicy + + computer = Computer.query.filter_by(assetid=assetid).first() + if computer is None: + return EffectivePolicy({}) + + business_policies = BusinessPolicy.query.filter_by(isactive=True).all() + return build_effective_policy(computer, business_policies) diff --git a/plugins/computers/services/policyscope.py b/plugins/computers/services/policyscope.py new file mode 100644 index 00000000..049c272d --- /dev/null +++ b/plugins/computers/services/policyscope.py @@ -0,0 +1,96 @@ +"""Boolean scope evaluation for Configuration Policies. + +A policy's scopejson is a small typed boolean AST over two endpoint dimensions, +PC type and installed application. This module is the ONLY place that interprets +that AST: it validates the shape (known node types, bounded depth) and evaluates +a validated tree against a resolved endpoint context. + +Node grammar (all dict nodes carry a "type"): + {"type": "and", "nodes": [, ...]} + {"type": "or", "nodes": [, ...]} + {"type": "not", "node": } + {"type": "pctype", "id": } + {"type": "app", "id": } + {"type": "true"} # matches everything (an unscoped/global policy) + +Kept pure and dependency-free (no db, no Flask) so it is trivially unit-tested +and so a future GE-Enforce assembly engine can reuse the exact same evaluator on +the exact same AST when it composes policies. +""" + +# Bound the tree so a hand-crafted or imported AST cannot cause pathological +# recursion. Ten levels is far past any policy a human would author. +MAX_SCOPE_DEPTH = 10 + +_LOGICAL = {'and', 'or', 'not'} +_LEAF = {'pctype', 'app', 'true'} + + +class ScopeError(ValueError): + """Raised when a scope AST is malformed. Carries a human-readable reason.""" + + +def validate_scope(node, _depth=0): + """Validate a scope AST, raising ScopeError on any problem. + + Returns None on success. Call this at write time (CRUD) so a bad tree never + reaches the database; evaluate_scope then trusts the stored shape. + """ + if _depth > MAX_SCOPE_DEPTH: + raise ScopeError(f'scope nested deeper than {MAX_SCOPE_DEPTH} levels') + if not isinstance(node, dict): + raise ScopeError('scope node must be an object') + + ntype = node.get('type') + if ntype not in _LOGICAL and ntype not in _LEAF: + raise ScopeError(f'unknown scope node type: {ntype!r}') + + if ntype in ('and', 'or'): + nodes = node.get('nodes') + if not isinstance(nodes, list) or not nodes: + raise ScopeError(f'{ntype} requires a non-empty "nodes" list') + for child in nodes: + validate_scope(child, _depth + 1) + elif ntype == 'not': + child = node.get('node') + if child is None: + raise ScopeError('not requires a "node"') + validate_scope(child, _depth + 1) + elif ntype in ('pctype', 'app'): + ident = node.get('id') + if not isinstance(ident, int) or isinstance(ident, bool): + raise ScopeError(f'{ntype} requires an integer "id"') + # 'true' carries no payload. + + +def evaluate_scope(node, context): + """Evaluate a validated scope AST against an endpoint context. + + context is expected to expose: + context['computertypeid'] -> int or None + context['appids'] -> set/collection of installed appids + + Returns True when the endpoint falls in scope. An empty/None scope means the + policy is unscoped and matches everything (same as {"type": "true"}), which + keeps a freshly-created policy usable before a scope is drawn. + """ + if node is None: + return True + + ntype = node['type'] + if ntype == 'true': + return True + if ntype == 'and': + return all(evaluate_scope(child, context) for child in node['nodes']) + if ntype == 'or': + return any(evaluate_scope(child, context) for child in node['nodes']) + if ntype == 'not': + return not evaluate_scope(node['node'], context) + if ntype == 'pctype': + return context.get('computertypeid') == node['id'] + if ntype == 'app': + return node['id'] in (context.get('appids') or ()) + + # validate_scope should have caught this; be strict rather than silently + # matching an endpoint against an unknown node. + raise ScopeError(f'unknown scope node type at evaluation: {ntype!r}') diff --git a/plugins/endpoints/AGENT-CONTRACT.md b/plugins/endpoints/AGENT-CONTRACT.md new file mode 100644 index 00000000..6c6b2764 --- /dev/null +++ b/plugins/endpoints/AGENT-CONTRACT.md @@ -0,0 +1,274 @@ +# Endpoint agent contract (C# build reference) + +Companion to DESIGN-DECISIONS-agent.md. This is the concrete wire contract the +future C# agent targets, derived from `EndpointsPlugin.get_collector_schema()` +and the generic collector dispatcher in `shopdb/core/api/collector.py`. When the +schema changes, update this file in the same change. + +## 1. Transport + +- Method/URL: `POST {baseurl}/api/collector/endpoints` +- Auth header: `X-API-Key: ` (Bearer token also accepted). The + key is a UI-managed, endpoints-only service token minted from Settings -> + Endpoint agent key (section 6). A bootstrap `.env` collector key still works + until the first UI key is generated, then it is switched off (section 6). No + querystring fallback. +- Body: `application/json`, a single flat object (NOT an array). One POST per + report per endpoint. +- Identity field: `hostname` (required, case-insensitive). + +## 2. Request payload + +Only `hostname` is required. Every other field is optional; the server upserts +only the keys that are present, so a lightweight report can omit inventory +arrays. `adopt` is first-contact only. + +```jsonc +{ + "hostname": "SHOP-PC-014", // required, resolved to computers.hostname + "adopt": true, // first run ONLY; delegates PC creation to computers plugin + "loggedinuser": "j.smith", + "serialnumber": "5CD1234ABC", // adoption subset (forwarded to computers) + "pctype": "CNC Controller", // adoption subset + "machinenumber": "12345", // adoption subset; "" or "9999" == none + "agentversion": "1.0.0", + "lastboottime": "2025-01-14T06:12:00Z", // ISO-8601 UTC + "uptimeseconds": 86400, + "idleseconds": 120, + "cpupercent": 12.5, + "memorypercent": 47.0, + "services": [ /* opaque array, stored as JSON */ ], + "ports": [ /* opaque array */ ], + "interfaces": [ + { + "alias": "Intel(R) Ethernet Connection", // adapter description + "ipv4": "10.20.30.40", // "" if none + "mac": "AA:BB:CC:DD:EE:FF", + "gateway": "10.20.30.1", // "" for the static machine-IPS NIC + "subnet": "255.255.255.0", + "dhcp": false + } + ], + "scheduledtasks": [ /* opaque array */ ], + "events": [ + { + "eventid": 6008, // required per item; non-catalog IDs are dropped, not errored + "eventchannel": "System", + "eventsource": "EventLog", + "eventtime": "2025-01-14T06:11:58Z", + "eventmessage": "The previous system shutdown was unexpected.", + "fields": { "any": "extra" } // opaque object, stored as JSON + } + ] +} +``` + +Notes: +- `adopt=true` is sent ONLY by the installer's first run. Steady-state reports + omit it (or send `false`). After adoption, an unknown hostname is a hard + validation error again (DESIGN-DECISIONS-agent.md 1a). +- The adoption subset (`serialnumber`, `pctype`, `machinenumber`, plus + `loggedinuser`) is forwarded to the `computers` collector so the newly created + PC is not a bare shell. Send them on first contact. +- `services`/`ports`/`interfaces`/`scheduledtasks` are stored verbatim as JSON; + the server does not validate their inner shape yet. Keep them stable so the UI + can render them later. `interfaces` reports EVERY hardware NIC (not just the + primary) so the gatewayless static NIC that links a PC to its machine's IPS is + preserved (DESIGN-DECISIONS-agent.md 9.3); each entry is + `{alias, ipv4, mac, gateway, subnet, dhcp}`. +- `machinenumber` is discovered on-box by the agent from the eDNC registry + (`HKLM\SOFTWARE\[WOW6432Node\]GE Aircraft Engines\DNC\General:MachineNo`) or + the UDC `udc_settings_*.json` (`GeneralSettings.MachineNumber`), in that order + (DESIGN-DECISIONS-agent.md 9.4/9.5). `""` or `"9999"` means none. + +## 3. Response + +Success (HTTP 200), ADR-006 envelope: + +```jsonc +{ + "status": "ok", + "action": "created", // "created" first time, "updated" thereafter + "assetid": 4821, + "identityvalue": "SHOP-PC-014", + "warnings": [ + "adopted new computer for hostname: 'SHOP-PC-014'" // present only on adoption + ] +} +``` + +Validation failure (HTTP 400) - e.g. unknown hostname with `adopt` absent: + +```jsonc +{ "status": "error", "error": { "code": "VALIDATION_ERROR", + "message": "could not resolve hostname to a computer: 'SHOP-PC-014'" } } +``` + +Agent handling: treat non-200 as retryable except a `VALIDATION_ERROR` that +names an unresolved hostname on a NON-first-contact report - that means the PC +was removed server-side; surface it, do not spin. + +## 4. C# DTOs + +Match the JSON keys EXACTLY (lowercase, no underscores) to satisfy the naming +rule; use attributes so C# properties can stay PascalCase without a camelCase +serializer changing the wire keys. + +```csharp +using System.Text.Json.Serialization; + +public sealed class EndpointReport +{ + [JsonPropertyName("hostname")] public string Hostname { get; set; } = ""; + [JsonPropertyName("adopt")] public bool? Adopt { get; set; } + [JsonPropertyName("loggedinuser")] public string? LoggedInUser { get; set; } + [JsonPropertyName("serialnumber")] public string? SerialNumber { get; set; } + [JsonPropertyName("pctype")] public string? PcType { get; set; } + [JsonPropertyName("machinenumber")] public string? MachineNumber { get; set; } + [JsonPropertyName("agentversion")] public string? AgentVersion { get; set; } + [JsonPropertyName("lastboottime")] public DateTimeOffset? LastBootTime { get; set; } + [JsonPropertyName("uptimeseconds")] public long? UptimeSeconds { get; set; } + [JsonPropertyName("idleseconds")] public long? IdleSeconds { get; set; } + [JsonPropertyName("cpupercent")] public double? CpuPercent { get; set; } + [JsonPropertyName("memorypercent")] public double? MemoryPercent { get; set; } + [JsonPropertyName("services")] public object[]? Services { get; set; } + [JsonPropertyName("ports")] public object[]? Ports { get; set; } + [JsonPropertyName("interfaces")] public object[]? Interfaces { get; set; } + [JsonPropertyName("scheduledtasks")]public object[]? ScheduledTasks { get; set; } + [JsonPropertyName("events")] public EndpointEventRecord[]? Events { get; set; } +} + +public sealed class EndpointEventRecord +{ + [JsonPropertyName("eventid")] public int EventId { get; set; } + [JsonPropertyName("eventchannel")] public string? EventChannel { get; set; } + [JsonPropertyName("eventsource")] public string? EventSource { get; set; } + [JsonPropertyName("eventtime")] public DateTimeOffset? EventTime { get; set; } + [JsonPropertyName("eventmessage")] public string? EventMessage { get; set; } + [JsonPropertyName("fields")] public Dictionary? Fields { get; set; } +} + +public sealed class CollectorResponse +{ + [JsonPropertyName("status")] public string Status { get; set; } = ""; + [JsonPropertyName("action")] public string? Action { get; set; } + [JsonPropertyName("assetid")] public int? AssetId { get; set; } + [JsonPropertyName("identityvalue")] public string? IdentityValue { get; set; } + [JsonPropertyName("warnings")] public string[]? Warnings { get; set; } +} +``` + +Serializer settings: `DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull` +so omitted fields are not sent (server upserts only present keys). Send UTC ISO-8601. + +## 5. "Build site agent" settings flow (build cadence) + +Goal: an operator in ShopDB Settings produces an instance-locked `.exe` that +knows its server URL and its collector key, needing no manual config on the +client. + +1. Settings -> "Build site agent": operator picks/creates a collector-scoped + managed token (the same token accepted by `X-API-Key` on + `/api/collector/endpoints`). This binds the build to one ShopDB instance. +2. Server stamps a small config (base URL + token + reporting cadence) into the + agent build artifact. Instance-locked = the token only works against that + server, and the server can revoke it to kill a leaked build. +3. First run on a client: + - Collect the ShopDB PowerShell asset-collector data set (minus the CSV + emit): hostname, logged-in user, serial number, pc type, machine number, + boot time, plus the curated event tail. + - POST once with `adopt=true`. The endpoints plugin delegates PC creation to + the `computers` collector, then writes the first `EndpointState` in the + same commit. The endpoint appears in the Fleet UI immediately - no second + call, no separate enrollment endpoint. +4. Subsequent runs POST on the configured cadence with `adopt` omitted; strict + hostname resolution applies from then on. + +Open item (from DESIGN-DECISIONS-agent.md 1a): whether `adopt` stays a +per-report flag or is promoted to a site setting (strict vs self-adopting +fleet). Default proposed: agent sends `adopt=true` on first run only. + +## 6. UI-managed agent key (kill switch) + +The endpoint agent authenticates with a single UI-managed key scoped to ONLY +`collector.ingest.endpoints`, so rotating or revoking it stops all +endpoint-agent traffic without affecting the computers or geenforce collectors. + +- Settings -> Endpoint agent key (admin only) generates the key. The full secret + is shown ONCE at generation and stored only as a hash afterwards; configure it + as the agent `X-API-Key`. +- Generating a key is a hard cutover: any previous endpoint key is revoked in the + same request, and the bootstrap `.env` fallback for endpoints is switched off + (`plugin.endpoints.envkeyfallback` -> false), so the managed key becomes the + sole credential. +- Regenerate to rotate (old key dies immediately). Revoke to stop all + endpoint-agent reporting until a new key is generated - the kill switch for a + leaked agent. +- Backend: `GET/POST /api/endpoints/agentkey`, `POST + /api/endpoints/agentkey/regenerate`, `DELETE /api/endpoints/agentkey`. These + use the `mint_service_token`/`list_service_tokens`/`revoke_service_tokens` + contract helpers (shopdb.api, contract 0.22.0). + +The bootstrap `.env` key (`COLLECTOR_API_KEY[_ENDPOINTS]`) remains only as a +first-run path before a UI key exists; it cannot be the runtime kill switch +because container env is injected at boot and cannot be rotated live. + +## 7. Command channel (WebUI -> agent) + +The WebUI can send an endpoint an ALLOWLISTED command (never an arbitrary +command string). Delivery is agent-initiated long-poll, so shop-floor PCs open +no inbound port: the agent makes an outbound request the server holds open until +a command is queued, runs it locally, and posts the result back. All three +routes reuse the same endpoints collector key (section 6) via `X-API-Key`, so +the command channel shares the kill switch and adds no second secret. + +Allowlisted actions (server source of truth: `plugins/endpoints/commands.py`): + +- `mtconnect_start` - run the MTConnect UDC agent start script + (`C:\ProgramData\UDC\MTConnect_UDC\Agent\run.bat`). +- `mtconnect_stop` - stop the running MTConnect UDC agent process. +- `mtconnect_status` - report whether the MTConnect UDC agent is running. + +### 7.1 Claim the next command (long-poll) + +`GET {baseurl}/api/endpoints/commands/next?hostname=` + +- Auth: `X-API-Key` (endpoints collector key). +- The server holds the request open up to ~25s. If a command is queued it is + atomically claimed (marked running) and returned; only one poll ever claims a + given command. +- Command queued (HTTP 200): + +```jsonc +{ "status": "ok", "data": { "endpointcommandid": 91, "action": "mtconnect_start" } } +``` + +- Nothing queued within the hold window: HTTP 204 (no body). Re-poll. + +### 7.2 Report a command result + +`POST {baseurl}/api/endpoints/commands//result` + +- Auth: `X-API-Key` (endpoints collector key). +- Body: + +```jsonc +{ + "succeeded": true, // required; false marks the command failed + "message": "MTConnect started", // short summary, optional (<= 2000 chars) + "mtconnectrunning": true, // optional; observed run state after the action + "detail": { "any": "extra" } // optional structured detail, stored as JSON +} +``` + +- When `mtconnectrunning` is present the server caches it on the endpoint's + `EndpointState` (`mtconnectrunning` + `mtconnectcheckedat`) so the WebUI toggle + shows a real state between polls. Every `mtconnect_*` action SHOULD probe and + report the resulting run state so the toggle never lies. + +### 7.3 Enqueue (WebUI side, for reference) + +`POST /api/endpoints//commands` with `{ "action": "mtconnect_start" }`, +gated by user JWT + `endpoints.command`. Only one command may be in flight per +endpoint (a second enqueue returns HTTP 409 until the first finalizes). + diff --git a/plugins/endpoints/DESIGN-DECISIONS-agent.md b/plugins/endpoints/DESIGN-DECISIONS-agent.md index 67e2da4b..ab50920b 100644 --- a/plugins/endpoints/DESIGN-DECISIONS-agent.md +++ b/plugins/endpoints/DESIGN-DECISIONS-agent.md @@ -22,10 +22,52 @@ agent changes. computers reporter does: out of band, e.g. `HKLM:\SOFTWARE\GE\ShopDB CollectorKey`. The key is never baked into a distributed script/binary. - Identity field is `hostname` (case-insensitive). The server resolves it to an - existing computer asset via the `computers` plugin. An unknown hostname is - REJECTED with a validation error, never auto-created. Decision: the fleet is - provisioned in ShopDB first (imports/Azure device sync); the agent does not - get to invent assets. Revisit only if bulk self-registration becomes a need. + existing computer asset via the `computers` plugin. In STEADY STATE an unknown + hostname is REJECTED with a validation error, never auto-created by this + plugin. Decision: the fleet is provisioned in ShopDB first (imports/Azure + device sync) OR adopted at first contact via the delegation shim in section + 1a; the `endpoints` plugin itself does not get to invent assets. + +## 1a. First-contact adoption (delegation shim, not a design reversal) + +The agent installer's first run reaches a PC that may not exist in ShopDB yet. +Rather than letting `endpoints` create assets (which would break ADR-001 and the +section 1 rule that this plugin never invents assets), first contact DELEGATES +adoption to the `computers` collector, which is already the sole authoritative +creator of computer Assets and PC rows. + +Decision: + +- The `endpoints` plugin NEVER constructs an `Asset` or a `Computer` row itself. + Section 1 stands. This is a clarification, not a reversal. +- When `apply_collector_payload` cannot resolve the hostname AND the payload + opts in to adoption (see the `adopt` flag below), the plugin calls the + `computers` plugin's `apply_collector_payload` with the PC-identifying subset + of the payload, flushes, then re-resolves the hostname and proceeds with + telemetry. All of this runs inside the ONE transaction the generic collector + dispatcher owns (`shopdb/core/api/collector.py`): a `ValueError` still rolls + the whole thing back, so a half-adopted PC can never be committed. +- Steady-state reports send `adopt=false` (or omit it once a site chooses strict + mode); only the installer's first run sends `adopt=true`. After adoption the + strict "must already resolve" invariant holds for every subsequent report. + +First-contact payload (the PC-identifying subset the shim forwards to the +`computers` collector - every field except `hostname` is optional but recommended +so the adopted PC is not a bare shell): + +- `hostname` (required), `loggedinuser`, `serialnumber`, `pctype`, + `machinenumber` (only when the PC is bound to a machine; `''`/`'9999'` are + treated as none by the computers collector). + +Net effect for the agent: + +- ONE POST to `/api/collector/endpoints` with `adopt=true` on first run creates + the Asset + PC (via the shim) and writes the first `EndpointState` in the same + commit, so the endpoint appears in the UI immediately (no second call, no + separate enrollment endpoint). +- OPEN: whether to keep `adopt` per-report or promote it to a site setting + (strict vs. self-adopting fleet). Default proposed: adoption allowed, agent + sends `adopt=true` only on first run. ## 2. Report shape (current server contract) @@ -117,6 +159,31 @@ endpoint agree: - Code signing and Intune deployment are agent-side concerns, out of scope for the server plugin but must be planned before fleet rollout. +### 7a. Tray deployment and on-demand "Send Status" (built) + +The tray icon is a PER-USER process, not a service. A Windows service runs in +session 0 and cannot draw a tray icon in the operator's session, so the tray +ships as a second executable (`shopdb-endpoint-tray.exe`) that the installer +copies next to `shopdbagent.exe`, registers for logon autostart via the HKCU +Run key (`ShopDbEndpointTray`), and launches for the current user. Because the +Run key is per-user, the installer must be run from the operator's own session; +`-NoTray` yields a headless, service-only install. + +DECISION - "Send Status now" reuses `--once`, it does not reimplement collection. +The tray's Send Status action shells out to `shopdbagent.exe --once`, the same +production collect-and-report path the service uses on its cadence. Consequences: +the tray holds NO collector key (the agent exe reads it from registry/appsettings +itself), there is one collection code path to maintain, and an operator can force +the ShopDB monitor page to refresh immediately instead of waiting for the next +cadence tick. The tray surfaces the agent's exit code as a balloon tip (0 = +sent, non-zero = check the rolling agent log). Note: `--once` honors +`adoptonfirstrun`, so an on-demand send can carry `adopt=true`; the server dedups +on hostname, so this is safe. + +The ServiceNow ticket action stays a scaffold (section 6). A future desktop +shortcut to the same tray exe is the intended explicitly-launchable entry point +for that workflow; nothing about the Send Status wiring blocks it. + ## 8. Web UI display (server-side, built) The collected data surfaces through ShopDB's existing declaration-driven UI @@ -153,3 +220,224 @@ Agent implication: the agent does not drive any UI refresh. Its only job is to report on its rhythm; the UI reflects whatever the last report wrote. The `services` payload should send a per-service run STATE (e.g. running/stopped) so the badge can color it; a missing/unknown state renders as a gray Unknown badge. + +## 9. Shop-floor collection knowledge base (reference, NOT a schema change) + +This section is a body of reference, not a change request. Nothing here alters +the shape of ShopDB as it stands. It records WHAT a shop-floor endpoint really +is, WHICH signals identify it, and WHY each signal matters to the enterprise, so +that whoever grows the collector understands the stakes behind each field before +touching the wire contract. The source material is the site's own extant +tooling: `scripts/site_imports/assetcollector/Collect-ShopDB.ps1` (the mature +CSV collector), the migration/assessment flow under `scripts/.temp` +(`Assess-Orchestrator.ps1`, `Setup_ShopPC.ps1`, `Setup_InspectionPC.ps1`, +`eDncRegToUDC.ps1`, `machinetypeselector.bat`), and the eDNC/UDC/MTConnect config +formats those scripts read. Treat it as the "why" that the terse contract in +sections 1 through 8 cannot carry. + +### 9.0 Why this matters beyond one plugin + +This collector is a wedge into an enterprise-wide body of knowledge. A shop-floor +PC is never just a PC: it is the human-facing half of a machine tool, the thing +that feeds NC programs to a controller and streams data back off it. Getting its +identity, hardware, network, and application picture right lets ShopDB answer +questions that today require a person to walk the floor: which controller does +this box talk to, is its data path healthy, is the hardware under warranty, what +is it actually running. Every signal below earns its place by answering one of +those questions. + +### 9.1 The endpoint taxonomy (how the floor classifies its own PCs) + +`machinetypeselector.bat` plus the two `Setup_*` scripts show the site already +thinks in three roles. The collector should be able to DERIVE the role from +on-box evidence rather than ask an operator: + +- Shop PC: runs UDC (Universal Data Collector) and/or eDNC, wired to one CNC + machine (FANUC / Siemens / Okuma controller). This is the ~2/3 majority. +- Inspection PC: metrology stack. Marked by `C:\GE Aerospace\Genspect\genspect.true` + (a flag file `Setup_InspectionPC.ps1` writes) and/or `C:\GE Aerospace\Gagecal` + (GageCal 6.4). +- Non-shop-floor PC: office box, no machine link, minimal shop apps. + +Role is derivable from application/marker presence (section 9.4), which is a more +honest signal than the raw WMI `Model` string the current `pctype` probe sends. + +### 9.2 Hardware facts, and WHY + +`Collect-ShopDB.ps1 :: Get-PcFacts` reads `Win32_ComputerSystem`, +`Win32_OperatingSystem`, and `Win32_BIOS` for: vendor/manufacturer, model, OS +caption, serial (BIOS serial first, ComputerSystem serial as fallback), last +boot, logged-in user. + +Why hardware is not cosmetic: + +- Dell Warranty API linkage. Vendor + model family + service tag (BIOS serial) + is exactly the tuple the Dell Warranty API keys on. Capturing it lets ShopDB + resolve warranty status and hardware line remotely, so a support tech can + diagnose a hardware-level fault (failing disk, EOL platform, out-of-warranty + chassis) without physically finding the machine. The serial is the join key; + the model family is the diagnostic context. +- Remote hardware diagnosis generally. Knowing the exact hardware line turns + "some PC is flaky" into "this is a known-bad model/BIOS combination," which is + an enterprise-scale pattern once the fleet reports uniformly. + +Collection note: prefer BIOS `SerialNumber` (the Dell service tag) and capture +manufacturer + model as distinct fields, since the warranty join needs the raw +service tag, not a normalized `pctype`. + +### 9.3 Network interfaces, and WHY (the second-NIC reality) + +`Get-PcFacts` and `Assess-Orchestrator.ps1 :: Get-EthernetNetworkSnapshot` +enumerate hardware adapters only and capture alias, IPv4, MAC, gateway, and +subnet (the orchestrator even reconstructs the mask from prefix length). + +Why the interface list is first-class, not trivia: + +- The second interface IS the machine link. Upwards of two-thirds of shop-floor + PCs have a SECOND network interface with a STATIC, manually mapped IP address + whose sole purpose is to talk to the related machine's onboard IPS + (industrial PC / controller network). That private NIC is physical evidence of + the PC-to-machine relationship and often the only place the machine-side + address is recorded. Losing it loses the wiring diagram of the floor. +- MAC as stable identity. MAC is the most stable hardware identifier across + re-image and rename, which is why every extant script grabs it. It is the + durable anchor when hostname/asset number churn during migration. + +Collection note: send ALL hardware interfaces (alias, IPv4, MAC, and where +available gateway/subnet), not just the primary. The primary/gateway NIC is the +corporate LAN; the gatewayless static NIC is very likely the machine IPS link and +is the interesting one. + +### 9.4 Installed applications, and WHY (the extrapolation engine) + +`Get-PcFacts` reads the registry Uninstall keys (name, version, publisher, +skipping `SystemComponent`). On these boxes the installed-app set is small and +highly diagnostic: they primarily run a handful of known apps. + +Why the app inventory is the richest signal we have: + +- Apps imply role (section 9.1): UDC/eDNC => Shop PC; Genspect/GageCal => + Inspection PC; neither => office. +- Apps imply the MACHINE binding. This is the crux. If UDC or eDNC (or MTConnect) + is installed, its config file carries the four-digit number ShopDB calls the + Machine Number, and from it we can extrapolate the ShopDB `machines` entry the + PC should be related to. The same value appears under different names across + the app generations: + - eDNC `.reg` / live registry: `MachineNo` (under + `HKLM\SOFTWARE\[WOW6432Node\]GE Aircraft Engines\DNC\General`). + - UDC `udc_settings_*.json`: `GeneralSettings.MachineNumber`. + - MTConnect: workstation / workstation ID / MachineNo, carried in the + adapter/agent config under the fixed directory tree documented in section + 9.9. + Treat "MachineNo", "workstation", "workstation ID", and "MachineNumber" as the + same concept: the ShopDB Machine Number. Version and publisher are worth + keeping too (they date the app generation, which tells you which config format + to parse). + +Collection note: the app inventory is not just an asset list; it is the pointer +that tells the collector WHICH config file to open next to find the machine +binding. + +### 9.5 The machine binding (DNC/UDC/MTConnect config), and WHY + +`Collect-ShopDB.ps1 :: Resolve-DncSource` resolves the machine config with a +SOURCE-PRIORITY chain, then three parsers normalize the formats into one shape: + +- Priority: explicit JSON path -> explicit `.reg` path -> live registry + (`HKLM\...\GE Aircraft Engines\DNC`) -> local `.reg` export -> local + `udc_settings_*.json`. This ordering makes one probe work across both the + eDNC-era and UDC-era boxes. +- Parsers: `ConvertFrom-RegExport` (parses exported `.reg`, including + `dword:` decoding), `ConvertFrom-LiveRegistry`, `ConvertFrom-UdcJson`. +- Extracted: machine number, site, control type, DNC type, and connection detail + (TCP address/port for ethernet; COM port/baud for serial). +- Controller vendor is DERIVED, not stored raw: `Resolve-ControllerVendor` maps + `NcIF = EFOCAS -> FANUC`, `Cnc = SIEMENS -> Siemens`, `Cnc = OKUMA -> Okuma`. + `eDncRegToUDC.ps1` confirms the same mapping and the eFocas/PPDCS key layout. + +Why it matters: this binding is what upgrades a bare endpoint into "the PC that +controls machine 1234 over eFOCAS at 10.x.x.x." That relationship is the point of +the whole exercise; the hardware and app facts are supporting evidence for it. + +### 9.6 Printers, and WHY + +`Assess-Orchestrator.ps1 :: Get-PrinterSnapshot` / `Get-PrinterPortSnapshot` +capture name, driver, port, host address, shared/published, and the default +flag, filtering out virtual outputs (Print to PDF, XPS, Fax, OneNote). Shop and +inspection PCs are tied to specific physical/label printers, so the printer set +is part of a workstation's operational identity and a common migration break +point (a replaced PC that lost its label printer is a floor stoppage). + +### 9.7 Patterns from the extant tooling worth carrying forward + +Two engineering habits in these scripts are worth honoring in the collector, +because they encode hard-won migration lessons: + +- Field-level idempotent merge (`Merge-CsvStore`): an empty incoming cell NEVER + overwrites a previously captured value; later partial runs fill gaps without + wiping earlier data. The server's endpoints ingest already mirrors this instinct + (`if 'field' in payload`), and it should stay that way: a partial report means + "leave the rest alone," not "clear it." +- Normalized intermediate shape + sidecars: heterogeneous sources (registry, + `.reg`, JSON, WMI) all converge on one normalized record before import, with + phase-2 data (interfaces, installed apps, PC-to-machine link) written as JSON + sidecars. The same decoupling lets a future non-agent source (Intune/Azure + `Get-MgDevice`, a MES/CNC database) feed the same model without a new pipeline. + +### 9.9 MTConnect footprint, and WHY (three parallel stacks per box) + +Source: `scripts/.temp/Install_MTConnect_ExisDir.ps1`, +`Install_MTConnect_ExisDir_BatConvert.ps1`, its usage guide +`Guide-Install_MTConnect_ExisDir_BatConvert.md`, and +`Firewall_Rules_MTConnect_All.bat`. MTConnect is the standardized data-off-the- +machine layer, and the install scripts reveal its exact on-disk shape, which is +what a collector needs to detect it reliably. + +The critical structural fact: an endpoint can host up to THREE parallel MTConnect +stacks, one per data path, each an Adapter (talks to the machine) plus an Agent +(serves the MTConnect HTTP feed): + +- Machine-direct: `C:\MTConnect\Adapter` + `C:\MTConnect\Agent` (Agent exe + `C:\MTConnect\Agent\Agent.exe`). +- eDNC-fed: `C:\MTConnect_eDNC\Adapter` + `C:\MTConnect_eDNC\Agent` + (`...\Agent\agent.exe`). The eDNC adapter is `DNCAdapter.exe` driven by + `DNCAdapter.ini`, tying this path back to the eDNC config world in section 9.5. +- UDC-fed: `C:\ProgramData\udc\mtconnect_udc\agent` (`...\agent\agent.exe`, + launched via `run.bat`). This binds MTConnect to the UDC install and therefore + to the same `udc_settings_*.json` that carries `GeneralSettings.MachineNumber`. + +Why the collector should care about these paths: + +- Presence is role/path evidence. Which of the three trees exist tells you HOW the + machine's data is being lifted (direct vs eDNC vs UDC), which is a real + operational distinction, not a cosmetic one. +- The MTConnect Agent is a network service. The firewall bat and the + `Ensure-FirewallProgramRule` calls open inbound rules named + `Allow_MTConnect_Machine`, `Allow_MTConnect_UDC`, `Allow_MTConnect_eDNC` for + each `Agent.exe`. That means each agent listens on an HTTP port, so its live + MTConnect feed (which publishes a device UUID / station identity) is itself a + future data source the collector could probe, not just a process to count. +- Machine Number lives here too. Because the eDNC path is `DNCAdapter.ini` and the + UDC path is the UDC settings JSON, the MTConnect footprint is a second route to + the same four-digit Machine Number described in 9.4/9.5, cross-checkable against + the eDNC/UDC values. + +Collection note: detect MTConnect by the three directory trees, capture which +Agent exes exist and whether their service/process is running (section 5 already +covers MTConnect as a monitored service), and record the data path +(direct/eDNC/UDC). +(`Install_MTConnect_ExisDir_BatConvert.ps1`) is a deployment concern, not a +collection concern, but it explains why these `install.bat`/`start.bat` files +were rewritten to self-elevate without a `pause`: so they can run unattended +under DSC on locked-down machines. Do not treat a self-elevating batch as +tampering. + +### 9.8 Boundary (explicit) + +None of section 9 is a ShopDB schema proposal. Where a signal above has no home +in the current model, that is fine: it stays reference until a separate, deliberate +decision (its own ADR / contract bump) decides to store it. The value here is +comprehension: anyone extending the collector should know that hardware feeds +warranty, the second NIC is the machine link, and the installed apps are the +thread that leads to the Machine Number. Build the agent with that understanding; +do not let this document, by itself, move the schema. diff --git a/plugins/endpoints/api/routes.py b/plugins/endpoints/api/routes.py index 650ad040..14f84474 100644 --- a/plugins/endpoints/api/routes.py +++ b/plugins/endpoints/api/routes.py @@ -7,17 +7,21 @@ These routes are the human-facing read surface, gated by user JWT + RBAC. """ import json +import time from datetime import datetime, timedelta, timezone -from flask import Blueprint, request -from flask_jwt_extended import jwt_required +from flask import Blueprint, request, current_app +from flask_jwt_extended import jwt_required, current_user from shopdb.api import ( db, Asset, Setting, success_response, error_response, paginated_response, ErrorCodes, get_pagination_params, paginate_query, require_permission, + require_role, mint_service_token, list_service_tokens, revoke_service_tokens, + AuditLog, collector_authorized, ) -from ..models import EndpointState, EndpointEvent +from ..models import EndpointState, EndpointEvent, EndpointCommand +from ..commands import COMMAND_ALLOWLIST, is_allowed_action, ALLOWED_ACTIONS_FOR, row_actions_for, requires_target, action_targetkind from ..eventcatalog import CRITICAL_EVENT_IDS, event_severity, event_title endpoints_bp = Blueprint('endpoints', __name__) @@ -188,14 +192,27 @@ def panel_services(asset_id: int): if isinstance(entry, dict): name = entry.get('name') or entry.get('service') or 'Unknown' raw = (entry.get('state') or entry.get('status') or '').strip().lower() + app = entry.get('app') + kind = entry.get('kind') else: - name, raw = str(entry), '' + name, raw, app, kind = str(entry), '', None, None running = raw in ('running', 'started', 'up', 'ok', 'active') - rows.append({ + row = { 'name': name, 'statelabel': 'Running' if running else (raw.title() or 'Unknown'), 'statecolor': 'green' if running else ('red' if raw else 'gray'), - }) + } + targets = row_actions_for(app, kind, name) + if targets: + row['action'] = { + 'running': running, + 'start': targets['start'], + 'stop': targets['stop'], + 'status': targets['status'], + 'target': targets['target'], + 'targetkind': targets['targetkind'], + } + rows.append(row) return success_response(rows) @@ -223,6 +240,48 @@ def panel_recent_events(asset_id: int): return success_response(rows) +@endpoints_bp.route('/asset//interfaces', methods=['GET']) +@jwt_required() +@require_permission('endpoints.view') +def panel_interfaces(asset_id: int): + """Table panel: every hardware NIC the agent last reported. + + Reshapes the latest interfacesjson snapshot into one row per adapter with + the {alias, ipv4, mac, gateway, subnet, dhcp} shape the collector emits. A + gatewayless adapter is flagged as the likely machine-IPS link (see + DESIGN-DECISIONS-agent.md 9.3): those NICs speak to the machine, not the + corporate LAN. Tolerant of a missing or unexpected snapshot: an unreadable + payload yields no rows so the panel hides itself. + """ + state = EndpointState.query.filter_by(assetid=asset_id).first() + if not state or not state.interfacesjson: + return success_response([]) + + try: + interfaces = json.loads(state.interfacesjson) + except (TypeError, ValueError): + return success_response([]) + + rows = [] + for entry in (interfaces or []): + if not isinstance(entry, dict): + continue + gateway = (entry.get('gateway') or '').strip() + ipv4 = (entry.get('ipv4') or '').strip() + rows.append({ + 'alias': entry.get('alias') or 'Unknown', + 'ipv4': ipv4 or '-', + 'mac': (entry.get('mac') or '').strip() or '-', + 'gateway': gateway or '-', + 'subnet': (entry.get('subnet') or '').strip() or '-', + 'dhcp': 'DHCP' if entry.get('dhcp') else 'Static', + # A gatewayless NIC with an address is the machine link; the gateway + # NIC is the corporate LAN. + 'role': 'Machine link' if (ipv4 and not gateway) else 'LAN', + }) + return success_response(rows) + + # ============================================================================= # Dashboard # ============================================================================= @@ -348,3 +407,304 @@ def _quiet_hours() -> int: except (TypeError, ValueError): pass return hours + + +# ============================================================================= +# Endpoint agent key (UI-managed collector.ingest.endpoints kill switch) +# ============================================================================= + +# The single permission the endpoint agent key authorizes. A managed token +# scoped to ONLY this authorizes the endpoints collector and nothing else, so +# rotating/revoking it stops endpoint-agent traffic without touching the +# computers/geenforce collectors. +AGENT_KEY_SCOPE = 'collector.ingest.endpoints' +AGENT_KEY_NAME = 'Endpoint agent key' + + +def _agentkey_status_payload(): + """Current state of the endpoint agent key for the settings page. + + Reports whether an active managed key exists (and its non-secret metadata) + plus whether the bootstrap .env fallback is still accepted. Never exposes a + secret; the full secret is only ever returned once, at regenerate time. + """ + tokens = list_service_tokens(AGENT_KEY_SCOPE) + active = tokens[0] if tokens else None + envfallback = Setting.get('plugin.endpoints.envkeyfallback', default=True) + provisioned = Setting.get('plugin.endpoints.agentkeyprovisioned', + default=False) + return { + 'provisioned': bool(provisioned) or active is not None, + 'envkeyfallback': bool(envfallback), + 'key': active.to_dict() if active is not None else None, + } + + +@endpoints_bp.route('/agentkey', methods=['GET']) +@jwt_required() +@require_role('admin') +def get_agentkey(): + """Report the endpoint agent key state (no secret).""" + return success_response(_agentkey_status_payload()) + + +@endpoints_bp.route('/agentkey/regenerate', methods=['POST']) +@jwt_required() +@require_role('admin') +def regenerate_agentkey(): + """Mint a fresh endpoint agent key, revoking any previous one. + + Hard cutover: the old key is deactivated in the same call, and the .env + fallback is switched off, so the new managed token becomes the sole + credential and revoking it is a true kill switch. The full secret is + returned ONCE here and never again. + """ + try: + token, secret = mint_service_token( + AGENT_KEY_SCOPE, AGENT_KEY_NAME, current_user.userid) + except PermissionError as error: + return error_response(ErrorCodes.FORBIDDEN, str(error), http_code=403) + + Setting.set('plugin.endpoints.agentkeyprovisioned', True, + valuetype='boolean', category='plugin.endpoints') + Setting.set('plugin.endpoints.envkeyfallback', False, + valuetype='boolean', category='plugin.endpoints') + + result = _agentkey_status_payload() + # The secret appears here and NOWHERE else, ever. Not stored, not logged. + result['secret'] = secret + result['warning'] = ('Save this key now. It will not be shown again. ' + 'Configure it as the agent X-API-Key.') + return success_response(result, message='Endpoint agent key generated', + http_code=201) + + +@endpoints_bp.route('/agentkey', methods=['DELETE']) +@jwt_required() +@require_role('admin') +def revoke_agentkey(): + """Revoke the endpoint agent key: stop all managed-token endpoint traffic. + + Leaves envkeyfallback off (the kill switch stays armed). To restore + reporting an operator regenerates a new key. + """ + revoked = revoke_service_tokens(AGENT_KEY_SCOPE) + Setting.set('plugin.endpoints.agentkeyprovisioned', False, + valuetype='boolean', category='plugin.endpoints') + result = _agentkey_status_payload() + result['revoked'] = revoked + return success_response(result, message='Endpoint agent key revoked') + + +# ============================================================================= +# Endpoint commands (WebUI -> agent, allowlisted actions only) +# ============================================================================= +# +# Operators enqueue a named, server-allowlisted action for one endpoint (JWT + +# endpoints.command). The agent long-polls /commands/next with its collector +# token, claims the single next pending command, runs it locally, and reports +# the outcome to /commands//result. The wire never carries a command string +# - only an allowlisted action name - so a compromised WebUI cannot ask an +# endpoint to run something arbitrary. + +# How long the agent claim poll is held open server-side before returning empty. +# Kept below the agent HttpClient timeout and typical proxy idle timeouts so a +# quiet fleet re-polls cleanly instead of tripping a read timeout. +_CLAIM_HOLD_SECONDS = 25 +_CLAIM_POLL_INTERVAL = 1.0 + + +def _collector_auth_or_error(): + """Authorize an agent-facing command route on the endpoints collector token. + + Reuses the collector auth path so the command channel shares the endpoint + agent key (kill switch) and introduces no second secret. + """ + return collector_authorized('endpoints') + + +def _resolve_asset_for_hostname(hostname): + """Resolve a reported hostname to its asset id via the computers plugin.""" + from plugins.computers.models import Computer + + computer = (Computer.query + .filter(db.func.lower(Computer.hostname) == hostname.lower()) + .first()) + return computer.assetid if computer is not None else None + + +@endpoints_bp.route('//commands', methods=['GET']) +@jwt_required() +@require_permission('endpoints.view') +def list_endpoint_commands(asset_id: int): + """Recent commands for one endpoint, newest first (operator view).""" + rows = (EndpointCommand.query + .filter_by(assetid=asset_id) + .order_by(EndpointCommand.createdat.desc()) + .limit(20) + .all()) + return success_response([row.to_dict() for row in rows]) + + +@endpoints_bp.route('//commands', methods=['POST']) +@jwt_required() +@require_permission('endpoints.command') +def enqueue_endpoint_command(asset_id: int): + """Queue an allowlisted command for one endpoint. + + Rejects unknown actions and refuses to stack a second pending/running + command on the same endpoint, so the queue stays at-most-one-in-flight and + the WebUI toggle cannot race itself. + """ + payload = request.get_json(silent=True) or {} + action = (payload.get('action') or '').strip() + if not is_allowed_action(action): + return error_response(ErrorCodes.VALIDATION_ERROR, + f'Unknown or disallowed action: {action!r}') + + if action not in ALLOWED_ACTIONS_FOR(assetid=asset_id): + return error_response( + ErrorCodes.VALIDATION_ERROR, + f'Action not permitted by policy for this endpoint: {action!r}') + + # Generic actions (service_* / process_*) must carry the subject to act on; + # fixed-target actions (mtconnect_*) must not. Validate both directions so a + # caller cannot send a targetless generic action (nothing to act on) or + # smuggle a target onto a pinned action. + target = (payload.get('target') or '').strip() or None + targetkind = action_targetkind(action) + if requires_target(action): + if not target: + return error_response( + ErrorCodes.VALIDATION_ERROR, + f'Action {action!r} requires a target') + elif target is not None: + return error_response( + ErrorCodes.VALIDATION_ERROR, + f'Action {action!r} does not accept a target') + + state = EndpointState.query.filter_by(assetid=asset_id).first() + if state is None: + return error_response(ErrorCodes.NOT_FOUND, + f'No endpoint state for asset {asset_id}', + http_code=404) + + pending = (EndpointCommand.query + .filter(EndpointCommand.assetid == asset_id, + EndpointCommand.status.in_(( + EndpointCommand.STATUS_PENDING, + EndpointCommand.STATUS_RUNNING))) + .first()) + if pending is not None: + return error_response( + ErrorCodes.VALIDATION_ERROR, + 'A command is already in flight for this endpoint', + http_code=409) + + command = EndpointCommand( + assetid=asset_id, + action=action, + target=target, + targetkind=targetkind if target else None, + status=EndpointCommand.STATUS_PENDING, + requestedby=getattr(current_user, 'userid', None), + ) + db.session.add(command) + + AuditLog.log('created', 'EndpointCommand', + entityname=action, + details=f'action={action} target={target} asset={asset_id}') + + db.session.commit() + return success_response(command.to_dict(), + message='Command queued', http_code=201) + + +@endpoints_bp.route('/commands/next', methods=['GET']) +def claim_next_command(): + """Agent long-poll: claim the next pending command for a hostname. + + Collector-token auth (not JWT). Holds the request open briefly and returns + the single oldest pending command for the caller's endpoint, atomically + marking it running so no other poll claims it. Returns 204 when nothing is + queued within the hold window - the agent simply re-polls. + """ + error = _collector_auth_or_error() + if error is not None: + return error + + hostname = (request.args.get('hostname') or '').strip() + if not hostname: + return error_response(ErrorCodes.VALIDATION_ERROR, + 'hostname is required') + + asset_id = _resolve_asset_for_hostname(hostname) + if asset_id is None: + return error_response(ErrorCodes.NOT_FOUND, + f'Unknown hostname: {hostname!r}', http_code=404) + + deadline = time.monotonic() + _CLAIM_HOLD_SECONDS + while True: + command = (EndpointCommand.query + .filter_by(assetid=asset_id, + status=EndpointCommand.STATUS_PENDING) + .order_by(EndpointCommand.createdat.asc()) + .first()) + if command is not None: + command.status = EndpointCommand.STATUS_RUNNING + command.claimedat = _naive_utcnow() + db.session.commit() + return success_response({ + 'endpointcommandid': command.endpointcommandid, + 'action': command.action, + 'target': command.target, + 'targetkind': command.targetkind, + }) + + # Nothing queued: release the session so a concurrent enqueue is visible + # on the next poll, wait a beat, then re-check until the hold expires. + db.session.rollback() + if time.monotonic() >= deadline: + return ('', 204) + time.sleep(_CLAIM_POLL_INTERVAL) + + +@endpoints_bp.route('/commands//result', methods=['POST']) +def report_command_result(command_id: int): + """Agent reports a claimed command's outcome (collector-token auth). + + Finalizes the command row and, for MTConnect actions that observed the run + state, caches it on the EndpointState so the WebUI toggle reflects reality + between polls. Both writes land in one commit. + """ + error = _collector_auth_or_error() + if error is not None: + return error + + command = db.session.get(EndpointCommand, command_id) + if command is None: + return error_response(ErrorCodes.NOT_FOUND, + f'No command {command_id}', http_code=404) + + payload = request.get_json(silent=True) or {} + succeeded = bool(payload.get('succeeded')) + command.status = (EndpointCommand.STATUS_SUCCEEDED if succeeded + else EndpointCommand.STATUS_FAILED) + command.resultmessage = (payload.get('message') or '')[:2000] or None + detail = payload.get('detail') + command.resultjson = json.dumps(detail, separators=(',', ':')) \ + if detail is not None else None + command.completedat = _naive_utcnow() + + # Cache the observed MTConnect state so the toggle shows a real value. The + # agent sends mtconnectrunning on any mtconnect_* result that probed it. + if 'mtconnectrunning' in payload: + state = EndpointState.query.filter_by(assetid=command.assetid).first() + if state is not None: + running = payload.get('mtconnectrunning') + state.mtconnectrunning = bool(running) if running is not None else None + state.mtconnectcheckedat = _naive_utcnow() + + db.session.commit() + return success_response(command.to_dict(), message='Result recorded') + diff --git a/plugins/endpoints/commands.py b/plugins/endpoints/commands.py new file mode 100644 index 00000000..a44569a2 --- /dev/null +++ b/plugins/endpoints/commands.py @@ -0,0 +1,177 @@ +"""Server-side allowlist of endpoint agent commands. + +The WebUI never sends an arbitrary command string. It sends an allowlisted +ACTION NAME; the concrete on-endpoint behavior (which script to run, how to stop +a process) lives in the agent. This module is the single source of truth for +which action names are legal and what they mean, so both the enqueue route and +the settings/UI surface validate against the same set. + +The structure anticipates per-PC-type allowlists: today every computer may run +every action in COMMAND_ALLOWLIST, but ALLOWED_ACTIONS_FOR is the seam a later +change swaps for a type-scoped lookup (inspection PCs vs shopfloor PCs). +""" + +# action name -> operator-facing metadata. label/description feed the UI; the +# agent maps the action name to its local handler (see CommandExecutor). +# +# Two families of action: +# * Named MTConnect actions - fixed target, no parameters. The original, +# bay-specific controls kept for the inline services-panel toggle. +# * Generic RMM actions (service_* / process_*) - the target is supplied by +# the policy action's params ('target' = a Windows service name or a process +# image like 'DncMain.exe'). This is what populates the Configuration Policy +# builder with reusable controls that any scope (PC type or application) can +# attach. The 'targetkind' hint tells the builder which target field to show. +# NOTE: agent-side execution of the generic actions is not wired yet; the +# server allowlist and policy plumbing land first (the whole feature is +# flag-gated), so a generic action saved today is authored and gated but is +# a no-op on the endpoint until the CommandExecutor handlers ship. +COMMAND_ALLOWLIST = { + 'service_start': { + 'label': 'Start service', + 'description': 'Start a Windows service by name on the endpoint.', + 'targetkind': 'service', + }, + 'service_stop': { + 'label': 'Stop service', + 'description': 'Stop a running Windows service by name on the endpoint.', + 'targetkind': 'service', + }, + 'service_status': { + 'label': 'Check service', + 'description': 'Report whether a named Windows service is running.', + 'targetkind': 'service', + }, + 'process_start': { + 'label': 'Start process', + 'description': 'Launch a process by executable path on the endpoint.', + 'targetkind': 'process', + }, + 'process_stop': { + 'label': 'Stop process', + 'description': 'Stop a running process by image name on the endpoint.', + 'targetkind': 'process', + }, + 'process_status': { + 'label': 'Check process', + 'description': 'Report whether a named process is running.', + 'targetkind': 'process', + }, + 'mtconnect_start': { + 'label': 'Start MTConnect', + 'description': 'Run the MTConnect UDC agent start script on the endpoint.', + }, + 'mtconnect_stop': { + 'label': 'Stop MTConnect', + 'description': 'Stop the running MTConnect UDC agent on the endpoint.', + }, + 'mtconnect_status': { + 'label': 'Check MTConnect', + 'description': 'Report whether the MTConnect UDC agent is running.', + }, +} + + +# Maps a monitored services/processes row to the start/stop/status actions that +# control it, so the services panel can render an inline toggle on exactly the +# rows that have a registered command action. The key is the agent-reported +# 'app' label; 'kind' pins the toggle to the authoritative row (for MTConnect +# the agent.exe PROCESS is the source of truth - run.bat launches it and the +# Windows service state is not reliable), so the service row stays badge-only. +ROW_COMMAND_TARGETS = { + ('MTConnect', 'process'): { + 'start': 'mtconnect_start', + 'stop': 'mtconnect_stop', + 'status': 'mtconnect_status', + }, +} + + +def row_actions_for(app, kind, name=None): + """Return the {start, stop, status, target, targetkind} map for a row. + + Two paths: + * A row registered in ROW_COMMAND_TARGETS (MTConnect) keeps its fixed, + named actions with no wire target - agent.exe is the source of truth and + the behavior is pinned in the agent. + * Any other process/service row maps to the generic primitives + (process_* / service_*) with the row's OWN reported name as the wire + target. This is what lets a new app light up start/stop/status without + an agent rebuild: the target rides the wire. + + Rows with no usable kind return None and render as a plain status badge. + """ + if not app and not name: + return None + fixed = ROW_COMMAND_TARGETS.get((app, (kind or '').lower())) + if fixed: + return {**fixed, 'target': None, 'targetkind': None} + normalized = (kind or '').lower() + if normalized == 'process' and name: + return { + 'start': 'process_start', + 'stop': 'process_stop', + 'status': 'process_status', + 'target': name, + 'targetkind': 'process', + } + if normalized == 'service' and name: + return { + 'start': 'service_start', + 'stop': 'service_stop', + 'status': 'service_status', + 'target': name, + 'targetkind': 'service', + } + return None + + +def is_allowed_action(action): + """True when action is a known, allowlisted command name.""" + return action in COMMAND_ALLOWLIST + + +def action_targetkind(action): + """The targetkind an action requires ('service'/'process'), or None. + + Generic actions declare a targetkind in COMMAND_ALLOWLIST; fixed-target + actions (mtconnect_*) declare none and must not carry a wire target. + """ + meta = COMMAND_ALLOWLIST.get(action) or {} + return meta.get('targetkind') + + +def requires_target(action): + """True when the action is generic and needs a wire target.""" + return action_targetkind(action) is not None + + +def ALLOWED_ACTIONS_FOR(pctype=None, assetid=None): + """The action names an endpoint may run. + + Base allowlist is flat (every action for every computer). When the computers + Configuration/Business Policy feature is enabled AND an assetid is given, the + result is narrowed to the actions the endpoint's effective policy permits - + this is the allowlist-gating half of the policy design. With the feature off + (or no assetid), it returns the full base allowlist, so nothing changes until + a site opts in. Policy resolution is imported lazily to keep this module free + of a hard dependency on the computers plugin. + """ + base = set(COMMAND_ALLOWLIST.keys()) + if assetid is None: + return base + try: + from plugins.computers.services.policyresolver import effective_policy_for + except Exception: + return base + effective = effective_policy_for(assetid) + allowed = effective.allowed_actions() + if not allowed: + # No policy spoke for this endpoint. With the feature off the resolver + # returns empty and we fall back to the base allowlist (unchanged + # behavior). This is intentionally permissive; deny-by-default would be + # a separate, explicit setting. + return base + # Never widen past the code-level allowlist: policy can only gate within it. + return base & allowed + diff --git a/plugins/endpoints/frontend/routes.js b/plugins/endpoints/frontend/routes.js index 1d51f0e2..3e07f25d 100644 --- a/plugins/endpoints/frontend/routes.js +++ b/plugins/endpoints/frontend/routes.js @@ -11,6 +11,12 @@ export default [ name: 'endpoints', component: () => import('./views/EndpointsList.vue'), meta: { plugin: 'endpoints' } + }, + { + path: 'settings/endpoints/agentkey', + name: 'endpoints-agentkey-settings', + component: () => import('./views/AgentKeySettings.vue'), + meta: { requiresAuth: true, requiresAdmin: true, plugin: 'endpoints' } } ] diff --git a/plugins/endpoints/frontend/views/AgentKeySettings.vue b/plugins/endpoints/frontend/views/AgentKeySettings.vue new file mode 100644 index 00000000..8e67d7e5 --- /dev/null +++ b/plugins/endpoints/frontend/views/AgentKeySettings.vue @@ -0,0 +1,293 @@ + + + + + diff --git a/plugins/endpoints/migrations/versions/0002_endpoint_commands.py b/plugins/endpoints/migrations/versions/0002_endpoint_commands.py new file mode 100644 index 00000000..d157e4cc --- /dev/null +++ b/plugins/endpoints/migrations/versions/0002_endpoint_commands.py @@ -0,0 +1,84 @@ +"""endpoint commands queue + cached MTConnect state. + +Adds the endpointcommands queue (one WebUI-issued, server-allowlisted action +per row, claimed and finalized by the agent long-poll) and two cached MTConnect +run-state columns on endpointstates so the WebUI toggle can show a real state +between command polls. + +Idempotent: create_table_if_missing guards the new table, and the column adds +check the live schema first so a db.create_all() site (which already built the +columns) re-runs clean on strict MySQL 8. +""" +from alembic import op + +from shopdb.plugins.alembic_template import ( + create_index_if_missing, + create_table_if_missing, +) +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'endpoints0002commands' +down_revision = 'endpoints0001baseline' +branch_labels = None +depends_on = None + + +def _has_column(table_name, column_name): + from sqlalchemy import inspect + + inspector = inspect(op.get_bind()) + if not inspector.has_table(table_name): + return False + return any(col['name'] == column_name + for col in inspector.get_columns(table_name)) + + +def upgrade(): + if not _has_column('endpointstates', 'mtconnectrunning'): + op.add_column('endpointstates', + sa.Column('mtconnectrunning', sa.Boolean(), + nullable=True)) + if not _has_column('endpointstates', 'mtconnectcheckedat'): + op.add_column('endpointstates', + sa.Column('mtconnectcheckedat', sa.DateTime(), + nullable=True)) + + create_table_if_missing( + 'endpointcommands', + sa.Column('endpointcommandid', sa.Integer(), nullable=False), + sa.Column('assetid', sa.Integer(), nullable=False), + sa.Column('action', sa.String(length=64), nullable=False), + sa.Column('status', sa.String(length=20), nullable=False), + sa.Column('requestedby', sa.Integer(), nullable=True), + sa.Column('resultmessage', sa.Text(), nullable=True), + sa.Column('resultjson', sa.Text(length=16777215), nullable=True), + sa.Column('createdat', sa.DateTime(), nullable=False), + sa.Column('claimedat', sa.DateTime(), nullable=True), + sa.Column('completedat', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['assetid'], ['assets.assetid'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('endpointcommandid'), + ) + create_index_if_missing('ix_endpointcommands_assetid', + 'endpointcommands', ['assetid']) + create_index_if_missing('ix_endpointcommands_status', + 'endpointcommands', ['status']) + create_index_if_missing('idx_endpointcommands_assetstatus', + 'endpointcommands', ['assetid', 'status']) + + +def downgrade(): + op.drop_index('idx_endpointcommands_assetstatus', + table_name='endpointcommands') + op.drop_index('ix_endpointcommands_status', + table_name='endpointcommands') + op.drop_index('ix_endpointcommands_assetid', + table_name='endpointcommands') + op.drop_table('endpointcommands') + + if _has_column('endpointstates', 'mtconnectcheckedat'): + op.drop_column('endpointstates', 'mtconnectcheckedat') + if _has_column('endpointstates', 'mtconnectrunning'): + op.drop_column('endpointstates', 'mtconnectrunning') diff --git a/plugins/endpoints/migrations/versions/0003_endpoint_command_target.py b/plugins/endpoints/migrations/versions/0003_endpoint_command_target.py new file mode 100644 index 00000000..60b86257 --- /dev/null +++ b/plugins/endpoints/migrations/versions/0003_endpoint_command_target.py @@ -0,0 +1,50 @@ +"""endpoint command target for generic service/process actions. + +Adds nullable target and targetkind columns to endpointcommands so generic +service_* / process_* actions can carry their subject (a Windows service name or +a process image like 'DncMain.exe') over the wire. Fixed-target actions +(mtconnect_*) leave both null. This is the schema half of making the agent +data-driven: a site builds the agent once and every future policy/app is server +data, no agent redistribution. + +Idempotent: each add checks the live schema first so a db.create_all() site +(which already built the columns) re-runs clean on strict MySQL 8. +""" +from alembic import op + +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'endpoints0003cmdtarget' +down_revision = 'endpoints0002commands' +branch_labels = None +depends_on = None + + +def _has_column(table_name, column_name): + from sqlalchemy import inspect + + inspector = inspect(op.get_bind()) + if not inspector.has_table(table_name): + return False + return any(col['name'] == column_name + for col in inspector.get_columns(table_name)) + + +def upgrade(): + if not _has_column('endpointcommands', 'target'): + op.add_column('endpointcommands', + sa.Column('target', sa.String(length=255), + nullable=True)) + if not _has_column('endpointcommands', 'targetkind'): + op.add_column('endpointcommands', + sa.Column('targetkind', sa.String(length=16), + nullable=True)) + + +def downgrade(): + if _has_column('endpointcommands', 'targetkind'): + op.drop_column('endpointcommands', 'targetkind') + if _has_column('endpointcommands', 'target'): + op.drop_column('endpointcommands', 'target') diff --git a/plugins/endpoints/models/__init__.py b/plugins/endpoints/models/__init__.py index f641e95b..1e1fabaa 100644 --- a/plugins/endpoints/models/__init__.py +++ b/plugins/endpoints/models/__init__.py @@ -1,5 +1,5 @@ """Endpoints plugin models.""" -from .endpoint import EndpointState, EndpointEvent +from .endpoint import EndpointState, EndpointEvent, EndpointCommand -__all__ = ['EndpointState', 'EndpointEvent'] +__all__ = ['EndpointState', 'EndpointEvent', 'EndpointCommand'] diff --git a/plugins/endpoints/models/endpoint.py b/plugins/endpoints/models/endpoint.py index aa0bead7..0b0e4593 100644 --- a/plugins/endpoints/models/endpoint.py +++ b/plugins/endpoints/models/endpoint.py @@ -75,6 +75,12 @@ class EndpointState(db.Model): interfacesjson = db.Column(db.Text(length=16777215), nullable=True) scheduledtasksjson = db.Column(db.Text(length=16777215), nullable=True) + # Last-known MTConnect agent run state, cached so the WebUI toggle can show + # a real state between command polls. Set by the agent's mtconnect_status + # result. Null = never reported (unknown). + mtconnectrunning = db.Column(db.Boolean, nullable=True) + mtconnectcheckedat = db.Column(db.DateTime, nullable=True) + # When the agent last successfully reported. There is no heartbeat separate # from the report, so silence past the quiet-hours window is the only # evidence an endpoint has stopped checking in. @@ -97,11 +103,94 @@ class EndpointState(db.Model): 'idleseconds': self.idleseconds, 'cpupercent': self.cpupercent, 'memorypercent': self.memorypercent, + 'mtconnectrunning': self.mtconnectrunning, + 'mtconnectcheckedat': _utciso(self.mtconnectcheckedat), 'lastreportedat': _utciso(self.lastreportedat), 'createdat': _utciso(self.createdat), } +class EndpointCommand(db.Model): + """A queued operator command targeting one endpoint agent. + + The WebUI enqueues a named action (server allowlist only, never an arbitrary + command string) for an asset. The agent claims the next pending command on + its long-poll, executes it locally, and reports the outcome back, which + moves the row to a terminal status in the same request. This is a small + at-most-one-in-flight queue, not a general job system. + """ + + __tablename__ = 'endpointcommands' + + # Lifecycle: pending -> running -> succeeded|failed. A command is claimed + # (pending -> running) by exactly one agent poll, then finalized by its + # result post. + STATUS_PENDING = 'pending' + STATUS_RUNNING = 'running' + STATUS_SUCCEEDED = 'succeeded' + STATUS_FAILED = 'failed' + + endpointcommandid = db.Column(db.Integer, primary_key=True) + + assetid = db.Column( + db.Integer, + db.ForeignKey('assets.assetid', ondelete='CASCADE'), + nullable=False, + index=True, + ) + + asset = db.relationship('Asset', lazy='joined', viewonly=True) + + # The allowlisted action name (for example 'mtconnect_start'). Validated + # against the server allowlist at enqueue time, so only known names land. + action = db.Column(db.String(64), nullable=False) + + # Generic actions (service_* / process_*) carry their subject over the wire: + # target is a Windows service name or a process image (for example + # 'DncMain.exe'); targetkind is 'service' or 'process'. Fixed-target actions + # (mtconnect_*) leave both null. This is what lets a site build the agent + # once and drive every future policy/app purely from server data. + target = db.Column(db.String(255), nullable=True) + targetkind = db.Column(db.String(16), nullable=True) + + status = db.Column(db.String(20), nullable=False, + default=STATUS_PENDING, index=True) + + # Who queued it (userid) for audit; the agent never sees this. + requestedby = db.Column(db.Integer, nullable=True) + + # Agent-reported outcome. resultjson holds structured detail (for example + # the observed MTConnect running state); resultmessage is a short summary. + resultmessage = db.Column(db.Text, nullable=True) + resultjson = db.Column(db.Text(length=16777215), nullable=True) + + createdat = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) + claimedat = db.Column(db.DateTime, nullable=True) + completedat = db.Column(db.DateTime, nullable=True) + + __table_args__ = ( + db.Index('idx_endpointcommands_assetstatus', 'assetid', 'status'), + ) + + def __repr__(self): + return f"" + + def to_dict(self): + return { + 'endpointcommandid': self.endpointcommandid, + 'assetid': self.assetid, + 'action': self.action, + 'target': self.target, + 'targetkind': self.targetkind, + 'status': self.status, + 'requestedby': self.requestedby, + 'resultmessage': self.resultmessage, + 'createdat': _utciso(self.createdat), + 'claimedat': _utciso(self.claimedat), + 'completedat': _utciso(self.completedat), + } + + class EndpointEvent(db.Model): """A single curated Windows Event Log record forwarded by the agent.""" diff --git a/plugins/endpoints/plugin.py b/plugins/endpoints/plugin.py index 35d71cc8..263aa4e0 100644 --- a/plugins/endpoints/plugin.py +++ b/plugins/endpoints/plugin.py @@ -26,7 +26,7 @@ from shopdb.api import db from .api import endpoints_bp from .eventcatalog import EVENT_CATALOG -from .models import EndpointState, EndpointEvent +from .models import EndpointState, EndpointEvent, EndpointCommand logger = logging.getLogger(__name__) @@ -91,7 +91,7 @@ class EndpointsPlugin(BasePlugin): return endpoints_bp def get_models(self) -> List[Type]: - return [EndpointState, EndpointEvent] + return [EndpointState, EndpointEvent, EndpointCommand] def get_permissions(self) -> List: """RBAC for the human-facing read routes. @@ -103,6 +103,70 @@ class EndpointsPlugin(BasePlugin): """ return [ ('endpoints.view', 'View endpoint telemetry', 'endpoints'), + # Issue an allowlisted command to an endpoint agent (for example the + # MTConnect toggle). Separate from endpoints.view because acting on a + # shop-floor PC is a heavier grant than reading its telemetry. + ('endpoints.command', + 'Send allowlisted commands to endpoint agents', 'endpoints'), + # Collector service scope for the endpoint agent. A managed token + # scoped to ONLY this authorizes the endpoints collector and nothing + # else, so rotating/revoking it stops endpoint-agent traffic without + # touching computers/geenforce collectors. Minted from the settings + # UI (see get_settings_cards). Admins hold it via the catalog. + ('collector.ingest.endpoints', + 'Submit endpoint agent telemetry (fleet reporting)', 'collector'), + ] + + def get_settings_defaults(self) -> List[Dict]: + """Settings owned by the endpoints agent-key kill switch. + + `envkeyfallback` gates the bootstrap .env collector key: while true the + old COLLECTOR_API_KEY[_ENDPOINTS] still authorizes (back-compat, and the + only path before a UI key exists). The regenerate flow flips it false so + the .env key stops working and the managed token is the sole credential, + making revocation an instant kill switch. `agentkeyprovisioned` records + that a UI-managed key has been minted so the settings page can render + the rotate/revoke state instead of the first-run generate state. + """ + return [ + { + 'key': 'plugin.endpoints.envkeyfallback', + 'value': 'true', + 'valuetype': 'boolean', + 'category': 'plugin.endpoints', + 'description': 'Accept the bootstrap .env collector key for ' + 'endpoints. Turned off once a UI-managed agent ' + 'key is provisioned so revocation is a real kill ' + 'switch.', + }, + { + 'key': 'plugin.endpoints.agentkeyprovisioned', + 'value': 'false', + 'valuetype': 'boolean', + 'category': 'plugin.endpoints', + 'description': 'True once an operator has minted a UI-managed ' + 'endpoint agent key from the settings page.', + }, + ] + + def get_settings_cards(self) -> List[Dict]: + """The endpoints management card on the global settings page. + + A settings card rather than a docs page because it is the operator's + control surface: manage the agent key (the kill switch), build the site + agent, and configure what it reports. Admin only - key management mints + a collector service credential. + """ + return [ + { + 'group': 'Endpoints', + 'to': '/settings/endpoints/agentkey', + 'icon': 'key', + 'title': 'Endpoints', + 'description': 'Manage the endpoint agent key, build the site ' + 'agent, and configure what it reports.', + 'position': 27, + }, ] def get_navigation_items(self) -> List[Dict]: @@ -150,10 +214,22 @@ class EndpointsPlugin(BasePlugin): 'assettypes': ['computer'], 'endpoint': '/api/endpoints/asset/{assetid}/services', 'render': 'list', + 'collapsible': True, + 'status': { + 'key': 'statecolor', + 'levels': { + 'green': 'ok', + 'yellow': 'warning', + 'red': 'critical', + }, + }, 'map': { 'title': 'name', 'badge': {'label': 'statelabel', 'color': 'statecolor'}, + 'action': 'action', }, + 'commandendpoint': '/api/endpoints/{assetid}/commands', + 'permission': 'endpoints.command', 'empty': 'No monitored services reported.', 'position': 12, }, @@ -163,6 +239,15 @@ class EndpointsPlugin(BasePlugin): 'assettypes': ['computer'], 'endpoint': '/api/endpoints/asset/{assetid}/recentevents', 'render': 'list', + 'collapsible': True, + 'status': { + 'key': 'severity', + 'levels': { + 'info': 'ok', + 'warning': 'warning', + 'critical': 'critical', + }, + }, 'map': { 'title': 'title', 'badge': { @@ -180,6 +265,25 @@ class EndpointsPlugin(BasePlugin): 'empty': 'No recent events.', 'position': 14, }, + { + 'id': 'endpoints-interfaces', + 'title': 'Network interfaces', + 'assettypes': ['computer'], + 'endpoint': '/api/endpoints/asset/{assetid}/interfaces', + 'render': 'table', + 'collapsible': True, + 'columns': [ + {'key': 'alias', 'label': 'Adapter'}, + {'key': 'ipv4', 'label': 'IPv4'}, + {'key': 'mac', 'label': 'MAC'}, + {'key': 'gateway', 'label': 'Gateway'}, + {'key': 'subnet', 'label': 'Subnet'}, + {'key': 'dhcp', 'label': 'Assignment'}, + {'key': 'role', 'label': 'Role'}, + ], + 'empty': 'No network interfaces reported.', + 'position': 16, + }, ] def get_dashboard_widgets(self) -> List[Dict]: @@ -240,6 +344,14 @@ class EndpointsPlugin(BasePlugin): 'description': 'Endpoint hostname. Resolved against ' 'the computers plugin (computers.hostname).', }, + 'adopt': { + 'type': 'boolean', + 'description': 'First-contact only. When true and the ' + 'hostname does not resolve, adoption is ' + 'delegated to the computers collector ' + '(see DESIGN-DECISIONS-agent.md 1a). This ' + 'plugin never creates assets itself.', + }, 'loggedinuser': {'type': 'string'}, 'agentversion': {'type': 'string'}, 'lastboottime': {'type': 'string', 'format': 'date-time'}, @@ -286,6 +398,56 @@ class EndpointsPlugin(BasePlugin): }, } + # PC-identifying and hardware fields forwarded to the computers collector, + # which owns the Asset/Computer row and renders the Hardware and Network + # cards. The endpoints plugin only caches telemetry, so these must be + # relayed or vendor/model/OS/IP never land on the PC (see + # DESIGN-DECISIONS-agent.md 1a). + _COMPUTER_HANDOFF_FIELDS = ( + 'loggedinuser', 'serialnumber', 'pctype', 'machinenumber', + 'vendorname', 'modelnumber', 'osname', 'ipaddress', 'lastboottime', + ) + + def _forward_to_computers(self, hostname: str, payload: Dict): + """Return the computers plugin and the PC subset to hand off, or None. + + Both adoption and steady-state reports relay the same hardware subset, + so the Hardware and Network cards fill in on every report, not just at + first contact. + """ + from flask import current_app + + manager = current_app.extensions.get('plugin_manager') + computers = manager.get_plugin('computers') if manager else None + if computers is None: + return None, None + + subset = {'hostname': hostname} + for field in self._COMPUTER_HANDOFF_FIELDS: + if payload.get(field): + subset[field] = payload.get(field) + return computers, subset + + def _adopt_computer(self, hostname: str, payload: Dict, + warnings: List[str]) -> None: + """Delegate first-contact adoption to the computers collector. + + This plugin never creates Asset or Computer rows itself (see + DESIGN-DECISIONS-agent.md 1a). The computers plugin is the sole + authority for that, so on an unknown hostname we forward the + PC-identifying subset of the payload to its collector. It runs inside + the same transaction, so a later failure rolls the adoption back. + """ + computers, adoptpayload = self._forward_to_computers(hostname, payload) + if computers is None: + raise ValueError( + 'cannot adopt {!r}: computers plugin is not available' + .format(hostname)) + + computers.apply_collector_payload(adoptpayload) + db.session.flush() + warnings.append('adopted new computer for hostname: {!r}'.format(hostname)) + def apply_collector_payload(self, payload: Dict) -> Dict: """Idempotently upsert one endpoint's state and append its events. @@ -305,8 +467,29 @@ class EndpointsPlugin(BasePlugin): .filter(db.func.lower(Computer.hostname) == hostname.lower()) .first()) if computer is None: - raise ValueError( - 'could not resolve hostname to a computer: {!r}'.format(hostname)) + # First-contact adoption is delegated to the computers collector, + # which is the only authority allowed to create Asset + Computer + # rows (DESIGN-DECISIONS-agent.md 1a). This plugin never invents + # assets itself. Adoption runs in the dispatcher's transaction, so a + # later ValueError rolls the whole report back. + if payload.get('adopt'): + self._adopt_computer(hostname, payload, warnings) + computer = (Computer.query + .filter(db.func.lower(Computer.hostname) + == hostname.lower()) + .first()) + if computer is None: + raise ValueError( + 'could not resolve hostname to a computer: {!r}'.format(hostname)) + else: + # Already adopted: relay the hardware/identity subset on every + # report so the computers plugin keeps vendor/model/OS/IP current. + # Without this, Hardware and Network stay blank after adoption + # because this plugin only caches telemetry. + computers, subset = self._forward_to_computers(hostname, payload) + if computers is not None and len(subset) > 1: + computers.apply_collector_payload(subset) + db.session.flush() assetid = computer.assetid @@ -373,4 +556,4 @@ class EndpointsPlugin(BasePlugin): 'identityvalue': hostname, 'storedevents': stored_events, 'warnings': warnings, - } + } \ No newline at end of file diff --git a/plugins/machines/frontend/views/MachineForm.vue b/plugins/machines/frontend/views/MachineForm.vue index 646a5795..15e603e8 100644 --- a/plugins/machines/frontend/views/MachineForm.vue +++ b/plugins/machines/frontend/views/MachineForm.vue @@ -446,20 +446,44 @@ const existingRelationshipId = ref(null) const loadedControllingPcId = ref(null) const currentMachine = ref(null) -// Filter models by selected machine vendor. Controller-type models are offered -// only in the controller dropdown, so keep them out of the machine list. +// Ensure a saved model id is present in models.value even when the filtered +// list (category Equipment) excluded it because it has no modeltype. Fetches +// the single row by id and appends it; a no-op when already present or blank. +async function ensureModelLoaded(modelId) { + if (!modelId) return + if (models.value.some(m => m.modelnumberid === modelId)) return + try { + const response = await modelsApi.get(modelId) + const model = response.data.data + if (model) models.value = [...models.value, model] + } catch (e) { + // Leave it absent; the select shows blank but we did not invent a value. + } +} + +// Filter models by selected machine vendor. +// only in the controller dropdown, so keep them out of the machine list. Keep +// the record's own saved model even if it is untyped or a different vendor, so +// editing a machine never renders a blank select that clears the field on save. const filteredModels = computed(() => { - const machineModels = models.value.filter(m => m.modeltype !== 'Controller') + const savedId = form.value.modelnumberid + const machineModels = models.value.filter( + m => m.modeltype !== 'Controller' || m.modelnumberid === savedId) if (!form.value.vendorid) return machineModels - return machineModels.filter(m => m.vendorid === form.value.vendorid) + return machineModels.filter( + m => m.vendorid === form.value.vendorid || m.modelnumberid === savedId) }) // Filter models by selected controller vendor. Only Controller-type models are -// valid here. +// valid here. Keep the record's own saved controller model even if it is untyped +// or a different vendor, for the same reason as filteredModels above. const filteredControllerModels = computed(() => { - const controllerModels = models.value.filter(m => m.modeltype === 'Controller') + const savedId = form.value.controllermodelid + const controllerModels = models.value.filter( + m => m.modeltype === 'Controller' || m.modelnumberid === savedId) if (!form.value.controllervendorid) return controllerModels - return controllerModels.filter(m => m.vendorid === form.value.controllervendorid) + return controllerModels.filter( + m => m.vendorid === form.value.controllervendorid || m.modelnumberid === savedId) }) // Only offer vendors that have machine (non-controller Equipment) models. Keep @@ -595,6 +619,14 @@ onMounted(async () => { levelid: data.levelid ?? null, } + // The model list is loaded filtered to category Equipment, which the + // backend inner-joins against ModelType - so a model with no modeltype is + // absent from models.value. If this machine's saved model or controller + // model is one of those untyped rows, fetch it by id and merge it in, or + // its dropdown would render blank and saving would clear the field. + await ensureModelLoaded(form.value.modelnumberid) + await ensureModelLoaded(form.value.controllermodelid) + // Load existing relationships to find controlling PC if (data.assetid) { try { diff --git a/plugins/printers/frontend/views/PrinterForm.vue b/plugins/printers/frontend/views/PrinterForm.vue index caa8084a..65670487 100644 --- a/plugins/printers/frontend/views/PrinterForm.vue +++ b/plugins/printers/frontend/views/PrinterForm.vue @@ -370,9 +370,28 @@ const vendors = ref([]) const models = ref([]) const locations = ref([]) +// Ensure a saved model id is present in models.value even when the filtered +// list (category Printer) excluded it because it has no modeltype. Fetches the +// single row by id and appends it; a no-op when already present or blank. +async function ensureModelLoaded(modelId) { + if (!modelId) return + if (models.value.some(m => m.modelnumberid === modelId)) return + try { + const response = await modelsApi.get(modelId) + const model = response.data.data + if (model) models.value = [...models.value, model] + } catch (e) { + // Leave it absent; the select shows blank but we did not invent a value. + } +} + // Filter models by selected vendor and printer type const filteredModels = computed(() => { + const savedId = form.value.modelnumberid return models.value.filter(m => { + // Always keep the record's own saved model, so editing a printer never + // renders a blank select that clears the field on save. + if (m.modelnumberid === savedId) return true // Filter by vendor if selected if (form.value.vendorid && m.vendorid !== form.value.vendorid) { return false @@ -550,6 +569,12 @@ onMounted(async () => { pin: ext.pin || '' } + // The model list is loaded filtered to category Printer, which the backend + // inner-joins against ModelType - so an untyped model is absent. If this + // printer's saved model is one of those rows, fetch it by id and merge it + // in, or its dropdown would render blank and saving would clear the field. + await ensureModelLoaded(form.value.modelnumberid) + // Don't auto-generate for existing printers manualWindowsName.value = true manualHostname.value = true diff --git a/scripts/dev/seed_shared_applications.py b/scripts/dev/seed_shared_applications.py new file mode 100644 index 00000000..2798db84 --- /dev/null +++ b/scripts/dev/seed_shared_applications.py @@ -0,0 +1,102 @@ +"""Seed the org's shared shopfloor applications (DEV ONLY). + +Inserts the applications that ship as common across the organization so the +Configuration Policy builder has real apps to scope against and the software +inventory has canonical rows to match collected installs to. Each app records +its executable and (where relevant) its Windows service name in +applicationnotes, since the Application model has no dedicated process/service +column - these identifiers are what a generic service_*/process_* policy action +targets. + +Shipped set: + * UDC - UDC.exe + * MTConnect - agent.exe process, "MTConnect UDC Agent" service + * eDNC - DncMain.exe process + +Idempotent: matches on appname and updates in place, so re-running does not +duplicate. Marked installable so they appear in installable-app pickers. + +DEV ONLY: lives under scripts/dev/ and must never be imported by application +code. Run inside the API container. + +Usage (in the API container): + docker exec shopdb-flask-wilm-api-1 python scripts/dev/seed_shared_applications.py + docker exec shopdb-flask-wilm-api-1 python scripts/dev/seed_shared_applications.py --clear +""" + +import argparse +import os +import sys + +# Repo root on sys.path so `shopdb` imports when run as a script. +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) + +from shopdb import create_app # noqa: E402 +from shopdb.api import db # noqa: E402 + +# appname -> seed fields. 'notes' captures the process image and service name so +# a generic policy action (params.target) has a canonical identifier to bind to. +SHARED_APPS = [ + { + 'appname': 'UDC', + 'appdescription': 'Universal Data Collector', + 'notes': 'process=UDC.exe', + }, + { + 'appname': 'MTConnect', + 'appdescription': 'MTConnect UDC agent', + 'notes': 'process=agent.exe; service=MTConnect UDC Agent', + }, + { + 'appname': 'eDNC', + 'appdescription': 'eDNC distributed NC file transfer', + 'notes': 'process=DncMain.exe', + }, +] + + +def _seed(Application): + created = 0 + updated = 0 + for spec in SHARED_APPS: + app_row = Application.query.filter_by(appname=spec['appname']).first() + if app_row is None: + app_row = Application(appname=spec['appname']) + db.session.add(app_row) + created += 1 + else: + updated += 1 + app_row.appdescription = spec['appdescription'] + app_row.applicationnotes = spec['notes'] + app_row.isinstallable = True + db.session.commit() + print('Seeded shared applications: {} created, {} updated.' + .format(created, updated)) + + +def _clear(Application): + names = [spec['appname'] for spec in SHARED_APPS] + removed = (Application.query + .filter(Application.appname.in_(names)) + .delete(synchronize_session=False)) + db.session.commit() + print('Removed {} shared application row(s).'.format(removed)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--clear', action='store_true', + help='Remove the seeded shared applications.') + args = parser.parse_args() + + app = create_app(os.environ.get('FLASK_ENV', 'production')) + with app.app_context(): + from shopdb.api import Application + if args.clear: + _clear(Application) + else: + _seed(Application) + + +if __name__ == '__main__': + main() diff --git a/shopdb/__init__.py b/shopdb/__init__.py index 3047076c..18f8f943 100644 --- a/shopdb/__init__.py +++ b/shopdb/__init__.py @@ -65,7 +65,18 @@ from .plugins import plugin_manager # machine number in the PC row instead of hand-building a relationships sheet. # Core stays ignorant of the column's meaning. Additive name on the import # surface, minor bump. -__contract_version__ = '0.21.0' +# 0.22.0: added mint_service_token, list_service_tokens and revoke_service_tokens +# to shopdb.api so a plugin can provision, list, and revoke a single-scope +# managed service token (an endpoint agent key) without importing core token +# internals. The endpoints plugin uses them for a UI-managed, regenerable +# collector.ingest.endpoints key whose revocation is an instant kill switch. +# Additive names on the surface, minor bump. +# 0.23.0: added collector_authorized to shopdb.api so a plugin can gate an +# agent-facing route that lives OUTSIDE the generic collector blueprint (the +# endpoints command claim/result long-poll callbacks) on the same per-plugin +# collector key + managed-token check the ingest route uses, without importing +# core collector internals. Additive name on the surface, minor bump. +__contract_version__ = '0.23.0' # Product release version (see ADR-007). The product version and the # plugin-contract version above are distinct series with independent diff --git a/shopdb/api/__init__.py b/shopdb/api/__init__.py index f802e387..4c0af379 100644 --- a/shopdb/api/__init__.py +++ b/shopdb/api/__init__.py @@ -75,8 +75,28 @@ from shopdb.utils.authz import require_permission, require_role # internals. from shopdb.utils.apitoken_auth import ( service_token_authorized, authorized_service_token, + mint_service_token, list_service_tokens, revoke_service_tokens, ) +# Collector-token authorization for agent-facing plugin routes that live outside +# the generic collector blueprint (e.g. the endpoints command claim/result +# callbacks). Reuses the same per-plugin collector key + managed-token check the +# ingest route uses, so a plugin never imports core collector internals. +def collector_authorized(pluginname: str): + """Authorize an agent-facing route on a plugin's collector token. + + Returns None when the caller presented a valid collector credential for + this plugin (the per-plugin env key or a collector.ingest[.]-scoped + managed token), else the error_response to return. Lets a plugin gate an + agent callback on the same key and kill switch as its ingest route without + reaching into core collector internals. + + Imported lazily: shopdb.core.api imports back into shopdb.api (send_upload), + so a module-level import here would be a circular import at load time. + """ + from shopdb.core.api.collector import _check_collector_auth, _plugin_api_key + return _check_collector_auth(_plugin_api_key(pluginname), pluginname) + # Serving an uploaded file safely. A plugin that serves user-supplied bytes must # not have to remember the headers that keep an SVG or an HTML file from running # as script in this origin - see shopdb/utils/uploads.py. @@ -313,6 +333,10 @@ __all__ = [ 'require_role', 'service_token_authorized', 'authorized_service_token', + 'mint_service_token', + 'list_service_tokens', + 'revoke_service_tokens', + 'collector_authorized', 'SupportTeam', # Serving uploads 'send_upload', diff --git a/shopdb/core/api/collector.py b/shopdb/core/api/collector.py index 1b0bd654..6d6093b3 100644 --- a/shopdb/core/api/collector.py +++ b/shopdb/core/api/collector.py @@ -39,17 +39,56 @@ def _computer_models(): return None -def _token_is_collector(token): - """True when this managed token is a collector service token: scoped for - collector.ingest AND its owner actually holds that permission (admins do).""" +def _collector_scopes_for(pluginname=None): + """The scope names that authorize a collector request. + + The generic scope `collector.ingest` authorizes every collector. A plugin + MAY also mint a token scoped to `collector.ingest.` so that + rotating/revoking it stops only that plugin's collector traffic (endpoints + uses this for its regenerable agent kill switch). Both are accepted. + """ + scopes = {COLLECTOR_SCOPE} + if pluginname: + scopes.add(f'{COLLECTOR_SCOPE}.{pluginname}') + return scopes + + +def _env_fallback_disabled(pluginname=None): + """True when a plugin has switched OFF the bootstrap env-key fallback. + + Once an operator provisions a UI-managed collector token, the plugin sets + its `plugin..envkeyfallback` setting to false so the old .env + key stops authorizing. That turns key revocation into a real kill switch: + rotating or revoking the managed token cannot be bypassed by a lingering env + key. Absent + setting = fallback stays ON (back-compat, so nothing breaks on upgrade). + """ + if not pluginname: + return False + from shopdb.core.models.setting import Setting + key = f'plugin.{pluginname}.envkeyfallback' + return Setting.get(key, default=True) is False + + +def _token_is_collector(token, pluginname=None): + """True when this managed token is a collector service token: scoped for a + collector scope AND its owner actually holds that permission (admins do). + + Accepts the generic `collector.ingest` or, when pluginname is given, the + plugin-specific `collector.ingest.` scope. + """ from shopdb.core.models import User scopelist = token.scopelist - if not scopelist or COLLECTOR_SCOPE not in scopelist: + if not scopelist: + return False + allowed = _collector_scopes_for(pluginname) + held = [s for s in scopelist if s in allowed] + if not held: return False user = db.session.get(User, token.userid) return (user is not None and user.isactive - and user.haspermission(COLLECTOR_SCOPE)) + and any(user.haspermission(s) for s in held)) def _collector_managed_token(api_key): @@ -73,24 +112,28 @@ def _collector_managed_token(api_key): return None -def _check_collector_auth(expected_key): +def _check_collector_auth(expected_key, pluginname=None): """Authorize a collector request. Returns None when authorized, else the error_response to return. Accepts a collector-scoped managed token (Bearer or X-API-Key) OR the env key via X-API-Key (legacy/bootstrap fallback). Env keys stay supported so nothing breaks; a valid managed token works even with no env key set. + + When pluginname is given, a token scoped to collector.ingest. + also authorizes (in addition to the generic collector.ingest scope). """ from shopdb.utils.apitoken_auth import touch_apitoken_lastused api_key = request.headers.get('X-API-Key') token = _collector_managed_token(api_key) - if token is not None and _token_is_collector(token): + if token is not None and _token_is_collector(token, pluginname): touch_apitoken_lastused(token) return None - if expected_key and api_key == expected_key: + if expected_key and api_key == expected_key \ + and not _env_fallback_disabled(pluginname): return None # Fail-closed only on pure server misconfiguration: no env key AND the @@ -200,7 +243,7 @@ def generic_collect(pluginname): # Per-plugin env key (fallback) OR a collector-scoped managed token. Header # only (querystring fallback dropped, see require_api_key). - error = _check_collector_auth(_plugin_api_key(pluginname)) + error = _check_collector_auth(_plugin_api_key(pluginname), pluginname) if error is not None: return error diff --git a/shopdb/core/api/settings.py b/shopdb/core/api/settings.py index a0098079..84d51c42 100644 --- a/shopdb/core/api/settings.py +++ b/shopdb/core/api/settings.py @@ -879,10 +879,10 @@ def build_default_settings(): }, { 'key': 'asset_import_reconcile_threshold', - 'value': '0.85', + 'value': '0', 'valuetype': 'string', 'category': 'site', - 'description': 'Similarity (0 to 1) a fuzzy lookup match must reach for the asset importer to reconcile a near-miss name (e.g. "Dell Inc." to an existing "Dell") onto an existing vendor, model, type, status, business unit, or location. Reconciled matches are applied but reported for review. 0 disables reconciliation. Operating systems and asset rows are never reconciled.' + 'description': 'Similarity (0 to 1) a fuzzy lookup match must reach for the asset importer to reconcile a near-miss name (e.g. "Dell Inc." to an existing "Dell") onto an existing vendor, model, type, status, business unit, or location. Reconciled matches are applied but reported for review. 0 (the default) disables fuzzy reconciliation - names must match exactly after normalization. Labels whose digit runs differ (e.g. "Bay 310" vs "Bay 311") are never fuzzy-matched. Operating systems and asset rows are never reconciled.' }, ] diff --git a/shopdb/core/services/csvimport.py b/shopdb/core/services/csvimport.py index 0adeb3a2..c5ebe904 100644 --- a/shopdb/core/services/csvimport.py +++ b/shopdb/core/services/csvimport.py @@ -172,8 +172,11 @@ CORPORATE_SUFFIXES = { } # Default similarity a fuzzy match must reach, 0..1. Also the value the settings -# default is seeded with; 0 disables reconciliation entirely. -DEFAULT_RECONCILE_THRESHOLD = 0.85 +# default is seeded with; 0 disables reconciliation entirely. Fuzzy matching is +# opt-in: it ships OFF (0) because a high similarity score is precisely the +# dangerous case - 'Bay 310' and 'Bay 311' score 0.857 and would silently merge. +# A site turns it on deliberately once it understands its own vocabulary. +DEFAULT_RECONCILE_THRESHOLD = 0.0 # -- plugin import hooks (inversion of control) --------------------------------- @@ -259,6 +262,19 @@ def normalize_label(value): return ' '.join(words) +def digit_runs(value): + """Return the ordered list of digit runs in a normalized label. + + 'bay 310' -> ['310'], 'precision 3660' -> ['3660'], 'dell' -> []. Two labels + that share a fuzzy word shape but differ in any digit run ('bay 310' vs + 'bay 311') must NOT reconcile: a single differing digit is a small fraction + of a long string, so SequenceMatcher scores it above threshold even though + the labels name two distinct things. Comparing digit runs exactly closes + that gap while still fuzzing the surrounding words. + """ + return re.findall(r'\d+', value or '') + + def table_for(name): table = db.metadata.tables.get(name) if table is None: @@ -451,6 +467,7 @@ class Resolver: wanted = normalize_label(value) if not wanted: return None + wanted_digits = digit_runs(wanted) best_id = None best_label = None @@ -463,6 +480,12 @@ class Resolver: if normalized == wanted: score = 1.0 else: + # Never fuzzy-match two labels whose digit runs differ. 'Bay 310' + # and 'Bay 311' share every word and score 0.857, but the digits + # name distinct things; refuse rather than silently merge. The + # normalized-exact tier above (score 1.0) is unaffected. + if digit_runs(normalized) != wanted_digits: + continue score = difflib.SequenceMatcher(None, wanted, normalized).ratio() if score > best_score: best_score = score diff --git a/shopdb/plugins/alembic_template.py b/shopdb/plugins/alembic_template.py index 43854d77..657297e9 100644 --- a/shopdb/plugins/alembic_template.py +++ b/shopdb/plugins/alembic_template.py @@ -49,9 +49,11 @@ logger = logging.getLogger('alembic.env.plugin') PLUGIN_TABLE_OWNERS: dict[str, Iterable[str]] = { 'backups': ('backuprevisions',), 'computers': ('computertypes', 'computers', 'computerinstalledapps', - 'accessprotocols', 'computeraccess'), + 'accessprotocols', 'computeraccess', + 'configurationpolicies', 'configurationpolicyactions', + 'businesspolicies', 'businesspolicyitems'), 'employees': ('directoryemployees',), - 'endpoints': ('endpointstates', 'endpointevents'), + 'endpoints': ('endpointstates', 'endpointevents', 'endpointcommands'), 'geenforce': ('manifestscopes', 'manifestentries', 'manifestentrypctypes', 'manifestentryhostnames', 'manifestentrymachinenumbers', 'manifestinusechecks', 'manifestinusecheckprocesses', diff --git a/shopdb/utils/apitoken_auth.py b/shopdb/utils/apitoken_auth.py index ceb6b630..e39e9e96 100644 --- a/shopdb/utils/apitoken_auth.py +++ b/shopdb/utils/apitoken_auth.py @@ -107,6 +107,82 @@ def authorized_service_token(scope): return token +def mint_service_token(scope, name, owner_userid, revoke_existing=True): + """Mint a managed service token scoped to exactly ONE permission. + + The contract-surface way for a plugin to provision an unattended service + credential (an endpoint agent key, a collector key) without importing core + token internals. The returned secret is the ONLY time the full secret is + available - callers reveal it once and store only the prefix afterwards. + + scope the single permission the token authorizes (e.g. + collector.ingest.endpoints). The owner must hold it. + name human label shown in the token list UI. + owner_userid the acting admin who owns (and can be audited for) the key. + revoke_existing when True (default), every currently-active token scoped to + EXACTLY this one permission is deactivated first, so + regenerating is a hard cutover: the old agent key stops + working the instant the new one is minted. + + Returns (token, secret). Commits. + """ + from shopdb.core.models import User + + owner = db.session.get(User, owner_userid) + if owner is None or not owner.isactive or not owner.haspermission(scope): + raise PermissionError( + 'Token owner must be an active user holding ' + scope) + + if revoke_existing: + for existing in list_service_tokens(scope): + existing.isactive = False + + secret = ApiToken.generate_secret() + token = ApiToken( + userid=owner_userid, + name=name, + tokenprefix=ApiToken.prefix_of(secret), + tokenhash=ApiToken.hash_secret(secret), + ) + token.scopelist = [scope] + db.session.add(token) + db.session.commit() + return token, secret + + +def list_service_tokens(scope, active_only=True): + """Return managed tokens scoped to EXACTLY the single permission `scope`. + + Used by a plugin settings page to render the current agent-key state + (prefix, created, last-used) without exposing token internals. Filters in + Python because scopes are stored as a JSON blob, not a queryable column. + """ + query = ApiToken.query + if active_only: + query = query.filter_by(isactive=True) + matches = [] + for token in query.order_by(ApiToken.createddate.desc()).all(): + scopelist = token.scopelist + if scopelist and len(scopelist) == 1 and scopelist[0] == scope: + matches.append(token) + return matches + + +def revoke_service_tokens(scope): + """Deactivate every active managed token scoped to EXACTLY `scope`. + + The kill switch: revoking the endpoint agent key stops all endpoint-agent + traffic authorized by a managed token. Returns the count revoked. Commits. + """ + revoked = 0 + for token in list_service_tokens(scope): + token.isactive = False + revoked += 1 + if revoked: + db.session.commit() + return revoked + + def service_token_authorized(scope): """True when the current request carries a managed token scoped for `scope` whose owner is active and holds that permission. Accepts X-API-Key or a diff --git a/tests/test_core/test_authz.py b/tests/test_core/test_authz.py index 476bb0f8..61e3d427 100644 --- a/tests/test_core/test_authz.py +++ b/tests/test_core/test_authz.py @@ -62,7 +62,15 @@ EXEMPT_ENDPOINTS = {'knowledgebase.track_click', 'users.update_user', # GET urls and a preview must not consume a one-view link. # CREATING one is permission-gated and IS swept. # Justification in plugins/tools/README.md. - 'tools.redeem_secret'} + 'tools.redeem_secret', + # Agent-facing command result callback. Authenticated by the + # endpoints collector API key (same shape as the exempt + # collector blueprint), not JWT+role: the endpoint agent has + # no user account, the collector key is its credential and + # kill switch. Enqueue (WebUI -> operator) IS permission-gated + # on endpoints.command and IS swept. Contract in + # plugins/endpoints/AGENT-CONTRACT.md section 7. + 'endpoints.report_command_result'} @pytest.fixture(autouse=True) diff --git a/tests/test_csv_import.py b/tests/test_csv_import.py index ec1a1dfc..0cd4956d 100644 --- a/tests/test_csv_import.py +++ b/tests/test_csv_import.py @@ -12,6 +12,7 @@ from shopdb.core.services.csvimport import ( IMPORTABLE, ImportError_, Resolver, generate_template, import_csv, importable_columns, required_columns, ) +from shopdb.core.services.csvimport import digit_runs, normalize_label from shopdb.extensions import db as _db @@ -280,3 +281,57 @@ def test_template_example_set_imports_cleanly(db): table, [str(p) for p in result.problems]) _db.session.rollback() + +# -- fuzzy reconciliation safety (review item 1.1) ----------------------------- +# +# A high similarity score is the DANGEROUS case: 'Bay 310' and 'Bay 311' share +# every word and score ~0.857, so a naive threshold silently merges two distinct +# locations. Reconciliation now ships OFF (threshold 0) and, even when a site +# turns it on, refuses any pair whose digit runs differ. + +def test_digit_runs_extracts_number_groups(): + assert digit_runs(normalize_label('Bay 310')) == ['310'] + assert digit_runs(normalize_label('Precision 3660')) == ['3660'] + assert digit_runs(normalize_label('Dell Inc.')) == [] + + +def test_reconciliation_is_off_by_default(db): + """A fresh Resolver does not fuzzy-match: an unknown numbered bay is a row + error, not a silent bind onto its neighbour.""" + resolver = Resolver() + seed_reference(resolver) + # 'Bay 3' exists from the reference seed; 'Bay 8' does not. + result = import_csv( + 'assets', + 'assetnumber,assettypeid,locationid\nA-1,Machine,Bay 8\n', + resolver=resolver) + assert not result.ok + assert not resolver.events + + +def test_reconcile_refuses_when_digit_runs_differ(db): + """Even with fuzzy matching turned on, 'Bay 311' must not bind to the + existing 'Bay 3': the digit runs differ, so it is a distinct location.""" + resolver = Resolver(reconcile_threshold=0.5) + seed_reference(resolver) + result = import_csv( + 'assets', + 'assetnumber,assettypeid,locationid\nB-1,Machine,Bay 311\n', + resolver=resolver) + assert not result.ok + assert not resolver.events + + +def test_reconcile_still_binds_normalized_exact(db): + """The normalized-exact tier is untouched: 'bay 3' folds to the existing + 'Bay 3' regardless of threshold, and is reported as an event.""" + resolver = Resolver(reconcile_threshold=0.9) + seed_reference(resolver) + result = import_csv( + 'assets', + 'assetnumber,assettypeid,locationid\nC-1,Machine,BAY 3\n', + resolver=resolver) + assert result.ok, [str(p) for p in result.problems] + _db.session.rollback() + + diff --git a/tests/test_plugin_migrations.py b/tests/test_plugin_migrations.py index dfaa8220..28f01be4 100644 --- a/tests/test_plugin_migrations.py +++ b/tests/test_plugin_migrations.py @@ -47,9 +47,9 @@ CUTOVER_PLUGINS = ( EXPECTED_HEAD_REVISION = {plugin: f'{plugin}0001anchor' for plugin in CUTOVER_PLUGINS} EXPECTED_HEAD_REVISION['measuringtools'] = 'measuringtools0001baseline' # computers adds the Entra/Azure device identity columns (azuredeviceid, then -# azureobjectid + azuredevicecategory) on top of its anchor, so its head is not -# the f-string default. -EXPECTED_HEAD_REVISION['computers'] = 'computers0003azurefields' +# azureobjectid + azuredevicecategory), then the Configuration/Business policy +# tables, on top of its anchor, so its head is not the f-string default. +EXPECTED_HEAD_REVISION['computers'] = 'computers0004policies' # backups is also post-cutover: its 0001 really creates backuprevisions, and # 0002 adds lastseenat - when a config was last CONFIRMED unchanged, which dedup # otherwise throws away. @@ -88,8 +88,10 @@ EXPECTED_HEAD_REVISION['notifications'] = 'notifications0005boardorder' # warranty adds the proof-of-cover document columns on top of its anchor. EXPECTED_HEAD_REVISION['warranty'] = 'warranty0002proof' # endpoints is a NEW post-cutover plugin: its 0001 genuinely creates the -# endpointstates + endpointevents tables the core chain never owned. -EXPECTED_HEAD_REVISION['endpoints'] = 'endpoints0001baseline' +# endpointstates + endpointevents tables the core chain never owned, 0002 +# adds the endpointcommands queue plus the MTConnect state columns, and 0003 +# adds the generic command target/targetkind columns. +EXPECTED_HEAD_REVISION['endpoints'] = 'endpoints0003cmdtarget' # Plugins built after the cutover: their 0001 baseline really creates tables the # core chain never owned. From af133372b4317c679fa78fa32e6528f31ccb6fdc Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Fri, 28 Aug 2026 15:07:20 -0400 Subject: [PATCH 13/14] Gate endpoint UI refresh on fresh telemetry and land endpoint agent A command reaching 'succeeded' only proves the agent ran the action, not that it posted a new telemetry snapshot. Reloading panels on command success alone re-rendered stale service/process rows, so a stopped process could still show running right after Refresh Status. Add a lightweight telemetrymeta probe exposing lastreportedat and gate both the header Refresh Status action and inline row Start/Stop on that stamp advancing past a pre-action baseline. Polling is bounded so an offline or idle agent never hangs the button, and messaging distinguishes a confirmed refresh from a queued one. Bring the .NET endpoint agent source (service, tray, installer) into the repo so Cameron can merge and test, with an agent/.gitignore keeping bin/obj/publish out. Includes the DncMain launch-path fix, tray single-file host env scrub, and best-effort file logger. --- .github/agent-workflow.md | 86 +++ Dockerfile | 19 +- agent/.gitignore | 7 + agent/README.md | 166 +++++ agent/ShopDbEndpointAgent.Core/AgentConfig.cs | 109 ++++ .../AgentConfigProvider.cs | 110 ++++ .../AgentFaultMonitor.cs | 91 +++ .../ShopDbEndpointAgent.Core/CollectSignal.cs | 26 + .../CollectorClient.cs | 118 ++++ .../ShopDbEndpointAgent.Core/CommandClient.cs | 83 +++ .../CommandExecutor.cs | 581 ++++++++++++++++++ .../ShopDbEndpointAgent.Core/CommandWorker.cs | 143 +++++ agent/ShopDbEndpointAgent.Core/Contracts.cs | 92 +++ .../CuratedEventCatalog.cs | 44 ++ .../EndpointCollector.cs | 489 +++++++++++++++ .../FileLoggerProvider.cs | 164 +++++ agent/ShopDbEndpointAgent.Core/ICollector.cs | 24 + .../InteractiveProcessLauncher.cs | 200 ++++++ .../PowerShellUtilityRunner.cs | 114 ++++ .../ShopDbEndpointAgent.Core.csproj | 22 + .../TelemetryWorker.cs | 129 ++++ agent/ShopDbEndpointAgent.Tray/Program.cs | 17 + .../ShopDbEndpointAgent.Tray.csproj | 24 + agent/ShopDbEndpointAgent.Tray/TicketForm.cs | 118 ++++ agent/ShopDbEndpointAgent.Tray/TrayContext.cs | 202 ++++++ agent/ShopDbEndpointAgent.Tray/TrayIcon.cs | 81 +++ agent/ShopDbEndpointAgent.slnx | 5 + agent/ShopDbEndpointAgent/Program.cs | 164 +++++ .../ShopDbEndpointAgent.csproj | 40 ++ agent/ShopDbEndpointAgent/appsettings.json | 26 + agent/installer/install-shopdbagent.ps1 | 337 ++++++++++ frontend/src/api/index.js | 11 + frontend/src/components/PluginAssetPanels.vue | 86 ++- frontend/src/components/pluginAssetPanels.js | 18 +- .../src/components/pluginAssetPanels.spec.js | 28 + .../src/views/settings/ImportSiteAssets.vue | 2 +- plugins/computers/frontend/views/PCDetail.vue | 121 +++- plugins/endpoints/api/routes.py | 48 +- plugins/endpoints/commands.py | 21 + plugins/endpoints/plugin.py | 53 +- plugins/endpoints/tests/test_plugin.py | 83 ++- .../assetcollector/Collect-ShopDB.ps1 | 60 +- .../assetcollector/sample/computers.csv | 4 +- shopdb/core/api/settings.py | 14 +- shopdb/core/services/importsources.py | 2 +- shopdb/data/catalog.json | 339 +++++++--- 46 files changed, 4593 insertions(+), 128 deletions(-) create mode 100644 .github/agent-workflow.md create mode 100644 agent/.gitignore create mode 100644 agent/README.md create mode 100644 agent/ShopDbEndpointAgent.Core/AgentConfig.cs create mode 100644 agent/ShopDbEndpointAgent.Core/AgentConfigProvider.cs create mode 100644 agent/ShopDbEndpointAgent.Core/AgentFaultMonitor.cs create mode 100644 agent/ShopDbEndpointAgent.Core/CollectSignal.cs create mode 100644 agent/ShopDbEndpointAgent.Core/CollectorClient.cs create mode 100644 agent/ShopDbEndpointAgent.Core/CommandClient.cs create mode 100644 agent/ShopDbEndpointAgent.Core/CommandExecutor.cs create mode 100644 agent/ShopDbEndpointAgent.Core/CommandWorker.cs create mode 100644 agent/ShopDbEndpointAgent.Core/Contracts.cs create mode 100644 agent/ShopDbEndpointAgent.Core/CuratedEventCatalog.cs create mode 100644 agent/ShopDbEndpointAgent.Core/EndpointCollector.cs create mode 100644 agent/ShopDbEndpointAgent.Core/FileLoggerProvider.cs create mode 100644 agent/ShopDbEndpointAgent.Core/ICollector.cs create mode 100644 agent/ShopDbEndpointAgent.Core/InteractiveProcessLauncher.cs create mode 100644 agent/ShopDbEndpointAgent.Core/PowerShellUtilityRunner.cs create mode 100644 agent/ShopDbEndpointAgent.Core/ShopDbEndpointAgent.Core.csproj create mode 100644 agent/ShopDbEndpointAgent.Core/TelemetryWorker.cs create mode 100644 agent/ShopDbEndpointAgent.Tray/Program.cs create mode 100644 agent/ShopDbEndpointAgent.Tray/ShopDbEndpointAgent.Tray.csproj create mode 100644 agent/ShopDbEndpointAgent.Tray/TicketForm.cs create mode 100644 agent/ShopDbEndpointAgent.Tray/TrayContext.cs create mode 100644 agent/ShopDbEndpointAgent.Tray/TrayIcon.cs create mode 100644 agent/ShopDbEndpointAgent.slnx create mode 100644 agent/ShopDbEndpointAgent/Program.cs create mode 100644 agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj create mode 100644 agent/ShopDbEndpointAgent/appsettings.json create mode 100644 agent/installer/install-shopdbagent.ps1 diff --git a/.github/agent-workflow.md b/.github/agent-workflow.md new file mode 100644 index 00000000..59b87aaf --- /dev/null +++ b/.github/agent-workflow.md @@ -0,0 +1,86 @@ +# Agent workflow rules + +Companion to `copilot-instructions.md` (which owns naming/style/architecture). +These cut wasted turns. All are lessons from real sessions. + +## Environment - know these, do not rediscover + +- Shell is **PowerShell**: use `;` not `&&`, `2>$null` not `2>/dev/null`. +- API container `shopdb-flask-wilm-api-1` (Flask, port `5001`). +- DB container `shopdb-flask-wilm-db-1` (MySQL 8, root pw `root_password_123`). +- SPA is **baked into the image** (not bind-mounted). Plugin code is + **bind-mounted read-only**. +- `docs/` is stripped to a public wiki mirror - keep working docs out of it. +- `requirements.txt` is hash-checked - never hand-edit. + +## Verify, don't react + +- The terminal "Command failed" banner fires on any non-zero exit or stderr + output and is **usually a false negative** (`curl --fail` on a 400, + pytest/npm/git writing to stderr). Judge success from real output. +- Read container health via + `docker inspect --format '{{.State.Health.Status}}'` plus a probe **without** + `--fail` - never eyeball the exit code. + +## Container runs the image, not your edits + +Before running anything in the container that must reflect a change: + +1. Frontend: `npm run build`, then + `docker cp frontend/dist/. shopdb-flask-wilm-api-1:/app/frontend/dist`. +2. Backend/scripts/templates: `docker cp` the files (or rebuild) first. + +If a result contradicts a change you just made, suspect stale container code +before re-deriving the fix. + +## Testing + +- Confirm the working directory once, up front. +- Run the smallest test that proves the point; find the minimal polluter for + isolation bugs. Run the full suite only at the end. +- Clear stale local state before plugin/migration tests: + `if (Test-Path instance/plugins.json) { Remove-Item instance/plugins.json -Force }`. +- Naming-gate hits inside local venvs (`.ci-venv/`, `env/`) are not repo code - + do not "fix" vendored libraries. +- Final gates, exactly as CI: `python -m pytest tests/ -q`; + `cd frontend; npx vitest run; npm run build; cd ..`; + `bash scripts/check-naming-and-style.sh`. + +## Migrations and imports + +- Every migration: idempotent guard + real downgrade; must run clean on strict + MySQL 8. If you rename indexes in `upgrade()`, update `downgrade()` to match. +- Avoid duplicate indexes: `index=True` plus an explicit `db.Index` on the same + column creates two. +- Importers/seeders match by natural key and must be idempotent - prove it by + running twice (expect `updated`, not `created`). Watch all-or-nothing writes. + +## Don't offer rollbacks you don't need + +State current state plainly; surface a rollback path only for a real regression. + +## Wilmington (wilm) local-run known failures - do not re-investigate + +Running the full `python -m pytest tests/ -q` on a Windows dev box (not CI) +produces a stable set of environmental failures that are NOT code regressions. +Treat these as expected locally; judge your change against CI/Linux instead: + +- `test_core/test_security_headers.py::test_a_served_upload_is_sandboxed` - + `PermissionError [WinError 32]` (served upload file still locked by another + process). Windows file-lock timing, not a headers bug. +- `test_plugin_contract.py::test_import_scan_covers_symlinked_plugins` - + `OSError [WinError 1314]` (symlink privilege not held). Needs admin/Developer + Mode to create symlinks; irrelevant to plugin logic. +- `test_plugin_importguard.py::test_verified_source_signed_ok_tampered_raises` - + CRLF vs LF (`b'# plugin\r\n' == b'# plugin\n'`). Git autocrlf checkout, not a + signing bug. +- `test_docs_publishable.py` and `test_docs_versions.py` - the local `docs/` + tree is not the stripped public wiki mirror, so doc-count / PROJECT-MAP + assertions fail. Expected off-CI. +- `test_core/test_public_read_surface.py::test_every_get_route_is_gated_or_named_public` + - pre-existing ungated-GET inventory drift; only act on it if YOUR change + added a new GET route. + +If a run shows ONLY these, the suite is green for our purposes. Anything outside +this list is real - investigate it. + diff --git a/Dockerfile b/Dockerfile index 034e0492..df0156d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,10 +23,21 @@ FROM node:20-slim AS frontendbuild WORKDIR /build -# Trust the site's corporate root CA so npm's TLS to the registry validates -# behind an inspecting proxy. Without it npm 10.x fails silently and leaves an -# incomplete node_modules. -COPY corp-root-ca.pem /usr/local/share/ca-certificates/corp-root-ca.crt +# Optionally trust the site's corporate root CA so npm's TLS to the registry +# validates behind an inspecting proxy. The .pem is gitignored and absent from a +# clean checkout, so the copy is made optional by pairing it with a file that +# always exists; the cert is installed only when actually provided. Without it +# npm 10.x behind an inspecting proxy fails silently and leaves an incomplete +# node_modules; a clean build with no proxy is unaffected. +COPY corp-root-ca.pem* frontend/package.json /tmp/corpca/ +# node:20-slim ships without the ca-certificates package (no update-ca-certificates +# and no /usr/local/share/ca-certificates), so install it before trusting the cert. +RUN if [ -f /tmp/corpca/corp-root-ca.pem ]; then \ + apt-get update && apt-get install -y --no-install-recommends ca-certificates && \ + rm -rf /var/lib/apt/lists/* && \ + cp /tmp/corpca/corp-root-ca.pem /usr/local/share/ca-certificates/corp-root-ca.crt && \ + update-ca-certificates; \ + fi ENV NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/corp-root-ca.crt # Copy only the manifests first so `npm ci` caches on dependency changes. diff --git a/agent/.gitignore b/agent/.gitignore new file mode 100644 index 00000000..aa65026a --- /dev/null +++ b/agent/.gitignore @@ -0,0 +1,7 @@ +# .NET build output - never tracked; produced by dotnet build/publish. +bin/ +obj/ +publish/ + +# Local publish scratch and command-output captures. +*.user diff --git a/agent/README.md b/agent/README.md new file mode 100644 index 00000000..0c43efe5 --- /dev/null +++ b/agent/README.md @@ -0,0 +1,166 @@ +# ShopDB Endpoint Agent + +A Windows telemetry agent that reports endpoint state to a ShopDB server's +`endpoints` plugin. It runs as a Windows service in production and as a console +app for development and testing. + +## Projects + +- `ShopDbEndpointAgent.Core` - contracts, config, logging, collection, and the + resilient collector HTTP client. No host concerns; unit-testable. +- `ShopDbEndpointAgent` - the service host (generic host + Windows service + lifetime). Composes Core through DI. This is the telemetry runtime. +- `ShopDbEndpointAgent.Tray` - interactive user-session tray app: a system-tray + icon with "Send Status now" (on-demand report) and the 5-click issue reporter + (ServiceNow flow is still a scaffold). Separate process from the service, + because a SYSTEM service cannot draw a tray icon in a user session. + +## Configuration + +Runtime config is layered. Non-secret defaults live in `appsettings.json` +(shipped with the build). The collector key and base URL are provisioned out of +band in the registry and win over `appsettings.json`, so a distributed build +never carries the secret and the site can rotate the key without a rebuild. + +### Registry (production) + +Create these values under `HKLM\SOFTWARE\GE\ShopDB` (64-bit view): + +| Value | Type | Meaning | +|----------------|--------|--------------------------------------------------| +| `BaseUrl` | String | `https://shopdb.example.net` (server root) | +| `CollectorKey` | String | The endpoint agent key from Settings -> Endpoints -> Key management | + +Example (elevated PowerShell): + +``` +New-Item -Path 'HKLM:\SOFTWARE\GE\ShopDB' -Force | Out-Null +Set-ItemProperty -Path 'HKLM:\SOFTWARE\GE\ShopDB' -Name 'BaseUrl' -Value 'https://shopdb.example.net' +Set-ItemProperty -Path 'HKLM:\SOFTWARE\GE\ShopDB' -Name 'CollectorKey' -Value '' +``` + +### appsettings.json (all environments) + +| Key | Default | Meaning | +|-------------------|-------------|------------------------------------------------| +| `baseurl` | "" | Dev fallback for the server root (registry wins). | +| `apikey` | placeholder | Dev fallback for the collector key (registry wins). | +| `cadenceseconds` | 900 | Report interval (15 min). | +| `jitterseconds` | 60 | Random +/- spread on the interval (anti-herd). | +| `adoptonfirstrun` | true | Send `adopt=true` on first contact only. | +| `utilitiespath` | "" | Directory of site PowerShell "agent utilities". | +| `logpath` | "" | Override the rolling log directory. | + +For a dev box, set `baseurl` and `apikey` here and skip the registry. + +## Running + +Console (dev), one report then exit: + +``` +dotnet run --project ShopDbEndpointAgent -- --once +``` + +Console (dev), continuous cadence loop: + +``` +dotnet run --project ShopDbEndpointAgent -- --console +``` + +## Install as a Windows service + +Publish self-contained, then register with `sc.exe` (elevated): + +``` +dotnet publish ShopDbEndpointAgent -c Release -r win-x64 --self-contained -o C:\ProgramData\ShopDB\EndpointAgent\bin +sc.exe create "ShopDBEndpointAgent" binPath= "C:\ProgramData\ShopDB\EndpointAgent\bin\shopdb-endpoint-agent.exe" start= auto DisplayName= "ShopDB Endpoint Agent" +sc.exe description "ShopDBEndpointAgent" "Reports endpoint telemetry to ShopDB." +sc.exe start "ShopDBEndpointAgent" +``` + +The service runs as LocalSystem by default, which it needs to read the Security +event channel and system WMI. + +## Tray app (system-tray icon) + +The tray is a per-user process, not a service, so it starts at sign-in rather +than under `sc.exe`. `install-shopdbagent.ps1` handles it: it copies +`shopdb-endpoint-tray.exe` next to the agent, registers an HKCU logon autostart +(`Software\Microsoft\Windows\CurrentVersion\Run\ShopDbEndpointTray`), and +launches it for the current user. Run the installer from the operator's own +session so the autostart lands in the right user hive. Pass `-NoTray` for a +headless, service-only install. + +Right-click the tray icon for: + +- `Send Status now` - reports this PC to ShopDB immediately so the monitor page + refreshes without waiting for the next cadence tick. It shells out to + `shopdbagent.exe --once`, reusing the exact production collection path; the + tray holds no collector key. A balloon tip reflects success or failure (on + failure, see the rolling agent log). +- `Report an issue...` - the ServiceNow ticket flow (scaffold; submission path + is an open decision). + +A desktop shortcut to `shopdb-endpoint-tray.exe` can be added later for an +explicitly launchable ServiceNow surface. + +## WebUI command execution model (security context) + +The WebUI can send an endpoint an allowlisted command (server source of truth: +`plugins/endpoints/commands.py`; agent handler: `CommandExecutor`). The agent +service runs as LocalSystem, but not every command runs in that context. The +launch context is chosen per action so processes surface to the operator while +privileged operations keep the token they need: + +- Generic `process_start` (for example DncMain, UDC) launches into the ACTIVE + CONSOLE USER's session at medium integrity, so the app is visible on the + operator's desktop. CyberArk/EPM brokers any elevation afterward. +- Generic `process_stop` / `process_status` run as SYSTEM (kill and probe). +- `service_start` / `service_stop` / `service_status` run as SYSTEM, because + service control requires the admin/SYSTEM token. +- `mtconnect_start` / `mtconnect_stop` / `mtconnect_status` all run as SYSTEM, + because MTConnect is the "UDC MTConnect Agent" Windows service and service + control requires the admin/SYSTEM token. `mtconnect_start` runs `install.bat` + (`Agent.exe install`) to install or repair the service, then ensures it is + running; `mtconnect_stop` stops the service; `mtconnect_status` queries it. + `run.bat` (`agent.exe run agent.cfg`) is a foreground TESTING launcher and is + deliberately NOT used by the agent. +- `system_restart` runs as SYSTEM and reboots the machine immediately via + `shutdown.exe /r /t 0` (no grace timer). It is a fixed action with no target, + gated server-side by the dedicated `endpoints.restart` permission on top of + `endpoints.command`, and fronted by a WebUI confirmation gate. + +Do not route MTConnect through the generic `process_start` user-session path or +service control breaks. MTConnect is intentionally absent from the +`process_start` launch map for the same reason; the fixed `mtconnect_*` handler +owns it. + +Behavior note for redeploys: the MTConnect service-control model is a runtime +change, so the agent must be rebuilt and redeployed for it to take effect. +Server-side command/allowlist/RBAC changes are picked up by the container +rebuild; rebuild and redeploy the agent whenever the runtime launch/control +logic itself changes. + +## Diagnostics + +- Rolling file log: `%ProgramData%\ShopDB\EndpointAgent\logs\agent-YYYYMMDD.log` + (14-day retention). +- Windows Event Log: agent runtime failures and unhandled exceptions are written + to the `Application` log under source `ShopDB Endpoint Agent`. The source is + created on first elevated run (or by the installer). Filter Event Viewer by + that source to see agent/system interop failures. + +## Agent utilities (PowerShell seam) + +If `utilitiespath` points at a directory, the runtime runs each `*.ps1` there +per report, parses its JSON stdout, and merges the result into the report. This +is the seam the future server-side "Agent configuration" settings page uses to +push site-authored collectors down. Scripts must be code-signed and +allow-listed before this is enabled in the field (rollout concern). + +## Logging extensibility + +All runtime code logs through `Microsoft.Extensions.Logging` (`ILogger`). Adding +a sink later (a ShopDB HTTP sink, Seq, syslog) is a host-builder change in +`Program.cs::ConfigureLogging` with no changes to the worker, collector, or +client. diff --git a/agent/ShopDbEndpointAgent.Core/AgentConfig.cs b/agent/ShopDbEndpointAgent.Core/AgentConfig.cs new file mode 100644 index 00000000..f8a8b9f9 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/AgentConfig.cs @@ -0,0 +1,109 @@ +using System.Text.Json.Serialization; + +namespace ShopDbEndpointAgent.Core; + +// Runtime configuration for the agent. Non-secret values come from +// appsettings.json; the collector key and base URL are provisioned out of band +// in the registry (HKLM:\SOFTWARE\GE\ShopDB) and layered on top - see +// RegistryConfigProvider. The registry wins so a distributed appsettings.json +// never carries the secret and the site can rotate the key without a rebuild. +public sealed class AgentConfig +{ + [JsonPropertyName("baseurl")] public string BaseUrl { get; set; } = ""; + [JsonPropertyName("apikey")] public string ApiKey { get; set; } = ""; + [JsonPropertyName("cadenceseconds")] public int CadenceSeconds { get; set; } = 900; + + // Jitter spreads a fleet's reports so they do not arrive in lockstep + // (thundering herd). Actual delay = cadence +/- up to jitterseconds. + [JsonPropertyName("jitterseconds")] public int JitterSeconds { get; set; } = 60; + + [JsonPropertyName("adoptonfirstrun")] public bool AdoptOnFirstRun { get; set; } = true; + + // Directory holding site-provided PowerShell "agent utilities" the runtime + // may invoke through the documented collection seam. Empty = none loaded. + [JsonPropertyName("utilitiespath")] public string UtilitiesPath { get; set; } = ""; + + // Absolute path for the rolling diagnostics log. Empty = the default under + // %ProgramData%\ShopDB\EndpointAgent\logs. + [JsonPropertyName("logpath")] public string LogPath { get; set; } = ""; + + // What this endpoint watches. Shipped in appsettings.json (non-secret) so a + // site can retarget monitors without a rebuild. Processes are interactive + // apps watched by image name; services are Windows services watched by their + // service key name. Empty lists = nothing watched. + [JsonPropertyName("monitors")] public MonitorConfig Monitors { get; set; } = new(); + + // Per-image launchable paths for the generic process_start action. The wire + // never carries a path (that would be a code-execution hole); Start sends + // only the image name and the agent maps it here to a known-good executable + // (or a .bat launcher). Anything not in the resolved map fails closed. A + // site overrides or extends the shipped defaults in appsettings.json without + // a rebuild. Keyed by process image name (case-insensitive, .exe optional). + [JsonPropertyName("launchpaths")] + public Dictionary LaunchPaths { get; set; } = new(); + + // Shipped defaults for the org's common shop-floor apps. A site's + // appsettings.json LaunchPaths entries win over these on a per-image basis. + // MTConnect is intentionally absent here: it is a Windows service controlled + // by the fixed mtconnect_* handler (install.bat + service control), not the + // generic process_start path. eDNC and UDC launch their runtime executables + // directly. + public static readonly IReadOnlyDictionary DefaultLaunchPaths = + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["DncMain.exe"] = @"C:\Program Files (x86)\Dnc\bin\DncMain.exe", + ["UDC.exe"] = @"C:\Program Files\UDC\UDC.exe", + }; + + // The effective launch map: shipped defaults overlaid with appsettings.json + // entries (site wins). Keys are normalized to bare image names so a lookup + // tolerates 'DncMain' or 'DncMain.exe'. + public IReadOnlyDictionary ResolveLaunchPaths() + { + var resolved = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var pair in DefaultLaunchPaths) + resolved[NormalizeImage(pair.Key)] = pair.Value; + foreach (var pair in LaunchPaths) + { + if (string.IsNullOrWhiteSpace(pair.Key) || string.IsNullOrWhiteSpace(pair.Value)) + continue; + resolved[NormalizeImage(pair.Key)] = pair.Value; + } + return resolved; + } + + private static string NormalizeImage(string image) + { + var bare = (image ?? "").Trim(); + if (bare.EndsWith(".exe", StringComparison.OrdinalIgnoreCase)) + bare = bare[..^4]; + return bare; + } + + public bool IsUsable() + => !string.IsNullOrWhiteSpace(BaseUrl) + && !string.IsNullOrWhiteSpace(ApiKey) + && ApiKey != "REPLACE_WITH_COLLECTOR_KEY"; +} + +// Watched targets for this endpoint. Both lists carry entries with a wire name +// (process image name or Windows service key name) and an optional friendly app +// label the server/UI can show (for example "eDNC" for dncMain.exe). +public sealed class MonitorConfig +{ + [JsonPropertyName("processes")] public MonitorTarget[] Processes { get; set; } + = Array.Empty(); + + [JsonPropertyName("services")] public MonitorTarget[] Services { get; set; } + = Array.Empty(); +} + +public sealed class MonitorTarget +{ + // Process image name (for example "udc.exe") or Windows service key name + // (for example "MTConnectAgent"). Matched case-insensitively. + [JsonPropertyName("name")] public string Name { get; set; } = ""; + + // Human-facing application label, optional. Falls back to Name when empty. + [JsonPropertyName("app")] public string App { get; set; } = ""; +} diff --git a/agent/ShopDbEndpointAgent.Core/AgentConfigProvider.cs b/agent/ShopDbEndpointAgent.Core/AgentConfigProvider.cs new file mode 100644 index 00000000..ede80f88 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/AgentConfigProvider.cs @@ -0,0 +1,110 @@ +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Win32; + +namespace ShopDbEndpointAgent.Core; + +// Builds the effective AgentConfig by layering two sources: +// 1. appsettings.json - non-secret runtime defaults, shipped with the build. +// 2. HKLM\SOFTWARE\GE\ShopDB - the collector key and base URL, provisioned +// out of band (Intune/DSC/GE-Enforce). Registry values WIN so a +// distributed appsettings.json never carries the secret and the site can +// rotate the key without a rebuild. +// +// Reading the registry never throws out to the caller: a missing hive or value +// simply leaves the appsettings value in place, which is what makes dev boxes +// (appsettings only) and fielded machines (registry provisioned) both work. +public static class AgentConfigProvider +{ + public const string RegistryPath = @"SOFTWARE\GE\ShopDB"; + + // Registry value names. These mirror the names the computers reporter uses + // (see plugins/endpoints/DESIGN-DECISIONS-agent.md section 1). + public const string BaseUrlValue = "BaseUrl"; + public const string CollectorKeyValue = "CollectorKey"; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + }; + + public static AgentConfig Load(string? appSettingsPath = null, + Microsoft.Extensions.Logging.ILogger? logger = null) + { + var config = LoadAppSettings(appSettingsPath) ?? new AgentConfig(); + ApplyRegistryOverrides(config, logger); + return config; + } + + private static AgentConfig? LoadAppSettings(string? path) + { + path ??= Path.Combine(AppContext.BaseDirectory, "appsettings.json"); + if (!File.Exists(path)) + return null; + try + { + var json = File.ReadAllText(path); + return JsonSerializer.Deserialize(json, JsonOptions); + } + catch + { + // Malformed appsettings: fall back to defaults + registry so the + // agent can still run if the secret is provisioned in the registry. + return null; + } + } + + private static void ApplyRegistryOverrides(AgentConfig config, + Microsoft.Extensions.Logging.ILogger? logger = null) + { + // Try 64-bit view first, then the default/32-bit view. On a locked-down + // box (CyberArk/EPM gating regedit and hive reads) the read can throw + // SecurityException/UnauthorizedAccessException even when the values + // exist. We log the exact outcome per view instead of swallowing it, so + // "Config incomplete" can be told apart from "access denied". + RegistryView[] views = { RegistryView.Registry64, RegistryView.Registry32 }; + foreach (var view in views) + { + try + { + using var machine = RegistryKey.OpenBaseKey( + RegistryHive.LocalMachine, view); + using var key = machine.OpenSubKey(RegistryPath); + if (key is null) + { + logger?.LogWarning( + "Registry key HKLM\\{Path} not found in {View} view.", + RegistryPath, view); + continue; + } + + var baseUrl = key.GetValue(BaseUrlValue) as string; + if (!string.IsNullOrWhiteSpace(baseUrl)) + config.BaseUrl = baseUrl.Trim(); + + var collectorKey = key.GetValue(CollectorKeyValue) as string; + if (!string.IsNullOrWhiteSpace(collectorKey)) + config.ApiKey = collectorKey.Trim(); + + logger?.LogInformation( + "Registry {View}: BaseUrl {BaseUrlState}, CollectorKey {KeyState}.", + view, + string.IsNullOrWhiteSpace(baseUrl) ? "MISSING" : "present", + string.IsNullOrWhiteSpace(collectorKey) ? "MISSING" : "present"); + + // Success reading this view; stop unless nothing was found. + if (!string.IsNullOrWhiteSpace(baseUrl) || + !string.IsNullOrWhiteSpace(collectorKey)) + return; + } + catch (Exception error) + { + logger?.LogError(error, + "Registry read of HKLM\\{Path} in {View} view was blocked " + + "({ErrorType}). If CyberArk/EPM is gating the hive, grant the " + + "agent read access or use appsettings.json for this test.", + RegistryPath, view, error.GetType().Name); + } + } + } +} diff --git a/agent/ShopDbEndpointAgent.Core/AgentFaultMonitor.cs b/agent/ShopDbEndpointAgent.Core/AgentFaultMonitor.cs new file mode 100644 index 00000000..a46e98df --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/AgentFaultMonitor.cs @@ -0,0 +1,91 @@ +using System.Diagnostics; +using Microsoft.Extensions.Logging; + +namespace ShopDbEndpointAgent.Core; + +// Agent self-monitoring (WER-style). Routes otherwise-unobserved failures - +// unhandled exceptions on any thread and faulted tasks nobody awaited - to the +// ILogger (so they land in the rolling file) AND directly to a dedicated +// Windows Event Log source so agent runtime failures and agent/system interop +// errors are visible in Event Viewer alongside the OS's own records. +// +// The Event Log source is created on first use; creating a source needs admin, +// so if it is absent and cannot be created we degrade to the file log only and +// never throw from a fault handler (that would mask the original failure). +public sealed class AgentFaultMonitor +{ + public const string EventLogSource = "ShopDB Endpoint Agent"; + public const string EventLogName = "Application"; + + private readonly ILogger logger; + private readonly bool eventLogReady; + + public AgentFaultMonitor(ILogger logger) + { + this.logger = logger; + eventLogReady = EnsureEventLogSource(); + } + + // Wire the process-wide fault hooks. Call once at startup. + public void Install() + { + AppDomain.CurrentDomain.UnhandledException += OnUnhandledException; + TaskScheduler.UnobservedTaskException += OnUnobservedTaskException; + } + + public void ReportFatal(string context, Exception error) + => Report(EventLogEntryType.Error, 1000, $"{context}: {error}"); + + private void OnUnhandledException(object sender, UnhandledExceptionEventArgs args) + { + var error = args.ExceptionObject as Exception; + var message = error?.ToString() ?? "unknown unhandled exception"; + logger.LogCritical(error, "Unhandled exception (terminating={Terminating})", + args.IsTerminating); + Report(EventLogEntryType.Error, 1001, + $"Unhandled exception (terminating={args.IsTerminating}): {message}"); + } + + private void OnUnobservedTaskException(object? sender, + UnobservedTaskExceptionEventArgs args) + { + logger.LogError(args.Exception, "Unobserved task exception"); + Report(EventLogEntryType.Warning, 1002, + $"Unobserved task exception: {args.Exception}"); + // Mark observed so it does not escalate to a process-level failure. + args.SetObserved(); + } + + private void Report(EventLogEntryType type, int eventId, string message) + { + if (!eventLogReady) + return; + try + { + EventLog.WriteEntry(EventLogSource, message, type, eventId); + } + catch + { + // Event Log write failed (source removed, log full): the file log + // already has this entry; a fault handler must never throw. + } + } + + private static bool EnsureEventLogSource() + { + try + { + if (EventLog.SourceExists(EventLogSource)) + return true; + EventLog.CreateEventSource(new EventSourceCreationData( + EventLogSource, EventLogName)); + return true; + } + catch + { + // Needs admin the first time; the installer creates the source, so + // in the field this succeeds. In dev without admin we log to file. + return false; + } + } +} diff --git a/agent/ShopDbEndpointAgent.Core/CollectSignal.cs b/agent/ShopDbEndpointAgent.Core/CollectSignal.cs new file mode 100644 index 00000000..4deea78b --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/CollectSignal.cs @@ -0,0 +1,26 @@ +namespace ShopDbEndpointAgent.Core; + +// A one-shot wake signal shared between the command worker and the telemetry +// worker. After a command runs (e.g. MTConnect start/stop), the command worker +// calls Request() so the telemetry loop collects immediately instead of waiting +// out the remaining cadence, and the WebUI card reflects the new state within a +// single report cycle rather than up to 15 minutes later. +public sealed class CollectSignal +{ + private readonly SemaphoreSlim gate = new(0, 1); + + // Wake the telemetry loop for one off-cadence collection. Extra calls while + // one is already pending collapse into a single wake (the gate caps at 1). + public void Request() + { + try { gate.Release(); } + catch (SemaphoreFullException) { /* a wake is already pending */ } + } + + // Wait until either the cadence delay elapses or a collect is requested. + // Returns true when woken by a request, false on a normal cadence timeout. + public async Task WaitAsync(TimeSpan delay, CancellationToken token) + { + return await gate.WaitAsync(delay, token); + } +} diff --git a/agent/ShopDbEndpointAgent.Core/CollectorClient.cs b/agent/ShopDbEndpointAgent.Core/CollectorClient.cs new file mode 100644 index 00000000..4c4d30d9 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/CollectorClient.cs @@ -0,0 +1,118 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Microsoft.Extensions.Logging; + +namespace ShopDbEndpointAgent.Core; + +// Posts one EndpointReport to POST {baseurl}/api/collector/endpoints with the +// X-API-Key header (plugins/endpoints/AGENT-CONTRACT.md section 1), unwrapping +// the ShopDB envelope { status, data, meta } to return the CollectorResponse. +// +// Resilience: transient failures (network errors, timeouts, 5xx, 429) are +// retried with exponential backoff plus jitter, bounded by maxAttempts. A 4xx +// other than 429 is a permanent error (bad key, unknown host in strict mode) +// and is surfaced immediately without retry - hammering the server changes +// nothing. +public sealed class CollectorClient : IDisposable +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + DefaultIgnoreCondition = System.Text.Json.Serialization + .JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly HttpClient http; + private readonly ILogger logger; + private readonly int maxAttempts; + private readonly Random jitter = new(); + + public CollectorClient(AgentConfig config, ILogger logger, + int maxAttempts = 4) + { + this.logger = logger; + this.maxAttempts = Math.Max(1, maxAttempts); + http = new HttpClient + { + BaseAddress = new Uri(config.BaseUrl.TrimEnd('/') + "/"), + Timeout = TimeSpan.FromSeconds(30), + }; + http.DefaultRequestHeaders.Add("X-API-Key", config.ApiKey); + } + + public async Task SendAsync( + EndpointReport report, CancellationToken token) + { + Exception? last = null; + for (int attempt = 1; attempt <= maxAttempts; attempt++) + { + token.ThrowIfCancellationRequested(); + try + { + return await SendOnceAsync(report, token); + } + catch (CollectorException error) when (error.IsPermanent) + { + // Bad credential or rejected payload: no retry. + throw; + } + catch (Exception error) when ( + error is CollectorException or HttpRequestException or TaskCanceledException + && !token.IsCancellationRequested) + { + last = error; + if (attempt == maxAttempts) + break; + var delay = BackoffDelay(attempt); + logger.LogWarning( + "Collector send attempt {Attempt}/{Max} failed: {Message}; retrying in {Delay}s", + attempt, maxAttempts, error.Message, delay.TotalSeconds); + await Task.Delay(delay, token); + } + } + throw last ?? new CollectorException(0, "collector send failed"); + } + + private async Task SendOnceAsync( + EndpointReport report, CancellationToken token) + { + using var response = await http.PostAsJsonAsync( + "api/collector/endpoints", report, JsonOptions, token); + + var body = await response.Content.ReadAsStringAsync(token); + if (!response.IsSuccessStatusCode) + throw new CollectorException((int)response.StatusCode, body); + + var envelope = JsonSerializer.Deserialize(body, JsonOptions); + if (envelope?.Data is null) + throw new CollectorException((int)response.StatusCode, "empty response body"); + return envelope.Data; + } + + private TimeSpan BackoffDelay(int attempt) + { + // 2^(attempt-1) seconds, capped, plus up to 1s jitter. + double seconds = Math.Min(30, Math.Pow(2, attempt - 1)); + return TimeSpan.FromSeconds(seconds + jitter.NextDouble()); + } + + public void Dispose() => http.Dispose(); +} + +public sealed class CollectorException : Exception +{ + public int StatusCode { get; } + + // A permanent failure is a 4xx that is not 429 (rate limit): retrying with + // the same key and payload cannot succeed, so the worker should back off to + // its normal cadence rather than fast-retry. + public bool IsPermanent => + StatusCode >= 400 && StatusCode < 500 + && StatusCode != (int)HttpStatusCode.TooManyRequests; + + public CollectorException(int statusCode, string body) + : base($"collector returned HTTP {statusCode}: {body}") + { + StatusCode = statusCode; + } +} diff --git a/agent/ShopDbEndpointAgent.Core/CommandClient.cs b/agent/ShopDbEndpointAgent.Core/CommandClient.cs new file mode 100644 index 00000000..296dc280 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/CommandClient.cs @@ -0,0 +1,83 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Microsoft.Extensions.Logging; + +namespace ShopDbEndpointAgent.Core; + +// Talks to the endpoint command channel (plugins/endpoints/AGENT-CONTRACT.md +// section 7). The agent opens no inbound port: it long-polls +// GET /api/endpoints/commands/next, which the server holds open until a command +// is queued (HTTP 200) or the hold window elapses (HTTP 204). After running the +// claimed command locally the agent posts the outcome to +// POST /api/endpoints/commands/{id}/result. Both share the endpoints collector +// key via X-API-Key, so the command channel rides the same kill switch as +// telemetry and adds no second secret. +public sealed class CommandClient : IDisposable +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + DefaultIgnoreCondition = System.Text.Json.Serialization + .JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly HttpClient http; + private readonly ILogger logger; + + public CommandClient(AgentConfig config, ILogger logger) + { + this.logger = logger; + http = new HttpClient + { + BaseAddress = new Uri(config.BaseUrl.TrimEnd('/') + "/"), + // Longer than the collector client: the claim request is held open + // server-side (~25s), so the read timeout must clear that plus slack. + Timeout = TimeSpan.FromSeconds(60), + }; + http.DefaultRequestHeaders.Add("X-API-Key", config.ApiKey); + } + + // Claim the next pending command for this hostname, or null when the server + // returned 204 (nothing queued within the hold window). A 4xx other than 429 + // is surfaced so the worker can back off instead of hot-looping a bad key or + // an unknown host. + public async Task ClaimNextAsync( + string hostname, CancellationToken token) + { + var url = "api/endpoints/commands/next?hostname=" + + Uri.EscapeDataString(hostname); + using var response = await http.GetAsync(url, token); + + if (response.StatusCode == HttpStatusCode.NoContent) + return null; + + var body = await response.Content.ReadAsStringAsync(token); + if (!response.IsSuccessStatusCode) + throw new CollectorException((int)response.StatusCode, body); + + var envelope = JsonSerializer.Deserialize( + body, JsonOptions); + return envelope?.Data; + } + + // Report a claimed command's outcome. Best-effort: a failure to post the + // result is logged and swallowed so a transient network blip does not crash + // the worker; the command simply stays 'running' server-side until a later + // operator retry supersedes it. + public async Task ReportResultAsync( + int commandId, CommandResult result, CancellationToken token) + { + var url = $"api/endpoints/commands/{commandId}/result"; + using var response = await http.PostAsJsonAsync( + url, result, JsonOptions, token); + if (!response.IsSuccessStatusCode) + { + var body = await response.Content.ReadAsStringAsync(token); + logger.LogWarning( + "Reporting result for command {CommandId} returned HTTP {Status}: {Body}", + commandId, (int)response.StatusCode, body); + } + } + + public void Dispose() => http.Dispose(); +} diff --git a/agent/ShopDbEndpointAgent.Core/CommandExecutor.cs b/agent/ShopDbEndpointAgent.Core/CommandExecutor.cs new file mode 100644 index 00000000..c739bdc7 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/CommandExecutor.cs @@ -0,0 +1,581 @@ +using System.Diagnostics; +using System.ServiceProcess; +using Microsoft.Extensions.Logging; + +namespace ShopDbEndpointAgent.Core; + +// Runs an allowlisted endpoint command locally and returns the result to post +// back. The allowlist here mirrors the server's (plugins/endpoints/commands.py): +// the wire carries an action NAME plus, for generic actions, a TARGET, and this +// is the only place that maps a name to concrete behavior. An unknown name +// fails closed - it is never treated as a command string to execute. +// +// Two families: +// Generic primitives (data-driven; NO agent rebuild for a new app/policy): +// service_start / service_stop / service_status - act on claim.Target, a +// Windows service name. +// process_stop / process_status - act on claim.Target, a process image name +// (for example 'DncMain.exe'). process_start also acts on the image name +// but launches ONLY through a per-image allowlist of known-good paths +// (AgentConfig.LaunchPaths, shipped defaults overridable per site); an +// unmapped target fails closed so the wire can never launch an arbitrary +// binary. process_start launches into the ACTIVE CONSOLE USER'S session +// (not session 0) at the user's medium integrity so the app is visible +// on their desktop; CyberArk/EPM brokers any elevation afterward. Stop +// and status stay as SYSTEM. +// Fixed MTConnect actions (target pinned in the agent, never on the wire): +// mtconnect_start / mtconnect_stop / mtconnect_status. +// MTConnect is a Windows service ("UDC MTConnect Agent"), so all three run +// as SYSTEM (service control needs the admin/SYSTEM token). Start runs +// install.bat (Agent.exe install) to install or repair the service, then +// ensures it is running; stop stops the service; status queries it. NOTE: +// run.bat (agent.exe run agent.cfg) is a foreground TESTING launcher and is +// deliberately NOT used - do not route MTConnect through the generic +// process_start user-session path or the service control breaks. +// +// Every action probes and reports the resulting run state (MtConnectRunning for +// MTConnect; the generic probes report via Message/Detail) so the WebUI toggle +// reflects reality rather than the action's intent. +public sealed class CommandExecutor +{ + // The UDC MTConnect agent is a Windows service. Its name is fixed here (not + // sent over the wire) so a compromised WebUI cannot redirect service control. + // A later per-site override belongs in AgentConfig, not the command payload. + private const string MtConnectServiceName = "UDC MTConnect Agent"; + + // The MTConnect install/repair seam: Agent.exe install. run.bat (agent.exe + // run agent.cfg) is a foreground TESTING launcher and is deliberately NOT + // used here. install.bat registers or repairs the service; the start handler + // runs it, then ensures the service is running. + private const string MtConnectInstallBat = + @"C:\ProgramData\UDC\MTConnect_UDC\Agent\install.bat"; + + private readonly ILogger logger; + private readonly TimeSpan startTimeout; + + // Resolved per-image launchable paths for process_start (shipped defaults + // overlaid with any appsettings.json overrides). A target absent from this + // map cannot be started - process_start fails closed. + private readonly IReadOnlyDictionary launchPaths; + + public CommandExecutor(ILogger logger, + AgentConfig config, TimeSpan? startTimeout = null) + { + this.logger = logger; + this.startTimeout = startTimeout ?? TimeSpan.FromSeconds(30); + this.launchPaths = (config ?? new AgentConfig()).ResolveLaunchPaths(); + } + + public async Task ExecuteAsync( + CommandClaim claim, CancellationToken token) + { + var action = claim.Action; + var target = (claim.Target ?? "").Trim(); + switch (action) + { + // Fixed MTConnect actions: target is pinned in the agent. + case "mtconnect_start": + return await MtConnectStartAsync(token); + case "mtconnect_stop": + return MtConnectStop(); + case "mtconnect_status": + return MtConnectStatus(); + + // Generic service primitives: act on the wire target. + case "service_start": + return RequireTarget(action, target) + ?? ServiceControl(target, ServiceControllerStatus.Running); + case "service_stop": + return RequireTarget(action, target) + ?? ServiceControl(target, ServiceControllerStatus.Stopped); + case "service_status": + return RequireTarget(action, target) + ?? ServiceStatus(target); + + // Generic process primitives: act on the wire target image name. + case "process_stop": + return RequireTarget(action, target) + ?? ProcessStop(target); + case "process_status": + return RequireTarget(action, target) + ?? ProcessStatus(target); + case "process_start": + return RequireTarget(action, target) + ?? await ProcessStart(target, token); + + // Fixed destructive action: reboot this endpoint immediately. Runs + // as the SYSTEM agent, so shutdown does not need a user session. + case "system_restart": + return SystemRestart(); + + // Force an immediate off-cadence telemetry report. The handler is a + // no-op success: CommandWorker fires collectSignal.Request() after + // every command, which wakes the telemetry loop to collect now, so + // the WebUI reflects live state instead of waiting for cadence. + case "collect_now": + logger.LogInformation("collect_now requested: forcing off-cadence telemetry"); + return new CommandResult + { + Succeeded = true, + Message = "Fresh telemetry collection requested", + }; + + default: + logger.LogWarning("Rejected unknown command action {Action}", action); + return new CommandResult + { + Succeeded = false, + Message = $"Unknown or disallowed action: {action}", + }; + } + } + + // Guard: a generic action with no target has nothing to act on. Returns a + // failed result to short-circuit, or null when the target is present. + private CommandResult? RequireTarget(string action, string target) + { + if (!string.IsNullOrWhiteSpace(target)) + return null; + logger.LogWarning("Action {Action} arrived with no target", action); + return new CommandResult + { + Succeeded = false, + Message = $"Action {action} requires a target", + }; + } + + // ---- Fixed destructive actions ---- + + // Reboot the machine now. shutdown.exe with /t 0 gives no grace timer; the + // WebUI confirmation gate is the safety, not a countdown. As SYSTEM the + // agent already holds the privilege, so no session or elevation dance. + private CommandResult SystemRestart() + { + try + { + var restart = new ProcessStartInfo + { + FileName = "shutdown.exe", + Arguments = "/r /t 0", + UseShellExecute = false, + CreateNoWindow = true, + }; + using var process = Process.Start(restart); + logger.LogWarning("system_restart issued: rebooting now"); + return new CommandResult + { + Succeeded = true, + Message = "Restart initiated", + }; + } + catch (Exception error) + { + logger.LogError(error, "system_restart failed"); + return new CommandResult + { + Succeeded = false, + Message = $"Failed to initiate restart: {error.Message}", + }; + } + } + + // ---- Generic service primitives ---- + + private CommandResult ServiceControl( + string serviceName, ServiceControllerStatus desired) + { + try + { + using var controller = new ServiceController(serviceName); + + if (controller.Status == desired) + { + return new CommandResult + { + Succeeded = true, + Message = $"Service {serviceName} already {desired}", + Detail = ServiceDetail(controller), + }; + } + + if (desired == ServiceControllerStatus.Running) + controller.Start(); + else + controller.Stop(); + + controller.WaitForStatus(desired, startTimeout); + controller.Refresh(); + var reached = controller.Status == desired; + return new CommandResult + { + Succeeded = reached, + Message = reached + ? $"Service {serviceName} {desired}" + : $"Service {serviceName} did not reach {desired} within timeout", + Detail = ServiceDetail(controller), + }; + } + catch (Exception error) + { + logger.LogWarning(error, "Failed to control service {Service}", serviceName); + return new CommandResult + { + Succeeded = false, + Message = $"Failed to control service {serviceName}: {error.Message}", + }; + } + } + + private CommandResult ServiceStatus(string serviceName) + { + try + { + using var controller = new ServiceController(serviceName); + return new CommandResult + { + Succeeded = true, + Message = $"Service {serviceName} is {controller.Status}", + Detail = ServiceDetail(controller), + }; + } + catch (Exception error) + { + logger.LogWarning(error, "Failed to read service {Service}", serviceName); + return new CommandResult + { + Succeeded = false, + Message = $"Failed to read service {serviceName}: {error.Message}", + }; + } + } + + private static Dictionary ServiceDetail(ServiceController controller) + { + controller.Refresh(); + return new Dictionary + { + ["kind"] = "service", + ["name"] = controller.ServiceName, + ["status"] = controller.Status.ToString(), + ["running"] = controller.Status == ServiceControllerStatus.Running, + }; + } + + // ---- Generic process primitives ---- + + private CommandResult ProcessStop(string image) + { + var processes = FindProcessesByImage(image); + if (processes.Count == 0) + { + return new CommandResult + { + Succeeded = true, + Message = $"Process {image} already stopped", + Detail = ProcessDetail(image, false), + }; + } + + var errors = new List(); + foreach (var process in processes) + { + try + { + process.Kill(entireProcessTree: true); + process.WaitForExit(10000); + } + catch (Exception error) + { + errors.Add(error.Message); + logger.LogWarning(error, "Failed to stop process {Pid}", process.Id); + } + finally + { + process.Dispose(); + } + } + + var running = IsProcessRunning(image); + return new CommandResult + { + Succeeded = !running, + Message = running + ? $"Process {image} still running after stop: {string.Join("; ", errors)}" + : $"Process {image} stopped", + Detail = ProcessDetail(image, running), + }; + } + + private CommandResult ProcessStatus(string image) + { + var running = IsProcessRunning(image); + return new CommandResult + { + Succeeded = true, + Message = running ? $"Process {image} running" : $"Process {image} stopped", + Detail = ProcessDetail(image, running), + }; + } + + private static Dictionary ProcessDetail(string image, bool running) + => new() + { + ["kind"] = "process", + ["name"] = image, + ["running"] = running, + }; + + // Start a process by image name, but ONLY if it resolves to a known-good + // launchable path (shipped defaults + appsettings overrides). The wire never + // carries a path, so a compromised WebUI cannot launch an arbitrary binary: + // an unmapped target fails closed. A mapped .bat is launched from its own + // directory; a mapped .exe is launched + // directly. Either way we do not wait for the launcher to exit - we poll for + // the image to appear and report the resulting run state. + private async Task ProcessStart(string image, CancellationToken token) + { + if (IsProcessRunning(image)) + { + return new CommandResult + { + Succeeded = true, + Message = $"Process {image} already running", + Detail = ProcessDetail(image, true), + }; + } + + var bare = image; + if (bare.EndsWith(".exe", StringComparison.OrdinalIgnoreCase)) + bare = bare[..^4]; + + if (!launchPaths.TryGetValue(bare, out var launchPath) + || string.IsNullOrWhiteSpace(launchPath)) + { + logger.LogWarning( + "Rejected process_start for {Image}: no launchable path configured", + image); + return new CommandResult + { + Succeeded = false, + Message = $"process_start is not permitted for {image}: no launch path configured", + Detail = ProcessDetail(image, false), + }; + } + + if (!File.Exists(launchPath)) + { + logger.LogWarning( + "process_start for {Image}: configured launch path does not exist: {Path}", + image, launchPath); + return new CommandResult + { + Succeeded = false, + Message = $"Launch path not found for {image}: {launchPath}", + Detail = ProcessDetail(image, false), + }; + } + + try + { + // Launch into the ACTIVE CONSOLE USER'S session, not session 0. The + // agent is a SYSTEM service; a plain Process.Start here would put the + // GUI app on the hidden session-0 desktop where the operator never + // sees it. The app starts at the user's medium integrity and + // CyberArk/EPM brokers any elevation afterward. + var workingDirectory = Path.GetDirectoryName(launchPath) ?? ""; + var outcome = InteractiveProcessLauncher.LaunchInActiveSession( + launchPath, workingDirectory); + if (!outcome.Launched) + { + logger.LogWarning( + "process_start could not launch {Image} in the user session: {Error}", + image, outcome.Error); + return new CommandResult + { + Succeeded = false, + Message = $"Failed to start {image}: {outcome.Error}", + Detail = ProcessDetail(image, false), + }; + } + + // The launcher (an .exe or a possibly long-lived run.bat) may not be + // the watched image itself, so poll for the target image to appear + // rather than waiting on the launcher to exit. + var deadline = DateTime.UtcNow + startTimeout; + while (DateTime.UtcNow < deadline) + { + if (IsProcessRunning(image)) + return new CommandResult + { + Succeeded = true, + Message = $"Process {image} started", + Detail = ProcessDetail(image, true), + }; + await Task.Delay(1000, token); + } + + return new CommandResult + { + Succeeded = false, + Message = $"Process {image} did not report running within the timeout", + Detail = ProcessDetail(image, IsProcessRunning(image)), + }; + } + catch (Exception ex) + { + logger.LogError(ex, "process_start failed for {Image} via {Path}", + image, launchPath); + return new CommandResult + { + Succeeded = false, + Message = $"Failed to start {image}: {ex.Message}", + Detail = ProcessDetail(image, false), + }; + } + } + + private bool IsProcessRunning(string image) + { + var processes = FindProcessesByImage(image); + try + { + return processes.Count > 0; + } + finally + { + foreach (var process in processes) + process.Dispose(); + } + } + + // Match a target image name against running processes. Accepts 'DncMain.exe' + // or 'DncMain' - Process.GetProcessesByName wants the bare name, so strip a + // trailing .exe before matching. + private static List FindProcessesByImage(string image) + { + var bare = image; + if (bare.EndsWith(".exe", StringComparison.OrdinalIgnoreCase)) + bare = bare[..^4]; + + var found = new List(); + try + { + found.AddRange(Process.GetProcessesByName(bare)); + } + catch + { + // Access-denied on a foreign session's process: not our agent's to + // see, so treat it as absent rather than failing the probe. + } + return found; + } + + // ---- Fixed MTConnect actions ---- + + private async Task MtConnectStartAsync(CancellationToken token) + { + if (MtConnectServiceRunning()) + { + return new CommandResult + { + Succeeded = true, + Message = "MTConnect already running", + MtConnectRunning = true, + }; + } + + // install.bat (Agent.exe install) installs or repairs the service. Run it + // as SYSTEM and wait for it to exit - unlike the long-lived run.bat, the + // installer is a quick one-shot. Missing installer is not fatal on its + // own: the service may already be registered, so fall through to start. + if (File.Exists(MtConnectInstallBat)) + { + try + { + var install = new ProcessStartInfo + { + FileName = MtConnectInstallBat, + WorkingDirectory = Path.GetDirectoryName(MtConnectInstallBat) ?? "", + UseShellExecute = false, + CreateNoWindow = true, + }; + using var installer = Process.Start(install); + if (installer is not null) + { + try + { + await installer.WaitForExitAsync(token); + } + catch (OperationCanceledException) + { + // Shutting down; leave the service in whatever state it + // reached and report the current status below. + } + } + } + catch (Exception error) + { + logger.LogWarning(error, "MTConnect install.bat failed"); + } + } + else + { + logger.LogWarning( + "MTConnect installer not found: {Installer}; attempting service start anyway", + MtConnectInstallBat); + } + + // install.bat may or may not leave the service started; ensure it is + // running via the same service-control path the generic primitives use. + var result = ServiceControl(MtConnectServiceName, ServiceControllerStatus.Running); + return new CommandResult + { + Succeeded = result.Succeeded, + Message = result.Succeeded ? "MTConnect started" : result.Message, + MtConnectRunning = MtConnectServiceRunning(), + Detail = result.Detail, + }; + } + + private CommandResult MtConnectStop() + { + var result = ServiceControl(MtConnectServiceName, ServiceControllerStatus.Stopped); + var running = MtConnectServiceRunning(); + return new CommandResult + { + Succeeded = result.Succeeded, + Message = running ? result.Message : "MTConnect stopped", + MtConnectRunning = running, + Detail = result.Detail, + }; + } + + private CommandResult MtConnectStatus() + { + var result = ServiceStatus(MtConnectServiceName); + var running = MtConnectServiceRunning(); + return new CommandResult + { + Succeeded = result.Succeeded, + Message = running ? "MTConnect running" : "MTConnect stopped", + MtConnectRunning = running, + Detail = result.Detail, + }; + } + + // Probe the MTConnect Windows service state. A missing/unqueryable service + // (not installed, access denied on a foreign session) reports not running + // rather than throwing, so the WebUI toggle degrades to "stopped". + private bool MtConnectServiceRunning() + { + try + { + using var controller = new ServiceController(MtConnectServiceName); + return controller.Status == ServiceControllerStatus.Running; + } + catch + { + return false; + } + } +} diff --git a/agent/ShopDbEndpointAgent.Core/CommandWorker.cs b/agent/ShopDbEndpointAgent.Core/CommandWorker.cs new file mode 100644 index 00000000..9389d805 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/CommandWorker.cs @@ -0,0 +1,143 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace ShopDbEndpointAgent.Core; + +// The command loop as a hosted BackgroundService, separate from TelemetryWorker +// so a slow or held-open command poll never delays a telemetry report and vice +// versa. It long-polls the server for an allowlisted command +// (plugins/endpoints/AGENT-CONTRACT.md section 7), runs it through +// CommandExecutor, and posts the result back. The server holds each poll open +// briefly, so this loop is effectively "wait for work, do it, immediately wait +// again" rather than a fixed-cadence poll. +// +// Like TelemetryWorker, one bad iteration never kills the loop: a permanent +// error (bad key, unknown host) backs off to a slow retry so it does not hammer +// the server, and any other failure is logged and retried after a short pause. +public sealed class CommandWorker : BackgroundService +{ + // Pause after a transient failure before re-polling, so a flapping network + // or a briefly-down server does not spin this loop. + private static readonly TimeSpan TransientRetryDelay = TimeSpan.FromSeconds(10); + + // Longer back-off after a permanent error (bad key, host removed): the + // operator has to fix something, so re-polling fast changes nothing. + private static readonly TimeSpan PermanentRetryDelay = TimeSpan.FromSeconds(300); + + private readonly CommandClient client; + private readonly CommandExecutor executor; + private readonly ILogger logger; + private readonly CollectSignal? collectSignal; + private readonly string hostname; + + public CommandWorker(CommandClient client, CommandExecutor executor, + ILogger logger, CollectSignal? collectSignal = null) + { + this.client = client; + this.executor = executor; + this.logger = logger; + this.collectSignal = collectSignal; + hostname = Environment.MachineName; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + logger.LogInformation("Command channel started for host {Host}", hostname); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + var claim = await client.ClaimNextAsync(hostname, stoppingToken); + if (claim is not null) + await RunClaimAsync(claim, stoppingToken); + // On a 204 (no work) the server already consumed the hold + // window, so re-poll immediately. + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (CollectorException error) when (error.IsPermanent) + { + logger.LogError(error, + "Permanent command-channel error; backing off"); + await SafeDelay(PermanentRetryDelay, stoppingToken); + } + catch (Exception error) when (IsIdlePollClose(error)) + { + // The server (or a proxy) closed the held-open long poll without + // a command. That is the normal end of a hold window, not a + // failure: log quietly at debug and re-poll immediately. + logger.LogDebug(error, "Command poll connection closed; re-polling"); + } + catch (Exception error) + { + logger.LogWarning(error, "Command poll failed; retrying shortly"); + await SafeDelay(TransientRetryDelay, stoppingToken); + } + } + + logger.LogInformation("Command channel stopped"); + } + + private async Task RunClaimAsync(CommandClaim claim, CancellationToken token) + { + logger.LogInformation("Running command {Id} action={Action}", + claim.EndpointCommandId, claim.Action); + + CommandResult result; + try + { + result = await executor.ExecuteAsync(claim, token); + } + catch (Exception error) + { + logger.LogError(error, "Command {Id} action={Action} threw", + claim.EndpointCommandId, claim.Action); + result = new CommandResult + { + Succeeded = false, + Message = $"Command failed: {error.Message}", + }; + } + + await client.ReportResultAsync(claim.EndpointCommandId, result, token); + logger.LogInformation("Command {Id} finished succeeded={Succeeded}", + claim.EndpointCommandId, result.Succeeded); + + // A command changed something on the endpoint (e.g. MTConnect start/stop); + // wake the telemetry loop to collect now so the WebUI card reflects the + // new state within one report cycle instead of after the full cadence. + collectSignal?.Request(); + } + + // A long poll that the server or an intermediary closes without sending a + // command surfaces as an HttpRequestException wrapping an HttpIOException + // (ResponseEnded), or a bare IOException. That is the expected end of a hold + // window, so it should not be treated as a transient failure. + private static bool IsIdlePollClose(Exception error) + { + for (Exception? current = error; current is not null; current = current.InnerException) + { + if (current is System.Net.Http.HttpIOException httpIo + && httpIo.HttpRequestError == System.Net.Http.HttpRequestError.ResponseEnded) + return true; + if (current is System.IO.IOException) + return true; + } + return false; + } + + private static async Task SafeDelay(TimeSpan delay, CancellationToken token) + { + try + { + await Task.Delay(delay, token); + } + catch (OperationCanceledException) + { + // Shutting down: let the loop condition end it. + } + } +} diff --git a/agent/ShopDbEndpointAgent.Core/Contracts.cs b/agent/ShopDbEndpointAgent.Core/Contracts.cs new file mode 100644 index 00000000..cb504a36 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/Contracts.cs @@ -0,0 +1,92 @@ +using System.Text.Json.Serialization; + +namespace ShopDbEndpointAgent.Core; + +// Wire keys MUST stay lowercase/no-underscore to match the ShopDB collector +// schema (plugins/endpoints/AGENT-CONTRACT.md). PascalCase C# properties are +// mapped back with JsonPropertyName so the serializer does not rename them. + +public sealed class EndpointReport +{ + [JsonPropertyName("hostname")] public string Hostname { get; set; } = ""; + [JsonPropertyName("adopt")] public bool? Adopt { get; set; } + [JsonPropertyName("loggedinuser")] public string? LoggedInUser { get; set; } + [JsonPropertyName("serialnumber")] public string? SerialNumber { get; set; } + [JsonPropertyName("pctype")] public string? PcType { get; set; } + [JsonPropertyName("vendorname")] public string? VendorName { get; set; } + [JsonPropertyName("modelnumber")] public string? ModelNumber { get; set; } + [JsonPropertyName("osname")] public string? OsName { get; set; } + [JsonPropertyName("ipaddress")] public string? IpAddress { get; set; } + [JsonPropertyName("machinenumber")] public string? MachineNumber { get; set; } + [JsonPropertyName("agentversion")] public string? AgentVersion { get; set; } + [JsonPropertyName("lastboottime")] public DateTimeOffset? LastBootTime { get; set; } + [JsonPropertyName("uptimeseconds")] public long? UptimeSeconds { get; set; } + [JsonPropertyName("idleseconds")] public long? IdleSeconds { get; set; } + [JsonPropertyName("cpupercent")] public double? CpuPercent { get; set; } + [JsonPropertyName("memorypercent")] public double? MemoryPercent { get; set; } + [JsonPropertyName("services")] public object[]? Services { get; set; } + [JsonPropertyName("ports")] public object[]? Ports { get; set; } + [JsonPropertyName("interfaces")] public object[]? Interfaces { get; set; } + [JsonPropertyName("scheduledtasks")] public object[]? ScheduledTasks { get; set; } + [JsonPropertyName("events")] public EndpointEventRecord[]? Events { get; set; } +} + +public sealed class EndpointEventRecord +{ + [JsonPropertyName("eventid")] public int EventId { get; set; } + [JsonPropertyName("eventchannel")] public string? EventChannel { get; set; } + [JsonPropertyName("eventsource")] public string? EventSource { get; set; } + [JsonPropertyName("eventtime")] public DateTimeOffset? EventTime { get; set; } + [JsonPropertyName("eventmessage")] public string? EventMessage { get; set; } + [JsonPropertyName("fields")] public Dictionary? Fields { get; set; } +} + +public sealed class CollectorResponse +{ + [JsonPropertyName("status")] public string Status { get; set; } = ""; + [JsonPropertyName("action")] public string? Action { get; set; } + [JsonPropertyName("assetid")] public int? AssetId { get; set; } + [JsonPropertyName("identityvalue")] public string? IdentityValue { get; set; } + [JsonPropertyName("warnings")] public string[]? Warnings { get; set; } +} + +// The collector wraps the plugin payload in the ShopDB envelope +// { status, data: { action, assetid, identityvalue, warnings }, meta }. +public sealed class CollectorEnvelope +{ + [JsonPropertyName("status")] public string Status { get; set; } = ""; + [JsonPropertyName("data")] public CollectorResponse? Data { get; set; } +} + +// A command the server claimed for this endpoint on the long-poll. The +// allowlisted action name crosses the wire (see AGENT-CONTRACT.md section 7); +// generic actions (service_* / process_*) also carry the subject to act on - +// Target is a Windows service name or a process image (for example +// 'DncMain.exe') and TargetKind is 'service' or 'process'. Fixed-target actions +// (mtconnect_*) leave both null. Carrying the target on the wire is what lets a +// site build the agent once and drive every future policy/app from server data. +public sealed class CommandClaim +{ + [JsonPropertyName("endpointcommandid")] public int EndpointCommandId { get; set; } + [JsonPropertyName("action")] public string Action { get; set; } = ""; + [JsonPropertyName("target")] public string? Target { get; set; } + [JsonPropertyName("targetkind")] public string? TargetKind { get; set; } +} + +// Envelope for the claim long-poll: { status, data: { endpointcommandid, action } }. +public sealed class CommandClaimEnvelope +{ + [JsonPropertyName("status")] public string Status { get; set; } = ""; + [JsonPropertyName("data")] public CommandClaim? Data { get; set; } +} + +// The outcome the agent posts back after running a claimed command. When the +// action probed MTConnect, MtConnectRunning carries the observed state so the +// server can cache it for the WebUI toggle. +public sealed class CommandResult +{ + [JsonPropertyName("succeeded")] public bool Succeeded { get; set; } + [JsonPropertyName("message")] public string? Message { get; set; } + [JsonPropertyName("mtconnectrunning")] public bool? MtConnectRunning { get; set; } + [JsonPropertyName("detail")] public Dictionary? Detail { get; set; } +} diff --git a/agent/ShopDbEndpointAgent.Core/CuratedEventCatalog.cs b/agent/ShopDbEndpointAgent.Core/CuratedEventCatalog.cs new file mode 100644 index 00000000..57b44d7f --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/CuratedEventCatalog.cs @@ -0,0 +1,44 @@ +namespace ShopDbEndpointAgent.Core; + +// The curated Windows Event catalog, LOCKED to the server list in +// plugins/endpoints/plugin.py (CURATED_EVENT_IDS) and documented in +// plugins/endpoints/DESIGN-DECISIONS-agent.md section 4. The server stores ONLY +// these IDs; anything else in a report's `events` is ignored (not an error). +// Sending only these keeps the payload small and mirrors the catalog intent. +// +// Channels matter: an ID is read from its own log, so the collector reads +// Security, System, and Application - not System alone. IDs are namespaced by +// channel here because the same numeric ID can mean different things across +// logs; we only harvest the (channel, id) pairs the server curates. +public static class CuratedEventCatalog +{ + public sealed record Entry(int EventId, string Channel, string Meaning); + + public static readonly IReadOnlyList Entries = new List + { + new(4624, "Security", "Successful logon"), + new(4625, "Security", "Failed logon"), + new(4634, "Security", "Logoff"), + new(4672, "Security", "Special privileges assigned to new logon"), + new(4697, "Security", "Service installed"), + new(4698, "Security", "Scheduled task created"), + new(4618, "Security", "Monitored security event pattern"), + new(41, "System", "Kernel-Power, unexpected shutdown"), + new(55, "System", "NTFS file system corruption"), + new(7011, "System", "Service Control Manager timeout"), + new(10016, "System", "DCOM permission error"), + new(1001, "Application", "Application error / bugcheck report"), + }; + + // The distinct channels the collector must read. + public static readonly IReadOnlyList Channels = + Entries.Select(entry => entry.Channel).Distinct().ToList(); + + // The set of curated IDs for a given channel, for a fast membership check + // while walking that channel's log newest-first. + public static IReadOnlySet IdsForChannel(string channel) + => Entries.Where(entry => + string.Equals(entry.Channel, channel, StringComparison.OrdinalIgnoreCase)) + .Select(entry => entry.EventId) + .ToHashSet(); +} diff --git a/agent/ShopDbEndpointAgent.Core/EndpointCollector.cs b/agent/ShopDbEndpointAgent.Core/EndpointCollector.cs new file mode 100644 index 00000000..4154db63 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/EndpointCollector.cs @@ -0,0 +1,489 @@ +using System.Diagnostics; +using System.Management; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Win32; + +namespace ShopDbEndpointAgent.Core; + +// Pure-C# endpoint collector implementing ICollector. Mirrors the ShopDB +// PowerShell asset-collector data set (minus the CSV emit): hostname, logged-in +// user, serial number, pc type, boot time, monitored processes, plus a curated +// Windows Event Log tail across the Security/System/Application channels the +// server curates (CuratedEventCatalog). WMI/registry/event access can throw or +// return null on a locked-down box, so every probe is defensive and never +// aborts the report. +public sealed class EndpointCollector : ICollector +{ + // Max curated events harvested per channel per report, newest-first, to + // keep the payload bounded on a chatty Security log. + private const int MaxEventsPerChannel = 25; + + private readonly ILogger logger; + private readonly AgentConfig config; + private readonly IAgentUtilityRunner? utilities; + + public EndpointCollector(ILogger logger, AgentConfig config, + IAgentUtilityRunner? utilities = null) + { + this.logger = logger; + this.config = config; + this.utilities = utilities; + } + + public EndpointReport Collect(string agentVersion, bool adopt) + { + var report = new EndpointReport + { + Hostname = Environment.MachineName, + AgentVersion = agentVersion, + Adopt = adopt ? true : null, + LoggedInUser = GetLoggedInUser(), + SerialNumber = GetSerialNumber(), + PcType = GetPcType(), + VendorName = GetVendor(), + ModelNumber = GetModelNumber(), + OsName = GetOsName(), + IpAddress = GetPrimaryIpAddress(), + MachineNumber = GetMachineNumber(), + LastBootTime = GetLastBootTime(), + Services = GetMonitoredTargets(), + Interfaces = GetInterfaces(), + Events = GetCuratedEvents(), + }; + + if (report.LastBootTime is DateTimeOffset boot) + report.UptimeSeconds = (long)(DateTimeOffset.UtcNow - boot).TotalSeconds; + + return report; + } + + // Combined watch list: interactive processes (by image name) and Windows + // services (by service key name). Both land in report.Services with a shared + // {name, app, kind, state, running, count} shape so the server/UI renders + // one list of health badges. + private object[] GetMonitoredTargets() + { + var result = new List(); + result.AddRange(GetMonitoredProcesses()); + result.AddRange(GetMonitoredServices()); + return result.ToArray(); + } + + private IEnumerable GetMonitoredProcesses() + { + var targets = config.Monitors?.Processes ?? Array.Empty(); + if (targets.Length == 0) + return Array.Empty(); + + var running = new Dictionary(StringComparer.OrdinalIgnoreCase); + try + { + foreach (var proc in Process.GetProcesses()) + { + string image; + try { image = proc.ProcessName + ".exe"; } + catch { continue; } + finally { proc.Dispose(); } + + if (Array.FindIndex(targets, t => + string.Equals(t.Name, image, StringComparison.OrdinalIgnoreCase)) < 0) + continue; + + running.TryGetValue(image, out int count); + running[image] = count + 1; + } + } + catch (Exception error) + { + logger.LogDebug(error, "Process enumeration blocked"); + } + + var result = new List(targets.Length); + foreach (var target in targets) + { + bool found = running.TryGetValue(target.Name, out int count); + result.Add(new + { + name = target.Name, + app = string.IsNullOrWhiteSpace(target.App) ? target.Name : target.App, + kind = "process", + // state string so the UI badge can color it; missing = unknown. + state = found ? "running" : "stopped", + running = found, + count = found ? count : 0, + }); + } + return result; + } + + // Windows services watched by service key name. Reports the reported state + // (running/stopped/etc.) from Win32_Service so the server can flag a + // collector service (MTConnect and the like) that has stopped or gone + // missing. Never aborts: a locked-down WMI hive leaves entries "unknown". + private IEnumerable GetMonitoredServices() + { + var targets = config.Monitors?.Services ?? Array.Empty(); + if (targets.Length == 0) + return Array.Empty(); + + var states = new Dictionary(StringComparer.OrdinalIgnoreCase); + try + { + using var searcher = new ManagementObjectSearcher( + "SELECT Name, DisplayName, State FROM Win32_Service"); + foreach (var mo in searcher.Get()) + { + var state = mo["State"]?.ToString()?.Trim() ?? ""; + // Index by both service key name and display name so a config + // that used either one still matches. + var name = mo["Name"]?.ToString(); + if (!string.IsNullOrWhiteSpace(name)) + states[name.Trim()] = state; + var display = mo["DisplayName"]?.ToString(); + if (!string.IsNullOrWhiteSpace(display)) + states[display.Trim()] = state; + } + } + catch (Exception error) + { + logger.LogDebug(error, "Service enumeration blocked"); + } + + var result = new List(targets.Length); + foreach (var target in targets) + { + bool found = states.TryGetValue(target.Name, out var raw); + bool running = string.Equals(raw, "Running", StringComparison.OrdinalIgnoreCase); + result.Add(new + { + name = target.Name, + app = string.IsNullOrWhiteSpace(target.App) ? target.Name : target.App, + kind = "service", + // "unknown" = not installed / WMI unreadable; else lowercased + // Win32_Service state (running, stopped, start pending, ...). + state = found ? (raw ?? "").ToLowerInvariant() : "unknown", + running, + count = running ? 1 : 0, + }); + } + return result; + } + + private string? GetLoggedInUser() + { + var user = QueryFirst("SELECT UserName FROM Win32_ComputerSystem", "UserName"); + return string.IsNullOrWhiteSpace(user) + ? $"{Environment.UserDomainName}\\{Environment.UserName}" + : user; + } + + private string? GetSerialNumber() + => QueryFirst("SELECT SerialNumber FROM Win32_BIOS", "SerialNumber"); + + private string? GetPcType() + { + // Raw WMI model; the ShopDB pctype map normalizes it server-side, so the + // current site's catalog stays the single source of that mapping. + var model = QueryFirst("SELECT Model FROM Win32_ComputerSystem", "Model"); + return string.IsNullOrWhiteSpace(model) ? null : model; + } + + // Manufacturer for the Asset vendor/model rails (Win32_ComputerSystem + // Manufacturer, for example "Dell Inc.", "HP"). Server upserts Vendor. + private string? GetVendor() + => QueryFirst("SELECT Manufacturer FROM Win32_ComputerSystem", "Manufacturer"); + + // Hardware model number. Prefer the human-readable Win32_ComputerSystem + // Model (for example "OptiPlex 7090", "Latitude 5540") - the same value + // pctype classifies on. SystemSKUNumber is an OEM SKU code (for example + // Dell's "0BCC"), not a model name, so it is only a last resort when Model + // is blank. + private string? GetModelNumber() + { + var model = QueryFirst("SELECT Model FROM Win32_ComputerSystem", "Model"); + if (!string.IsNullOrWhiteSpace(model)) + return model; + return QueryFirst("SELECT SystemSKUNumber FROM Win32_ComputerSystem", + "SystemSKUNumber"); + } + + // OS caption, for example "Microsoft Windows 10 Enterprise". The server + // matches this against its controlled OperatingSystem vocab (warns if the + // exact string is unknown), so it may need a catalog entry to bind. + private string? GetOsName() + => QueryFirst("SELECT Caption FROM Win32_OperatingSystem", "Caption"); + + // Primary IPv4 address of the first enabled, non-loopback adapter with a + // default gateway. Feeds the Communication (IP) row server-side. + private string? GetPrimaryIpAddress() + { + try + { + using var searcher = new ManagementObjectSearcher( + "SELECT IPAddress, DefaultIPGateway, IPEnabled " + + "FROM Win32_NetworkAdapterConfiguration WHERE IPEnabled = True"); + string? firstEnabled = null; + foreach (var mo in searcher.Get()) + { + var addresses = mo["IPAddress"] as string[]; + if (addresses is null) + continue; + var ipv4 = Array.Find(addresses, a => + !string.IsNullOrWhiteSpace(a) && a.IndexOf(':') < 0); + if (string.IsNullOrWhiteSpace(ipv4)) + continue; + + firstEnabled ??= ipv4; + // Prefer an adapter that actually has a default gateway (the + // real LAN NIC, not a virtual/VPN interface). + if (mo["DefaultIPGateway"] is string[] gw && gw.Length > 0) + return ipv4; + } + return firstEnabled; + } + catch (Exception error) + { + logger.LogDebug(error, "IP address query failed"); + return null; + } + } + + // All hardware NICs, not just the primary. DESIGN-DECISIONS-agent.md 9.3: + // upwards of two-thirds of shop-floor PCs carry a SECOND, static, gatewayless + // NIC whose only job is to talk to the machine's onboard IPS. That interface + // is the physical PC-to-machine link and is exactly the one a single + // primary-IP probe throws away, so we report every enabled adapter with its + // alias/ipv4/mac/gateway/subnet and let the server keep the whole picture. + // Stored verbatim as JSON server-side (AGENT-CONTRACT.md), so keys are + // lowercase/no-underscore. + private object[] GetInterfaces() + { + var result = new List(); + try + { + using var searcher = new ManagementObjectSearcher( + "SELECT Description, IPAddress, IPSubnet, DefaultIPGateway, " + + "MACAddress, DHCPEnabled FROM Win32_NetworkAdapterConfiguration " + + "WHERE IPEnabled = True"); + foreach (var mo in searcher.Get()) + { + var mac = mo["MACAddress"] as string; + // No MAC = not a real hardware adapter (some tunnels/virtuals). + if (string.IsNullOrWhiteSpace(mac)) + continue; + + var addresses = mo["IPAddress"] as string[]; + var ipv4 = addresses is null ? null : Array.Find(addresses, + a => !string.IsNullOrWhiteSpace(a) && a.IndexOf(':') < 0); + + var subnets = mo["IPSubnet"] as string[]; + var subnet = subnets is null ? null : Array.Find(subnets, + s => !string.IsNullOrWhiteSpace(s) && s.IndexOf(':') < 0); + + var gateways = mo["DefaultIPGateway"] as string[]; + var gateway = gateways is null || gateways.Length == 0 + ? null : gateways[0]; + + result.Add(new + { + alias = mo["Description"]?.ToString()?.Trim() ?? "", + ipv4 = ipv4 ?? "", + mac = mac.Trim(), + // A gatewayless NIC is the likely machine-IPS link; the + // gateway NIC is the corporate LAN. Server/UI can tell them + // apart from these two fields. + gateway = gateway ?? "", + subnet = subnet ?? "", + dhcp = mo["DHCPEnabled"] as bool? ?? false, + }); + } + } + catch (Exception error) + { + logger.LogDebug(error, "Interface enumeration blocked"); + } + return result.ToArray(); + } + + // The four-digit ShopDB Machine Number, discovered from on-box app config. + // DESIGN-DECISIONS-agent.md 9.4/9.5: when eDNC or UDC is installed its config + // carries the number that binds this PC to a ShopDB machines row. Source + // priority: eDNC live registry first, then UDC settings JSON. "" or "9999" + // means none (AGENT-CONTRACT.md). Never throws out to the caller. + private string? GetMachineNumber() + { + var fromRegistry = GetMachineNumberFromEdncRegistry(); + if (IsUsableMachineNumber(fromRegistry)) + return fromRegistry!.Trim(); + + var fromUdc = GetMachineNumberFromUdcSettings(); + if (IsUsableMachineNumber(fromUdc)) + return fromUdc!.Trim(); + + return null; + } + + private static bool IsUsableMachineNumber(string? value) + => !string.IsNullOrWhiteSpace(value) && value.Trim() != "9999"; + + // eDNC writes MachineNo under the GE Aircraft Engines\DNC\General key. Try + // the 64-bit view then the 32-bit (WOW6432Node) view, mirroring + // AgentConfigProvider so a locked-down or 32-bit-installed box still reads. + private string? GetMachineNumberFromEdncRegistry() + { + const string subKey = @"SOFTWARE\GE Aircraft Engines\DNC\General"; + RegistryView[] views = { RegistryView.Registry64, RegistryView.Registry32 }; + foreach (var view in views) + { + try + { + using var machine = RegistryKey.OpenBaseKey( + RegistryHive.LocalMachine, view); + using var key = machine.OpenSubKey(subKey); + var value = key?.GetValue("MachineNo")?.ToString(); + if (IsUsableMachineNumber(value)) + return value; + } + catch (Exception error) + { + logger.LogDebug(error, + "eDNC registry read blocked in {View} view", view); + } + } + return null; + } + + // UDC stores GeneralSettings.MachineNumber in udc_settings_*.json under + // %ProgramData%\udc. Scan for the newest settings file and read the number. + private string? GetMachineNumberFromUdcSettings() + { + try + { + var root = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), + "udc"); + if (!Directory.Exists(root)) + return null; + + var files = Directory.EnumerateFiles( + root, "udc_settings*.json", SearchOption.AllDirectories) + .OrderByDescending(File.GetLastWriteTimeUtc); + + foreach (var file in files) + { + try + { + using var doc = JsonDocument.Parse(File.ReadAllText(file)); + if (doc.RootElement.TryGetProperty("GeneralSettings", out var general) + && general.TryGetProperty("MachineNumber", out var number)) + { + var value = number.ValueKind == JsonValueKind.String + ? number.GetString() + : number.ToString(); + if (IsUsableMachineNumber(value)) + return value; + } + } + catch (Exception error) + { + logger.LogDebug(error, "UDC settings parse failed: {File}", file); + } + } + } + catch (Exception error) + { + logger.LogDebug(error, "UDC settings scan failed"); + } + return null; + } + + private DateTimeOffset? GetLastBootTime() + { + var raw = QueryFirst( + "SELECT LastBootUpTime FROM Win32_OperatingSystem", "LastBootUpTime"); + if (string.IsNullOrWhiteSpace(raw)) + return null; + try + { + return ManagementDateTimeConverter.ToDateTime(raw).ToUniversalTime(); + } + catch (Exception error) + { + logger.LogDebug(error, "LastBootUpTime parse failed"); + return null; + } + } + + // Walk each curated channel newest-first, keep only the curated IDs for + // that channel, cap per channel. Reads Security/System/Application - the + // old scaffold read System only and filtered by the wrong IDs. + private EndpointEventRecord[] GetCuratedEvents() + { + var records = new List(); + foreach (var channel in CuratedEventCatalog.Channels) + { + var wanted = CuratedEventCatalog.IdsForChannel(channel); + try + { + using var log = new EventLog(channel); + int kept = 0; + for (int i = log.Entries.Count - 1; + i >= 0 && kept < MaxEventsPerChannel; i--) + { + EventLogEntry entry; + try { entry = log.Entries[i]; } catch { continue; } + + int id = (int)(entry.InstanceId & 0xFFFF); + if (!wanted.Contains(id)) + continue; + + records.Add(new EndpointEventRecord + { + EventId = id, + EventChannel = channel, + EventSource = entry.Source, + EventTime = new DateTimeOffset( + entry.TimeGenerated.ToUniversalTime(), TimeSpan.Zero), + EventMessage = Truncate(entry.Message, 1000), + }); + kept++; + } + } + catch (Exception error) + { + // Channel unreadable (Security needs elevation; a log may be + // off): log and continue with the other channels. + logger.LogDebug(error, "Event channel {Channel} unreadable", channel); + } + } + return records.ToArray(); + } + + private string? QueryFirst(string wql, string property) + { + try + { + using var searcher = new ManagementObjectSearcher(wql); + foreach (var mo in searcher.Get()) + { + var value = mo[property]?.ToString(); + if (!string.IsNullOrWhiteSpace(value)) + return value.Trim(); + } + } + catch (Exception error) + { + logger.LogDebug(error, "WMI query failed: {Query}", wql); + } + return null; + } + + private static string Truncate(string? text, int max) + { + if (string.IsNullOrEmpty(text)) + return ""; + text = text.Trim(); + return text.Length <= max ? text : text[..max]; + } +} diff --git a/agent/ShopDbEndpointAgent.Core/FileLoggerProvider.cs b/agent/ShopDbEndpointAgent.Core/FileLoggerProvider.cs new file mode 100644 index 00000000..fa374105 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/FileLoggerProvider.cs @@ -0,0 +1,164 @@ +using System.Collections.Concurrent; +using System.Text; +using Microsoft.Extensions.Logging; + +namespace ShopDbEndpointAgent.Core; + +// A dependency-free rolling-file ILoggerProvider. The runtime code only ever +// talks to ILogger, so swapping or adding sinks later (a ShopDB HTTP sink, Seq, +// syslog) is a host-builder change with zero worker changes. This provider +// writes one line per entry to a daily file under the configured log directory +// and prunes files older than the retention window. +public sealed class FileLoggerProvider : ILoggerProvider +{ + private readonly string directory; + private readonly LogLevel minLevel; + private readonly int retentionDays; + private readonly object gate = new(); + private readonly ConcurrentDictionary loggers = new(); + private string currentDateStamp = ""; + private StreamWriter? writer; + + public FileLoggerProvider(string directory, LogLevel minLevel = LogLevel.Information, + int retentionDays = 14) + { + this.directory = directory; + this.minLevel = minLevel; + this.retentionDays = retentionDays; + // Best-effort: creating the log directory can fail on a locked-down host + // or when a non-privileged user targets a SYSTEM-owned path. Never let + // that abort construction; Write() degrades to a no-op if the sink is + // unusable. + try { Directory.CreateDirectory(directory); } + catch (Exception error) when (error is IOException or UnauthorizedAccessException) { } + } + + // The default log directory when config leaves LogPath empty: + // %ProgramData%\ShopDB\EndpointAgent\logs + public static string DefaultDirectory() + => Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), + "ShopDB", "EndpointAgent", "logs"); + + public ILogger CreateLogger(string categoryName) + => loggers.GetOrAdd(categoryName, name => new FileLogger(this, name)); + + internal bool IsEnabled(LogLevel level) => level >= minLevel && level != LogLevel.None; + + internal void Write(string category, LogLevel level, string message, Exception? error) + { + var now = DateTimeOffset.Now; + var line = new StringBuilder() + .Append(now.ToString("yyyy-MM-dd HH:mm:ss.fff zzz")) + .Append(" [").Append(LevelTag(level)).Append("] ") + .Append(ShortCategory(category)).Append(": ") + .Append(message); + if (error is not null) + line.Append(Environment.NewLine).Append(error); + + lock (gate) + { + // Logging must never take down the agent. If the log file cannot be + // opened or written (for example a non-privileged user running the + // exe against a logs directory owned by the SYSTEM service, which + // yields UnauthorizedAccessException on open), swallow it and drop + // the line rather than letting it unwind through bootstrap. A failed + // sink is a degraded state, not a fatal one. + try + { + RollIfNeeded(now); + writer?.WriteLine(line.ToString()); + } + catch (Exception sinkError) when (sinkError is IOException or UnauthorizedAccessException) + { + writer = null; + } + } + } + + private void RollIfNeeded(DateTimeOffset now) + { + var stamp = now.ToString("yyyyMMdd"); + if (stamp == currentDateStamp && writer is not null) + return; + + writer?.Flush(); + writer?.Dispose(); + currentDateStamp = stamp; + var path = Path.Combine(directory, $"agent-{stamp}.log"); + writer = new StreamWriter(new FileStream( + path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite)) + { + AutoFlush = true, + }; + PruneOldFiles(); + } + + private void PruneOldFiles() + { + try + { + var cutoff = DateTime.Now.AddDays(-retentionDays); + foreach (var file in Directory.EnumerateFiles(directory, "agent-*.log")) + { + if (File.GetLastWriteTime(file) < cutoff) + File.Delete(file); + } + } + catch + { + // Pruning is best-effort; never let housekeeping break logging. + } + } + + private static string LevelTag(LogLevel level) => level switch + { + LogLevel.Trace => "TRC", + LogLevel.Debug => "DBG", + LogLevel.Information => "INF", + LogLevel.Warning => "WRN", + LogLevel.Error => "ERR", + LogLevel.Critical => "CRT", + _ => "---", + }; + + private static string ShortCategory(string category) + { + var dot = category.LastIndexOf('.'); + return dot >= 0 && dot < category.Length - 1 ? category[(dot + 1)..] : category; + } + + public void Dispose() + { + lock (gate) + { + writer?.Flush(); + writer?.Dispose(); + writer = null; + } + } + + private sealed class FileLogger : ILogger + { + private readonly FileLoggerProvider provider; + private readonly string category; + + public FileLogger(FileLoggerProvider provider, string category) + { + this.provider = provider; + this.category = category; + } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => provider.IsEnabled(logLevel); + + public void Log(LogLevel logLevel, EventId eventId, TState state, + Exception? exception, Func formatter) + { + if (!IsEnabled(logLevel)) + return; + provider.Write(category, logLevel, formatter(state, exception), exception); + } + } +} diff --git a/agent/ShopDbEndpointAgent.Core/ICollector.cs b/agent/ShopDbEndpointAgent.Core/ICollector.cs new file mode 100644 index 00000000..7be48930 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/ICollector.cs @@ -0,0 +1,24 @@ +namespace ShopDbEndpointAgent.Core; + +// The collection surface. The runtime worker depends only on this, so the +// concrete implementation (pure C# today, possibly C# + PowerShell utilities +// later) can change without touching the worker or the HTTP client. +public interface ICollector +{ + EndpointReport Collect(string agentVersion, bool adopt); +} + +// The documented seam for site-provided PowerShell "agent utilities". The +// agent configuration settings page (server-side, future) lets site staff load +// their own .ps1 scripts as utilities; the runtime discovers them under the +// configured utilities directory and merges their JSON output into the report. +// +// This iteration ships the interface and a discovery contract only - the +// concrete runner is added when the settings page can push utilities down. +public interface IAgentUtilityRunner +{ + // Run every discovered utility and return a map of utility name -> parsed + // JSON result, to be attached to the report. Never throws: a failing + // utility is logged and skipped so it cannot break the telemetry run. + IReadOnlyDictionary RunAll(CancellationToken token); +} diff --git a/agent/ShopDbEndpointAgent.Core/InteractiveProcessLauncher.cs b/agent/ShopDbEndpointAgent.Core/InteractiveProcessLauncher.cs new file mode 100644 index 00000000..3b028bcb --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/InteractiveProcessLauncher.cs @@ -0,0 +1,200 @@ +using System.ComponentModel; +using System.Runtime.InteropServices; + +namespace ShopDbEndpointAgent.Core; + +// Launches a program in the ACTIVE CONSOLE USER'S session at that user's +// integrity, from the agent's LocalSystem service (session 0). +// +// Why this exists: the agent runs as a session-0 service. A plain +// Process.Start() there launches GUI apps (DncMain.exe, UDC.exe) onto the hidden +// session-0 desktop, so the logged-in operator never sees a window even though +// the process is running. process_start needs the app to appear on the user's +// desktop, so we obtain the interactive user's token and CreateProcessAsUser. +// +// Elevation is deliberately NOT handled here: the process starts at the user's +// normal (medium) integrity and CyberArk/EPM brokers any elevation afterward. +// +// Only process_start uses this. Stop/status/service actions stay as SYSTEM. +public static class InteractiveProcessLauncher +{ + public readonly record struct LaunchOutcome(bool Launched, string? Error); + + private const int TokenDuplicate = 0x0002; + private const int TokenQuery = 0x0008; + private const int TokenAssignPrimary = 0x0001; + private const int TokenAdjustPrivileges = 0x0020; + private const int TokenAdjustDefault = 0x0080; + private const int TokenAdjustSessionId = 0x0100; + + private const uint CreateUnicodeEnvironment = 0x00000400; + private const uint CreateNoWindow = 0x08000000; + + private const uint InvalidSessionId = 0xFFFFFFFF; + + // Returns Launched=true when CreateProcessAsUser succeeds. Never throws; + // failures come back as Launched=false with a descriptive Error so the + // caller can surface a clean CommandResult. + public static LaunchOutcome LaunchInActiveSession( + string applicationPath, string? workingDirectory) + { + var sessionId = WTSGetActiveConsoleSessionId(); + if (sessionId == InvalidSessionId) + return new LaunchOutcome(false, + "No active console session (no interactive user is signed in)."); + + if (!WTSQueryUserToken(sessionId, out var userToken)) + return new LaunchOutcome(false, + "Could not obtain the interactive user token (WTSQueryUserToken failed): " + + new Win32Exception(Marshal.GetLastWin32Error()).Message); + + var duplicatedToken = IntPtr.Zero; + var environmentBlock = IntPtr.Zero; + try + { + var desiredAccess = TokenAssignPrimary | TokenDuplicate | TokenQuery + | TokenAdjustPrivileges | TokenAdjustDefault | TokenAdjustSessionId; + if (!DuplicateTokenEx(userToken, (uint)desiredAccess, IntPtr.Zero, + SecurityImpersonationLevel.SecurityImpersonation, + TokenType.TokenPrimary, out duplicatedToken)) + { + return new LaunchOutcome(false, + "Could not duplicate the user token (DuplicateTokenEx failed): " + + new Win32Exception(Marshal.GetLastWin32Error()).Message); + } + + // A per-user environment block so the launched app sees the user's + // profile variables rather than SYSTEM's. + if (!CreateEnvironmentBlock(out environmentBlock, duplicatedToken, false)) + environmentBlock = IntPtr.Zero; + + var startupInfo = new STARTUPINFO(); + startupInfo.cb = Marshal.SizeOf(startupInfo); + // Target the interactive desktop so the window is visible. + startupInfo.lpDesktop = @"winsta0\default"; + + var creationFlags = CreateUnicodeEnvironment | CreateNoWindow; + + var launched = CreateProcessAsUser( + duplicatedToken, + applicationPath, + null, + IntPtr.Zero, + IntPtr.Zero, + false, + creationFlags, + environmentBlock, + workingDirectory, + ref startupInfo, + out var processInformation); + + if (!launched) + { + return new LaunchOutcome(false, + "CreateProcessAsUser failed: " + + new Win32Exception(Marshal.GetLastWin32Error()).Message); + } + + if (processInformation.hProcess != IntPtr.Zero) + CloseHandle(processInformation.hProcess); + if (processInformation.hThread != IntPtr.Zero) + CloseHandle(processInformation.hThread); + + return new LaunchOutcome(true, null); + } + finally + { + if (environmentBlock != IntPtr.Zero) + DestroyEnvironmentBlock(environmentBlock); + if (duplicatedToken != IntPtr.Zero) + CloseHandle(duplicatedToken); + if (userToken != IntPtr.Zero) + CloseHandle(userToken); + } + } + + private enum SecurityImpersonationLevel + { + SecurityAnonymous, + SecurityIdentification, + SecurityImpersonation, + SecurityDelegation, + } + + private enum TokenType + { + TokenPrimary = 1, + TokenImpersonation, + } + + [StructLayout(LayoutKind.Sequential)] + private struct PROCESS_INFORMATION + { + public IntPtr hProcess; + public IntPtr hThread; + public uint dwProcessId; + public uint dwThreadId; + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] + private struct STARTUPINFO + { + public int cb; + public string? lpReserved; + public string? lpDesktop; + public string? lpTitle; + public uint dwX; + public uint dwY; + public uint dwXSize; + public uint dwYSize; + public uint dwXCountChars; + public uint dwYCountChars; + public uint dwFillAttribute; + public uint dwFlags; + public ushort wShowWindow; + public ushort cbReserved2; + public IntPtr lpReserved2; + public IntPtr hStdInput; + public IntPtr hStdOutput; + public IntPtr hStdError; + } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern uint WTSGetActiveConsoleSessionId(); + + [DllImport("wtsapi32.dll", SetLastError = true)] + private static extern bool WTSQueryUserToken(uint sessionId, out IntPtr token); + + [DllImport("advapi32.dll", SetLastError = true)] + private static extern bool DuplicateTokenEx( + IntPtr existingToken, + uint desiredAccess, + IntPtr tokenAttributes, + SecurityImpersonationLevel impersonationLevel, + TokenType tokenType, + out IntPtr newToken); + + [DllImport("userenv.dll", SetLastError = true)] + private static extern bool CreateEnvironmentBlock( + out IntPtr environment, IntPtr token, bool inherit); + + [DllImport("userenv.dll", SetLastError = true)] + private static extern bool DestroyEnvironmentBlock(IntPtr environment); + + [DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Unicode)] + private static extern bool CreateProcessAsUser( + IntPtr token, + string? applicationName, + string? commandLine, + IntPtr processAttributes, + IntPtr threadAttributes, + bool inheritHandles, + uint creationFlags, + IntPtr environment, + string? currentDirectory, + ref STARTUPINFO startupInfo, + out PROCESS_INFORMATION processInformation); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool CloseHandle(IntPtr handle); +} diff --git a/agent/ShopDbEndpointAgent.Core/PowerShellUtilityRunner.cs b/agent/ShopDbEndpointAgent.Core/PowerShellUtilityRunner.cs new file mode 100644 index 00000000..e7c624b3 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/PowerShellUtilityRunner.cs @@ -0,0 +1,114 @@ +using System.Diagnostics; +using System.Text.Json; +using Microsoft.Extensions.Logging; + +namespace ShopDbEndpointAgent.Core; + +// Site-provided PowerShell "agent utilities" runner. Discovers *.ps1 under the +// configured utilities directory and runs each with the OS powershell.exe, +// parsing its stdout as JSON. This is the documented collection seam: the +// future agent-configuration settings page lets site staff push their own +// scripts down, and their JSON output is merged into the report under +// `scheduledtasks`-adjacent extension keys without the runtime knowing what any +// script means. +// +// Safety posture for this iteration: +// - Disabled unless a utilities directory is configured AND exists. +// - Each script is bounded by a timeout and its failure is isolated. +// - Output must be a single JSON value; non-JSON output is logged and dropped. +// Script SIGNING/allow-listing is a rollout concern (code signing + Intune) and +// is called out in the design doc; do not enable this in production until that +// is in place. +public sealed class PowerShellUtilityRunner : IAgentUtilityRunner +{ + private readonly string utilitiesPath; + private readonly ILogger logger; + private readonly TimeSpan perScriptTimeout; + + public PowerShellUtilityRunner(string utilitiesPath, + ILogger logger, TimeSpan? perScriptTimeout = null) + { + this.utilitiesPath = utilitiesPath; + this.logger = logger; + this.perScriptTimeout = perScriptTimeout ?? TimeSpan.FromSeconds(30); + } + + public IReadOnlyDictionary RunAll(CancellationToken token) + { + var results = new Dictionary(); + if (string.IsNullOrWhiteSpace(utilitiesPath) || !Directory.Exists(utilitiesPath)) + return results; + + foreach (var script in Directory.EnumerateFiles(utilitiesPath, "*.ps1")) + { + token.ThrowIfCancellationRequested(); + var name = Path.GetFileNameWithoutExtension(script); + try + { + var parsed = RunOne(script, token); + if (parsed is not null) + results[name] = parsed; + } + catch (Exception error) + { + logger.LogWarning(error, "Agent utility {Utility} failed", name); + } + } + return results; + } + + private object? RunOne(string scriptPath, CancellationToken token) + { + var start = new ProcessStartInfo + { + FileName = "powershell.exe", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + start.ArgumentList.Add("-NoProfile"); + start.ArgumentList.Add("-NonInteractive"); + start.ArgumentList.Add("-ExecutionPolicy"); + start.ArgumentList.Add("Bypass"); + start.ArgumentList.Add("-File"); + start.ArgumentList.Add(scriptPath); + + using var process = new Process { StartInfo = start }; + if (!process.Start()) + return null; + + var stdout = process.StandardOutput.ReadToEnd(); + var stderr = process.StandardError.ReadToEnd(); + + if (!process.WaitForExit((int)perScriptTimeout.TotalMilliseconds)) + { + try { process.Kill(entireProcessTree: true); } catch { } + logger.LogWarning("Agent utility {Script} timed out", scriptPath); + return null; + } + + if (process.ExitCode != 0) + { + logger.LogWarning("Agent utility {Script} exit {Code}: {Error}", + scriptPath, process.ExitCode, stderr.Trim()); + return null; + } + + stdout = stdout.Trim(); + if (stdout.Length == 0) + return null; + + try + { + using var document = JsonDocument.Parse(stdout); + return JsonSerializer.Deserialize(stdout); + } + catch (JsonException) + { + logger.LogWarning("Agent utility {Script} produced non-JSON output", + scriptPath); + return null; + } + } +} diff --git a/agent/ShopDbEndpointAgent.Core/ShopDbEndpointAgent.Core.csproj b/agent/ShopDbEndpointAgent.Core/ShopDbEndpointAgent.Core.csproj new file mode 100644 index 00000000..013080b0 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/ShopDbEndpointAgent.Core.csproj @@ -0,0 +1,22 @@ + + + + net8.0-windows + enable + enable + ShopDbEndpointAgent.Core + ShopDbEndpointAgent.Core + true + + + + + + + + + + + + + diff --git a/agent/ShopDbEndpointAgent.Core/TelemetryWorker.cs b/agent/ShopDbEndpointAgent.Core/TelemetryWorker.cs new file mode 100644 index 00000000..1fcd3412 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Core/TelemetryWorker.cs @@ -0,0 +1,129 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace ShopDbEndpointAgent.Core; + +// The telemetry loop as a hosted BackgroundService. On first contact it sends +// adopt=true (once) so the server's computers collector creates the PC; every +// run after that omits adopt and relies on strict hostname resolution. Between +// runs it waits the configured cadence plus/minus jitter so a fleet does not +// report in lockstep (thundering herd). +// +// The worker never lets one bad run kill the loop: a transient failure is +// already retried inside CollectorClient, and a permanent failure (bad key, +// PC removed server-side) is logged and the loop simply waits for the next +// cadence tick rather than hammering the server. +public sealed class TelemetryWorker : BackgroundService +{ + private readonly ICollector collector; + private readonly CollectorClient client; + private readonly IAgentUtilityRunner? utilities; + private readonly AgentConfig config; + private readonly string agentVersion; + private readonly ILogger logger; + private readonly CollectSignal? collectSignal; + private readonly Random jitter = new(); + + public TelemetryWorker(ICollector collector, CollectorClient client, + AgentConfig config, string agentVersion, ILogger logger, + IAgentUtilityRunner? utilities = null, CollectSignal? collectSignal = null) + { + this.collector = collector; + this.client = client; + this.config = config; + this.agentVersion = agentVersion; + this.logger = logger; + this.utilities = utilities; + this.collectSignal = collectSignal; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + logger.LogInformation( + "Endpoint agent {Version} -> {BaseUrl}, cadence {Cadence}s +/-{Jitter}s", + agentVersion, config.BaseUrl, config.CadenceSeconds, config.JitterSeconds); + + bool firstRun = true; + while (!stoppingToken.IsCancellationRequested) + { + bool adopt = firstRun && config.AdoptOnFirstRun; + try + { + await RunOnceAsync(adopt, stoppingToken); + firstRun = false; + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (CollectorException error) when (error.IsPermanent) + { + // Bad key or PC removed server-side: surface it, do not retry + // faster than cadence. If it was a first-run adopt, keep trying + // to adopt next tick (the operator may be provisioning the PC). + logger.LogError(error, "Permanent collector error; waiting for next cadence"); + } + catch (Exception error) + { + logger.LogError(error, "Telemetry run failed"); + } + + try + { + if (collectSignal is not null) + { + // Wake early if a command asked for an immediate re-collect, + // otherwise fall through when the cadence delay elapses. + bool woken = await collectSignal.WaitAsync(NextDelay(), stoppingToken); + if (woken) + logger.LogInformation("Off-cadence collect requested after a command"); + } + else + { + await Task.Delay(NextDelay(), stoppingToken); + } + } + catch (OperationCanceledException) + { + break; + } + } + + logger.LogInformation("Endpoint agent stopped"); + } + + // Public so a --once console run and future on-demand triggers can reuse the + // exact production collection path. + public async Task RunOnceAsync(bool adopt, CancellationToken token) + { + var report = collector.Collect(agentVersion, adopt); + + if (utilities is not null) + { + var extra = utilities.RunAll(token); + if (extra.Count > 0) + logger.LogDebug("Loaded {Count} agent utility result(s)", extra.Count); + } + + logger.LogInformation( + "Reporting (adopt={Adopt}) user={User} serial={Serial} events={Events}", + adopt, report.LoggedInUser, report.SerialNumber, report.Events?.Length ?? 0); + + var result = await client.SendAsync(report, token); + + logger.LogInformation("OK action={Action} assetid={AssetId} identity={Identity}", + result.Action, result.AssetId, result.IdentityValue); + foreach (var warning in result.Warnings ?? Array.Empty()) + logger.LogWarning("Server warning: {Warning}", warning); + + return result; + } + + private TimeSpan NextDelay() + { + int spread = Math.Max(0, config.JitterSeconds); + int offset = spread == 0 ? 0 : jitter.Next(-spread, spread + 1); + int seconds = Math.Max(1, config.CadenceSeconds + offset); + return TimeSpan.FromSeconds(seconds); + } +} diff --git a/agent/ShopDbEndpointAgent.Tray/Program.cs b/agent/ShopDbEndpointAgent.Tray/Program.cs new file mode 100644 index 00000000..80e0f0cd --- /dev/null +++ b/agent/ShopDbEndpointAgent.Tray/Program.cs @@ -0,0 +1,17 @@ +namespace ShopDbEndpointAgent.Tray; + +// Interactive user-session tray app (SCAFFOLD). Separate process from the +// headless Windows Service: the service owns telemetry, this owns the operator +// surface - a system-tray icon and the 5-click-max ServiceNow ticket flow +// (DESIGN-DECISIONS-agent.md section 6). This iteration wires the tray and a +// stubbed ticket dialog; the ServiceNow submission (direct vs server-proxied) +// is deferred until that decision is made server-side. +internal static class Program +{ + [STAThread] + private static void Main() + { + ApplicationConfiguration.Initialize(); + Application.Run(new TrayContext()); + } +} diff --git a/agent/ShopDbEndpointAgent.Tray/ShopDbEndpointAgent.Tray.csproj b/agent/ShopDbEndpointAgent.Tray/ShopDbEndpointAgent.Tray.csproj new file mode 100644 index 00000000..365c1e5f --- /dev/null +++ b/agent/ShopDbEndpointAgent.Tray/ShopDbEndpointAgent.Tray.csproj @@ -0,0 +1,24 @@ + + + + WinExe + net8.0-windows + enable + enable + true + ShopDbEndpointAgent.Tray + shopdb-endpoint-tray + true + win-x64 + true + true + true + true + none + + + + + + + diff --git a/agent/ShopDbEndpointAgent.Tray/TicketForm.cs b/agent/ShopDbEndpointAgent.Tray/TicketForm.cs new file mode 100644 index 00000000..392418e2 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Tray/TicketForm.cs @@ -0,0 +1,118 @@ +namespace ShopDbEndpointAgent.Tray; + +// The 5-click-max issue reporter (SCAFFOLD). Target flow from +// DESIGN-DECISIONS-agent.md section 6: +// network precheck -> issue dropdown -> optional screenshot -> submit. +// This iteration renders the dropdown of fixed reasons and a submit button but +// does NOT talk to ServiceNow yet - the submission path (direct API vs proxied +// through a ShopDB server endpoint so ServiceNow creds stay server-side) is an +// open decision. Submit currently shows what WOULD be sent. +internal sealed class TicketForm : Form +{ + // Fixed dropdown reasons from the source conversation (section 6). + private static readonly string[] Reasons = + { + "Can't logon to CSF", + "Data collections down", + "Can't download program", + "Machine communication failing", + "Network drives down", + "Other (free text)", + }; + + private readonly ComboBox reasonBox; + private readonly TextBox detailBox; + private readonly CheckBox screenshotBox; + + public TicketForm() + { + Text = "Report an issue"; + FormBorderStyle = FormBorderStyle.FixedDialog; + StartPosition = FormStartPosition.CenterScreen; + MinimizeBox = false; + MaximizeBox = false; + ClientSize = new Size(420, 260); + + var reasonLabel = new Label + { + Text = "What is happening?", + Left = 16, + Top = 16, + Width = 388, + }; + reasonBox = new ComboBox + { + Left = 16, + Top = 40, + Width = 388, + DropDownStyle = ComboBoxStyle.DropDownList, + }; + reasonBox.Items.AddRange(Reasons); + reasonBox.SelectedIndex = 0; + + var detailLabel = new Label + { + Text = "Details (optional)", + Left = 16, + Top = 76, + Width = 388, + }; + detailBox = new TextBox + { + Left = 16, + Top = 100, + Width = 388, + Height = 80, + Multiline = true, + }; + + screenshotBox = new CheckBox + { + Text = "Attach a screenshot of my screen", + Left = 16, + Top = 188, + Width = 388, + }; + + var submit = new Button + { + Text = "Submit", + Left = 244, + Top = 220, + Width = 76, + DialogResult = DialogResult.OK, + }; + submit.Click += OnSubmit; + + var cancel = new Button + { + Text = "Cancel", + Left = 328, + Top = 220, + Width = 76, + DialogResult = DialogResult.Cancel, + }; + + Controls.AddRange(new Control[] + { + reasonLabel, reasonBox, detailLabel, detailBox, screenshotBox, + submit, cancel, + }); + AcceptButton = submit; + CancelButton = cancel; + } + + private void OnSubmit(object? sender, EventArgs e) + { + // Placeholder: show what would be submitted. The real path (screenshot + // capture to Pictures\Screenshots, then ServiceNow submit) lands once + // the direct-vs-proxied decision is made. + var summary = + $"Reason: {reasonBox.SelectedItem}\n" + + $"Details: {(string.IsNullOrWhiteSpace(detailBox.Text) ? "(none)" : detailBox.Text)}\n" + + $"Screenshot: {(screenshotBox.Checked ? "yes" : "no")}"; + MessageBox.Show( + "Ticketing is not wired up yet. This would be submitted:\n\n" + summary, + "ShopDB agent (scaffold)", MessageBoxButtons.OK, MessageBoxIcon.Information); + } +} diff --git a/agent/ShopDbEndpointAgent.Tray/TrayContext.cs b/agent/ShopDbEndpointAgent.Tray/TrayContext.cs new file mode 100644 index 00000000..93827706 --- /dev/null +++ b/agent/ShopDbEndpointAgent.Tray/TrayContext.cs @@ -0,0 +1,202 @@ +using System.Diagnostics; + +namespace ShopDbEndpointAgent.Tray; + +// Owns the tray icon and its context menu for the app's lifetime. Kept minimal +// and headless-friendly: no main window, the icon is the whole UI surface until +// the operator opens the ticket dialog. +// +// "Send Status now" reuses the exact production collection path by shelling out +// to the installed shopdbagent.exe with --once. That keeps the collector key and +// all collection logic in the service binary: the tray never holds a secret and +// never duplicates wiring, it just asks the agent to report on demand so the +// ShopDB monitor page reflects the box immediately instead of at next cadence. +internal sealed class TrayContext : ApplicationContext +{ + // The headless agent/service executable. The tray ships next to it in the + // install directory, so same-dir resolution is the primary path. + private const string AgentExeName = "shopdbagent.exe"; + private const string DefaultInstallDir = @"C:\Program Files\ShopDB\EndpointAgent"; + + private readonly NotifyIcon trayIcon; + private readonly Icon brandIcon; + private bool sending; + + public TrayContext() + { + var menu = new ContextMenuStrip(); + menu.Items.Add("Send Status now", null, OnSendStatus); + menu.Items.Add("Report an issue...", null, OnReportIssue); + menu.Items.Add(new ToolStripSeparator()); + menu.Items.Add("About ShopDB agent", null, OnAbout); + menu.Items.Add("Exit", null, OnExit); + + // Branded tray icon drawn from the ShopDB WebUI favicon (see TrayIcon). + brandIcon = TrayIcon.Create(); + + trayIcon = new NotifyIcon + { + Icon = brandIcon, + Text = "ShopDB Endpoint Agent", + Visible = true, + ContextMenuStrip = menu, + }; + trayIcon.DoubleClick += OnSendStatus; + } + + private void OnReportIssue(object? sender, EventArgs e) + { + using var dialog = new TicketForm(); + dialog.ShowDialog(); + } + + // Runs shopdbagent.exe --once out of process so the operator can push a + // fresh report without waiting for the next cadence tick. The exit code is + // the agent's own contract: 0 = reported OK, non-zero = failed (bad key, + // host removed, network) and the detail is in the rolling agent log. + private async void OnSendStatus(object? sender, EventArgs e) + { + if (sending) + return; + + var agentExe = ResolveAgentExe(); + if (agentExe is null) + { + ShowBalloon("Send Status", "Could not find " + AgentExeName + + ". Is the ShopDB Endpoint Agent installed?", ToolTipIcon.Error); + return; + } + + sending = true; + trayIcon.Text = "ShopDB Endpoint Agent (sending status...)"; + ShowBalloon("Send Status", "Sending this PC's status to ShopDB...", + ToolTipIcon.Info); + + try + { + int exitCode = await RunAgentOnceAsync(agentExe); + if (exitCode == 0) + ShowBalloon("Send Status", "Status sent to ShopDB.", + ToolTipIcon.Info); + else + ShowBalloon("Send Status", + "Status send failed (exit " + exitCode + + "). See the agent log under ProgramData\\ShopDB.", + ToolTipIcon.Warning); + } + catch (Exception error) + { + ShowBalloon("Send Status", "Could not run the agent: " + + error.Message, ToolTipIcon.Error); + } + finally + { + sending = false; + trayIcon.Text = "ShopDB Endpoint Agent"; + } + } + + private static Task RunAgentOnceAsync(string agentExe) + { + var startInfo = new ProcessStartInfo + { + FileName = agentExe, + Arguments = "--once", + WorkingDirectory = Path.GetDirectoryName(agentExe) ?? Environment.CurrentDirectory, + UseShellExecute = false, + CreateNoWindow = true, + }; + + // The tray and the agent are both single-file, self-contained apps. When + // the tray host started, the .NET single-file bootstrapper set DOTNET_* + // and bundle-extraction variables in this process's environment. With + // UseShellExecute=false the child inherits those variables, so the + // freshly installed agent tries to bootstrap against the tray's stale + // extraction state and dies during host startup (exit 0xE0434352) before + // any managed code - or the file log - runs. Stripping these inherited + // markers gives the child the same clean host environment it gets when + // launched from a shell, and it also removes the need to restart the tray + // after every agent rebuild. The agent re-derives whatever it needs. + ScrubInheritedDotnetHostVariables(startInfo); + + var completion = new TaskCompletionSource(); + var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + process.Exited += (_, _) => + { + completion.TrySetResult(process.ExitCode); + process.Dispose(); + }; + + if (!process.Start()) + { + process.Dispose(); + completion.TrySetException( + new InvalidOperationException("failed to start " + agentExe)); + } + + return completion.Task; + } + + // Remove the .NET host / single-file bundle markers this tray process + // inherited at startup so a spawned single-file child bootstraps cleanly + // instead of adopting the tray's extraction state. Names are matched by a + // DOTNET_/COREHOST_ prefix (covers DOTNET_BUNDLE_EXTRACT_BASE_DIR, + // DOTNET_ROOT, the internal startup hooks, etc.) so this stays correct as + // the runtime adds new host variables. + private static void ScrubInheritedDotnetHostVariables(ProcessStartInfo startInfo) + { + var toRemove = startInfo.Environment.Keys + .Where(key => + key.StartsWith("DOTNET_", StringComparison.OrdinalIgnoreCase) || + key.StartsWith("COREHOST_", StringComparison.OrdinalIgnoreCase)) + .ToList(); + foreach (var key in toRemove) + startInfo.Environment.Remove(key); + } + + // Same directory first (the installer ships the tray next to the agent), + // then the known Program Files install directory as a fallback. + private static string? ResolveAgentExe() + { + var beside = Path.Combine(AppContext.BaseDirectory, AgentExeName); + if (File.Exists(beside)) + return beside; + + var installed = Path.Combine(DefaultInstallDir, AgentExeName); + if (File.Exists(installed)) + return installed; + + return null; + } + + private void ShowBalloon(string title, string text, ToolTipIcon icon) + { + trayIcon.BalloonTipTitle = title; + trayIcon.BalloonTipText = text; + trayIcon.BalloonTipIcon = icon; + trayIcon.ShowBalloonTip(4000); + } + + private void OnAbout(object? sender, EventArgs e) + => MessageBox.Show( + "ShopDB Endpoint Agent tray.\n" + + "Telemetry runs as the ShopDB Endpoint Agent Windows service.\n" + + "Send Status now reports this PC to ShopDB on demand.", + "About", MessageBoxButtons.OK, MessageBoxIcon.Information); + + private void OnExit(object? sender, EventArgs e) + { + trayIcon.Visible = false; + ExitThread(); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + trayIcon.Dispose(); + brandIcon.Dispose(); + } + base.Dispose(disposing); + } +} diff --git a/agent/ShopDbEndpointAgent.Tray/TrayIcon.cs b/agent/ShopDbEndpointAgent.Tray/TrayIcon.cs new file mode 100644 index 00000000..dbc0957b --- /dev/null +++ b/agent/ShopDbEndpointAgent.Tray/TrayIcon.cs @@ -0,0 +1,81 @@ +using System.Drawing.Drawing2D; +using System.Drawing.Imaging; +using System.Runtime.InteropServices; + +namespace ShopDbEndpointAgent.Tray; + +// Builds the tray icon from the ShopDB WebUI favicon (frontend/public/favicon.svg: +// a #4181ff rounded square with a white "S") by drawing it with GDI+ at runtime. +// Drawing it here rather than shipping a binary .ico keeps the brand in one +// place (these values mirror the SVG) and avoids adding an SVG rasterizer or a +// checked-in binary to the build. +internal static class TrayIcon +{ + // Favicon colors (frontend/public/favicon.svg). + private static readonly Color Background = ColorTranslator.FromHtml("#4181ff"); + private static readonly Color Foreground = Color.White; + + // Renders the branded icon at a tray-friendly size. 32px covers standard and + // most high-DPI tray scaling; the OS downsamples as needed. + public static Icon Create(int size = 32) + { + using var bitmap = new Bitmap(size, size, PixelFormat.Format32bppArgb); + using (var graphics = Graphics.FromImage(bitmap)) + { + graphics.SmoothingMode = SmoothingMode.AntiAlias; + graphics.TextRenderingHint = + System.Drawing.Text.TextRenderingHint.AntiAlias; + graphics.Clear(Color.Transparent); + + // Rounded square (SVG rect rx=14 on a 64 viewport -> ~0.22 radius). + float inset = size * 0.03f; + var rect = new RectangleF(inset, inset, + size - inset * 2, size - inset * 2); + float radius = size * 0.22f; + using (var path = RoundedRect(rect, radius)) + using (var fill = new SolidBrush(Background)) + graphics.FillPath(fill, path); + + // White "S", centered. + using var font = new Font("Arial", size * 0.58f, + FontStyle.Bold, GraphicsUnit.Pixel); + using var text = new SolidBrush(Foreground); + using var format = new StringFormat + { + Alignment = StringAlignment.Center, + LineAlignment = StringAlignment.Center, + }; + graphics.DrawString("S", font, text, rect, format); + } + + // Icon.FromHandle borrows the GDI HICON, so clone into a managed Icon + // and free the handle to avoid leaking one per launch. + IntPtr hicon = bitmap.GetHicon(); + try + { + using var borrowed = Icon.FromHandle(hicon); + return (Icon)borrowed.Clone(); + } + finally + { + DestroyIcon(hicon); + } + } + + private static GraphicsPath RoundedRect(RectangleF rect, float radius) + { + float diameter = radius * 2; + var path = new GraphicsPath(); + path.AddArc(rect.Left, rect.Top, diameter, diameter, 180, 90); + path.AddArc(rect.Right - diameter, rect.Top, diameter, diameter, 270, 90); + path.AddArc(rect.Right - diameter, rect.Bottom - diameter, diameter, + diameter, 0, 90); + path.AddArc(rect.Left, rect.Bottom - diameter, diameter, diameter, 90, 90); + path.CloseFigure(); + return path; + } + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool DestroyIcon(IntPtr handle); +} diff --git a/agent/ShopDbEndpointAgent.slnx b/agent/ShopDbEndpointAgent.slnx new file mode 100644 index 00000000..1383261a --- /dev/null +++ b/agent/ShopDbEndpointAgent.slnx @@ -0,0 +1,5 @@ + + + + + diff --git a/agent/ShopDbEndpointAgent/Program.cs b/agent/ShopDbEndpointAgent/Program.cs new file mode 100644 index 00000000..c97a8839 --- /dev/null +++ b/agent/ShopDbEndpointAgent/Program.cs @@ -0,0 +1,164 @@ +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using ShopDbEndpointAgent.Core; + +// Headless Windows Service host for the ShopDB endpoint agent. Composes the +// Core runtime (config -> collector -> client -> worker) behind the generic +// host, adds durable file logging and a WER-style fault monitor, and supports: +// (default) run as a Windows Service (AddWindowsService) +// --console run interactively in the foreground for debugging +// --once collect and report exactly once, then exit (smoke test / cron) +// +// Secrets are never read here: AgentConfigProvider layers the registry +// (HKLM\SOFTWARE\GE\ShopDB: CollectorKey, BaseUrl) over the shipped, non-secret +// appsettings.json. The executable is intentionally named shopdbagent.exe so it +// never collides with third-party agent.exe processes/services (for example the +// MTConnect agent this same build can monitor). + +var runOnce = args.Any(a => string.Equals(a, "--once", StringComparison.OrdinalIgnoreCase)); +var forceConsole = args.Any(a => string.Equals(a, "--console", StringComparison.OrdinalIgnoreCase)); + +var agentVersion = Assembly.GetExecutingAssembly() + .GetName().Version?.ToString() ?? "0.0.0"; + +// Bootstrap logger first so config loading (including the registry read that +// CyberArk/EPM may gate) is fully diagnosable before the host starts. +using var bootstrapFileLogger = new FileLoggerProvider(FileLoggerProvider.DefaultDirectory()); +var bootstrapLogger = bootstrapFileLogger.CreateLogger("Startup"); + +var faultMonitor = new AgentFaultMonitor(bootstrapLogger); +faultMonitor.Install(); + +var config = AgentConfigProvider.Load(logger: bootstrapLogger); + +var logDirectory = string.IsNullOrWhiteSpace(config.LogPath) + ? FileLoggerProvider.DefaultDirectory() + : config.LogPath; + +if (!config.IsUsable()) +{ + bootstrapLogger.LogError( + "Config incomplete: provision CollectorKey and BaseUrl in " + + "HKLM\\SOFTWARE\\GE\\ShopDB (or baseurl/apikey in appsettings.json for dev). " + + "See the registry lines above to tell 'missing' from 'access denied'."); + return 1; +} + +try +{ + if (runOnce) + return await RunOnceAsync(config, agentVersion, logDirectory, faultMonitor); + + await RunHostAsync(config, agentVersion, logDirectory, forceConsole); + return 0; +} +catch (Exception error) +{ + faultMonitor.ReportFatal("host", error); + return 1; +} + +static async Task RunHostAsync(AgentConfig config, string agentVersion, + string logDirectory, bool forceConsole) +{ + var builder = Host.CreateApplicationBuilder(); + + if (!forceConsole) + builder.Services.AddWindowsService(options => + options.ServiceName = "ShopDbEndpointAgent"); + + ConfigureLogging(builder.Logging, logDirectory); + RegisterCoreServices(builder.Services, config, agentVersion); + builder.Services.AddHostedService(sp => sp.GetRequiredService()); + builder.Services.AddHostedService(sp => sp.GetRequiredService()); + + await builder.Build().RunAsync(); +} + +static async Task RunOnceAsync(AgentConfig config, string agentVersion, + string logDirectory, AgentFaultMonitor faultMonitor) +{ + using var services = BuildOnceServices(config, agentVersion, logDirectory); + var worker = services.GetRequiredService(); + var logger = services.GetRequiredService>(); + try + { + var result = await worker.RunOnceAsync(config.AdoptOnFirstRun, CancellationToken.None); + logger.LogInformation("Single report complete: action={Action}", result.Action); + return 0; + } + catch (Exception error) + { + faultMonitor.ReportFatal("--once", error); + return 1; + } +} + +static ServiceProvider BuildOnceServices(AgentConfig config, string agentVersion, + string logDirectory) +{ + var services = new ServiceCollection(); + services.AddLogging(logging => ConfigureLogging(logging, logDirectory)); + RegisterCoreServices(services, config, agentVersion); + return services.BuildServiceProvider(); +} + +static void RegisterCoreServices(IServiceCollection services, AgentConfig config, + string agentVersion) +{ + services.AddSingleton(config); + + // Shared wake used by the command worker to trigger an immediate telemetry + // collect after a command changes endpoint state (e.g. MTConnect toggle). + services.AddSingleton(); + + if (!string.IsNullOrWhiteSpace(config.UtilitiesPath) && + Directory.Exists(config.UtilitiesPath)) + { + services.AddSingleton(sp => + new PowerShellUtilityRunner(config.UtilitiesPath, + sp.GetRequiredService>())); + } + + services.AddSingleton(sp => new EndpointCollector( + sp.GetRequiredService>(), + config, + sp.GetService())); + + services.AddSingleton(sp => new CollectorClient( + config, sp.GetRequiredService>())); + + services.AddSingleton(sp => new TelemetryWorker( + sp.GetRequiredService(), + sp.GetRequiredService(), + config, + agentVersion, + sp.GetRequiredService>(), + sp.GetService(), + sp.GetRequiredService())); + + // Command channel: outbound long-poll for allowlisted WebUI commands, run + // through the fail-closed executor. Hosted alongside telemetry but on its + // own loop so a held-open poll never delays a report. + services.AddSingleton(sp => new CommandClient( + config, sp.GetRequiredService>())); + + services.AddSingleton(sp => new CommandExecutor( + sp.GetRequiredService>(), + config)); + + services.AddSingleton(sp => new CommandWorker( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>(), + sp.GetRequiredService())); +} + +static void ConfigureLogging(ILoggingBuilder logging, string logDirectory) +{ + logging.ClearProviders(); + logging.AddProvider(new FileLoggerProvider(logDirectory)); + logging.SetMinimumLevel(LogLevel.Information); +} diff --git a/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj b/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj new file mode 100644 index 00000000..3612a79a --- /dev/null +++ b/agent/ShopDbEndpointAgent/ShopDbEndpointAgent.csproj @@ -0,0 +1,40 @@ + + + + Exe + net8.0-windows + enable + enable + ShopDbEndpointAgent + shopdbagent + + 1.3.0 + true + win-x64 + true + true + true + true + none + + + + + + + + + + + + + + + PreserveNewest + + + + diff --git a/agent/ShopDbEndpointAgent/appsettings.json b/agent/ShopDbEndpointAgent/appsettings.json new file mode 100644 index 00000000..162913e3 --- /dev/null +++ b/agent/ShopDbEndpointAgent/appsettings.json @@ -0,0 +1,26 @@ +{ + "baseurl": "", + "apikey": "REPLACE_WITH_COLLECTOR_KEY", + "cadenceseconds": 900, + "jitterseconds": 60, + "adoptonfirstrun": true, + "utilitiespath": "", + "logpath": "", + "monitors": { + "processes": [ + { "name": "dncMain.exe", "app": "eDNC" }, + { "name": "udc.exe", "app": "Universal Data Collector" }, + { "name": "agent.exe", "app": "MTConnect" } + ], + "services": [ + { "name": "MTConnect UDC Agent", "app": "MTConnect" } + ] + }, + "launchpaths": { + "DncMain.exe": "C:\\Program Files (x86)\\Dnc\\bin\\DncMain.exe", + "UDC.exe": "C:\\Program Files\\UDC\\UDC.exe", + "agent.exe": "C:\\ProgramData\\udc\\mtconnect_udc\\agent\\run.bat" + } +} + + diff --git a/agent/installer/install-shopdbagent.ps1 b/agent/installer/install-shopdbagent.ps1 new file mode 100644 index 00000000..4fcbea71 --- /dev/null +++ b/agent/installer/install-shopdbagent.ps1 @@ -0,0 +1,337 @@ +# install-shopdbagent.ps1 +# ShopDB Endpoint Agent installer for the test endpoint. Run from an ELEVATED +# (admin) PowerShell on the endpoint. Provisions the registry secrets, copies +# the agent files to Program Files, runs a one-shot smoke test, then installs +# and starts the Windows service. +# +# Secrets are provisioned to HKLM\SOFTWARE\GE\ShopDB. On locked-down hosts +# (CyberArk/EPM) the agent may be blocked from READING that hive, so the same +# values are also stamped into the installed appsettings.json as a fallback. +# +# The agent and tray are published as single-file self-contained executables, +# so copying the two .exe files plus appsettings.json is the complete payload. + +[CmdletBinding()] +param( + # ShopDB server root URL for this instance. Defaulted to the test instance; + # pass -BaseUrl "http://:" to override. + [string] $BaseUrl = "http://H1HWNL84.device.geaerospace.net:5001", + + # Endpoints-only collector key (this instance's .env + # COLLECTOR_API_KEY_ENDPOINTS). + [string] $CollectorKey = "l755ddRw1iSeOcSTh9N9wXM4gGT7N6WIVN9crOY6uZU", + + [string] $InstallDir = "C:\Program Files\ShopDB\EndpointAgent", + [string] $ServiceName = "ShopDbEndpointAgent", + + # Interactive tray app that shows the system-tray icon and the on-demand + # "Send Status now" action. Separate process from the SYSTEM service (a + # service cannot draw a tray icon in a user session). Pass -NoTray to skip. + [string] $TrayExeName = "shopdb-endpoint-tray.exe", + [switch] $NoTray, + + # Add a Microsoft Defender path exclusion for the install directory. The + # agent and tray are unsigned single-file self-contained executables, which + # Defender heuristics (and the tray's GDI/user32 icon P/Invoke) can flag as + # a false positive and quarantine, killing the tray before its icon appears. + # Pass -NoDefenderExclusion to skip (e.g. where policy manages exclusions + # centrally or the binaries are code-signed). + [switch] $NoDefenderExclusion +) + +$ErrorActionPreference = "Stop" +$sourceDir = Split-Path -Parent $MyInvocation.MyCommand.Path + +function Assert-Admin { + $identity = [Security.Principal.WindowsIdentity]::GetCurrent() + $principal = New-Object Security.Principal.WindowsPrincipal($identity) + if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { + throw "This script must be run from an elevated (Run as administrator) PowerShell." + } +} + +Assert-Admin + +if ([string]::IsNullOrWhiteSpace($BaseUrl)) { + throw "BaseUrl is not set. Pass -BaseUrl 'http://:' when running the script." +} + +# Write HKLM values through the explicit 64-bit registry view. If this script +# runs under 32-bit PowerShell (some management tooling launches x86), the +# PSDrive HKLM: silently redirects writes into WOW6432Node. The 64-bit agent +# service and 64-bit Explorer (the logon Run key) then read the 64-bit view and +# see nothing, so config "goes missing" and the tray never autostarts. Pinning +# the view removes that dependency on the launching shell's bitness. +function Set-Hklm64Value { + param( + [Parameter(Mandatory)] [string] $SubKey, + [Parameter(Mandatory)] [string] $Name, + [Parameter(Mandatory)] [string] $Value + ) + $base = [Microsoft.Win32.RegistryKey]::OpenBaseKey( + [Microsoft.Win32.RegistryHive]::LocalMachine, + [Microsoft.Win32.RegistryView]::Registry64) + try { + $key = $base.CreateSubKey($SubKey) + try { + $key.SetValue($Name, $Value, [Microsoft.Win32.RegistryValueKind]::String) + } + finally { $key.Dispose() } + } + finally { $base.Dispose() } +} + +# Tear down any previously installed agent BEFORE touching files. The agent and +# tray are single-file self-contained exes; while the old service or tray is +# running it holds those .exe files open, so Copy-Item over them fails with a +# file-in-use error. Stopping the service, deleting it, and killing any lingering +# agent/tray processes up front makes the install a clean overwrite. Best-effort +# throughout: a missing service or process is the normal fresh-install case. +function Uninstall-PreviousAgent { + param( + [Parameter(Mandatory)] [string] $Service, + [Parameter(Mandatory)] [string] $TrayExe, + [Parameter(Mandatory)] [string] $TargetDir + ) + + $existing = Get-Service -Name $Service -ErrorAction SilentlyContinue + if ($existing) { + Write-Host " stopping and removing existing service $Service ..." + if ($existing.Status -ne "Stopped") { + try { Stop-Service -Name $Service -Force -ErrorAction Stop } + catch { Write-Warning " could not stop service: $($_.Exception.Message)" } + } + # sc.exe delete only marks the service for deletion; the SCM finishes the + # removal once every handle closes. Give it a moment so the subsequent + # sc.exe create does not hit a "marked for deletion" (1072) error. + sc.exe delete $Service | Out-Null + Start-Sleep -Seconds 2 + } + + # Kill any lingering worker or tray processes that would keep the exes + # locked. Match by the installed exe base names; the service host process is + # shopdbagent, the tray is the TrayExe base name. + $trayBase = [System.IO.Path]::GetFileNameWithoutExtension($TrayExe) + foreach ($name in @("shopdbagent", $trayBase)) { + $procs = Get-Process -Name $name -ErrorAction SilentlyContinue + foreach ($p in $procs) { + Write-Host " killing lingering process $($p.ProcessName) (pid $($p.Id)) ..." + try { Stop-Process -Id $p.Id -Force -ErrorAction Stop } + catch { Write-Warning " could not kill $($p.ProcessName): $($_.Exception.Message)" } + } + } + # Also kill anything still running out of the target install directory, + # regardless of exe name. A tray from an older build (different exe name) or + # one launched from a stale path would otherwise survive the name-based sweep + # above, keep its exe locked, and leave a duplicate icon once step 5 starts + # the fresh tray. Match on the executable path so we only touch our own dir. + $targetPrefix = $TargetDir.TrimEnd('\') + '\' + foreach ($p in (Get-Process -ErrorAction SilentlyContinue)) { + $path = $null + try { $path = $p.Path } catch { $path = $null } + if ($path -and $path.StartsWith($targetPrefix, [System.StringComparison]::OrdinalIgnoreCase)) { + Write-Host " killing process from install dir $($p.ProcessName) (pid $($p.Id)) ..." + try { Stop-Process -Id $p.Id -Force -ErrorAction Stop } + catch { Write-Warning " could not kill $($p.ProcessName): $($_.Exception.Message)" } + } + } + # Brief settle so file handles are released before the copy overwrites them. + Start-Sleep -Milliseconds 500 +} + +Write-Host "== ShopDB Endpoint Agent install ==" -ForegroundColor Cyan +Write-Host "BaseUrl : $BaseUrl" +Write-Host "InstallDir : $InstallDir" +Write-Host "ServiceName : $ServiceName" + +# 0. Clear the Mark-of-the-Web on the SOURCE payload first. The operator +# typically extracts installer.zip into Downloads, so every file carries a +# Zone.Identifier ADS. SmartScreen reads that and blocks the unsigned exes with +# "Windows protected your PC" the first time they run - including the [3/5] +# smoke test below, which launches shopdbagent.exe out of $sourceDir before the +# installed copy exists. Unblocking the source folder up front keeps the smoke +# test (and any manual run from the extract folder) clean; the installed copies +# get unblocked again in step 2a after they land in Program Files. Best-effort: +# a file with no MotW is a no-op. +Write-Host "`n[0a] Clearing Mark-of-the-Web on the extracted installer files ..." +Get-ChildItem -Path $sourceDir -File | ForEach-Object { + try { Unblock-File -Path $_.FullName -ErrorAction Stop } + catch { Write-Warning " could not unblock $($_.Name): $($_.Exception.Message)" } +} + +# 0. Uninstall any previous agent first, so the file copy is a clean overwrite +# and does not fail on exes locked by a running service or tray. +Write-Host "`n[0/5] Removing any previously installed agent ..." +Uninstall-PreviousAgent -Service $ServiceName -TrayExe $TrayExeName -TargetDir $InstallDir + +# 1. Provision registry secrets (64-bit view, see Set-Hklm64Value). +$regPath = "HKLM:\SOFTWARE\GE\ShopDB" +Write-Host "`n[1/5] Provisioning registry $regPath (Registry64) ..." +Set-Hklm64Value -SubKey "SOFTWARE\GE\ShopDB" -Name "BaseUrl" -Value $BaseUrl +Set-Hklm64Value -SubKey "SOFTWARE\GE\ShopDB" -Name "CollectorKey" -Value $CollectorKey + +# 2. Copy agent files to the install directory. The agent and tray are +# single-file self-contained executables, so the .exe plus appsettings.json is +# the whole payload; there are no side-by-side runtime DLLs to copy. +Write-Host "[2/5] Copying agent files to $InstallDir ..." +New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null +$agentSource = Join-Path $sourceDir "shopdbagent.exe" +if (-not (Test-Path $agentSource)) { + throw "shopdbagent.exe not found in $sourceDir. Unzip the full installer package and run from that folder." +} +Copy-Item -Path $agentSource -Destination $InstallDir -Force +$appsettings = Join-Path $sourceDir "appsettings.json" +$installedSettings = Join-Path $InstallDir "appsettings.json" +if (Test-Path $appsettings) { + Copy-Item -Path $appsettings -Destination $InstallDir -Force +} + +# Copy the interactive tray app next to the agent so its same-dir resolution +# finds shopdbagent.exe for the "Send Status now" action. Optional: absent on a +# service-only (headless) build. +$trayInstalled = $null +if (-not $NoTray) { + $traySource = Join-Path $sourceDir $TrayExeName + if (Test-Path $traySource) { + Copy-Item -Path $traySource -Destination $InstallDir -Force + $trayInstalled = Join-Path $InstallDir $TrayExeName + Write-Host " copied tray app $TrayExeName" + } + else { + Write-Warning " tray app $TrayExeName not found in $sourceDir; skipping tray (service still installs)" + } +} + +# Fallback for locked-down endpoints: some hosts (CyberArk/EPM) gate the HKLM +# hive so the agent cannot READ the values the registry step just wrote. Stamp +# BaseUrl/CollectorKey into the installed appsettings.json too. The agent reads +# appsettings first, then lets the registry override it, so this is a safe +# no-op where the registry read works and the rescue where it does not. +if (Test-Path $installedSettings) { + try { + $json = Get-Content $installedSettings -Raw | ConvertFrom-Json + $json.baseurl = $BaseUrl + $json.apikey = $CollectorKey + $json | ConvertTo-Json -Depth 10 | Set-Content -Path $installedSettings -Encoding UTF8 + Write-Host " stamped BaseUrl/CollectorKey into appsettings.json (registry-read fallback)" + } + catch { + Write-Warning " could not stamp appsettings.json: $($_.Exception.Message)" + } +} +$exePath = Join-Path $InstallDir "shopdbagent.exe" + +# 2a. Strip the Mark-of-the-Web from the copied files. Windows tags files +# extracted from a downloaded zip with a Zone.Identifier alternate data stream; +# SmartScreen reads that and blocks the unsigned exe with "Windows protected +# your PC" the first time it runs, and the operator sees only a "Don't run" +# button. Unblock-File removes the ADS so the smoke test and the tray launch +# start clean. Best-effort: a file with no MotW is a no-op. +Write-Host "[2a/5] Clearing Mark-of-the-Web on installed files ..." +Get-ChildItem -Path $InstallDir -File | ForEach-Object { + try { Unblock-File -Path $_.FullName -ErrorAction Stop } + catch { Write-Warning " could not unblock $($_.Name): $($_.Exception.Message)" } +} + +# 2b. Defender path exclusion for the install dir. Unsigned single-file +# self-contained binaries (and the tray's icon P/Invoke) can trip Defender +# heuristics and get quarantined, which is why the tray dies before showing an +# icon. Excluding the fixed install directory is the least-privilege fix short +# of code-signing. Best-effort: Set-MpPreference is absent on servers without +# Defender and can be blocked by tamper protection; a failure is not fatal. +if (-not $NoDefenderExclusion) { + Write-Host "[2b/5] Adding Defender path exclusion for $InstallDir ..." + if (Get-Command Add-MpPreference -ErrorAction SilentlyContinue) { + try { + Add-MpPreference -ExclusionPath $InstallDir -ErrorAction Stop + Write-Host " excluded $InstallDir from Defender scanning." + } + catch { + Write-Warning " could not add Defender exclusion (tamper protection or policy?): $($_.Exception.Message)" + Write-Warning " if the tray is quarantined, code-sign the binaries or add the exclusion via policy." + } + } + else { + Write-Host " Defender cmdlets not present; skipping exclusion." + } +} + +# 3. One-shot smoke test (exact production collection path, then exits). +Write-Host "[3/5] Running one-shot smoke test (--once) ..." +$proc = Start-Process -FilePath $exePath -ArgumentList "--once" -WorkingDirectory $InstallDir -Wait -PassThru -NoNewWindow +Write-Host " smoke test exit code: $($proc.ExitCode)" + +$logDir = Join-Path $env:ProgramData "ShopDB\EndpointAgent\logs" +$logFile = Join-Path $logDir ("agent-{0}.log" -f (Get-Date -Format "yyyyMMdd")) +if (Test-Path $logFile) { + Write-Host " --- last log lines ---" -ForegroundColor DarkGray + Get-Content $logFile -Tail 20 | ForEach-Object { Write-Host " $_" -ForegroundColor DarkGray } +} +if ($proc.ExitCode -ne 0) { + throw "Smoke test failed (exit $($proc.ExitCode)). Check $logFile before installing the service." +} + +# 4. Install the Windows service. Any previous instance was already stopped, +# deleted, and its processes killed in step 0, so this is a clean create. Guard +# against the rare case where the SCM has not finished a pending deletion yet. +Write-Host "[4/5] Installing service $ServiceName ..." +$existing = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue +if ($existing) { + Write-Host " service still present (pending deletion); removing again ..." + if ($existing.Status -ne "Stopped") { Stop-Service -Name $ServiceName -Force } + sc.exe delete $ServiceName | Out-Null + Start-Sleep -Seconds 2 +} +$binPath = '"' + $exePath + '"' +sc.exe create $ServiceName binPath= $binPath start= auto DisplayName= "ShopDB Endpoint Agent" | Out-Null +sc.exe description $ServiceName "Collects endpoint telemetry for ShopDB and runs allowlisted WebUI commands." | Out-Null +sc.exe failure $ServiceName reset= 86400 actions= restart/5000/restart/5000/restart/5000 | Out-Null + +Start-Service -Name $ServiceName +Write-Host " service started." + +# 5. Register the interactive tray app to launch at user logon (per-machine). +Write-Host "[5/5] Registering tray app for logon ..." +if ($trayInstalled -and (Test-Path $trayInstalled)) { + # Register in the 64-bit Run key so 64-bit Explorer fires it at logon. A + # 32-bit write would land in WOW6432Node\...\Run, which Explorer ignores. + Set-Hklm64Value -SubKey "SOFTWARE\Microsoft\Windows\CurrentVersion\Run" ` + -Name "ShopDbEndpointTray" -Value ('"' + $trayInstalled + '"') + Write-Host " tray app registered to start at logon (Registry64): $trayInstalled" + + # Start it now so the icon appears without waiting for the next logon. The + # Run key only fires at logon, so a fresh install would otherwise show no + # tray until the operator signs out and back in. + # + # The installer runs elevated, so a direct Start-Process would launch the + # tray ELEVATED too. That is wrong for a tray app (it should run at the + # user's medium integrity) and, on EPM-managed hosts (CyberArk), elevating an + # unsigned exe triggers a broker prompt that comes back as Win32 1223 + # "The operation was canceled by the user" - which is exactly what we hit. + # Launching through explorer.exe hands the exe to the already-running user + # shell, which spawns it de-elevated in the interactive session, sidestepping + # the elevation broker entirely. + $interactiveUser = $env:USERNAME + if ($interactiveUser -and $interactiveUser -ne "SYSTEM") { + try { + # explorer.exe re-parents the launch under the user shell at + # medium integrity; it returns immediately regardless of the tray's + # own lifetime, so there is nothing to wait on here. + Start-Process -FilePath "explorer.exe" -ArgumentList "`"$trayInstalled`"" | Out-Null + Write-Host " tray app started for the current session (de-elevated via explorer)." + } + catch { + Write-Warning " could not start tray now: $($_.Exception.Message); it will start at next logon." + } + } + else { + Write-Host " no interactive user session; tray will start at next logon." + } +} +else { + Write-Host " tray skipped (headless install)." +} + +Write-Host "`nInstall complete." -ForegroundColor Green +Write-Host "Service '$ServiceName' is running and set to auto-start." +Write-Host "Logs: $logDir" diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 31daa2b3..715161b3 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -153,6 +153,17 @@ export const endpointsApi = { }, get(assetId) { return api.get(`/endpoints/${assetId}`) + }, + enqueueCommand(assetId, action, target = null) { + const body = { action } + if (target) body.target = target + return api.post(`/endpoints/${assetId}/commands`, body) + }, + listCommands(assetId) { + return api.get(`/endpoints/${assetId}/commands`) + }, + telemetryMeta(assetId) { + return api.get(`/endpoints/asset/${assetId}/telemetrymeta`) } } diff --git a/frontend/src/components/PluginAssetPanels.vue b/frontend/src/components/PluginAssetPanels.vue index 4a4d1725..f8e4d3ea 100644 --- a/frontend/src/components/PluginAssetPanels.vue +++ b/frontend/src/components/PluginAssetPanels.vue @@ -150,12 +150,18 @@ const tabs = ref({}) function activeTab(panel) { return tabs.value[panel.id] || 0 } function setTab(panel, index) { tabs.value = { ...tabs.value, [panel.id]: index } } -// Collapsed state per panel id (default expanded), kept here for the same -// reason as tabs: panels are replaced wholesale on refetch. +// Collapsed state per panel id, kept here for the same reason as tabs: panels +// are replaced wholesale on refetch. A panel may declare collapsedDefault to +// start collapsed (for example a low-priority events card); once the user +// toggles it, their explicit choice (stored by id) wins. const collapsed = ref({}) -function isCollapsed(panel) { return !!collapsed.value[panel.id] } +function isCollapsed(panel) { + const state = collapsed.value[panel.id] + if (state === undefined) return !!panel.collapsedDefault + return !!state +} function toggleCollapse(panel) { - collapsed.value = { ...collapsed.value, [panel.id]: !collapsed.value[panel.id] } + collapsed.value = { ...collapsed.value, [panel.id]: !isCollapsed(panel) } } // Per-panel action state: busy guards a double-click while an enqueue POST is @@ -179,7 +185,26 @@ async function runRowAction(panel, item) { try { const body = { action: info.toggle } if (info.target) body.target = info.target - await api.post(toApiPath(info.endpoint, props.assetid), body) + // Baseline telemetry freshness before queuing so we can tell a genuinely + // new snapshot from the stale cached one after the command settles. + const baseline = await currentTelemetryStamp() + const response = await api.post(toApiPath(info.endpoint, props.assetid), body) + const commandId = response?.data?.data?.endpointcommandid + // The POST only queues the command (201). The agent runs it asynchronously, + // so poll for the terminal result and surface the agent's own message when + // it fails (e.g. a bad launch path), rather than reporting a false success. + const settled = await waitForCommandResult(commandId) + if (settled && settled.status === 'failed') { + actionError.value = { + ...actionError.value, + [panel.id]: settled.resultmessage || 'The endpoint agent could not run the command.', + } + } else if (settled && settled.status === 'succeeded') { + // The command ran, but the fresh run state only lands when the agent's + // follow-up telemetry post arrives. Wait for lastreportedat to advance so + // the reloaded row reflects reality instead of the pre-command snapshot. + await waitForFreshTelemetry(baseline) + } await load() } catch (err) { const message = err?.response?.data?.message || 'Command failed to queue.' @@ -189,6 +214,53 @@ async function runRowAction(panel, item) { } } +// Poll the endpoint's recent commands until the given command reaches a terminal +// state (succeeded/failed) or the attempt budget runs out. Returns the command +// row or null. Uses the same commands endpoint the operator view reads. +async function waitForCommandResult(commandId, { attempts = 15, delayMs = 1000 } = {}) { + if (!commandId || !props.assetid) return null + for (let i = 0; i < attempts; i += 1) { + await new Promise((resolve) => setTimeout(resolve, delayMs)) + try { + const response = await api.get(`/endpoints/${props.assetid}/commands`) + const command = (response?.data?.data || []).find( + (row) => row.endpointcommandid === commandId) + if (command && (command.status === 'succeeded' || command.status === 'failed')) { + return command + } + } catch { + // Transient poll failure: keep trying until the attempt budget runs out. + } + } + return null +} + +// Read the endpoint's current lastreportedat (ISO string or null) so a row +// action can baseline freshness before it forces a state change. +async function currentTelemetryStamp() { + if (!props.assetid) return null + try { + const response = await api.get(`/endpoints/asset/${props.assetid}/telemetrymeta`) + return response?.data?.data?.lastreportedat || null + } catch { + return null + } +} + +// Poll lastreportedat until it advances past the baseline (proof the agent has +// posted a fresh snapshot reflecting the command) or the attempt budget runs +// out. Bounded so an idle/offline agent never hangs the row button; the caller +// reloads regardless once this returns. +async function waitForFreshTelemetry(baseline, { attempts = 15, delayMs = 1000 } = {}) { + const baseMs = baseline ? Date.parse(baseline) : 0 + for (let i = 0; i < attempts; i += 1) { + await new Promise((resolve) => setTimeout(resolve, delayMs)) + const stamp = await currentTelemetryStamp() + if (stamp && Date.parse(stamp) > baseMs) return true + } + return false +} + // Fetch the declared panels for this asset, then each panel's data endpoint. async function load() { panels.value = [] @@ -216,6 +288,10 @@ async function load() { } watch(() => props.assetid, load, { immediate: true }) + +// Let a parent detail page (for example PCDetail's header Refresh button) +// trigger a panel reload without duplicating the fetch plumbing. +defineExpose({ refresh: load })