From 4a1b97362b59205ed85ba3384c9ee1e96a08efae Mon Sep 17 00:00:00 2001 From: "benjamin.baker" Date: Mon, 24 Aug 2026 14:29:02 -0400 Subject: [PATCH] Add site asset CSV import with fuzzy reconciliation Sites seed inventory from the Asset Collector script or spreadsheet exports, but lookup names drift between sources (Dell vs Dell Inc.), which would otherwise create duplicate vendors/models/types on every import. The importer now normalizes and fuzzy-matches near-miss references onto existing lookup rows within an operator-set similarity threshold, auto-applying the match while reporting it for review so nothing merges silently. Operating systems and asset rows are never reconciled. Adds the dedicated Import Site Assets page with script/template download and a multi-file dry-run/commit workflow. --- frontend/src/api/index.js | 24 + frontend/src/router/routes/core.js | 6 + .../src/views/settings/ImportSiteAssets.vue | 659 ++++++++++++++++++ frontend/src/views/settings/SiteSettings.vue | 8 +- frontend/src/views/settings/settingsNav.js | 3 +- .../assetcollector/Collect-ShopDB.ps1 | 542 ++++++++++++++ .../site_imports/assetcollector/overview.md | 91 +++ .../assetcollector/sample/assets.csv | 5 + .../assetcollector/sample/computers.csv | 3 + .../assetcollector/sample/machines.csv | 3 + .../assetcollector/sample/models.csv | 5 + .../sample/operatingsystems.csv | 3 + .../assetcollector/sample/vendors.csv | 4 + shopdb/core/api/settings.py | 227 ++++++ shopdb/core/services/csvimport.py | 102 ++- 15 files changed, 1681 insertions(+), 4 deletions(-) create mode 100644 frontend/src/views/settings/ImportSiteAssets.vue create mode 100644 scripts/site_imports/assetcollector/Collect-ShopDB.ps1 create mode 100644 scripts/site_imports/assetcollector/overview.md create mode 100644 scripts/site_imports/assetcollector/sample/assets.csv create mode 100644 scripts/site_imports/assetcollector/sample/computers.csv create mode 100644 scripts/site_imports/assetcollector/sample/machines.csv create mode 100644 scripts/site_imports/assetcollector/sample/models.csv create mode 100644 scripts/site_imports/assetcollector/sample/operatingsystems.csv create mode 100644 scripts/site_imports/assetcollector/sample/vendors.csv diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 0e9e54bb..6712925c 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -1014,6 +1014,30 @@ export const settingsApi = { form.append('kind', kind) form.append('file', file) return api.post('/settings/branding-logo', form, { headers: { 'Content-Type': 'multipart/form-data' } }) + }, + downloadCollectorScript() { + // Collect-ShopDB.ps1 with this site's CSV path baked into $NetworkStoreRoot + return api.get('/settings/asset-import/script', { responseType: 'blob' }) + }, + listImportTemplates() { + // Importable CSV tables in the order they must be uploaded + return api.get('/settings/asset-import/templates') + }, + downloadImportTemplate(tablename) { + // Generated CSV template (header + example row) for one table + return api.get(`/settings/asset-import/templates/${encodeURIComponent(tablename)}`, { responseType: 'blob' }) + }, + importAssetCsvs(files, commit = false) { + // Multipart upload of one or more CSVs. commit=false is a dry-run preview + // that writes nothing; commit=true applies the whole set in one transaction. + const form = new FormData() + for (const file of files) { + form.append('files', file) + } + form.append('commit', commit ? 'true' : 'false') + return api.post('/settings/asset-import/import', form, { + headers: { 'Content-Type': 'multipart/form-data' } + }) } } diff --git a/frontend/src/router/routes/core.js b/frontend/src/router/routes/core.js index 0d98321a..b2bc3864 100644 --- a/frontend/src/router/routes/core.js +++ b/frontend/src/router/routes/core.js @@ -150,6 +150,12 @@ export default [ component: () => import('../../views/settings/BrandingSettings.vue'), meta: { requiresAuth: true, requiresAdmin: true } }, + { + path: 'settings/importsiteassets', + name: 'import-site-assets', + component: () => import('../../views/settings/ImportSiteAssets.vue'), + meta: { requiresAuth: true, requiresAdmin: true } + }, { path: 'settings/printing', name: 'printing-settings', diff --git a/frontend/src/views/settings/ImportSiteAssets.vue b/frontend/src/views/settings/ImportSiteAssets.vue new file mode 100644 index 00000000..60ed215c --- /dev/null +++ b/frontend/src/views/settings/ImportSiteAssets.vue @@ -0,0 +1,659 @@ + + + + + diff --git a/frontend/src/views/settings/SiteSettings.vue b/frontend/src/views/settings/SiteSettings.vue index 77a77221..7be8eb45 100644 --- a/frontend/src/views/settings/SiteSettings.vue +++ b/frontend/src/views/settings/SiteSettings.vue @@ -75,7 +75,8 @@ const LABELS = { employee_directory_mode: 'Employee Directory Mode', usb_directory_mode: 'USB Directory Mode', site_timezone: 'Site Timezone', - setup_complete: 'Setup Complete' + setup_complete: 'Setup Complete', + asset_import_csv_location: 'Asset Import CSV Location' } function prettyLabel(key) { return LABELS[key] || key @@ -127,7 +128,10 @@ async function load() { try { const response = await settingsApi.list() const all = response.data?.data || response.data || [] - items.value = all.filter(s => s.category === 'site') + // The Asset Collector CSV location is managed on the dedicated Import Site + // Assets page. + const importKeys = ['asset_import_csv_location'] + items.value = all.filter(s => s.category === 'site' && !importKeys.includes(s.key)) } catch (err) { console.error('Error loading site settings:', err) error.value = 'Could not load settings.' diff --git a/frontend/src/views/settings/settingsNav.js b/frontend/src/views/settings/settingsNav.js index c1fb32d9..a5b87238 100644 --- a/frontend/src/views/settings/settingsNav.js +++ b/frontend/src/views/settings/settingsNav.js @@ -4,7 +4,7 @@ // Order: site identity + the reference-data catalogs users touch daily come // first, then the platform/system groups (integrations, communication, search, // plugins, access) cluster together at the end. -import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key } from 'lucide-vue-next' +import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key, Upload } from 'lucide-vue-next' export const settingsGroups = [ { @@ -13,6 +13,7 @@ export const settingsGroups = [ { to: '/settings/site', icon: Home, title: 'Site & Facility', description: 'Site URL/FQDN, facility name, PC access domain, employee-ID pattern, printer hostname template' }, { to: '/settings/floormap', icon: MapPin, title: 'Floor Map', description: 'Facility floor-plan blueprint image and pixel dimensions (width, height) for the site map' }, { to: '/settings/branding', icon: Palette, title: 'Branding', description: 'Site, QR, and badge logos, favicon, and brand primary, accent, sidebar, and theme colors' }, + { to: '/settings/importsiteassets', icon: Upload, title: 'Import Site Assets', description: 'Seed Machines, PCs, and Applications via the Asset Collector script, existing databases, or Intune/Azure exports; set the shared CSV collection location and download collected files' }, ], }, { diff --git a/scripts/site_imports/assetcollector/Collect-ShopDB.ps1 b/scripts/site_imports/assetcollector/Collect-ShopDB.ps1 new file mode 100644 index 00000000..f66c4b8d --- /dev/null +++ b/scripts/site_imports/assetcollector/Collect-ShopDB.ps1 @@ -0,0 +1,542 @@ +#Requires -Version 5.1 +[CmdletBinding()] +param( + # Where CSVs are collected/accumulated. Overrides the in-script CONFIG block + # below. See the CONFIG block for the easy place to set this permanently. + [Parameter(Mandatory = $false)] + [string]$OutputRoot, + + # Optional network share to mirror the store into after a run. Overrides the + # in-script CONFIG block below. + [Parameter(Mandatory = $false)] + [string]$PublishRoot, + + # Business identifier for this PC (workstation number). Defaults to hostname. + [Parameter(Mandatory = $false)] + [string]$AssetNumber, + + # Value written into assets.assettypeid for the PC (resolved by name). + [Parameter(Mandatory = $false)] + [string]$AssetType = 'Computer', + + # Value written into computers.computertypeid (resolved by name). + [Parameter(Mandatory = $false)] + [string]$ComputerType, + + # Value written into machines.machinetypeid (resolved by name). + [Parameter(Mandatory = $false)] + [string]$MachineType, + + # Explicit eDNC/UDC source overrides. When omitted, standard on-asset + # locations are searched. + [Parameter(Mandatory = $false)] + [string]$RegPath, + [Parameter(Mandatory = $false)] + [string]$JsonPath, + + # Skip switches for partial runs. + [switch]$SkipMachine, + [switch]$SkipApps +) + +<# +.SYNOPSIS + Unified ShopDB site-adoption collector. Runs on any PC, collects the PC and any + attached eDNC/DNC machine, and UPSERTS the results into a shared set of + importable CSVs so a site can build its inventory PC by PC. + +.DESCRIPTION + Accumulating store under OutputRoot: + vendors.csv, models.csv, operatingsystems.csv, assets.csv, + computers.csv, machines.csv + Rows are merged field-by-field on each table's natural key (a later run fills + gaps and refreshes values without wiping data an earlier run captured). + + Phase-2 data that CSV cannot carry is written per entity under OutputRoot\sidecars: + computer_.json (interfaces, installed apps) + machine_.json (PC<->Machine link, connection detail) + + Read-only against the machine. Local reads may run elevated; publishing to a + share runs in the current user context only. + +.NOTES + PS 5.1 compatible. Headers match the ShopDB importer (see CSV-SCHEMA.md). +#> + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# ============================================================================= +# CONFIG - SET YOUR CSV DESTINATIONS HERE +# ----------------------------------------------------------------------------- +# Two-stage flow so you can collect locally (possibly elevated) and then append +# the captured rows to a CSV store that lives on a network share: +# +# 1) LocalStageRoot Fast, always-writable local folder where CSVs are built +# and merged. Collection can run elevated here. +# 2) NetworkStoreRoot The shared CSV store the whole site accumulates into and +# that you later point 'flask csv import --dir' at. Leave +# blank ('') to keep everything local. When set, the merged +# store is copied here at the end of the run. Network +# writes must happen as a STANDARD user - do NOT run the +# publish step elevated. +# +# Example network path: '\\fileserver\shopdb$\intake' +# +# Command-line -OutputRoot / -PublishRoot still override these values. +# ============================================================================= + +$LocalStageRoot = Join-Path $env:ProgramData 'ShopDB\intake' +$NetworkStoreRoot = '\\wi001dataaege.wlm.geaerospace.net\shared\TEAMSYS\.sfscripts\shopdb' + +# Resolve effective destinations (command-line params win over CONFIG above). +if (-not $OutputRoot) { $OutputRoot = $LocalStageRoot } +if (-not $PublishRoot) { $PublishRoot = $NetworkStoreRoot } + +# --- Store definitions (header order + natural key per importable table) ------- + +$script:Stores = [ordered]@{ + 'vendors.csv' = @{ key = 'vendor'; header = @('vendor') } + 'models.csv' = @{ key = 'modelnumber'; header = @('modelnumber','vendorid','modeltypeid') } + 'operatingsystems.csv' = @{ key = 'osname'; header = @('osname') } + 'assets.csv' = @{ key = 'assetnumber'; header = @('assetnumber','assettypeid','name','statusid','serialnumber','gaugelabreference','maintenancereference','locationid','businessunitid','notes') } + 'computers.csv' = @{ key = 'assetid'; header = @('assetid','hostname','computertypeid','osid','vendorid','modelnumberid','loggedinuser','lastreporteddate','lastboottime') } + 'machines.csv' = @{ key = 'assetid'; header = @('assetid','machinetypeid','vendorid','modelnumberid','controllervendorid','controllermodelid','requiresmanualconfig','islocationonly','notes') } +} + +# --- Shared helpers ----------------------------------------------------------- + +function ConvertTo-CsvField { + param([AllowNull()]$Value) + if ($null -eq $Value) { return '' } + $text = [string]$Value + if ($text -match '[",\r\n]') { return '"' + ($text -replace '"', '""') + '"' } + return $text +} + +function Get-Prop { + param([AllowNull()]$Object, [Parameter(Mandatory = $true)][string]$Name) + if ($null -eq $Object) { return $null } + $prop = $Object.PSObject.Properties[$Name] + if ($null -eq $prop) { return $null } + return $prop.Value +} + +function Get-NonEmpty { + param([AllowNull()]$Value) + if ($null -ne $Value -and -not [string]::IsNullOrWhiteSpace([string]$Value)) { + return ([string]$Value).Trim() + } + return $null +} + +function Get-FirstNonEmpty { + param([object[]]$Values) + foreach ($v in $Values) { $n = Get-NonEmpty $v; if ($n) { return $n } } + return $null +} + +function Format-ShopDbDate { + param([AllowNull()]$Value) + if ($null -eq $Value) { return $null } + try { return ([datetime]$Value).ToString('yyyy-MM-dd HH:mm:ss') } catch { return $null } +} + +function Write-Utf8NoBom { + param([string]$Path, [string]$Content) + $encoding = New-Object System.Text.UTF8Encoding($false) + [System.IO.File]::WriteAllText($Path, $Content, $encoding) +} + +# Field-level upsert into a persistent CSV store, keyed by natural key. Existing +# values are kept when the incoming cell is empty, so a partial run never blanks +# out data an earlier run captured. +function Merge-CsvStore { + param( + [Parameter(Mandatory = $true)][string]$Path, + [Parameter(Mandatory = $true)][string[]]$Header, + [Parameter(Mandatory = $true)][string]$Key, + [Parameter(Mandatory = $true)][AllowNull()][object[]]$IncomingRows + ) + + $index = [ordered]@{} + if (Test-Path -LiteralPath $Path) { + foreach ($existing in @(Import-Csv -LiteralPath $Path)) { + $keyVal = Get-NonEmpty (Get-Prop $existing $Key) + if (-not $keyVal) { continue } + $index[$keyVal.ToLowerInvariant()] = $existing + } + } + + $changed = 0 + foreach ($row in $IncomingRows) { + if ($null -eq $row) { continue } + $keyVal = Get-NonEmpty (Get-Prop $row $Key) + if (-not $keyVal) { continue } + $lc = $keyVal.ToLowerInvariant() + if ($index.Contains($lc)) { + $target = $index[$lc] + foreach ($col in $Header) { + $incoming = Get-NonEmpty (Get-Prop $row $col) + if ($incoming) { + if ($target.PSObject.Properties[$col]) { $target.$col = $incoming } + else { $target | Add-Member -NotePropertyName $col -NotePropertyValue $incoming -Force } + } + } + } else { + $fresh = [ordered]@{} + foreach ($col in $Header) { $fresh[$col] = Get-NonEmpty (Get-Prop $row $col) } + $index[$lc] = [pscustomobject]$fresh + } + $changed++ + } + + $lines = New-Object System.Collections.Generic.List[string] + $lines.Add(($Header -join ',')) + foreach ($stored in $index.Values) { + $cells = foreach ($col in $Header) { ConvertTo-CsvField (Get-Prop $stored $col) } + $lines.Add(($cells -join ',')) + } + Write-Utf8NoBom -Path $Path -Content (($lines -join "`r`n") + "`r`n") + return $changed +} + +# --- eDNC/UDC parsing (registry, .reg export, UDC settings json) -------------- + +function Resolve-ControllerVendor { + param([AllowNull()][string]$ControlType, [AllowNull()][string]$NcIF) + $ct = ('' + $ControlType).Trim().ToUpperInvariant() + $nc = ('' + $NcIF).Trim().ToUpperInvariant() + if ($nc -eq 'EFOCAS' -or $ct -eq 'EFOCAS') { return 'FANUC' } + if ($ct -eq 'SIEMENS') { return 'Siemens' } + if ($ct -eq 'OKUMA') { return 'Okuma' } + return $null +} + +function Read-RegExport { + param([Parameter(Mandatory = $true)][string]$Path) + $lines = @(Get-Content -LiteralPath $Path -Encoding Unicode -ErrorAction Stop) + $firstMeaningful = ($lines | Where-Object { $_ -match '\S' } | Select-Object -First 1) + if (-not ($firstMeaningful -match '^(REGEDIT4|Windows Registry Editor Version)')) { + $lines = @(Get-Content -LiteralPath $Path -Encoding Default -ErrorAction Stop) + } + $data = @{} + $currentKey = $null + foreach ($raw in $lines) { + $line = $raw.Trim() + if ([string]::IsNullOrWhiteSpace($line)) { continue } + if ($line -match '^(REGEDIT4|Windows Registry Editor Version)') { continue } + if ($line -match '^\s*;') { continue } + if ($line -match '^\[(?[^\]]+)\]$') { + $currentKey = $Matches['key'] + if (-not $data.ContainsKey($currentKey)) { $data[$currentKey] = @{} } + continue + } + if (-not $currentKey) { continue } + if ($line -match '^"(?[^"]+)"=(?.+)$') { + $name = $Matches['name'] + $rhs = $Matches['rhs'].Trim() + if ($rhs -match '^"(?.*)"$') { $val = $Matches['s'] } + elseif ($rhs -match '^dword:(?[0-9a-fA-F]{8})$') { $val = [Convert]::ToInt32($Matches['hex'], 16) } + else { $val = $rhs } + $data[$currentKey][$name] = $val + } + } + return $data +} + +function Get-RegExportValue { + param([hashtable]$Reg, [string]$KeySuffix, [string]$Name) + foreach ($key in $Reg.Keys) { + if ($key -match ('\\' + [regex]::Escape($KeySuffix) + '$')) { + if ($Reg[$key].ContainsKey($Name)) { return $Reg[$key][$Name] } + } + } + return $null +} + +function ConvertFrom-RegExport { + param([hashtable]$Reg) + [ordered]@{ + source = 'reg' + machinenumber = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'MachineNo') + site = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'Site') + controltype = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'Cnc') + ncif = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'NcIF') + dnctype = 'eDNC' + tcpaddress = Get-NonEmpty (Get-RegExportValue $Reg 'eFocas' 'IpAddr') + tcpport = Get-NonEmpty (Get-RegExportValue $Reg 'eFocas' 'SocketNo') + comport = Get-NonEmpty (Get-RegExportValue $Reg 'PPDCS' 'Port Id') + baudrate = Get-NonEmpty (Get-RegExportValue $Reg 'PPDCS' 'Baud') + } +} + +function Get-RegistryValue { + param([string]$Path, [string]$Name) + if (-not (Test-Path -LiteralPath $Path)) { return $null } + return Get-Prop (Get-ItemProperty -LiteralPath $Path -ErrorAction SilentlyContinue) $Name +} + +function ConvertFrom-LiveRegistry { + param([string]$Base) + if (-not (Test-Path -LiteralPath $Base)) { return $null } + $general = Join-Path $Base 'General' + $efocas = Join-Path $Base 'eFocas' + $ppdcs = Join-Path $Base 'PPDCS' + [ordered]@{ + source = 'registry' + machinenumber = Get-NonEmpty (Get-RegistryValue $general 'MachineNo') + site = Get-NonEmpty (Get-RegistryValue $general 'Site') + controltype = Get-NonEmpty (Get-RegistryValue $general 'Cnc') + ncif = Get-NonEmpty (Get-RegistryValue $general 'NcIF') + dnctype = 'eDNC' + tcpaddress = Get-NonEmpty (Get-RegistryValue $efocas 'IpAddr') + tcpport = Get-NonEmpty (Get-RegistryValue $efocas 'SocketNo') + comport = Get-NonEmpty (Get-RegistryValue $ppdcs 'Port Id') + baudrate = Get-NonEmpty (Get-RegistryValue $ppdcs 'Baud') + } +} + +function ConvertFrom-UdcJson { + param([string]$Path) + $obj = Get-Content -LiteralPath $Path -Raw -ErrorAction Stop | ConvertFrom-Json + $general = Get-Prop $obj 'GeneralSettings' + $ethernet = Get-Prop $obj 'EthernetSettings' + $serial = Get-Prop $obj 'SerialSettings' + [ordered]@{ + source = 'json' + machinenumber = Get-NonEmpty (Get-Prop $general 'MachineNumber') + site = Get-NonEmpty (Get-Prop $general 'Site') + controltype = Get-NonEmpty (Get-Prop $general 'ControlType') + ncif = $null + dnctype = Get-NonEmpty (Get-Prop $general 'DncType') + tcpaddress = Get-NonEmpty (Get-Prop $ethernet 'TCPAddress') + tcpport = Get-NonEmpty (Get-Prop $ethernet 'TCPPort') + comport = Get-NonEmpty (Get-Prop $serial 'ComPort') + baudrate = Get-NonEmpty (Get-Prop $serial 'BaudRate') + } +} + +function Find-FirstFile { + param([string[]]$Roots, [string]$Filter) + foreach ($root in $Roots) { + if (-not $root -or -not (Test-Path -LiteralPath $root)) { continue } + $hit = Get-ChildItem -LiteralPath $root -Filter $Filter -File -ErrorAction SilentlyContinue | + Sort-Object LastWriteTime -Descending | Select-Object -First 1 + if ($hit) { return $hit.FullName } + } + return $null +} + +function Resolve-DncSource { + param([string]$RegPath, [string]$JsonPath) + if ($JsonPath -and (Test-Path -LiteralPath $JsonPath)) { + Write-Host (' machine source: JSON (explicit) ' + $JsonPath) -ForegroundColor DarkGray + return ConvertFrom-UdcJson -Path $JsonPath + } + if ($RegPath -and (Test-Path -LiteralPath $RegPath)) { + Write-Host (' machine source: .reg (explicit) ' + $RegPath) -ForegroundColor DarkGray + return ConvertFrom-RegExport -Reg (Read-RegExport -Path $RegPath) + } + foreach ($base in @( + 'HKLM:\SOFTWARE\GE Aircraft Engines\DNC', + 'HKLM:\SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC')) { + $fromReg = ConvertFrom-LiveRegistry -Base $base + if ($fromReg -and $fromReg.machinenumber) { + Write-Host (' machine source: live registry ' + $base) -ForegroundColor DarkGray + return $fromReg + } + } + $localReg = Find-FirstFile -Roots @('C:\Program Files (x86)\Dnc', (Join-Path $env:SystemDrive 'GE Aerospace')) -Filter '*.reg' + if ($localReg) { + Write-Host (' machine source: local .reg ' + $localReg) -ForegroundColor DarkGray + return ConvertFrom-RegExport -Reg (Read-RegExport -Path $localReg) + } + $localJson = Find-FirstFile -Roots @((Join-Path $env:SystemDrive 'GE Aerospace'), 'C:\Program Files (x86)\UDC', 'C:\Program Files (x86)\Dnc') -Filter 'udc_settings_*.json' + if ($localJson) { + Write-Host (' machine source: local UDC json ' + $localJson) -ForegroundColor DarkGray + return ConvertFrom-UdcJson -Path $localJson + } + return $null +} + +# --- PC collection (read-only) ------------------------------------------------ + +function Get-PcFacts { + $computerSystem = Get-CimInstance Win32_ComputerSystem -ErrorAction SilentlyContinue + $operatingSys = Get-CimInstance Win32_OperatingSystem -ErrorAction SilentlyContinue + $bios = Get-CimInstance Win32_BIOS -ErrorAction SilentlyContinue + + $interfaces = @() + try { + foreach ($cfg in @(Get-NetIPConfiguration -ErrorAction SilentlyContinue)) { + $adapter = $null + try { $adapter = Get-NetAdapter -InterfaceAlias $cfg.InterfaceAlias -ErrorAction SilentlyContinue } catch {} + if (-not $adapter -or (Get-Prop $adapter 'HardwareInterface') -ne $true) { continue } + $ipv4 = @(Get-Prop $cfg 'IPv4Address') | Select-Object -First 1 + $interfaces += [ordered]@{ + interfacealias = [string]$cfg.InterfaceAlias + ipaddress = if ($ipv4) { [string](Get-Prop $ipv4 'IPAddress') } else { $null } + macaddress = [string](Get-Prop $adapter 'MacAddress') + } + } + } catch {} + + $installedApps = @() + if (-not $SkipApps) { + foreach ($keyPath in @( + 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\*', + 'HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\*')) { + foreach ($entry in @(Get-ItemProperty -Path $keyPath -ErrorAction SilentlyContinue)) { + $displayName = Get-NonEmpty (Get-Prop $entry 'DisplayName') + if (-not $displayName) { continue } + if ((Get-Prop $entry 'SystemComponent') -eq 1) { continue } + $installedApps += [ordered]@{ + appname = $displayName + installedversion = Get-NonEmpty (Get-Prop $entry 'DisplayVersion') + publisher = Get-NonEmpty (Get-Prop $entry 'Publisher') + } + } + } + $installedApps = @($installedApps | Sort-Object { $_.appname } -Unique) + } + + [ordered]@{ + hostname = Get-FirstNonEmpty @($env:COMPUTERNAME, (Get-Prop $computerSystem 'Name')) + vendor = Get-NonEmpty (Get-Prop $computerSystem 'Manufacturer') + model = Get-NonEmpty (Get-Prop $computerSystem 'Model') + osname = Get-NonEmpty (Get-Prop $operatingSys 'Caption') + serial = Get-FirstNonEmpty @((Get-Prop $bios 'SerialNumber'), (Get-Prop $computerSystem 'SerialNumber')) + lastboot = Format-ShopDbDate (Get-Prop $operatingSys 'LastBootUpTime') + loggeduser = Get-NonEmpty (Get-Prop $computerSystem 'UserName') + interfaces = $interfaces + installedapps = $installedApps + } +} + +# --- Main --------------------------------------------------------------------- + +Write-Host '' +Write-Host ' ShopDB unified collector' -ForegroundColor Cyan +Write-Host (' ' + ('-' * 56)) -ForegroundColor DarkGray + +if (-not (Test-Path -LiteralPath $OutputRoot)) { New-Item -ItemType Directory -Path $OutputRoot -Force | Out-Null } +$sidecarDir = Join-Path $OutputRoot 'sidecars' +if (-not (Test-Path -LiteralPath $sidecarDir)) { New-Item -ItemType Directory -Path $sidecarDir -Force | Out-Null } + +$collectedAt = (Get-Date).ToString('yyyy-MM-dd HH:mm:ss') + +$pc = Get-PcFacts +if (-not $AssetNumber) { $AssetNumber = $pc.hostname } + +# Row sets keyed to the shared stores. +$vendorRows = @() +$modelRows = @() +$osRows = @() +$assetRows = @() +$computerRows = @() +$machineRows = @() + +if ($pc.vendor) { $vendorRows += [pscustomobject]@{ vendor = $pc.vendor } } +if ($pc.model) { $modelRows += [pscustomobject]@{ modelnumber = $pc.model; vendorid = $pc.vendor; modeltypeid = $null } } +if ($pc.osname) { $osRows += [pscustomobject]@{ osname = $pc.osname } } + +$assetRows += [pscustomobject]@{ + assetnumber = $AssetNumber; assettypeid = $AssetType; name = $pc.hostname; statusid = $null + serialnumber = $pc.serial; gaugelabreference = $null; maintenancereference = $null + locationid = $null; businessunitid = $null + notes = 'PC collected ' + $collectedAt +} +$computerRows += [pscustomobject]@{ + assetid = $AssetNumber; hostname = $pc.hostname; computertypeid = $ComputerType; osid = $pc.osname + vendorid = $pc.vendor; modelnumberid = $pc.model; loggedinuser = $pc.loggeduser + lastreporteddate = $collectedAt; lastboottime = $pc.lastboot +} + +# Machine (best-effort). +$machine = $null +if (-not $SkipMachine) { + try { $machine = Resolve-DncSource -RegPath $RegPath -JsonPath $JsonPath } catch { $machine = $null } +} +if ($machine -and $machine.machinenumber) { + $controllerVendor = Resolve-ControllerVendor -ControlType $machine.controltype -NcIF $machine.ncif + $transport = if ($machine.tcpaddress) { 'ethernet' } elseif ($machine.comport) { 'serial' } else { 'unknown' } + $noteBits = @("DNC=$($machine.dnctype)") + if ($machine.controltype) { $noteBits += "control=$($machine.controltype)" } + $noteBits += "transport=$transport" + $machineNote = 'eDNC collected ' + $collectedAt + '; ' + ($noteBits -join '; ') + + if ($controllerVendor) { $vendorRows += [pscustomobject]@{ vendor = $controllerVendor } } + $assetRows += [pscustomobject]@{ + assetnumber = $machine.machinenumber; assettypeid = 'Machine'; name = $machine.machinenumber; statusid = $null + serialnumber = $null; gaugelabreference = $null; maintenancereference = $null + locationid = $null; businessunitid = $null; notes = $machineNote + } + $machineRows += [pscustomobject]@{ + assetid = $machine.machinenumber; machinetypeid = $MachineType; vendorid = $null; modelnumberid = $null + controllervendorid = $controllerVendor; controllermodelid = $null + requiresmanualconfig = $null; islocationonly = $null; notes = $machineNote + } + + $link = [ordered]@{ + pcassetnumber = $AssetNumber; machineassetnumber = $machine.machinenumber; relationship = 'controls' + dnctype = $machine.dnctype; controltype = $machine.controltype; controllervendor = $controllerVendor + site = $machine.site; source = $machine.source; collectedat = $collectedAt + connection = [ordered]@{ + transport = $transport; tcpaddress = $machine.tcpaddress; tcpport = $machine.tcpport + comport = $machine.comport; baudrate = $machine.baudrate + } + } + Write-Utf8NoBom -Path (Join-Path $sidecarDir ('machine_' + $machine.machinenumber + '.json')) -Content ($link | ConvertTo-Json -Depth 6) + Write-Host (' linked machine ' + $machine.machinenumber + ' (' + $controllerVendor + ', ' + $transport + ')') -ForegroundColor DarkGray +} elseif (-not $SkipMachine) { + Write-Host ' no eDNC/DNC/UDC machine config found on this PC' -ForegroundColor DarkGray +} + +# --- Upsert into the shared store -------------------------------------------- + +$rowsByStore = @{ + 'vendors.csv' = $vendorRows + 'models.csv' = $modelRows + 'operatingsystems.csv' = $osRows + 'assets.csv' = $assetRows + 'computers.csv' = $computerRows + 'machines.csv' = $machineRows +} + +Write-Host '' +Write-Host (' Store: ' + $OutputRoot) -ForegroundColor Green +foreach ($fileName in $script:Stores.Keys) { + $spec = $script:Stores[$fileName] + $rows = @($rowsByStore[$fileName]) + if ($rows.Count -eq 0) { continue } + $path = Join-Path $OutputRoot $fileName + $n = Merge-CsvStore -Path $path -Header $spec.header -Key $spec.key -IncomingRows $rows + Write-Host (' merged ' + $n + ' row(s) into ' + $fileName) -ForegroundColor DarkGray +} + +# PC phase-2 sidecar. +$pcSidecar = [ordered]@{ + assetnumber = $AssetNumber; hostname = $pc.hostname; collectedat = $collectedAt + interfaces = $pc.interfaces; installedapps = $pc.installedapps +} +Write-Utf8NoBom -Path (Join-Path $sidecarDir ('computer_' + $AssetNumber + '.json')) -Content ($pcSidecar | ConvertTo-Json -Depth 6) + +# --- Publish (standard-user network mirror, optional) ------------------------- + +if ($PublishRoot) { + if (([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole( + [Security.Principal.WindowsBuiltInRole]::Administrator)) { + Write-Host '' + Write-Host ' WARNING: publishing while elevated. Network writes should run as the standard user.' -ForegroundColor Yellow + } + if (-not (Test-Path -LiteralPath $PublishRoot)) { New-Item -ItemType Directory -Path $PublishRoot -Force | Out-Null } + Copy-Item -Path (Join-Path $OutputRoot '*') -Destination $PublishRoot -Recurse -Force + Write-Host '' + Write-Host (' Published store to: ' + $PublishRoot) -ForegroundColor Green +} + +Write-Host '' +Write-Host ' Done. Dry-run validate the accumulated store with:' -ForegroundColor Cyan +Write-Host (' flask csv import --dir "' + $OutputRoot + '"') -ForegroundColor White +Write-Host '' diff --git a/scripts/site_imports/assetcollector/overview.md b/scripts/site_imports/assetcollector/overview.md new file mode 100644 index 00000000..1587c2f3 --- /dev/null +++ b/scripts/site_imports/assetcollector/overview.md @@ -0,0 +1,91 @@ +# Collect-ShopDB.ps1 Overview + +## Overview + +- **Purpose**: A unified, read-only "site adoption" collector that runs on any + Windows PC, gathers facts about that PC plus any attached eDNC/DNC/UDC machine, + and merges them into a shared set of importable CSVs so a site can build its + ShopDB inventory one PC at a time. +- **Compatibility**: PowerShell 5.1+; CSV headers are aligned to the ShopDB + importer (`flask csv import --dir`). + +## Configuration and destinations + +- **Two-stage flow**: a fast local stage folder (`%ProgramData%\ShopDB\intake`) + where CSVs are built/merged, and an optional network store (a file share) that + the whole site accumulates into. +- Command-line params `-OutputRoot` and `-PublishRoot` override the in-script + CONFIG block. +- Other params: `-AssetNumber` (defaults to hostname), + `-AssetType`/`-ComputerType`/`-MachineType` (resolved by name), explicit + `-RegPath`/`-JsonPath` machine source overrides, and `-SkipMachine`/`-SkipApps` + switches for partial runs. + +## Data model (the stores) + +- Six importable CSVs, each with a defined header order and natural key: + `vendors.csv`, `models.csv`, `operatingsystems.csv`, `assets.csv`, + `computers.csv`, `machines.csv`. +- Column names follow ShopDB conventions (e.g., `assetid`, `modelnumberid`, + `controllervendorid`, `lastreporteddate`). + +## PC collection (`Get-PcFacts`, read-only) + +- Uses CIM/WMI (`Win32_ComputerSystem`, `Win32_OperatingSystem`, `Win32_BIOS`) + to gather hostname, vendor/manufacturer, model, OS caption, serial number, + last boot time, and logged-in user. +- Enumerates **network interfaces** (hardware adapters only: alias, IPv4, MAC). +- Enumerates **installed applications** from the registry Uninstall keys (name, + version, publisher), skipping system components - unless `-SkipApps` is set. + +## Machine (eDNC/DNC/UDC) collection + +- `Resolve-DncSource` searches in priority order: explicit JSON path -> explicit + .reg path -> live registry (`HKLM\...\GE Aircraft Engines\DNC`) -> local `.reg` + export files -> local `udc_settings_*.json` files. +- Three parsers normalize the different source formats into one shape: + `ConvertFrom-RegExport` (parses exported `.reg` files including dword decoding), + `ConvertFrom-LiveRegistry`, and `ConvertFrom-UdcJson`. +- Extracts machine number, site, control type, DNC type, and connection detail + (TCP address/port for ethernet, COM port/baud for serial). +- `Resolve-ControllerVendor` maps control type / NcIF to a controller vendor + (e.g., eFOCAS -> FANUC, Siemens, Okuma). + +## Merge / upsert logic (`Merge-CsvStore`) + +- Field-level UPSERT keyed on each table's natural key: existing rows are matched + case-insensitively, and empty incoming cells never overwrite previously captured + values - so partial or repeated runs fill gaps without wiping earlier data. +- CSV writing is UTF-8 without BOM, with proper quoting/escaping. + +## Sidecar files (phase-2 data CSV can't carry) + +- Per-PC `sidecars\computer_.json`: interfaces + installed apps. +- Per-machine `sidecars\machine_.json`: PC-to-machine link + (relationship "controls"), DNC/control details, and connection transport info. + +## Publishing and output + +- Optionally mirrors the whole local store to the network share + (`Copy-Item -Recurse -Force`), with a warning if run elevated (network writes + should run as standard user). +- Prints a summary and the follow-up dry-run command: + `flask csv import --dir ""`. + +--- + +## Relevance to an "Asset Collector" package + +This script is a solid template for one collector in a broader package. Notable +patterns worth generalizing across other collectors (existing databases, +Intune/Azure exports, etc.): + +- **Normalized intermediate schema** (the six CSV stores + JSON sidecars) that + all sources converge on, decoupling collection from import. +- **Idempotent field-level merge** so multiple heterogeneous sources can + contribute to the same asset without clobbering each other. +- **Source-priority resolution** and per-format parsers - the same shape could + wrap an Intune/Azure `Get-MgDevice` export or a SQL query. +- **Machines, PCs, Applications** are already modeled here; an Intune/Azure + source would map naturally onto `computers.csv` + the installed-apps sidecar, + while a MES/CNC database could feed `machines.csv`. diff --git a/scripts/site_imports/assetcollector/sample/assets.csv b/scripts/site_imports/assetcollector/sample/assets.csv new file mode 100644 index 00000000..a4db46bd --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/assets.csv @@ -0,0 +1,5 @@ +assetnumber,name,serialnumber,assettypeid,statusid,locationid,businessunitid,notes +SAMPLE-PC-01,Shopfloor PC 01,DELLSN0001,computer,In Use,A1,Gage Lab,Sample collected device +SAMPLE-PC-02,CMM PC 02,DELLSN0002,computer,In Use,A1,Gage Lab,Sample collected device +SAMPLE-MC-01,Zeiss CMM 01,ZEISSSN0001,machine,In Use,A1,Gage Lab,Sample collected device attached to SAMPLE-PC-02 +SAMPLE-MC-02,Haas VF-2SS 02,HAASSN0002,machine,In Use,A1,Gage Lab,Sample collected standalone CNC diff --git a/scripts/site_imports/assetcollector/sample/computers.csv b/scripts/site_imports/assetcollector/sample/computers.csv new file mode 100644 index 00000000..692119f0 --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/computers.csv @@ -0,0 +1,3 @@ +assetid,computertypeid,hostname,osid,vendorid,modelnumberid,loggedinuser +SAMPLE-PC-01,Shopfloor PC,WKS-SAMPLE01,Windows 10 Enterprise,Dell,OptiPlex 7090,AEROAD\operator1 +SAMPLE-PC-02,CMM PC,WKS-SAMPLE02,Windows 11 Enterprise,Dell,Precision 3660,AEROAD\operator2 diff --git a/scripts/site_imports/assetcollector/sample/machines.csv b/scripts/site_imports/assetcollector/sample/machines.csv new file mode 100644 index 00000000..d55e537f --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/machines.csv @@ -0,0 +1,3 @@ +assetid,machinetypeid,vendorid,modelnumberid,requiresmanualconfig,islocationonly +SAMPLE-MC-01,CMM,Zeiss,CONTURA G2,0,0 +SAMPLE-MC-02,CNC,Haas Automation,VF-2SS,0,0 diff --git a/scripts/site_imports/assetcollector/sample/models.csv b/scripts/site_imports/assetcollector/sample/models.csv new file mode 100644 index 00000000..18f29ac6 --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/models.csv @@ -0,0 +1,5 @@ +modelnumber,vendorid +OptiPlex 7090,Dell +Precision 3660,Dell +VF-2SS,Haas Automation +CONTURA G2,Zeiss diff --git a/scripts/site_imports/assetcollector/sample/operatingsystems.csv b/scripts/site_imports/assetcollector/sample/operatingsystems.csv new file mode 100644 index 00000000..f5ca264e --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/operatingsystems.csv @@ -0,0 +1,3 @@ +osname +Windows 10 Enterprise +Windows 11 Enterprise diff --git a/scripts/site_imports/assetcollector/sample/vendors.csv b/scripts/site_imports/assetcollector/sample/vendors.csv new file mode 100644 index 00000000..a6591f59 --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/vendors.csv @@ -0,0 +1,4 @@ +vendor +Dell +Haas Automation +Zeiss diff --git a/shopdb/core/api/settings.py b/shopdb/core/api/settings.py index 60c919e5..75fe8e14 100644 --- a/shopdb/core/api/settings.py +++ b/shopdb/core/api/settings.py @@ -1,6 +1,7 @@ """Settings API routes.""" import os +import re from flask import Blueprint, request, current_app from flask_jwt_extended import jwt_required, get_jwt_identity @@ -240,6 +241,218 @@ def serve_branding_logo(filename): return send_upload(_branding_dir(), filename) +def _asset_import_dir(): + """Configured Asset Collector CSV location, or '' when unset.""" + setting = Setting.query.filter_by(key='asset_import_csv_location').first() + return (setting.value or '').strip() if setting else '' + + +# The collector script that gets served (with the site's CSV path baked in) from +# the Import Site Assets page. Copied into the image under scripts/ (see +# Dockerfile), so it is always available next to the running API. +_COLLECTOR_SCRIPT = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), + 'scripts', 'site_imports', 'assetcollector', 'Collect-ShopDB.ps1') + + +def _embed_store_path(script_text, path): + """Return the collector script with $NetworkStoreRoot set to path. + + The script ships with a site-specific default on that line; rewriting it + means an operator downloads a script already pointed at this facility's CSV + store and does not have to edit PowerShell by hand. A blank path leaves the + shipped default untouched. + """ + if not path: + return script_text + literal = path.replace("'", "''") + replacement = "$NetworkStoreRoot = '%s'" % literal + # A function replacement is used deliberately: a UNC path is full of + # backslashes, and a string replacement runs them through re's escape + # processing (so \s, \w, a trailing \1 all raise or corrupt). A callable + # returns the text verbatim. + return re.sub( + r"(?m)^\$NetworkStoreRoot\s*=.*$", + lambda _match: replacement, + script_text, + count=1) + + +@settings_bp.route('/asset-import/script', methods=['GET']) +@jwt_required() +@require_role('admin') +def download_collector_script(): + """Download Collect-ShopDB.ps1 with this site's CSV path embedded.""" + try: + with open(_COLLECTOR_SCRIPT, 'r', encoding='utf-8') as handle: + script_text = handle.read() + except OSError: + return error_response(ErrorCodes.NOT_FOUND, + 'Collector script is not available on the server', + http_code=404) + + script_text = _embed_store_path(script_text, _asset_import_dir()) + response = current_app.response_class(script_text, mimetype='text/plain') + response.headers['Content-Disposition'] = 'attachment; filename=Collect-ShopDB.ps1' + return response + + +@settings_bp.route('/asset-import/templates', methods=['GET']) +@jwt_required() +@require_role('admin') +def list_import_templates(): + """List the importable CSV tables in the order they must be uploaded.""" + from shopdb.core.services.csvimport import IMPORTABLE + + tables = [{'table': name, 'filename': '%s.csv' % name} + for name in IMPORTABLE] + return success_response({'tables': tables}) + + +@settings_bp.route('/asset-import/templates/', methods=['GET']) +@jwt_required() +@require_role('admin') +def download_import_template(tablename): + """Download a generated CSV template for a single importable table.""" + from shopdb.core.services.csvimport import ( + IMPORTABLE, generate_template, ImportError_) + + if tablename not in IMPORTABLE: + return error_response(ErrorCodes.NOT_FOUND, + 'No such importable table: %s' % tablename, + http_code=404) + try: + template_text = generate_template(tablename) + except ImportError_ as exc: + return error_response(ErrorCodes.VALIDATION_ERROR, str(exc)) + + response = current_app.response_class(template_text, mimetype='text/csv') + response.headers['Content-Disposition'] = ( + 'attachment; filename=%s.csv' % tablename) + return response + + +@settings_bp.route('/asset-import/import', methods=['POST']) +@jwt_required() +@require_role('admin') +def import_asset_csvs(): + """Validate uploaded CSVs and, when commit=true, load them. + + Mirrors the 'flask csv import' CLI: every file runs through import_csv inside + ONE transaction with a shared resolver, so a computers.csv can resolve the + assets.csv uploaded alongside it. A dry run (the default) rolls the whole + transaction back and reports what WOULD change; commit=true persists it. The + whole run is all-or-nothing - if any file has a row problem, nothing is + written even on a commit. + """ + from sqlalchemy.exc import SQLAlchemyError + from shopdb.core.services.csvimport import ( + ImportError_, Resolver, dependency_order, import_csv, table_from_filename) + + uploads = request.files.getlist('files') + if not uploads: + return error_response(ErrorCodes.VALIDATION_ERROR, 'No CSV files were uploaded') + + commit = str(request.form.get('commit', '')).strip().lower() in ( + '1', 'true', 'yes', 'on') + + # Map each upload to a table by filename. A later file with the same table + # name wins, matching the folder-import behaviour (one file per table). + texts = {} + filenames = {} + for upload in uploads: + name = upload.filename or '' + if not name.lower().endswith('.csv'): + continue + table = table_from_filename(name) + try: + texts[table] = upload.read().decode('utf-8-sig') + except (UnicodeDecodeError, OSError): + return error_response(ErrorCodes.VALIDATION_ERROR, + 'Could not read %s as UTF-8 text' % name) + filenames[table] = name + + if not texts: + return error_response(ErrorCodes.VALIDATION_ERROR, + 'None of the uploaded files were CSV files') + + ordered = dependency_order(list(texts)) + skipped = sorted(set(texts) - set(ordered)) + + reconcile_setting = Setting.query.filter_by( + key='asset_import_reconcile_threshold').first() + try: + reconcile_threshold = float(reconcile_setting.value) if reconcile_setting and reconcile_setting.value else 0.0 + except (TypeError, ValueError): + reconcile_threshold = 0.0 + + resolver = Resolver(reconcile_threshold=reconcile_threshold) + files_report = [] + problem_count = 0 + reconciled_count = 0 + request_error = None + + try: + for table in ordered: + try: + result = import_csv(table, texts[table], resolver=resolver, commit=False) + except ImportError_ as exc: + request_error = '%s: %s' % (filenames[table], str(exc)) + break + except SQLAlchemyError as exc: + db.session.rollback() + request_error = '%s: database error: %s' % ( + filenames[table], str(exc).split(chr(10))[0]) + break + + problems = [ + {'line': p.line, 'column': p.column, 'message': p.message} + for p in result.problems + ] + problem_count += len(problems) + reconciled_count += len(result.reconciliations) + files_report.append({ + 'table': table, + 'filename': filenames[table], + 'created': result.created, + 'updated': result.updated, + 'problems': problems, + 'reconciliations': result.reconciliations, + }) + + applied = False + if request_error is not None: + db.session.rollback() + elif problem_count > 0: + # A row problem anywhere fails the whole run, so nothing is written + # even when commit was asked for. + db.session.rollback() + elif commit: + db.session.commit() + applied = True + else: + db.session.rollback() + except Exception: + db.session.rollback() + raise + + total_created = sum(f['created'] for f in files_report) + total_updated = sum(f['updated'] for f in files_report) + + return success_response({ + 'committed': applied, + 'requested_commit': commit, + 'request_error': request_error, + 'files': files_report, + 'skipped': skipped, + 'problem_count': problem_count, + 'reconciled_count': reconciled_count, + 'reconcile_threshold': reconcile_threshold, + 'total_created': total_created, + 'total_updated': total_updated, + }) + + @settings_bp.route('', methods=['GET']) @jwt_required(optional=True) def list_settings(): @@ -556,6 +769,20 @@ def build_default_settings(): 'category': 'site', 'description': 'Email domain appended to a contact SSO to build email/Teams links (sso@domain). Blank disables contact action buttons.' }, + { + 'key': 'asset_import_csv_location', + 'value': '', + 'valuetype': 'string', + 'category': 'site', + 'description': r'Shared network location where the Asset Collector script deposits its CSVs and where ShopDB reads them for import (e.g. \\server.fqdn\share\folder). This path is baked into the downloadable collector script. Blank = not configured.' + }, + { + 'key': 'asset_import_reconcile_threshold', + 'value': '0.85', + 'valuetype': 'string', + 'category': 'site', + 'description': 'Similarity (0 to 1) a fuzzy lookup match must reach for the asset importer to reconcile a near-miss name (e.g. "Dell Inc." to an existing "Dell") onto an existing vendor, model, type, status, business unit, or location. Reconciled matches are applied but reported for review. 0 disables reconciliation. Operating systems and asset rows are never reconciled.' + }, ] # Site branding. Each instance (ADR-004) can replace the shipped GE defaults diff --git a/shopdb/core/services/csvimport.py b/shopdb/core/services/csvimport.py index d114965f..462e8b57 100644 --- a/shopdb/core/services/csvimport.py +++ b/shopdb/core/services/csvimport.py @@ -34,6 +34,7 @@ not at all. from __future__ import annotations import csv +import difflib import io import re from collections import OrderedDict @@ -86,6 +87,43 @@ PASSWORD_COLUMNS = {'passwordhash', 'password'} TRUE_VALUES = {'1', 'true', 'yes', 'y', 't'} FALSE_VALUES = {'0', 'false', 'no', 'n', 'f', ''} +# Reconciliation: when a name does not match a lookup row exactly, the resolver +# may still bind it to an existing row that is close enough. Only lookup tables +# take part - the small reference tables an operator picks values from. Rows that +# ARE the data being imported (assets, computers, machines) and the fast-moving +# operatingsystems list are excluded, so a typo there is a new row, not a silent +# merge onto something unrelated. +RECONCILABLE_TABLES = { + 'assetstatuses', 'assettypes', 'locationtypes', 'modeltypes', + 'computertypes', 'machinetypes', 'businessunits', 'locations', + 'vendors', 'models', +} + +# Corporate suffixes stripped before comparison so 'Dell Inc.' reconciles to +# 'Dell'. Lowercase, punctuation already removed by the time these are checked. +CORPORATE_SUFFIXES = { + 'inc', 'incorporated', 'llc', 'llp', 'ltd', 'limited', 'corp', + 'corporation', 'co', 'company', 'gmbh', 'ag', 'sa', 'plc', +} + +# Default similarity a fuzzy match must reach, 0..1. Also the value the settings +# default is seeded with; 0 disables reconciliation entirely. +DEFAULT_RECONCILE_THRESHOLD = 0.85 + + +def normalize_label(value): + """Fold a label to its comparison form: lowercase, no punctuation, no + corporate suffix, single-spaced. 'Dell Inc.' and 'DELL' both become 'dell'. + """ + text = (value or '').strip().lower() + text = re.sub(r'[^a-z0-9]+', ' ', text).strip() + if not text: + return '' + words = [w for w in text.split(' ') if w] + while len(words) > 1 and words[-1] in CORPORATE_SUFFIXES: + words.pop() + return ' '.join(words) + def table_for(name): table = db.metadata.tables.get(name) @@ -221,10 +259,17 @@ class Resolver: Rows created earlier in the SAME import are resolvable, so one file can reference another that was loaded moments ago without a commit in between. + + When a name does not match exactly and the table is reconcilable, a + normalized then fuzzy comparison against the existing rows may bind it to a + close-enough match. Every such binding is recorded so the caller can report + it for review; nothing is merged silently. """ - def __init__(self): + def __init__(self, reconcile_threshold=0.0): self._cache = {} + self.reconcile_threshold = reconcile_threshold or 0.0 + self.events = [] def resolve(self, tablename, idcolumn, value): if value is None: @@ -249,6 +294,10 @@ class Resolver: db.func.lower(db.func.trim(table.c[label])) == str(value).strip().lower()) ).first() if row is None: + found = self._reconcile(tablename, idcolumn, label, table, value) + if found is not None: + self._cache[key] = found + return found raise ValueError( "nothing in %s is named '%s' - add it to %s.csv, or import that file first" % (tablename, value, tablename)) @@ -256,6 +305,52 @@ class Resolver: self._cache[key] = found return found + def _reconcile(self, tablename, idcolumn, label, table, value): + """Bind value to the closest existing row when within threshold. + + Returns the matched id and records an event, or None when reconciliation + is off, the table is not reconcilable, or nothing is close enough. + """ + if self.reconcile_threshold <= 0 or tablename not in RECONCILABLE_TABLES: + return None + + wanted = normalize_label(value) + if not wanted: + return None + + best_id = None + best_label = None + best_score = 0.0 + for existing in db.session.execute(table.select()): + candidate = getattr(existing, label, None) + normalized = normalize_label(candidate) + if not normalized: + continue + if normalized == wanted: + score = 1.0 + else: + score = difflib.SequenceMatcher(None, wanted, normalized).ratio() + if score > best_score: + best_score = score + best_id = getattr(existing, idcolumn) + best_label = candidate + + if best_id is not None and best_score >= self.reconcile_threshold: + self.events.append({ + 'table': tablename, + 'input': str(value).strip(), + 'matched': best_label, + 'score': round(best_score, 3), + }) + return best_id + return None + + def take_events(self): + """Return and clear reconciliations recorded since the last call.""" + events = self.events + self.events = [] + return events + def remember(self, tablename, name, rowid): if name is None: return @@ -282,6 +377,7 @@ class ImportResult(object): self.updated = 0 self.problems = [] self.skipped = 0 + self.reconciliations = [] @property def ok(self): @@ -381,6 +477,10 @@ def import_csv(tablename, text, resolver=None, commit=False): continue try: values[name] = resolver.resolve(target[0], target[1], text) + for event in resolver.take_events(): + event['line'] = line + event['column'] = name + result.reconciliations.append(event) except ValueError as exc: result.problems.append(RowProblem(line, name, str(exc))) failed = True