diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 0e9e54bb..6712925c 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -1014,6 +1014,30 @@ export const settingsApi = { form.append('kind', kind) form.append('file', file) return api.post('/settings/branding-logo', form, { headers: { 'Content-Type': 'multipart/form-data' } }) + }, + downloadCollectorScript() { + // Collect-ShopDB.ps1 with this site's CSV path baked into $NetworkStoreRoot + return api.get('/settings/asset-import/script', { responseType: 'blob' }) + }, + listImportTemplates() { + // Importable CSV tables in the order they must be uploaded + return api.get('/settings/asset-import/templates') + }, + downloadImportTemplate(tablename) { + // Generated CSV template (header + example row) for one table + return api.get(`/settings/asset-import/templates/${encodeURIComponent(tablename)}`, { responseType: 'blob' }) + }, + importAssetCsvs(files, commit = false) { + // Multipart upload of one or more CSVs. commit=false is a dry-run preview + // that writes nothing; commit=true applies the whole set in one transaction. + const form = new FormData() + for (const file of files) { + form.append('files', file) + } + form.append('commit', commit ? 'true' : 'false') + return api.post('/settings/asset-import/import', form, { + headers: { 'Content-Type': 'multipart/form-data' } + }) } } diff --git a/frontend/src/router/routes/core.js b/frontend/src/router/routes/core.js index 0d98321a..b2bc3864 100644 --- a/frontend/src/router/routes/core.js +++ b/frontend/src/router/routes/core.js @@ -150,6 +150,12 @@ export default [ component: () => import('../../views/settings/BrandingSettings.vue'), meta: { requiresAuth: true, requiresAdmin: true } }, + { + path: 'settings/importsiteassets', + name: 'import-site-assets', + component: () => import('../../views/settings/ImportSiteAssets.vue'), + meta: { requiresAuth: true, requiresAdmin: true } + }, { path: 'settings/printing', name: 'printing-settings', diff --git a/frontend/src/views/settings/ImportSiteAssets.vue b/frontend/src/views/settings/ImportSiteAssets.vue new file mode 100644 index 00000000..60ed215c --- /dev/null +++ b/frontend/src/views/settings/ImportSiteAssets.vue @@ -0,0 +1,659 @@ + + + + + diff --git a/frontend/src/views/settings/SiteSettings.vue b/frontend/src/views/settings/SiteSettings.vue index 77a77221..7be8eb45 100644 --- a/frontend/src/views/settings/SiteSettings.vue +++ b/frontend/src/views/settings/SiteSettings.vue @@ -75,7 +75,8 @@ const LABELS = { employee_directory_mode: 'Employee Directory Mode', usb_directory_mode: 'USB Directory Mode', site_timezone: 'Site Timezone', - setup_complete: 'Setup Complete' + setup_complete: 'Setup Complete', + asset_import_csv_location: 'Asset Import CSV Location' } function prettyLabel(key) { return LABELS[key] || key @@ -127,7 +128,10 @@ async function load() { try { const response = await settingsApi.list() const all = response.data?.data || response.data || [] - items.value = all.filter(s => s.category === 'site') + // The Asset Collector CSV location is managed on the dedicated Import Site + // Assets page. + const importKeys = ['asset_import_csv_location'] + items.value = all.filter(s => s.category === 'site' && !importKeys.includes(s.key)) } catch (err) { console.error('Error loading site settings:', err) error.value = 'Could not load settings.' diff --git a/frontend/src/views/settings/settingsNav.js b/frontend/src/views/settings/settingsNav.js index c1fb32d9..a5b87238 100644 --- a/frontend/src/views/settings/settingsNav.js +++ b/frontend/src/views/settings/settingsNav.js @@ -4,7 +4,7 @@ // Order: site identity + the reference-data catalogs users touch daily come // first, then the platform/system groups (integrations, communication, search, // plugins, access) cluster together at the end. -import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key } from 'lucide-vue-next' +import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key, Upload } from 'lucide-vue-next' export const settingsGroups = [ { @@ -13,6 +13,7 @@ export const settingsGroups = [ { to: '/settings/site', icon: Home, title: 'Site & Facility', description: 'Site URL/FQDN, facility name, PC access domain, employee-ID pattern, printer hostname template' }, { to: '/settings/floormap', icon: MapPin, title: 'Floor Map', description: 'Facility floor-plan blueprint image and pixel dimensions (width, height) for the site map' }, { to: '/settings/branding', icon: Palette, title: 'Branding', description: 'Site, QR, and badge logos, favicon, and brand primary, accent, sidebar, and theme colors' }, + { to: '/settings/importsiteassets', icon: Upload, title: 'Import Site Assets', description: 'Seed Machines, PCs, and Applications via the Asset Collector script, existing databases, or Intune/Azure exports; set the shared CSV collection location and download collected files' }, ], }, { diff --git a/scripts/site_imports/assetcollector/Collect-ShopDB.ps1 b/scripts/site_imports/assetcollector/Collect-ShopDB.ps1 new file mode 100644 index 00000000..f66c4b8d --- /dev/null +++ b/scripts/site_imports/assetcollector/Collect-ShopDB.ps1 @@ -0,0 +1,542 @@ +#Requires -Version 5.1 +[CmdletBinding()] +param( + # Where CSVs are collected/accumulated. Overrides the in-script CONFIG block + # below. See the CONFIG block for the easy place to set this permanently. + [Parameter(Mandatory = $false)] + [string]$OutputRoot, + + # Optional network share to mirror the store into after a run. Overrides the + # in-script CONFIG block below. + [Parameter(Mandatory = $false)] + [string]$PublishRoot, + + # Business identifier for this PC (workstation number). Defaults to hostname. + [Parameter(Mandatory = $false)] + [string]$AssetNumber, + + # Value written into assets.assettypeid for the PC (resolved by name). + [Parameter(Mandatory = $false)] + [string]$AssetType = 'Computer', + + # Value written into computers.computertypeid (resolved by name). + [Parameter(Mandatory = $false)] + [string]$ComputerType, + + # Value written into machines.machinetypeid (resolved by name). + [Parameter(Mandatory = $false)] + [string]$MachineType, + + # Explicit eDNC/UDC source overrides. When omitted, standard on-asset + # locations are searched. + [Parameter(Mandatory = $false)] + [string]$RegPath, + [Parameter(Mandatory = $false)] + [string]$JsonPath, + + # Skip switches for partial runs. + [switch]$SkipMachine, + [switch]$SkipApps +) + +<# +.SYNOPSIS + Unified ShopDB site-adoption collector. Runs on any PC, collects the PC and any + attached eDNC/DNC machine, and UPSERTS the results into a shared set of + importable CSVs so a site can build its inventory PC by PC. + +.DESCRIPTION + Accumulating store under OutputRoot: + vendors.csv, models.csv, operatingsystems.csv, assets.csv, + computers.csv, machines.csv + Rows are merged field-by-field on each table's natural key (a later run fills + gaps and refreshes values without wiping data an earlier run captured). + + Phase-2 data that CSV cannot carry is written per entity under OutputRoot\sidecars: + computer_.json (interfaces, installed apps) + machine_.json (PC<->Machine link, connection detail) + + Read-only against the machine. Local reads may run elevated; publishing to a + share runs in the current user context only. + +.NOTES + PS 5.1 compatible. Headers match the ShopDB importer (see CSV-SCHEMA.md). +#> + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# ============================================================================= +# CONFIG - SET YOUR CSV DESTINATIONS HERE +# ----------------------------------------------------------------------------- +# Two-stage flow so you can collect locally (possibly elevated) and then append +# the captured rows to a CSV store that lives on a network share: +# +# 1) LocalStageRoot Fast, always-writable local folder where CSVs are built +# and merged. Collection can run elevated here. +# 2) NetworkStoreRoot The shared CSV store the whole site accumulates into and +# that you later point 'flask csv import --dir' at. Leave +# blank ('') to keep everything local. When set, the merged +# store is copied here at the end of the run. Network +# writes must happen as a STANDARD user - do NOT run the +# publish step elevated. +# +# Example network path: '\\fileserver\shopdb$\intake' +# +# Command-line -OutputRoot / -PublishRoot still override these values. +# ============================================================================= + +$LocalStageRoot = Join-Path $env:ProgramData 'ShopDB\intake' +$NetworkStoreRoot = '\\wi001dataaege.wlm.geaerospace.net\shared\TEAMSYS\.sfscripts\shopdb' + +# Resolve effective destinations (command-line params win over CONFIG above). +if (-not $OutputRoot) { $OutputRoot = $LocalStageRoot } +if (-not $PublishRoot) { $PublishRoot = $NetworkStoreRoot } + +# --- Store definitions (header order + natural key per importable table) ------- + +$script:Stores = [ordered]@{ + 'vendors.csv' = @{ key = 'vendor'; header = @('vendor') } + 'models.csv' = @{ key = 'modelnumber'; header = @('modelnumber','vendorid','modeltypeid') } + 'operatingsystems.csv' = @{ key = 'osname'; header = @('osname') } + 'assets.csv' = @{ key = 'assetnumber'; header = @('assetnumber','assettypeid','name','statusid','serialnumber','gaugelabreference','maintenancereference','locationid','businessunitid','notes') } + 'computers.csv' = @{ key = 'assetid'; header = @('assetid','hostname','computertypeid','osid','vendorid','modelnumberid','loggedinuser','lastreporteddate','lastboottime') } + 'machines.csv' = @{ key = 'assetid'; header = @('assetid','machinetypeid','vendorid','modelnumberid','controllervendorid','controllermodelid','requiresmanualconfig','islocationonly','notes') } +} + +# --- Shared helpers ----------------------------------------------------------- + +function ConvertTo-CsvField { + param([AllowNull()]$Value) + if ($null -eq $Value) { return '' } + $text = [string]$Value + if ($text -match '[",\r\n]') { return '"' + ($text -replace '"', '""') + '"' } + return $text +} + +function Get-Prop { + param([AllowNull()]$Object, [Parameter(Mandatory = $true)][string]$Name) + if ($null -eq $Object) { return $null } + $prop = $Object.PSObject.Properties[$Name] + if ($null -eq $prop) { return $null } + return $prop.Value +} + +function Get-NonEmpty { + param([AllowNull()]$Value) + if ($null -ne $Value -and -not [string]::IsNullOrWhiteSpace([string]$Value)) { + return ([string]$Value).Trim() + } + return $null +} + +function Get-FirstNonEmpty { + param([object[]]$Values) + foreach ($v in $Values) { $n = Get-NonEmpty $v; if ($n) { return $n } } + return $null +} + +function Format-ShopDbDate { + param([AllowNull()]$Value) + if ($null -eq $Value) { return $null } + try { return ([datetime]$Value).ToString('yyyy-MM-dd HH:mm:ss') } catch { return $null } +} + +function Write-Utf8NoBom { + param([string]$Path, [string]$Content) + $encoding = New-Object System.Text.UTF8Encoding($false) + [System.IO.File]::WriteAllText($Path, $Content, $encoding) +} + +# Field-level upsert into a persistent CSV store, keyed by natural key. Existing +# values are kept when the incoming cell is empty, so a partial run never blanks +# out data an earlier run captured. +function Merge-CsvStore { + param( + [Parameter(Mandatory = $true)][string]$Path, + [Parameter(Mandatory = $true)][string[]]$Header, + [Parameter(Mandatory = $true)][string]$Key, + [Parameter(Mandatory = $true)][AllowNull()][object[]]$IncomingRows + ) + + $index = [ordered]@{} + if (Test-Path -LiteralPath $Path) { + foreach ($existing in @(Import-Csv -LiteralPath $Path)) { + $keyVal = Get-NonEmpty (Get-Prop $existing $Key) + if (-not $keyVal) { continue } + $index[$keyVal.ToLowerInvariant()] = $existing + } + } + + $changed = 0 + foreach ($row in $IncomingRows) { + if ($null -eq $row) { continue } + $keyVal = Get-NonEmpty (Get-Prop $row $Key) + if (-not $keyVal) { continue } + $lc = $keyVal.ToLowerInvariant() + if ($index.Contains($lc)) { + $target = $index[$lc] + foreach ($col in $Header) { + $incoming = Get-NonEmpty (Get-Prop $row $col) + if ($incoming) { + if ($target.PSObject.Properties[$col]) { $target.$col = $incoming } + else { $target | Add-Member -NotePropertyName $col -NotePropertyValue $incoming -Force } + } + } + } else { + $fresh = [ordered]@{} + foreach ($col in $Header) { $fresh[$col] = Get-NonEmpty (Get-Prop $row $col) } + $index[$lc] = [pscustomobject]$fresh + } + $changed++ + } + + $lines = New-Object System.Collections.Generic.List[string] + $lines.Add(($Header -join ',')) + foreach ($stored in $index.Values) { + $cells = foreach ($col in $Header) { ConvertTo-CsvField (Get-Prop $stored $col) } + $lines.Add(($cells -join ',')) + } + Write-Utf8NoBom -Path $Path -Content (($lines -join "`r`n") + "`r`n") + return $changed +} + +# --- eDNC/UDC parsing (registry, .reg export, UDC settings json) -------------- + +function Resolve-ControllerVendor { + param([AllowNull()][string]$ControlType, [AllowNull()][string]$NcIF) + $ct = ('' + $ControlType).Trim().ToUpperInvariant() + $nc = ('' + $NcIF).Trim().ToUpperInvariant() + if ($nc -eq 'EFOCAS' -or $ct -eq 'EFOCAS') { return 'FANUC' } + if ($ct -eq 'SIEMENS') { return 'Siemens' } + if ($ct -eq 'OKUMA') { return 'Okuma' } + return $null +} + +function Read-RegExport { + param([Parameter(Mandatory = $true)][string]$Path) + $lines = @(Get-Content -LiteralPath $Path -Encoding Unicode -ErrorAction Stop) + $firstMeaningful = ($lines | Where-Object { $_ -match '\S' } | Select-Object -First 1) + if (-not ($firstMeaningful -match '^(REGEDIT4|Windows Registry Editor Version)')) { + $lines = @(Get-Content -LiteralPath $Path -Encoding Default -ErrorAction Stop) + } + $data = @{} + $currentKey = $null + foreach ($raw in $lines) { + $line = $raw.Trim() + if ([string]::IsNullOrWhiteSpace($line)) { continue } + if ($line -match '^(REGEDIT4|Windows Registry Editor Version)') { continue } + if ($line -match '^\s*;') { continue } + if ($line -match '^\[(?[^\]]+)\]$') { + $currentKey = $Matches['key'] + if (-not $data.ContainsKey($currentKey)) { $data[$currentKey] = @{} } + continue + } + if (-not $currentKey) { continue } + if ($line -match '^"(?[^"]+)"=(?.+)$') { + $name = $Matches['name'] + $rhs = $Matches['rhs'].Trim() + if ($rhs -match '^"(?.*)"$') { $val = $Matches['s'] } + elseif ($rhs -match '^dword:(?[0-9a-fA-F]{8})$') { $val = [Convert]::ToInt32($Matches['hex'], 16) } + else { $val = $rhs } + $data[$currentKey][$name] = $val + } + } + return $data +} + +function Get-RegExportValue { + param([hashtable]$Reg, [string]$KeySuffix, [string]$Name) + foreach ($key in $Reg.Keys) { + if ($key -match ('\\' + [regex]::Escape($KeySuffix) + '$')) { + if ($Reg[$key].ContainsKey($Name)) { return $Reg[$key][$Name] } + } + } + return $null +} + +function ConvertFrom-RegExport { + param([hashtable]$Reg) + [ordered]@{ + source = 'reg' + machinenumber = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'MachineNo') + site = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'Site') + controltype = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'Cnc') + ncif = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'NcIF') + dnctype = 'eDNC' + tcpaddress = Get-NonEmpty (Get-RegExportValue $Reg 'eFocas' 'IpAddr') + tcpport = Get-NonEmpty (Get-RegExportValue $Reg 'eFocas' 'SocketNo') + comport = Get-NonEmpty (Get-RegExportValue $Reg 'PPDCS' 'Port Id') + baudrate = Get-NonEmpty (Get-RegExportValue $Reg 'PPDCS' 'Baud') + } +} + +function Get-RegistryValue { + param([string]$Path, [string]$Name) + if (-not (Test-Path -LiteralPath $Path)) { return $null } + return Get-Prop (Get-ItemProperty -LiteralPath $Path -ErrorAction SilentlyContinue) $Name +} + +function ConvertFrom-LiveRegistry { + param([string]$Base) + if (-not (Test-Path -LiteralPath $Base)) { return $null } + $general = Join-Path $Base 'General' + $efocas = Join-Path $Base 'eFocas' + $ppdcs = Join-Path $Base 'PPDCS' + [ordered]@{ + source = 'registry' + machinenumber = Get-NonEmpty (Get-RegistryValue $general 'MachineNo') + site = Get-NonEmpty (Get-RegistryValue $general 'Site') + controltype = Get-NonEmpty (Get-RegistryValue $general 'Cnc') + ncif = Get-NonEmpty (Get-RegistryValue $general 'NcIF') + dnctype = 'eDNC' + tcpaddress = Get-NonEmpty (Get-RegistryValue $efocas 'IpAddr') + tcpport = Get-NonEmpty (Get-RegistryValue $efocas 'SocketNo') + comport = Get-NonEmpty (Get-RegistryValue $ppdcs 'Port Id') + baudrate = Get-NonEmpty (Get-RegistryValue $ppdcs 'Baud') + } +} + +function ConvertFrom-UdcJson { + param([string]$Path) + $obj = Get-Content -LiteralPath $Path -Raw -ErrorAction Stop | ConvertFrom-Json + $general = Get-Prop $obj 'GeneralSettings' + $ethernet = Get-Prop $obj 'EthernetSettings' + $serial = Get-Prop $obj 'SerialSettings' + [ordered]@{ + source = 'json' + machinenumber = Get-NonEmpty (Get-Prop $general 'MachineNumber') + site = Get-NonEmpty (Get-Prop $general 'Site') + controltype = Get-NonEmpty (Get-Prop $general 'ControlType') + ncif = $null + dnctype = Get-NonEmpty (Get-Prop $general 'DncType') + tcpaddress = Get-NonEmpty (Get-Prop $ethernet 'TCPAddress') + tcpport = Get-NonEmpty (Get-Prop $ethernet 'TCPPort') + comport = Get-NonEmpty (Get-Prop $serial 'ComPort') + baudrate = Get-NonEmpty (Get-Prop $serial 'BaudRate') + } +} + +function Find-FirstFile { + param([string[]]$Roots, [string]$Filter) + foreach ($root in $Roots) { + if (-not $root -or -not (Test-Path -LiteralPath $root)) { continue } + $hit = Get-ChildItem -LiteralPath $root -Filter $Filter -File -ErrorAction SilentlyContinue | + Sort-Object LastWriteTime -Descending | Select-Object -First 1 + if ($hit) { return $hit.FullName } + } + return $null +} + +function Resolve-DncSource { + param([string]$RegPath, [string]$JsonPath) + if ($JsonPath -and (Test-Path -LiteralPath $JsonPath)) { + Write-Host (' machine source: JSON (explicit) ' + $JsonPath) -ForegroundColor DarkGray + return ConvertFrom-UdcJson -Path $JsonPath + } + if ($RegPath -and (Test-Path -LiteralPath $RegPath)) { + Write-Host (' machine source: .reg (explicit) ' + $RegPath) -ForegroundColor DarkGray + return ConvertFrom-RegExport -Reg (Read-RegExport -Path $RegPath) + } + foreach ($base in @( + 'HKLM:\SOFTWARE\GE Aircraft Engines\DNC', + 'HKLM:\SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC')) { + $fromReg = ConvertFrom-LiveRegistry -Base $base + if ($fromReg -and $fromReg.machinenumber) { + Write-Host (' machine source: live registry ' + $base) -ForegroundColor DarkGray + return $fromReg + } + } + $localReg = Find-FirstFile -Roots @('C:\Program Files (x86)\Dnc', (Join-Path $env:SystemDrive 'GE Aerospace')) -Filter '*.reg' + if ($localReg) { + Write-Host (' machine source: local .reg ' + $localReg) -ForegroundColor DarkGray + return ConvertFrom-RegExport -Reg (Read-RegExport -Path $localReg) + } + $localJson = Find-FirstFile -Roots @((Join-Path $env:SystemDrive 'GE Aerospace'), 'C:\Program Files (x86)\UDC', 'C:\Program Files (x86)\Dnc') -Filter 'udc_settings_*.json' + if ($localJson) { + Write-Host (' machine source: local UDC json ' + $localJson) -ForegroundColor DarkGray + return ConvertFrom-UdcJson -Path $localJson + } + return $null +} + +# --- PC collection (read-only) ------------------------------------------------ + +function Get-PcFacts { + $computerSystem = Get-CimInstance Win32_ComputerSystem -ErrorAction SilentlyContinue + $operatingSys = Get-CimInstance Win32_OperatingSystem -ErrorAction SilentlyContinue + $bios = Get-CimInstance Win32_BIOS -ErrorAction SilentlyContinue + + $interfaces = @() + try { + foreach ($cfg in @(Get-NetIPConfiguration -ErrorAction SilentlyContinue)) { + $adapter = $null + try { $adapter = Get-NetAdapter -InterfaceAlias $cfg.InterfaceAlias -ErrorAction SilentlyContinue } catch {} + if (-not $adapter -or (Get-Prop $adapter 'HardwareInterface') -ne $true) { continue } + $ipv4 = @(Get-Prop $cfg 'IPv4Address') | Select-Object -First 1 + $interfaces += [ordered]@{ + interfacealias = [string]$cfg.InterfaceAlias + ipaddress = if ($ipv4) { [string](Get-Prop $ipv4 'IPAddress') } else { $null } + macaddress = [string](Get-Prop $adapter 'MacAddress') + } + } + } catch {} + + $installedApps = @() + if (-not $SkipApps) { + foreach ($keyPath in @( + 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\*', + 'HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\*')) { + foreach ($entry in @(Get-ItemProperty -Path $keyPath -ErrorAction SilentlyContinue)) { + $displayName = Get-NonEmpty (Get-Prop $entry 'DisplayName') + if (-not $displayName) { continue } + if ((Get-Prop $entry 'SystemComponent') -eq 1) { continue } + $installedApps += [ordered]@{ + appname = $displayName + installedversion = Get-NonEmpty (Get-Prop $entry 'DisplayVersion') + publisher = Get-NonEmpty (Get-Prop $entry 'Publisher') + } + } + } + $installedApps = @($installedApps | Sort-Object { $_.appname } -Unique) + } + + [ordered]@{ + hostname = Get-FirstNonEmpty @($env:COMPUTERNAME, (Get-Prop $computerSystem 'Name')) + vendor = Get-NonEmpty (Get-Prop $computerSystem 'Manufacturer') + model = Get-NonEmpty (Get-Prop $computerSystem 'Model') + osname = Get-NonEmpty (Get-Prop $operatingSys 'Caption') + serial = Get-FirstNonEmpty @((Get-Prop $bios 'SerialNumber'), (Get-Prop $computerSystem 'SerialNumber')) + lastboot = Format-ShopDbDate (Get-Prop $operatingSys 'LastBootUpTime') + loggeduser = Get-NonEmpty (Get-Prop $computerSystem 'UserName') + interfaces = $interfaces + installedapps = $installedApps + } +} + +# --- Main --------------------------------------------------------------------- + +Write-Host '' +Write-Host ' ShopDB unified collector' -ForegroundColor Cyan +Write-Host (' ' + ('-' * 56)) -ForegroundColor DarkGray + +if (-not (Test-Path -LiteralPath $OutputRoot)) { New-Item -ItemType Directory -Path $OutputRoot -Force | Out-Null } +$sidecarDir = Join-Path $OutputRoot 'sidecars' +if (-not (Test-Path -LiteralPath $sidecarDir)) { New-Item -ItemType Directory -Path $sidecarDir -Force | Out-Null } + +$collectedAt = (Get-Date).ToString('yyyy-MM-dd HH:mm:ss') + +$pc = Get-PcFacts +if (-not $AssetNumber) { $AssetNumber = $pc.hostname } + +# Row sets keyed to the shared stores. +$vendorRows = @() +$modelRows = @() +$osRows = @() +$assetRows = @() +$computerRows = @() +$machineRows = @() + +if ($pc.vendor) { $vendorRows += [pscustomobject]@{ vendor = $pc.vendor } } +if ($pc.model) { $modelRows += [pscustomobject]@{ modelnumber = $pc.model; vendorid = $pc.vendor; modeltypeid = $null } } +if ($pc.osname) { $osRows += [pscustomobject]@{ osname = $pc.osname } } + +$assetRows += [pscustomobject]@{ + assetnumber = $AssetNumber; assettypeid = $AssetType; name = $pc.hostname; statusid = $null + serialnumber = $pc.serial; gaugelabreference = $null; maintenancereference = $null + locationid = $null; businessunitid = $null + notes = 'PC collected ' + $collectedAt +} +$computerRows += [pscustomobject]@{ + assetid = $AssetNumber; hostname = $pc.hostname; computertypeid = $ComputerType; osid = $pc.osname + vendorid = $pc.vendor; modelnumberid = $pc.model; loggedinuser = $pc.loggeduser + lastreporteddate = $collectedAt; lastboottime = $pc.lastboot +} + +# Machine (best-effort). +$machine = $null +if (-not $SkipMachine) { + try { $machine = Resolve-DncSource -RegPath $RegPath -JsonPath $JsonPath } catch { $machine = $null } +} +if ($machine -and $machine.machinenumber) { + $controllerVendor = Resolve-ControllerVendor -ControlType $machine.controltype -NcIF $machine.ncif + $transport = if ($machine.tcpaddress) { 'ethernet' } elseif ($machine.comport) { 'serial' } else { 'unknown' } + $noteBits = @("DNC=$($machine.dnctype)") + if ($machine.controltype) { $noteBits += "control=$($machine.controltype)" } + $noteBits += "transport=$transport" + $machineNote = 'eDNC collected ' + $collectedAt + '; ' + ($noteBits -join '; ') + + if ($controllerVendor) { $vendorRows += [pscustomobject]@{ vendor = $controllerVendor } } + $assetRows += [pscustomobject]@{ + assetnumber = $machine.machinenumber; assettypeid = 'Machine'; name = $machine.machinenumber; statusid = $null + serialnumber = $null; gaugelabreference = $null; maintenancereference = $null + locationid = $null; businessunitid = $null; notes = $machineNote + } + $machineRows += [pscustomobject]@{ + assetid = $machine.machinenumber; machinetypeid = $MachineType; vendorid = $null; modelnumberid = $null + controllervendorid = $controllerVendor; controllermodelid = $null + requiresmanualconfig = $null; islocationonly = $null; notes = $machineNote + } + + $link = [ordered]@{ + pcassetnumber = $AssetNumber; machineassetnumber = $machine.machinenumber; relationship = 'controls' + dnctype = $machine.dnctype; controltype = $machine.controltype; controllervendor = $controllerVendor + site = $machine.site; source = $machine.source; collectedat = $collectedAt + connection = [ordered]@{ + transport = $transport; tcpaddress = $machine.tcpaddress; tcpport = $machine.tcpport + comport = $machine.comport; baudrate = $machine.baudrate + } + } + Write-Utf8NoBom -Path (Join-Path $sidecarDir ('machine_' + $machine.machinenumber + '.json')) -Content ($link | ConvertTo-Json -Depth 6) + Write-Host (' linked machine ' + $machine.machinenumber + ' (' + $controllerVendor + ', ' + $transport + ')') -ForegroundColor DarkGray +} elseif (-not $SkipMachine) { + Write-Host ' no eDNC/DNC/UDC machine config found on this PC' -ForegroundColor DarkGray +} + +# --- Upsert into the shared store -------------------------------------------- + +$rowsByStore = @{ + 'vendors.csv' = $vendorRows + 'models.csv' = $modelRows + 'operatingsystems.csv' = $osRows + 'assets.csv' = $assetRows + 'computers.csv' = $computerRows + 'machines.csv' = $machineRows +} + +Write-Host '' +Write-Host (' Store: ' + $OutputRoot) -ForegroundColor Green +foreach ($fileName in $script:Stores.Keys) { + $spec = $script:Stores[$fileName] + $rows = @($rowsByStore[$fileName]) + if ($rows.Count -eq 0) { continue } + $path = Join-Path $OutputRoot $fileName + $n = Merge-CsvStore -Path $path -Header $spec.header -Key $spec.key -IncomingRows $rows + Write-Host (' merged ' + $n + ' row(s) into ' + $fileName) -ForegroundColor DarkGray +} + +# PC phase-2 sidecar. +$pcSidecar = [ordered]@{ + assetnumber = $AssetNumber; hostname = $pc.hostname; collectedat = $collectedAt + interfaces = $pc.interfaces; installedapps = $pc.installedapps +} +Write-Utf8NoBom -Path (Join-Path $sidecarDir ('computer_' + $AssetNumber + '.json')) -Content ($pcSidecar | ConvertTo-Json -Depth 6) + +# --- Publish (standard-user network mirror, optional) ------------------------- + +if ($PublishRoot) { + if (([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole( + [Security.Principal.WindowsBuiltInRole]::Administrator)) { + Write-Host '' + Write-Host ' WARNING: publishing while elevated. Network writes should run as the standard user.' -ForegroundColor Yellow + } + if (-not (Test-Path -LiteralPath $PublishRoot)) { New-Item -ItemType Directory -Path $PublishRoot -Force | Out-Null } + Copy-Item -Path (Join-Path $OutputRoot '*') -Destination $PublishRoot -Recurse -Force + Write-Host '' + Write-Host (' Published store to: ' + $PublishRoot) -ForegroundColor Green +} + +Write-Host '' +Write-Host ' Done. Dry-run validate the accumulated store with:' -ForegroundColor Cyan +Write-Host (' flask csv import --dir "' + $OutputRoot + '"') -ForegroundColor White +Write-Host '' diff --git a/scripts/site_imports/assetcollector/overview.md b/scripts/site_imports/assetcollector/overview.md new file mode 100644 index 00000000..1587c2f3 --- /dev/null +++ b/scripts/site_imports/assetcollector/overview.md @@ -0,0 +1,91 @@ +# Collect-ShopDB.ps1 Overview + +## Overview + +- **Purpose**: A unified, read-only "site adoption" collector that runs on any + Windows PC, gathers facts about that PC plus any attached eDNC/DNC/UDC machine, + and merges them into a shared set of importable CSVs so a site can build its + ShopDB inventory one PC at a time. +- **Compatibility**: PowerShell 5.1+; CSV headers are aligned to the ShopDB + importer (`flask csv import --dir`). + +## Configuration and destinations + +- **Two-stage flow**: a fast local stage folder (`%ProgramData%\ShopDB\intake`) + where CSVs are built/merged, and an optional network store (a file share) that + the whole site accumulates into. +- Command-line params `-OutputRoot` and `-PublishRoot` override the in-script + CONFIG block. +- Other params: `-AssetNumber` (defaults to hostname), + `-AssetType`/`-ComputerType`/`-MachineType` (resolved by name), explicit + `-RegPath`/`-JsonPath` machine source overrides, and `-SkipMachine`/`-SkipApps` + switches for partial runs. + +## Data model (the stores) + +- Six importable CSVs, each with a defined header order and natural key: + `vendors.csv`, `models.csv`, `operatingsystems.csv`, `assets.csv`, + `computers.csv`, `machines.csv`. +- Column names follow ShopDB conventions (e.g., `assetid`, `modelnumberid`, + `controllervendorid`, `lastreporteddate`). + +## PC collection (`Get-PcFacts`, read-only) + +- Uses CIM/WMI (`Win32_ComputerSystem`, `Win32_OperatingSystem`, `Win32_BIOS`) + to gather hostname, vendor/manufacturer, model, OS caption, serial number, + last boot time, and logged-in user. +- Enumerates **network interfaces** (hardware adapters only: alias, IPv4, MAC). +- Enumerates **installed applications** from the registry Uninstall keys (name, + version, publisher), skipping system components - unless `-SkipApps` is set. + +## Machine (eDNC/DNC/UDC) collection + +- `Resolve-DncSource` searches in priority order: explicit JSON path -> explicit + .reg path -> live registry (`HKLM\...\GE Aircraft Engines\DNC`) -> local `.reg` + export files -> local `udc_settings_*.json` files. +- Three parsers normalize the different source formats into one shape: + `ConvertFrom-RegExport` (parses exported `.reg` files including dword decoding), + `ConvertFrom-LiveRegistry`, and `ConvertFrom-UdcJson`. +- Extracts machine number, site, control type, DNC type, and connection detail + (TCP address/port for ethernet, COM port/baud for serial). +- `Resolve-ControllerVendor` maps control type / NcIF to a controller vendor + (e.g., eFOCAS -> FANUC, Siemens, Okuma). + +## Merge / upsert logic (`Merge-CsvStore`) + +- Field-level UPSERT keyed on each table's natural key: existing rows are matched + case-insensitively, and empty incoming cells never overwrite previously captured + values - so partial or repeated runs fill gaps without wiping earlier data. +- CSV writing is UTF-8 without BOM, with proper quoting/escaping. + +## Sidecar files (phase-2 data CSV can't carry) + +- Per-PC `sidecars\computer_.json`: interfaces + installed apps. +- Per-machine `sidecars\machine_.json`: PC-to-machine link + (relationship "controls"), DNC/control details, and connection transport info. + +## Publishing and output + +- Optionally mirrors the whole local store to the network share + (`Copy-Item -Recurse -Force`), with a warning if run elevated (network writes + should run as standard user). +- Prints a summary and the follow-up dry-run command: + `flask csv import --dir ""`. + +--- + +## Relevance to an "Asset Collector" package + +This script is a solid template for one collector in a broader package. Notable +patterns worth generalizing across other collectors (existing databases, +Intune/Azure exports, etc.): + +- **Normalized intermediate schema** (the six CSV stores + JSON sidecars) that + all sources converge on, decoupling collection from import. +- **Idempotent field-level merge** so multiple heterogeneous sources can + contribute to the same asset without clobbering each other. +- **Source-priority resolution** and per-format parsers - the same shape could + wrap an Intune/Azure `Get-MgDevice` export or a SQL query. +- **Machines, PCs, Applications** are already modeled here; an Intune/Azure + source would map naturally onto `computers.csv` + the installed-apps sidecar, + while a MES/CNC database could feed `machines.csv`. diff --git a/scripts/site_imports/assetcollector/sample/assets.csv b/scripts/site_imports/assetcollector/sample/assets.csv new file mode 100644 index 00000000..a4db46bd --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/assets.csv @@ -0,0 +1,5 @@ +assetnumber,name,serialnumber,assettypeid,statusid,locationid,businessunitid,notes +SAMPLE-PC-01,Shopfloor PC 01,DELLSN0001,computer,In Use,A1,Gage Lab,Sample collected device +SAMPLE-PC-02,CMM PC 02,DELLSN0002,computer,In Use,A1,Gage Lab,Sample collected device +SAMPLE-MC-01,Zeiss CMM 01,ZEISSSN0001,machine,In Use,A1,Gage Lab,Sample collected device attached to SAMPLE-PC-02 +SAMPLE-MC-02,Haas VF-2SS 02,HAASSN0002,machine,In Use,A1,Gage Lab,Sample collected standalone CNC diff --git a/scripts/site_imports/assetcollector/sample/computers.csv b/scripts/site_imports/assetcollector/sample/computers.csv new file mode 100644 index 00000000..692119f0 --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/computers.csv @@ -0,0 +1,3 @@ +assetid,computertypeid,hostname,osid,vendorid,modelnumberid,loggedinuser +SAMPLE-PC-01,Shopfloor PC,WKS-SAMPLE01,Windows 10 Enterprise,Dell,OptiPlex 7090,AEROAD\operator1 +SAMPLE-PC-02,CMM PC,WKS-SAMPLE02,Windows 11 Enterprise,Dell,Precision 3660,AEROAD\operator2 diff --git a/scripts/site_imports/assetcollector/sample/machines.csv b/scripts/site_imports/assetcollector/sample/machines.csv new file mode 100644 index 00000000..d55e537f --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/machines.csv @@ -0,0 +1,3 @@ +assetid,machinetypeid,vendorid,modelnumberid,requiresmanualconfig,islocationonly +SAMPLE-MC-01,CMM,Zeiss,CONTURA G2,0,0 +SAMPLE-MC-02,CNC,Haas Automation,VF-2SS,0,0 diff --git a/scripts/site_imports/assetcollector/sample/models.csv b/scripts/site_imports/assetcollector/sample/models.csv new file mode 100644 index 00000000..18f29ac6 --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/models.csv @@ -0,0 +1,5 @@ +modelnumber,vendorid +OptiPlex 7090,Dell +Precision 3660,Dell +VF-2SS,Haas Automation +CONTURA G2,Zeiss diff --git a/scripts/site_imports/assetcollector/sample/operatingsystems.csv b/scripts/site_imports/assetcollector/sample/operatingsystems.csv new file mode 100644 index 00000000..f5ca264e --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/operatingsystems.csv @@ -0,0 +1,3 @@ +osname +Windows 10 Enterprise +Windows 11 Enterprise diff --git a/scripts/site_imports/assetcollector/sample/vendors.csv b/scripts/site_imports/assetcollector/sample/vendors.csv new file mode 100644 index 00000000..a6591f59 --- /dev/null +++ b/scripts/site_imports/assetcollector/sample/vendors.csv @@ -0,0 +1,4 @@ +vendor +Dell +Haas Automation +Zeiss diff --git a/shopdb/core/api/settings.py b/shopdb/core/api/settings.py index 60c919e5..75fe8e14 100644 --- a/shopdb/core/api/settings.py +++ b/shopdb/core/api/settings.py @@ -1,6 +1,7 @@ """Settings API routes.""" import os +import re from flask import Blueprint, request, current_app from flask_jwt_extended import jwt_required, get_jwt_identity @@ -240,6 +241,218 @@ def serve_branding_logo(filename): return send_upload(_branding_dir(), filename) +def _asset_import_dir(): + """Configured Asset Collector CSV location, or '' when unset.""" + setting = Setting.query.filter_by(key='asset_import_csv_location').first() + return (setting.value or '').strip() if setting else '' + + +# The collector script that gets served (with the site's CSV path baked in) from +# the Import Site Assets page. Copied into the image under scripts/ (see +# Dockerfile), so it is always available next to the running API. +_COLLECTOR_SCRIPT = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), + 'scripts', 'site_imports', 'assetcollector', 'Collect-ShopDB.ps1') + + +def _embed_store_path(script_text, path): + """Return the collector script with $NetworkStoreRoot set to path. + + The script ships with a site-specific default on that line; rewriting it + means an operator downloads a script already pointed at this facility's CSV + store and does not have to edit PowerShell by hand. A blank path leaves the + shipped default untouched. + """ + if not path: + return script_text + literal = path.replace("'", "''") + replacement = "$NetworkStoreRoot = '%s'" % literal + # A function replacement is used deliberately: a UNC path is full of + # backslashes, and a string replacement runs them through re's escape + # processing (so \s, \w, a trailing \1 all raise or corrupt). A callable + # returns the text verbatim. + return re.sub( + r"(?m)^\$NetworkStoreRoot\s*=.*$", + lambda _match: replacement, + script_text, + count=1) + + +@settings_bp.route('/asset-import/script', methods=['GET']) +@jwt_required() +@require_role('admin') +def download_collector_script(): + """Download Collect-ShopDB.ps1 with this site's CSV path embedded.""" + try: + with open(_COLLECTOR_SCRIPT, 'r', encoding='utf-8') as handle: + script_text = handle.read() + except OSError: + return error_response(ErrorCodes.NOT_FOUND, + 'Collector script is not available on the server', + http_code=404) + + script_text = _embed_store_path(script_text, _asset_import_dir()) + response = current_app.response_class(script_text, mimetype='text/plain') + response.headers['Content-Disposition'] = 'attachment; filename=Collect-ShopDB.ps1' + return response + + +@settings_bp.route('/asset-import/templates', methods=['GET']) +@jwt_required() +@require_role('admin') +def list_import_templates(): + """List the importable CSV tables in the order they must be uploaded.""" + from shopdb.core.services.csvimport import IMPORTABLE + + tables = [{'table': name, 'filename': '%s.csv' % name} + for name in IMPORTABLE] + return success_response({'tables': tables}) + + +@settings_bp.route('/asset-import/templates/', methods=['GET']) +@jwt_required() +@require_role('admin') +def download_import_template(tablename): + """Download a generated CSV template for a single importable table.""" + from shopdb.core.services.csvimport import ( + IMPORTABLE, generate_template, ImportError_) + + if tablename not in IMPORTABLE: + return error_response(ErrorCodes.NOT_FOUND, + 'No such importable table: %s' % tablename, + http_code=404) + try: + template_text = generate_template(tablename) + except ImportError_ as exc: + return error_response(ErrorCodes.VALIDATION_ERROR, str(exc)) + + response = current_app.response_class(template_text, mimetype='text/csv') + response.headers['Content-Disposition'] = ( + 'attachment; filename=%s.csv' % tablename) + return response + + +@settings_bp.route('/asset-import/import', methods=['POST']) +@jwt_required() +@require_role('admin') +def import_asset_csvs(): + """Validate uploaded CSVs and, when commit=true, load them. + + Mirrors the 'flask csv import' CLI: every file runs through import_csv inside + ONE transaction with a shared resolver, so a computers.csv can resolve the + assets.csv uploaded alongside it. A dry run (the default) rolls the whole + transaction back and reports what WOULD change; commit=true persists it. The + whole run is all-or-nothing - if any file has a row problem, nothing is + written even on a commit. + """ + from sqlalchemy.exc import SQLAlchemyError + from shopdb.core.services.csvimport import ( + ImportError_, Resolver, dependency_order, import_csv, table_from_filename) + + uploads = request.files.getlist('files') + if not uploads: + return error_response(ErrorCodes.VALIDATION_ERROR, 'No CSV files were uploaded') + + commit = str(request.form.get('commit', '')).strip().lower() in ( + '1', 'true', 'yes', 'on') + + # Map each upload to a table by filename. A later file with the same table + # name wins, matching the folder-import behaviour (one file per table). + texts = {} + filenames = {} + for upload in uploads: + name = upload.filename or '' + if not name.lower().endswith('.csv'): + continue + table = table_from_filename(name) + try: + texts[table] = upload.read().decode('utf-8-sig') + except (UnicodeDecodeError, OSError): + return error_response(ErrorCodes.VALIDATION_ERROR, + 'Could not read %s as UTF-8 text' % name) + filenames[table] = name + + if not texts: + return error_response(ErrorCodes.VALIDATION_ERROR, + 'None of the uploaded files were CSV files') + + ordered = dependency_order(list(texts)) + skipped = sorted(set(texts) - set(ordered)) + + reconcile_setting = Setting.query.filter_by( + key='asset_import_reconcile_threshold').first() + try: + reconcile_threshold = float(reconcile_setting.value) if reconcile_setting and reconcile_setting.value else 0.0 + except (TypeError, ValueError): + reconcile_threshold = 0.0 + + resolver = Resolver(reconcile_threshold=reconcile_threshold) + files_report = [] + problem_count = 0 + reconciled_count = 0 + request_error = None + + try: + for table in ordered: + try: + result = import_csv(table, texts[table], resolver=resolver, commit=False) + except ImportError_ as exc: + request_error = '%s: %s' % (filenames[table], str(exc)) + break + except SQLAlchemyError as exc: + db.session.rollback() + request_error = '%s: database error: %s' % ( + filenames[table], str(exc).split(chr(10))[0]) + break + + problems = [ + {'line': p.line, 'column': p.column, 'message': p.message} + for p in result.problems + ] + problem_count += len(problems) + reconciled_count += len(result.reconciliations) + files_report.append({ + 'table': table, + 'filename': filenames[table], + 'created': result.created, + 'updated': result.updated, + 'problems': problems, + 'reconciliations': result.reconciliations, + }) + + applied = False + if request_error is not None: + db.session.rollback() + elif problem_count > 0: + # A row problem anywhere fails the whole run, so nothing is written + # even when commit was asked for. + db.session.rollback() + elif commit: + db.session.commit() + applied = True + else: + db.session.rollback() + except Exception: + db.session.rollback() + raise + + total_created = sum(f['created'] for f in files_report) + total_updated = sum(f['updated'] for f in files_report) + + return success_response({ + 'committed': applied, + 'requested_commit': commit, + 'request_error': request_error, + 'files': files_report, + 'skipped': skipped, + 'problem_count': problem_count, + 'reconciled_count': reconciled_count, + 'reconcile_threshold': reconcile_threshold, + 'total_created': total_created, + 'total_updated': total_updated, + }) + + @settings_bp.route('', methods=['GET']) @jwt_required(optional=True) def list_settings(): @@ -556,6 +769,20 @@ def build_default_settings(): 'category': 'site', 'description': 'Email domain appended to a contact SSO to build email/Teams links (sso@domain). Blank disables contact action buttons.' }, + { + 'key': 'asset_import_csv_location', + 'value': '', + 'valuetype': 'string', + 'category': 'site', + 'description': r'Shared network location where the Asset Collector script deposits its CSVs and where ShopDB reads them for import (e.g. \\server.fqdn\share\folder). This path is baked into the downloadable collector script. Blank = not configured.' + }, + { + 'key': 'asset_import_reconcile_threshold', + 'value': '0.85', + 'valuetype': 'string', + 'category': 'site', + 'description': 'Similarity (0 to 1) a fuzzy lookup match must reach for the asset importer to reconcile a near-miss name (e.g. "Dell Inc." to an existing "Dell") onto an existing vendor, model, type, status, business unit, or location. Reconciled matches are applied but reported for review. 0 disables reconciliation. Operating systems and asset rows are never reconciled.' + }, ] # Site branding. Each instance (ADR-004) can replace the shipped GE defaults diff --git a/shopdb/core/services/csvimport.py b/shopdb/core/services/csvimport.py index d114965f..462e8b57 100644 --- a/shopdb/core/services/csvimport.py +++ b/shopdb/core/services/csvimport.py @@ -34,6 +34,7 @@ not at all. from __future__ import annotations import csv +import difflib import io import re from collections import OrderedDict @@ -86,6 +87,43 @@ PASSWORD_COLUMNS = {'passwordhash', 'password'} TRUE_VALUES = {'1', 'true', 'yes', 'y', 't'} FALSE_VALUES = {'0', 'false', 'no', 'n', 'f', ''} +# Reconciliation: when a name does not match a lookup row exactly, the resolver +# may still bind it to an existing row that is close enough. Only lookup tables +# take part - the small reference tables an operator picks values from. Rows that +# ARE the data being imported (assets, computers, machines) and the fast-moving +# operatingsystems list are excluded, so a typo there is a new row, not a silent +# merge onto something unrelated. +RECONCILABLE_TABLES = { + 'assetstatuses', 'assettypes', 'locationtypes', 'modeltypes', + 'computertypes', 'machinetypes', 'businessunits', 'locations', + 'vendors', 'models', +} + +# Corporate suffixes stripped before comparison so 'Dell Inc.' reconciles to +# 'Dell'. Lowercase, punctuation already removed by the time these are checked. +CORPORATE_SUFFIXES = { + 'inc', 'incorporated', 'llc', 'llp', 'ltd', 'limited', 'corp', + 'corporation', 'co', 'company', 'gmbh', 'ag', 'sa', 'plc', +} + +# Default similarity a fuzzy match must reach, 0..1. Also the value the settings +# default is seeded with; 0 disables reconciliation entirely. +DEFAULT_RECONCILE_THRESHOLD = 0.85 + + +def normalize_label(value): + """Fold a label to its comparison form: lowercase, no punctuation, no + corporate suffix, single-spaced. 'Dell Inc.' and 'DELL' both become 'dell'. + """ + text = (value or '').strip().lower() + text = re.sub(r'[^a-z0-9]+', ' ', text).strip() + if not text: + return '' + words = [w for w in text.split(' ') if w] + while len(words) > 1 and words[-1] in CORPORATE_SUFFIXES: + words.pop() + return ' '.join(words) + def table_for(name): table = db.metadata.tables.get(name) @@ -221,10 +259,17 @@ class Resolver: Rows created earlier in the SAME import are resolvable, so one file can reference another that was loaded moments ago without a commit in between. + + When a name does not match exactly and the table is reconcilable, a + normalized then fuzzy comparison against the existing rows may bind it to a + close-enough match. Every such binding is recorded so the caller can report + it for review; nothing is merged silently. """ - def __init__(self): + def __init__(self, reconcile_threshold=0.0): self._cache = {} + self.reconcile_threshold = reconcile_threshold or 0.0 + self.events = [] def resolve(self, tablename, idcolumn, value): if value is None: @@ -249,6 +294,10 @@ class Resolver: db.func.lower(db.func.trim(table.c[label])) == str(value).strip().lower()) ).first() if row is None: + found = self._reconcile(tablename, idcolumn, label, table, value) + if found is not None: + self._cache[key] = found + return found raise ValueError( "nothing in %s is named '%s' - add it to %s.csv, or import that file first" % (tablename, value, tablename)) @@ -256,6 +305,52 @@ class Resolver: self._cache[key] = found return found + def _reconcile(self, tablename, idcolumn, label, table, value): + """Bind value to the closest existing row when within threshold. + + Returns the matched id and records an event, or None when reconciliation + is off, the table is not reconcilable, or nothing is close enough. + """ + if self.reconcile_threshold <= 0 or tablename not in RECONCILABLE_TABLES: + return None + + wanted = normalize_label(value) + if not wanted: + return None + + best_id = None + best_label = None + best_score = 0.0 + for existing in db.session.execute(table.select()): + candidate = getattr(existing, label, None) + normalized = normalize_label(candidate) + if not normalized: + continue + if normalized == wanted: + score = 1.0 + else: + score = difflib.SequenceMatcher(None, wanted, normalized).ratio() + if score > best_score: + best_score = score + best_id = getattr(existing, idcolumn) + best_label = candidate + + if best_id is not None and best_score >= self.reconcile_threshold: + self.events.append({ + 'table': tablename, + 'input': str(value).strip(), + 'matched': best_label, + 'score': round(best_score, 3), + }) + return best_id + return None + + def take_events(self): + """Return and clear reconciliations recorded since the last call.""" + events = self.events + self.events = [] + return events + def remember(self, tablename, name, rowid): if name is None: return @@ -282,6 +377,7 @@ class ImportResult(object): self.updated = 0 self.problems = [] self.skipped = 0 + self.reconciliations = [] @property def ok(self): @@ -381,6 +477,10 @@ def import_csv(tablename, text, resolver=None, commit=False): continue try: values[name] = resolver.resolve(target[0], target[1], text) + for event in resolver.take_events(): + event['line'] = line + event['column'] = name + result.reconciliations.append(event) except ValueError as exc: result.problems.append(RowProblem(line, name, str(exc))) failed = True