Add site asset CSV import with fuzzy reconciliation

Sites seed inventory from the Asset Collector script or spreadsheet exports, but lookup names drift between sources (Dell vs Dell Inc.), which would otherwise create duplicate vendors/models/types on every import. The importer now normalizes and fuzzy-matches near-miss references onto existing lookup rows within an operator-set similarity threshold, auto-applying the match while reporting it for review so nothing merges silently. Operating systems and asset rows are never reconciled. Adds the dedicated Import Site Assets page with script/template download and a multi-file dry-run/commit workflow.
This commit is contained in:
benjamin.baker
2026-08-24 14:29:02 -04:00
parent 1b5b28acff
commit 4a1b97362b
15 changed files with 1681 additions and 4 deletions

View File

@@ -1014,6 +1014,30 @@ export const settingsApi = {
form.append('kind', kind)
form.append('file', file)
return api.post('/settings/branding-logo', form, { headers: { 'Content-Type': 'multipart/form-data' } })
},
downloadCollectorScript() {
// Collect-ShopDB.ps1 with this site's CSV path baked into $NetworkStoreRoot
return api.get('/settings/asset-import/script', { responseType: 'blob' })
},
listImportTemplates() {
// Importable CSV tables in the order they must be uploaded
return api.get('/settings/asset-import/templates')
},
downloadImportTemplate(tablename) {
// Generated CSV template (header + example row) for one table
return api.get(`/settings/asset-import/templates/${encodeURIComponent(tablename)}`, { responseType: 'blob' })
},
importAssetCsvs(files, commit = false) {
// Multipart upload of one or more CSVs. commit=false is a dry-run preview
// that writes nothing; commit=true applies the whole set in one transaction.
const form = new FormData()
for (const file of files) {
form.append('files', file)
}
form.append('commit', commit ? 'true' : 'false')
return api.post('/settings/asset-import/import', form, {
headers: { 'Content-Type': 'multipart/form-data' }
})
}
}

View File

@@ -150,6 +150,12 @@ export default [
component: () => import('../../views/settings/BrandingSettings.vue'),
meta: { requiresAuth: true, requiresAdmin: true }
},
{
path: 'settings/importsiteassets',
name: 'import-site-assets',
component: () => import('../../views/settings/ImportSiteAssets.vue'),
meta: { requiresAuth: true, requiresAdmin: true }
},
{
path: 'settings/printing',
name: 'printing-settings',

View File

@@ -0,0 +1,659 @@
<template>
<div>
<div class="page-header">
<h1>Import Site Assets</h1>
<router-link to="/settings" class="btn btn-secondary">Back to Settings</router-link>
</div>
<div class="card">
<p class="lead">
There are several ways to seed this facility's inventory - Machines, PCs,
Applications, and their properties - into ShopDB. You can combine them:
each source writes to the same set of importable CSVs, and a later run
fills gaps without wiping data an earlier run captured.
</p>
<div class="methods">
<div class="method is-primary">
<div class="method-head">
<span class="badge">Recommended</span>
<h3>Asset Collector PowerShell script</h3>
</div>
<p>
Deploy <code>Collect-ShopDB.ps1</code> to your existing endpoints in a
way of your choosing (logon script, scheduled task, remote execution,
or a manual run). It runs read-only, extracts each PC and any attached
eDNC/DNC machine, and extrapolates vendor, model, OS, network, and
installed-application details. Every run UPSERTs rows into the shared
CSV store below, so a site builds its inventory one PC at a time.
</p>
</div>
<div class="method">
<div class="method-head">
<h3>Existing databases</h3>
</div>
<p>
Export from an existing MES, CMDB, or asset spreadsheet into the same
CSV schema and drop the files in the collection location.
</p>
</div>
<div class="method">
<div class="method-head">
<h3>Intune / Azure exports</h3>
</div>
<p>
Map a device or application export (for example from Microsoft Graph)
onto the computers and applications CSVs, then place them alongside the
script output.
</p>
</div>
</div>
</div>
<div class="card">
<h3 class="section-title">Asset Collector Script</h3>
<p class="muted">
Download <code>Collect-ShopDB.ps1</code> and deploy it to your endpoints
(logon script, scheduled task, remote execution, or a manual run). The CSV
Collections Location you save below is baked into the script's
<code>$NetworkStoreRoot</code> variable, so every endpoint publishes into
the same store without editing PowerShell by hand.
</p>
<div v-if="loading" class="muted">Loading...</div>
<template v-else>
<label class="field">
<span>CSV Collections Location</span>
<input
v-model="location"
type="text"
placeholder="\\server.fqdn\share\folder"
/>
<small class="muted">
UNC path or folder where the script deposits its CSVs (e.g.
\\server.fqdn\share\folder). Saved here and embedded into the script
download.
</small>
</label>
<div class="actions">
<button class="btn btn-primary" :disabled="saving" @click="save">
{{ saving ? 'Saving...' : 'Save Location' }}
</button>
<button class="btn btn-secondary" :disabled="downloadingScript" @click="downloadScript">
{{ downloadingScript ? 'Preparing...' : 'Download Script' }}
</button>
<span v-if="saved" class="muted">Saved.</span>
<span v-if="error" class="error">{{ error }}</span>
</div>
</template>
</div>
<div class="card">
<h3 class="section-title">CSV Import Templates</h3>
<p class="muted">
These are early-draft import templates generated from the live ShopDB
schema. Each file has a header row and a commented example. Fill one out
(or let the collector script populate them) and import them in the order
below - a table never references something that has not been loaded yet.
Foreign keys accept the referenced row's name or its numeric id, and
re-importing a file updates matching rows instead of duplicating them.
</p>
<p v-if="templatesLoading" class="muted">Loading...</p>
<ol v-else class="templates">
<li v-for="t in templates" :key="t.table">
<div class="template-row">
<code>{{ t.filename }}</code>
<button class="btn btn-secondary btn-sm" @click="downloadTemplate(t)">
Download Template
</button>
</div>
</li>
</ol>
<p v-if="templatesError" class="error">{{ templatesError }}</p>
</div>
<div class="card">
<h3 class="section-title">Import CSVs</h3>
<p class="muted">
Select one or more filled-in CSVs. They are ordered by dependency
automatically, so you can pick the whole set at once. Preview first: it
writes nothing and reports exactly what each file would create or update,
plus any rows that need fixing. Nothing is imported until you press
Commit, and the whole set is applied together - if any row has a problem,
nothing is written.
</p>
<div class="import-controls">
<input
ref="fileInput"
type="file"
accept=".csv"
multiple
@change="onFilesPicked"
/>
<div class="actions">
<button
class="btn btn-secondary"
:disabled="!selectedFiles.length || previewing || committing"
@click="preview"
>
{{ previewing ? 'Checking...' : 'Preview (dry run)' }}
</button>
<button
class="btn btn-primary"
:disabled="!canCommit"
@click="commit"
>
{{ committing ? 'Importing...' : 'Commit Import' }}
</button>
<button
v-if="selectedFiles.length"
class="btn btn-secondary btn-sm"
:disabled="previewing || committing"
@click="clearSelection"
>
Clear
</button>
</div>
</div>
<p v-if="selectedFiles.length" class="muted">
{{ selectedFiles.length }} file(s) selected:
{{ selectedFiles.map(f => f.name).join(', ') }}
</p>
<p v-if="importError" class="error">{{ importError }}</p>
<div v-if="report" class="report">
<p v-if="report.request_error" class="error">
{{ report.request_error }}
</p>
<p
v-else
:class="['report-banner', report.committed ? 'ok' : (report.problem_count ? 'warn' : 'info')]"
>
<template v-if="report.committed">
Imported: {{ report.total_created }} created,
{{ report.total_updated }} updated.
</template>
<template v-else-if="report.problem_count">
{{ report.problem_count }} problem(s) found - nothing was written.
Fix the rows below and preview again.
</template>
<template v-else>
Looks good: {{ report.total_created }} would be created,
{{ report.total_updated }} updated. Press Commit Import to apply.
</template>
</p>
<p v-if="report.reconciled_count" class="report-banner reconcile">
{{ report.reconciled_count }} name(s) reconciled to an existing record
within the similarity threshold ({{ report.reconcile_threshold }}).
Review the matches below before committing.
</p>
<table v-if="report.files && report.files.length" class="report-table">
<thead>
<tr>
<th>File</th>
<th>Table</th>
<th>Created</th>
<th>Updated</th>
<th>Reconciled</th>
<th>Problems</th>
</tr>
</thead>
<tbody>
<tr v-for="f in report.files" :key="f.table">
<td><code>{{ f.filename }}</code></td>
<td>{{ f.table }}</td>
<td>{{ f.created }}</td>
<td>{{ f.updated }}</td>
<td :class="{ 'has-reconcile': (f.reconciliations || []).length }">
{{ (f.reconciliations || []).length }}
</td>
<td :class="{ 'has-problems': f.problems.length }">
{{ f.problems.length }}
</td>
</tr>
</tbody>
</table>
<div v-if="allReconciliations.length" class="reconciliations">
<h4>Reconciled matches</h4>
<p class="muted">
These names did not match exactly but were close enough to bind to an
existing record. Confirm each is correct; if any is wrong, add the
exact name to the matching CSV (or lower/raise the reconciliation
threshold in Site settings) and preview again.
</p>
<ul>
<li v-for="(r, i) in allReconciliations" :key="i">
<code>{{ r.filename }}</code>
line {{ r.line }}<template v-if="r.column">, column
<code>{{ r.column }}</code></template>:
"{{ r.input }}" matched <strong>{{ r.matched }}</strong>
({{ r.table }}, {{ Math.round(r.score * 100) }}%)
</li>
</ul>
</div>
<div v-if="allProblems.length" class="problems">
<h4>Problems to fix</h4>
<ul>
<li v-for="(p, i) in allProblems" :key="i">
<code>{{ p.filename }}</code>
line {{ p.line }}<template v-if="p.column">, column
<code>{{ p.column }}</code></template>: {{ p.message }}
</li>
</ul>
</div>
<p v-if="report.skipped && report.skipped.length" class="muted warn">
Skipped (not importable): {{ report.skipped.join(', ') }}
</p>
</div>
</div>
<div class="card">
<h3 class="section-title">PC to Machine Relationships</h3>
<p class="muted">
A PC that drives a machine (a CMM PC, a DNC/eDNC host) is linked to that
machine so the two show up connected across ShopDB. That link is built
automatically by the Asset Collector, not by this CSV import: the script
reads the four-digit workstation number from the PC it runs on and
reports it, and ShopDB matches that number to a machine's
<code>assetnumber</code> to create the connection.
</p>
<p class="muted">
What this means for import order: load your machines first (so their
<code>assetnumber</code> values exist), then let the collector run on
each PC. A machine number the collector reports that does not match an
existing machine is flagged rather than guessed, so nothing links to the
wrong asset. Building this relationship straight from a CSV column is
planned as a follow-up.
</p>
</div>
</div>
</template>
<script setup>
import { ref, computed, onMounted } from 'vue'
import { settingsApi } from '@/api'
import { apiError } from '../../utils/apiError'
const LOCATION_KEY = 'asset_import_csv_location'
const location = ref('')
const loading = ref(true)
const saving = ref(false)
const saved = ref(false)
const error = ref('')
const downloadingScript = ref(false)
const templates = ref([])
const templatesLoading = ref(false)
const templatesError = ref('')
const fileInput = ref(null)
const selectedFiles = ref([])
const previewing = ref(false)
const committing = ref(false)
const importError = ref('')
const report = ref(null)
// Commit is allowed only after a clean preview of the CURRENT selection: a
// report with no problems and no request-level error that has not already been
// committed.
const canCommit = computed(() => {
if (!report.value || committing.value || previewing.value) return false
if (report.value.committed) return false
if (report.value.request_error) return false
return report.value.problem_count === 0
})
const allProblems = computed(() => {
if (!report.value?.files) return []
const out = []
for (const file of report.value.files) {
for (const problem of file.problems) {
out.push({ filename: file.filename, ...problem })
}
}
return out
})
const allReconciliations = computed(() => {
if (!report.value?.files) return []
const out = []
for (const file of report.value.files) {
for (const event of (file.reconciliations || [])) {
out.push({ filename: file.filename, ...event })
}
}
return out
})
async function loadLocation() {
loading.value = true
try {
const response = await settingsApi.get(LOCATION_KEY).catch(nullOn404)
location.value = readValue(response) || ''
} catch (err) {
error.value = apiError(err, 'Could not load the CSV location.')
} finally {
loading.value = false
}
}
function nullOn404(err) {
if (err.response?.status === 404) return null
throw err
}
function readValue(response) {
if (!response) return ''
const setting = response.data?.data || response.data
return setting?.value ?? ''
}
async function save() {
saving.value = true
saved.value = false
error.value = ''
try {
await settingsApi.update(LOCATION_KEY, location.value)
saved.value = true
} catch (err) {
error.value = apiError(err, 'Save failed.')
} finally {
saving.value = false
}
}
function saveBlob(data, filename) {
const url = window.URL.createObjectURL(new Blob([data]))
const link = document.createElement('a')
link.href = url
link.download = filename
document.body.appendChild(link)
link.click()
link.remove()
window.URL.revokeObjectURL(url)
}
async function downloadScript() {
downloadingScript.value = true
error.value = ''
try {
const response = await settingsApi.downloadCollectorScript()
saveBlob(response.data, 'Collect-ShopDB.ps1')
} catch (err) {
error.value = apiError(err, 'Script download failed.')
} finally {
downloadingScript.value = false
}
}
async function loadTemplates() {
templatesLoading.value = true
templatesError.value = ''
try {
const response = await settingsApi.listImportTemplates()
const data = response.data?.data || response.data || {}
templates.value = data.tables || []
} catch (err) {
templatesError.value = apiError(err, 'Could not list import templates.')
} finally {
templatesLoading.value = false
}
}
async function downloadTemplate(t) {
templatesError.value = ''
try {
const response = await settingsApi.downloadImportTemplate(t.table)
saveBlob(response.data, t.filename)
} catch (err) {
templatesError.value = apiError(err, 'Template download failed.')
}
}
function onFilesPicked(event) {
selectedFiles.value = Array.from(event.target.files || [])
// A new selection invalidates any previous preview so a stale clean report
// cannot be committed against different files.
report.value = null
importError.value = ''
}
function clearSelection() {
selectedFiles.value = []
report.value = null
importError.value = ''
if (fileInput.value) fileInput.value.value = ''
}
async function runImport(commit) {
importError.value = ''
try {
const response = await settingsApi.importAssetCsvs(selectedFiles.value, commit)
report.value = response.data?.data || response.data || null
} catch (err) {
importError.value = apiError(err, commit ? 'Import failed.' : 'Preview failed.')
report.value = null
}
}
async function preview() {
previewing.value = true
try {
await runImport(false)
} finally {
previewing.value = false
}
}
async function commit() {
committing.value = true
try {
await runImport(true)
} finally {
committing.value = false
}
}
onMounted(async () => {
await loadLocation()
await loadTemplates()
})
</script>
<style scoped>
.lead {
margin-top: 0;
color: var(--text);
}
.methods {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(240px, 1fr));
gap: 16px;
margin-top: 12px;
}
.method {
border: 1px solid var(--border);
border-radius: 8px;
padding: 14px 16px;
background: var(--bg);
}
.method.is-primary {
border-color: var(--primary);
}
.method-head {
display: flex;
align-items: center;
gap: 8px;
}
.method-head h3 {
margin: 0;
font-size: 1rem;
}
.method p {
margin: 8px 0 0;
color: var(--text-light);
font-size: 0.9rem;
}
.badge {
background: var(--primary);
color: #fff;
border-radius: 10px;
padding: 2px 8px;
font-size: 0.7rem;
font-weight: 700;
text-transform: uppercase;
letter-spacing: 0.04em;
}
.section-title {
margin: 0 0 8px;
}
.field {
display: flex;
flex-direction: column;
gap: 4px;
max-width: 640px;
margin-top: 12px;
}
.field > span {
font-weight: 600;
}
.field input[type='text'] {
padding: 8px 10px;
border: 1px solid var(--border);
border-radius: 6px;
background: var(--bg);
color: var(--text);
font-family: var(--font-mono, monospace);
}
.actions {
display: flex;
align-items: center;
gap: 12px;
margin-top: 12px;
}
.templates {
margin: 8px 0 0;
padding-left: 20px;
}
.templates li {
margin: 6px 0;
}
.template-row {
display: flex;
align-items: center;
gap: 12px;
}
.template-row code {
min-width: 200px;
}
.btn-sm {
padding: 4px 10px;
font-size: 0.8rem;
}
.muted {
color: var(--text-light);
font-size: 0.9rem;
}
.muted.warn {
color: var(--warning, #c47f00);
}
.error {
color: var(--danger);
margin-top: 8px;
}
.import-controls {
display: flex;
flex-direction: column;
gap: 10px;
margin-top: 10px;
}
.report {
margin-top: 14px;
}
.report-banner {
padding: 8px 12px;
border-radius: 6px;
font-weight: 600;
}
.report-banner.ok {
background: var(--success-bg, rgba(40, 160, 80, 0.15));
color: var(--success, #2e7d32);
}
.report-banner.info {
background: var(--info-bg, rgba(60, 120, 200, 0.15));
color: var(--text);
}
.report-banner.warn {
background: var(--warning-bg, rgba(196, 127, 0, 0.15));
color: var(--warning, #c47f00);
}
.report-table {
width: 100%;
border-collapse: collapse;
margin-top: 12px;
}
.report-table th,
.report-table td {
text-align: left;
padding: 6px 10px;
border-bottom: 1px solid var(--border);
font-size: 0.9rem;
}
.report-table td.has-problems {
color: var(--danger);
font-weight: 700;
}
.report-table td.has-reconcile {
color: var(--warning, #c47f00);
font-weight: 700;
}
.report-banner.reconcile {
background: var(--warning-bg, rgba(196, 127, 0, 0.15));
color: var(--warning, #c47f00);
}
.reconciliations {
margin-top: 12px;
}
.reconciliations h4 {
margin: 0 0 6px;
color: var(--warning, #c47f00);
}
.reconciliations ul {
margin: 6px 0 0;
padding-left: 20px;
}
.reconciliations li {
margin: 4px 0;
font-size: 0.9rem;
}
.problems {
margin-top: 12px;
}
.problems h4 {
margin: 0 0 6px;
color: var(--danger);
}
.problems ul {
margin: 0;
padding-left: 20px;
}
.problems li {
margin: 4px 0;
font-size: 0.9rem;
}
</style>

View File

@@ -75,7 +75,8 @@ const LABELS = {
employee_directory_mode: 'Employee Directory Mode',
usb_directory_mode: 'USB Directory Mode',
site_timezone: 'Site Timezone',
setup_complete: 'Setup Complete'
setup_complete: 'Setup Complete',
asset_import_csv_location: 'Asset Import CSV Location'
}
function prettyLabel(key) {
return LABELS[key] || key
@@ -127,7 +128,10 @@ async function load() {
try {
const response = await settingsApi.list()
const all = response.data?.data || response.data || []
items.value = all.filter(s => s.category === 'site')
// The Asset Collector CSV location is managed on the dedicated Import Site
// Assets page.
const importKeys = ['asset_import_csv_location']
items.value = all.filter(s => s.category === 'site' && !importKeys.includes(s.key))
} catch (err) {
console.error('Error loading site settings:', err)
error.value = 'Could not load settings.'

View File

@@ -4,7 +4,7 @@
// Order: site identity + the reference-data catalogs users touch daily come
// first, then the platform/system groups (integrations, communication, search,
// plugins, access) cluster together at the end.
import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key } from 'lucide-vue-next'
import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key, Upload } from 'lucide-vue-next'
export const settingsGroups = [
{
@@ -13,6 +13,7 @@ export const settingsGroups = [
{ to: '/settings/site', icon: Home, title: 'Site & Facility', description: 'Site URL/FQDN, facility name, PC access domain, employee-ID pattern, printer hostname template' },
{ to: '/settings/floormap', icon: MapPin, title: 'Floor Map', description: 'Facility floor-plan blueprint image and pixel dimensions (width, height) for the site map' },
{ to: '/settings/branding', icon: Palette, title: 'Branding', description: 'Site, QR, and badge logos, favicon, and brand primary, accent, sidebar, and theme colors' },
{ to: '/settings/importsiteassets', icon: Upload, title: 'Import Site Assets', description: 'Seed Machines, PCs, and Applications via the Asset Collector script, existing databases, or Intune/Azure exports; set the shared CSV collection location and download collected files' },
],
},
{

View File

@@ -0,0 +1,542 @@
#Requires -Version 5.1
[CmdletBinding()]
param(
# Where CSVs are collected/accumulated. Overrides the in-script CONFIG block
# below. See the CONFIG block for the easy place to set this permanently.
[Parameter(Mandatory = $false)]
[string]$OutputRoot,
# Optional network share to mirror the store into after a run. Overrides the
# in-script CONFIG block below.
[Parameter(Mandatory = $false)]
[string]$PublishRoot,
# Business identifier for this PC (workstation number). Defaults to hostname.
[Parameter(Mandatory = $false)]
[string]$AssetNumber,
# Value written into assets.assettypeid for the PC (resolved by name).
[Parameter(Mandatory = $false)]
[string]$AssetType = 'Computer',
# Value written into computers.computertypeid (resolved by name).
[Parameter(Mandatory = $false)]
[string]$ComputerType,
# Value written into machines.machinetypeid (resolved by name).
[Parameter(Mandatory = $false)]
[string]$MachineType,
# Explicit eDNC/UDC source overrides. When omitted, standard on-asset
# locations are searched.
[Parameter(Mandatory = $false)]
[string]$RegPath,
[Parameter(Mandatory = $false)]
[string]$JsonPath,
# Skip switches for partial runs.
[switch]$SkipMachine,
[switch]$SkipApps
)
<#
.SYNOPSIS
Unified ShopDB site-adoption collector. Runs on any PC, collects the PC and any
attached eDNC/DNC machine, and UPSERTS the results into a shared set of
importable CSVs so a site can build its inventory PC by PC.
.DESCRIPTION
Accumulating store under OutputRoot:
vendors.csv, models.csv, operatingsystems.csv, assets.csv,
computers.csv, machines.csv
Rows are merged field-by-field on each table's natural key (a later run fills
gaps and refreshes values without wiping data an earlier run captured).
Phase-2 data that CSV cannot carry is written per entity under OutputRoot\sidecars:
computer_<assetnumber>.json (interfaces, installed apps)
machine_<machinenumber>.json (PC<->Machine link, connection detail)
Read-only against the machine. Local reads may run elevated; publishing to a
share runs in the current user context only.
.NOTES
PS 5.1 compatible. Headers match the ShopDB importer (see CSV-SCHEMA.md).
#>
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
# =============================================================================
# CONFIG - SET YOUR CSV DESTINATIONS HERE
# -----------------------------------------------------------------------------
# Two-stage flow so you can collect locally (possibly elevated) and then append
# the captured rows to a CSV store that lives on a network share:
#
# 1) LocalStageRoot Fast, always-writable local folder where CSVs are built
# and merged. Collection can run elevated here.
# 2) NetworkStoreRoot The shared CSV store the whole site accumulates into and
# that you later point 'flask csv import --dir' at. Leave
# blank ('') to keep everything local. When set, the merged
# store is copied here at the end of the run. Network
# writes must happen as a STANDARD user - do NOT run the
# publish step elevated.
#
# Example network path: '\\fileserver\shopdb$\intake'
#
# Command-line -OutputRoot / -PublishRoot still override these values.
# =============================================================================
$LocalStageRoot = Join-Path $env:ProgramData 'ShopDB\intake'
$NetworkStoreRoot = '\\wi001dataaege.wlm.geaerospace.net\shared\TEAMSYS\.sfscripts\shopdb'
# Resolve effective destinations (command-line params win over CONFIG above).
if (-not $OutputRoot) { $OutputRoot = $LocalStageRoot }
if (-not $PublishRoot) { $PublishRoot = $NetworkStoreRoot }
# --- Store definitions (header order + natural key per importable table) -------
$script:Stores = [ordered]@{
'vendors.csv' = @{ key = 'vendor'; header = @('vendor') }
'models.csv' = @{ key = 'modelnumber'; header = @('modelnumber','vendorid','modeltypeid') }
'operatingsystems.csv' = @{ key = 'osname'; header = @('osname') }
'assets.csv' = @{ key = 'assetnumber'; header = @('assetnumber','assettypeid','name','statusid','serialnumber','gaugelabreference','maintenancereference','locationid','businessunitid','notes') }
'computers.csv' = @{ key = 'assetid'; header = @('assetid','hostname','computertypeid','osid','vendorid','modelnumberid','loggedinuser','lastreporteddate','lastboottime') }
'machines.csv' = @{ key = 'assetid'; header = @('assetid','machinetypeid','vendorid','modelnumberid','controllervendorid','controllermodelid','requiresmanualconfig','islocationonly','notes') }
}
# --- Shared helpers -----------------------------------------------------------
function ConvertTo-CsvField {
param([AllowNull()]$Value)
if ($null -eq $Value) { return '' }
$text = [string]$Value
if ($text -match '[",\r\n]') { return '"' + ($text -replace '"', '""') + '"' }
return $text
}
function Get-Prop {
param([AllowNull()]$Object, [Parameter(Mandatory = $true)][string]$Name)
if ($null -eq $Object) { return $null }
$prop = $Object.PSObject.Properties[$Name]
if ($null -eq $prop) { return $null }
return $prop.Value
}
function Get-NonEmpty {
param([AllowNull()]$Value)
if ($null -ne $Value -and -not [string]::IsNullOrWhiteSpace([string]$Value)) {
return ([string]$Value).Trim()
}
return $null
}
function Get-FirstNonEmpty {
param([object[]]$Values)
foreach ($v in $Values) { $n = Get-NonEmpty $v; if ($n) { return $n } }
return $null
}
function Format-ShopDbDate {
param([AllowNull()]$Value)
if ($null -eq $Value) { return $null }
try { return ([datetime]$Value).ToString('yyyy-MM-dd HH:mm:ss') } catch { return $null }
}
function Write-Utf8NoBom {
param([string]$Path, [string]$Content)
$encoding = New-Object System.Text.UTF8Encoding($false)
[System.IO.File]::WriteAllText($Path, $Content, $encoding)
}
# Field-level upsert into a persistent CSV store, keyed by natural key. Existing
# values are kept when the incoming cell is empty, so a partial run never blanks
# out data an earlier run captured.
function Merge-CsvStore {
param(
[Parameter(Mandatory = $true)][string]$Path,
[Parameter(Mandatory = $true)][string[]]$Header,
[Parameter(Mandatory = $true)][string]$Key,
[Parameter(Mandatory = $true)][AllowNull()][object[]]$IncomingRows
)
$index = [ordered]@{}
if (Test-Path -LiteralPath $Path) {
foreach ($existing in @(Import-Csv -LiteralPath $Path)) {
$keyVal = Get-NonEmpty (Get-Prop $existing $Key)
if (-not $keyVal) { continue }
$index[$keyVal.ToLowerInvariant()] = $existing
}
}
$changed = 0
foreach ($row in $IncomingRows) {
if ($null -eq $row) { continue }
$keyVal = Get-NonEmpty (Get-Prop $row $Key)
if (-not $keyVal) { continue }
$lc = $keyVal.ToLowerInvariant()
if ($index.Contains($lc)) {
$target = $index[$lc]
foreach ($col in $Header) {
$incoming = Get-NonEmpty (Get-Prop $row $col)
if ($incoming) {
if ($target.PSObject.Properties[$col]) { $target.$col = $incoming }
else { $target | Add-Member -NotePropertyName $col -NotePropertyValue $incoming -Force }
}
}
} else {
$fresh = [ordered]@{}
foreach ($col in $Header) { $fresh[$col] = Get-NonEmpty (Get-Prop $row $col) }
$index[$lc] = [pscustomobject]$fresh
}
$changed++
}
$lines = New-Object System.Collections.Generic.List[string]
$lines.Add(($Header -join ','))
foreach ($stored in $index.Values) {
$cells = foreach ($col in $Header) { ConvertTo-CsvField (Get-Prop $stored $col) }
$lines.Add(($cells -join ','))
}
Write-Utf8NoBom -Path $Path -Content (($lines -join "`r`n") + "`r`n")
return $changed
}
# --- eDNC/UDC parsing (registry, .reg export, UDC settings json) --------------
function Resolve-ControllerVendor {
param([AllowNull()][string]$ControlType, [AllowNull()][string]$NcIF)
$ct = ('' + $ControlType).Trim().ToUpperInvariant()
$nc = ('' + $NcIF).Trim().ToUpperInvariant()
if ($nc -eq 'EFOCAS' -or $ct -eq 'EFOCAS') { return 'FANUC' }
if ($ct -eq 'SIEMENS') { return 'Siemens' }
if ($ct -eq 'OKUMA') { return 'Okuma' }
return $null
}
function Read-RegExport {
param([Parameter(Mandatory = $true)][string]$Path)
$lines = @(Get-Content -LiteralPath $Path -Encoding Unicode -ErrorAction Stop)
$firstMeaningful = ($lines | Where-Object { $_ -match '\S' } | Select-Object -First 1)
if (-not ($firstMeaningful -match '^(REGEDIT4|Windows Registry Editor Version)')) {
$lines = @(Get-Content -LiteralPath $Path -Encoding Default -ErrorAction Stop)
}
$data = @{}
$currentKey = $null
foreach ($raw in $lines) {
$line = $raw.Trim()
if ([string]::IsNullOrWhiteSpace($line)) { continue }
if ($line -match '^(REGEDIT4|Windows Registry Editor Version)') { continue }
if ($line -match '^\s*;') { continue }
if ($line -match '^\[(?<key>[^\]]+)\]$') {
$currentKey = $Matches['key']
if (-not $data.ContainsKey($currentKey)) { $data[$currentKey] = @{} }
continue
}
if (-not $currentKey) { continue }
if ($line -match '^"(?<name>[^"]+)"=(?<rhs>.+)$') {
$name = $Matches['name']
$rhs = $Matches['rhs'].Trim()
if ($rhs -match '^"(?<s>.*)"$') { $val = $Matches['s'] }
elseif ($rhs -match '^dword:(?<hex>[0-9a-fA-F]{8})$') { $val = [Convert]::ToInt32($Matches['hex'], 16) }
else { $val = $rhs }
$data[$currentKey][$name] = $val
}
}
return $data
}
function Get-RegExportValue {
param([hashtable]$Reg, [string]$KeySuffix, [string]$Name)
foreach ($key in $Reg.Keys) {
if ($key -match ('\\' + [regex]::Escape($KeySuffix) + '$')) {
if ($Reg[$key].ContainsKey($Name)) { return $Reg[$key][$Name] }
}
}
return $null
}
function ConvertFrom-RegExport {
param([hashtable]$Reg)
[ordered]@{
source = 'reg'
machinenumber = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'MachineNo')
site = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'Site')
controltype = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'Cnc')
ncif = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'NcIF')
dnctype = 'eDNC'
tcpaddress = Get-NonEmpty (Get-RegExportValue $Reg 'eFocas' 'IpAddr')
tcpport = Get-NonEmpty (Get-RegExportValue $Reg 'eFocas' 'SocketNo')
comport = Get-NonEmpty (Get-RegExportValue $Reg 'PPDCS' 'Port Id')
baudrate = Get-NonEmpty (Get-RegExportValue $Reg 'PPDCS' 'Baud')
}
}
function Get-RegistryValue {
param([string]$Path, [string]$Name)
if (-not (Test-Path -LiteralPath $Path)) { return $null }
return Get-Prop (Get-ItemProperty -LiteralPath $Path -ErrorAction SilentlyContinue) $Name
}
function ConvertFrom-LiveRegistry {
param([string]$Base)
if (-not (Test-Path -LiteralPath $Base)) { return $null }
$general = Join-Path $Base 'General'
$efocas = Join-Path $Base 'eFocas'
$ppdcs = Join-Path $Base 'PPDCS'
[ordered]@{
source = 'registry'
machinenumber = Get-NonEmpty (Get-RegistryValue $general 'MachineNo')
site = Get-NonEmpty (Get-RegistryValue $general 'Site')
controltype = Get-NonEmpty (Get-RegistryValue $general 'Cnc')
ncif = Get-NonEmpty (Get-RegistryValue $general 'NcIF')
dnctype = 'eDNC'
tcpaddress = Get-NonEmpty (Get-RegistryValue $efocas 'IpAddr')
tcpport = Get-NonEmpty (Get-RegistryValue $efocas 'SocketNo')
comport = Get-NonEmpty (Get-RegistryValue $ppdcs 'Port Id')
baudrate = Get-NonEmpty (Get-RegistryValue $ppdcs 'Baud')
}
}
function ConvertFrom-UdcJson {
param([string]$Path)
$obj = Get-Content -LiteralPath $Path -Raw -ErrorAction Stop | ConvertFrom-Json
$general = Get-Prop $obj 'GeneralSettings'
$ethernet = Get-Prop $obj 'EthernetSettings'
$serial = Get-Prop $obj 'SerialSettings'
[ordered]@{
source = 'json'
machinenumber = Get-NonEmpty (Get-Prop $general 'MachineNumber')
site = Get-NonEmpty (Get-Prop $general 'Site')
controltype = Get-NonEmpty (Get-Prop $general 'ControlType')
ncif = $null
dnctype = Get-NonEmpty (Get-Prop $general 'DncType')
tcpaddress = Get-NonEmpty (Get-Prop $ethernet 'TCPAddress')
tcpport = Get-NonEmpty (Get-Prop $ethernet 'TCPPort')
comport = Get-NonEmpty (Get-Prop $serial 'ComPort')
baudrate = Get-NonEmpty (Get-Prop $serial 'BaudRate')
}
}
function Find-FirstFile {
param([string[]]$Roots, [string]$Filter)
foreach ($root in $Roots) {
if (-not $root -or -not (Test-Path -LiteralPath $root)) { continue }
$hit = Get-ChildItem -LiteralPath $root -Filter $Filter -File -ErrorAction SilentlyContinue |
Sort-Object LastWriteTime -Descending | Select-Object -First 1
if ($hit) { return $hit.FullName }
}
return $null
}
function Resolve-DncSource {
param([string]$RegPath, [string]$JsonPath)
if ($JsonPath -and (Test-Path -LiteralPath $JsonPath)) {
Write-Host (' machine source: JSON (explicit) ' + $JsonPath) -ForegroundColor DarkGray
return ConvertFrom-UdcJson -Path $JsonPath
}
if ($RegPath -and (Test-Path -LiteralPath $RegPath)) {
Write-Host (' machine source: .reg (explicit) ' + $RegPath) -ForegroundColor DarkGray
return ConvertFrom-RegExport -Reg (Read-RegExport -Path $RegPath)
}
foreach ($base in @(
'HKLM:\SOFTWARE\GE Aircraft Engines\DNC',
'HKLM:\SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC')) {
$fromReg = ConvertFrom-LiveRegistry -Base $base
if ($fromReg -and $fromReg.machinenumber) {
Write-Host (' machine source: live registry ' + $base) -ForegroundColor DarkGray
return $fromReg
}
}
$localReg = Find-FirstFile -Roots @('C:\Program Files (x86)\Dnc', (Join-Path $env:SystemDrive 'GE Aerospace')) -Filter '*.reg'
if ($localReg) {
Write-Host (' machine source: local .reg ' + $localReg) -ForegroundColor DarkGray
return ConvertFrom-RegExport -Reg (Read-RegExport -Path $localReg)
}
$localJson = Find-FirstFile -Roots @((Join-Path $env:SystemDrive 'GE Aerospace'), 'C:\Program Files (x86)\UDC', 'C:\Program Files (x86)\Dnc') -Filter 'udc_settings_*.json'
if ($localJson) {
Write-Host (' machine source: local UDC json ' + $localJson) -ForegroundColor DarkGray
return ConvertFrom-UdcJson -Path $localJson
}
return $null
}
# --- PC collection (read-only) ------------------------------------------------
function Get-PcFacts {
$computerSystem = Get-CimInstance Win32_ComputerSystem -ErrorAction SilentlyContinue
$operatingSys = Get-CimInstance Win32_OperatingSystem -ErrorAction SilentlyContinue
$bios = Get-CimInstance Win32_BIOS -ErrorAction SilentlyContinue
$interfaces = @()
try {
foreach ($cfg in @(Get-NetIPConfiguration -ErrorAction SilentlyContinue)) {
$adapter = $null
try { $adapter = Get-NetAdapter -InterfaceAlias $cfg.InterfaceAlias -ErrorAction SilentlyContinue } catch {}
if (-not $adapter -or (Get-Prop $adapter 'HardwareInterface') -ne $true) { continue }
$ipv4 = @(Get-Prop $cfg 'IPv4Address') | Select-Object -First 1
$interfaces += [ordered]@{
interfacealias = [string]$cfg.InterfaceAlias
ipaddress = if ($ipv4) { [string](Get-Prop $ipv4 'IPAddress') } else { $null }
macaddress = [string](Get-Prop $adapter 'MacAddress')
}
}
} catch {}
$installedApps = @()
if (-not $SkipApps) {
foreach ($keyPath in @(
'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\*',
'HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\*')) {
foreach ($entry in @(Get-ItemProperty -Path $keyPath -ErrorAction SilentlyContinue)) {
$displayName = Get-NonEmpty (Get-Prop $entry 'DisplayName')
if (-not $displayName) { continue }
if ((Get-Prop $entry 'SystemComponent') -eq 1) { continue }
$installedApps += [ordered]@{
appname = $displayName
installedversion = Get-NonEmpty (Get-Prop $entry 'DisplayVersion')
publisher = Get-NonEmpty (Get-Prop $entry 'Publisher')
}
}
}
$installedApps = @($installedApps | Sort-Object { $_.appname } -Unique)
}
[ordered]@{
hostname = Get-FirstNonEmpty @($env:COMPUTERNAME, (Get-Prop $computerSystem 'Name'))
vendor = Get-NonEmpty (Get-Prop $computerSystem 'Manufacturer')
model = Get-NonEmpty (Get-Prop $computerSystem 'Model')
osname = Get-NonEmpty (Get-Prop $operatingSys 'Caption')
serial = Get-FirstNonEmpty @((Get-Prop $bios 'SerialNumber'), (Get-Prop $computerSystem 'SerialNumber'))
lastboot = Format-ShopDbDate (Get-Prop $operatingSys 'LastBootUpTime')
loggeduser = Get-NonEmpty (Get-Prop $computerSystem 'UserName')
interfaces = $interfaces
installedapps = $installedApps
}
}
# --- Main ---------------------------------------------------------------------
Write-Host ''
Write-Host ' ShopDB unified collector' -ForegroundColor Cyan
Write-Host (' ' + ('-' * 56)) -ForegroundColor DarkGray
if (-not (Test-Path -LiteralPath $OutputRoot)) { New-Item -ItemType Directory -Path $OutputRoot -Force | Out-Null }
$sidecarDir = Join-Path $OutputRoot 'sidecars'
if (-not (Test-Path -LiteralPath $sidecarDir)) { New-Item -ItemType Directory -Path $sidecarDir -Force | Out-Null }
$collectedAt = (Get-Date).ToString('yyyy-MM-dd HH:mm:ss')
$pc = Get-PcFacts
if (-not $AssetNumber) { $AssetNumber = $pc.hostname }
# Row sets keyed to the shared stores.
$vendorRows = @()
$modelRows = @()
$osRows = @()
$assetRows = @()
$computerRows = @()
$machineRows = @()
if ($pc.vendor) { $vendorRows += [pscustomobject]@{ vendor = $pc.vendor } }
if ($pc.model) { $modelRows += [pscustomobject]@{ modelnumber = $pc.model; vendorid = $pc.vendor; modeltypeid = $null } }
if ($pc.osname) { $osRows += [pscustomobject]@{ osname = $pc.osname } }
$assetRows += [pscustomobject]@{
assetnumber = $AssetNumber; assettypeid = $AssetType; name = $pc.hostname; statusid = $null
serialnumber = $pc.serial; gaugelabreference = $null; maintenancereference = $null
locationid = $null; businessunitid = $null
notes = 'PC collected ' + $collectedAt
}
$computerRows += [pscustomobject]@{
assetid = $AssetNumber; hostname = $pc.hostname; computertypeid = $ComputerType; osid = $pc.osname
vendorid = $pc.vendor; modelnumberid = $pc.model; loggedinuser = $pc.loggeduser
lastreporteddate = $collectedAt; lastboottime = $pc.lastboot
}
# Machine (best-effort).
$machine = $null
if (-not $SkipMachine) {
try { $machine = Resolve-DncSource -RegPath $RegPath -JsonPath $JsonPath } catch { $machine = $null }
}
if ($machine -and $machine.machinenumber) {
$controllerVendor = Resolve-ControllerVendor -ControlType $machine.controltype -NcIF $machine.ncif
$transport = if ($machine.tcpaddress) { 'ethernet' } elseif ($machine.comport) { 'serial' } else { 'unknown' }
$noteBits = @("DNC=$($machine.dnctype)")
if ($machine.controltype) { $noteBits += "control=$($machine.controltype)" }
$noteBits += "transport=$transport"
$machineNote = 'eDNC collected ' + $collectedAt + '; ' + ($noteBits -join '; ')
if ($controllerVendor) { $vendorRows += [pscustomobject]@{ vendor = $controllerVendor } }
$assetRows += [pscustomobject]@{
assetnumber = $machine.machinenumber; assettypeid = 'Machine'; name = $machine.machinenumber; statusid = $null
serialnumber = $null; gaugelabreference = $null; maintenancereference = $null
locationid = $null; businessunitid = $null; notes = $machineNote
}
$machineRows += [pscustomobject]@{
assetid = $machine.machinenumber; machinetypeid = $MachineType; vendorid = $null; modelnumberid = $null
controllervendorid = $controllerVendor; controllermodelid = $null
requiresmanualconfig = $null; islocationonly = $null; notes = $machineNote
}
$link = [ordered]@{
pcassetnumber = $AssetNumber; machineassetnumber = $machine.machinenumber; relationship = 'controls'
dnctype = $machine.dnctype; controltype = $machine.controltype; controllervendor = $controllerVendor
site = $machine.site; source = $machine.source; collectedat = $collectedAt
connection = [ordered]@{
transport = $transport; tcpaddress = $machine.tcpaddress; tcpport = $machine.tcpport
comport = $machine.comport; baudrate = $machine.baudrate
}
}
Write-Utf8NoBom -Path (Join-Path $sidecarDir ('machine_' + $machine.machinenumber + '.json')) -Content ($link | ConvertTo-Json -Depth 6)
Write-Host (' linked machine ' + $machine.machinenumber + ' (' + $controllerVendor + ', ' + $transport + ')') -ForegroundColor DarkGray
} elseif (-not $SkipMachine) {
Write-Host ' no eDNC/DNC/UDC machine config found on this PC' -ForegroundColor DarkGray
}
# --- Upsert into the shared store --------------------------------------------
$rowsByStore = @{
'vendors.csv' = $vendorRows
'models.csv' = $modelRows
'operatingsystems.csv' = $osRows
'assets.csv' = $assetRows
'computers.csv' = $computerRows
'machines.csv' = $machineRows
}
Write-Host ''
Write-Host (' Store: ' + $OutputRoot) -ForegroundColor Green
foreach ($fileName in $script:Stores.Keys) {
$spec = $script:Stores[$fileName]
$rows = @($rowsByStore[$fileName])
if ($rows.Count -eq 0) { continue }
$path = Join-Path $OutputRoot $fileName
$n = Merge-CsvStore -Path $path -Header $spec.header -Key $spec.key -IncomingRows $rows
Write-Host (' merged ' + $n + ' row(s) into ' + $fileName) -ForegroundColor DarkGray
}
# PC phase-2 sidecar.
$pcSidecar = [ordered]@{
assetnumber = $AssetNumber; hostname = $pc.hostname; collectedat = $collectedAt
interfaces = $pc.interfaces; installedapps = $pc.installedapps
}
Write-Utf8NoBom -Path (Join-Path $sidecarDir ('computer_' + $AssetNumber + '.json')) -Content ($pcSidecar | ConvertTo-Json -Depth 6)
# --- Publish (standard-user network mirror, optional) -------------------------
if ($PublishRoot) {
if (([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole(
[Security.Principal.WindowsBuiltInRole]::Administrator)) {
Write-Host ''
Write-Host ' WARNING: publishing while elevated. Network writes should run as the standard user.' -ForegroundColor Yellow
}
if (-not (Test-Path -LiteralPath $PublishRoot)) { New-Item -ItemType Directory -Path $PublishRoot -Force | Out-Null }
Copy-Item -Path (Join-Path $OutputRoot '*') -Destination $PublishRoot -Recurse -Force
Write-Host ''
Write-Host (' Published store to: ' + $PublishRoot) -ForegroundColor Green
}
Write-Host ''
Write-Host ' Done. Dry-run validate the accumulated store with:' -ForegroundColor Cyan
Write-Host (' flask csv import --dir "' + $OutputRoot + '"') -ForegroundColor White
Write-Host ''

View File

@@ -0,0 +1,91 @@
# Collect-ShopDB.ps1 Overview
## Overview
- **Purpose**: A unified, read-only "site adoption" collector that runs on any
Windows PC, gathers facts about that PC plus any attached eDNC/DNC/UDC machine,
and merges them into a shared set of importable CSVs so a site can build its
ShopDB inventory one PC at a time.
- **Compatibility**: PowerShell 5.1+; CSV headers are aligned to the ShopDB
importer (`flask csv import --dir`).
## Configuration and destinations
- **Two-stage flow**: a fast local stage folder (`%ProgramData%\ShopDB\intake`)
where CSVs are built/merged, and an optional network store (a file share) that
the whole site accumulates into.
- Command-line params `-OutputRoot` and `-PublishRoot` override the in-script
CONFIG block.
- Other params: `-AssetNumber` (defaults to hostname),
`-AssetType`/`-ComputerType`/`-MachineType` (resolved by name), explicit
`-RegPath`/`-JsonPath` machine source overrides, and `-SkipMachine`/`-SkipApps`
switches for partial runs.
## Data model (the stores)
- Six importable CSVs, each with a defined header order and natural key:
`vendors.csv`, `models.csv`, `operatingsystems.csv`, `assets.csv`,
`computers.csv`, `machines.csv`.
- Column names follow ShopDB conventions (e.g., `assetid`, `modelnumberid`,
`controllervendorid`, `lastreporteddate`).
## PC collection (`Get-PcFacts`, read-only)
- Uses CIM/WMI (`Win32_ComputerSystem`, `Win32_OperatingSystem`, `Win32_BIOS`)
to gather hostname, vendor/manufacturer, model, OS caption, serial number,
last boot time, and logged-in user.
- Enumerates **network interfaces** (hardware adapters only: alias, IPv4, MAC).
- Enumerates **installed applications** from the registry Uninstall keys (name,
version, publisher), skipping system components - unless `-SkipApps` is set.
## Machine (eDNC/DNC/UDC) collection
- `Resolve-DncSource` searches in priority order: explicit JSON path -> explicit
.reg path -> live registry (`HKLM\...\GE Aircraft Engines\DNC`) -> local `.reg`
export files -> local `udc_settings_*.json` files.
- Three parsers normalize the different source formats into one shape:
`ConvertFrom-RegExport` (parses exported `.reg` files including dword decoding),
`ConvertFrom-LiveRegistry`, and `ConvertFrom-UdcJson`.
- Extracts machine number, site, control type, DNC type, and connection detail
(TCP address/port for ethernet, COM port/baud for serial).
- `Resolve-ControllerVendor` maps control type / NcIF to a controller vendor
(e.g., eFOCAS -> FANUC, Siemens, Okuma).
## Merge / upsert logic (`Merge-CsvStore`)
- Field-level UPSERT keyed on each table's natural key: existing rows are matched
case-insensitively, and empty incoming cells never overwrite previously captured
values - so partial or repeated runs fill gaps without wiping earlier data.
- CSV writing is UTF-8 without BOM, with proper quoting/escaping.
## Sidecar files (phase-2 data CSV can't carry)
- Per-PC `sidecars\computer_<assetnumber>.json`: interfaces + installed apps.
- Per-machine `sidecars\machine_<machinenumber>.json`: PC-to-machine link
(relationship "controls"), DNC/control details, and connection transport info.
## Publishing and output
- Optionally mirrors the whole local store to the network share
(`Copy-Item -Recurse -Force`), with a warning if run elevated (network writes
should run as standard user).
- Prints a summary and the follow-up dry-run command:
`flask csv import --dir "<OutputRoot>"`.
---
## Relevance to an "Asset Collector" package
This script is a solid template for one collector in a broader package. Notable
patterns worth generalizing across other collectors (existing databases,
Intune/Azure exports, etc.):
- **Normalized intermediate schema** (the six CSV stores + JSON sidecars) that
all sources converge on, decoupling collection from import.
- **Idempotent field-level merge** so multiple heterogeneous sources can
contribute to the same asset without clobbering each other.
- **Source-priority resolution** and per-format parsers - the same shape could
wrap an Intune/Azure `Get-MgDevice` export or a SQL query.
- **Machines, PCs, Applications** are already modeled here; an Intune/Azure
source would map naturally onto `computers.csv` + the installed-apps sidecar,
while a MES/CNC database could feed `machines.csv`.

View File

@@ -0,0 +1,5 @@
assetnumber,name,serialnumber,assettypeid,statusid,locationid,businessunitid,notes
SAMPLE-PC-01,Shopfloor PC 01,DELLSN0001,computer,In Use,A1,Gage Lab,Sample collected device
SAMPLE-PC-02,CMM PC 02,DELLSN0002,computer,In Use,A1,Gage Lab,Sample collected device
SAMPLE-MC-01,Zeiss CMM 01,ZEISSSN0001,machine,In Use,A1,Gage Lab,Sample collected device attached to SAMPLE-PC-02
SAMPLE-MC-02,Haas VF-2SS 02,HAASSN0002,machine,In Use,A1,Gage Lab,Sample collected standalone CNC
1 assetnumber name serialnumber assettypeid statusid locationid businessunitid notes
2 SAMPLE-PC-01 Shopfloor PC 01 DELLSN0001 computer In Use A1 Gage Lab Sample collected device
3 SAMPLE-PC-02 CMM PC 02 DELLSN0002 computer In Use A1 Gage Lab Sample collected device
4 SAMPLE-MC-01 Zeiss CMM 01 ZEISSSN0001 machine In Use A1 Gage Lab Sample collected device attached to SAMPLE-PC-02
5 SAMPLE-MC-02 Haas VF-2SS 02 HAASSN0002 machine In Use A1 Gage Lab Sample collected standalone CNC

View File

@@ -0,0 +1,3 @@
assetid,computertypeid,hostname,osid,vendorid,modelnumberid,loggedinuser
SAMPLE-PC-01,Shopfloor PC,WKS-SAMPLE01,Windows 10 Enterprise,Dell,OptiPlex 7090,AEROAD\operator1
SAMPLE-PC-02,CMM PC,WKS-SAMPLE02,Windows 11 Enterprise,Dell,Precision 3660,AEROAD\operator2
1 assetid computertypeid hostname osid vendorid modelnumberid loggedinuser
2 SAMPLE-PC-01 Shopfloor PC WKS-SAMPLE01 Windows 10 Enterprise Dell OptiPlex 7090 AEROAD\operator1
3 SAMPLE-PC-02 CMM PC WKS-SAMPLE02 Windows 11 Enterprise Dell Precision 3660 AEROAD\operator2

View File

@@ -0,0 +1,3 @@
assetid,machinetypeid,vendorid,modelnumberid,requiresmanualconfig,islocationonly
SAMPLE-MC-01,CMM,Zeiss,CONTURA G2,0,0
SAMPLE-MC-02,CNC,Haas Automation,VF-2SS,0,0
1 assetid machinetypeid vendorid modelnumberid requiresmanualconfig islocationonly
2 SAMPLE-MC-01 CMM Zeiss CONTURA G2 0 0
3 SAMPLE-MC-02 CNC Haas Automation VF-2SS 0 0

View File

@@ -0,0 +1,5 @@
modelnumber,vendorid
OptiPlex 7090,Dell
Precision 3660,Dell
VF-2SS,Haas Automation
CONTURA G2,Zeiss
1 modelnumber vendorid
2 OptiPlex 7090 Dell
3 Precision 3660 Dell
4 VF-2SS Haas Automation
5 CONTURA G2 Zeiss

View File

@@ -0,0 +1,3 @@
osname
Windows 10 Enterprise
Windows 11 Enterprise
1 osname
2 Windows 10 Enterprise
3 Windows 11 Enterprise

View File

@@ -0,0 +1,4 @@
vendor
Dell
Haas Automation
Zeiss
1 vendor
2 Dell
3 Haas Automation
4 Zeiss

View File

@@ -1,6 +1,7 @@
"""Settings API routes."""
import os
import re
from flask import Blueprint, request, current_app
from flask_jwt_extended import jwt_required, get_jwt_identity
@@ -240,6 +241,218 @@ def serve_branding_logo(filename):
return send_upload(_branding_dir(), filename)
def _asset_import_dir():
"""Configured Asset Collector CSV location, or '' when unset."""
setting = Setting.query.filter_by(key='asset_import_csv_location').first()
return (setting.value or '').strip() if setting else ''
# The collector script that gets served (with the site's CSV path baked in) from
# the Import Site Assets page. Copied into the image under scripts/ (see
# Dockerfile), so it is always available next to the running API.
_COLLECTOR_SCRIPT = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))),
'scripts', 'site_imports', 'assetcollector', 'Collect-ShopDB.ps1')
def _embed_store_path(script_text, path):
"""Return the collector script with $NetworkStoreRoot set to path.
The script ships with a site-specific default on that line; rewriting it
means an operator downloads a script already pointed at this facility's CSV
store and does not have to edit PowerShell by hand. A blank path leaves the
shipped default untouched.
"""
if not path:
return script_text
literal = path.replace("'", "''")
replacement = "$NetworkStoreRoot = '%s'" % literal
# A function replacement is used deliberately: a UNC path is full of
# backslashes, and a string replacement runs them through re's escape
# processing (so \s, \w, a trailing \1 all raise or corrupt). A callable
# returns the text verbatim.
return re.sub(
r"(?m)^\$NetworkStoreRoot\s*=.*$",
lambda _match: replacement,
script_text,
count=1)
@settings_bp.route('/asset-import/script', methods=['GET'])
@jwt_required()
@require_role('admin')
def download_collector_script():
"""Download Collect-ShopDB.ps1 with this site's CSV path embedded."""
try:
with open(_COLLECTOR_SCRIPT, 'r', encoding='utf-8') as handle:
script_text = handle.read()
except OSError:
return error_response(ErrorCodes.NOT_FOUND,
'Collector script is not available on the server',
http_code=404)
script_text = _embed_store_path(script_text, _asset_import_dir())
response = current_app.response_class(script_text, mimetype='text/plain')
response.headers['Content-Disposition'] = 'attachment; filename=Collect-ShopDB.ps1'
return response
@settings_bp.route('/asset-import/templates', methods=['GET'])
@jwt_required()
@require_role('admin')
def list_import_templates():
"""List the importable CSV tables in the order they must be uploaded."""
from shopdb.core.services.csvimport import IMPORTABLE
tables = [{'table': name, 'filename': '%s.csv' % name}
for name in IMPORTABLE]
return success_response({'tables': tables})
@settings_bp.route('/asset-import/templates/<tablename>', methods=['GET'])
@jwt_required()
@require_role('admin')
def download_import_template(tablename):
"""Download a generated CSV template for a single importable table."""
from shopdb.core.services.csvimport import (
IMPORTABLE, generate_template, ImportError_)
if tablename not in IMPORTABLE:
return error_response(ErrorCodes.NOT_FOUND,
'No such importable table: %s' % tablename,
http_code=404)
try:
template_text = generate_template(tablename)
except ImportError_ as exc:
return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
response = current_app.response_class(template_text, mimetype='text/csv')
response.headers['Content-Disposition'] = (
'attachment; filename=%s.csv' % tablename)
return response
@settings_bp.route('/asset-import/import', methods=['POST'])
@jwt_required()
@require_role('admin')
def import_asset_csvs():
"""Validate uploaded CSVs and, when commit=true, load them.
Mirrors the 'flask csv import' CLI: every file runs through import_csv inside
ONE transaction with a shared resolver, so a computers.csv can resolve the
assets.csv uploaded alongside it. A dry run (the default) rolls the whole
transaction back and reports what WOULD change; commit=true persists it. The
whole run is all-or-nothing - if any file has a row problem, nothing is
written even on a commit.
"""
from sqlalchemy.exc import SQLAlchemyError
from shopdb.core.services.csvimport import (
ImportError_, Resolver, dependency_order, import_csv, table_from_filename)
uploads = request.files.getlist('files')
if not uploads:
return error_response(ErrorCodes.VALIDATION_ERROR, 'No CSV files were uploaded')
commit = str(request.form.get('commit', '')).strip().lower() in (
'1', 'true', 'yes', 'on')
# Map each upload to a table by filename. A later file with the same table
# name wins, matching the folder-import behaviour (one file per table).
texts = {}
filenames = {}
for upload in uploads:
name = upload.filename or ''
if not name.lower().endswith('.csv'):
continue
table = table_from_filename(name)
try:
texts[table] = upload.read().decode('utf-8-sig')
except (UnicodeDecodeError, OSError):
return error_response(ErrorCodes.VALIDATION_ERROR,
'Could not read %s as UTF-8 text' % name)
filenames[table] = name
if not texts:
return error_response(ErrorCodes.VALIDATION_ERROR,
'None of the uploaded files were CSV files')
ordered = dependency_order(list(texts))
skipped = sorted(set(texts) - set(ordered))
reconcile_setting = Setting.query.filter_by(
key='asset_import_reconcile_threshold').first()
try:
reconcile_threshold = float(reconcile_setting.value) if reconcile_setting and reconcile_setting.value else 0.0
except (TypeError, ValueError):
reconcile_threshold = 0.0
resolver = Resolver(reconcile_threshold=reconcile_threshold)
files_report = []
problem_count = 0
reconciled_count = 0
request_error = None
try:
for table in ordered:
try:
result = import_csv(table, texts[table], resolver=resolver, commit=False)
except ImportError_ as exc:
request_error = '%s: %s' % (filenames[table], str(exc))
break
except SQLAlchemyError as exc:
db.session.rollback()
request_error = '%s: database error: %s' % (
filenames[table], str(exc).split(chr(10))[0])
break
problems = [
{'line': p.line, 'column': p.column, 'message': p.message}
for p in result.problems
]
problem_count += len(problems)
reconciled_count += len(result.reconciliations)
files_report.append({
'table': table,
'filename': filenames[table],
'created': result.created,
'updated': result.updated,
'problems': problems,
'reconciliations': result.reconciliations,
})
applied = False
if request_error is not None:
db.session.rollback()
elif problem_count > 0:
# A row problem anywhere fails the whole run, so nothing is written
# even when commit was asked for.
db.session.rollback()
elif commit:
db.session.commit()
applied = True
else:
db.session.rollback()
except Exception:
db.session.rollback()
raise
total_created = sum(f['created'] for f in files_report)
total_updated = sum(f['updated'] for f in files_report)
return success_response({
'committed': applied,
'requested_commit': commit,
'request_error': request_error,
'files': files_report,
'skipped': skipped,
'problem_count': problem_count,
'reconciled_count': reconciled_count,
'reconcile_threshold': reconcile_threshold,
'total_created': total_created,
'total_updated': total_updated,
})
@settings_bp.route('', methods=['GET'])
@jwt_required(optional=True)
def list_settings():
@@ -556,6 +769,20 @@ def build_default_settings():
'category': 'site',
'description': 'Email domain appended to a contact SSO to build email/Teams links (sso@domain). Blank disables contact action buttons.'
},
{
'key': 'asset_import_csv_location',
'value': '',
'valuetype': 'string',
'category': 'site',
'description': r'Shared network location where the Asset Collector script deposits its CSVs and where ShopDB reads them for import (e.g. \\server.fqdn\share\folder). This path is baked into the downloadable collector script. Blank = not configured.'
},
{
'key': 'asset_import_reconcile_threshold',
'value': '0.85',
'valuetype': 'string',
'category': 'site',
'description': 'Similarity (0 to 1) a fuzzy lookup match must reach for the asset importer to reconcile a near-miss name (e.g. "Dell Inc." to an existing "Dell") onto an existing vendor, model, type, status, business unit, or location. Reconciled matches are applied but reported for review. 0 disables reconciliation. Operating systems and asset rows are never reconciled.'
},
]
# Site branding. Each instance (ADR-004) can replace the shipped GE defaults

View File

@@ -34,6 +34,7 @@ not at all.
from __future__ import annotations
import csv
import difflib
import io
import re
from collections import OrderedDict
@@ -86,6 +87,43 @@ PASSWORD_COLUMNS = {'passwordhash', 'password'}
TRUE_VALUES = {'1', 'true', 'yes', 'y', 't'}
FALSE_VALUES = {'0', 'false', 'no', 'n', 'f', ''}
# Reconciliation: when a name does not match a lookup row exactly, the resolver
# may still bind it to an existing row that is close enough. Only lookup tables
# take part - the small reference tables an operator picks values from. Rows that
# ARE the data being imported (assets, computers, machines) and the fast-moving
# operatingsystems list are excluded, so a typo there is a new row, not a silent
# merge onto something unrelated.
RECONCILABLE_TABLES = {
'assetstatuses', 'assettypes', 'locationtypes', 'modeltypes',
'computertypes', 'machinetypes', 'businessunits', 'locations',
'vendors', 'models',
}
# Corporate suffixes stripped before comparison so 'Dell Inc.' reconciles to
# 'Dell'. Lowercase, punctuation already removed by the time these are checked.
CORPORATE_SUFFIXES = {
'inc', 'incorporated', 'llc', 'llp', 'ltd', 'limited', 'corp',
'corporation', 'co', 'company', 'gmbh', 'ag', 'sa', 'plc',
}
# Default similarity a fuzzy match must reach, 0..1. Also the value the settings
# default is seeded with; 0 disables reconciliation entirely.
DEFAULT_RECONCILE_THRESHOLD = 0.85
def normalize_label(value):
"""Fold a label to its comparison form: lowercase, no punctuation, no
corporate suffix, single-spaced. 'Dell Inc.' and 'DELL' both become 'dell'.
"""
text = (value or '').strip().lower()
text = re.sub(r'[^a-z0-9]+', ' ', text).strip()
if not text:
return ''
words = [w for w in text.split(' ') if w]
while len(words) > 1 and words[-1] in CORPORATE_SUFFIXES:
words.pop()
return ' '.join(words)
def table_for(name):
table = db.metadata.tables.get(name)
@@ -221,10 +259,17 @@ class Resolver:
Rows created earlier in the SAME import are resolvable, so one file can
reference another that was loaded moments ago without a commit in between.
When a name does not match exactly and the table is reconcilable, a
normalized then fuzzy comparison against the existing rows may bind it to a
close-enough match. Every such binding is recorded so the caller can report
it for review; nothing is merged silently.
"""
def __init__(self):
def __init__(self, reconcile_threshold=0.0):
self._cache = {}
self.reconcile_threshold = reconcile_threshold or 0.0
self.events = []
def resolve(self, tablename, idcolumn, value):
if value is None:
@@ -249,6 +294,10 @@ class Resolver:
db.func.lower(db.func.trim(table.c[label])) == str(value).strip().lower())
).first()
if row is None:
found = self._reconcile(tablename, idcolumn, label, table, value)
if found is not None:
self._cache[key] = found
return found
raise ValueError(
"nothing in %s is named '%s' - add it to %s.csv, or import that file first"
% (tablename, value, tablename))
@@ -256,6 +305,52 @@ class Resolver:
self._cache[key] = found
return found
def _reconcile(self, tablename, idcolumn, label, table, value):
"""Bind value to the closest existing row when within threshold.
Returns the matched id and records an event, or None when reconciliation
is off, the table is not reconcilable, or nothing is close enough.
"""
if self.reconcile_threshold <= 0 or tablename not in RECONCILABLE_TABLES:
return None
wanted = normalize_label(value)
if not wanted:
return None
best_id = None
best_label = None
best_score = 0.0
for existing in db.session.execute(table.select()):
candidate = getattr(existing, label, None)
normalized = normalize_label(candidate)
if not normalized:
continue
if normalized == wanted:
score = 1.0
else:
score = difflib.SequenceMatcher(None, wanted, normalized).ratio()
if score > best_score:
best_score = score
best_id = getattr(existing, idcolumn)
best_label = candidate
if best_id is not None and best_score >= self.reconcile_threshold:
self.events.append({
'table': tablename,
'input': str(value).strip(),
'matched': best_label,
'score': round(best_score, 3),
})
return best_id
return None
def take_events(self):
"""Return and clear reconciliations recorded since the last call."""
events = self.events
self.events = []
return events
def remember(self, tablename, name, rowid):
if name is None:
return
@@ -282,6 +377,7 @@ class ImportResult(object):
self.updated = 0
self.problems = []
self.skipped = 0
self.reconciliations = []
@property
def ok(self):
@@ -381,6 +477,10 @@ def import_csv(tablename, text, resolver=None, commit=False):
continue
try:
values[name] = resolver.resolve(target[0], target[1], text)
for event in resolver.take_events():
event['line'] = line
event['column'] = name
result.reconciliations.append(event)
except ValueError as exc:
result.problems.append(RowProblem(line, name, str(exc)))
failed = True