Add site asset CSV import with fuzzy reconciliation
Sites seed inventory from the Asset Collector script or spreadsheet exports, but lookup names drift between sources (Dell vs Dell Inc.), which would otherwise create duplicate vendors/models/types on every import. The importer now normalizes and fuzzy-matches near-miss references onto existing lookup rows within an operator-set similarity threshold, auto-applying the match while reporting it for review so nothing merges silently. Operating systems and asset rows are never reconciled. Adds the dedicated Import Site Assets page with script/template download and a multi-file dry-run/commit workflow.
This commit is contained in:
@@ -1014,6 +1014,30 @@ export const settingsApi = {
|
||||
form.append('kind', kind)
|
||||
form.append('file', file)
|
||||
return api.post('/settings/branding-logo', form, { headers: { 'Content-Type': 'multipart/form-data' } })
|
||||
},
|
||||
downloadCollectorScript() {
|
||||
// Collect-ShopDB.ps1 with this site's CSV path baked into $NetworkStoreRoot
|
||||
return api.get('/settings/asset-import/script', { responseType: 'blob' })
|
||||
},
|
||||
listImportTemplates() {
|
||||
// Importable CSV tables in the order they must be uploaded
|
||||
return api.get('/settings/asset-import/templates')
|
||||
},
|
||||
downloadImportTemplate(tablename) {
|
||||
// Generated CSV template (header + example row) for one table
|
||||
return api.get(`/settings/asset-import/templates/${encodeURIComponent(tablename)}`, { responseType: 'blob' })
|
||||
},
|
||||
importAssetCsvs(files, commit = false) {
|
||||
// Multipart upload of one or more CSVs. commit=false is a dry-run preview
|
||||
// that writes nothing; commit=true applies the whole set in one transaction.
|
||||
const form = new FormData()
|
||||
for (const file of files) {
|
||||
form.append('files', file)
|
||||
}
|
||||
form.append('commit', commit ? 'true' : 'false')
|
||||
return api.post('/settings/asset-import/import', form, {
|
||||
headers: { 'Content-Type': 'multipart/form-data' }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -150,6 +150,12 @@ export default [
|
||||
component: () => import('../../views/settings/BrandingSettings.vue'),
|
||||
meta: { requiresAuth: true, requiresAdmin: true }
|
||||
},
|
||||
{
|
||||
path: 'settings/importsiteassets',
|
||||
name: 'import-site-assets',
|
||||
component: () => import('../../views/settings/ImportSiteAssets.vue'),
|
||||
meta: { requiresAuth: true, requiresAdmin: true }
|
||||
},
|
||||
{
|
||||
path: 'settings/printing',
|
||||
name: 'printing-settings',
|
||||
|
||||
659
frontend/src/views/settings/ImportSiteAssets.vue
Normal file
659
frontend/src/views/settings/ImportSiteAssets.vue
Normal file
@@ -0,0 +1,659 @@
|
||||
<template>
|
||||
<div>
|
||||
<div class="page-header">
|
||||
<h1>Import Site Assets</h1>
|
||||
<router-link to="/settings" class="btn btn-secondary">Back to Settings</router-link>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<p class="lead">
|
||||
There are several ways to seed this facility's inventory - Machines, PCs,
|
||||
Applications, and their properties - into ShopDB. You can combine them:
|
||||
each source writes to the same set of importable CSVs, and a later run
|
||||
fills gaps without wiping data an earlier run captured.
|
||||
</p>
|
||||
|
||||
<div class="methods">
|
||||
<div class="method is-primary">
|
||||
<div class="method-head">
|
||||
<span class="badge">Recommended</span>
|
||||
<h3>Asset Collector PowerShell script</h3>
|
||||
</div>
|
||||
<p>
|
||||
Deploy <code>Collect-ShopDB.ps1</code> to your existing endpoints in a
|
||||
way of your choosing (logon script, scheduled task, remote execution,
|
||||
or a manual run). It runs read-only, extracts each PC and any attached
|
||||
eDNC/DNC machine, and extrapolates vendor, model, OS, network, and
|
||||
installed-application details. Every run UPSERTs rows into the shared
|
||||
CSV store below, so a site builds its inventory one PC at a time.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="method">
|
||||
<div class="method-head">
|
||||
<h3>Existing databases</h3>
|
||||
</div>
|
||||
<p>
|
||||
Export from an existing MES, CMDB, or asset spreadsheet into the same
|
||||
CSV schema and drop the files in the collection location.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="method">
|
||||
<div class="method-head">
|
||||
<h3>Intune / Azure exports</h3>
|
||||
</div>
|
||||
<p>
|
||||
Map a device or application export (for example from Microsoft Graph)
|
||||
onto the computers and applications CSVs, then place them alongside the
|
||||
script output.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h3 class="section-title">Asset Collector Script</h3>
|
||||
<p class="muted">
|
||||
Download <code>Collect-ShopDB.ps1</code> and deploy it to your endpoints
|
||||
(logon script, scheduled task, remote execution, or a manual run). The CSV
|
||||
Collections Location you save below is baked into the script's
|
||||
<code>$NetworkStoreRoot</code> variable, so every endpoint publishes into
|
||||
the same store without editing PowerShell by hand.
|
||||
</p>
|
||||
|
||||
<div v-if="loading" class="muted">Loading...</div>
|
||||
<template v-else>
|
||||
<label class="field">
|
||||
<span>CSV Collections Location</span>
|
||||
<input
|
||||
v-model="location"
|
||||
type="text"
|
||||
placeholder="\\server.fqdn\share\folder"
|
||||
/>
|
||||
<small class="muted">
|
||||
UNC path or folder where the script deposits its CSVs (e.g.
|
||||
\\server.fqdn\share\folder). Saved here and embedded into the script
|
||||
download.
|
||||
</small>
|
||||
</label>
|
||||
|
||||
<div class="actions">
|
||||
<button class="btn btn-primary" :disabled="saving" @click="save">
|
||||
{{ saving ? 'Saving...' : 'Save Location' }}
|
||||
</button>
|
||||
<button class="btn btn-secondary" :disabled="downloadingScript" @click="downloadScript">
|
||||
{{ downloadingScript ? 'Preparing...' : 'Download Script' }}
|
||||
</button>
|
||||
<span v-if="saved" class="muted">Saved.</span>
|
||||
<span v-if="error" class="error">{{ error }}</span>
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h3 class="section-title">CSV Import Templates</h3>
|
||||
<p class="muted">
|
||||
These are early-draft import templates generated from the live ShopDB
|
||||
schema. Each file has a header row and a commented example. Fill one out
|
||||
(or let the collector script populate them) and import them in the order
|
||||
below - a table never references something that has not been loaded yet.
|
||||
Foreign keys accept the referenced row's name or its numeric id, and
|
||||
re-importing a file updates matching rows instead of duplicating them.
|
||||
</p>
|
||||
|
||||
<p v-if="templatesLoading" class="muted">Loading...</p>
|
||||
<ol v-else class="templates">
|
||||
<li v-for="t in templates" :key="t.table">
|
||||
<div class="template-row">
|
||||
<code>{{ t.filename }}</code>
|
||||
<button class="btn btn-secondary btn-sm" @click="downloadTemplate(t)">
|
||||
Download Template
|
||||
</button>
|
||||
</div>
|
||||
</li>
|
||||
</ol>
|
||||
|
||||
<p v-if="templatesError" class="error">{{ templatesError }}</p>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h3 class="section-title">Import CSVs</h3>
|
||||
<p class="muted">
|
||||
Select one or more filled-in CSVs. They are ordered by dependency
|
||||
automatically, so you can pick the whole set at once. Preview first: it
|
||||
writes nothing and reports exactly what each file would create or update,
|
||||
plus any rows that need fixing. Nothing is imported until you press
|
||||
Commit, and the whole set is applied together - if any row has a problem,
|
||||
nothing is written.
|
||||
</p>
|
||||
|
||||
<div class="import-controls">
|
||||
<input
|
||||
ref="fileInput"
|
||||
type="file"
|
||||
accept=".csv"
|
||||
multiple
|
||||
@change="onFilesPicked"
|
||||
/>
|
||||
<div class="actions">
|
||||
<button
|
||||
class="btn btn-secondary"
|
||||
:disabled="!selectedFiles.length || previewing || committing"
|
||||
@click="preview"
|
||||
>
|
||||
{{ previewing ? 'Checking...' : 'Preview (dry run)' }}
|
||||
</button>
|
||||
<button
|
||||
class="btn btn-primary"
|
||||
:disabled="!canCommit"
|
||||
@click="commit"
|
||||
>
|
||||
{{ committing ? 'Importing...' : 'Commit Import' }}
|
||||
</button>
|
||||
<button
|
||||
v-if="selectedFiles.length"
|
||||
class="btn btn-secondary btn-sm"
|
||||
:disabled="previewing || committing"
|
||||
@click="clearSelection"
|
||||
>
|
||||
Clear
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p v-if="selectedFiles.length" class="muted">
|
||||
{{ selectedFiles.length }} file(s) selected:
|
||||
{{ selectedFiles.map(f => f.name).join(', ') }}
|
||||
</p>
|
||||
|
||||
<p v-if="importError" class="error">{{ importError }}</p>
|
||||
|
||||
<div v-if="report" class="report">
|
||||
<p v-if="report.request_error" class="error">
|
||||
{{ report.request_error }}
|
||||
</p>
|
||||
|
||||
<p
|
||||
v-else
|
||||
:class="['report-banner', report.committed ? 'ok' : (report.problem_count ? 'warn' : 'info')]"
|
||||
>
|
||||
<template v-if="report.committed">
|
||||
Imported: {{ report.total_created }} created,
|
||||
{{ report.total_updated }} updated.
|
||||
</template>
|
||||
<template v-else-if="report.problem_count">
|
||||
{{ report.problem_count }} problem(s) found - nothing was written.
|
||||
Fix the rows below and preview again.
|
||||
</template>
|
||||
<template v-else>
|
||||
Looks good: {{ report.total_created }} would be created,
|
||||
{{ report.total_updated }} updated. Press Commit Import to apply.
|
||||
</template>
|
||||
</p>
|
||||
|
||||
<p v-if="report.reconciled_count" class="report-banner reconcile">
|
||||
{{ report.reconciled_count }} name(s) reconciled to an existing record
|
||||
within the similarity threshold ({{ report.reconcile_threshold }}).
|
||||
Review the matches below before committing.
|
||||
</p>
|
||||
|
||||
<table v-if="report.files && report.files.length" class="report-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>File</th>
|
||||
<th>Table</th>
|
||||
<th>Created</th>
|
||||
<th>Updated</th>
|
||||
<th>Reconciled</th>
|
||||
<th>Problems</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="f in report.files" :key="f.table">
|
||||
<td><code>{{ f.filename }}</code></td>
|
||||
<td>{{ f.table }}</td>
|
||||
<td>{{ f.created }}</td>
|
||||
<td>{{ f.updated }}</td>
|
||||
<td :class="{ 'has-reconcile': (f.reconciliations || []).length }">
|
||||
{{ (f.reconciliations || []).length }}
|
||||
</td>
|
||||
<td :class="{ 'has-problems': f.problems.length }">
|
||||
{{ f.problems.length }}
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<div v-if="allReconciliations.length" class="reconciliations">
|
||||
<h4>Reconciled matches</h4>
|
||||
<p class="muted">
|
||||
These names did not match exactly but were close enough to bind to an
|
||||
existing record. Confirm each is correct; if any is wrong, add the
|
||||
exact name to the matching CSV (or lower/raise the reconciliation
|
||||
threshold in Site settings) and preview again.
|
||||
</p>
|
||||
<ul>
|
||||
<li v-for="(r, i) in allReconciliations" :key="i">
|
||||
<code>{{ r.filename }}</code>
|
||||
line {{ r.line }}<template v-if="r.column">, column
|
||||
<code>{{ r.column }}</code></template>:
|
||||
"{{ r.input }}" matched <strong>{{ r.matched }}</strong>
|
||||
({{ r.table }}, {{ Math.round(r.score * 100) }}%)
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div v-if="allProblems.length" class="problems">
|
||||
<h4>Problems to fix</h4>
|
||||
<ul>
|
||||
<li v-for="(p, i) in allProblems" :key="i">
|
||||
<code>{{ p.filename }}</code>
|
||||
line {{ p.line }}<template v-if="p.column">, column
|
||||
<code>{{ p.column }}</code></template>: {{ p.message }}
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<p v-if="report.skipped && report.skipped.length" class="muted warn">
|
||||
Skipped (not importable): {{ report.skipped.join(', ') }}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h3 class="section-title">PC to Machine Relationships</h3>
|
||||
<p class="muted">
|
||||
A PC that drives a machine (a CMM PC, a DNC/eDNC host) is linked to that
|
||||
machine so the two show up connected across ShopDB. That link is built
|
||||
automatically by the Asset Collector, not by this CSV import: the script
|
||||
reads the four-digit workstation number from the PC it runs on and
|
||||
reports it, and ShopDB matches that number to a machine's
|
||||
<code>assetnumber</code> to create the connection.
|
||||
</p>
|
||||
<p class="muted">
|
||||
What this means for import order: load your machines first (so their
|
||||
<code>assetnumber</code> values exist), then let the collector run on
|
||||
each PC. A machine number the collector reports that does not match an
|
||||
existing machine is flagged rather than guessed, so nothing links to the
|
||||
wrong asset. Building this relationship straight from a CSV column is
|
||||
planned as a follow-up.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref, computed, onMounted } from 'vue'
|
||||
import { settingsApi } from '@/api'
|
||||
import { apiError } from '../../utils/apiError'
|
||||
|
||||
const LOCATION_KEY = 'asset_import_csv_location'
|
||||
|
||||
const location = ref('')
|
||||
const loading = ref(true)
|
||||
const saving = ref(false)
|
||||
const saved = ref(false)
|
||||
const error = ref('')
|
||||
const downloadingScript = ref(false)
|
||||
|
||||
const templates = ref([])
|
||||
const templatesLoading = ref(false)
|
||||
const templatesError = ref('')
|
||||
|
||||
const fileInput = ref(null)
|
||||
const selectedFiles = ref([])
|
||||
const previewing = ref(false)
|
||||
const committing = ref(false)
|
||||
const importError = ref('')
|
||||
const report = ref(null)
|
||||
|
||||
// Commit is allowed only after a clean preview of the CURRENT selection: a
|
||||
// report with no problems and no request-level error that has not already been
|
||||
// committed.
|
||||
const canCommit = computed(() => {
|
||||
if (!report.value || committing.value || previewing.value) return false
|
||||
if (report.value.committed) return false
|
||||
if (report.value.request_error) return false
|
||||
return report.value.problem_count === 0
|
||||
})
|
||||
|
||||
const allProblems = computed(() => {
|
||||
if (!report.value?.files) return []
|
||||
const out = []
|
||||
for (const file of report.value.files) {
|
||||
for (const problem of file.problems) {
|
||||
out.push({ filename: file.filename, ...problem })
|
||||
}
|
||||
}
|
||||
return out
|
||||
})
|
||||
|
||||
const allReconciliations = computed(() => {
|
||||
if (!report.value?.files) return []
|
||||
const out = []
|
||||
for (const file of report.value.files) {
|
||||
for (const event of (file.reconciliations || [])) {
|
||||
out.push({ filename: file.filename, ...event })
|
||||
}
|
||||
}
|
||||
return out
|
||||
})
|
||||
|
||||
async function loadLocation() {
|
||||
loading.value = true
|
||||
try {
|
||||
const response = await settingsApi.get(LOCATION_KEY).catch(nullOn404)
|
||||
location.value = readValue(response) || ''
|
||||
} catch (err) {
|
||||
error.value = apiError(err, 'Could not load the CSV location.')
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function nullOn404(err) {
|
||||
if (err.response?.status === 404) return null
|
||||
throw err
|
||||
}
|
||||
|
||||
function readValue(response) {
|
||||
if (!response) return ''
|
||||
const setting = response.data?.data || response.data
|
||||
return setting?.value ?? ''
|
||||
}
|
||||
|
||||
async function save() {
|
||||
saving.value = true
|
||||
saved.value = false
|
||||
error.value = ''
|
||||
try {
|
||||
await settingsApi.update(LOCATION_KEY, location.value)
|
||||
saved.value = true
|
||||
} catch (err) {
|
||||
error.value = apiError(err, 'Save failed.')
|
||||
} finally {
|
||||
saving.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function saveBlob(data, filename) {
|
||||
const url = window.URL.createObjectURL(new Blob([data]))
|
||||
const link = document.createElement('a')
|
||||
link.href = url
|
||||
link.download = filename
|
||||
document.body.appendChild(link)
|
||||
link.click()
|
||||
link.remove()
|
||||
window.URL.revokeObjectURL(url)
|
||||
}
|
||||
|
||||
async function downloadScript() {
|
||||
downloadingScript.value = true
|
||||
error.value = ''
|
||||
try {
|
||||
const response = await settingsApi.downloadCollectorScript()
|
||||
saveBlob(response.data, 'Collect-ShopDB.ps1')
|
||||
} catch (err) {
|
||||
error.value = apiError(err, 'Script download failed.')
|
||||
} finally {
|
||||
downloadingScript.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function loadTemplates() {
|
||||
templatesLoading.value = true
|
||||
templatesError.value = ''
|
||||
try {
|
||||
const response = await settingsApi.listImportTemplates()
|
||||
const data = response.data?.data || response.data || {}
|
||||
templates.value = data.tables || []
|
||||
} catch (err) {
|
||||
templatesError.value = apiError(err, 'Could not list import templates.')
|
||||
} finally {
|
||||
templatesLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadTemplate(t) {
|
||||
templatesError.value = ''
|
||||
try {
|
||||
const response = await settingsApi.downloadImportTemplate(t.table)
|
||||
saveBlob(response.data, t.filename)
|
||||
} catch (err) {
|
||||
templatesError.value = apiError(err, 'Template download failed.')
|
||||
}
|
||||
}
|
||||
|
||||
function onFilesPicked(event) {
|
||||
selectedFiles.value = Array.from(event.target.files || [])
|
||||
// A new selection invalidates any previous preview so a stale clean report
|
||||
// cannot be committed against different files.
|
||||
report.value = null
|
||||
importError.value = ''
|
||||
}
|
||||
|
||||
function clearSelection() {
|
||||
selectedFiles.value = []
|
||||
report.value = null
|
||||
importError.value = ''
|
||||
if (fileInput.value) fileInput.value.value = ''
|
||||
}
|
||||
|
||||
async function runImport(commit) {
|
||||
importError.value = ''
|
||||
try {
|
||||
const response = await settingsApi.importAssetCsvs(selectedFiles.value, commit)
|
||||
report.value = response.data?.data || response.data || null
|
||||
} catch (err) {
|
||||
importError.value = apiError(err, commit ? 'Import failed.' : 'Preview failed.')
|
||||
report.value = null
|
||||
}
|
||||
}
|
||||
|
||||
async function preview() {
|
||||
previewing.value = true
|
||||
try {
|
||||
await runImport(false)
|
||||
} finally {
|
||||
previewing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function commit() {
|
||||
committing.value = true
|
||||
try {
|
||||
await runImport(true)
|
||||
} finally {
|
||||
committing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(async () => {
|
||||
await loadLocation()
|
||||
await loadTemplates()
|
||||
})
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.lead {
|
||||
margin-top: 0;
|
||||
color: var(--text);
|
||||
}
|
||||
.methods {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(240px, 1fr));
|
||||
gap: 16px;
|
||||
margin-top: 12px;
|
||||
}
|
||||
.method {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
padding: 14px 16px;
|
||||
background: var(--bg);
|
||||
}
|
||||
.method.is-primary {
|
||||
border-color: var(--primary);
|
||||
}
|
||||
.method-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
.method-head h3 {
|
||||
margin: 0;
|
||||
font-size: 1rem;
|
||||
}
|
||||
.method p {
|
||||
margin: 8px 0 0;
|
||||
color: var(--text-light);
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
.badge {
|
||||
background: var(--primary);
|
||||
color: #fff;
|
||||
border-radius: 10px;
|
||||
padding: 2px 8px;
|
||||
font-size: 0.7rem;
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.section-title {
|
||||
margin: 0 0 8px;
|
||||
}
|
||||
.field {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
max-width: 640px;
|
||||
margin-top: 12px;
|
||||
}
|
||||
.field > span {
|
||||
font-weight: 600;
|
||||
}
|
||||
.field input[type='text'] {
|
||||
padding: 8px 10px;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 6px;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font-family: var(--font-mono, monospace);
|
||||
}
|
||||
.actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-top: 12px;
|
||||
}
|
||||
.templates {
|
||||
margin: 8px 0 0;
|
||||
padding-left: 20px;
|
||||
}
|
||||
.templates li {
|
||||
margin: 6px 0;
|
||||
}
|
||||
.template-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
.template-row code {
|
||||
min-width: 200px;
|
||||
}
|
||||
.btn-sm {
|
||||
padding: 4px 10px;
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
.muted {
|
||||
color: var(--text-light);
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
.muted.warn {
|
||||
color: var(--warning, #c47f00);
|
||||
}
|
||||
.error {
|
||||
color: var(--danger);
|
||||
margin-top: 8px;
|
||||
}
|
||||
.import-controls {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
.report {
|
||||
margin-top: 14px;
|
||||
}
|
||||
.report-banner {
|
||||
padding: 8px 12px;
|
||||
border-radius: 6px;
|
||||
font-weight: 600;
|
||||
}
|
||||
.report-banner.ok {
|
||||
background: var(--success-bg, rgba(40, 160, 80, 0.15));
|
||||
color: var(--success, #2e7d32);
|
||||
}
|
||||
.report-banner.info {
|
||||
background: var(--info-bg, rgba(60, 120, 200, 0.15));
|
||||
color: var(--text);
|
||||
}
|
||||
.report-banner.warn {
|
||||
background: var(--warning-bg, rgba(196, 127, 0, 0.15));
|
||||
color: var(--warning, #c47f00);
|
||||
}
|
||||
.report-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin-top: 12px;
|
||||
}
|
||||
.report-table th,
|
||||
.report-table td {
|
||||
text-align: left;
|
||||
padding: 6px 10px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
.report-table td.has-problems {
|
||||
color: var(--danger);
|
||||
font-weight: 700;
|
||||
}
|
||||
.report-table td.has-reconcile {
|
||||
color: var(--warning, #c47f00);
|
||||
font-weight: 700;
|
||||
}
|
||||
.report-banner.reconcile {
|
||||
background: var(--warning-bg, rgba(196, 127, 0, 0.15));
|
||||
color: var(--warning, #c47f00);
|
||||
}
|
||||
.reconciliations {
|
||||
margin-top: 12px;
|
||||
}
|
||||
.reconciliations h4 {
|
||||
margin: 0 0 6px;
|
||||
color: var(--warning, #c47f00);
|
||||
}
|
||||
.reconciliations ul {
|
||||
margin: 6px 0 0;
|
||||
padding-left: 20px;
|
||||
}
|
||||
.reconciliations li {
|
||||
margin: 4px 0;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
.problems {
|
||||
margin-top: 12px;
|
||||
}
|
||||
.problems h4 {
|
||||
margin: 0 0 6px;
|
||||
color: var(--danger);
|
||||
}
|
||||
.problems ul {
|
||||
margin: 0;
|
||||
padding-left: 20px;
|
||||
}
|
||||
.problems li {
|
||||
margin: 4px 0;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
</style>
|
||||
@@ -75,7 +75,8 @@ const LABELS = {
|
||||
employee_directory_mode: 'Employee Directory Mode',
|
||||
usb_directory_mode: 'USB Directory Mode',
|
||||
site_timezone: 'Site Timezone',
|
||||
setup_complete: 'Setup Complete'
|
||||
setup_complete: 'Setup Complete',
|
||||
asset_import_csv_location: 'Asset Import CSV Location'
|
||||
}
|
||||
function prettyLabel(key) {
|
||||
return LABELS[key] || key
|
||||
@@ -127,7 +128,10 @@ async function load() {
|
||||
try {
|
||||
const response = await settingsApi.list()
|
||||
const all = response.data?.data || response.data || []
|
||||
items.value = all.filter(s => s.category === 'site')
|
||||
// The Asset Collector CSV location is managed on the dedicated Import Site
|
||||
// Assets page.
|
||||
const importKeys = ['asset_import_csv_location']
|
||||
items.value = all.filter(s => s.category === 'site' && !importKeys.includes(s.key))
|
||||
} catch (err) {
|
||||
console.error('Error loading site settings:', err)
|
||||
error.value = 'Could not load settings.'
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
// Order: site identity + the reference-data catalogs users touch daily come
|
||||
// first, then the platform/system groups (integrations, communication, search,
|
||||
// plugins, access) cluster together at the end.
|
||||
import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key } from 'lucide-vue-next'
|
||||
import { Factory, MapPin, Tag, Package, Droplets, Monitor, MonitorSmartphone, Laptop, Cog, Building, Globe, Link, History, FileText, Users, Puzzle, Bell, Network, Home, Wrench, Printer, Router, Palette, SlidersHorizontal, Contact, Mail, ShieldCheck, KeyRound, Fingerprint, Search, Key, Upload } from 'lucide-vue-next'
|
||||
|
||||
export const settingsGroups = [
|
||||
{
|
||||
@@ -13,6 +13,7 @@ export const settingsGroups = [
|
||||
{ to: '/settings/site', icon: Home, title: 'Site & Facility', description: 'Site URL/FQDN, facility name, PC access domain, employee-ID pattern, printer hostname template' },
|
||||
{ to: '/settings/floormap', icon: MapPin, title: 'Floor Map', description: 'Facility floor-plan blueprint image and pixel dimensions (width, height) for the site map' },
|
||||
{ to: '/settings/branding', icon: Palette, title: 'Branding', description: 'Site, QR, and badge logos, favicon, and brand primary, accent, sidebar, and theme colors' },
|
||||
{ to: '/settings/importsiteassets', icon: Upload, title: 'Import Site Assets', description: 'Seed Machines, PCs, and Applications via the Asset Collector script, existing databases, or Intune/Azure exports; set the shared CSV collection location and download collected files' },
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
542
scripts/site_imports/assetcollector/Collect-ShopDB.ps1
Normal file
542
scripts/site_imports/assetcollector/Collect-ShopDB.ps1
Normal file
@@ -0,0 +1,542 @@
|
||||
#Requires -Version 5.1
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
# Where CSVs are collected/accumulated. Overrides the in-script CONFIG block
|
||||
# below. See the CONFIG block for the easy place to set this permanently.
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$OutputRoot,
|
||||
|
||||
# Optional network share to mirror the store into after a run. Overrides the
|
||||
# in-script CONFIG block below.
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$PublishRoot,
|
||||
|
||||
# Business identifier for this PC (workstation number). Defaults to hostname.
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$AssetNumber,
|
||||
|
||||
# Value written into assets.assettypeid for the PC (resolved by name).
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$AssetType = 'Computer',
|
||||
|
||||
# Value written into computers.computertypeid (resolved by name).
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$ComputerType,
|
||||
|
||||
# Value written into machines.machinetypeid (resolved by name).
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$MachineType,
|
||||
|
||||
# Explicit eDNC/UDC source overrides. When omitted, standard on-asset
|
||||
# locations are searched.
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$RegPath,
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$JsonPath,
|
||||
|
||||
# Skip switches for partial runs.
|
||||
[switch]$SkipMachine,
|
||||
[switch]$SkipApps
|
||||
)
|
||||
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Unified ShopDB site-adoption collector. Runs on any PC, collects the PC and any
|
||||
attached eDNC/DNC machine, and UPSERTS the results into a shared set of
|
||||
importable CSVs so a site can build its inventory PC by PC.
|
||||
|
||||
.DESCRIPTION
|
||||
Accumulating store under OutputRoot:
|
||||
vendors.csv, models.csv, operatingsystems.csv, assets.csv,
|
||||
computers.csv, machines.csv
|
||||
Rows are merged field-by-field on each table's natural key (a later run fills
|
||||
gaps and refreshes values without wiping data an earlier run captured).
|
||||
|
||||
Phase-2 data that CSV cannot carry is written per entity under OutputRoot\sidecars:
|
||||
computer_<assetnumber>.json (interfaces, installed apps)
|
||||
machine_<machinenumber>.json (PC<->Machine link, connection detail)
|
||||
|
||||
Read-only against the machine. Local reads may run elevated; publishing to a
|
||||
share runs in the current user context only.
|
||||
|
||||
.NOTES
|
||||
PS 5.1 compatible. Headers match the ShopDB importer (see CSV-SCHEMA.md).
|
||||
#>
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# =============================================================================
|
||||
# CONFIG - SET YOUR CSV DESTINATIONS HERE
|
||||
# -----------------------------------------------------------------------------
|
||||
# Two-stage flow so you can collect locally (possibly elevated) and then append
|
||||
# the captured rows to a CSV store that lives on a network share:
|
||||
#
|
||||
# 1) LocalStageRoot Fast, always-writable local folder where CSVs are built
|
||||
# and merged. Collection can run elevated here.
|
||||
# 2) NetworkStoreRoot The shared CSV store the whole site accumulates into and
|
||||
# that you later point 'flask csv import --dir' at. Leave
|
||||
# blank ('') to keep everything local. When set, the merged
|
||||
# store is copied here at the end of the run. Network
|
||||
# writes must happen as a STANDARD user - do NOT run the
|
||||
# publish step elevated.
|
||||
#
|
||||
# Example network path: '\\fileserver\shopdb$\intake'
|
||||
#
|
||||
# Command-line -OutputRoot / -PublishRoot still override these values.
|
||||
# =============================================================================
|
||||
|
||||
$LocalStageRoot = Join-Path $env:ProgramData 'ShopDB\intake'
|
||||
$NetworkStoreRoot = '\\wi001dataaege.wlm.geaerospace.net\shared\TEAMSYS\.sfscripts\shopdb'
|
||||
|
||||
# Resolve effective destinations (command-line params win over CONFIG above).
|
||||
if (-not $OutputRoot) { $OutputRoot = $LocalStageRoot }
|
||||
if (-not $PublishRoot) { $PublishRoot = $NetworkStoreRoot }
|
||||
|
||||
# --- Store definitions (header order + natural key per importable table) -------
|
||||
|
||||
$script:Stores = [ordered]@{
|
||||
'vendors.csv' = @{ key = 'vendor'; header = @('vendor') }
|
||||
'models.csv' = @{ key = 'modelnumber'; header = @('modelnumber','vendorid','modeltypeid') }
|
||||
'operatingsystems.csv' = @{ key = 'osname'; header = @('osname') }
|
||||
'assets.csv' = @{ key = 'assetnumber'; header = @('assetnumber','assettypeid','name','statusid','serialnumber','gaugelabreference','maintenancereference','locationid','businessunitid','notes') }
|
||||
'computers.csv' = @{ key = 'assetid'; header = @('assetid','hostname','computertypeid','osid','vendorid','modelnumberid','loggedinuser','lastreporteddate','lastboottime') }
|
||||
'machines.csv' = @{ key = 'assetid'; header = @('assetid','machinetypeid','vendorid','modelnumberid','controllervendorid','controllermodelid','requiresmanualconfig','islocationonly','notes') }
|
||||
}
|
||||
|
||||
# --- Shared helpers -----------------------------------------------------------
|
||||
|
||||
function ConvertTo-CsvField {
|
||||
param([AllowNull()]$Value)
|
||||
if ($null -eq $Value) { return '' }
|
||||
$text = [string]$Value
|
||||
if ($text -match '[",\r\n]') { return '"' + ($text -replace '"', '""') + '"' }
|
||||
return $text
|
||||
}
|
||||
|
||||
function Get-Prop {
|
||||
param([AllowNull()]$Object, [Parameter(Mandatory = $true)][string]$Name)
|
||||
if ($null -eq $Object) { return $null }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $null }
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
function Get-NonEmpty {
|
||||
param([AllowNull()]$Value)
|
||||
if ($null -ne $Value -and -not [string]::IsNullOrWhiteSpace([string]$Value)) {
|
||||
return ([string]$Value).Trim()
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Get-FirstNonEmpty {
|
||||
param([object[]]$Values)
|
||||
foreach ($v in $Values) { $n = Get-NonEmpty $v; if ($n) { return $n } }
|
||||
return $null
|
||||
}
|
||||
|
||||
function Format-ShopDbDate {
|
||||
param([AllowNull()]$Value)
|
||||
if ($null -eq $Value) { return $null }
|
||||
try { return ([datetime]$Value).ToString('yyyy-MM-dd HH:mm:ss') } catch { return $null }
|
||||
}
|
||||
|
||||
function Write-Utf8NoBom {
|
||||
param([string]$Path, [string]$Content)
|
||||
$encoding = New-Object System.Text.UTF8Encoding($false)
|
||||
[System.IO.File]::WriteAllText($Path, $Content, $encoding)
|
||||
}
|
||||
|
||||
# Field-level upsert into a persistent CSV store, keyed by natural key. Existing
|
||||
# values are kept when the incoming cell is empty, so a partial run never blanks
|
||||
# out data an earlier run captured.
|
||||
function Merge-CsvStore {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$Path,
|
||||
[Parameter(Mandatory = $true)][string[]]$Header,
|
||||
[Parameter(Mandatory = $true)][string]$Key,
|
||||
[Parameter(Mandatory = $true)][AllowNull()][object[]]$IncomingRows
|
||||
)
|
||||
|
||||
$index = [ordered]@{}
|
||||
if (Test-Path -LiteralPath $Path) {
|
||||
foreach ($existing in @(Import-Csv -LiteralPath $Path)) {
|
||||
$keyVal = Get-NonEmpty (Get-Prop $existing $Key)
|
||||
if (-not $keyVal) { continue }
|
||||
$index[$keyVal.ToLowerInvariant()] = $existing
|
||||
}
|
||||
}
|
||||
|
||||
$changed = 0
|
||||
foreach ($row in $IncomingRows) {
|
||||
if ($null -eq $row) { continue }
|
||||
$keyVal = Get-NonEmpty (Get-Prop $row $Key)
|
||||
if (-not $keyVal) { continue }
|
||||
$lc = $keyVal.ToLowerInvariant()
|
||||
if ($index.Contains($lc)) {
|
||||
$target = $index[$lc]
|
||||
foreach ($col in $Header) {
|
||||
$incoming = Get-NonEmpty (Get-Prop $row $col)
|
||||
if ($incoming) {
|
||||
if ($target.PSObject.Properties[$col]) { $target.$col = $incoming }
|
||||
else { $target | Add-Member -NotePropertyName $col -NotePropertyValue $incoming -Force }
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$fresh = [ordered]@{}
|
||||
foreach ($col in $Header) { $fresh[$col] = Get-NonEmpty (Get-Prop $row $col) }
|
||||
$index[$lc] = [pscustomobject]$fresh
|
||||
}
|
||||
$changed++
|
||||
}
|
||||
|
||||
$lines = New-Object System.Collections.Generic.List[string]
|
||||
$lines.Add(($Header -join ','))
|
||||
foreach ($stored in $index.Values) {
|
||||
$cells = foreach ($col in $Header) { ConvertTo-CsvField (Get-Prop $stored $col) }
|
||||
$lines.Add(($cells -join ','))
|
||||
}
|
||||
Write-Utf8NoBom -Path $Path -Content (($lines -join "`r`n") + "`r`n")
|
||||
return $changed
|
||||
}
|
||||
|
||||
# --- eDNC/UDC parsing (registry, .reg export, UDC settings json) --------------
|
||||
|
||||
function Resolve-ControllerVendor {
|
||||
param([AllowNull()][string]$ControlType, [AllowNull()][string]$NcIF)
|
||||
$ct = ('' + $ControlType).Trim().ToUpperInvariant()
|
||||
$nc = ('' + $NcIF).Trim().ToUpperInvariant()
|
||||
if ($nc -eq 'EFOCAS' -or $ct -eq 'EFOCAS') { return 'FANUC' }
|
||||
if ($ct -eq 'SIEMENS') { return 'Siemens' }
|
||||
if ($ct -eq 'OKUMA') { return 'Okuma' }
|
||||
return $null
|
||||
}
|
||||
|
||||
function Read-RegExport {
|
||||
param([Parameter(Mandatory = $true)][string]$Path)
|
||||
$lines = @(Get-Content -LiteralPath $Path -Encoding Unicode -ErrorAction Stop)
|
||||
$firstMeaningful = ($lines | Where-Object { $_ -match '\S' } | Select-Object -First 1)
|
||||
if (-not ($firstMeaningful -match '^(REGEDIT4|Windows Registry Editor Version)')) {
|
||||
$lines = @(Get-Content -LiteralPath $Path -Encoding Default -ErrorAction Stop)
|
||||
}
|
||||
$data = @{}
|
||||
$currentKey = $null
|
||||
foreach ($raw in $lines) {
|
||||
$line = $raw.Trim()
|
||||
if ([string]::IsNullOrWhiteSpace($line)) { continue }
|
||||
if ($line -match '^(REGEDIT4|Windows Registry Editor Version)') { continue }
|
||||
if ($line -match '^\s*;') { continue }
|
||||
if ($line -match '^\[(?<key>[^\]]+)\]$') {
|
||||
$currentKey = $Matches['key']
|
||||
if (-not $data.ContainsKey($currentKey)) { $data[$currentKey] = @{} }
|
||||
continue
|
||||
}
|
||||
if (-not $currentKey) { continue }
|
||||
if ($line -match '^"(?<name>[^"]+)"=(?<rhs>.+)$') {
|
||||
$name = $Matches['name']
|
||||
$rhs = $Matches['rhs'].Trim()
|
||||
if ($rhs -match '^"(?<s>.*)"$') { $val = $Matches['s'] }
|
||||
elseif ($rhs -match '^dword:(?<hex>[0-9a-fA-F]{8})$') { $val = [Convert]::ToInt32($Matches['hex'], 16) }
|
||||
else { $val = $rhs }
|
||||
$data[$currentKey][$name] = $val
|
||||
}
|
||||
}
|
||||
return $data
|
||||
}
|
||||
|
||||
function Get-RegExportValue {
|
||||
param([hashtable]$Reg, [string]$KeySuffix, [string]$Name)
|
||||
foreach ($key in $Reg.Keys) {
|
||||
if ($key -match ('\\' + [regex]::Escape($KeySuffix) + '$')) {
|
||||
if ($Reg[$key].ContainsKey($Name)) { return $Reg[$key][$Name] }
|
||||
}
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function ConvertFrom-RegExport {
|
||||
param([hashtable]$Reg)
|
||||
[ordered]@{
|
||||
source = 'reg'
|
||||
machinenumber = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'MachineNo')
|
||||
site = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'Site')
|
||||
controltype = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'Cnc')
|
||||
ncif = Get-NonEmpty (Get-RegExportValue $Reg 'General' 'NcIF')
|
||||
dnctype = 'eDNC'
|
||||
tcpaddress = Get-NonEmpty (Get-RegExportValue $Reg 'eFocas' 'IpAddr')
|
||||
tcpport = Get-NonEmpty (Get-RegExportValue $Reg 'eFocas' 'SocketNo')
|
||||
comport = Get-NonEmpty (Get-RegExportValue $Reg 'PPDCS' 'Port Id')
|
||||
baudrate = Get-NonEmpty (Get-RegExportValue $Reg 'PPDCS' 'Baud')
|
||||
}
|
||||
}
|
||||
|
||||
function Get-RegistryValue {
|
||||
param([string]$Path, [string]$Name)
|
||||
if (-not (Test-Path -LiteralPath $Path)) { return $null }
|
||||
return Get-Prop (Get-ItemProperty -LiteralPath $Path -ErrorAction SilentlyContinue) $Name
|
||||
}
|
||||
|
||||
function ConvertFrom-LiveRegistry {
|
||||
param([string]$Base)
|
||||
if (-not (Test-Path -LiteralPath $Base)) { return $null }
|
||||
$general = Join-Path $Base 'General'
|
||||
$efocas = Join-Path $Base 'eFocas'
|
||||
$ppdcs = Join-Path $Base 'PPDCS'
|
||||
[ordered]@{
|
||||
source = 'registry'
|
||||
machinenumber = Get-NonEmpty (Get-RegistryValue $general 'MachineNo')
|
||||
site = Get-NonEmpty (Get-RegistryValue $general 'Site')
|
||||
controltype = Get-NonEmpty (Get-RegistryValue $general 'Cnc')
|
||||
ncif = Get-NonEmpty (Get-RegistryValue $general 'NcIF')
|
||||
dnctype = 'eDNC'
|
||||
tcpaddress = Get-NonEmpty (Get-RegistryValue $efocas 'IpAddr')
|
||||
tcpport = Get-NonEmpty (Get-RegistryValue $efocas 'SocketNo')
|
||||
comport = Get-NonEmpty (Get-RegistryValue $ppdcs 'Port Id')
|
||||
baudrate = Get-NonEmpty (Get-RegistryValue $ppdcs 'Baud')
|
||||
}
|
||||
}
|
||||
|
||||
function ConvertFrom-UdcJson {
|
||||
param([string]$Path)
|
||||
$obj = Get-Content -LiteralPath $Path -Raw -ErrorAction Stop | ConvertFrom-Json
|
||||
$general = Get-Prop $obj 'GeneralSettings'
|
||||
$ethernet = Get-Prop $obj 'EthernetSettings'
|
||||
$serial = Get-Prop $obj 'SerialSettings'
|
||||
[ordered]@{
|
||||
source = 'json'
|
||||
machinenumber = Get-NonEmpty (Get-Prop $general 'MachineNumber')
|
||||
site = Get-NonEmpty (Get-Prop $general 'Site')
|
||||
controltype = Get-NonEmpty (Get-Prop $general 'ControlType')
|
||||
ncif = $null
|
||||
dnctype = Get-NonEmpty (Get-Prop $general 'DncType')
|
||||
tcpaddress = Get-NonEmpty (Get-Prop $ethernet 'TCPAddress')
|
||||
tcpport = Get-NonEmpty (Get-Prop $ethernet 'TCPPort')
|
||||
comport = Get-NonEmpty (Get-Prop $serial 'ComPort')
|
||||
baudrate = Get-NonEmpty (Get-Prop $serial 'BaudRate')
|
||||
}
|
||||
}
|
||||
|
||||
function Find-FirstFile {
|
||||
param([string[]]$Roots, [string]$Filter)
|
||||
foreach ($root in $Roots) {
|
||||
if (-not $root -or -not (Test-Path -LiteralPath $root)) { continue }
|
||||
$hit = Get-ChildItem -LiteralPath $root -Filter $Filter -File -ErrorAction SilentlyContinue |
|
||||
Sort-Object LastWriteTime -Descending | Select-Object -First 1
|
||||
if ($hit) { return $hit.FullName }
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Resolve-DncSource {
|
||||
param([string]$RegPath, [string]$JsonPath)
|
||||
if ($JsonPath -and (Test-Path -LiteralPath $JsonPath)) {
|
||||
Write-Host (' machine source: JSON (explicit) ' + $JsonPath) -ForegroundColor DarkGray
|
||||
return ConvertFrom-UdcJson -Path $JsonPath
|
||||
}
|
||||
if ($RegPath -and (Test-Path -LiteralPath $RegPath)) {
|
||||
Write-Host (' machine source: .reg (explicit) ' + $RegPath) -ForegroundColor DarkGray
|
||||
return ConvertFrom-RegExport -Reg (Read-RegExport -Path $RegPath)
|
||||
}
|
||||
foreach ($base in @(
|
||||
'HKLM:\SOFTWARE\GE Aircraft Engines\DNC',
|
||||
'HKLM:\SOFTWARE\WOW6432Node\GE Aircraft Engines\DNC')) {
|
||||
$fromReg = ConvertFrom-LiveRegistry -Base $base
|
||||
if ($fromReg -and $fromReg.machinenumber) {
|
||||
Write-Host (' machine source: live registry ' + $base) -ForegroundColor DarkGray
|
||||
return $fromReg
|
||||
}
|
||||
}
|
||||
$localReg = Find-FirstFile -Roots @('C:\Program Files (x86)\Dnc', (Join-Path $env:SystemDrive 'GE Aerospace')) -Filter '*.reg'
|
||||
if ($localReg) {
|
||||
Write-Host (' machine source: local .reg ' + $localReg) -ForegroundColor DarkGray
|
||||
return ConvertFrom-RegExport -Reg (Read-RegExport -Path $localReg)
|
||||
}
|
||||
$localJson = Find-FirstFile -Roots @((Join-Path $env:SystemDrive 'GE Aerospace'), 'C:\Program Files (x86)\UDC', 'C:\Program Files (x86)\Dnc') -Filter 'udc_settings_*.json'
|
||||
if ($localJson) {
|
||||
Write-Host (' machine source: local UDC json ' + $localJson) -ForegroundColor DarkGray
|
||||
return ConvertFrom-UdcJson -Path $localJson
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
# --- PC collection (read-only) ------------------------------------------------
|
||||
|
||||
function Get-PcFacts {
|
||||
$computerSystem = Get-CimInstance Win32_ComputerSystem -ErrorAction SilentlyContinue
|
||||
$operatingSys = Get-CimInstance Win32_OperatingSystem -ErrorAction SilentlyContinue
|
||||
$bios = Get-CimInstance Win32_BIOS -ErrorAction SilentlyContinue
|
||||
|
||||
$interfaces = @()
|
||||
try {
|
||||
foreach ($cfg in @(Get-NetIPConfiguration -ErrorAction SilentlyContinue)) {
|
||||
$adapter = $null
|
||||
try { $adapter = Get-NetAdapter -InterfaceAlias $cfg.InterfaceAlias -ErrorAction SilentlyContinue } catch {}
|
||||
if (-not $adapter -or (Get-Prop $adapter 'HardwareInterface') -ne $true) { continue }
|
||||
$ipv4 = @(Get-Prop $cfg 'IPv4Address') | Select-Object -First 1
|
||||
$interfaces += [ordered]@{
|
||||
interfacealias = [string]$cfg.InterfaceAlias
|
||||
ipaddress = if ($ipv4) { [string](Get-Prop $ipv4 'IPAddress') } else { $null }
|
||||
macaddress = [string](Get-Prop $adapter 'MacAddress')
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
$installedApps = @()
|
||||
if (-not $SkipApps) {
|
||||
foreach ($keyPath in @(
|
||||
'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\*',
|
||||
'HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\*')) {
|
||||
foreach ($entry in @(Get-ItemProperty -Path $keyPath -ErrorAction SilentlyContinue)) {
|
||||
$displayName = Get-NonEmpty (Get-Prop $entry 'DisplayName')
|
||||
if (-not $displayName) { continue }
|
||||
if ((Get-Prop $entry 'SystemComponent') -eq 1) { continue }
|
||||
$installedApps += [ordered]@{
|
||||
appname = $displayName
|
||||
installedversion = Get-NonEmpty (Get-Prop $entry 'DisplayVersion')
|
||||
publisher = Get-NonEmpty (Get-Prop $entry 'Publisher')
|
||||
}
|
||||
}
|
||||
}
|
||||
$installedApps = @($installedApps | Sort-Object { $_.appname } -Unique)
|
||||
}
|
||||
|
||||
[ordered]@{
|
||||
hostname = Get-FirstNonEmpty @($env:COMPUTERNAME, (Get-Prop $computerSystem 'Name'))
|
||||
vendor = Get-NonEmpty (Get-Prop $computerSystem 'Manufacturer')
|
||||
model = Get-NonEmpty (Get-Prop $computerSystem 'Model')
|
||||
osname = Get-NonEmpty (Get-Prop $operatingSys 'Caption')
|
||||
serial = Get-FirstNonEmpty @((Get-Prop $bios 'SerialNumber'), (Get-Prop $computerSystem 'SerialNumber'))
|
||||
lastboot = Format-ShopDbDate (Get-Prop $operatingSys 'LastBootUpTime')
|
||||
loggeduser = Get-NonEmpty (Get-Prop $computerSystem 'UserName')
|
||||
interfaces = $interfaces
|
||||
installedapps = $installedApps
|
||||
}
|
||||
}
|
||||
|
||||
# --- Main ---------------------------------------------------------------------
|
||||
|
||||
Write-Host ''
|
||||
Write-Host ' ShopDB unified collector' -ForegroundColor Cyan
|
||||
Write-Host (' ' + ('-' * 56)) -ForegroundColor DarkGray
|
||||
|
||||
if (-not (Test-Path -LiteralPath $OutputRoot)) { New-Item -ItemType Directory -Path $OutputRoot -Force | Out-Null }
|
||||
$sidecarDir = Join-Path $OutputRoot 'sidecars'
|
||||
if (-not (Test-Path -LiteralPath $sidecarDir)) { New-Item -ItemType Directory -Path $sidecarDir -Force | Out-Null }
|
||||
|
||||
$collectedAt = (Get-Date).ToString('yyyy-MM-dd HH:mm:ss')
|
||||
|
||||
$pc = Get-PcFacts
|
||||
if (-not $AssetNumber) { $AssetNumber = $pc.hostname }
|
||||
|
||||
# Row sets keyed to the shared stores.
|
||||
$vendorRows = @()
|
||||
$modelRows = @()
|
||||
$osRows = @()
|
||||
$assetRows = @()
|
||||
$computerRows = @()
|
||||
$machineRows = @()
|
||||
|
||||
if ($pc.vendor) { $vendorRows += [pscustomobject]@{ vendor = $pc.vendor } }
|
||||
if ($pc.model) { $modelRows += [pscustomobject]@{ modelnumber = $pc.model; vendorid = $pc.vendor; modeltypeid = $null } }
|
||||
if ($pc.osname) { $osRows += [pscustomobject]@{ osname = $pc.osname } }
|
||||
|
||||
$assetRows += [pscustomobject]@{
|
||||
assetnumber = $AssetNumber; assettypeid = $AssetType; name = $pc.hostname; statusid = $null
|
||||
serialnumber = $pc.serial; gaugelabreference = $null; maintenancereference = $null
|
||||
locationid = $null; businessunitid = $null
|
||||
notes = 'PC collected ' + $collectedAt
|
||||
}
|
||||
$computerRows += [pscustomobject]@{
|
||||
assetid = $AssetNumber; hostname = $pc.hostname; computertypeid = $ComputerType; osid = $pc.osname
|
||||
vendorid = $pc.vendor; modelnumberid = $pc.model; loggedinuser = $pc.loggeduser
|
||||
lastreporteddate = $collectedAt; lastboottime = $pc.lastboot
|
||||
}
|
||||
|
||||
# Machine (best-effort).
|
||||
$machine = $null
|
||||
if (-not $SkipMachine) {
|
||||
try { $machine = Resolve-DncSource -RegPath $RegPath -JsonPath $JsonPath } catch { $machine = $null }
|
||||
}
|
||||
if ($machine -and $machine.machinenumber) {
|
||||
$controllerVendor = Resolve-ControllerVendor -ControlType $machine.controltype -NcIF $machine.ncif
|
||||
$transport = if ($machine.tcpaddress) { 'ethernet' } elseif ($machine.comport) { 'serial' } else { 'unknown' }
|
||||
$noteBits = @("DNC=$($machine.dnctype)")
|
||||
if ($machine.controltype) { $noteBits += "control=$($machine.controltype)" }
|
||||
$noteBits += "transport=$transport"
|
||||
$machineNote = 'eDNC collected ' + $collectedAt + '; ' + ($noteBits -join '; ')
|
||||
|
||||
if ($controllerVendor) { $vendorRows += [pscustomobject]@{ vendor = $controllerVendor } }
|
||||
$assetRows += [pscustomobject]@{
|
||||
assetnumber = $machine.machinenumber; assettypeid = 'Machine'; name = $machine.machinenumber; statusid = $null
|
||||
serialnumber = $null; gaugelabreference = $null; maintenancereference = $null
|
||||
locationid = $null; businessunitid = $null; notes = $machineNote
|
||||
}
|
||||
$machineRows += [pscustomobject]@{
|
||||
assetid = $machine.machinenumber; machinetypeid = $MachineType; vendorid = $null; modelnumberid = $null
|
||||
controllervendorid = $controllerVendor; controllermodelid = $null
|
||||
requiresmanualconfig = $null; islocationonly = $null; notes = $machineNote
|
||||
}
|
||||
|
||||
$link = [ordered]@{
|
||||
pcassetnumber = $AssetNumber; machineassetnumber = $machine.machinenumber; relationship = 'controls'
|
||||
dnctype = $machine.dnctype; controltype = $machine.controltype; controllervendor = $controllerVendor
|
||||
site = $machine.site; source = $machine.source; collectedat = $collectedAt
|
||||
connection = [ordered]@{
|
||||
transport = $transport; tcpaddress = $machine.tcpaddress; tcpport = $machine.tcpport
|
||||
comport = $machine.comport; baudrate = $machine.baudrate
|
||||
}
|
||||
}
|
||||
Write-Utf8NoBom -Path (Join-Path $sidecarDir ('machine_' + $machine.machinenumber + '.json')) -Content ($link | ConvertTo-Json -Depth 6)
|
||||
Write-Host (' linked machine ' + $machine.machinenumber + ' (' + $controllerVendor + ', ' + $transport + ')') -ForegroundColor DarkGray
|
||||
} elseif (-not $SkipMachine) {
|
||||
Write-Host ' no eDNC/DNC/UDC machine config found on this PC' -ForegroundColor DarkGray
|
||||
}
|
||||
|
||||
# --- Upsert into the shared store --------------------------------------------
|
||||
|
||||
$rowsByStore = @{
|
||||
'vendors.csv' = $vendorRows
|
||||
'models.csv' = $modelRows
|
||||
'operatingsystems.csv' = $osRows
|
||||
'assets.csv' = $assetRows
|
||||
'computers.csv' = $computerRows
|
||||
'machines.csv' = $machineRows
|
||||
}
|
||||
|
||||
Write-Host ''
|
||||
Write-Host (' Store: ' + $OutputRoot) -ForegroundColor Green
|
||||
foreach ($fileName in $script:Stores.Keys) {
|
||||
$spec = $script:Stores[$fileName]
|
||||
$rows = @($rowsByStore[$fileName])
|
||||
if ($rows.Count -eq 0) { continue }
|
||||
$path = Join-Path $OutputRoot $fileName
|
||||
$n = Merge-CsvStore -Path $path -Header $spec.header -Key $spec.key -IncomingRows $rows
|
||||
Write-Host (' merged ' + $n + ' row(s) into ' + $fileName) -ForegroundColor DarkGray
|
||||
}
|
||||
|
||||
# PC phase-2 sidecar.
|
||||
$pcSidecar = [ordered]@{
|
||||
assetnumber = $AssetNumber; hostname = $pc.hostname; collectedat = $collectedAt
|
||||
interfaces = $pc.interfaces; installedapps = $pc.installedapps
|
||||
}
|
||||
Write-Utf8NoBom -Path (Join-Path $sidecarDir ('computer_' + $AssetNumber + '.json')) -Content ($pcSidecar | ConvertTo-Json -Depth 6)
|
||||
|
||||
# --- Publish (standard-user network mirror, optional) -------------------------
|
||||
|
||||
if ($PublishRoot) {
|
||||
if (([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole(
|
||||
[Security.Principal.WindowsBuiltInRole]::Administrator)) {
|
||||
Write-Host ''
|
||||
Write-Host ' WARNING: publishing while elevated. Network writes should run as the standard user.' -ForegroundColor Yellow
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $PublishRoot)) { New-Item -ItemType Directory -Path $PublishRoot -Force | Out-Null }
|
||||
Copy-Item -Path (Join-Path $OutputRoot '*') -Destination $PublishRoot -Recurse -Force
|
||||
Write-Host ''
|
||||
Write-Host (' Published store to: ' + $PublishRoot) -ForegroundColor Green
|
||||
}
|
||||
|
||||
Write-Host ''
|
||||
Write-Host ' Done. Dry-run validate the accumulated store with:' -ForegroundColor Cyan
|
||||
Write-Host (' flask csv import --dir "' + $OutputRoot + '"') -ForegroundColor White
|
||||
Write-Host ''
|
||||
91
scripts/site_imports/assetcollector/overview.md
Normal file
91
scripts/site_imports/assetcollector/overview.md
Normal file
@@ -0,0 +1,91 @@
|
||||
# Collect-ShopDB.ps1 Overview
|
||||
|
||||
## Overview
|
||||
|
||||
- **Purpose**: A unified, read-only "site adoption" collector that runs on any
|
||||
Windows PC, gathers facts about that PC plus any attached eDNC/DNC/UDC machine,
|
||||
and merges them into a shared set of importable CSVs so a site can build its
|
||||
ShopDB inventory one PC at a time.
|
||||
- **Compatibility**: PowerShell 5.1+; CSV headers are aligned to the ShopDB
|
||||
importer (`flask csv import --dir`).
|
||||
|
||||
## Configuration and destinations
|
||||
|
||||
- **Two-stage flow**: a fast local stage folder (`%ProgramData%\ShopDB\intake`)
|
||||
where CSVs are built/merged, and an optional network store (a file share) that
|
||||
the whole site accumulates into.
|
||||
- Command-line params `-OutputRoot` and `-PublishRoot` override the in-script
|
||||
CONFIG block.
|
||||
- Other params: `-AssetNumber` (defaults to hostname),
|
||||
`-AssetType`/`-ComputerType`/`-MachineType` (resolved by name), explicit
|
||||
`-RegPath`/`-JsonPath` machine source overrides, and `-SkipMachine`/`-SkipApps`
|
||||
switches for partial runs.
|
||||
|
||||
## Data model (the stores)
|
||||
|
||||
- Six importable CSVs, each with a defined header order and natural key:
|
||||
`vendors.csv`, `models.csv`, `operatingsystems.csv`, `assets.csv`,
|
||||
`computers.csv`, `machines.csv`.
|
||||
- Column names follow ShopDB conventions (e.g., `assetid`, `modelnumberid`,
|
||||
`controllervendorid`, `lastreporteddate`).
|
||||
|
||||
## PC collection (`Get-PcFacts`, read-only)
|
||||
|
||||
- Uses CIM/WMI (`Win32_ComputerSystem`, `Win32_OperatingSystem`, `Win32_BIOS`)
|
||||
to gather hostname, vendor/manufacturer, model, OS caption, serial number,
|
||||
last boot time, and logged-in user.
|
||||
- Enumerates **network interfaces** (hardware adapters only: alias, IPv4, MAC).
|
||||
- Enumerates **installed applications** from the registry Uninstall keys (name,
|
||||
version, publisher), skipping system components - unless `-SkipApps` is set.
|
||||
|
||||
## Machine (eDNC/DNC/UDC) collection
|
||||
|
||||
- `Resolve-DncSource` searches in priority order: explicit JSON path -> explicit
|
||||
.reg path -> live registry (`HKLM\...\GE Aircraft Engines\DNC`) -> local `.reg`
|
||||
export files -> local `udc_settings_*.json` files.
|
||||
- Three parsers normalize the different source formats into one shape:
|
||||
`ConvertFrom-RegExport` (parses exported `.reg` files including dword decoding),
|
||||
`ConvertFrom-LiveRegistry`, and `ConvertFrom-UdcJson`.
|
||||
- Extracts machine number, site, control type, DNC type, and connection detail
|
||||
(TCP address/port for ethernet, COM port/baud for serial).
|
||||
- `Resolve-ControllerVendor` maps control type / NcIF to a controller vendor
|
||||
(e.g., eFOCAS -> FANUC, Siemens, Okuma).
|
||||
|
||||
## Merge / upsert logic (`Merge-CsvStore`)
|
||||
|
||||
- Field-level UPSERT keyed on each table's natural key: existing rows are matched
|
||||
case-insensitively, and empty incoming cells never overwrite previously captured
|
||||
values - so partial or repeated runs fill gaps without wiping earlier data.
|
||||
- CSV writing is UTF-8 without BOM, with proper quoting/escaping.
|
||||
|
||||
## Sidecar files (phase-2 data CSV can't carry)
|
||||
|
||||
- Per-PC `sidecars\computer_<assetnumber>.json`: interfaces + installed apps.
|
||||
- Per-machine `sidecars\machine_<machinenumber>.json`: PC-to-machine link
|
||||
(relationship "controls"), DNC/control details, and connection transport info.
|
||||
|
||||
## Publishing and output
|
||||
|
||||
- Optionally mirrors the whole local store to the network share
|
||||
(`Copy-Item -Recurse -Force`), with a warning if run elevated (network writes
|
||||
should run as standard user).
|
||||
- Prints a summary and the follow-up dry-run command:
|
||||
`flask csv import --dir "<OutputRoot>"`.
|
||||
|
||||
---
|
||||
|
||||
## Relevance to an "Asset Collector" package
|
||||
|
||||
This script is a solid template for one collector in a broader package. Notable
|
||||
patterns worth generalizing across other collectors (existing databases,
|
||||
Intune/Azure exports, etc.):
|
||||
|
||||
- **Normalized intermediate schema** (the six CSV stores + JSON sidecars) that
|
||||
all sources converge on, decoupling collection from import.
|
||||
- **Idempotent field-level merge** so multiple heterogeneous sources can
|
||||
contribute to the same asset without clobbering each other.
|
||||
- **Source-priority resolution** and per-format parsers - the same shape could
|
||||
wrap an Intune/Azure `Get-MgDevice` export or a SQL query.
|
||||
- **Machines, PCs, Applications** are already modeled here; an Intune/Azure
|
||||
source would map naturally onto `computers.csv` + the installed-apps sidecar,
|
||||
while a MES/CNC database could feed `machines.csv`.
|
||||
5
scripts/site_imports/assetcollector/sample/assets.csv
Normal file
5
scripts/site_imports/assetcollector/sample/assets.csv
Normal file
@@ -0,0 +1,5 @@
|
||||
assetnumber,name,serialnumber,assettypeid,statusid,locationid,businessunitid,notes
|
||||
SAMPLE-PC-01,Shopfloor PC 01,DELLSN0001,computer,In Use,A1,Gage Lab,Sample collected device
|
||||
SAMPLE-PC-02,CMM PC 02,DELLSN0002,computer,In Use,A1,Gage Lab,Sample collected device
|
||||
SAMPLE-MC-01,Zeiss CMM 01,ZEISSSN0001,machine,In Use,A1,Gage Lab,Sample collected device attached to SAMPLE-PC-02
|
||||
SAMPLE-MC-02,Haas VF-2SS 02,HAASSN0002,machine,In Use,A1,Gage Lab,Sample collected standalone CNC
|
||||
|
3
scripts/site_imports/assetcollector/sample/computers.csv
Normal file
3
scripts/site_imports/assetcollector/sample/computers.csv
Normal file
@@ -0,0 +1,3 @@
|
||||
assetid,computertypeid,hostname,osid,vendorid,modelnumberid,loggedinuser
|
||||
SAMPLE-PC-01,Shopfloor PC,WKS-SAMPLE01,Windows 10 Enterprise,Dell,OptiPlex 7090,AEROAD\operator1
|
||||
SAMPLE-PC-02,CMM PC,WKS-SAMPLE02,Windows 11 Enterprise,Dell,Precision 3660,AEROAD\operator2
|
||||
|
3
scripts/site_imports/assetcollector/sample/machines.csv
Normal file
3
scripts/site_imports/assetcollector/sample/machines.csv
Normal file
@@ -0,0 +1,3 @@
|
||||
assetid,machinetypeid,vendorid,modelnumberid,requiresmanualconfig,islocationonly
|
||||
SAMPLE-MC-01,CMM,Zeiss,CONTURA G2,0,0
|
||||
SAMPLE-MC-02,CNC,Haas Automation,VF-2SS,0,0
|
||||
|
5
scripts/site_imports/assetcollector/sample/models.csv
Normal file
5
scripts/site_imports/assetcollector/sample/models.csv
Normal file
@@ -0,0 +1,5 @@
|
||||
modelnumber,vendorid
|
||||
OptiPlex 7090,Dell
|
||||
Precision 3660,Dell
|
||||
VF-2SS,Haas Automation
|
||||
CONTURA G2,Zeiss
|
||||
|
@@ -0,0 +1,3 @@
|
||||
osname
|
||||
Windows 10 Enterprise
|
||||
Windows 11 Enterprise
|
||||
|
4
scripts/site_imports/assetcollector/sample/vendors.csv
Normal file
4
scripts/site_imports/assetcollector/sample/vendors.csv
Normal file
@@ -0,0 +1,4 @@
|
||||
vendor
|
||||
Dell
|
||||
Haas Automation
|
||||
Zeiss
|
||||
|
@@ -1,6 +1,7 @@
|
||||
"""Settings API routes."""
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from flask import Blueprint, request, current_app
|
||||
from flask_jwt_extended import jwt_required, get_jwt_identity
|
||||
@@ -240,6 +241,218 @@ def serve_branding_logo(filename):
|
||||
return send_upload(_branding_dir(), filename)
|
||||
|
||||
|
||||
def _asset_import_dir():
|
||||
"""Configured Asset Collector CSV location, or '' when unset."""
|
||||
setting = Setting.query.filter_by(key='asset_import_csv_location').first()
|
||||
return (setting.value or '').strip() if setting else ''
|
||||
|
||||
|
||||
# The collector script that gets served (with the site's CSV path baked in) from
|
||||
# the Import Site Assets page. Copied into the image under scripts/ (see
|
||||
# Dockerfile), so it is always available next to the running API.
|
||||
_COLLECTOR_SCRIPT = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))),
|
||||
'scripts', 'site_imports', 'assetcollector', 'Collect-ShopDB.ps1')
|
||||
|
||||
|
||||
def _embed_store_path(script_text, path):
|
||||
"""Return the collector script with $NetworkStoreRoot set to path.
|
||||
|
||||
The script ships with a site-specific default on that line; rewriting it
|
||||
means an operator downloads a script already pointed at this facility's CSV
|
||||
store and does not have to edit PowerShell by hand. A blank path leaves the
|
||||
shipped default untouched.
|
||||
"""
|
||||
if not path:
|
||||
return script_text
|
||||
literal = path.replace("'", "''")
|
||||
replacement = "$NetworkStoreRoot = '%s'" % literal
|
||||
# A function replacement is used deliberately: a UNC path is full of
|
||||
# backslashes, and a string replacement runs them through re's escape
|
||||
# processing (so \s, \w, a trailing \1 all raise or corrupt). A callable
|
||||
# returns the text verbatim.
|
||||
return re.sub(
|
||||
r"(?m)^\$NetworkStoreRoot\s*=.*$",
|
||||
lambda _match: replacement,
|
||||
script_text,
|
||||
count=1)
|
||||
|
||||
|
||||
@settings_bp.route('/asset-import/script', methods=['GET'])
|
||||
@jwt_required()
|
||||
@require_role('admin')
|
||||
def download_collector_script():
|
||||
"""Download Collect-ShopDB.ps1 with this site's CSV path embedded."""
|
||||
try:
|
||||
with open(_COLLECTOR_SCRIPT, 'r', encoding='utf-8') as handle:
|
||||
script_text = handle.read()
|
||||
except OSError:
|
||||
return error_response(ErrorCodes.NOT_FOUND,
|
||||
'Collector script is not available on the server',
|
||||
http_code=404)
|
||||
|
||||
script_text = _embed_store_path(script_text, _asset_import_dir())
|
||||
response = current_app.response_class(script_text, mimetype='text/plain')
|
||||
response.headers['Content-Disposition'] = 'attachment; filename=Collect-ShopDB.ps1'
|
||||
return response
|
||||
|
||||
|
||||
@settings_bp.route('/asset-import/templates', methods=['GET'])
|
||||
@jwt_required()
|
||||
@require_role('admin')
|
||||
def list_import_templates():
|
||||
"""List the importable CSV tables in the order they must be uploaded."""
|
||||
from shopdb.core.services.csvimport import IMPORTABLE
|
||||
|
||||
tables = [{'table': name, 'filename': '%s.csv' % name}
|
||||
for name in IMPORTABLE]
|
||||
return success_response({'tables': tables})
|
||||
|
||||
|
||||
@settings_bp.route('/asset-import/templates/<tablename>', methods=['GET'])
|
||||
@jwt_required()
|
||||
@require_role('admin')
|
||||
def download_import_template(tablename):
|
||||
"""Download a generated CSV template for a single importable table."""
|
||||
from shopdb.core.services.csvimport import (
|
||||
IMPORTABLE, generate_template, ImportError_)
|
||||
|
||||
if tablename not in IMPORTABLE:
|
||||
return error_response(ErrorCodes.NOT_FOUND,
|
||||
'No such importable table: %s' % tablename,
|
||||
http_code=404)
|
||||
try:
|
||||
template_text = generate_template(tablename)
|
||||
except ImportError_ as exc:
|
||||
return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
|
||||
|
||||
response = current_app.response_class(template_text, mimetype='text/csv')
|
||||
response.headers['Content-Disposition'] = (
|
||||
'attachment; filename=%s.csv' % tablename)
|
||||
return response
|
||||
|
||||
|
||||
@settings_bp.route('/asset-import/import', methods=['POST'])
|
||||
@jwt_required()
|
||||
@require_role('admin')
|
||||
def import_asset_csvs():
|
||||
"""Validate uploaded CSVs and, when commit=true, load them.
|
||||
|
||||
Mirrors the 'flask csv import' CLI: every file runs through import_csv inside
|
||||
ONE transaction with a shared resolver, so a computers.csv can resolve the
|
||||
assets.csv uploaded alongside it. A dry run (the default) rolls the whole
|
||||
transaction back and reports what WOULD change; commit=true persists it. The
|
||||
whole run is all-or-nothing - if any file has a row problem, nothing is
|
||||
written even on a commit.
|
||||
"""
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from shopdb.core.services.csvimport import (
|
||||
ImportError_, Resolver, dependency_order, import_csv, table_from_filename)
|
||||
|
||||
uploads = request.files.getlist('files')
|
||||
if not uploads:
|
||||
return error_response(ErrorCodes.VALIDATION_ERROR, 'No CSV files were uploaded')
|
||||
|
||||
commit = str(request.form.get('commit', '')).strip().lower() in (
|
||||
'1', 'true', 'yes', 'on')
|
||||
|
||||
# Map each upload to a table by filename. A later file with the same table
|
||||
# name wins, matching the folder-import behaviour (one file per table).
|
||||
texts = {}
|
||||
filenames = {}
|
||||
for upload in uploads:
|
||||
name = upload.filename or ''
|
||||
if not name.lower().endswith('.csv'):
|
||||
continue
|
||||
table = table_from_filename(name)
|
||||
try:
|
||||
texts[table] = upload.read().decode('utf-8-sig')
|
||||
except (UnicodeDecodeError, OSError):
|
||||
return error_response(ErrorCodes.VALIDATION_ERROR,
|
||||
'Could not read %s as UTF-8 text' % name)
|
||||
filenames[table] = name
|
||||
|
||||
if not texts:
|
||||
return error_response(ErrorCodes.VALIDATION_ERROR,
|
||||
'None of the uploaded files were CSV files')
|
||||
|
||||
ordered = dependency_order(list(texts))
|
||||
skipped = sorted(set(texts) - set(ordered))
|
||||
|
||||
reconcile_setting = Setting.query.filter_by(
|
||||
key='asset_import_reconcile_threshold').first()
|
||||
try:
|
||||
reconcile_threshold = float(reconcile_setting.value) if reconcile_setting and reconcile_setting.value else 0.0
|
||||
except (TypeError, ValueError):
|
||||
reconcile_threshold = 0.0
|
||||
|
||||
resolver = Resolver(reconcile_threshold=reconcile_threshold)
|
||||
files_report = []
|
||||
problem_count = 0
|
||||
reconciled_count = 0
|
||||
request_error = None
|
||||
|
||||
try:
|
||||
for table in ordered:
|
||||
try:
|
||||
result = import_csv(table, texts[table], resolver=resolver, commit=False)
|
||||
except ImportError_ as exc:
|
||||
request_error = '%s: %s' % (filenames[table], str(exc))
|
||||
break
|
||||
except SQLAlchemyError as exc:
|
||||
db.session.rollback()
|
||||
request_error = '%s: database error: %s' % (
|
||||
filenames[table], str(exc).split(chr(10))[0])
|
||||
break
|
||||
|
||||
problems = [
|
||||
{'line': p.line, 'column': p.column, 'message': p.message}
|
||||
for p in result.problems
|
||||
]
|
||||
problem_count += len(problems)
|
||||
reconciled_count += len(result.reconciliations)
|
||||
files_report.append({
|
||||
'table': table,
|
||||
'filename': filenames[table],
|
||||
'created': result.created,
|
||||
'updated': result.updated,
|
||||
'problems': problems,
|
||||
'reconciliations': result.reconciliations,
|
||||
})
|
||||
|
||||
applied = False
|
||||
if request_error is not None:
|
||||
db.session.rollback()
|
||||
elif problem_count > 0:
|
||||
# A row problem anywhere fails the whole run, so nothing is written
|
||||
# even when commit was asked for.
|
||||
db.session.rollback()
|
||||
elif commit:
|
||||
db.session.commit()
|
||||
applied = True
|
||||
else:
|
||||
db.session.rollback()
|
||||
except Exception:
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
total_created = sum(f['created'] for f in files_report)
|
||||
total_updated = sum(f['updated'] for f in files_report)
|
||||
|
||||
return success_response({
|
||||
'committed': applied,
|
||||
'requested_commit': commit,
|
||||
'request_error': request_error,
|
||||
'files': files_report,
|
||||
'skipped': skipped,
|
||||
'problem_count': problem_count,
|
||||
'reconciled_count': reconciled_count,
|
||||
'reconcile_threshold': reconcile_threshold,
|
||||
'total_created': total_created,
|
||||
'total_updated': total_updated,
|
||||
})
|
||||
|
||||
|
||||
@settings_bp.route('', methods=['GET'])
|
||||
@jwt_required(optional=True)
|
||||
def list_settings():
|
||||
@@ -556,6 +769,20 @@ def build_default_settings():
|
||||
'category': 'site',
|
||||
'description': 'Email domain appended to a contact SSO to build email/Teams links (sso@domain). Blank disables contact action buttons.'
|
||||
},
|
||||
{
|
||||
'key': 'asset_import_csv_location',
|
||||
'value': '',
|
||||
'valuetype': 'string',
|
||||
'category': 'site',
|
||||
'description': r'Shared network location where the Asset Collector script deposits its CSVs and where ShopDB reads them for import (e.g. \\server.fqdn\share\folder). This path is baked into the downloadable collector script. Blank = not configured.'
|
||||
},
|
||||
{
|
||||
'key': 'asset_import_reconcile_threshold',
|
||||
'value': '0.85',
|
||||
'valuetype': 'string',
|
||||
'category': 'site',
|
||||
'description': 'Similarity (0 to 1) a fuzzy lookup match must reach for the asset importer to reconcile a near-miss name (e.g. "Dell Inc." to an existing "Dell") onto an existing vendor, model, type, status, business unit, or location. Reconciled matches are applied but reported for review. 0 disables reconciliation. Operating systems and asset rows are never reconciled.'
|
||||
},
|
||||
]
|
||||
|
||||
# Site branding. Each instance (ADR-004) can replace the shipped GE defaults
|
||||
|
||||
@@ -34,6 +34,7 @@ not at all.
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import difflib
|
||||
import io
|
||||
import re
|
||||
from collections import OrderedDict
|
||||
@@ -86,6 +87,43 @@ PASSWORD_COLUMNS = {'passwordhash', 'password'}
|
||||
TRUE_VALUES = {'1', 'true', 'yes', 'y', 't'}
|
||||
FALSE_VALUES = {'0', 'false', 'no', 'n', 'f', ''}
|
||||
|
||||
# Reconciliation: when a name does not match a lookup row exactly, the resolver
|
||||
# may still bind it to an existing row that is close enough. Only lookup tables
|
||||
# take part - the small reference tables an operator picks values from. Rows that
|
||||
# ARE the data being imported (assets, computers, machines) and the fast-moving
|
||||
# operatingsystems list are excluded, so a typo there is a new row, not a silent
|
||||
# merge onto something unrelated.
|
||||
RECONCILABLE_TABLES = {
|
||||
'assetstatuses', 'assettypes', 'locationtypes', 'modeltypes',
|
||||
'computertypes', 'machinetypes', 'businessunits', 'locations',
|
||||
'vendors', 'models',
|
||||
}
|
||||
|
||||
# Corporate suffixes stripped before comparison so 'Dell Inc.' reconciles to
|
||||
# 'Dell'. Lowercase, punctuation already removed by the time these are checked.
|
||||
CORPORATE_SUFFIXES = {
|
||||
'inc', 'incorporated', 'llc', 'llp', 'ltd', 'limited', 'corp',
|
||||
'corporation', 'co', 'company', 'gmbh', 'ag', 'sa', 'plc',
|
||||
}
|
||||
|
||||
# Default similarity a fuzzy match must reach, 0..1. Also the value the settings
|
||||
# default is seeded with; 0 disables reconciliation entirely.
|
||||
DEFAULT_RECONCILE_THRESHOLD = 0.85
|
||||
|
||||
|
||||
def normalize_label(value):
|
||||
"""Fold a label to its comparison form: lowercase, no punctuation, no
|
||||
corporate suffix, single-spaced. 'Dell Inc.' and 'DELL' both become 'dell'.
|
||||
"""
|
||||
text = (value or '').strip().lower()
|
||||
text = re.sub(r'[^a-z0-9]+', ' ', text).strip()
|
||||
if not text:
|
||||
return ''
|
||||
words = [w for w in text.split(' ') if w]
|
||||
while len(words) > 1 and words[-1] in CORPORATE_SUFFIXES:
|
||||
words.pop()
|
||||
return ' '.join(words)
|
||||
|
||||
|
||||
def table_for(name):
|
||||
table = db.metadata.tables.get(name)
|
||||
@@ -221,10 +259,17 @@ class Resolver:
|
||||
|
||||
Rows created earlier in the SAME import are resolvable, so one file can
|
||||
reference another that was loaded moments ago without a commit in between.
|
||||
|
||||
When a name does not match exactly and the table is reconcilable, a
|
||||
normalized then fuzzy comparison against the existing rows may bind it to a
|
||||
close-enough match. Every such binding is recorded so the caller can report
|
||||
it for review; nothing is merged silently.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, reconcile_threshold=0.0):
|
||||
self._cache = {}
|
||||
self.reconcile_threshold = reconcile_threshold or 0.0
|
||||
self.events = []
|
||||
|
||||
def resolve(self, tablename, idcolumn, value):
|
||||
if value is None:
|
||||
@@ -249,6 +294,10 @@ class Resolver:
|
||||
db.func.lower(db.func.trim(table.c[label])) == str(value).strip().lower())
|
||||
).first()
|
||||
if row is None:
|
||||
found = self._reconcile(tablename, idcolumn, label, table, value)
|
||||
if found is not None:
|
||||
self._cache[key] = found
|
||||
return found
|
||||
raise ValueError(
|
||||
"nothing in %s is named '%s' - add it to %s.csv, or import that file first"
|
||||
% (tablename, value, tablename))
|
||||
@@ -256,6 +305,52 @@ class Resolver:
|
||||
self._cache[key] = found
|
||||
return found
|
||||
|
||||
def _reconcile(self, tablename, idcolumn, label, table, value):
|
||||
"""Bind value to the closest existing row when within threshold.
|
||||
|
||||
Returns the matched id and records an event, or None when reconciliation
|
||||
is off, the table is not reconcilable, or nothing is close enough.
|
||||
"""
|
||||
if self.reconcile_threshold <= 0 or tablename not in RECONCILABLE_TABLES:
|
||||
return None
|
||||
|
||||
wanted = normalize_label(value)
|
||||
if not wanted:
|
||||
return None
|
||||
|
||||
best_id = None
|
||||
best_label = None
|
||||
best_score = 0.0
|
||||
for existing in db.session.execute(table.select()):
|
||||
candidate = getattr(existing, label, None)
|
||||
normalized = normalize_label(candidate)
|
||||
if not normalized:
|
||||
continue
|
||||
if normalized == wanted:
|
||||
score = 1.0
|
||||
else:
|
||||
score = difflib.SequenceMatcher(None, wanted, normalized).ratio()
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_id = getattr(existing, idcolumn)
|
||||
best_label = candidate
|
||||
|
||||
if best_id is not None and best_score >= self.reconcile_threshold:
|
||||
self.events.append({
|
||||
'table': tablename,
|
||||
'input': str(value).strip(),
|
||||
'matched': best_label,
|
||||
'score': round(best_score, 3),
|
||||
})
|
||||
return best_id
|
||||
return None
|
||||
|
||||
def take_events(self):
|
||||
"""Return and clear reconciliations recorded since the last call."""
|
||||
events = self.events
|
||||
self.events = []
|
||||
return events
|
||||
|
||||
def remember(self, tablename, name, rowid):
|
||||
if name is None:
|
||||
return
|
||||
@@ -282,6 +377,7 @@ class ImportResult(object):
|
||||
self.updated = 0
|
||||
self.problems = []
|
||||
self.skipped = 0
|
||||
self.reconciliations = []
|
||||
|
||||
@property
|
||||
def ok(self):
|
||||
@@ -381,6 +477,10 @@ def import_csv(tablename, text, resolver=None, commit=False):
|
||||
continue
|
||||
try:
|
||||
values[name] = resolver.resolve(target[0], target[1], text)
|
||||
for event in resolver.take_events():
|
||||
event['line'] = line
|
||||
event['column'] = name
|
||||
result.reconciliations.append(event)
|
||||
except ValueError as exc:
|
||||
result.problems.append(RowProblem(line, name, str(exc)))
|
||||
failed = True
|
||||
|
||||
Reference in New Issue
Block a user