ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add Entra device identity to computers and Azure/Thor import sources

Browse Source 
PC records now carry Entra/Azure device identity so imported machines keep a durable link back to their directory object instead of matching on hostname alone. Adds azuredeviceid, azureobjectid, and azuredevicecategory to the Computer model with matching migrations, surfaces them read-only in PCDetail as an Entra Device Info card, and lets PCForm edit the device id directly. The computers plugin registers a machineassetnumber import hook that reconciles PC-to-machine links during bulk import and warns on unresolved or conflicting relationships.

Adds importsources adapters that normalize Azure device-group and Thor PC inventory exports into canonical ShopDB CSVs via hostname alignment, exposes register_import_hook and RowWarning on the shopdb.api contract surface (contract 0.21.0), and refreshes the Import Site Assets page with clearer source instructions. Includes reconcile_locations maintenance script and coverage for the new import paths.
This commit is contained in:
benjamin.baker
2026-08-25 16:46:00 -04:00
parent f4d8868463
commit bdef21da50
21 changed files with 2064 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
.dockerignore Normal file
View File

@@ -0,0 +1,23 @@
# Keep the build context lean and prevent host artifacts from clobbering
# in-image installs. node_modules is platform-specific: copying the host's
# (Windows) tree over the container's `npm ci` breaks rollup's native binary.
**/node_modules
frontend/dist
frontend/src/.plugins-staged
frontend/src/router/routes.gen.js
# Python caches and virtualenvs
**/__pycache__
**/*.pyc
.venv
venv
# Local dev and VCS noise
.git
.gitignore
*.log
.env.local
# Installer build artifacts (not part of the app image)
installer-arch-dev/build
installer-arch-dev/dist

6
Dockerfile
View File

@@ -23,6 +23,12 @@ FROM node:20-slim AS frontendbuild
WORKDIR /build
# Trust the site's corporate root CA so npm's TLS to the registry validates
# behind an inspecting proxy. Without it npm 10.x fails silently and leaves an
# incomplete node_modules.
COPY corp-root-ca.pem /usr/local/share/ca-certificates/corp-root-ca.crt
ENV NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/corp-root-ca.crt
# Copy only the manifests first so `npm ci` caches on dependency changes.
COPY frontend/package*.json ./frontend/
RUN cd frontend && npm ci

4
frontend/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "shopdb-frontend",
"version": "0.7.0",
"version": "0.12.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "shopdb-frontend",
"version": "0.7.0",
"version": "0.12.0",
"dependencies": {
"@fontsource-variable/inter": "^5.2.8",
"@fullcalendar/core": "^6.1.20",

14
frontend/src/api/index.js
View File

@@ -1027,14 +1027,26 @@ export const settingsApi = {
// Generated CSV template (header + example row) for one table
return api.get(`/settings/asset-import/templates/${encodeURIComponent(tablename)}`, { responseType: 'blob' })
},
importAssetCsvs(files, commit = false) {
importAssetCsvs(files, commit = false, options = {}) {
// Multipart upload of one or more CSVs. commit=false is a dry-run preview
// that writes nothing; commit=true applies the whole set in one transaction.
// options.source picks a foreign adapter (azure, thor); default native means
// the files are already canonical per-table CSVs. options.computertype is
// stamped onto every imported computer when a source assigns a PC type.
const form = new FormData()
for (const file of files) {
form.append('files', file)
}
form.append('commit', commit ? 'true' : 'false')
if (options.source) {
form.append('source', options.source)
}
if (options.computertype) {
form.append('computertype', options.computertype)
}
if (options.category) {
form.append('category', options.category)
}
return api.post('/settings/asset-import/import', form, {
headers: { 'Content-Type': 'multipart/form-data' }
})

385
frontend/src/views/settings/ImportSiteAssets.vue
View File

@@ -91,32 +91,6 @@
</template>
</div>
<div class="card">
<h3 class="section-title">CSV Import Templates</h3>
<p class="muted">
These are early-draft import templates generated from the live ShopDB
schema. Each file has a header row and a commented example. Fill one out
(or let the collector script populate them) and import them in the order
below - a table never references something that has not been loaded yet.
Foreign keys accept the referenced row's name or its numeric id, and
re-importing a file updates matching rows instead of duplicating them.
</p>
<p v-if="templatesLoading" class="muted">Loading...</p>
<ol v-else class="templates">
<li v-for="t in templates" :key="t.table">
<div class="template-row">
<code>{{ t.filename }}</code>
<button class="btn btn-secondary btn-sm" @click="downloadTemplate(t)">
Download Template
</button>
</div>
</li>
</ol>
<p v-if="templatesError" class="error">{{ templatesError }}</p>
</div>
<div class="card">
<h3 class="section-title">Import CSVs</h3>
<p class="muted">
@@ -128,6 +102,108 @@
nothing is written.
</p>
<label class="field">
<span>Source format</span>
<div class="source-cards">
<button
v-for="s in IMPORT_SOURCES"
:key="s.key"
type="button"
:class="['source-card', { active: importSource === s.key }]"
@click="selectSource(s.key)"
>
<span class="source-card-title">{{ s.label }}</span>
<span class="source-card-desc">{{ s.description }}</span>
</button>
</div>
<small class="muted">
ShopDB native expects the canonical per-table CSVs below. Azure and
Thor accept those tools' own exports and convert them for you.
</small>
</label>
<div v-if="importSource === 'azure'" class="source-help">
<p class="muted">
An Azure device-group export lists device hostnames plus their Entra
deviceId. The device displayName is matched to an existing PC by
hostname (case-insensitive): a match updates that PC in place and
stores its deviceId, object id, and category; a device with no
matching hostname creates a new computer (assettype computer, status
In Use). It carries no model, OS, location, or user data - enrich
those later with the collector or a Thor import.
</p>
<label class="field">
<span>Assign computer type (optional)</span>
<select v-model="azureComputerType">
<option value="">No type</option>
<option v-for="t in computerTypes" :key="t.computertypeid" :value="t.computertype">
{{ t.computertype }}
</option>
<option value="__custom__">Custom (add a new type)...</option>
</select>
<small class="muted">
Stamped onto every device in this file. For example import the SHP
group with "Shopfloor PC" and the BUS group with "Inspection PC".
Picking from the list prevents typos labelling hundreds of machines.
</small>
</label>
<label v-if="azureComputerType === '__custom__'" class="field">
<span>New computer type name</span>
<div class="custom-type-row">
<input
v-model="customComputerType"
type="text"
placeholder="e.g. Inspection PC"
/>
<button
class="btn btn-secondary btn-sm"
:disabled="!customComputerType.trim() || addingType"
@click="addComputerType"
>
{{ addingType ? 'Adding...' : 'Add type' }}
</button>
</div>
<small v-if="customTypeError" class="error">{{ customTypeError }}</small>
<small v-else class="muted">
Adds the type to your settings now and selects it for this import.
</small>
</label>
<label class="field">
<span>Azure device category (optional)</span>
<input
v-model="azureDeviceCategory"
type="text"
placeholder="e.g. SFLD_DYN_Device_Group_Wilmington_BUS_NC_COUNTRY:USA"
/>
<small class="muted">
The Entra group/category these devices belong to. Stored on every
imported device and shown under Entra Device Info on the PC card.
</small>
</label>
<details class="instructions">
<summary>How to export group members from Azure</summary>
<ol>
<li>Open the Microsoft Entra admin center (entra.microsoft.com).</li>
<li>Go to Groups, then open the device group you want to import.</li>
<li>Select Members in the left menu.</li>
<li>Click Bulk Operations-&gt;Download Members to export the CSV.</li>
<li>Upload that CSV here with the source set to Azure device group.</li>
</ol>
</details>
</div>
<div v-else-if="importSource === 'thor'" class="source-help">
<p class="muted">
A Thor PC-inventory export is a wide, fixed-format dump. This maps the
hostname, BIOS manufacturer, model, OS, logged-in user, last-reported
and boot times, and location into ShopDB. Each host is matched to an
existing PC by hostname (case-insensitive): a match updates that PC in
place; a host with no matching hostname creates a new computer.
Network details (MAC, switch, VLAN) and the machine relationship are
not imported in this pass.
</p>
</div>
<div class="import-controls">
<input
ref="fileInput"
@@ -198,6 +274,12 @@
Review the matches below before committing.
</p>
<p v-if="report.warning_count" class="report-banner amber">
{{ report.warning_count }} warning(s): these rows still import, but
something needs a look - most often a machine link that could not be
made. Review them below; each says what to do.
</p>
<table v-if="report.files && report.files.length" class="report-table">
<thead>
<tr>
@@ -206,6 +288,7 @@
<th>Created</th>
<th>Updated</th>
<th>Reconciled</th>
<th>Warnings</th>
<th>Problems</th>
</tr>
</thead>
@@ -218,6 +301,9 @@
<td :class="{ 'has-reconcile': (f.reconciliations || []).length }">
{{ (f.reconciliations || []).length }}
</td>
<td :class="{ 'has-warnings': (f.warnings || []).length }">
{{ (f.warnings || []).length }}
</td>
<td :class="{ 'has-problems': f.problems.length }">
{{ f.problems.length }}
</td>
@@ -244,6 +330,23 @@
</ul>
</div>
<div v-if="allWarnings.length" class="warnings">
<h4>Warnings</h4>
<p class="muted">
Each row below imported, but something could not be completed - most
often a PC's machine link. Nothing here blocks the import; fix what
you can and re-import to complete the links.
</p>
<ul>
<li v-for="(w, i) in allWarnings" :key="i">
<code>{{ w.filename }}</code>
line {{ w.line }}<template v-if="w.column">, column
<code>{{ w.column }}</code></template>:
{{ w.reason }}<template v-if="w.action"> - <em>{{ w.action }}</em></template>
</li>
</ul>
</div>
<div v-if="allProblems.length" class="problems">
<h4>Problems to fix</h4>
<ul>
@@ -261,23 +364,57 @@
</div>
</div>
<div class="card">
<h3 class="section-title">CSV Import Templates</h3>
<p class="muted">
These import templates are generated from the live ShopDB schema. Each
file has a header row plus a small worked example - commented rows you
can uncomment and edit. The examples are internally consistent across
every file (the same locations, vendors, and types are referenced), so
you can download the full set, keep it as a starter workbook, and fill
in your own assets right away. Import them in the order below - a table
never references something that has not been loaded yet. Foreign keys
accept the referenced row's name or its numeric id, and re-importing a
file updates matching rows instead of duplicating them.
</p>
<p v-if="templatesLoading" class="muted">Loading...</p>
<ol v-else class="templates">
<li v-for="t in templates" :key="t.table">
<div class="template-row">
<code>{{ t.filename }}</code>
<button class="btn btn-secondary btn-sm" @click="downloadTemplate(t)">
Download Template
</button>
</div>
</li>
</ol>
<p v-if="templatesError" class="error">{{ templatesError }}</p>
</div>
<div class="card">
<h3 class="section-title">PC to Machine Relationships</h3>
<p class="muted">
A PC that drives a machine (a CMM PC, a DNC/eDNC host) is linked to that
machine so the two show up connected across ShopDB. That link is built
automatically by the Asset Collector, not by this CSV import: the script
reads the four-digit workstation number from the PC it runs on and
reports it, and ShopDB matches that number to a machine's
<code>assetnumber</code> to create the connection.
machine so the two show up connected across ShopDB. You can build that
link two ways. The Asset Collector builds it automatically: the script
reads the four-digit workstation number from the PC it runs on, reports
it, and ShopDB matches that number to a machine's
<code>assetnumber</code>.
</p>
<p class="muted">
What this means for import order: load your machines first (so their
<code>assetnumber</code> values exist), then let the collector run on
each PC. A machine number the collector reports that does not match an
existing machine is flagged rather than guessed, so nothing links to the
wrong asset. Building this relationship straight from a CSV column is
planned as a follow-up.
Or build it straight from the spreadsheet: put the machine's
<code>assetnumber</code> in the <code>machineassetnumber</code> column on
<code>computers.csv</code>, and the import creates the link for you.
</p>
<p class="muted">
Either way, load your machines first (so their
<code>assetnumber</code> values exist), then the PCs. A machine number
that does not match an existing machine - or one already driven by
another PC - is reported as a warning and skipped, not guessed, so
nothing links to the wrong asset. The PC still imports; fix the number
and re-import to complete the link.
</p>
</div>
</div>
@@ -285,7 +422,7 @@
<script setup>
import { ref, computed, onMounted } from 'vue'
import { settingsApi } from '@/api'
import { settingsApi, computersApi } from '@/api'
import { apiError } from '../../utils/apiError'
const LOCATION_KEY = 'asset_import_csv_location'
@@ -308,6 +445,24 @@ const committing = ref(false)
const importError = ref('')
const report = ref(null)
// Which format the selected files are in. 'native' means the uploads are
// already canonical per-table CSVs; 'azure' and 'thor' route through a
// server-side adapter that transforms the foreign export first.
const IMPORT_SOURCES = [
{ key: 'native', label: 'ShopDB native', description: 'The canonical per-table CSVs below (assets, computers, vendors, ...).' },
{ key: 'azure', label: 'Azure / Intune device group', description: 'A device-group member export from Entra. Creates PCs by hostname and stores the deviceId.' },
{ key: 'thor', label: 'Thor PC inventory', description: 'A Thor inventory export. Maps hostname, vendor, model, OS, user, and location.' }
]
const importSource = ref('native')
// Optional computer type stamped onto every computer an Azure import creates or
// updates, so a SHP group becomes Shopfloor PCs and a BUS group Inspection PCs.
const azureComputerType = ref('')
const customComputerType = ref('')
const customTypeError = ref('')
const addingType = ref(false)
const computerTypes = ref([])
const azureDeviceCategory = ref('')
// Commit is allowed only after a clean preview of the CURRENT selection: a
// report with no problems and no request-level error that has not already been
// committed.
@@ -329,6 +484,17 @@ const allProblems = computed(() => {
return out
})
const allWarnings = computed(() => {
if (!report.value?.files) return []
const out = []
for (const file of report.value.files) {
for (const warning of (file.warnings || [])) {
out.push({ filename: file.filename, ...warning })
}
}
return out
})
const allReconciliations = computed(() => {
if (!report.value?.files) return []
const out = []
@@ -440,10 +606,26 @@ function clearSelection() {
if (fileInput.value) fileInput.value.value = ''
}
function selectSource(key) {
importSource.value = key
// Switching format invalidates a preview built against the previous adapter.
report.value = null
importError.value = ''
}
async function runImport(commit) {
importError.value = ''
try {
const response = await settingsApi.importAssetCsvs(selectedFiles.value, commit)
const options = { source: importSource.value }
if (importSource.value === 'azure') {
if (azureComputerType.value && azureComputerType.value !== '__custom__') {
options.computertype = azureComputerType.value.trim()
}
if (azureDeviceCategory.value.trim()) {
options.category = azureDeviceCategory.value.trim()
}
}
const response = await settingsApi.importAssetCsvs(selectedFiles.value, commit, options)
report.value = response.data?.data || response.data || null
} catch (err) {
importError.value = apiError(err, commit ? 'Import failed.' : 'Preview failed.')
@@ -469,9 +651,38 @@ async function commit() {
}
}
async function addComputerType() {
const name = customComputerType.value.trim()
if (!name) return
customTypeError.value = ''
addingType.value = true
try {
const response = await computersApi.types.create({ computertype: name })
const created = response.data?.data || response.data
await loadComputerTypes()
// Select the freshly added type by its name.
azureComputerType.value = created?.computertype || name
customComputerType.value = ''
} catch (err) {
customTypeError.value = apiError(err, 'Could not add the computer type.')
} finally {
addingType.value = false
}
}
async function loadComputerTypes() {
try {
const response = await computersApi.types.list({ per_page: 200 })
computerTypes.value = response.data?.data || response.data || []
} catch {
computerTypes.value = []
}
}
onMounted(async () => {
await loadLocation()
await loadTemplates()
await loadComputerTypes()
})
</script>
@@ -540,6 +751,75 @@ onMounted(async () => {
color: var(--text);
font-family: var(--font-mono, monospace);
}
.field select {
padding: 8px 10px;
border: 1px solid var(--border);
border-radius: 6px;
background: var(--bg);
color: var(--text);
}
.custom-type-row {
display: flex;
gap: 8px;
align-items: center;
}
.custom-type-row input {
flex: 1;
}
.source-cards {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 10px;
margin: 4px 0;
}
.source-card {
display: flex;
flex-direction: column;
gap: 4px;
text-align: left;
padding: 12px 14px;
border: 1px solid var(--border);
border-radius: 8px;
background: var(--bg);
color: var(--text);
cursor: pointer;
}
.source-card:hover {
border-color: var(--primary);
}
.source-card.active {
border-color: var(--primary);
box-shadow: 0 0 0 1px var(--primary) inset;
}
.source-card-title {
font-weight: 600;
}
.source-card-desc {
font-size: 0.85rem;
color: var(--text-light);
}
.source-help {
margin-top: 12px;
border-left: 3px solid var(--border);
padding-left: 12px;
}
.source-help .instructions {
margin-top: 8px;
}
.source-help .instructions summary {
cursor: pointer;
font-weight: 600;
color: var(--text);
}
.source-help .instructions ol {
margin: 8px 0 0;
padding-left: 20px;
color: var(--text-light);
font-size: 0.9rem;
}
.source-help .instructions li {
margin: 4px 0;
}
.actions {
display: flex;
align-items: center;
@@ -641,6 +921,33 @@ onMounted(async () => {
margin: 4px 0;
font-size: 0.9rem;
}
.report-table td.has-warnings {
color: var(--amber, #d18b16);
font-weight: 700;
}
.report-banner.amber {
background: var(--amber-bg, rgba(209, 139, 22, 0.15));
color: var(--amber, #d18b16);
}
.warnings {
margin-top: 12px;
}
.warnings h4 {
margin: 0 0 6px;
color: var(--amber, #d18b16);
}
.warnings ul {
margin: 6px 0 0;
padding-left: 20px;
}
.warnings li {
margin: 4px 0;
font-size: 0.9rem;
}
.warnings li em {
color: var(--text-muted, #9aa4b2);
font-style: normal;
}
.problems {
margin-top: 12px;
}

8
plugins/computers/api/routes.py
View File

@@ -582,6 +582,9 @@ def create_computer():
vendorid=data.get('vendorid'),
modelnumberid=data.get('modelnumberid'),
loggedinuser=data.get('loggedinuser'),
azuredeviceid=data.get('azuredeviceid'),
azureobjectid=data.get('azureobjectid'),
azuredevicecategory=data.get('azuredevicecategory'),
lastreporteddate=data.get('lastreporteddate'),
lastboottime=data.get('lastboottime')
)
@@ -686,8 +689,9 @@ def update_computer(computer_id: int):
# Update computer fields
computer_fields = ['computertypeid', 'hostname', 'osid', 'vendorid',
'modelnumberid', 'loggedinuser', 'lastreporteddate',
'lastboottime']
'modelnumberid', 'loggedinuser', 'azuredeviceid',
'azureobjectid', 'azuredevicecategory',
'lastreporteddate', 'lastboottime']
for key in computer_fields:
if key in data:
old_val = getattr(comp, key)

22
plugins/computers/frontend/views/PCDetail.vue
View File

@@ -152,6 +152,28 @@
</div>
</div>
</div>
<!-- Entra / Azure device info -->
<div
v-if="computer.computer?.azuredeviceid || computer.computer?.azureobjectid || computer.computer?.azuredevicecategory"
class="section-card"
>
<h3 class="section-title">Entra Device Info</h3>
<div class="info-list">
<div v-if="computer.computer?.azuredeviceid" class="info-row">
<span class="info-label">Device ID</span>
<span class="info-value">{{ computer.computer.azuredeviceid }}</span>
</div>
<div v-if="computer.computer?.azureobjectid" class="info-row">
<span class="info-label">Object ID</span>
<span class="info-value">{{ computer.computer.azureobjectid }}</span>
</div>
<div v-if="computer.computer?.azuredevicecategory" class="info-row">
<span class="info-label">Device Category</span>
<span class="info-value">{{ computer.computer.azuredevicecategory }}</span>
</div>
</div>
</div>
</div>
<!-- Right Column -->

14
plugins/computers/frontend/views/PCForm.vue
View File

@@ -223,6 +223,17 @@
</div>
</div>
<div class="form-group">
<label for="azuredeviceid">Azure Device ID</label>
<input
id="azuredeviceid"
v-model="form.azuredeviceid"
type="text"
class="form-control"
placeholder="Entra/Azure deviceId GUID (set by Azure imports)"
/>
</div>
<div class="form-group">
<label>Remote Access Protocols</label>
<div class="protocol-list">
@@ -383,6 +394,7 @@ const form = ref({
locationid: '',
osid: '',
loggedinuser: '',
azuredeviceid: '',
accessmethods: [],
notes: '',
mapx: null,
@@ -486,6 +498,7 @@ onMounted(async () => {
locationid: pc.locationid || '',
osid: ext.osid || '',
loggedinuser: ext.loggedinuser || '',
azuredeviceid: ext.azuredeviceid || '',
accessmethods: (pc.accessmethods || []).map(a => ({
protocolid: a.protocolid,
portoverride: a.portoverride ?? null
@@ -555,6 +568,7 @@ async function savePC() {
locationid: form.value.locationid || null,
osid: form.value.osid || null,
loggedinuser: form.value.loggedinuser || null,
azuredeviceid: form.value.azuredeviceid || null,
accessmethods: form.value.accessmethods,
notes: form.value.notes || null,
ipaddress: form.value.ipaddress || null,

40
plugins/computers/migrations/versions/0002_computer_azuredeviceid.py Normal file
View File

@@ -0,0 +1,40 @@
"""Add azuredeviceid to computers (Entra/Azure device-group imports).
The core chain created computers WITHOUT an azuredeviceid column. This plugin
revision adds it so BOTH fresh installs (core chain builds the table, then this
adds the column) and existing installs get it. Guarded/idempotent: skips when
the table is absent (plugin disabled) or the column already exists (e.g. a test
DB built by db.create_all() from the model).
Revision ID: computers0002azureid
Revises: computers0001anchor
"""
from alembic import op
import sqlalchemy as sa
revision = 'computers0002azureid'
down_revision = 'computers0001anchor'
branch_labels = None
depends_on = None
def upgrade():
bind = op.get_bind()
insp = sa.inspect(bind)
if 'computers' not in insp.get_table_names():
return
cols = {c['name'] for c in insp.get_columns('computers')}
if 'azuredeviceid' not in cols:
op.add_column('computers',
sa.Column('azuredeviceid', sa.String(length=64), nullable=True))
def downgrade():
bind = op.get_bind()
insp = sa.inspect(bind)
if 'computers' not in insp.get_table_names():
return
cols = {c['name'] for c in insp.get_columns('computers')}
if 'azuredeviceid' in cols:
op.drop_column('computers', 'azuredeviceid')

45
plugins/computers/migrations/versions/0003_computer_azure_fields.py Normal file
View File

@@ -0,0 +1,45 @@
"""Add azureobjectid and azuredevicecategory to computers.
The Azure device-group import now also records the Entra directory object id
(the CSV 'id' column) and an operator-supplied device category (the group
membership label, e.g. SFLD_DYN_Device_Group_...). This revision adds both
columns. Guarded/idempotent: skips when the table is absent (plugin disabled)
or a column already exists (e.g. a test DB built by db.create_all()).
Revision ID: computers0003azurefields
Revises: computers0002azureid
"""
from alembic import op
import sqlalchemy as sa
revision = 'computers0003azurefields'
down_revision = 'computers0002azureid'
branch_labels = None
depends_on = None
def upgrade():
bind = op.get_bind()
insp = sa.inspect(bind)
if 'computers' not in insp.get_table_names():
return
cols = {c['name'] for c in insp.get_columns('computers')}
if 'azureobjectid' not in cols:
op.add_column('computers',
sa.Column('azureobjectid', sa.String(length=64), nullable=True))
if 'azuredevicecategory' not in cols:
op.add_column('computers',
sa.Column('azuredevicecategory', sa.String(length=255), nullable=True))
def downgrade():
bind = op.get_bind()
insp = sa.inspect(bind)
if 'computers' not in insp.get_table_names():
return
cols = {c['name'] for c in insp.get_columns('computers')}
if 'azuredevicecategory' in cols:
op.drop_column('computers', 'azuredevicecategory')
if 'azureobjectid' in cols:
op.drop_column('computers', 'azureobjectid')

17
plugins/computers/models/computer.py
View File

@@ -85,6 +85,23 @@ class Computer(BaseModel):
lastreporteddate = db.Column(db.DateTime, nullable=True)
lastboottime = db.Column(db.DateTime, nullable=True)
# Entra/Azure device identity (set by Azure device-group imports)
azuredeviceid = db.Column(
db.String(64),
nullable=True,
comment='Entra/Azure deviceId GUID'
)
azureobjectid = db.Column(
db.String(64),
nullable=True,
comment='Entra/Azure directory object id GUID'
)
azuredevicecategory = db.Column(
db.String(255),
nullable=True,
comment='Entra device category / group membership label set at import'
)
# Remote access is now modeled per-protocol via the accessmethods
# relationship (AccessProtocol / ComputerAccess), replacing isvnc/iswinrm.

284
plugins/computers/plugin.py
View File

@@ -34,6 +34,14 @@ MEASURINGTOOL_LINK_ORIGIN = 'collector:measuringtool'
# touched.
MACHINE_LINK_ORIGIN = 'collector:machine'
# Marker stamped on the PC->machine link built from a CSV import's machine
# number column. A DISTINCT origin from the collector's so the two provenances
# stay separable: an import establishes a link once from a spreadsheet, the
# collector maintains it continuously from the floor, and a site may use one,
# the other, or both. Same archive discipline - only rows carrying this label
# are touched by a re-import.
MACHINE_LINK_IMPORT_ORIGIN = 'import:machine'
# Marker stamped on both links a part-marker PC produces: PC controls marker,
# and marker partof the operation it serves. Same archive discipline again.
PARTMARKER_LINK_ORIGIN = 'collector:partmarker'
@@ -90,8 +98,224 @@ class ComputersPlugin(BasePlugin):
def init_app(self, app: Flask, db_instance) -> None:
"""Initialize plugin with Flask app."""
self._register_import_hook()
logger.info(f"Computers plugin initialized (v{self.meta.version})")
# -- bulk-import machine link -------------------------------------------
def _register_import_hook(self):
"""Own a `machineassetnumber` column on computers.csv.
A site with a spreadsheet knows "this PC drives machine 3015" and wants
to write that number in the PC row, not hand-build an assetrelationships
sheet keyed on ids it never sees. The core importer stays ignorant of
what the column means: it accepts the header, keeps it out of the
computers write, and hands the value to this hook after each PC row is
written, so the PC's asset id exists. Registration is idempotent, so a
plugin reload or a test re-init does not double-fire.
"""
from shopdb.api import register_import_hook
register_import_hook(
'computers',
{'machineassetnumber':
'the asset number of the machine this PC drives (from '
'machines.csv); builds the PC-controls-machine link. A number '
'that has no machine, or one already driven by another PC, is '
'reported as a warning and skipped - the PC still imports.'},
self._import_machine_link)
def _import_machine_link(self, rowvalues, virtualvalues, line):
"""Build a PC->machine controls link from an import row's machine number.
Runs inside the importer's transaction - a dry run rolls it back - so it
reuses the collector's mature _sync_machine_link rather than inventing a
second linker, but under the import origin (so an import never archives
a collector's rows and vice versa) and in preview mode (so a bulk import
never emails an alert per row; the findings are the report's amber
warnings instead).
Returns RowWarning list, never raises: a machine number that resolves to
no asset is an amber note beside a PC that still imports, not a failed
file. Every warning names the PC, the number, and the one fix.
"""
from shopdb.api import RowWarning
machinenumber = (virtualvalues.get('machineassetnumber') or '').strip()
if not machinenumber:
return []
assetid = rowvalues.get('assetid')
comp = Computer.query.filter_by(assetid=assetid).first() if assetid else None
if comp is None:
# The PC row was written but its Computer extension is not readable
# here - nothing to link to. Amber, not fatal.
return [RowWarning(
line, 'machineassetnumber',
reason='PC row not found to link to machine {!r}'.format(
machinenumber),
action='re-run the import; if it recurs, check the computers '
'row imported',
target=machinenumber)]
hostname = comp.hostname or (assetid and str(assetid)) or '?'
# IN-FILE FAN-OUT GUARD. Two rows in one computers.csv both naming the
# same machine is a spreadsheet mistake - a copied cell, a wrong
# workstation number - not a swap. But _incumbent_has_yielded reads a
# freshly-imported PC (no lastreporteddate) as having yielded, so
# without this the SECOND row would silently archive the first row's
# link and steal the machine, and the file would import clean with the
# wrong PC on the bay. So the first claim on a machine this run wins and
# is remembered; a later row for the same machine is warned and skipped.
# The claim set lives on db.session.info, which Flask-SQLAlchemy resets
# at request teardown, so it is scoped to exactly this one import and
# never leaks into the next.
machine = self._asset_by_number(machinenumber)
if machine is not None and machine.assetid == assetid:
# The number names this very PC. Happens on a PC first seen before
# the machine-number convention, whose own asset number IS the
# machine number. Skip - a PC does not control itself.
return [RowWarning(
line, 'machineassetnumber',
reason='machine number {!r} is this PC\'s own asset number; '
'not linked'.format(machinenumber),
action='give this PC a hostname-based asset number, or point '
'machineassetnumber at the machine it drives',
subject=hostname, target=machinenumber)]
if machine is not None:
# A location-only machine is a virtual bay marker, not a real
# machine, so nothing controls it. Skip rather than build a link
# that would read as a PC driving a location.
skip = self._location_only_skip(
machine, hostname, machinenumber, line)
if skip is not None:
return [skip]
if machine is not None and machine.assetid != assetid:
# ALREADY-CONTROLS-DIFFERENT. Unlike the collector - which reads a
# new machine number as a PC moved to a new bay and silently
# re-bays it - an import naming a machine when this PC already
# actively controls a DIFFERENT one is far more likely a wrong
# workstation number in the sheet than a real move. Warn and skip
# rather than archive a link a person may have set on purpose.
other = self._active_controlled_machine(assetid, machine.assetid)
if other is not None:
return [RowWarning(
line, 'machineassetnumber',
reason='{} already controls machine {}; not moved to {}'
.format(hostname, other, machine.assetnumber),
action='if this PC really moved, clear the existing link '
'first; otherwise correct the machine number on '
'this row',
subject=hostname, target=machinenumber)]
if machine is not None and machine.assetid != assetid:
claimed = db.session.info.setdefault('import_machine_claims', {})
# Keyed by machine asset id -> (claimant PC asset id, its hostname).
# The id is what the guard compares on (the same PC re-stating its
# own link is not a conflict); the hostname is only for the message.
firstclaimant = claimed.get(machine.assetid)
if firstclaimant is not None and firstclaimant[0] != assetid:
return [RowWarning(
line, 'machineassetnumber',
reason='machine {} was already claimed by {} earlier in '
'this import; {} not linked'.format(
machine.assetnumber, firstclaimant[1], hostname),
action='a machine drives one PC - correct the machine '
'number on one of these rows so each PC names its '
'own machine',
subject=hostname, target=machinenumber)]
claimed[machine.assetid] = (assetid, hostname)
# _sync_machine_link appends plain strings to its warnings list. Collect
# them and translate to structured, actionable RowWarnings so the UI can
# show the PC, the machine number, and the corrective step.
rawwarnings = []
self._sync_machine_link(
comp, machinenumber, rawwarnings,
origin_label=MACHINE_LINK_IMPORT_ORIGIN, preview=True)
out = []
for text in rawwarnings:
out.append(RowWarning(
line, 'machineassetnumber',
reason=text,
action=self._machine_link_action(text, machinenumber),
subject=hostname, target=machinenumber))
return out
@staticmethod
def _active_controlled_machine(pcassetid, excludeassetid):
"""Asset number of a machine this PC actively controls under the import
origin, other than excludeassetid, or None.
Scoped to MACHINE_LINK_IMPORT_ORIGIN so an import only reasons about its
own links: a collector link or a hand-made one is the collector's or a
person's to move, not the import's to block on.
"""
from shopdb.api import AssetRelationship, RelationshipType, Asset
controls = RelationshipType.query.filter_by(
relationshiptype='controls').first()
if not controls:
return None
rel = AssetRelationship.query.filter(
AssetRelationship.sourceassetid == pcassetid,
AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
AssetRelationship.label == MACHINE_LINK_IMPORT_ORIGIN,
AssetRelationship.targetassetid != excludeassetid,
AssetRelationship.isactive.is_(True),
).first()
if rel is None:
return None
asset = db.session.get(Asset, rel.targetassetid)
return asset.assetnumber if asset else str(rel.targetassetid)
@staticmethod
def _location_only_skip(machine, hostname, machinenumber, line):
"""A RowWarning if the resolved asset is a location-only marker, else None.
A machine flagged islocationonly is a virtual bay, not a real machine,
so a PC does not control it. Guarded behind the machines plugin being
installed - a site without it cannot have a location-only machine, and
the import must not fail because the model is absent.
"""
from shopdb.api import RowWarning
try:
from plugins.machines.models import Machine
except ImportError:
return None
row = Machine.query.filter_by(assetid=machine.assetid).first()
if row is None or not row.islocationonly:
return None
return RowWarning(
line, 'machineassetnumber',
reason='machine {} is a location-only marker, not a real machine; '
'not linked'.format(machine.assetnumber),
action='point machineassetnumber at a real machine, or clear it '
'for this PC',
subject=hostname, target=machinenumber)
@staticmethod
def _machine_link_action(text, machinenumber):
"""The one corrective step for a machine-link warning, by its shape."""
lowered = text.lower()
if 'no asset for machine number' in lowered:
return ('create the machine asset with number {!r} (import '
'machines.csv first), or correct the number on this PC '
'row'.format(machinenumber))
if 'still held by' in lowered:
return ('confirm which PC drives this machine; retire or move the '
'old PC off In Use to hand the machine over')
if 'taken over from' in lowered:
return ('expected on a PC swap; set the old PC to Retired, '
'Inventory or In Repair')
if 'relationship type missing' in lowered:
return 'run flask seed reference-data, then re-import'
return 'review this PC and its machine number'
# -- ADR-006 collector contract -----------------------------------------
def get_collector_schema(self) -> Optional[Dict]:
@@ -647,7 +871,8 @@ class ComputersPlugin(BasePlugin):
# -- measuring-tool sync -----------------------------------------------
def _sync_machine_link(self, comp, machinenumber, warnings):
def _sync_machine_link(self, comp, machinenumber, warnings,
origin_label=MACHINE_LINK_ORIGIN, preview=False):
"""Link a PC to the machine it drives, from the reported machine number.
Until this existed the machine number was collected and then discarded,
@@ -669,6 +894,15 @@ class ComputersPlugin(BasePlugin):
cadence, alerting on every pass. The incumbent therefore keeps the
machine while it is still alive, and the challenger is recorded as a
dormant link. See _incumbent_has_yielded for what alive means.
origin_label separates provenance: the collector stamps
MACHINE_LINK_ORIGIN, a CSV import stamps MACHINE_LINK_IMPORT_ORIGIN, and
each only ever archives its own rows so the two sources coexist.
preview=True is for a dry-run import: DB writes still happen in the
caller's transaction (which it rolls back), but the alert fan-out - the
one side effect a rollback cannot undo - is suppressed so a preview
never emails anyone.
"""
from shopdb.api import AssetRelationship, RelationshipType, Asset
@@ -700,7 +934,7 @@ class ComputersPlugin(BasePlugin):
mine = AssetRelationship.query.filter(
AssetRelationship.sourceassetid == pcasset.assetid,
AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
AssetRelationship.label == MACHINE_LINK_ORIGIN,
AssetRelationship.label == origin_label,
).all()
link = None
@@ -710,11 +944,23 @@ class ComputersPlugin(BasePlugin):
else:
rel.isactive = False
# The unique constraint on assetrelationships is (source, target, type)
# and does NOT include label, so an import (import:machine) and the
# collector (collector:machine) cannot each hold their OWN row for the
# same PC->machine control. If our label-scoped lookup found nothing but
# a row for this exact triple exists under the other provenance, adopt
# it rather than insert a duplicate that would violate the constraint.
if link is None:
link = AssetRelationship.query.filter_by(
sourceassetid=pcasset.assetid,
targetassetid=machine.assetid,
relationshiptypeid=controls.relationshiptypeid).first()
# Whoever actively holds this machine now, if it is not this PC.
held = AssetRelationship.query.filter(
AssetRelationship.targetassetid == machine.assetid,
AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
AssetRelationship.label == MACHINE_LINK_ORIGIN,
AssetRelationship.label == origin_label,
AssetRelationship.sourceassetid != pcasset.assetid,
AssetRelationship.isactive.is_(True),
).all()
@@ -737,9 +983,10 @@ class ComputersPlugin(BasePlugin):
sourceassetid=pcasset.assetid,
targetassetid=machine.assetid,
relationshiptypeid=controls.relationshiptypeid,
label=MACHINE_LINK_ORIGIN,
label=origin_label,
isactive=False))
self._alert_machine_contested(names, comp, machine)
if not preview:
self._alert_machine_contested(names, comp, machine)
else:
link.isactive = False
return [{'assetid': machine.assetid,
@@ -753,7 +1000,7 @@ class ComputersPlugin(BasePlugin):
sourceassetid=pcasset.assetid,
targetassetid=machine.assetid,
relationshiptypeid=controls.relationshiptypeid,
label=MACHINE_LINK_ORIGIN,
label=origin_label,
isactive=True)
db.session.add(link)
else:
@@ -767,7 +1014,8 @@ class ComputersPlugin(BasePlugin):
warnings.append(
'machine {} was taken over from {}; check that PC'.format(
machine.assetnumber, oldname))
self._alert_pc_superseded(oldname, comp, machine)
if not preview:
self._alert_pc_superseded(oldname, comp, machine)
return [{'assetid': machine.assetid,
'machinenumber': machine.assetnumber,
@@ -870,8 +1118,8 @@ class ComputersPlugin(BasePlugin):
#
# A row made by hand carries neither label and is deliberately NOT
# counted: a person's link is not the collector's to archive.
ourlabels = (label, MACHINE_LINK_ORIGIN, MEASURINGTOOL_LINK_ORIGIN,
PARTMARKER_LINK_ORIGIN)
ourlabels = (label, MACHINE_LINK_ORIGIN, MACHINE_LINK_IMPORT_ORIGIN,
MEASURINGTOOL_LINK_ORIGIN, PARTMARKER_LINK_ORIGIN)
others = AssetRelationship.query.filter(
AssetRelationship.targetassetid == deviceasset.assetid,
AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
@@ -1423,15 +1671,17 @@ class ComputersPlugin(BasePlugin):
sourceassetid=pcasset.assetid,
targetassetid=adopted.assetid,
relationshiptypeid=controls.relationshiptypeid).first()
if claimed is not None and claimed.label == MACHINE_LINK_ORIGIN:
if claimed is not None and claimed.label in (
MACHINE_LINK_ORIGIN, MACHINE_LINK_IMPORT_ORIGIN):
# LEAVE IT ALONE. On a CMM the instrument IS the reported bay,
# so the machine sync has already made the row for this exact
# triple - and it finds its own rows BY that label. Relabelling
# hid the link from it, so the next cycle built a second row for
# the same triple and MySQL rejected it: the PC reported 200,
# then 500 forever after. The row's lifecycle (including staying
# dormant while a bay is contested) belongs to the machine sync;
# adoption only needs the identity, which it has.
# so the machine sync (or a CSV import's machine link) has
# already made the row for this exact triple - and each finds
# its own rows BY that label. Relabelling hid the link from it,
# so the next cycle built a second row for the same triple and
# MySQL rejected it: the PC reported 200, then 500 forever
# after. The row's lifecycle (including staying dormant while a
# bay is contested) belongs to the machine sync; adoption only
# needs the identity, which it has.
pass
elif claimed is not None:
# Relabelling is what adoption MEANS for any other row: the link

156
scripts/reconcile_locations.py Normal file
View File

@@ -0,0 +1,156 @@
"""Generate reconciliation SQL for the locations table.
Reads a TSV export (locationid, locationname, description, locationtypeid,
parentlocationid) and emits a single transaction-wrapped SQL script that:
- Treats bare grid coordinates ([A-V][0-9]{1,2}W? and LD[0-9]+W?) as the
canonical identity for a cell.
- Folds coordinate-plus-suffix rows into the canonical coordinate, moving the
suffix text into the canonical row's description (never the name).
- Folds named-area suffix variants ("NDE - 6409") into the parent named area
("NDE"); named areas are kept as-is and never forced into coordinates.
- Repoints assets.locationid, subnets.locationid and child
locations.parentlocationid onto the canonical row, then deletes duplicates.
Bare coordinates with no variants are left untouched. The trailing W (west
column, e.g. G2W, K3W) is part of the coordinate and is never stripped, so
G2W is kept distinct from a hypothetical G2.
"""
import re
import sys
COORD = re.compile(r'^(LD|[ABCDEFGHJKLMNPQRSTUV])(\d{1,2})(W)?(?=$|[ \-])')
def esc(s):
return s.replace("'", "''")
def load(path):
rows = []
with open(path, encoding='utf-8-sig') as fh:
for line in fh:
line = line.rstrip('\n').rstrip('\r')
if not line:
continue
parts = line.split('\t')
if len(parts) < 5:
parts += [''] * (5 - len(parts))
lid, name, desc, typeid, parentid = parts[:5]
rows.append({
'id': int(lid),
'name': name,
'desc': desc,
'typeid': typeid,
'parentid': parentid,
})
return rows
def classify(name):
"""Return (canonical, suffix). canonical is the bare coordinate or named
parent; suffix is the trailing detail (may be '')."""
m = COORD.match(name)
if m:
prefix, digits, west = m.group(1), m.group(2), m.group(3) or ''
canonical = f"{prefix}{digits}{west}"
suffix = name[m.end():].strip()
suffix = re.sub(r'^[\-\s]+', '', suffix).strip()
return canonical, suffix, 'coord'
# named area: fold on ' - ' separator only
if ' - ' in name:
base, _, rest = name.partition(' - ')
return base.strip(), rest.strip(), 'named'
return name.strip(), '', 'named'
def main(tsv, out):
rows = load(tsv)
# Build canonical groups. Key is casefolded canonical for grouping, but we
# preserve a display canonical (prefer an existing exact bare/standalone).
groups = {}
for r in rows:
canonical, suffix, kind = classify(r['name'])
key = (kind, canonical.casefold())
g = groups.setdefault(key, {
'canonical': canonical,
'kind': kind,
'members': [],
'exact_id': None,
})
r['_suffix'] = suffix
r['_canonical'] = canonical
g['members'].append(r)
# exact row: name equals canonical (no suffix)
if suffix == '' and r['name'].strip() == canonical:
g['exact_id'] = r['id']
g['canonical'] = r['name'].strip()
lines = []
lines.append('-- Auto-generated locations reconciliation. Review before running.')
lines.append('START TRANSACTION;')
touched = 0
for key, g in sorted(groups.items(), key=lambda kv: kv[0][1]):
members = sorted(g['members'], key=lambda r: r['id'])
# nothing to do: single member that is already the exact canonical
if len(members) == 1 and g['exact_id'] == members[0]['id']:
continue
touched += 1
canonical = g['canonical']
# choose target id: prefer exact bare row, else promote smallest id
if g['exact_id'] is not None:
target = g['exact_id']
promote = None
else:
target = members[0]['id']
promote = members[0]
# accumulate suffix/description text from all members (dedup, ordered)
details = []
for m in members:
for chunk in (m['_suffix'], m['desc']):
chunk = (chunk or '').strip()
if chunk and chunk not in details:
details.append(chunk)
desc_val = '; '.join(details)
lines.append('')
lines.append(f"-- {canonical} (target locationid {target}) <= "
+ ', '.join(str(m['id']) for m in members))
if promote is not None:
lines.append(
f"UPDATE locations SET locationname='{esc(canonical)}' "
f"WHERE locationid={target};")
if desc_val:
lines.append(
f"UPDATE locations SET description='{esc(desc_val)}' "
f"WHERE locationid={target};")
others = [m['id'] for m in members if m['id'] != target]
if others:
ids = ','.join(str(i) for i in others)
lines.append(
f"UPDATE assets SET locationid={target} "
f"WHERE locationid IN ({ids});")
lines.append(
f"UPDATE subnets SET locationid={target} "
f"WHERE locationid IN ({ids});")
lines.append(
f"UPDATE locations SET parentlocationid={target} "
f"WHERE parentlocationid IN ({ids});")
lines.append(
f"DELETE FROM locations WHERE locationid IN ({ids});")
lines.append('')
lines.append('COMMIT;')
lines.append(f'-- groups touched: {touched}')
with open(out, 'w', encoding='utf-8') as fh:
fh.write('\n'.join(lines) + '\n')
print(f'wrote {out}: {touched} groups touched, {len(rows)} source rows')
if __name__ == '__main__':
main(sys.argv[1], sys.argv[2])

10
shopdb/__init__.py
View File

@@ -57,7 +57,15 @@ from .plugins import plugin_manager
# permission / empty / position. BREAKING for any plugin still using the old
# shape, which is why it is recorded here: the change itself shipped earlier
# without a bump, and a contract that changes silently is not a contract.
__contract_version__ = '0.20.0'
# 0.21.0: added register_import_hook to shopdb.api so a plugin can own a VIRTUAL
# COLUMN on an importable CSV table - a header the bulk importer accepts but
# never writes - and a post-write row hook that turns that value into a plugin
# concept. The computers plugin uses it for a machineassetnumber column on
# computers.csv that builds the PC->machine controls link, so a site writes the
# machine number in the PC row instead of hand-building a relationships sheet.
# Core stays ignorant of the column's meaning. Additive name on the import
# surface, minor bump.
__contract_version__ = '0.21.0'
# Product release version (see ADR-007). The product version and the
# plugin-contract version above are distinct series with independent

12
shopdb/api/__init__.py
View File

@@ -95,6 +95,15 @@ from shopdb.core.services.dualpath import (
dualpath_single_machine_enabled,
)
# Bulk-import extension point: a plugin registers a virtual column on an
# importable CSV table plus a post-write row hook, so a relationship a site
# would otherwise hand-build (a PC's machine number -> a controls link) can be
# written in the row itself. Core never learns what the column means.
from shopdb.core.services.csvimport import (
register_import_hook,
RowWarning,
)
# Legacy employee directory lookup (read-only) used by notifications
from shopdb.utils.employee_db import employee_connection
from shopdb.utils.mailer import send_email, send_alert, send_webhook
@@ -261,6 +270,9 @@ __all__ = [
'resolve_asset_position',
'resolve_dualpath_pairs',
'dualpath_single_machine_enabled',
# Bulk-import extension point
'register_import_hook',
'RowWarning',
# Infrastructure
'db',
'cache',

112
shopdb/core/api/settings.py
View File

@@ -6,6 +6,7 @@ import re
from flask import Blueprint, request, current_app
from flask_jwt_extended import jwt_required, get_jwt_identity
from werkzeug.utils import secure_filename
from sqlalchemy import select as sa_select
from shopdb.extensions import db, cache
from shopdb.core.models import Setting, AuditLog
@@ -332,6 +333,27 @@ def download_import_template(tablename):
return response
def _existing_hostname_map():
"""Case-insensitive hostname -> assetnumber map for existing computers.
Azure device-group exports align on hostname (their displayName), while the
import engine keys assets on assetnumber. This lets the Azure adapter route
a matched device to its real asset so the import updates in place.
"""
assets = db.metadata.tables['assets']
computers = db.metadata.tables['computers']
query = sa_select(
computers.c.hostname, assets.c.assetnumber
).select_from(
computers.join(assets, computers.c.assetid == assets.c.assetid)
).where(computers.c.hostname.isnot(None))
mapping = {}
for hostname, assetnumber in db.session.execute(query):
if hostname:
mapping[hostname.strip().lower()] = assetnumber
return mapping
@settings_bp.route('/asset-import/import', methods=['POST'])
@jwt_required()
@require_role('admin')
@@ -344,10 +366,16 @@ def import_asset_csvs():
transaction back and reports what WOULD change; commit=true persists it. The
whole run is all-or-nothing - if any file has a row problem, nothing is
written even on a commit.
A 'source' form field selects a foreign adapter (azure, thor); it defaults
to 'native' where uploads are already canonical per-table files. Foreign
uploads are transformed into canonical files first, then run identically.
"""
from sqlalchemy.exc import SQLAlchemyError
from shopdb.core.services.csvimport import (
ImportError_, Resolver, dependency_order, import_csv, table_from_filename)
from shopdb.core.services.importsources import (
ImportSourceError, get_source)
uploads = request.files.getlist('files')
if not uploads:
@@ -356,21 +384,66 @@ def import_asset_csvs():
commit = str(request.form.get('commit', '')).strip().lower() in (
'1', 'true', 'yes', 'on')
# Map each upload to a table by filename. A later file with the same table
# name wins, matching the folder-import behaviour (one file per table).
# A foreign source (azure, thor) is transformed into canonical files before
# import. 'native' (or nothing) means the files already ARE canonical.
source_key = (request.form.get('source') or 'native').strip().lower()
source = None
if source_key and source_key != 'native':
try:
source = get_source(source_key)
except ImportSourceError as exc:
return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
source_options = {
'computertype': (request.form.get('computertype') or '').strip(),
'category': (request.form.get('category') or '').strip(),
}
# Azure and Thor align on hostname: their device name is the ShopDB
# computers.hostname, not the assetnumber. Hand the adapter a
# case-insensitive hostname -> existing assetnumber map so devices that
# already exist update in place (keyed to their real asset) instead of
# minting a duplicate asset.
if source is not None and getattr(source, 'key', None) in ('azure', 'thor'):
source_options['hostname_map'] = _existing_hostname_map()
texts = {}
filenames = {}
for upload in uploads:
name = upload.filename or ''
if not name.lower().endswith('.csv'):
continue
table = table_from_filename(name)
try:
texts[table] = upload.read().decode('utf-8-sig')
except (UnicodeDecodeError, OSError):
return error_response(ErrorCodes.VALIDATION_ERROR,
'Could not read %s as UTF-8 text' % name)
filenames[table] = name
if source is not None:
# One foreign export becomes several canonical files. Each upload is
# transformed and its canonical outputs merged; a later file's table
# wins, matching the native one-file-per-table rule.
for upload in uploads:
name = upload.filename or ''
if not name.lower().endswith('.csv'):
continue
try:
raw = upload.read().decode('utf-8-sig')
except (UnicodeDecodeError, OSError):
return error_response(ErrorCodes.VALIDATION_ERROR,
'Could not read %s as UTF-8 text' % name)
try:
produced = source.transform(raw, options=source_options)
except ImportSourceError as exc:
return error_response(ErrorCodes.VALIDATION_ERROR,
'%s: %s' % (name, str(exc)))
for table, csv_text in produced.items():
texts[table] = csv_text
filenames[table] = '%s (from %s)' % (table, name)
else:
# Map each upload to a table by filename. A later file with the same
# table name wins, matching the folder-import behaviour.
for upload in uploads:
name = upload.filename or ''
if not name.lower().endswith('.csv'):
continue
table = table_from_filename(name)
try:
texts[table] = upload.read().decode('utf-8-sig')
except (UnicodeDecodeError, OSError):
return error_response(ErrorCodes.VALIDATION_ERROR,
'Could not read %s as UTF-8 text' % name)
filenames[table] = name
if not texts:
return error_response(ErrorCodes.VALIDATION_ERROR,
@@ -389,6 +462,7 @@ def import_asset_csvs():
resolver = Resolver(reconcile_threshold=reconcile_threshold)
files_report = []
problem_count = 0
warning_count = 0
reconciled_count = 0
request_error = None
@@ -409,7 +483,17 @@ def import_asset_csvs():
{'line': p.line, 'column': p.column, 'message': p.message}
for p in result.problems
]
# Amber tier: a warning is a finding beside a row that still
# imported (an unresolved machine link, a contested bay), NOT a
# reason to fail the file. Reported alongside problems so the UI can
# show Errors / Warnings / Info distinctly, but never gates commit.
warnings = [
{'line': w.line, 'column': w.column, 'reason': w.reason,
'action': w.action, 'subject': w.subject, 'target': w.target}
for w in result.warnings
]
problem_count += len(problems)
warning_count += len(warnings)
reconciled_count += len(result.reconciliations)
files_report.append({
'table': table,
@@ -417,6 +501,7 @@ def import_asset_csvs():
'created': result.created,
'updated': result.updated,
'problems': problems,
'warnings': warnings,
'reconciliations': result.reconciliations,
})
@@ -446,6 +531,7 @@ def import_asset_csvs():
'files': files_report,
'skipped': skipped,
'problem_count': problem_count,
'warning_count': warning_count,
'reconciled_count': reconciled_count,
'reconcile_threshold': reconcile_threshold,
'total_created': total_created,

233
shopdb/core/services/csvimport.py
View File

@@ -74,6 +74,71 @@ IMPORTABLE = OrderedDict([
('machines', {'label': None, 'natural': 'assetid'}),
])
# A small, INTERNALLY CONSISTENT worked example per table, in import order. A
# site can download the set, see the pattern, and keep it as a starter workbook:
# every name referenced by a later file (Bay 3, Dell, Computer, ...) is defined
# in an earlier one, so the whole set imports cleanly as a demo before a single
# real row is typed. Emitted as commented rows the engine skips, so a freshly
# downloaded template never writes example data by accident. Values avoid commas
# on purpose - the generator writes them without CSV quoting.
EXAMPLE_ROWS = OrderedDict([
('assetstatuses', [
{'status': 'In Use', 'description': 'Actively deployed', 'color': '#22c55e', 'isactive': '1'},
{'status': 'In Storage', 'description': 'Spare - not deployed', 'color': '#eab308', 'isactive': '1'},
{'status': 'Retired', 'description': 'Removed from service', 'color': '#ef4444', 'isactive': '1'},
]),
('assettypes', [
{'assettype': 'Computer', 'pluginname': 'computers', 'tablename': 'computers', 'description': 'PCs - servers - workstations', 'icon': 'laptop', 'color': '#3b82f6', 'isactive': '1'},
{'assettype': 'Machine', 'pluginname': 'machines', 'tablename': 'machines', 'description': 'Shop-floor machines', 'icon': 'cpu', 'color': '#8b5cf6', 'isactive': '1'},
]),
('locationtypes', [
{'locationtype': 'Building', 'description': 'A physical building', 'color': '#64748b', 'isactive': '1'},
{'locationtype': 'Bay', 'description': 'A shop-floor bay', 'color': '#0ea5e9', 'isactive': '1'},
]),
('modeltypes', [
{'modeltype': 'Desktop', 'category': 'Computer', 'description': 'Desktop PC chassis', 'icon': 'desktop', 'isactive': '1'},
{'modeltype': 'CNC Machine', 'category': 'Machine', 'description': 'CNC machining center', 'icon': 'cpu', 'isactive': '1'},
]),
('computertypes', [
{'computertype': 'Shopfloor PC', 'description': 'PC on the shop floor', 'icon': 'monitor', 'color': '#3b82f6', 'isactive': '1'},
{'computertype': 'Engineer Workstation', 'description': 'CAD/CAM workstation', 'icon': 'laptop', 'color': '#6366f1', 'isactive': '1'},
]),
('machinetypes', [
{'machinetype': 'CNC Mill', 'description': 'Computer numerical control mill', 'icon': 'cpu', 'color': '#8b5cf6', 'isactive': '1'},
{'machinetype': 'CMM', 'description': 'Coordinate measuring machine', 'icon': 'ruler', 'color': '#14b8a6', 'isactive': '1'},
]),
('businessunits', [
{'businessunit': 'Machining', 'code': 'MACH', 'description': 'Machining department', 'isactive': '1'},
{'businessunit': 'Quality', 'code': 'QUAL', 'description': 'Quality and metrology', 'isactive': '1'},
]),
('locations', [
{'locationname': 'Building 1', 'building': 'Building 1', 'floor': '1', 'description': 'Main manufacturing building', 'locationtypeid': 'Building', 'isactive': '1'},
{'locationname': 'Bay 3', 'building': 'Building 1', 'floor': '1', 'room': 'Machining bay 3', 'description': 'Machining bay 3', 'locationtypeid': 'Bay', 'isactive': '1'},
]),
('vendors', [
{'vendor': 'Dell', 'description': 'Computer hardware', 'website': 'https://www.dell.com', 'supportphone': '800-555-0100', 'supportemail': 'support@dell.com', 'isactive': '1'},
{'vendor': 'Haas Automation', 'description': 'CNC machine tools', 'website': 'https://www.haascnc.com', 'supportphone': '800-555-0199', 'isactive': '1'},
]),
('models', [
{'modelnumber': 'OptiPlex 7010', 'modeltypeid': 'Desktop', 'vendorid': 'Dell', 'description': 'Small form factor desktop', 'isactive': '1'},
{'modelnumber': 'VF-2', 'modeltypeid': 'CNC Machine', 'vendorid': 'Haas Automation', 'description': '40-taper vertical mill', 'isactive': '1'},
]),
('operatingsystems', [
{'osname': 'Windows 11 (24H2)', 'osversion': '24H2', 'architecture': 'x64', 'endoflife': '2027-10-12', 'isactive': '1'},
{'osname': 'Windows 10 (22H2)', 'osversion': '22H2', 'architecture': 'x64', 'endoflife': '2025-10-14', 'isactive': '1'},
]),
('assets', [
{'assetnumber': 'PC-001', 'name': 'Shopfloor PC 001', 'serialnumber': 'SN-PC-001', 'assettypeid': 'Computer', 'statusid': 'In Use', 'locationid': 'Bay 3', 'businessunitid': 'Machining', 'isactive': '1'},
{'assetnumber': 'CNC-014', 'name': 'Haas VF-2 #14', 'serialnumber': 'SN-CNC-014', 'assettypeid': 'Machine', 'statusid': 'In Use', 'locationid': 'Bay 3', 'businessunitid': 'Machining', 'isactive': '1'},
]),
('computers', [
{'assetid': 'PC-001', 'computertypeid': 'Shopfloor PC', 'hostname': 'WS-PC-001', 'osid': 'Windows 11 (24H2)', 'vendorid': 'Dell', 'modelnumberid': 'OptiPlex 7010', 'loggedinuser': 'jdoe', 'lastreporteddate': '2026-08-01', 'lastboottime': '2026-08-01', 'isactive': '1', 'machineassetnumber': 'CNC-014'},
]),
('machines', [
{'assetid': 'CNC-014', 'machinetypeid': 'CNC Mill', 'vendorid': 'Haas Automation', 'modelnumberid': 'VF-2', 'requiresmanualconfig': '0', 'islocationonly': '0', 'lastmaintenancedate': '2026-06-01', 'nextmaintenancedate': '2026-12-01', 'maintenanceintervaldays': '180', 'isactive': '1'},
]),
])
# Columns a CSV may never set: surrogate keys the database owns, and audit
# stamps. Accepting an id from a spreadsheet is how two sites end up with
# colliding primary keys the first time anyone merges data.
@@ -111,6 +176,75 @@ CORPORATE_SUFFIXES = {
DEFAULT_RECONCILE_THRESHOLD = 0.85
# -- plugin import hooks (inversion of control) ---------------------------------
#
# A spreadsheet is the wrong place to type an integer foreign key, and it is
# also the wrong place to split a relationship across two files: a site that
# knows "this PC drives machine 3015" wants to write that number in the PC row,
# not hand-build an assetrelationships sheet keyed on ids it never sees.
#
# But the machine link is a computers-plugin concept, and this module is core:
# it must not import a plugin. So a plugin REGISTERS itself here instead. It
# declares a VIRTUAL COLUMN - a header this table accepts but never writes to
# the database - and a HOOK that runs once per written row to turn that column's
# value into whatever the plugin means by it (here, a PC->machine controls link).
# Core stays ignorant of what the column is for; the plugin owns the meaning.
_VIRTUAL_COLUMNS = {} # tablename -> {columnname: description}
_ROW_HOOKS = {} # tablename -> [hook, ...]
def register_import_hook(tablename, virtual_columns, hook):
"""Let a plugin own virtual column(s) on an importable table.
virtual_columns: either a sequence of header names, or a mapping of
name -> one-line description. Each is accepted for ``tablename`` but never
written - it is pulled out of the row and handed to the hook instead of the
table. A description, when given, is shown in the generated template's
column notes so an operator knows what to type.
hook(rowvalues, virtualvalues, line) -> list[RowWarning]: called once per
written row, AFTER the row is inserted/updated, so its id exists. It runs
inside the caller's transaction - which a dry run rolls back - so it must
not commit, and must not do anything a rollback cannot undo (no emails). A
hook reports trouble by returning RowWarning, never by raising: a bad
machine number is an amber note beside a PC that still imports, not a failed
file.
Idempotent by contract: registering the same (table, column, hook) twice is
a no-op, so a plugin re-init during tests or a reload does not double-fire.
"""
cols = _VIRTUAL_COLUMNS.setdefault(tablename, {})
if isinstance(virtual_columns, dict):
for name, description in virtual_columns.items():
cols[name] = description or ''
else:
for name in virtual_columns:
cols.setdefault(name, '')
hooks = _ROW_HOOKS.setdefault(tablename, [])
# Dedupe by the underlying function's qualified name, not by the bound
# method: a plugin re-init (a new app context in the test suite, a reload in
# prod) makes a fresh instance whose bound method is != the old one, and a
# symlinked plugin can even be imported under two module paths, so an
# identity check would append a second hook and the row would be processed -
# and warned about - more than once.
def _identity(fn):
func = getattr(fn, '__func__', fn)
return getattr(func, '__qualname__', repr(func))
if _identity(hook) not in [_identity(existing) for existing in hooks]:
hooks.append(hook)
def virtual_columns_for(tablename):
"""Header names this table accepts but does not store. Never None."""
return set(_VIRTUAL_COLUMNS.get(tablename, ()))
def virtual_column_notes(tablename):
"""(name, description) for each virtual column, in registration order."""
return list(_VIRTUAL_COLUMNS.get(tablename, {}).items())
def normalize_label(value):
"""Fold a label to its comparison form: lowercase, no punctuation, no
corporate suffix, single-spaced. 'Dell Inc.' and 'DELL' both become 'dell'.
@@ -370,6 +504,32 @@ class RowProblem(object):
return "%s: %s" % (where, self.message)
class RowWarning(object):
"""A non-blocking, amber-tier finding.
Errors (RowProblem) stop the whole import; a RowWarning does NOT - the row
it belongs to still imports. It exists so a caveat (an unresolved or
conflicting machine link, say) is loud and REVIEWABLE without failing an
otherwise-clean file. Every warning names WHERE it happened, WHY, and the
single ACTION that fixes it, so a cold user can correct it in place - the
convention IT Glue / RMM imports train.
"""
def __init__(self, line, column, reason, action, subject=None, target=None):
self.line = line
self.column = column
self.reason = reason # what is wrong, in plain terms
self.action = action # the one corrective step to take
self.subject = subject # the row's actor (e.g. the PC hostname)
self.target = target # what it referenced (e.g. the machine number)
def __str__(self):
where = "line %d" % self.line
if self.column:
where += ", column '%s'" % self.column
return "%s: %s - %s" % (where, self.reason, self.action)
class ImportResult(object):
def __init__(self, tablename):
self.tablename = tablename
@@ -378,14 +538,18 @@ class ImportResult(object):
self.problems = []
self.skipped = 0
self.reconciliations = []
self.warnings = []
@property
def ok(self):
# Warnings are amber, not red: a file with only warnings is still ok to
# commit. Only problems (errors) block.
return not self.problems
def summary(self):
return "%s: %d new, %d updated, %d problem(s)" % (
self.tablename, self.created, self.updated, len(self.problems))
return "%s: %d new, %d updated, %d problem(s), %d warning(s)" % (
self.tablename, self.created, self.updated,
len(self.problems), len(self.warnings))
def read_rows(text):
@@ -439,9 +603,10 @@ def import_csv(tablename, text, resolver=None, commit=False):
header, rows = read_rows(text)
allowed = {c.name: c for c in importable_columns(tablename)}
unknown = [h for h in header if h not in allowed]
virtual = virtual_columns_for(tablename)
unknown = [h for h in header if h not in allowed and h not in virtual]
if unknown:
known = ', '.join(sorted(allowed))
known = ', '.join(sorted(set(allowed) | virtual))
result.problems.append(RowProblem(
1, None, "unknown column(s): %s\n this table accepts: %s" % (', '.join(unknown), known)))
return result
@@ -461,7 +626,17 @@ def import_csv(tablename, text, resolver=None, commit=False):
prepared = []
for line, raw in rows:
values = {}
virtualvalues = {}
failed = False
# Virtual columns belong to a plugin hook, not the table. Pull them out
# first so they never reach an insert/update, and stash them for the
# post-write hook. Whitespace-trimmed; an empty cell is dropped so the
# hook only sees values a row actually carried.
for name in virtual:
if name in raw:
text = (raw[name] or '').strip()
if text != '':
virtualvalues[name] = text
for name, column in allowed.items():
if name not in raw:
continue
@@ -496,16 +671,18 @@ def import_csv(tablename, text, resolver=None, commit=False):
if values.get(natural) in (None, ''):
result.problems.append(RowProblem(line, natural, 'must have a value'))
continue
prepared.append((line, values))
prepared.append((line, values, virtualvalues))
if not result.ok:
return result
hooks = _ROW_HOOKS.get(tablename, ())
# Second pass: decide insert vs update. Counted even on a dry run, because
# "this will change 4000 rows" is the number an operator needs before saying
# yes.
label = _label_column(tablename)
for line, values in prepared:
for line, values, virtualvalues in prepared:
existing = db.session.execute(
table.select().where(table.c[natural] == values[natural])).first()
if existing is None:
@@ -522,6 +699,15 @@ def import_csv(tablename, text, resolver=None, commit=False):
primary = list(table.primary_key)[0].name
resolver.remember(tablename, values[label], getattr(existing, primary))
# Fire plugin hooks AFTER the write, so the row's id exists. Only when
# the row actually carried a virtual value - a hook that has nothing to
# act on should never run. Warnings are collected, never raised: an
# unresolved machine number is amber, not a failed file.
if hooks and virtualvalues:
for hook in hooks:
result.warnings.extend(
hook(values, virtualvalues, line) or [])
return result
@@ -556,24 +742,55 @@ def generate_template(tablename):
bits.append('text, max %d' % column.type.length)
notes.append("# %-24s %s" % (column.name, ', '.join(bits)))
# Virtual columns a plugin owns: accepted on this table but written by the
# plugin's hook, not the schema. Appended after the real columns so the
# header carries them and the notes explain what they do.
virtualnames = []
for name, description in virtual_column_notes(tablename):
virtualnames.append(name)
header.append(name)
note = 'optional, ' + description if description else 'optional'
notes.append("# %-24s %s" % (name, note))
spec = IMPORTABLE[tablename]
examples = EXAMPLE_ROWS.get(tablename, [])
exampleorder = [c.name for c in columns] + virtualnames
lines = [
"# %s.csv - generated from the ShopDB schema, do not hand-maintain" % tablename,
"#",
"# Rows are matched on '%s'. Re-importing updates a matching row rather" % spec['natural'],
"# than creating a second one, so it is safe to correct this file and run again.",
"#",
"# HOW TO USE THIS TEMPLATE:",
"# 1. Delete everything above the header row (every line starting with #).",
"# 2. Fill in your own rows using the example row(s) below as a guide.",
"# 3. DELETE THE EXAMPLE DATA before you import - the example rows are",
"# live data and WILL be imported if you leave them in.",
"#",
"# Columns:",
] + notes + [
"#",
"# Lines starting with # are ignored. Delete the example row before importing.",
"# --- header and example rows follow. Keep the header; replace the examples. ---",
','.join(header),
'# ' + ','.join(_example(c, required) for c in columns),
] + [
','.join(_example_value(row.get(name, '')) for name in exampleorder)
for row in examples
]
return '\n'.join(lines) + '\n'
def _example_value(value):
"""A single example cell. Commas would break the unquoted CSV the template
writes, so they are replaced with a dash; example data is authored without
them, this is only a guard.
"""
return str(value).replace(',', ' -')
def _example(column, required):
"""Kept for callers/tests that ask for a single synthetic cell. The template
generator now emits the curated EXAMPLE_ROWS instead.
"""
target = _fk_target(column)
if target:
label = _label_column(target[0])

325
shopdb/core/services/importsources.py Normal file
View File

@@ -0,0 +1,325 @@
"""Foreign import-source adapters.
The CSV import engine in csvimport.py speaks exactly one dialect: the canonical
per-table files (assets.csv, computers.csv, vendors.csv, ...). Foreign exports -
an Entra/Azure device-group member list, a Thor PC inventory dump - do not match
that dialect and never will; they are owned by other systems.
Rather than teach the engine every foreign schema, each adapter here TRANSFORMS
a foreign export into the canonical files and hands them back as
{tablename: csv_text}. The caller then runs those through the ordinary
import_csv path, so dry-run/commit, dependency ordering, and reconciliation all
work unchanged.
An adapter declares a header signature so an upload can be auto-detected, and a
transform() that yields the canonical files. Adapters never touch the database.
"""
import csv
import io
class ImportSourceError(Exception):
"""A foreign file could not be transformed (bad header, empty, etc.)."""
# Canonical defaults for a discovered PC. Azure carries only a hostname, so
# every device it creates is a computer that is In Use until something richer
# (Thor, the collector) says otherwise.
DEFAULT_ASSETTYPE = 'computer'
DEFAULT_STATUS = 'In Use'
def _read(text):
"""Parse CSV text into (header, list-of-dicts), tolerating a UTF-8 BOM."""
stream = io.StringIO(text)
reader = csv.DictReader(stream)
header = [h.strip() for h in (reader.fieldnames or [])]
rows = []
for raw in reader:
rows.append({(k.strip() if k else k): (v or '').strip()
for k, v in raw.items()})
return header, rows
def _write(header, rows):
"""Render canonical rows back to CSV text with a trailing newline."""
out = io.StringIO()
writer = csv.DictWriter(out, fieldnames=header, extrasaction='ignore')
writer.writeheader()
for row in rows:
writer.writerow(row)
return out.getvalue()
def _has_columns(header, required):
lowered = {h.lower() for h in header}
return all(col.lower() in lowered for col in required)
class AzureDeviceGroupSource:
"""Entra/Azure device-group member export.
Columns: displayName, objectType, mail, userType, id, deviceId,
userPrincipalName, isUser, isGroup, isGuest. Only device rows carry signal;
displayName is the hostname and deviceId is the Entra device GUID. The export
has no model, OS, location or user data, so this creates/updates minimal
asset + computer rows keyed on hostname.
options:
computertype - optional name stamped onto every computer's computertypeid
(e.g. 'Shopfloor PC' for a SHP group, 'Inspection PC' for
BUS). Resolved by the engine against computertypes.
category - optional Entra device category / group membership label
stamped onto every computer's azuredevicecategory (e.g.
SFLD_DYN_Device_Group_Wilmington_BUS_NC_COUNTRY:USA).
hostname_map - optional {lowercased hostname: existing assetnumber}. A
device whose displayName matches an existing computer's
hostname is routed to that asset and updated in place
(no new asset row); unmatched devices create a new
asset + computer keyed on the hostname.
"""
key = 'azure'
label = 'Azure device group'
signature = ('displayName', 'deviceId', 'objectType')
@classmethod
def matches(cls, header):
return _has_columns(header, cls.signature)
@classmethod
def transform(cls, text, options=None):
options = options or {}
computertype = (options.get('computertype') or '').strip()
category = (options.get('category') or '').strip()
hostname_map = options.get('hostname_map') or {}
header, rows = _read(text)
if not _has_columns(header, cls.signature):
raise ImportSourceError(
'not an Azure device-group export (expected displayName, '
'deviceId, objectType columns)')
assets = []
computers = []
seen = set()
for row in rows:
objecttype = (row.get('objectType') or '').strip().lower()
if objecttype and objecttype != 'device':
continue
hostname = (row.get('displayName') or '').strip()
if not hostname or hostname.lower() in seen:
continue
seen.add(hostname.lower())
# Align on hostname: if this device already exists in ShopDB, point
# the computer row at its real asset so the engine updates it in
# place. Only create a new asset when there is no hostname match.
existing_assetnumber = hostname_map.get(hostname.lower())
computer = {
'assetid': existing_assetnumber if existing_assetnumber is not None else hostname,
'hostname': hostname,
'azuredeviceid': (row.get('deviceId') or '').strip(),
'azureobjectid': (row.get('id') or '').strip(),
}
if computertype:
computer['computertypeid'] = computertype
if category:
computer['azuredevicecategory'] = category
computers.append(computer)
if existing_assetnumber is None:
assets.append({
'assetnumber': hostname,
'name': hostname,
'assettypeid': DEFAULT_ASSETTYPE,
'statusid': DEFAULT_STATUS,
})
if not computers:
raise ImportSourceError('no device rows found in the Azure export')
asset_header = ['assetnumber', 'name', 'assettypeid', 'statusid']
computer_header = ['assetid', 'hostname', 'azuredeviceid', 'azureobjectid']
if computertype:
computer_header.append('computertypeid')
if category:
computer_header.append('azuredevicecategory')
produced = {
'computers': _write(computer_header, computers),
}
# Only emit an assets file when there are new devices to create; an
# empty assets.csv would just be an extra clean no-op table.
if assets:
produced['assets'] = _write(asset_header, assets)
return produced
class ThorSource:
"""Thor PC-inventory export (internal GE Aerospace tool).
A wide, fixed-format PC inventory. Maps the columns ShopDB owns into
canonical rows: hostname, vendor (BIOS Manf.), model, operating system,
logged-in user, last-reported and last-boot timestamps, and location. The
network columns (MAC, switch, VLAN, IP) and the machine 'Attached' link are
out of scope for this first pass.
options:
computertype - optional name stamped onto every computer's computertypeid.
Resolved by the engine against computertypes.
hostname_map - optional {lowercased hostname: existing assetnumber}. A
host whose Host Name matches an existing computer's hostname
is routed to that asset and updated in place (no new asset
row); unmatched hosts create a new asset + computer keyed on
the hostname.
"""
key = 'thor'
label = 'Thor PC inventory'
signature = ('Host Name', 'BIOS Manf.', 'OS Version')
@classmethod
def matches(cls, header):
return _has_columns(header, cls.signature)
@classmethod
def transform(cls, text, options=None):
options = options or {}
computertype = (options.get('computertype') or '').strip()
hostname_map = options.get('hostname_map') or {}
header, rows = _read(text)
if not _has_columns(header, cls.signature):
raise ImportSourceError(
'not a Thor export (expected Host Name, BIOS Manf., OS Version '
'columns)')
vendors = {}
models = {}
operatingsystems = {}
assets = []
computers = []
seen = set()
for row in rows:
hostname = (row.get('Host Name') or '').strip()
if not hostname or hostname.lower() in seen:
continue
seen.add(hostname.lower())
vendor = (row.get('BIOS Manf.') or '').strip()
model = (row.get('Model') or '').strip()
osname = cls._osname(row)
location = (row.get('Location') or '').strip()
loggedinuser = cls._logon(row)
if vendor:
vendors[vendor.lower()] = vendor
if model:
models[model.lower()] = model
if osname:
operatingsystems[osname.lower()] = osname
# Align on hostname: if this host already exists in ShopDB, point
# the computer row at its real asset so the engine updates it in
# place. Only create a new asset when there is no hostname match.
existing_assetnumber = hostname_map.get(hostname.lower())
if existing_assetnumber is None:
assets.append({
'assetnumber': hostname,
'name': hostname,
'assettypeid': DEFAULT_ASSETTYPE,
'statusid': DEFAULT_STATUS,
'locationid': location,
})
computer = {
'assetid': existing_assetnumber if existing_assetnumber is not None else hostname,
'hostname': hostname,
'osid': osname,
'vendorid': vendor,
'modelnumberid': model,
'loggedinuser': loggedinuser,
'lastreporteddate': cls._date(row.get('Last Updated')),
'lastboottime': cls._date(row.get('Boot Time')),
}
if computertype:
computer['computertypeid'] = computertype
computers.append(computer)
if not computers:
raise ImportSourceError('no host rows found in the Thor export')
out = {}
if vendors:
out['vendors'] = _write(['vendor'],
[{'vendor': v} for v in vendors.values()])
if models:
out['models'] = _write(['modelnumber'],
[{'modelnumber': m} for m in models.values()])
if operatingsystems:
out['operatingsystems'] = _write(
['osname'], [{'osname': o} for o in operatingsystems.values()])
asset_header = ['assetnumber', 'name', 'assettypeid', 'statusid', 'locationid']
computer_header = ['assetid', 'hostname', 'osid', 'vendorid',
'modelnumberid', 'loggedinuser', 'lastreporteddate',
'lastboottime']
if computertype:
computer_header.append('computertypeid')
out['computers'] = _write(computer_header, computers)
# Only emit an assets file when there are new hosts to create; an empty
# assets.csv would just be an extra clean no-op table.
if assets:
out['assets'] = _write(asset_header, assets)
return out
@staticmethod
def _osname(row):
"""Combine OS Version and OS ID into a single OS name.
Thor splits the OS into a build ('10.0.26100') and a marketing id
('24H2'); neither alone is meaningful, so join them.
"""
version = (row.get('OS Version') or '').strip()
osid = (row.get('OS ID') or '').strip()
if version and osid:
return '%s (%s)' % (version, osid)
return version or osid
@staticmethod
def _logon(row):
"""Take the account name out of a DOMAIN\\user or HOST\\user LOGON."""
logon = (row.get('LOGON') or '').strip()
if '\\' in logon:
return logon.rsplit('\\', 1)[-1]
return logon
@staticmethod
def _date(value):
"""Normalize a Thor timestamp to a form the engine's date coercion
accepts, or empty when it cannot be parsed. Leaves already-ISO values
alone; the engine tries several formats itself.
"""
return (value or '').strip()
SOURCES = {
AzureDeviceGroupSource.key: AzureDeviceGroupSource,
ThorSource.key: ThorSource,
}
def get_source(key):
source = SOURCES.get((key or '').strip().lower())
if source is None:
raise ImportSourceError(
"unknown import source '%s' (known: %s)"
% (key, ', '.join(sorted(SOURCES))))
return source
def detect_source(header):
"""Return the adapter whose signature matches this header, or None."""
for source in SOURCES.values():
if source.matches(header):
return source
return None

52
tests/test_csv_import.py
View File

@@ -173,14 +173,17 @@ def test_generated_templates_only_contain_real_columns(db):
A hand-written template set had invented columns on 7 of 11 tables and named
a table that does not exist. Generating from the schema is what makes that
impossible - so assert the generator actually does it.
impossible - so assert the generator actually does it. Plugin-registered
virtual columns (machineassetnumber) are legitimate, not invented, so they
are allowed alongside the table's real columns.
"""
from shopdb.core.services.csvimport import virtual_columns_for
for table in IMPORTABLE:
text = generate_template(table)
header = [line for line in text.splitlines() if line and not line.startswith('#')][0]
real = {c.name for c in importable_columns(table)}
allowed = {c.name for c in importable_columns(table)} | virtual_columns_for(table)
for column in header.split(','):
assert column in real, '%s.csv offers a column that does not exist: %s' % (table, column)
assert column in allowed, '%s.csv offers a column that does not exist: %s' % (table, column)
def test_every_template_includes_its_required_columns(db):
@@ -234,3 +237,46 @@ def test_a_dry_run_over_several_files_resolves_across_them(db):
assert result.ok, [str(p) for p in result.problems]
_db.session.rollback()
assert count('assets') == 0
def _example_files():
"""Reconstruct each template's example rows as an importable CSV: the header
plus the live (uncommented) example data rows that follow it."""
files = {}
for table in IMPORTABLE:
header = None
rows = []
for line in generate_template(table).splitlines():
if not line.strip():
continue
if line.startswith('#'):
continue
if header is None:
header = line
else:
rows.append(line)
files[table] = '\n'.join([header] + rows) + '\n'
return files
def test_template_examples_are_present_for_every_table(db):
"""Each template ships at least one worked example row, so a downloaded
set is a usable starter workbook rather than a bare header."""
files = _example_files()
for table in IMPORTABLE:
rows = files[table].splitlines()[1:]
assert rows, '%s.csv has no example row' % table
def test_template_example_set_imports_cleanly(db):
"""The whole curated example set must import as-is: every name a later file
references is defined by an earlier one, with no self-references that the
validation pass cannot resolve. This is the promise the workbook makes."""
resolver = Resolver()
files = _example_files()
for table in IMPORTABLE:
result = import_csv(table, files[table], resolver=resolver)
assert result.ok, '%s.csv example failed: %s' % (
table, [str(p) for p in result.problems])
_db.session.rollback()

201
tests/test_import_sources.py Normal file
View File

@@ -0,0 +1,201 @@
"""Foreign import-source adapters: transforming Azure and Thor exports.
These adapters are pure - they turn a foreign CSV into the canonical per-table
CSVs the import engine already understands, and touch no database. The tests
check the transform output directly, so they run without app context.
"""
import csv
import io
import pytest
from shopdb.core.services.importsources import (
AzureDeviceGroupSource, ImportSourceError, ThorSource, detect_source,
get_source,
)
def _rows(text):
return list(csv.DictReader(io.StringIO(text)))
AZURE_CSV = (
'displayName,objectType,mail,userType,id,deviceId,userPrincipalName,isUser,isGroup,isGuest\n'
'F13HSGH4,device,,,11111111-1111-1111-1111-111111111111,aaaa1111-bbbb-2222-cccc-333333333333,,False,False,False\n'
'E3P5M0B4,device,,,22222222-2222-2222-2222-222222222222,dddd4444-eeee-5555-ffff-666666666666,,False,False,False\n'
)
def test_azure_matches_signature():
header = ['displayName', 'objectType', 'deviceId', 'id']
assert AzureDeviceGroupSource.matches(header)
assert detect_source(header) is AzureDeviceGroupSource
def test_azure_transform_builds_assets_and_computers():
produced = AzureDeviceGroupSource.transform(AZURE_CSV)
assert set(produced) == {'assets', 'computers'}
assets = _rows(produced['assets'])
assert [a['assetnumber'] for a in assets] == ['F13HSGH4', 'E3P5M0B4']
assert all(a['assettypeid'] == 'computer' for a in assets)
assert all(a['statusid'] == 'In Use' for a in assets)
computers = _rows(produced['computers'])
assert computers[0]['assetid'] == 'F13HSGH4'
assert computers[0]['hostname'] == 'F13HSGH4'
assert computers[0]['azuredeviceid'] == 'aaaa1111-bbbb-2222-cccc-333333333333'
assert 'computertypeid' not in computers[0]
def test_azure_transform_stamps_computertype():
produced = AzureDeviceGroupSource.transform(
AZURE_CSV, options={'computertype': 'Shopfloor PC'})
computers = _rows(produced['computers'])
assert all(c['computertypeid'] == 'Shopfloor PC' for c in computers)
def test_azure_transform_carries_objectid():
produced = AzureDeviceGroupSource.transform(AZURE_CSV)
computers = _rows(produced['computers'])
# The CSV 'id' column is the Entra directory object id.
assert computers[0]['azureobjectid'] == '11111111-1111-1111-1111-111111111111'
def test_azure_transform_stamps_category():
produced = AzureDeviceGroupSource.transform(
AZURE_CSV, options={'category': 'SFLD_DYN_Device_Group_X'})
computers = _rows(produced['computers'])
assert all(c['azuredevicecategory'] == 'SFLD_DYN_Device_Group_X'
for c in computers)
def test_azure_aligns_matched_hostname_to_existing_asset():
# A case-insensitive hostname match routes the device to its real asset and
# emits NO new asset row, so the import updates in place instead of creating
# a duplicate. Unmatched devices still create a new asset keyed on hostname.
produced = AzureDeviceGroupSource.transform(
AZURE_CSV, options={'hostname_map': {'f13hsgh4': '13NSGH4'}})
computers = _rows(produced['computers'])
by_host = {c['hostname']: c for c in computers}
# Matched device points at the existing assetnumber, not its hostname.
assert by_host['F13HSGH4']['assetid'] == '13NSGH4'
# Unmatched device falls back to its hostname (new asset).
assert by_host['E3P5M0B4']['assetid'] == 'E3P5M0B4'
# Only the unmatched device produces an asset row.
assets = _rows(produced['assets'])
assert [a['assetnumber'] for a in assets] == ['E3P5M0B4']
def test_azure_all_matched_emits_no_assets_file():
produced = AzureDeviceGroupSource.transform(
AZURE_CSV,
options={'hostname_map': {'f13hsgh4': '13NSGH4', 'e3p5m0b4': 'E9Q'}})
# Every device matched, so there is nothing to create and no assets file.
assert 'assets' not in produced
computers = _rows(produced['computers'])
assert {c['assetid'] for c in computers} == {'13NSGH4', 'E9Q'}
def test_azure_skips_non_device_rows_and_dedupes():
text = (
'displayName,objectType,mail,userType,id,deviceId,userPrincipalName,isUser,isGroup,isGuest\n'
'F13HSGH4,device,,,1,aaaa,,False,False,False\n'
'F13HSGH4,device,,,1,aaaa,,False,False,False\n'
'Some User,user,u@x,,2,,u@x,True,False,False\n'
)
computers = _rows(AzureDeviceGroupSource.transform(text)['computers'])
assert [c['hostname'] for c in computers] == ['F13HSGH4']
def test_azure_rejects_wrong_header():
with pytest.raises(ImportSourceError):
AzureDeviceGroupSource.transform('a,b,c\n1,2,3\n')
THOR_CSV = (
'"Host Name","IP Address","LOGON","OS Version","OS ID","Last Updated","Boot Time","Location","Model","BIOS Manf."\n'
'"F67HSGH4","10.0.0.5","F67HSGH4\\ShopFloor","10.0.26100","24H2","2026-01-02 03:04:05","2026-01-01 06:00:00","S13","Dell Pro Tower Plus QBT1250","Dell Inc."\n'
)
def test_thor_matches_signature():
header = ['Host Name', 'BIOS Manf.', 'OS Version', 'Model']
assert ThorSource.matches(header)
assert detect_source(header) is ThorSource
def test_thor_transform_builds_lookup_and_subtype_files():
produced = ThorSource.transform(THOR_CSV)
assert 'vendors' in produced
assert 'models' in produced
assert 'operatingsystems' in produced
assert 'assets' in produced
assert 'computers' in produced
assert _rows(produced['vendors'])[0]['vendor'] == 'Dell Inc.'
assert _rows(produced['models'])[0]['modelnumber'] == 'Dell Pro Tower Plus QBT1250'
assert _rows(produced['operatingsystems'])[0]['osname'] == '10.0.26100 (24H2)'
computer = _rows(produced['computers'])[0]
assert computer['hostname'] == 'F67HSGH4'
assert computer['vendorid'] == 'Dell Inc.'
assert computer['modelnumberid'] == 'Dell Pro Tower Plus QBT1250'
assert computer['osid'] == '10.0.26100 (24H2)'
assert computer['loggedinuser'] == 'ShopFloor'
assert computer['lastreporteddate'] == '2026-01-02 03:04:05'
assert computer['lastboottime'] == '2026-01-01 06:00:00'
asset = _rows(produced['assets'])[0]
assert asset['locationid'] == 'S13'
assert asset['assettypeid'] == 'computer'
assert asset['statusid'] == 'In Use'
THOR_TWO_CSV = (
'"Host Name","IP Address","LOGON","OS Version","OS ID","Last Updated","Boot Time","Location","Model","BIOS Manf."\n'
'"F67HSGH4","10.0.0.5","F67HSGH4\\ShopFloor","10.0.26100","24H2","2026-01-02 03:04:05","2026-01-01 06:00:00","S13","Dell Pro Tower Plus QBT1250","Dell Inc."\n'
'"FDKKSGH4","10.0.0.6","FDKKSGH4\\ShopFloor","10.0.26100","24H2","2026-01-03 03:04:05","2026-01-02 06:00:00","G8","Dell Pro Tower Plus QBT1250","Dell Inc."\n'
)
def test_thor_stamps_computertype():
computer = _rows(
ThorSource.transform(THOR_CSV, options={'computertype': 'Shop Floor PC'})['computers'])[0]
assert computer['computertypeid'] == 'Shop Floor PC'
def test_thor_aligns_matched_hostname_to_existing_asset():
# A case-insensitive hostname match routes the host to its real asset and
# emits NO new asset row, so the import updates in place instead of creating
# a duplicate. Unmatched hosts still create a new asset keyed on hostname.
produced = ThorSource.transform(
THOR_TWO_CSV, options={'hostname_map': {'f67hsgh4': 'PC-1001'}})
computers = _rows(produced['computers'])
by_host = {c['hostname']: c for c in computers}
assert by_host['F67HSGH4']['assetid'] == 'PC-1001'
assert by_host['FDKKSGH4']['assetid'] == 'FDKKSGH4'
assets = _rows(produced['assets'])
assert [a['assetnumber'] for a in assets] == ['FDKKSGH4']
def test_thor_all_matched_emits_no_assets_file():
produced = ThorSource.transform(
THOR_TWO_CSV,
options={'hostname_map': {'f67hsgh4': 'PC-1001', 'fdkksgh4': 'PC-1002'}})
assert 'assets' not in produced
computers = _rows(produced['computers'])
assert {c['assetid'] for c in computers} == {'PC-1001', 'PC-1002'}
def test_get_source_unknown_raises():
with pytest.raises(ImportSourceError):
get_source('nope')
def test_detect_source_none_for_unknown_header():
assert detect_source(['foo', 'bar']) is None

187
tests/test_plugins/test_computers_import_machine_link.py Normal file
View File

@@ -0,0 +1,187 @@
"""computers.csv can carry the machine it drives, and the link is built on import.
A site with a spreadsheet knows "this PC drives machine CNC-014" and wants to
write that in the PC row, not hand-build an assetrelationships sheet keyed on
ids it never sees. The importer accepts a virtual `machineassetnumber` column,
keeps it out of the computers write, and hands it to the computers plugin after
each PC row is written. The rules that matter for a cold user driving this off a
spreadsheet:
- a good number builds the PC-controls-machine link;
- a number that has no machine, names this PC itself, names a location-only
marker, or is claimed twice in one file, is an AMBER warning and is skipped -
the PC still imports, the file is still ok;
- a dry run builds nothing and, critically, sends no alert;
- re-running the same file does not duplicate the link.
"""
import pytest
from shopdb.core.services.csvimport import (
Resolver, import_csv, virtual_columns_for,
)
from shopdb.extensions import db as _db
REFERENCE = [
('assettypes', 'assettype\nComputer\nMachine\n'),
('assetstatuses', 'status\nIn Use\n'),
('locationtypes', 'locationtype\nBay\n'),
]
def _seed(resolver):
"""Reference rows, a location, and the 'controls' relationship type."""
from shopdb.core.models.relationship import RelationshipType
for table, text in REFERENCE:
result = import_csv(table, text, resolver=resolver, commit=True)
assert result.ok, [str(p) for p in result.problems]
result = import_csv(
'locations', 'locationname,locationtypeid\nBay 3,Bay\n',
resolver=resolver, commit=True)
assert result.ok, [str(p) for p in result.problems]
if not RelationshipType.query.filter_by(relationshiptype='controls').first():
_db.session.add(RelationshipType(relationshiptype='controls'))
# import_csv writes into the session but never commits; commit the setup so
# a dry-run import's rollback only unwinds the dry run, not the fixtures.
_db.session.commit()
def _assets(resolver, *rows):
"""Create the given assets. Each row is (assetnumber, assettype)."""
body = 'assetnumber,assettypeid,statusid,locationid\n'
body += ''.join('%s,%s,In Use,Bay 3\n' % r for r in rows)
result = import_csv('assets', body, resolver=resolver, commit=True)
assert result.ok, [str(p) for p in result.problems]
_db.session.commit()
def _import_computers(resolver, body, commit=True):
return import_csv('computers', body, resolver=resolver, commit=commit)
def _controls_target(pc_assetnumber):
"""Asset number of the machine this PC actively controls, or None."""
from shopdb.core.models import Asset
from shopdb.core.models.relationship import RelationshipType, AssetRelationship
pc = Asset.query.filter_by(assetnumber=pc_assetnumber).first()
controls = RelationshipType.query.filter_by(relationshiptype='controls').first()
rel = AssetRelationship.query.filter(
AssetRelationship.sourceassetid == pc.assetid,
AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
AssetRelationship.isactive.is_(True)).first()
if rel is None:
return None
return _db.session.get(Asset, rel.targetassetid).assetnumber
def test_the_column_is_virtual_and_owned_by_the_computers_plugin(db):
"""The plugin registers machineassetnumber; core stays ignorant of it."""
assert 'machineassetnumber' in virtual_columns_for('computers')
def test_a_good_machine_number_builds_the_link(db):
resolver = Resolver()
_seed(resolver)
_assets(resolver, ('PC-001', 'Computer'), ('CNC-014', 'Machine'))
result = _import_computers(
resolver,
'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,CNC-014\n')
assert result.ok, [str(p) for p in result.problems]
assert result.warnings == []
assert _controls_target('PC-001') == 'CNC-014'
def test_an_empty_machine_number_links_nothing_and_does_not_warn(db):
resolver = Resolver()
_seed(resolver)
_assets(resolver, ('PC-001', 'Computer'))
result = _import_computers(
resolver, 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,\n')
assert result.ok
assert result.warnings == []
assert _controls_target('PC-001') is None
def test_a_machine_number_with_no_machine_warns_and_skips(db):
"""The PC still imports; the unresolved number is amber, not a failed file."""
resolver = Resolver()
_seed(resolver)
_assets(resolver, ('PC-001', 'Computer'))
result = _import_computers(
resolver, 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,NOPE-99\n')
assert result.ok, 'an unresolved machine number must not fail the file'
assert len(result.warnings) == 1
warning = result.warnings[0]
assert warning.column == 'machineassetnumber'
assert warning.target == 'NOPE-99'
assert warning.action, 'a warning must carry the one corrective step'
assert _controls_target('PC-001') is None
def test_a_pc_naming_its_own_asset_number_is_not_self_linked(db):
resolver = Resolver()
_seed(resolver)
_assets(resolver, ('PC-001', 'Computer'))
result = _import_computers(
resolver, 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,PC-001\n')
assert result.ok
assert len(result.warnings) == 1
assert 'own asset number' in result.warnings[0].reason
assert _controls_target('PC-001') is None
def test_two_rows_claiming_one_machine_link_only_the_first(db):
"""The in-file fan-out guard: a copied cell must not steal the bay."""
resolver = Resolver()
_seed(resolver)
_assets(resolver,
('PC-001', 'Computer'), ('PC-002', 'Computer'), ('CNC-014', 'Machine'))
result = _import_computers(
resolver,
'assetid,hostname,machineassetnumber\n'
'PC-001,WS-PC-001,CNC-014\n'
'PC-002,WS-PC-002,CNC-014\n')
assert result.ok
assert len(result.warnings) == 1
assert 'already claimed' in result.warnings[0].reason
assert _controls_target('PC-001') == 'CNC-014'
assert _controls_target('PC-002') is None
def test_a_dry_run_builds_no_link_and_leaves_no_trace(db):
resolver = Resolver()
_seed(resolver)
_assets(resolver, ('PC-001', 'Computer'), ('CNC-014', 'Machine'))
result = _import_computers(
resolver,
'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,CNC-014\n',
commit=False)
assert result.ok
assert result.created == 1
_db.session.rollback()
assert _controls_target('PC-001') is None
def test_reimporting_the_same_link_does_not_duplicate(db):
resolver = Resolver()
_seed(resolver)
_assets(resolver, ('PC-001', 'Computer'), ('CNC-014', 'Machine'))
body = 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,CNC-014\n'
first = _import_computers(resolver, body)
assert first.ok and first.warnings == []
second = _import_computers(resolver, body)
assert second.ok and second.warnings == []
assert _controls_target('PC-001') == 'CNC-014'
from shopdb.core.models import Asset
from shopdb.core.models.relationship import RelationshipType, AssetRelationship
pc = Asset.query.filter_by(assetnumber='PC-001').first()
controls = RelationshipType.query.filter_by(relationshiptype='controls').first()
active = AssetRelationship.query.filter(
AssetRelationship.sourceassetid == pc.assetid,
AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
AssetRelationship.isactive.is_(True)).count()
assert active == 1, 'a re-import must not stack a second active link'