diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000..3ea9bfaa
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,23 @@
+# Keep the build context lean and prevent host artifacts from clobbering
+# in-image installs. node_modules is platform-specific: copying the host's
+# (Windows) tree over the container's `npm ci` breaks rollup's native binary.
+**/node_modules
+frontend/dist
+frontend/src/.plugins-staged
+frontend/src/router/routes.gen.js
+
+# Python caches and virtualenvs
+**/__pycache__
+**/*.pyc
+.venv
+venv
+
+# Local dev and VCS noise
+.git
+.gitignore
+*.log
+.env.local
+
+# Installer build artifacts (not part of the app image)
+installer-arch-dev/build
+installer-arch-dev/dist
diff --git a/Dockerfile b/Dockerfile
index c6567d3c..a747c3fe 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -23,6 +23,12 @@ FROM node:20-slim AS frontendbuild
WORKDIR /build
+# Trust the site's corporate root CA so npm's TLS to the registry validates
+# behind an inspecting proxy. Without it npm 10.x fails silently and leaves an
+# incomplete node_modules.
+COPY corp-root-ca.pem /usr/local/share/ca-certificates/corp-root-ca.crt
+ENV NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/corp-root-ca.crt
+
# Copy only the manifests first so `npm ci` caches on dependency changes.
COPY frontend/package*.json ./frontend/
RUN cd frontend && npm ci
diff --git a/frontend/package-lock.json b/frontend/package-lock.json
index 9b279c06..c069d5c7 100644
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "shopdb-frontend",
- "version": "0.7.0",
+ "version": "0.12.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "shopdb-frontend",
- "version": "0.7.0",
+ "version": "0.12.0",
"dependencies": {
"@fontsource-variable/inter": "^5.2.8",
"@fullcalendar/core": "^6.1.20",
diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js
index 6712925c..0f0fccd3 100644
--- a/frontend/src/api/index.js
+++ b/frontend/src/api/index.js
@@ -1027,14 +1027,26 @@ export const settingsApi = {
// Generated CSV template (header + example row) for one table
return api.get(`/settings/asset-import/templates/${encodeURIComponent(tablename)}`, { responseType: 'blob' })
},
- importAssetCsvs(files, commit = false) {
+ importAssetCsvs(files, commit = false, options = {}) {
// Multipart upload of one or more CSVs. commit=false is a dry-run preview
// that writes nothing; commit=true applies the whole set in one transaction.
+ // options.source picks a foreign adapter (azure, thor); default native means
+ // the files are already canonical per-table CSVs. options.computertype is
+ // stamped onto every imported computer when a source assigns a PC type.
const form = new FormData()
for (const file of files) {
form.append('files', file)
}
form.append('commit', commit ? 'true' : 'false')
+ if (options.source) {
+ form.append('source', options.source)
+ }
+ if (options.computertype) {
+ form.append('computertype', options.computertype)
+ }
+ if (options.category) {
+ form.append('category', options.category)
+ }
return api.post('/settings/asset-import/import', form, {
headers: { 'Content-Type': 'multipart/form-data' }
})
diff --git a/frontend/src/views/settings/ImportSiteAssets.vue b/frontend/src/views/settings/ImportSiteAssets.vue
index 60ed215c..8cfeeba2 100644
--- a/frontend/src/views/settings/ImportSiteAssets.vue
+++ b/frontend/src/views/settings/ImportSiteAssets.vue
@@ -91,32 +91,6 @@
-
-
CSV Import Templates
-
- These are early-draft import templates generated from the live ShopDB
- schema. Each file has a header row and a commented example. Fill one out
- (or let the collector script populate them) and import them in the order
- below - a table never references something that has not been loaded yet.
- Foreign keys accept the referenced row's name or its numeric id, and
- re-importing a file updates matching rows instead of duplicating them.
-
-
-
Loading...
-
-
-
- {{ t.filename }}
-
-
-
-
-
-
{{ templatesError }}
-
-
Import CSVs
@@ -128,6 +102,108 @@
nothing is written.
+
+
+
+
+ An Azure device-group export lists device hostnames plus their Entra
+ deviceId. The device displayName is matched to an existing PC by
+ hostname (case-insensitive): a match updates that PC in place and
+ stores its deviceId, object id, and category; a device with no
+ matching hostname creates a new computer (assettype computer, status
+ In Use). It carries no model, OS, location, or user data - enrich
+ those later with the collector or a Thor import.
+
+
+
+
+
+ How to export group members from Azure
+
+
Open the Microsoft Entra admin center (entra.microsoft.com).
+
Go to Groups, then open the device group you want to import.
+
Select Members in the left menu.
+
Click Bulk Operations->Download Members to export the CSV.
+
Upload that CSV here with the source set to Azure device group.
+
+
+
+
+
+
+ A Thor PC-inventory export is a wide, fixed-format dump. This maps the
+ hostname, BIOS manufacturer, model, OS, logged-in user, last-reported
+ and boot times, and location into ShopDB. Each host is matched to an
+ existing PC by hostname (case-insensitive): a match updates that PC in
+ place; a host with no matching hostname creates a new computer.
+ Network details (MAC, switch, VLAN) and the machine relationship are
+ not imported in this pass.
+
+
+
+
+ {{ report.warning_count }} warning(s): these rows still import, but
+ something needs a look - most often a machine link that could not be
+ made. Review them below; each says what to do.
+
+
@@ -206,6 +288,7 @@
Created
Updated
Reconciled
+
Warnings
Problems
@@ -218,6 +301,9 @@
{{ (f.reconciliations || []).length }}
+
+ {{ (f.warnings || []).length }}
+
{{ f.problems.length }}
@@ -244,6 +330,23 @@
+
+
Warnings
+
+ Each row below imported, but something could not be completed - most
+ often a PC's machine link. Nothing here blocks the import; fix what
+ you can and re-import to complete the links.
+
+ These import templates are generated from the live ShopDB schema. Each
+ file has a header row plus a small worked example - commented rows you
+ can uncomment and edit. The examples are internally consistent across
+ every file (the same locations, vendors, and types are referenced), so
+ you can download the full set, keep it as a starter workbook, and fill
+ in your own assets right away. Import them in the order below - a table
+ never references something that has not been loaded yet. Foreign keys
+ accept the referenced row's name or its numeric id, and re-importing a
+ file updates matching rows instead of duplicating them.
+
+
+
Loading...
+
+
+
+ {{ t.filename }}
+
+
+
+
+
+
{{ templatesError }}
+
+
PC to Machine Relationships
A PC that drives a machine (a CMM PC, a DNC/eDNC host) is linked to that
- machine so the two show up connected across ShopDB. That link is built
- automatically by the Asset Collector, not by this CSV import: the script
- reads the four-digit workstation number from the PC it runs on and
- reports it, and ShopDB matches that number to a machine's
- assetnumber to create the connection.
+ machine so the two show up connected across ShopDB. You can build that
+ link two ways. The Asset Collector builds it automatically: the script
+ reads the four-digit workstation number from the PC it runs on, reports
+ it, and ShopDB matches that number to a machine's
+ assetnumber.
- What this means for import order: load your machines first (so their
- assetnumber values exist), then let the collector run on
- each PC. A machine number the collector reports that does not match an
- existing machine is flagged rather than guessed, so nothing links to the
- wrong asset. Building this relationship straight from a CSV column is
- planned as a follow-up.
+ Or build it straight from the spreadsheet: put the machine's
+ assetnumber in the machineassetnumber column on
+ computers.csv, and the import creates the link for you.
+
+
+ Either way, load your machines first (so their
+ assetnumber values exist), then the PCs. A machine number
+ that does not match an existing machine - or one already driven by
+ another PC - is reported as a warning and skipped, not guessed, so
+ nothing links to the wrong asset. The PC still imports; fix the number
+ and re-import to complete the link.
@@ -383,6 +394,7 @@ const form = ref({
locationid: '',
osid: '',
loggedinuser: '',
+ azuredeviceid: '',
accessmethods: [],
notes: '',
mapx: null,
@@ -486,6 +498,7 @@ onMounted(async () => {
locationid: pc.locationid || '',
osid: ext.osid || '',
loggedinuser: ext.loggedinuser || '',
+ azuredeviceid: ext.azuredeviceid || '',
accessmethods: (pc.accessmethods || []).map(a => ({
protocolid: a.protocolid,
portoverride: a.portoverride ?? null
@@ -555,6 +568,7 @@ async function savePC() {
locationid: form.value.locationid || null,
osid: form.value.osid || null,
loggedinuser: form.value.loggedinuser || null,
+ azuredeviceid: form.value.azuredeviceid || null,
accessmethods: form.value.accessmethods,
notes: form.value.notes || null,
ipaddress: form.value.ipaddress || null,
diff --git a/plugins/computers/migrations/versions/0002_computer_azuredeviceid.py b/plugins/computers/migrations/versions/0002_computer_azuredeviceid.py
new file mode 100644
index 00000000..72c5fe0b
--- /dev/null
+++ b/plugins/computers/migrations/versions/0002_computer_azuredeviceid.py
@@ -0,0 +1,40 @@
+"""Add azuredeviceid to computers (Entra/Azure device-group imports).
+
+The core chain created computers WITHOUT an azuredeviceid column. This plugin
+revision adds it so BOTH fresh installs (core chain builds the table, then this
+adds the column) and existing installs get it. Guarded/idempotent: skips when
+the table is absent (plugin disabled) or the column already exists (e.g. a test
+DB built by db.create_all() from the model).
+
+Revision ID: computers0002azureid
+Revises: computers0001anchor
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = 'computers0002azureid'
+down_revision = 'computers0001anchor'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ bind = op.get_bind()
+ insp = sa.inspect(bind)
+ if 'computers' not in insp.get_table_names():
+ return
+ cols = {c['name'] for c in insp.get_columns('computers')}
+ if 'azuredeviceid' not in cols:
+ op.add_column('computers',
+ sa.Column('azuredeviceid', sa.String(length=64), nullable=True))
+
+
+def downgrade():
+ bind = op.get_bind()
+ insp = sa.inspect(bind)
+ if 'computers' not in insp.get_table_names():
+ return
+ cols = {c['name'] for c in insp.get_columns('computers')}
+ if 'azuredeviceid' in cols:
+ op.drop_column('computers', 'azuredeviceid')
diff --git a/plugins/computers/migrations/versions/0003_computer_azure_fields.py b/plugins/computers/migrations/versions/0003_computer_azure_fields.py
new file mode 100644
index 00000000..6ce93b29
--- /dev/null
+++ b/plugins/computers/migrations/versions/0003_computer_azure_fields.py
@@ -0,0 +1,45 @@
+"""Add azureobjectid and azuredevicecategory to computers.
+
+The Azure device-group import now also records the Entra directory object id
+(the CSV 'id' column) and an operator-supplied device category (the group
+membership label, e.g. SFLD_DYN_Device_Group_...). This revision adds both
+columns. Guarded/idempotent: skips when the table is absent (plugin disabled)
+or a column already exists (e.g. a test DB built by db.create_all()).
+
+Revision ID: computers0003azurefields
+Revises: computers0002azureid
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = 'computers0003azurefields'
+down_revision = 'computers0002azureid'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ bind = op.get_bind()
+ insp = sa.inspect(bind)
+ if 'computers' not in insp.get_table_names():
+ return
+ cols = {c['name'] for c in insp.get_columns('computers')}
+ if 'azureobjectid' not in cols:
+ op.add_column('computers',
+ sa.Column('azureobjectid', sa.String(length=64), nullable=True))
+ if 'azuredevicecategory' not in cols:
+ op.add_column('computers',
+ sa.Column('azuredevicecategory', sa.String(length=255), nullable=True))
+
+
+def downgrade():
+ bind = op.get_bind()
+ insp = sa.inspect(bind)
+ if 'computers' not in insp.get_table_names():
+ return
+ cols = {c['name'] for c in insp.get_columns('computers')}
+ if 'azuredevicecategory' in cols:
+ op.drop_column('computers', 'azuredevicecategory')
+ if 'azureobjectid' in cols:
+ op.drop_column('computers', 'azureobjectid')
diff --git a/plugins/computers/models/computer.py b/plugins/computers/models/computer.py
index 6a35b4fc..c0ca3463 100644
--- a/plugins/computers/models/computer.py
+++ b/plugins/computers/models/computer.py
@@ -85,6 +85,23 @@ class Computer(BaseModel):
lastreporteddate = db.Column(db.DateTime, nullable=True)
lastboottime = db.Column(db.DateTime, nullable=True)
+ # Entra/Azure device identity (set by Azure device-group imports)
+ azuredeviceid = db.Column(
+ db.String(64),
+ nullable=True,
+ comment='Entra/Azure deviceId GUID'
+ )
+ azureobjectid = db.Column(
+ db.String(64),
+ nullable=True,
+ comment='Entra/Azure directory object id GUID'
+ )
+ azuredevicecategory = db.Column(
+ db.String(255),
+ nullable=True,
+ comment='Entra device category / group membership label set at import'
+ )
+
# Remote access is now modeled per-protocol via the accessmethods
# relationship (AccessProtocol / ComputerAccess), replacing isvnc/iswinrm.
diff --git a/plugins/computers/plugin.py b/plugins/computers/plugin.py
index 96f1e6b6..086836aa 100644
--- a/plugins/computers/plugin.py
+++ b/plugins/computers/plugin.py
@@ -34,6 +34,14 @@ MEASURINGTOOL_LINK_ORIGIN = 'collector:measuringtool'
# touched.
MACHINE_LINK_ORIGIN = 'collector:machine'
+# Marker stamped on the PC->machine link built from a CSV import's machine
+# number column. A DISTINCT origin from the collector's so the two provenances
+# stay separable: an import establishes a link once from a spreadsheet, the
+# collector maintains it continuously from the floor, and a site may use one,
+# the other, or both. Same archive discipline - only rows carrying this label
+# are touched by a re-import.
+MACHINE_LINK_IMPORT_ORIGIN = 'import:machine'
+
# Marker stamped on both links a part-marker PC produces: PC controls marker,
# and marker partof the operation it serves. Same archive discipline again.
PARTMARKER_LINK_ORIGIN = 'collector:partmarker'
@@ -90,8 +98,224 @@ class ComputersPlugin(BasePlugin):
def init_app(self, app: Flask, db_instance) -> None:
"""Initialize plugin with Flask app."""
+ self._register_import_hook()
logger.info(f"Computers plugin initialized (v{self.meta.version})")
+ # -- bulk-import machine link -------------------------------------------
+
+ def _register_import_hook(self):
+ """Own a `machineassetnumber` column on computers.csv.
+
+ A site with a spreadsheet knows "this PC drives machine 3015" and wants
+ to write that number in the PC row, not hand-build an assetrelationships
+ sheet keyed on ids it never sees. The core importer stays ignorant of
+ what the column means: it accepts the header, keeps it out of the
+ computers write, and hands the value to this hook after each PC row is
+ written, so the PC's asset id exists. Registration is idempotent, so a
+ plugin reload or a test re-init does not double-fire.
+ """
+ from shopdb.api import register_import_hook
+
+ register_import_hook(
+ 'computers',
+ {'machineassetnumber':
+ 'the asset number of the machine this PC drives (from '
+ 'machines.csv); builds the PC-controls-machine link. A number '
+ 'that has no machine, or one already driven by another PC, is '
+ 'reported as a warning and skipped - the PC still imports.'},
+ self._import_machine_link)
+
+ def _import_machine_link(self, rowvalues, virtualvalues, line):
+ """Build a PC->machine controls link from an import row's machine number.
+
+ Runs inside the importer's transaction - a dry run rolls it back - so it
+ reuses the collector's mature _sync_machine_link rather than inventing a
+ second linker, but under the import origin (so an import never archives
+ a collector's rows and vice versa) and in preview mode (so a bulk import
+ never emails an alert per row; the findings are the report's amber
+ warnings instead).
+
+ Returns RowWarning list, never raises: a machine number that resolves to
+ no asset is an amber note beside a PC that still imports, not a failed
+ file. Every warning names the PC, the number, and the one fix.
+ """
+ from shopdb.api import RowWarning
+
+ machinenumber = (virtualvalues.get('machineassetnumber') or '').strip()
+ if not machinenumber:
+ return []
+
+ assetid = rowvalues.get('assetid')
+ comp = Computer.query.filter_by(assetid=assetid).first() if assetid else None
+ if comp is None:
+ # The PC row was written but its Computer extension is not readable
+ # here - nothing to link to. Amber, not fatal.
+ return [RowWarning(
+ line, 'machineassetnumber',
+ reason='PC row not found to link to machine {!r}'.format(
+ machinenumber),
+ action='re-run the import; if it recurs, check the computers '
+ 'row imported',
+ target=machinenumber)]
+
+ hostname = comp.hostname or (assetid and str(assetid)) or '?'
+
+ # IN-FILE FAN-OUT GUARD. Two rows in one computers.csv both naming the
+ # same machine is a spreadsheet mistake - a copied cell, a wrong
+ # workstation number - not a swap. But _incumbent_has_yielded reads a
+ # freshly-imported PC (no lastreporteddate) as having yielded, so
+ # without this the SECOND row would silently archive the first row's
+ # link and steal the machine, and the file would import clean with the
+ # wrong PC on the bay. So the first claim on a machine this run wins and
+ # is remembered; a later row for the same machine is warned and skipped.
+ # The claim set lives on db.session.info, which Flask-SQLAlchemy resets
+ # at request teardown, so it is scoped to exactly this one import and
+ # never leaks into the next.
+ machine = self._asset_by_number(machinenumber)
+ if machine is not None and machine.assetid == assetid:
+ # The number names this very PC. Happens on a PC first seen before
+ # the machine-number convention, whose own asset number IS the
+ # machine number. Skip - a PC does not control itself.
+ return [RowWarning(
+ line, 'machineassetnumber',
+ reason='machine number {!r} is this PC\'s own asset number; '
+ 'not linked'.format(machinenumber),
+ action='give this PC a hostname-based asset number, or point '
+ 'machineassetnumber at the machine it drives',
+ subject=hostname, target=machinenumber)]
+
+ if machine is not None:
+ # A location-only machine is a virtual bay marker, not a real
+ # machine, so nothing controls it. Skip rather than build a link
+ # that would read as a PC driving a location.
+ skip = self._location_only_skip(
+ machine, hostname, machinenumber, line)
+ if skip is not None:
+ return [skip]
+
+ if machine is not None and machine.assetid != assetid:
+ # ALREADY-CONTROLS-DIFFERENT. Unlike the collector - which reads a
+ # new machine number as a PC moved to a new bay and silently
+ # re-bays it - an import naming a machine when this PC already
+ # actively controls a DIFFERENT one is far more likely a wrong
+ # workstation number in the sheet than a real move. Warn and skip
+ # rather than archive a link a person may have set on purpose.
+ other = self._active_controlled_machine(assetid, machine.assetid)
+ if other is not None:
+ return [RowWarning(
+ line, 'machineassetnumber',
+ reason='{} already controls machine {}; not moved to {}'
+ .format(hostname, other, machine.assetnumber),
+ action='if this PC really moved, clear the existing link '
+ 'first; otherwise correct the machine number on '
+ 'this row',
+ subject=hostname, target=machinenumber)]
+
+ if machine is not None and machine.assetid != assetid:
+ claimed = db.session.info.setdefault('import_machine_claims', {})
+ # Keyed by machine asset id -> (claimant PC asset id, its hostname).
+ # The id is what the guard compares on (the same PC re-stating its
+ # own link is not a conflict); the hostname is only for the message.
+ firstclaimant = claimed.get(machine.assetid)
+ if firstclaimant is not None and firstclaimant[0] != assetid:
+ return [RowWarning(
+ line, 'machineassetnumber',
+ reason='machine {} was already claimed by {} earlier in '
+ 'this import; {} not linked'.format(
+ machine.assetnumber, firstclaimant[1], hostname),
+ action='a machine drives one PC - correct the machine '
+ 'number on one of these rows so each PC names its '
+ 'own machine',
+ subject=hostname, target=machinenumber)]
+ claimed[machine.assetid] = (assetid, hostname)
+
+ # _sync_machine_link appends plain strings to its warnings list. Collect
+ # them and translate to structured, actionable RowWarnings so the UI can
+ # show the PC, the machine number, and the corrective step.
+ rawwarnings = []
+ self._sync_machine_link(
+ comp, machinenumber, rawwarnings,
+ origin_label=MACHINE_LINK_IMPORT_ORIGIN, preview=True)
+
+ out = []
+ for text in rawwarnings:
+ out.append(RowWarning(
+ line, 'machineassetnumber',
+ reason=text,
+ action=self._machine_link_action(text, machinenumber),
+ subject=hostname, target=machinenumber))
+ return out
+
+ @staticmethod
+ def _active_controlled_machine(pcassetid, excludeassetid):
+ """Asset number of a machine this PC actively controls under the import
+ origin, other than excludeassetid, or None.
+
+ Scoped to MACHINE_LINK_IMPORT_ORIGIN so an import only reasons about its
+ own links: a collector link or a hand-made one is the collector's or a
+ person's to move, not the import's to block on.
+ """
+ from shopdb.api import AssetRelationship, RelationshipType, Asset
+
+ controls = RelationshipType.query.filter_by(
+ relationshiptype='controls').first()
+ if not controls:
+ return None
+ rel = AssetRelationship.query.filter(
+ AssetRelationship.sourceassetid == pcassetid,
+ AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
+ AssetRelationship.label == MACHINE_LINK_IMPORT_ORIGIN,
+ AssetRelationship.targetassetid != excludeassetid,
+ AssetRelationship.isactive.is_(True),
+ ).first()
+ if rel is None:
+ return None
+ asset = db.session.get(Asset, rel.targetassetid)
+ return asset.assetnumber if asset else str(rel.targetassetid)
+
+ @staticmethod
+ def _location_only_skip(machine, hostname, machinenumber, line):
+ """A RowWarning if the resolved asset is a location-only marker, else None.
+
+ A machine flagged islocationonly is a virtual bay, not a real machine,
+ so a PC does not control it. Guarded behind the machines plugin being
+ installed - a site without it cannot have a location-only machine, and
+ the import must not fail because the model is absent.
+ """
+ from shopdb.api import RowWarning
+ try:
+ from plugins.machines.models import Machine
+ except ImportError:
+ return None
+ row = Machine.query.filter_by(assetid=machine.assetid).first()
+ if row is None or not row.islocationonly:
+ return None
+ return RowWarning(
+ line, 'machineassetnumber',
+ reason='machine {} is a location-only marker, not a real machine; '
+ 'not linked'.format(machine.assetnumber),
+ action='point machineassetnumber at a real machine, or clear it '
+ 'for this PC',
+ subject=hostname, target=machinenumber)
+
+ @staticmethod
+ def _machine_link_action(text, machinenumber):
+ """The one corrective step for a machine-link warning, by its shape."""
+ lowered = text.lower()
+ if 'no asset for machine number' in lowered:
+ return ('create the machine asset with number {!r} (import '
+ 'machines.csv first), or correct the number on this PC '
+ 'row'.format(machinenumber))
+ if 'still held by' in lowered:
+ return ('confirm which PC drives this machine; retire or move the '
+ 'old PC off In Use to hand the machine over')
+ if 'taken over from' in lowered:
+ return ('expected on a PC swap; set the old PC to Retired, '
+ 'Inventory or In Repair')
+ if 'relationship type missing' in lowered:
+ return 'run flask seed reference-data, then re-import'
+ return 'review this PC and its machine number'
+
# -- ADR-006 collector contract -----------------------------------------
def get_collector_schema(self) -> Optional[Dict]:
@@ -647,7 +871,8 @@ class ComputersPlugin(BasePlugin):
# -- measuring-tool sync -----------------------------------------------
- def _sync_machine_link(self, comp, machinenumber, warnings):
+ def _sync_machine_link(self, comp, machinenumber, warnings,
+ origin_label=MACHINE_LINK_ORIGIN, preview=False):
"""Link a PC to the machine it drives, from the reported machine number.
Until this existed the machine number was collected and then discarded,
@@ -669,6 +894,15 @@ class ComputersPlugin(BasePlugin):
cadence, alerting on every pass. The incumbent therefore keeps the
machine while it is still alive, and the challenger is recorded as a
dormant link. See _incumbent_has_yielded for what alive means.
+
+ origin_label separates provenance: the collector stamps
+ MACHINE_LINK_ORIGIN, a CSV import stamps MACHINE_LINK_IMPORT_ORIGIN, and
+ each only ever archives its own rows so the two sources coexist.
+
+ preview=True is for a dry-run import: DB writes still happen in the
+ caller's transaction (which it rolls back), but the alert fan-out - the
+ one side effect a rollback cannot undo - is suppressed so a preview
+ never emails anyone.
"""
from shopdb.api import AssetRelationship, RelationshipType, Asset
@@ -700,7 +934,7 @@ class ComputersPlugin(BasePlugin):
mine = AssetRelationship.query.filter(
AssetRelationship.sourceassetid == pcasset.assetid,
AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
- AssetRelationship.label == MACHINE_LINK_ORIGIN,
+ AssetRelationship.label == origin_label,
).all()
link = None
@@ -710,11 +944,23 @@ class ComputersPlugin(BasePlugin):
else:
rel.isactive = False
+ # The unique constraint on assetrelationships is (source, target, type)
+ # and does NOT include label, so an import (import:machine) and the
+ # collector (collector:machine) cannot each hold their OWN row for the
+ # same PC->machine control. If our label-scoped lookup found nothing but
+ # a row for this exact triple exists under the other provenance, adopt
+ # it rather than insert a duplicate that would violate the constraint.
+ if link is None:
+ link = AssetRelationship.query.filter_by(
+ sourceassetid=pcasset.assetid,
+ targetassetid=machine.assetid,
+ relationshiptypeid=controls.relationshiptypeid).first()
+
# Whoever actively holds this machine now, if it is not this PC.
held = AssetRelationship.query.filter(
AssetRelationship.targetassetid == machine.assetid,
AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
- AssetRelationship.label == MACHINE_LINK_ORIGIN,
+ AssetRelationship.label == origin_label,
AssetRelationship.sourceassetid != pcasset.assetid,
AssetRelationship.isactive.is_(True),
).all()
@@ -737,9 +983,10 @@ class ComputersPlugin(BasePlugin):
sourceassetid=pcasset.assetid,
targetassetid=machine.assetid,
relationshiptypeid=controls.relationshiptypeid,
- label=MACHINE_LINK_ORIGIN,
+ label=origin_label,
isactive=False))
- self._alert_machine_contested(names, comp, machine)
+ if not preview:
+ self._alert_machine_contested(names, comp, machine)
else:
link.isactive = False
return [{'assetid': machine.assetid,
@@ -753,7 +1000,7 @@ class ComputersPlugin(BasePlugin):
sourceassetid=pcasset.assetid,
targetassetid=machine.assetid,
relationshiptypeid=controls.relationshiptypeid,
- label=MACHINE_LINK_ORIGIN,
+ label=origin_label,
isactive=True)
db.session.add(link)
else:
@@ -767,7 +1014,8 @@ class ComputersPlugin(BasePlugin):
warnings.append(
'machine {} was taken over from {}; check that PC'.format(
machine.assetnumber, oldname))
- self._alert_pc_superseded(oldname, comp, machine)
+ if not preview:
+ self._alert_pc_superseded(oldname, comp, machine)
return [{'assetid': machine.assetid,
'machinenumber': machine.assetnumber,
@@ -870,8 +1118,8 @@ class ComputersPlugin(BasePlugin):
#
# A row made by hand carries neither label and is deliberately NOT
# counted: a person's link is not the collector's to archive.
- ourlabels = (label, MACHINE_LINK_ORIGIN, MEASURINGTOOL_LINK_ORIGIN,
- PARTMARKER_LINK_ORIGIN)
+ ourlabels = (label, MACHINE_LINK_ORIGIN, MACHINE_LINK_IMPORT_ORIGIN,
+ MEASURINGTOOL_LINK_ORIGIN, PARTMARKER_LINK_ORIGIN)
others = AssetRelationship.query.filter(
AssetRelationship.targetassetid == deviceasset.assetid,
AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
@@ -1423,15 +1671,17 @@ class ComputersPlugin(BasePlugin):
sourceassetid=pcasset.assetid,
targetassetid=adopted.assetid,
relationshiptypeid=controls.relationshiptypeid).first()
- if claimed is not None and claimed.label == MACHINE_LINK_ORIGIN:
+ if claimed is not None and claimed.label in (
+ MACHINE_LINK_ORIGIN, MACHINE_LINK_IMPORT_ORIGIN):
# LEAVE IT ALONE. On a CMM the instrument IS the reported bay,
- # so the machine sync has already made the row for this exact
- # triple - and it finds its own rows BY that label. Relabelling
- # hid the link from it, so the next cycle built a second row for
- # the same triple and MySQL rejected it: the PC reported 200,
- # then 500 forever after. The row's lifecycle (including staying
- # dormant while a bay is contested) belongs to the machine sync;
- # adoption only needs the identity, which it has.
+ # so the machine sync (or a CSV import's machine link) has
+ # already made the row for this exact triple - and each finds
+ # its own rows BY that label. Relabelling hid the link from it,
+ # so the next cycle built a second row for the same triple and
+ # MySQL rejected it: the PC reported 200, then 500 forever
+ # after. The row's lifecycle (including staying dormant while a
+ # bay is contested) belongs to the machine sync; adoption only
+ # needs the identity, which it has.
pass
elif claimed is not None:
# Relabelling is what adoption MEANS for any other row: the link
diff --git a/scripts/reconcile_locations.py b/scripts/reconcile_locations.py
new file mode 100644
index 00000000..becb3511
--- /dev/null
+++ b/scripts/reconcile_locations.py
@@ -0,0 +1,156 @@
+"""Generate reconciliation SQL for the locations table.
+
+Reads a TSV export (locationid, locationname, description, locationtypeid,
+parentlocationid) and emits a single transaction-wrapped SQL script that:
+ - Treats bare grid coordinates ([A-V][0-9]{1,2}W? and LD[0-9]+W?) as the
+ canonical identity for a cell.
+ - Folds coordinate-plus-suffix rows into the canonical coordinate, moving the
+ suffix text into the canonical row's description (never the name).
+ - Folds named-area suffix variants ("NDE - 6409") into the parent named area
+ ("NDE"); named areas are kept as-is and never forced into coordinates.
+ - Repoints assets.locationid, subnets.locationid and child
+ locations.parentlocationid onto the canonical row, then deletes duplicates.
+
+Bare coordinates with no variants are left untouched. The trailing W (west
+column, e.g. G2W, K3W) is part of the coordinate and is never stripped, so
+G2W is kept distinct from a hypothetical G2.
+"""
+import re
+import sys
+
+COORD = re.compile(r'^(LD|[ABCDEFGHJKLMNPQRSTUV])(\d{1,2})(W)?(?=$|[ \-])')
+
+
+def esc(s):
+ return s.replace("'", "''")
+
+
+def load(path):
+ rows = []
+ with open(path, encoding='utf-8-sig') as fh:
+ for line in fh:
+ line = line.rstrip('\n').rstrip('\r')
+ if not line:
+ continue
+ parts = line.split('\t')
+ if len(parts) < 5:
+ parts += [''] * (5 - len(parts))
+ lid, name, desc, typeid, parentid = parts[:5]
+ rows.append({
+ 'id': int(lid),
+ 'name': name,
+ 'desc': desc,
+ 'typeid': typeid,
+ 'parentid': parentid,
+ })
+ return rows
+
+
+def classify(name):
+ """Return (canonical, suffix). canonical is the bare coordinate or named
+ parent; suffix is the trailing detail (may be '')."""
+ m = COORD.match(name)
+ if m:
+ prefix, digits, west = m.group(1), m.group(2), m.group(3) or ''
+ canonical = f"{prefix}{digits}{west}"
+ suffix = name[m.end():].strip()
+ suffix = re.sub(r'^[\-\s]+', '', suffix).strip()
+ return canonical, suffix, 'coord'
+ # named area: fold on ' - ' separator only
+ if ' - ' in name:
+ base, _, rest = name.partition(' - ')
+ return base.strip(), rest.strip(), 'named'
+ return name.strip(), '', 'named'
+
+
+def main(tsv, out):
+ rows = load(tsv)
+
+ # Build canonical groups. Key is casefolded canonical for grouping, but we
+ # preserve a display canonical (prefer an existing exact bare/standalone).
+ groups = {}
+ for r in rows:
+ canonical, suffix, kind = classify(r['name'])
+ key = (kind, canonical.casefold())
+ g = groups.setdefault(key, {
+ 'canonical': canonical,
+ 'kind': kind,
+ 'members': [],
+ 'exact_id': None,
+ })
+ r['_suffix'] = suffix
+ r['_canonical'] = canonical
+ g['members'].append(r)
+ # exact row: name equals canonical (no suffix)
+ if suffix == '' and r['name'].strip() == canonical:
+ g['exact_id'] = r['id']
+ g['canonical'] = r['name'].strip()
+
+ lines = []
+ lines.append('-- Auto-generated locations reconciliation. Review before running.')
+ lines.append('START TRANSACTION;')
+
+ touched = 0
+ for key, g in sorted(groups.items(), key=lambda kv: kv[0][1]):
+ members = sorted(g['members'], key=lambda r: r['id'])
+ # nothing to do: single member that is already the exact canonical
+ if len(members) == 1 and g['exact_id'] == members[0]['id']:
+ continue
+ touched += 1
+ canonical = g['canonical']
+
+ # choose target id: prefer exact bare row, else promote smallest id
+ if g['exact_id'] is not None:
+ target = g['exact_id']
+ promote = None
+ else:
+ target = members[0]['id']
+ promote = members[0]
+
+ # accumulate suffix/description text from all members (dedup, ordered)
+ details = []
+ for m in members:
+ for chunk in (m['_suffix'], m['desc']):
+ chunk = (chunk or '').strip()
+ if chunk and chunk not in details:
+ details.append(chunk)
+ desc_val = '; '.join(details)
+
+ lines.append('')
+ lines.append(f"-- {canonical} (target locationid {target}) <= "
+ + ', '.join(str(m['id']) for m in members))
+ if promote is not None:
+ lines.append(
+ f"UPDATE locations SET locationname='{esc(canonical)}' "
+ f"WHERE locationid={target};")
+ if desc_val:
+ lines.append(
+ f"UPDATE locations SET description='{esc(desc_val)}' "
+ f"WHERE locationid={target};")
+
+ others = [m['id'] for m in members if m['id'] != target]
+ if others:
+ ids = ','.join(str(i) for i in others)
+ lines.append(
+ f"UPDATE assets SET locationid={target} "
+ f"WHERE locationid IN ({ids});")
+ lines.append(
+ f"UPDATE subnets SET locationid={target} "
+ f"WHERE locationid IN ({ids});")
+ lines.append(
+ f"UPDATE locations SET parentlocationid={target} "
+ f"WHERE parentlocationid IN ({ids});")
+ lines.append(
+ f"DELETE FROM locations WHERE locationid IN ({ids});")
+
+ lines.append('')
+ lines.append('COMMIT;')
+ lines.append(f'-- groups touched: {touched}')
+
+ with open(out, 'w', encoding='utf-8') as fh:
+ fh.write('\n'.join(lines) + '\n')
+ print(f'wrote {out}: {touched} groups touched, {len(rows)} source rows')
+
+
+if __name__ == '__main__':
+ main(sys.argv[1], sys.argv[2])
diff --git a/shopdb/__init__.py b/shopdb/__init__.py
index 3e7ee884..8cc89ae8 100644
--- a/shopdb/__init__.py
+++ b/shopdb/__init__.py
@@ -57,7 +57,15 @@ from .plugins import plugin_manager
# permission / empty / position. BREAKING for any plugin still using the old
# shape, which is why it is recorded here: the change itself shipped earlier
# without a bump, and a contract that changes silently is not a contract.
-__contract_version__ = '0.20.0'
+# 0.21.0: added register_import_hook to shopdb.api so a plugin can own a VIRTUAL
+# COLUMN on an importable CSV table - a header the bulk importer accepts but
+# never writes - and a post-write row hook that turns that value into a plugin
+# concept. The computers plugin uses it for a machineassetnumber column on
+# computers.csv that builds the PC->machine controls link, so a site writes the
+# machine number in the PC row instead of hand-building a relationships sheet.
+# Core stays ignorant of the column's meaning. Additive name on the import
+# surface, minor bump.
+__contract_version__ = '0.21.0'
# Product release version (see ADR-007). The product version and the
# plugin-contract version above are distinct series with independent
diff --git a/shopdb/api/__init__.py b/shopdb/api/__init__.py
index 5a11f324..f802e387 100644
--- a/shopdb/api/__init__.py
+++ b/shopdb/api/__init__.py
@@ -95,6 +95,15 @@ from shopdb.core.services.dualpath import (
dualpath_single_machine_enabled,
)
+# Bulk-import extension point: a plugin registers a virtual column on an
+# importable CSV table plus a post-write row hook, so a relationship a site
+# would otherwise hand-build (a PC's machine number -> a controls link) can be
+# written in the row itself. Core never learns what the column means.
+from shopdb.core.services.csvimport import (
+ register_import_hook,
+ RowWarning,
+)
+
# Legacy employee directory lookup (read-only) used by notifications
from shopdb.utils.employee_db import employee_connection
from shopdb.utils.mailer import send_email, send_alert, send_webhook
@@ -261,6 +270,9 @@ __all__ = [
'resolve_asset_position',
'resolve_dualpath_pairs',
'dualpath_single_machine_enabled',
+ # Bulk-import extension point
+ 'register_import_hook',
+ 'RowWarning',
# Infrastructure
'db',
'cache',
diff --git a/shopdb/core/api/settings.py b/shopdb/core/api/settings.py
index 75fe8e14..d2f2830c 100644
--- a/shopdb/core/api/settings.py
+++ b/shopdb/core/api/settings.py
@@ -6,6 +6,7 @@ import re
from flask import Blueprint, request, current_app
from flask_jwt_extended import jwt_required, get_jwt_identity
from werkzeug.utils import secure_filename
+from sqlalchemy import select as sa_select
from shopdb.extensions import db, cache
from shopdb.core.models import Setting, AuditLog
@@ -332,6 +333,27 @@ def download_import_template(tablename):
return response
+def _existing_hostname_map():
+ """Case-insensitive hostname -> assetnumber map for existing computers.
+
+ Azure device-group exports align on hostname (their displayName), while the
+ import engine keys assets on assetnumber. This lets the Azure adapter route
+ a matched device to its real asset so the import updates in place.
+ """
+ assets = db.metadata.tables['assets']
+ computers = db.metadata.tables['computers']
+ query = sa_select(
+ computers.c.hostname, assets.c.assetnumber
+ ).select_from(
+ computers.join(assets, computers.c.assetid == assets.c.assetid)
+ ).where(computers.c.hostname.isnot(None))
+ mapping = {}
+ for hostname, assetnumber in db.session.execute(query):
+ if hostname:
+ mapping[hostname.strip().lower()] = assetnumber
+ return mapping
+
+
@settings_bp.route('/asset-import/import', methods=['POST'])
@jwt_required()
@require_role('admin')
@@ -344,10 +366,16 @@ def import_asset_csvs():
transaction back and reports what WOULD change; commit=true persists it. The
whole run is all-or-nothing - if any file has a row problem, nothing is
written even on a commit.
+
+ A 'source' form field selects a foreign adapter (azure, thor); it defaults
+ to 'native' where uploads are already canonical per-table files. Foreign
+ uploads are transformed into canonical files first, then run identically.
"""
from sqlalchemy.exc import SQLAlchemyError
from shopdb.core.services.csvimport import (
ImportError_, Resolver, dependency_order, import_csv, table_from_filename)
+ from shopdb.core.services.importsources import (
+ ImportSourceError, get_source)
uploads = request.files.getlist('files')
if not uploads:
@@ -356,21 +384,66 @@ def import_asset_csvs():
commit = str(request.form.get('commit', '')).strip().lower() in (
'1', 'true', 'yes', 'on')
- # Map each upload to a table by filename. A later file with the same table
- # name wins, matching the folder-import behaviour (one file per table).
+ # A foreign source (azure, thor) is transformed into canonical files before
+ # import. 'native' (or nothing) means the files already ARE canonical.
+ source_key = (request.form.get('source') or 'native').strip().lower()
+ source = None
+ if source_key and source_key != 'native':
+ try:
+ source = get_source(source_key)
+ except ImportSourceError as exc:
+ return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
+
+ source_options = {
+ 'computertype': (request.form.get('computertype') or '').strip(),
+ 'category': (request.form.get('category') or '').strip(),
+ }
+
+ # Azure and Thor align on hostname: their device name is the ShopDB
+ # computers.hostname, not the assetnumber. Hand the adapter a
+ # case-insensitive hostname -> existing assetnumber map so devices that
+ # already exist update in place (keyed to their real asset) instead of
+ # minting a duplicate asset.
+ if source is not None and getattr(source, 'key', None) in ('azure', 'thor'):
+ source_options['hostname_map'] = _existing_hostname_map()
+
texts = {}
filenames = {}
- for upload in uploads:
- name = upload.filename or ''
- if not name.lower().endswith('.csv'):
- continue
- table = table_from_filename(name)
- try:
- texts[table] = upload.read().decode('utf-8-sig')
- except (UnicodeDecodeError, OSError):
- return error_response(ErrorCodes.VALIDATION_ERROR,
- 'Could not read %s as UTF-8 text' % name)
- filenames[table] = name
+ if source is not None:
+ # One foreign export becomes several canonical files. Each upload is
+ # transformed and its canonical outputs merged; a later file's table
+ # wins, matching the native one-file-per-table rule.
+ for upload in uploads:
+ name = upload.filename or ''
+ if not name.lower().endswith('.csv'):
+ continue
+ try:
+ raw = upload.read().decode('utf-8-sig')
+ except (UnicodeDecodeError, OSError):
+ return error_response(ErrorCodes.VALIDATION_ERROR,
+ 'Could not read %s as UTF-8 text' % name)
+ try:
+ produced = source.transform(raw, options=source_options)
+ except ImportSourceError as exc:
+ return error_response(ErrorCodes.VALIDATION_ERROR,
+ '%s: %s' % (name, str(exc)))
+ for table, csv_text in produced.items():
+ texts[table] = csv_text
+ filenames[table] = '%s (from %s)' % (table, name)
+ else:
+ # Map each upload to a table by filename. A later file with the same
+ # table name wins, matching the folder-import behaviour.
+ for upload in uploads:
+ name = upload.filename or ''
+ if not name.lower().endswith('.csv'):
+ continue
+ table = table_from_filename(name)
+ try:
+ texts[table] = upload.read().decode('utf-8-sig')
+ except (UnicodeDecodeError, OSError):
+ return error_response(ErrorCodes.VALIDATION_ERROR,
+ 'Could not read %s as UTF-8 text' % name)
+ filenames[table] = name
if not texts:
return error_response(ErrorCodes.VALIDATION_ERROR,
@@ -389,6 +462,7 @@ def import_asset_csvs():
resolver = Resolver(reconcile_threshold=reconcile_threshold)
files_report = []
problem_count = 0
+ warning_count = 0
reconciled_count = 0
request_error = None
@@ -409,7 +483,17 @@ def import_asset_csvs():
{'line': p.line, 'column': p.column, 'message': p.message}
for p in result.problems
]
+ # Amber tier: a warning is a finding beside a row that still
+ # imported (an unresolved machine link, a contested bay), NOT a
+ # reason to fail the file. Reported alongside problems so the UI can
+ # show Errors / Warnings / Info distinctly, but never gates commit.
+ warnings = [
+ {'line': w.line, 'column': w.column, 'reason': w.reason,
+ 'action': w.action, 'subject': w.subject, 'target': w.target}
+ for w in result.warnings
+ ]
problem_count += len(problems)
+ warning_count += len(warnings)
reconciled_count += len(result.reconciliations)
files_report.append({
'table': table,
@@ -417,6 +501,7 @@ def import_asset_csvs():
'created': result.created,
'updated': result.updated,
'problems': problems,
+ 'warnings': warnings,
'reconciliations': result.reconciliations,
})
@@ -446,6 +531,7 @@ def import_asset_csvs():
'files': files_report,
'skipped': skipped,
'problem_count': problem_count,
+ 'warning_count': warning_count,
'reconciled_count': reconciled_count,
'reconcile_threshold': reconcile_threshold,
'total_created': total_created,
diff --git a/shopdb/core/services/csvimport.py b/shopdb/core/services/csvimport.py
index 462e8b57..0adeb3a2 100644
--- a/shopdb/core/services/csvimport.py
+++ b/shopdb/core/services/csvimport.py
@@ -74,6 +74,71 @@ IMPORTABLE = OrderedDict([
('machines', {'label': None, 'natural': 'assetid'}),
])
+# A small, INTERNALLY CONSISTENT worked example per table, in import order. A
+# site can download the set, see the pattern, and keep it as a starter workbook:
+# every name referenced by a later file (Bay 3, Dell, Computer, ...) is defined
+# in an earlier one, so the whole set imports cleanly as a demo before a single
+# real row is typed. Emitted as commented rows the engine skips, so a freshly
+# downloaded template never writes example data by accident. Values avoid commas
+# on purpose - the generator writes them without CSV quoting.
+EXAMPLE_ROWS = OrderedDict([
+ ('assetstatuses', [
+ {'status': 'In Use', 'description': 'Actively deployed', 'color': '#22c55e', 'isactive': '1'},
+ {'status': 'In Storage', 'description': 'Spare - not deployed', 'color': '#eab308', 'isactive': '1'},
+ {'status': 'Retired', 'description': 'Removed from service', 'color': '#ef4444', 'isactive': '1'},
+ ]),
+ ('assettypes', [
+ {'assettype': 'Computer', 'pluginname': 'computers', 'tablename': 'computers', 'description': 'PCs - servers - workstations', 'icon': 'laptop', 'color': '#3b82f6', 'isactive': '1'},
+ {'assettype': 'Machine', 'pluginname': 'machines', 'tablename': 'machines', 'description': 'Shop-floor machines', 'icon': 'cpu', 'color': '#8b5cf6', 'isactive': '1'},
+ ]),
+ ('locationtypes', [
+ {'locationtype': 'Building', 'description': 'A physical building', 'color': '#64748b', 'isactive': '1'},
+ {'locationtype': 'Bay', 'description': 'A shop-floor bay', 'color': '#0ea5e9', 'isactive': '1'},
+ ]),
+ ('modeltypes', [
+ {'modeltype': 'Desktop', 'category': 'Computer', 'description': 'Desktop PC chassis', 'icon': 'desktop', 'isactive': '1'},
+ {'modeltype': 'CNC Machine', 'category': 'Machine', 'description': 'CNC machining center', 'icon': 'cpu', 'isactive': '1'},
+ ]),
+ ('computertypes', [
+ {'computertype': 'Shopfloor PC', 'description': 'PC on the shop floor', 'icon': 'monitor', 'color': '#3b82f6', 'isactive': '1'},
+ {'computertype': 'Engineer Workstation', 'description': 'CAD/CAM workstation', 'icon': 'laptop', 'color': '#6366f1', 'isactive': '1'},
+ ]),
+ ('machinetypes', [
+ {'machinetype': 'CNC Mill', 'description': 'Computer numerical control mill', 'icon': 'cpu', 'color': '#8b5cf6', 'isactive': '1'},
+ {'machinetype': 'CMM', 'description': 'Coordinate measuring machine', 'icon': 'ruler', 'color': '#14b8a6', 'isactive': '1'},
+ ]),
+ ('businessunits', [
+ {'businessunit': 'Machining', 'code': 'MACH', 'description': 'Machining department', 'isactive': '1'},
+ {'businessunit': 'Quality', 'code': 'QUAL', 'description': 'Quality and metrology', 'isactive': '1'},
+ ]),
+ ('locations', [
+ {'locationname': 'Building 1', 'building': 'Building 1', 'floor': '1', 'description': 'Main manufacturing building', 'locationtypeid': 'Building', 'isactive': '1'},
+ {'locationname': 'Bay 3', 'building': 'Building 1', 'floor': '1', 'room': 'Machining bay 3', 'description': 'Machining bay 3', 'locationtypeid': 'Bay', 'isactive': '1'},
+ ]),
+ ('vendors', [
+ {'vendor': 'Dell', 'description': 'Computer hardware', 'website': 'https://www.dell.com', 'supportphone': '800-555-0100', 'supportemail': 'support@dell.com', 'isactive': '1'},
+ {'vendor': 'Haas Automation', 'description': 'CNC machine tools', 'website': 'https://www.haascnc.com', 'supportphone': '800-555-0199', 'isactive': '1'},
+ ]),
+ ('models', [
+ {'modelnumber': 'OptiPlex 7010', 'modeltypeid': 'Desktop', 'vendorid': 'Dell', 'description': 'Small form factor desktop', 'isactive': '1'},
+ {'modelnumber': 'VF-2', 'modeltypeid': 'CNC Machine', 'vendorid': 'Haas Automation', 'description': '40-taper vertical mill', 'isactive': '1'},
+ ]),
+ ('operatingsystems', [
+ {'osname': 'Windows 11 (24H2)', 'osversion': '24H2', 'architecture': 'x64', 'endoflife': '2027-10-12', 'isactive': '1'},
+ {'osname': 'Windows 10 (22H2)', 'osversion': '22H2', 'architecture': 'x64', 'endoflife': '2025-10-14', 'isactive': '1'},
+ ]),
+ ('assets', [
+ {'assetnumber': 'PC-001', 'name': 'Shopfloor PC 001', 'serialnumber': 'SN-PC-001', 'assettypeid': 'Computer', 'statusid': 'In Use', 'locationid': 'Bay 3', 'businessunitid': 'Machining', 'isactive': '1'},
+ {'assetnumber': 'CNC-014', 'name': 'Haas VF-2 #14', 'serialnumber': 'SN-CNC-014', 'assettypeid': 'Machine', 'statusid': 'In Use', 'locationid': 'Bay 3', 'businessunitid': 'Machining', 'isactive': '1'},
+ ]),
+ ('computers', [
+ {'assetid': 'PC-001', 'computertypeid': 'Shopfloor PC', 'hostname': 'WS-PC-001', 'osid': 'Windows 11 (24H2)', 'vendorid': 'Dell', 'modelnumberid': 'OptiPlex 7010', 'loggedinuser': 'jdoe', 'lastreporteddate': '2026-08-01', 'lastboottime': '2026-08-01', 'isactive': '1', 'machineassetnumber': 'CNC-014'},
+ ]),
+ ('machines', [
+ {'assetid': 'CNC-014', 'machinetypeid': 'CNC Mill', 'vendorid': 'Haas Automation', 'modelnumberid': 'VF-2', 'requiresmanualconfig': '0', 'islocationonly': '0', 'lastmaintenancedate': '2026-06-01', 'nextmaintenancedate': '2026-12-01', 'maintenanceintervaldays': '180', 'isactive': '1'},
+ ]),
+])
+
# Columns a CSV may never set: surrogate keys the database owns, and audit
# stamps. Accepting an id from a spreadsheet is how two sites end up with
# colliding primary keys the first time anyone merges data.
@@ -111,6 +176,75 @@ CORPORATE_SUFFIXES = {
DEFAULT_RECONCILE_THRESHOLD = 0.85
+# -- plugin import hooks (inversion of control) ---------------------------------
+#
+# A spreadsheet is the wrong place to type an integer foreign key, and it is
+# also the wrong place to split a relationship across two files: a site that
+# knows "this PC drives machine 3015" wants to write that number in the PC row,
+# not hand-build an assetrelationships sheet keyed on ids it never sees.
+#
+# But the machine link is a computers-plugin concept, and this module is core:
+# it must not import a plugin. So a plugin REGISTERS itself here instead. It
+# declares a VIRTUAL COLUMN - a header this table accepts but never writes to
+# the database - and a HOOK that runs once per written row to turn that column's
+# value into whatever the plugin means by it (here, a PC->machine controls link).
+# Core stays ignorant of what the column is for; the plugin owns the meaning.
+_VIRTUAL_COLUMNS = {} # tablename -> {columnname: description}
+_ROW_HOOKS = {} # tablename -> [hook, ...]
+
+
+def register_import_hook(tablename, virtual_columns, hook):
+ """Let a plugin own virtual column(s) on an importable table.
+
+ virtual_columns: either a sequence of header names, or a mapping of
+ name -> one-line description. Each is accepted for ``tablename`` but never
+ written - it is pulled out of the row and handed to the hook instead of the
+ table. A description, when given, is shown in the generated template's
+ column notes so an operator knows what to type.
+
+ hook(rowvalues, virtualvalues, line) -> list[RowWarning]: called once per
+ written row, AFTER the row is inserted/updated, so its id exists. It runs
+ inside the caller's transaction - which a dry run rolls back - so it must
+ not commit, and must not do anything a rollback cannot undo (no emails). A
+ hook reports trouble by returning RowWarning, never by raising: a bad
+ machine number is an amber note beside a PC that still imports, not a failed
+ file.
+
+ Idempotent by contract: registering the same (table, column, hook) twice is
+ a no-op, so a plugin re-init during tests or a reload does not double-fire.
+ """
+ cols = _VIRTUAL_COLUMNS.setdefault(tablename, {})
+ if isinstance(virtual_columns, dict):
+ for name, description in virtual_columns.items():
+ cols[name] = description or ''
+ else:
+ for name in virtual_columns:
+ cols.setdefault(name, '')
+ hooks = _ROW_HOOKS.setdefault(tablename, [])
+ # Dedupe by the underlying function's qualified name, not by the bound
+ # method: a plugin re-init (a new app context in the test suite, a reload in
+ # prod) makes a fresh instance whose bound method is != the old one, and a
+ # symlinked plugin can even be imported under two module paths, so an
+ # identity check would append a second hook and the row would be processed -
+ # and warned about - more than once.
+ def _identity(fn):
+ func = getattr(fn, '__func__', fn)
+ return getattr(func, '__qualname__', repr(func))
+
+ if _identity(hook) not in [_identity(existing) for existing in hooks]:
+ hooks.append(hook)
+
+
+def virtual_columns_for(tablename):
+ """Header names this table accepts but does not store. Never None."""
+ return set(_VIRTUAL_COLUMNS.get(tablename, ()))
+
+
+def virtual_column_notes(tablename):
+ """(name, description) for each virtual column, in registration order."""
+ return list(_VIRTUAL_COLUMNS.get(tablename, {}).items())
+
+
def normalize_label(value):
"""Fold a label to its comparison form: lowercase, no punctuation, no
corporate suffix, single-spaced. 'Dell Inc.' and 'DELL' both become 'dell'.
@@ -370,6 +504,32 @@ class RowProblem(object):
return "%s: %s" % (where, self.message)
+class RowWarning(object):
+ """A non-blocking, amber-tier finding.
+
+ Errors (RowProblem) stop the whole import; a RowWarning does NOT - the row
+ it belongs to still imports. It exists so a caveat (an unresolved or
+ conflicting machine link, say) is loud and REVIEWABLE without failing an
+ otherwise-clean file. Every warning names WHERE it happened, WHY, and the
+ single ACTION that fixes it, so a cold user can correct it in place - the
+ convention IT Glue / RMM imports train.
+ """
+
+ def __init__(self, line, column, reason, action, subject=None, target=None):
+ self.line = line
+ self.column = column
+ self.reason = reason # what is wrong, in plain terms
+ self.action = action # the one corrective step to take
+ self.subject = subject # the row's actor (e.g. the PC hostname)
+ self.target = target # what it referenced (e.g. the machine number)
+
+ def __str__(self):
+ where = "line %d" % self.line
+ if self.column:
+ where += ", column '%s'" % self.column
+ return "%s: %s - %s" % (where, self.reason, self.action)
+
+
class ImportResult(object):
def __init__(self, tablename):
self.tablename = tablename
@@ -378,14 +538,18 @@ class ImportResult(object):
self.problems = []
self.skipped = 0
self.reconciliations = []
+ self.warnings = []
@property
def ok(self):
+ # Warnings are amber, not red: a file with only warnings is still ok to
+ # commit. Only problems (errors) block.
return not self.problems
def summary(self):
- return "%s: %d new, %d updated, %d problem(s)" % (
- self.tablename, self.created, self.updated, len(self.problems))
+ return "%s: %d new, %d updated, %d problem(s), %d warning(s)" % (
+ self.tablename, self.created, self.updated,
+ len(self.problems), len(self.warnings))
def read_rows(text):
@@ -439,9 +603,10 @@ def import_csv(tablename, text, resolver=None, commit=False):
header, rows = read_rows(text)
allowed = {c.name: c for c in importable_columns(tablename)}
- unknown = [h for h in header if h not in allowed]
+ virtual = virtual_columns_for(tablename)
+ unknown = [h for h in header if h not in allowed and h not in virtual]
if unknown:
- known = ', '.join(sorted(allowed))
+ known = ', '.join(sorted(set(allowed) | virtual))
result.problems.append(RowProblem(
1, None, "unknown column(s): %s\n this table accepts: %s" % (', '.join(unknown), known)))
return result
@@ -461,7 +626,17 @@ def import_csv(tablename, text, resolver=None, commit=False):
prepared = []
for line, raw in rows:
values = {}
+ virtualvalues = {}
failed = False
+ # Virtual columns belong to a plugin hook, not the table. Pull them out
+ # first so they never reach an insert/update, and stash them for the
+ # post-write hook. Whitespace-trimmed; an empty cell is dropped so the
+ # hook only sees values a row actually carried.
+ for name in virtual:
+ if name in raw:
+ text = (raw[name] or '').strip()
+ if text != '':
+ virtualvalues[name] = text
for name, column in allowed.items():
if name not in raw:
continue
@@ -496,16 +671,18 @@ def import_csv(tablename, text, resolver=None, commit=False):
if values.get(natural) in (None, ''):
result.problems.append(RowProblem(line, natural, 'must have a value'))
continue
- prepared.append((line, values))
+ prepared.append((line, values, virtualvalues))
if not result.ok:
return result
+ hooks = _ROW_HOOKS.get(tablename, ())
+
# Second pass: decide insert vs update. Counted even on a dry run, because
# "this will change 4000 rows" is the number an operator needs before saying
# yes.
label = _label_column(tablename)
- for line, values in prepared:
+ for line, values, virtualvalues in prepared:
existing = db.session.execute(
table.select().where(table.c[natural] == values[natural])).first()
if existing is None:
@@ -522,6 +699,15 @@ def import_csv(tablename, text, resolver=None, commit=False):
primary = list(table.primary_key)[0].name
resolver.remember(tablename, values[label], getattr(existing, primary))
+ # Fire plugin hooks AFTER the write, so the row's id exists. Only when
+ # the row actually carried a virtual value - a hook that has nothing to
+ # act on should never run. Warnings are collected, never raised: an
+ # unresolved machine number is amber, not a failed file.
+ if hooks and virtualvalues:
+ for hook in hooks:
+ result.warnings.extend(
+ hook(values, virtualvalues, line) or [])
+
return result
@@ -556,24 +742,55 @@ def generate_template(tablename):
bits.append('text, max %d' % column.type.length)
notes.append("# %-24s %s" % (column.name, ', '.join(bits)))
+ # Virtual columns a plugin owns: accepted on this table but written by the
+ # plugin's hook, not the schema. Appended after the real columns so the
+ # header carries them and the notes explain what they do.
+ virtualnames = []
+ for name, description in virtual_column_notes(tablename):
+ virtualnames.append(name)
+ header.append(name)
+ note = 'optional, ' + description if description else 'optional'
+ notes.append("# %-24s %s" % (name, note))
+
spec = IMPORTABLE[tablename]
+ examples = EXAMPLE_ROWS.get(tablename, [])
+ exampleorder = [c.name for c in columns] + virtualnames
lines = [
"# %s.csv - generated from the ShopDB schema, do not hand-maintain" % tablename,
"#",
"# Rows are matched on '%s'. Re-importing updates a matching row rather" % spec['natural'],
"# than creating a second one, so it is safe to correct this file and run again.",
"#",
+ "# HOW TO USE THIS TEMPLATE:",
+ "# 1. Delete everything above the header row (every line starting with #).",
+ "# 2. Fill in your own rows using the example row(s) below as a guide.",
+ "# 3. DELETE THE EXAMPLE DATA before you import - the example rows are",
+ "# live data and WILL be imported if you leave them in.",
+ "#",
"# Columns:",
] + notes + [
"#",
- "# Lines starting with # are ignored. Delete the example row before importing.",
+ "# --- header and example rows follow. Keep the header; replace the examples. ---",
','.join(header),
- '# ' + ','.join(_example(c, required) for c in columns),
+ ] + [
+ ','.join(_example_value(row.get(name, '')) for name in exampleorder)
+ for row in examples
]
return '\n'.join(lines) + '\n'
+def _example_value(value):
+ """A single example cell. Commas would break the unquoted CSV the template
+ writes, so they are replaced with a dash; example data is authored without
+ them, this is only a guard.
+ """
+ return str(value).replace(',', ' -')
+
+
def _example(column, required):
+ """Kept for callers/tests that ask for a single synthetic cell. The template
+ generator now emits the curated EXAMPLE_ROWS instead.
+ """
target = _fk_target(column)
if target:
label = _label_column(target[0])
diff --git a/shopdb/core/services/importsources.py b/shopdb/core/services/importsources.py
new file mode 100644
index 00000000..6aa56041
--- /dev/null
+++ b/shopdb/core/services/importsources.py
@@ -0,0 +1,325 @@
+"""Foreign import-source adapters.
+
+The CSV import engine in csvimport.py speaks exactly one dialect: the canonical
+per-table files (assets.csv, computers.csv, vendors.csv, ...). Foreign exports -
+an Entra/Azure device-group member list, a Thor PC inventory dump - do not match
+that dialect and never will; they are owned by other systems.
+
+Rather than teach the engine every foreign schema, each adapter here TRANSFORMS
+a foreign export into the canonical files and hands them back as
+{tablename: csv_text}. The caller then runs those through the ordinary
+import_csv path, so dry-run/commit, dependency ordering, and reconciliation all
+work unchanged.
+
+An adapter declares a header signature so an upload can be auto-detected, and a
+transform() that yields the canonical files. Adapters never touch the database.
+"""
+
+import csv
+import io
+
+
+class ImportSourceError(Exception):
+ """A foreign file could not be transformed (bad header, empty, etc.)."""
+
+
+# Canonical defaults for a discovered PC. Azure carries only a hostname, so
+# every device it creates is a computer that is In Use until something richer
+# (Thor, the collector) says otherwise.
+DEFAULT_ASSETTYPE = 'computer'
+DEFAULT_STATUS = 'In Use'
+
+
+def _read(text):
+ """Parse CSV text into (header, list-of-dicts), tolerating a UTF-8 BOM."""
+ stream = io.StringIO(text)
+ reader = csv.DictReader(stream)
+ header = [h.strip() for h in (reader.fieldnames or [])]
+ rows = []
+ for raw in reader:
+ rows.append({(k.strip() if k else k): (v or '').strip()
+ for k, v in raw.items()})
+ return header, rows
+
+
+def _write(header, rows):
+ """Render canonical rows back to CSV text with a trailing newline."""
+ out = io.StringIO()
+ writer = csv.DictWriter(out, fieldnames=header, extrasaction='ignore')
+ writer.writeheader()
+ for row in rows:
+ writer.writerow(row)
+ return out.getvalue()
+
+
+def _has_columns(header, required):
+ lowered = {h.lower() for h in header}
+ return all(col.lower() in lowered for col in required)
+
+
+class AzureDeviceGroupSource:
+ """Entra/Azure device-group member export.
+
+ Columns: displayName, objectType, mail, userType, id, deviceId,
+ userPrincipalName, isUser, isGroup, isGuest. Only device rows carry signal;
+ displayName is the hostname and deviceId is the Entra device GUID. The export
+ has no model, OS, location or user data, so this creates/updates minimal
+ asset + computer rows keyed on hostname.
+
+ options:
+ computertype - optional name stamped onto every computer's computertypeid
+ (e.g. 'Shopfloor PC' for a SHP group, 'Inspection PC' for
+ BUS). Resolved by the engine against computertypes.
+ category - optional Entra device category / group membership label
+ stamped onto every computer's azuredevicecategory (e.g.
+ SFLD_DYN_Device_Group_Wilmington_BUS_NC_COUNTRY:USA).
+ hostname_map - optional {lowercased hostname: existing assetnumber}. A
+ device whose displayName matches an existing computer's
+ hostname is routed to that asset and updated in place
+ (no new asset row); unmatched devices create a new
+ asset + computer keyed on the hostname.
+ """
+
+ key = 'azure'
+ label = 'Azure device group'
+ signature = ('displayName', 'deviceId', 'objectType')
+
+ @classmethod
+ def matches(cls, header):
+ return _has_columns(header, cls.signature)
+
+ @classmethod
+ def transform(cls, text, options=None):
+ options = options or {}
+ computertype = (options.get('computertype') or '').strip()
+ category = (options.get('category') or '').strip()
+ hostname_map = options.get('hostname_map') or {}
+ header, rows = _read(text)
+ if not _has_columns(header, cls.signature):
+ raise ImportSourceError(
+ 'not an Azure device-group export (expected displayName, '
+ 'deviceId, objectType columns)')
+
+ assets = []
+ computers = []
+ seen = set()
+ for row in rows:
+ objecttype = (row.get('objectType') or '').strip().lower()
+ if objecttype and objecttype != 'device':
+ continue
+ hostname = (row.get('displayName') or '').strip()
+ if not hostname or hostname.lower() in seen:
+ continue
+ seen.add(hostname.lower())
+
+ # Align on hostname: if this device already exists in ShopDB, point
+ # the computer row at its real asset so the engine updates it in
+ # place. Only create a new asset when there is no hostname match.
+ existing_assetnumber = hostname_map.get(hostname.lower())
+ computer = {
+ 'assetid': existing_assetnumber if existing_assetnumber is not None else hostname,
+ 'hostname': hostname,
+ 'azuredeviceid': (row.get('deviceId') or '').strip(),
+ 'azureobjectid': (row.get('id') or '').strip(),
+ }
+ if computertype:
+ computer['computertypeid'] = computertype
+ if category:
+ computer['azuredevicecategory'] = category
+ computers.append(computer)
+
+ if existing_assetnumber is None:
+ assets.append({
+ 'assetnumber': hostname,
+ 'name': hostname,
+ 'assettypeid': DEFAULT_ASSETTYPE,
+ 'statusid': DEFAULT_STATUS,
+ })
+
+ if not computers:
+ raise ImportSourceError('no device rows found in the Azure export')
+
+ asset_header = ['assetnumber', 'name', 'assettypeid', 'statusid']
+ computer_header = ['assetid', 'hostname', 'azuredeviceid', 'azureobjectid']
+ if computertype:
+ computer_header.append('computertypeid')
+ if category:
+ computer_header.append('azuredevicecategory')
+ produced = {
+ 'computers': _write(computer_header, computers),
+ }
+ # Only emit an assets file when there are new devices to create; an
+ # empty assets.csv would just be an extra clean no-op table.
+ if assets:
+ produced['assets'] = _write(asset_header, assets)
+ return produced
+
+
+class ThorSource:
+ """Thor PC-inventory export (internal GE Aerospace tool).
+
+ A wide, fixed-format PC inventory. Maps the columns ShopDB owns into
+ canonical rows: hostname, vendor (BIOS Manf.), model, operating system,
+ logged-in user, last-reported and last-boot timestamps, and location. The
+ network columns (MAC, switch, VLAN, IP) and the machine 'Attached' link are
+ out of scope for this first pass.
+
+ options:
+ computertype - optional name stamped onto every computer's computertypeid.
+ Resolved by the engine against computertypes.
+ hostname_map - optional {lowercased hostname: existing assetnumber}. A
+ host whose Host Name matches an existing computer's hostname
+ is routed to that asset and updated in place (no new asset
+ row); unmatched hosts create a new asset + computer keyed on
+ the hostname.
+ """
+
+ key = 'thor'
+ label = 'Thor PC inventory'
+ signature = ('Host Name', 'BIOS Manf.', 'OS Version')
+
+ @classmethod
+ def matches(cls, header):
+ return _has_columns(header, cls.signature)
+
+ @classmethod
+ def transform(cls, text, options=None):
+ options = options or {}
+ computertype = (options.get('computertype') or '').strip()
+ hostname_map = options.get('hostname_map') or {}
+ header, rows = _read(text)
+ if not _has_columns(header, cls.signature):
+ raise ImportSourceError(
+ 'not a Thor export (expected Host Name, BIOS Manf., OS Version '
+ 'columns)')
+
+ vendors = {}
+ models = {}
+ operatingsystems = {}
+ assets = []
+ computers = []
+ seen = set()
+ for row in rows:
+ hostname = (row.get('Host Name') or '').strip()
+ if not hostname or hostname.lower() in seen:
+ continue
+ seen.add(hostname.lower())
+
+ vendor = (row.get('BIOS Manf.') or '').strip()
+ model = (row.get('Model') or '').strip()
+ osname = cls._osname(row)
+ location = (row.get('Location') or '').strip()
+ loggedinuser = cls._logon(row)
+
+ if vendor:
+ vendors[vendor.lower()] = vendor
+ if model:
+ models[model.lower()] = model
+ if osname:
+ operatingsystems[osname.lower()] = osname
+
+ # Align on hostname: if this host already exists in ShopDB, point
+ # the computer row at its real asset so the engine updates it in
+ # place. Only create a new asset when there is no hostname match.
+ existing_assetnumber = hostname_map.get(hostname.lower())
+ if existing_assetnumber is None:
+ assets.append({
+ 'assetnumber': hostname,
+ 'name': hostname,
+ 'assettypeid': DEFAULT_ASSETTYPE,
+ 'statusid': DEFAULT_STATUS,
+ 'locationid': location,
+ })
+
+ computer = {
+ 'assetid': existing_assetnumber if existing_assetnumber is not None else hostname,
+ 'hostname': hostname,
+ 'osid': osname,
+ 'vendorid': vendor,
+ 'modelnumberid': model,
+ 'loggedinuser': loggedinuser,
+ 'lastreporteddate': cls._date(row.get('Last Updated')),
+ 'lastboottime': cls._date(row.get('Boot Time')),
+ }
+ if computertype:
+ computer['computertypeid'] = computertype
+ computers.append(computer)
+
+ if not computers:
+ raise ImportSourceError('no host rows found in the Thor export')
+
+ out = {}
+ if vendors:
+ out['vendors'] = _write(['vendor'],
+ [{'vendor': v} for v in vendors.values()])
+ if models:
+ out['models'] = _write(['modelnumber'],
+ [{'modelnumber': m} for m in models.values()])
+ if operatingsystems:
+ out['operatingsystems'] = _write(
+ ['osname'], [{'osname': o} for o in operatingsystems.values()])
+
+ asset_header = ['assetnumber', 'name', 'assettypeid', 'statusid', 'locationid']
+ computer_header = ['assetid', 'hostname', 'osid', 'vendorid',
+ 'modelnumberid', 'loggedinuser', 'lastreporteddate',
+ 'lastboottime']
+ if computertype:
+ computer_header.append('computertypeid')
+ out['computers'] = _write(computer_header, computers)
+ # Only emit an assets file when there are new hosts to create; an empty
+ # assets.csv would just be an extra clean no-op table.
+ if assets:
+ out['assets'] = _write(asset_header, assets)
+ return out
+
+ @staticmethod
+ def _osname(row):
+ """Combine OS Version and OS ID into a single OS name.
+
+ Thor splits the OS into a build ('10.0.26100') and a marketing id
+ ('24H2'); neither alone is meaningful, so join them.
+ """
+ version = (row.get('OS Version') or '').strip()
+ osid = (row.get('OS ID') or '').strip()
+ if version and osid:
+ return '%s (%s)' % (version, osid)
+ return version or osid
+
+ @staticmethod
+ def _logon(row):
+ """Take the account name out of a DOMAIN\\user or HOST\\user LOGON."""
+ logon = (row.get('LOGON') or '').strip()
+ if '\\' in logon:
+ return logon.rsplit('\\', 1)[-1]
+ return logon
+
+ @staticmethod
+ def _date(value):
+ """Normalize a Thor timestamp to a form the engine's date coercion
+ accepts, or empty when it cannot be parsed. Leaves already-ISO values
+ alone; the engine tries several formats itself.
+ """
+ return (value or '').strip()
+
+
+SOURCES = {
+ AzureDeviceGroupSource.key: AzureDeviceGroupSource,
+ ThorSource.key: ThorSource,
+}
+
+
+def get_source(key):
+ source = SOURCES.get((key or '').strip().lower())
+ if source is None:
+ raise ImportSourceError(
+ "unknown import source '%s' (known: %s)"
+ % (key, ', '.join(sorted(SOURCES))))
+ return source
+
+
+def detect_source(header):
+ """Return the adapter whose signature matches this header, or None."""
+ for source in SOURCES.values():
+ if source.matches(header):
+ return source
+ return None
diff --git a/tests/test_csv_import.py b/tests/test_csv_import.py
index d75fdec3..ec1a1dfc 100644
--- a/tests/test_csv_import.py
+++ b/tests/test_csv_import.py
@@ -173,14 +173,17 @@ def test_generated_templates_only_contain_real_columns(db):
A hand-written template set had invented columns on 7 of 11 tables and named
a table that does not exist. Generating from the schema is what makes that
- impossible - so assert the generator actually does it.
+ impossible - so assert the generator actually does it. Plugin-registered
+ virtual columns (machineassetnumber) are legitimate, not invented, so they
+ are allowed alongside the table's real columns.
"""
+ from shopdb.core.services.csvimport import virtual_columns_for
for table in IMPORTABLE:
text = generate_template(table)
header = [line for line in text.splitlines() if line and not line.startswith('#')][0]
- real = {c.name for c in importable_columns(table)}
+ allowed = {c.name for c in importable_columns(table)} | virtual_columns_for(table)
for column in header.split(','):
- assert column in real, '%s.csv offers a column that does not exist: %s' % (table, column)
+ assert column in allowed, '%s.csv offers a column that does not exist: %s' % (table, column)
def test_every_template_includes_its_required_columns(db):
@@ -234,3 +237,46 @@ def test_a_dry_run_over_several_files_resolves_across_them(db):
assert result.ok, [str(p) for p in result.problems]
_db.session.rollback()
assert count('assets') == 0
+
+
+def _example_files():
+ """Reconstruct each template's example rows as an importable CSV: the header
+ plus the live (uncommented) example data rows that follow it."""
+ files = {}
+ for table in IMPORTABLE:
+ header = None
+ rows = []
+ for line in generate_template(table).splitlines():
+ if not line.strip():
+ continue
+ if line.startswith('#'):
+ continue
+ if header is None:
+ header = line
+ else:
+ rows.append(line)
+ files[table] = '\n'.join([header] + rows) + '\n'
+ return files
+
+
+def test_template_examples_are_present_for_every_table(db):
+ """Each template ships at least one worked example row, so a downloaded
+ set is a usable starter workbook rather than a bare header."""
+ files = _example_files()
+ for table in IMPORTABLE:
+ rows = files[table].splitlines()[1:]
+ assert rows, '%s.csv has no example row' % table
+
+
+def test_template_example_set_imports_cleanly(db):
+ """The whole curated example set must import as-is: every name a later file
+ references is defined by an earlier one, with no self-references that the
+ validation pass cannot resolve. This is the promise the workbook makes."""
+ resolver = Resolver()
+ files = _example_files()
+ for table in IMPORTABLE:
+ result = import_csv(table, files[table], resolver=resolver)
+ assert result.ok, '%s.csv example failed: %s' % (
+ table, [str(p) for p in result.problems])
+ _db.session.rollback()
+
diff --git a/tests/test_import_sources.py b/tests/test_import_sources.py
new file mode 100644
index 00000000..f844c9b1
--- /dev/null
+++ b/tests/test_import_sources.py
@@ -0,0 +1,201 @@
+"""Foreign import-source adapters: transforming Azure and Thor exports.
+
+These adapters are pure - they turn a foreign CSV into the canonical per-table
+CSVs the import engine already understands, and touch no database. The tests
+check the transform output directly, so they run without app context.
+"""
+import csv
+import io
+
+import pytest
+
+from shopdb.core.services.importsources import (
+ AzureDeviceGroupSource, ImportSourceError, ThorSource, detect_source,
+ get_source,
+)
+
+
+def _rows(text):
+ return list(csv.DictReader(io.StringIO(text)))
+
+
+AZURE_CSV = (
+ 'displayName,objectType,mail,userType,id,deviceId,userPrincipalName,isUser,isGroup,isGuest\n'
+ 'F13HSGH4,device,,,11111111-1111-1111-1111-111111111111,aaaa1111-bbbb-2222-cccc-333333333333,,False,False,False\n'
+ 'E3P5M0B4,device,,,22222222-2222-2222-2222-222222222222,dddd4444-eeee-5555-ffff-666666666666,,False,False,False\n'
+)
+
+
+def test_azure_matches_signature():
+ header = ['displayName', 'objectType', 'deviceId', 'id']
+ assert AzureDeviceGroupSource.matches(header)
+ assert detect_source(header) is AzureDeviceGroupSource
+
+
+def test_azure_transform_builds_assets_and_computers():
+ produced = AzureDeviceGroupSource.transform(AZURE_CSV)
+ assert set(produced) == {'assets', 'computers'}
+
+ assets = _rows(produced['assets'])
+ assert [a['assetnumber'] for a in assets] == ['F13HSGH4', 'E3P5M0B4']
+ assert all(a['assettypeid'] == 'computer' for a in assets)
+ assert all(a['statusid'] == 'In Use' for a in assets)
+
+ computers = _rows(produced['computers'])
+ assert computers[0]['assetid'] == 'F13HSGH4'
+ assert computers[0]['hostname'] == 'F13HSGH4'
+ assert computers[0]['azuredeviceid'] == 'aaaa1111-bbbb-2222-cccc-333333333333'
+ assert 'computertypeid' not in computers[0]
+
+
+def test_azure_transform_stamps_computertype():
+ produced = AzureDeviceGroupSource.transform(
+ AZURE_CSV, options={'computertype': 'Shopfloor PC'})
+ computers = _rows(produced['computers'])
+ assert all(c['computertypeid'] == 'Shopfloor PC' for c in computers)
+
+
+def test_azure_transform_carries_objectid():
+ produced = AzureDeviceGroupSource.transform(AZURE_CSV)
+ computers = _rows(produced['computers'])
+ # The CSV 'id' column is the Entra directory object id.
+ assert computers[0]['azureobjectid'] == '11111111-1111-1111-1111-111111111111'
+
+
+def test_azure_transform_stamps_category():
+ produced = AzureDeviceGroupSource.transform(
+ AZURE_CSV, options={'category': 'SFLD_DYN_Device_Group_X'})
+ computers = _rows(produced['computers'])
+ assert all(c['azuredevicecategory'] == 'SFLD_DYN_Device_Group_X'
+ for c in computers)
+
+
+def test_azure_aligns_matched_hostname_to_existing_asset():
+ # A case-insensitive hostname match routes the device to its real asset and
+ # emits NO new asset row, so the import updates in place instead of creating
+ # a duplicate. Unmatched devices still create a new asset keyed on hostname.
+ produced = AzureDeviceGroupSource.transform(
+ AZURE_CSV, options={'hostname_map': {'f13hsgh4': '13NSGH4'}})
+
+ computers = _rows(produced['computers'])
+ by_host = {c['hostname']: c for c in computers}
+ # Matched device points at the existing assetnumber, not its hostname.
+ assert by_host['F13HSGH4']['assetid'] == '13NSGH4'
+ # Unmatched device falls back to its hostname (new asset).
+ assert by_host['E3P5M0B4']['assetid'] == 'E3P5M0B4'
+
+ # Only the unmatched device produces an asset row.
+ assets = _rows(produced['assets'])
+ assert [a['assetnumber'] for a in assets] == ['E3P5M0B4']
+
+
+def test_azure_all_matched_emits_no_assets_file():
+ produced = AzureDeviceGroupSource.transform(
+ AZURE_CSV,
+ options={'hostname_map': {'f13hsgh4': '13NSGH4', 'e3p5m0b4': 'E9Q'}})
+ # Every device matched, so there is nothing to create and no assets file.
+ assert 'assets' not in produced
+ computers = _rows(produced['computers'])
+ assert {c['assetid'] for c in computers} == {'13NSGH4', 'E9Q'}
+
+
+def test_azure_skips_non_device_rows_and_dedupes():
+ text = (
+ 'displayName,objectType,mail,userType,id,deviceId,userPrincipalName,isUser,isGroup,isGuest\n'
+ 'F13HSGH4,device,,,1,aaaa,,False,False,False\n'
+ 'F13HSGH4,device,,,1,aaaa,,False,False,False\n'
+ 'Some User,user,u@x,,2,,u@x,True,False,False\n'
+ )
+ computers = _rows(AzureDeviceGroupSource.transform(text)['computers'])
+ assert [c['hostname'] for c in computers] == ['F13HSGH4']
+
+
+def test_azure_rejects_wrong_header():
+ with pytest.raises(ImportSourceError):
+ AzureDeviceGroupSource.transform('a,b,c\n1,2,3\n')
+
+
+THOR_CSV = (
+ '"Host Name","IP Address","LOGON","OS Version","OS ID","Last Updated","Boot Time","Location","Model","BIOS Manf."\n'
+ '"F67HSGH4","10.0.0.5","F67HSGH4\\ShopFloor","10.0.26100","24H2","2026-01-02 03:04:05","2026-01-01 06:00:00","S13","Dell Pro Tower Plus QBT1250","Dell Inc."\n'
+)
+
+
+def test_thor_matches_signature():
+ header = ['Host Name', 'BIOS Manf.', 'OS Version', 'Model']
+ assert ThorSource.matches(header)
+ assert detect_source(header) is ThorSource
+
+
+def test_thor_transform_builds_lookup_and_subtype_files():
+ produced = ThorSource.transform(THOR_CSV)
+ assert 'vendors' in produced
+ assert 'models' in produced
+ assert 'operatingsystems' in produced
+ assert 'assets' in produced
+ assert 'computers' in produced
+
+ assert _rows(produced['vendors'])[0]['vendor'] == 'Dell Inc.'
+ assert _rows(produced['models'])[0]['modelnumber'] == 'Dell Pro Tower Plus QBT1250'
+ assert _rows(produced['operatingsystems'])[0]['osname'] == '10.0.26100 (24H2)'
+
+ computer = _rows(produced['computers'])[0]
+ assert computer['hostname'] == 'F67HSGH4'
+ assert computer['vendorid'] == 'Dell Inc.'
+ assert computer['modelnumberid'] == 'Dell Pro Tower Plus QBT1250'
+ assert computer['osid'] == '10.0.26100 (24H2)'
+ assert computer['loggedinuser'] == 'ShopFloor'
+ assert computer['lastreporteddate'] == '2026-01-02 03:04:05'
+ assert computer['lastboottime'] == '2026-01-01 06:00:00'
+
+ asset = _rows(produced['assets'])[0]
+ assert asset['locationid'] == 'S13'
+ assert asset['assettypeid'] == 'computer'
+ assert asset['statusid'] == 'In Use'
+
+
+THOR_TWO_CSV = (
+ '"Host Name","IP Address","LOGON","OS Version","OS ID","Last Updated","Boot Time","Location","Model","BIOS Manf."\n'
+ '"F67HSGH4","10.0.0.5","F67HSGH4\\ShopFloor","10.0.26100","24H2","2026-01-02 03:04:05","2026-01-01 06:00:00","S13","Dell Pro Tower Plus QBT1250","Dell Inc."\n'
+ '"FDKKSGH4","10.0.0.6","FDKKSGH4\\ShopFloor","10.0.26100","24H2","2026-01-03 03:04:05","2026-01-02 06:00:00","G8","Dell Pro Tower Plus QBT1250","Dell Inc."\n'
+)
+
+
+def test_thor_stamps_computertype():
+ computer = _rows(
+ ThorSource.transform(THOR_CSV, options={'computertype': 'Shop Floor PC'})['computers'])[0]
+ assert computer['computertypeid'] == 'Shop Floor PC'
+
+
+def test_thor_aligns_matched_hostname_to_existing_asset():
+ # A case-insensitive hostname match routes the host to its real asset and
+ # emits NO new asset row, so the import updates in place instead of creating
+ # a duplicate. Unmatched hosts still create a new asset keyed on hostname.
+ produced = ThorSource.transform(
+ THOR_TWO_CSV, options={'hostname_map': {'f67hsgh4': 'PC-1001'}})
+
+ computers = _rows(produced['computers'])
+ by_host = {c['hostname']: c for c in computers}
+ assert by_host['F67HSGH4']['assetid'] == 'PC-1001'
+ assert by_host['FDKKSGH4']['assetid'] == 'FDKKSGH4'
+
+ assets = _rows(produced['assets'])
+ assert [a['assetnumber'] for a in assets] == ['FDKKSGH4']
+
+
+def test_thor_all_matched_emits_no_assets_file():
+ produced = ThorSource.transform(
+ THOR_TWO_CSV,
+ options={'hostname_map': {'f67hsgh4': 'PC-1001', 'fdkksgh4': 'PC-1002'}})
+ assert 'assets' not in produced
+ computers = _rows(produced['computers'])
+ assert {c['assetid'] for c in computers} == {'PC-1001', 'PC-1002'}
+
+
+def test_get_source_unknown_raises():
+ with pytest.raises(ImportSourceError):
+ get_source('nope')
+
+
+def test_detect_source_none_for_unknown_header():
+ assert detect_source(['foo', 'bar']) is None
diff --git a/tests/test_plugins/test_computers_import_machine_link.py b/tests/test_plugins/test_computers_import_machine_link.py
new file mode 100644
index 00000000..28162565
--- /dev/null
+++ b/tests/test_plugins/test_computers_import_machine_link.py
@@ -0,0 +1,187 @@
+"""computers.csv can carry the machine it drives, and the link is built on import.
+
+A site with a spreadsheet knows "this PC drives machine CNC-014" and wants to
+write that in the PC row, not hand-build an assetrelationships sheet keyed on
+ids it never sees. The importer accepts a virtual `machineassetnumber` column,
+keeps it out of the computers write, and hands it to the computers plugin after
+each PC row is written. The rules that matter for a cold user driving this off a
+spreadsheet:
+
+- a good number builds the PC-controls-machine link;
+- a number that has no machine, names this PC itself, names a location-only
+ marker, or is claimed twice in one file, is an AMBER warning and is skipped -
+ the PC still imports, the file is still ok;
+- a dry run builds nothing and, critically, sends no alert;
+- re-running the same file does not duplicate the link.
+"""
+import pytest
+
+from shopdb.core.services.csvimport import (
+ Resolver, import_csv, virtual_columns_for,
+)
+from shopdb.extensions import db as _db
+
+
+REFERENCE = [
+ ('assettypes', 'assettype\nComputer\nMachine\n'),
+ ('assetstatuses', 'status\nIn Use\n'),
+ ('locationtypes', 'locationtype\nBay\n'),
+]
+
+
+def _seed(resolver):
+ """Reference rows, a location, and the 'controls' relationship type."""
+ from shopdb.core.models.relationship import RelationshipType
+
+ for table, text in REFERENCE:
+ result = import_csv(table, text, resolver=resolver, commit=True)
+ assert result.ok, [str(p) for p in result.problems]
+ result = import_csv(
+ 'locations', 'locationname,locationtypeid\nBay 3,Bay\n',
+ resolver=resolver, commit=True)
+ assert result.ok, [str(p) for p in result.problems]
+ if not RelationshipType.query.filter_by(relationshiptype='controls').first():
+ _db.session.add(RelationshipType(relationshiptype='controls'))
+ # import_csv writes into the session but never commits; commit the setup so
+ # a dry-run import's rollback only unwinds the dry run, not the fixtures.
+ _db.session.commit()
+
+
+def _assets(resolver, *rows):
+ """Create the given assets. Each row is (assetnumber, assettype)."""
+ body = 'assetnumber,assettypeid,statusid,locationid\n'
+ body += ''.join('%s,%s,In Use,Bay 3\n' % r for r in rows)
+ result = import_csv('assets', body, resolver=resolver, commit=True)
+ assert result.ok, [str(p) for p in result.problems]
+ _db.session.commit()
+
+
+def _import_computers(resolver, body, commit=True):
+ return import_csv('computers', body, resolver=resolver, commit=commit)
+
+
+def _controls_target(pc_assetnumber):
+ """Asset number of the machine this PC actively controls, or None."""
+ from shopdb.core.models import Asset
+ from shopdb.core.models.relationship import RelationshipType, AssetRelationship
+
+ pc = Asset.query.filter_by(assetnumber=pc_assetnumber).first()
+ controls = RelationshipType.query.filter_by(relationshiptype='controls').first()
+ rel = AssetRelationship.query.filter(
+ AssetRelationship.sourceassetid == pc.assetid,
+ AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
+ AssetRelationship.isactive.is_(True)).first()
+ if rel is None:
+ return None
+ return _db.session.get(Asset, rel.targetassetid).assetnumber
+
+
+def test_the_column_is_virtual_and_owned_by_the_computers_plugin(db):
+ """The plugin registers machineassetnumber; core stays ignorant of it."""
+ assert 'machineassetnumber' in virtual_columns_for('computers')
+
+
+def test_a_good_machine_number_builds_the_link(db):
+ resolver = Resolver()
+ _seed(resolver)
+ _assets(resolver, ('PC-001', 'Computer'), ('CNC-014', 'Machine'))
+ result = _import_computers(
+ resolver,
+ 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,CNC-014\n')
+ assert result.ok, [str(p) for p in result.problems]
+ assert result.warnings == []
+ assert _controls_target('PC-001') == 'CNC-014'
+
+
+def test_an_empty_machine_number_links_nothing_and_does_not_warn(db):
+ resolver = Resolver()
+ _seed(resolver)
+ _assets(resolver, ('PC-001', 'Computer'))
+ result = _import_computers(
+ resolver, 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,\n')
+ assert result.ok
+ assert result.warnings == []
+ assert _controls_target('PC-001') is None
+
+
+def test_a_machine_number_with_no_machine_warns_and_skips(db):
+ """The PC still imports; the unresolved number is amber, not a failed file."""
+ resolver = Resolver()
+ _seed(resolver)
+ _assets(resolver, ('PC-001', 'Computer'))
+ result = _import_computers(
+ resolver, 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,NOPE-99\n')
+ assert result.ok, 'an unresolved machine number must not fail the file'
+ assert len(result.warnings) == 1
+ warning = result.warnings[0]
+ assert warning.column == 'machineassetnumber'
+ assert warning.target == 'NOPE-99'
+ assert warning.action, 'a warning must carry the one corrective step'
+ assert _controls_target('PC-001') is None
+
+
+def test_a_pc_naming_its_own_asset_number_is_not_self_linked(db):
+ resolver = Resolver()
+ _seed(resolver)
+ _assets(resolver, ('PC-001', 'Computer'))
+ result = _import_computers(
+ resolver, 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,PC-001\n')
+ assert result.ok
+ assert len(result.warnings) == 1
+ assert 'own asset number' in result.warnings[0].reason
+ assert _controls_target('PC-001') is None
+
+
+def test_two_rows_claiming_one_machine_link_only_the_first(db):
+ """The in-file fan-out guard: a copied cell must not steal the bay."""
+ resolver = Resolver()
+ _seed(resolver)
+ _assets(resolver,
+ ('PC-001', 'Computer'), ('PC-002', 'Computer'), ('CNC-014', 'Machine'))
+ result = _import_computers(
+ resolver,
+ 'assetid,hostname,machineassetnumber\n'
+ 'PC-001,WS-PC-001,CNC-014\n'
+ 'PC-002,WS-PC-002,CNC-014\n')
+ assert result.ok
+ assert len(result.warnings) == 1
+ assert 'already claimed' in result.warnings[0].reason
+ assert _controls_target('PC-001') == 'CNC-014'
+ assert _controls_target('PC-002') is None
+
+
+def test_a_dry_run_builds_no_link_and_leaves_no_trace(db):
+ resolver = Resolver()
+ _seed(resolver)
+ _assets(resolver, ('PC-001', 'Computer'), ('CNC-014', 'Machine'))
+ result = _import_computers(
+ resolver,
+ 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,CNC-014\n',
+ commit=False)
+ assert result.ok
+ assert result.created == 1
+ _db.session.rollback()
+ assert _controls_target('PC-001') is None
+
+
+def test_reimporting_the_same_link_does_not_duplicate(db):
+ resolver = Resolver()
+ _seed(resolver)
+ _assets(resolver, ('PC-001', 'Computer'), ('CNC-014', 'Machine'))
+ body = 'assetid,hostname,machineassetnumber\nPC-001,WS-PC-001,CNC-014\n'
+ first = _import_computers(resolver, body)
+ assert first.ok and first.warnings == []
+
+ second = _import_computers(resolver, body)
+ assert second.ok and second.warnings == []
+ assert _controls_target('PC-001') == 'CNC-014'
+
+ from shopdb.core.models import Asset
+ from shopdb.core.models.relationship import RelationshipType, AssetRelationship
+ pc = Asset.query.filter_by(assetnumber='PC-001').first()
+ controls = RelationshipType.query.filter_by(relationshiptype='controls').first()
+ active = AssetRelationship.query.filter(
+ AssetRelationship.sourceassetid == pc.assetid,
+ AssetRelationship.relationshiptypeid == controls.relationshiptypeid,
+ AssetRelationship.isactive.is_(True)).count()
+ assert active == 1, 'a re-import must not stack a second active link'