ge-aerospace/shopdb-flask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add Entra device identity to computers and Azure/Thor import sources

Browse Source 
PC records now carry Entra/Azure device identity so imported machines keep a durable link back to their directory object instead of matching on hostname alone. Adds azuredeviceid, azureobjectid, and azuredevicecategory to the Computer model with matching migrations, surfaces them read-only in PCDetail as an Entra Device Info card, and lets PCForm edit the device id directly. The computers plugin registers a machineassetnumber import hook that reconciles PC-to-machine links during bulk import and warns on unresolved or conflicting relationships.

Adds importsources adapters that normalize Azure device-group and Thor PC inventory exports into canonical ShopDB CSVs via hostname alignment, exposes register_import_hook and RowWarning on the shopdb.api contract surface (contract 0.21.0), and refreshes the Import Site Assets page with clearer source instructions. Includes reconcile_locations maintenance script and coverage for the new import paths.
This commit is contained in:
benjamin.baker
2026-08-25 16:46:00 -04:00
parent f4d8868463
commit bdef21da50
21 changed files with 2064 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
shopdb/core/api/settings.py
View File

@@ -6,6 +6,7 @@ import re
from flask import Blueprint, request, current_app
from flask_jwt_extended import jwt_required, get_jwt_identity
from werkzeug.utils import secure_filename
from sqlalchemy import select as sa_select
from shopdb.extensions import db, cache
from shopdb.core.models import Setting, AuditLog
@@ -332,6 +333,27 @@ def download_import_template(tablename):
return response
def _existing_hostname_map():
"""Case-insensitive hostname -> assetnumber map for existing computers.
Azure device-group exports align on hostname (their displayName), while the
import engine keys assets on assetnumber. This lets the Azure adapter route
a matched device to its real asset so the import updates in place.
"""
assets = db.metadata.tables['assets']
computers = db.metadata.tables['computers']
query = sa_select(
computers.c.hostname, assets.c.assetnumber
).select_from(
computers.join(assets, computers.c.assetid == assets.c.assetid)
).where(computers.c.hostname.isnot(None))
mapping = {}
for hostname, assetnumber in db.session.execute(query):
if hostname:
mapping[hostname.strip().lower()] = assetnumber
return mapping
@settings_bp.route('/asset-import/import', methods=['POST'])
@jwt_required()
@require_role('admin')
@@ -344,10 +366,16 @@ def import_asset_csvs():
transaction back and reports what WOULD change; commit=true persists it. The
whole run is all-or-nothing - if any file has a row problem, nothing is
written even on a commit.
A 'source' form field selects a foreign adapter (azure, thor); it defaults
to 'native' where uploads are already canonical per-table files. Foreign
uploads are transformed into canonical files first, then run identically.
"""
from sqlalchemy.exc import SQLAlchemyError
from shopdb.core.services.csvimport import (
ImportError_, Resolver, dependency_order, import_csv, table_from_filename)
from shopdb.core.services.importsources import (
ImportSourceError, get_source)
uploads = request.files.getlist('files')
if not uploads:
@@ -356,21 +384,66 @@ def import_asset_csvs():
commit = str(request.form.get('commit', '')).strip().lower() in (
'1', 'true', 'yes', 'on')
# Map each upload to a table by filename. A later file with the same table
# name wins, matching the folder-import behaviour (one file per table).
# A foreign source (azure, thor) is transformed into canonical files before
# import. 'native' (or nothing) means the files already ARE canonical.
source_key = (request.form.get('source') or 'native').strip().lower()
source = None
if source_key and source_key != 'native':
try:
source = get_source(source_key)
except ImportSourceError as exc:
return error_response(ErrorCodes.VALIDATION_ERROR, str(exc))
source_options = {
'computertype': (request.form.get('computertype') or '').strip(),
'category': (request.form.get('category') or '').strip(),
}
# Azure and Thor align on hostname: their device name is the ShopDB
# computers.hostname, not the assetnumber. Hand the adapter a
# case-insensitive hostname -> existing assetnumber map so devices that
# already exist update in place (keyed to their real asset) instead of
# minting a duplicate asset.
if source is not None and getattr(source, 'key', None) in ('azure', 'thor'):
source_options['hostname_map'] = _existing_hostname_map()
texts = {}
filenames = {}
for upload in uploads:
name = upload.filename or ''
if not name.lower().endswith('.csv'):
continue
table = table_from_filename(name)
try:
texts[table] = upload.read().decode('utf-8-sig')
except (UnicodeDecodeError, OSError):
return error_response(ErrorCodes.VALIDATION_ERROR,
'Could not read %s as UTF-8 text' % name)
filenames[table] = name
if source is not None:
# One foreign export becomes several canonical files. Each upload is
# transformed and its canonical outputs merged; a later file's table
# wins, matching the native one-file-per-table rule.
for upload in uploads:
name = upload.filename or ''
if not name.lower().endswith('.csv'):
continue
try:
raw = upload.read().decode('utf-8-sig')
except (UnicodeDecodeError, OSError):
return error_response(ErrorCodes.VALIDATION_ERROR,
'Could not read %s as UTF-8 text' % name)
try:
produced = source.transform(raw, options=source_options)
except ImportSourceError as exc:
return error_response(ErrorCodes.VALIDATION_ERROR,
'%s: %s' % (name, str(exc)))
for table, csv_text in produced.items():
texts[table] = csv_text
filenames[table] = '%s (from %s)' % (table, name)
else:
# Map each upload to a table by filename. A later file with the same
# table name wins, matching the folder-import behaviour.
for upload in uploads:
name = upload.filename or ''
if not name.lower().endswith('.csv'):
continue
table = table_from_filename(name)
try:
texts[table] = upload.read().decode('utf-8-sig')
except (UnicodeDecodeError, OSError):
return error_response(ErrorCodes.VALIDATION_ERROR,
'Could not read %s as UTF-8 text' % name)
filenames[table] = name
if not texts:
return error_response(ErrorCodes.VALIDATION_ERROR,
@@ -389,6 +462,7 @@ def import_asset_csvs():
resolver = Resolver(reconcile_threshold=reconcile_threshold)
files_report = []
problem_count = 0
warning_count = 0
reconciled_count = 0
request_error = None
@@ -409,7 +483,17 @@ def import_asset_csvs():
{'line': p.line, 'column': p.column, 'message': p.message}
for p in result.problems
]
# Amber tier: a warning is a finding beside a row that still
# imported (an unresolved machine link, a contested bay), NOT a
# reason to fail the file. Reported alongside problems so the UI can
# show Errors / Warnings / Info distinctly, but never gates commit.
warnings = [
{'line': w.line, 'column': w.column, 'reason': w.reason,
'action': w.action, 'subject': w.subject, 'target': w.target}
for w in result.warnings
]
problem_count += len(problems)
warning_count += len(warnings)
reconciled_count += len(result.reconciliations)
files_report.append({
'table': table,
@@ -417,6 +501,7 @@ def import_asset_csvs():
'created': result.created,
'updated': result.updated,
'problems': problems,
'warnings': warnings,
'reconciliations': result.reconciliations,
})
@@ -446,6 +531,7 @@ def import_asset_csvs():
'files': files_report,
'skipped': skipped,
'problem_count': problem_count,
'warning_count': warning_count,
'reconciled_count': reconciled_count,
'reconcile_threshold': reconcile_threshold,
'total_created': total_created,

233
shopdb/core/services/csvimport.py
View File

@@ -74,6 +74,71 @@ IMPORTABLE = OrderedDict([
('machines', {'label': None, 'natural': 'assetid'}),
])
# A small, INTERNALLY CONSISTENT worked example per table, in import order. A
# site can download the set, see the pattern, and keep it as a starter workbook:
# every name referenced by a later file (Bay 3, Dell, Computer, ...) is defined
# in an earlier one, so the whole set imports cleanly as a demo before a single
# real row is typed. Emitted as commented rows the engine skips, so a freshly
# downloaded template never writes example data by accident. Values avoid commas
# on purpose - the generator writes them without CSV quoting.
EXAMPLE_ROWS = OrderedDict([
('assetstatuses', [
{'status': 'In Use', 'description': 'Actively deployed', 'color': '#22c55e', 'isactive': '1'},
{'status': 'In Storage', 'description': 'Spare - not deployed', 'color': '#eab308', 'isactive': '1'},
{'status': 'Retired', 'description': 'Removed from service', 'color': '#ef4444', 'isactive': '1'},
]),
('assettypes', [
{'assettype': 'Computer', 'pluginname': 'computers', 'tablename': 'computers', 'description': 'PCs - servers - workstations', 'icon': 'laptop', 'color': '#3b82f6', 'isactive': '1'},
{'assettype': 'Machine', 'pluginname': 'machines', 'tablename': 'machines', 'description': 'Shop-floor machines', 'icon': 'cpu', 'color': '#8b5cf6', 'isactive': '1'},
]),
('locationtypes', [
{'locationtype': 'Building', 'description': 'A physical building', 'color': '#64748b', 'isactive': '1'},
{'locationtype': 'Bay', 'description': 'A shop-floor bay', 'color': '#0ea5e9', 'isactive': '1'},
]),
('modeltypes', [
{'modeltype': 'Desktop', 'category': 'Computer', 'description': 'Desktop PC chassis', 'icon': 'desktop', 'isactive': '1'},
{'modeltype': 'CNC Machine', 'category': 'Machine', 'description': 'CNC machining center', 'icon': 'cpu', 'isactive': '1'},
]),
('computertypes', [
{'computertype': 'Shopfloor PC', 'description': 'PC on the shop floor', 'icon': 'monitor', 'color': '#3b82f6', 'isactive': '1'},
{'computertype': 'Engineer Workstation', 'description': 'CAD/CAM workstation', 'icon': 'laptop', 'color': '#6366f1', 'isactive': '1'},
]),
('machinetypes', [
{'machinetype': 'CNC Mill', 'description': 'Computer numerical control mill', 'icon': 'cpu', 'color': '#8b5cf6', 'isactive': '1'},
{'machinetype': 'CMM', 'description': 'Coordinate measuring machine', 'icon': 'ruler', 'color': '#14b8a6', 'isactive': '1'},
]),
('businessunits', [
{'businessunit': 'Machining', 'code': 'MACH', 'description': 'Machining department', 'isactive': '1'},
{'businessunit': 'Quality', 'code': 'QUAL', 'description': 'Quality and metrology', 'isactive': '1'},
]),
('locations', [
{'locationname': 'Building 1', 'building': 'Building 1', 'floor': '1', 'description': 'Main manufacturing building', 'locationtypeid': 'Building', 'isactive': '1'},
{'locationname': 'Bay 3', 'building': 'Building 1', 'floor': '1', 'room': 'Machining bay 3', 'description': 'Machining bay 3', 'locationtypeid': 'Bay', 'isactive': '1'},
]),
('vendors', [
{'vendor': 'Dell', 'description': 'Computer hardware', 'website': 'https://www.dell.com', 'supportphone': '800-555-0100', 'supportemail': 'support@dell.com', 'isactive': '1'},
{'vendor': 'Haas Automation', 'description': 'CNC machine tools', 'website': 'https://www.haascnc.com', 'supportphone': '800-555-0199', 'isactive': '1'},
]),
('models', [
{'modelnumber': 'OptiPlex 7010', 'modeltypeid': 'Desktop', 'vendorid': 'Dell', 'description': 'Small form factor desktop', 'isactive': '1'},
{'modelnumber': 'VF-2', 'modeltypeid': 'CNC Machine', 'vendorid': 'Haas Automation', 'description': '40-taper vertical mill', 'isactive': '1'},
]),
('operatingsystems', [
{'osname': 'Windows 11 (24H2)', 'osversion': '24H2', 'architecture': 'x64', 'endoflife': '2027-10-12', 'isactive': '1'},
{'osname': 'Windows 10 (22H2)', 'osversion': '22H2', 'architecture': 'x64', 'endoflife': '2025-10-14', 'isactive': '1'},
]),
('assets', [
{'assetnumber': 'PC-001', 'name': 'Shopfloor PC 001', 'serialnumber': 'SN-PC-001', 'assettypeid': 'Computer', 'statusid': 'In Use', 'locationid': 'Bay 3', 'businessunitid': 'Machining', 'isactive': '1'},
{'assetnumber': 'CNC-014', 'name': 'Haas VF-2 #14', 'serialnumber': 'SN-CNC-014', 'assettypeid': 'Machine', 'statusid': 'In Use', 'locationid': 'Bay 3', 'businessunitid': 'Machining', 'isactive': '1'},
]),
('computers', [
{'assetid': 'PC-001', 'computertypeid': 'Shopfloor PC', 'hostname': 'WS-PC-001', 'osid': 'Windows 11 (24H2)', 'vendorid': 'Dell', 'modelnumberid': 'OptiPlex 7010', 'loggedinuser': 'jdoe', 'lastreporteddate': '2026-08-01', 'lastboottime': '2026-08-01', 'isactive': '1', 'machineassetnumber': 'CNC-014'},
]),
('machines', [
{'assetid': 'CNC-014', 'machinetypeid': 'CNC Mill', 'vendorid': 'Haas Automation', 'modelnumberid': 'VF-2', 'requiresmanualconfig': '0', 'islocationonly': '0', 'lastmaintenancedate': '2026-06-01', 'nextmaintenancedate': '2026-12-01', 'maintenanceintervaldays': '180', 'isactive': '1'},
]),
])
# Columns a CSV may never set: surrogate keys the database owns, and audit
# stamps. Accepting an id from a spreadsheet is how two sites end up with
# colliding primary keys the first time anyone merges data.
@@ -111,6 +176,75 @@ CORPORATE_SUFFIXES = {
DEFAULT_RECONCILE_THRESHOLD = 0.85
# -- plugin import hooks (inversion of control) ---------------------------------
#
# A spreadsheet is the wrong place to type an integer foreign key, and it is
# also the wrong place to split a relationship across two files: a site that
# knows "this PC drives machine 3015" wants to write that number in the PC row,
# not hand-build an assetrelationships sheet keyed on ids it never sees.
#
# But the machine link is a computers-plugin concept, and this module is core:
# it must not import a plugin. So a plugin REGISTERS itself here instead. It
# declares a VIRTUAL COLUMN - a header this table accepts but never writes to
# the database - and a HOOK that runs once per written row to turn that column's
# value into whatever the plugin means by it (here, a PC->machine controls link).
# Core stays ignorant of what the column is for; the plugin owns the meaning.
_VIRTUAL_COLUMNS = {} # tablename -> {columnname: description}
_ROW_HOOKS = {} # tablename -> [hook, ...]
def register_import_hook(tablename, virtual_columns, hook):
"""Let a plugin own virtual column(s) on an importable table.
virtual_columns: either a sequence of header names, or a mapping of
name -> one-line description. Each is accepted for ``tablename`` but never
written - it is pulled out of the row and handed to the hook instead of the
table. A description, when given, is shown in the generated template's
column notes so an operator knows what to type.
hook(rowvalues, virtualvalues, line) -> list[RowWarning]: called once per
written row, AFTER the row is inserted/updated, so its id exists. It runs
inside the caller's transaction - which a dry run rolls back - so it must
not commit, and must not do anything a rollback cannot undo (no emails). A
hook reports trouble by returning RowWarning, never by raising: a bad
machine number is an amber note beside a PC that still imports, not a failed
file.
Idempotent by contract: registering the same (table, column, hook) twice is
a no-op, so a plugin re-init during tests or a reload does not double-fire.
"""
cols = _VIRTUAL_COLUMNS.setdefault(tablename, {})
if isinstance(virtual_columns, dict):
for name, description in virtual_columns.items():
cols[name] = description or ''
else:
for name in virtual_columns:
cols.setdefault(name, '')
hooks = _ROW_HOOKS.setdefault(tablename, [])
# Dedupe by the underlying function's qualified name, not by the bound
# method: a plugin re-init (a new app context in the test suite, a reload in
# prod) makes a fresh instance whose bound method is != the old one, and a
# symlinked plugin can even be imported under two module paths, so an
# identity check would append a second hook and the row would be processed -
# and warned about - more than once.
def _identity(fn):
func = getattr(fn, '__func__', fn)
return getattr(func, '__qualname__', repr(func))
if _identity(hook) not in [_identity(existing) for existing in hooks]:
hooks.append(hook)
def virtual_columns_for(tablename):
"""Header names this table accepts but does not store. Never None."""
return set(_VIRTUAL_COLUMNS.get(tablename, ()))
def virtual_column_notes(tablename):
"""(name, description) for each virtual column, in registration order."""
return list(_VIRTUAL_COLUMNS.get(tablename, {}).items())
def normalize_label(value):
"""Fold a label to its comparison form: lowercase, no punctuation, no
corporate suffix, single-spaced. 'Dell Inc.' and 'DELL' both become 'dell'.
@@ -370,6 +504,32 @@ class RowProblem(object):
return "%s: %s" % (where, self.message)
class RowWarning(object):
"""A non-blocking, amber-tier finding.
Errors (RowProblem) stop the whole import; a RowWarning does NOT - the row
it belongs to still imports. It exists so a caveat (an unresolved or
conflicting machine link, say) is loud and REVIEWABLE without failing an
otherwise-clean file. Every warning names WHERE it happened, WHY, and the
single ACTION that fixes it, so a cold user can correct it in place - the
convention IT Glue / RMM imports train.
"""
def __init__(self, line, column, reason, action, subject=None, target=None):
self.line = line
self.column = column
self.reason = reason # what is wrong, in plain terms
self.action = action # the one corrective step to take
self.subject = subject # the row's actor (e.g. the PC hostname)
self.target = target # what it referenced (e.g. the machine number)
def __str__(self):
where = "line %d" % self.line
if self.column:
where += ", column '%s'" % self.column
return "%s: %s - %s" % (where, self.reason, self.action)
class ImportResult(object):
def __init__(self, tablename):
self.tablename = tablename
@@ -378,14 +538,18 @@ class ImportResult(object):
self.problems = []
self.skipped = 0
self.reconciliations = []
self.warnings = []
@property
def ok(self):
# Warnings are amber, not red: a file with only warnings is still ok to
# commit. Only problems (errors) block.
return not self.problems
def summary(self):
return "%s: %d new, %d updated, %d problem(s)" % (
self.tablename, self.created, self.updated, len(self.problems))
return "%s: %d new, %d updated, %d problem(s), %d warning(s)" % (
self.tablename, self.created, self.updated,
len(self.problems), len(self.warnings))
def read_rows(text):
@@ -439,9 +603,10 @@ def import_csv(tablename, text, resolver=None, commit=False):
header, rows = read_rows(text)
allowed = {c.name: c for c in importable_columns(tablename)}
unknown = [h for h in header if h not in allowed]
virtual = virtual_columns_for(tablename)
unknown = [h for h in header if h not in allowed and h not in virtual]
if unknown:
known = ', '.join(sorted(allowed))
known = ', '.join(sorted(set(allowed) | virtual))
result.problems.append(RowProblem(
1, None, "unknown column(s): %s\n this table accepts: %s" % (', '.join(unknown), known)))
return result
@@ -461,7 +626,17 @@ def import_csv(tablename, text, resolver=None, commit=False):
prepared = []
for line, raw in rows:
values = {}
virtualvalues = {}
failed = False
# Virtual columns belong to a plugin hook, not the table. Pull them out
# first so they never reach an insert/update, and stash them for the
# post-write hook. Whitespace-trimmed; an empty cell is dropped so the
# hook only sees values a row actually carried.
for name in virtual:
if name in raw:
text = (raw[name] or '').strip()
if text != '':
virtualvalues[name] = text
for name, column in allowed.items():
if name not in raw:
continue
@@ -496,16 +671,18 @@ def import_csv(tablename, text, resolver=None, commit=False):
if values.get(natural) in (None, ''):
result.problems.append(RowProblem(line, natural, 'must have a value'))
continue
prepared.append((line, values))
prepared.append((line, values, virtualvalues))
if not result.ok:
return result
hooks = _ROW_HOOKS.get(tablename, ())
# Second pass: decide insert vs update. Counted even on a dry run, because
# "this will change 4000 rows" is the number an operator needs before saying
# yes.
label = _label_column(tablename)
for line, values in prepared:
for line, values, virtualvalues in prepared:
existing = db.session.execute(
table.select().where(table.c[natural] == values[natural])).first()
if existing is None:
@@ -522,6 +699,15 @@ def import_csv(tablename, text, resolver=None, commit=False):
primary = list(table.primary_key)[0].name
resolver.remember(tablename, values[label], getattr(existing, primary))
# Fire plugin hooks AFTER the write, so the row's id exists. Only when
# the row actually carried a virtual value - a hook that has nothing to
# act on should never run. Warnings are collected, never raised: an
# unresolved machine number is amber, not a failed file.
if hooks and virtualvalues:
for hook in hooks:
result.warnings.extend(
hook(values, virtualvalues, line) or [])
return result
@@ -556,24 +742,55 @@ def generate_template(tablename):
bits.append('text, max %d' % column.type.length)
notes.append("# %-24s %s" % (column.name, ', '.join(bits)))
# Virtual columns a plugin owns: accepted on this table but written by the
# plugin's hook, not the schema. Appended after the real columns so the
# header carries them and the notes explain what they do.
virtualnames = []
for name, description in virtual_column_notes(tablename):
virtualnames.append(name)
header.append(name)
note = 'optional, ' + description if description else 'optional'
notes.append("# %-24s %s" % (name, note))
spec = IMPORTABLE[tablename]
examples = EXAMPLE_ROWS.get(tablename, [])
exampleorder = [c.name for c in columns] + virtualnames
lines = [
"# %s.csv - generated from the ShopDB schema, do not hand-maintain" % tablename,
"#",
"# Rows are matched on '%s'. Re-importing updates a matching row rather" % spec['natural'],
"# than creating a second one, so it is safe to correct this file and run again.",
"#",
"# HOW TO USE THIS TEMPLATE:",
"# 1. Delete everything above the header row (every line starting with #).",
"# 2. Fill in your own rows using the example row(s) below as a guide.",
"# 3. DELETE THE EXAMPLE DATA before you import - the example rows are",
"# live data and WILL be imported if you leave them in.",
"#",
"# Columns:",
] + notes + [
"#",
"# Lines starting with # are ignored. Delete the example row before importing.",
"# --- header and example rows follow. Keep the header; replace the examples. ---",
','.join(header),
'# ' + ','.join(_example(c, required) for c in columns),
] + [
','.join(_example_value(row.get(name, '')) for name in exampleorder)
for row in examples
]
return '\n'.join(lines) + '\n'
def _example_value(value):
"""A single example cell. Commas would break the unquoted CSV the template
writes, so they are replaced with a dash; example data is authored without
them, this is only a guard.
"""
return str(value).replace(',', ' -')
def _example(column, required):
"""Kept for callers/tests that ask for a single synthetic cell. The template
generator now emits the curated EXAMPLE_ROWS instead.
"""
target = _fk_target(column)
if target:
label = _label_column(target[0])

325
shopdb/core/services/importsources.py Normal file
View File

@@ -0,0 +1,325 @@
"""Foreign import-source adapters.
The CSV import engine in csvimport.py speaks exactly one dialect: the canonical
per-table files (assets.csv, computers.csv, vendors.csv, ...). Foreign exports -
an Entra/Azure device-group member list, a Thor PC inventory dump - do not match
that dialect and never will; they are owned by other systems.
Rather than teach the engine every foreign schema, each adapter here TRANSFORMS
a foreign export into the canonical files and hands them back as
{tablename: csv_text}. The caller then runs those through the ordinary
import_csv path, so dry-run/commit, dependency ordering, and reconciliation all
work unchanged.
An adapter declares a header signature so an upload can be auto-detected, and a
transform() that yields the canonical files. Adapters never touch the database.
"""
import csv
import io
class ImportSourceError(Exception):
"""A foreign file could not be transformed (bad header, empty, etc.)."""
# Canonical defaults for a discovered PC. Azure carries only a hostname, so
# every device it creates is a computer that is In Use until something richer
# (Thor, the collector) says otherwise.
DEFAULT_ASSETTYPE = 'computer'
DEFAULT_STATUS = 'In Use'
def _read(text):
"""Parse CSV text into (header, list-of-dicts), tolerating a UTF-8 BOM."""
stream = io.StringIO(text)
reader = csv.DictReader(stream)
header = [h.strip() for h in (reader.fieldnames or [])]
rows = []
for raw in reader:
rows.append({(k.strip() if k else k): (v or '').strip()
for k, v in raw.items()})
return header, rows
def _write(header, rows):
"""Render canonical rows back to CSV text with a trailing newline."""
out = io.StringIO()
writer = csv.DictWriter(out, fieldnames=header, extrasaction='ignore')
writer.writeheader()
for row in rows:
writer.writerow(row)
return out.getvalue()
def _has_columns(header, required):
lowered = {h.lower() for h in header}
return all(col.lower() in lowered for col in required)
class AzureDeviceGroupSource:
"""Entra/Azure device-group member export.
Columns: displayName, objectType, mail, userType, id, deviceId,
userPrincipalName, isUser, isGroup, isGuest. Only device rows carry signal;
displayName is the hostname and deviceId is the Entra device GUID. The export
has no model, OS, location or user data, so this creates/updates minimal
asset + computer rows keyed on hostname.
options:
computertype - optional name stamped onto every computer's computertypeid
(e.g. 'Shopfloor PC' for a SHP group, 'Inspection PC' for
BUS). Resolved by the engine against computertypes.
category - optional Entra device category / group membership label
stamped onto every computer's azuredevicecategory (e.g.
SFLD_DYN_Device_Group_Wilmington_BUS_NC_COUNTRY:USA).
hostname_map - optional {lowercased hostname: existing assetnumber}. A
device whose displayName matches an existing computer's
hostname is routed to that asset and updated in place
(no new asset row); unmatched devices create a new
asset + computer keyed on the hostname.
"""
key = 'azure'
label = 'Azure device group'
signature = ('displayName', 'deviceId', 'objectType')
@classmethod
def matches(cls, header):
return _has_columns(header, cls.signature)
@classmethod
def transform(cls, text, options=None):
options = options or {}
computertype = (options.get('computertype') or '').strip()
category = (options.get('category') or '').strip()
hostname_map = options.get('hostname_map') or {}
header, rows = _read(text)
if not _has_columns(header, cls.signature):
raise ImportSourceError(
'not an Azure device-group export (expected displayName, '
'deviceId, objectType columns)')
assets = []
computers = []
seen = set()
for row in rows:
objecttype = (row.get('objectType') or '').strip().lower()
if objecttype and objecttype != 'device':
continue
hostname = (row.get('displayName') or '').strip()
if not hostname or hostname.lower() in seen:
continue
seen.add(hostname.lower())
# Align on hostname: if this device already exists in ShopDB, point
# the computer row at its real asset so the engine updates it in
# place. Only create a new asset when there is no hostname match.
existing_assetnumber = hostname_map.get(hostname.lower())
computer = {
'assetid': existing_assetnumber if existing_assetnumber is not None else hostname,
'hostname': hostname,
'azuredeviceid': (row.get('deviceId') or '').strip(),
'azureobjectid': (row.get('id') or '').strip(),
}
if computertype:
computer['computertypeid'] = computertype
if category:
computer['azuredevicecategory'] = category
computers.append(computer)
if existing_assetnumber is None:
assets.append({
'assetnumber': hostname,
'name': hostname,
'assettypeid': DEFAULT_ASSETTYPE,
'statusid': DEFAULT_STATUS,
})
if not computers:
raise ImportSourceError('no device rows found in the Azure export')
asset_header = ['assetnumber', 'name', 'assettypeid', 'statusid']
computer_header = ['assetid', 'hostname', 'azuredeviceid', 'azureobjectid']
if computertype:
computer_header.append('computertypeid')
if category:
computer_header.append('azuredevicecategory')
produced = {
'computers': _write(computer_header, computers),
}
# Only emit an assets file when there are new devices to create; an
# empty assets.csv would just be an extra clean no-op table.
if assets:
produced['assets'] = _write(asset_header, assets)
return produced
class ThorSource:
"""Thor PC-inventory export (internal GE Aerospace tool).
A wide, fixed-format PC inventory. Maps the columns ShopDB owns into
canonical rows: hostname, vendor (BIOS Manf.), model, operating system,
logged-in user, last-reported and last-boot timestamps, and location. The
network columns (MAC, switch, VLAN, IP) and the machine 'Attached' link are
out of scope for this first pass.
options:
computertype - optional name stamped onto every computer's computertypeid.
Resolved by the engine against computertypes.
hostname_map - optional {lowercased hostname: existing assetnumber}. A
host whose Host Name matches an existing computer's hostname
is routed to that asset and updated in place (no new asset
row); unmatched hosts create a new asset + computer keyed on
the hostname.
"""
key = 'thor'
label = 'Thor PC inventory'
signature = ('Host Name', 'BIOS Manf.', 'OS Version')
@classmethod
def matches(cls, header):
return _has_columns(header, cls.signature)
@classmethod
def transform(cls, text, options=None):
options = options or {}
computertype = (options.get('computertype') or '').strip()
hostname_map = options.get('hostname_map') or {}
header, rows = _read(text)
if not _has_columns(header, cls.signature):
raise ImportSourceError(
'not a Thor export (expected Host Name, BIOS Manf., OS Version '
'columns)')
vendors = {}
models = {}
operatingsystems = {}
assets = []
computers = []
seen = set()
for row in rows:
hostname = (row.get('Host Name') or '').strip()
if not hostname or hostname.lower() in seen:
continue
seen.add(hostname.lower())
vendor = (row.get('BIOS Manf.') or '').strip()
model = (row.get('Model') or '').strip()
osname = cls._osname(row)
location = (row.get('Location') or '').strip()
loggedinuser = cls._logon(row)
if vendor:
vendors[vendor.lower()] = vendor
if model:
models[model.lower()] = model
if osname:
operatingsystems[osname.lower()] = osname
# Align on hostname: if this host already exists in ShopDB, point
# the computer row at its real asset so the engine updates it in
# place. Only create a new asset when there is no hostname match.
existing_assetnumber = hostname_map.get(hostname.lower())
if existing_assetnumber is None:
assets.append({
'assetnumber': hostname,
'name': hostname,
'assettypeid': DEFAULT_ASSETTYPE,
'statusid': DEFAULT_STATUS,
'locationid': location,
})
computer = {
'assetid': existing_assetnumber if existing_assetnumber is not None else hostname,
'hostname': hostname,
'osid': osname,
'vendorid': vendor,
'modelnumberid': model,
'loggedinuser': loggedinuser,
'lastreporteddate': cls._date(row.get('Last Updated')),
'lastboottime': cls._date(row.get('Boot Time')),
}
if computertype:
computer['computertypeid'] = computertype
computers.append(computer)
if not computers:
raise ImportSourceError('no host rows found in the Thor export')
out = {}
if vendors:
out['vendors'] = _write(['vendor'],
[{'vendor': v} for v in vendors.values()])
if models:
out['models'] = _write(['modelnumber'],
[{'modelnumber': m} for m in models.values()])
if operatingsystems:
out['operatingsystems'] = _write(
['osname'], [{'osname': o} for o in operatingsystems.values()])
asset_header = ['assetnumber', 'name', 'assettypeid', 'statusid', 'locationid']
computer_header = ['assetid', 'hostname', 'osid', 'vendorid',
'modelnumberid', 'loggedinuser', 'lastreporteddate',
'lastboottime']
if computertype:
computer_header.append('computertypeid')
out['computers'] = _write(computer_header, computers)
# Only emit an assets file when there are new hosts to create; an empty
# assets.csv would just be an extra clean no-op table.
if assets:
out['assets'] = _write(asset_header, assets)
return out
@staticmethod
def _osname(row):
"""Combine OS Version and OS ID into a single OS name.
Thor splits the OS into a build ('10.0.26100') and a marketing id
('24H2'); neither alone is meaningful, so join them.
"""
version = (row.get('OS Version') or '').strip()
osid = (row.get('OS ID') or '').strip()
if version and osid:
return '%s (%s)' % (version, osid)
return version or osid
@staticmethod
def _logon(row):
"""Take the account name out of a DOMAIN\\user or HOST\\user LOGON."""
logon = (row.get('LOGON') or '').strip()
if '\\' in logon:
return logon.rsplit('\\', 1)[-1]
return logon
@staticmethod
def _date(value):
"""Normalize a Thor timestamp to a form the engine's date coercion
accepts, or empty when it cannot be parsed. Leaves already-ISO values
alone; the engine tries several formats itself.
"""
return (value or '').strip()
SOURCES = {
AzureDeviceGroupSource.key: AzureDeviceGroupSource,
ThorSource.key: ThorSource,
}
def get_source(key):
source = SOURCES.get((key or '').strip().lower())
if source is None:
raise ImportSourceError(
"unknown import source '%s' (known: %s)"
% (key, ', '.join(sorted(SOURCES))))
return source
def detect_source(header):
"""Return the adapter whose signature matches this header, or None."""
for source in SOURCES.values():
if source.matches(header):
return source
return None